diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 85c5459251fd..cd6bf080dfaa 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -8,37 +8,11 @@ body: value: > DuckDB has several repositories for different components, please make sure you're raising your issue in the correct one: - * [Documentation/website](https://github.com/duckdb/duckdb-web/issues/new) - * APIs: - * [duckdb-java](https://github.com/duckdb/duckdb-java/issues/new) - * [duckdb-node](https://github.com/duckdb/duckdb-node/issues/new) - * [duckdb-node-neo](https://github.com/duckdb/duckdb-node-neo/issues/new) - * [duckdb-odbc](https://github.com/duckdb/duckdb-odbc/issues/new) - * [duckdb-r](https://github.com/duckdb/duckdb-r/issues/new) - * [duckdb-rs](https://github.com/duckdb/duckdb-rs/issues/new) - * [duckdb-swift](https://github.com/duckdb/duckdb-swift/issues/new) - * [duckdb-wasm](https://github.com/duckdb/duckdb-wasm/issues/new) - * [go-duckdb](https://github.com/marcboeker/go-duckdb/issues/new) - * Extensions: - * [AWS extension](https://github.com/duckdb/duckdb-aws/issues/new) - * [Azure extension](https://github.com/duckdb/duckdb-azure/issues/new) - * [Delta extension](https://github.com/duckdb/duckdb-delta/issues/new) - * [Encodings extension](https://github.com/duckdb/duckdb-encodings/issues/new) - * [Excel extension](https://github.com/duckdb/duckdb-excel/issues/new) - * [fts (full text search) extension](https://github.com/duckdb/duckdb-fts/issues/new) - * [httpfs extension](https://github.com/duckdb/duckdb-httpfs/issues/new) - * [Iceberg extension](https://github.com/duckdb/duckdb-iceberg/issues/new) - * [inet extension](https://github.com/duckdb/duckdb-inet/issues/new) - * [MySQL extension](https://github.com/duckdb/duckdb-mysql/issues/new) - * [Postgres scanner](https://github.com/duckdb/duckdb-postgres/issues/new) - * [Spatial extension](https://github.com/duckdb/duckdb-spatial/issues/new) - * [SQLite scanner](https://github.com/duckdb/duckdb-sqlite/issues/new) - * [UI](https://github.com/duckdb/duckdb-ui/issues/new) - * [VSS extension](https://github.com/duckdb/duckdb-vss/issues/new) - * Connectors: - * [dbt-duckdb](https://github.com/duckdb/dbt-duckdb) + * [Documentation](https://github.com/duckdb/duckdb-web/issues/new) + * Clients: [Go](https://github.com/duckdb/duckdb-go/issues/new), [Java (JDBC)](https://github.com/duckdb/duckdb-java/issues/new), [Node.js](https://github.com/duckdb/duckdb-node-neo/issues/new), [ODBC](https://github.com/duckdb/duckdb-odbc/issues/new), [Python](https://github.com/duckdb/duckdb-python/issues/new), [R](https://github.com/duckdb/duckdb-r/issues/new), [Rust](https://github.com/duckdb/duckdb-rs/issues/new), [WebAssembly (Wasm)](https://github.com/duckdb/duckdb-wasm/issues/new) + * Extensions: [`aws`](https://github.com/duckdb/duckdb-aws/issues/new), [`azure`](https://github.com/duckdb/duckdb-azure/issues/new), [`delta`](https://github.com/duckdb/duckdb-delta/issues/new), [`ducklake`](https://github.com/duckdb/duckdb-ducklake/issues/new), [`encodings`](https://github.com/duckdb/duckdb-encodings/issues/new), [`excel`](https://github.com/duckdb/duckdb-excel/issues/new), [`fts`](https://github.com/duckdb/duckdb-fts/issues/new), [`httpfs`](https://github.com/duckdb/duckdb-httpfs/issues/new), [`iceberg`](https://github.com/duckdb/duckdb-iceberg/issues/new), [`inet`](https://github.com/duckdb/duckdb-inet/issues/new), [`mysql`](https://github.com/duckdb/duckdb-mysql/issues/new), [`postgres`](https://github.com/duckdb/duckdb-postgres/issues/new), [`spatial`](https://github.com/duckdb/duckdb-spatial/issues/new), [`sqlite`](https://github.com/duckdb/duckdb-sqlite/issues/new), [`ui`](https://github.com/duckdb/duckdb-ui/issues/new), [`vss`](https://github.com/duckdb/duckdb-vss/issues/new) - If none of the above repositories are applicable, feel free to raise it in this one + If the issue occurs in core DuckDB (e.g., a SQL query crashes or returns incorrect results) or if the issue is in the DuckDB command line client, feel free to raise it in this repository. Please report security vulnerabilities using GitHub's [report vulnerability form](https://github.com/duckdb/duckdb/security/advisories/new). @@ -120,18 +94,17 @@ body: # Before Submitting: - - type: dropdown + - type: checkboxes attributes: - label: What is the latest build you tested with? If possible, we recommend testing with the latest nightly build. - description: | - Visit the [installation page](https://duckdb.org/docs/installation/) for instructions. + label: Did you include all relevant configuration (e.g., CPU architecture, Linux distribution) to reproduce the issue? options: - - I have not tested with any build - - I have tested with a stable release - - I have tested with a nightly build - - I have tested with a source build - validations: - required: true + - label: Yes, I have + + - type: checkboxes + attributes: + label: Did you include all code required to reproduce the issue? + options: + - label: Yes, I have - type: dropdown attributes: @@ -145,15 +118,3 @@ body: default: 0 validations: required: true - - - type: checkboxes - attributes: - label: Did you include all code required to reproduce the issue? - options: - - label: Yes, I have - - - type: checkboxes - attributes: - label: Did you include all relevant configuration (e.g., CPU architecture, Python version, Linux distribution) to reproduce the issue? - options: - - label: Yes, I have diff --git a/.github/actions/build_extensions/action.yml b/.github/actions/build_extensions/action.yml index 79b5517a6f52..83cd3f28189d 100644 --- a/.github/actions/build_extensions/action.yml +++ b/.github/actions/build_extensions/action.yml @@ -166,8 +166,6 @@ runs: run: | ls - mkdir -p ~/.ssh - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts cd ${{ inputs.build_dir}} ls -al pwd diff --git a/.github/config/extensions/avro.cmake b/.github/config/extensions/avro.cmake index 75e509ea17a7..d4fdcf9c0e31 100644 --- a/.github/config/extensions/avro.cmake +++ b/.github/config/extensions/avro.cmake @@ -2,7 +2,7 @@ if (NOT MINGW) duckdb_extension_load(avro LOAD_TESTS DONT_LINK GIT_URL https://github.com/duckdb/duckdb-avro - GIT_TAG b75cb5cea43d3e2bee6f5d0f377aa99354dd868a + GIT_TAG 7b75062f6345d11c5342c09216a75c57342c2e82 APPLY_PATCHES ) endif() diff --git a/.github/config/extensions/aws.cmake b/.github/config/extensions/aws.cmake index 47deb231f8d8..ee0547d8089e 100644 --- a/.github/config/extensions/aws.cmake +++ b/.github/config/extensions/aws.cmake @@ -1,8 +1,7 @@ if (NOT MINGW AND NOT ${WASM_ENABLED}) duckdb_extension_load(aws - APPLY_PATCHES ### TODO: re-enable LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb-aws - GIT_TAG 4f318ebd088e464266c511abe2f70bbdeee2fcd8 + GIT_TAG 18803d5e55b9f9f6dda5047d0fdb4f4238b6801d ) endif() diff --git a/.github/config/extensions/azure.cmake b/.github/config/extensions/azure.cmake index 621b01c5e4e4..fe6dcfd19914 100644 --- a/.github/config/extensions/azure.cmake +++ b/.github/config/extensions/azure.cmake @@ -1,8 +1,7 @@ if (NOT MINGW AND NOT ${WASM_ENABLED}) duckdb_extension_load(azure - APPLY_PATCHES LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb-azure - GIT_TAG 8bac45ad4bb858a4b46e9eb3e03cf3a59e3f8df4 + GIT_TAG 0709c0fa1cf67a668b58b1f06ff3e5fc1696e10a ) endif() diff --git a/.github/config/extensions/delta.cmake b/.github/config/extensions/delta.cmake index e0a8e010fbe4..d45a09fc5f0e 100644 --- a/.github/config/extensions/delta.cmake +++ b/.github/config/extensions/delta.cmake @@ -1,7 +1,7 @@ if (NOT MINGW AND NOT ${WASM_ENABLED}) duckdb_extension_load(delta GIT_URL https://github.com/duckdb/duckdb-delta - GIT_TAG b0b32dbd30561dfc0db6399f5305fd14c04ec89d # currently latest commit of v1.4-andium branch + GIT_TAG 03aaf0f073bc622ade27c158d32473588b32aa8b SUBMODULES extension-ci-tools APPLY_PATCHES ) diff --git a/.github/config/extensions/ducklake.cmake b/.github/config/extensions/ducklake.cmake index a55e360f5f06..42ff34e3c3e6 100644 --- a/.github/config/extensions/ducklake.cmake +++ b/.github/config/extensions/ducklake.cmake @@ -1,6 +1,5 @@ duckdb_extension_load(ducklake DONT_LINK GIT_URL https://github.com/duckdb/ducklake - GIT_TAG d2392c36f33151cf5cdd7d006375b0b669bd44ac - APPLY_PATCHES + GIT_TAG f134ad86f2f6e7cdf4133086c38ecd9c48f1a772 ) diff --git a/.github/config/extensions/encodings.cmake b/.github/config/extensions/encodings.cmake index 683abb3956f8..251390e692f3 100644 --- a/.github/config/extensions/encodings.cmake +++ b/.github/config/extensions/encodings.cmake @@ -3,8 +3,7 @@ duckdb_extension_load(encodings LOAD_TESTS DONT_LINK GIT_URL https://github.com/duckdb/duckdb-encodings - GIT_TAG dc3c206e237b517abcdd95ebe40d02dcd0f71084 + GIT_TAG b5a547ec74fad87698ed3142033d7b9cf86e0b2f TEST_DIR test/sql - APPLY_PATCHES ) endif() diff --git a/.github/config/extensions/excel.cmake b/.github/config/extensions/excel.cmake index 5265c2a4eb35..e2403efe3ea8 100644 --- a/.github/config/extensions/excel.cmake +++ b/.github/config/extensions/excel.cmake @@ -1,7 +1,6 @@ duckdb_extension_load(excel - APPLY_PATCHES LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb-excel - GIT_TAG cf00672f2d16685d9aefcca48c6a04d8c37d7015 + GIT_TAG 8504be9ec8183e4082141f9359b53a64d3a440b7 INCLUDE_DIR src/excel/include ) diff --git a/.github/config/extensions/fts.cmake b/.github/config/extensions/fts.cmake index c10aafe5c2d9..6ced0868bfd8 100644 --- a/.github/config/extensions/fts.cmake +++ b/.github/config/extensions/fts.cmake @@ -2,7 +2,6 @@ duckdb_extension_load(fts LOAD_TESTS DONT_LINK GIT_URL https://github.com/duckdb/duckdb-fts - GIT_TAG 3aa6a180b9c101d78070f5f7214c27552bb091c8 + GIT_TAG 39376623630a968154bef4e6930d12ad0b59d7fb TEST_DIR test/sql - APPLY_PATCHES ) diff --git a/.github/config/extensions/httpfs.cmake b/.github/config/extensions/httpfs.cmake index d79065e4285f..7cf901e05a2a 100644 --- a/.github/config/extensions/httpfs.cmake +++ b/.github/config/extensions/httpfs.cmake @@ -1,7 +1,7 @@ duckdb_extension_load(httpfs LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb-httpfs - GIT_TAG c31f9e922a5cc99c5854fd9529954cb364afd1f1 - INCLUDE_DIR extension/httpfs/include + GIT_TAG 8356a9017444f54018159718c8017ff7db4ea756 APPLY_PATCHES + INCLUDE_DIR src/include ) diff --git a/.github/config/extensions/iceberg.cmake b/.github/config/extensions/iceberg.cmake index 47d0bde7f00c..ac47b9f604c6 100644 --- a/.github/config/extensions/iceberg.cmake +++ b/.github/config/extensions/iceberg.cmake @@ -4,12 +4,11 @@ IF (NOT WIN32) else () set(LOAD_ICEBERG_TESTS "") endif() - if (NOT MINGW AND NOT ${WASM_ENABLED}) duckdb_extension_load(iceberg - APPLY_PATCHES # ${LOAD_ICEBERG_TESTS} TODO: re-enable once autoloading test is fixed GIT_URL https://github.com/duckdb/duckdb-iceberg - GIT_TAG 24dd874bee165661f6c3c79ee2a823f02941ed94 + GIT_TAG 49d67e45a6f15ad855f3760658b4ab42967d9cdc + APPLY_PATCHES ) endif() diff --git a/.github/config/extensions/inet.cmake b/.github/config/extensions/inet.cmake index f77911685d38..7b112317c3fa 100644 --- a/.github/config/extensions/inet.cmake +++ b/.github/config/extensions/inet.cmake @@ -1,7 +1,7 @@ duckdb_extension_load(inet LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb-inet - GIT_TAG eb2455703ca0665e69b9fd20fd1d8816c547cb49 + GIT_TAG f6a2a14f061d2dfccdb4283800b55fef3fcbb128 INCLUDE_DIR src/include TEST_DIR test/sql APPLY_PATCHES diff --git a/.github/config/extensions/mysql_scanner.cmake b/.github/config/extensions/mysql_scanner.cmake index ea24702314d2..581cac266d26 100644 --- a/.github/config/extensions/mysql_scanner.cmake +++ b/.github/config/extensions/mysql_scanner.cmake @@ -3,7 +3,6 @@ if (NOT MINGW AND NOT ${WASM_ENABLED} AND NOT ${MUSL_ENABLED}) DONT_LINK LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb-mysql - GIT_TAG b79ef7e2dde1f9253f9ad584883b029eba8d29a4 - APPLY_PATCHES + GIT_TAG c80647b33972c150f0bd0001c35085cefdc82d1e ) endif() diff --git a/.github/config/extensions/postgres_scanner.cmake b/.github/config/extensions/postgres_scanner.cmake index 4417434223a5..d99e014da81d 100644 --- a/.github/config/extensions/postgres_scanner.cmake +++ b/.github/config/extensions/postgres_scanner.cmake @@ -4,7 +4,6 @@ if (NOT MINGW AND NOT ${WASM_ENABLED}) duckdb_extension_load(postgres_scanner DONT_LINK GIT_URL https://github.com/duckdb/duckdb-postgres - GIT_TAG e58cd1dfed98a04ce3e928d5b941e5e2c533ba12 - APPLY_PATCHES + GIT_TAG f012a4f99cea1d276d1787d0dc84b1f1a0e0f0b2 ) endif() diff --git a/.github/config/extensions/spatial.cmake b/.github/config/extensions/spatial.cmake index a4a8895cc874..5d27cf96f70d 100644 --- a/.github/config/extensions/spatial.cmake +++ b/.github/config/extensions/spatial.cmake @@ -1,10 +1,11 @@ -if (NOT MINGW AND ${BUILD_COMPLETE_EXTENSION_SET}) +if (${BUILD_COMPLETE_EXTENSION_SET}) ################# SPATIAL duckdb_extension_load(spatial DONT_LINK LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb-spatial - GIT_TAG 35a3635d6e51151134226f38ffe1c4368a5a5292 + GIT_TAG a6a607fe3a98ef9ad4bed218490b770f725fbc12 INCLUDE_DIR src/spatial TEST_DIR test/sql + APPLY_PATCHES ) endif() diff --git a/.github/config/extensions/sqlite_scanner.cmake b/.github/config/extensions/sqlite_scanner.cmake index c4765f480a87..59d852240978 100644 --- a/.github/config/extensions/sqlite_scanner.cmake +++ b/.github/config/extensions/sqlite_scanner.cmake @@ -8,6 +8,5 @@ endif() duckdb_extension_load(sqlite_scanner ${STATIC_LINK_SQLITE} LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb-sqlite - GIT_TAG ed38d770e0bbf1d5a6660ec1887cc5abef65be15 - APPLY_PATCHES + GIT_TAG 0c93d610af1e1f66292559fcf0f01a93597a98b6 ) diff --git a/.github/config/extensions/sqlsmith.cmake b/.github/config/extensions/sqlsmith.cmake index 9212ace11f9b..1b75231ca0ac 100644 --- a/.github/config/extensions/sqlsmith.cmake +++ b/.github/config/extensions/sqlsmith.cmake @@ -1,6 +1,6 @@ duckdb_extension_load(sqlsmith DONT_LINK LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb-sqlsmith - GIT_TAG 06e8da8a95710c996fcd62f385962ccd36a363f6 + GIT_TAG e6e6750ceb91e1869b2c736abb70d818bac73e9a APPLY_PATCHES ) diff --git a/.github/config/extensions/vss.cmake b/.github/config/extensions/vss.cmake index 84b434bb15bd..d1babd61c369 100644 --- a/.github/config/extensions/vss.cmake +++ b/.github/config/extensions/vss.cmake @@ -2,7 +2,6 @@ duckdb_extension_load(vss LOAD_TESTS DONT_LINK GIT_URL https://github.com/duckdb/duckdb-vss - GIT_TAG ccfa7c9c1f1f540fa7f433a93d32bed772aa44f4 + GIT_TAG c8a4efe05003d8ef6eaad34f5521cf50126c9967 TEST_DIR test/sql - APPLY_PATCHES ) diff --git a/.github/config/in_tree_extensions.cmake b/.github/config/in_tree_extensions.cmake index 4751a3d28478..d8c51afa6e17 100644 --- a/.github/config/in_tree_extensions.cmake +++ b/.github/config/in_tree_extensions.cmake @@ -14,4 +14,4 @@ duckdb_extension_load(tpcds) duckdb_extension_load(tpch) # Test extension for the upcoming C CAPI extensions -duckdb_extension_load(demo_capi DONT_LINK) \ No newline at end of file +duckdb_extension_load(demo_capi DONT_LINK) diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index fc29a85ce53b..ecd463a72962 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -27,7 +27,7 @@ include("${EXTENSION_CONFIG_BASE_DIR}/iceberg.cmake") include("${EXTENSION_CONFIG_BASE_DIR}/inet.cmake") include("${EXTENSION_CONFIG_BASE_DIR}/mysql_scanner.cmake") include("${EXTENSION_CONFIG_BASE_DIR}/postgres_scanner.cmake") -include("${EXTENSION_CONFIG_BASE_DIR}/spatial.cmake") +# include("${EXTENSION_CONFIG_BASE_DIR}/spatial.cmake") Remove spatial until the geometry refactor is done include("${EXTENSION_CONFIG_BASE_DIR}/sqlite_scanner.cmake") include("${EXTENSION_CONFIG_BASE_DIR}/sqlsmith.cmake") include("${EXTENSION_CONFIG_BASE_DIR}/vss.cmake") diff --git a/.github/config/uncovered_files.csv b/.github/config/uncovered_files.csv index 000773130f51..394fa3c4afbe 100644 --- a/.github/config/uncovered_files.csv +++ b/.github/config/uncovered_files.csv @@ -60,7 +60,6 @@ common/serializer/binary_serializer.cpp 11 common/serializer/memory_stream.cpp 5 common/sort/comparators.cpp 103 common/sort/merge_sorter.cpp 100 -common/sort/partition_state.cpp 44 common/sort/sort_state.cpp 10 common/sort/sorted_block.cpp 20 common/string_util.cpp 15 diff --git a/.github/patches/duckdb-wasm/README.md b/.github/patches/duckdb-wasm/README.md deleted file mode 100644 index bf8cd8f6587b..000000000000 --- a/.github/patches/duckdb-wasm/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# WASM patches -Patches in this directory are used to smoothen the process of introducing changes to DuckDB that break compatibility with -the current duckdb-wasm version as is pinned in the `.github/workflows/NightlyTests.yml` workflow. - -# Workflow -Imagine a change to DuckDB is introduced that breaks compatibility with current WASM. The -workflow for this is as follows: - -### PR #1: breaking change to DuckDB -- Commit breaking change to DuckDB -- Fix breakage in duckdb-wasm, producing a patch with fix (be wary of already existing patches) -- Commit patch in `.github/patches/duckdb-wasm/*.patch` using a descriptive name - -### PR #2: patch to duckdb-wasm -- Apply (all) the patch(es) in `.github/patches/duckdb-wasm/*.patch` to duckdb-wasm. - -### PR #3: update extension X in DuckDB -- Remove patches in `.github/patches/duckdb-wasm/*.patch` -- Update hash of duckdb-wasm in `.github/workflows/NightlyTests.yml` \ No newline at end of file diff --git a/.github/patches/extensions/avro/fix.patch b/.github/patches/extensions/avro/fix.patch index f72e40a1e666..5f1f295d11a0 100644 --- a/.github/patches/extensions/avro/fix.patch +++ b/.github/patches/extensions/avro/fix.patch @@ -1,27 +1,13 @@ -diff --git a/src/avro_multi_file_info.cpp b/src/avro_multi_file_info.cpp -index 02dc660..d67d744 100644 ---- a/src/avro_multi_file_info.cpp -+++ b/src/avro_multi_file_info.cpp -@@ -4,7 +4,7 @@ - namespace duckdb { +diff --git a/src/field_ids.cpp b/src/field_ids.cpp +index d197f8d..52fb48c 100644 +--- a/src/field_ids.cpp ++++ b/src/field_ids.cpp +@@ -5,6 +5,8 @@ namespace duckdb { - unique_ptr --AvroMultiFileInfo::InitializeInterface(ClientContext &context, MultiFileReader &reader, MultiFileList &file_list) { -+AvroMultiFileInfo::CreateInterface(ClientContext &context) { - return make_uniq(); - } - -diff --git a/src/include/avro_multi_file_info.hpp b/src/include/avro_multi_file_info.hpp -index 5202c8b..dac2c2c 100644 ---- a/src/include/avro_multi_file_info.hpp -+++ b/src/include/avro_multi_file_info.hpp -@@ -16,8 +16,7 @@ namespace duckdb { - class AvroFileReaderOptions : public BaseFileReaderOptions {}; + namespace avro { - struct AvroMultiFileInfo : MultiFileReaderInterface { -- static unique_ptr InitializeInterface(ClientContext &context, MultiFileReader &reader, -- MultiFileList &file_list); -+ static unique_ptr CreateInterface(ClientContext &context); ++constexpr const char *FieldID::DUCKDB_FIELD_ID; ++ + FieldID::FieldID() : set(false) { + } - unique_ptr InitializeOptions(ClientContext &context, - optional_ptr info) override; diff --git a/.github/patches/extensions/aws/aws_loader.patch b/.github/patches/extensions/aws/aws_loader.patch deleted file mode 100644 index e73ca910ccfc..000000000000 --- a/.github/patches/extensions/aws/aws_loader.patch +++ /dev/null @@ -1,133 +0,0 @@ -diff --git a/src/aws_extension.cpp b/src/aws_extension.cpp -index 1130a1b..08c2e69 100644 ---- a/src/aws_extension.cpp -+++ b/src/aws_extension.cpp -@@ -1,12 +1,10 @@ --#define DUCKDB_EXTENSION_MAIN -- - #include "aws_secret.hpp" - #include "aws_extension.hpp" - - #include "duckdb.hpp" - #include "duckdb/common/exception.hpp" - #include "duckdb/catalog/catalog.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include - #include - #include -@@ -117,11 +115,11 @@ static void LoadAWSCredentialsFun(ClientContext &context, TableFunctionInput &da - - data.finished = true; - } --static void LoadInternal(DuckDB &db) { -+static void LoadInternal(ExtensionLoader &loader) { - Aws::SDKOptions options; - Aws::InitAPI(options); - -- CreateAwsSecretFunctions::InitializeCurlCertificates(*db.instance); -+ CreateAwsSecretFunctions::InitializeCurlCertificates(loader.GetDatabaseInstance()); - - TableFunctionSet function_set("load_aws_credentials"); - auto base_fun = TableFunction("load_aws_credentials", {}, LoadAWSCredentialsFun, LoadAWSCredentialsBind); -@@ -136,13 +134,13 @@ static void LoadInternal(DuckDB &db) { - function_set.AddFunction(base_fun); - function_set.AddFunction(profile_fun); - -- ExtensionUtil::RegisterFunction(*db.instance, function_set); -+ loader.RegisterFunction(function_set); - -- CreateAwsSecretFunctions::Register(*db.instance); -+ CreateAwsSecretFunctions::Register(loader); - } - --void AwsExtension::Load(DuckDB &db) { -- LoadInternal(db); -+void AwsExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - std::string AwsExtension::Name() { - return "aws"; -@@ -152,16 +150,8 @@ std::string AwsExtension::Name() { - - extern "C" { - --DUCKDB_EXTENSION_API void aws_init(duckdb::DatabaseInstance &db) { -- duckdb::DuckDB db_wrapper(db); -- db_wrapper.LoadExtension(); --} -- --DUCKDB_EXTENSION_API const char *aws_version() { -- return duckdb::DuckDB::LibraryVersion(); --} -+DUCKDB_CPP_EXTENSION_ENTRY(aws, loader) { -+ duckdb::LoadInternal(loader); - } - --#ifndef DUCKDB_EXTENSION_MAIN --#error DUCKDB_EXTENSION_MAIN not defined --#endif -+} -\ No newline at end of file -diff --git a/src/aws_secret.cpp b/src/aws_secret.cpp -index bbfc852..87de1bb 100644 ---- a/src/aws_secret.cpp -+++ b/src/aws_secret.cpp -@@ -1,7 +1,7 @@ - #include "aws_secret.hpp" - - #include "duckdb/common/case_insensitive_map.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - - #include - #include -@@ -243,7 +243,7 @@ void CreateAwsSecretFunctions::InitializeCurlCertificates(DatabaseInstance &db) - } - } - --void CreateAwsSecretFunctions::Register(DatabaseInstance &instance) { -+void CreateAwsSecretFunctions::Register(ExtensionLoader &loader) { - vector types = {"s3", "r2", "gcs", "aws"}; - - for (const auto &type : types) { -@@ -275,7 +275,7 @@ void CreateAwsSecretFunctions::Register(DatabaseInstance &instance) { - // Params for configuring the credential loading - cred_chain_function.named_parameters["profile"] = LogicalType::VARCHAR; - -- ExtensionUtil::RegisterFunction(instance, cred_chain_function); -+ loader.RegisterFunction(cred_chain_function); - } - } - -diff --git a/src/include/aws_extension.hpp b/src/include/aws_extension.hpp -index eeb7b57..fcd695e 100644 ---- a/src/include/aws_extension.hpp -+++ b/src/include/aws_extension.hpp -@@ -13,7 +13,7 @@ struct AwsSetCredentialsResult { - - class AwsExtension : public Extension { - public: -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &db) override; - std::string Name() override; - }; - -diff --git a/src/include/aws_secret.hpp b/src/include/aws_secret.hpp -index e16c301..f593e6e 100644 ---- a/src/include/aws_secret.hpp -+++ b/src/include/aws_secret.hpp -@@ -6,10 +6,12 @@ - - namespace duckdb { - -+class ExtensionLoader; -+ - struct CreateAwsSecretFunctions { - public: - //! Register all CreateSecretFunctions -- static void Register(DatabaseInstance &instance); -+ static void Register(ExtensionLoader &instance); - - //! WARNING: not thread-safe, to be called on extension initialization once - static void InitializeCurlCertificates(DatabaseInstance &db); diff --git a/.github/patches/extensions/azure/azure_loader_and_timestamp.patch b/.github/patches/extensions/azure/azure_loader_and_timestamp.patch deleted file mode 100644 index f146943e4a47..000000000000 --- a/.github/patches/extensions/azure/azure_loader_and_timestamp.patch +++ /dev/null @@ -1,277 +0,0 @@ -diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp -index ee19c8c..4116a59 100644 ---- a/src/azure_blob_filesystem.cpp -+++ b/src/azure_blob_filesystem.cpp -@@ -12,7 +12,7 @@ - #include "duckdb/main/secret/secret_manager.hpp" - #include "duckdb/function/scalar/string_common.hpp" - #include "duckdb/function/scalar_function.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/main/client_data.hpp" - #include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" - #include -@@ -151,8 +151,8 @@ vector AzureBlobStorageFileSystem::Glob(const string &path, FileOp - info.extended_info = make_shared_ptr(); - auto &options = info.extended_info->options; - options.emplace("file_size", Value::BIGINT(key.BlobSize)); -- options.emplace("last_modified", Value::TIMESTAMP(Timestamp::FromTimeT( -- AzureStorageFileSystem::ToTimeT(key.Details.LastModified)))); -+ options.emplace("last_modified", Value::TIMESTAMP( -+ AzureStorageFileSystem::ToTimestamp(key.Details.LastModified))); - result.push_back(info); - } - } -@@ -171,10 +171,10 @@ vector AzureBlobStorageFileSystem::Glob(const string &path, FileOp - void AzureBlobStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) { - auto &hfh = handle.Cast(); - -- if (hfh.length == 0 && hfh.last_modified == 0) { -+ if (hfh.length == 0 && hfh.last_modified == timestamp_t(0)) { - auto res = hfh.blob_client.GetProperties(); - hfh.length = res.Value.BlobSize; -- hfh.last_modified = ToTimeT(res.Value.LastModified); -+ hfh.last_modified = ToTimestamp(res.Value.LastModified); - } - } - -diff --git a/src/azure_dfs_filesystem.cpp b/src/azure_dfs_filesystem.cpp -index 5feb445..af8f03c 100644 ---- a/src/azure_dfs_filesystem.cpp -+++ b/src/azure_dfs_filesystem.cpp -@@ -67,8 +67,8 @@ static void Walk(const Azure::Storage::Files::DataLake::DataLakeFileSystemClient - info.extended_info = make_shared_ptr(); - auto &options = info.extended_info->options; - options.emplace("file_size", Value::BIGINT(elt.FileSize)); -- options.emplace("last_modified", Value::TIMESTAMP(Timestamp::FromTimeT( -- AzureStorageFileSystem::ToTimeT(elt.LastModified)))); -+ options.emplace("last_modified", Value::TIMESTAMP( -+ AzureStorageFileSystem::ToTimestamp(elt.LastModified))); - out_result->push_back(info); - } - } -@@ -172,10 +172,10 @@ vector AzureDfsStorageFileSystem::Glob(const string &path, FileOpe - void AzureDfsStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) { - auto &hfh = handle.Cast(); - -- if (hfh.length == 0 && hfh.last_modified == 0) { -+ if (hfh.length == 0 && hfh.last_modified == timestamp_t(0)) { - auto res = hfh.file_client.GetProperties(); - hfh.length = res.Value.FileSize; -- hfh.last_modified = ToTimeT(res.Value.LastModified); -+ hfh.last_modified = ToTimestamp(res.Value.LastModified); - } - } - -diff --git a/src/azure_extension.cpp b/src/azure_extension.cpp -index 5ce0b8d..d0758e2 100644 ---- a/src/azure_extension.cpp -+++ b/src/azure_extension.cpp -@@ -1,5 +1,3 @@ --#define DUCKDB_EXTENSION_MAIN -- - #include "azure_extension.hpp" - #include "azure_blob_filesystem.hpp" - #include "azure_dfs_filesystem.hpp" -@@ -7,14 +5,15 @@ - - namespace duckdb { - --static void LoadInternal(DatabaseInstance &instance) { -+static void LoadInternal(ExtensionLoader &loader) { - // Load filesystem -+ auto &instance = loader.GetDatabaseInstance(); - auto &fs = instance.GetFileSystem(); - fs.RegisterSubSystem(make_uniq()); - fs.RegisterSubSystem(make_uniq()); - - // Load Secret functions -- CreateAzureSecretFunctions::Register(instance); -+ CreateAzureSecretFunctions::Register(loader); - - // Load extension config - auto &config = DBConfig::GetConfig(instance); -@@ -77,8 +76,8 @@ static void LoadInternal(DatabaseInstance &instance) { - Value(nullptr)); - } - --void AzureExtension::Load(DuckDB &db) { -- LoadInternal(*db.instance); -+void AzureExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - std::string AzureExtension::Name() { - return "azure"; -@@ -88,15 +87,8 @@ std::string AzureExtension::Name() { - - extern "C" { - --DUCKDB_EXTENSION_API void azure_init(duckdb::DatabaseInstance &db) { -- LoadInternal(db); --} -- --DUCKDB_EXTENSION_API const char *azure_version() { -- return duckdb::DuckDB::LibraryVersion(); --} -+DUCKDB_CPP_EXTENSION_ENTRY(azure, loader) { -+ duckdb::LoadInternal(loader); - } - --#ifndef DUCKDB_EXTENSION_MAIN --#error DUCKDB_EXTENSION_MAIN not defined --#endif -+} -\ No newline at end of file -diff --git a/src/azure_filesystem.cpp b/src/azure_filesystem.cpp -index d7ed201..07f84eb 100644 ---- a/src/azure_filesystem.cpp -+++ b/src/azure_filesystem.cpp -@@ -40,7 +40,7 @@ AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, const OpenFileInfo - } - auto entry2 = info.extended_info->options.find("last_modified"); - if (entry2 != info.extended_info->options.end()) { -- last_modified = Timestamp::ToTimeT(entry2->second.GetValue()); -+ last_modified = entry2->second.GetValue(); - } - } - } -@@ -93,7 +93,7 @@ int64_t AzureStorageFileSystem::GetFileSize(FileHandle &handle) { - return afh.length; - } - --time_t AzureStorageFileSystem::GetLastModifiedTime(FileHandle &handle) { -+timestamp_t AzureStorageFileSystem::GetLastModifiedTime(FileHandle &handle) { - auto &afh = handle.Cast(); - return afh.last_modified; - } -@@ -228,9 +228,10 @@ AzureReadOptions AzureStorageFileSystem::ParseAzureReadOptions(optional_ptr(dt); -- return std::chrono::system_clock::to_time_t(time_point); -+ auto micros = std::chrono::duration_cast(time_point.time_since_epoch()).count(); -+ return timestamp_t(micros); - } - - } // namespace duckdb -diff --git a/src/azure_secret.cpp b/src/azure_secret.cpp -index e9f08f0..7b1919a 100644 ---- a/src/azure_secret.cpp -+++ b/src/azure_secret.cpp -@@ -2,7 +2,7 @@ - #include "azure_dfs_filesystem.hpp" - #include "duckdb/common/types.hpp" - #include "duckdb/common/unique_ptr.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/main/secret/secret.hpp" - #include - #include -@@ -148,7 +148,7 @@ static void RegisterCommonSecretParameters(CreateSecretFunction &function) { - function.named_parameters["proxy_password"] = LogicalType::VARCHAR; - } - --void CreateAzureSecretFunctions::Register(DatabaseInstance &instance) { -+void CreateAzureSecretFunctions::Register(ExtensionLoader &loader) { - string type = "azure"; - - // Register the new type -@@ -156,19 +156,19 @@ void CreateAzureSecretFunctions::Register(DatabaseInstance &instance) { - secret_type.name = type; - secret_type.deserializer = KeyValueSecret::Deserialize; - secret_type.default_provider = "config"; -- ExtensionUtil::RegisterSecretType(instance, secret_type); -+ loader.RegisterSecretType(secret_type); - - // Register the connection string secret provider - CreateSecretFunction connection_string_function = {type, "config", CreateAzureSecretFromConfig}; - connection_string_function.named_parameters["connection_string"] = LogicalType::VARCHAR; - RegisterCommonSecretParameters(connection_string_function); -- ExtensionUtil::RegisterFunction(instance, connection_string_function); -+ loader.RegisterFunction(connection_string_function); - - // Register the credential_chain secret provider - CreateSecretFunction cred_chain_function = {type, "credential_chain", CreateAzureSecretFromCredentialChain}; - cred_chain_function.named_parameters["chain"] = LogicalType::VARCHAR; - RegisterCommonSecretParameters(cred_chain_function); -- ExtensionUtil::RegisterFunction(instance, cred_chain_function); -+ loader.RegisterFunction(cred_chain_function); - - // Register the service_principal secret provider - CreateSecretFunction service_principal_function = {type, "service_principal", -@@ -178,13 +178,13 @@ void CreateAzureSecretFunctions::Register(DatabaseInstance &instance) { - service_principal_function.named_parameters["client_secret"] = LogicalType::VARCHAR; - service_principal_function.named_parameters["client_certificate_path"] = LogicalType::VARCHAR; - RegisterCommonSecretParameters(service_principal_function); -- ExtensionUtil::RegisterFunction(instance, service_principal_function); -+ loader.RegisterFunction(service_principal_function); - - // Register the access_token secret provider - CreateSecretFunction access_token_function = {type, "access_token", CreateAzureSecretFromAccessToken}; - access_token_function.named_parameters["access_token"] = LogicalType::VARCHAR; - RegisterCommonSecretParameters(access_token_function); -- ExtensionUtil::RegisterFunction(instance, access_token_function); -+ loader.RegisterFunction(access_token_function); - } - - } // namespace duckdb -diff --git a/src/include/azure_extension.hpp b/src/include/azure_extension.hpp -index c9be37e..1a8404f 100644 ---- a/src/include/azure_extension.hpp -+++ b/src/include/azure_extension.hpp -@@ -5,7 +5,7 @@ - namespace duckdb { - class AzureExtension : public Extension { - public: -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &loader) override; - std::string Name() override; - }; - -diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp -index 32c26bf..e29cf9a 100644 ---- a/src/include/azure_filesystem.hpp -+++ b/src/include/azure_filesystem.hpp -@@ -61,7 +61,7 @@ public: - - // File info - idx_t length; -- time_t last_modified; -+ timestamp_t last_modified; - - // Read buffer - duckdb::unique_ptr read_buffer; -@@ -93,7 +93,7 @@ public: - return false; - } - int64_t GetFileSize(FileHandle &handle) override; -- time_t GetLastModifiedTime(FileHandle &handle) override; -+ timestamp_t GetLastModifiedTime(FileHandle &handle) override; - void Seek(FileHandle &handle, idx_t location) override; - void FileSync(FileHandle &handle) override; - -@@ -125,7 +125,7 @@ protected: - static AzureReadOptions ParseAzureReadOptions(optional_ptr opener); - - public: -- static time_t ToTimeT(const Azure::DateTime &dt); -+ static timestamp_t ToTimestamp(const Azure::DateTime &dt); - }; - - } // namespace duckdb -diff --git a/src/include/azure_secret.hpp b/src/include/azure_secret.hpp -index 3dc39e9..6b81816 100644 ---- a/src/include/azure_secret.hpp -+++ b/src/include/azure_secret.hpp -@@ -18,7 +18,7 @@ class CreateSecretFunction; - struct CreateAzureSecretFunctions { - public: - //! Register all CreateSecretFunctions -- static void Register(DatabaseInstance &instance); -+ static void Register(ExtensionLoader &instance); - }; - - } // namespace duckdb diff --git a/.github/patches/extensions/delta/logging_pragma.patch b/.github/patches/extensions/delta/logging_pragma.patch deleted file mode 100644 index f0aaa4b05c5d..000000000000 --- a/.github/patches/extensions/delta/logging_pragma.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/src/functions/delta_scan/delta_multi_file_reader.cpp b/src/functions/delta_scan/delta_multi_file_reader.cpp -index de3b219..cda479c 100644 ---- a/src/functions/delta_scan/delta_multi_file_reader.cpp -+++ b/src/functions/delta_scan/delta_multi_file_reader.cpp -@@ -226,7 +226,7 @@ void DeltaMultiFileReader::FinalizeBind(MultiFileReaderData &reader_data, const - } - - shared_ptr DeltaMultiFileReader::CreateFileList(ClientContext &context, const vector &paths, -- FileGlobOptions options) { -+ const FileGlobInput &glob_input) { - if (paths.size() != 1) { - throw BinderException("'delta_scan' only supports single path as input"); - } -diff --git a/src/include/functions/delta_scan/delta_multi_file_reader.hpp b/src/include/functions/delta_scan/delta_multi_file_reader.hpp -index da600ff..d56b7a0 100644 ---- a/src/include/functions/delta_scan/delta_multi_file_reader.hpp -+++ b/src/include/functions/delta_scan/delta_multi_file_reader.hpp -@@ -29,7 +29,7 @@ struct DeltaMultiFileReader : public MultiFileReader { - static unique_ptr CreateInstance(const TableFunction &table_function); - //! Return a DeltaMultiFileList - shared_ptr CreateFileList(ClientContext &context, const vector &paths, -- FileGlobOptions options) override; -+ const FileGlobInput &glob_input) override; - - //! Override the regular parquet bind using the MultiFileReader Bind. The bind from these are what DuckDB's file - //! readers will try read diff --git a/.github/patches/extensions/delta/remove_include.patch b/.github/patches/extensions/delta/remove_include.patch new file mode 100644 index 000000000000..bab81fd2e26b --- /dev/null +++ b/.github/patches/extensions/delta/remove_include.patch @@ -0,0 +1,12 @@ +diff --git a/src/storage/delta_insert.cpp b/src/storage/delta_insert.cpp +index 93ebf9f..8eea9f5 100644 +--- a/src/storage/delta_insert.cpp ++++ b/src/storage/delta_insert.cpp +@@ -1,7 +1,5 @@ + #include "storage/delta_insert.hpp" + +-#include +- + #include "duckdb/catalog/catalog_entry/copy_function_catalog_entry.hpp" + #include "duckdb/main/client_data.hpp" + #include "duckdb/planner/operator/logical_copy_to_file.hpp" diff --git a/.github/patches/extensions/ducklake/fix.patch b/.github/patches/extensions/ducklake/fix.patch deleted file mode 100644 index 2c54b1b81c26..000000000000 --- a/.github/patches/extensions/ducklake/fix.patch +++ /dev/null @@ -1,39 +0,0 @@ -diff --git a/src/include/storage/ducklake_multi_file_reader.hpp b/src/include/storage/ducklake_multi_file_reader.hpp -index 4ee6910..9150fe5 100644 ---- a/src/include/storage/ducklake_multi_file_reader.hpp -+++ b/src/include/storage/ducklake_multi_file_reader.hpp -@@ -32,7 +32,7 @@ public: - static unique_ptr CreateInstance(const TableFunction &table_function); - //! Return a DuckLakeMultiFileList - shared_ptr CreateFileList(ClientContext &context, const vector &paths, -- FileGlobOptions options) override; -+ const FileGlobInput &glob_input) override; - - //! Override the regular parquet bind using the MultiFileReader Bind. The bind from these are what DuckDB's file - //! readers will try read -diff --git a/src/storage/ducklake_insert.cpp b/src/storage/ducklake_insert.cpp -index bebf523..2b16d6f 100644 ---- a/src/storage/ducklake_insert.cpp -+++ b/src/storage/ducklake_insert.cpp -@@ -661,7 +661,7 @@ PhysicalOperator &DuckLakeCatalog::PlanInsert(ClientContext &context, PhysicalPl - if (op.return_chunk) { - throw BinderException("RETURNING clause not yet supported for insertion into DuckLake table"); - } -- if (op.action_type != OnConflictAction::THROW) { -+ if (op.on_conflict_info.action_type != OnConflictAction::THROW) { - throw BinderException("ON CONFLICT clause not yet supported for insertion into DuckLake table"); - } - if (!op.column_index_map.empty()) { -diff --git a/src/storage/ducklake_multi_file_reader.cpp b/src/storage/ducklake_multi_file_reader.cpp -index d952f84..865e575 100644 ---- a/src/storage/ducklake_multi_file_reader.cpp -+++ b/src/storage/ducklake_multi_file_reader.cpp -@@ -50,7 +50,7 @@ unique_ptr DuckLakeMultiFileReader::CreateInstance(const TableF - } - - shared_ptr DuckLakeMultiFileReader::CreateFileList(ClientContext &context, const vector &paths, -- FileGlobOptions options) { -+ const FileGlobInput &glob_input) { - auto &transaction = DuckLakeTransaction::Get(context, read_info.table.ParentCatalog()); - auto transaction_local_files = transaction.GetTransactionLocalFiles(read_info.table_id); - transaction_local_data = transaction.GetTransactionLocalInlinedData(read_info.table_id); diff --git a/.github/patches/extensions/encodings/encodings_loader.patch b/.github/patches/extensions/encodings/encodings_loader.patch deleted file mode 100644 index 63ab9914ff12..000000000000 --- a/.github/patches/extensions/encodings/encodings_loader.patch +++ /dev/null @@ -1,67 +0,0 @@ -diff --git a/src/encodings_extension.cpp b/src/encodings_extension.cpp -index 4549afa..ae64a56 100644 ---- a/src/encodings_extension.cpp -+++ b/src/encodings_extension.cpp -@@ -1,24 +1,23 @@ --#define DUCKDB_EXTENSION_MAIN -- - #include "encodings_extension.hpp" - #include "duckdb.hpp" - #include "duckdb/common/exception.hpp" - #include "duckdb/common/string_util.hpp" - #include "duckdb/function/scalar_function.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - - #include "generated_encoded_function.hpp" - #include "generated/registration.hpp" - - namespace duckdb { - --static void LoadInternal(DatabaseInstance &instance) { -+static void LoadInternal(ExtensionLoader &loader) { - // Register a scalar function -- duckdb_encodings::RegistrationEncodedFunctions::RegisterFunctions(instance.config); -+ auto &config = loader.GetDatabaseInstance().config; -+ duckdb_encodings::RegistrationEncodedFunctions::RegisterFunctions(config); - } - --void EncodingsExtension::Load(DuckDB &db) { -- LoadInternal(*db.instance); -+void EncodingsExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - std::string EncodingsExtension::Name() { - return "encodings"; -@@ -36,16 +35,8 @@ std::string EncodingsExtension::Version() const { - - extern "C" { - --DUCKDB_EXTENSION_API void encodings_init(duckdb::DatabaseInstance &db) { -- duckdb::DuckDB db_wrapper(db); -- db_wrapper.LoadExtension(); -+DUCKDB_CPP_EXTENSION_ENTRY(encodings, loader) { -+ duckdb::LoadInternal(loader); - } - --DUCKDB_EXTENSION_API const char *encodings_version() { -- return duckdb::DuckDB::LibraryVersion(); --} - } -- --#ifndef DUCKDB_EXTENSION_MAIN --#error DUCKDB_EXTENSION_MAIN not defined --#endif -diff --git a/src/include/encodings_extension.hpp b/src/include/encodings_extension.hpp -index 2c3559c..bcf5538 100644 ---- a/src/include/encodings_extension.hpp -+++ b/src/include/encodings_extension.hpp -@@ -6,7 +6,7 @@ namespace duckdb { - - class EncodingsExtension : public Extension { - public: -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &loader) override; - std::string Name() override; - std::string Version() const override; - }; diff --git a/.github/patches/extensions/excel/excel_copy.patch b/.github/patches/extensions/excel/excel_copy.patch deleted file mode 100644 index 65ee79bf8b8a..000000000000 --- a/.github/patches/extensions/excel/excel_copy.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/src/excel/xlsx/copy_xlsx.cpp b/src/excel/xlsx/copy_xlsx.cpp -index 33890b5..f91adf6 100644 ---- a/src/excel/xlsx/copy_xlsx.cpp -+++ b/src/excel/xlsx/copy_xlsx.cpp -@@ -392,7 +392,7 @@ static void SetVarcharValue(named_parameter_map_t ¶ms, const string &key, co - params[key] = val.back(); - } - --static void ParseCopyFromOptions(XLSXReadData &data, case_insensitive_map_t> &options) { -+static void ParseCopyFromOptions(XLSXReadData &data, const case_insensitive_map_t> &options) { - - // Just make it really easy for us, extract everything into a named parameter map - named_parameter_map_t named_parameters; -@@ -421,16 +421,16 @@ static void ParseCopyFromOptions(XLSXReadData &data, case_insensitive_map_t CopyFromBind(ClientContext &context, CopyInfo &info, vector &expected_names, -+static unique_ptr CopyFromBind(ClientContext &context, CopyFromFunctionBindInput &input, vector &expected_names, - vector &expected_types) { - - auto result = make_uniq(); -- result->file_path = info.file_path; -+ result->file_path = input.info.file_path; - - // TODO: Parse options -- ParseCopyFromOptions(*result, info.options); -+ ParseCopyFromOptions(*result, input.info.options); - -- ZipFileReader archive(context, info.file_path); -+ ZipFileReader archive(context, input.info.file_path); - ReadXLSX::ResolveSheet(result, archive); - - // Column count mismatch! diff --git a/.github/patches/extensions/excel/excel_loader.patch b/.github/patches/extensions/excel/excel_loader.patch deleted file mode 100644 index d8b731bde835..000000000000 --- a/.github/patches/extensions/excel/excel_loader.patch +++ /dev/null @@ -1,168 +0,0 @@ -diff --git a/src/excel/excel_extension.cpp b/src/excel/excel_extension.cpp -index 84a4e4a..9dce971 100644 ---- a/src/excel/excel_extension.cpp -+++ b/src/excel/excel_extension.cpp -@@ -1,12 +1,10 @@ --#define DUCKDB_EXTENSION_MAIN -- - #include "excel_extension.hpp" - - #include "duckdb.hpp" - #include "duckdb/common/exception.hpp" - #include "duckdb/common/string_util.hpp" - #include "duckdb/function/scalar_function.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "nf_calendar.h" - #include "nf_localedata.h" - #include "nf_zformat.h" -@@ -70,21 +68,24 @@ static void NumberFormatFunction(DataChunk &args, ExpressionState &state, Vector - // Load - //-------------------------------------------------------------------------------------------------- - --void ExcelExtension::Load(DuckDB &db) { -- auto &db_instance = *db.instance; -+static void LoadInternal(ExtensionLoader &loader) { - - ScalarFunction text_func("text", {LogicalType::DOUBLE, LogicalType::VARCHAR}, LogicalType::VARCHAR, -- NumberFormatFunction); -- ExtensionUtil::RegisterFunction(db_instance, text_func); -+ NumberFormatFunction); -+ loader.RegisterFunction(text_func); - - ScalarFunction excel_text_func("excel_text", {LogicalType::DOUBLE, LogicalType::VARCHAR}, LogicalType::VARCHAR, -- NumberFormatFunction); -+ NumberFormatFunction); - -- ExtensionUtil::RegisterFunction(db_instance, excel_text_func); -+ loader.RegisterFunction(excel_text_func); - - // Register the XLSX functions -- ReadXLSX::Register(db_instance); -- WriteXLSX::Register(db_instance); -+ ReadXLSX::Register(loader); -+ WriteXLSX::Register(loader); -+} -+ -+void ExcelExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - - std::string ExcelExtension::Name() { -@@ -95,16 +96,8 @@ std::string ExcelExtension::Name() { - - extern "C" { - --DUCKDB_EXTENSION_API void excel_init(duckdb::DatabaseInstance &db) { -- duckdb::DuckDB db_wrapper(db); -- db_wrapper.LoadExtension(); -+DUCKDB_CPP_EXTENSION_ENTRY(excel, loader) { -+ duckdb::LoadInternal(loader); - } - --DUCKDB_EXTENSION_API const char *excel_version() { -- return duckdb::DuckDB::LibraryVersion(); - } --} -- --#ifndef DUCKDB_EXTENSION_MAIN --#error DUCKDB_EXTENSION_MAIN not defined --#endif -diff --git a/src/excel/include/excel_extension.hpp b/src/excel/include/excel_extension.hpp -index 156bb7e..5b711aa 100644 ---- a/src/excel/include/excel_extension.hpp -+++ b/src/excel/include/excel_extension.hpp -@@ -15,7 +15,7 @@ namespace duckdb { - - class ExcelExtension : public Extension { - public: -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &loader) override; - std::string Name() override; - }; - -diff --git a/src/excel/include/xlsx/read_xlsx.hpp b/src/excel/include/xlsx/read_xlsx.hpp -index fb1a323..f3901d6 100644 ---- a/src/excel/include/xlsx/read_xlsx.hpp -+++ b/src/excel/include/xlsx/read_xlsx.hpp -@@ -6,10 +6,10 @@ - - namespace duckdb { - --class DatabaseInstance; -+class ExtensionLoader; - - struct WriteXLSX { -- static void Register(DatabaseInstance &db); -+ static void Register(ExtensionLoader &loader); - }; - - enum class XLSXHeaderMode : uint8_t { NEVER, MAYBE, FORCE }; -@@ -47,7 +47,7 @@ struct ReadXLSX { - static void ParseOptions(XLSXReadOptions &options, const named_parameter_map_t &input); - static void ResolveSheet(const unique_ptr &result, ZipFileReader &archive); - -- static void Register(DatabaseInstance &db); -+ static void Register(ExtensionLoader &loader); - static TableFunction GetFunction(); - }; - -diff --git a/src/excel/xlsx/copy_xlsx.cpp b/src/excel/xlsx/copy_xlsx.cpp -index 33890b5..94a0ac9 100644 ---- a/src/excel/xlsx/copy_xlsx.cpp -+++ b/src/excel/xlsx/copy_xlsx.cpp -@@ -1,7 +1,7 @@ - #include "duckdb/common/exception/conversion_exception.hpp" - #include "duckdb/function/copy_function.hpp" - #include "duckdb/main/database.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/planner/expression/bound_cast_expression.hpp" - #include "duckdb/planner/expression/bound_function_expression.hpp" - #include "duckdb/planner/expression/bound_reference_expression.hpp" -@@ -470,7 +470,7 @@ static unique_ptr CopyFromBind(ClientContext &context, CopyInfo &i - //------------------------------------------------------------------------------ - // Register - //------------------------------------------------------------------------------ --void WriteXLSX::Register(DatabaseInstance &db) { -+void WriteXLSX::Register(ExtensionLoader &loader) { - CopyFunction info("xlsx"); - - info.copy_to_bind = Bind; -@@ -485,7 +485,7 @@ void WriteXLSX::Register(DatabaseInstance &db) { - info.copy_from_function = ReadXLSX::GetFunction(); - - info.extension = "xlsx"; -- ExtensionUtil::RegisterFunction(db, info); -+ loader.RegisterFunction(info); - } - - } // namespace duckdb -\ No newline at end of file -diff --git a/src/excel/xlsx/read_xlsx.cpp b/src/excel/xlsx/read_xlsx.cpp -index ebd621b..10b10e8 100644 ---- a/src/excel/xlsx/read_xlsx.cpp -+++ b/src/excel/xlsx/read_xlsx.cpp -@@ -5,7 +5,7 @@ - #include "duckdb/function/replacement_scan.hpp" - #include "duckdb/function/table_function.hpp" - #include "duckdb/main/database.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/main/query_result.hpp" - #include "duckdb/parser/expression/constant_expression.hpp" - #include "duckdb/parser/expression/function_expression.hpp" -@@ -726,9 +726,9 @@ TableFunction ReadXLSX::GetFunction() { - return read_xlsx; - } - --void ReadXLSX::Register(DatabaseInstance &db) { -- ExtensionUtil::RegisterFunction(db, GetFunction()); -- db.config.replacement_scans.emplace_back(XLSXReplacementScan); -+void ReadXLSX::Register(ExtensionLoader &loader) { -+ loader.RegisterFunction(GetFunction()); -+ loader.GetDatabaseInstance().config.replacement_scans.emplace_back(XLSXReplacementScan); - } - - } // namespace duckdb -\ No newline at end of file diff --git a/.github/patches/extensions/fix.patch b/.github/patches/extensions/fix.patch deleted file mode 100644 index f72e40a1e666..000000000000 --- a/.github/patches/extensions/fix.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/src/avro_multi_file_info.cpp b/src/avro_multi_file_info.cpp -index 02dc660..d67d744 100644 ---- a/src/avro_multi_file_info.cpp -+++ b/src/avro_multi_file_info.cpp -@@ -4,7 +4,7 @@ - namespace duckdb { - - unique_ptr --AvroMultiFileInfo::InitializeInterface(ClientContext &context, MultiFileReader &reader, MultiFileList &file_list) { -+AvroMultiFileInfo::CreateInterface(ClientContext &context) { - return make_uniq(); - } - -diff --git a/src/include/avro_multi_file_info.hpp b/src/include/avro_multi_file_info.hpp -index 5202c8b..dac2c2c 100644 ---- a/src/include/avro_multi_file_info.hpp -+++ b/src/include/avro_multi_file_info.hpp -@@ -16,8 +16,7 @@ namespace duckdb { - class AvroFileReaderOptions : public BaseFileReaderOptions {}; - - struct AvroMultiFileInfo : MultiFileReaderInterface { -- static unique_ptr InitializeInterface(ClientContext &context, MultiFileReader &reader, -- MultiFileList &file_list); -+ static unique_ptr CreateInterface(ClientContext &context); - - unique_ptr InitializeOptions(ClientContext &context, - optional_ptr info) override; diff --git a/.github/patches/extensions/fts/fts_loader.patch b/.github/patches/extensions/fts/fts_loader.patch deleted file mode 100644 index 66f8b19b07be..000000000000 --- a/.github/patches/extensions/fts/fts_loader.patch +++ /dev/null @@ -1,80 +0,0 @@ -diff --git a/extension/fts/fts_extension.cpp b/extension/fts/fts_extension.cpp -index 3cd6e75..4f4d0b2 100644 ---- a/extension/fts/fts_extension.cpp -+++ b/extension/fts/fts_extension.cpp -@@ -1,4 +1,3 @@ --#define DUCKDB_EXTENSION_MAIN - #include "fts_extension.hpp" - - #include "duckdb.hpp" -@@ -6,7 +5,7 @@ - #include "duckdb/common/string_util.hpp" - #include "duckdb/function/pragma_function.hpp" - #include "duckdb/function/scalar_function.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "fts_indexing.hpp" - #include "libstemmer.h" - -@@ -46,8 +45,8 @@ static void StemFunction(DataChunk &args, ExpressionState &state, Vector &result - }); - } - --static void LoadInternal(DuckDB &db) { -- auto &db_instance = *db.instance; -+static void LoadInternal(ExtensionLoader &loader) { -+ - ScalarFunction stem_func("stem", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, StemFunction); - - auto create_fts_index_func = -@@ -63,13 +62,13 @@ static void LoadInternal(DuckDB &db) { - auto drop_fts_index_func = - PragmaFunction::PragmaCall("drop_fts_index", FTSIndexing::DropFTSIndexQuery, {LogicalType::VARCHAR}); - -- ExtensionUtil::RegisterFunction(db_instance, stem_func); -- ExtensionUtil::RegisterFunction(db_instance, create_fts_index_func); -- ExtensionUtil::RegisterFunction(db_instance, drop_fts_index_func); -+ loader.RegisterFunction(stem_func); -+ loader.RegisterFunction(create_fts_index_func); -+ loader.RegisterFunction(drop_fts_index_func); - } - --void FtsExtension::Load(DuckDB &db) { -- LoadInternal(db); -+void FtsExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - - std::string FtsExtension::Name() { -@@ -88,16 +87,8 @@ std::string FtsExtension::Version() const { - - extern "C" { - --DUCKDB_EXTENSION_API void fts_init(duckdb::DatabaseInstance &db) { -- duckdb::DuckDB db_wrapper(db); -- duckdb::LoadInternal(db_wrapper); -+DUCKDB_CPP_EXTENSION_ENTRY(fts, loader) { -+ duckdb::LoadInternal(loader); - } - --DUCKDB_EXTENSION_API const char *fts_version() { -- return duckdb::DuckDB::LibraryVersion(); --} - } -- --#ifndef DUCKDB_EXTENSION_MAIN --#error DUCKDB_EXTENSION_MAIN not defined --#endif -diff --git a/extension/fts/include/fts_extension.hpp b/extension/fts/include/fts_extension.hpp -index 389ffd7..048d4ea 100644 ---- a/extension/fts/include/fts_extension.hpp -+++ b/extension/fts/include/fts_extension.hpp -@@ -14,7 +14,7 @@ namespace duckdb { - - class FtsExtension : public Extension { - public: -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &loader) override; - std::string Name() override; - std::string Version() const override; - }; diff --git a/.github/patches/extensions/httpfs/fix.patch b/.github/patches/extensions/httpfs/fix.patch index 9ca17967d0b4..0327dd513e33 100644 --- a/.github/patches/extensions/httpfs/fix.patch +++ b/.github/patches/extensions/httpfs/fix.patch @@ -1,26 +1,38 @@ -diff --git a/test/sql/copy/csv/test_csv_httpfs.test b/test/sql/copy/csv/test_csv_httpfs.test -index 76b92bd..d461416 100644 ---- a/test/sql/copy/csv/test_csv_httpfs.test -+++ b/test/sql/copy/csv/test_csv_httpfs.test -@@ -10,7 +10,7 @@ statement ok - PRAGMA enable_verification - - statement ok --pragma enable_logging('HTTP'); -+CALL enable_logging('HTTP'); - - foreach httpfs_implementation curl httplib - -diff --git a/test/sql/test_headers_parsed.test b/test/sql/test_headers_parsed.test -index 317ec82..d0e76bd 100644 ---- a/test/sql/test_headers_parsed.test -+++ b/test/sql/test_headers_parsed.test -@@ -10,7 +10,7 @@ statement ok - SET httpfs_client_implementation='curl'; - - statement ok --pragma enable_logging('HTTP'); -+CALL enable_logging('HTTP'); - - query II - select * from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/job_role_type.parquet' order by all; +diff --git a/src/s3fs.cpp b/src/s3fs.cpp +index 72eddc3..601ecba 100644 +--- a/src/s3fs.cpp ++++ b/src/s3fs.cpp +@@ -895,7 +895,7 @@ void S3FileHandle::Initialize(optional_ptr opener) { + correct_region = new_region->second; + } + auto extra_text = S3FileSystem::GetS3BadRequestError(auth_params, correct_region); +- throw Exception(error.Type(), error.RawMessage() + extra_text, extra_info); ++ throw Exception(extra_info, error.Type(), error.RawMessage() + extra_text); + } + if (entry->second == "403") { + // 403: FORBIDDEN +@@ -905,7 +905,7 @@ void S3FileHandle::Initialize(optional_ptr opener) { + } else { + extra_text = S3FileSystem::GetS3AuthError(auth_params); + } +- throw Exception(error.Type(), error.RawMessage() + extra_text, extra_info); ++ throw Exception(extra_info, error.Type(), error.RawMessage() + extra_text); + } + } + throw; +@@ -941,13 +941,13 @@ bool S3FileSystem::CanHandleFile(const string &fpath) { + void S3FileSystem::RemoveFile(const string &path, optional_ptr opener) { + auto handle = OpenFile(path, FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS, opener); + if (!handle) { +- throw IOException("Could not remove file \"%s\": %s", {{"errno", "404"}}, path, "No such file or directory"); ++ throw IOException({{"errno", "404"}}, "Could not remove file \"%s\": %s", path, "No such file or directory"); + } + + auto &s3fh = handle->Cast(); + auto res = DeleteRequest(*handle, s3fh.path, {}); + if (res->status != HTTPStatusCode::OK_200 && res->status != HTTPStatusCode::NoContent_204) { +- throw IOException("Could not remove file \"%s\": %s", {{"errno", to_string(static_cast(res->status))}}, ++ throw IOException({{"errno", to_string(static_cast(res->status))}}, "Could not remove file \"%s\": %s", + path, res->GetError()); + } + } diff --git a/.github/patches/extensions/iceberg/fix.patch b/.github/patches/extensions/iceberg/fix.patch deleted file mode 100644 index 4449a2217e2f..000000000000 --- a/.github/patches/extensions/iceberg/fix.patch +++ /dev/null @@ -1,485 +0,0 @@ -diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp -index 0b278db6..83d26410 100644 ---- a/src/iceberg_extension.cpp -+++ b/src/iceberg_extension.cpp -@@ -1,5 +1,3 @@ --#define DUCKDB_EXTENSION_MAIN -- - #include "iceberg_extension.hpp" - #include "storage/irc_catalog.hpp" - #include "storage/irc_transaction_manager.hpp" -@@ -9,7 +7,7 @@ - #include "duckdb/common/exception/http_exception.hpp" - #include "duckdb/common/string_util.hpp" - #include "duckdb/function/scalar_function.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/catalog/catalog_entry/macro_catalog_entry.hpp" - #include "duckdb/catalog/default/default_functions.hpp" - #include "duckdb/parser/parsed_data/attach_info.hpp" -@@ -122,9 +120,9 @@ static void GlueAttach(ClientContext &context, IcebergAttachOptions &input) { - - //! Base attach method - --static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, -+static unique_ptr IcebergCatalogAttach(optional_ptr storage_info, ClientContext &context, - AttachedDatabase &db, const string &name, AttachInfo &info, -- AccessMode access_mode) { -+ AttachOptions &options) { - IRCEndpointBuilder endpoint_builder; - - string endpoint_type_string; -@@ -213,13 +211,13 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in - } - - D_ASSERT(auth_handler); -- auto catalog = make_uniq(db, access_mode, std::move(auth_handler), attach_options.warehouse, -+ auto catalog = make_uniq(db, options.access_mode, std::move(auth_handler), attach_options.warehouse, - attach_options.endpoint); - catalog->GetConfig(context); - return std::move(catalog); - } - --static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, -+static unique_ptr CreateTransactionManager(optional_ptr storage_info, AttachedDatabase &db, - Catalog &catalog) { - auto &ic_catalog = catalog.Cast(); - return make_uniq(db, ic_catalog); -@@ -233,10 +231,12 @@ public: - } - }; - --static void LoadInternal(DatabaseInstance &instance) { -+static void LoadInternal(ExtensionLoader &loader) { - Aws::SDKOptions options; - Aws::InitAPI(options); // Should only be called once. - -+ auto &instance = loader.GetDatabaseInstance(); -+ - ExtensionHelper::AutoLoadExtension(instance, "parquet"); - if (!instance.ExtensionIsLoaded("parquet")) { - throw MissingExtensionException("The iceberg extension requires the parquet extension to be loaded!"); -@@ -250,13 +250,13 @@ static void LoadInternal(DatabaseInstance &instance) { - LogicalType::BOOLEAN, Value::BOOLEAN(false)); - - // Iceberg Table Functions -- for (auto &fun : IcebergFunctions::GetTableFunctions(instance)) { -- ExtensionUtil::RegisterFunction(instance, fun); -+ for (auto &fun : IcebergFunctions::GetTableFunctions(loader)) { -+ loader.RegisterFunction(fun); - } - - // Iceberg Scalar Functions - for (auto &fun : IcebergFunctions::GetScalarFunctions()) { -- ExtensionUtil::RegisterFunction(instance, fun); -+ loader.RegisterFunction(fun); - } - - SecretType secret_type; -@@ -264,16 +264,16 @@ static void LoadInternal(DatabaseInstance &instance) { - secret_type.deserializer = KeyValueSecret::Deserialize; - secret_type.default_provider = "config"; - -- ExtensionUtil::RegisterSecretType(instance, secret_type); -+ loader.RegisterSecretType(secret_type); - CreateSecretFunction secret_function = {"iceberg", "config", OAuth2Authorization::CreateCatalogSecretFunction}; - OAuth2Authorization::SetCatalogSecretParameters(secret_function); -- ExtensionUtil::RegisterFunction(instance, secret_function); -+ loader.RegisterFunction(secret_function); - - config.storage_extensions["iceberg"] = make_uniq(); - } - --void IcebergExtension::Load(DuckDB &db) { -- LoadInternal(*db.instance); -+void IcebergExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - std::string IcebergExtension::Name() { - return "iceberg"; -@@ -283,15 +283,8 @@ std::string IcebergExtension::Name() { - - extern "C" { - --DUCKDB_EXTENSION_API void iceberg_init(duckdb::DatabaseInstance &db) { -- LoadInternal(db); --} -- --DUCKDB_EXTENSION_API const char *iceberg_version() { -- return duckdb::DuckDB::LibraryVersion(); --} -+DUCKDB_CPP_EXTENSION_ENTRY(iceberg, loader) { -+ LoadInternal(loader); - } - --#ifndef DUCKDB_EXTENSION_MAIN --#error DUCKDB_EXTENSION_MAIN not defined --#endif -+} -\ No newline at end of file -diff --git a/src/iceberg_functions.cpp b/src/iceberg_functions.cpp -index d3ea52fb..fda85203 100644 ---- a/src/iceberg_functions.cpp -+++ b/src/iceberg_functions.cpp -@@ -7,11 +7,13 @@ - - namespace duckdb { - --vector IcebergFunctions::GetTableFunctions(DatabaseInstance &instance) { -+class ExtensionLoader; -+ -+vector IcebergFunctions::GetTableFunctions(ExtensionLoader &loader) { - vector functions; - - functions.push_back(GetIcebergSnapshotsFunction()); -- functions.push_back(GetIcebergScanFunction(instance)); -+ functions.push_back(GetIcebergScanFunction(loader)); - functions.push_back(GetIcebergMetadataFunction()); - - return functions; -diff --git a/src/iceberg_functions/iceberg_multi_file_reader.cpp b/src/iceberg_functions/iceberg_multi_file_reader.cpp -index 10978072..a0ec3f31 100644 ---- a/src/iceberg_functions/iceberg_multi_file_reader.cpp -+++ b/src/iceberg_functions/iceberg_multi_file_reader.cpp -@@ -7,7 +7,6 @@ - #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" - #include "duckdb/common/exception.hpp" - #include "duckdb/execution/execution_context.hpp" --#include "duckdb/main/extension_util.hpp" - #include "duckdb/parallel/thread_context.hpp" - #include "duckdb/parser/tableref/table_function_ref.hpp" - #include "duckdb/parser/expression/constant_expression.hpp" -@@ -532,7 +531,7 @@ unique_ptr IcebergMultiFileReader::CreateInstance(const TableFu - } - - shared_ptr IcebergMultiFileReader::CreateFileList(ClientContext &context, const vector &paths, -- FileGlobOptions options) { -+ const FileGlobInput &input) { - if (paths.size() != 1) { - throw BinderException("'iceberg_scan' only supports single path as input"); - } -@@ -919,7 +918,15 @@ void IcebergMultiFileList::ScanDeleteFile(const IcebergManifestEntry &entry, - auto &instance = DatabaseInstance::GetDatabase(context); - //! FIXME: delete files could also be made without row_ids, - //! in which case we need to rely on the `'schema.column-mapping.default'` property just like data files do. -- auto &parquet_scan_entry = ExtensionUtil::GetTableFunction(instance, "parquet_scan"); -+ -+ auto &system_catalog = Catalog::GetSystemCatalog(instance); -+ auto data = CatalogTransaction::GetSystemTransaction(instance); -+ auto &schema = system_catalog.GetSchema(data, DEFAULT_SCHEMA); -+ auto catalog_entry = schema.GetEntry(data, CatalogType::TABLE_FUNCTION_ENTRY, "parquet_scan"); -+ if (!catalog_entry) { -+ throw InvalidInputException("Function with name \"parquet_scan\" not found!"); -+ } -+ auto &parquet_scan_entry = catalog_entry->Cast(); - auto &parquet_scan = parquet_scan_entry.functions.functions[0]; - - // Prepare the inputs for the bind -diff --git a/src/iceberg_functions/iceberg_scan.cpp b/src/iceberg_functions/iceberg_scan.cpp -index 223c66f5..bdb80d4e 100644 ---- a/src/iceberg_functions/iceberg_scan.cpp -+++ b/src/iceberg_functions/iceberg_scan.cpp -@@ -19,7 +19,7 @@ - #include "duckdb/planner/operator/logical_comparison_join.hpp" - #include "duckdb/common/file_opener.hpp" - #include "duckdb/common/file_system.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "iceberg_metadata.hpp" - #include "iceberg_utils.hpp" - #include "iceberg_multi_file_reader.hpp" -@@ -43,11 +43,11 @@ static void AddNamedParameters(TableFunction &fun) { - fun.named_parameters["snapshot_from_id"] = LogicalType::UBIGINT; - } - --TableFunctionSet IcebergFunctions::GetIcebergScanFunction(DatabaseInstance &instance) { -+TableFunctionSet IcebergFunctions::GetIcebergScanFunction(ExtensionLoader &loader) { - // The iceberg_scan function is constructed by grabbing the parquet scan from the Catalog, then injecting the - // IcebergMultiFileReader into it to create a Iceberg-based multi file read - -- auto &parquet_scan = ExtensionUtil::GetTableFunction(instance, "parquet_scan"); -+ auto &parquet_scan = loader.GetTableFunction("parquet_scan"); - auto parquet_scan_copy = parquet_scan.functions; - - for (auto &function : parquet_scan_copy.functions) { -diff --git a/src/iceberg_manifest.cpp b/src/iceberg_manifest.cpp -index 74fabd72..06d4352e 100644 ---- a/src/iceberg_manifest.cpp -+++ b/src/iceberg_manifest.cpp -@@ -426,9 +426,17 @@ AvroScan::AvroScan(const string &scan_name, ClientContext &context, const string - auto &instance = DatabaseInstance::GetDatabase(context); - ExtensionHelper::AutoLoadExtension(instance, "avro"); - -- auto &avro_scan_entry = ExtensionUtil::GetTableFunction(instance, "read_avro"); -+ auto &system_catalog = Catalog::GetSystemCatalog(instance); -+ auto data = CatalogTransaction::GetSystemTransaction(instance); -+ auto &schema = system_catalog.GetSchema(data, DEFAULT_SCHEMA); -+ auto catalog_entry = schema.GetEntry(data, CatalogType::TABLE_FUNCTION_ENTRY, "read_avro"); -+ if (!catalog_entry) { -+ throw InvalidInputException("Function with name \"read_avro\" not found!"); -+ } -+ auto &avro_scan_entry = catalog_entry->Cast(); - avro_scan = avro_scan_entry.functions.functions[0]; - -+ - // Prepare the inputs for the bind - vector children; - children.reserve(1); -diff --git a/src/include/iceberg_extension.hpp b/src/include/iceberg_extension.hpp -index b0f51ceb..021aa0e6 100644 ---- a/src/include/iceberg_extension.hpp -+++ b/src/include/iceberg_extension.hpp -@@ -6,7 +6,7 @@ namespace duckdb { - - class IcebergExtension : public Extension { - public: -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &db) override; - std::string Name() override; - }; - -diff --git a/src/include/iceberg_functions.hpp b/src/include/iceberg_functions.hpp -index cd85a8dd..4becfa81 100644 ---- a/src/include/iceberg_functions.hpp -+++ b/src/include/iceberg_functions.hpp -@@ -13,15 +13,16 @@ - #include "duckdb/parser/parsed_data/create_table_function_info.hpp" - - namespace duckdb { -+class ExtensionLoader; - - class IcebergFunctions { - public: -- static vector GetTableFunctions(DatabaseInstance &instance); -+ static vector GetTableFunctions(ExtensionLoader &loader); - static vector GetScalarFunctions(); - - private: - static TableFunctionSet GetIcebergSnapshotsFunction(); -- static TableFunctionSet GetIcebergScanFunction(DatabaseInstance &instance); -+ static TableFunctionSet GetIcebergScanFunction(ExtensionLoader &instance); - static TableFunctionSet GetIcebergMetadataFunction(); - }; - -diff --git a/src/include/iceberg_manifest.hpp b/src/include/iceberg_manifest.hpp -index 2f50b6d3..20e0da35 100644 ---- a/src/include/iceberg_manifest.hpp -+++ b/src/include/iceberg_manifest.hpp -@@ -11,7 +11,7 @@ - #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" - #include "duckdb/common/exception.hpp" - #include "duckdb/execution/execution_context.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/parallel/thread_context.hpp" - #include "duckdb/parser/tableref/table_function_ref.hpp" - #include "duckdb/parser/expression/constant_expression.hpp" -diff --git a/src/include/iceberg_multi_file_reader.hpp b/src/include/iceberg_multi_file_reader.hpp -index d962306b..3fe3a5a1 100644 ---- a/src/include/iceberg_multi_file_reader.hpp -+++ b/src/include/iceberg_multi_file_reader.hpp -@@ -182,7 +182,7 @@ public: - static unique_ptr CreateInstance(const TableFunction &table); - //! Return a IcebergSnapshot - shared_ptr CreateFileList(ClientContext &context, const vector &paths, -- FileGlobOptions options) override; -+ const FileGlobInput &input) override; - - //! Override the regular parquet bind using the MultiFileReader Bind. The bind from these are what DuckDB's file - //! readers will try read -diff --git a/src/storage/irc_clear_cache.cpp b/src/storage/irc_clear_cache.cpp -index 4e1da4d8..168929b4 100644 ---- a/src/storage/irc_clear_cache.cpp -+++ b/src/storage/irc_clear_cache.cpp -@@ -23,7 +23,7 @@ static unique_ptr ClearCacheBind(ClientContext &context, TableFunc - static void ClearIRCCaches(ClientContext &context) { - auto databases = DatabaseManager::Get(context).GetDatabases(context); - for (auto &db_ref : databases) { -- auto &db = db_ref.get(); -+ auto &db = *db_ref; - auto &catalog = db.GetCatalog(); - if (catalog.GetCatalogType() != "iceberg") { - continue; -diff --git a/src/storage/irc_table_entry.cpp b/src/storage/irc_table_entry.cpp -index f3a78161..e6066dea 100644 ---- a/src/storage/irc_table_entry.cpp -+++ b/src/storage/irc_table_entry.cpp -@@ -3,7 +3,7 @@ - #include "storage/irc_table_entry.hpp" - #include "duckdb/storage/statistics/base_statistics.hpp" - #include "duckdb/storage/table_storage_info.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/main/database.hpp" - #include "duckdb/main/secret/secret_manager.hpp" - #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" -@@ -123,7 +123,15 @@ string ICTableEntry::PrepareIcebergScanFromEntry(ClientContext &context) { - TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data, - const EntryLookupInfo &lookup) { - auto &db = DatabaseInstance::GetDatabase(context); -- auto &iceberg_scan_function_set = ExtensionUtil::GetTableFunction(db, "iceberg_scan"); -+ -+ auto &system_catalog = Catalog::GetSystemCatalog(db); -+ auto data = CatalogTransaction::GetSystemTransaction(db); -+ auto &schema = system_catalog.GetSchema(data, DEFAULT_SCHEMA); -+ auto catalog_entry = schema.GetEntry(data, CatalogType::TABLE_FUNCTION_ENTRY, "iceberg_scan"); -+ if (!catalog_entry) { -+ throw InvalidInputException("Function with name \"iceberg_scan\" not found!"); -+ } -+ auto &iceberg_scan_function_set = catalog_entry->Cast(); - auto iceberg_scan_function = - iceberg_scan_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); - auto storage_location = PrepareIcebergScanFromEntry(context); -diff --git a/test/sql/local/iceberg_scans/filtering_on_bounds.test b/test/sql/local/iceberg_scans/filtering_on_bounds.test -index e9a862ee..3248e621 100644 ---- a/test/sql/local/iceberg_scans/filtering_on_bounds.test -+++ b/test/sql/local/iceberg_scans/filtering_on_bounds.test -@@ -19,7 +19,7 @@ statement ok - create view my_datalake.default.filtering_on_bounds as select * from ICEBERG_SCAN('data/generated/iceberg/spark-local/default/filtering_on_bounds'); - - statement ok --pragma enable_logging=true; -+CALL enable_logging(); - - statement ok - set enable_logging=false; -@@ -34,7 +34,7 @@ select count(*) from my_datalake.default.filtering_on_bounds; - 5000 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select count(*) from my_datalake.default.filtering_on_bounds where col1 > 500; -@@ -56,7 +56,7 @@ ON logs.msg = meta.file_path; - 500 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select count(*) from my_datalake.default.filtering_on_bounds where col1 > 1500; -@@ -77,7 +77,7 @@ ON logs.msg = meta.file_path; - 1500 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select count(*) from my_datalake.default.filtering_on_bounds where col1 >= 2300 and col1 < 3500; -diff --git a/test/sql/local/iceberg_scans/filtering_on_partition_bounds.test b/test/sql/local/iceberg_scans/filtering_on_partition_bounds.test -index ca105539..07ec4457 100644 ---- a/test/sql/local/iceberg_scans/filtering_on_partition_bounds.test -+++ b/test/sql/local/iceberg_scans/filtering_on_partition_bounds.test -@@ -19,7 +19,7 @@ statement ok - create view my_datalake.default.filtering_on_partition_bounds as select * from ICEBERG_SCAN('data/generated/iceberg/spark-local/default/filtering_on_partition_bounds'); - - statement ok --pragma enable_logging=true; -+CALL enable_logging(); - - statement ok - set enable_logging=false; -@@ -34,7 +34,7 @@ select count(*) from my_datalake.default.filtering_on_partition_bounds; - 5000 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select count(*) from my_datalake.default.filtering_on_partition_bounds where seq = 1 -@@ -56,7 +56,7 @@ ON logs.msg = meta.manifest_path; - 4000 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select count(*) from my_datalake.default.filtering_on_partition_bounds where seq > 1 and seq < 4; -@@ -77,7 +77,7 @@ ON logs.msg = meta.manifest_path; - 3000 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select count(*) from my_datalake.default.filtering_on_partition_bounds where seq >= 2 and seq <= 3; -diff --git a/test/sql/local/partitioning/year/year_timestamp.test b/test/sql/local/partitioning/year/year_timestamp.test -index 8dccd449..e006226d 100644 ---- a/test/sql/local/partitioning/year/year_timestamp.test -+++ b/test/sql/local/partitioning/year/year_timestamp.test -@@ -12,7 +12,7 @@ require httpfs - require-env DUCKDB_ICEBERG_HAVE_GENERATED_DATA - - statement ok --pragma enable_logging=true; -+CALL enable_logging(); - - statement ok - set enable_logging=false; -@@ -31,7 +31,7 @@ select * from ICEBERG_SCAN('data/generated/iceberg/spark-local/default/year_time - NULL 99999 null_event - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select user_id from ICEBERG_SCAN('data/generated/iceberg/spark-local/default/year_timestamp') -@@ -65,7 +65,7 @@ ON logs.msg = meta.manifest_path; - 5 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select user_id from ICEBERG_SCAN('data/generated/iceberg/spark-local/default/year_timestamp') -@@ -92,7 +92,7 @@ ON logs.msg = meta.manifest_path; - 1 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - # Test filtering by year - query I -@@ -115,7 +115,7 @@ ON logs.msg = meta.manifest_path; - 2 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select user_id from ICEBERG_SCAN('data/generated/iceberg/spark-local/default/year_timestamp') -@@ -137,7 +137,7 @@ ON logs.msg = meta.manifest_path; - 2 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - query I - select user_id from ICEBERG_SCAN('data/generated/iceberg/spark-local/default/year_timestamp') -@@ -159,7 +159,7 @@ ON logs.msg = meta.manifest_path; - 2 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); - - # Test filtering by timestamp range - query I -@@ -182,4 +182,4 @@ ON logs.msg = meta.manifest_path; - 2 - - statement ok --pragma truncate_duckdb_logs; -+CALL truncate_duckdb_logs(); diff --git a/.github/patches/extensions/iceberg/remove_include.patch b/.github/patches/extensions/iceberg/remove_include.patch new file mode 100644 index 000000000000..243e37b245b3 --- /dev/null +++ b/.github/patches/extensions/iceberg/remove_include.patch @@ -0,0 +1,12 @@ +diff --git a/src/storage/iceberg_insert.cpp b/src/storage/iceberg_insert.cpp +index aa2371e8..cccc82d6 100644 +--- a/src/storage/iceberg_insert.cpp ++++ b/src/storage/iceberg_insert.cpp +@@ -7,7 +7,6 @@ + + #include "iceberg_multi_file_list.hpp" + +-#include "duckdb/common/sort/partition_state.hpp" + #include "duckdb/catalog/catalog_entry/copy_function_catalog_entry.hpp" + #include "duckdb/main/client_data.hpp" + #include "duckdb/planner/operator/logical_copy_to_file.hpp" diff --git a/.github/patches/extensions/inet/hugeint_fixes.patch b/.github/patches/extensions/inet/hugeint_fixes.patch new file mode 100644 index 000000000000..4b4375d116d7 --- /dev/null +++ b/.github/patches/extensions/inet/hugeint_fixes.patch @@ -0,0 +1,19 @@ +diff --git a/src/inet_functions.cpp b/src/inet_functions.cpp +index da92a4c..afa7446 100644 +--- a/src/inet_functions.cpp ++++ b/src/inet_functions.cpp +@@ -185,11 +185,12 @@ static INET_TYPE AddImplementation(INET_TYPE ip, hugeint_t val) { + if (val > 0) { + address_out = + AddOperatorOverflowCheck::Operation( +- address_in, val); ++ address_in, (uhugeint_t)val); + } else { ++ // TODO: this is off for when val is the minimal uhugeint_t value + address_out = + SubtractOperatorOverflowCheck::Operation(address_in, -val); ++ uhugeint_t>(address_in, (uhugeint_t)(-val)); + } + + if (addr_type == IPAddressType::IP_ADDRESS_V4 && diff --git a/.github/patches/extensions/inet/inet_loader.patch b/.github/patches/extensions/inet/inet_loader.patch deleted file mode 100644 index 39d582081cb5..000000000000 --- a/.github/patches/extensions/inet/inet_loader.patch +++ /dev/null @@ -1,157 +0,0 @@ -diff --git a/src/include/inet_extension.hpp b/src/include/inet_extension.hpp -index bb9b18b..fbd22d6 100644 ---- a/src/include/inet_extension.hpp -+++ b/src/include/inet_extension.hpp -@@ -15,13 +15,12 @@ namespace duckdb { - - class InetExtension : public Extension { - public: -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &loader) override; - std::string Name() override; - std::string Version() const override; - --protected: -- ScalarFunctionSet GetEscapeFunctionSet(); -- ScalarFunction GetUnescapeFunction(); -+ static ScalarFunctionSet GetEscapeFunctionSet(); -+ static ScalarFunction GetUnescapeFunction(); - }; - - } // namespace duckdb -diff --git a/src/inet_escape_functions.cpp b/src/inet_escape_functions.cpp -index 0be7481..1503440 100644 ---- a/src/inet_escape_functions.cpp -+++ b/src/inet_escape_functions.cpp -@@ -1,5 +1,5 @@ - #include "duckdb/planner/expression/bound_function_expression.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/common/types/blob.hpp" - #include "utf8proc_wrapper.hpp" - #include "inet_extension.hpp" -diff --git a/src/inet_extension.cpp b/src/inet_extension.cpp -index 4eb3a86..ac0f107 100644 ---- a/src/inet_extension.cpp -+++ b/src/inet_extension.cpp -@@ -1,10 +1,8 @@ --#define DUCKDB_EXTENSION_MAIN -- - #include "duckdb.hpp" - #include "duckdb/common/exception.hpp" - #include "duckdb/common/string_util.hpp" - #include "duckdb/common/pair.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/function/scalar_function.hpp" - #include "duckdb/parser/parsed_data/create_type_info.hpp" - #include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" -@@ -17,7 +15,7 @@ namespace duckdb { - - static constexpr auto INET_TYPE_NAME = "INET"; - --void InetExtension::Load(DuckDB &db) { -+static void LoadInternal(ExtensionLoader &loader) { - // add the "inet" type - child_list_t children; - children.push_back(make_pair("ip_type", LogicalType::UTINYINT)); -@@ -28,55 +26,55 @@ void InetExtension::Load(DuckDB &db) { - children.push_back(make_pair("mask", LogicalType::USMALLINT)); - auto inet_type = LogicalType::STRUCT(std::move(children)); - inet_type.SetAlias(INET_TYPE_NAME); -- ExtensionUtil::RegisterType(*db.instance, INET_TYPE_NAME, inet_type); -+ loader.RegisterType(INET_TYPE_NAME, inet_type); - - // add the casts to and from INET type -- ExtensionUtil::RegisterCastFunction(*db.instance, LogicalType::VARCHAR, -- inet_type, -- INetFunctions::CastVarcharToINET); -- ExtensionUtil::RegisterCastFunction(*db.instance, inet_type, -- LogicalType::VARCHAR, -- INetFunctions::CastINETToVarchar); -+ loader.RegisterCastFunction(LogicalType::VARCHAR, inet_type, -+ INetFunctions::CastVarcharToINET); -+ loader.RegisterCastFunction(inet_type, LogicalType::VARCHAR, -+ INetFunctions::CastINETToVarchar); - - // add inet functions -- ExtensionUtil::RegisterFunction( -- *db.instance, ScalarFunction("host", {inet_type}, LogicalType::VARCHAR, -+ loader.RegisterFunction( -+ ScalarFunction("host", {inet_type}, LogicalType::VARCHAR, - INetFunctions::Host)); -- ExtensionUtil::RegisterFunction( -- *db.instance, ScalarFunction("family", {inet_type}, LogicalType::UTINYINT, -+ loader.RegisterFunction( -+ ScalarFunction("family", {inet_type}, LogicalType::UTINYINT, - INetFunctions::Family)); -- ExtensionUtil::RegisterFunction( -- *db.instance, ScalarFunction("netmask", {inet_type}, {inet_type}, -+ loader.RegisterFunction( -+ ScalarFunction("netmask", {inet_type}, {inet_type}, - INetFunctions::Netmask)); -- ExtensionUtil::RegisterFunction( -- *db.instance, ScalarFunction("network", {inet_type}, {inet_type}, -+ loader.RegisterFunction( -+ ScalarFunction("network", {inet_type}, {inet_type}, - INetFunctions::Network)); -- ExtensionUtil::RegisterFunction( -- *db.instance, ScalarFunction("broadcast", {inet_type}, {inet_type}, -+ loader.RegisterFunction( -+ ScalarFunction("broadcast", {inet_type}, {inet_type}, - INetFunctions::Broadcast)); -- ExtensionUtil::RegisterFunction(*db.instance, GetEscapeFunctionSet()); -- ExtensionUtil::RegisterFunction(*db.instance, GetUnescapeFunction()); -+ loader.RegisterFunction(InetExtension::GetEscapeFunctionSet()); -+ loader.RegisterFunction(InetExtension::GetUnescapeFunction()); - - // Add - function with ALTER_ON_CONFLICT - ScalarFunction substract_fun("-", {inet_type, LogicalType::HUGEINT}, - inet_type, INetFunctions::Subtract); -- ExtensionUtil::AddFunctionOverload(*db.instance, substract_fun); -+ loader.AddFunctionOverload(substract_fun); - - ScalarFunction add_fun("+", {inet_type, LogicalType::HUGEINT}, inet_type, - INetFunctions::Add); -- ExtensionUtil::AddFunctionOverload(*db.instance, add_fun); -+ loader.AddFunctionOverload(add_fun); - - // Add IP range operators -- ExtensionUtil::RegisterFunction(*db.instance, -- ScalarFunction("<<=", {inet_type, inet_type}, -+ loader.RegisterFunction(ScalarFunction("<<=", {inet_type, inet_type}, - LogicalType::BOOLEAN, - INetFunctions::ContainsLeft)); -- ExtensionUtil::RegisterFunction(*db.instance, -- ScalarFunction(">>=", {inet_type, inet_type}, -+ loader.RegisterFunction(ScalarFunction(">>=", {inet_type, inet_type}, - LogicalType::BOOLEAN, - INetFunctions::ContainsRight)); - } - -+void InetExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); -+} -+ - std::string InetExtension::Name() { return "inet"; } - - std::string InetExtension::Version() const { -@@ -91,16 +89,8 @@ std::string InetExtension::Version() const { - - extern "C" { - --DUCKDB_EXTENSION_API void inet_init(duckdb::DatabaseInstance &db) { -- duckdb::DuckDB db_wrapper(db); -- db_wrapper.LoadExtension(); -+DUCKDB_CPP_EXTENSION_ENTRY(inet, loader) { -+ duckdb::LoadInternal(loader); - } - --DUCKDB_EXTENSION_API const char *inet_version() { -- return duckdb::DuckDB::LibraryVersion(); - } --} -- --#ifndef DUCKDB_EXTENSION_MAIN --#error DUCKDB_EXTENSION_MAIN not defined --#endif diff --git a/.github/patches/extensions/mysql_scanner/fix.patch b/.github/patches/extensions/mysql_scanner/fix.patch deleted file mode 100644 index 841b9a374cf9..000000000000 --- a/.github/patches/extensions/mysql_scanner/fix.patch +++ /dev/null @@ -1,264 +0,0 @@ -diff --git a/src/include/mysql_scanner_extension.hpp b/src/include/mysql_scanner_extension.hpp -index dc69b08..ba2c8a7 100644 ---- a/src/include/mysql_scanner_extension.hpp -+++ b/src/include/mysql_scanner_extension.hpp -@@ -13,11 +13,9 @@ public: - std::string Name() override { - return "mysql_scanner"; - } -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &loader) override; - }; - - extern "C" { --DUCKDB_EXTENSION_API void mysql_scanner_init(duckdb::DatabaseInstance &db); --DUCKDB_EXTENSION_API const char *mysql_scanner_version(); --DUCKDB_EXTENSION_API void mysql_scanner_storage_init(DBConfig &config); -+ DUCKDB_CPP_EXTENSION_ENTRY(mysql_scanner, loader); - } -diff --git a/src/include/storage/mysql_execute_query.hpp b/src/include/storage/mysql_execute_query.hpp -index f9e02cd..848deb3 100644 ---- a/src/include/storage/mysql_execute_query.hpp -+++ b/src/include/storage/mysql_execute_query.hpp -@@ -15,7 +15,7 @@ class TableCatalogEntry; - - class MySQLExecuteQuery : public PhysicalOperator { - public: -- MySQLExecuteQuery(LogicalOperator &op, string op_name, TableCatalogEntry &table, string query); -+ MySQLExecuteQuery(PhysicalPlan &physical_plan, LogicalOperator &op, string op_name, TableCatalogEntry &table, string query); - - //! The table to delete from - string op_name; -diff --git a/src/include/storage/mysql_index.hpp b/src/include/storage/mysql_index.hpp -index fbc4836..a674069 100644 ---- a/src/include/storage/mysql_index.hpp -+++ b/src/include/storage/mysql_index.hpp -@@ -16,7 +16,7 @@ namespace duckdb { - //! PhysicalCreateSequence represents a CREATE SEQUENCE command - class MySQLCreateIndex : public PhysicalOperator { - public: -- explicit MySQLCreateIndex(unique_ptr info, TableCatalogEntry &table); -+ MySQLCreateIndex(PhysicalPlan &physical_plan, unique_ptr info, TableCatalogEntry &table); - - unique_ptr info; - TableCatalogEntry &table; -diff --git a/src/include/storage/mysql_insert.hpp b/src/include/storage/mysql_insert.hpp -index 3c3213d..45c9ba7 100644 ---- a/src/include/storage/mysql_insert.hpp -+++ b/src/include/storage/mysql_insert.hpp -@@ -16,9 +16,9 @@ namespace duckdb { - class MySQLInsert : public PhysicalOperator { - public: - //! INSERT INTO -- MySQLInsert(LogicalOperator &op, TableCatalogEntry &table, physical_index_vector_t column_index_map); -+ MySQLInsert(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, physical_index_vector_t column_index_map); - //! CREATE TABLE AS -- MySQLInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info); -+ MySQLInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info); - - //! The table to insert into - optional_ptr table; -diff --git a/src/mysql_extension.cpp b/src/mysql_extension.cpp -index 9d51436..b2cfa8a 100644 ---- a/src/mysql_extension.cpp -+++ b/src/mysql_extension.cpp -@@ -7,7 +7,7 @@ - - #include "duckdb/catalog/catalog.hpp" - #include "duckdb/parser/parsed_data/create_table_function_info.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/main/database_manager.hpp" - #include "duckdb/main/attached_database.hpp" - #include "storage/mysql_catalog.hpp" -@@ -83,27 +83,29 @@ void SetMySQLSecretParameters(CreateSecretFunction &function) { - function.named_parameters["ssl_key"] = LogicalType::VARCHAR; - } - --static void LoadInternal(DatabaseInstance &db) { -+static void LoadInternal(ExtensionLoader &loader) { - mysql_library_init(0, NULL, NULL); - MySQLClearCacheFunction clear_cache_func; -- ExtensionUtil::RegisterFunction(db, clear_cache_func); -+ loader.RegisterFunction(clear_cache_func); - - MySQLExecuteFunction execute_function; -- ExtensionUtil::RegisterFunction(db, execute_function); -+ loader.RegisterFunction(execute_function); - - MySQLQueryFunction query_function; -- ExtensionUtil::RegisterFunction(db, query_function); -+ loader.RegisterFunction(query_function); - - SecretType secret_type; - secret_type.name = "mysql"; - secret_type.deserializer = KeyValueSecret::Deserialize; - secret_type.default_provider = "config"; - -- ExtensionUtil::RegisterSecretType(db, secret_type); -+ loader.RegisterSecretType(secret_type); - - CreateSecretFunction mysql_secret_function = {"mysql", "config", CreateMySQLSecretFunction}; - SetMySQLSecretParameters(mysql_secret_function); -- ExtensionUtil::RegisterFunction(db, mysql_secret_function); -+ loader.RegisterFunction(mysql_secret_function); -+ -+ auto &db = loader.GetDatabaseInstance(); - - auto &config = DBConfig::GetConfig(db); - config.storage_extensions["mysql_scanner"] = make_uniq(); -@@ -123,22 +125,14 @@ static void LoadInternal(DatabaseInstance &db) { - config.optimizer_extensions.push_back(std::move(mysql_optimizer)); - } - --void MysqlScannerExtension::Load(DuckDB &db) { -- LoadInternal(*db.instance); -+void MysqlScannerExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - - extern "C" { - --DUCKDB_EXTENSION_API void mysql_scanner_init(duckdb::DatabaseInstance &db) { -- LoadInternal(db); --} -- --DUCKDB_EXTENSION_API const char *mysql_scanner_version() { -- return DuckDB::LibraryVersion(); -+DUCKDB_CPP_EXTENSION_ENTRY(mysql_scanner, loader) { -+ LoadInternal(loader); - } - --DUCKDB_EXTENSION_API void mysql_scanner_storage_init(DBConfig &config) { -- mysql_library_init(0, NULL, NULL); -- config.storage_extensions["mysql_scanner"] = make_uniq(); --} - } -diff --git a/src/mysql_scanner.cpp b/src/mysql_scanner.cpp -index a239dfb..46df8b7 100644 ---- a/src/mysql_scanner.cpp -+++ b/src/mysql_scanner.cpp -@@ -1,6 +1,6 @@ - #include "duckdb.hpp" - --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/parser/parsed_data/create_table_function_info.hpp" - #include "mysql_scanner.hpp" - #include "mysql_result.hpp" -diff --git a/src/mysql_storage.cpp b/src/mysql_storage.cpp -index 7ee6fe6..fde8d08 100644 ---- a/src/mysql_storage.cpp -+++ b/src/mysql_storage.cpp -@@ -7,19 +7,17 @@ - - namespace duckdb { - --static unique_ptr MySQLAttach(StorageExtensionInfo *storage_info, ClientContext &context, AttachedDatabase &db, -- const string &name, AttachInfo &info, AccessMode access_mode) { -+static unique_ptr MySQLAttach(optional_ptr storage_info, ClientContext &context, AttachedDatabase &db, -+ const string &name, AttachInfo &info, AttachOptions &attach_options) { - auto &config = DBConfig::GetConfig(context); - if (!config.options.enable_external_access) { - throw PermissionException("Attaching MySQL databases is disabled through configuration"); - } - // check if we have a secret provided - string secret_name; -- for (auto &entry : info.options) { -+ for (auto &entry : attach_options.options) { - auto lower_name = StringUtil::Lower(entry.first); -- if (lower_name == "type" || lower_name == "read_only") { -- // already handled -- } else if (lower_name == "secret") { -+ if (lower_name == "secret") { - secret_name = entry.second.ToString(); - } else { - throw BinderException("Unrecognized option for MySQL attach: %s", entry.first); -@@ -28,10 +26,10 @@ static unique_ptr MySQLAttach(StorageExtensionInfo *storage_info, Clien - - string attach_path = info.path; - auto connection_string = MySQLCatalog::GetConnectionString(context, attach_path, secret_name); -- return make_uniq(db, std::move(connection_string), std::move(attach_path), access_mode); -+ return make_uniq(db, std::move(connection_string), std::move(attach_path), attach_options.access_mode); - } - --static unique_ptr MySQLCreateTransactionManager(StorageExtensionInfo *storage_info, -+static unique_ptr MySQLCreateTransactionManager(optional_ptr storage_info, - AttachedDatabase &db, Catalog &catalog) { - auto &mysql_catalog = catalog.Cast(); - return make_uniq(db, mysql_catalog); -diff --git a/src/storage/mysql_clear_cache.cpp b/src/storage/mysql_clear_cache.cpp -index bf288c0..882c1c2 100644 ---- a/src/storage/mysql_clear_cache.cpp -+++ b/src/storage/mysql_clear_cache.cpp -@@ -24,7 +24,7 @@ static unique_ptr ClearCacheBind(ClientContext &context, TableFunc - static void ClearMySQLCaches(ClientContext &context) { - auto databases = DatabaseManager::Get(context).GetDatabases(context); - for (auto &db_ref : databases) { -- auto &db = db_ref.get(); -+ auto &db = *db_ref; - auto &catalog = db.GetCatalog(); - if (catalog.GetCatalogType() != "mysql") { - continue; -diff --git a/src/storage/mysql_execute_query.cpp b/src/storage/mysql_execute_query.cpp -index 8bf9426..aaeb3c7 100644 ---- a/src/storage/mysql_execute_query.cpp -+++ b/src/storage/mysql_execute_query.cpp -@@ -12,8 +12,8 @@ - - namespace duckdb { - --MySQLExecuteQuery::MySQLExecuteQuery(LogicalOperator &op, string op_name_p, TableCatalogEntry &table, string query_p) -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, op.types, 1), op_name(std::move(op_name_p)), table(table), -+MySQLExecuteQuery::MySQLExecuteQuery(PhysicalPlan &physical_plan, LogicalOperator &op, string op_name_p, TableCatalogEntry &table, string query_p) -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), op_name(std::move(op_name_p)), table(table), - query(std::move(query_p)) { - } - -diff --git a/src/storage/mysql_index.cpp b/src/storage/mysql_index.cpp -index 587cdf2..ce95653 100644 ---- a/src/storage/mysql_index.cpp -+++ b/src/storage/mysql_index.cpp -@@ -6,8 +6,8 @@ - - namespace duckdb { - --MySQLCreateIndex::MySQLCreateIndex(unique_ptr info, TableCatalogEntry &table) -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, {LogicalType::BIGINT}, 1), info(std::move(info)), table(table) { -+MySQLCreateIndex::MySQLCreateIndex(PhysicalPlan &physical_plan, unique_ptr info, TableCatalogEntry &table) -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, {LogicalType::BIGINT}, 1), info(std::move(info)), table(table) { - } - - //===--------------------------------------------------------------------===// -diff --git a/src/storage/mysql_insert.cpp b/src/storage/mysql_insert.cpp -index 121d85a..395111b 100644 ---- a/src/storage/mysql_insert.cpp -+++ b/src/storage/mysql_insert.cpp -@@ -14,14 +14,14 @@ - - namespace duckdb { - --MySQLInsert::MySQLInsert(LogicalOperator &op, TableCatalogEntry &table, -+MySQLInsert::MySQLInsert(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, - physical_index_vector_t column_index_map_p) -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, op.types, 1), table(&table), schema(nullptr), -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(&table), schema(nullptr), - column_index_map(std::move(column_index_map_p)) { - } - --MySQLInsert::MySQLInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info) -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, op.types, 1), table(nullptr), schema(&schema), -+MySQLInsert::MySQLInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info) -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(nullptr), schema(&schema), - info(std::move(info)) { - } - -@@ -307,7 +307,7 @@ PhysicalOperator &MySQLCatalog::PlanInsert(ClientContext &context, PhysicalPlanG - if (op.return_chunk) { - throw BinderException("RETURNING clause not yet supported for insertion into MySQL table"); - } -- if (op.action_type != OnConflictAction::THROW) { -+ if (op.on_conflict_info.action_type != OnConflictAction::THROW) { - throw BinderException("ON CONFLICT clause not yet supported for insertion into MySQL table"); - } - diff --git a/.github/patches/extensions/postgres_scanner/fix.patch b/.github/patches/extensions/postgres_scanner/fix.patch deleted file mode 100644 index bc87def1dda6..000000000000 --- a/.github/patches/extensions/postgres_scanner/fix.patch +++ /dev/null @@ -1,39 +0,0 @@ -diff --git a/src/postgres_extension.cpp b/src/postgres_extension.cpp -index a12f431..b0591d5 100644 ---- a/src/postgres_extension.cpp -+++ b/src/postgres_extension.cpp -@@ -58,7 +58,7 @@ static void SetPostgresConnectionLimit(ClientContext &context, SetScope scope, V - } - auto databases = DatabaseManager::Get(context).GetDatabases(context); - for (auto &db_ref : databases) { -- auto &db = db_ref.get(); -+ auto &db = *db_ref; - auto &catalog = db.GetCatalog(); - if (catalog.GetCatalogType() != "postgres") { - continue; -diff --git a/src/storage/postgres_clear_cache.cpp b/src/storage/postgres_clear_cache.cpp -index 3e16e48..878cf77 100644 ---- a/src/storage/postgres_clear_cache.cpp -+++ b/src/storage/postgres_clear_cache.cpp -@@ -24,7 +24,7 @@ static unique_ptr ClearCacheBind(ClientContext &context, TableFunc - void PostgresClearCacheFunction::ClearPostgresCaches(ClientContext &context) { - auto databases = DatabaseManager::Get(context).GetDatabases(context); - for (auto &db_ref : databases) { -- auto &db = db_ref.get(); -+ auto &db = *db_ref; - auto &catalog = db.GetCatalog(); - if (catalog.GetCatalogType() != "postgres") { - continue; -diff --git a/src/storage/postgres_insert.cpp b/src/storage/postgres_insert.cpp -index fcd7109..77a3372 100644 ---- a/src/storage/postgres_insert.cpp -+++ b/src/storage/postgres_insert.cpp -@@ -218,7 +218,7 @@ PhysicalOperator &PostgresCatalog::PlanInsert(ClientContext &context, PhysicalPl - if (op.return_chunk) { - throw BinderException("RETURNING clause not yet supported for insertion into Postgres table"); - } -- if (op.action_type != OnConflictAction::THROW) { -+ if (op.on_conflict_info.action_type != OnConflictAction::THROW) { - throw BinderException("ON CONFLICT clause not yet supported for insertion into Postgres table"); - } - diff --git a/.github/patches/extensions/spatial/fix.patch b/.github/patches/extensions/spatial/fix.patch new file mode 100644 index 000000000000..26a4b8ddf223 --- /dev/null +++ b/.github/patches/extensions/spatial/fix.patch @@ -0,0 +1,16 @@ +diff --git a/src/spatial/modules/main/spatial_functions_scalar.cpp b/src/spatial/modules/main/spatial_functions_scalar.cpp +index 60ca7373ce..a44cfc7a82 100644 +--- a/src/spatial/modules/main/spatial_functions_scalar.cpp ++++ b/src/spatial/modules/main/spatial_functions_scalar.cpp +@@ -9243,6 +9243,11 @@ struct ST_MMin : VertexAggFunctionBase { + static constexpr auto ORDINATE = VertexOrdinate::M; + }; + ++constexpr const char * ST_M::NAME; ++constexpr const char * ST_X::NAME; ++constexpr const char * ST_Y::NAME; ++constexpr const char * ST_Z::NAME; ++ + } // namespace + + // Helper to access the constant distance from the bind data diff --git a/.github/patches/extensions/sqlite_scanner/fix.patch b/.github/patches/extensions/sqlite_scanner/fix.patch deleted file mode 100644 index 383a653ec404..000000000000 --- a/.github/patches/extensions/sqlite_scanner/fix.patch +++ /dev/null @@ -1,263 +0,0 @@ -diff --git a/src/include/sqlite_scanner_extension.hpp b/src/include/sqlite_scanner_extension.hpp -index 40dfb9a..d290788 100644 ---- a/src/include/sqlite_scanner_extension.hpp -+++ b/src/include/sqlite_scanner_extension.hpp -@@ -13,11 +13,9 @@ public: - std::string Name() override { - return "sqlite_scanner"; - } -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &loader) override; - }; - - extern "C" { --DUCKDB_EXTENSION_API void sqlite_scanner_init(duckdb::DatabaseInstance &db); --DUCKDB_EXTENSION_API const char *sqlite_scanner_version(); --DUCKDB_EXTENSION_API void sqlite_scanner_storage_init(DBConfig &config); -+ DUCKDB_CPP_EXTENSION_ENTRY(sqlite_scanner, loader); - } -\ No newline at end of file -diff --git a/src/include/storage/sqlite_delete.hpp b/src/include/storage/sqlite_delete.hpp -index 3c07df3..7315ebe 100644 ---- a/src/include/storage/sqlite_delete.hpp -+++ b/src/include/storage/sqlite_delete.hpp -@@ -14,7 +14,7 @@ namespace duckdb { - - class SQLiteDelete : public PhysicalOperator { - public: -- SQLiteDelete(LogicalOperator &op, TableCatalogEntry &table, idx_t row_id_index); -+ SQLiteDelete(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, idx_t row_id_index); - - //! The table to delete from - TableCatalogEntry &table; -diff --git a/src/include/storage/sqlite_index.hpp b/src/include/storage/sqlite_index.hpp -index 4400497..f60a33a 100644 ---- a/src/include/storage/sqlite_index.hpp -+++ b/src/include/storage/sqlite_index.hpp -@@ -16,7 +16,7 @@ namespace duckdb { - //! PhysicalCreateSequence represents a CREATE SEQUENCE command - class SQLiteCreateIndex : public PhysicalOperator { - public: -- explicit SQLiteCreateIndex(unique_ptr info, TableCatalogEntry &table); -+ SQLiteCreateIndex(PhysicalPlan &physical_plan, unique_ptr info, TableCatalogEntry &table); - - unique_ptr info; - TableCatalogEntry &table; -diff --git a/src/include/storage/sqlite_insert.hpp b/src/include/storage/sqlite_insert.hpp -index 65ec357..2a5d181 100644 ---- a/src/include/storage/sqlite_insert.hpp -+++ b/src/include/storage/sqlite_insert.hpp -@@ -16,9 +16,9 @@ namespace duckdb { - class SQLiteInsert : public PhysicalOperator { - public: - //! INSERT INTO -- SQLiteInsert(LogicalOperator &op, TableCatalogEntry &table, physical_index_vector_t column_index_map); -+ SQLiteInsert(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, physical_index_vector_t column_index_map); - //! CREATE TABLE AS -- SQLiteInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info); -+ SQLiteInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info); - - //! The table to insert into - optional_ptr table; -diff --git a/src/include/storage/sqlite_update.hpp b/src/include/storage/sqlite_update.hpp -index 0061620..92ed76e 100644 ---- a/src/include/storage/sqlite_update.hpp -+++ b/src/include/storage/sqlite_update.hpp -@@ -15,7 +15,7 @@ namespace duckdb { - - class SQLiteUpdate : public PhysicalOperator { - public: -- SQLiteUpdate(LogicalOperator &op, TableCatalogEntry &table, vector columns); -+ SQLiteUpdate(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, vector columns); - - //! The table to delete from - TableCatalogEntry &table; -diff --git a/src/sqlite_extension.cpp b/src/sqlite_extension.cpp -index f85328f..c2f7c81 100644 ---- a/src/sqlite_extension.cpp -+++ b/src/sqlite_extension.cpp -@@ -9,7 +9,7 @@ - #include "sqlite_scanner_extension.hpp" - - #include "duckdb/catalog/catalog.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/parser/parsed_data/create_table_function_info.hpp" - - using namespace duckdb; -@@ -20,16 +20,17 @@ static void SetSqliteDebugQueryPrint(ClientContext &context, SetScope scope, Val - SQLiteDB::DebugSetPrintQueries(BooleanValue::Get(parameter)); - } - --static void LoadInternal(DatabaseInstance &db) { -+static void LoadInternal(ExtensionLoader &loader) { - SqliteScanFunction sqlite_fun; -- ExtensionUtil::RegisterFunction(db, sqlite_fun); -+ loader.RegisterFunction(sqlite_fun); - - SqliteAttachFunction attach_func; -- ExtensionUtil::RegisterFunction(db, attach_func); -+ loader.RegisterFunction(attach_func); - - SQLiteQueryFunction query_func; -- ExtensionUtil::RegisterFunction(db, query_func); -+ loader.RegisterFunction(query_func); - -+ auto &db = loader.GetDatabaseInstance(); - auto &config = DBConfig::GetConfig(db); - config.AddExtensionOption("sqlite_all_varchar", "Load all SQLite columns as VARCHAR columns", LogicalType::BOOLEAN); - -@@ -39,19 +40,12 @@ static void LoadInternal(DatabaseInstance &db) { - config.storage_extensions["sqlite_scanner"] = make_uniq(); - } - --void SqliteScannerExtension::Load(DuckDB &db) { -- LoadInternal(*db.instance); -+void SqliteScannerExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - --DUCKDB_EXTENSION_API void sqlite_scanner_init(duckdb::DatabaseInstance &db) { -- LoadInternal(db); -+DUCKDB_CPP_EXTENSION_ENTRY(sqlite_scanner, loader) { -+ LoadInternal(loader); - } - --DUCKDB_EXTENSION_API const char *sqlite_scanner_version() { -- return DuckDB::LibraryVersion(); --} -- --DUCKDB_EXTENSION_API void sqlite_scanner_storage_init(DBConfig &config) { -- config.storage_extensions["sqlite_scanner"] = make_uniq(); --} - } -diff --git a/src/sqlite_storage.cpp b/src/sqlite_storage.cpp -index 09d212d..38255f6 100644 ---- a/src/sqlite_storage.cpp -+++ b/src/sqlite_storage.cpp -@@ -12,22 +12,24 @@ - - namespace duckdb { - --static unique_ptr SQLiteAttach(StorageExtensionInfo *storage_info, ClientContext &context, -+static unique_ptr SQLiteAttach(optional_ptr storage_info, ClientContext &context, - AttachedDatabase &db, const string &name, AttachInfo &info, -- AccessMode access_mode) { -+ AttachOptions &attach_options) { - SQLiteOpenOptions options; -- options.access_mode = access_mode; -- for (auto &entry : info.options) { -+ options.access_mode = attach_options.access_mode; -+ for (auto &entry : attach_options.options) { - if (StringUtil::CIEquals(entry.first, "busy_timeout")) { - options.busy_timeout = entry.second.GetValue(); - } else if (StringUtil::CIEquals(entry.first, "journal_mode")) { - options.journal_mode = entry.second.ToString(); -+ } else { -+ throw NotImplementedException("Unsupported parameter for SQLite Attach: %s", entry.first); - } - } - return make_uniq(db, info.path, std::move(options)); - } - --static unique_ptr SQLiteCreateTransactionManager(StorageExtensionInfo *storage_info, -+static unique_ptr SQLiteCreateTransactionManager(optional_ptr storage_info, - AttachedDatabase &db, Catalog &catalog) { - auto &sqlite_catalog = catalog.Cast(); - return make_uniq(db, sqlite_catalog); -diff --git a/src/storage/sqlite_delete.cpp b/src/storage/sqlite_delete.cpp -index c19f45b..bde562f 100644 ---- a/src/storage/sqlite_delete.cpp -+++ b/src/storage/sqlite_delete.cpp -@@ -9,8 +9,8 @@ - - namespace duckdb { - --SQLiteDelete::SQLiteDelete(LogicalOperator &op, TableCatalogEntry &table, idx_t row_id_index) -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, op.types, 1), table(table), row_id_index(row_id_index) { -+SQLiteDelete::SQLiteDelete(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, idx_t row_id_index) -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(table), row_id_index(row_id_index) { - } - - //===--------------------------------------------------------------------===// -diff --git a/src/storage/sqlite_index.cpp b/src/storage/sqlite_index.cpp -index 440db80..e076502 100644 ---- a/src/storage/sqlite_index.cpp -+++ b/src/storage/sqlite_index.cpp -@@ -6,8 +6,8 @@ - - namespace duckdb { - --SQLiteCreateIndex::SQLiteCreateIndex(unique_ptr info, TableCatalogEntry &table) -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, {LogicalType::BIGINT}, 1), info(std::move(info)), table(table) { -+SQLiteCreateIndex::SQLiteCreateIndex(PhysicalPlan &physical_plan, unique_ptr info, TableCatalogEntry &table) -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, {LogicalType::BIGINT}, 1), info(std::move(info)), table(table) { - } - - //===--------------------------------------------------------------------===// -diff --git a/src/storage/sqlite_insert.cpp b/src/storage/sqlite_insert.cpp -index 9cd4786..0339e6b 100644 ---- a/src/storage/sqlite_insert.cpp -+++ b/src/storage/sqlite_insert.cpp -@@ -13,14 +13,14 @@ - - namespace duckdb { - --SQLiteInsert::SQLiteInsert(LogicalOperator &op, TableCatalogEntry &table, -+SQLiteInsert::SQLiteInsert(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, - physical_index_vector_t column_index_map_p) -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, op.types, 1), table(&table), schema(nullptr), -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(&table), schema(nullptr), - column_index_map(std::move(column_index_map_p)) { - } - --SQLiteInsert::SQLiteInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info) -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, op.types, 1), table(nullptr), schema(&schema), -+SQLiteInsert::SQLiteInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info) -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(nullptr), schema(&schema), - info(std::move(info)) { - } - -@@ -184,7 +184,7 @@ PhysicalOperator &SQLiteCatalog::PlanInsert(ClientContext &context, PhysicalPlan - if (op.return_chunk) { - throw BinderException("RETURNING clause not yet supported for insertion into SQLite table"); - } -- if (op.action_type != OnConflictAction::THROW) { -+ if (op.on_conflict_info.action_type != OnConflictAction::THROW) { - throw BinderException("ON CONFLICT clause not yet supported for insertion into SQLite table"); - } - -diff --git a/src/storage/sqlite_update.cpp b/src/storage/sqlite_update.cpp -index b550549..6c071d7 100644 ---- a/src/storage/sqlite_update.cpp -+++ b/src/storage/sqlite_update.cpp -@@ -8,8 +8,8 @@ - - namespace duckdb { - --SQLiteUpdate::SQLiteUpdate(LogicalOperator &op, TableCatalogEntry &table, vector columns_p) -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, op.types, 1), table(table), columns(std::move(columns_p)) { -+SQLiteUpdate::SQLiteUpdate(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table, vector columns_p) -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, op.types, 1), table(table), columns(std::move(columns_p)) { - } - - //===--------------------------------------------------------------------===// -diff --git a/test/sql/storage/attach_on_conflict.test b/test/sql/storage/attach_on_conflict.test -index 6f7e678..7c13892 100644 ---- a/test/sql/storage/attach_on_conflict.test -+++ b/test/sql/storage/attach_on_conflict.test -@@ -21,7 +21,7 @@ UNIQUE constraint failed - statement error - INSERT OR IGNORE INTO s.tbl VALUES (1) - ---- --ON CONFLICT clause not yet supported for insertion into SQLite table -+does not support - - # INSERT OR IGNORE in a table without primary key constraints - statement ok -@@ -30,4 +30,4 @@ CREATE TABLE s.tbl2(i INTEGER) - statement error - INSERT OR REPLACE INTO s.tbl2 VALUES (1) - ---- --There are no UNIQUE/PRIMARY KEY Indexes -+no UNIQUE/PRIMARY KEY constraints diff --git a/.github/patches/extensions/sqlsmith/fix.patch b/.github/patches/extensions/sqlsmith/fix.patch index 37bae0e08b1a..e9c9c53dd728 100644 --- a/.github/patches/extensions/sqlsmith/fix.patch +++ b/.github/patches/extensions/sqlsmith/fix.patch @@ -1,118 +1,44 @@ -diff --git a/src/include/sqlsmith_extension.hpp b/src/include/sqlsmith_extension.hpp -index 4cd16f7..1e43766 100644 ---- a/src/include/sqlsmith_extension.hpp -+++ b/src/include/sqlsmith_extension.hpp -@@ -14,7 +14,7 @@ namespace duckdb { - - class SqlsmithExtension : public Extension { - public: -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &loader) override; - std::string Name() override; - std::string Version() const override; - }; -diff --git a/src/sqlsmith_extension.cpp b/src/sqlsmith_extension.cpp -index f995e73..29b7c37 100644 ---- a/src/sqlsmith_extension.cpp -+++ b/src/sqlsmith_extension.cpp -@@ -1,5 +1,3 @@ --#define DUCKDB_EXTENSION_MAIN -- - #include "sqlsmith_extension.hpp" - #include "statement_simplifier.hpp" - #include "fuzzyduck.hpp" -@@ -8,7 +6,7 @@ - #include "duckdb.hpp" - #include "duckdb/function/table_function.hpp" - #include "duckdb/parser/parser.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - - namespace duckdb { - -@@ -170,8 +168,7 @@ static void FuzzAllFunctions(ClientContext &context, TableFunctionInput &data_p, - data.finished = true; - } - --void SqlsmithExtension::Load(DuckDB &db) { -- auto &db_instance = *db.instance; -+static void LoadInternal(ExtensionLoader &loader) { - - TableFunction sqlsmith_func("sqlsmith", {}, SQLSmithFunction, SQLSmithBind); - sqlsmith_func.named_parameters["seed"] = LogicalType::INTEGER; -@@ -182,7 +179,7 @@ void SqlsmithExtension::Load(DuckDB &db) { - sqlsmith_func.named_parameters["verbose_output"] = LogicalType::BOOLEAN; - sqlsmith_func.named_parameters["complete_log"] = LogicalType::VARCHAR; - sqlsmith_func.named_parameters["log"] = LogicalType::VARCHAR; -- ExtensionUtil::RegisterFunction(db_instance, sqlsmith_func); -+ loader.RegisterFunction(sqlsmith_func); - - TableFunction fuzzy_duck_fun("fuzzyduck", {}, FuzzyDuckFunction, FuzzyDuckBind); - fuzzy_duck_fun.named_parameters["seed"] = LogicalType::INTEGER; -@@ -191,17 +188,20 @@ void SqlsmithExtension::Load(DuckDB &db) { - fuzzy_duck_fun.named_parameters["complete_log"] = LogicalType::VARCHAR; - fuzzy_duck_fun.named_parameters["verbose_output"] = LogicalType::BOOLEAN; - fuzzy_duck_fun.named_parameters["enable_verification"] = LogicalType::BOOLEAN; -- ExtensionUtil::RegisterFunction(db_instance, fuzzy_duck_fun); -+ loader.RegisterFunction(fuzzy_duck_fun); - - TableFunction fuzz_all_functions("fuzz_all_functions", {}, FuzzAllFunctions, FuzzyDuckBind); - fuzz_all_functions.named_parameters["seed"] = LogicalType::INTEGER; - fuzz_all_functions.named_parameters["log"] = LogicalType::VARCHAR; - fuzz_all_functions.named_parameters["complete_log"] = LogicalType::VARCHAR; - fuzz_all_functions.named_parameters["verbose_output"] = LogicalType::BOOLEAN; -- ExtensionUtil::RegisterFunction(db_instance, fuzz_all_functions); -+ loader.RegisterFunction(fuzz_all_functions); - - TableFunction reduce_sql_function("reduce_sql_statement", {LogicalType::VARCHAR}, ReduceSQLFunction, ReduceSQLBind); -- ExtensionUtil::RegisterFunction(db_instance, reduce_sql_function); -+ loader.RegisterFunction(reduce_sql_function); -+} -+void SqlsmithExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - - std::string SqlsmithExtension::Name() { -@@ -220,16 +220,8 @@ std::string SqlsmithExtension::Version() const { - - extern "C" { - --DUCKDB_EXTENSION_API void sqlsmith_init(duckdb::DatabaseInstance &db) { -- duckdb::DuckDB db_wrapper(db); -- db_wrapper.LoadExtension(); -+DUCKDB_CPP_EXTENSION_ENTRY(sqlsmith, loader) { -+ duckdb::LoadInternal(loader); - } - --DUCKDB_EXTENSION_API const char *sqlsmith_version() { -- return duckdb::DuckDB::LibraryVersion(); --} - } -- --#ifndef DUCKDB_EXTENSION_MAIN --#error DUCKDB_EXTENSION_MAIN not defined --#endif diff --git a/src/statement_generator.cpp b/src/statement_generator.cpp -index 325fb0a..fc34c7c 100644 +index fc34c7c..5defc4e 100644 --- a/src/statement_generator.cpp +++ b/src/statement_generator.cpp -@@ -34,7 +34,7 @@ struct GeneratorContext { - vector> table_functions; - vector> pragma_functions; - vector> tables_and_views; -- vector> attached_databases; -+ vector> attached_databases; - }; - - StatementGenerator::StatementGenerator(ClientContext &context) : context(context), parent(nullptr), depth(0) { -@@ -216,8 +216,8 @@ unique_ptr StatementGenerator::GenerateDetachInfo() { +@@ -373,8 +373,9 @@ unique_ptr StatementGenerator::GenerateQueryNode() { + GenerateCTEs(*setop); + setop->setop_type = Choose({SetOperationType::EXCEPT, SetOperationType::INTERSECT, + SetOperationType::UNION, SetOperationType::UNION_BY_NAME}); +- setop->left = GenerateQueryNode(); +- setop->right = GenerateQueryNode(); ++ for(idx_t i = 0; i < 2; i++) { ++ setop->children.push_back(GenerateQueryNode()); ++ } + switch (setop->setop_type) { + case SetOperationType::EXCEPT: + case SetOperationType::INTERSECT: +diff --git a/src/statement_simplifier.cpp b/src/statement_simplifier.cpp +index 2cd7f06..4602928 100644 +--- a/src/statement_simplifier.cpp ++++ b/src/statement_simplifier.cpp +@@ -196,8 +196,9 @@ void StatementSimplifier::Simplify(SelectNode &node) { + } - std::string StatementGenerator::GetRandomAttachedDataBase() { - auto state = GetDatabaseState(context); -- auto st_name = state->attached_databases[RandomValue(state->attached_databases.size())]; -- auto name = st_name.get().name; -+ auto &st_name = *state->attached_databases[RandomValue(state->attached_databases.size())]; -+ auto name = st_name.name; - return name; + void StatementSimplifier::Simplify(SetOperationNode &node) { +- Simplify(node.left); +- Simplify(node.right); ++ for(auto &child : node.children) { ++ Simplify(child); ++ } } + void StatementSimplifier::Simplify(CommonTableExpressionMap &cte) { +@@ -218,8 +219,9 @@ void StatementSimplifier::Simplify(unique_ptr &node) { + break; + case QueryNodeType::SET_OPERATION_NODE: { + auto &setop = node->Cast(); +- SimplifyReplace(node, setop.left); +- SimplifyReplace(node, setop.right); ++ for(auto &child : setop.children) { ++ SimplifyReplace(node, child); ++ } + Simplify(setop); + break; + } diff --git a/.github/patches/extensions/vss/fix.patch b/.github/patches/extensions/vss/fix.patch deleted file mode 100644 index b1064b063bfd..000000000000 --- a/.github/patches/extensions/vss/fix.patch +++ /dev/null @@ -1,578 +0,0 @@ -diff --git a/src/hnsw/hnsw_index.cpp b/src/hnsw/hnsw_index.cpp -index ce89f08..5ef1fc1 100644 ---- a/src/hnsw/hnsw_index.cpp -+++ b/src/hnsw/hnsw_index.cpp -@@ -523,7 +523,7 @@ void HNSWIndex::PersistToDisk() { - is_dirty = false; - } - --IndexStorageInfo HNSWIndex::GetStorageInfo(const case_insensitive_map_t &options, const bool to_wal) { -+IndexStorageInfo HNSWIndex::SerializeToDisk(QueryContext context, const case_insensitive_map_t &options) { - - PersistToDisk(); - -@@ -531,17 +531,26 @@ IndexStorageInfo HNSWIndex::GetStorageInfo(const case_insensitive_map_t & - info.name = name; - info.root = root_block_ptr.Get(); - -- if (!to_wal) { -- // use the partial block manager to serialize all allocator data -- auto &block_manager = table_io_manager.GetIndexBlockManager(); -- PartialBlockManager partial_block_manager(block_manager, PartialBlockType::FULL_CHECKPOINT); -- linked_block_allocator->SerializeBuffers(partial_block_manager); -- partial_block_manager.FlushPartialBlocks(); -- } else { -- info.buffers.push_back(linked_block_allocator->InitSerializationToWAL()); -- } -+ // Use the partial block manager to serialize allocator data. -+ auto &block_manager = table_io_manager.GetIndexBlockManager(); -+ PartialBlockManager partial_block_manager(context, block_manager, PartialBlockType::FULL_CHECKPOINT); -+ linked_block_allocator->SerializeBuffers(partial_block_manager); -+ partial_block_manager.FlushPartialBlocks(); -+ info.allocator_infos.push_back(linked_block_allocator->GetInfo()); -+ -+ return info; -+} - -+IndexStorageInfo HNSWIndex::SerializeToWAL(const case_insensitive_map_t &options) { -+ -+ PersistToDisk(); -+ -+ IndexStorageInfo info; -+ info.name = name; -+ info.root = root_block_ptr.Get(); -+ info.buffers.push_back(linked_block_allocator->InitSerializationToWAL()); - info.allocator_infos.push_back(linked_block_allocator->GetInfo()); -+ - return info; - } - -diff --git a/src/hnsw/hnsw_index_macros.cpp b/src/hnsw/hnsw_index_macros.cpp -index 774b409..450f75a 100644 ---- a/src/hnsw/hnsw_index_macros.cpp -+++ b/src/hnsw/hnsw_index_macros.cpp -@@ -1,5 +1,5 @@ - #include "duckdb/function/table_macro_function.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/optimizer/matcher/expression_matcher.hpp" - #include "hnsw/hnsw.hpp" - #include "hnsw/hnsw_index.hpp" -@@ -77,7 +77,7 @@ FROM - //------------------------------------------------------------------------- - // Register - //------------------------------------------------------------------------- --static void RegisterTableMacro(DatabaseInstance &db, const string &name, const string &query, -+static void RegisterTableMacro(ExtensionLoader &loader, const string &name, const string &query, - const vector ¶ms, const child_list_t &named_params) { - - Parser parser; -@@ -101,15 +101,15 @@ static void RegisterTableMacro(DatabaseInstance &db, const string &name, const s - info.internal = true; - info.macros.push_back(std::move(func)); - -- ExtensionUtil::RegisterFunction(db, info); -+ loader.RegisterFunction(info); - } - --void HNSWModule::RegisterMacros(DatabaseInstance &db) { -+void HNSWModule::RegisterMacros(ExtensionLoader &loader) { - -- RegisterTableMacro(db, "vss_join", VSS_JOIN_MACRO, {"left_table", "right_table", "left_col", "right_col", "k"}, -+ RegisterTableMacro(loader, "vss_join", VSS_JOIN_MACRO, {"left_table", "right_table", "left_col", "right_col", "k"}, - {{"metric", Value("l2sq")}}); - -- RegisterTableMacro(db, "vss_match", VSS_MATCH_MACRO, {"right_table", "left_col", "right_col", "k"}, -+ RegisterTableMacro(loader, "vss_match", VSS_MATCH_MACRO, {"right_table", "left_col", "right_col", "k"}, - {{"metric", Value("l2sq")}}); - } - -diff --git a/src/hnsw/hnsw_index_physical_create.cpp b/src/hnsw/hnsw_index_physical_create.cpp -index 72225ec..8bd423e 100644 ---- a/src/hnsw/hnsw_index_physical_create.cpp -+++ b/src/hnsw/hnsw_index_physical_create.cpp -@@ -14,12 +14,12 @@ - - namespace duckdb { - --PhysicalCreateHNSWIndex::PhysicalCreateHNSWIndex(const vector &types_p, TableCatalogEntry &table_p, -+PhysicalCreateHNSWIndex::PhysicalCreateHNSWIndex(PhysicalPlan &physical_plan, const vector &types_p, TableCatalogEntry &table_p, - const vector &column_ids, unique_ptr info, - vector> unbound_expressions, - idx_t estimated_cardinality) - // Declare this operators as a EXTENSION operator -- : PhysicalOperator(PhysicalOperatorType::EXTENSION, types_p, estimated_cardinality), -+ : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, types_p, estimated_cardinality), - table(table_p.Cast()), info(std::move(info)), unbound_expressions(std::move(unbound_expressions)), - sorted(false) { - -diff --git a/src/hnsw/hnsw_index_pragmas.cpp b/src/hnsw/hnsw_index_pragmas.cpp -index 50b5779..95e02f1 100644 ---- a/src/hnsw/hnsw_index_pragmas.cpp -+++ b/src/hnsw/hnsw_index_pragmas.cpp -@@ -8,7 +8,7 @@ - #include "duckdb/storage/table/scan_state.hpp" - #include "duckdb/transaction/duck_transaction.hpp" - #include "duckdb/transaction/local_storage.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/catalog/catalog_entry/duck_index_entry.hpp" - #include "duckdb/storage/data_table.hpp" - -@@ -99,9 +99,15 @@ static void HNSWIndexInfoExecute(ClientContext &context, TableFunctionInput &dat - HNSWIndex *hnsw_index = nullptr; - - auto &table_info = *storage.GetDataTableInfo(); -- table_info.GetIndexes().BindAndScan(context, table_info, [&](HNSWIndex &index) { -- if (index.name == index_entry.name) { -- hnsw_index = &index; -+ -+ table_info.BindIndexes(context, HNSWIndex::TYPE_NAME); -+ table_info.GetIndexes().Scan([&](Index &index) { -+ if (!index.IsBound() || HNSWIndex::TYPE_NAME != index.GetIndexType()) { -+ return false; -+ } -+ auto &cast_index = index.Cast(); -+ if (cast_index.name == index_entry.name) { -+ hnsw_index = &cast_index; - return true; - } - return false; -@@ -175,9 +181,14 @@ static void CompactIndexPragma(ClientContext &context, const FunctionParameters - bool found_index = false; - - auto &table_info = *storage.GetDataTableInfo(); -- table_info.GetIndexes().BindAndScan(context, table_info, [&](HNSWIndex &hnsw_index) { -- if (index_entry.name == index_name) { -- hnsw_index.Compact(); -+ table_info.BindIndexes(context, HNSWIndex::TYPE_NAME); -+ table_info.GetIndexes().Scan([&](Index &index) { -+ if (!index.IsBound() || HNSWIndex::TYPE_NAME != index.GetIndexType()) { -+ return false; -+ } -+ auto &cast_index = index.Cast(); -+ if (cast_index.name == index_entry.name) { -+ cast_index.Compact(); - found_index = true; - return true; - } -@@ -192,14 +203,13 @@ static void CompactIndexPragma(ClientContext &context, const FunctionParameters - //------------------------------------------------------------------------- - // Register - //------------------------------------------------------------------------- --void HNSWModule::RegisterIndexPragmas(DatabaseInstance &db) { -- ExtensionUtil::RegisterFunction( -- db, PragmaFunction::PragmaCall("hnsw_compact_index", CompactIndexPragma, {LogicalType::VARCHAR})); -+void HNSWModule::RegisterIndexPragmas(ExtensionLoader &loader) { -+ loader.RegisterFunction(PragmaFunction::PragmaCall("hnsw_compact_index", CompactIndexPragma, {LogicalType::VARCHAR})); - - // TODO: This is kind of ugly and maybe should just take a parameter instead... - TableFunction info_function("pragma_hnsw_index_info", {}, HNSWIndexInfoExecute, HNSWindexInfoBind, - HNSWIndexInfoInitGlobal); -- ExtensionUtil::RegisterFunction(db, info_function); -+ loader.RegisterFunction(info_function); - } - - } // namespace duckdb -\ No newline at end of file -diff --git a/src/hnsw/hnsw_index_scan.cpp b/src/hnsw/hnsw_index_scan.cpp -index edd808e..73ab224 100644 ---- a/src/hnsw/hnsw_index_scan.cpp -+++ b/src/hnsw/hnsw_index_scan.cpp -@@ -8,7 +8,7 @@ - #include "duckdb/storage/table/scan_state.hpp" - #include "duckdb/transaction/duck_transaction.hpp" - #include "duckdb/transaction/local_storage.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/catalog/catalog_entry/duck_index_entry.hpp" - #include "duckdb/storage/data_table.hpp" - -@@ -155,8 +155,8 @@ TableFunction HNSWIndexScanFunction::GetFunction() { - //------------------------------------------------------------------------- - // Register - //------------------------------------------------------------------------- --void HNSWModule::RegisterIndexScan(DatabaseInstance &db) { -- ExtensionUtil::RegisterFunction(db, HNSWIndexScanFunction::GetFunction()); -+void HNSWModule::RegisterIndexScan(ExtensionLoader &loader) { -+ loader.RegisterFunction(HNSWIndexScanFunction::GetFunction()); - } - - } // namespace duckdb -\ No newline at end of file -diff --git a/src/hnsw/hnsw_optimize_expr.cpp b/src/hnsw/hnsw_optimize_expr.cpp -index a05f1a6..573593e 100644 ---- a/src/hnsw/hnsw_optimize_expr.cpp -+++ b/src/hnsw/hnsw_optimize_expr.cpp -@@ -6,6 +6,7 @@ - #include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp" - #include "duckdb/optimizer/column_binding_replacer.hpp" - #include "duckdb/optimizer/optimizer.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - - #include "hnsw/hnsw.hpp" - #include "hnsw/hnsw_index.hpp" -diff --git a/src/hnsw/hnsw_optimize_join.cpp b/src/hnsw/hnsw_optimize_join.cpp -index 1a8d93b..cf2eb77 100644 ---- a/src/hnsw/hnsw_optimize_join.cpp -+++ b/src/hnsw/hnsw_optimize_join.cpp -@@ -34,9 +34,9 @@ class PhysicalHNSWIndexJoin final : public PhysicalOperator { - public: - static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::EXTENSION; - -- PhysicalHNSWIndexJoin(const vector &types_p, const idx_t estimated_cardinality, -+ PhysicalHNSWIndexJoin(PhysicalPlan &physical_plan, const vector &types_p, const idx_t estimated_cardinality, - DuckTableEntry &table_p, HNSWIndex &hnsw_index_p, const idx_t limit_p) -- : PhysicalOperator(TYPE, types_p, estimated_cardinality), table(table_p), hnsw_index(hnsw_index_p), -+ : PhysicalOperator(physical_plan, TYPE, types_p, estimated_cardinality), table(table_p), hnsw_index(hnsw_index_p), - limit(limit_p) { - } - -@@ -129,7 +129,7 @@ OperatorResultType PhysicalHNSWIndexJoin::Execute(ExecutionContext &context, Dat - const auto rhs_vector_ptr = FlatVector::GetData(rhs_vector_child); - - // We mimic the window row_number() operator here and output the row number in each batch, basically. -- const auto row_number_vector = FlatVector::GetData(chunk.data[MATCH_COLUMN_OFFSET]); -+ const auto row_number_vector = FlatVector::GetData(chunk.data[MATCH_COLUMN_OFFSET]); - - hnsw_index.ResetMultiScan(*state.index_state); - -@@ -512,13 +512,21 @@ bool HNSWIndexJoinOptimizer::TryOptimize(Binder &binder, ClientContext &context, - - HNSWIndex *index_ptr = nullptr; - vector> bindings; -- table_info.GetIndexes().BindAndScan(context, table_info, [&](HNSWIndex &hnsw_index) { -+ -+ table_info.BindIndexes(context, HNSWIndex::TYPE_NAME); -+ table_info.GetIndexes().Scan([&](Index &index) { -+ if (!index.IsBound() || HNSWIndex::TYPE_NAME != index.GetIndexType()) { -+ return false; -+ } -+ auto &cast_index = index.Cast(); -+ -+ // Reset the bindings - bindings.clear(); -- if (!hnsw_index.TryMatchDistanceFunction(distance_expr_ptr, bindings)) { -+ if (!cast_index.TryMatchDistanceFunction(distance_expr_ptr, bindings)) { - return false; - } - unique_ptr bound_index_expr = nullptr; -- if (!hnsw_index.TryBindIndexExpression(inner_get, bound_index_expr)) { -+ if (!cast_index.TryBindIndexExpression(inner_get, bound_index_expr)) { - return false; - } - -@@ -546,7 +554,7 @@ bool HNSWIndexJoinOptimizer::TryOptimize(Binder &binder, ClientContext &context, - } - - // Save the pointer to the index -- index_ptr = &hnsw_index; -+ index_ptr = &cast_index; - return true; - }); - if (!index_ptr) { -diff --git a/src/hnsw/hnsw_optimize_scan.cpp b/src/hnsw/hnsw_optimize_scan.cpp -index d5aded4..dc9ae3f 100644 ---- a/src/hnsw/hnsw_optimize_scan.cpp -+++ b/src/hnsw/hnsw_optimize_scan.cpp -@@ -10,6 +10,7 @@ - #include "duckdb/planner/operator/logical_top_n.hpp" - #include "duckdb/planner/operator/logical_filter.hpp" - #include "duckdb/storage/data_table.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - - #include "hnsw/hnsw.hpp" - #include "hnsw/hnsw_index.hpp" -@@ -100,17 +101,23 @@ public: - unique_ptr bind_data = nullptr; - vector> bindings; - -- table_info.GetIndexes().BindAndScan(context, table_info, [&](HNSWIndex &hnsw_index) { -+ table_info.BindIndexes(context, HNSWIndex::TYPE_NAME); -+ table_info.GetIndexes().Scan([&](Index &index) { -+ if (!index.IsBound() || HNSWIndex::TYPE_NAME != index.GetIndexType()) { -+ return false; -+ } -+ auto &cast_index = index.Cast(); -+ - // Reset the bindings - bindings.clear(); - - // Check that the projection expression is a distance function that matches the index -- if (!hnsw_index.TryMatchDistanceFunction(projection_expr, bindings)) { -+ if (!cast_index.TryMatchDistanceFunction(projection_expr, bindings)) { - return false; - } - // Check that the HNSW index actually indexes the expression - unique_ptr index_expr; -- if (!hnsw_index.TryBindIndexExpression(get, index_expr)) { -+ if (!cast_index.TryBindIndexExpression(get, index_expr)) { - return false; - } - -@@ -128,7 +135,7 @@ public: - } - } - -- const auto vector_size = hnsw_index.GetVectorSize(); -+ const auto vector_size = cast_index.GetVectorSize(); - const auto &matched_vector = const_expr_ref.get().Cast().value; - auto query_vector = make_unsafe_uniq_array(vector_size); - auto vector_elements = ArrayValue::GetChildren(matched_vector); -@@ -136,7 +143,7 @@ public: - query_vector[i] = vector_elements[i].GetValue(); - } - -- bind_data = make_uniq(duck_table, hnsw_index, top_n.limit, std::move(query_vector)); -+ bind_data = make_uniq(duck_table, cast_index, top_n.limit, std::move(query_vector)); - return true; - }); - -diff --git a/src/hnsw/hnsw_optimize_topk.cpp b/src/hnsw/hnsw_optimize_topk.cpp -index 14967d3..a099d75 100644 ---- a/src/hnsw/hnsw_optimize_topk.cpp -+++ b/src/hnsw/hnsw_optimize_topk.cpp -@@ -8,7 +8,7 @@ - #include "duckdb/optimizer/optimizer.hpp" - #include "duckdb/planner/expression/bound_aggregate_expression.hpp" - #include "duckdb/optimizer/matcher/expression_matcher.hpp" -- -+#include "duckdb/main/extension/extension_loader.hpp" - #include "hnsw/hnsw.hpp" - #include "hnsw/hnsw_index.hpp" - #include "hnsw/hnsw_index_scan.hpp" -@@ -123,17 +123,23 @@ public: - unique_ptr bind_data = nullptr; - vector> bindings; - -- table_info.GetIndexes().BindAndScan(context, table_info, [&](HNSWIndex &hnsw_index) { -+ table_info.BindIndexes(context, HNSWIndex::TYPE_NAME); -+ table_info.GetIndexes().Scan([&](Index &index) { -+ if (!index.IsBound() || HNSWIndex::TYPE_NAME != index.GetIndexType()) { -+ return false; -+ } -+ auto &cast_index = index.Cast(); -+ - // Reset the bindings - bindings.clear(); - - // Check that the projection expression is a distance function that matches the index -- if (!hnsw_index.TryMatchDistanceFunction(dist_expr, bindings)) { -+ if (!cast_index.TryMatchDistanceFunction(dist_expr, bindings)) { - return false; - } - // Check that the HNSW index actually indexes the expression - unique_ptr index_expr; -- if (!hnsw_index.TryBindIndexExpression(get, index_expr)) { -+ if (!cast_index.TryBindIndexExpression(get, index_expr)) { - return false; - } - -@@ -151,7 +157,7 @@ public: - } - } - -- const auto vector_size = hnsw_index.GetVectorSize(); -+ const auto vector_size = cast_index.GetVectorSize(); - const auto &matched_vector = const_expr_ref.get().Cast().value; - - auto query_vector = make_unsafe_uniq_array(vector_size); -@@ -163,7 +169,7 @@ public: - if (k_limit <= 0 || k_limit >= STANDARD_VECTOR_SIZE) { - return false; - } -- bind_data = make_uniq(duck_table, hnsw_index, k_limit, std::move(query_vector)); -+ bind_data = make_uniq(duck_table, cast_index, k_limit, std::move(query_vector)); - return true; - }); - -diff --git a/src/include/hnsw/hnsw.hpp b/src/include/hnsw/hnsw.hpp -index 5f0b55e..c5dd550 100644 ---- a/src/include/hnsw/hnsw.hpp -+++ b/src/include/hnsw/hnsw.hpp -@@ -1,16 +1,19 @@ - #pragma once - - #include "duckdb.hpp" -- -+#include "duckdb/main/extension/extension_loader.hpp" - namespace duckdb { - - struct HNSWModule { - public: -- static void Register(DatabaseInstance &db) { -+ static void Register(ExtensionLoader &loader) { -+ -+ auto &db = loader.GetDatabaseInstance(); -+ - RegisterIndex(db); -- RegisterIndexScan(db); -- RegisterIndexPragmas(db); -- RegisterMacros(db); -+ RegisterIndexScan(loader); -+ RegisterIndexPragmas(loader); -+ RegisterMacros(loader); - - // Optimizers - RegisterExprOptimizer(db); -@@ -20,15 +23,14 @@ public: - } - - private: -- static void RegisterIndex(DatabaseInstance &db); -- static void RegisterIndexScan(DatabaseInstance &db); -- static void RegisterMultiScan(DatabaseInstance &db); -- static void RegisterIndexPragmas(DatabaseInstance &db); -- static void RegisterMacros(DatabaseInstance &db); -- static void RegisterTopKOptimizer(DatabaseInstance &db); -+ static void RegisterIndex(DatabaseInstance &ldb); -+ static void RegisterIndexScan(ExtensionLoader &loader); -+ static void RegisterIndexPragmas(ExtensionLoader &loader); -+ static void RegisterMacros(ExtensionLoader &loader); -+ static void RegisterTopKOperator(DatabaseInstance &db); - - static void RegisterExprOptimizer(DatabaseInstance &db); -- static void RegisterTopKOperator(DatabaseInstance &db); -+ static void RegisterTopKOptimizer(DatabaseInstance &db); - static void RegisterScanOptimizer(DatabaseInstance &db); - static void RegisterJoinOptimizer(DatabaseInstance &db); - }; -diff --git a/src/include/hnsw/hnsw_index.hpp b/src/include/hnsw/hnsw_index.hpp -index 75783a0..b9ce61a 100644 ---- a/src/include/hnsw/hnsw_index.hpp -+++ b/src/include/hnsw/hnsw_index.hpp -@@ -77,7 +77,11 @@ public: - //! Insert a chunk of entries into the index - ErrorData Insert(IndexLock &lock, DataChunk &data, Vector &row_ids) override; - -- IndexStorageInfo GetStorageInfo(const case_insensitive_map_t &options, const bool to_wal) override; -+ //! Serializes HNSW memory to disk and returns the index storage information. -+ IndexStorageInfo SerializeToDisk(QueryContext context, const case_insensitive_map_t &options) override; -+ //! Serializes HNSW memory to the WAL and returns the index storage information. -+ IndexStorageInfo SerializeToWAL(const case_insensitive_map_t &options) override; -+ - idx_t GetInMemorySize(IndexLock &state) override; - - //! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other -diff --git a/src/include/hnsw/hnsw_index_physical_create.hpp b/src/include/hnsw/hnsw_index_physical_create.hpp -index 8b3e19d..2b1cc6a 100644 ---- a/src/include/hnsw/hnsw_index_physical_create.hpp -+++ b/src/include/hnsw/hnsw_index_physical_create.hpp -@@ -12,7 +12,7 @@ public: - static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::EXTENSION; - - public: -- PhysicalCreateHNSWIndex(const vector &types_p, TableCatalogEntry &table, -+ PhysicalCreateHNSWIndex(PhysicalPlan &physical_plan, const vector &types_p, TableCatalogEntry &table, - const vector &column_ids, unique_ptr info, - vector> unbound_expressions, idx_t estimated_cardinality); - -diff --git a/src/include/usearch/index.hpp b/src/include/usearch/index.hpp -index ef79586..d1e71f3 100644 ---- a/src/include/usearch/index.hpp -+++ b/src/include/usearch/index.hpp -@@ -3363,7 +3363,7 @@ class index_gt { - bool operator==(candidates_iterator_t const& other) noexcept { return current_ == other.current_; } - bool operator!=(candidates_iterator_t const& other) noexcept { return current_ != other.current_; } - -- vector_key_t key() const noexcept { return index_->node_at_(slot()).key(); } -+ vector_key_t key() const noexcept { return index_.node_at_(slot()).key(); } - compressed_slot_t slot() const noexcept { return neighbors_[current_]; } - friend inline std::size_t get_slot(candidates_iterator_t const& it) noexcept { return it.slot(); } - friend inline vector_key_t get_key(candidates_iterator_t const& it) noexcept { return it.key(); } -diff --git a/src/include/vss_extension.hpp b/src/include/vss_extension.hpp -index a941b5e..e34943c 100644 ---- a/src/include/vss_extension.hpp -+++ b/src/include/vss_extension.hpp -@@ -6,7 +6,7 @@ namespace duckdb { - - class VssExtension : public Extension { - public: -- void Load(DuckDB &db) override; -+ void Load(ExtensionLoader &loader) override; - std::string Name() override; - }; - -diff --git a/src/vss_extension.cpp b/src/vss_extension.cpp -index c010292..0a73510 100644 ---- a/src/vss_extension.cpp -+++ b/src/vss_extension.cpp -@@ -1,24 +1,22 @@ --#define DUCKDB_EXTENSION_MAIN -- - #include "vss_extension.hpp" - #include "duckdb.hpp" - #include "duckdb/common/exception.hpp" - #include "duckdb/common/string_util.hpp" - #include "duckdb/function/scalar_function.hpp" --#include "duckdb/main/extension_util.hpp" -+#include "duckdb/main/extension/extension_loader.hpp" - #include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" - - #include "hnsw/hnsw.hpp" - - namespace duckdb { - --static void LoadInternal(DatabaseInstance &instance) { -+static void LoadInternal(ExtensionLoader &loader) { - // Register the HNSW index module -- HNSWModule::Register(instance); -+ HNSWModule::Register(loader); - } - --void VssExtension::Load(DuckDB &db) { -- LoadInternal(*db.instance); -+void VssExtension::Load(ExtensionLoader &loader) { -+ LoadInternal(loader); - } - - std::string VssExtension::Name() { -@@ -29,16 +27,8 @@ std::string VssExtension::Name() { - - extern "C" { - --DUCKDB_EXTENSION_API void vss_init(duckdb::DatabaseInstance &db) { -- duckdb::DuckDB db_wrapper(db); -- db_wrapper.LoadExtension(); -+DUCKDB_CPP_EXTENSION_ENTRY(vss, loader) { -+ duckdb::LoadInternal(loader); - } - --DUCKDB_EXTENSION_API const char *vss_version() { -- return duckdb::DuckDB::LibraryVersion(); --} - } -- --#ifndef DUCKDB_EXTENSION_MAIN --#error DUCKDB_EXTENSION_MAIN not defined --#endif -diff --git a/test/sql/hnsw/hnsw_lateral_join.test b/test/sql/hnsw/hnsw_lateral_join.test -index 0362f3c..5a8bc92 100644 ---- a/test/sql/hnsw/hnsw_lateral_join.test -+++ b/test/sql/hnsw/hnsw_lateral_join.test -@@ -58,18 +58,18 @@ statement ok - INSERT INTO a VALUES (NULL, 3); - - query IIIIII rowsort a_has_null --select * from a, lateral (select *, a_id as id_dup from b order by array_distance(a.a_vec, b.b_vec) limit 2); -+select * from a, lateral (select *, a_id as id_dup from b order by array_distance(a.a_vec, b.b_vec), b_str DESC limit 2); - - statement ok - CREATE INDEX my_idx ON b USING HNSW (b_vec); - - query IIIIII rowsort a_has_null --select * from a, lateral (select *, a_id as id_dup from b order by array_distance(a.a_vec, b.b_vec) limit 2); -+select * from a, lateral (select *, a_id as id_dup from b order by array_distance(a.a_vec, b.b_vec), b_str DESC limit 2); - - # Test with a grouping function - query II rowsort --select a_id, list(b_str) from a, lateral (select *, a_id as id_dup from b order by array_distance(a.a_vec, b.b_vec) limit 2) GROUP BY a_id; -+select a_id, list(b_str) from a, lateral (select *, a_id as id_dup from b order by array_distance(a.a_vec, b.b_vec), b_str limit 2) GROUP BY a_id; - ---- --1 [a, b] --2 [b, a] --3 [a, b] -\ No newline at end of file -+1 [b, a] -+2 [a, b] -+3 [b, a] diff --git a/.github/patches/extensions/vss/macros.patch b/.github/patches/extensions/vss/macros.patch deleted file mode 100644 index f0d8212ea615..000000000000 --- a/.github/patches/extensions/vss/macros.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/src/hnsw/hnsw_index_macros.cpp b/src/hnsw/hnsw_index_macros.cpp -index 774b409..226a2f0 100644 ---- a/src/hnsw/hnsw_index_macros.cpp -+++ b/src/hnsw/hnsw_index_macros.cpp -@@ -91,6 +91,7 @@ static void RegisterTableMacro(DatabaseInstance &db, const string &name, const s - } - - for (auto ¶m : named_params) { -+ func->parameters.push_back(make_uniq(param.first)); - func->default_parameters[param.first] = make_uniq(param.second); - } - -@@ -113,4 +114,4 @@ void HNSWModule::RegisterMacros(DatabaseInstance &db) { - {{"metric", Value("l2sq")}}); - } - --} // namespace duckdb -\ No newline at end of file -+} // namespace duckdb diff --git a/.github/regression/micro.csv b/.github/regression/micro.csv index 485f820a8286..e29628575e74 100644 --- a/.github/regression/micro.csv +++ b/.github/regression/micro.csv @@ -32,4 +32,6 @@ benchmark/micro/logger/logging_overhead/parquet_q1_with_default_logging.benchmar benchmark/micro/logger/logging_overhead/duckdb_persistent_q1_with_default_logging.benchmark benchmark/micro/logger/storage/file/log_message_size/huge_string.benchmark benchmark/micro/logger/storage/file/log_message_size/small_string.benchmark -benchmark/micro/filter/choose_correct_filter_function.benchmark \ No newline at end of file +benchmark/micro/filter/choose_correct_filter_function.benchmark +benchmark/micro/optimizer/topn_window_elimination.benchmark +benchmark/micro/aggregate/group_two_string_dictionaries.benchmark diff --git a/.github/workflows/BundleStaticLibs.yml b/.github/workflows/BundleStaticLibs.yml index b8aa73cc7d18..7b25b5eb04a0 100644 --- a/.github/workflows/BundleStaticLibs.yml +++ b/.github/workflows/BundleStaticLibs.yml @@ -44,16 +44,18 @@ jobs: strategy: matrix: include: - - version: "macos-13" + - xcode_target_flag: "x86_64" architecture: "amd64" - - version: "macos-14" + - xcode_target_flag: "arm64" architecture: "arm64" - runs-on: ${{ matrix.version }} + runs-on: macos-latest env: EXTENSION_CONFIGS: '${GITHUB_WORKSPACE}/.github/config/bundled_extensions.cmake' ENABLE_EXTENSION_AUTOLOADING: 1 ENABLE_EXTENSION_AUTOINSTALL: 1 GEN: ninja + OSX_BUILD_ARCH: ${{ matrix.xcode_target_flag }} + DUCKDB_PLATFORM: osx_${{ matrix.architecture }} steps: - uses: actions/checkout@v4 @@ -72,7 +74,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -83,10 +85,6 @@ jobs: run: | make gather-libs - - name: Print platform - shell: bash - run: ./build/release/duckdb -c "PRAGMA platform;" - - name: Deploy shell: bash env: @@ -103,6 +101,7 @@ jobs: path: | static-libs-osx-${{ matrix.architecture }}.zip + bundle-mingw-static-lib: name: Windows MingW static libs runs-on: windows-latest diff --git a/.github/workflows/CodeQuality.yml b/.github/workflows/CodeQuality.yml index 4e127c6d268d..67f410030ba6 100644 --- a/.github/workflows/CodeQuality.yml +++ b/.github/workflows/CodeQuality.yml @@ -18,7 +18,8 @@ on: - '.github/workflows/**' - '!.github/workflows/lcov_exclude' - '!.github/workflows/CodeQuality.yml' - - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' merge_group: pull_request: types: [opened, reopened, ready_for_review, converted_to_draft] @@ -29,7 +30,8 @@ on: - '.github/workflows/**' - '!.github/workflows/lcov_exclude' - '!.github/workflows/CodeQuality.yml' - - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} @@ -134,7 +136,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Download clang-tidy-cache if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feature' }} diff --git a/.github/workflows/CrossVersion.yml b/.github/workflows/CrossVersion.yml index 3616bc07ab16..7703fe7af077 100644 --- a/.github/workflows/CrossVersion.yml +++ b/.github/workflows/CrossVersion.yml @@ -57,7 +57,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -113,7 +113,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -212,7 +212,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -269,7 +269,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash diff --git a/.github/workflows/ExtendedTests.yml b/.github/workflows/ExtendedTests.yml index 9800ccef4e3c..e8cbbcc89ce7 100644 --- a/.github/workflows/ExtendedTests.yml +++ b/.github/workflows/ExtendedTests.yml @@ -51,7 +51,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -131,7 +131,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -211,7 +211,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -296,7 +296,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash diff --git a/.github/workflows/Extensions.yml b/.github/workflows/Extensions.yml index 6184bae8527c..6944f533881b 100644 --- a/.github/workflows/Extensions.yml +++ b/.github/workflows/Extensions.yml @@ -142,7 +142,7 @@ jobs: override_tag: ${{ inputs.override_git_describe }} duckdb_ref: ${{ inputs.git_ref }} skip_tests: ${{ inputs.skip_tests && true || false }} - save_cache: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save_cache: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} # Build the extensions from .github/config/rust_based_extensions.cmake rust-based-extensions: @@ -158,7 +158,7 @@ jobs: override_tag: ${{ inputs.override_git_describe }} duckdb_ref: ${{ inputs.git_ref }} skip_tests: ${{ inputs.skip_tests && true || false }} - save_cache: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save_cache: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} # Merge all extensions into a single, versioned repository create-extension-repository: @@ -225,11 +225,11 @@ jobs: - name: Deploy extensions shell: bash env: - AWS_ACCESS_KEY_ID: ${{secrets.S3_ID}} - AWS_SECRET_ACCESS_KEY: ${{secrets.S3_KEY}} + AWS_ENDPOINT_URL: ${{ secrets.DUCKDB_CORE_EXTENSION_S3_ENDPOINT }} + AWS_ACCESS_KEY_ID: ${{secrets.DUCKDB_CORE_EXTENSION_S3_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.DUCKDB_CORE_EXTENSION_S3_SECRET}} DUCKDB_DEPLOY_SCRIPT_MODE: for_real DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} - AWS_DEFAULT_REGION: us-east-1 run: | pip install awscli ./scripts/extension-upload-repository.sh /tmp/extension-repository-${{ github.sha }} @@ -255,7 +255,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - uses: actions/download-artifact@v4 with: @@ -313,7 +313,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash diff --git a/.github/workflows/ExtraTests.yml b/.github/workflows/ExtraTests.yml index 7d1818e1f2ee..7d06db9d6a4a 100644 --- a/.github/workflows/ExtraTests.yml +++ b/.github/workflows/ExtraTests.yml @@ -38,7 +38,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build Last Release shell: bash diff --git a/.github/workflows/IssuesCloseStale.yml b/.github/workflows/IssuesCloseStale.yml index a8c4c718d667..ca09f7a23ea8 100644 --- a/.github/workflows/IssuesCloseStale.yml +++ b/.github/workflows/IssuesCloseStale.yml @@ -23,7 +23,7 @@ jobs: close-pr-message: 'This pull request was closed because it has been stale for 30 days with no activity.' exempt-issue-labels: 'no stale' exempt-pr-labels: 'no stale' - days-before-stale: 180 + days-before-stale: 365 days-before-close: 30 operations-per-run: 500 stale-issue-label: stale diff --git a/.github/workflows/Julia.yml b/.github/workflows/Julia.yml index 0ba8661ce73a..abb3a78bfab6 100644 --- a/.github/workflows/Julia.yml +++ b/.github/workflows/Julia.yml @@ -16,7 +16,8 @@ on: - '.github/patches/duckdb-wasm/**' - '.github/workflows/**' - '!.github/workflows/Julia.yml' - - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' merge_group: pull_request: types: [opened, reopened, ready_for_review, converted_to_draft] @@ -29,7 +30,8 @@ on: - '.github/patches/duckdb-wasm/**' - '.github/workflows/**' - '!.github/workflows/Julia.yml' - - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} @@ -95,7 +97,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }}-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.version }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build DuckDB shell: bash diff --git a/.github/workflows/LinuxRelease.yml b/.github/workflows/LinuxRelease.yml index 6a2062e17706..0ab52bc2ef3f 100644 --- a/.github/workflows/LinuxRelease.yml +++ b/.github/workflows/LinuxRelease.yml @@ -200,7 +200,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -209,66 +209,3 @@ jobs: - name: Symbol Leakage Test shell: bash run: python3 scripts/exported_symbols_check.py build/release/src/libduckdb*.so - - linux-httpfs: - name: Linux HTTPFS - runs-on: ubuntu-22.04 - needs: linux-release-cli - env: - CORE_EXTENSIONS: "json;parquet;tpch;tpcds;httpfs" - S3_TEST_SERVER_AVAILABLE: 1 - AWS_DEFAULT_REGION: eu-west-1 - AWS_ACCESS_KEY_ID: minio_duckdb_user - AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password - DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000 - DUCKDB_S3_USE_SSL: false - HTTP_PROXY_PUBLIC: localhost:3128 - TEST_PERSISTENT_SECRETS_AVAILABLE: true - GEN: ninja - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install Ninja - shell: bash - run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build libcurl4-openssl-dev - - - name: Fix permissions of test secrets - shell: bash - run: chmod -R 700 data/secrets - - # TODO: fix the authenticated proxy here - - name: Install and run http proxy squid - shell: bash - run: | - sudo apt-get install squid - ./scripts/run_squid.sh --port 3128 --log_dir squid_logs & - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - with: - key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} - - - name: Build - shell: bash - run: make - - - name: Start test server & run tests - shell: bash - run: | - sudo ./scripts/install_s3_test_server.sh - ./scripts/generate_presigned_url.sh - source ./scripts/run_s3_test_server.sh - source ./scripts/set_s3_test_server_variables.sh - sleep 60 - - python3 scripts/get_test_list.py --file-contains 'require httpfs' --list '"*"' > test.list - python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '-f test.list' - python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '[secret]' diff --git a/.github/workflows/Main.yml b/.github/workflows/Main.yml index 1b4068c3c02e..bc3ad5fd8c1a 100644 --- a/.github/workflows/Main.yml +++ b/.github/workflows/Main.yml @@ -14,7 +14,10 @@ on: - '.github/patches/duckdb-wasm/**' - '.github/workflows/**' - '!.github/workflows/Main.yml' - - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' + - '!.github/patches/extensions/fts/*.patch' # fts used in some jobs + - '!.github/config/extensions/fts.cmake' merge_group: pull_request: types: [opened, reopened, ready_for_review, converted_to_draft] @@ -25,7 +28,10 @@ on: - '.github/patches/duckdb-wasm/**' - '.github/workflows/**' - '!.github/workflows/Main.yml' - - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' + - '!.github/patches/extensions/fts/*.patch' # fts used in some jobs + - '!.github/config/extensions/fts.cmake' concurrency: @@ -43,7 +49,7 @@ jobs: steps: - name: Preliminary checks on CI run: echo "Event name is ${{ github.event_name }}" - + linux-debug: name: Linux Debug # This tests release build while enabling slow verifiers (masked by #ifdef DEBUG) and sanitizers @@ -72,7 +78,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -115,7 +121,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -152,7 +158,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -162,40 +168,6 @@ jobs: shell: bash run: build/relassert/test/unittest - hash-zero: - name: Hash Zero - runs-on: ubuntu-24.04 - needs: linux-configs - env: - GEN: ninja - CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json;inet" - HASH_ZERO: 1 - LSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-leak-suppressions.txt - DUCKDB_TEST_DESCRIPTION: 'Compiled with HASH_ZERO=1. Use require no_hash_zero to skip.' - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install - shell: bash - run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - with: - key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} - - - name: Build - shell: bash - run: make relassert - - - name: Test - shell: bash - run: build/relassert/test/unittest --test-config test/configs/hash_zero.json - vector-sizes: name: Vector Sizes runs-on: ubuntu-22.04 @@ -228,7 +200,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -263,7 +235,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -303,7 +275,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -375,7 +347,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build id: build @@ -486,3 +458,26 @@ jobs: shell: bash run: | ./build/release/test/unittest --test-config test/configs/variant_vector.json + + - name: Test variant_vector + if: (success() || failure()) && steps.build.conclusion == 'success' + shell: bash + run: | + ./build/release/test/unittest --test-config test/configs/compressed_in_memory.json + + - name: Test block prefetching + if: (success() || failure()) && steps.build.conclusion == 'success' + shell: bash + run: | + ./build/release/test/unittest --test-config test/configs/prefetch_all_storage.json + + - name: Test peg_parser + if: (success() || failure()) && steps.build.conclusion == 'success' + shell: bash + run: | + ./build/release/test/unittest --test-config test/configs/peg_parser.json + - name: Forwards compatibility tests + if: (success() || failure()) && steps.build.conclusion == 'success' + shell: bash + run: | + python3 scripts/test_storage_compatibility.py --versions "1.2.1|1.3.2" --new-unittest build/release/test/unittest diff --git a/.github/workflows/NightlyTests.yml b/.github/workflows/NightlyTests.yml index 6012dad28d64..227e7fcb4278 100644 --- a/.github/workflows/NightlyTests.yml +++ b/.github/workflows/NightlyTests.yml @@ -25,8 +25,7 @@ concurrency: env: GH_TOKEN: ${{ secrets.GH_TOKEN }} DUCKDB_WASM_VERSION: "cf2048bd6d669ffa05c56d7d453e09e99de8b87e" - CCACHE_SAVE: ${{ github.repository != 'duckdb/duckdb' }} - BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} + CCACHE_SAVE: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} jobs: check-draft: @@ -326,6 +325,20 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Cleanup disk before build + run: | + echo "Disk usage before clean up:" + df -h + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + docker system prune -af || true + rm -rf ~/.cache + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + echo "Disk usage after clean up:" + df -h - name: Install shell: bash @@ -628,6 +641,40 @@ jobs: path: | duckdb-wasm32.zip + hash-zero: + name: Hash Zero + runs-on: ubuntu-24.04 + needs: linux-memory-leaks + env: + GEN: ninja + CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json;inet" + HASH_ZERO: 1 + LSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-leak-suppressions.txt + DUCKDB_TEST_DESCRIPTION: 'Compiled with HASH_ZERO=1. Use require no_hash_zero to skip.' + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install + shell: bash + run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + with: + key: ${{ github.job }} + save: ${{ env.CCACHE_SAVE }} + + - name: Build + shell: bash + run: make relassert + + - name: Test + shell: bash + run: build/relassert/test/unittest --test-config test/configs/hash_zero.json + codecov: name: Code Coverage runs-on: ubuntu-22.04 diff --git a/.github/workflows/NotifyExternalRepositories.yml b/.github/workflows/NotifyExternalRepositories.yml index c30b82895173..44de96fd0cf1 100644 --- a/.github/workflows/NotifyExternalRepositories.yml +++ b/.github/workflows/NotifyExternalRepositories.yml @@ -108,8 +108,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Call /dispatch - if: ${{ github.repository == 'duckdb/duckdb' }} + if: ${{ github.repository == 'duckdb/duckdb' && inputs.override-git-describe == '' }} run: | export URL=https://api.github.com/repos/duckdb/duckdb-python/actions/workflows/release.yml/dispatches - export DATA='{"ref": "${{ inputs.target-branch }}", "inputs": {"duckdb-sha": "${{ inputs.duckdb-sha }}", "stable-version": "${{ inputs.override-git-describe }}", "pypi-index": "prod" }}' + export DATA='{"ref": "${{ inputs.target-branch }}", "inputs": {"duckdb-sha": "${{ inputs.duckdb-sha }}", "pypi-index": "prod" }}' curl -v -XPOST -u "${PAT_USER}:${PAT_TOKEN}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" $URL --data "$DATA" diff --git a/.github/workflows/OSX.yml b/.github/workflows/OSX.yml index 46522bfb0fc8..7e296f1bbb49 100644 --- a/.github/workflows/OSX.yml +++ b/.github/workflows/OSX.yml @@ -67,7 +67,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Install ninja shell: bash @@ -128,7 +128,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Install pytest run: | diff --git a/.github/workflows/Regression.yml b/.github/workflows/Regression.yml index 14ebb44054c2..1145a155090a 100644 --- a/.github/workflows/Regression.yml +++ b/.github/workflows/Regression.yml @@ -23,6 +23,10 @@ on: - '.github/workflows/**' - '!.github/workflows/Regression.yml' - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' + - '!.github/patches/extensions/httpfs/*.patch' # httpfs used in some jobs + - '!.github/config/extensions/httpfs.cmake' merge_group: pull_request: types: [opened, reopened, ready_for_review, converted_to_draft] @@ -33,7 +37,10 @@ on: - '.github/patches/duckdb-wasm/**' - '.github/workflows/**' - '!.github/workflows/Regression.yml' - - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' + - '!.github/patches/extensions/httpfs/*.patch' # httpfs used in some jobs + - '!.github/config/extensions/httpfs.cmake' concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} @@ -82,7 +89,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Checkout Private Regression if: ${{ github.repository == 'duckdb/duckdb' && github.ref == 'refs/heads/main' }} @@ -187,7 +194,7 @@ jobs: mkdir -p duckdb_benchmark_data rm -R duckdb/duckdb_benchmark_data mkdir -p duckdb/duckdb_benchmark_data - wget -q https://duckdb-blobs.s3.amazonaws.com/data/realnest/realnest.duckdb --output-document=duckdb_benchmark_data/real_nest.duckdb + wget -q https://blobs.duckdb.org/data/realnest/realnest.duckdb --output-document=duckdb_benchmark_data/real_nest.duckdb cp duckdb_benchmark_data/real_nest.duckdb duckdb/duckdb_benchmark_data/real_nest.duckdb python scripts/regression/test_runner.py --old duckdb/build/release/benchmark/benchmark_runner --new build/release/benchmark/benchmark_runner --benchmarks .github/regression/realnest.csv --verbose --threads 2 @@ -218,7 +225,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -299,7 +306,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -342,7 +349,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash diff --git a/.github/workflows/Swift.yml b/.github/workflows/Swift.yml index a58a3e953e96..a8e9dba1bb7c 100644 --- a/.github/workflows/Swift.yml +++ b/.github/workflows/Swift.yml @@ -16,7 +16,8 @@ on: - '.github/patches/duckdb-wasm/**' - '.github/workflows/**' - '!.github/workflows/Swift.yml' - - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' merge_group: pull_request: types: [opened, reopened, ready_for_review, converted_to_draft] @@ -29,7 +30,8 @@ on: - '.github/patches/duckdb-wasm/**' - '.github/workflows/**' - '!.github/workflows/Swift.yml' - - '.github/config/out_of_tree_extensions.cmake' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml index ea47d0eb069f..fb0c10f4d125 100644 --- a/.github/workflows/Windows.yml +++ b/.github/workflows/Windows.yml @@ -33,6 +33,8 @@ on: - '.github/patches/duckdb-wasm/**' - '.github/workflows/**' - '!.github/workflows/Windows.yml' + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' merge_group: pull_request: @@ -45,7 +47,8 @@ on: - '.github/patches/duckdb-wasm/**' - '.github/workflows/**' - '!.github/workflows/Windows.yml' - + - '.github/config/extensions/*.cmake' + - '.github/patches/extensions/**/*.patch' concurrency: group: windows-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }} @@ -86,7 +89,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -190,7 +193,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -232,7 +235,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: bash @@ -304,7 +307,7 @@ jobs: uses: hendrikmuhs/ccache-action@main with: key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + save: ${{ vars.BRANCHES_TO_BE_CACHED == '' || contains(vars.BRANCHES_TO_BE_CACHED, github.ref) }} - name: Build shell: msys2 {0} diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b81cd989d87..94e16b44e440 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,14 +37,24 @@ set(CMAKE_VERBOSE_MAKEFILE OFF) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_MACOSX_RPATH 1) -find_program(CCACHE_PROGRAM ccache) -if(CCACHE_PROGRAM) - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") -else() - find_program(CCACHE_PROGRAM sccache) - if(CCACHE_PROGRAM) - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") - endif() +if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER) + find_program(COMPILER_LAUNCHER NAMES ccache sccache) + if(COMPILER_LAUNCHER) + message(STATUS "Using ${COMPILER_LAUNCHER} as C compiler launcher") + set(CMAKE_C_COMPILER_LAUNCHER + "${COMPILER_LAUNCHER}" + CACHE STRING "" FORCE) + endif() +endif() + +if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER) + find_program(COMPILER_LAUNCHER NAMES ccache sccache) + if(COMPILER_LAUNCHER) + message(STATUS "Using ${COMPILER_LAUNCHER} as C++ compiler launcher") + set(CMAKE_CXX_COMPILER_LAUNCHER + "${COMPILER_LAUNCHER}" + CACHE STRING "" FORCE) + endif() endif() # Determine install paths @@ -443,6 +453,7 @@ option(EXTENSION_CONFIG_BUILD "Produce extension configuration artifacts instead option(CUSTOM_LINKER "Use a custom linker program" "") option(CRASH_ON_ASSERT "Trigger a sigabort on an assert failing, instead of throwing an exception" FALSE) option(FORCE_ASSERT "Enable checking of assertions, even in release mode" FALSE) +option(FORCE_DEBUG "Force adding a debug define, even in release mode" FALSE) option(TREAT_WARNINGS_AS_ERRORS "Treat warnings as errors" FALSE) option(EXPORT_DLL_SYMBOLS "Export dll symbols on Windows, else import" TRUE) @@ -450,6 +461,7 @@ option(BUILD_RDTSC "Enable the rdtsc instruction." FALSE) option(SMALLER_BINARY "Produce a smaller binary by trimming specialized code paths. This can negatively affect performance." FALSE) option(NATIVE_ARCH "Compile targeting the native architecture" FALSE) option(OVERRIDE_NEW_DELETE "Override C++ new/delete (only when jemalloc is enabled)" FALSE) +option(SET_DUCKDB_LIBRARY_VERSION "Whether or not to set the DuckDB Library version and emit versioned shared libraries" FALSE) if(${BUILD_RDTSC}) add_compile_definitions(RDTSC) @@ -553,6 +565,10 @@ if(FORCE_ASSERT) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_FORCE_ASSERT") endif() +if(FORCE_DEBUG) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDEBUG") +endif() + if(STANDALONE_DEBUG) if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug" OR NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang$") message(FATAL_ERROR "Only clang compiler supports -fstandalone-debug") @@ -1076,17 +1092,25 @@ endfunction() # Downloads the external extension repo at the specified commit and calls register_extension macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PATH TEST_PATH APPLY_PATCHES LINKED_LIBS SUBMODULES EXTENSION_VERSION) include(FetchContent) - if (${APPLY_PATCHES}) - set(PATCH_COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/scripts/apply_extension_patches.py ${CMAKE_SOURCE_DIR}/.github/patches/extensions/${NAME}/) - endif() - FETCHCONTENT_DECLARE( - ${NAME}_extension_fc - GIT_REPOSITORY ${URL} - GIT_TAG ${COMMIT} - GIT_SUBMODULES "${SUBMODULES}" - PATCH_COMMAND ${PATCH_COMMAND} - ) - FETCHCONTENT_POPULATE(${NAME}_EXTENSION_FC) + + string(TOUPPER "DUCKDB_${NAME}_DIRECTORY" DIRECTORY_OVERRIDE) + if(DEFINED ENV{${DIRECTORY_OVERRIDE}}) + set("${NAME}_extension_fc_SOURCE_DIR" "$ENV{${DIRECTORY_OVERRIDE}}") + message(STATUS "Load extension '${NAME}' from local path \"${${NAME}_extension_fc_SOURCE_DIR}\"") + else() + if (${APPLY_PATCHES}) + set(PATCH_COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/scripts/apply_extension_patches.py ${CMAKE_SOURCE_DIR}/.github/patches/extensions/${NAME}/) + endif() + FETCHCONTENT_DECLARE( + ${NAME}_extension_fc + GIT_REPOSITORY ${URL} + GIT_TAG ${COMMIT} + GIT_SUBMODULES "${SUBMODULES}" + PATCH_COMMAND ${PATCH_COMMAND} + ) + FETCHCONTENT_POPULATE(${NAME}_EXTENSION_FC) + message(STATUS "Load extension '${NAME}' from ${URL} @ ${EXTERNAL_EXTENSION_VERSION}") + endif() # Autogenerate version tag if not provided if ("${EXTENSION_VERSION}" STREQUAL "") @@ -1095,8 +1119,6 @@ macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TEST set(EXTERNAL_EXTENSION_VERSION "${EXTENSION_VERSION}") endif() - message(STATUS "Load extension '${NAME}' from ${URL} @ ${EXTERNAL_EXTENSION_VERSION}") - string(TOUPPER ${NAME} EXTENSION_NAME_UPPERCASE) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${EXTERNAL_EXTENSION_VERSION}" PARENT_SCOPE) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4838f8fa1f95..7ef7765dceb8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,4 +129,4 @@ This project and everyone participating in it is governed by a [Code of Conduct] ## Generative AI Policy Please do not submit pull requests generated by AI (LLMs). -Reviewing such PRs put a considerable burden on the maintainers. +Reviewing such PRs puts a considerable burden on the maintainers. diff --git a/Makefile b/Makefile index 9e0af673b335..a414c98d75ba 100644 --- a/Makefile +++ b/Makefile @@ -200,6 +200,9 @@ endif ifeq (${FORCE_ASSERT}, 1) CMAKE_VARS:=${CMAKE_VARS} -DFORCE_ASSERT=1 endif +ifeq (${FORCE_DEBUG}, 1) + CMAKE_VARS:=${CMAKE_VARS} -DFORCE_DEBUG=1 +endif ifeq (${SMALLER_BINARY}, 1) CMAKE_VARS:=${CMAKE_VARS} -DSMALLER_BINARY=1 endif @@ -525,6 +528,7 @@ bundle-setup: cp src/libduckdb_static.a bundle/. && \ cp third_party/*/libduckdb_*.a bundle/. && \ cp extension/*/lib*_extension.a bundle/. && \ + mkdir -p vcpkg_installed && \ find vcpkg_installed -name '*.a' -exec cp {} bundle/. \; && \ cd bundle && \ find . -name '*.a' -exec mkdir -p {}.objects \; -exec mv {} {}.objects \; && \ diff --git a/benchmark/appian_benchmarks/appian.benchmark.in b/benchmark/appian_benchmarks/appian.benchmark.in index 742f64b11bfc..50985f28120c 100644 --- a/benchmark/appian_benchmarks/appian.benchmark.in +++ b/benchmark/appian_benchmarks/appian.benchmark.in @@ -11,7 +11,7 @@ cache ads.5M.duck load LOAD httpfs; -attach 'https://duckdb-blobs.s3.amazonaws.com/data/appian_benchmark_data.duckdb' as appian_db (READ_ONLY); +attach 'https://blobs.duckdb.org/data/appian_benchmark_data.duckdb' as appian_db (READ_ONLY); create table AddressView as select * from appian_db.AddressView; create table CustomerView as select * from appian_db.CustomerView; create table OrderView as select * from appian_db.OrderView; diff --git a/benchmark/micro/optimizer/topn_window_elimination.benchmark b/benchmark/micro/optimizer/topn_window_elimination.benchmark new file mode 100644 index 000000000000..a91df5703f0d --- /dev/null +++ b/benchmark/micro/optimizer/topn_window_elimination.benchmark @@ -0,0 +1,23 @@ +# name: benchmark/micro/optimizer/topn_window_elimination.benchmark +# description: Benchmark of top n window elimination +# group: [optimizer] + +name TopN Window Elimination +group micro +subgroup optimizer + +load +CREATE TABLE metrics AS ( + SELECT k, '2001-01-01 00:00:00'::TIMESTAMP + INTERVAL (v) MINUTE AS tm, v AS v1, v % 1000 AS v2, v % 100 as v3 + FROM range(0,100000) vals(v), range(0,100) keys(k) +); +CREATE TABLE tags AS ( + SELECT k, CAST(hash(k+1) AS VARCHAR) t1, CAST(hash(k+2) AS VARCHAR) t2, CAST(hash(k+3) AS VARCHAR) t3, + FROM range(0,100) keys(k) +); + +run +SELECT * FROM tags t INNER JOIN LATERAL (SELECT * FROM metrics m WHERE m.k = t.k ORDER BY tm DESC LIMIT 1) AS b ON true ORDER BY t.k, b.tm DESC; +SELECT * FROM tags t INNER JOIN LATERAL (SELECT * FROM metrics m WHERE m.k = t.k ORDER BY tm DESC LIMIT 3) AS b ON true ORDER BY t.k, b.tm DESC; +SELECT * FROM tags t, (SELECT *, row_number() OVER (PARTITION BY m.k ORDER BY m.tm DESC) rn FROM metrics m QUALIFY rn <= 1) m WHERE t.k = m.k; +SELECT * FROM tags t, (SELECT *, row_number() OVER (PARTITION BY m.k ORDER BY m.tm DESC) rn FROM metrics m QUALIFY rn <= 3) m WHERE t.k = m.k; diff --git a/benchmark/micro/optimizer/unnest_join_filter_pushdown.benchmark b/benchmark/micro/optimizer/unnest_join_filter_pushdown.benchmark new file mode 100644 index 000000000000..601bab76784e --- /dev/null +++ b/benchmark/micro/optimizer/unnest_join_filter_pushdown.benchmark @@ -0,0 +1,14 @@ +# name: benchmark/micro/optimizer/unnest_join_filter_pushdown.benchmark +# description: Benchmark filter pushdown with unnest +# group: [optimizer] + +name Unnest Filter Pushdown +group micro +subgroup optimizer + +load +CREATE TABLE test_table as +SELECT s.i as id, [1, 2, 3]::bigint[] AS VALUES FROM generate_series(1, 10000000) AS s(i); + +run +SELECT id, value FROM test_table CROSS JOIN UNNEST(values) AS values(value) WHERE id = 87100; \ No newline at end of file diff --git a/benchmark/realnest/hep/load.sql b/benchmark/realnest/hep/load.sql index c7bfb6044455..057e08386062 100644 --- a/benchmark/realnest/hep/load.sql +++ b/benchmark/realnest/hep/load.sql @@ -1,4 +1,4 @@ -CREATE TABLE hep_singleMu AS SELECT * FROM READ_PARQUET('s3://duckdb-blobs/data/realnest/Run2012B_SingleMu_restructured_1000.parquet'); +CREATE TABLE hep_singleMu AS SELECT * FROM READ_PARQUET('https://blobs.duckdb.org/data/realnest/Run2012B_SingleMu_restructured_1000.parquet'); CREATE FUNCTION Pi() AS (ACOS(-1)); @@ -75,4 +75,4 @@ CREATE OR REPLACE FUNCTION HistogramBinHelper( CREATE OR REPLACE FUNCTION HistogramBin( value, lo, hi, num_bins) AS ( HistogramBinHelper(value, lo, hi, (hi - lo) / num_bins) -); \ No newline at end of file +); diff --git a/benchmark/realnest/micro/load.sql b/benchmark/realnest/micro/load.sql index de06b949bd57..ae22ed6b1e1b 100644 --- a/benchmark/realnest/micro/load.sql +++ b/benchmark/realnest/micro/load.sql @@ -1,14 +1,14 @@ -ATTACH 'https://duckdb-blobs.s3.amazonaws.com/data/realnest/cord_10k.duckdb' AS cord (READ_ONLY); +ATTACH 'https://blobs.duckdb.org/data/realnest/cord_10k.duckdb' AS cord (READ_ONLY); CREATE TABLE cord AS SELECT * FROM cord.cord; -ATTACH 'https://duckdb-blobs.s3.amazonaws.com/data/realnest/open_street_map_524k.duckdb' AS osm (READ_ONLY); +ATTACH 'https://blobs.duckdb.org/data/realnest/open_street_map_524k.duckdb' AS osm (READ_ONLY); CREATE TABLE open_street_map AS SELECT * FROM osm.open_street_map; -ATTACH 'https://duckdb-blobs.s3.amazonaws.com/data/realnest/pull_131k.duckdb' AS gh_pull (READ_ONLY); +ATTACH 'https://blobs.duckdb.org/data/realnest/pull_131k.duckdb' AS gh_pull (READ_ONLY); CREATE TABLE gh_pull AS SELECT * FROM gh_pull.gh_pull; -ATTACH 'https://duckdb-blobs.s3.amazonaws.com/data/realnest/issue_131k.duckdb' AS gh_issue (READ_ONLY); +ATTACH 'https://blobs.duckdb.org/data/realnest/issue_131k.duckdb' AS gh_issue (READ_ONLY); CREATE TABLE gh_issue AS SELECT * FROM gh_issue.gh_issue; -ATTACH 'https://duckdb-blobs.s3.amazonaws.com/data/realnest/twitter_131k.duckdb' AS tw (READ_ONLY); +ATTACH 'https://blobs.duckdb.org/data/realnest/twitter_131k.duckdb' AS tw (READ_ONLY); CREATE TABLE twitter AS SELECT * FROM tw.twitter; -ATTACH 'https://duckdb-blobs.s3.amazonaws.com/data/realnest/singleMu_524k.duckdb' AS rn_singleMu (READ_ONLY); +ATTACH 'https://blobs.duckdb.org/data/realnest/singleMu_524k.duckdb' AS rn_singleMu (READ_ONLY); CREATE TABLE run2012B_singleMu AS SELECT * FROM rn_singleMu.run2012B_singleMu; CREATE TABLE single_mu_lists AS SELECT * REPLACE( list_resize(Jet, 10, NULL) AS Jet, list_resize(Muon, 10, NULL) AS Muon, diff --git a/data/csv/afl/3981/case_0.csv b/data/csv/afl/3981/case_0.csv deleted file mode 100644 index 59390ec49901..000000000000 Binary files a/data/csv/afl/3981/case_0.csv and /dev/null differ diff --git a/data/csv/afl/3981/case_1.csv b/data/csv/afl/3981/case_1.csv deleted file mode 100644 index a8919290cb72..000000000000 Binary files a/data/csv/afl/3981/case_1.csv and /dev/null differ diff --git a/data/csv/afl/3981/case_2.csv b/data/csv/afl/3981/case_2.csv deleted file mode 100644 index 2154533db63a..000000000000 Binary files a/data/csv/afl/3981/case_2.csv and /dev/null differ diff --git a/data/csv/afl/3981/case_3.csv b/data/csv/afl/3981/case_3.csv deleted file mode 100644 index 7fb006c47f72..000000000000 Binary files a/data/csv/afl/3981/case_3.csv and /dev/null differ diff --git a/data/csv/afl/3981/case_4.csv b/data/csv/afl/3981/case_4.csv deleted file mode 100644 index d73484ecd2ed..000000000000 Binary files a/data/csv/afl/3981/case_4.csv and /dev/null differ diff --git a/data/csv/afl/3981/case_5.csv b/data/csv/afl/3981/case_5.csv deleted file mode 100644 index 7e5b80b63bdd..000000000000 Binary files a/data/csv/afl/3981/case_5.csv and /dev/null differ diff --git a/data/csv/afl/3981/case_6.csv b/data/csv/afl/3981/case_6.csv deleted file mode 100644 index 3a200bab6ab5..000000000000 Binary files a/data/csv/afl/3981/case_6.csv and /dev/null differ diff --git a/data/parquet-testing/broken/internal_6129.parquet b/data/parquet-testing/broken/internal_6129.parquet new file mode 100644 index 000000000000..bf426a659c49 Binary files /dev/null and b/data/parquet-testing/broken/internal_6129.parquet differ diff --git a/data/parquet-testing/broken/internal_6165.parquet b/data/parquet-testing/broken/internal_6165.parquet new file mode 100644 index 000000000000..00f245b9e9ab Binary files /dev/null and b/data/parquet-testing/broken/internal_6165.parquet differ diff --git a/data/parquet-testing/internal_5994.parquet b/data/parquet-testing/internal_5994.parquet new file mode 100644 index 000000000000..8dba5bdcadf1 Binary files /dev/null and b/data/parquet-testing/internal_5994.parquet differ diff --git a/data/parquet-testing/internal_6129.parquet b/data/parquet-testing/internal_6129.parquet new file mode 100644 index 000000000000..bf426a659c49 Binary files /dev/null and b/data/parquet-testing/internal_6129.parquet differ diff --git a/data/parquet-testing/variant_array_array_string.parquet b/data/parquet-testing/variant_array_array_string.parquet index d6934634c0a1..3a69c824b9d5 100644 Binary files a/data/parquet-testing/variant_array_array_string.parquet and b/data/parquet-testing/variant_array_array_string.parquet differ diff --git a/data/parquet-testing/variant_array_array_string_and_integer.parquet b/data/parquet-testing/variant_array_array_string_and_integer.parquet index 6c8bd4bef26f..d1df57eed094 100644 Binary files a/data/parquet-testing/variant_array_array_string_and_integer.parquet and b/data/parquet-testing/variant_array_array_string_and_integer.parquet differ diff --git a/data/parquet-testing/variant_array_empty.parquet b/data/parquet-testing/variant_array_empty.parquet index 13add28ef1f8..aa6e7db1cb65 100644 Binary files a/data/parquet-testing/variant_array_empty.parquet and b/data/parquet-testing/variant_array_empty.parquet differ diff --git a/data/parquet-testing/variant_array_object_string.parquet b/data/parquet-testing/variant_array_object_string.parquet index eb3d7f3071ce..12590ce0d622 100644 Binary files a/data/parquet-testing/variant_array_object_string.parquet and b/data/parquet-testing/variant_array_object_string.parquet differ diff --git a/data/parquet-testing/variant_array_object_string_and_integer.parquet b/data/parquet-testing/variant_array_object_string_and_integer.parquet index 0aa803d9ef77..ab750a58360f 100644 Binary files a/data/parquet-testing/variant_array_object_string_and_integer.parquet and b/data/parquet-testing/variant_array_object_string_and_integer.parquet differ diff --git a/data/parquet-testing/variant_array_string.parquet b/data/parquet-testing/variant_array_string.parquet index e31c07aa1a93..98fc7d3c8990 100644 Binary files a/data/parquet-testing/variant_array_string.parquet and b/data/parquet-testing/variant_array_string.parquet differ diff --git a/data/parquet-testing/variant_binary.parquet b/data/parquet-testing/variant_binary.parquet index 7b5bcc777d60..a0687858e187 100644 Binary files a/data/parquet-testing/variant_binary.parquet and b/data/parquet-testing/variant_binary.parquet differ diff --git a/data/parquet-testing/variant_bool_false.parquet b/data/parquet-testing/variant_bool_false.parquet index 8225f8bd135b..c70c48ce472e 100644 Binary files a/data/parquet-testing/variant_bool_false.parquet and b/data/parquet-testing/variant_bool_false.parquet differ diff --git a/data/parquet-testing/variant_bool_true.parquet b/data/parquet-testing/variant_bool_true.parquet index ae18fd45f206..6a74a5ed1ad1 100644 Binary files a/data/parquet-testing/variant_bool_true.parquet and b/data/parquet-testing/variant_bool_true.parquet differ diff --git a/data/parquet-testing/variant_date_negative.parquet b/data/parquet-testing/variant_date_negative.parquet index 52a630d7234d..49ca71406f0b 100644 Binary files a/data/parquet-testing/variant_date_negative.parquet and b/data/parquet-testing/variant_date_negative.parquet differ diff --git a/data/parquet-testing/variant_date_positive.parquet b/data/parquet-testing/variant_date_positive.parquet index 0ec114498068..86f08a6ecde6 100644 Binary files a/data/parquet-testing/variant_date_positive.parquet and b/data/parquet-testing/variant_date_positive.parquet differ diff --git a/data/parquet-testing/variant_decimal16.parquet b/data/parquet-testing/variant_decimal16.parquet index a519da5ce8e2..a54dc1d647db 100644 Binary files a/data/parquet-testing/variant_decimal16.parquet and b/data/parquet-testing/variant_decimal16.parquet differ diff --git a/data/parquet-testing/variant_decimal16_negative.parquet b/data/parquet-testing/variant_decimal16_negative.parquet index f1f3311e4862..2d232801ba66 100644 Binary files a/data/parquet-testing/variant_decimal16_negative.parquet and b/data/parquet-testing/variant_decimal16_negative.parquet differ diff --git a/data/parquet-testing/variant_decimal4_negative.parquet b/data/parquet-testing/variant_decimal4_negative.parquet index d7c02e84c865..fe5219b6bde9 100644 Binary files a/data/parquet-testing/variant_decimal4_negative.parquet and b/data/parquet-testing/variant_decimal4_negative.parquet differ diff --git a/data/parquet-testing/variant_decimal4_positive.parquet b/data/parquet-testing/variant_decimal4_positive.parquet index 96767ec586c8..92ae431725fd 100644 Binary files a/data/parquet-testing/variant_decimal4_positive.parquet and b/data/parquet-testing/variant_decimal4_positive.parquet differ diff --git a/data/parquet-testing/variant_decimal8_negative.parquet b/data/parquet-testing/variant_decimal8_negative.parquet index d73e13e5fb42..b942ce92d694 100644 Binary files a/data/parquet-testing/variant_decimal8_negative.parquet and b/data/parquet-testing/variant_decimal8_negative.parquet differ diff --git a/data/parquet-testing/variant_double_negative.parquet b/data/parquet-testing/variant_double_negative.parquet index f684206e0cb9..9f877d77166e 100644 Binary files a/data/parquet-testing/variant_double_negative.parquet and b/data/parquet-testing/variant_double_negative.parquet differ diff --git a/data/parquet-testing/variant_double_positive.parquet b/data/parquet-testing/variant_double_positive.parquet index 1b5ad53c98b1..035064dcddda 100644 Binary files a/data/parquet-testing/variant_double_positive.parquet and b/data/parquet-testing/variant_double_positive.parquet differ diff --git a/data/parquet-testing/variant_float_negative.parquet b/data/parquet-testing/variant_float_negative.parquet index a23eb3f977c1..3bbe66cdb7a8 100644 Binary files a/data/parquet-testing/variant_float_negative.parquet and b/data/parquet-testing/variant_float_negative.parquet differ diff --git a/data/parquet-testing/variant_int16.parquet b/data/parquet-testing/variant_int16.parquet index 31a5bebaf3d1..baa7cbace20b 100644 Binary files a/data/parquet-testing/variant_int16.parquet and b/data/parquet-testing/variant_int16.parquet differ diff --git a/data/parquet-testing/variant_int16_positive.parquet b/data/parquet-testing/variant_int16_positive.parquet index 2b0dee0892f7..d8536907bda0 100644 Binary files a/data/parquet-testing/variant_int16_positive.parquet and b/data/parquet-testing/variant_int16_positive.parquet differ diff --git a/data/parquet-testing/variant_int32.parquet b/data/parquet-testing/variant_int32.parquet index 9e60f20fd629..a604654d2168 100644 Binary files a/data/parquet-testing/variant_int32.parquet and b/data/parquet-testing/variant_int32.parquet differ diff --git a/data/parquet-testing/variant_int32_positive.parquet b/data/parquet-testing/variant_int32_positive.parquet index aa689d1eec1a..78d057ef63ad 100644 Binary files a/data/parquet-testing/variant_int32_positive.parquet and b/data/parquet-testing/variant_int32_positive.parquet differ diff --git a/data/parquet-testing/variant_int64.parquet b/data/parquet-testing/variant_int64.parquet index 3b69b23b7a55..45927679d85e 100644 Binary files a/data/parquet-testing/variant_int64.parquet and b/data/parquet-testing/variant_int64.parquet differ diff --git a/data/parquet-testing/variant_int64_positive.parquet b/data/parquet-testing/variant_int64_positive.parquet index 3d31a6aa087d..6f1d51c9936d 100644 Binary files a/data/parquet-testing/variant_int64_positive.parquet and b/data/parquet-testing/variant_int64_positive.parquet differ diff --git a/data/parquet-testing/variant_int8_negative.parquet b/data/parquet-testing/variant_int8_negative.parquet index cf944f2d24af..0420eb46bd42 100644 Binary files a/data/parquet-testing/variant_int8_negative.parquet and b/data/parquet-testing/variant_int8_negative.parquet differ diff --git a/data/parquet-testing/variant_int8_positive.parquet b/data/parquet-testing/variant_int8_positive.parquet index 48d3cdf497b2..c02e19db404a 100644 Binary files a/data/parquet-testing/variant_int8_positive.parquet and b/data/parquet-testing/variant_int8_positive.parquet differ diff --git a/data/parquet-testing/variant_null.parquet b/data/parquet-testing/variant_null.parquet index 7fb04bab7aa9..89132f751303 100644 Binary files a/data/parquet-testing/variant_null.parquet and b/data/parquet-testing/variant_null.parquet differ diff --git a/data/parquet-testing/variant_object_empty.parquet b/data/parquet-testing/variant_object_empty.parquet index 25f091f86e67..897c33d81566 100644 Binary files a/data/parquet-testing/variant_object_empty.parquet and b/data/parquet-testing/variant_object_empty.parquet differ diff --git a/data/parquet-testing/variant_object_null_and_string.parquet b/data/parquet-testing/variant_object_null_and_string.parquet index b599a03c5f63..fdf70cd4b9be 100644 Binary files a/data/parquet-testing/variant_object_null_and_string.parquet and b/data/parquet-testing/variant_object_null_and_string.parquet differ diff --git a/data/parquet-testing/variant_object_primitives.parquet b/data/parquet-testing/variant_object_primitives.parquet index 85082eebcf9b..730b49a09561 100644 Binary files a/data/parquet-testing/variant_object_primitives.parquet and b/data/parquet-testing/variant_object_primitives.parquet differ diff --git a/data/parquet-testing/variant_object_string_and_array.parquet b/data/parquet-testing/variant_object_string_and_array.parquet index edd4f20061e0..ec723790e20e 100644 Binary files a/data/parquet-testing/variant_object_string_and_array.parquet and b/data/parquet-testing/variant_object_string_and_array.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded0.parquet b/data/parquet-testing/variant_partial_shredded0.parquet deleted file mode 100644 index bee3475b31af..000000000000 Binary files a/data/parquet-testing/variant_partial_shredded0.parquet and /dev/null differ diff --git a/data/parquet-testing/variant_partial_shredded1.parquet b/data/parquet-testing/variant_partial_shredded1.parquet index 2bedcfa396eb..b5fd347bce1c 100644 Binary files a/data/parquet-testing/variant_partial_shredded1.parquet and b/data/parquet-testing/variant_partial_shredded1.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded10.parquet b/data/parquet-testing/variant_partial_shredded10.parquet index 2fbdd1524b6e..540f365c4d0f 100644 Binary files a/data/parquet-testing/variant_partial_shredded10.parquet and b/data/parquet-testing/variant_partial_shredded10.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded11.parquet b/data/parquet-testing/variant_partial_shredded11.parquet index fa062b7e67e6..4395b6a7aa8b 100644 Binary files a/data/parquet-testing/variant_partial_shredded11.parquet and b/data/parquet-testing/variant_partial_shredded11.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded12.parquet b/data/parquet-testing/variant_partial_shredded12.parquet index 015b268fd8fa..886eba044e1e 100644 Binary files a/data/parquet-testing/variant_partial_shredded12.parquet and b/data/parquet-testing/variant_partial_shredded12.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded13.parquet b/data/parquet-testing/variant_partial_shredded13.parquet index fa062b7e67e6..3348b552b64a 100644 Binary files a/data/parquet-testing/variant_partial_shredded13.parquet and b/data/parquet-testing/variant_partial_shredded13.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded14.parquet b/data/parquet-testing/variant_partial_shredded14.parquet index 6cf0b58661c1..5f6910e45318 100644 Binary files a/data/parquet-testing/variant_partial_shredded14.parquet and b/data/parquet-testing/variant_partial_shredded14.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded15.parquet b/data/parquet-testing/variant_partial_shredded15.parquet index ce8296fbe8da..b05cc2913e7e 100644 Binary files a/data/parquet-testing/variant_partial_shredded15.parquet and b/data/parquet-testing/variant_partial_shredded15.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded16.parquet b/data/parquet-testing/variant_partial_shredded16.parquet index 4b9e29ffe50f..681cd9609c28 100644 Binary files a/data/parquet-testing/variant_partial_shredded16.parquet and b/data/parquet-testing/variant_partial_shredded16.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded17.parquet b/data/parquet-testing/variant_partial_shredded17.parquet index 2bfb70f60b64..55fc79667cd4 100644 Binary files a/data/parquet-testing/variant_partial_shredded17.parquet and b/data/parquet-testing/variant_partial_shredded17.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded18.parquet b/data/parquet-testing/variant_partial_shredded18.parquet index 24940d7eafd4..0e86944ea332 100644 Binary files a/data/parquet-testing/variant_partial_shredded18.parquet and b/data/parquet-testing/variant_partial_shredded18.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded19.parquet b/data/parquet-testing/variant_partial_shredded19.parquet index 7b46b84f8072..5f6910e45318 100644 Binary files a/data/parquet-testing/variant_partial_shredded19.parquet and b/data/parquet-testing/variant_partial_shredded19.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded2.parquet b/data/parquet-testing/variant_partial_shredded2.parquet index 070d756527cf..c53f4985cdba 100644 Binary files a/data/parquet-testing/variant_partial_shredded2.parquet and b/data/parquet-testing/variant_partial_shredded2.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded20.parquet b/data/parquet-testing/variant_partial_shredded20.parquet index cda196fb59f4..b5fd347bce1c 100644 Binary files a/data/parquet-testing/variant_partial_shredded20.parquet and b/data/parquet-testing/variant_partial_shredded20.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded21.parquet b/data/parquet-testing/variant_partial_shredded21.parquet index db9b8ca80ac4..9afc1c04bb96 100644 Binary files a/data/parquet-testing/variant_partial_shredded21.parquet and b/data/parquet-testing/variant_partial_shredded21.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded22.parquet b/data/parquet-testing/variant_partial_shredded22.parquet index cda196fb59f4..c15ebeb8e032 100644 Binary files a/data/parquet-testing/variant_partial_shredded22.parquet and b/data/parquet-testing/variant_partial_shredded22.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded23.parquet b/data/parquet-testing/variant_partial_shredded23.parquet index 2fbdd1524b6e..73f1d2dd72e8 100644 Binary files a/data/parquet-testing/variant_partial_shredded23.parquet and b/data/parquet-testing/variant_partial_shredded23.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded24.parquet b/data/parquet-testing/variant_partial_shredded24.parquet index 2bedcfa396eb..b05cc2913e7e 100644 Binary files a/data/parquet-testing/variant_partial_shredded24.parquet and b/data/parquet-testing/variant_partial_shredded24.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded25.parquet b/data/parquet-testing/variant_partial_shredded25.parquet index 6cf0b58661c1..444ef94018df 100644 Binary files a/data/parquet-testing/variant_partial_shredded25.parquet and b/data/parquet-testing/variant_partial_shredded25.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded26.parquet b/data/parquet-testing/variant_partial_shredded26.parquet index b1d7d9dd38ed..0e86944ea332 100644 Binary files a/data/parquet-testing/variant_partial_shredded26.parquet and b/data/parquet-testing/variant_partial_shredded26.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded27.parquet b/data/parquet-testing/variant_partial_shredded27.parquet index aa3f9d461d17..cd4c7540f623 100644 Binary files a/data/parquet-testing/variant_partial_shredded27.parquet and b/data/parquet-testing/variant_partial_shredded27.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded28.parquet b/data/parquet-testing/variant_partial_shredded28.parquet index cda196fb59f4..31dd31fdc66f 100644 Binary files a/data/parquet-testing/variant_partial_shredded28.parquet and b/data/parquet-testing/variant_partial_shredded28.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded29.parquet b/data/parquet-testing/variant_partial_shredded29.parquet index 7b46b84f8072..c15ebeb8e032 100644 Binary files a/data/parquet-testing/variant_partial_shredded29.parquet and b/data/parquet-testing/variant_partial_shredded29.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded3.parquet b/data/parquet-testing/variant_partial_shredded3.parquet index cda196fb59f4..6d7468991f79 100644 Binary files a/data/parquet-testing/variant_partial_shredded3.parquet and b/data/parquet-testing/variant_partial_shredded3.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded30.parquet b/data/parquet-testing/variant_partial_shredded30.parquet index 42247e00a92c..73f1d2dd72e8 100644 Binary files a/data/parquet-testing/variant_partial_shredded30.parquet and b/data/parquet-testing/variant_partial_shredded30.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded31.parquet b/data/parquet-testing/variant_partial_shredded31.parquet index b6215f42a12f..82f0cdc548d5 100644 Binary files a/data/parquet-testing/variant_partial_shredded31.parquet and b/data/parquet-testing/variant_partial_shredded31.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded32.parquet b/data/parquet-testing/variant_partial_shredded32.parquet index cda196fb59f4..55fc79667cd4 100644 Binary files a/data/parquet-testing/variant_partial_shredded32.parquet and b/data/parquet-testing/variant_partial_shredded32.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded33.parquet b/data/parquet-testing/variant_partial_shredded33.parquet index b1d7d9dd38ed..f87526f7b723 100644 Binary files a/data/parquet-testing/variant_partial_shredded33.parquet and b/data/parquet-testing/variant_partial_shredded33.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded34.parquet b/data/parquet-testing/variant_partial_shredded34.parquet index 16b1afd42455..69371dd49346 100644 Binary files a/data/parquet-testing/variant_partial_shredded34.parquet and b/data/parquet-testing/variant_partial_shredded34.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded35.parquet b/data/parquet-testing/variant_partial_shredded35.parquet index cda196fb59f4..9afc1c04bb96 100644 Binary files a/data/parquet-testing/variant_partial_shredded35.parquet and b/data/parquet-testing/variant_partial_shredded35.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded36.parquet b/data/parquet-testing/variant_partial_shredded36.parquet index ce305dc4a38e..9afc1c04bb96 100644 Binary files a/data/parquet-testing/variant_partial_shredded36.parquet and b/data/parquet-testing/variant_partial_shredded36.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded37.parquet b/data/parquet-testing/variant_partial_shredded37.parquet index c57cf9869d83..bd7886efc57d 100644 Binary files a/data/parquet-testing/variant_partial_shredded37.parquet and b/data/parquet-testing/variant_partial_shredded37.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded38.parquet b/data/parquet-testing/variant_partial_shredded38.parquet index 219d890a7e3b..3348b552b64a 100644 Binary files a/data/parquet-testing/variant_partial_shredded38.parquet and b/data/parquet-testing/variant_partial_shredded38.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded39.parquet b/data/parquet-testing/variant_partial_shredded39.parquet index 639822199d82..bd7886efc57d 100644 Binary files a/data/parquet-testing/variant_partial_shredded39.parquet and b/data/parquet-testing/variant_partial_shredded39.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded4.parquet b/data/parquet-testing/variant_partial_shredded4.parquet index e8b252ca9e95..9afc1c04bb96 100644 Binary files a/data/parquet-testing/variant_partial_shredded4.parquet and b/data/parquet-testing/variant_partial_shredded4.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded40.parquet b/data/parquet-testing/variant_partial_shredded40.parquet index 2bfb70f60b64..097f77084d52 100644 Binary files a/data/parquet-testing/variant_partial_shredded40.parquet and b/data/parquet-testing/variant_partial_shredded40.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded41.parquet b/data/parquet-testing/variant_partial_shredded41.parquet index 24940d7eafd4..9afc1c04bb96 100644 Binary files a/data/parquet-testing/variant_partial_shredded41.parquet and b/data/parquet-testing/variant_partial_shredded41.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded42.parquet b/data/parquet-testing/variant_partial_shredded42.parquet index ce305dc4a38e..5c660e0235cf 100644 Binary files a/data/parquet-testing/variant_partial_shredded42.parquet and b/data/parquet-testing/variant_partial_shredded42.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded43.parquet b/data/parquet-testing/variant_partial_shredded43.parquet index 4b9e29ffe50f..6d7468991f79 100644 Binary files a/data/parquet-testing/variant_partial_shredded43.parquet and b/data/parquet-testing/variant_partial_shredded43.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded44.parquet b/data/parquet-testing/variant_partial_shredded44.parquet index 63879ac551b3..398f0f3441fc 100644 Binary files a/data/parquet-testing/variant_partial_shredded44.parquet and b/data/parquet-testing/variant_partial_shredded44.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded45.parquet b/data/parquet-testing/variant_partial_shredded45.parquet index 16b1afd42455..8f09bd60aac2 100644 Binary files a/data/parquet-testing/variant_partial_shredded45.parquet and b/data/parquet-testing/variant_partial_shredded45.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded46.parquet b/data/parquet-testing/variant_partial_shredded46.parquet new file mode 100644 index 000000000000..cd4c7540f623 Binary files /dev/null and b/data/parquet-testing/variant_partial_shredded46.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded5.parquet b/data/parquet-testing/variant_partial_shredded5.parquet index b6215f42a12f..886eba044e1e 100644 Binary files a/data/parquet-testing/variant_partial_shredded5.parquet and b/data/parquet-testing/variant_partial_shredded5.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded6.parquet b/data/parquet-testing/variant_partial_shredded6.parquet index 639822199d82..540f365c4d0f 100644 Binary files a/data/parquet-testing/variant_partial_shredded6.parquet and b/data/parquet-testing/variant_partial_shredded6.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded7.parquet b/data/parquet-testing/variant_partial_shredded7.parquet index 29e84b444bc0..9afc1c04bb96 100644 Binary files a/data/parquet-testing/variant_partial_shredded7.parquet and b/data/parquet-testing/variant_partial_shredded7.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded8.parquet b/data/parquet-testing/variant_partial_shredded8.parquet index c57cf9869d83..444ef94018df 100644 Binary files a/data/parquet-testing/variant_partial_shredded8.parquet and b/data/parquet-testing/variant_partial_shredded8.parquet differ diff --git a/data/parquet-testing/variant_partial_shredded9.parquet b/data/parquet-testing/variant_partial_shredded9.parquet index 42247e00a92c..097f77084d52 100644 Binary files a/data/parquet-testing/variant_partial_shredded9.parquet and b/data/parquet-testing/variant_partial_shredded9.parquet differ diff --git a/data/parquet-testing/variant_shredded_array1.parquet b/data/parquet-testing/variant_shredded_array1.parquet index 636f041804ef..1efc9bec49ed 100644 Binary files a/data/parquet-testing/variant_shredded_array1.parquet and b/data/parquet-testing/variant_shredded_array1.parquet differ diff --git a/data/parquet-testing/variant_shredded_array2.parquet b/data/parquet-testing/variant_shredded_array2.parquet index 6a70c95d6ce6..12590ce0d622 100644 Binary files a/data/parquet-testing/variant_shredded_array2.parquet and b/data/parquet-testing/variant_shredded_array2.parquet differ diff --git a/data/parquet-testing/variant_shredded_array3.parquet b/data/parquet-testing/variant_shredded_array3.parquet index d5961872a76e..03500e561543 100644 Binary files a/data/parquet-testing/variant_shredded_array3.parquet and b/data/parquet-testing/variant_shredded_array3.parquet differ diff --git a/data/parquet-testing/variant_shredded_binary.parquet b/data/parquet-testing/variant_shredded_binary.parquet index 366c725a0926..4c3b7409a236 100644 Binary files a/data/parquet-testing/variant_shredded_binary.parquet and b/data/parquet-testing/variant_shredded_binary.parquet differ diff --git a/data/parquet-testing/variant_shredded_bool_false.parquet b/data/parquet-testing/variant_shredded_bool_false.parquet index b97ec4867e69..f3ba9041ff1c 100644 Binary files a/data/parquet-testing/variant_shredded_bool_false.parquet and b/data/parquet-testing/variant_shredded_bool_false.parquet differ diff --git a/data/parquet-testing/variant_shredded_bool_true.parquet b/data/parquet-testing/variant_shredded_bool_true.parquet index 990a3ab6f1e7..48010dcf7ab4 100644 Binary files a/data/parquet-testing/variant_shredded_bool_true.parquet and b/data/parquet-testing/variant_shredded_bool_true.parquet differ diff --git a/data/parquet-testing/variant_shredded_date_negative.parquet b/data/parquet-testing/variant_shredded_date_negative.parquet index 2a0fa6be1ae3..49ca71406f0b 100644 Binary files a/data/parquet-testing/variant_shredded_date_negative.parquet and b/data/parquet-testing/variant_shredded_date_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_date_positive.parquet b/data/parquet-testing/variant_shredded_date_positive.parquet index edc1de77b5ec..86f08a6ecde6 100644 Binary files a/data/parquet-testing/variant_shredded_date_positive.parquet and b/data/parquet-testing/variant_shredded_date_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_decimal16_negative.parquet b/data/parquet-testing/variant_shredded_decimal16_negative.parquet index 8b4fe4cb198e..2d232801ba66 100644 Binary files a/data/parquet-testing/variant_shredded_decimal16_negative.parquet and b/data/parquet-testing/variant_shredded_decimal16_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_decimal16_positive.parquet b/data/parquet-testing/variant_shredded_decimal16_positive.parquet index 6c7658b938ac..7c78e7c43501 100644 Binary files a/data/parquet-testing/variant_shredded_decimal16_positive.parquet and b/data/parquet-testing/variant_shredded_decimal16_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_decimal4_negative.parquet b/data/parquet-testing/variant_shredded_decimal4_negative.parquet index 78fefc0f99a8..5c979ab7835e 100644 Binary files a/data/parquet-testing/variant_shredded_decimal4_negative.parquet and b/data/parquet-testing/variant_shredded_decimal4_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_decimal4_positive.parquet b/data/parquet-testing/variant_shredded_decimal4_positive.parquet index 441c70504450..f3aad6f7fa59 100644 Binary files a/data/parquet-testing/variant_shredded_decimal4_positive.parquet and b/data/parquet-testing/variant_shredded_decimal4_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_decimal8_negative.parquet b/data/parquet-testing/variant_shredded_decimal8_negative.parquet index 81ba1e9bd1d1..d16966654f8d 100644 Binary files a/data/parquet-testing/variant_shredded_decimal8_negative.parquet and b/data/parquet-testing/variant_shredded_decimal8_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_decimal8_positive.parquet b/data/parquet-testing/variant_shredded_decimal8_positive.parquet index 7e2abd2d37b2..36d47db3140b 100644 Binary files a/data/parquet-testing/variant_shredded_decimal8_positive.parquet and b/data/parquet-testing/variant_shredded_decimal8_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_double_negative.parquet b/data/parquet-testing/variant_shredded_double_negative.parquet index 424f27e1259e..2697af1f2f08 100644 Binary files a/data/parquet-testing/variant_shredded_double_negative.parquet and b/data/parquet-testing/variant_shredded_double_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_double_positive.parquet b/data/parquet-testing/variant_shredded_double_positive.parquet index bda4dadfcdd0..5ed87ae8a235 100644 Binary files a/data/parquet-testing/variant_shredded_double_positive.parquet and b/data/parquet-testing/variant_shredded_double_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_float_negative.parquet b/data/parquet-testing/variant_shredded_float_negative.parquet index 67caf8463bbb..6b3f27edf42e 100644 Binary files a/data/parquet-testing/variant_shredded_float_negative.parquet and b/data/parquet-testing/variant_shredded_float_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_float_positive.parquet b/data/parquet-testing/variant_shredded_float_positive.parquet index dbac72a1f664..39e103a57d19 100644 Binary files a/data/parquet-testing/variant_shredded_float_positive.parquet and b/data/parquet-testing/variant_shredded_float_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_int16_negative.parquet b/data/parquet-testing/variant_shredded_int16_negative.parquet index 107311941776..fe7fa002dcd9 100644 Binary files a/data/parquet-testing/variant_shredded_int16_negative.parquet and b/data/parquet-testing/variant_shredded_int16_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_int16_positive.parquet b/data/parquet-testing/variant_shredded_int16_positive.parquet index 2c42eeaf0f7d..d8536907bda0 100644 Binary files a/data/parquet-testing/variant_shredded_int16_positive.parquet and b/data/parquet-testing/variant_shredded_int16_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_int32_negative.parquet b/data/parquet-testing/variant_shredded_int32_negative.parquet index f83ba3e2b64b..eaf5817eb833 100644 Binary files a/data/parquet-testing/variant_shredded_int32_negative.parquet and b/data/parquet-testing/variant_shredded_int32_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_int32_positive.parquet b/data/parquet-testing/variant_shredded_int32_positive.parquet index 8fc805ce76af..24303005a298 100644 Binary files a/data/parquet-testing/variant_shredded_int32_positive.parquet and b/data/parquet-testing/variant_shredded_int32_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_int64_negative.parquet b/data/parquet-testing/variant_shredded_int64_negative.parquet index feff7d46df19..45927679d85e 100644 Binary files a/data/parquet-testing/variant_shredded_int64_negative.parquet and b/data/parquet-testing/variant_shredded_int64_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_int64_positive.parquet b/data/parquet-testing/variant_shredded_int64_positive.parquet index 7cc2bcd7faa8..6f1d51c9936d 100644 Binary files a/data/parquet-testing/variant_shredded_int64_positive.parquet and b/data/parquet-testing/variant_shredded_int64_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_int8_negative.parquet b/data/parquet-testing/variant_shredded_int8_negative.parquet index d0630b2a407e..d54edf50ce65 100644 Binary files a/data/parquet-testing/variant_shredded_int8_negative.parquet and b/data/parquet-testing/variant_shredded_int8_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_int8_positive.parquet b/data/parquet-testing/variant_shredded_int8_positive.parquet index 0073802aadbf..adc984fc596c 100644 Binary files a/data/parquet-testing/variant_shredded_int8_positive.parquet and b/data/parquet-testing/variant_shredded_int8_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_object1.parquet b/data/parquet-testing/variant_shredded_object1.parquet index 404c3729ae6e..730b49a09561 100644 Binary files a/data/parquet-testing/variant_shredded_object1.parquet and b/data/parquet-testing/variant_shredded_object1.parquet differ diff --git a/data/parquet-testing/variant_shredded_object2.parquet b/data/parquet-testing/variant_shredded_object2.parquet index da8151283ad8..fc18ff26c4d1 100644 Binary files a/data/parquet-testing/variant_shredded_object2.parquet and b/data/parquet-testing/variant_shredded_object2.parquet differ diff --git a/data/parquet-testing/variant_shredded_object3.parquet b/data/parquet-testing/variant_shredded_object3.parquet index 756bc43c9fd7..b6f839c736ff 100644 Binary files a/data/parquet-testing/variant_shredded_object3.parquet and b/data/parquet-testing/variant_shredded_object3.parquet differ diff --git a/data/parquet-testing/variant_shredded_string.parquet b/data/parquet-testing/variant_shredded_string.parquet index 348b0fa4feb4..5d8da51dfed8 100644 Binary files a/data/parquet-testing/variant_shredded_string.parquet and b/data/parquet-testing/variant_shredded_string.parquet differ diff --git a/data/parquet-testing/variant_shredded_time_micros_ntz.parquet b/data/parquet-testing/variant_shredded_time_micros_ntz.parquet index 838b98d89f42..f45a8f36428d 100644 Binary files a/data/parquet-testing/variant_shredded_time_micros_ntz.parquet and b/data/parquet-testing/variant_shredded_time_micros_ntz.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_micros_ntz_negative.parquet b/data/parquet-testing/variant_shredded_timestamp_micros_ntz_negative.parquet index b79914cc5791..6cff3b796e06 100644 Binary files a/data/parquet-testing/variant_shredded_timestamp_micros_ntz_negative.parquet and b/data/parquet-testing/variant_shredded_timestamp_micros_ntz_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_micros_ntz_positive.parquet b/data/parquet-testing/variant_shredded_timestamp_micros_ntz_positive.parquet index 1c8071ae26c1..77fe2070401d 100644 Binary files a/data/parquet-testing/variant_shredded_timestamp_micros_ntz_positive.parquet and b/data/parquet-testing/variant_shredded_timestamp_micros_ntz_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_micros_tz_negative.parquet b/data/parquet-testing/variant_shredded_timestamp_micros_tz_negative.parquet index 3c0cfbdf2573..0454b720cb15 100644 Binary files a/data/parquet-testing/variant_shredded_timestamp_micros_tz_negative.parquet and b/data/parquet-testing/variant_shredded_timestamp_micros_tz_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_micros_tz_negative2.parquet b/data/parquet-testing/variant_shredded_timestamp_micros_tz_negative2.parquet index 6be4b49e71c6..f393c10d9aa2 100644 Binary files a/data/parquet-testing/variant_shredded_timestamp_micros_tz_negative2.parquet and b/data/parquet-testing/variant_shredded_timestamp_micros_tz_negative2.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_micros_tz_positive2.parquet b/data/parquet-testing/variant_shredded_timestamp_micros_tz_positive2.parquet index 3c95fad760b3..1c87cb26de15 100644 Binary files a/data/parquet-testing/variant_shredded_timestamp_micros_tz_positive2.parquet and b/data/parquet-testing/variant_shredded_timestamp_micros_tz_positive2.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_nanos_ntz_negative.parquet b/data/parquet-testing/variant_shredded_timestamp_nanos_ntz_negative.parquet index 3d1e1af18a77..ceef68b11e7c 100644 Binary files a/data/parquet-testing/variant_shredded_timestamp_nanos_ntz_negative.parquet and b/data/parquet-testing/variant_shredded_timestamp_nanos_ntz_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_nanos_ntz_positive.parquet b/data/parquet-testing/variant_shredded_timestamp_nanos_ntz_positive.parquet index a8025ba3df6c..d63b0f32f9a8 100644 Binary files a/data/parquet-testing/variant_shredded_timestamp_nanos_ntz_positive.parquet and b/data/parquet-testing/variant_shredded_timestamp_nanos_ntz_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_nanos_tz_negative.parquet b/data/parquet-testing/variant_shredded_timestamp_nanos_tz_negative.parquet new file mode 100644 index 000000000000..f393c10d9aa2 Binary files /dev/null and b/data/parquet-testing/variant_shredded_timestamp_nanos_tz_negative.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_nanos_tz_negative_no_logical_type.parquet b/data/parquet-testing/variant_shredded_timestamp_nanos_tz_negative_no_logical_type.parquet deleted file mode 100644 index 75dd1d663e40..000000000000 Binary files a/data/parquet-testing/variant_shredded_timestamp_nanos_tz_negative_no_logical_type.parquet and /dev/null differ diff --git a/data/parquet-testing/variant_shredded_timestamp_nanos_tz_positive.parquet b/data/parquet-testing/variant_shredded_timestamp_nanos_tz_positive.parquet new file mode 100644 index 000000000000..1c87cb26de15 Binary files /dev/null and b/data/parquet-testing/variant_shredded_timestamp_nanos_tz_positive.parquet differ diff --git a/data/parquet-testing/variant_shredded_timestamp_nanos_tz_positive_no_logical_type.parquet b/data/parquet-testing/variant_shredded_timestamp_nanos_tz_positive_no_logical_type.parquet deleted file mode 100644 index 610512898fa9..000000000000 Binary files a/data/parquet-testing/variant_shredded_timestamp_nanos_tz_positive_no_logical_type.parquet and /dev/null differ diff --git a/data/parquet-testing/variant_shredded_uuid.parquet b/data/parquet-testing/variant_shredded_uuid.parquet index a672b2e35cc1..adc1d89d3fb9 100644 Binary files a/data/parquet-testing/variant_shredded_uuid.parquet and b/data/parquet-testing/variant_shredded_uuid.parquet differ diff --git a/data/parquet-testing/variant_string.parquet b/data/parquet-testing/variant_string.parquet index 14fc2008f1a3..bd3071bcf971 100644 Binary files a/data/parquet-testing/variant_string.parquet and b/data/parquet-testing/variant_string.parquet differ diff --git a/data/parquet-testing/variant_time_ntz.parquet b/data/parquet-testing/variant_time_ntz.parquet index 2458e927a86f..5e6b6d8bd99a 100644 Binary files a/data/parquet-testing/variant_time_ntz.parquet and b/data/parquet-testing/variant_time_ntz.parquet differ diff --git a/data/parquet-testing/variant_timestamp_micros.parquet b/data/parquet-testing/variant_timestamp_micros.parquet index 0aa60742a72e..c803bf94e276 100644 Binary files a/data/parquet-testing/variant_timestamp_micros.parquet and b/data/parquet-testing/variant_timestamp_micros.parquet differ diff --git a/data/parquet-testing/variant_timestamp_micros_negative.parquet b/data/parquet-testing/variant_timestamp_micros_negative.parquet index 1f998325ed46..d58d87f44928 100644 Binary files a/data/parquet-testing/variant_timestamp_micros_negative.parquet and b/data/parquet-testing/variant_timestamp_micros_negative.parquet differ diff --git a/data/parquet-testing/variant_timestamp_micros_ntz_positive.parquet b/data/parquet-testing/variant_timestamp_micros_ntz_positive.parquet index 7644535f5a52..0039862a888f 100644 Binary files a/data/parquet-testing/variant_timestamp_micros_ntz_positive.parquet and b/data/parquet-testing/variant_timestamp_micros_ntz_positive.parquet differ diff --git a/data/parquet-testing/variant_timestamp_nanos1.parquet b/data/parquet-testing/variant_timestamp_nanos1.parquet index c05bc05d2baa..0860570803fa 100644 Binary files a/data/parquet-testing/variant_timestamp_nanos1.parquet and b/data/parquet-testing/variant_timestamp_nanos1.parquet differ diff --git a/data/parquet-testing/variant_timestamp_nanos2.parquet b/data/parquet-testing/variant_timestamp_nanos2.parquet index df4e52a9948b..648b7dde5947 100644 Binary files a/data/parquet-testing/variant_timestamp_nanos2.parquet and b/data/parquet-testing/variant_timestamp_nanos2.parquet differ diff --git a/data/parquet-testing/variant_timestamp_nanos_ntz.parquet b/data/parquet-testing/variant_timestamp_nanos_ntz.parquet index 588a10b3e063..b3ed9016027a 100644 Binary files a/data/parquet-testing/variant_timestamp_nanos_ntz.parquet and b/data/parquet-testing/variant_timestamp_nanos_ntz.parquet differ diff --git a/data/parquet-testing/variant_timestamp_nanos_tz_negative.parquet b/data/parquet-testing/variant_timestamp_nanos_tz_negative.parquet new file mode 100644 index 000000000000..0860570803fa Binary files /dev/null and b/data/parquet-testing/variant_timestamp_nanos_tz_negative.parquet differ diff --git a/data/parquet-testing/variant_timestamp_nanos_tz_negative_no_logical_type.parquet b/data/parquet-testing/variant_timestamp_nanos_tz_negative_no_logical_type.parquet deleted file mode 100644 index 7fb26872a852..000000000000 Binary files a/data/parquet-testing/variant_timestamp_nanos_tz_negative_no_logical_type.parquet and /dev/null differ diff --git a/data/parquet-testing/variant_timestamp_nanos_tz_positive.parquet b/data/parquet-testing/variant_timestamp_nanos_tz_positive.parquet new file mode 100644 index 000000000000..648b7dde5947 Binary files /dev/null and b/data/parquet-testing/variant_timestamp_nanos_tz_positive.parquet differ diff --git a/data/parquet-testing/variant_timestamp_nanos_tz_positive_no_logical_type.parquet b/data/parquet-testing/variant_timestamp_nanos_tz_positive_no_logical_type.parquet deleted file mode 100644 index ed54510b8a2c..000000000000 Binary files a/data/parquet-testing/variant_timestamp_nanos_tz_positive_no_logical_type.parquet and /dev/null differ diff --git a/data/parquet-testing/variant_unshredded_nested_nulls.parquet b/data/parquet-testing/variant_unshredded_nested_nulls.parquet deleted file mode 100644 index 549ef7d66554..000000000000 Binary files a/data/parquet-testing/variant_unshredded_nested_nulls.parquet and /dev/null differ diff --git a/data/parquet-testing/variant_uuid.parquet b/data/parquet-testing/variant_uuid.parquet index bb8a514f32f3..534647ce8203 100644 Binary files a/data/parquet-testing/variant_uuid.parquet and b/data/parquet-testing/variant_uuid.parquet differ diff --git a/data/secrets/httpfs/s3_config_secret_v1_1_2.duckdb_secret b/data/secrets/httpfs/s3_config_secret_v1_1_2.duckdb_secret deleted file mode 100755 index 07998250b523..000000000000 Binary files a/data/secrets/httpfs/s3_config_secret_v1_1_2.duckdb_secret and /dev/null differ diff --git a/data/secrets/httpfs/s3_config_secret_v1_1_3.duckdb_secret b/data/secrets/httpfs/s3_config_secret_v1_1_3.duckdb_secret deleted file mode 100755 index a3e161bc17bd..000000000000 Binary files a/data/secrets/httpfs/s3_config_secret_v1_1_3.duckdb_secret and /dev/null differ diff --git a/data/secrets/httpfs/s3_config_secret_v_1_0_0.duckdb_secret b/data/secrets/httpfs/s3_config_secret_v_1_0_0.duckdb_secret deleted file mode 100755 index 7ff366232729..000000000000 Binary files a/data/secrets/httpfs/s3_config_secret_v_1_0_0.duckdb_secret and /dev/null differ diff --git a/data/secrets/httpfs/s3_secret_chain_v_1_0_0.duckdb_secret b/data/secrets/httpfs/s3_secret_chain_v_1_0_0.duckdb_secret deleted file mode 100755 index 212fb180563b..000000000000 Binary files a/data/secrets/httpfs/s3_secret_chain_v_1_0_0.duckdb_secret and /dev/null differ diff --git a/data/secrets/httpfs/s3_secret_chain_v_1_1_2.duckdb_secret b/data/secrets/httpfs/s3_secret_chain_v_1_1_2.duckdb_secret deleted file mode 100755 index 9cc6ef5ad869..000000000000 Binary files a/data/secrets/httpfs/s3_secret_chain_v_1_1_2.duckdb_secret and /dev/null differ diff --git a/data/secrets/httpfs/s3_secret_chain_v_1_1_3.duckdb_secret b/data/secrets/httpfs/s3_secret_chain_v_1_1_3.duckdb_secret deleted file mode 100755 index b13faa6a48dd..000000000000 Binary files a/data/secrets/httpfs/s3_secret_chain_v_1_1_3.duckdb_secret and /dev/null differ diff --git a/data/storage/cte_v1.db.gz b/data/storage/cte_v1.db.gz new file mode 100644 index 000000000000..c04621a345c0 Binary files /dev/null and b/data/storage/cte_v1.db.gz differ diff --git a/data/storage/cte_v1_4.db.gz b/data/storage/cte_v1_4.db.gz new file mode 100644 index 000000000000..caf302585d57 Binary files /dev/null and b/data/storage/cte_v1_4.db.gz differ diff --git a/extension/autocomplete/CMakeLists.txt b/extension/autocomplete/CMakeLists.txt index 544e65aae9fa..741975363037 100644 --- a/extension/autocomplete/CMakeLists.txt +++ b/extension/autocomplete/CMakeLists.txt @@ -8,6 +8,9 @@ set(AUTOCOMPLETE_EXTENSION_FILES autocomplete_extension.cpp matcher.cpp tokenizer.cpp keyword_helper.cpp keyword_map.cpp) +add_subdirectory(transformer) +add_subdirectory(parser) + build_static_extension(autocomplete ${AUTOCOMPLETE_EXTENSION_FILES}) set(PARAMETERS "-warnings") build_loadable_extension(autocomplete ${PARAMETERS} diff --git a/extension/autocomplete/autocomplete_extension.cpp b/extension/autocomplete/autocomplete_extension.cpp index 580f43dd1190..ad22863159ca 100644 --- a/extension/autocomplete/autocomplete_extension.cpp +++ b/extension/autocomplete/autocomplete_extension.cpp @@ -10,6 +10,7 @@ #include "duckdb/main/client_context.hpp" #include "duckdb/main/client_data.hpp" #include "duckdb/main/extension/extension_loader.hpp" +#include "transformer/peg_transformer.hpp" #include "duckdb/parser/keyword_helper.hpp" #include "matcher.hpp" #include "duckdb/catalog/default/builtin_types/types.hpp" @@ -491,7 +492,8 @@ static duckdb::unique_ptr GenerateSuggestions(Clien // tokenize the input vector tokens; vector suggestions; - MatchState state(tokens, suggestions); + ParseResultAllocator parse_allocator; + MatchState state(tokens, suggestions, parse_allocator); vector unicode_spaces; string clean_sql; const string &sql_ref = StripUnicodeSpaces(sql, clean_sql) ? clean_sql : sql; @@ -618,11 +620,11 @@ class ParserTokenizer : public BaseTokenizer { statements.push_back(std::move(tokens)); tokens.clear(); } - void OnLastToken(TokenizeState state, string last_word, idx_t) override { + void OnLastToken(TokenizeState state, string last_word, idx_t last_pos) override { if (last_word.empty()) { return; } - tokens.push_back(std::move(last_word)); + tokens.emplace_back(std::move(last_word), last_pos); } vector> statements; @@ -654,7 +656,8 @@ static duckdb::unique_ptr CheckPEGParserBind(ClientContext &contex continue; } vector suggestions; - MatchState state(tokens, suggestions); + ParseResultAllocator parse_allocator; + MatchState state(tokens, suggestions, parse_allocator); MatcherAllocator allocator; auto &matcher = Matcher::RootMatcher(allocator); @@ -681,6 +684,44 @@ static duckdb::unique_ptr CheckPEGParserBind(ClientContext &contex void CheckPEGParserFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { } +class PEGParserExtension : public ParserExtension { +public: + PEGParserExtension() { + parser_override = PEGParser; + } + + static ParserOverrideResult PEGParser(ParserExtensionInfo *info, const string &query) { + vector root_tokens; + string clean_sql; + + ParserTokenizer tokenizer(query, root_tokens); + tokenizer.TokenizeInput(); + tokenizer.statements.push_back(std::move(root_tokens)); + + vector> result; + try { + for (auto &tokenized_statement : tokenizer.statements) { + if (tokenized_statement.empty()) { + continue; + } + auto &transformer = PEGTransformerFactory::GetInstance(); + auto statement = transformer.Transform(tokenized_statement, "Statement"); + if (statement) { + statement->stmt_location = NumericCast(tokenized_statement[0].offset); + statement->stmt_length = + NumericCast(tokenized_statement[tokenized_statement.size() - 1].offset + + tokenized_statement[tokenized_statement.size() - 1].length); + } + statement->query = query; + result.push_back(std::move(statement)); + } + return ParserOverrideResult(std::move(result)); + } catch (std::exception &e) { + return ParserOverrideResult(e); + } + } +}; + static void LoadInternal(ExtensionLoader &loader) { TableFunction auto_complete_fun("sql_auto_complete", {LogicalType::VARCHAR}, SQLAutoCompleteFunction, SQLAutoCompleteBind, SQLAutoCompleteInit); @@ -689,6 +730,9 @@ static void LoadInternal(ExtensionLoader &loader) { TableFunction check_peg_parser_fun("check_peg_parser", {LogicalType::VARCHAR}, CheckPEGParserFunction, CheckPEGParserBind, nullptr); loader.RegisterFunction(check_peg_parser_fun); + + auto &config = DBConfig::GetConfig(loader.GetDatabaseInstance()); + config.parser_extensions.push_back(PEGParserExtension()); } void AutocompleteExtension::Load(ExtensionLoader &loader) { diff --git a/extension/autocomplete/grammar/statements/alter.gram b/extension/autocomplete/grammar/statements/alter.gram index 5b2453191c08..5512faba8b98 100644 --- a/extension/autocomplete/grammar/statements/alter.gram +++ b/extension/autocomplete/grammar/statements/alter.gram @@ -1,41 +1,46 @@ -AlterStatement <- 'ALTER'i AlterOptions +AlterStatement <- 'ALTER' AlterOptions -AlterOptions <- AlterTableStmt / AlterViewStmt / AlterSequenceStmt +AlterOptions <- AlterTableStmt / AlterViewStmt / AlterSequenceStmt / AlterDatabaseStmt / AlterSchemaStmt -AlterTableStmt <- 'TABLE'i IfExists? BaseTableName AlterTableOptions +AlterTableStmt <- 'TABLE' IfExists? BaseTableName AlterTableOptions +AlterSchemaStmt <- 'SCHEMA' IfExists? QualifiedName RenameAlter AlterTableOptions <- AddColumn / DropColumn / AlterColumn / AddConstraint / ChangeNullability / RenameColumn / RenameAlter / SetPartitionedBy / ResetPartitionedBy / SetSortedBy / ResetSortedBy -AddConstraint <- 'ADD'i TopLevelConstraint -AddColumn <- 'ADD'i 'COLUMN'i? IfNotExists? ColumnDefinition -DropColumn <- 'DROP'i 'COLUMN'i? IfExists? NestedColumnName DropBehavior? -AlterColumn <- 'ALTER'i 'COLUMN'i? NestedColumnName AlterColumnEntry -RenameColumn <- 'RENAME'i 'COLUMN'i? NestedColumnName 'TO'i Identifier +AddConstraint <- 'ADD' TopLevelConstraint +AddColumn <- 'ADD' 'COLUMN'? IfNotExists? ColumnDefinition +DropColumn <- 'DROP' 'COLUMN'? IfExists? NestedColumnName DropBehavior? +AlterColumn <- 'ALTER' 'COLUMN'? NestedColumnName AlterColumnEntry +RenameColumn <- 'RENAME' 'COLUMN'? NestedColumnName 'TO' Identifier NestedColumnName <- (Identifier '.')* ColumnName -RenameAlter <- 'RENAME'i 'TO'i Identifier -SetPartitionedBy <- 'SET'i 'PARTITIONED'i 'BY'i Parens(List(Expression)) -ResetPartitionedBy <- 'RESET'i 'PARTITIONED'i 'BY'i -SetSortedBy <- 'SET'i 'SORTED'i 'BY'i OrderByExpressions -ResetSortedBy <- 'RESET'i 'SORTED'i 'BY'i +RenameAlter <- 'RENAME' 'TO' Identifier +SetPartitionedBy <- 'SET' 'PARTITIONED' 'BY' Parens(List(Expression)) +ResetPartitionedBy <- 'RESET' 'PARTITIONED' 'BY' +SetSortedBy <- 'SET' 'SORTED' 'BY' Parens(OrderByExpressions) +ResetSortedBy <- 'RESET' 'SORTED' 'BY' AlterColumnEntry <- AddOrDropDefault / ChangeNullability / AlterType AddOrDropDefault <- AddDefault / DropDefault -AddDefault <- 'SET'i 'DEFAULT'i Expression -DropDefault <- 'DROP'i 'DEFAULT'i +AddDefault <- 'SET' 'DEFAULT' Expression +DropDefault <- 'DROP' 'DEFAULT' -ChangeNullability <- ('DROP'i / 'SET'i) 'NOT'i 'NULL'i +ChangeNullability <- ('DROP' / 'SET') 'NOT' 'NULL' -AlterType <- SetData? 'TYPE'i Type? UsingExpression? -SetData <- 'SET'i 'DATA'i? -UsingExpression <- 'USING'i Expression +AlterType <- SetData? 'TYPE' Type? UsingExpression? +SetData <- 'SET' 'DATA'? +UsingExpression <- 'USING' Expression -AlterViewStmt <- 'VIEW'i IfExists? BaseTableName RenameAlter +AlterViewStmt <- 'VIEW' IfExists? BaseTableName RenameAlter -AlterSequenceStmt <- 'SEQUENCE'i IfExists? QualifiedSequenceName AlterSequenceOptions +AlterSequenceStmt <- 'SEQUENCE' IfExists? QualifiedSequenceName AlterSequenceOptions QualifiedSequenceName <- CatalogQualification? SchemaQualification? SequenceName AlterSequenceOptions <- RenameAlter / SetSequenceOption SetSequenceOption <- List(SequenceOption) + +AlterDatabaseStmt <- 'DATABASE' IfExists? Identifier RenameDatabaseAlter + +RenameDatabaseAlter <- 'RENAME' 'TO' Identifier diff --git a/extension/autocomplete/grammar/statements/analyze.gram b/extension/autocomplete/grammar/statements/analyze.gram index 67b699f24e1a..36f08ea5b9c3 100644 --- a/extension/autocomplete/grammar/statements/analyze.gram +++ b/extension/autocomplete/grammar/statements/analyze.gram @@ -1,3 +1,3 @@ -AnalyzeStatement <- 'ANALYZE'i 'VERBOSE'i? AnalyzeTarget? +AnalyzeStatement <- 'ANALYZE' 'VERBOSE'? AnalyzeTarget? AnalyzeTarget <- QualifiedName Parens(List(Name))? Name <- ColId ('.' ColLabel)* \ No newline at end of file diff --git a/extension/autocomplete/grammar/statements/attach.gram b/extension/autocomplete/grammar/statements/attach.gram index 92ef72484bbf..977583b6fea8 100644 --- a/extension/autocomplete/grammar/statements/attach.gram +++ b/extension/autocomplete/grammar/statements/attach.gram @@ -1,6 +1,6 @@ -AttachStatement <- 'ATTACH'i OrReplace? IfNotExists? Database? DatabasePath AttachAlias? AttachOptions? +AttachStatement <- 'ATTACH' OrReplace? IfNotExists? Database? DatabasePath AttachAlias? AttachOptions? -Database <- 'DATABASE'i +Database <- 'DATABASE' DatabasePath <- StringLiteral -AttachAlias <- 'AS'i ColId +AttachAlias <- 'AS' ColId AttachOptions <- Parens(GenericCopyOptionList) diff --git a/extension/autocomplete/grammar/statements/call.gram b/extension/autocomplete/grammar/statements/call.gram index cdad1d2efe9c..d6a2b6db5014 100644 --- a/extension/autocomplete/grammar/statements/call.gram +++ b/extension/autocomplete/grammar/statements/call.gram @@ -1 +1 @@ -CallStatement <- 'CALL'i TableFunctionName TableFunctionArguments +CallStatement <- 'CALL' TableFunctionName TableFunctionArguments diff --git a/extension/autocomplete/grammar/statements/checkpoint.gram b/extension/autocomplete/grammar/statements/checkpoint.gram index 946dfa5377c7..ffb4924727dc 100644 --- a/extension/autocomplete/grammar/statements/checkpoint.gram +++ b/extension/autocomplete/grammar/statements/checkpoint.gram @@ -1 +1 @@ -CheckpointStatement <- 'FORCE'i? 'CHECKPOINT'i CatalogName? +CheckpointStatement <- 'FORCE'? 'CHECKPOINT' CatalogName? diff --git a/extension/autocomplete/grammar/statements/comment.gram b/extension/autocomplete/grammar/statements/comment.gram index 4807e686605f..786133b20e2d 100644 --- a/extension/autocomplete/grammar/statements/comment.gram +++ b/extension/autocomplete/grammar/statements/comment.gram @@ -1,5 +1,5 @@ -CommentStatement <- 'COMMENT'i 'ON'i CommentOnType ColumnReference 'IS'i CommentValue +CommentStatement <- 'COMMENT' 'ON' CommentOnType ColumnReference 'IS' CommentValue -CommentOnType <- 'TABLE'i / 'SEQUENCE'i / 'FUNCTION'i / ('MACRO'i 'TABLE'i?) / 'VIEW'i / 'DATABASE'i / 'INDEX'i / 'SCHEMA'i / 'TYPE'i / 'COLUMN'i -CommentValue <- 'NULL'i / StringLiteral +CommentOnType <- 'TABLE' / 'SEQUENCE' / 'FUNCTION' / ('MACRO' 'TABLE'?) / 'VIEW' / 'DATABASE' / 'INDEX' / 'SCHEMA' / 'TYPE' / 'COLUMN' +CommentValue <- 'NULL' / StringLiteral diff --git a/extension/autocomplete/grammar/statements/common.gram b/extension/autocomplete/grammar/statements/common.gram index ce56b632d93f..5e3d8e52010d 100644 --- a/extension/autocomplete/grammar/statements/common.gram +++ b/extension/autocomplete/grammar/statements/common.gram @@ -53,29 +53,29 @@ SecretName <- ColId NumberLiteral <- < [+-]?[0-9]*([.][0-9]*)? > StringLiteral <- '\'' [^\']* '\'' -Type <- (TimeType / IntervalType / BitType / RowType / MapType / UnionType / NumericType / SimpleType) ArrayBounds* +Type <- (TimeType / IntervalType / BitType / RowType / MapType / UnionType / NumericType / SetofType / SimpleType) ArrayBounds* SimpleType <- (QualifiedTypeName / CharacterType) TypeModifiers? -CharacterType <- ('CHARACTER'i 'VARYING'i?) / - ('CHAR'i 'VARYING'i?) / - ('NATIONAL'i 'CHARACTER'i 'VARYING'i?) / - ('NATIONAL'i 'CHAR'i 'VARYING'i?) / - ('NCHAR'i 'VARYING'i?) / - 'VARCHAR'i -IntervalType <- ('INTERVAL'i Interval?) / ('INTERVAL'i Parens(NumberLiteral)) +CharacterType <- ('CHARACTER' 'VARYING'?) / + ('CHAR' 'VARYING'?) / + ('NATIONAL' 'CHARACTER' 'VARYING'?) / + ('NATIONAL' 'CHAR' 'VARYING'?) / + ('NCHAR' 'VARYING'?) / + 'VARCHAR' +IntervalType <- ('INTERVAL' Interval?) / ('INTERVAL' Parens(NumberLiteral)) -YearKeyword <- 'YEAR'i / 'YEARS'i -MonthKeyword <- 'MONTH'i / 'MONTHS'i -DayKeyword <- 'DAY'i / 'DAYS'i -HourKeyword <- 'HOUR'i / 'HOURS'i -MinuteKeyword <- 'MINUTE'i / 'MINUTES'i -SecondKeyword <- 'SECOND'i / 'SECONDS'i -MillisecondKeyword <- 'MILLISECOND'i / 'MILLISECONDS'i -MicrosecondKeyword <- 'MICROSECOND'i / 'MICROSECONDS'i -WeekKeyword <- 'WEEK'i / 'WEEKS'i -QuarterKeyword <- 'QUARTER'i / 'QUARTERS'i -DecadeKeyword <- 'DECADE'i / 'DECADES'i -CenturyKeyword <- 'CENTURY'i / 'CENTURIES'i -MillenniumKeyword <- 'MILLENNIUM'i / 'MILLENNIA'i +YearKeyword <- 'YEAR' / 'YEARS' +MonthKeyword <- 'MONTH' / 'MONTHS' +DayKeyword <- 'DAY' / 'DAYS' +HourKeyword <- 'HOUR' / 'HOURS' +MinuteKeyword <- 'MINUTE' / 'MINUTES' +SecondKeyword <- 'SECOND' / 'SECONDS' +MillisecondKeyword <- 'MILLISECOND' / 'MILLISECONDS' +MicrosecondKeyword <- 'MICROSECOND' / 'MICROSECONDS' +WeekKeyword <- 'WEEK' / 'WEEKS' +QuarterKeyword <- 'QUARTER' / 'QUARTERS' +DecadeKeyword <- 'DECADE' / 'DECADES' +CenturyKeyword <- 'CENTURY' / 'CENTURIES' +MillenniumKeyword <- 'MILLENNIUM' / 'MILLENNIA' Interval <- YearKeyword / MonthKeyword / @@ -90,41 +90,42 @@ Interval <- YearKeyword / DecadeKeyword / CenturyKeyword / MillenniumKeyword / - (YearKeyword 'TO'i MonthKeyword) / - (DayKeyword 'TO'i HourKeyword) / - (DayKeyword 'TO'i MinuteKeyword) / - (DayKeyword 'TO'i SecondKeyword) / - (HourKeyword 'TO'i MinuteKeyword) / - (HourKeyword 'TO'i SecondKeyword) / - (MinuteKeyword 'TO'i SecondKeyword) + (YearKeyword 'TO' MonthKeyword) / + (DayKeyword 'TO' HourKeyword) / + (DayKeyword 'TO' MinuteKeyword) / + (DayKeyword 'TO' SecondKeyword) / + (HourKeyword 'TO' MinuteKeyword) / + (HourKeyword 'TO' SecondKeyword) / + (MinuteKeyword 'TO' SecondKeyword) -BitType <- 'BIT'i 'VARYING'i? Parens(List(Expression))? +BitType <- 'BIT' 'VARYING'? Parens(List(Expression))? -NumericType <- 'INT'i / - 'INTEGER'i / - 'SMALLINT'i / - 'BIGINT'i / - 'REAL'i / - 'BOOLEAN'i / - ('FLOAT'i Parens(NumberLiteral)?) / - ('DOUBLE'i 'PRECISION'i) / - ('DECIMAL'i TypeModifiers?) / - ('DEC'i TypeModifiers?) / - ('NUMERIC'i TypeModifiers?) +NumericType <- 'INT' / + 'INTEGER' / + 'SMALLINT' / + 'BIGINT' / + 'REAL' / + 'BOOLEAN' / + ('FLOAT' Parens(NumberLiteral)?) / + ('DOUBLE' 'PRECISION') / + ('DECIMAL' TypeModifiers?) / + ('DEC' TypeModifiers?) / + ('NUMERIC' TypeModifiers?) QualifiedTypeName <- CatalogQualification? SchemaQualification? TypeName TypeModifiers <- Parens(List(Expression)?) RowType <- RowOrStruct Parens(List(ColIdType)) -UnionType <- 'UNION'i Parens(List(ColIdType)) -MapType <- 'MAP'i Parens(List(Type)) +UnionType <- 'UNION' Parens(List(ColIdType)) +SetofType <- 'SETOF' Type +MapType <- 'MAP' Parens(List(Type)) ColIdType <- ColId Type -ArrayBounds <- ('[' NumberLiteral? ']') / 'ARRAY'i +ArrayBounds <- ('[' NumberLiteral? ']') / 'ARRAY' TimeType <- TimeOrTimestamp TypeModifiers? TimeZone? -TimeOrTimestamp <- 'TIME'i / 'TIMESTAMP'i -TimeZone <- WithOrWithout 'TIME'i 'ZONE'i -WithOrWithout <- 'WITH'i / 'WITHOUT'i +TimeOrTimestamp <- 'TIME' / 'TIMESTAMP' +TimeZone <- WithOrWithout 'TIME' 'ZONE' +WithOrWithout <- 'WITH' / 'WITHOUT' -RowOrStruct <- 'ROW'i / 'STRUCT'i +RowOrStruct <- 'ROW' / 'STRUCT' # internal definitions %whitespace <- [ \t\n\r]* diff --git a/extension/autocomplete/grammar/statements/copy.gram b/extension/autocomplete/grammar/statements/copy.gram index 2350fece9ad9..e1dc7d347b87 100644 --- a/extension/autocomplete/grammar/statements/copy.gram +++ b/extension/autocomplete/grammar/statements/copy.gram @@ -1,30 +1,30 @@ -CopyStatement <- 'COPY'i (CopyTable / CopySelect / CopyFromDatabase) +CopyStatement <- 'COPY' (CopyTable / CopySelect / CopyFromDatabase) CopyTable <- BaseTableName InsertColumnList? FromOrTo CopyFileName CopyOptions? -FromOrTo <- 'FROM'i / 'TO'i +FromOrTo <- 'FROM' / 'TO' -CopySelect <- Parens(SelectStatement) 'TO'i CopyFileName CopyOptions? +CopySelect <- Parens(SelectStatement) 'TO' CopyFileName CopyOptions? -CopyFileName <- StringLiteral / Identifier / (Identifier '.' ColId) -CopyOptions <- 'WITH'i? (Parens(GenericCopyOptionList) / (SpecializedOptions*)) +CopyFileName <- Expression / StringLiteral / Identifier / (Identifier '.' ColId) +CopyOptions <- 'WITH'? (Parens(GenericCopyOptionList) / (SpecializedOptions*)) SpecializedOptions <- - 'BINARY'i / 'FREEZE'i / 'OIDS'i / 'CSV'i / 'HEADER'i / + 'BINARY' / 'FREEZE' / 'OIDS' / 'CSV' / 'HEADER' / SpecializedStringOption / - ('ENCODING'i StringLiteral) / - ('FORCE'i 'QUOTE'i StarOrColumnList) / - ('PARTITION'i 'BY'i StarOrColumnList) / - ('FORCE'i 'NOT'i? 'NULL'i ColumnList) + ('ENCODING' StringLiteral) / + ('FORCE' 'QUOTE' StarOrColumnList) / + ('PARTITION' 'BY' StarOrColumnList) / + ('FORCE' 'NOT'? 'NULL' ColumnList) -SpecializedStringOption <- ('DELIMITER'i / 'NULL'i / 'QUOTE'i / 'ESCAPE'i) 'AS'i? StringLiteral +SpecializedStringOption <- ('DELIMITER' / 'NULL' / 'QUOTE' / 'ESCAPE') 'AS'? StringLiteral StarOrColumnList <- '*' / ColumnList GenericCopyOptionList <- List(GenericCopyOption) GenericCopyOption <- GenericCopyOptionName Expression? # FIXME: should not need to hard-code options here -GenericCopyOptionName <- 'ARRAY'i / 'NULL'i / 'ANALYZE'i / CopyOptionName +GenericCopyOptionName <- 'ARRAY' / 'NULL' / 'ANALYZE' / CopyOptionName -CopyFromDatabase <- 'FROM'i 'DATABASE'i ColId 'TO'i ColId CopyDatabaseFlag? +CopyFromDatabase <- 'FROM' 'DATABASE' ColId 'TO' ColId CopyDatabaseFlag? CopyDatabaseFlag <- Parens(SchemaOrData) -SchemaOrData <- 'SCHEMA'i / 'DATA'i +SchemaOrData <- 'SCHEMA' / 'DATA' diff --git a/extension/autocomplete/grammar/statements/create_index.gram b/extension/autocomplete/grammar/statements/create_index.gram index 97ea0d1252fb..b86c2aa055ab 100644 --- a/extension/autocomplete/grammar/statements/create_index.gram +++ b/extension/autocomplete/grammar/statements/create_index.gram @@ -1,10 +1,10 @@ -CreateIndexStmt <- Unique? 'INDEX'i IfNotExists? IndexName? 'ON'i BaseTableName IndexType? Parens(List(IndexElement)) WithList? +CreateIndexStmt <- Unique? 'INDEX' IfNotExists? IndexName? 'ON' BaseTableName IndexType? Parens(List(IndexElement)) WithList? WhereClause? -WithList <- 'WITH'i Parens(List(RelOption)) / Oids -Oids <- ('WITH'i / 'WITHOUT'i) 'OIDS'i +WithList <- 'WITH' Parens(List(RelOption)) / Oids +Oids <- ('WITH' / 'WITHOUT') 'OIDS' IndexElement <- Expression DescOrAsc? NullsFirstOrLast? -Unique <- 'UNIQUE'i -IndexType <- 'USING'i Identifier +Unique <- 'UNIQUE' +IndexType <- 'USING' Identifier RelOption <- ColLabel ('.' ColLabel)* ('=' DefArg)? -DefArg <- FuncType / ReservedKeyword / StringLiteral / NumberLiteral / 'NONE'i -FuncType <- Type / ('SETOF'i? TypeFuncName '%' 'TYPE'i) +DefArg <- FuncType / ReservedKeyword / StringLiteral / NumberLiteral / 'NONE' +FuncType <- Type / ('SETOF'? TypeFuncName '%' 'TYPE') diff --git a/extension/autocomplete/grammar/statements/create_macro.gram b/extension/autocomplete/grammar/statements/create_macro.gram index c387437bf461..9ceba441df96 100644 --- a/extension/autocomplete/grammar/statements/create_macro.gram +++ b/extension/autocomplete/grammar/statements/create_macro.gram @@ -1,11 +1,11 @@ CreateMacroStmt <- MacroOrFunction IfNotExists? QualifiedName List(MacroDefinition) -MacroOrFunction <- 'MACRO'i / 'FUNCTION'i +MacroOrFunction <- 'MACRO' / 'FUNCTION' -MacroDefinition <- Parens(MacroParameters?) 'AS'i (TableMacroDefinition / ScalarMacroDefinition) +MacroDefinition <- Parens(MacroParameters?) 'AS' (TableMacroDefinition / ScalarMacroDefinition) MacroParameters <- List(MacroParameter) -MacroParameter <- NamedParameter / TypeFuncName +MacroParameter <- NamedParameter / (TypeFuncName Type?) ScalarMacroDefinition <- Expression -TableMacroDefinition <- 'TABLE'i SelectStatement +TableMacroDefinition <- 'TABLE' SelectStatement diff --git a/extension/autocomplete/grammar/statements/create_schema.gram b/extension/autocomplete/grammar/statements/create_schema.gram index b6e3b4a23108..5044cc8ab848 100644 --- a/extension/autocomplete/grammar/statements/create_schema.gram +++ b/extension/autocomplete/grammar/statements/create_schema.gram @@ -1 +1 @@ -CreateSchemaStmt <- 'SCHEMA'i IfNotExists? QualifiedName \ No newline at end of file +CreateSchemaStmt <- 'SCHEMA' IfNotExists? QualifiedName \ No newline at end of file diff --git a/extension/autocomplete/grammar/statements/create_secret.gram b/extension/autocomplete/grammar/statements/create_secret.gram index 412a3decaecb..6ce478f3bd7f 100644 --- a/extension/autocomplete/grammar/statements/create_secret.gram +++ b/extension/autocomplete/grammar/statements/create_secret.gram @@ -1,3 +1,3 @@ -CreateSecretStmt <- 'SECRET'i IfNotExists? SecretName? SecretStorageSpecifier? Parens(GenericCopyOptionList) +CreateSecretStmt <- 'SECRET' IfNotExists? SecretName? SecretStorageSpecifier? Parens(GenericCopyOptionList) -SecretStorageSpecifier <- 'IN'i Identifier +SecretStorageSpecifier <- 'IN' Identifier diff --git a/extension/autocomplete/grammar/statements/create_sequence.gram b/extension/autocomplete/grammar/statements/create_sequence.gram index 5fead5dde3a2..819f32cf54d0 100644 --- a/extension/autocomplete/grammar/statements/create_sequence.gram +++ b/extension/autocomplete/grammar/statements/create_sequence.gram @@ -1,4 +1,4 @@ -CreateSequenceStmt <- 'SEQUENCE'i IfNotExists? QualifiedName SequenceOption* +CreateSequenceStmt <- 'SEQUENCE' IfNotExists? QualifiedName SequenceOption* SequenceOption <- SeqSetCycle / @@ -8,13 +8,13 @@ SequenceOption <- SeqStartWith / SeqOwnedBy -SeqSetCycle <- 'NO'i? 'CYCLE'i -SeqSetIncrement <- 'INCREMENT'i 'BY'i? Expression +SeqSetCycle <- 'NO'? 'CYCLE' +SeqSetIncrement <- 'INCREMENT' 'BY'? Expression SeqSetMinMax <- SeqMinOrMax Expression -SeqNoMinMax <- 'NO'i SeqMinOrMax -SeqStartWith <- 'START'i 'WITH'i? Expression -SeqOwnedBy <- 'OWNED'i 'BY'i QualifiedName +SeqNoMinMax <- 'NO' SeqMinOrMax +SeqStartWith <- 'START' 'WITH'? Expression +SeqOwnedBy <- 'OWNED' 'BY' QualifiedName -SeqMinOrMax <- 'MINVALUE'i / 'MAXVALUE'i +SeqMinOrMax <- 'MINVALUE' / 'MAXVALUE' diff --git a/extension/autocomplete/grammar/statements/create_table.gram b/extension/autocomplete/grammar/statements/create_table.gram index d02043655c82..be3f9aece014 100644 --- a/extension/autocomplete/grammar/statements/create_table.gram +++ b/extension/autocomplete/grammar/statements/create_table.gram @@ -1,14 +1,14 @@ -CreateStatement <- 'CREATE'i OrReplace? Temporary? (CreateTableStmt / CreateMacroStmt / CreateSequenceStmt / CreateTypeStmt / CreateSchemaStmt / CreateViewStmt / CreateIndexStmt / CreateSecretStmt) -OrReplace <- 'OR'i 'REPLACE'i -Temporary <- 'TEMP'i / 'TEMPORARY'i / 'PERSISTENT'i +CreateStatement <- 'CREATE' OrReplace? Temporary? (CreateTableStmt / CreateMacroStmt / CreateSequenceStmt / CreateTypeStmt / CreateSchemaStmt / CreateViewStmt / CreateIndexStmt / CreateSecretStmt) +OrReplace <- 'OR' 'REPLACE' +Temporary <- 'TEMP' / 'TEMPORARY' / 'PERSISTENT' -CreateTableStmt <- 'TABLE'i IfNotExists? QualifiedName (CreateTableAs / CreateColumnList) CommitAction? +CreateTableStmt <- 'TABLE' IfNotExists? QualifiedName (CreateTableAs / CreateColumnList) CommitAction? -CreateTableAs <- IdentifierList? 'AS'i SelectStatement WithData? -WithData <- 'WITH'i 'NO'i? 'DATA'i +CreateTableAs <- IdentifierList? 'AS' SelectStatement WithData? +WithData <- 'WITH' 'NO'? 'DATA' IdentifierList <- Parens(List(Identifier)) CreateColumnList <- Parens(CreateTableColumnList) -IfNotExists <- 'IF'i 'NOT'i 'EXISTS'i +IfNotExists <- 'IF' 'NOT' 'EXISTS' QualifiedName <- CatalogReservedSchemaIdentifier / SchemaReservedIdentifierOrStringLiteral / IdentifierOrStringLiteral SchemaReservedIdentifierOrStringLiteral <- SchemaQualification ReservedIdentifierOrStringLiteral CatalogReservedSchemaIdentifier <- CatalogQualification ReservedSchemaQualification ReservedIdentifierOrStringLiteral @@ -25,26 +25,26 @@ CreateTableColumnElement <- ColumnDefinition / TopLevelConstraint ColumnDefinition <- DottedIdentifier TypeOrGenerated ColumnConstraint* TypeOrGenerated <- Type? GeneratedColumn? ColumnConstraint <- NotNullConstraint / UniqueConstraint / PrimaryKeyConstraint / DefaultValue / CheckConstraint / ForeignKeyConstraint / ColumnCollation / ColumnCompression -NotNullConstraint <- 'NOT'i? 'NULL'i -UniqueConstraint <- 'UNIQUE'i -PrimaryKeyConstraint <- 'PRIMARY'i 'KEY'i -DefaultValue <- 'DEFAULT'i Expression -CheckConstraint <- 'CHECK'i Parens(Expression) -ForeignKeyConstraint <- 'REFERENCES'i BaseTableName Parens(ColumnList)? KeyActions? -ColumnCollation <- 'COLLATE'i Expression -ColumnCompression <- 'USING'i 'COMPRESSION'i ColIdOrString +NotNullConstraint <- 'NOT'? 'NULL' +UniqueConstraint <- 'UNIQUE' +PrimaryKeyConstraint <- 'PRIMARY' 'KEY' +DefaultValue <- 'DEFAULT' Expression +CheckConstraint <- 'CHECK' Parens(Expression) +ForeignKeyConstraint <- 'REFERENCES' BaseTableName Parens(ColumnList)? KeyActions? +ColumnCollation <- 'COLLATE' Expression +ColumnCompression <- 'USING' 'COMPRESSION' ColIdOrString KeyActions <- UpdateAction? DeleteAction? UpdateAction <- 'ON' 'UPDATE' KeyAction DeleteAction <- 'ON' 'DELETE' KeyAction -KeyAction <- ('NO'i 'ACTION'i) / 'RESTRICT'i / 'CASCADE'i / ('SET'i 'NULL'i) / ('SET'i 'DEFAULT'i) +KeyAction <- ('NO' 'ACTION') / 'RESTRICT' / 'CASCADE' / ('SET' 'NULL') / ('SET' 'DEFAULT') TopLevelConstraint <- ConstraintNameClause? TopLevelConstraintList TopLevelConstraintList <- TopPrimaryKeyConstraint / CheckConstraint / TopUniqueConstraint / TopForeignKeyConstraint -ConstraintNameClause <- 'CONSTRAINT'i Identifier -TopPrimaryKeyConstraint <- 'PRIMARY'i 'KEY'i ColumnIdList -TopUniqueConstraint <- 'UNIQUE'i ColumnIdList -TopForeignKeyConstraint <- 'FOREIGN'i 'KEY'i ColumnIdList ForeignKeyConstraint +ConstraintNameClause <- 'CONSTRAINT' Identifier +TopPrimaryKeyConstraint <- 'PRIMARY' 'KEY' ColumnIdList +TopUniqueConstraint <- 'UNIQUE' ColumnIdList +TopForeignKeyConstraint <- 'FOREIGN' 'KEY' ColumnIdList ForeignKeyConstraint ColumnIdList <- Parens(List(ColId)) PlainIdentifier <- !ReservedKeyword <[a-z_]i[a-z0-9_]i*> @@ -59,10 +59,11 @@ TypeFuncName <- UnreservedKeyword / TypeNameKeyword / FuncNameKeyword / Identifi TypeName <- UnreservedKeyword / TypeNameKeyword / Identifier ColLabel <- ReservedKeyword / UnreservedKeyword / ColumnNameKeyword / FuncNameKeyword / TypeNameKeyword / Identifier ColLabelOrString <- ColLabel / StringLiteral -GeneratedColumn <- Generated? 'AS'i Parens(Expression) GeneratedColumnType? +GeneratedColumn <- Generated? 'AS' Parens(Expression) GeneratedColumnType? -Generated <- 'GENERATED'i AlwaysOrByDefault? -AlwaysOrByDefault <- 'ALWAYS'i / ('BY'i 'DEFAULT'i) -GeneratedColumnType <- 'VIRTUAL'i / 'STORED'i +Generated <- 'GENERATED' AlwaysOrByDefault? +AlwaysOrByDefault <- 'ALWAYS' / ('BY' 'DEFAULT') +GeneratedColumnType <- 'VIRTUAL' / 'STORED' -CommitAction <- 'ON'i 'COMMIT'i 'PRESERVE'i 'ROWS'i +CommitAction <- 'ON' 'COMMIT' PreserveOrDelete +PreserveOrDelete <- ('PRESERVE' / 'DELETE') 'ROWS' diff --git a/extension/autocomplete/grammar/statements/create_type.gram b/extension/autocomplete/grammar/statements/create_type.gram index 2fe5947138a6..2ae28bbf9bea 100644 --- a/extension/autocomplete/grammar/statements/create_type.gram +++ b/extension/autocomplete/grammar/statements/create_type.gram @@ -1,4 +1,4 @@ -CreateTypeStmt <- 'TYPE'i IfNotExists? QualifiedName 'AS'i CreateType -CreateType <- ('ENUM'i Parens(SelectStatement)) / - ('ENUM'i Parens(List(StringLiteral))) / +CreateTypeStmt <- 'TYPE' IfNotExists? QualifiedName 'AS' CreateType +CreateType <- ('ENUM' Parens(SelectStatement)) / + ('ENUM' Parens(List(StringLiteral))) / Type diff --git a/extension/autocomplete/grammar/statements/create_view.gram b/extension/autocomplete/grammar/statements/create_view.gram index 9f373a7e5e39..2dcce300a0b0 100644 --- a/extension/autocomplete/grammar/statements/create_view.gram +++ b/extension/autocomplete/grammar/statements/create_view.gram @@ -1 +1 @@ -CreateViewStmt <- 'RECURSIVE'i? 'VIEW'i IfNotExists? QualifiedName InsertColumnList? 'AS'i SelectStatement +CreateViewStmt <- 'RECURSIVE'? 'VIEW' IfNotExists? QualifiedName InsertColumnList? 'AS' SelectStatement diff --git a/extension/autocomplete/grammar/statements/deallocate.gram b/extension/autocomplete/grammar/statements/deallocate.gram index c6aeb584b737..3b2e0cca3120 100644 --- a/extension/autocomplete/grammar/statements/deallocate.gram +++ b/extension/autocomplete/grammar/statements/deallocate.gram @@ -1 +1 @@ -DeallocateStatement <- 'DEALLOCATE'i 'PREPARE'i? Identifier +DeallocateStatement <- 'DEALLOCATE' 'PREPARE'? Identifier diff --git a/extension/autocomplete/grammar/statements/delete.gram b/extension/autocomplete/grammar/statements/delete.gram index 83ce7c3967b7..f609c2f64dc3 100644 --- a/extension/autocomplete/grammar/statements/delete.gram +++ b/extension/autocomplete/grammar/statements/delete.gram @@ -1,4 +1,4 @@ -DeleteStatement <- WithClause? 'DELETE'i 'FROM'i TargetOptAlias DeleteUsingClause? WhereClause? ReturningClause? -TruncateStatement <- 'TRUNCATE'i 'TABLE'i? BaseTableName -TargetOptAlias <- BaseTableName 'AS'i? ColId? -DeleteUsingClause <- 'USING'i List(TableRef) +DeleteStatement <- WithClause? 'DELETE' 'FROM' TargetOptAlias DeleteUsingClause? WhereClause? ReturningClause? +TruncateStatement <- 'TRUNCATE' 'TABLE'? BaseTableName +TargetOptAlias <- BaseTableName 'AS'? ColId? +DeleteUsingClause <- 'USING' List(TableRef) diff --git a/extension/autocomplete/grammar/statements/describe.gram b/extension/autocomplete/grammar/statements/describe.gram index 84bc6f5bc381..804efd52a6e8 100644 --- a/extension/autocomplete/grammar/statements/describe.gram +++ b/extension/autocomplete/grammar/statements/describe.gram @@ -1,8 +1,9 @@ -DescribeStatement <- ShowSelect / ShowAllTables / ShowQualifiedName +DescribeStatement <- ShowTables / ShowSelect / ShowAllTables / ShowQualifiedName ShowSelect <- ShowOrDescribeOrSummarize SelectStatement -ShowAllTables <- ShowOrDescribe 'ALL'i 'TABLES' +ShowAllTables <- ShowOrDescribe 'ALL' 'TABLES' ShowQualifiedName <- ShowOrDescribeOrSummarize (BaseTableName / StringLiteral)? +ShowTables <- ShowOrDescribe 'TABLES' 'FROM' QualifiedName -ShowOrDescribeOrSummarize <- ShowOrDescribe / 'SUMMARIZE'i -ShowOrDescribe <- 'SHOW'i / 'DESCRIBE'i / 'DESC'i +ShowOrDescribeOrSummarize <- ShowOrDescribe / 'SUMMARIZE' +ShowOrDescribe <- 'SHOW' / 'DESCRIBE' / 'DESC' diff --git a/extension/autocomplete/grammar/statements/detach.gram b/extension/autocomplete/grammar/statements/detach.gram index d34c973bf3c4..560d319acfdb 100644 --- a/extension/autocomplete/grammar/statements/detach.gram +++ b/extension/autocomplete/grammar/statements/detach.gram @@ -1 +1 @@ -DetachStatement <- 'DETACH'i Database? IfExists? CatalogName +DetachStatement <- 'DETACH' Database? IfExists? CatalogName diff --git a/extension/autocomplete/grammar/statements/drop.gram b/extension/autocomplete/grammar/statements/drop.gram index 42a4fb956f45..1e70aae4b9da 100644 --- a/extension/autocomplete/grammar/statements/drop.gram +++ b/extension/autocomplete/grammar/statements/drop.gram @@ -1,4 +1,4 @@ -DropStatement <- 'DROP'i DropEntries DropBehavior? +DropStatement <- 'DROP' DropEntries DropBehavior? DropEntries <- DropTable / @@ -12,22 +12,22 @@ DropEntries <- DropSecret DropTable <- TableOrView IfExists? List(BaseTableName) -DropTableFunction <- 'MACRO'i 'TABLE'i IfExists? List(TableFunctionName) +DropTableFunction <- 'MACRO' 'TABLE' IfExists? List(TableFunctionName) DropFunction <- FunctionType IfExists? List(FunctionIdentifier) -DropSchema <- 'SCHEMA'i IfExists? List(QualifiedSchemaName) -DropIndex <- 'INDEX'i IfExists? List(QualifiedIndexName) +DropSchema <- 'SCHEMA' IfExists? List(QualifiedSchemaName) +DropIndex <- 'INDEX' IfExists? List(QualifiedIndexName) QualifiedIndexName <- CatalogQualification? SchemaQualification? IndexName -DropSequence <- 'SEQUENCE'i IfExists? List(QualifiedSequenceName) -DropCollation <- 'COLLATION'i IfExists? List(CollationName) -DropType <- 'TYPE'i IfExists? List(QualifiedTypeName) -DropSecret <- Temporary? 'SECRET'i IfExists? SecretName DropSecretStorage? +DropSequence <- 'SEQUENCE' IfExists? List(QualifiedSequenceName) +DropCollation <- 'COLLATION' IfExists? List(CollationName) +DropType <- 'TYPE' IfExists? List(QualifiedTypeName) +DropSecret <- Temporary? 'SECRET' IfExists? SecretName DropSecretStorage? -TableOrView <- 'TABLE'i / 'VIEW'i / ('MATERIALIZED'i 'VIEW'i) -FunctionType <- 'MACRO'i / 'FUNCTION'i +TableOrView <- 'TABLE' / 'VIEW' / ('MATERIALIZED' 'VIEW') +FunctionType <- 'MACRO' / 'FUNCTION' -DropBehavior <- 'CASCADE'i / 'RESTRICT'i +DropBehavior <- 'CASCADE' / 'RESTRICT' -IfExists <- 'IF'i 'EXISTS'i +IfExists <- 'IF' 'EXISTS' QualifiedSchemaName <- CatalogQualification? SchemaName -DropSecretStorage <- 'FROM'i Identifier +DropSecretStorage <- 'FROM' Identifier diff --git a/extension/autocomplete/grammar/statements/execute.gram b/extension/autocomplete/grammar/statements/execute.gram index b640b635b3e4..13e5ca18df41 100644 --- a/extension/autocomplete/grammar/statements/execute.gram +++ b/extension/autocomplete/grammar/statements/execute.gram @@ -1 +1 @@ -ExecuteStatement <- 'EXECUTE'i Identifier TableFunctionArguments? \ No newline at end of file +ExecuteStatement <- 'EXECUTE' Identifier TableFunctionArguments? \ No newline at end of file diff --git a/extension/autocomplete/grammar/statements/explain.gram b/extension/autocomplete/grammar/statements/explain.gram index e59f03550172..9c811a054c3f 100644 --- a/extension/autocomplete/grammar/statements/explain.gram +++ b/extension/autocomplete/grammar/statements/explain.gram @@ -1,3 +1,3 @@ -ExplainStatement <- 'EXPLAIN'i 'ANALYZE'i? ExplainOptions? Statement +ExplainStatement <- 'EXPLAIN' 'ANALYZE'? ExplainOptions? Statement ExplainOptions <- Parens(GenericCopyOptionList) diff --git a/extension/autocomplete/grammar/statements/export.gram b/extension/autocomplete/grammar/statements/export.gram index c4afafe23118..43a927b0f261 100644 --- a/extension/autocomplete/grammar/statements/export.gram +++ b/extension/autocomplete/grammar/statements/export.gram @@ -1,5 +1,5 @@ -ExportStatement <- 'EXPORT'i 'DATABASE'i ExportSource? StringLiteral Parens(GenericCopyOptionList)? +ExportStatement <- 'EXPORT' 'DATABASE' ExportSource? StringLiteral Parens(GenericCopyOptionList)? -ExportSource <- CatalogName 'TO'i +ExportSource <- CatalogName 'TO' -ImportStatement <- 'IMPORT'i 'DATABASE'i StringLiteral +ImportStatement <- 'IMPORT' 'DATABASE' StringLiteral diff --git a/extension/autocomplete/grammar/statements/expression.gram b/extension/autocomplete/grammar/statements/expression.gram index 709d4e53eccb..95d961d7ee97 100644 --- a/extension/autocomplete/grammar/statements/expression.gram +++ b/extension/autocomplete/grammar/statements/expression.gram @@ -8,61 +8,65 @@ FunctionIdentifier <- CatalogReservedSchemaFunctionName / SchemaReservedFunction CatalogReservedSchemaFunctionName <- CatalogQualification ReservedSchemaQualification? ReservedFunctionName SchemaReservedFunctionName <- SchemaQualification ReservedFunctionName -DistinctOrAll <- 'DISTINCT'i / 'ALL'i -ExportClause <- 'EXPORT_STATE'i -WithinGroupClause <- 'WITHIN'i 'GROUP'i Parens(OrderByClause) -FilterClause <- 'FILTER' Parens('WHERE'i? Expression) -IgnoreNulls <- ('IGNORE'i 'NULLS'i) / ('RESPECT'i 'NULLS'i) +DistinctOrAll <- 'DISTINCT' / 'ALL' +ExportClause <- 'EXPORT_STATE' +WithinGroupClause <- 'WITHIN' 'GROUP' Parens(OrderByClause) +FilterClause <- 'FILTER' Parens('WHERE'? Expression) +IgnoreNulls <- ('IGNORE' 'NULLS') / ('RESPECT' 'NULLS') ParenthesisExpression <- Parens(List(Expression)) -LiteralExpression <- StringLiteral / NumberLiteral / 'NULL'i / 'TRUE'i / 'FALSE'i -CastExpression <- CastOrTryCast Parens(Expression 'AS'i Type) -CastOrTryCast <- 'CAST'i / 'TRY_CAST'i +LiteralExpression <- StringLiteral / NumberLiteral / ConstantLiteral +ConstantLiteral <- NullLiteral / TrueLiteral / FalseLiteral +NullLiteral <- 'NULL' +TrueLiteral <- 'TRUE' +FalseLiteral <- 'FALSE' +CastExpression <- CastOrTryCast Parens(Expression 'AS' Type) +CastOrTryCast <- 'CAST' / 'TRY_CAST' -StarExpression <- (ColId '.')* '*'i ExcludeList? ReplaceList? RenameList? -ExcludeList <- 'EXCLUDE'i (Parens(List(ExcludeName)) / ExcludeName) +StarExpression <- (ColId '.')* '*' ExcludeList? ReplaceList? RenameList? +ExcludeList <- 'EXCLUDE' (Parens(List(ExcludeName)) / ExcludeName) ExcludeName <- DottedIdentifier / ColIdOrString -ReplaceList <- 'REPLACE'i (Parens(List(ReplaceEntry)) / ReplaceEntry) -ReplaceEntry <- Expression 'AS'i ColumnReference -RenameList <- 'RENAME'i (Parens(List(RenameEntry)) / RenameEntry) -RenameEntry <- ColumnReference 'AS'i Identifier -SubqueryExpression <- 'NOT'i? 'EXISTS'i? SubqueryReference -CaseExpression <- 'CASE'i Expression? CaseWhenThen CaseWhenThen* CaseElse? 'END'i -CaseWhenThen <- 'WHEN'i Expression 'THEN'i Expression -CaseElse <- 'ELSE'i Expression +ReplaceList <- 'REPLACE' (Parens(List(ReplaceEntry)) / ReplaceEntry) +ReplaceEntry <- Expression 'AS' ColumnReference +RenameList <- 'RENAME' (Parens(List(RenameEntry)) / RenameEntry) +RenameEntry <- ColumnReference 'AS' Identifier +SubqueryExpression <- 'NOT'? 'EXISTS'? SubqueryReference +CaseExpression <- 'CASE' Expression? CaseWhenThen CaseWhenThen* CaseElse? 'END' +CaseWhenThen <- 'WHEN' Expression 'THEN' Expression +CaseElse <- 'ELSE' Expression TypeLiteral <- ColId StringLiteral -IntervalLiteral <- 'INTERVAL'i IntervalParameter IntervalUnit? +IntervalLiteral <- 'INTERVAL' IntervalParameter IntervalUnit? IntervalParameter <- StringLiteral / NumberLiteral / Parens(Expression) IntervalUnit <- ColId FrameClause <- Framing FrameExtent WindowExcludeClause? -Framing <- 'ROWS'i / 'RANGE'i / 'GROUPS'i -FrameExtent <- ('BETWEEN'i FrameBound 'AND'i FrameBound) / FrameBound -FrameBound <- ('UNBOUNDED'i 'PRECEDING'i) / ('UNBOUNDED'i 'FOLLOWING'i) / ('CURRENT'i 'ROW'i) / (Expression 'PRECEDING'i) / (Expression 'FOLLOWING'i) -WindowExcludeClause <- 'EXCLUDE'i WindowExcludeElement -WindowExcludeElement <- ('CURRENT'i 'ROW'i) / 'GROUP'i / 'TIES'i / ('NO'i 'OTHERS'i) -OverClause <- 'OVER'i WindowFrame +Framing <- 'ROWS' / 'RANGE' / 'GROUPS' +FrameExtent <- ('BETWEEN' FrameBound 'AND' FrameBound) / FrameBound +FrameBound <- ('UNBOUNDED' 'PRECEDING') / ('UNBOUNDED' 'FOLLOWING') / ('CURRENT' 'ROW') / (Expression 'PRECEDING') / (Expression 'FOLLOWING') +WindowExcludeClause <- 'EXCLUDE' WindowExcludeElement +WindowExcludeElement <- ('CURRENT' 'ROW') / 'GROUP' / 'TIES' / ('NO' 'OTHERS') +OverClause <- 'OVER' WindowFrame WindowFrame <- WindowFrameDefinition / Identifier / Parens(Identifier) WindowFrameDefinition <- Parens(BaseWindowName? WindowFrameContents) / Parens(WindowFrameContents) WindowFrameContents <- WindowPartition? OrderByClause? FrameClause? BaseWindowName <- Identifier -WindowPartition <- 'PARTITION'i 'BY'i List(Expression) +WindowPartition <- 'PARTITION' 'BY' List(Expression) PrefixExpression <- PrefixOperator Expression -PrefixOperator <- 'NOT'i / '-' / '+' / '~' -ListExpression <- 'ARRAY'i? (BoundedListExpression / SelectStatement) +PrefixOperator <- 'NOT' / '-' / '+' / '~' +ListExpression <- 'ARRAY'? (BoundedListExpression / SelectStatement) BoundedListExpression <- '[' List(Expression)? ']' StructExpression <- '{' List(StructField)? '}' -StructField <- Expression ':'i Expression -MapExpression <- 'MAP'i StructExpression +StructField <- Expression ':' Expression +MapExpression <- 'MAP' StructExpression GroupingExpression <- GroupingOrGroupingId Parens(List(Expression)) -GroupingOrGroupingId <- 'GROUPING'i / 'GROUPING_ID'i +GroupingOrGroupingId <- 'GROUPING' / 'GROUPING_ID' Parameter <- '?' / NumberedParameter / ColLabelParameter NumberedParameter <- '$' NumberLiteral ColLabelParameter <- '$' ColLabel PositionalExpression <- '#' NumberLiteral -DefaultExpression <- 'DEFAULT'i +DefaultExpression <- 'DEFAULT' -ListComprehensionExpression <- '['i Expression 'FOR'i List(Expression) ListComprehensionFilter? ']' -ListComprehensionFilter <- 'IF'i Expression +ListComprehensionExpression <- '[' Expression 'FOR' List(Expression) ListComprehensionFilter? ']' +ListComprehensionFilter <- 'IF' Expression SingleExpression <- LiteralExpression / @@ -89,18 +93,18 @@ SingleExpression <- OperatorLiteral <- <[\+\-\*\/\%\^\<\>\=\~\!\@\&\|\`]+> -LikeOperator <- 'NOT'i? LikeOrSimilarTo -LikeOrSimilarTo <- 'LIKE'i / 'ILIKE'i / 'GLOB'i / ('SIMILAR'i 'TO'i) -InOperator <- 'NOT'i? 'IN'i -IsOperator <- 'IS'i 'NOT'i? DistinctFrom? -DistinctFrom <- 'DISTINCT'i 'FROM'i -ConjunctionOperator <- 'OR'i / 'AND'i +LikeOperator <- 'NOT'? LikeOrSimilarTo +LikeOrSimilarTo <- 'LIKE' / 'ILIKE' / 'GLOB' / ('SIMILAR' 'TO') +InOperator <- 'NOT'? 'IN' +IsOperator <- 'IS' 'NOT'? DistinctFrom? +DistinctFrom <- 'DISTINCT' 'FROM' +ConjunctionOperator <- 'OR' / 'AND' ComparisonOperator <- '=' / '<=' / '>=' / '<' / '>' / '<>' / '!=' / '==' -BetweenOperator <- 'NOT'i? 'BETWEEN'i -CollateOperator <- 'COLLATE'i +BetweenOperator <- 'NOT'? 'BETWEEN' +CollateOperator <- 'COLLATE' LambdaOperator <- '->' -EscapeOperator <- 'ESCAPE'i -AtTimeZoneOperator <- 'AT'i 'TIME'i 'ZONE'i +EscapeOperator <- 'ESCAPE' +AtTimeZoneOperator <- 'AT' 'TIME' 'ZONE' PostfixOperator <- '!' AnyAllOperator <- ComparisonOperator AnyOrAll AnyOrAll <- 'ANY' / 'ALL' @@ -120,7 +124,7 @@ Operator <- CastOperator <- '::' Type DotOperator <- '.' (FunctionExpression / ColLabel) -NotNull <- 'NOT'i 'NULL'i +NotNull <- 'NOT' 'NULL' Indirection <- CastOperator / DotOperator / SliceExpression / NotNull / PostfixOperator BaseExpression <- SingleExpression Indirection* @@ -130,17 +134,17 @@ SliceExpression <- '[' SliceBound ']' SliceBound <- Expression? (':' (Expression / '-')?)? (':' Expression?)? SpecialFunctionExpression <- CoalesceExpression / UnpackExpression / ColumnsExpression / ExtractExpression / LambdaExpression / NullIfExpression / PositionExpression / RowExpression / SubstringExpression / TrimExpression -CoalesceExpression <- 'COALESCE'i Parens(List(Expression)) -UnpackExpression <- 'UNPACK'i Parens(Expression) -ColumnsExpression <- '*'? 'COLUMNS'i Parens(Expression) -ExtractExpression <- 'EXTRACT'i Parens(Expression 'FROM'i Expression) -LambdaExpression <- 'LAMBDA'i List(ColIdOrString) ':' Expression -NullIfExpression <- 'NULLIF'i Parens(Expression ',' Expression) -PositionExpression <- 'POSITION'i Parens(Expression) -RowExpression <- 'ROW'i Parens(List(Expression)) -SubstringExpression <- 'SUBSTRING'i Parens(SubstringParameters / List(Expression)) -SubstringParameters <- Expression 'FROM'i NumberLiteral 'FOR'i NumberLiteral -TrimExpression <- 'TRIM'i Parens(TrimDirection? TrimSource? List(Expression)) +CoalesceExpression <- 'COALESCE' Parens(List(Expression)) +UnpackExpression <- 'UNPACK' Parens(Expression) +ColumnsExpression <- '*'? 'COLUMNS' Parens(Expression) +ExtractExpression <- 'EXTRACT' Parens(Expression 'FROM' Expression) +LambdaExpression <- 'LAMBDA' List(ColIdOrString) ':' Expression +NullIfExpression <- 'NULLIF' Parens(Expression ',' Expression) +PositionExpression <- 'POSITION' Parens(Expression) +RowExpression <- 'ROW' Parens(List(Expression)) +SubstringExpression <- 'SUBSTRING' Parens(SubstringParameters / List(Expression)) +SubstringParameters <- Expression 'FROM' NumberLiteral 'FOR' NumberLiteral +TrimExpression <- 'TRIM' Parens(TrimDirection? TrimSource? List(Expression)) -TrimDirection <- 'BOTH'i / 'LEADING'i / 'TRAILING'i -TrimSource <- Expression? 'FROM'i +TrimDirection <- 'BOTH' / 'LEADING' / 'TRAILING' +TrimSource <- Expression? 'FROM' diff --git a/extension/autocomplete/grammar/statements/insert.gram b/extension/autocomplete/grammar/statements/insert.gram index 23d609d27d87..9b8289baf862 100644 --- a/extension/autocomplete/grammar/statements/insert.gram +++ b/extension/autocomplete/grammar/statements/insert.gram @@ -1,27 +1,27 @@ -InsertStatement <- WithClause? 'INSERT'i OrAction? 'INTO'i InsertTarget ByNameOrPosition? InsertColumnList? InsertValues OnConflictClause? ReturningClause? +InsertStatement <- WithClause? 'INSERT' OrAction? 'INTO' InsertTarget ByNameOrPosition? InsertColumnList? InsertValues OnConflictClause? ReturningClause? -OrAction <- 'OR'i 'REPLACE'i / 'IGNORE'i -ByNameOrPosition <- 'BY'i 'NAME'i / 'POSITION'i +OrAction <- 'OR' 'REPLACE' / 'IGNORE' +ByNameOrPosition <- 'BY' 'NAME' / 'POSITION' InsertTarget <- BaseTableName InsertAlias? -InsertAlias <- 'AS'i Identifier +InsertAlias <- 'AS' Identifier ColumnList <- List(ColId) InsertColumnList <- Parens(ColumnList) InsertValues <- SelectStatement / DefaultValues -DefaultValues <- 'DEFAULT'i 'VALUES'i +DefaultValues <- 'DEFAULT' 'VALUES' -OnConflictClause <- 'ON'i 'CONFLICT'i OnConflictTarget? OnConflictAction +OnConflictClause <- 'ON' 'CONFLICT' OnConflictTarget? OnConflictAction OnConflictTarget <- OnConflictExpressionTarget / OnConflictIndexTarget OnConflictExpressionTarget <- Parens(List(ColId)) WhereClause? -OnConflictIndexTarget <- 'ON'i 'CONSTRAINT'i ConstraintName +OnConflictIndexTarget <- 'ON' 'CONSTRAINT' ConstraintName OnConflictAction <- OnConflictUpdate / OnConflictNothing -OnConflictUpdate <- 'DO'i 'UPDATE'i 'SET'i UpdateSetClause WhereClause? -OnConflictNothing <- 'DO'i 'NOTHING'i +OnConflictUpdate <- 'DO' 'UPDATE' 'SET' UpdateSetClause WhereClause? +OnConflictNothing <- 'DO' 'NOTHING' -ReturningClause <- 'RETURNING'i TargetList +ReturningClause <- 'RETURNING' TargetList diff --git a/extension/autocomplete/grammar/statements/load.gram b/extension/autocomplete/grammar/statements/load.gram index 95393f76e219..f23700a35202 100644 --- a/extension/autocomplete/grammar/statements/load.gram +++ b/extension/autocomplete/grammar/statements/load.gram @@ -1,4 +1,4 @@ -LoadStatement <- 'LOAD'i ColIdOrString -InstallStatement <- 'FORCE'i? 'INSTALL'i Identifier FromSource? VersionNumber? -FromSource <- 'FROM'i (Identifier / StringLiteral) +LoadStatement <- 'LOAD' ColIdOrString +InstallStatement <- 'FORCE'? 'INSTALL' Identifier FromSource? VersionNumber? +FromSource <- 'FROM' (Identifier / StringLiteral) VersionNumber <- Identifier \ No newline at end of file diff --git a/extension/autocomplete/grammar/statements/merge_into.gram b/extension/autocomplete/grammar/statements/merge_into.gram index 67094a5e397e..a2a6c21b43f8 100644 --- a/extension/autocomplete/grammar/statements/merge_into.gram +++ b/extension/autocomplete/grammar/statements/merge_into.gram @@ -1,19 +1,19 @@ -MergeIntoStatement <- WithClause? 'MERGE'i 'INTO'i TargetOptAlias MergeIntoUsingClause MergeMatch* ReturningClause? -MergeIntoUsingClause <- 'USING'i TableRef JoinQualifier +MergeIntoStatement <- WithClause? 'MERGE' 'INTO' TargetOptAlias MergeIntoUsingClause MergeMatch* ReturningClause? +MergeIntoUsingClause <- 'USING' TableRef JoinQualifier MergeMatch <- MatchedClause / NotMatchedClause -MatchedClause <- 'WHEN'i 'MATCHED'i AndExpression? 'THEN'i MatchedClauseAction +MatchedClause <- 'WHEN' 'MATCHED' AndExpression? 'THEN' MatchedClauseAction MatchedClauseAction <- UpdateMatchClause / DeleteMatchClause / InsertMatchClause / DoNothingMatchClause / ErrorMatchClause -UpdateMatchClause <- 'UPDATE'i (UpdateMatchSetClause / ByNameOrPosition?) -DeleteMatchClause <- 'DELETE'i -InsertMatchClause <- 'INSERT'i (InsertValuesList / DefaultValues / InsertByNameOrPosition)? +UpdateMatchClause <- 'UPDATE' (UpdateMatchSetClause / ByNameOrPosition?) +DeleteMatchClause <- 'DELETE' +InsertMatchClause <- 'INSERT' (InsertValuesList / DefaultValues / InsertByNameOrPosition)? InsertByNameOrPosition <- ByNameOrPosition? '*'? -InsertValuesList <- InsertColumnList? 'VALUES'i Parens(List(Expression)) -DoNothingMatchClause <- 'DO'i 'NOTHING'i -ErrorMatchClause <- 'ERROR'i Expression? -UpdateMatchSetClause <- 'SET'i (UpdateSetClause / '*') -AndExpression <- 'AND'i Expression -NotMatchedClause <- 'WHEN'i 'NOT'i 'MATCHED'i BySourceOrTarget? AndExpression? 'THEN'i MatchedClauseAction -BySourceOrTarget <- 'BY'i ('SOURCE'i / 'TARGET'i) +InsertValuesList <- InsertColumnList? 'VALUES' Parens(List(Expression)) +DoNothingMatchClause <- 'DO' 'NOTHING' +ErrorMatchClause <- 'ERROR' Expression? +UpdateMatchSetClause <- 'SET' (UpdateSetClause / '*') +AndExpression <- 'AND' Expression +NotMatchedClause <- 'WHEN' 'NOT' 'MATCHED' BySourceOrTarget? AndExpression? 'THEN' MatchedClauseAction +BySourceOrTarget <- 'BY' ('SOURCE' / 'TARGET') diff --git a/extension/autocomplete/grammar/statements/pivot.gram b/extension/autocomplete/grammar/statements/pivot.gram index e1f8efd697e5..136315e03bd8 100644 --- a/extension/autocomplete/grammar/statements/pivot.gram +++ b/extension/autocomplete/grammar/statements/pivot.gram @@ -5,8 +5,8 @@ PivotUsing <- 'USING' TargetList PivotColumnList <- List(Expression) -PivotKeyword <- 'PIVOT'i / 'PIVOT_WIDER'i -UnpivotKeyword <- 'UNPIVOT'i / 'PIVOT_LONGER'i +PivotKeyword <- 'PIVOT' / 'PIVOT_WIDER' +UnpivotKeyword <- 'UNPIVOT' / 'PIVOT_LONGER' UnpivotStatement <- UnpivotKeyword TableRef 'ON' TargetList IntoNameValues? diff --git a/extension/autocomplete/grammar/statements/pragma.gram b/extension/autocomplete/grammar/statements/pragma.gram index b63dece9892c..80fcdf505da7 100644 --- a/extension/autocomplete/grammar/statements/pragma.gram +++ b/extension/autocomplete/grammar/statements/pragma.gram @@ -1,4 +1,4 @@ -PragmaStatement <- 'PRAGMA'i (PragmaAssign / PragmaFunction) +PragmaStatement <- 'PRAGMA' (PragmaAssign / PragmaFunction) PragmaAssign <- SettingName '=' VariableList PragmaFunction <- PragmaName PragmaParameters? diff --git a/extension/autocomplete/grammar/statements/prepare.gram b/extension/autocomplete/grammar/statements/prepare.gram index f71ac16ad4cc..269b69f7e3cb 100644 --- a/extension/autocomplete/grammar/statements/prepare.gram +++ b/extension/autocomplete/grammar/statements/prepare.gram @@ -1,3 +1,3 @@ -PrepareStatement <- 'PREPARE'i Identifier TypeList? 'AS'i Statement +PrepareStatement <- 'PREPARE' Identifier TypeList? 'AS' Statement TypeList <- Parens(List(Type)) diff --git a/extension/autocomplete/grammar/statements/select.gram b/extension/autocomplete/grammar/statements/select.gram index 59ed22ae34e4..c0e6ae68e5fd 100644 --- a/extension/autocomplete/grammar/statements/select.gram +++ b/extension/autocomplete/grammar/statements/select.gram @@ -1,7 +1,7 @@ SelectStatement <- SelectOrParens (SetopClause SelectStatement)* ResultModifiers -SetopClause <- ('UNION'i / 'EXCEPT'i / 'INTERSECT'i) DistinctOrAll? ByName? -ByName <- 'BY'i 'NAME'i +SetopClause <- ('UNION' / 'EXCEPT' / 'INTERSECT') DistinctOrAll? ByName? +ByName <- 'BY' 'NAME' SelectOrParens <- BaseSelect / Parens(SelectStatement) BaseSelect <- WithClause? (OptionalParensSimpleSelect / ValuesClause / DescribeStatement / TableStatement / PivotStatement / UnpivotStatement) ResultModifiers @@ -11,17 +11,17 @@ OptionalParensSimpleSelect <- Parens(SimpleSelect) / SimpleSelect SimpleSelect <- SelectFrom WhereClause? GroupByClause? HavingClause? WindowClause? QualifyClause? SampleClause? SelectFrom <- (SelectClause FromClause?) / (FromClause SelectClause?) -WithStatement <- ColIdOrString InsertColumnList? UsingKey? 'AS'i Materialized? SubqueryReference -UsingKey <- 'USING'i 'KEY'i Parens(List(ColId)) -Materialized <- 'NOT'i? 'MATERIALIZED'i -WithClause <- 'WITH'i Recursive? List(WithStatement) -Recursive <- 'RECURSIVE'i -SelectClause <- 'SELECT'i DistinctClause? TargetList +WithStatement <- ColIdOrString InsertColumnList? UsingKey? 'AS' Materialized? SubqueryReference +UsingKey <- 'USING' 'KEY' Parens(List(ColId)) +Materialized <- 'NOT'? 'MATERIALIZED' +WithClause <- 'WITH' Recursive? List(WithStatement) +Recursive <- 'RECURSIVE' +SelectClause <- 'SELECT' DistinctClause? TargetList TargetList <- List(AliasedExpression) ColumnAliases <- Parens(List(ColIdOrString)) -DistinctClause <- ('DISTINCT'i DistinctOn?) / 'ALL'i -DistinctOn <- 'ON'i Parens(List(Expression)) +DistinctClause <- ('DISTINCT' DistinctOn?) / 'ALL' +DistinctOn <- 'ON' Parens(List(Expression)) InnerTableRef <- ValuesRef / TableFunction / TableSubquery / BaseTableRef / ParensTableRef @@ -42,81 +42,85 @@ PivotValueLists <- PivotValueList PivotValueList* PivotValueList <- PivotHeader 'IN' PivotTargetList PivotTargetList <- Identifier / Parens(TargetList) -Lateral <- 'LATERAL'i +Lateral <- 'LATERAL' BaseTableName <- CatalogReservedSchemaTable / SchemaReservedTable / TableName SchemaReservedTable <- SchemaQualification ReservedTableName CatalogReservedSchemaTable <- CatalogQualification ReservedSchemaQualification ReservedTableName TableFunction <- TableFunctionLateralOpt / TableFunctionAliasColon -TableFunctionLateralOpt <- Lateral? QualifiedTableFunction TableFunctionArguments TableAlias? -TableFunctionAliasColon <- TableAliasColon QualifiedTableFunction TableFunctionArguments +TableFunctionLateralOpt <- Lateral? QualifiedTableFunction TableFunctionArguments WithOrdinality? TableAlias? +TableFunctionAliasColon <- TableAliasColon QualifiedTableFunction TableFunctionArguments WithOrdinality? +WithOrdinality <- 'WITH' 'ORDINALITY' QualifiedTableFunction <- CatalogQualification? SchemaQualification? TableFunctionName TableFunctionArguments <- Parens(List(FunctionArgument)?) FunctionArgument <- NamedParameter / Expression -NamedParameter <- TypeName NamedParameterAssignment Expression +NamedParameter <- TypeName Type? NamedParameterAssignment Expression NamedParameterAssignment <- ':=' / '=>' -TableAlias <- 'AS'i? (Identifier / StringLiteral) ColumnAliases? +TableAlias <- 'AS'? (Identifier / StringLiteral) ColumnAliases? -AtClause <- 'AT'i Parens(AtSpecifier) +AtClause <- 'AT' Parens(AtSpecifier) AtSpecifier <- AtUnit '=>' Expression -AtUnit <- 'VERSION'i / 'TIMESTAMP'i +AtUnit <- 'VERSION' / 'TIMESTAMP' JoinClause <- RegularJoinClause / JoinWithoutOnClause -RegularJoinClause <- 'ASOF'i? JoinType? 'JOIN'i TableRef JoinQualifier -JoinWithoutOnClause <- JoinPrefix 'JOIN'i TableRef +RegularJoinClause <- 'ASOF'? JoinType? 'JOIN' TableRef JoinQualifier +JoinWithoutOnClause <- JoinPrefix 'JOIN' TableRef JoinQualifier <- OnClause / UsingClause -OnClause <- 'ON'i Expression -UsingClause <- 'USING'i Parens(List(ColumnName)) - -OuterJoinType <- 'FULL'i / 'LEFT'i / 'RIGHT'i -JoinType <- (OuterJoinType 'OUTER'i?) / 'SEMI'i / 'ANTI'i / 'INNER'i -JoinPrefix <- 'CROSS'i / ('NATURAL'i JoinType?) / 'POSITIONAL'i - -FromClause <- 'FROM'i List(TableRef) -WhereClause <- 'WHERE'i Expression -GroupByClause <- 'GROUP'i 'BY'i GroupByExpressions -HavingClause <- 'HAVING'i Expression -QualifyClause <- 'QUALIFY'i Expression +OnClause <- 'ON' Expression +UsingClause <- 'USING' Parens(List(ColumnName)) + +OuterJoinType <- 'FULL' / 'LEFT' / 'RIGHT' +JoinType <- (OuterJoinType 'OUTER'?) / 'SEMI' / 'ANTI' / 'INNER' +JoinPrefix <- 'CROSS' / ('NATURAL' JoinType?) / 'POSITIONAL' + +FromClause <- 'FROM' List(TableRef) +WhereClause <- 'WHERE' Expression +GroupByClause <- 'GROUP' 'BY' GroupByExpressions +HavingClause <- 'HAVING' Expression +QualifyClause <- 'QUALIFY' Expression SampleClause <- (TableSample / UsingSample) SampleEntry -UsingSample <- 'USING'i 'SAMPLE'i -TableSample <- 'TABLESAMPLE'i -WindowClause <- 'WINDOW'i List(WindowDefinition) -WindowDefinition <- Identifier 'AS'i WindowFrameDefinition +UsingSample <- 'USING' 'SAMPLE' +TableSample <- 'TABLESAMPLE' +WindowClause <- 'WINDOW' List(WindowDefinition) +WindowDefinition <- Identifier 'AS' WindowFrameDefinition SampleEntry <- SampleEntryFunction / SampleEntryCount SampleEntryCount <- SampleCount Parens(SampleProperties)? SampleEntryFunction <- SampleFunction? Parens(SampleCount) RepeatableSample? SampleFunction <- ColId SampleProperties <- ColId (',' NumberLiteral)? -RepeatableSample <- 'REPEATABLE'i Parens(NumberLiteral) +RepeatableSample <- 'REPEATABLE' Parens(NumberLiteral) SampleCount <- Expression SampleUnit? -SampleUnit <- '%' / 'PERCENT'i / 'ROWS'i +SampleUnit <- '%' / 'PERCENT' / 'ROWS' -GroupByExpressions <- GroupByList / 'ALL'i +GroupByExpressions <- GroupByList / 'ALL' GroupByList <- List(GroupByExpression) GroupByExpression <- EmptyGroupingItem / CubeOrRollupClause / GroupingSetsClause / Expression EmptyGroupingItem <- '(' ')' CubeOrRollupClause <- CubeOrRollup Parens(List(Expression)) -CubeOrRollup <- 'CUBE'i / 'ROLLUP'i -GroupingSetsClause <- 'GROUPING'i 'SETS'i Parens(GroupByList) +CubeOrRollup <- 'CUBE' / 'ROLLUP' +GroupingSetsClause <- 'GROUPING' 'SETS' Parens(GroupByList) SubqueryReference <- Parens(SelectStatement) OrderByExpression <- Expression DescOrAsc? NullsFirstOrLast? -DescOrAsc <- 'DESC'i / 'DESCENDING'i / 'ASC'i / 'ASCENDING'i -NullsFirstOrLast <- 'NULLS'i 'FIRST'i / 'LAST'i -OrderByClause <- 'ORDER'i 'BY'i OrderByExpressions +DescOrAsc <- 'DESC' / 'DESCENDING' / 'ASC' / 'ASCENDING' +NullsFirstOrLast <- 'NULLS' 'FIRST' / 'LAST' +OrderByClause <- 'ORDER' 'BY' OrderByExpressions OrderByExpressions <- List(OrderByExpression) / OrderByAll -OrderByAll <- 'ALL'i DescOrAsc? NullsFirstOrLast? +OrderByAll <- 'ALL' DescOrAsc? NullsFirstOrLast? -LimitClause <- 'LIMIT'i LimitValue -OffsetClause <- 'OFFSET'i LimitValue -LimitValue <- 'ALL'i / (NumberLiteral 'PERCENT'i) / (Expression '%'?) +LimitClause <- 'LIMIT' LimitValue +OffsetClause <- 'OFFSET' OffsetValue +LimitValue <- 'ALL' / (NumberLiteral 'PERCENT') / (Expression '%'?) +OffsetValue <- Expression RowOrRows? +RowOrRows <- 'ROW' / 'ROWS' -AliasedExpression <- (ColId ':' Expression) / (Expression 'AS'i ColLabelOrString) / (Expression Identifier?) -ValuesClause <- 'VALUES'i List(ValuesExpressions) +AliasedExpression <- (ColId ':' Expression) / (Expression 'AS' ColLabelOrString) / (Expression Identifier?) + +ValuesClause <- 'VALUES' List(ValuesExpressions) ValuesExpressions <- Parens(List(Expression)) diff --git a/extension/autocomplete/grammar/statements/set.gram b/extension/autocomplete/grammar/statements/set.gram index a79b93371a3d..561f115d6a17 100644 --- a/extension/autocomplete/grammar/statements/set.gram +++ b/extension/autocomplete/grammar/statements/set.gram @@ -1,15 +1,19 @@ -SetStatement <- 'SET'i (StandardAssignment / SetTimeZone) +SetStatement <- 'SET' (StandardAssignment / SetTimeZone) StandardAssignment <- (SetVariable / SetSetting) SetAssignment -SetTimeZone <- 'TIME'i 'ZONE'i Expression +SetTimeZone <- 'TIME' 'ZONE' Expression SetSetting <- SettingScope? SettingName -SetVariable <- 'VARIABLE'i Identifier +SetVariable <- VariableScope Identifier +VariableScope <- 'VARIABLE' -SettingScope <- 'LOCAL'i / 'SESSION'i / 'GLOBAL'i +SettingScope <- LocalScope / SessionScope / GlobalScope +LocalScope <- 'LOCAL' +SessionScope <- 'SESSION' +GlobalScope <- 'GLOBAL' SetAssignment <- VariableAssign VariableList VariableAssign <- '=' / 'TO' VariableList <- List(Expression) -ResetStatement <- 'RESET'i (SetVariable / SetSetting) +ResetStatement <- 'RESET' (SetVariable / SetSetting) diff --git a/extension/autocomplete/grammar/statements/transaction.gram b/extension/autocomplete/grammar/statements/transaction.gram index 67cf5817fc78..ea8935ba26d0 100644 --- a/extension/autocomplete/grammar/statements/transaction.gram +++ b/extension/autocomplete/grammar/statements/transaction.gram @@ -4,8 +4,8 @@ BeginTransaction <- StartOrBegin Transaction? ReadOrWrite? RollbackTransaction <- AbortOrRollback Transaction? CommitTransaction <- CommitOrEnd Transaction? -StartOrBegin <- 'START'i / 'BEGIN'i -Transaction <- 'WORK'i / 'TRANSACTION'i -ReadOrWrite <- 'READ'i ('ONLY'i / 'WRITE'i) -AbortOrRollback <- 'ABORT'i / 'ROLLBACK'i -CommitOrEnd <- 'COMMIT'i / 'END'i +StartOrBegin <- 'START' / 'BEGIN' +Transaction <- 'WORK' / 'TRANSACTION' +ReadOrWrite <- 'READ' ('ONLY' / 'WRITE') +AbortOrRollback <- 'ABORT' / 'ROLLBACK' +CommitOrEnd <- 'COMMIT' / 'END' diff --git a/extension/autocomplete/grammar/statements/update.gram b/extension/autocomplete/grammar/statements/update.gram index f93dad41faab..19ea549604b7 100644 --- a/extension/autocomplete/grammar/statements/update.gram +++ b/extension/autocomplete/grammar/statements/update.gram @@ -1,6 +1,6 @@ -UpdateStatement <- WithClause? 'UPDATE'i UpdateTarget UpdateSetClause FromClause? WhereClause? ReturningClause? +UpdateStatement <- WithClause? 'UPDATE' UpdateTarget UpdateSetClause FromClause? WhereClause? ReturningClause? -UpdateTarget <- (BaseTableName 'SET'i) / (BaseTableName UpdateAlias? 'SET'i) -UpdateAlias <- 'AS'i? ColId +UpdateTarget <- (BaseTableName 'SET') / (BaseTableName UpdateAlias? 'SET') +UpdateAlias <- 'AS'? ColId UpdateSetClause <- List(UpdateSetElement) / (Parens(List(ColumnName)) '=' Expression) UpdateSetElement <- ColumnName '=' Expression diff --git a/extension/autocomplete/grammar/statements/use.gram b/extension/autocomplete/grammar/statements/use.gram index 5b15463523ad..c4eaee169405 100644 --- a/extension/autocomplete/grammar/statements/use.gram +++ b/extension/autocomplete/grammar/statements/use.gram @@ -1,3 +1,3 @@ -UseStatement <- 'USE'i UseTarget +UseStatement <- 'USE' UseTarget UseTarget <- (CatalogName '.' ReservedSchemaName) / SchemaName / CatalogName diff --git a/extension/autocomplete/grammar/statements/vacuum.gram b/extension/autocomplete/grammar/statements/vacuum.gram index 271c69203ca4..0ba6b74dbc51 100644 --- a/extension/autocomplete/grammar/statements/vacuum.gram +++ b/extension/autocomplete/grammar/statements/vacuum.gram @@ -1,12 +1,12 @@ -VacuumStatement <- 'VACUUM'i (VacuumLegacyOptions AnalyzeStatement / VacuumLegacyOptions QualifiedTarget / VacuumLegacyOptions / VacuumParensOptions QualifiedTarget?)? +VacuumStatement <- 'VACUUM' (VacuumLegacyOptions AnalyzeStatement / VacuumLegacyOptions QualifiedTarget / VacuumLegacyOptions / VacuumParensOptions QualifiedTarget?)? VacuumLegacyOptions <- OptFull OptFreeze OptVerbose VacuumParensOptions <- Parens(List(VacuumOption)) -VacuumOption <- 'ANALYZE'i / 'VERBOSE'i / 'FREEZE'i / 'FULL'i / Identifier +VacuumOption <- 'ANALYZE' / 'VERBOSE' / 'FREEZE' / 'FULL' / Identifier -OptFull <- 'FULL'i? -OptFreeze <- 'FREEZE'i? -OptVerbose <- 'VERBOSE'i? +OptFull <- 'FULL'? +OptFreeze <- 'FREEZE'? +OptVerbose <- 'VERBOSE'? QualifiedTarget <- QualifiedName OptNameList OptNameList <- Parens(List(Name))? \ No newline at end of file diff --git a/extension/autocomplete/include/ast/setting_info.hpp b/extension/autocomplete/include/ast/setting_info.hpp new file mode 100644 index 000000000000..16e41d813983 --- /dev/null +++ b/extension/autocomplete/include/ast/setting_info.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "duckdb/common/enums/set_scope.hpp" +#include "duckdb/common/string.hpp" + +namespace duckdb { + +struct SettingInfo { + string name; + SetScope scope = SetScope::AUTOMATIC; // Default value is defined here +}; + +} // namespace duckdb diff --git a/extension/autocomplete/include/inlined_grammar.gram b/extension/autocomplete/include/inlined_grammar.gram index 6d651d04e26b..677d42ac5892 100644 --- a/extension/autocomplete/include/inlined_grammar.gram +++ b/extension/autocomplete/include/inlined_grammar.gram @@ -1,523 +1,523 @@ -UnreservedKeyword <- 'ABORT'i / -'ABSOLUTE'i / -'ACCESS'i / -'ACTION'i / -'ADD'i / -'ADMIN'i / -'AFTER'i / -'AGGREGATE'i / -'ALSO'i / -'ALTER'i / -'ALWAYS'i / -'ASSERTION'i / -'ASSIGNMENT'i / -'ATTACH'i / -'ATTRIBUTE'i / -'BACKWARD'i / -'BEFORE'i / -'BEGIN'i / -'CACHE'i / -'CALL'i / -'CALLED'i / -'CASCADE'i / -'CASCADED'i / -'CATALOG'i / -'CENTURY'i / -'CENTURIES'i / -'CHAIN'i / -'CHARACTERISTICS'i / -'CHECKPOINT'i / -'CLASS'i / -'CLOSE'i / -'CLUSTER'i / -'COMMENT'i / -'COMMENTS'i / -'COMMIT'i / -'COMMITTED'i / -'COMPRESSION'i / -'CONFIGURATION'i / -'CONFLICT'i / -'CONNECTION'i / -'CONSTRAINTS'i / -'CONTENT'i / -'CONTINUE'i / -'CONVERSION'i / -'COPY'i / -'COST'i / -'CSV'i / -'CUBE'i / -'CURRENT'i / -'CURSOR'i / -'CYCLE'i / -'DATA'i / -'DATABASE'i / -'DAY'i / -'DAYS'i / -'DEALLOCATE'i / -'DECADE'i / -'DECADES'i / -'DECLARE'i / -'DEFAULTS'i / -'DEFERRED'i / -'DEFINER'i / -'DELETE'i / -'DELIMITER'i / -'DELIMITERS'i / -'DEPENDS'i / -'DETACH'i / -'DICTIONARY'i / -'DISABLE'i / -'DISCARD'i / -'DOCUMENT'i / -'DOMAIN'i / -'DOUBLE'i / -'DROP'i / -'EACH'i / -'ENABLE'i / -'ENCODING'i / -'ENCRYPTED'i / -'ENUM'i / -'ERROR'i / -'ESCAPE'i / -'EVENT'i / -'EXCLUDE'i / -'EXCLUDING'i / -'EXCLUSIVE'i / -'EXECUTE'i / -'EXPLAIN'i / -'EXPORT'i / -'EXPORT_STATE'i / -'EXTENSION'i / -'EXTENSIONS'i / -'EXTERNAL'i / -'FAMILY'i / -'FILTER'i / -'FIRST'i / -'FOLLOWING'i / -'FORCE'i / -'FORWARD'i / -'FUNCTION'i / -'FUNCTIONS'i / -'GLOBAL'i / -'GRANT'i / -'GRANTED'i / -'GROUPS'i / -'HANDLER'i / -'HEADER'i / -'HOLD'i / -'HOUR'i / -'HOURS'i / -'IDENTITY'i / -'IF'i / -'IGNORE'i / -'IMMEDIATE'i / -'IMMUTABLE'i / -'IMPLICIT'i / -'IMPORT'i / -'INCLUDE'i / -'INCLUDING'i / -'INCREMENT'i / -'INDEX'i / -'INDEXES'i / -'INHERIT'i / -'INHERITS'i / -'INLINE'i / -'INPUT'i / -'INSENSITIVE'i / -'INSERT'i / -'INSTALL'i / -'INSTEAD'i / -'INVOKER'i / -'JSON'i / -'ISOLATION'i / -'KEY'i / -'LABEL'i / -'LANGUAGE'i / -'LARGE'i / -'LAST'i / -'LEAKPROOF'i / -'LEVEL'i / -'LISTEN'i / -'LOAD'i / -'LOCAL'i / -'LOCATION'i / -'LOCK'i / -'LOCKED'i / -'LOGGED'i / -'MACRO'i / -'MAPPING'i / -'MATCH'i / -'MATCHED'i / -'MATERIALIZED'i / -'MAXVALUE'i / -'MERGE'i / -'METHOD'i / -'MICROSECOND'i / -'MICROSECONDS'i / -'MILLENNIUM'i / -'MILLENNIA'i / -'MILLISECOND'i / -'MILLISECONDS'i / -'MINUTE'i / -'MINUTES'i / -'MINVALUE'i / -'MODE'i / -'MONTH'i / -'MONTHS'i / -'MOVE'i / -'NAME'i / -'NAMES'i / -'NEW'i / -'NEXT'i / -'NO'i / -'NOTHING'i / -'NOTIFY'i / -'NOWAIT'i / -'NULLS'i / -'OBJECT'i / -'OF'i / -'OFF'i / -'OIDS'i / -'OLD'i / -'OPERATOR'i / -'OPTION'i / -'OPTIONS'i / -'ORDINALITY'i / -'OTHERS'i / -'OVER'i / -'OVERRIDING'i / -'OWNED'i / -'OWNER'i / -'PARALLEL'i / -'PARSER'i / -'PARTIAL'i / -'PARTITION'i / -'PARTITIONED'i / -'PASSING'i / -'PASSWORD'i / -'PERCENT'i / -'PERSISTENT'i / -'PLANS'i / -'POLICY'i / -'PRAGMA'i / -'PRECEDING'i / -'PREPARE'i / -'PREPARED'i / -'PRESERVE'i / -'PRIOR'i / -'PRIVILEGES'i / -'PROCEDURAL'i / -'PROCEDURE'i / -'PROGRAM'i / -'PUBLICATION'i / -'QUARTER'i / -'QUARTERS'i / -'QUOTE'i / -'RANGE'i / -'READ'i / -'REASSIGN'i / -'RECHECK'i / -'RECURSIVE'i / -'REF'i / -'REFERENCING'i / -'REFRESH'i / -'REINDEX'i / -'RELATIVE'i / -'RELEASE'i / -'RENAME'i / -'REPEATABLE'i / -'REPLACE'i / -'REPLICA'i / -'RESET'i / -'RESPECT'i / -'RESTART'i / -'RESTRICT'i / -'RETURNS'i / -'REVOKE'i / -'ROLE'i / -'ROLLBACK'i / -'ROLLUP'i / -'ROWS'i / -'RULE'i / -'SAMPLE'i / -'SAVEPOINT'i / -'SCHEMA'i / -'SCHEMAS'i / -'SCOPE'i / -'SCROLL'i / -'SEARCH'i / -'SECRET'i / -'SECOND'i / -'SECONDS'i / -'SECURITY'i / -'SEQUENCE'i / -'SEQUENCES'i / -'SERIALIZABLE'i / -'SERVER'i / -'SESSION'i / -'SET'i / -'SETS'i / -'SHARE'i / -'SIMPLE'i / -'SKIP'i / -'SNAPSHOT'i / -'SORTED'i / -'SOURCE'i / -'SQL'i / -'STABLE'i / -'STANDALONE'i / -'START'i / -'STATEMENT'i / -'STATISTICS'i / -'STDIN'i / -'STDOUT'i / -'STORAGE'i / -'STORED'i / -'STRICT'i / -'STRIP'i / -'SUBSCRIPTION'i / -'SYSID'i / -'SYSTEM'i / -'TABLES'i / -'TABLESPACE'i / -'TARGET'i / -'TEMP'i / -'TEMPLATE'i / -'TEMPORARY'i / -'TEXT'i / -'TIES'i / -'TRANSACTION'i / -'TRANSFORM'i / -'TRIGGER'i / -'TRUNCATE'i / -'TRUSTED'i / -'TYPE'i / -'TYPES'i / -'UNBOUNDED'i / -'UNCOMMITTED'i / -'UNENCRYPTED'i / -'UNKNOWN'i / -'UNLISTEN'i / -'UNLOGGED'i / -'UNTIL'i / -'UPDATE'i / -'USE'i / -'USER'i / -'VACUUM'i / -'VALID'i / -'VALIDATE'i / -'VALIDATOR'i / -'VALUE'i / -'VARIABLE'i / -'VARYING'i / -'VERSION'i / -'VIEW'i / -'VIEWS'i / -'VIRTUAL'i / -'VOLATILE'i / -'WEEK'i / -'WEEKS'i / -'WHITESPACE'i / -'WITHIN'i / -'WITHOUT'i / -'WORK'i / -'WRAPPER'i / -'WRITE'i / -'XML'i / -'YEAR'i / -'YEARS'i / -'YES'i / -'ZONE'i -ReservedKeyword <- 'ALL'i / -'ANALYSE'i / -'ANALYZE'i / -'AND'i / -'ANY'i / -'ARRAY'i / -'AS'i / -'ASC'i / -'ASYMMETRIC'i / -'BOTH'i / -'CASE'i / -'CAST'i / -'CHECK'i / -'COLLATE'i / -'COLUMN'i / -'CONSTRAINT'i / -'CREATE'i / -'DEFAULT'i / -'DEFERRABLE'i / -'DESC'i / -'DESCRIBE'i / -'DISTINCT'i / -'DO'i / -'ELSE'i / -'END'i / -'EXCEPT'i / -'FALSE'i / -'FETCH'i / -'FOR'i / -'FOREIGN'i / -'FROM'i / -'GROUP'i / -'HAVING'i / -'QUALIFY'i / -'IN'i / -'INITIALLY'i / -'INTERSECT'i / -'INTO'i / -'LAMBDA'i / -'LATERAL'i / -'LEADING'i / -'LIMIT'i / -'NOT'i / -'NULL'i / -'OFFSET'i / -'ON'i / -'ONLY'i / -'OR'i / -'ORDER'i / -'PIVOT'i / -'PIVOT_WIDER'i / -'PIVOT_LONGER'i / -'PLACING'i / -'PRIMARY'i / -'REFERENCES'i / -'RETURNING'i / -'SELECT'i / -'SHOW'i / -'SOME'i / -'SUMMARIZE'i / -'SYMMETRIC'i / -'TABLE'i / -'THEN'i / -'TO'i / -'TRAILING'i / -'TRUE'i / -'UNION'i / -'UNIQUE'i / -'UNPIVOT'i / -'USING'i / -'VARIADIC'i / -'WHEN'i / -'WHERE'i / -'WINDOW'i / -'WITH'i -ColumnNameKeyword <- 'BETWEEN'i / -'BIGINT'i / -'BIT'i / -'BOOLEAN'i / -'CHAR'i / -'CHARACTER'i / -'COALESCE'i / -'COLUMNS'i / -'DEC'i / -'DECIMAL'i / -'EXISTS'i / -'EXTRACT'i / -'FLOAT'i / -'GENERATED'i / -'GROUPING'i / -'GROUPING_ID'i / -'INOUT'i / -'INT'i / -'INTEGER'i / -'INTERVAL'i / -'MAP'i / -'NATIONAL'i / -'NCHAR'i / -'NONE'i / -'NULLIF'i / -'NUMERIC'i / -'OUT'i / -'OVERLAY'i / -'POSITION'i / -'PRECISION'i / -'REAL'i / -'ROW'i / -'SETOF'i / -'SMALLINT'i / -'SUBSTRING'i / -'STRUCT'i / -'TIME'i / -'TIMESTAMP'i / -'TREAT'i / -'TRIM'i / -'TRY_CAST'i / -'VALUES'i / -'VARCHAR'i / -'XMLATTRIBUTES'i / -'XMLCONCAT'i / -'XMLELEMENT'i / -'XMLEXISTS'i / -'XMLFOREST'i / -'XMLNAMESPACES'i / -'XMLPARSE'i / -'XMLPI'i / -'XMLROOT'i / -'XMLSERIALIZE'i / -'XMLTABLE'i -FuncNameKeyword <- 'ASOF'i / -'AT'i / -'AUTHORIZATION'i / -'BINARY'i / -'COLLATION'i / -'CONCURRENTLY'i / -'CROSS'i / -'FREEZE'i / -'FULL'i / -'GENERATED'i / -'GLOB'i / -'ILIKE'i / -'INNER'i / -'IS'i / -'ISNULL'i / -'JOIN'i / -'LEFT'i / -'LIKE'i / -'MAP'i / -'NATURAL'i / -'NOTNULL'i / -'OUTER'i / -'OVERLAPS'i / -'POSITIONAL'i / -'RIGHT'i / -'SIMILAR'i / -'STRUCT'i / -'TABLESAMPLE'i / -'VERBOSE'i -TypeNameKeyword <- 'ASOF'i / -'AT'i / -'AUTHORIZATION'i / -'BINARY'i / -'BY'i / -'COLLATION'i / -'COLUMNS'i / -'CONCURRENTLY'i / -'CROSS'i / -'FREEZE'i / -'FULL'i / -'GLOB'i / -'ILIKE'i / -'INNER'i / -'IS'i / -'ISNULL'i / -'JOIN'i / -'LEFT'i / -'LIKE'i / -'NATURAL'i / -'NOTNULL'i / -'OUTER'i / -'OVERLAPS'i / -'POSITIONAL'i / -'RIGHT'i / -'UNPACK'i / -'SIMILAR'i / -'TABLESAMPLE'i / -'TRY_CAST'i / -'VERBOSE'i / -'SEMI'i / -'ANTI'i +UnreservedKeyword <- 'ABORT' / +'ABSOLUTE' / +'ACCESS' / +'ACTION' / +'ADD' / +'ADMIN' / +'AFTER' / +'AGGREGATE' / +'ALSO' / +'ALTER' / +'ALWAYS' / +'ASSERTION' / +'ASSIGNMENT' / +'ATTACH' / +'ATTRIBUTE' / +'BACKWARD' / +'BEFORE' / +'BEGIN' / +'CACHE' / +'CALL' / +'CALLED' / +'CASCADE' / +'CASCADED' / +'CATALOG' / +'CENTURY' / +'CENTURIES' / +'CHAIN' / +'CHARACTERISTICS' / +'CHECKPOINT' / +'CLASS' / +'CLOSE' / +'CLUSTER' / +'COMMENT' / +'COMMENTS' / +'COMMIT' / +'COMMITTED' / +'COMPRESSION' / +'CONFIGURATION' / +'CONFLICT' / +'CONNECTION' / +'CONSTRAINTS' / +'CONTENT' / +'CONTINUE' / +'CONVERSION' / +'COPY' / +'COST' / +'CSV' / +'CUBE' / +'CURRENT' / +'CURSOR' / +'CYCLE' / +'DATA' / +'DATABASE' / +'DAY' / +'DAYS' / +'DEALLOCATE' / +'DECADE' / +'DECADES' / +'DECLARE' / +'DEFAULTS' / +'DEFERRED' / +'DEFINER' / +'DELETE' / +'DELIMITER' / +'DELIMITERS' / +'DEPENDS' / +'DETACH' / +'DICTIONARY' / +'DISABLE' / +'DISCARD' / +'DOCUMENT' / +'DOMAIN' / +'DOUBLE' / +'DROP' / +'EACH' / +'ENABLE' / +'ENCODING' / +'ENCRYPTED' / +'ENUM' / +'ERROR' / +'ESCAPE' / +'EVENT' / +'EXCLUDE' / +'EXCLUDING' / +'EXCLUSIVE' / +'EXECUTE' / +'EXPLAIN' / +'EXPORT' / +'EXPORT_STATE' / +'EXTENSION' / +'EXTENSIONS' / +'EXTERNAL' / +'FAMILY' / +'FILTER' / +'FIRST' / +'FOLLOWING' / +'FORCE' / +'FORWARD' / +'FUNCTION' / +'FUNCTIONS' / +'GLOBAL' / +'GRANT' / +'GRANTED' / +'GROUPS' / +'HANDLER' / +'HEADER' / +'HOLD' / +'HOUR' / +'HOURS' / +'IDENTITY' / +'IF' / +'IGNORE' / +'IMMEDIATE' / +'IMMUTABLE' / +'IMPLICIT' / +'IMPORT' / +'INCLUDE' / +'INCLUDING' / +'INCREMENT' / +'INDEX' / +'INDEXES' / +'INHERIT' / +'INHERITS' / +'INLINE' / +'INPUT' / +'INSENSITIVE' / +'INSERT' / +'INSTALL' / +'INSTEAD' / +'INVOKER' / +'JSON' / +'ISOLATION' / +'KEY' / +'LABEL' / +'LANGUAGE' / +'LARGE' / +'LAST' / +'LEAKPROOF' / +'LEVEL' / +'LISTEN' / +'LOAD' / +'LOCAL' / +'LOCATION' / +'LOCK' / +'LOCKED' / +'LOGGED' / +'MACRO' / +'MAPPING' / +'MATCH' / +'MATCHED' / +'MATERIALIZED' / +'MAXVALUE' / +'MERGE' / +'METHOD' / +'MICROSECOND' / +'MICROSECONDS' / +'MILLENNIUM' / +'MILLENNIA' / +'MILLISECOND' / +'MILLISECONDS' / +'MINUTE' / +'MINUTES' / +'MINVALUE' / +'MODE' / +'MONTH' / +'MONTHS' / +'MOVE' / +'NAME' / +'NAMES' / +'NEW' / +'NEXT' / +'NO' / +'NOTHING' / +'NOTIFY' / +'NOWAIT' / +'NULLS' / +'OBJECT' / +'OF' / +'OFF' / +'OIDS' / +'OLD' / +'OPERATOR' / +'OPTION' / +'OPTIONS' / +'ORDINALITY' / +'OTHERS' / +'OVER' / +'OVERRIDING' / +'OWNED' / +'OWNER' / +'PARALLEL' / +'PARSER' / +'PARTIAL' / +'PARTITION' / +'PARTITIONED' / +'PASSING' / +'PASSWORD' / +'PERCENT' / +'PERSISTENT' / +'PLANS' / +'POLICY' / +'PRAGMA' / +'PRECEDING' / +'PREPARE' / +'PREPARED' / +'PRESERVE' / +'PRIOR' / +'PRIVILEGES' / +'PROCEDURAL' / +'PROCEDURE' / +'PROGRAM' / +'PUBLICATION' / +'QUARTER' / +'QUARTERS' / +'QUOTE' / +'RANGE' / +'READ' / +'REASSIGN' / +'RECHECK' / +'RECURSIVE' / +'REF' / +'REFERENCING' / +'REFRESH' / +'REINDEX' / +'RELATIVE' / +'RELEASE' / +'RENAME' / +'REPEATABLE' / +'REPLACE' / +'REPLICA' / +'RESET' / +'RESPECT' / +'RESTART' / +'RESTRICT' / +'RETURNS' / +'REVOKE' / +'ROLE' / +'ROLLBACK' / +'ROLLUP' / +'ROWS' / +'RULE' / +'SAMPLE' / +'SAVEPOINT' / +'SCHEMA' / +'SCHEMAS' / +'SCOPE' / +'SCROLL' / +'SEARCH' / +'SECRET' / +'SECOND' / +'SECONDS' / +'SECURITY' / +'SEQUENCE' / +'SEQUENCES' / +'SERIALIZABLE' / +'SERVER' / +'SESSION' / +'SET' / +'SETS' / +'SHARE' / +'SIMPLE' / +'SKIP' / +'SNAPSHOT' / +'SORTED' / +'SOURCE' / +'SQL' / +'STABLE' / +'STANDALONE' / +'START' / +'STATEMENT' / +'STATISTICS' / +'STDIN' / +'STDOUT' / +'STORAGE' / +'STORED' / +'STRICT' / +'STRIP' / +'SUBSCRIPTION' / +'SYSID' / +'SYSTEM' / +'TABLES' / +'TABLESPACE' / +'TARGET' / +'TEMP' / +'TEMPLATE' / +'TEMPORARY' / +'TEXT' / +'TIES' / +'TRANSACTION' / +'TRANSFORM' / +'TRIGGER' / +'TRUNCATE' / +'TRUSTED' / +'TYPE' / +'TYPES' / +'UNBOUNDED' / +'UNCOMMITTED' / +'UNENCRYPTED' / +'UNKNOWN' / +'UNLISTEN' / +'UNLOGGED' / +'UNTIL' / +'UPDATE' / +'USE' / +'USER' / +'VACUUM' / +'VALID' / +'VALIDATE' / +'VALIDATOR' / +'VALUE' / +'VARIABLE' / +'VARYING' / +'VERSION' / +'VIEW' / +'VIEWS' / +'VIRTUAL' / +'VOLATILE' / +'WEEK' / +'WEEKS' / +'WHITESPACE' / +'WITHIN' / +'WITHOUT' / +'WORK' / +'WRAPPER' / +'WRITE' / +'XML' / +'YEAR' / +'YEARS' / +'YES' / +'ZONE' +ReservedKeyword <- 'ALL' / +'ANALYSE' / +'ANALYZE' / +'AND' / +'ANY' / +'ARRAY' / +'AS' / +'ASC' / +'ASYMMETRIC' / +'BOTH' / +'CASE' / +'CAST' / +'CHECK' / +'COLLATE' / +'COLUMN' / +'CONSTRAINT' / +'CREATE' / +'DEFAULT' / +'DEFERRABLE' / +'DESC' / +'DESCRIBE' / +'DISTINCT' / +'DO' / +'ELSE' / +'END' / +'EXCEPT' / +'FALSE' / +'FETCH' / +'FOR' / +'FOREIGN' / +'FROM' / +'GROUP' / +'HAVING' / +'QUALIFY' / +'IN' / +'INITIALLY' / +'INTERSECT' / +'INTO' / +'LAMBDA' / +'LATERAL' / +'LEADING' / +'LIMIT' / +'NOT' / +'NULL' / +'OFFSET' / +'ON' / +'ONLY' / +'OR' / +'ORDER' / +'PIVOT' / +'PIVOT_WIDER' / +'PIVOT_LONGER' / +'PLACING' / +'PRIMARY' / +'REFERENCES' / +'RETURNING' / +'SELECT' / +'SHOW' / +'SOME' / +'SUMMARIZE' / +'SYMMETRIC' / +'TABLE' / +'THEN' / +'TO' / +'TRAILING' / +'TRUE' / +'UNION' / +'UNIQUE' / +'UNPIVOT' / +'USING' / +'VARIADIC' / +'WHEN' / +'WHERE' / +'WINDOW' / +'WITH' +ColumnNameKeyword <- 'BETWEEN' / +'BIGINT' / +'BIT' / +'BOOLEAN' / +'CHAR' / +'CHARACTER' / +'COALESCE' / +'COLUMNS' / +'DEC' / +'DECIMAL' / +'EXISTS' / +'EXTRACT' / +'FLOAT' / +'GENERATED' / +'GROUPING' / +'GROUPING_ID' / +'INOUT' / +'INT' / +'INTEGER' / +'INTERVAL' / +'MAP' / +'NATIONAL' / +'NCHAR' / +'NONE' / +'NULLIF' / +'NUMERIC' / +'OUT' / +'OVERLAY' / +'POSITION' / +'PRECISION' / +'REAL' / +'ROW' / +'SETOF' / +'SMALLINT' / +'SUBSTRING' / +'STRUCT' / +'TIME' / +'TIMESTAMP' / +'TREAT' / +'TRIM' / +'TRY_CAST' / +'VALUES' / +'VARCHAR' / +'XMLATTRIBUTES' / +'XMLCONCAT' / +'XMLELEMENT' / +'XMLEXISTS' / +'XMLFOREST' / +'XMLNAMESPACES' / +'XMLPARSE' / +'XMLPI' / +'XMLROOT' / +'XMLSERIALIZE' / +'XMLTABLE' +FuncNameKeyword <- 'ASOF' / +'AT' / +'AUTHORIZATION' / +'BINARY' / +'COLLATION' / +'CONCURRENTLY' / +'CROSS' / +'FREEZE' / +'FULL' / +'GENERATED' / +'GLOB' / +'ILIKE' / +'INNER' / +'IS' / +'ISNULL' / +'JOIN' / +'LEFT' / +'LIKE' / +'MAP' / +'NATURAL' / +'NOTNULL' / +'OUTER' / +'OVERLAPS' / +'POSITIONAL' / +'RIGHT' / +'SIMILAR' / +'STRUCT' / +'TABLESAMPLE' / +'VERBOSE' +TypeNameKeyword <- 'ASOF' / +'AT' / +'AUTHORIZATION' / +'BINARY' / +'BY' / +'COLLATION' / +'COLUMNS' / +'CONCURRENTLY' / +'CROSS' / +'FREEZE' / +'FULL' / +'GLOB' / +'ILIKE' / +'INNER' / +'IS' / +'ISNULL' / +'JOIN' / +'LEFT' / +'LIKE' / +'NATURAL' / +'NOTNULL' / +'OUTER' / +'OVERLAPS' / +'POSITIONAL' / +'RIGHT' / +'UNPACK' / +'SIMILAR' / +'TABLESAMPLE' / +'TRY_CAST' / +'VERBOSE' / +'SEMI' / +'ANTI' PivotStatement <- PivotKeyword TableRef PivotOn? PivotUsing? GroupByClause? PivotOn <- 'ON' PivotColumnList @@ -525,8 +525,8 @@ PivotUsing <- 'USING' TargetList PivotColumnList <- List(Expression) -PivotKeyword <- 'PIVOT'i / 'PIVOT_WIDER'i -UnpivotKeyword <- 'UNPIVOT'i / 'PIVOT_LONGER'i +PivotKeyword <- 'PIVOT' / 'PIVOT_WIDER' +UnpivotKeyword <- 'UNPIVOT' / 'PIVOT_LONGER' UnpivotStatement <- UnpivotKeyword TableRef 'ON' TargetList IntoNameValues? @@ -547,61 +547,65 @@ FunctionIdentifier <- CatalogReservedSchemaFunctionName / SchemaReservedFunction CatalogReservedSchemaFunctionName <- CatalogQualification ReservedSchemaQualification? ReservedFunctionName SchemaReservedFunctionName <- SchemaQualification ReservedFunctionName -DistinctOrAll <- 'DISTINCT'i / 'ALL'i -ExportClause <- 'EXPORT_STATE'i -WithinGroupClause <- 'WITHIN'i 'GROUP'i Parens(OrderByClause) -FilterClause <- 'FILTER' Parens('WHERE'i? Expression) -IgnoreNulls <- ('IGNORE'i 'NULLS'i) / ('RESPECT'i 'NULLS'i) +DistinctOrAll <- 'DISTINCT' / 'ALL' +ExportClause <- 'EXPORT_STATE' +WithinGroupClause <- 'WITHIN' 'GROUP' Parens(OrderByClause) +FilterClause <- 'FILTER' Parens('WHERE'? Expression) +IgnoreNulls <- ('IGNORE' 'NULLS') / ('RESPECT' 'NULLS') ParenthesisExpression <- Parens(List(Expression)) -LiteralExpression <- StringLiteral / NumberLiteral / 'NULL'i / 'TRUE'i / 'FALSE'i -CastExpression <- CastOrTryCast Parens(Expression 'AS'i Type) -CastOrTryCast <- 'CAST'i / 'TRY_CAST'i - -StarExpression <- (ColId '.')* '*'i ExcludeList? ReplaceList? RenameList? -ExcludeList <- 'EXCLUDE'i (Parens(List(ExcludeName)) / ExcludeName) +LiteralExpression <- StringLiteral / NumberLiteral / ConstantLiteral +ConstantLiteral <- NullLiteral / TrueLiteral / FalseLiteral +NullLiteral <- 'NULL' +TrueLiteral <- 'TRUE' +FalseLiteral <- 'FALSE' +CastExpression <- CastOrTryCast Parens(Expression 'AS' Type) +CastOrTryCast <- 'CAST' / 'TRY_CAST' + +StarExpression <- (ColId '.')* '*' ExcludeList? ReplaceList? RenameList? +ExcludeList <- 'EXCLUDE' (Parens(List(ExcludeName)) / ExcludeName) ExcludeName <- DottedIdentifier / ColIdOrString -ReplaceList <- 'REPLACE'i (Parens(List(ReplaceEntry)) / ReplaceEntry) -ReplaceEntry <- Expression 'AS'i ColumnReference -RenameList <- 'RENAME'i (Parens(List(RenameEntry)) / RenameEntry) -RenameEntry <- ColumnReference 'AS'i Identifier -SubqueryExpression <- 'NOT'i? 'EXISTS'i? SubqueryReference -CaseExpression <- 'CASE'i Expression? CaseWhenThen CaseWhenThen* CaseElse? 'END'i -CaseWhenThen <- 'WHEN'i Expression 'THEN'i Expression -CaseElse <- 'ELSE'i Expression +ReplaceList <- 'REPLACE' (Parens(List(ReplaceEntry)) / ReplaceEntry) +ReplaceEntry <- Expression 'AS' ColumnReference +RenameList <- 'RENAME' (Parens(List(RenameEntry)) / RenameEntry) +RenameEntry <- ColumnReference 'AS' Identifier +SubqueryExpression <- 'NOT'? 'EXISTS'? SubqueryReference +CaseExpression <- 'CASE' Expression? CaseWhenThen CaseWhenThen* CaseElse? 'END' +CaseWhenThen <- 'WHEN' Expression 'THEN' Expression +CaseElse <- 'ELSE' Expression TypeLiteral <- ColId StringLiteral -IntervalLiteral <- 'INTERVAL'i IntervalParameter IntervalUnit? +IntervalLiteral <- 'INTERVAL' IntervalParameter IntervalUnit? IntervalParameter <- StringLiteral / NumberLiteral / Parens(Expression) IntervalUnit <- ColId FrameClause <- Framing FrameExtent WindowExcludeClause? -Framing <- 'ROWS'i / 'RANGE'i / 'GROUPS'i -FrameExtent <- ('BETWEEN'i FrameBound 'AND'i FrameBound) / FrameBound -FrameBound <- ('UNBOUNDED'i 'PRECEDING'i) / ('UNBOUNDED'i 'FOLLOWING'i) / ('CURRENT'i 'ROW'i) / (Expression 'PRECEDING'i) / (Expression 'FOLLOWING'i) -WindowExcludeClause <- 'EXCLUDE'i WindowExcludeElement -WindowExcludeElement <- ('CURRENT'i 'ROW'i) / 'GROUP'i / 'TIES'i / ('NO'i 'OTHERS'i) -OverClause <- 'OVER'i WindowFrame +Framing <- 'ROWS' / 'RANGE' / 'GROUPS' +FrameExtent <- ('BETWEEN' FrameBound 'AND' FrameBound) / FrameBound +FrameBound <- ('UNBOUNDED' 'PRECEDING') / ('UNBOUNDED' 'FOLLOWING') / ('CURRENT' 'ROW') / (Expression 'PRECEDING') / (Expression 'FOLLOWING') +WindowExcludeClause <- 'EXCLUDE' WindowExcludeElement +WindowExcludeElement <- ('CURRENT' 'ROW') / 'GROUP' / 'TIES' / ('NO' 'OTHERS') +OverClause <- 'OVER' WindowFrame WindowFrame <- WindowFrameDefinition / Identifier / Parens(Identifier) WindowFrameDefinition <- Parens(BaseWindowName? WindowFrameContents) / Parens(WindowFrameContents) WindowFrameContents <- WindowPartition? OrderByClause? FrameClause? BaseWindowName <- Identifier -WindowPartition <- 'PARTITION'i 'BY'i List(Expression) +WindowPartition <- 'PARTITION' 'BY' List(Expression) PrefixExpression <- PrefixOperator Expression -PrefixOperator <- 'NOT'i / '-' / '+' / '~' -ListExpression <- 'ARRAY'i? (BoundedListExpression / SelectStatement) +PrefixOperator <- 'NOT' / '-' / '+' / '~' +ListExpression <- 'ARRAY'? (BoundedListExpression / SelectStatement) BoundedListExpression <- '[' List(Expression)? ']' StructExpression <- '{' List(StructField)? '}' -StructField <- Expression ':'i Expression -MapExpression <- 'MAP'i StructExpression +StructField <- Expression ':' Expression +MapExpression <- 'MAP' StructExpression GroupingExpression <- GroupingOrGroupingId Parens(List(Expression)) -GroupingOrGroupingId <- 'GROUPING'i / 'GROUPING_ID'i +GroupingOrGroupingId <- 'GROUPING' / 'GROUPING_ID' Parameter <- '?' / NumberedParameter / ColLabelParameter NumberedParameter <- '$' NumberLiteral ColLabelParameter <- '$' ColLabel PositionalExpression <- '#' NumberLiteral -DefaultExpression <- 'DEFAULT'i +DefaultExpression <- 'DEFAULT' -ListComprehensionExpression <- '['i Expression 'FOR'i List(Expression) ListComprehensionFilter? ']' -ListComprehensionFilter <- 'IF'i Expression +ListComprehensionExpression <- '[' Expression 'FOR' List(Expression) ListComprehensionFilter? ']' +ListComprehensionFilter <- 'IF' Expression SingleExpression <- LiteralExpression / @@ -628,18 +632,18 @@ SingleExpression <- OperatorLiteral <- <[\+\-\*\/\%\^\<\>\=\~\!\@\&\|\`]+> -LikeOperator <- 'NOT'i? LikeOrSimilarTo -LikeOrSimilarTo <- 'LIKE'i / 'ILIKE'i / 'GLOB'i / ('SIMILAR'i 'TO'i) -InOperator <- 'NOT'i? 'IN'i -IsOperator <- 'IS'i 'NOT'i? DistinctFrom? -DistinctFrom <- 'DISTINCT'i 'FROM'i -ConjunctionOperator <- 'OR'i / 'AND'i +LikeOperator <- 'NOT'? LikeOrSimilarTo +LikeOrSimilarTo <- 'LIKE' / 'ILIKE' / 'GLOB' / ('SIMILAR' 'TO') +InOperator <- 'NOT'? 'IN' +IsOperator <- 'IS' 'NOT'? DistinctFrom? +DistinctFrom <- 'DISTINCT' 'FROM' +ConjunctionOperator <- 'OR' / 'AND' ComparisonOperator <- '=' / '<=' / '>=' / '<' / '>' / '<>' / '!=' / '==' -BetweenOperator <- 'NOT'i? 'BETWEEN'i -CollateOperator <- 'COLLATE'i +BetweenOperator <- 'NOT'? 'BETWEEN' +CollateOperator <- 'COLLATE' LambdaOperator <- '->' -EscapeOperator <- 'ESCAPE'i -AtTimeZoneOperator <- 'AT'i 'TIME'i 'ZONE'i +EscapeOperator <- 'ESCAPE' +AtTimeZoneOperator <- 'AT' 'TIME' 'ZONE' PostfixOperator <- '!' AnyAllOperator <- ComparisonOperator AnyOrAll AnyOrAll <- 'ANY' / 'ALL' @@ -659,7 +663,7 @@ Operator <- CastOperator <- '::' Type DotOperator <- '.' (FunctionExpression / ColLabel) -NotNull <- 'NOT'i 'NULL'i +NotNull <- 'NOT' 'NULL' Indirection <- CastOperator / DotOperator / SliceExpression / NotNull / PostfixOperator BaseExpression <- SingleExpression Indirection* @@ -669,94 +673,95 @@ SliceExpression <- '[' SliceBound ']' SliceBound <- Expression? (':' (Expression / '-')?)? (':' Expression?)? SpecialFunctionExpression <- CoalesceExpression / UnpackExpression / ColumnsExpression / ExtractExpression / LambdaExpression / NullIfExpression / PositionExpression / RowExpression / SubstringExpression / TrimExpression -CoalesceExpression <- 'COALESCE'i Parens(List(Expression)) -UnpackExpression <- 'UNPACK'i Parens(Expression) -ColumnsExpression <- '*'? 'COLUMNS'i Parens(Expression) -ExtractExpression <- 'EXTRACT'i Parens(Expression 'FROM'i Expression) -LambdaExpression <- 'LAMBDA'i List(ColIdOrString) ':' Expression -NullIfExpression <- 'NULLIF'i Parens(Expression ',' Expression) -PositionExpression <- 'POSITION'i Parens(Expression) -RowExpression <- 'ROW'i Parens(List(Expression)) -SubstringExpression <- 'SUBSTRING'i Parens(SubstringParameters / List(Expression)) -SubstringParameters <- Expression 'FROM'i NumberLiteral 'FOR'i NumberLiteral -TrimExpression <- 'TRIM'i Parens(TrimDirection? TrimSource? List(Expression)) +CoalesceExpression <- 'COALESCE' Parens(List(Expression)) +UnpackExpression <- 'UNPACK' Parens(Expression) +ColumnsExpression <- '*'? 'COLUMNS' Parens(Expression) +ExtractExpression <- 'EXTRACT' Parens(Expression 'FROM' Expression) +LambdaExpression <- 'LAMBDA' List(ColIdOrString) ':' Expression +NullIfExpression <- 'NULLIF' Parens(Expression ',' Expression) +PositionExpression <- 'POSITION' Parens(Expression) +RowExpression <- 'ROW' Parens(List(Expression)) +SubstringExpression <- 'SUBSTRING' Parens(SubstringParameters / List(Expression)) +SubstringParameters <- Expression 'FROM' NumberLiteral 'FOR' NumberLiteral +TrimExpression <- 'TRIM' Parens(TrimDirection? TrimSource? List(Expression)) -TrimDirection <- 'BOTH'i / 'LEADING'i / 'TRAILING'i -TrimSource <- Expression? 'FROM'i +TrimDirection <- 'BOTH' / 'LEADING' / 'TRAILING' +TrimSource <- Expression? 'FROM' -ExecuteStatement <- 'EXECUTE'i Identifier TableFunctionArguments? -CreateSecretStmt <- 'SECRET'i IfNotExists? SecretName? SecretStorageSpecifier? Parens(GenericCopyOptionList) +ExecuteStatement <- 'EXECUTE' Identifier TableFunctionArguments? +CreateSecretStmt <- 'SECRET' IfNotExists? SecretName? SecretStorageSpecifier? Parens(GenericCopyOptionList) -SecretStorageSpecifier <- 'IN'i Identifier +SecretStorageSpecifier <- 'IN' Identifier -CreateViewStmt <- 'RECURSIVE'i? 'VIEW'i IfNotExists? QualifiedName InsertColumnList? 'AS'i SelectStatement +CreateViewStmt <- 'RECURSIVE'? 'VIEW' IfNotExists? QualifiedName InsertColumnList? 'AS' SelectStatement -DescribeStatement <- ShowSelect / ShowAllTables / ShowQualifiedName +DescribeStatement <- ShowTables / ShowSelect / ShowAllTables / ShowQualifiedName ShowSelect <- ShowOrDescribeOrSummarize SelectStatement -ShowAllTables <- ShowOrDescribe 'ALL'i 'TABLES' +ShowAllTables <- ShowOrDescribe 'ALL' 'TABLES' ShowQualifiedName <- ShowOrDescribeOrSummarize (BaseTableName / StringLiteral)? +ShowTables <- ShowOrDescribe 'TABLES' 'FROM' QualifiedName -ShowOrDescribeOrSummarize <- ShowOrDescribe / 'SUMMARIZE'i -ShowOrDescribe <- 'SHOW'i / 'DESCRIBE'i / 'DESC'i +ShowOrDescribeOrSummarize <- ShowOrDescribe / 'SUMMARIZE' +ShowOrDescribe <- 'SHOW' / 'DESCRIBE' / 'DESC' -VacuumStatement <- 'VACUUM'i (VacuumLegacyOptions AnalyzeStatement / VacuumLegacyOptions QualifiedTarget / VacuumLegacyOptions / VacuumParensOptions QualifiedTarget?)? +VacuumStatement <- 'VACUUM' (VacuumLegacyOptions AnalyzeStatement / VacuumLegacyOptions QualifiedTarget / VacuumLegacyOptions / VacuumParensOptions QualifiedTarget?)? VacuumLegacyOptions <- OptFull OptFreeze OptVerbose VacuumParensOptions <- Parens(List(VacuumOption)) -VacuumOption <- 'ANALYZE'i / 'VERBOSE'i / 'FREEZE'i / 'FULL'i / Identifier +VacuumOption <- 'ANALYZE' / 'VERBOSE' / 'FREEZE' / 'FULL' / Identifier -OptFull <- 'FULL'i? -OptFreeze <- 'FREEZE'i? -OptVerbose <- 'VERBOSE'i? +OptFull <- 'FULL'? +OptFreeze <- 'FREEZE'? +OptVerbose <- 'VERBOSE'? QualifiedTarget <- QualifiedName OptNameList OptNameList <- Parens(List(Name))? -MergeIntoStatement <- WithClause? 'MERGE'i 'INTO'i TargetOptAlias MergeIntoUsingClause MergeMatch* ReturningClause? -MergeIntoUsingClause <- 'USING'i TableRef JoinQualifier +MergeIntoStatement <- WithClause? 'MERGE' 'INTO' TargetOptAlias MergeIntoUsingClause MergeMatch* ReturningClause? +MergeIntoUsingClause <- 'USING' TableRef JoinQualifier MergeMatch <- MatchedClause / NotMatchedClause -MatchedClause <- 'WHEN'i 'MATCHED'i AndExpression? 'THEN'i MatchedClauseAction +MatchedClause <- 'WHEN' 'MATCHED' AndExpression? 'THEN' MatchedClauseAction MatchedClauseAction <- UpdateMatchClause / DeleteMatchClause / InsertMatchClause / DoNothingMatchClause / ErrorMatchClause -UpdateMatchClause <- 'UPDATE'i (UpdateMatchSetClause / ByNameOrPosition?) -DeleteMatchClause <- 'DELETE'i -InsertMatchClause <- 'INSERT'i (InsertValuesList / DefaultValues / InsertByNameOrPosition)? +UpdateMatchClause <- 'UPDATE' (UpdateMatchSetClause / ByNameOrPosition?) +DeleteMatchClause <- 'DELETE' +InsertMatchClause <- 'INSERT' (InsertValuesList / DefaultValues / InsertByNameOrPosition)? InsertByNameOrPosition <- ByNameOrPosition? '*'? -InsertValuesList <- InsertColumnList? 'VALUES'i Parens(List(Expression)) -DoNothingMatchClause <- 'DO'i 'NOTHING'i -ErrorMatchClause <- 'ERROR'i Expression? -UpdateMatchSetClause <- 'SET'i (UpdateSetClause / '*') -AndExpression <- 'AND'i Expression -NotMatchedClause <- 'WHEN'i 'NOT'i 'MATCHED'i BySourceOrTarget? AndExpression? 'THEN'i MatchedClauseAction -BySourceOrTarget <- 'BY'i ('SOURCE'i / 'TARGET'i) +InsertValuesList <- InsertColumnList? 'VALUES' Parens(List(Expression)) +DoNothingMatchClause <- 'DO' 'NOTHING' +ErrorMatchClause <- 'ERROR' Expression? +UpdateMatchSetClause <- 'SET' (UpdateSetClause / '*') +AndExpression <- 'AND' Expression +NotMatchedClause <- 'WHEN' 'NOT' 'MATCHED' BySourceOrTarget? AndExpression? 'THEN' MatchedClauseAction +BySourceOrTarget <- 'BY' ('SOURCE' / 'TARGET') -PragmaStatement <- 'PRAGMA'i (PragmaAssign / PragmaFunction) +PragmaStatement <- 'PRAGMA' (PragmaAssign / PragmaFunction) PragmaAssign <- SettingName '=' VariableList PragmaFunction <- PragmaName PragmaParameters? PragmaParameters <- List(Expression) -DeallocateStatement <- 'DEALLOCATE'i 'PREPARE'i? Identifier +DeallocateStatement <- 'DEALLOCATE' 'PREPARE'? Identifier -PrepareStatement <- 'PREPARE'i Identifier TypeList? 'AS'i Statement +PrepareStatement <- 'PREPARE' Identifier TypeList? 'AS' Statement TypeList <- Parens(List(Type)) -CreateStatement <- 'CREATE'i OrReplace? Temporary? (CreateTableStmt / CreateMacroStmt / CreateSequenceStmt / CreateTypeStmt / CreateSchemaStmt / CreateViewStmt / CreateIndexStmt / CreateSecretStmt) -OrReplace <- 'OR'i 'REPLACE'i -Temporary <- 'TEMP'i / 'TEMPORARY'i / 'PERSISTENT'i +CreateStatement <- 'CREATE' OrReplace? Temporary? (CreateTableStmt / CreateMacroStmt / CreateSequenceStmt / CreateTypeStmt / CreateSchemaStmt / CreateViewStmt / CreateIndexStmt / CreateSecretStmt) +OrReplace <- 'OR' 'REPLACE' +Temporary <- 'TEMP' / 'TEMPORARY' / 'PERSISTENT' -CreateTableStmt <- 'TABLE'i IfNotExists? QualifiedName (CreateTableAs / CreateColumnList) CommitAction? +CreateTableStmt <- 'TABLE' IfNotExists? QualifiedName (CreateTableAs / CreateColumnList) CommitAction? -CreateTableAs <- IdentifierList? 'AS'i SelectStatement WithData? -WithData <- 'WITH'i 'NO'i? 'DATA'i +CreateTableAs <- IdentifierList? 'AS' SelectStatement WithData? +WithData <- 'WITH' 'NO'? 'DATA' IdentifierList <- Parens(List(Identifier)) CreateColumnList <- Parens(CreateTableColumnList) -IfNotExists <- 'IF'i 'NOT'i 'EXISTS'i +IfNotExists <- 'IF' 'NOT' 'EXISTS' QualifiedName <- CatalogReservedSchemaIdentifier / SchemaReservedIdentifierOrStringLiteral / IdentifierOrStringLiteral SchemaReservedIdentifierOrStringLiteral <- SchemaQualification ReservedIdentifierOrStringLiteral CatalogReservedSchemaIdentifier <- CatalogQualification ReservedSchemaQualification ReservedIdentifierOrStringLiteral @@ -773,26 +778,26 @@ CreateTableColumnElement <- ColumnDefinition / TopLevelConstraint ColumnDefinition <- DottedIdentifier TypeOrGenerated ColumnConstraint* TypeOrGenerated <- Type? GeneratedColumn? ColumnConstraint <- NotNullConstraint / UniqueConstraint / PrimaryKeyConstraint / DefaultValue / CheckConstraint / ForeignKeyConstraint / ColumnCollation / ColumnCompression -NotNullConstraint <- 'NOT'i? 'NULL'i -UniqueConstraint <- 'UNIQUE'i -PrimaryKeyConstraint <- 'PRIMARY'i 'KEY'i -DefaultValue <- 'DEFAULT'i Expression -CheckConstraint <- 'CHECK'i Parens(Expression) -ForeignKeyConstraint <- 'REFERENCES'i BaseTableName Parens(ColumnList)? KeyActions? -ColumnCollation <- 'COLLATE'i Expression -ColumnCompression <- 'USING'i 'COMPRESSION'i ColIdOrString +NotNullConstraint <- 'NOT'? 'NULL' +UniqueConstraint <- 'UNIQUE' +PrimaryKeyConstraint <- 'PRIMARY' 'KEY' +DefaultValue <- 'DEFAULT' Expression +CheckConstraint <- 'CHECK' Parens(Expression) +ForeignKeyConstraint <- 'REFERENCES' BaseTableName Parens(ColumnList)? KeyActions? +ColumnCollation <- 'COLLATE' Expression +ColumnCompression <- 'USING' 'COMPRESSION' ColIdOrString KeyActions <- UpdateAction? DeleteAction? UpdateAction <- 'ON' 'UPDATE' KeyAction DeleteAction <- 'ON' 'DELETE' KeyAction -KeyAction <- ('NO'i 'ACTION'i) / 'RESTRICT'i / 'CASCADE'i / ('SET'i 'NULL'i) / ('SET'i 'DEFAULT'i) +KeyAction <- ('NO' 'ACTION') / 'RESTRICT' / 'CASCADE' / ('SET' 'NULL') / ('SET' 'DEFAULT') TopLevelConstraint <- ConstraintNameClause? TopLevelConstraintList TopLevelConstraintList <- TopPrimaryKeyConstraint / CheckConstraint / TopUniqueConstraint / TopForeignKeyConstraint -ConstraintNameClause <- 'CONSTRAINT'i Identifier -TopPrimaryKeyConstraint <- 'PRIMARY'i 'KEY'i ColumnIdList -TopUniqueConstraint <- 'UNIQUE'i ColumnIdList -TopForeignKeyConstraint <- 'FOREIGN'i 'KEY'i ColumnIdList ForeignKeyConstraint +ConstraintNameClause <- 'CONSTRAINT' Identifier +TopPrimaryKeyConstraint <- 'PRIMARY' 'KEY' ColumnIdList +TopUniqueConstraint <- 'UNIQUE' ColumnIdList +TopForeignKeyConstraint <- 'FOREIGN' 'KEY' ColumnIdList ForeignKeyConstraint ColumnIdList <- Parens(List(ColId)) PlainIdentifier <- !ReservedKeyword <[a-z_]i[a-z0-9_]i*> @@ -807,30 +812,31 @@ TypeFuncName <- UnreservedKeyword / TypeNameKeyword / FuncNameKeyword / Identifi TypeName <- UnreservedKeyword / TypeNameKeyword / Identifier ColLabel <- ReservedKeyword / UnreservedKeyword / ColumnNameKeyword / FuncNameKeyword / TypeNameKeyword / Identifier ColLabelOrString <- ColLabel / StringLiteral -GeneratedColumn <- Generated? 'AS'i Parens(Expression) GeneratedColumnType? +GeneratedColumn <- Generated? 'AS' Parens(Expression) GeneratedColumnType? -Generated <- 'GENERATED'i AlwaysOrByDefault? -AlwaysOrByDefault <- 'ALWAYS'i / ('BY'i 'DEFAULT'i) -GeneratedColumnType <- 'VIRTUAL'i / 'STORED'i +Generated <- 'GENERATED' AlwaysOrByDefault? +AlwaysOrByDefault <- 'ALWAYS' / ('BY' 'DEFAULT') +GeneratedColumnType <- 'VIRTUAL' / 'STORED' -CommitAction <- 'ON'i 'COMMIT'i 'PRESERVE'i 'ROWS'i +CommitAction <- 'ON' 'COMMIT' PreserveOrDelete +PreserveOrDelete <- ('PRESERVE' / 'DELETE') 'ROWS' -CreateIndexStmt <- Unique? 'INDEX'i IfNotExists? IndexName? 'ON'i BaseTableName IndexType? Parens(List(IndexElement)) WithList? +CreateIndexStmt <- Unique? 'INDEX' IfNotExists? IndexName? 'ON' BaseTableName IndexType? Parens(List(IndexElement)) WithList? WhereClause? -WithList <- 'WITH'i Parens(List(RelOption)) / Oids -Oids <- ('WITH'i / 'WITHOUT'i) 'OIDS'i +WithList <- 'WITH' Parens(List(RelOption)) / Oids +Oids <- ('WITH' / 'WITHOUT') 'OIDS' IndexElement <- Expression DescOrAsc? NullsFirstOrLast? -Unique <- 'UNIQUE'i -IndexType <- 'USING'i Identifier +Unique <- 'UNIQUE' +IndexType <- 'USING' Identifier RelOption <- ColLabel ('.' ColLabel)* ('=' DefArg)? -DefArg <- FuncType / ReservedKeyword / StringLiteral / NumberLiteral / 'NONE'i -FuncType <- Type / ('SETOF'i? TypeFuncName '%' 'TYPE'i) +DefArg <- FuncType / ReservedKeyword / StringLiteral / NumberLiteral / 'NONE' +FuncType <- Type / ('SETOF'? TypeFuncName '%' 'TYPE') -LoadStatement <- 'LOAD'i ColIdOrString -InstallStatement <- 'FORCE'i? 'INSTALL'i Identifier FromSource? VersionNumber? -FromSource <- 'FROM'i (Identifier / StringLiteral) +LoadStatement <- 'LOAD' ColIdOrString +InstallStatement <- 'FORCE'? 'INSTALL' Identifier FromSource? VersionNumber? +FromSource <- 'FROM' (Identifier / StringLiteral) VersionNumber <- Identifier -DropStatement <- 'DROP'i DropEntries DropBehavior? +DropStatement <- 'DROP' DropEntries DropBehavior? DropEntries <- DropTable / @@ -844,66 +850,66 @@ DropEntries <- DropSecret DropTable <- TableOrView IfExists? List(BaseTableName) -DropTableFunction <- 'MACRO'i 'TABLE'i IfExists? List(TableFunctionName) +DropTableFunction <- 'MACRO' 'TABLE' IfExists? List(TableFunctionName) DropFunction <- FunctionType IfExists? List(FunctionIdentifier) -DropSchema <- 'SCHEMA'i IfExists? List(QualifiedSchemaName) -DropIndex <- 'INDEX'i IfExists? List(QualifiedIndexName) +DropSchema <- 'SCHEMA' IfExists? List(QualifiedSchemaName) +DropIndex <- 'INDEX' IfExists? List(QualifiedIndexName) QualifiedIndexName <- CatalogQualification? SchemaQualification? IndexName -DropSequence <- 'SEQUENCE'i IfExists? List(QualifiedSequenceName) -DropCollation <- 'COLLATION'i IfExists? List(CollationName) -DropType <- 'TYPE'i IfExists? List(QualifiedTypeName) -DropSecret <- Temporary? 'SECRET'i IfExists? SecretName DropSecretStorage? +DropSequence <- 'SEQUENCE' IfExists? List(QualifiedSequenceName) +DropCollation <- 'COLLATION' IfExists? List(CollationName) +DropType <- 'TYPE' IfExists? List(QualifiedTypeName) +DropSecret <- Temporary? 'SECRET' IfExists? SecretName DropSecretStorage? -TableOrView <- 'TABLE'i / 'VIEW'i / ('MATERIALIZED'i 'VIEW'i) -FunctionType <- 'MACRO'i / 'FUNCTION'i +TableOrView <- 'TABLE' / 'VIEW' / ('MATERIALIZED' 'VIEW') +FunctionType <- 'MACRO' / 'FUNCTION' -DropBehavior <- 'CASCADE'i / 'RESTRICT'i +DropBehavior <- 'CASCADE' / 'RESTRICT' -IfExists <- 'IF'i 'EXISTS'i +IfExists <- 'IF' 'EXISTS' QualifiedSchemaName <- CatalogQualification? SchemaName -DropSecretStorage <- 'FROM'i Identifier +DropSecretStorage <- 'FROM' Identifier -UpdateStatement <- WithClause? 'UPDATE'i UpdateTarget UpdateSetClause FromClause? WhereClause? ReturningClause? +UpdateStatement <- WithClause? 'UPDATE' UpdateTarget UpdateSetClause FromClause? WhereClause? ReturningClause? -UpdateTarget <- (BaseTableName 'SET'i) / (BaseTableName UpdateAlias? 'SET'i) -UpdateAlias <- 'AS'i? ColId +UpdateTarget <- (BaseTableName 'SET') / (BaseTableName UpdateAlias? 'SET') +UpdateAlias <- 'AS'? ColId UpdateSetClause <- List(UpdateSetElement) / (Parens(List(ColumnName)) '=' Expression) UpdateSetElement <- ColumnName '=' Expression -InsertStatement <- WithClause? 'INSERT'i OrAction? 'INTO'i InsertTarget ByNameOrPosition? InsertColumnList? InsertValues OnConflictClause? ReturningClause? +InsertStatement <- WithClause? 'INSERT' OrAction? 'INTO' InsertTarget ByNameOrPosition? InsertColumnList? InsertValues OnConflictClause? ReturningClause? -OrAction <- 'OR'i 'REPLACE'i / 'IGNORE'i -ByNameOrPosition <- 'BY'i 'NAME'i / 'POSITION'i +OrAction <- 'OR' 'REPLACE' / 'IGNORE' +ByNameOrPosition <- 'BY' 'NAME' / 'POSITION' InsertTarget <- BaseTableName InsertAlias? -InsertAlias <- 'AS'i Identifier +InsertAlias <- 'AS' Identifier ColumnList <- List(ColId) InsertColumnList <- Parens(ColumnList) InsertValues <- SelectStatement / DefaultValues -DefaultValues <- 'DEFAULT'i 'VALUES'i +DefaultValues <- 'DEFAULT' 'VALUES' -OnConflictClause <- 'ON'i 'CONFLICT'i OnConflictTarget? OnConflictAction +OnConflictClause <- 'ON' 'CONFLICT' OnConflictTarget? OnConflictAction OnConflictTarget <- OnConflictExpressionTarget / OnConflictIndexTarget OnConflictExpressionTarget <- Parens(List(ColId)) WhereClause? -OnConflictIndexTarget <- 'ON'i 'CONSTRAINT'i ConstraintName +OnConflictIndexTarget <- 'ON' 'CONSTRAINT' ConstraintName OnConflictAction <- OnConflictUpdate / OnConflictNothing -OnConflictUpdate <- 'DO'i 'UPDATE'i 'SET'i UpdateSetClause WhereClause? -OnConflictNothing <- 'DO'i 'NOTHING'i +OnConflictUpdate <- 'DO' 'UPDATE' 'SET' UpdateSetClause WhereClause? +OnConflictNothing <- 'DO' 'NOTHING' -ReturningClause <- 'RETURNING'i TargetList +ReturningClause <- 'RETURNING' TargetList -CreateSchemaStmt <- 'SCHEMA'i IfNotExists? QualifiedName +CreateSchemaStmt <- 'SCHEMA' IfNotExists? QualifiedName SelectStatement <- SelectOrParens (SetopClause SelectStatement)* ResultModifiers -SetopClause <- ('UNION'i / 'EXCEPT'i / 'INTERSECT'i) DistinctOrAll? ByName? -ByName <- 'BY'i 'NAME'i +SetopClause <- ('UNION' / 'EXCEPT' / 'INTERSECT') DistinctOrAll? ByName? +ByName <- 'BY' 'NAME' SelectOrParens <- BaseSelect / Parens(SelectStatement) BaseSelect <- WithClause? (OptionalParensSimpleSelect / ValuesClause / DescribeStatement / TableStatement / PivotStatement / UnpivotStatement) ResultModifiers @@ -913,17 +919,17 @@ OptionalParensSimpleSelect <- Parens(SimpleSelect) / SimpleSelect SimpleSelect <- SelectFrom WhereClause? GroupByClause? HavingClause? WindowClause? QualifyClause? SampleClause? SelectFrom <- (SelectClause FromClause?) / (FromClause SelectClause?) -WithStatement <- ColIdOrString InsertColumnList? UsingKey? 'AS'i Materialized? SubqueryReference -UsingKey <- 'USING'i 'KEY'i Parens(List(ColId)) -Materialized <- 'NOT'i? 'MATERIALIZED'i -WithClause <- 'WITH'i Recursive? List(WithStatement) -Recursive <- 'RECURSIVE'i -SelectClause <- 'SELECT'i DistinctClause? TargetList +WithStatement <- ColIdOrString InsertColumnList? UsingKey? 'AS' Materialized? SubqueryReference +UsingKey <- 'USING' 'KEY' Parens(List(ColId)) +Materialized <- 'NOT'? 'MATERIALIZED' +WithClause <- 'WITH' Recursive? List(WithStatement) +Recursive <- 'RECURSIVE' +SelectClause <- 'SELECT' DistinctClause? TargetList TargetList <- List(AliasedExpression) ColumnAliases <- Parens(List(ColIdOrString)) -DistinctClause <- ('DISTINCT'i DistinctOn?) / 'ALL'i -DistinctOn <- 'ON'i Parens(List(Expression)) +DistinctClause <- ('DISTINCT' DistinctOn?) / 'ALL' +DistinctOn <- 'ON' Parens(List(Expression)) InnerTableRef <- ValuesRef / TableFunction / TableSubquery / BaseTableRef / ParensTableRef @@ -944,83 +950,87 @@ PivotValueLists <- PivotValueList PivotValueList* PivotValueList <- PivotHeader 'IN' PivotTargetList PivotTargetList <- Identifier / Parens(TargetList) -Lateral <- 'LATERAL'i +Lateral <- 'LATERAL' BaseTableName <- CatalogReservedSchemaTable / SchemaReservedTable / TableName SchemaReservedTable <- SchemaQualification ReservedTableName CatalogReservedSchemaTable <- CatalogQualification ReservedSchemaQualification ReservedTableName TableFunction <- TableFunctionLateralOpt / TableFunctionAliasColon -TableFunctionLateralOpt <- Lateral? QualifiedTableFunction TableFunctionArguments TableAlias? -TableFunctionAliasColon <- TableAliasColon QualifiedTableFunction TableFunctionArguments +TableFunctionLateralOpt <- Lateral? QualifiedTableFunction TableFunctionArguments WithOrdinality? TableAlias? +TableFunctionAliasColon <- TableAliasColon QualifiedTableFunction TableFunctionArguments WithOrdinality? +WithOrdinality <- 'WITH' 'ORDINALITY' QualifiedTableFunction <- CatalogQualification? SchemaQualification? TableFunctionName TableFunctionArguments <- Parens(List(FunctionArgument)?) FunctionArgument <- NamedParameter / Expression -NamedParameter <- TypeName NamedParameterAssignment Expression +NamedParameter <- TypeName Type? NamedParameterAssignment Expression NamedParameterAssignment <- ':=' / '=>' -TableAlias <- 'AS'i? (Identifier / StringLiteral) ColumnAliases? +TableAlias <- 'AS'? (Identifier / StringLiteral) ColumnAliases? -AtClause <- 'AT'i Parens(AtSpecifier) +AtClause <- 'AT' Parens(AtSpecifier) AtSpecifier <- AtUnit '=>' Expression -AtUnit <- 'VERSION'i / 'TIMESTAMP'i +AtUnit <- 'VERSION' / 'TIMESTAMP' JoinClause <- RegularJoinClause / JoinWithoutOnClause -RegularJoinClause <- 'ASOF'i? JoinType? 'JOIN'i TableRef JoinQualifier -JoinWithoutOnClause <- JoinPrefix 'JOIN'i TableRef +RegularJoinClause <- 'ASOF'? JoinType? 'JOIN' TableRef JoinQualifier +JoinWithoutOnClause <- JoinPrefix 'JOIN' TableRef JoinQualifier <- OnClause / UsingClause -OnClause <- 'ON'i Expression -UsingClause <- 'USING'i Parens(List(ColumnName)) - -OuterJoinType <- 'FULL'i / 'LEFT'i / 'RIGHT'i -JoinType <- (OuterJoinType 'OUTER'i?) / 'SEMI'i / 'ANTI'i / 'INNER'i -JoinPrefix <- 'CROSS'i / ('NATURAL'i JoinType?) / 'POSITIONAL'i - -FromClause <- 'FROM'i List(TableRef) -WhereClause <- 'WHERE'i Expression -GroupByClause <- 'GROUP'i 'BY'i GroupByExpressions -HavingClause <- 'HAVING'i Expression -QualifyClause <- 'QUALIFY'i Expression +OnClause <- 'ON' Expression +UsingClause <- 'USING' Parens(List(ColumnName)) + +OuterJoinType <- 'FULL' / 'LEFT' / 'RIGHT' +JoinType <- (OuterJoinType 'OUTER'?) / 'SEMI' / 'ANTI' / 'INNER' +JoinPrefix <- 'CROSS' / ('NATURAL' JoinType?) / 'POSITIONAL' + +FromClause <- 'FROM' List(TableRef) +WhereClause <- 'WHERE' Expression +GroupByClause <- 'GROUP' 'BY' GroupByExpressions +HavingClause <- 'HAVING' Expression +QualifyClause <- 'QUALIFY' Expression SampleClause <- (TableSample / UsingSample) SampleEntry -UsingSample <- 'USING'i 'SAMPLE'i -TableSample <- 'TABLESAMPLE'i -WindowClause <- 'WINDOW'i List(WindowDefinition) -WindowDefinition <- Identifier 'AS'i WindowFrameDefinition +UsingSample <- 'USING' 'SAMPLE' +TableSample <- 'TABLESAMPLE' +WindowClause <- 'WINDOW' List(WindowDefinition) +WindowDefinition <- Identifier 'AS' WindowFrameDefinition SampleEntry <- SampleEntryFunction / SampleEntryCount SampleEntryCount <- SampleCount Parens(SampleProperties)? SampleEntryFunction <- SampleFunction? Parens(SampleCount) RepeatableSample? SampleFunction <- ColId SampleProperties <- ColId (',' NumberLiteral)? -RepeatableSample <- 'REPEATABLE'i Parens(NumberLiteral) +RepeatableSample <- 'REPEATABLE' Parens(NumberLiteral) SampleCount <- Expression SampleUnit? -SampleUnit <- '%' / 'PERCENT'i / 'ROWS'i +SampleUnit <- '%' / 'PERCENT' / 'ROWS' -GroupByExpressions <- GroupByList / 'ALL'i +GroupByExpressions <- GroupByList / 'ALL' GroupByList <- List(GroupByExpression) GroupByExpression <- EmptyGroupingItem / CubeOrRollupClause / GroupingSetsClause / Expression EmptyGroupingItem <- '(' ')' CubeOrRollupClause <- CubeOrRollup Parens(List(Expression)) -CubeOrRollup <- 'CUBE'i / 'ROLLUP'i -GroupingSetsClause <- 'GROUPING'i 'SETS'i Parens(GroupByList) +CubeOrRollup <- 'CUBE' / 'ROLLUP' +GroupingSetsClause <- 'GROUPING' 'SETS' Parens(GroupByList) SubqueryReference <- Parens(SelectStatement) OrderByExpression <- Expression DescOrAsc? NullsFirstOrLast? -DescOrAsc <- 'DESC'i / 'DESCENDING'i / 'ASC'i / 'ASCENDING'i -NullsFirstOrLast <- 'NULLS'i 'FIRST'i / 'LAST'i -OrderByClause <- 'ORDER'i 'BY'i OrderByExpressions +DescOrAsc <- 'DESC' / 'DESCENDING' / 'ASC' / 'ASCENDING' +NullsFirstOrLast <- 'NULLS' 'FIRST' / 'LAST' +OrderByClause <- 'ORDER' 'BY' OrderByExpressions OrderByExpressions <- List(OrderByExpression) / OrderByAll -OrderByAll <- 'ALL'i DescOrAsc? NullsFirstOrLast? +OrderByAll <- 'ALL' DescOrAsc? NullsFirstOrLast? + +LimitClause <- 'LIMIT' LimitValue +OffsetClause <- 'OFFSET' OffsetValue +LimitValue <- 'ALL' / (NumberLiteral 'PERCENT') / (Expression '%'?) +OffsetValue <- Expression RowOrRows? +RowOrRows <- 'ROW' / 'ROWS' -LimitClause <- 'LIMIT'i LimitValue -OffsetClause <- 'OFFSET'i LimitValue -LimitValue <- 'ALL'i / (NumberLiteral 'PERCENT'i) / (Expression '%'?) -AliasedExpression <- (ColId ':' Expression) / (Expression 'AS'i ColLabelOrString) / (Expression Identifier?) +AliasedExpression <- (ColId ':' Expression) / (Expression 'AS' ColLabelOrString) / (Expression Identifier?) -ValuesClause <- 'VALUES'i List(ValuesExpressions) +ValuesClause <- 'VALUES' List(ValuesExpressions) ValuesExpressions <- Parens(List(Expression)) TransactionStatement <- BeginTransaction / RollbackTransaction / CommitTransaction @@ -1029,120 +1039,129 @@ BeginTransaction <- StartOrBegin Transaction? ReadOrWrite? RollbackTransaction <- AbortOrRollback Transaction? CommitTransaction <- CommitOrEnd Transaction? -StartOrBegin <- 'START'i / 'BEGIN'i -Transaction <- 'WORK'i / 'TRANSACTION'i -ReadOrWrite <- 'READ'i ('ONLY'i / 'WRITE'i) -AbortOrRollback <- 'ABORT'i / 'ROLLBACK'i -CommitOrEnd <- 'COMMIT'i / 'END'i +StartOrBegin <- 'START' / 'BEGIN' +Transaction <- 'WORK' / 'TRANSACTION' +ReadOrWrite <- 'READ' ('ONLY' / 'WRITE') +AbortOrRollback <- 'ABORT' / 'ROLLBACK' +CommitOrEnd <- 'COMMIT' / 'END' -DeleteStatement <- WithClause? 'DELETE'i 'FROM'i TargetOptAlias DeleteUsingClause? WhereClause? ReturningClause? -TruncateStatement <- 'TRUNCATE'i 'TABLE'i? BaseTableName -TargetOptAlias <- BaseTableName 'AS'i? ColId? -DeleteUsingClause <- 'USING'i List(TableRef) +DeleteStatement <- WithClause? 'DELETE' 'FROM' TargetOptAlias DeleteUsingClause? WhereClause? ReturningClause? +TruncateStatement <- 'TRUNCATE' 'TABLE'? BaseTableName +TargetOptAlias <- BaseTableName 'AS'? ColId? +DeleteUsingClause <- 'USING' List(TableRef) -CreateTypeStmt <- 'TYPE'i IfNotExists? QualifiedName 'AS'i CreateType -CreateType <- ('ENUM'i Parens(SelectStatement)) / - ('ENUM'i Parens(List(StringLiteral))) / +CreateTypeStmt <- 'TYPE' IfNotExists? QualifiedName 'AS' CreateType +CreateType <- ('ENUM' Parens(SelectStatement)) / + ('ENUM' Parens(List(StringLiteral))) / Type -SetStatement <- 'SET'i (StandardAssignment / SetTimeZone) +SetStatement <- 'SET' (StandardAssignment / SetTimeZone) StandardAssignment <- (SetVariable / SetSetting) SetAssignment -SetTimeZone <- 'TIME'i 'ZONE'i Expression +SetTimeZone <- 'TIME' 'ZONE' Expression SetSetting <- SettingScope? SettingName -SetVariable <- 'VARIABLE'i Identifier +SetVariable <- VariableScope Identifier +VariableScope <- 'VARIABLE' -SettingScope <- 'LOCAL'i / 'SESSION'i / 'GLOBAL'i +SettingScope <- LocalScope / SessionScope / GlobalScope +LocalScope <- 'LOCAL' +SessionScope <- 'SESSION' +GlobalScope <- 'GLOBAL' SetAssignment <- VariableAssign VariableList VariableAssign <- '=' / 'TO' VariableList <- List(Expression) -ResetStatement <- 'RESET'i (SetVariable / SetSetting) +ResetStatement <- 'RESET' (SetVariable / SetSetting) -ExportStatement <- 'EXPORT'i 'DATABASE'i ExportSource? StringLiteral Parens(GenericCopyOptionList)? +ExportStatement <- 'EXPORT' 'DATABASE' ExportSource? StringLiteral Parens(GenericCopyOptionList)? -ExportSource <- CatalogName 'TO'i +ExportSource <- CatalogName 'TO' -ImportStatement <- 'IMPORT'i 'DATABASE'i StringLiteral +ImportStatement <- 'IMPORT' 'DATABASE' StringLiteral -CheckpointStatement <- 'FORCE'i? 'CHECKPOINT'i CatalogName? +CheckpointStatement <- 'FORCE'? 'CHECKPOINT' CatalogName? -CopyStatement <- 'COPY'i (CopyTable / CopySelect / CopyFromDatabase) +CopyStatement <- 'COPY' (CopyTable / CopySelect / CopyFromDatabase) CopyTable <- BaseTableName InsertColumnList? FromOrTo CopyFileName CopyOptions? -FromOrTo <- 'FROM'i / 'TO'i +FromOrTo <- 'FROM' / 'TO' -CopySelect <- Parens(SelectStatement) 'TO'i CopyFileName CopyOptions? +CopySelect <- Parens(SelectStatement) 'TO' CopyFileName CopyOptions? -CopyFileName <- StringLiteral / Identifier / (Identifier '.' ColId) -CopyOptions <- 'WITH'i? (Parens(GenericCopyOptionList) / (SpecializedOptions*)) +CopyFileName <- Expression / StringLiteral / Identifier / (Identifier '.' ColId) +CopyOptions <- 'WITH'? (Parens(GenericCopyOptionList) / (SpecializedOptions*)) SpecializedOptions <- - 'BINARY'i / 'FREEZE'i / 'OIDS'i / 'CSV'i / 'HEADER'i / + 'BINARY' / 'FREEZE' / 'OIDS' / 'CSV' / 'HEADER' / SpecializedStringOption / - ('ENCODING'i StringLiteral) / - ('FORCE'i 'QUOTE'i StarOrColumnList) / - ('PARTITION'i 'BY'i StarOrColumnList) / - ('FORCE'i 'NOT'i? 'NULL'i ColumnList) + ('ENCODING' StringLiteral) / + ('FORCE' 'QUOTE' StarOrColumnList) / + ('PARTITION' 'BY' StarOrColumnList) / + ('FORCE' 'NOT'? 'NULL' ColumnList) -SpecializedStringOption <- ('DELIMITER'i / 'NULL'i / 'QUOTE'i / 'ESCAPE'i) 'AS'i? StringLiteral +SpecializedStringOption <- ('DELIMITER' / 'NULL' / 'QUOTE' / 'ESCAPE') 'AS'? StringLiteral StarOrColumnList <- '*' / ColumnList GenericCopyOptionList <- List(GenericCopyOption) GenericCopyOption <- GenericCopyOptionName Expression? # FIXME: should not need to hard-code options here -GenericCopyOptionName <- 'ARRAY'i / 'NULL'i / 'ANALYZE'i / CopyOptionName +GenericCopyOptionName <- 'ARRAY' / 'NULL' / 'ANALYZE' / CopyOptionName -CopyFromDatabase <- 'FROM'i 'DATABASE'i ColId 'TO'i ColId CopyDatabaseFlag? +CopyFromDatabase <- 'FROM' 'DATABASE' ColId 'TO' ColId CopyDatabaseFlag? CopyDatabaseFlag <- Parens(SchemaOrData) -SchemaOrData <- 'SCHEMA'i / 'DATA'i +SchemaOrData <- 'SCHEMA' / 'DATA' -AlterStatement <- 'ALTER'i AlterOptions +AlterStatement <- 'ALTER' AlterOptions -AlterOptions <- AlterTableStmt / AlterViewStmt / AlterSequenceStmt +AlterOptions <- AlterTableStmt / AlterViewStmt / AlterSequenceStmt / AlterDatabaseStmt / AlterSchemaStmt -AlterTableStmt <- 'TABLE'i IfExists? BaseTableName AlterTableOptions +AlterTableStmt <- 'TABLE' IfExists? BaseTableName AlterTableOptions +AlterSchemaStmt <- 'SCHEMA' IfExists? QualifiedName RenameAlter AlterTableOptions <- AddColumn / DropColumn / AlterColumn / AddConstraint / ChangeNullability / RenameColumn / RenameAlter / SetPartitionedBy / ResetPartitionedBy / SetSortedBy / ResetSortedBy -AddConstraint <- 'ADD'i TopLevelConstraint -AddColumn <- 'ADD'i 'COLUMN'i? IfNotExists? ColumnDefinition -DropColumn <- 'DROP'i 'COLUMN'i? IfExists? NestedColumnName DropBehavior? -AlterColumn <- 'ALTER'i 'COLUMN'i? NestedColumnName AlterColumnEntry -RenameColumn <- 'RENAME'i 'COLUMN'i? NestedColumnName 'TO'i Identifier +AddConstraint <- 'ADD' TopLevelConstraint +AddColumn <- 'ADD' 'COLUMN'? IfNotExists? ColumnDefinition +DropColumn <- 'DROP' 'COLUMN'? IfExists? NestedColumnName DropBehavior? +AlterColumn <- 'ALTER' 'COLUMN'? NestedColumnName AlterColumnEntry +RenameColumn <- 'RENAME' 'COLUMN'? NestedColumnName 'TO' Identifier NestedColumnName <- (Identifier '.')* ColumnName -RenameAlter <- 'RENAME'i 'TO'i Identifier -SetPartitionedBy <- 'SET'i 'PARTITIONED'i 'BY'i Parens(List(Expression)) -ResetPartitionedBy <- 'RESET'i 'PARTITIONED'i 'BY'i -SetSortedBy <- 'SET'i 'SORTED'i 'BY'i OrderByExpressions -ResetSortedBy <- 'RESET'i 'SORTED'i 'BY'i +RenameAlter <- 'RENAME' 'TO' Identifier +SetPartitionedBy <- 'SET' 'PARTITIONED' 'BY' Parens(List(Expression)) +ResetPartitionedBy <- 'RESET' 'PARTITIONED' 'BY' +SetSortedBy <- 'SET' 'SORTED' 'BY' Parens(OrderByExpressions) +ResetSortedBy <- 'RESET' 'SORTED' 'BY' AlterColumnEntry <- AddOrDropDefault / ChangeNullability / AlterType AddOrDropDefault <- AddDefault / DropDefault -AddDefault <- 'SET'i 'DEFAULT'i Expression -DropDefault <- 'DROP'i 'DEFAULT'i +AddDefault <- 'SET' 'DEFAULT' Expression +DropDefault <- 'DROP' 'DEFAULT' -ChangeNullability <- ('DROP'i / 'SET'i) 'NOT'i 'NULL'i +ChangeNullability <- ('DROP' / 'SET') 'NOT' 'NULL' -AlterType <- SetData? 'TYPE'i Type? UsingExpression? -SetData <- 'SET'i 'DATA'i? -UsingExpression <- 'USING'i Expression +AlterType <- SetData? 'TYPE' Type? UsingExpression? +SetData <- 'SET' 'DATA'? +UsingExpression <- 'USING' Expression -AlterViewStmt <- 'VIEW'i IfExists? BaseTableName RenameAlter +AlterViewStmt <- 'VIEW' IfExists? BaseTableName RenameAlter -AlterSequenceStmt <- 'SEQUENCE'i IfExists? QualifiedSequenceName AlterSequenceOptions +AlterSequenceStmt <- 'SEQUENCE' IfExists? QualifiedSequenceName AlterSequenceOptions QualifiedSequenceName <- CatalogQualification? SchemaQualification? SequenceName AlterSequenceOptions <- RenameAlter / SetSequenceOption SetSequenceOption <- List(SequenceOption) -CreateSequenceStmt <- 'SEQUENCE'i IfNotExists? QualifiedName SequenceOption* +AlterDatabaseStmt <- 'DATABASE' IfExists? Identifier RenameDatabaseAlter + +RenameDatabaseAlter <- 'RENAME' 'TO' Identifier + +CreateSequenceStmt <- 'SEQUENCE' IfNotExists? QualifiedName SequenceOption* SequenceOption <- SeqSetCycle / @@ -1152,15 +1171,15 @@ SequenceOption <- SeqStartWith / SeqOwnedBy -SeqSetCycle <- 'NO'i? 'CYCLE'i -SeqSetIncrement <- 'INCREMENT'i 'BY'i? Expression +SeqSetCycle <- 'NO'? 'CYCLE' +SeqSetIncrement <- 'INCREMENT' 'BY'? Expression SeqSetMinMax <- SeqMinOrMax Expression -SeqNoMinMax <- 'NO'i SeqMinOrMax -SeqStartWith <- 'START'i 'WITH'i? Expression -SeqOwnedBy <- 'OWNED'i 'BY'i QualifiedName +SeqNoMinMax <- 'NO' SeqMinOrMax +SeqStartWith <- 'START' 'WITH'? Expression +SeqOwnedBy <- 'OWNED' 'BY' QualifiedName -SeqMinOrMax <- 'MINVALUE'i / 'MAXVALUE'i +SeqMinOrMax <- 'MINVALUE' / 'MAXVALUE' Statement <- @@ -1218,29 +1237,29 @@ SecretName <- ColId NumberLiteral <- < [+-]?[0-9]*([.][0-9]*)? > StringLiteral <- '\'' [^\']* '\'' -Type <- (TimeType / IntervalType / BitType / RowType / MapType / UnionType / NumericType / SimpleType) ArrayBounds* +Type <- (TimeType / IntervalType / BitType / RowType / MapType / UnionType / NumericType / SetofType / SimpleType) ArrayBounds* SimpleType <- (QualifiedTypeName / CharacterType) TypeModifiers? -CharacterType <- ('CHARACTER'i 'VARYING'i?) / - ('CHAR'i 'VARYING'i?) / - ('NATIONAL'i 'CHARACTER'i 'VARYING'i?) / - ('NATIONAL'i 'CHAR'i 'VARYING'i?) / - ('NCHAR'i 'VARYING'i?) / - 'VARCHAR'i -IntervalType <- ('INTERVAL'i Interval?) / ('INTERVAL'i Parens(NumberLiteral)) - -YearKeyword <- 'YEAR'i / 'YEARS'i -MonthKeyword <- 'MONTH'i / 'MONTHS'i -DayKeyword <- 'DAY'i / 'DAYS'i -HourKeyword <- 'HOUR'i / 'HOURS'i -MinuteKeyword <- 'MINUTE'i / 'MINUTES'i -SecondKeyword <- 'SECOND'i / 'SECONDS'i -MillisecondKeyword <- 'MILLISECOND'i / 'MILLISECONDS'i -MicrosecondKeyword <- 'MICROSECOND'i / 'MICROSECONDS'i -WeekKeyword <- 'WEEK'i / 'WEEKS'i -QuarterKeyword <- 'QUARTER'i / 'QUARTERS'i -DecadeKeyword <- 'DECADE'i / 'DECADES'i -CenturyKeyword <- 'CENTURY'i / 'CENTURIES'i -MillenniumKeyword <- 'MILLENNIUM'i / 'MILLENNIA'i +CharacterType <- ('CHARACTER' 'VARYING'?) / + ('CHAR' 'VARYING'?) / + ('NATIONAL' 'CHARACTER' 'VARYING'?) / + ('NATIONAL' 'CHAR' 'VARYING'?) / + ('NCHAR' 'VARYING'?) / + 'VARCHAR' +IntervalType <- ('INTERVAL' Interval?) / ('INTERVAL' Parens(NumberLiteral)) + +YearKeyword <- 'YEAR' / 'YEARS' +MonthKeyword <- 'MONTH' / 'MONTHS' +DayKeyword <- 'DAY' / 'DAYS' +HourKeyword <- 'HOUR' / 'HOURS' +MinuteKeyword <- 'MINUTE' / 'MINUTES' +SecondKeyword <- 'SECOND' / 'SECONDS' +MillisecondKeyword <- 'MILLISECOND' / 'MILLISECONDS' +MicrosecondKeyword <- 'MICROSECOND' / 'MICROSECONDS' +WeekKeyword <- 'WEEK' / 'WEEKS' +QuarterKeyword <- 'QUARTER' / 'QUARTERS' +DecadeKeyword <- 'DECADE' / 'DECADES' +CenturyKeyword <- 'CENTURY' / 'CENTURIES' +MillenniumKeyword <- 'MILLENNIUM' / 'MILLENNIA' Interval <- YearKeyword / MonthKeyword / @@ -1255,84 +1274,85 @@ Interval <- YearKeyword / DecadeKeyword / CenturyKeyword / MillenniumKeyword / - (YearKeyword 'TO'i MonthKeyword) / - (DayKeyword 'TO'i HourKeyword) / - (DayKeyword 'TO'i MinuteKeyword) / - (DayKeyword 'TO'i SecondKeyword) / - (HourKeyword 'TO'i MinuteKeyword) / - (HourKeyword 'TO'i SecondKeyword) / - (MinuteKeyword 'TO'i SecondKeyword) - -BitType <- 'BIT'i 'VARYING'i? Parens(List(Expression))? - -NumericType <- 'INT'i / - 'INTEGER'i / - 'SMALLINT'i / - 'BIGINT'i / - 'REAL'i / - 'BOOLEAN'i / - ('FLOAT'i Parens(NumberLiteral)?) / - ('DOUBLE'i 'PRECISION'i) / - ('DECIMAL'i TypeModifiers?) / - ('DEC'i TypeModifiers?) / - ('NUMERIC'i TypeModifiers?) + (YearKeyword 'TO' MonthKeyword) / + (DayKeyword 'TO' HourKeyword) / + (DayKeyword 'TO' MinuteKeyword) / + (DayKeyword 'TO' SecondKeyword) / + (HourKeyword 'TO' MinuteKeyword) / + (HourKeyword 'TO' SecondKeyword) / + (MinuteKeyword 'TO' SecondKeyword) + +BitType <- 'BIT' 'VARYING'? Parens(List(Expression))? + +NumericType <- 'INT' / + 'INTEGER' / + 'SMALLINT' / + 'BIGINT' / + 'REAL' / + 'BOOLEAN' / + ('FLOAT' Parens(NumberLiteral)?) / + ('DOUBLE' 'PRECISION') / + ('DECIMAL' TypeModifiers?) / + ('DEC' TypeModifiers?) / + ('NUMERIC' TypeModifiers?) QualifiedTypeName <- CatalogQualification? SchemaQualification? TypeName TypeModifiers <- Parens(List(Expression)?) RowType <- RowOrStruct Parens(List(ColIdType)) -UnionType <- 'UNION'i Parens(List(ColIdType)) -MapType <- 'MAP'i Parens(List(Type)) +UnionType <- 'UNION' Parens(List(ColIdType)) +SetofType <- 'SETOF' Type +MapType <- 'MAP' Parens(List(Type)) ColIdType <- ColId Type -ArrayBounds <- ('[' NumberLiteral? ']') / 'ARRAY'i +ArrayBounds <- ('[' NumberLiteral? ']') / 'ARRAY' TimeType <- TimeOrTimestamp TypeModifiers? TimeZone? -TimeOrTimestamp <- 'TIME'i / 'TIMESTAMP'i -TimeZone <- WithOrWithout 'TIME'i 'ZONE'i -WithOrWithout <- 'WITH'i / 'WITHOUT'i +TimeOrTimestamp <- 'TIME' / 'TIMESTAMP' +TimeZone <- WithOrWithout 'TIME' 'ZONE' +WithOrWithout <- 'WITH' / 'WITHOUT' -RowOrStruct <- 'ROW'i / 'STRUCT'i +RowOrStruct <- 'ROW' / 'STRUCT' # internal definitions %whitespace <- [ \t\n\r]* List(D) <- D (',' D)* ','? Parens(D) <- '(' D ')' -ExplainStatement <- 'EXPLAIN'i 'ANALYZE'i? ExplainOptions? Statement +ExplainStatement <- 'EXPLAIN' 'ANALYZE'? ExplainOptions? Statement ExplainOptions <- Parens(GenericCopyOptionList) -AnalyzeStatement <- 'ANALYZE'i 'VERBOSE'i? AnalyzeTarget? +AnalyzeStatement <- 'ANALYZE' 'VERBOSE'? AnalyzeTarget? AnalyzeTarget <- QualifiedName Parens(List(Name))? Name <- ColId ('.' ColLabel)* CreateMacroStmt <- MacroOrFunction IfNotExists? QualifiedName List(MacroDefinition) -MacroOrFunction <- 'MACRO'i / 'FUNCTION'i +MacroOrFunction <- 'MACRO' / 'FUNCTION' -MacroDefinition <- Parens(MacroParameters?) 'AS'i (TableMacroDefinition / ScalarMacroDefinition) +MacroDefinition <- Parens(MacroParameters?) 'AS' (TableMacroDefinition / ScalarMacroDefinition) MacroParameters <- List(MacroParameter) -MacroParameter <- NamedParameter / TypeFuncName +MacroParameter <- NamedParameter / (TypeFuncName Type?) ScalarMacroDefinition <- Expression -TableMacroDefinition <- 'TABLE'i SelectStatement +TableMacroDefinition <- 'TABLE' SelectStatement -CommentStatement <- 'COMMENT'i 'ON'i CommentOnType ColumnReference 'IS'i CommentValue +CommentStatement <- 'COMMENT' 'ON' CommentOnType ColumnReference 'IS' CommentValue -CommentOnType <- 'TABLE'i / 'SEQUENCE'i / 'FUNCTION'i / ('MACRO'i 'TABLE'i?) / 'VIEW'i / 'DATABASE'i / 'INDEX'i / 'SCHEMA'i / 'TYPE'i / 'COLUMN'i -CommentValue <- 'NULL'i / StringLiteral +CommentOnType <- 'TABLE' / 'SEQUENCE' / 'FUNCTION' / ('MACRO' 'TABLE'?) / 'VIEW' / 'DATABASE' / 'INDEX' / 'SCHEMA' / 'TYPE' / 'COLUMN' +CommentValue <- 'NULL' / StringLiteral -AttachStatement <- 'ATTACH'i OrReplace? IfNotExists? Database? DatabasePath AttachAlias? AttachOptions? +AttachStatement <- 'ATTACH' OrReplace? IfNotExists? Database? DatabasePath AttachAlias? AttachOptions? -Database <- 'DATABASE'i +Database <- 'DATABASE' DatabasePath <- StringLiteral -AttachAlias <- 'AS'i ColId +AttachAlias <- 'AS' ColId AttachOptions <- Parens(GenericCopyOptionList) -DetachStatement <- 'DETACH'i Database? IfExists? CatalogName +DetachStatement <- 'DETACH' Database? IfExists? CatalogName -UseStatement <- 'USE'i UseTarget +UseStatement <- 'USE' UseTarget UseTarget <- (CatalogName '.' ReservedSchemaName) / SchemaName / CatalogName -CallStatement <- 'CALL'i TableFunctionName TableFunctionArguments +CallStatement <- 'CALL' TableFunctionName TableFunctionArguments diff --git a/extension/autocomplete/include/inlined_grammar.hpp b/extension/autocomplete/include/inlined_grammar.hpp index 66a0a81051be..4ac2928d00e6 100644 --- a/extension/autocomplete/include/inlined_grammar.hpp +++ b/extension/autocomplete/include/inlined_grammar.hpp @@ -4,532 +4,532 @@ namespace duckdb { const char INLINED_PEG_GRAMMAR[] = { - "UnreservedKeyword <- 'ABORT'i /\n" - "'ABSOLUTE'i /\n" - "'ACCESS'i /\n" - "'ACTION'i /\n" - "'ADD'i /\n" - "'ADMIN'i /\n" - "'AFTER'i /\n" - "'AGGREGATE'i /\n" - "'ALSO'i /\n" - "'ALTER'i /\n" - "'ALWAYS'i /\n" - "'ASSERTION'i /\n" - "'ASSIGNMENT'i /\n" - "'ATTACH'i /\n" - "'ATTRIBUTE'i /\n" - "'BACKWARD'i /\n" - "'BEFORE'i /\n" - "'BEGIN'i /\n" - "'CACHE'i /\n" - "'CALL'i /\n" - "'CALLED'i /\n" - "'CASCADE'i /\n" - "'CASCADED'i /\n" - "'CATALOG'i /\n" - "'CENTURY'i /\n" - "'CENTURIES'i /\n" - "'CHAIN'i /\n" - "'CHARACTERISTICS'i /\n" - "'CHECKPOINT'i /\n" - "'CLASS'i /\n" - "'CLOSE'i /\n" - "'CLUSTER'i /\n" - "'COMMENT'i /\n" - "'COMMENTS'i /\n" - "'COMMIT'i /\n" - "'COMMITTED'i /\n" - "'COMPRESSION'i /\n" - "'CONFIGURATION'i /\n" - "'CONFLICT'i /\n" - "'CONNECTION'i /\n" - "'CONSTRAINTS'i /\n" - "'CONTENT'i /\n" - "'CONTINUE'i /\n" - "'CONVERSION'i /\n" - "'COPY'i /\n" - "'COST'i /\n" - "'CSV'i /\n" - "'CUBE'i /\n" - "'CURRENT'i /\n" - "'CURSOR'i /\n" - "'CYCLE'i /\n" - "'DATA'i /\n" - "'DATABASE'i /\n" - "'DAY'i /\n" - "'DAYS'i /\n" - "'DEALLOCATE'i /\n" - "'DECADE'i /\n" - "'DECADES'i /\n" - "'DECLARE'i /\n" - "'DEFAULTS'i /\n" - "'DEFERRED'i /\n" - "'DEFINER'i /\n" - "'DELETE'i /\n" - "'DELIMITER'i /\n" - "'DELIMITERS'i /\n" - "'DEPENDS'i /\n" - "'DETACH'i /\n" - "'DICTIONARY'i /\n" - "'DISABLE'i /\n" - "'DISCARD'i /\n" - "'DOCUMENT'i /\n" - "'DOMAIN'i /\n" - "'DOUBLE'i /\n" - "'DROP'i /\n" - "'EACH'i /\n" - "'ENABLE'i /\n" - "'ENCODING'i /\n" - "'ENCRYPTED'i /\n" - "'ENUM'i /\n" - "'ERROR'i /\n" - "'ESCAPE'i /\n" - "'EVENT'i /\n" - "'EXCLUDE'i /\n" - "'EXCLUDING'i /\n" - "'EXCLUSIVE'i /\n" - "'EXECUTE'i /\n" - "'EXPLAIN'i /\n" - "'EXPORT'i /\n" - "'EXPORT_STATE'i /\n" - "'EXTENSION'i /\n" - "'EXTENSIONS'i /\n" - "'EXTERNAL'i /\n" - "'FAMILY'i /\n" - "'FILTER'i /\n" - "'FIRST'i /\n" - "'FOLLOWING'i /\n" - "'FORCE'i /\n" - "'FORWARD'i /\n" - "'FUNCTION'i /\n" - "'FUNCTIONS'i /\n" - "'GLOBAL'i /\n" - "'GRANT'i /\n" - "'GRANTED'i /\n" - "'GROUPS'i /\n" - "'HANDLER'i /\n" - "'HEADER'i /\n" - "'HOLD'i /\n" - "'HOUR'i /\n" - "'HOURS'i /\n" - "'IDENTITY'i /\n" - "'IF'i /\n" - "'IGNORE'i /\n" - "'IMMEDIATE'i /\n" - "'IMMUTABLE'i /\n" - "'IMPLICIT'i /\n" - "'IMPORT'i /\n" - "'INCLUDE'i /\n" - "'INCLUDING'i /\n" - "'INCREMENT'i /\n" - "'INDEX'i /\n" - "'INDEXES'i /\n" - "'INHERIT'i /\n" - "'INHERITS'i /\n" - "'INLINE'i /\n" - "'INPUT'i /\n" - "'INSENSITIVE'i /\n" - "'INSERT'i /\n" - "'INSTALL'i /\n" - "'INSTEAD'i /\n" - "'INVOKER'i /\n" - "'JSON'i /\n" - "'ISOLATION'i /\n" - "'KEY'i /\n" - "'LABEL'i /\n" - "'LANGUAGE'i /\n" - "'LARGE'i /\n" - "'LAST'i /\n" - "'LEAKPROOF'i /\n" - "'LEVEL'i /\n" - "'LISTEN'i /\n" - "'LOAD'i /\n" - "'LOCAL'i /\n" - "'LOCATION'i /\n" - "'LOCK'i /\n" - "'LOCKED'i /\n" - "'LOGGED'i /\n" - "'MACRO'i /\n" - "'MAPPING'i /\n" - "'MATCH'i /\n" - "'MATCHED'i /\n" - "'MATERIALIZED'i /\n" - "'MAXVALUE'i /\n" - "'MERGE'i /\n" - "'METHOD'i /\n" - "'MICROSECOND'i /\n" - "'MICROSECONDS'i /\n" - "'MILLENNIUM'i /\n" - "'MILLENNIA'i /\n" - "'MILLISECOND'i /\n" - "'MILLISECONDS'i /\n" - "'MINUTE'i /\n" - "'MINUTES'i /\n" - "'MINVALUE'i /\n" - "'MODE'i /\n" - "'MONTH'i /\n" - "'MONTHS'i /\n" - "'MOVE'i /\n" - "'NAME'i /\n" - "'NAMES'i /\n" - "'NEW'i /\n" - "'NEXT'i /\n" - "'NO'i /\n" - "'NOTHING'i /\n" - "'NOTIFY'i /\n" - "'NOWAIT'i /\n" - "'NULLS'i /\n" - "'OBJECT'i /\n" - "'OF'i /\n" - "'OFF'i /\n" - "'OIDS'i /\n" - "'OLD'i /\n" - "'OPERATOR'i /\n" - "'OPTION'i /\n" - "'OPTIONS'i /\n" - "'ORDINALITY'i /\n" - "'OTHERS'i /\n" - "'OVER'i /\n" - "'OVERRIDING'i /\n" - "'OWNED'i /\n" - "'OWNER'i /\n" - "'PARALLEL'i /\n" - "'PARSER'i /\n" - "'PARTIAL'i /\n" - "'PARTITION'i /\n" - "'PARTITIONED'i /\n" - "'PASSING'i /\n" - "'PASSWORD'i /\n" - "'PERCENT'i /\n" - "'PERSISTENT'i /\n" - "'PLANS'i /\n" - "'POLICY'i /\n" - "'PRAGMA'i /\n" - "'PRECEDING'i /\n" - "'PREPARE'i /\n" - "'PREPARED'i /\n" - "'PRESERVE'i /\n" - "'PRIOR'i /\n" - "'PRIVILEGES'i /\n" - "'PROCEDURAL'i /\n" - "'PROCEDURE'i /\n" - "'PROGRAM'i /\n" - "'PUBLICATION'i /\n" - "'QUARTER'i /\n" - "'QUARTERS'i /\n" - "'QUOTE'i /\n" - "'RANGE'i /\n" - "'READ'i /\n" - "'REASSIGN'i /\n" - "'RECHECK'i /\n" - "'RECURSIVE'i /\n" - "'REF'i /\n" - "'REFERENCING'i /\n" - "'REFRESH'i /\n" - "'REINDEX'i /\n" - "'RELATIVE'i /\n" - "'RELEASE'i /\n" - "'RENAME'i /\n" - "'REPEATABLE'i /\n" - "'REPLACE'i /\n" - "'REPLICA'i /\n" - "'RESET'i /\n" - "'RESPECT'i /\n" - "'RESTART'i /\n" - "'RESTRICT'i /\n" - "'RETURNS'i /\n" - "'REVOKE'i /\n" - "'ROLE'i /\n" - "'ROLLBACK'i /\n" - "'ROLLUP'i /\n" - "'ROWS'i /\n" - "'RULE'i /\n" - "'SAMPLE'i /\n" - "'SAVEPOINT'i /\n" - "'SCHEMA'i /\n" - "'SCHEMAS'i /\n" - "'SCOPE'i /\n" - "'SCROLL'i /\n" - "'SEARCH'i /\n" - "'SECRET'i /\n" - "'SECOND'i /\n" - "'SECONDS'i /\n" - "'SECURITY'i /\n" - "'SEQUENCE'i /\n" - "'SEQUENCES'i /\n" - "'SERIALIZABLE'i /\n" - "'SERVER'i /\n" - "'SESSION'i /\n" - "'SET'i /\n" - "'SETS'i /\n" - "'SHARE'i /\n" - "'SIMPLE'i /\n" - "'SKIP'i /\n" - "'SNAPSHOT'i /\n" - "'SORTED'i /\n" - "'SOURCE'i /\n" - "'SQL'i /\n" - "'STABLE'i /\n" - "'STANDALONE'i /\n" - "'START'i /\n" - "'STATEMENT'i /\n" - "'STATISTICS'i /\n" - "'STDIN'i /\n" - "'STDOUT'i /\n" - "'STORAGE'i /\n" - "'STORED'i /\n" - "'STRICT'i /\n" - "'STRIP'i /\n" - "'SUBSCRIPTION'i /\n" - "'SYSID'i /\n" - "'SYSTEM'i /\n" - "'TABLES'i /\n" - "'TABLESPACE'i /\n" - "'TARGET'i /\n" - "'TEMP'i /\n" - "'TEMPLATE'i /\n" - "'TEMPORARY'i /\n" - "'TEXT'i /\n" - "'TIES'i /\n" - "'TRANSACTION'i /\n" - "'TRANSFORM'i /\n" - "'TRIGGER'i /\n" - "'TRUNCATE'i /\n" - "'TRUSTED'i /\n" - "'TYPE'i /\n" - "'TYPES'i /\n" - "'UNBOUNDED'i /\n" - "'UNCOMMITTED'i /\n" - "'UNENCRYPTED'i /\n" - "'UNKNOWN'i /\n" - "'UNLISTEN'i /\n" - "'UNLOGGED'i /\n" - "'UNTIL'i /\n" - "'UPDATE'i /\n" - "'USE'i /\n" - "'USER'i /\n" - "'VACUUM'i /\n" - "'VALID'i /\n" - "'VALIDATE'i /\n" - "'VALIDATOR'i /\n" - "'VALUE'i /\n" - "'VARIABLE'i /\n" - "'VARYING'i /\n" - "'VERSION'i /\n" - "'VIEW'i /\n" - "'VIEWS'i /\n" - "'VIRTUAL'i /\n" - "'VOLATILE'i /\n" - "'WEEK'i /\n" - "'WEEKS'i /\n" - "'WHITESPACE'i /\n" - "'WITHIN'i /\n" - "'WITHOUT'i /\n" - "'WORK'i /\n" - "'WRAPPER'i /\n" - "'WRITE'i /\n" - "'XML'i /\n" - "'YEAR'i /\n" - "'YEARS'i /\n" - "'YES'i /\n" - "'ZONE'i\n" - "ReservedKeyword <- 'ALL'i /\n" - "'ANALYSE'i /\n" - "'ANALYZE'i /\n" - "'AND'i /\n" - "'ANY'i /\n" - "'ARRAY'i /\n" - "'AS'i /\n" - "'ASC'i /\n" - "'ASYMMETRIC'i /\n" - "'BOTH'i /\n" - "'CASE'i /\n" - "'CAST'i /\n" - "'CHECK'i /\n" - "'COLLATE'i /\n" - "'COLUMN'i /\n" - "'CONSTRAINT'i /\n" - "'CREATE'i /\n" - "'DEFAULT'i /\n" - "'DEFERRABLE'i /\n" - "'DESC'i /\n" - "'DESCRIBE'i /\n" - "'DISTINCT'i /\n" - "'DO'i /\n" - "'ELSE'i /\n" - "'END'i /\n" - "'EXCEPT'i /\n" - "'FALSE'i /\n" - "'FETCH'i /\n" - "'FOR'i /\n" - "'FOREIGN'i /\n" - "'FROM'i /\n" - "'GROUP'i /\n" - "'HAVING'i /\n" - "'QUALIFY'i /\n" - "'IN'i /\n" - "'INITIALLY'i /\n" - "'INTERSECT'i /\n" - "'INTO'i /\n" - "'LAMBDA'i /\n" - "'LATERAL'i /\n" - "'LEADING'i /\n" - "'LIMIT'i /\n" - "'NOT'i /\n" - "'NULL'i /\n" - "'OFFSET'i /\n" - "'ON'i /\n" - "'ONLY'i /\n" - "'OR'i /\n" - "'ORDER'i /\n" - "'PIVOT'i /\n" - "'PIVOT_WIDER'i /\n" - "'PIVOT_LONGER'i /\n" - "'PLACING'i /\n" - "'PRIMARY'i /\n" - "'REFERENCES'i /\n" - "'RETURNING'i /\n" - "'SELECT'i /\n" - "'SHOW'i /\n" - "'SOME'i /\n" - "'SUMMARIZE'i /\n" - "'SYMMETRIC'i /\n" - "'TABLE'i /\n" - "'THEN'i /\n" - "'TO'i /\n" - "'TRAILING'i /\n" - "'TRUE'i /\n" - "'UNION'i /\n" - "'UNIQUE'i /\n" - "'UNPIVOT'i /\n" - "'USING'i /\n" - "'VARIADIC'i /\n" - "'WHEN'i /\n" - "'WHERE'i /\n" - "'WINDOW'i /\n" - "'WITH'i\n" - "ColumnNameKeyword <- 'BETWEEN'i /\n" - "'BIGINT'i /\n" - "'BIT'i /\n" - "'BOOLEAN'i /\n" - "'CHAR'i /\n" - "'CHARACTER'i /\n" - "'COALESCE'i /\n" - "'COLUMNS'i /\n" - "'DEC'i /\n" - "'DECIMAL'i /\n" - "'EXISTS'i /\n" - "'EXTRACT'i /\n" - "'FLOAT'i /\n" - "'GENERATED'i /\n" - "'GROUPING'i /\n" - "'GROUPING_ID'i /\n" - "'INOUT'i /\n" - "'INT'i /\n" - "'INTEGER'i /\n" - "'INTERVAL'i /\n" - "'MAP'i /\n" - "'NATIONAL'i /\n" - "'NCHAR'i /\n" - "'NONE'i /\n" - "'NULLIF'i /\n" - "'NUMERIC'i /\n" - "'OUT'i /\n" - "'OVERLAY'i /\n" - "'POSITION'i /\n" - "'PRECISION'i /\n" - "'REAL'i /\n" - "'ROW'i /\n" - "'SETOF'i /\n" - "'SMALLINT'i /\n" - "'SUBSTRING'i /\n" - "'STRUCT'i /\n" - "'TIME'i /\n" - "'TIMESTAMP'i /\n" - "'TREAT'i /\n" - "'TRIM'i /\n" - "'TRY_CAST'i /\n" - "'VALUES'i /\n" - "'VARCHAR'i /\n" - "'XMLATTRIBUTES'i /\n" - "'XMLCONCAT'i /\n" - "'XMLELEMENT'i /\n" - "'XMLEXISTS'i /\n" - "'XMLFOREST'i /\n" - "'XMLNAMESPACES'i /\n" - "'XMLPARSE'i /\n" - "'XMLPI'i /\n" - "'XMLROOT'i /\n" - "'XMLSERIALIZE'i /\n" - "'XMLTABLE'i\n" - "FuncNameKeyword <- 'ASOF'i /\n" - "'AT'i /\n" - "'AUTHORIZATION'i /\n" - "'BINARY'i /\n" - "'COLLATION'i /\n" - "'CONCURRENTLY'i /\n" - "'CROSS'i /\n" - "'FREEZE'i /\n" - "'FULL'i /\n" - "'GENERATED'i /\n" - "'GLOB'i /\n" - "'ILIKE'i /\n" - "'INNER'i /\n" - "'IS'i /\n" - "'ISNULL'i /\n" - "'JOIN'i /\n" - "'LEFT'i /\n" - "'LIKE'i /\n" - "'MAP'i /\n" - "'NATURAL'i /\n" - "'NOTNULL'i /\n" - "'OUTER'i /\n" - "'OVERLAPS'i /\n" - "'POSITIONAL'i /\n" - "'RIGHT'i /\n" - "'SIMILAR'i /\n" - "'STRUCT'i /\n" - "'TABLESAMPLE'i /\n" - "'VERBOSE'i\n" - "TypeNameKeyword <- 'ASOF'i /\n" - "'AT'i /\n" - "'AUTHORIZATION'i /\n" - "'BINARY'i /\n" - "'BY'i /\n" - "'COLLATION'i /\n" - "'COLUMNS'i /\n" - "'CONCURRENTLY'i /\n" - "'CROSS'i /\n" - "'FREEZE'i /\n" - "'FULL'i /\n" - "'GLOB'i /\n" - "'ILIKE'i /\n" - "'INNER'i /\n" - "'IS'i /\n" - "'ISNULL'i /\n" - "'JOIN'i /\n" - "'LEFT'i /\n" - "'LIKE'i /\n" - "'NATURAL'i /\n" - "'NOTNULL'i /\n" - "'OUTER'i /\n" - "'OVERLAPS'i /\n" - "'POSITIONAL'i /\n" - "'RIGHT'i /\n" - "'UNPACK'i /\n" - "'SIMILAR'i /\n" - "'TABLESAMPLE'i /\n" - "'TRY_CAST'i /\n" - "'VERBOSE'i /\n" - "'SEMI'i /\n" - "'ANTI'i\n" + "UnreservedKeyword <- 'ABORT' /\n" + "'ABSOLUTE' /\n" + "'ACCESS' /\n" + "'ACTION' /\n" + "'ADD' /\n" + "'ADMIN' /\n" + "'AFTER' /\n" + "'AGGREGATE' /\n" + "'ALSO' /\n" + "'ALTER' /\n" + "'ALWAYS' /\n" + "'ASSERTION' /\n" + "'ASSIGNMENT' /\n" + "'ATTACH' /\n" + "'ATTRIBUTE' /\n" + "'BACKWARD' /\n" + "'BEFORE' /\n" + "'BEGIN' /\n" + "'CACHE' /\n" + "'CALL' /\n" + "'CALLED' /\n" + "'CASCADE' /\n" + "'CASCADED' /\n" + "'CATALOG' /\n" + "'CENTURY' /\n" + "'CENTURIES' /\n" + "'CHAIN' /\n" + "'CHARACTERISTICS' /\n" + "'CHECKPOINT' /\n" + "'CLASS' /\n" + "'CLOSE' /\n" + "'CLUSTER' /\n" + "'COMMENT' /\n" + "'COMMENTS' /\n" + "'COMMIT' /\n" + "'COMMITTED' /\n" + "'COMPRESSION' /\n" + "'CONFIGURATION' /\n" + "'CONFLICT' /\n" + "'CONNECTION' /\n" + "'CONSTRAINTS' /\n" + "'CONTENT' /\n" + "'CONTINUE' /\n" + "'CONVERSION' /\n" + "'COPY' /\n" + "'COST' /\n" + "'CSV' /\n" + "'CUBE' /\n" + "'CURRENT' /\n" + "'CURSOR' /\n" + "'CYCLE' /\n" + "'DATA' /\n" + "'DATABASE' /\n" + "'DAY' /\n" + "'DAYS' /\n" + "'DEALLOCATE' /\n" + "'DECADE' /\n" + "'DECADES' /\n" + "'DECLARE' /\n" + "'DEFAULTS' /\n" + "'DEFERRED' /\n" + "'DEFINER' /\n" + "'DELETE' /\n" + "'DELIMITER' /\n" + "'DELIMITERS' /\n" + "'DEPENDS' /\n" + "'DETACH' /\n" + "'DICTIONARY' /\n" + "'DISABLE' /\n" + "'DISCARD' /\n" + "'DOCUMENT' /\n" + "'DOMAIN' /\n" + "'DOUBLE' /\n" + "'DROP' /\n" + "'EACH' /\n" + "'ENABLE' /\n" + "'ENCODING' /\n" + "'ENCRYPTED' /\n" + "'ENUM' /\n" + "'ERROR' /\n" + "'ESCAPE' /\n" + "'EVENT' /\n" + "'EXCLUDE' /\n" + "'EXCLUDING' /\n" + "'EXCLUSIVE' /\n" + "'EXECUTE' /\n" + "'EXPLAIN' /\n" + "'EXPORT' /\n" + "'EXPORT_STATE' /\n" + "'EXTENSION' /\n" + "'EXTENSIONS' /\n" + "'EXTERNAL' /\n" + "'FAMILY' /\n" + "'FILTER' /\n" + "'FIRST' /\n" + "'FOLLOWING' /\n" + "'FORCE' /\n" + "'FORWARD' /\n" + "'FUNCTION' /\n" + "'FUNCTIONS' /\n" + "'GLOBAL' /\n" + "'GRANT' /\n" + "'GRANTED' /\n" + "'GROUPS' /\n" + "'HANDLER' /\n" + "'HEADER' /\n" + "'HOLD' /\n" + "'HOUR' /\n" + "'HOURS' /\n" + "'IDENTITY' /\n" + "'IF' /\n" + "'IGNORE' /\n" + "'IMMEDIATE' /\n" + "'IMMUTABLE' /\n" + "'IMPLICIT' /\n" + "'IMPORT' /\n" + "'INCLUDE' /\n" + "'INCLUDING' /\n" + "'INCREMENT' /\n" + "'INDEX' /\n" + "'INDEXES' /\n" + "'INHERIT' /\n" + "'INHERITS' /\n" + "'INLINE' /\n" + "'INPUT' /\n" + "'INSENSITIVE' /\n" + "'INSERT' /\n" + "'INSTALL' /\n" + "'INSTEAD' /\n" + "'INVOKER' /\n" + "'JSON' /\n" + "'ISOLATION' /\n" + "'KEY' /\n" + "'LABEL' /\n" + "'LANGUAGE' /\n" + "'LARGE' /\n" + "'LAST' /\n" + "'LEAKPROOF' /\n" + "'LEVEL' /\n" + "'LISTEN' /\n" + "'LOAD' /\n" + "'LOCAL' /\n" + "'LOCATION' /\n" + "'LOCK' /\n" + "'LOCKED' /\n" + "'LOGGED' /\n" + "'MACRO' /\n" + "'MAPPING' /\n" + "'MATCH' /\n" + "'MATCHED' /\n" + "'MATERIALIZED' /\n" + "'MAXVALUE' /\n" + "'MERGE' /\n" + "'METHOD' /\n" + "'MICROSECOND' /\n" + "'MICROSECONDS' /\n" + "'MILLENNIUM' /\n" + "'MILLENNIA' /\n" + "'MILLISECOND' /\n" + "'MILLISECONDS' /\n" + "'MINUTE' /\n" + "'MINUTES' /\n" + "'MINVALUE' /\n" + "'MODE' /\n" + "'MONTH' /\n" + "'MONTHS' /\n" + "'MOVE' /\n" + "'NAME' /\n" + "'NAMES' /\n" + "'NEW' /\n" + "'NEXT' /\n" + "'NO' /\n" + "'NOTHING' /\n" + "'NOTIFY' /\n" + "'NOWAIT' /\n" + "'NULLS' /\n" + "'OBJECT' /\n" + "'OF' /\n" + "'OFF' /\n" + "'OIDS' /\n" + "'OLD' /\n" + "'OPERATOR' /\n" + "'OPTION' /\n" + "'OPTIONS' /\n" + "'ORDINALITY' /\n" + "'OTHERS' /\n" + "'OVER' /\n" + "'OVERRIDING' /\n" + "'OWNED' /\n" + "'OWNER' /\n" + "'PARALLEL' /\n" + "'PARSER' /\n" + "'PARTIAL' /\n" + "'PARTITION' /\n" + "'PARTITIONED' /\n" + "'PASSING' /\n" + "'PASSWORD' /\n" + "'PERCENT' /\n" + "'PERSISTENT' /\n" + "'PLANS' /\n" + "'POLICY' /\n" + "'PRAGMA' /\n" + "'PRECEDING' /\n" + "'PREPARE' /\n" + "'PREPARED' /\n" + "'PRESERVE' /\n" + "'PRIOR' /\n" + "'PRIVILEGES' /\n" + "'PROCEDURAL' /\n" + "'PROCEDURE' /\n" + "'PROGRAM' /\n" + "'PUBLICATION' /\n" + "'QUARTER' /\n" + "'QUARTERS' /\n" + "'QUOTE' /\n" + "'RANGE' /\n" + "'READ' /\n" + "'REASSIGN' /\n" + "'RECHECK' /\n" + "'RECURSIVE' /\n" + "'REF' /\n" + "'REFERENCING' /\n" + "'REFRESH' /\n" + "'REINDEX' /\n" + "'RELATIVE' /\n" + "'RELEASE' /\n" + "'RENAME' /\n" + "'REPEATABLE' /\n" + "'REPLACE' /\n" + "'REPLICA' /\n" + "'RESET' /\n" + "'RESPECT' /\n" + "'RESTART' /\n" + "'RESTRICT' /\n" + "'RETURNS' /\n" + "'REVOKE' /\n" + "'ROLE' /\n" + "'ROLLBACK' /\n" + "'ROLLUP' /\n" + "'ROWS' /\n" + "'RULE' /\n" + "'SAMPLE' /\n" + "'SAVEPOINT' /\n" + "'SCHEMA' /\n" + "'SCHEMAS' /\n" + "'SCOPE' /\n" + "'SCROLL' /\n" + "'SEARCH' /\n" + "'SECRET' /\n" + "'SECOND' /\n" + "'SECONDS' /\n" + "'SECURITY' /\n" + "'SEQUENCE' /\n" + "'SEQUENCES' /\n" + "'SERIALIZABLE' /\n" + "'SERVER' /\n" + "'SESSION' /\n" + "'SET' /\n" + "'SETS' /\n" + "'SHARE' /\n" + "'SIMPLE' /\n" + "'SKIP' /\n" + "'SNAPSHOT' /\n" + "'SORTED' /\n" + "'SOURCE' /\n" + "'SQL' /\n" + "'STABLE' /\n" + "'STANDALONE' /\n" + "'START' /\n" + "'STATEMENT' /\n" + "'STATISTICS' /\n" + "'STDIN' /\n" + "'STDOUT' /\n" + "'STORAGE' /\n" + "'STORED' /\n" + "'STRICT' /\n" + "'STRIP' /\n" + "'SUBSCRIPTION' /\n" + "'SYSID' /\n" + "'SYSTEM' /\n" + "'TABLES' /\n" + "'TABLESPACE' /\n" + "'TARGET' /\n" + "'TEMP' /\n" + "'TEMPLATE' /\n" + "'TEMPORARY' /\n" + "'TEXT' /\n" + "'TIES' /\n" + "'TRANSACTION' /\n" + "'TRANSFORM' /\n" + "'TRIGGER' /\n" + "'TRUNCATE' /\n" + "'TRUSTED' /\n" + "'TYPE' /\n" + "'TYPES' /\n" + "'UNBOUNDED' /\n" + "'UNCOMMITTED' /\n" + "'UNENCRYPTED' /\n" + "'UNKNOWN' /\n" + "'UNLISTEN' /\n" + "'UNLOGGED' /\n" + "'UNTIL' /\n" + "'UPDATE' /\n" + "'USE' /\n" + "'USER' /\n" + "'VACUUM' /\n" + "'VALID' /\n" + "'VALIDATE' /\n" + "'VALIDATOR' /\n" + "'VALUE' /\n" + "'VARIABLE' /\n" + "'VARYING' /\n" + "'VERSION' /\n" + "'VIEW' /\n" + "'VIEWS' /\n" + "'VIRTUAL' /\n" + "'VOLATILE' /\n" + "'WEEK' /\n" + "'WEEKS' /\n" + "'WHITESPACE' /\n" + "'WITHIN' /\n" + "'WITHOUT' /\n" + "'WORK' /\n" + "'WRAPPER' /\n" + "'WRITE' /\n" + "'XML' /\n" + "'YEAR' /\n" + "'YEARS' /\n" + "'YES' /\n" + "'ZONE'\n" + "ReservedKeyword <- 'ALL' /\n" + "'ANALYSE' /\n" + "'ANALYZE' /\n" + "'AND' /\n" + "'ANY' /\n" + "'ARRAY' /\n" + "'AS' /\n" + "'ASC' /\n" + "'ASYMMETRIC' /\n" + "'BOTH' /\n" + "'CASE' /\n" + "'CAST' /\n" + "'CHECK' /\n" + "'COLLATE' /\n" + "'COLUMN' /\n" + "'CONSTRAINT' /\n" + "'CREATE' /\n" + "'DEFAULT' /\n" + "'DEFERRABLE' /\n" + "'DESC' /\n" + "'DESCRIBE' /\n" + "'DISTINCT' /\n" + "'DO' /\n" + "'ELSE' /\n" + "'END' /\n" + "'EXCEPT' /\n" + "'FALSE' /\n" + "'FETCH' /\n" + "'FOR' /\n" + "'FOREIGN' /\n" + "'FROM' /\n" + "'GROUP' /\n" + "'HAVING' /\n" + "'QUALIFY' /\n" + "'IN' /\n" + "'INITIALLY' /\n" + "'INTERSECT' /\n" + "'INTO' /\n" + "'LAMBDA' /\n" + "'LATERAL' /\n" + "'LEADING' /\n" + "'LIMIT' /\n" + "'NOT' /\n" + "'NULL' /\n" + "'OFFSET' /\n" + "'ON' /\n" + "'ONLY' /\n" + "'OR' /\n" + "'ORDER' /\n" + "'PIVOT' /\n" + "'PIVOT_WIDER' /\n" + "'PIVOT_LONGER' /\n" + "'PLACING' /\n" + "'PRIMARY' /\n" + "'REFERENCES' /\n" + "'RETURNING' /\n" + "'SELECT' /\n" + "'SHOW' /\n" + "'SOME' /\n" + "'SUMMARIZE' /\n" + "'SYMMETRIC' /\n" + "'TABLE' /\n" + "'THEN' /\n" + "'TO' /\n" + "'TRAILING' /\n" + "'TRUE' /\n" + "'UNION' /\n" + "'UNIQUE' /\n" + "'UNPIVOT' /\n" + "'USING' /\n" + "'VARIADIC' /\n" + "'WHEN' /\n" + "'WHERE' /\n" + "'WINDOW' /\n" + "'WITH'\n" + "ColumnNameKeyword <- 'BETWEEN' /\n" + "'BIGINT' /\n" + "'BIT' /\n" + "'BOOLEAN' /\n" + "'CHAR' /\n" + "'CHARACTER' /\n" + "'COALESCE' /\n" + "'COLUMNS' /\n" + "'DEC' /\n" + "'DECIMAL' /\n" + "'EXISTS' /\n" + "'EXTRACT' /\n" + "'FLOAT' /\n" + "'GENERATED' /\n" + "'GROUPING' /\n" + "'GROUPING_ID' /\n" + "'INOUT' /\n" + "'INT' /\n" + "'INTEGER' /\n" + "'INTERVAL' /\n" + "'MAP' /\n" + "'NATIONAL' /\n" + "'NCHAR' /\n" + "'NONE' /\n" + "'NULLIF' /\n" + "'NUMERIC' /\n" + "'OUT' /\n" + "'OVERLAY' /\n" + "'POSITION' /\n" + "'PRECISION' /\n" + "'REAL' /\n" + "'ROW' /\n" + "'SETOF' /\n" + "'SMALLINT' /\n" + "'SUBSTRING' /\n" + "'STRUCT' /\n" + "'TIME' /\n" + "'TIMESTAMP' /\n" + "'TREAT' /\n" + "'TRIM' /\n" + "'TRY_CAST' /\n" + "'VALUES' /\n" + "'VARCHAR' /\n" + "'XMLATTRIBUTES' /\n" + "'XMLCONCAT' /\n" + "'XMLELEMENT' /\n" + "'XMLEXISTS' /\n" + "'XMLFOREST' /\n" + "'XMLNAMESPACES' /\n" + "'XMLPARSE' /\n" + "'XMLPI' /\n" + "'XMLROOT' /\n" + "'XMLSERIALIZE' /\n" + "'XMLTABLE'\n" + "FuncNameKeyword <- 'ASOF' /\n" + "'AT' /\n" + "'AUTHORIZATION' /\n" + "'BINARY' /\n" + "'COLLATION' /\n" + "'CONCURRENTLY' /\n" + "'CROSS' /\n" + "'FREEZE' /\n" + "'FULL' /\n" + "'GENERATED' /\n" + "'GLOB' /\n" + "'ILIKE' /\n" + "'INNER' /\n" + "'IS' /\n" + "'ISNULL' /\n" + "'JOIN' /\n" + "'LEFT' /\n" + "'LIKE' /\n" + "'MAP' /\n" + "'NATURAL' /\n" + "'NOTNULL' /\n" + "'OUTER' /\n" + "'OVERLAPS' /\n" + "'POSITIONAL' /\n" + "'RIGHT' /\n" + "'SIMILAR' /\n" + "'STRUCT' /\n" + "'TABLESAMPLE' /\n" + "'VERBOSE'\n" + "TypeNameKeyword <- 'ASOF' /\n" + "'AT' /\n" + "'AUTHORIZATION' /\n" + "'BINARY' /\n" + "'BY' /\n" + "'COLLATION' /\n" + "'COLUMNS' /\n" + "'CONCURRENTLY' /\n" + "'CROSS' /\n" + "'FREEZE' /\n" + "'FULL' /\n" + "'GLOB' /\n" + "'ILIKE' /\n" + "'INNER' /\n" + "'IS' /\n" + "'ISNULL' /\n" + "'JOIN' /\n" + "'LEFT' /\n" + "'LIKE' /\n" + "'NATURAL' /\n" + "'NOTNULL' /\n" + "'OUTER' /\n" + "'OVERLAPS' /\n" + "'POSITIONAL' /\n" + "'RIGHT' /\n" + "'UNPACK' /\n" + "'SIMILAR' /\n" + "'TABLESAMPLE' /\n" + "'TRY_CAST' /\n" + "'VERBOSE' /\n" + "'SEMI' /\n" + "'ANTI'\n" "PivotStatement <- PivotKeyword TableRef PivotOn? PivotUsing? GroupByClause?\n" "PivotOn <- 'ON' PivotColumnList\n" "PivotUsing <- 'USING' TargetList\n" "PivotColumnList <- List(Expression)\n" - "PivotKeyword <- 'PIVOT'i / 'PIVOT_WIDER'i\n" - "UnpivotKeyword <- 'UNPIVOT'i / 'PIVOT_LONGER'i\n" + "PivotKeyword <- 'PIVOT' / 'PIVOT_WIDER'\n" + "UnpivotKeyword <- 'UNPIVOT' / 'PIVOT_LONGER'\n" "UnpivotStatement <- UnpivotKeyword TableRef 'ON' TargetList IntoNameValues?\n" "IntoNameValues <- 'INTO' 'NAME' ColIdOrString ValueOrValues List(Identifier)\n" "ValueOrValues <- 'VALUE' / 'VALUES'\n" @@ -543,58 +543,62 @@ const char INLINED_PEG_GRAMMAR[] = { "FunctionIdentifier <- CatalogReservedSchemaFunctionName / SchemaReservedFunctionName / FunctionName\n" "CatalogReservedSchemaFunctionName <- CatalogQualification ReservedSchemaQualification? ReservedFunctionName\n" "SchemaReservedFunctionName <- SchemaQualification ReservedFunctionName\n" - "DistinctOrAll <- 'DISTINCT'i / 'ALL'i\n" - "ExportClause <- 'EXPORT_STATE'i\n" - "WithinGroupClause <- 'WITHIN'i 'GROUP'i Parens(OrderByClause)\n" - "FilterClause <- 'FILTER' Parens('WHERE'i? Expression)\n" - "IgnoreNulls <- ('IGNORE'i 'NULLS'i) / ('RESPECT'i 'NULLS'i)\n" + "DistinctOrAll <- 'DISTINCT' / 'ALL'\n" + "ExportClause <- 'EXPORT_STATE'\n" + "WithinGroupClause <- 'WITHIN' 'GROUP' Parens(OrderByClause)\n" + "FilterClause <- 'FILTER' Parens('WHERE'? Expression)\n" + "IgnoreNulls <- ('IGNORE' 'NULLS') / ('RESPECT' 'NULLS')\n" "ParenthesisExpression <- Parens(List(Expression))\n" - "LiteralExpression <- StringLiteral / NumberLiteral / 'NULL'i / 'TRUE'i / 'FALSE'i\n" - "CastExpression <- CastOrTryCast Parens(Expression 'AS'i Type)\n" - "CastOrTryCast <- 'CAST'i / 'TRY_CAST'i\n" - "StarExpression <- (ColId '.')* '*'i ExcludeList? ReplaceList? RenameList?\n" - "ExcludeList <- 'EXCLUDE'i (Parens(List(ExcludeName)) / ExcludeName)\n" + "LiteralExpression <- StringLiteral / NumberLiteral / ConstantLiteral\n" + "ConstantLiteral <- NullLiteral / TrueLiteral / FalseLiteral\n" + "NullLiteral <- 'NULL'\n" + "TrueLiteral <- 'TRUE'\n" + "FalseLiteral <- 'FALSE'\n" + "CastExpression <- CastOrTryCast Parens(Expression 'AS' Type)\n" + "CastOrTryCast <- 'CAST' / 'TRY_CAST'\n" + "StarExpression <- (ColId '.')* '*' ExcludeList? ReplaceList? RenameList?\n" + "ExcludeList <- 'EXCLUDE' (Parens(List(ExcludeName)) / ExcludeName)\n" "ExcludeName <- DottedIdentifier / ColIdOrString\n" - "ReplaceList <- 'REPLACE'i (Parens(List(ReplaceEntry)) / ReplaceEntry)\n" - "ReplaceEntry <- Expression 'AS'i ColumnReference\n" - "RenameList <- 'RENAME'i (Parens(List(RenameEntry)) / RenameEntry)\n" - "RenameEntry <- ColumnReference 'AS'i Identifier\n" - "SubqueryExpression <- 'NOT'i? 'EXISTS'i? SubqueryReference\n" - "CaseExpression <- 'CASE'i Expression? CaseWhenThen CaseWhenThen* CaseElse? 'END'i\n" - "CaseWhenThen <- 'WHEN'i Expression 'THEN'i Expression\n" - "CaseElse <- 'ELSE'i Expression\n" + "ReplaceList <- 'REPLACE' (Parens(List(ReplaceEntry)) / ReplaceEntry)\n" + "ReplaceEntry <- Expression 'AS' ColumnReference\n" + "RenameList <- 'RENAME' (Parens(List(RenameEntry)) / RenameEntry)\n" + "RenameEntry <- ColumnReference 'AS' Identifier\n" + "SubqueryExpression <- 'NOT'? 'EXISTS'? SubqueryReference\n" + "CaseExpression <- 'CASE' Expression? CaseWhenThen CaseWhenThen* CaseElse? 'END'\n" + "CaseWhenThen <- 'WHEN' Expression 'THEN' Expression\n" + "CaseElse <- 'ELSE' Expression\n" "TypeLiteral <- ColId StringLiteral\n" - "IntervalLiteral <- 'INTERVAL'i IntervalParameter IntervalUnit?\n" + "IntervalLiteral <- 'INTERVAL' IntervalParameter IntervalUnit?\n" "IntervalParameter <- StringLiteral / NumberLiteral / Parens(Expression)\n" "IntervalUnit <- ColId\n" "FrameClause <- Framing FrameExtent WindowExcludeClause?\n" - "Framing <- 'ROWS'i / 'RANGE'i / 'GROUPS'i\n" - "FrameExtent <- ('BETWEEN'i FrameBound 'AND'i FrameBound) / FrameBound\n" - "FrameBound <- ('UNBOUNDED'i 'PRECEDING'i) / ('UNBOUNDED'i 'FOLLOWING'i) / ('CURRENT'i 'ROW'i) / (Expression 'PRECEDING'i) / (Expression 'FOLLOWING'i)\n" - "WindowExcludeClause <- 'EXCLUDE'i WindowExcludeElement\n" - "WindowExcludeElement <- ('CURRENT'i 'ROW'i) / 'GROUP'i / 'TIES'i / ('NO'i 'OTHERS'i)\n" - "OverClause <- 'OVER'i WindowFrame\n" + "Framing <- 'ROWS' / 'RANGE' / 'GROUPS'\n" + "FrameExtent <- ('BETWEEN' FrameBound 'AND' FrameBound) / FrameBound\n" + "FrameBound <- ('UNBOUNDED' 'PRECEDING') / ('UNBOUNDED' 'FOLLOWING') / ('CURRENT' 'ROW') / (Expression 'PRECEDING') / (Expression 'FOLLOWING')\n" + "WindowExcludeClause <- 'EXCLUDE' WindowExcludeElement\n" + "WindowExcludeElement <- ('CURRENT' 'ROW') / 'GROUP' / 'TIES' / ('NO' 'OTHERS')\n" + "OverClause <- 'OVER' WindowFrame\n" "WindowFrame <- WindowFrameDefinition / Identifier / Parens(Identifier)\n" "WindowFrameDefinition <- Parens(BaseWindowName? WindowFrameContents) / Parens(WindowFrameContents)\n" "WindowFrameContents <- WindowPartition? OrderByClause? FrameClause?\n" "BaseWindowName <- Identifier\n" - "WindowPartition <- 'PARTITION'i 'BY'i List(Expression)\n" + "WindowPartition <- 'PARTITION' 'BY' List(Expression)\n" "PrefixExpression <- PrefixOperator Expression\n" - "PrefixOperator <- 'NOT'i / '-' / '+' / '~'\n" - "ListExpression <- 'ARRAY'i? (BoundedListExpression / SelectStatement)\n" + "PrefixOperator <- 'NOT' / '-' / '+' / '~'\n" + "ListExpression <- 'ARRAY'? (BoundedListExpression / SelectStatement)\n" "BoundedListExpression <- '[' List(Expression)? ']'\n" "StructExpression <- '{' List(StructField)? '}'\n" - "StructField <- Expression ':'i Expression\n" - "MapExpression <- 'MAP'i StructExpression\n" + "StructField <- Expression ':' Expression\n" + "MapExpression <- 'MAP' StructExpression\n" "GroupingExpression <- GroupingOrGroupingId Parens(List(Expression))\n" - "GroupingOrGroupingId <- 'GROUPING'i / 'GROUPING_ID'i\n" + "GroupingOrGroupingId <- 'GROUPING' / 'GROUPING_ID'\n" "Parameter <- '?' / NumberedParameter / ColLabelParameter\n" "NumberedParameter <- '$' NumberLiteral\n" "ColLabelParameter <- '$' ColLabel\n" "PositionalExpression <- '#' NumberLiteral\n" - "DefaultExpression <- 'DEFAULT'i\n" - "ListComprehensionExpression <- '['i Expression 'FOR'i List(Expression) ListComprehensionFilter? ']'\n" - "ListComprehensionFilter <- 'IF'i Expression\n" + "DefaultExpression <- 'DEFAULT'\n" + "ListComprehensionExpression <- '[' Expression 'FOR' List(Expression) ListComprehensionFilter? ']'\n" + "ListComprehensionFilter <- 'IF' Expression\n" "SingleExpression <-\n" " LiteralExpression /\n" " Parameter /\n" @@ -617,18 +621,18 @@ const char INLINED_PEG_GRAMMAR[] = { " PositionalExpression /\n" " DefaultExpression\n" "OperatorLiteral <- <[\\+\\-\\*\\/\\%\\^\\<\\>\\=\\~\\!\\@\\&\\|\\`]+>\n" - "LikeOperator <- 'NOT'i? LikeOrSimilarTo\n" - "LikeOrSimilarTo <- 'LIKE'i / 'ILIKE'i / 'GLOB'i / ('SIMILAR'i 'TO'i)\n" - "InOperator <- 'NOT'i? 'IN'i\n" - "IsOperator <- 'IS'i 'NOT'i? DistinctFrom?\n" - "DistinctFrom <- 'DISTINCT'i 'FROM'i\n" - "ConjunctionOperator <- 'OR'i / 'AND'i\n" + "LikeOperator <- 'NOT'? LikeOrSimilarTo\n" + "LikeOrSimilarTo <- 'LIKE' / 'ILIKE' / 'GLOB' / ('SIMILAR' 'TO')\n" + "InOperator <- 'NOT'? 'IN'\n" + "IsOperator <- 'IS' 'NOT'? DistinctFrom?\n" + "DistinctFrom <- 'DISTINCT' 'FROM'\n" + "ConjunctionOperator <- 'OR' / 'AND'\n" "ComparisonOperator <- '=' / '<=' / '>=' / '<' / '>' / '<>' / '!=' / '=='\n" - "BetweenOperator <- 'NOT'i? 'BETWEEN'i\n" - "CollateOperator <- 'COLLATE'i\n" + "BetweenOperator <- 'NOT'? 'BETWEEN'\n" + "CollateOperator <- 'COLLATE'\n" "LambdaOperator <- '->'\n" - "EscapeOperator <- 'ESCAPE'i\n" - "AtTimeZoneOperator <- 'AT'i 'TIME'i 'ZONE'i\n" + "EscapeOperator <- 'ESCAPE'\n" + "AtTimeZoneOperator <- 'AT' 'TIME' 'ZONE'\n" "PostfixOperator <- '!'\n" "AnyAllOperator <- ComparisonOperator AnyOrAll\n" "AnyOrAll <- 'ANY' / 'ALL'\n" @@ -646,7 +650,7 @@ const char INLINED_PEG_GRAMMAR[] = { " OperatorLiteral\n" "CastOperator <- '::' Type\n" "DotOperator <- '.' (FunctionExpression / ColLabel)\n" - "NotNull <- 'NOT'i 'NULL'i\n" + "NotNull <- 'NOT' 'NULL'\n" "Indirection <- CastOperator / DotOperator / SliceExpression / NotNull / PostfixOperator\n" "BaseExpression <- SingleExpression Indirection*\n" "Expression <- BaseExpression RecursiveExpression*\n" @@ -654,70 +658,71 @@ const char INLINED_PEG_GRAMMAR[] = { "SliceExpression <- '[' SliceBound ']'\n" "SliceBound <- Expression? (':' (Expression / '-')?)? (':' Expression?)?\n" "SpecialFunctionExpression <- CoalesceExpression / UnpackExpression / ColumnsExpression / ExtractExpression / LambdaExpression / NullIfExpression / PositionExpression / RowExpression / SubstringExpression / TrimExpression\n" - "CoalesceExpression <- 'COALESCE'i Parens(List(Expression))\n" - "UnpackExpression <- 'UNPACK'i Parens(Expression)\n" - "ColumnsExpression <- '*'? 'COLUMNS'i Parens(Expression)\n" - "ExtractExpression <- 'EXTRACT'i Parens(Expression 'FROM'i Expression)\n" - "LambdaExpression <- 'LAMBDA'i List(ColIdOrString) ':' Expression\n" - "NullIfExpression <- 'NULLIF'i Parens(Expression ',' Expression)\n" - "PositionExpression <- 'POSITION'i Parens(Expression)\n" - "RowExpression <- 'ROW'i Parens(List(Expression))\n" - "SubstringExpression <- 'SUBSTRING'i Parens(SubstringParameters / List(Expression))\n" - "SubstringParameters <- Expression 'FROM'i NumberLiteral 'FOR'i NumberLiteral\n" - "TrimExpression <- 'TRIM'i Parens(TrimDirection? TrimSource? List(Expression))\n" - "TrimDirection <- 'BOTH'i / 'LEADING'i / 'TRAILING'i\n" - "TrimSource <- Expression? 'FROM'i\n" - "ExecuteStatement <- 'EXECUTE'i Identifier TableFunctionArguments?\n" - "CreateSecretStmt <- 'SECRET'i IfNotExists? SecretName? SecretStorageSpecifier? Parens(GenericCopyOptionList)\n" - "SecretStorageSpecifier <- 'IN'i Identifier\n" - "CreateViewStmt <- 'RECURSIVE'i? 'VIEW'i IfNotExists? QualifiedName InsertColumnList? 'AS'i SelectStatement\n" - "DescribeStatement <- ShowSelect / ShowAllTables / ShowQualifiedName\n" + "CoalesceExpression <- 'COALESCE' Parens(List(Expression))\n" + "UnpackExpression <- 'UNPACK' Parens(Expression)\n" + "ColumnsExpression <- '*'? 'COLUMNS' Parens(Expression)\n" + "ExtractExpression <- 'EXTRACT' Parens(Expression 'FROM' Expression)\n" + "LambdaExpression <- 'LAMBDA' List(ColIdOrString) ':' Expression\n" + "NullIfExpression <- 'NULLIF' Parens(Expression ',' Expression)\n" + "PositionExpression <- 'POSITION' Parens(Expression)\n" + "RowExpression <- 'ROW' Parens(List(Expression))\n" + "SubstringExpression <- 'SUBSTRING' Parens(SubstringParameters / List(Expression))\n" + "SubstringParameters <- Expression 'FROM' NumberLiteral 'FOR' NumberLiteral\n" + "TrimExpression <- 'TRIM' Parens(TrimDirection? TrimSource? List(Expression))\n" + "TrimDirection <- 'BOTH' / 'LEADING' / 'TRAILING'\n" + "TrimSource <- Expression? 'FROM'\n" + "ExecuteStatement <- 'EXECUTE' Identifier TableFunctionArguments?\n" + "CreateSecretStmt <- 'SECRET' IfNotExists? SecretName? SecretStorageSpecifier? Parens(GenericCopyOptionList)\n" + "SecretStorageSpecifier <- 'IN' Identifier\n" + "CreateViewStmt <- 'RECURSIVE'? 'VIEW' IfNotExists? QualifiedName InsertColumnList? 'AS' SelectStatement\n" + "DescribeStatement <- ShowTables / ShowSelect / ShowAllTables / ShowQualifiedName\n" "ShowSelect <- ShowOrDescribeOrSummarize SelectStatement\n" - "ShowAllTables <- ShowOrDescribe 'ALL'i 'TABLES'\n" + "ShowAllTables <- ShowOrDescribe 'ALL' 'TABLES'\n" "ShowQualifiedName <- ShowOrDescribeOrSummarize (BaseTableName / StringLiteral)?\n" - "ShowOrDescribeOrSummarize <- ShowOrDescribe / 'SUMMARIZE'i\n" - "ShowOrDescribe <- 'SHOW'i / 'DESCRIBE'i / 'DESC'i\n" - "VacuumStatement <- 'VACUUM'i (VacuumLegacyOptions AnalyzeStatement / VacuumLegacyOptions QualifiedTarget / VacuumLegacyOptions / VacuumParensOptions QualifiedTarget?)?\n" + "ShowTables <- ShowOrDescribe 'TABLES' 'FROM' QualifiedName\n" + "ShowOrDescribeOrSummarize <- ShowOrDescribe / 'SUMMARIZE'\n" + "ShowOrDescribe <- 'SHOW' / 'DESCRIBE' / 'DESC'\n" + "VacuumStatement <- 'VACUUM' (VacuumLegacyOptions AnalyzeStatement / VacuumLegacyOptions QualifiedTarget / VacuumLegacyOptions / VacuumParensOptions QualifiedTarget?)?\n" "VacuumLegacyOptions <- OptFull OptFreeze OptVerbose\n" "VacuumParensOptions <- Parens(List(VacuumOption))\n" - "VacuumOption <- 'ANALYZE'i / 'VERBOSE'i / 'FREEZE'i / 'FULL'i / Identifier\n" - "OptFull <- 'FULL'i?\n" - "OptFreeze <- 'FREEZE'i?\n" - "OptVerbose <- 'VERBOSE'i?\n" + "VacuumOption <- 'ANALYZE' / 'VERBOSE' / 'FREEZE' / 'FULL' / Identifier\n" + "OptFull <- 'FULL'?\n" + "OptFreeze <- 'FREEZE'?\n" + "OptVerbose <- 'VERBOSE'?\n" "QualifiedTarget <- QualifiedName OptNameList\n" "OptNameList <- Parens(List(Name))?\n" - "MergeIntoStatement <- WithClause? 'MERGE'i 'INTO'i TargetOptAlias MergeIntoUsingClause MergeMatch* ReturningClause?\n" - "MergeIntoUsingClause <- 'USING'i TableRef JoinQualifier\n" + "MergeIntoStatement <- WithClause? 'MERGE' 'INTO' TargetOptAlias MergeIntoUsingClause MergeMatch* ReturningClause?\n" + "MergeIntoUsingClause <- 'USING' TableRef JoinQualifier\n" "MergeMatch <- MatchedClause / NotMatchedClause\n" - "MatchedClause <- 'WHEN'i 'MATCHED'i AndExpression? 'THEN'i MatchedClauseAction\n" + "MatchedClause <- 'WHEN' 'MATCHED' AndExpression? 'THEN' MatchedClauseAction\n" "MatchedClauseAction <- UpdateMatchClause / DeleteMatchClause / InsertMatchClause / DoNothingMatchClause / ErrorMatchClause\n" - "UpdateMatchClause <- 'UPDATE'i (UpdateMatchSetClause / ByNameOrPosition?)\n" - "DeleteMatchClause <- 'DELETE'i\n" - "InsertMatchClause <- 'INSERT'i (InsertValuesList / DefaultValues / InsertByNameOrPosition)?\n" + "UpdateMatchClause <- 'UPDATE' (UpdateMatchSetClause / ByNameOrPosition?)\n" + "DeleteMatchClause <- 'DELETE'\n" + "InsertMatchClause <- 'INSERT' (InsertValuesList / DefaultValues / InsertByNameOrPosition)?\n" "InsertByNameOrPosition <- ByNameOrPosition? '*'?\n" - "InsertValuesList <- InsertColumnList? 'VALUES'i Parens(List(Expression))\n" - "DoNothingMatchClause <- 'DO'i 'NOTHING'i\n" - "ErrorMatchClause <- 'ERROR'i Expression?\n" - "UpdateMatchSetClause <- 'SET'i (UpdateSetClause / '*')\n" - "AndExpression <- 'AND'i Expression\n" - "NotMatchedClause <- 'WHEN'i 'NOT'i 'MATCHED'i BySourceOrTarget? AndExpression? 'THEN'i MatchedClauseAction\n" - "BySourceOrTarget <- 'BY'i ('SOURCE'i / 'TARGET'i)\n" - "PragmaStatement <- 'PRAGMA'i (PragmaAssign / PragmaFunction)\n" + "InsertValuesList <- InsertColumnList? 'VALUES' Parens(List(Expression))\n" + "DoNothingMatchClause <- 'DO' 'NOTHING'\n" + "ErrorMatchClause <- 'ERROR' Expression?\n" + "UpdateMatchSetClause <- 'SET' (UpdateSetClause / '*')\n" + "AndExpression <- 'AND' Expression\n" + "NotMatchedClause <- 'WHEN' 'NOT' 'MATCHED' BySourceOrTarget? AndExpression? 'THEN' MatchedClauseAction\n" + "BySourceOrTarget <- 'BY' ('SOURCE' / 'TARGET')\n" + "PragmaStatement <- 'PRAGMA' (PragmaAssign / PragmaFunction)\n" "PragmaAssign <- SettingName '=' VariableList\n" "PragmaFunction <- PragmaName PragmaParameters?\n" "PragmaParameters <- List(Expression)\n" - "DeallocateStatement <- 'DEALLOCATE'i 'PREPARE'i? Identifier\n" - "PrepareStatement <- 'PREPARE'i Identifier TypeList? 'AS'i Statement\n" + "DeallocateStatement <- 'DEALLOCATE' 'PREPARE'? Identifier\n" + "PrepareStatement <- 'PREPARE' Identifier TypeList? 'AS' Statement\n" "TypeList <- Parens(List(Type))\n" - "CreateStatement <- 'CREATE'i OrReplace? Temporary? (CreateTableStmt / CreateMacroStmt / CreateSequenceStmt / CreateTypeStmt / CreateSchemaStmt / CreateViewStmt / CreateIndexStmt / CreateSecretStmt)\n" - "OrReplace <- 'OR'i 'REPLACE'i\n" - "Temporary <- 'TEMP'i / 'TEMPORARY'i / 'PERSISTENT'i\n" - "CreateTableStmt <- 'TABLE'i IfNotExists? QualifiedName (CreateTableAs / CreateColumnList) CommitAction?\n" - "CreateTableAs <- IdentifierList? 'AS'i SelectStatement WithData?\n" - "WithData <- 'WITH'i 'NO'i? 'DATA'i\n" + "CreateStatement <- 'CREATE' OrReplace? Temporary? (CreateTableStmt / CreateMacroStmt / CreateSequenceStmt / CreateTypeStmt / CreateSchemaStmt / CreateViewStmt / CreateIndexStmt / CreateSecretStmt)\n" + "OrReplace <- 'OR' 'REPLACE'\n" + "Temporary <- 'TEMP' / 'TEMPORARY' / 'PERSISTENT'\n" + "CreateTableStmt <- 'TABLE' IfNotExists? QualifiedName (CreateTableAs / CreateColumnList) CommitAction?\n" + "CreateTableAs <- IdentifierList? 'AS' SelectStatement WithData?\n" + "WithData <- 'WITH' 'NO'? 'DATA'\n" "IdentifierList <- Parens(List(Identifier))\n" "CreateColumnList <- Parens(CreateTableColumnList)\n" - "IfNotExists <- 'IF'i 'NOT'i 'EXISTS'i\n" + "IfNotExists <- 'IF' 'NOT' 'EXISTS'\n" "QualifiedName <- CatalogReservedSchemaIdentifier / SchemaReservedIdentifierOrStringLiteral / IdentifierOrStringLiteral\n" "SchemaReservedIdentifierOrStringLiteral <- SchemaQualification ReservedIdentifierOrStringLiteral\n" "CatalogReservedSchemaIdentifier <- CatalogQualification ReservedSchemaQualification ReservedIdentifierOrStringLiteral\n" @@ -733,24 +738,24 @@ const char INLINED_PEG_GRAMMAR[] = { "ColumnDefinition <- DottedIdentifier TypeOrGenerated ColumnConstraint*\n" "TypeOrGenerated <- Type? GeneratedColumn?\n" "ColumnConstraint <- NotNullConstraint / UniqueConstraint / PrimaryKeyConstraint / DefaultValue / CheckConstraint / ForeignKeyConstraint / ColumnCollation / ColumnCompression\n" - "NotNullConstraint <- 'NOT'i? 'NULL'i\n" - "UniqueConstraint <- 'UNIQUE'i\n" - "PrimaryKeyConstraint <- 'PRIMARY'i 'KEY'i\n" - "DefaultValue <- 'DEFAULT'i Expression\n" - "CheckConstraint <- 'CHECK'i Parens(Expression)\n" - "ForeignKeyConstraint <- 'REFERENCES'i BaseTableName Parens(ColumnList)? KeyActions?\n" - "ColumnCollation <- 'COLLATE'i Expression\n" - "ColumnCompression <- 'USING'i 'COMPRESSION'i ColIdOrString\n" + "NotNullConstraint <- 'NOT'? 'NULL'\n" + "UniqueConstraint <- 'UNIQUE'\n" + "PrimaryKeyConstraint <- 'PRIMARY' 'KEY'\n" + "DefaultValue <- 'DEFAULT' Expression\n" + "CheckConstraint <- 'CHECK' Parens(Expression)\n" + "ForeignKeyConstraint <- 'REFERENCES' BaseTableName Parens(ColumnList)? KeyActions?\n" + "ColumnCollation <- 'COLLATE' Expression\n" + "ColumnCompression <- 'USING' 'COMPRESSION' ColIdOrString\n" "KeyActions <- UpdateAction? DeleteAction?\n" "UpdateAction <- 'ON' 'UPDATE' KeyAction\n" "DeleteAction <- 'ON' 'DELETE' KeyAction\n" - "KeyAction <- ('NO'i 'ACTION'i) / 'RESTRICT'i / 'CASCADE'i / ('SET'i 'NULL'i) / ('SET'i 'DEFAULT'i)\n" + "KeyAction <- ('NO' 'ACTION') / 'RESTRICT' / 'CASCADE' / ('SET' 'NULL') / ('SET' 'DEFAULT')\n" "TopLevelConstraint <- ConstraintNameClause? TopLevelConstraintList\n" "TopLevelConstraintList <- TopPrimaryKeyConstraint / CheckConstraint / TopUniqueConstraint / TopForeignKeyConstraint\n" - "ConstraintNameClause <- 'CONSTRAINT'i Identifier\n" - "TopPrimaryKeyConstraint <- 'PRIMARY'i 'KEY'i ColumnIdList\n" - "TopUniqueConstraint <- 'UNIQUE'i ColumnIdList\n" - "TopForeignKeyConstraint <- 'FOREIGN'i 'KEY'i ColumnIdList ForeignKeyConstraint\n" + "ConstraintNameClause <- 'CONSTRAINT' Identifier\n" + "TopPrimaryKeyConstraint <- 'PRIMARY' 'KEY' ColumnIdList\n" + "TopUniqueConstraint <- 'UNIQUE' ColumnIdList\n" + "TopForeignKeyConstraint <- 'FOREIGN' 'KEY' ColumnIdList ForeignKeyConstraint\n" "ColumnIdList <- Parens(List(ColId))\n" "PlainIdentifier <- !ReservedKeyword <[a-z_]i[a-z0-9_]i*>\n" "QuotedIdentifier <- '\"' [^\"]* '\"'\n" @@ -763,25 +768,26 @@ const char INLINED_PEG_GRAMMAR[] = { "TypeName <- UnreservedKeyword / TypeNameKeyword / Identifier\n" "ColLabel <- ReservedKeyword / UnreservedKeyword / ColumnNameKeyword / FuncNameKeyword / TypeNameKeyword / Identifier\n" "ColLabelOrString <- ColLabel / StringLiteral\n" - "GeneratedColumn <- Generated? 'AS'i Parens(Expression) GeneratedColumnType?\n" - "Generated <- 'GENERATED'i AlwaysOrByDefault?\n" - "AlwaysOrByDefault <- 'ALWAYS'i / ('BY'i 'DEFAULT'i)\n" - "GeneratedColumnType <- 'VIRTUAL'i / 'STORED'i\n" - "CommitAction <- 'ON'i 'COMMIT'i 'PRESERVE'i 'ROWS'i\n" - "CreateIndexStmt <- Unique? 'INDEX'i IfNotExists? IndexName? 'ON'i BaseTableName IndexType? Parens(List(IndexElement)) WithList?\n" - "WithList <- 'WITH'i Parens(List(RelOption)) / Oids\n" - "Oids <- ('WITH'i / 'WITHOUT'i) 'OIDS'i\n" + "GeneratedColumn <- Generated? 'AS' Parens(Expression) GeneratedColumnType?\n" + "Generated <- 'GENERATED' AlwaysOrByDefault?\n" + "AlwaysOrByDefault <- 'ALWAYS' / ('BY' 'DEFAULT')\n" + "GeneratedColumnType <- 'VIRTUAL' / 'STORED'\n" + "CommitAction <- 'ON' 'COMMIT' PreserveOrDelete\n" + "PreserveOrDelete <- ('PRESERVE' / 'DELETE') 'ROWS'\n" + "CreateIndexStmt <- Unique? 'INDEX' IfNotExists? IndexName? 'ON' BaseTableName IndexType? Parens(List(IndexElement)) WithList? WhereClause?\n" + "WithList <- 'WITH' Parens(List(RelOption)) / Oids\n" + "Oids <- ('WITH' / 'WITHOUT') 'OIDS'\n" "IndexElement <- Expression DescOrAsc? NullsFirstOrLast?\n" - "Unique <- 'UNIQUE'i\n" - "IndexType <- 'USING'i Identifier\n" + "Unique <- 'UNIQUE'\n" + "IndexType <- 'USING' Identifier\n" "RelOption <- ColLabel ('.' ColLabel)* ('=' DefArg)?\n" - "DefArg <- FuncType / ReservedKeyword / StringLiteral / NumberLiteral / 'NONE'i\n" - "FuncType <- Type / ('SETOF'i? TypeFuncName '%' 'TYPE'i)\n" - "LoadStatement <- 'LOAD'i ColIdOrString\n" - "InstallStatement <- 'FORCE'i? 'INSTALL'i Identifier FromSource? VersionNumber?\n" - "FromSource <- 'FROM'i (Identifier / StringLiteral)\n" + "DefArg <- FuncType / ReservedKeyword / StringLiteral / NumberLiteral / 'NONE'\n" + "FuncType <- Type / ('SETOF'? TypeFuncName '%' 'TYPE')\n" + "LoadStatement <- 'LOAD' ColIdOrString\n" + "InstallStatement <- 'FORCE'? 'INSTALL' Identifier FromSource? VersionNumber?\n" + "FromSource <- 'FROM' (Identifier / StringLiteral)\n" "VersionNumber <- Identifier\n" - "DropStatement <- 'DROP'i DropEntries DropBehavior?\n" + "DropStatement <- 'DROP' DropEntries DropBehavior?\n" "DropEntries <-\n" " DropTable /\n" " DropTableFunction /\n" @@ -793,47 +799,47 @@ const char INLINED_PEG_GRAMMAR[] = { " DropType /\n" " DropSecret\n" "DropTable <- TableOrView IfExists? List(BaseTableName)\n" - "DropTableFunction <- 'MACRO'i 'TABLE'i IfExists? List(TableFunctionName)\n" + "DropTableFunction <- 'MACRO' 'TABLE' IfExists? List(TableFunctionName)\n" "DropFunction <- FunctionType IfExists? List(FunctionIdentifier)\n" - "DropSchema <- 'SCHEMA'i IfExists? List(QualifiedSchemaName)\n" - "DropIndex <- 'INDEX'i IfExists? List(QualifiedIndexName)\n" + "DropSchema <- 'SCHEMA' IfExists? List(QualifiedSchemaName)\n" + "DropIndex <- 'INDEX' IfExists? List(QualifiedIndexName)\n" "QualifiedIndexName <- CatalogQualification? SchemaQualification? IndexName\n" - "DropSequence <- 'SEQUENCE'i IfExists? List(QualifiedSequenceName)\n" - "DropCollation <- 'COLLATION'i IfExists? List(CollationName)\n" - "DropType <- 'TYPE'i IfExists? List(QualifiedTypeName)\n" - "DropSecret <- Temporary? 'SECRET'i IfExists? SecretName DropSecretStorage?\n" - "TableOrView <- 'TABLE'i / 'VIEW'i / ('MATERIALIZED'i 'VIEW'i)\n" - "FunctionType <- 'MACRO'i / 'FUNCTION'i\n" - "DropBehavior <- 'CASCADE'i / 'RESTRICT'i\n" - "IfExists <- 'IF'i 'EXISTS'i\n" + "DropSequence <- 'SEQUENCE' IfExists? List(QualifiedSequenceName)\n" + "DropCollation <- 'COLLATION' IfExists? List(CollationName)\n" + "DropType <- 'TYPE' IfExists? List(QualifiedTypeName)\n" + "DropSecret <- Temporary? 'SECRET' IfExists? SecretName DropSecretStorage?\n" + "TableOrView <- 'TABLE' / 'VIEW' / ('MATERIALIZED' 'VIEW')\n" + "FunctionType <- 'MACRO' / 'FUNCTION'\n" + "DropBehavior <- 'CASCADE' / 'RESTRICT'\n" + "IfExists <- 'IF' 'EXISTS'\n" "QualifiedSchemaName <- CatalogQualification? SchemaName\n" - "DropSecretStorage <- 'FROM'i Identifier\n" - "UpdateStatement <- WithClause? 'UPDATE'i UpdateTarget UpdateSetClause FromClause? WhereClause? ReturningClause?\n" - "UpdateTarget <- (BaseTableName 'SET'i) / (BaseTableName UpdateAlias? 'SET'i)\n" - "UpdateAlias <- 'AS'i? ColId\n" + "DropSecretStorage <- 'FROM' Identifier\n" + "UpdateStatement <- WithClause? 'UPDATE' UpdateTarget UpdateSetClause FromClause? WhereClause? ReturningClause?\n" + "UpdateTarget <- (BaseTableName 'SET') / (BaseTableName UpdateAlias? 'SET')\n" + "UpdateAlias <- 'AS'? ColId\n" "UpdateSetClause <- List(UpdateSetElement) / (Parens(List(ColumnName)) '=' Expression)\n" "UpdateSetElement <- ColumnName '=' Expression\n" - "InsertStatement <- WithClause? 'INSERT'i OrAction? 'INTO'i InsertTarget ByNameOrPosition? InsertColumnList? InsertValues OnConflictClause? ReturningClause?\n" - "OrAction <- 'OR'i 'REPLACE'i / 'IGNORE'i\n" - "ByNameOrPosition <- 'BY'i 'NAME'i / 'POSITION'i\n" + "InsertStatement <- WithClause? 'INSERT' OrAction? 'INTO' InsertTarget ByNameOrPosition? InsertColumnList? InsertValues OnConflictClause? ReturningClause?\n" + "OrAction <- 'OR' 'REPLACE' / 'IGNORE'\n" + "ByNameOrPosition <- 'BY' 'NAME' / 'POSITION'\n" "InsertTarget <- BaseTableName InsertAlias?\n" - "InsertAlias <- 'AS'i Identifier\n" + "InsertAlias <- 'AS' Identifier\n" "ColumnList <- List(ColId)\n" "InsertColumnList <- Parens(ColumnList)\n" "InsertValues <- SelectStatement / DefaultValues\n" - "DefaultValues <- 'DEFAULT'i 'VALUES'i\n" - "OnConflictClause <- 'ON'i 'CONFLICT'i OnConflictTarget? OnConflictAction\n" + "DefaultValues <- 'DEFAULT' 'VALUES'\n" + "OnConflictClause <- 'ON' 'CONFLICT' OnConflictTarget? OnConflictAction\n" "OnConflictTarget <- OnConflictExpressionTarget / OnConflictIndexTarget\n" "OnConflictExpressionTarget <- Parens(List(ColId)) WhereClause?\n" - "OnConflictIndexTarget <- 'ON'i 'CONSTRAINT'i ConstraintName\n" + "OnConflictIndexTarget <- 'ON' 'CONSTRAINT' ConstraintName\n" "OnConflictAction <- OnConflictUpdate / OnConflictNothing\n" - "OnConflictUpdate <- 'DO'i 'UPDATE'i 'SET'i UpdateSetClause WhereClause?\n" - "OnConflictNothing <- 'DO'i 'NOTHING'i\n" - "ReturningClause <- 'RETURNING'i TargetList\n" - "CreateSchemaStmt <- 'SCHEMA'i IfNotExists? QualifiedName\n" + "OnConflictUpdate <- 'DO' 'UPDATE' 'SET' UpdateSetClause WhereClause?\n" + "OnConflictNothing <- 'DO' 'NOTHING'\n" + "ReturningClause <- 'RETURNING' TargetList\n" + "CreateSchemaStmt <- 'SCHEMA' IfNotExists? QualifiedName\n" "SelectStatement <- SelectOrParens (SetopClause SelectStatement)* ResultModifiers\n" - "SetopClause <- ('UNION'i / 'EXCEPT'i / 'INTERSECT'i) DistinctOrAll? ByName?\n" - "ByName <- 'BY'i 'NAME'i\n" + "SetopClause <- ('UNION' / 'EXCEPT' / 'INTERSECT') DistinctOrAll? ByName?\n" + "ByName <- 'BY' 'NAME'\n" "SelectOrParens <- BaseSelect / Parens(SelectStatement)\n" "BaseSelect <- WithClause? (OptionalParensSimpleSelect / ValuesClause / DescribeStatement / TableStatement / PivotStatement / UnpivotStatement) ResultModifiers\n" "ResultModifiers <- OrderByClause? LimitClause? OffsetClause?\n" @@ -841,16 +847,16 @@ const char INLINED_PEG_GRAMMAR[] = { "OptionalParensSimpleSelect <- Parens(SimpleSelect) / SimpleSelect\n" "SimpleSelect <- SelectFrom WhereClause? GroupByClause? HavingClause? WindowClause? QualifyClause? SampleClause?\n" "SelectFrom <- (SelectClause FromClause?) / (FromClause SelectClause?)\n" - "WithStatement <- ColIdOrString InsertColumnList? UsingKey? 'AS'i Materialized? SubqueryReference\n" - "UsingKey <- 'USING'i 'KEY'i Parens(List(ColId))\n" - "Materialized <- 'NOT'i? 'MATERIALIZED'i\n" - "WithClause <- 'WITH'i Recursive? List(WithStatement)\n" - "Recursive <- 'RECURSIVE'i\n" - "SelectClause <- 'SELECT'i DistinctClause? TargetList\n" + "WithStatement <- ColIdOrString InsertColumnList? UsingKey? 'AS' Materialized? SubqueryReference\n" + "UsingKey <- 'USING' 'KEY' Parens(List(ColId))\n" + "Materialized <- 'NOT'? 'MATERIALIZED'\n" + "WithClause <- 'WITH' Recursive? List(WithStatement)\n" + "Recursive <- 'RECURSIVE'\n" + "SelectClause <- 'SELECT' DistinctClause? TargetList\n" "TargetList <- List(AliasedExpression)\n" "ColumnAliases <- Parens(List(ColIdOrString))\n" - "DistinctClause <- ('DISTINCT'i DistinctOn?) / 'ALL'i\n" - "DistinctOn <- 'ON'i Parens(List(Expression))\n" + "DistinctClause <- ('DISTINCT' DistinctOn?) / 'ALL'\n" + "DistinctOn <- 'ON' Parens(List(Expression))\n" "InnerTableRef <- ValuesRef / TableFunction / TableSubquery / BaseTableRef / ParensTableRef\n" "TableRef <- InnerTableRef JoinOrPivot* TableAlias?\n" "TableSubquery <- Lateral? SubqueryReference TableAlias?\n" @@ -865,151 +871,161 @@ const char INLINED_PEG_GRAMMAR[] = { "PivotValueLists <- PivotValueList PivotValueList*\n" "PivotValueList <- PivotHeader 'IN' PivotTargetList\n" "PivotTargetList <- Identifier / Parens(TargetList)\n" - "Lateral <- 'LATERAL'i\n" + "Lateral <- 'LATERAL'\n" "BaseTableName <- CatalogReservedSchemaTable / SchemaReservedTable / TableName\n" "SchemaReservedTable <- SchemaQualification ReservedTableName\n" "CatalogReservedSchemaTable <- CatalogQualification ReservedSchemaQualification ReservedTableName\n" "TableFunction <- TableFunctionLateralOpt / TableFunctionAliasColon\n" - "TableFunctionLateralOpt <- Lateral? QualifiedTableFunction TableFunctionArguments TableAlias?\n" - "TableFunctionAliasColon <- TableAliasColon QualifiedTableFunction TableFunctionArguments\n" + "TableFunctionLateralOpt <- Lateral? QualifiedTableFunction TableFunctionArguments WithOrdinality? TableAlias?\n" + "TableFunctionAliasColon <- TableAliasColon QualifiedTableFunction TableFunctionArguments WithOrdinality?\n" + "WithOrdinality <- 'WITH' 'ORDINALITY'\n" "QualifiedTableFunction <- CatalogQualification? SchemaQualification? TableFunctionName\n" "TableFunctionArguments <- Parens(List(FunctionArgument)?)\n" "FunctionArgument <- NamedParameter / Expression\n" - "NamedParameter <- TypeName NamedParameterAssignment Expression\n" + "NamedParameter <- TypeName Type? NamedParameterAssignment Expression\n" "NamedParameterAssignment <- ':=' / '=>'\n" - "TableAlias <- 'AS'i? (Identifier / StringLiteral) ColumnAliases?\n" - "AtClause <- 'AT'i Parens(AtSpecifier)\n" + "TableAlias <- 'AS'? (Identifier / StringLiteral) ColumnAliases?\n" + "AtClause <- 'AT' Parens(AtSpecifier)\n" "AtSpecifier <- AtUnit '=>' Expression\n" - "AtUnit <- 'VERSION'i / 'TIMESTAMP'i\n" + "AtUnit <- 'VERSION' / 'TIMESTAMP'\n" "JoinClause <- RegularJoinClause / JoinWithoutOnClause\n" - "RegularJoinClause <- 'ASOF'i? JoinType? 'JOIN'i TableRef JoinQualifier\n" - "JoinWithoutOnClause <- JoinPrefix 'JOIN'i TableRef\n" + "RegularJoinClause <- 'ASOF'? JoinType? 'JOIN' TableRef JoinQualifier\n" + "JoinWithoutOnClause <- JoinPrefix 'JOIN' TableRef\n" "JoinQualifier <- OnClause / UsingClause\n" - "OnClause <- 'ON'i Expression\n" - "UsingClause <- 'USING'i Parens(List(ColumnName))\n" - "OuterJoinType <- 'FULL'i / 'LEFT'i / 'RIGHT'i\n" - "JoinType <- (OuterJoinType 'OUTER'i?) / 'SEMI'i / 'ANTI'i / 'INNER'i\n" - "JoinPrefix <- 'CROSS'i / ('NATURAL'i JoinType?) / 'POSITIONAL'i\n" - "FromClause <- 'FROM'i List(TableRef)\n" - "WhereClause <- 'WHERE'i Expression\n" - "GroupByClause <- 'GROUP'i 'BY'i GroupByExpressions\n" - "HavingClause <- 'HAVING'i Expression\n" - "QualifyClause <- 'QUALIFY'i Expression\n" + "OnClause <- 'ON' Expression\n" + "UsingClause <- 'USING' Parens(List(ColumnName))\n" + "OuterJoinType <- 'FULL' / 'LEFT' / 'RIGHT'\n" + "JoinType <- (OuterJoinType 'OUTER'?) / 'SEMI' / 'ANTI' / 'INNER'\n" + "JoinPrefix <- 'CROSS' / ('NATURAL' JoinType?) / 'POSITIONAL'\n" + "FromClause <- 'FROM' List(TableRef)\n" + "WhereClause <- 'WHERE' Expression\n" + "GroupByClause <- 'GROUP' 'BY' GroupByExpressions\n" + "HavingClause <- 'HAVING' Expression\n" + "QualifyClause <- 'QUALIFY' Expression\n" "SampleClause <- (TableSample / UsingSample) SampleEntry\n" - "UsingSample <- 'USING'i 'SAMPLE'i\n" - "TableSample <- 'TABLESAMPLE'i\n" - "WindowClause <- 'WINDOW'i List(WindowDefinition)\n" - "WindowDefinition <- Identifier 'AS'i WindowFrameDefinition\n" + "UsingSample <- 'USING' 'SAMPLE'\n" + "TableSample <- 'TABLESAMPLE'\n" + "WindowClause <- 'WINDOW' List(WindowDefinition)\n" + "WindowDefinition <- Identifier 'AS' WindowFrameDefinition\n" "SampleEntry <- SampleEntryFunction / SampleEntryCount\n" "SampleEntryCount <- SampleCount Parens(SampleProperties)?\n" "SampleEntryFunction <- SampleFunction? Parens(SampleCount) RepeatableSample?\n" "SampleFunction <- ColId\n" "SampleProperties <- ColId (',' NumberLiteral)?\n" - "RepeatableSample <- 'REPEATABLE'i Parens(NumberLiteral)\n" + "RepeatableSample <- 'REPEATABLE' Parens(NumberLiteral)\n" "SampleCount <- Expression SampleUnit?\n" - "SampleUnit <- '%' / 'PERCENT'i / 'ROWS'i\n" - "GroupByExpressions <- GroupByList / 'ALL'i\n" + "SampleUnit <- '%' / 'PERCENT' / 'ROWS'\n" + "GroupByExpressions <- GroupByList / 'ALL'\n" "GroupByList <- List(GroupByExpression)\n" "GroupByExpression <- EmptyGroupingItem / CubeOrRollupClause / GroupingSetsClause / Expression\n" "EmptyGroupingItem <- '(' ')'\n" "CubeOrRollupClause <- CubeOrRollup Parens(List(Expression))\n" - "CubeOrRollup <- 'CUBE'i / 'ROLLUP'i\n" - "GroupingSetsClause <- 'GROUPING'i 'SETS'i Parens(GroupByList)\n" + "CubeOrRollup <- 'CUBE' / 'ROLLUP'\n" + "GroupingSetsClause <- 'GROUPING' 'SETS' Parens(GroupByList)\n" "SubqueryReference <- Parens(SelectStatement)\n" "OrderByExpression <- Expression DescOrAsc? NullsFirstOrLast?\n" - "DescOrAsc <- 'DESC'i / 'DESCENDING'i / 'ASC'i / 'ASCENDING'i\n" - "NullsFirstOrLast <- 'NULLS'i 'FIRST'i / 'LAST'i\n" - "OrderByClause <- 'ORDER'i 'BY'i OrderByExpressions\n" + "DescOrAsc <- 'DESC' / 'DESCENDING' / 'ASC' / 'ASCENDING'\n" + "NullsFirstOrLast <- 'NULLS' 'FIRST' / 'LAST'\n" + "OrderByClause <- 'ORDER' 'BY' OrderByExpressions\n" "OrderByExpressions <- List(OrderByExpression) / OrderByAll\n" - "OrderByAll <- 'ALL'i DescOrAsc? NullsFirstOrLast?\n" - "LimitClause <- 'LIMIT'i LimitValue\n" - "OffsetClause <- 'OFFSET'i LimitValue\n" - "LimitValue <- 'ALL'i / (NumberLiteral 'PERCENT'i) / (Expression '%'?)\n" - "AliasedExpression <- (ColId ':' Expression) / (Expression 'AS'i ColLabelOrString) / (Expression Identifier?)\n" - "ValuesClause <- 'VALUES'i List(ValuesExpressions)\n" + "OrderByAll <- 'ALL' DescOrAsc? NullsFirstOrLast?\n" + "LimitClause <- 'LIMIT' LimitValue\n" + "OffsetClause <- 'OFFSET' OffsetValue\n" + "LimitValue <- 'ALL' / (NumberLiteral 'PERCENT') / (Expression '%'?)\n" + "OffsetValue <- Expression RowOrRows?\n" + "RowOrRows <- 'ROW' / 'ROWS'\n" + "AliasedExpression <- (ColId ':' Expression) / (Expression 'AS' ColLabelOrString) / (Expression Identifier?)\n" + "ValuesClause <- 'VALUES' List(ValuesExpressions)\n" "ValuesExpressions <- Parens(List(Expression))\n" "TransactionStatement <- BeginTransaction / RollbackTransaction / CommitTransaction\n" "BeginTransaction <- StartOrBegin Transaction? ReadOrWrite?\n" "RollbackTransaction <- AbortOrRollback Transaction?\n" "CommitTransaction <- CommitOrEnd Transaction?\n" - "StartOrBegin <- 'START'i / 'BEGIN'i\n" - "Transaction <- 'WORK'i / 'TRANSACTION'i\n" - "ReadOrWrite <- 'READ'i ('ONLY'i / 'WRITE'i)\n" - "AbortOrRollback <- 'ABORT'i / 'ROLLBACK'i\n" - "CommitOrEnd <- 'COMMIT'i / 'END'i\n" - "DeleteStatement <- WithClause? 'DELETE'i 'FROM'i TargetOptAlias DeleteUsingClause? WhereClause? ReturningClause?\n" - "TruncateStatement <- 'TRUNCATE'i 'TABLE'i? BaseTableName\n" - "TargetOptAlias <- BaseTableName 'AS'i? ColId?\n" - "DeleteUsingClause <- 'USING'i List(TableRef)\n" - "CreateTypeStmt <- 'TYPE'i IfNotExists? QualifiedName 'AS'i CreateType\n" - "CreateType <- ('ENUM'i Parens(SelectStatement)) /\n" - " ('ENUM'i Parens(List(StringLiteral))) /\n" + "StartOrBegin <- 'START' / 'BEGIN'\n" + "Transaction <- 'WORK' / 'TRANSACTION'\n" + "ReadOrWrite <- 'READ' ('ONLY' / 'WRITE')\n" + "AbortOrRollback <- 'ABORT' / 'ROLLBACK'\n" + "CommitOrEnd <- 'COMMIT' / 'END'\n" + "DeleteStatement <- WithClause? 'DELETE' 'FROM' TargetOptAlias DeleteUsingClause? WhereClause? ReturningClause?\n" + "TruncateStatement <- 'TRUNCATE' 'TABLE'? BaseTableName\n" + "TargetOptAlias <- BaseTableName 'AS'? ColId?\n" + "DeleteUsingClause <- 'USING' List(TableRef)\n" + "CreateTypeStmt <- 'TYPE' IfNotExists? QualifiedName 'AS' CreateType\n" + "CreateType <- ('ENUM' Parens(SelectStatement)) /\n" + " ('ENUM' Parens(List(StringLiteral))) /\n" " Type\n" - "SetStatement <- 'SET'i (StandardAssignment / SetTimeZone)\n" + "SetStatement <- 'SET' (StandardAssignment / SetTimeZone)\n" "StandardAssignment <- (SetVariable / SetSetting) SetAssignment\n" - "SetTimeZone <- 'TIME'i 'ZONE'i Expression\n" + "SetTimeZone <- 'TIME' 'ZONE' Expression\n" "SetSetting <- SettingScope? SettingName\n" - "SetVariable <- 'VARIABLE'i Identifier\n" - "SettingScope <- 'LOCAL'i / 'SESSION'i / 'GLOBAL'i\n" + "SetVariable <- VariableScope Identifier\n" + "VariableScope <- 'VARIABLE'\n" + "SettingScope <- LocalScope / SessionScope / GlobalScope\n" + "LocalScope <- 'LOCAL'\n" + "SessionScope <- 'SESSION'\n" + "GlobalScope <- 'GLOBAL'\n" "SetAssignment <- VariableAssign VariableList\n" "VariableAssign <- '=' / 'TO'\n" "VariableList <- List(Expression)\n" - "ResetStatement <- 'RESET'i (SetVariable / SetSetting)\n" - "ExportStatement <- 'EXPORT'i 'DATABASE'i ExportSource? StringLiteral Parens(GenericCopyOptionList)?\n" - "ExportSource <- CatalogName 'TO'i\n" - "ImportStatement <- 'IMPORT'i 'DATABASE'i StringLiteral\n" - "CheckpointStatement <- 'FORCE'i? 'CHECKPOINT'i CatalogName?\n" - "CopyStatement <- 'COPY'i (CopyTable / CopySelect / CopyFromDatabase)\n" + "ResetStatement <- 'RESET' (SetVariable / SetSetting)\n" + "ExportStatement <- 'EXPORT' 'DATABASE' ExportSource? StringLiteral Parens(GenericCopyOptionList)?\n" + "ExportSource <- CatalogName 'TO'\n" + "ImportStatement <- 'IMPORT' 'DATABASE' StringLiteral\n" + "CheckpointStatement <- 'FORCE'? 'CHECKPOINT' CatalogName?\n" + "CopyStatement <- 'COPY' (CopyTable / CopySelect / CopyFromDatabase)\n" "CopyTable <- BaseTableName InsertColumnList? FromOrTo CopyFileName CopyOptions?\n" - "FromOrTo <- 'FROM'i / 'TO'i\n" - "CopySelect <- Parens(SelectStatement) 'TO'i CopyFileName CopyOptions?\n" - "CopyFileName <- StringLiteral / Identifier / (Identifier '.' ColId)\n" - "CopyOptions <- 'WITH'i? (Parens(GenericCopyOptionList) / (SpecializedOptions*))\n" + "FromOrTo <- 'FROM' / 'TO'\n" + "CopySelect <- Parens(SelectStatement) 'TO' CopyFileName CopyOptions?\n" + "CopyFileName <- Expression / StringLiteral / Identifier / (Identifier '.' ColId)\n" + "CopyOptions <- 'WITH'? (Parens(GenericCopyOptionList) / (SpecializedOptions*))\n" "SpecializedOptions <-\n" - " 'BINARY'i / 'FREEZE'i / 'OIDS'i / 'CSV'i / 'HEADER'i /\n" + " 'BINARY' / 'FREEZE' / 'OIDS' / 'CSV' / 'HEADER' /\n" " SpecializedStringOption /\n" - " ('ENCODING'i StringLiteral) /\n" - " ('FORCE'i 'QUOTE'i StarOrColumnList) /\n" - " ('PARTITION'i 'BY'i StarOrColumnList) /\n" - " ('FORCE'i 'NOT'i? 'NULL'i ColumnList)\n" - "SpecializedStringOption <- ('DELIMITER'i / 'NULL'i / 'QUOTE'i / 'ESCAPE'i) 'AS'i? StringLiteral\n" + " ('ENCODING' StringLiteral) /\n" + " ('FORCE' 'QUOTE' StarOrColumnList) /\n" + " ('PARTITION' 'BY' StarOrColumnList) /\n" + " ('FORCE' 'NOT'? 'NULL' ColumnList)\n" + "SpecializedStringOption <- ('DELIMITER' / 'NULL' / 'QUOTE' / 'ESCAPE') 'AS'? StringLiteral\n" "StarOrColumnList <- '*' / ColumnList\n" "GenericCopyOptionList <- List(GenericCopyOption)\n" "GenericCopyOption <- GenericCopyOptionName Expression?\n" "# FIXME: should not need to hard-code options here\n" - "GenericCopyOptionName <- 'ARRAY'i / 'NULL'i / 'ANALYZE'i / CopyOptionName\n" - "CopyFromDatabase <- 'FROM'i 'DATABASE'i ColId 'TO'i ColId CopyDatabaseFlag?\n" + "GenericCopyOptionName <- 'ARRAY' / 'NULL' / 'ANALYZE' / CopyOptionName\n" + "CopyFromDatabase <- 'FROM' 'DATABASE' ColId 'TO' ColId CopyDatabaseFlag?\n" "CopyDatabaseFlag <- Parens(SchemaOrData)\n" - "SchemaOrData <- 'SCHEMA'i / 'DATA'i\n" - "AlterStatement <- 'ALTER'i AlterOptions\n" - "AlterOptions <- AlterTableStmt / AlterViewStmt / AlterSequenceStmt\n" - "AlterTableStmt <- 'TABLE'i IfExists? BaseTableName AlterTableOptions\n" + "SchemaOrData <- 'SCHEMA' / 'DATA'\n" + "AlterStatement <- 'ALTER' AlterOptions\n" + "AlterOptions <- AlterTableStmt / AlterViewStmt / AlterSequenceStmt / AlterDatabaseStmt / AlterSchemaStmt\n" + "AlterTableStmt <- 'TABLE' IfExists? BaseTableName AlterTableOptions\n" + "AlterSchemaStmt <- 'SCHEMA' IfExists? QualifiedName RenameAlter\n" "AlterTableOptions <- AddColumn / DropColumn / AlterColumn / AddConstraint / ChangeNullability / RenameColumn / RenameAlter / SetPartitionedBy / ResetPartitionedBy / SetSortedBy / ResetSortedBy\n" - "AddConstraint <- 'ADD'i TopLevelConstraint\n" - "AddColumn <- 'ADD'i 'COLUMN'i? IfNotExists? ColumnDefinition\n" - "DropColumn <- 'DROP'i 'COLUMN'i? IfExists? NestedColumnName DropBehavior?\n" - "AlterColumn <- 'ALTER'i 'COLUMN'i? NestedColumnName AlterColumnEntry\n" - "RenameColumn <- 'RENAME'i 'COLUMN'i? NestedColumnName 'TO'i Identifier\n" + "AddConstraint <- 'ADD' TopLevelConstraint\n" + "AddColumn <- 'ADD' 'COLUMN'? IfNotExists? ColumnDefinition\n" + "DropColumn <- 'DROP' 'COLUMN'? IfExists? NestedColumnName DropBehavior?\n" + "AlterColumn <- 'ALTER' 'COLUMN'? NestedColumnName AlterColumnEntry\n" + "RenameColumn <- 'RENAME' 'COLUMN'? NestedColumnName 'TO' Identifier\n" "NestedColumnName <- (Identifier '.')* ColumnName\n" - "RenameAlter <- 'RENAME'i 'TO'i Identifier\n" - "SetPartitionedBy <- 'SET'i 'PARTITIONED'i 'BY'i Parens(List(Expression))\n" - "ResetPartitionedBy <- 'RESET'i 'PARTITIONED'i 'BY'i\n" - "SetSortedBy <- 'SET'i 'SORTED'i 'BY'i OrderByExpressions\n" - "ResetSortedBy <- 'RESET'i 'SORTED'i 'BY'i\n" + "RenameAlter <- 'RENAME' 'TO' Identifier\n" + "SetPartitionedBy <- 'SET' 'PARTITIONED' 'BY' Parens(List(Expression))\n" + "ResetPartitionedBy <- 'RESET' 'PARTITIONED' 'BY'\n" + "SetSortedBy <- 'SET' 'SORTED' 'BY' Parens(OrderByExpressions)\n" + "ResetSortedBy <- 'RESET' 'SORTED' 'BY'\n" "AlterColumnEntry <- AddOrDropDefault / ChangeNullability / AlterType\n" "AddOrDropDefault <- AddDefault / DropDefault\n" - "AddDefault <- 'SET'i 'DEFAULT'i Expression\n" - "DropDefault <- 'DROP'i 'DEFAULT'i\n" - "ChangeNullability <- ('DROP'i / 'SET'i) 'NOT'i 'NULL'i\n" - "AlterType <- SetData? 'TYPE'i Type? UsingExpression?\n" - "SetData <- 'SET'i 'DATA'i?\n" - "UsingExpression <- 'USING'i Expression\n" - "AlterViewStmt <- 'VIEW'i IfExists? BaseTableName RenameAlter\n" - "AlterSequenceStmt <- 'SEQUENCE'i IfExists? QualifiedSequenceName AlterSequenceOptions\n" + "AddDefault <- 'SET' 'DEFAULT' Expression\n" + "DropDefault <- 'DROP' 'DEFAULT'\n" + "ChangeNullability <- ('DROP' / 'SET') 'NOT' 'NULL'\n" + "AlterType <- SetData? 'TYPE' Type? UsingExpression?\n" + "SetData <- 'SET' 'DATA'?\n" + "UsingExpression <- 'USING' Expression\n" + "AlterViewStmt <- 'VIEW' IfExists? BaseTableName RenameAlter\n" + "AlterSequenceStmt <- 'SEQUENCE' IfExists? QualifiedSequenceName AlterSequenceOptions\n" "QualifiedSequenceName <- CatalogQualification? SchemaQualification? SequenceName\n" "AlterSequenceOptions <- RenameAlter / SetSequenceOption\n" "SetSequenceOption <- List(SequenceOption)\n" - "CreateSequenceStmt <- 'SEQUENCE'i IfNotExists? QualifiedName SequenceOption*\n" + "AlterDatabaseStmt <- 'DATABASE' IfExists? Identifier RenameDatabaseAlter\n" + "RenameDatabaseAlter <- 'RENAME' 'TO' Identifier\n" + "CreateSequenceStmt <- 'SEQUENCE' IfNotExists? QualifiedName SequenceOption*\n" "SequenceOption <-\n" " SeqSetCycle /\n" " SeqSetIncrement /\n" @@ -1017,13 +1033,13 @@ const char INLINED_PEG_GRAMMAR[] = { " SeqNoMinMax /\n" " SeqStartWith /\n" " SeqOwnedBy\n" - "SeqSetCycle <- 'NO'i? 'CYCLE'i\n" - "SeqSetIncrement <- 'INCREMENT'i 'BY'i? Expression\n" + "SeqSetCycle <- 'NO'? 'CYCLE'\n" + "SeqSetIncrement <- 'INCREMENT' 'BY'? Expression\n" "SeqSetMinMax <- SeqMinOrMax Expression\n" - "SeqNoMinMax <- 'NO'i SeqMinOrMax\n" - "SeqStartWith <- 'START'i 'WITH'i? Expression\n" - "SeqOwnedBy <- 'OWNED'i 'BY'i QualifiedName\n" - "SeqMinOrMax <- 'MINVALUE'i / 'MAXVALUE'i\n" + "SeqNoMinMax <- 'NO' SeqMinOrMax\n" + "SeqStartWith <- 'START' 'WITH'? Expression\n" + "SeqOwnedBy <- 'OWNED' 'BY' QualifiedName\n" + "SeqMinOrMax <- 'MINVALUE' / 'MAXVALUE'\n" "Statement <-\n" " CreateStatement /\n" " SelectStatement /\n" @@ -1076,28 +1092,28 @@ const char INLINED_PEG_GRAMMAR[] = { "SecretName <- ColId\n" "NumberLiteral <- < [+-]?[0-9]*([.][0-9]*)? >\n" "StringLiteral <- '\\'' [^\\']* '\\''\n" - "Type <- (TimeType / IntervalType / BitType / RowType / MapType / UnionType / NumericType / SimpleType) ArrayBounds*\n" + "Type <- (TimeType / IntervalType / BitType / RowType / MapType / UnionType / NumericType / SetofType / SimpleType) ArrayBounds*\n" "SimpleType <- (QualifiedTypeName / CharacterType) TypeModifiers?\n" - "CharacterType <- ('CHARACTER'i 'VARYING'i?) /\n" - " ('CHAR'i 'VARYING'i?) /\n" - " ('NATIONAL'i 'CHARACTER'i 'VARYING'i?) /\n" - " ('NATIONAL'i 'CHAR'i 'VARYING'i?) /\n" - " ('NCHAR'i 'VARYING'i?) /\n" - " 'VARCHAR'i\n" - "IntervalType <- ('INTERVAL'i Interval?) / ('INTERVAL'i Parens(NumberLiteral))\n" - "YearKeyword <- 'YEAR'i / 'YEARS'i\n" - "MonthKeyword <- 'MONTH'i / 'MONTHS'i\n" - "DayKeyword <- 'DAY'i / 'DAYS'i\n" - "HourKeyword <- 'HOUR'i / 'HOURS'i\n" - "MinuteKeyword <- 'MINUTE'i / 'MINUTES'i\n" - "SecondKeyword <- 'SECOND'i / 'SECONDS'i\n" - "MillisecondKeyword <- 'MILLISECOND'i / 'MILLISECONDS'i\n" - "MicrosecondKeyword <- 'MICROSECOND'i / 'MICROSECONDS'i\n" - "WeekKeyword <- 'WEEK'i / 'WEEKS'i\n" - "QuarterKeyword <- 'QUARTER'i / 'QUARTERS'i\n" - "DecadeKeyword <- 'DECADE'i / 'DECADES'i\n" - "CenturyKeyword <- 'CENTURY'i / 'CENTURIES'i\n" - "MillenniumKeyword <- 'MILLENNIUM'i / 'MILLENNIA'i\n" + "CharacterType <- ('CHARACTER' 'VARYING'?) /\n" + " ('CHAR' 'VARYING'?) /\n" + " ('NATIONAL' 'CHARACTER' 'VARYING'?) /\n" + " ('NATIONAL' 'CHAR' 'VARYING'?) /\n" + " ('NCHAR' 'VARYING'?) /\n" + " 'VARCHAR'\n" + "IntervalType <- ('INTERVAL' Interval?) / ('INTERVAL' Parens(NumberLiteral))\n" + "YearKeyword <- 'YEAR' / 'YEARS'\n" + "MonthKeyword <- 'MONTH' / 'MONTHS'\n" + "DayKeyword <- 'DAY' / 'DAYS'\n" + "HourKeyword <- 'HOUR' / 'HOURS'\n" + "MinuteKeyword <- 'MINUTE' / 'MINUTES'\n" + "SecondKeyword <- 'SECOND' / 'SECONDS'\n" + "MillisecondKeyword <- 'MILLISECOND' / 'MILLISECONDS'\n" + "MicrosecondKeyword <- 'MICROSECOND' / 'MICROSECONDS'\n" + "WeekKeyword <- 'WEEK' / 'WEEKS'\n" + "QuarterKeyword <- 'QUARTER' / 'QUARTERS'\n" + "DecadeKeyword <- 'DECADE' / 'DECADES'\n" + "CenturyKeyword <- 'CENTURY' / 'CENTURIES'\n" + "MillenniumKeyword <- 'MILLENNIUM' / 'MILLENNIA'\n" "Interval <- YearKeyword /\n" " MonthKeyword /\n" " DayKeyword /\n" @@ -1111,65 +1127,66 @@ const char INLINED_PEG_GRAMMAR[] = { " DecadeKeyword /\n" " CenturyKeyword /\n" " MillenniumKeyword /\n" - " (YearKeyword 'TO'i MonthKeyword) /\n" - " (DayKeyword 'TO'i HourKeyword) /\n" - " (DayKeyword 'TO'i MinuteKeyword) /\n" - " (DayKeyword 'TO'i SecondKeyword) /\n" - " (HourKeyword 'TO'i MinuteKeyword) /\n" - " (HourKeyword 'TO'i SecondKeyword) /\n" - " (MinuteKeyword 'TO'i SecondKeyword)\n" - "BitType <- 'BIT'i 'VARYING'i? Parens(List(Expression))?\n" - "NumericType <- 'INT'i /\n" - " 'INTEGER'i /\n" - " 'SMALLINT'i /\n" - " 'BIGINT'i /\n" - " 'REAL'i /\n" - " 'BOOLEAN'i /\n" - " ('FLOAT'i Parens(NumberLiteral)?) /\n" - " ('DOUBLE'i 'PRECISION'i) /\n" - " ('DECIMAL'i TypeModifiers?) /\n" - " ('DEC'i TypeModifiers?) /\n" - " ('NUMERIC'i TypeModifiers?)\n" + " (YearKeyword 'TO' MonthKeyword) /\n" + " (DayKeyword 'TO' HourKeyword) /\n" + " (DayKeyword 'TO' MinuteKeyword) /\n" + " (DayKeyword 'TO' SecondKeyword) /\n" + " (HourKeyword 'TO' MinuteKeyword) /\n" + " (HourKeyword 'TO' SecondKeyword) /\n" + " (MinuteKeyword 'TO' SecondKeyword)\n" + "BitType <- 'BIT' 'VARYING'? Parens(List(Expression))?\n" + "NumericType <- 'INT' /\n" + " 'INTEGER' /\n" + " 'SMALLINT' /\n" + " 'BIGINT' /\n" + " 'REAL' /\n" + " 'BOOLEAN' /\n" + " ('FLOAT' Parens(NumberLiteral)?) /\n" + " ('DOUBLE' 'PRECISION') /\n" + " ('DECIMAL' TypeModifiers?) /\n" + " ('DEC' TypeModifiers?) /\n" + " ('NUMERIC' TypeModifiers?)\n" "QualifiedTypeName <- CatalogQualification? SchemaQualification? TypeName\n" "TypeModifiers <- Parens(List(Expression)?)\n" "RowType <- RowOrStruct Parens(List(ColIdType))\n" - "UnionType <- 'UNION'i Parens(List(ColIdType))\n" - "MapType <- 'MAP'i Parens(List(Type))\n" + "UnionType <- 'UNION' Parens(List(ColIdType))\n" + "SetofType <- 'SETOF' Type\n" + "MapType <- 'MAP' Parens(List(Type))\n" "ColIdType <- ColId Type\n" - "ArrayBounds <- ('[' NumberLiteral? ']') / 'ARRAY'i\n" + "ArrayBounds <- ('[' NumberLiteral? ']') / 'ARRAY'\n" "TimeType <- TimeOrTimestamp TypeModifiers? TimeZone?\n" - "TimeOrTimestamp <- 'TIME'i / 'TIMESTAMP'i\n" - "TimeZone <- WithOrWithout 'TIME'i 'ZONE'i\n" - "WithOrWithout <- 'WITH'i / 'WITHOUT'i\n" - "RowOrStruct <- 'ROW'i / 'STRUCT'i\n" + "TimeOrTimestamp <- 'TIME' / 'TIMESTAMP'\n" + "TimeZone <- WithOrWithout 'TIME' 'ZONE'\n" + "WithOrWithout <- 'WITH' / 'WITHOUT'\n" + "RowOrStruct <- 'ROW' / 'STRUCT'\n" "# internal definitions\n" "%whitespace <- [ \\t\\n\\r]*\n" "List(D) <- D (',' D)* ','?\n" "Parens(D) <- '(' D ')'\n" - "ExplainStatement <- 'EXPLAIN'i 'ANALYZE'i? ExplainOptions? Statement\n" + "ExplainStatement <- 'EXPLAIN' 'ANALYZE'? ExplainOptions? Statement\n" "ExplainOptions <- Parens(GenericCopyOptionList)\n" - "AnalyzeStatement <- 'ANALYZE'i 'VERBOSE'i? AnalyzeTarget?\n" + "AnalyzeStatement <- 'ANALYZE' 'VERBOSE'? AnalyzeTarget?\n" "AnalyzeTarget <- QualifiedName Parens(List(Name))?\n" "Name <- ColId ('.' ColLabel)*\n" "CreateMacroStmt <- MacroOrFunction IfNotExists? QualifiedName List(MacroDefinition)\n" - "MacroOrFunction <- 'MACRO'i / 'FUNCTION'i\n" - "MacroDefinition <- Parens(MacroParameters?) 'AS'i (TableMacroDefinition / ScalarMacroDefinition)\n" + "MacroOrFunction <- 'MACRO' / 'FUNCTION'\n" + "MacroDefinition <- Parens(MacroParameters?) 'AS' (TableMacroDefinition / ScalarMacroDefinition)\n" "MacroParameters <- List(MacroParameter)\n" - "MacroParameter <- NamedParameter / TypeFuncName\n" + "MacroParameter <- NamedParameter / (TypeFuncName Type?)\n" "ScalarMacroDefinition <- Expression\n" - "TableMacroDefinition <- 'TABLE'i SelectStatement\n" - "CommentStatement <- 'COMMENT'i 'ON'i CommentOnType ColumnReference 'IS'i CommentValue\n" - "CommentOnType <- 'TABLE'i / 'SEQUENCE'i / 'FUNCTION'i / ('MACRO'i 'TABLE'i?) / 'VIEW'i / 'DATABASE'i / 'INDEX'i / 'SCHEMA'i / 'TYPE'i / 'COLUMN'i\n" - "CommentValue <- 'NULL'i / StringLiteral\n" - "AttachStatement <- 'ATTACH'i OrReplace? IfNotExists? Database? DatabasePath AttachAlias? AttachOptions?\n" - "Database <- 'DATABASE'i\n" + "TableMacroDefinition <- 'TABLE' SelectStatement\n" + "CommentStatement <- 'COMMENT' 'ON' CommentOnType ColumnReference 'IS' CommentValue\n" + "CommentOnType <- 'TABLE' / 'SEQUENCE' / 'FUNCTION' / ('MACRO' 'TABLE'?) / 'VIEW' / 'DATABASE' / 'INDEX' / 'SCHEMA' / 'TYPE' / 'COLUMN'\n" + "CommentValue <- 'NULL' / StringLiteral\n" + "AttachStatement <- 'ATTACH' OrReplace? IfNotExists? Database? DatabasePath AttachAlias? AttachOptions?\n" + "Database <- 'DATABASE'\n" "DatabasePath <- StringLiteral\n" - "AttachAlias <- 'AS'i ColId\n" + "AttachAlias <- 'AS' ColId\n" "AttachOptions <- Parens(GenericCopyOptionList)\n" - "DetachStatement <- 'DETACH'i Database? IfExists? CatalogName\n" - "UseStatement <- 'USE'i UseTarget\n" + "DetachStatement <- 'DETACH' Database? IfExists? CatalogName\n" + "UseStatement <- 'USE' UseTarget\n" "UseTarget <- (CatalogName '.' ReservedSchemaName) / SchemaName / CatalogName\n" - "CallStatement <- 'CALL'i TableFunctionName TableFunctionArguments\n" + "CallStatement <- 'CALL' TableFunctionName TableFunctionArguments\n" }; diff --git a/extension/autocomplete/include/keyword_helper.hpp b/extension/autocomplete/include/keyword_helper.hpp index 1de4c220045e..fad021f06bf0 100644 --- a/extension/autocomplete/include/keyword_helper.hpp +++ b/extension/autocomplete/include/keyword_helper.hpp @@ -4,7 +4,7 @@ #include "duckdb/common/string.hpp" namespace duckdb { -enum class KeywordCategory : uint8_t { +enum class PEGKeywordCategory : uint8_t { KEYWORD_NONE, KEYWORD_UNRESERVED, KEYWORD_RESERVED, @@ -12,14 +12,14 @@ enum class KeywordCategory : uint8_t { KEYWORD_COL_NAME }; -class KeywordHelper { +class PEGKeywordHelper { public: - static KeywordHelper &Instance(); - bool KeywordCategoryType(const string &text, KeywordCategory type) const; + static PEGKeywordHelper &Instance(); + bool KeywordCategoryType(const string &text, PEGKeywordCategory type) const; void InitializeKeywordMaps(); private: - KeywordHelper(); + PEGKeywordHelper(); bool initialized; case_insensitive_set_t reserved_keyword_map; case_insensitive_set_t unreserved_keyword_map; diff --git a/extension/autocomplete/include/matcher.hpp b/extension/autocomplete/include/matcher.hpp index 35f3037eeb44..7eb58119bd2f 100644 --- a/extension/autocomplete/include/matcher.hpp +++ b/extension/autocomplete/include/matcher.hpp @@ -11,8 +11,10 @@ #include "duckdb/common/string_util.hpp" #include "duckdb/common/vector.hpp" #include "duckdb/common/reference_map.hpp" +#include "transformer/parse_result.hpp" namespace duckdb { +class ParseResultAllocator; class Matcher; class MatcherAllocator; @@ -73,11 +75,14 @@ enum class TokenType { WORD }; struct MatcherToken { // NOLINTNEXTLINE: allow implicit conversion from text - MatcherToken(string text_p) : text(std::move(text_p)) { + MatcherToken(string text_p, idx_t offset_p) : text(std::move(text_p)), offset(offset_p) { + length = text.length(); } TokenType type = TokenType::WORD; string text; + idx_t offset = 0; + idx_t length = 0; }; struct MatcherSuggestion { @@ -96,17 +101,19 @@ struct MatcherSuggestion { }; struct MatchState { - MatchState(vector &tokens, vector &suggestions) - : tokens(tokens), suggestions(suggestions), token_index(0) { + MatchState(vector &tokens, vector &suggestions, ParseResultAllocator &allocator) + : tokens(tokens), suggestions(suggestions), token_index(0), allocator(allocator) { } MatchState(MatchState &state) - : tokens(state.tokens), suggestions(state.suggestions), token_index(state.token_index) { + : tokens(state.tokens), suggestions(state.suggestions), token_index(state.token_index), + allocator(state.allocator) { } vector &tokens; vector &suggestions; reference_set_t added_suggestions; idx_t token_index; + ParseResultAllocator &allocator; void AddSuggestion(MatcherSuggestion suggestion); }; @@ -121,6 +128,7 @@ class Matcher { //! Match virtual MatchResultType Match(MatchState &state) const = 0; + virtual optional_ptr MatchParseResult(MatchState &state) const = 0; virtual SuggestionType AddSuggestion(MatchState &state) const; virtual SuggestionType AddSuggestionInternal(MatchState &state) const = 0; virtual string ToString() const = 0; @@ -166,4 +174,12 @@ class MatcherAllocator { vector> matchers; }; +class ParseResultAllocator { +public: + optional_ptr Allocate(unique_ptr parse_result); + +private: + vector> parse_results; +}; + } // namespace duckdb diff --git a/extension/autocomplete/include/parser/peg_parser.hpp b/extension/autocomplete/include/parser/peg_parser.hpp new file mode 100644 index 000000000000..b6723cdc86a3 --- /dev/null +++ b/extension/autocomplete/include/parser/peg_parser.hpp @@ -0,0 +1,66 @@ + +#pragma once +#include "duckdb/common/case_insensitive_map.hpp" +#include "duckdb/common/string_map_set.hpp" + +namespace duckdb { +enum class PEGRuleType { + LITERAL, // literal rule ('Keyword'i) + REFERENCE, // reference to another rule (Rule) + OPTIONAL, // optional rule (Rule?) + OR, // or rule (Rule1 / Rule2) + REPEAT // repeat rule (Rule1* +}; + +enum class PEGTokenType { + LITERAL, // literal token ('Keyword'i) + REFERENCE, // reference token (Rule) + OPERATOR, // operator token (/ or ) + FUNCTION_CALL, // start of function call (i.e. Function(...)) + REGEX // regular expression ([ \t\n\r] or <[a-z_]i[a-z0-9_]i>) +}; + +struct PEGToken { + PEGTokenType type; + string_t text; +}; + +struct PEGRule { + string_map_t parameters; + vector tokens; + + void Clear() { + parameters.clear(); + tokens.clear(); + } +}; + +struct PEGParser { +public: + void ParseRules(const char *grammar); + void AddRule(string_t rule_name, PEGRule rule); + + case_insensitive_map_t rules; +}; + +enum class PEGParseState { + RULE_NAME, // Rule name + RULE_SEPARATOR, // look for <- + RULE_DEFINITION // part of rule definition +}; + +inline bool IsPEGOperator(char c) { + switch (c) { + case '/': + case '?': + case '(': + case ')': + case '*': + case '!': + return true; + default: + return false; + } +} + +} // namespace duckdb diff --git a/extension/autocomplete/include/transformer/parse_result.hpp b/extension/autocomplete/include/transformer/parse_result.hpp new file mode 100644 index 000000000000..b7d364eac9ed --- /dev/null +++ b/extension/autocomplete/include/transformer/parse_result.hpp @@ -0,0 +1,325 @@ +#pragma once +#include "duckdb/common/arena_linked_list.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/common/optional_ptr.hpp" +#include "duckdb/common/string.hpp" +#include "duckdb/common/types/string_type.hpp" +#include "duckdb/parser/parsed_expression.hpp" + +namespace duckdb { + +class PEGTransformer; // Forward declaration + +enum class ParseResultType : uint8_t { + LIST, + OPTIONAL, + REPEAT, + CHOICE, + EXPRESSION, + IDENTIFIER, + KEYWORD, + OPERATOR, + STATEMENT, + EXTENSION, + NUMBER, + STRING, + INVALID +}; + +inline const char *ParseResultToString(ParseResultType type) { + switch (type) { + case ParseResultType::LIST: + return "LIST"; + case ParseResultType::OPTIONAL: + return "OPTIONAL"; + case ParseResultType::REPEAT: + return "REPEAT"; + case ParseResultType::CHOICE: + return "CHOICE"; + case ParseResultType::EXPRESSION: + return "EXPRESSION"; + case ParseResultType::IDENTIFIER: + return "IDENTIFIER"; + case ParseResultType::KEYWORD: + return "KEYWORD"; + case ParseResultType::OPERATOR: + return "OPERATOR"; + case ParseResultType::STATEMENT: + return "STATEMENT"; + case ParseResultType::EXTENSION: + return "EXTENSION"; + case ParseResultType::NUMBER: + return "NUMBER"; + case ParseResultType::STRING: + return "STRING"; + case ParseResultType::INVALID: + return "INVALID"; + } + return "INVALID"; +} + +class ParseResult { +public: + explicit ParseResult(ParseResultType type) : type(type) { + } + virtual ~ParseResult() = default; + + template + TARGET &Cast() { + if (TARGET::TYPE != ParseResultType::INVALID && type != TARGET::TYPE) { + throw InternalException("Failed to cast parse result of type %s to type %s for rule %s", + ParseResultToString(TARGET::TYPE), ParseResultToString(type), name); + } + return reinterpret_cast(*this); + } + + ParseResultType type; + string name; + + virtual void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const { + ss << indent << (is_last ? "└─" : "├─") << " " << ParseResultToString(type); + if (!name.empty()) { + ss << " (" << name << ")"; + } + } + + // The public entry point + std::string ToString() const { + std::stringstream ss; + std::unordered_set visited; + // The root is always the "last" element at its level + ToStringInternal(ss, visited, "", true); + return ss.str(); + } +}; + +struct IdentifierParseResult : ParseResult { + static constexpr ParseResultType TYPE = ParseResultType::IDENTIFIER; + string identifier; + + explicit IdentifierParseResult(string identifier_p) : ParseResult(TYPE), identifier(std::move(identifier_p)) { + } + + void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const override { + ParseResult::ToStringInternal(ss, visited, indent, is_last); + ss << ": \"" << identifier << "\"\n"; + } +}; + +struct KeywordParseResult : ParseResult { + static constexpr ParseResultType TYPE = ParseResultType::KEYWORD; + string keyword; + + explicit KeywordParseResult(string keyword_p) : ParseResult(TYPE), keyword(std::move(keyword_p)) { + } + + void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const override { + ParseResult::ToStringInternal(ss, visited, indent, is_last); + ss << ": \"" << keyword << "\"\n"; + } +}; + +struct ListParseResult : ParseResult { + static constexpr ParseResultType TYPE = ParseResultType::LIST; + +public: + explicit ListParseResult(vector> results_p, string name_p) + : ParseResult(TYPE), children(std::move(results_p)) { + name = name_p; + } + + vector> GetChildren() const { + return children; + } + + optional_ptr GetChild(idx_t index) { + if (index >= children.size()) { + throw InternalException("Child index out of bounds"); + } + return children[index]; + } + + template + T &Child(idx_t index) { + auto child_ptr = GetChild(index); + return child_ptr->Cast(); + } + + void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const override { + ss << indent << (is_last ? "└─" : "├─"); + + if (visited.count(this)) { + ss << " List (" << name << ") [... already printed ...]\n"; + return; + } + visited.insert(this); + + ss << " " << ParseResultToString(type); + if (!name.empty()) { + ss << " (" << name << ")"; + } + ss << " [" << children.size() << " children]\n"; + + std::string child_indent = indent + (is_last ? " " : "│ "); + for (size_t i = 0; i < children.size(); ++i) { + if (children[i]) { + children[i]->ToStringInternal(ss, visited, child_indent, i == children.size() - 1); + } else { + ss << child_indent << (i == children.size() - 1 ? "└─" : "├─") << " [nullptr]\n"; + } + } + } + +private: + vector> children; +}; + +struct RepeatParseResult : ParseResult { + static constexpr ParseResultType TYPE = ParseResultType::REPEAT; + vector> children; + + explicit RepeatParseResult(vector> results_p) + : ParseResult(TYPE), children(std::move(results_p)) { + } + + template + T &Child(idx_t index) { + if (index >= children.size()) { + throw InternalException("Child index out of bounds"); + } + return children[index]->Cast(); + } + + void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const override { + ss << indent << (is_last ? "└─" : "├─"); + + if (visited.count(this)) { + ss << " Repeat (" << name << ") [... already printed ...]\n"; + return; + } + visited.insert(this); + + ss << " " << ParseResultToString(type); + if (!name.empty()) { + ss << " (" << name << ")"; + } + ss << " [" << children.size() << " children]\n"; + + std::string child_indent = indent + (is_last ? " " : "│ "); + for (size_t i = 0; i < children.size(); ++i) { + if (children[i]) { + children[i]->ToStringInternal(ss, visited, child_indent, i == children.size() - 1); + } else { + ss << child_indent << (i == children.size() - 1 ? "└─" : "├─") << " [nullptr]\n"; + } + } + } +}; + +struct OptionalParseResult : ParseResult { + static constexpr ParseResultType TYPE = ParseResultType::OPTIONAL; + optional_ptr optional_result; + + explicit OptionalParseResult() : ParseResult(TYPE), optional_result(nullptr) { + } + explicit OptionalParseResult(optional_ptr result_p) : ParseResult(TYPE), optional_result(result_p) { + name = result_p->name; + } + + bool HasResult() const { + return optional_result != nullptr; + } + + void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const override { + if (HasResult()) { + // The optional node has a value, so we "collapse" it by just printing its child. + // We pass the same indentation and is_last status, so it takes the place of the Optional node. + optional_result->ToStringInternal(ss, visited, indent, is_last); + } else { + // The optional node is empty, which is useful information, so we print it. + ss << indent << (is_last ? "└─" : "├─") << " " << ParseResultToString(type) << " [empty]\n"; + } + } +}; + +class ChoiceParseResult : public ParseResult { +public: + static constexpr ParseResultType TYPE = ParseResultType::CHOICE; + + explicit ChoiceParseResult(optional_ptr parse_result_p, idx_t selected_idx_p) + : ParseResult(TYPE), result(parse_result_p), selected_idx(selected_idx_p) { + name = parse_result_p->name; + } + + optional_ptr result; + idx_t selected_idx; + + void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const override { + if (result) { + // The choice was resolved. We print a marker and then print the child below it. + ss << indent << (is_last ? "└─" : "├─") << " [" << ParseResultToString(type) << " (idx: " << selected_idx + << ")] ->\n"; + + // The child is now on a new indentation level and is the only child of our marker. + std::string child_indent = indent + (is_last ? " " : "│ "); + result->ToStringInternal(ss, visited, child_indent, true); + } else { + // The choice had no result. + ss << indent << (is_last ? "└─" : "├─") << " " << ParseResultToString(type) << " [no result]\n"; + } + } +}; + +class NumberParseResult : public ParseResult { +public: + static constexpr ParseResultType TYPE = ParseResultType::NUMBER; + + explicit NumberParseResult(string number_p) : ParseResult(TYPE), number(std::move(number_p)) { + } + string number; + + void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const override { + ParseResult::ToStringInternal(ss, visited, indent, is_last); + ss << ": " << number << "\n"; + } +}; + +class StringLiteralParseResult : public ParseResult { +public: + static constexpr ParseResultType TYPE = ParseResultType::STRING; + + explicit StringLiteralParseResult(string string_p) : ParseResult(TYPE), result(std::move(string_p)) { + } + string result; + + void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const override { + ParseResult::ToStringInternal(ss, visited, indent, is_last); + ss << ": \"" << result << "\"\n"; + } +}; + +class OperatorParseResult : public ParseResult { +public: + static constexpr ParseResultType TYPE = ParseResultType::OPERATOR; + + explicit OperatorParseResult(string operator_p) : ParseResult(TYPE), operator_token(std::move(operator_p)) { + } + string operator_token; + + void ToStringInternal(std::stringstream &ss, std::unordered_set &visited, + const std::string &indent, bool is_last) const override { + ParseResult::ToStringInternal(ss, visited, indent, is_last); + ss << ": " << operator_token << "\n"; + } +}; + +} // namespace duckdb diff --git a/extension/autocomplete/include/transformer/peg_transformer.hpp b/extension/autocomplete/include/transformer/peg_transformer.hpp new file mode 100644 index 000000000000..d8cefa0d0b7b --- /dev/null +++ b/extension/autocomplete/include/transformer/peg_transformer.hpp @@ -0,0 +1,208 @@ +#pragma once + +#include "tokenizer.hpp" +#include "parse_result.hpp" +#include "transform_enum_result.hpp" +#include "transform_result.hpp" +#include "ast/setting_info.hpp" +#include "duckdb/function/macro_function.hpp" +#include "duckdb/parser/expression/case_expression.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/expression/parameter_expression.hpp" +#include "duckdb/parser/expression/window_expression.hpp" +#include "duckdb/parser/parsed_data/create_type_info.hpp" +#include "duckdb/parser/parsed_data/transaction_info.hpp" +#include "duckdb/parser/statement/copy_database_statement.hpp" +#include "duckdb/parser/statement/set_statement.hpp" +#include "duckdb/parser/statement/create_statement.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "parser/peg_parser.hpp" +#include "duckdb/storage/arena_allocator.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" +#include "duckdb/parser/statement/insert_statement.hpp" + +namespace duckdb { + +// Forward declare +struct QualifiedName; +struct MatcherToken; + +struct PEGTransformerState { + explicit PEGTransformerState(const vector &tokens_p) : tokens(tokens_p), token_index(0) { + } + + const vector &tokens; + idx_t token_index; +}; + +class PEGTransformer { +public: + using AnyTransformFunction = + std::function(PEGTransformer &, optional_ptr)>; + + PEGTransformer(ArenaAllocator &allocator, PEGTransformerState &state, + const case_insensitive_map_t &transform_functions, + const case_insensitive_map_t &grammar_rules, + const case_insensitive_map_t> &enum_mappings) + : allocator(allocator), state(state), grammar_rules(grammar_rules), transform_functions(transform_functions), + enum_mappings(enum_mappings) { + } + +public: + template + T Transform(optional_ptr parse_result) { + auto it = transform_functions.find(parse_result->name); + if (it == transform_functions.end()) { + throw NotImplementedException("No transformer function found for rule '%s'", parse_result->name); + } + auto &func = it->second; + + unique_ptr base_result = func(*this, parse_result); + if (!base_result) { + throw InternalException("Transformer for rule '%s' returned a nullptr.", parse_result->name); + } + + auto *typed_result_ptr = dynamic_cast *>(base_result.get()); + if (!typed_result_ptr) { + throw InternalException("Transformer for rule '" + parse_result->name + "' returned an unexpected type."); + } + + return std::move(typed_result_ptr->value); + } + + template + T Transform(ListParseResult &parse_result, idx_t child_index) { + auto child_parse_result = parse_result.GetChild(child_index); + return Transform(child_parse_result); + } + + template + T TransformEnum(optional_ptr parse_result) { + auto enum_rule_name = parse_result->name; + + auto rule_value = enum_mappings.find(enum_rule_name); + if (rule_value == enum_mappings.end()) { + throw ParserException("Enum transform failed: could not find mapping for '%s'", enum_rule_name); + } + + auto *typed_enum_ptr = dynamic_cast *>(rule_value->second.get()); + if (!typed_enum_ptr) { + throw InternalException("Enum mapping for rule '%s' has an unexpected type.", enum_rule_name); + } + + return typed_enum_ptr->value; + } + + template + void TransformOptional(ListParseResult &list_pr, idx_t child_idx, T &target) { + auto &opt = list_pr.Child(child_idx); + if (opt.HasResult()) { + target = Transform(opt.optional_result); + } + } + + // Make overloads return raw pointers, as ownership is handled by the ArenaAllocator. + template + T *Make(Args &&...args) { + return allocator.Make(std::forward(args)...); + } + + void ClearParameters(); + static void ParamTypeCheck(PreparedParamType last_type, PreparedParamType new_type); + void SetParam(const string &name, idx_t index, PreparedParamType type); + bool GetParam(const string &name, idx_t &index, PreparedParamType type); + +public: + ArenaAllocator &allocator; + PEGTransformerState &state; + const case_insensitive_map_t &grammar_rules; + const case_insensitive_map_t &transform_functions; + const case_insensitive_map_t> &enum_mappings; + case_insensitive_map_t named_parameter_map; + idx_t prepared_statement_parameter_index = 0; + PreparedParamType last_param_type = PreparedParamType::INVALID; +}; + +class PEGTransformerFactory { +public: + static PEGTransformerFactory &GetInstance(); + explicit PEGTransformerFactory(); + static unique_ptr Transform(vector &tokens, const char *root_rule = "Statement"); + +private: + template + void RegisterEnum(const string &rule_name, T value) { + auto existing_rule = enum_mappings.find(rule_name); + if (existing_rule != enum_mappings.end()) { + throw InternalException("EnumRule %s already exists", rule_name); + } + enum_mappings[rule_name] = make_uniq>(value); + } + + template + void Register(const string &rule_name, FUNC function) { + auto existing_rule = sql_transform_functions.find(rule_name); + if (existing_rule != sql_transform_functions.end()) { + throw InternalException("Rule %s already exists", rule_name); + } + sql_transform_functions[rule_name] = + [function](PEGTransformer &transformer, + optional_ptr parse_result) -> unique_ptr { + auto result_value = function(transformer, parse_result); + return make_uniq>(std::move(result_value)); + }; + } + + PEGTransformerFactory(const PEGTransformerFactory &) = delete; + + static unique_ptr TransformStatement(PEGTransformer &, optional_ptr list); + + // common.gram + static unique_ptr TransformNumberLiteral(PEGTransformer &transformer, + optional_ptr parse_result); + static string TransformStringLiteral(PEGTransformer &transformer, optional_ptr parse_result); + + // expression.gram + static unique_ptr TransformBaseExpression(PEGTransformer &transformer, + optional_ptr parse_result); + static unique_ptr TransformExpression(PEGTransformer &transformer, + optional_ptr parse_result); + static unique_ptr TransformConstantLiteral(PEGTransformer &transformer, + optional_ptr parse_result); + static unique_ptr TransformLiteralExpression(PEGTransformer &transformer, + optional_ptr parse_result); + static unique_ptr TransformSingleExpression(PEGTransformer &transformer, + optional_ptr parse_result); + + // use.gram + static unique_ptr TransformUseStatement(PEGTransformer &transformer, + optional_ptr parse_result); + static QualifiedName TransformUseTarget(PEGTransformer &transformer, optional_ptr parse_result); + + // set.gram + static unique_ptr TransformResetStatement(PEGTransformer &transformer, + optional_ptr parse_result); + static vector> TransformSetAssignment(PEGTransformer &transformer, + optional_ptr parse_result); + static SettingInfo TransformSetSetting(PEGTransformer &transformer, optional_ptr parse_result); + static unique_ptr TransformSetStatement(PEGTransformer &transformer, + optional_ptr parse_result); + static unique_ptr TransformSetTimeZone(PEGTransformer &transformer, + optional_ptr parse_result); + static SettingInfo TransformSetVariable(PEGTransformer &transformer, optional_ptr parse_result); + static unique_ptr TransformStandardAssignment(PEGTransformer &transformer, + optional_ptr parse_result); + static vector> TransformVariableList(PEGTransformer &transformer, + optional_ptr parse_result); + + //! Helper functions + static vector> ExtractParseResultsFromList(optional_ptr parse_result); + +private: + PEGParser parser; + case_insensitive_map_t sql_transform_functions; + case_insensitive_map_t> enum_mappings; +}; + +} // namespace duckdb diff --git a/extension/autocomplete/include/transformer/transform_enum_result.hpp b/extension/autocomplete/include/transformer/transform_enum_result.hpp new file mode 100644 index 000000000000..31e058997938 --- /dev/null +++ b/extension/autocomplete/include/transformer/transform_enum_result.hpp @@ -0,0 +1,15 @@ +#pragma once + +namespace duckdb { +struct TransformEnumValue { + virtual ~TransformEnumValue() = default; +}; + +template +struct TypedTransformEnumResult : public TransformEnumValue { + explicit TypedTransformEnumResult(T value_p) : value(std::move(value_p)) { + } + T value; +}; + +} // namespace duckdb diff --git a/extension/autocomplete/include/transformer/transform_result.hpp b/extension/autocomplete/include/transformer/transform_result.hpp new file mode 100644 index 000000000000..2b9529eccc04 --- /dev/null +++ b/extension/autocomplete/include/transformer/transform_result.hpp @@ -0,0 +1,16 @@ +#pragma once + +namespace duckdb { + +struct TransformResultValue { + virtual ~TransformResultValue() = default; +}; + +template +struct TypedTransformResult : public TransformResultValue { + explicit TypedTransformResult(T value_p) : value(std::move(value_p)) { + } + T value; +}; + +} // namespace duckdb diff --git a/extension/autocomplete/inline_grammar.py b/extension/autocomplete/inline_grammar.py index 9d39b2fe04b6..2b9699382c27 100644 --- a/extension/autocomplete/inline_grammar.py +++ b/extension/autocomplete/inline_grammar.py @@ -82,7 +82,7 @@ def load_keywords(filepath): f.write("/* THIS FILE WAS AUTOMATICALLY GENERATED BY inline_grammar.py */\n") f.write("#include \"keyword_helper.hpp\"\n\n") f.write("namespace duckdb {\n") - f.write("void KeywordHelper::InitializeKeywordMaps() { // Renamed for clarity\n") + f.write("void PEGKeywordHelper::InitializeKeywordMaps() { // Renamed for clarity\n") f.write("\tif (initialized) {\n\t\treturn;\n\t};\n") f.write("\tinitialized = true;\n\n") @@ -119,7 +119,7 @@ def filename_to_upper_camel(file): rule_name = filename_to_upper_camel(file) rule = f"{rule_name} <- " with open(os.path.join(keywords_dir, file), 'r') as f: - lines = [f"'{line.strip()}'i" for line in f if line.strip()] + lines = [f"'{line.strip()}'" for line in f if line.strip()] rule += " /\n".join(lines) + "\n" contents += rule diff --git a/extension/autocomplete/keyword_helper.cpp b/extension/autocomplete/keyword_helper.cpp index dda2e4c4a7dd..893c96671283 100644 --- a/extension/autocomplete/keyword_helper.cpp +++ b/extension/autocomplete/keyword_helper.cpp @@ -1,30 +1,30 @@ #include "keyword_helper.hpp" namespace duckdb { -KeywordHelper &KeywordHelper::Instance() { - static KeywordHelper instance; +PEGKeywordHelper &PEGKeywordHelper::Instance() { + static PEGKeywordHelper instance; return instance; } -KeywordHelper::KeywordHelper() { +PEGKeywordHelper::PEGKeywordHelper() { InitializeKeywordMaps(); } -bool KeywordHelper::KeywordCategoryType(const std::string &text, const KeywordCategory type) const { +bool PEGKeywordHelper::KeywordCategoryType(const std::string &text, const PEGKeywordCategory type) const { switch (type) { - case KeywordCategory::KEYWORD_RESERVED: { + case PEGKeywordCategory::KEYWORD_RESERVED: { auto it = reserved_keyword_map.find(text); return it != reserved_keyword_map.end(); } - case KeywordCategory::KEYWORD_UNRESERVED: { + case PEGKeywordCategory::KEYWORD_UNRESERVED: { auto it = unreserved_keyword_map.find(text); return it != unreserved_keyword_map.end(); } - case KeywordCategory::KEYWORD_TYPE_FUNC: { + case PEGKeywordCategory::KEYWORD_TYPE_FUNC: { auto it = typefunc_keyword_map.find(text); return it != typefunc_keyword_map.end(); } - case KeywordCategory::KEYWORD_COL_NAME: { + case PEGKeywordCategory::KEYWORD_COL_NAME: { auto it = colname_keyword_map.find(text); return it != colname_keyword_map.end(); } diff --git a/extension/autocomplete/keyword_map.cpp b/extension/autocomplete/keyword_map.cpp index 5531b0142e8d..622474585051 100644 --- a/extension/autocomplete/keyword_map.cpp +++ b/extension/autocomplete/keyword_map.cpp @@ -2,7 +2,7 @@ #include "keyword_helper.hpp" namespace duckdb { -void KeywordHelper::InitializeKeywordMaps() { // Renamed for clarity +void PEGKeywordHelper::InitializeKeywordMaps() { // Renamed for clarity if (initialized) { return; }; diff --git a/extension/autocomplete/matcher.cpp b/extension/autocomplete/matcher.cpp index 95e9d7f3791b..2312eaafedf8 100644 --- a/extension/autocomplete/matcher.cpp +++ b/extension/autocomplete/matcher.cpp @@ -10,6 +10,8 @@ #include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/exception/parser_exception.hpp" #include "tokenizer.hpp" +#include "parser/peg_parser.hpp" +#include "transformer/parse_result.hpp" #ifdef PEG_PARSER_SOURCE_FILE #include #else @@ -17,7 +19,6 @@ #endif namespace duckdb { -struct PEGParser; SuggestionType Matcher::AddSuggestion(MatchState &state) const { auto entry = state.added_suggestions.find(*this); @@ -53,14 +54,19 @@ class KeywordMatcher : public Matcher { } MatchResultType Match(MatchState &state) const override { - auto &token = state.tokens[state.token_index]; - if (StringUtil::CIEquals(keyword, token.text)) { - // move to the next token - state.token_index++; - return MatchResultType::SUCCESS; - } else { + if (!MatchKeyword(state)) { return MatchResultType::FAIL; } + return MatchResultType::SUCCESS; + } + + optional_ptr MatchParseResult(MatchState &state) const override { + if (!MatchKeyword(state)) { + return nullptr; + } + auto result = state.allocator.Allocate(make_uniq(keyword)); + result->name = name; + return result; } SuggestionType AddSuggestionInternal(MatchState &state) const override { @@ -74,6 +80,20 @@ class KeywordMatcher : public Matcher { return "'" + keyword + "'"; } +private: + bool MatchKeyword(MatchState &state) const { + if (state.token_index >= state.tokens.size()) { + return false; + } + auto &token = state.tokens[state.token_index]; + if (StringUtil::CIEquals(keyword, token.text)) { + // move to the next token + state.token_index++; + return true; + } + return false; + } + private: string keyword; int32_t score_bonus; @@ -122,6 +142,22 @@ class ListMatcher : public Matcher { return MatchResultType::SUCCESS; } + optional_ptr MatchParseResult(MatchState &state) const override { + MatchState list_state(state); + vector> results; + + for (const auto &child_matcher : matchers) { + auto child_result = child_matcher.get().MatchParseResult(list_state); + if (!child_result) { + return nullptr; + } + results.push_back(child_result); + } + state.token_index = list_state.token_index; + // Empty name implies it's a subrule, e.g. 'SET'i (StandardAssignment / SetTimeZone) + return state.allocator.Allocate(make_uniq(std::move(results), name)); + } + SuggestionType AddSuggestionInternal(MatchState &state) const override { for (auto &matcher : matchers) { auto suggestion_result = matcher.get().AddSuggestion(state); @@ -160,7 +196,7 @@ class OptionalMatcher : public Matcher { MatchResultType Match(MatchState &state) const override { MatchState child_state(state); auto child_match = matcher.Match(child_state); - if (child_match != MatchResultType::SUCCESS) { + if (child_match == MatchResultType::FAIL) { // did not succeed in matching - go back up (but return success anyway) return MatchResultType::SUCCESS; } @@ -169,6 +205,18 @@ class OptionalMatcher : public Matcher { return MatchResultType::SUCCESS; } + optional_ptr MatchParseResult(MatchState &state) const override { + MatchState child_state(state); + auto child_match = matcher.MatchParseResult(child_state); + if (child_match == nullptr) { + // did not succeed in matching - go back up (simply return a nullptr) + return state.allocator.Allocate(make_uniq()); + } + // propagate the child state upwards + state.token_index = child_state.token_index; + return state.allocator.Allocate(make_uniq(child_match)); + } + SuggestionType AddSuggestionInternal(MatchState &state) const override { matcher.AddSuggestion(state); return SuggestionType::OPTIONAL; @@ -205,6 +253,20 @@ class ChoiceMatcher : public Matcher { return MatchResultType::FAIL; } + optional_ptr MatchParseResult(MatchState &state) const override { + for (idx_t i = 0; i < matchers.size(); i++) { + MatchState choice_state(state); + auto child_result = matchers[i].get().MatchParseResult(choice_state); + if (child_result != nullptr) { + // we matched this child - propagate upwards + state.token_index = choice_state.token_index; + auto result = state.allocator.Allocate(make_uniq(child_result, i)); + return result; + } + } + return nullptr; + } + SuggestionType AddSuggestionInternal(MatchState &state) const override { for (auto &child_matcher : matchers) { child_matcher.get().AddSuggestion(state); @@ -266,6 +328,41 @@ class RepeatMatcher : public Matcher { } } + optional_ptr MatchParseResult(MatchState &state) const override { + MatchState repeat_state(state); + vector> results; + + // First, we MUST match the element at least once. + auto first_result = element.MatchParseResult(repeat_state); + if (!first_result) { + // The first match failed, so the whole repeat fails. + return nullptr; + } + results.push_back(first_result); + + // After the first success, the overall result is a success. + // Now, we continue matching the element as many times as possible. + while (true) { + // Propagate the new state upwards. + state.token_index = repeat_state.token_index; + + // Check if there are any tokens left. + if (repeat_state.token_index >= state.tokens.size()) { + break; + } + + // Try to match the element again. + auto next_result = element.MatchParseResult(repeat_state); + if (!next_result) { + break; + } + results.push_back(next_result); + } + + // Return all collected results in a RepeatParseResult. + return state.allocator.Allocate(make_uniq(std::move(results))); + } + SuggestionType AddSuggestionInternal(MatchState &state) const override { element.AddSuggestion(state); return SuggestionType::MANDATORY; @@ -301,44 +398,23 @@ class IdentifierMatcher : public Matcher { } MatchResultType Match(MatchState &state) const override { - // variable matchers match anything except for reserved keywords - auto &token_text = state.tokens[state.token_index].text; - const auto &keyword_helper = KeywordHelper::Instance(); - switch (suggestion_type) { - case SuggestionState::SUGGEST_TYPE_NAME: - if (keyword_helper.KeywordCategoryType(token_text, KeywordCategory::KEYWORD_RESERVED) || - keyword_helper.KeywordCategoryType(token_text, GetBannedCategory())) { - return MatchResultType::FAIL; - } - break; - default: { - const auto banned_category = GetBannedCategory(); - const auto allowed_override_category = banned_category == KeywordCategory::KEYWORD_COL_NAME - ? KeywordCategory::KEYWORD_TYPE_FUNC - : KeywordCategory::KEYWORD_COL_NAME; - - const bool is_reserved = keyword_helper.KeywordCategoryType(token_text, KeywordCategory::KEYWORD_RESERVED); - const bool has_extra_banned_category = keyword_helper.KeywordCategoryType(token_text, banned_category); - const bool has_banned_flag = is_reserved || has_extra_banned_category; - - const bool is_unreserved = - keyword_helper.KeywordCategoryType(token_text, KeywordCategory::KEYWORD_UNRESERVED); - const bool has_override_flag = keyword_helper.KeywordCategoryType(token_text, allowed_override_category); - const bool has_allowed_flag = is_unreserved || has_override_flag; - - if (has_banned_flag && !has_allowed_flag) { - return MatchResultType::FAIL; - } - break; - } - } - if (!IsIdentifier(token_text)) { + if (!MatchIdentifier(state)) { return MatchResultType::FAIL; } - state.token_index++; return MatchResultType::SUCCESS; } + optional_ptr MatchParseResult(MatchState &state) const override { + if (state.token_index >= state.tokens.size()) { + return nullptr; + } + auto &token_text = state.tokens[state.token_index].text; + if (!MatchIdentifier(state)) { + return nullptr; + } + return state.allocator.Allocate(make_uniq(token_text)); + } + bool SupportsStringLiteral() const { switch (suggestion_type) { case SuggestionState::SUGGEST_TABLE_NAME: @@ -349,13 +425,13 @@ class IdentifierMatcher : public Matcher { } } - KeywordCategory GetBannedCategory() const { + PEGKeywordCategory GetBannedCategory() const { switch (suggestion_type) { case SuggestionState::SUGGEST_SCALAR_FUNCTION_NAME: case SuggestionState::SUGGEST_TABLE_FUNCTION_NAME: - return KeywordCategory::KEYWORD_COL_NAME; + return PEGKeywordCategory::KEYWORD_COL_NAME; default: - return KeywordCategory::KEYWORD_TYPE_FUNC; + return PEGKeywordCategory::KEYWORD_TYPE_FUNC; } } @@ -395,6 +471,47 @@ class IdentifierMatcher : public Matcher { } } +private: + bool MatchIdentifier(MatchState &state) const { + // variable matchers match anything except for reserved keywords + auto &token_text = state.tokens[state.token_index].text; + const auto &keyword_helper = PEGKeywordHelper::Instance(); + switch (suggestion_type) { + case SuggestionState::SUGGEST_TYPE_NAME: + if (keyword_helper.KeywordCategoryType(token_text, PEGKeywordCategory::KEYWORD_RESERVED) || + keyword_helper.KeywordCategoryType(token_text, GetBannedCategory())) { + return false; + } + break; + default: { + const auto banned_category = GetBannedCategory(); + const auto allowed_override_category = banned_category == PEGKeywordCategory::KEYWORD_COL_NAME + ? PEGKeywordCategory::KEYWORD_TYPE_FUNC + : PEGKeywordCategory::KEYWORD_COL_NAME; + + const bool is_reserved = + keyword_helper.KeywordCategoryType(token_text, PEGKeywordCategory::KEYWORD_RESERVED); + const bool has_extra_banned_category = keyword_helper.KeywordCategoryType(token_text, banned_category); + const bool has_banned_flag = is_reserved || has_extra_banned_category; + + const bool is_unreserved = + keyword_helper.KeywordCategoryType(token_text, PEGKeywordCategory::KEYWORD_UNRESERVED); + const bool has_override_flag = keyword_helper.KeywordCategoryType(token_text, allowed_override_category); + const bool has_allowed_flag = is_unreserved || has_override_flag; + + if (has_banned_flag && !has_allowed_flag) { + return false; + } + break; + } + } + if (!IsIdentifier(token_text)) { + return false; + } + state.token_index++; + return true; + } + SuggestionState suggestion_type; }; @@ -407,13 +524,28 @@ class ReservedIdentifierMatcher : public IdentifierMatcher { } MatchResultType Match(MatchState &state) const override { - // reserved variable matchers match anything + if (!MatchReservedIdentifier(state)) { + return MatchResultType::FAIL; + } + return MatchResultType::SUCCESS; + } + + optional_ptr MatchParseResult(MatchState &state) const override { + auto &token_text = state.tokens[state.token_index].text; + if (!MatchReservedIdentifier(state)) { + return nullptr; + } + return state.allocator.Allocate(make_uniq(token_text)); + } + +private: + bool MatchReservedIdentifier(MatchState &state) const { auto &token_text = state.tokens[state.token_index].text; if (!IsIdentifier(token_text)) { - return MatchResultType::FAIL; + return false; } state.token_index++; - return MatchResultType::SUCCESS; + return true; } }; @@ -423,16 +555,30 @@ class StringLiteralMatcher : public Matcher { public: explicit StringLiteralMatcher() : Matcher(TYPE) { + name = "StringLiteral"; } MatchResultType Match(MatchState &state) const override { // variable matchers match anything except for reserved keywords + if (!MatchStringLiteral(state)) { + return MatchResultType::FAIL; + } + return MatchResultType::SUCCESS; + } + + optional_ptr MatchParseResult(MatchState &state) const override { + if (state.token_index >= state.tokens.size()) { + return nullptr; + } auto &token_text = state.tokens[state.token_index].text; - if (token_text.size() >= 2 && token_text.front() == '\'' && token_text.back() == '\'') { - state.token_index++; - return MatchResultType::SUCCESS; + if (!MatchStringLiteral(state)) { + return nullptr; } - return MatchResultType::FAIL; + string stripped_string = token_text.substr(1, token_text.length() - 2); + + auto result = state.allocator.Allocate(make_uniq(stripped_string)); + result->name = name; + return result; } SuggestionType AddSuggestionInternal(MatchState &state) const override { @@ -442,6 +588,16 @@ class StringLiteralMatcher : public Matcher { string ToString() const override { return "STRING_LITERAL"; } + +private: + static bool MatchStringLiteral(MatchState &state) { + auto &token_text = state.tokens[state.token_index].text; + if (token_text.size() >= 2 && token_text.front() == '\'' && token_text.back() == '\'') { + state.token_index++; + return true; + } + return false; + } }; class NumberLiteralMatcher : public Matcher { @@ -450,23 +606,30 @@ class NumberLiteralMatcher : public Matcher { public: explicit NumberLiteralMatcher() : Matcher(TYPE) { + name = "NumberLiteral"; } MatchResultType Match(MatchState &state) const override { // variable matchers match anything except for reserved keywords - auto &token_text = state.tokens[state.token_index].text; - if (!BaseTokenizer::CharacterIsInitialNumber(token_text[0])) { + if (!MatchNumberLiteral(state)) { return MatchResultType::FAIL; } - for (idx_t i = 1; i < token_text.size(); i++) { - if (!BaseTokenizer::CharacterIsNumber(token_text[i])) { - return MatchResultType::FAIL; - } - } - state.token_index++; return MatchResultType::SUCCESS; } + optional_ptr MatchParseResult(MatchState &state) const override { + if (state.token_index >= state.tokens.size()) { + return nullptr; + } + auto &token_text = state.tokens[state.token_index].text; + if (!MatchNumberLiteral(state)) { + return nullptr; + } + auto result = state.allocator.Allocate(make_uniq(token_text)); + result->name = name; + return result; + } + SuggestionType AddSuggestionInternal(MatchState &state) const override { return SuggestionType::MANDATORY; } @@ -474,6 +637,21 @@ class NumberLiteralMatcher : public Matcher { string ToString() const override { return "NUMBER_LITERAL"; } + +private: + static bool MatchNumberLiteral(MatchState &state) { + auto &token_text = state.tokens[state.token_index].text; + if (!BaseTokenizer::CharacterIsInitialNumber(token_text[0])) { + return false; + } + for (idx_t i = 1; i < token_text.size(); i++) { + if (!BaseTokenizer::CharacterIsNumber(token_text[i])) { + return false; + } + } + state.token_index++; + return true; + } }; class OperatorMatcher : public Matcher { @@ -485,6 +663,33 @@ class OperatorMatcher : public Matcher { } MatchResultType Match(MatchState &state) const override { + if (!MatchOperator(state)) { + return MatchResultType::FAIL; + } + return MatchResultType::SUCCESS; + } + + optional_ptr MatchParseResult(MatchState &state) const override { + if (state.token_index >= state.tokens.size()) { + return nullptr; + } + auto &token_text = state.tokens[state.token_index].text; + if (!MatchOperator(state)) { + return nullptr; + } + return state.allocator.Allocate(make_uniq(token_text)); + } + + SuggestionType AddSuggestionInternal(MatchState &state) const override { + return SuggestionType::MANDATORY; + } + + string ToString() const override { + return "OPERATOR"; + } + +private: + static bool MatchOperator(MatchState &state) { auto &token_text = state.tokens[state.token_index].text; for (auto &c : token_text) { switch (c) { @@ -504,19 +709,11 @@ class OperatorMatcher : public Matcher { case '|': break; default: - return MatchResultType::FAIL; + return false; } } state.token_index++; - return MatchResultType::SUCCESS; - } - - SuggestionType AddSuggestionInternal(MatchState &state) const override { - return SuggestionType::MANDATORY; - } - - string ToString() const override { - return "OPERATOR"; + return true; } }; @@ -526,6 +723,12 @@ Matcher &MatcherAllocator::Allocate(unique_ptr matcher) { return result; } +optional_ptr ParseResultAllocator::Allocate(unique_ptr parse_result) { + auto result_ptr = parse_result.get(); + parse_results.push_back(std::move(parse_result)); + return optional_ptr(result_ptr); +} + //! Class for building matchers class MatcherFactory { friend struct MatcherList; @@ -674,258 +877,6 @@ Matcher &MatcherFactory::Operator() const { return allocator.Allocate(make_uniq()); } -enum class PEGRuleType { - LITERAL, // literal rule ('Keyword'i) - REFERENCE, // reference to another rule (Rule) - OPTIONAL, // optional rule (Rule?) - OR, // or rule (Rule1 / Rule2) - REPEAT // repeat rule (Rule1* -}; - -enum class PEGTokenType { - LITERAL, // literal token ('Keyword'i) - REFERENCE, // reference token (Rule) - OPERATOR, // operator token (/ or ) - FUNCTION_CALL, // start of function call (i.e. Function(...)) - REGEX // regular expression ([ \t\n\r] or <[a-z_]i[a-z0-9_]i>) -}; - -struct PEGToken { - PEGTokenType type; - string_t text; -}; - -struct PEGRule { - string_map_t parameters; - vector tokens; - - void Clear() { - parameters.clear(); - tokens.clear(); - } -}; - -struct PEGParser { -public: - void ParseRules(const char *grammar); - - void AddRule(string_t rule_name, PEGRule rule) { - auto entry = rules.find(rule_name); - if (entry != rules.end()) { - throw InternalException("Failed to parse grammar - duplicate rule name %s", rule_name.GetString()); - } - rules.insert(make_pair(rule_name, std::move(rule))); - } - - string_map_t rules; -}; - -enum class PEGParseState { - RULE_NAME, // Rule name - RULE_SEPARATOR, // look for <- - RULE_DEFINITION // part of rule definition -}; - -bool IsPEGOperator(char c) { - switch (c) { - case '/': - case '?': - case '(': - case ')': - case '*': - case '!': - return true; - default: - return false; - } -} - -void PEGParser::ParseRules(const char *grammar) { - string_t rule_name; - PEGRule rule; - PEGParseState parse_state = PEGParseState::RULE_NAME; - idx_t bracket_count = 0; - bool in_or_clause = false; - // look for the rules - idx_t c = 0; - while (grammar[c]) { - if (grammar[c] == '#') { - // comment - ignore until EOL - while (grammar[c] && !StringUtil::CharacterIsNewline(grammar[c])) { - c++; - } - continue; - } - if (parse_state == PEGParseState::RULE_DEFINITION && StringUtil::CharacterIsNewline(grammar[c]) && - bracket_count == 0 && !in_or_clause && !rule.tokens.empty()) { - // if we see a newline while we are parsing a rule definition we can complete the rule - AddRule(rule_name, std::move(rule)); - rule_name = string_t(); - rule.Clear(); - // look for the subsequent rule - parse_state = PEGParseState::RULE_NAME; - c++; - continue; - } - if (StringUtil::CharacterIsSpace(grammar[c])) { - // skip whitespace - c++; - continue; - } - switch (parse_state) { - case PEGParseState::RULE_NAME: { - // look for alpha-numerics - idx_t start_pos = c; - if (grammar[c] == '%') { - // rules can start with % (%whitespace) - c++; - } - while (grammar[c] && StringUtil::CharacterIsAlphaNumeric(grammar[c])) { - c++; - } - if (c == start_pos) { - throw InternalException("Failed to parse grammar - expected an alpha-numeric rule name (pos %d)", c); - } - rule_name = string_t(grammar + start_pos, c - start_pos); - rule.Clear(); - parse_state = PEGParseState::RULE_SEPARATOR; - break; - } - case PEGParseState::RULE_SEPARATOR: { - if (grammar[c] == '(') { - if (!rule.parameters.empty()) { - throw InternalException("Failed to parse grammar - multiple parameters at position %d", c); - } - // parameter - c++; - idx_t parameter_start = c; - while (grammar[c] && StringUtil::CharacterIsAlphaNumeric(grammar[c])) { - c++; - } - if (parameter_start == c) { - throw InternalException("Failed to parse grammar - expected a parameter at position %d", c); - } - rule.parameters.insert( - make_pair(string_t(grammar + parameter_start, c - parameter_start), rule.parameters.size())); - if (grammar[c] != ')') { - throw InternalException("Failed to parse grammar - expected closing bracket at position %d", c); - } - c++; - } else { - if (grammar[c] != '<' || grammar[c + 1] != '-') { - throw InternalException("Failed to parse grammar - expected a rule definition (<-) (pos %d)", c); - } - c += 2; - parse_state = PEGParseState::RULE_DEFINITION; - } - break; - } - case PEGParseState::RULE_DEFINITION: { - // we parse either: - // (1) a literal ('Keyword'i) - // (2) a rule reference (Rule) - // (3) an operator ( '(' '/' '?' '*' ')') - in_or_clause = false; - if (grammar[c] == '\'') { - // parse literal - c++; - idx_t literal_start = c; - while (grammar[c] && grammar[c] != '\'') { - if (grammar[c] == '\\') { - // escape - c++; - } - c++; - } - if (!grammar[c]) { - throw InternalException("Failed to parse grammar - did not find closing ' (pos %d)", c); - } - PEGToken token; - token.text = string_t(grammar + literal_start, c - literal_start); - token.type = PEGTokenType::LITERAL; - rule.tokens.push_back(token); - c++; - if (grammar[c] == 'i') { - // skip optional case insensitive marker - // note: all keywords we parse are case insensitive so we just ignore this marker - c++; - } - } else if (StringUtil::CharacterIsAlphaNumeric(grammar[c])) { - // alphanumeric character - this is a rule reference - idx_t rule_start = c; - while (grammar[c] && StringUtil::CharacterIsAlphaNumeric(grammar[c])) { - c++; - } - PEGToken token; - token.text = string_t(grammar + rule_start, c - rule_start); - if (grammar[c] == '(') { - // this is a function call - c++; - bracket_count++; - token.type = PEGTokenType::FUNCTION_CALL; - } else { - token.type = PEGTokenType::REFERENCE; - } - rule.tokens.push_back(token); - } else if (grammar[c] == '[' || grammar[c] == '<') { - // regular expression- [^"] or <...> - idx_t rule_start = c; - char final_char = grammar[c] == '[' ? ']' : '>'; - while (grammar[c] && grammar[c] != final_char) { - if (grammar[c] == '\\') { - // handle escapes - c++; - } - if (grammar[c]) { - c++; - } - } - c++; - PEGToken token; - token.text = string_t(grammar + rule_start, c - rule_start); - token.type = PEGTokenType::REGEX; - rule.tokens.push_back(token); - } else if (IsPEGOperator(grammar[c])) { - if (grammar[c] == '(') { - bracket_count++; - } else if (grammar[c] == ')') { - if (bracket_count == 0) { - throw InternalException("Failed to parse grammar - unclosed bracket at position %d in rule %s", - c, rule_name.GetString()); - } - bracket_count--; - } else if (grammar[c] == '/') { - in_or_clause = true; - } - // operator - operators are always length 1 - PEGToken token; - token.text = string_t(grammar + c, 1); - token.type = PEGTokenType::OPERATOR; - rule.tokens.push_back(token); - c++; - } else { - throw InternalException("Unrecognized rule contents in rule %s (character %s)", rule_name.GetString(), - string(1, grammar[c])); - } - } - default: - break; - } - if (!grammar[c]) { - break; - } - } - if (parse_state == PEGParseState::RULE_SEPARATOR) { - throw InternalException("Failed to parse grammar - rule %s does not have a definition", rule_name.GetString()); - } - if (parse_state == PEGParseState::RULE_DEFINITION) { - if (rule.tokens.empty()) { - throw InternalException("Failed to parse grammar - rule %s is empty", rule_name.GetString()); - } - AddRule(rule_name, std::move(rule)); - } -} - Matcher &MatcherFactory::CreateMatcher(PEGParser &parser, string_t rule_name) { vector> parameters; return CreateMatcher(parser, rule_name, parameters); @@ -1027,7 +978,7 @@ Matcher &MatcherFactory::CreateMatcher(PEGParser &parser, string_t rule_name, ve } } // look up the rule - auto entry = parser.rules.find(rule_name); + auto entry = parser.rules.find(rule_name.GetString()); if (entry == parser.rules.end()) { throw InternalException("Failed to create matcher for rule %s - rule is missing", rule_name.GetString()); } @@ -1100,25 +1051,28 @@ Matcher &MatcherFactory::CreateMatcher(PEGParser &parser, string_t rule_name, ve } case '/': { // OR operator - this signifies a choice between the last rule and the next rule - auto &last_matcher = list.GetLastRootMatcher().matcher; - if (last_matcher.Type() != MatcherType::LIST) { + auto &last_root_matcher = list.GetLastRootMatcher().matcher; + if (last_root_matcher.Type() != MatcherType::LIST) { throw InternalException("OR expected a list matcher"); } - auto &list_matcher = last_matcher.Cast(); + auto &list_matcher = last_root_matcher.Cast(); if (list_matcher.matchers.empty()) { throw InternalException("OR rule found as first token"); } - auto &final_matcher = list_matcher.matchers.back(); - vector> choice_matchers; - choice_matchers.push_back(final_matcher); - auto &choice_matcher = Choice(choice_matchers); + auto &previous_matcher = list_matcher.matchers.back(); - // the choice matcher gets added to the list matcher (instead of the previous matcher) - list_matcher.matchers.pop_back(); - list_matcher.matchers.push_back(choice_matcher); - // then it gets pushed onto the stack of matchers - // the next rule will then get pushed onto the choice matcher - list.AddRootMatcher(choice_matcher); + if (previous_matcher.get().Type() == MatcherType::CHOICE) { + list.AddRootMatcher(previous_matcher); + } else { + vector> choice_options; + choice_options.push_back(previous_matcher); + auto &new_choice_matcher = Choice(choice_options); + + list_matcher.matchers.pop_back(); + list_matcher.matchers.push_back(new_choice_matcher); + + list.AddRootMatcher(new_choice_matcher); + } break; } case '(': { diff --git a/extension/autocomplete/parser/CMakeLists.txt b/extension/autocomplete/parser/CMakeLists.txt new file mode 100644 index 000000000000..8486f809fb0f --- /dev/null +++ b/extension/autocomplete/parser/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library_unity(duckdb_peg_parser OBJECT peg_parser.cpp) +set(AUTOCOMPLETE_EXTENSION_FILES + ${AUTOCOMPLETE_EXTENSION_FILES} $ + PARENT_SCOPE) diff --git a/extension/autocomplete/parser/peg_parser.cpp b/extension/autocomplete/parser/peg_parser.cpp new file mode 100644 index 000000000000..f0f72f2456bc --- /dev/null +++ b/extension/autocomplete/parser/peg_parser.cpp @@ -0,0 +1,194 @@ +#include "parser/peg_parser.hpp" + +namespace duckdb { + +void PEGParser::AddRule(string_t rule_name, PEGRule rule) { + auto entry = rules.find(rule_name.GetString()); + if (entry != rules.end()) { + throw InternalException("Failed to parse grammar - duplicate rule name %s", rule_name.GetString()); + } + rules.insert(make_pair(rule_name, std::move(rule))); +} + +void PEGParser::ParseRules(const char *grammar) { + string_t rule_name; + PEGRule rule; + PEGParseState parse_state = PEGParseState::RULE_NAME; + idx_t bracket_count = 0; + bool in_or_clause = false; + // look for the rules + idx_t c = 0; + while (grammar[c]) { + if (grammar[c] == '#') { + // comment - ignore until EOL + while (grammar[c] && !StringUtil::CharacterIsNewline(grammar[c])) { + c++; + } + continue; + } + if (parse_state == PEGParseState::RULE_DEFINITION && StringUtil::CharacterIsNewline(grammar[c]) && + bracket_count == 0 && !in_or_clause && !rule.tokens.empty()) { + // if we see a newline while we are parsing a rule definition we can complete the rule + AddRule(rule_name, std::move(rule)); + rule_name = string_t(); + rule.Clear(); + // look for the subsequent rule + parse_state = PEGParseState::RULE_NAME; + c++; + continue; + } + if (StringUtil::CharacterIsSpace(grammar[c])) { + // skip whitespace + c++; + continue; + } + switch (parse_state) { + case PEGParseState::RULE_NAME: { + // look for alpha-numerics + idx_t start_pos = c; + if (grammar[c] == '%') { + // rules can start with % (%whitespace) + c++; + } + while (grammar[c] && StringUtil::CharacterIsAlphaNumeric(grammar[c])) { + c++; + } + if (c == start_pos) { + throw InternalException("Failed to parse grammar - expected an alpha-numeric rule name (pos %d)", c); + } + rule_name = string_t(grammar + start_pos, c - start_pos); + rule.Clear(); + parse_state = PEGParseState::RULE_SEPARATOR; + break; + } + case PEGParseState::RULE_SEPARATOR: { + if (grammar[c] == '(') { + if (!rule.parameters.empty()) { + throw InternalException("Failed to parse grammar - multiple parameters at position %d", c); + } + // parameter + c++; + idx_t parameter_start = c; + while (grammar[c] && StringUtil::CharacterIsAlphaNumeric(grammar[c])) { + c++; + } + if (parameter_start == c) { + throw InternalException("Failed to parse grammar - expected a parameter at position %d", c); + } + rule.parameters.insert( + make_pair(string_t(grammar + parameter_start, c - parameter_start), rule.parameters.size())); + if (grammar[c] != ')') { + throw InternalException("Failed to parse grammar - expected closing bracket at position %d", c); + } + c++; + } else { + if (grammar[c] != '<' || grammar[c + 1] != '-') { + throw InternalException("Failed to parse grammar - expected a rule definition (<-) (pos %d)", c); + } + c += 2; + parse_state = PEGParseState::RULE_DEFINITION; + } + break; + } + case PEGParseState::RULE_DEFINITION: { + // we parse either: + // (1) a literal ('Keyword'i) + // (2) a rule reference (Rule) + // (3) an operator ( '(' '/' '?' '*' ')') + in_or_clause = false; + if (grammar[c] == '\'') { + // parse literal + c++; + idx_t literal_start = c; + while (grammar[c] && grammar[c] != '\'') { + if (grammar[c] == '\\') { + // escape + c++; + } + c++; + } + if (!grammar[c]) { + throw InternalException("Failed to parse grammar - did not find closing ' (pos %d)", c); + } + PEGToken token; + token.text = string_t(grammar + literal_start, c - literal_start); + token.type = PEGTokenType::LITERAL; + rule.tokens.push_back(token); + c++; + } else if (StringUtil::CharacterIsAlphaNumeric(grammar[c])) { + // alphanumeric character - this is a rule reference + idx_t rule_start = c; + while (grammar[c] && StringUtil::CharacterIsAlphaNumeric(grammar[c])) { + c++; + } + PEGToken token; + token.text = string_t(grammar + rule_start, c - rule_start); + if (grammar[c] == '(') { + // this is a function call + c++; + bracket_count++; + token.type = PEGTokenType::FUNCTION_CALL; + } else { + token.type = PEGTokenType::REFERENCE; + } + rule.tokens.push_back(token); + } else if (grammar[c] == '[' || grammar[c] == '<') { + // regular expression- [^"] or <...> + idx_t rule_start = c; + char final_char = grammar[c] == '[' ? ']' : '>'; + while (grammar[c] && grammar[c] != final_char) { + if (grammar[c] == '\\') { + // handle escapes + c++; + } + if (grammar[c]) { + c++; + } + } + c++; + PEGToken token; + token.text = string_t(grammar + rule_start, c - rule_start); + token.type = PEGTokenType::REGEX; + rule.tokens.push_back(token); + } else if (IsPEGOperator(grammar[c])) { + if (grammar[c] == '(') { + bracket_count++; + } else if (grammar[c] == ')') { + if (bracket_count == 0) { + throw InternalException("Failed to parse grammar - unclosed bracket at position %d in rule %s", + c, rule_name.GetString()); + } + bracket_count--; + } else if (grammar[c] == '/') { + in_or_clause = true; + } + // operator - operators are always length 1 + PEGToken token; + token.text = string_t(grammar + c, 1); + token.type = PEGTokenType::OPERATOR; + rule.tokens.push_back(token); + c++; + } else { + throw InternalException("Unrecognized rule contents in rule %s (character %s)", rule_name.GetString(), + string(1, grammar[c])); + } + } + default: + break; + } + if (!grammar[c]) { + break; + } + } + if (parse_state == PEGParseState::RULE_SEPARATOR) { + throw InternalException("Failed to parse grammar - rule %s does not have a definition", rule_name.GetString()); + } + if (parse_state == PEGParseState::RULE_DEFINITION) { + if (rule.tokens.empty()) { + throw InternalException("Failed to parse grammar - rule %s is empty", rule_name.GetString()); + } + AddRule(rule_name, std::move(rule)); + } +} + +} // namespace duckdb diff --git a/extension/autocomplete/tokenizer.cpp b/extension/autocomplete/tokenizer.cpp index e6c63ce7d881..8331b1973b8e 100644 --- a/extension/autocomplete/tokenizer.cpp +++ b/extension/autocomplete/tokenizer.cpp @@ -134,7 +134,7 @@ void BaseTokenizer::PushToken(idx_t start, idx_t end) { return; } string last_token = sql.substr(start, end - start); - tokens.emplace_back(std::move(last_token)); + tokens.emplace_back(std::move(last_token), start); } bool BaseTokenizer::IsValidDollarTagCharacter(char c) { @@ -229,14 +229,14 @@ bool BaseTokenizer::TokenizeInput() { idx_t op_len; if (IsSpecialOperator(i, op_len)) { // special operator - push the special operator - tokens.emplace_back(sql.substr(i, op_len)); + tokens.emplace_back(sql.substr(i, op_len), last_pos); i += op_len - 1; last_pos = i + 1; break; } if (IsSingleByteOperator(c)) { // single-byte operator - directly push the token - tokens.emplace_back(string(1, c)); + tokens.emplace_back(string(1, c), last_pos); last_pos = i + 1; break; } @@ -358,7 +358,7 @@ bool BaseTokenizer::TokenizeInput() { size_t full_marker_len = dollar_quote_marker.size() + 2; string quoted = sql.substr(last_pos, (start + dollar_quote_marker.size() + 1) - last_pos); quoted = "'" + quoted.substr(full_marker_len, quoted.size() - 2 * full_marker_len) + "'"; - tokens.emplace_back(quoted); + tokens.emplace_back(quoted, full_marker_len); dollar_quote_marker = string(); state = TokenizeState::STANDARD; i = end; diff --git a/extension/autocomplete/transformer/CMakeLists.txt b/extension/autocomplete/transformer/CMakeLists.txt new file mode 100644 index 000000000000..c5d0146d9516 --- /dev/null +++ b/extension/autocomplete/transformer/CMakeLists.txt @@ -0,0 +1,12 @@ +add_library_unity( + duckdb_peg_transformer + OBJECT + peg_transformer.cpp + peg_transformer_factory.cpp + transform_common.cpp + transform_expression.cpp + transform_set.cpp + transform_use.cpp) +set(AUTOCOMPLETE_EXTENSION_FILES + ${AUTOCOMPLETE_EXTENSION_FILES} $ + PARENT_SCOPE) diff --git a/extension/autocomplete/transformer/peg_transformer.cpp b/extension/autocomplete/transformer/peg_transformer.cpp new file mode 100644 index 000000000000..5d7bee835070 --- /dev/null +++ b/extension/autocomplete/transformer/peg_transformer.cpp @@ -0,0 +1,47 @@ +#include "transformer/peg_transformer.hpp" + +#include "duckdb/parser/statement/set_statement.hpp" +#include "duckdb/common/string_util.hpp" + +namespace duckdb { + +void PEGTransformer::ParamTypeCheck(PreparedParamType last_type, PreparedParamType new_type) { + // Mixing positional/auto-increment and named parameters is not supported + if (last_type == PreparedParamType::INVALID) { + return; + } + if (last_type == PreparedParamType::NAMED) { + if (new_type != PreparedParamType::NAMED) { + throw NotImplementedException("Mixing named and positional parameters is not supported yet"); + } + } + if (last_type != PreparedParamType::NAMED) { + if (new_type == PreparedParamType::NAMED) { + throw NotImplementedException("Mixing named and positional parameters is not supported yet"); + } + } +} + +bool PEGTransformer::GetParam(const string &identifier, idx_t &index, PreparedParamType type) { + ParamTypeCheck(last_param_type, type); + auto entry = named_parameter_map.find(identifier); + if (entry == named_parameter_map.end()) { + return false; + } + index = entry->second; + return true; +} + +void PEGTransformer::SetParam(const string &identifier, idx_t index, PreparedParamType type) { + ParamTypeCheck(last_param_type, type); + last_param_type = type; + D_ASSERT(!named_parameter_map.count(identifier)); + named_parameter_map[identifier] = index; +} + +void PEGTransformer::ClearParameters() { + prepared_statement_parameter_index = 0; + named_parameter_map.clear(); +} + +} // namespace duckdb diff --git a/extension/autocomplete/transformer/peg_transformer_factory.cpp b/extension/autocomplete/transformer/peg_transformer_factory.cpp new file mode 100644 index 000000000000..3b34f9e6e8be --- /dev/null +++ b/extension/autocomplete/transformer/peg_transformer_factory.cpp @@ -0,0 +1,116 @@ +#include "transformer/peg_transformer.hpp" +#include "matcher.hpp" +#include "duckdb/common/to_string.hpp" +#include "duckdb/parser/sql_statement.hpp" +#include "duckdb/parser/tableref/showref.hpp" + +namespace duckdb { + +unique_ptr PEGTransformerFactory::TransformStatement(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + auto &choice_pr = list_pr.Child(0); + return transformer.Transform>(choice_pr.result); +} + +unique_ptr PEGTransformerFactory::Transform(vector &tokens, const char *root_rule) { + string token_stream; + for (auto &token : tokens) { + token_stream += token.text + " "; + } + + vector suggestions; + ParseResultAllocator parse_result_allocator; + MatchState state(tokens, suggestions, parse_result_allocator); + MatcherAllocator allocator; + auto &matcher = Matcher::RootMatcher(allocator); + auto match_result = matcher.MatchParseResult(state); + if (match_result == nullptr || state.token_index < state.tokens.size()) { + // TODO(dtenwolde) add error handling + string token_list; + for (idx_t i = 0; i < tokens.size(); i++) { + if (!token_list.empty()) { + token_list += "\n"; + } + if (i < 10) { + token_list += " "; + } + token_list += to_string(i) + ":" + tokens[i].text; + } + throw ParserException("Failed to parse query - did not consume all tokens (got to token %d - %s)\nTokens:\n%s", + state.token_index, tokens[state.token_index].text, token_list); + } + + match_result->name = root_rule; + ArenaAllocator transformer_allocator(Allocator::DefaultAllocator()); + PEGTransformerState transformer_state(tokens); + auto &factory = GetInstance(); + PEGTransformer transformer(transformer_allocator, transformer_state, factory.sql_transform_functions, + factory.parser.rules, factory.enum_mappings); + auto result = transformer.Transform>(match_result); + return transformer.Transform>(match_result); +} + +#define REGISTER_TRANSFORM(FUNCTION) Register(string(#FUNCTION).substr(9), &FUNCTION) + +PEGTransformerFactory &PEGTransformerFactory::GetInstance() { + static PEGTransformerFactory instance; + return instance; +} + +PEGTransformerFactory::PEGTransformerFactory() { + REGISTER_TRANSFORM(TransformStatement); + + // common.gram + REGISTER_TRANSFORM(TransformNumberLiteral); + REGISTER_TRANSFORM(TransformStringLiteral); + + // expression.gram + REGISTER_TRANSFORM(TransformBaseExpression); + REGISTER_TRANSFORM(TransformExpression); + REGISTER_TRANSFORM(TransformLiteralExpression); + REGISTER_TRANSFORM(TransformSingleExpression); + REGISTER_TRANSFORM(TransformConstantLiteral); + + // use.gram + REGISTER_TRANSFORM(TransformUseStatement); + REGISTER_TRANSFORM(TransformUseTarget); + + // set.gram + REGISTER_TRANSFORM(TransformResetStatement); + REGISTER_TRANSFORM(TransformSetAssignment); + REGISTER_TRANSFORM(TransformSetSetting); + REGISTER_TRANSFORM(TransformSetStatement); + REGISTER_TRANSFORM(TransformSetTimeZone); + REGISTER_TRANSFORM(TransformSetVariable); + REGISTER_TRANSFORM(TransformStandardAssignment); + REGISTER_TRANSFORM(TransformVariableList); + + RegisterEnum("LocalScope", SetScope::LOCAL); + RegisterEnum("GlobalScope", SetScope::GLOBAL); + RegisterEnum("SessionScope", SetScope::SESSION); + RegisterEnum("VariableScope", SetScope::VARIABLE); + + RegisterEnum("FalseLiteral", Value(false)); + RegisterEnum("TrueLiteral", Value(true)); + RegisterEnum("NullLiteral", Value()); +} + +vector> +PEGTransformerFactory::ExtractParseResultsFromList(optional_ptr parse_result) { + // List(D) <- D (',' D)* ','? + vector> result; + auto &list_pr = parse_result->Cast(); + result.push_back(list_pr.GetChild(0)); + auto opt_child = list_pr.Child(1); + if (opt_child.HasResult()) { + auto repeat_result = opt_child.optional_result->Cast(); + for (auto &child : repeat_result.children) { + auto &list_child = child->Cast(); + result.push_back(list_child.GetChild(1)); + } + } + + return result; +} +} // namespace duckdb diff --git a/extension/autocomplete/transformer/transform_common.cpp b/extension/autocomplete/transformer/transform_common.cpp new file mode 100644 index 000000000000..1d4aa01d09cd --- /dev/null +++ b/extension/autocomplete/transformer/transform_common.cpp @@ -0,0 +1,82 @@ +#include "duckdb/common/operator/cast_operators.hpp" +#include "duckdb/common/types/decimal.hpp" +#include "transformer/peg_transformer.hpp" + +namespace duckdb { + +// NumberLiteral <- < [+-]?[0-9]*([.][0-9]*)? > +unique_ptr PEGTransformerFactory::TransformNumberLiteral(PEGTransformer &transformer, + optional_ptr parse_result) { + auto literal_pr = parse_result->Cast(); + string_t str_val(literal_pr.number); + bool try_cast_as_integer = true; + bool try_cast_as_decimal = true; + optional_idx decimal_position = optional_idx::Invalid(); + idx_t num_underscores = 0; + idx_t num_integer_underscores = 0; + for (idx_t i = 0; i < str_val.GetSize(); i++) { + if (literal_pr.number[i] == '.') { + // decimal point: cast as either decimal or double + try_cast_as_integer = false; + decimal_position = i; + } + if (literal_pr.number[i] == 'e' || literal_pr.number[i] == 'E') { + // found exponent, cast as double + try_cast_as_integer = false; + try_cast_as_decimal = false; + } + if (literal_pr.number[i] == '_') { + num_underscores++; + if (!decimal_position.IsValid()) { + num_integer_underscores++; + } + } + } + if (try_cast_as_integer) { + int64_t bigint_value; + // try to cast as bigint first + if (TryCast::Operation(str_val, bigint_value)) { + // successfully cast to bigint: bigint value + return make_uniq(Value::BIGINT(bigint_value)); + } + hugeint_t hugeint_value; + // if that is not successful; try to cast as hugeint + if (TryCast::Operation(str_val, hugeint_value)) { + // successfully cast to bigint: bigint value + return make_uniq(Value::HUGEINT(hugeint_value)); + } + uhugeint_t uhugeint_value; + // if that is not successful; try to cast as uhugeint + if (TryCast::Operation(str_val, uhugeint_value)) { + // successfully cast to bigint: bigint value + return make_uniq(Value::UHUGEINT(uhugeint_value)); + } + } + idx_t decimal_offset = literal_pr.number[0] == '-' ? 3 : 2; + if (try_cast_as_decimal && decimal_position.IsValid() && + str_val.GetSize() - num_underscores < Decimal::MAX_WIDTH_DECIMAL + decimal_offset) { + // figure out the width/scale based on the decimal position + auto width = NumericCast(str_val.GetSize() - 1 - num_underscores); + auto scale = NumericCast(width - decimal_position.GetIndex() + num_integer_underscores); + if (literal_pr.number[0] == '-') { + width--; + } + if (width <= Decimal::MAX_WIDTH_DECIMAL) { + // we can cast the value as a decimal + Value val = Value(str_val); + val = val.DefaultCastAs(LogicalType::DECIMAL(width, scale)); + return make_uniq(std::move(val)); + } + } + // if there is a decimal or the value is too big to cast as either hugeint or bigint + double dbl_value = Cast::Operation(str_val); + return make_uniq(Value::DOUBLE(dbl_value)); +} + +// StringLiteral <- '\'' [^\']* '\'' +string PEGTransformerFactory::TransformStringLiteral(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &string_literal_pr = parse_result->Cast(); + return string_literal_pr.result; +} +} // namespace duckdb diff --git a/extension/autocomplete/transformer/transform_expression.cpp b/extension/autocomplete/transformer/transform_expression.cpp new file mode 100644 index 000000000000..3298d875f300 --- /dev/null +++ b/extension/autocomplete/transformer/transform_expression.cpp @@ -0,0 +1,118 @@ +#include "transformer/peg_transformer.hpp" +#include "duckdb/parser/expression/comparison_expression.hpp" +#include "duckdb/parser/expression/between_expression.hpp" +#include "duckdb/parser/expression/operator_expression.hpp" +#include "duckdb/parser/expression/cast_expression.hpp" + +namespace duckdb { + +// BaseExpression <- SingleExpression Indirection* +unique_ptr PEGTransformerFactory::TransformBaseExpression(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + auto expr = transformer.Transform>(list_pr.Child(0)); + auto indirection_opt = list_pr.Child(1); + if (indirection_opt.HasResult()) { + auto indirection_repeat = indirection_opt.optional_result->Cast(); + for (auto child : indirection_repeat.children) { + auto indirection_expr = transformer.Transform>(child); + if (indirection_expr->GetExpressionClass() == ExpressionClass::CAST) { + auto cast_expr = unique_ptr_cast(std::move(indirection_expr)); + cast_expr->child = std::move(expr); + expr = std::move(cast_expr); + } else if (indirection_expr->GetExpressionClass() == ExpressionClass::OPERATOR) { + auto operator_expr = unique_ptr_cast(std::move(indirection_expr)); + operator_expr->children.insert(operator_expr->children.begin(), std::move(expr)); + expr = std::move(operator_expr); + } else if (indirection_expr->GetExpressionClass() == ExpressionClass::FUNCTION) { + auto function_expr = unique_ptr_cast(std::move(indirection_expr)); + function_expr->children.push_back(std::move(expr)); + expr = std::move(function_expr); + } + } + } + + return expr; +} + +// Expression <- BaseExpression RecursiveExpression* +unique_ptr PEGTransformerFactory::TransformExpression(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + auto &base_expr_pr = list_pr.Child(0); + unique_ptr base_expr = transformer.Transform>(base_expr_pr); + auto &indirection_pr = list_pr.Child(1); + if (indirection_pr.HasResult()) { + auto repeat_expression_pr = indirection_pr.optional_result->Cast(); + vector> expr_children; + for (auto &child : repeat_expression_pr.children) { + auto expr = transformer.Transform>(child); + if (expr->expression_class == ExpressionClass::COMPARISON) { + auto compare_expr = unique_ptr_cast(std::move(expr)); + compare_expr->left = std::move(base_expr); + base_expr = std::move(compare_expr); + } else if (expr->expression_class == ExpressionClass::FUNCTION) { + auto func_expr = unique_ptr_cast(std::move(expr)); + func_expr->children.insert(func_expr->children.begin(), std::move(base_expr)); + base_expr = std::move(func_expr); + } else if (expr->expression_class == ExpressionClass::LAMBDA) { + auto lambda_expr = unique_ptr_cast(std::move(expr)); + lambda_expr->lhs = std::move(base_expr); + base_expr = std::move(lambda_expr); + } else if (expr->expression_class == ExpressionClass::BETWEEN) { + auto between_expr = unique_ptr_cast(std::move(expr)); + between_expr->input = std::move(base_expr); + base_expr = std::move(between_expr); + } else { + base_expr = make_uniq(expr->type, std::move(base_expr), std::move(expr)); + } + } + } + + return base_expr; +} + +// LiteralExpression <- StringLiteral / NumberLiteral / 'NULL' / 'TRUE' / 'FALSE' +unique_ptr PEGTransformerFactory::TransformLiteralExpression(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &choice_result = parse_result->Cast(); + auto &matched_rule_result = choice_result.Child(0); + if (matched_rule_result.name == "StringLiteral") { + return make_uniq(Value(transformer.Transform(matched_rule_result.result))); + } + return transformer.Transform>(matched_rule_result.result); +} + +unique_ptr PEGTransformerFactory::TransformConstantLiteral(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + return make_uniq(transformer.TransformEnum(list_pr.Child(0).result)); +} + +// SingleExpression <- LiteralExpression / +// Parameter / +// SubqueryExpression / +// SpecialFunctionExpression / +// ParenthesisExpression / +// IntervalLiteral / +// TypeLiteral / +// CaseExpression / +// StarExpression / +// CastExpression / +// GroupingExpression / +// MapExpression / +// FunctionExpression / +// ColumnReference / +// PrefixExpression / +// ListComprehensionExpression / +// ListExpression / +// StructExpression / +// PositionalExpression / +// DefaultExpression +unique_ptr PEGTransformerFactory::TransformSingleExpression(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + return transformer.Transform>(list_pr.Child(0).result); +} + +} // namespace duckdb diff --git a/extension/autocomplete/transformer/transform_set.cpp b/extension/autocomplete/transformer/transform_set.cpp new file mode 100644 index 000000000000..250a4e3adfb1 --- /dev/null +++ b/extension/autocomplete/transformer/transform_set.cpp @@ -0,0 +1,93 @@ +#include "transformer/peg_transformer.hpp" + +namespace duckdb { + +// ResetStatement <- 'RESET' (SetVariable / SetSetting) +unique_ptr PEGTransformerFactory::TransformResetStatement(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + auto &child_pr = list_pr.Child(1); + auto &choice_pr = child_pr.Child(0); + + SettingInfo setting_info = transformer.Transform(choice_pr.result); + return make_uniq(setting_info.name, setting_info.scope); +} + +// SetAssignment <- VariableAssign VariableList +vector> +PEGTransformerFactory::TransformSetAssignment(PEGTransformer &transformer, optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + return transformer.Transform>>(list_pr, 1); +} + +// SetSetting <- SettingScope? SettingName +SettingInfo PEGTransformerFactory::TransformSetSetting(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + auto &optional_scope_pr = list_pr.Child(0); + + SettingInfo result; + result.name = list_pr.Child(1).identifier; + if (optional_scope_pr.optional_result) { + auto setting_scope = optional_scope_pr.optional_result->Cast(); + auto scope_value = setting_scope.Child(0); + result.scope = transformer.TransformEnum(scope_value); + } + return result; +} + +// SetStatement <- 'SET' (StandardAssignment / SetTimeZone) +unique_ptr PEGTransformerFactory::TransformSetStatement(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + auto &child_pr = list_pr.Child(1); + auto &assignment_or_timezone = child_pr.Child(0); + return transformer.Transform>(assignment_or_timezone); +} + +// SetTimeZone <- 'TIME' 'ZONE' Expression +unique_ptr PEGTransformerFactory::TransformSetTimeZone(PEGTransformer &transformer, + optional_ptr parse_result) { + throw NotImplementedException("Rule 'SetTimeZone' has not been implemented yet"); +} + +// SetVariable <- VariableScope Identifier +SettingInfo PEGTransformerFactory::TransformSetVariable(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + + SettingInfo result; + result.scope = transformer.TransformEnum(list_pr.Child(0)); + result.name = list_pr.Child(1).identifier; + return result; +} + +// StandardAssignment <- (SetVariable / SetSetting) SetAssignment +unique_ptr +PEGTransformerFactory::TransformStandardAssignment(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &choice_pr = parse_result->Cast(); + auto &list_pr = choice_pr.result->Cast(); + auto &first_sub_rule = list_pr.Child(0); + + auto &setting_or_var_pr = first_sub_rule.Child(0); + SettingInfo setting_info = transformer.Transform(setting_or_var_pr.result); + + auto &set_assignment_pr = list_pr.Child(1); + auto value = transformer.Transform>>(set_assignment_pr); + // TODO(dtenwolde) Needs to throw error if more than 1 value (e.g. set threads=1,2;) + return make_uniq(setting_info.name, std::move(value[0]), setting_info.scope); +} + +// VariableList <- List(Expression) +vector> +PEGTransformerFactory::TransformVariableList(PEGTransformer &transformer, optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + auto expr_list = ExtractParseResultsFromList(list_pr.Child(0)); + vector> expressions; + for (auto &expr : expr_list) { + expressions.push_back(transformer.Transform>(expr)); + } + return expressions; +} +} // namespace duckdb diff --git a/extension/autocomplete/transformer/transform_use.cpp b/extension/autocomplete/transformer/transform_use.cpp new file mode 100644 index 000000000000..a2a7d7ad7b3f --- /dev/null +++ b/extension/autocomplete/transformer/transform_use.cpp @@ -0,0 +1,51 @@ +#include "transformer/peg_transformer.hpp" +#include "duckdb/parser/sql_statement.hpp" + +namespace duckdb { + +// UseStatement <- 'USE' UseTarget +unique_ptr PEGTransformerFactory::TransformUseStatement(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + auto qn = transformer.Transform(list_pr, 1); + + string value_str; + if (IsInvalidSchema(qn.schema)) { + value_str = qn.name; + } else { + value_str = qn.schema + "." + qn.name; + } + + auto value_expr = make_uniq(Value(value_str)); + return make_uniq("schema", std::move(value_expr), SetScope::AUTOMATIC); +} + +// UseTarget <- (CatalogName '.' ReservedSchemaName) / SchemaName / CatalogName +QualifiedName PEGTransformerFactory::TransformUseTarget(PEGTransformer &transformer, + optional_ptr parse_result) { + auto &list_pr = parse_result->Cast(); + auto &choice_pr = list_pr.Child(0); + QualifiedName result; + if (choice_pr.result->type == ParseResultType::LIST) { + vector entries; + auto use_target_children = choice_pr.result->Cast(); + for (auto &child : use_target_children.GetChildren()) { + if (child->type == ParseResultType::IDENTIFIER) { + entries.push_back(child->Cast().identifier); + } + } + if (entries.size() == 2) { + result.catalog = INVALID_CATALOG; + result.schema = entries[0]; + result.name = entries[1]; + } else { + throw InternalException("Invalid amount of entries for use statement"); + } + } else if (choice_pr.result->type == ParseResultType::IDENTIFIER) { + result.name = choice_pr.result->Cast().identifier; + } else { + throw InternalException("Unexpected parse result type encountered in UseTarget"); + } + return result; +} +} // namespace duckdb diff --git a/extension/core_functions/aggregate/distributive/arg_min_max.cpp b/extension/core_functions/aggregate/distributive/arg_min_max.cpp index d2bdfbe54443..2265d3daaf28 100644 --- a/extension/core_functions/aggregate/distributive/arg_min_max.cpp +++ b/extension/core_functions/aggregate/distributive/arg_min_max.cpp @@ -15,7 +15,7 @@ namespace duckdb { namespace { struct ArgMinMaxStateBase { - ArgMinMaxStateBase() : is_initialized(false), arg_null(false) { + ArgMinMaxStateBase() : is_initialized(false), arg_null(false), val_null(false) { } template @@ -34,6 +34,7 @@ struct ArgMinMaxStateBase { bool is_initialized; bool arg_null; + bool val_null; }; // Out-of-line specialisations @@ -81,7 +82,7 @@ struct ArgMinMaxState : public ArgMinMaxStateBase { } }; -template +template struct ArgMinMaxBase { template static void Initialize(STATE &state) { @@ -94,25 +95,48 @@ struct ArgMinMaxBase { } template - static void Assign(STATE &state, const A_TYPE &x, const B_TYPE &y, const bool x_null, + static void Assign(STATE &state, const A_TYPE &x, const B_TYPE &y, const bool x_null, const bool y_null, AggregateInputData &aggregate_input_data) { - if (IGNORE_NULL) { + D_ASSERT(aggregate_input_data.bind_data); + const auto &bind_data = aggregate_input_data.bind_data->Cast(); + + if (bind_data.null_handling == ArgMinMaxNullHandling::IGNORE_ANY_NULL) { STATE::template AssignValue(state.arg, x, aggregate_input_data); STATE::template AssignValue(state.value, y, aggregate_input_data); } else { state.arg_null = x_null; + state.val_null = y_null; if (!state.arg_null) { STATE::template AssignValue(state.arg, x, aggregate_input_data); } - STATE::template AssignValue(state.value, y, aggregate_input_data); + if (!state.val_null) { + STATE::template AssignValue(state.value, y, aggregate_input_data); + } } } template static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &binary) { + D_ASSERT(binary.input.bind_data); + const auto &bind_data = binary.input.bind_data->Cast(); if (!state.is_initialized) { - if (IGNORE_NULL || binary.right_mask.RowIsValid(binary.ridx)) { - Assign(state, x, y, !binary.left_mask.RowIsValid(binary.lidx), binary.input); + if (bind_data.null_handling == ArgMinMaxNullHandling::IGNORE_ANY_NULL && + binary.left_mask.RowIsValid(binary.lidx) && binary.right_mask.RowIsValid(binary.ridx)) { + Assign(state, x, y, !binary.left_mask.RowIsValid(binary.lidx), + !binary.right_mask.RowIsValid(binary.ridx), binary.input); + state.is_initialized = true; + return; + } + if (bind_data.null_handling == ArgMinMaxNullHandling::HANDLE_ARG_NULL && + binary.right_mask.RowIsValid(binary.ridx)) { + Assign(state, x, y, !binary.left_mask.RowIsValid(binary.lidx), + !binary.right_mask.RowIsValid(binary.ridx), binary.input); + state.is_initialized = true; + return; + } + if (bind_data.null_handling == ArgMinMaxNullHandling::HANDLE_ANY_NULL) { + Assign(state, x, y, !binary.left_mask.RowIsValid(binary.lidx), + !binary.right_mask.RowIsValid(binary.ridx), binary.input); state.is_initialized = true; } } else { @@ -122,8 +146,14 @@ struct ArgMinMaxBase { template static void Execute(STATE &state, A_TYPE x_data, B_TYPE y_data, AggregateBinaryInput &binary) { - if ((IGNORE_NULL || binary.right_mask.RowIsValid(binary.ridx)) && COMPARATOR::Operation(y_data, state.value)) { - Assign(state, x_data, y_data, !binary.left_mask.RowIsValid(binary.lidx), binary.input); + D_ASSERT(binary.input.bind_data); + const auto &bind_data = binary.input.bind_data->Cast(); + + if (binary.right_mask.RowIsValid(binary.ridx) && COMPARATOR::Operation(y_data, state.value)) { + if (bind_data.null_handling != ArgMinMaxNullHandling::IGNORE_ANY_NULL || + binary.left_mask.RowIsValid(binary.lidx)) { + Assign(state, x_data, y_data, !binary.left_mask.RowIsValid(binary.lidx), false, binary.input); + } } } @@ -132,8 +162,10 @@ struct ArgMinMaxBase { if (!source.is_initialized) { return; } - if (!target.is_initialized || COMPARATOR::Operation(source.value, target.value)) { - Assign(target, source.arg, source.value, source.arg_null, aggregate_input_data); + + if (!target.is_initialized || target.val_null || + (!source.val_null && COMPARATOR::Operation(source.value, target.value))) { + Assign(target, source.arg, source.value, source.arg_null, false, aggregate_input_data); target.is_initialized = true; } } @@ -148,9 +180,10 @@ struct ArgMinMaxBase { } static bool IgnoreNull() { - return IGNORE_NULL; + return false; } + template static unique_ptr Bind(ClientContext &context, AggregateFunction &function, vector> &arguments) { if (arguments[1]->return_type.InternalType() == PhysicalType::VARCHAR) { @@ -158,7 +191,9 @@ struct ArgMinMaxBase { } function.arguments[0] = arguments[0]->return_type; function.return_type = arguments[0]->return_type; - return nullptr; + + auto function_data = make_uniq(NULL_HANDLING); + return unique_ptr(std::move(function_data)); } }; @@ -186,12 +221,14 @@ struct GenericArgMinMaxState { } }; -template -struct VectorArgMinMaxBase : ArgMinMaxBase { +template +struct VectorArgMinMaxBase : ArgMinMaxBase { template static void Update(Vector inputs[], AggregateInputData &aggregate_input_data, idx_t input_count, Vector &state_vector, idx_t count) { + D_ASSERT(aggregate_input_data.bind_data); + const auto &bind_data = aggregate_input_data.bind_data->Cast(); + auto &arg = inputs[0]; UnifiedVectorFormat adata; arg.ToUnifiedFormat(count, adata); @@ -213,21 +250,36 @@ struct VectorArgMinMaxBase : ArgMinMaxBase { auto states = UnifiedVectorFormat::GetData(sdata); for (idx_t i = 0; i < count; i++) { - const auto bidx = bdata.sel->get_index(i); - if (!bdata.validity.RowIsValid(bidx)) { - continue; - } - const auto bval = bys[bidx]; + const auto sidx = sdata.sel->get_index(i); + auto &state = *states[sidx]; const auto aidx = adata.sel->get_index(i); const auto arg_null = !adata.validity.RowIsValid(aidx); - if (IGNORE_NULL && arg_null) { + + if (bind_data.null_handling == ArgMinMaxNullHandling::IGNORE_ANY_NULL && arg_null) { continue; } - const auto sidx = sdata.sel->get_index(i); - auto &state = *states[sidx]; - if (!state.is_initialized || COMPARATOR::template Operation(bval, state.value)) { + const auto bidx = bdata.sel->get_index(i); + + if (!bdata.validity.RowIsValid(bidx)) { + if (bind_data.null_handling == ArgMinMaxNullHandling::HANDLE_ANY_NULL && !state.is_initialized) { + state.is_initialized = true; + state.val_null = true; + if (!arg_null) { + if (&state == last_state) { + assign_count--; + } + assign_sel[assign_count++] = UnsafeNumericCast(i); + last_state = &state; + } + } + continue; + } + + const auto bval = bys[bidx]; + + if (!state.is_initialized || state.val_null || COMPARATOR::template Operation(bval, state.value)) { STATE::template AssignValue(state.value, bval, aggregate_input_data); state.arg_null = arg_null; // micro-adaptivity: it is common we overwrite the same state repeatedly @@ -270,8 +322,12 @@ struct VectorArgMinMaxBase : ArgMinMaxBase { if (!source.is_initialized) { return; } - if (!target.is_initialized || COMPARATOR::Operation(source.value, target.value)) { - STATE::template AssignValue(target.value, source.value, aggregate_input_data); + if (!target.is_initialized || target.val_null || + (!source.val_null && COMPARATOR::Operation(source.value, target.value))) { + target.val_null = source.val_null; + if (!target.val_null) { + STATE::template AssignValue(target.value, source.value, aggregate_input_data); + } target.arg_null = source.arg_null; if (!target.arg_null) { STATE::template AssignValue(target.arg, source.arg, aggregate_input_data); @@ -290,6 +346,7 @@ struct VectorArgMinMaxBase : ArgMinMaxBase { } } + template static unique_ptr Bind(ClientContext &context, AggregateFunction &function, vector> &arguments) { if (arguments[1]->return_type.InternalType() == PhysicalType::VARCHAR) { @@ -297,31 +354,48 @@ struct VectorArgMinMaxBase : ArgMinMaxBase { } function.arguments[0] = arguments[0]->return_type; function.return_type = arguments[0]->return_type; - return nullptr; + + auto function_data = make_uniq(NULL_HANDLING); + return unique_ptr(std::move(function_data)); } }; template -AggregateFunction GetGenericArgMinMaxFunction() { +bind_aggregate_function_t GetBindFunction(const ArgMinMaxNullHandling null_handling) { + switch (null_handling) { + case ArgMinMaxNullHandling::HANDLE_ARG_NULL: + return OP::template Bind; + case ArgMinMaxNullHandling::HANDLE_ANY_NULL: + return OP::template Bind; + default: + return OP::template Bind; + } +} + +template +AggregateFunction GetGenericArgMinMaxFunction(const ArgMinMaxNullHandling null_handling) { using STATE = ArgMinMaxState; + auto bind = GetBindFunction(null_handling); return AggregateFunction( {LogicalType::ANY, LogicalType::ANY}, LogicalType::ANY, AggregateFunction::StateSize, AggregateFunction::StateInitialize, OP::template Update, - AggregateFunction::StateCombine, AggregateFunction::StateVoidFinalize, nullptr, OP::Bind, + AggregateFunction::StateCombine, AggregateFunction::StateVoidFinalize, nullptr, bind, AggregateFunction::StateDestroy); } template -AggregateFunction GetVectorArgMinMaxFunctionInternal(const LogicalType &by_type, const LogicalType &type) { +AggregateFunction GetVectorArgMinMaxFunctionInternal(const LogicalType &by_type, const LogicalType &type, + const ArgMinMaxNullHandling null_handling) { #ifndef DUCKDB_SMALLER_BINARY using STATE = ArgMinMaxState; + auto bind = GetBindFunction(null_handling); return AggregateFunction({type, by_type}, type, AggregateFunction::StateSize, AggregateFunction::StateInitialize, OP::template Update, AggregateFunction::StateCombine, - AggregateFunction::StateVoidFinalize, nullptr, OP::Bind, + AggregateFunction::StateVoidFinalize, nullptr, bind, AggregateFunction::StateDestroy); #else - auto function = GetGenericArgMinMaxFunction(); + auto function = GetGenericArgMinMaxFunction(null_handling); function.arguments = {type, by_type}; function.return_type = type; return function; @@ -330,18 +404,19 @@ AggregateFunction GetVectorArgMinMaxFunctionInternal(const LogicalType &by_type, #ifndef DUCKDB_SMALLER_BINARY template -AggregateFunction GetVectorArgMinMaxFunctionBy(const LogicalType &by_type, const LogicalType &type) { +AggregateFunction GetVectorArgMinMaxFunctionBy(const LogicalType &by_type, const LogicalType &type, + const ArgMinMaxNullHandling null_handling) { switch (by_type.InternalType()) { case PhysicalType::INT32: - return GetVectorArgMinMaxFunctionInternal(by_type, type); + return GetVectorArgMinMaxFunctionInternal(by_type, type, null_handling); case PhysicalType::INT64: - return GetVectorArgMinMaxFunctionInternal(by_type, type); + return GetVectorArgMinMaxFunctionInternal(by_type, type, null_handling); case PhysicalType::INT128: - return GetVectorArgMinMaxFunctionInternal(by_type, type); + return GetVectorArgMinMaxFunctionInternal(by_type, type, null_handling); case PhysicalType::DOUBLE: - return GetVectorArgMinMaxFunctionInternal(by_type, type); + return GetVectorArgMinMaxFunctionInternal(by_type, type, null_handling); case PhysicalType::VARCHAR: - return GetVectorArgMinMaxFunctionInternal(by_type, type); + return GetVectorArgMinMaxFunctionInternal(by_type, type, null_handling); default: throw InternalException("Unimplemented arg_min/arg_max aggregate"); } @@ -356,19 +431,21 @@ const vector ArgMaxByTypes() { } template -void AddVectorArgMinMaxFunctionBy(AggregateFunctionSet &fun, const LogicalType &type) { +void AddVectorArgMinMaxFunctionBy(AggregateFunctionSet &fun, const LogicalType &type, + const ArgMinMaxNullHandling null_handling) { auto by_types = ArgMaxByTypes(); for (const auto &by_type : by_types) { #ifndef DUCKDB_SMALLER_BINARY - fun.AddFunction(GetVectorArgMinMaxFunctionBy(by_type, type)); + fun.AddFunction(GetVectorArgMinMaxFunctionBy(by_type, type, null_handling)); #else - fun.AddFunction(GetVectorArgMinMaxFunctionInternal(by_type, type)); + fun.AddFunction(GetVectorArgMinMaxFunctionInternal(by_type, type, null_handling)); #endif } } template -AggregateFunction GetArgMinMaxFunctionInternal(const LogicalType &by_type, const LogicalType &type) { +AggregateFunction GetArgMinMaxFunctionInternal(const LogicalType &by_type, const LogicalType &type, + const ArgMinMaxNullHandling null_handling) { #ifndef DUCKDB_SMALLER_BINARY using STATE = ArgMinMaxState; auto function = @@ -377,9 +454,9 @@ AggregateFunction GetArgMinMaxFunctionInternal(const LogicalType &by_type, const if (type.InternalType() == PhysicalType::VARCHAR || by_type.InternalType() == PhysicalType::VARCHAR) { function.destructor = AggregateFunction::StateDestroy; } - function.bind = OP::Bind; + function.bind = GetBindFunction(null_handling); #else - auto function = GetGenericArgMinMaxFunction(); + auto function = GetGenericArgMinMaxFunction(null_handling); function.arguments = {type, by_type}; function.return_type = type; #endif @@ -388,18 +465,19 @@ AggregateFunction GetArgMinMaxFunctionInternal(const LogicalType &by_type, const #ifndef DUCKDB_SMALLER_BINARY template -AggregateFunction GetArgMinMaxFunctionBy(const LogicalType &by_type, const LogicalType &type) { +AggregateFunction GetArgMinMaxFunctionBy(const LogicalType &by_type, const LogicalType &type, + const ArgMinMaxNullHandling null_handling) { switch (by_type.InternalType()) { case PhysicalType::INT32: - return GetArgMinMaxFunctionInternal(by_type, type); + return GetArgMinMaxFunctionInternal(by_type, type, null_handling); case PhysicalType::INT64: - return GetArgMinMaxFunctionInternal(by_type, type); + return GetArgMinMaxFunctionInternal(by_type, type, null_handling); case PhysicalType::INT128: - return GetArgMinMaxFunctionInternal(by_type, type); + return GetArgMinMaxFunctionInternal(by_type, type, null_handling); case PhysicalType::DOUBLE: - return GetArgMinMaxFunctionInternal(by_type, type); + return GetArgMinMaxFunctionInternal(by_type, type, null_handling); case PhysicalType::VARCHAR: - return GetArgMinMaxFunctionInternal(by_type, type); + return GetArgMinMaxFunctionInternal(by_type, type, null_handling); default: throw InternalException("Unimplemented arg_min/arg_max by aggregate"); } @@ -407,37 +485,38 @@ AggregateFunction GetArgMinMaxFunctionBy(const LogicalType &by_type, const Logic #endif template -void AddArgMinMaxFunctionBy(AggregateFunctionSet &fun, const LogicalType &type) { +void AddArgMinMaxFunctionBy(AggregateFunctionSet &fun, const LogicalType &type, ArgMinMaxNullHandling null_handling) { auto by_types = ArgMaxByTypes(); for (const auto &by_type : by_types) { #ifndef DUCKDB_SMALLER_BINARY - fun.AddFunction(GetArgMinMaxFunctionBy(by_type, type)); + fun.AddFunction(GetArgMinMaxFunctionBy(by_type, type, null_handling)); #else - fun.AddFunction(GetArgMinMaxFunctionInternal(by_type, type)); + fun.AddFunction(GetArgMinMaxFunctionInternal(by_type, type, null_handling)); #endif } } template -AggregateFunction GetDecimalArgMinMaxFunction(const LogicalType &by_type, const LogicalType &type) { +AggregateFunction GetDecimalArgMinMaxFunction(const LogicalType &by_type, const LogicalType &type, + ArgMinMaxNullHandling null_handling) { D_ASSERT(type.id() == LogicalTypeId::DECIMAL); #ifndef DUCKDB_SMALLER_BINARY switch (type.InternalType()) { case PhysicalType::INT16: - return GetArgMinMaxFunctionBy(by_type, type); + return GetArgMinMaxFunctionBy(by_type, type, null_handling); case PhysicalType::INT32: - return GetArgMinMaxFunctionBy(by_type, type); + return GetArgMinMaxFunctionBy(by_type, type, null_handling); case PhysicalType::INT64: - return GetArgMinMaxFunctionBy(by_type, type); + return GetArgMinMaxFunctionBy(by_type, type, null_handling); default: - return GetArgMinMaxFunctionBy(by_type, type); + return GetArgMinMaxFunctionBy(by_type, type, null_handling); } #else - return GetArgMinMaxFunctionInternal(by_type, type); + return GetArgMinMaxFunctionInternal(by_type, type, null_handling); #endif } -template +template unique_ptr BindDecimalArgMinMax(ClientContext &context, AggregateFunction &function, vector> &arguments) { auto decimal_type = arguments[0]->return_type; @@ -469,51 +548,69 @@ unique_ptr BindDecimalArgMinMax(ClientContext &context, AggregateF } auto name = std::move(function.name); - function = GetDecimalArgMinMaxFunction(by_type, decimal_type); + function = GetDecimalArgMinMaxFunction(by_type, decimal_type, NULL_HANDLING); function.name = std::move(name); function.return_type = decimal_type; - return nullptr; + + auto function_data = make_uniq(NULL_HANDLING); + return unique_ptr(std::move(function_data)); } template -void AddDecimalArgMinMaxFunctionBy(AggregateFunctionSet &fun, const LogicalType &by_type) { - fun.AddFunction(AggregateFunction({LogicalTypeId::DECIMAL, by_type}, LogicalTypeId::DECIMAL, nullptr, nullptr, - nullptr, nullptr, nullptr, nullptr, BindDecimalArgMinMax)); +void AddDecimalArgMinMaxFunctionBy(AggregateFunctionSet &fun, const LogicalType &by_type, + const ArgMinMaxNullHandling null_handling) { + switch (null_handling) { + case ArgMinMaxNullHandling::IGNORE_ANY_NULL: + fun.AddFunction(AggregateFunction({LogicalTypeId::DECIMAL, by_type}, LogicalTypeId::DECIMAL, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, + BindDecimalArgMinMax)); + break; + case ArgMinMaxNullHandling::HANDLE_ARG_NULL: + fun.AddFunction(AggregateFunction({LogicalTypeId::DECIMAL, by_type}, LogicalTypeId::DECIMAL, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, + BindDecimalArgMinMax)); + break; + case ArgMinMaxNullHandling::HANDLE_ANY_NULL: + fun.AddFunction(AggregateFunction({LogicalTypeId::DECIMAL, by_type}, LogicalTypeId::DECIMAL, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, + BindDecimalArgMinMax)); + break; + } } template -void AddGenericArgMinMaxFunction(AggregateFunctionSet &fun) { - fun.AddFunction(GetGenericArgMinMaxFunction()); +void AddGenericArgMinMaxFunction(AggregateFunctionSet &fun, const ArgMinMaxNullHandling null_handling) { + fun.AddFunction(GetGenericArgMinMaxFunction(null_handling)); } -template -void AddArgMinMaxFunctions(AggregateFunctionSet &fun) { - using GENERIC_VECTOR_OP = VectorArgMinMaxBase>; +template +void AddArgMinMaxFunctions(AggregateFunctionSet &fun, const ArgMinMaxNullHandling null_handling) { + using GENERIC_VECTOR_OP = VectorArgMinMaxBase>; #ifndef DUCKDB_SMALLER_BINARY - using OP = ArgMinMaxBase; - using VECTOR_OP = VectorArgMinMaxBase; + using OP = ArgMinMaxBase; + using VECTOR_OP = VectorArgMinMaxBase; #else using OP = GENERIC_VECTOR_OP; using VECTOR_OP = GENERIC_VECTOR_OP; #endif - AddArgMinMaxFunctionBy(fun, LogicalType::INTEGER); - AddArgMinMaxFunctionBy(fun, LogicalType::BIGINT); - AddArgMinMaxFunctionBy(fun, LogicalType::DOUBLE); - AddArgMinMaxFunctionBy(fun, LogicalType::VARCHAR); - AddArgMinMaxFunctionBy(fun, LogicalType::DATE); - AddArgMinMaxFunctionBy(fun, LogicalType::TIMESTAMP); - AddArgMinMaxFunctionBy(fun, LogicalType::TIMESTAMP_TZ); - AddArgMinMaxFunctionBy(fun, LogicalType::BLOB); + AddArgMinMaxFunctionBy(fun, LogicalType::INTEGER, null_handling); + AddArgMinMaxFunctionBy(fun, LogicalType::BIGINT, null_handling); + AddArgMinMaxFunctionBy(fun, LogicalType::DOUBLE, null_handling); + AddArgMinMaxFunctionBy(fun, LogicalType::VARCHAR, null_handling); + AddArgMinMaxFunctionBy(fun, LogicalType::DATE, null_handling); + AddArgMinMaxFunctionBy(fun, LogicalType::TIMESTAMP, null_handling); + AddArgMinMaxFunctionBy(fun, LogicalType::TIMESTAMP_TZ, null_handling); + AddArgMinMaxFunctionBy(fun, LogicalType::BLOB, null_handling); auto by_types = ArgMaxByTypes(); for (const auto &by_type : by_types) { - AddDecimalArgMinMaxFunctionBy(fun, by_type); + AddDecimalArgMinMaxFunctionBy(fun, by_type, null_handling); } - AddVectorArgMinMaxFunctionBy(fun, LogicalType::ANY); + AddVectorArgMinMaxFunctionBy(fun, LogicalType::ANY, null_handling); // we always use LessThan when using sort keys because the ORDER_TYPE takes care of selecting the lowest or highest - AddGenericArgMinMaxFunction(fun); + AddGenericArgMinMaxFunction(fun, null_handling); } //------------------------------------------------------------------------------ @@ -547,6 +644,8 @@ class ArgMinMaxNState { template void ArgMinMaxNUpdate(Vector inputs[], AggregateInputData &aggr_input, idx_t input_count, Vector &state_vector, idx_t count) { + D_ASSERT(aggr_input.bind_data); + const auto &bind_data = aggr_input.bind_data->Cast(); auto &val_vector = inputs[0]; auto &arg_vector = inputs[1]; @@ -560,8 +659,8 @@ void ArgMinMaxNUpdate(Vector inputs[], AggregateInputData &aggr_input, idx_t inp auto val_extra_state = STATE::VAL_TYPE::CreateExtraState(val_vector, count); auto arg_extra_state = STATE::ARG_TYPE::CreateExtraState(arg_vector, count); - STATE::VAL_TYPE::PrepareData(val_vector, count, val_extra_state, val_format); - STATE::ARG_TYPE::PrepareData(arg_vector, count, arg_extra_state, arg_format); + STATE::VAL_TYPE::PrepareData(val_vector, count, val_extra_state, val_format, bind_data.nulls_last); + STATE::ARG_TYPE::PrepareData(arg_vector, count, arg_extra_state, arg_format, bind_data.nulls_last); n_vector.ToUnifiedFormat(count, n_format); state_vector.ToUnifiedFormat(count, state_format); @@ -571,9 +670,16 @@ void ArgMinMaxNUpdate(Vector inputs[], AggregateInputData &aggr_input, idx_t inp for (idx_t i = 0; i < count; i++) { const auto arg_idx = arg_format.sel->get_index(i); const auto val_idx = val_format.sel->get_index(i); - if (!arg_format.validity.RowIsValid(arg_idx) || !val_format.validity.RowIsValid(val_idx)) { + + if (bind_data.null_handling == ArgMinMaxNullHandling::IGNORE_ANY_NULL && + (!arg_format.validity.RowIsValid(arg_idx) || !val_format.validity.RowIsValid(val_idx))) { + continue; + } + if (bind_data.null_handling == ArgMinMaxNullHandling::HANDLE_ARG_NULL && + !val_format.validity.RowIsValid(val_idx)) { continue; } + const auto state_idx = state_format.sel->get_index(i); auto &state = *states[state_idx]; @@ -671,7 +777,77 @@ void SpecializeArgMinMaxNFunction(PhysicalType val_type, PhysicalType arg_type, } } -template +template +void SpecializeArgMinMaxNullNFunction(AggregateFunction &function) { + using STATE = ArgMinMaxNState; + using OP = MinMaxNOperation; + + function.state_size = AggregateFunction::StateSize; + function.initialize = AggregateFunction::StateInitialize; + function.combine = AggregateFunction::StateCombine; + function.destructor = AggregateFunction::StateDestroy; + + function.finalize = MinMaxNOperation::Finalize; + function.update = ArgMinMaxNUpdate; +} + +template +void SpecializeArgMinMaxNullNFunction(PhysicalType arg_type, AggregateFunction &function) { + switch (arg_type) { +#ifndef DUCKDB_SMALLER_BINARY + case PhysicalType::VARCHAR: + SpecializeArgMinMaxNullNFunction(function); + break; + case PhysicalType::INT32: + SpecializeArgMinMaxNullNFunction, COMPARATOR>(function); + break; + case PhysicalType::INT64: + SpecializeArgMinMaxNullNFunction, COMPARATOR>(function); + break; + case PhysicalType::FLOAT: + SpecializeArgMinMaxNullNFunction, COMPARATOR>(function); + break; + case PhysicalType::DOUBLE: + SpecializeArgMinMaxNullNFunction, COMPARATOR>(function); + break; +#endif + default: + SpecializeArgMinMaxNullNFunction(function); + break; + } +} + +template +void SpecializeArgMinMaxNullNFunction(PhysicalType val_type, PhysicalType arg_type, AggregateFunction &function) { + switch (val_type) { +#ifndef DUCKDB_SMALLER_BINARY + case PhysicalType::VARCHAR: + SpecializeArgMinMaxNullNFunction(arg_type, function); + break; + case PhysicalType::INT32: + SpecializeArgMinMaxNullNFunction, NULLS_LAST, COMPARATOR>(arg_type, + function); + break; + case PhysicalType::INT64: + SpecializeArgMinMaxNullNFunction, NULLS_LAST, COMPARATOR>(arg_type, + function); + break; + case PhysicalType::FLOAT: + SpecializeArgMinMaxNullNFunction, NULLS_LAST, COMPARATOR>(arg_type, + function); + break; + case PhysicalType::DOUBLE: + SpecializeArgMinMaxNullNFunction, NULLS_LAST, COMPARATOR>(arg_type, + function); + break; +#endif + default: + SpecializeArgMinMaxNullNFunction(arg_type, function); + break; + } +} + +template unique_ptr ArgMinMaxNBind(ClientContext &context, AggregateFunction &function, vector> &arguments) { for (auto &arg : arguments) { @@ -682,19 +858,24 @@ unique_ptr ArgMinMaxNBind(ClientContext &context, AggregateFunctio const auto val_type = arguments[0]->return_type.InternalType(); const auto arg_type = arguments[1]->return_type.InternalType(); + function.return_type = LogicalType::LIST(arguments[0]->return_type); // Specialize the function based on the input types - SpecializeArgMinMaxNFunction(val_type, arg_type, function); + auto function_data = make_uniq(NULL_HANDLING, NULLS_LAST); + if (NULL_HANDLING != ArgMinMaxNullHandling::IGNORE_ANY_NULL) { + SpecializeArgMinMaxNullNFunction(val_type, arg_type, function); + } else { + SpecializeArgMinMaxNFunction(val_type, arg_type, function); + } - function.return_type = LogicalType::LIST(arguments[0]->return_type); - return nullptr; + return unique_ptr(std::move(function_data)); } -template +template void AddArgMinMaxNFunction(AggregateFunctionSet &set) { AggregateFunction function({LogicalTypeId::ANY, LogicalTypeId::ANY, LogicalType::BIGINT}, LogicalType::LIST(LogicalType::ANY), nullptr, nullptr, nullptr, nullptr, nullptr, - nullptr, ArgMinMaxNBind); + nullptr, ArgMinMaxNBind); return set.AddFunction(function); } @@ -707,27 +888,41 @@ void AddArgMinMaxNFunction(AggregateFunctionSet &set) { AggregateFunctionSet ArgMinFun::GetFunctions() { AggregateFunctionSet fun; - AddArgMinMaxFunctions(fun); - AddArgMinMaxNFunction(fun); + AddArgMinMaxFunctions(fun, ArgMinMaxNullHandling::IGNORE_ANY_NULL); + AddArgMinMaxNFunction(fun); return fun; } AggregateFunctionSet ArgMaxFun::GetFunctions() { AggregateFunctionSet fun; - AddArgMinMaxFunctions(fun); - AddArgMinMaxNFunction(fun); + AddArgMinMaxFunctions(fun, ArgMinMaxNullHandling::IGNORE_ANY_NULL); + AddArgMinMaxNFunction(fun); return fun; } AggregateFunctionSet ArgMinNullFun::GetFunctions() { AggregateFunctionSet fun; - AddArgMinMaxFunctions(fun); + AddArgMinMaxFunctions(fun, ArgMinMaxNullHandling::HANDLE_ARG_NULL); return fun; } AggregateFunctionSet ArgMaxNullFun::GetFunctions() { AggregateFunctionSet fun; - AddArgMinMaxFunctions(fun); + AddArgMinMaxFunctions(fun, ArgMinMaxNullHandling::HANDLE_ARG_NULL); + return fun; +} + +AggregateFunctionSet ArgMinNullsLastFun::GetFunctions() { + AggregateFunctionSet fun; + AddArgMinMaxFunctions(fun, ArgMinMaxNullHandling::HANDLE_ANY_NULL); + AddArgMinMaxNFunction(fun); + return fun; +} + +AggregateFunctionSet ArgMaxNullsLastFun::GetFunctions() { + AggregateFunctionSet fun; + AddArgMinMaxFunctions(fun, ArgMinMaxNullHandling::HANDLE_ANY_NULL); + AddArgMinMaxNFunction(fun); return fun; } diff --git a/extension/core_functions/aggregate/distributive/functions.json b/extension/core_functions/aggregate/distributive/functions.json index d8e614c50243..9dcc4910660f 100644 --- a/extension/core_functions/aggregate/distributive/functions.json +++ b/extension/core_functions/aggregate/distributive/functions.json @@ -21,6 +21,13 @@ "example": "arg_min_null(A, B)", "type": "aggregate_function_set" }, + { + "name": "arg_min_nulls_last", + "parameters": "arg,val,N", + "description": "Finds the rows with N minimum vals, including nulls. Calculates the arg expression at that row.", + "example": "arg_min_null_val(A, B, N)", + "type": "aggregate_function_set" + }, { "name": "arg_max", "parameters": "arg,val", @@ -36,6 +43,13 @@ "example": "arg_max_null(A, B)", "type": "aggregate_function_set" }, + { + "name": "arg_max_nulls_last", + "parameters": "arg,val,N", + "description": "Finds the rows with N maximum vals, including nulls. Calculates the arg expression at that row.", + "example": "arg_min_null_val(A, B, N)", + "type": "aggregate_function_set" + }, { "name": "bit_and", "parameters": "arg", diff --git a/extension/core_functions/function_list.cpp b/extension/core_functions/function_list.cpp index a8ba52658155..810b020ab880 100644 --- a/extension/core_functions/function_list.cpp +++ b/extension/core_functions/function_list.cpp @@ -73,8 +73,10 @@ static const StaticFunctionDefinition core_functions[] = { DUCKDB_AGGREGATE_FUNCTION(ApproxTopKFun), DUCKDB_AGGREGATE_FUNCTION_SET(ArgMaxFun), DUCKDB_AGGREGATE_FUNCTION_SET(ArgMaxNullFun), + DUCKDB_AGGREGATE_FUNCTION_SET(ArgMaxNullsLastFun), DUCKDB_AGGREGATE_FUNCTION_SET(ArgMinFun), DUCKDB_AGGREGATE_FUNCTION_SET(ArgMinNullFun), + DUCKDB_AGGREGATE_FUNCTION_SET(ArgMinNullsLastFun), DUCKDB_AGGREGATE_FUNCTION_SET_ALIAS(ArgmaxFun), DUCKDB_AGGREGATE_FUNCTION_SET_ALIAS(ArgminFun), DUCKDB_AGGREGATE_FUNCTION_ALIAS(ArrayAggFun), diff --git a/extension/core_functions/include/core_functions/aggregate/distributive_functions.hpp b/extension/core_functions/include/core_functions/aggregate/distributive_functions.hpp index 39bc9459c0af..4add0a00db27 100644 --- a/extension/core_functions/include/core_functions/aggregate/distributive_functions.hpp +++ b/extension/core_functions/include/core_functions/aggregate/distributive_functions.hpp @@ -57,6 +57,16 @@ struct ArgMinNullFun { static AggregateFunctionSet GetFunctions(); }; +struct ArgMinNullsLastFun { + static constexpr const char *Name = "arg_min_nulls_last"; + static constexpr const char *Parameters = "arg,val,N"; + static constexpr const char *Description = "Finds the rows with N minimum vals, including nulls. Calculates the arg expression at that row."; + static constexpr const char *Example = "arg_min_null_val(A, B, N)"; + static constexpr const char *Categories = ""; + + static AggregateFunctionSet GetFunctions(); +}; + struct ArgMaxFun { static constexpr const char *Name = "arg_max"; static constexpr const char *Parameters = "arg,val"; @@ -89,6 +99,16 @@ struct ArgMaxNullFun { static AggregateFunctionSet GetFunctions(); }; +struct ArgMaxNullsLastFun { + static constexpr const char *Name = "arg_max_nulls_last"; + static constexpr const char *Parameters = "arg,val,N"; + static constexpr const char *Description = "Finds the rows with N maximum vals, including nulls. Calculates the arg expression at that row."; + static constexpr const char *Example = "arg_min_null_val(A, B, N)"; + static constexpr const char *Categories = ""; + + static AggregateFunctionSet GetFunctions(); +}; + struct BitAndFun { static constexpr const char *Name = "bit_and"; static constexpr const char *Parameters = "arg"; diff --git a/extension/core_functions/scalar/generic/current_setting.cpp b/extension/core_functions/scalar/generic/current_setting.cpp index 4464f0544710..31e1afe1722f 100644 --- a/extension/core_functions/scalar/generic/current_setting.cpp +++ b/extension/core_functions/scalar/generic/current_setting.cpp @@ -53,10 +53,7 @@ unique_ptr CurrentSettingBind(ClientContext &context, ScalarFuncti if (!context.TryGetCurrentSetting(key, val)) { auto extension_name = Catalog::AutoloadExtensionByConfigName(context, key); // If autoloader didn't throw, the config is now available - if (!context.TryGetCurrentSetting(key, val)) { - throw InternalException("Extension %s did not provide the '%s' config setting", - extension_name.ToStdString(), key); - } + context.TryGetCurrentSetting(key, val); } bound_function.return_type = val.type(); diff --git a/extension/icu/icu-strptime.cpp b/extension/icu/icu-strptime.cpp index d7e1283c9b50..63e383dcd5d1 100644 --- a/extension/icu/icu-strptime.cpp +++ b/extension/icu/icu-strptime.cpp @@ -221,8 +221,9 @@ struct ICUStrptime : public ICUDateFunc { if (!error.empty()) { throw InvalidInputException("Failed to parse format specifier %s: %s", format_string, error); } - // If any format has UTC offsets, then we have to produce TSTZ + // If any format has UTC offsets or names, then we have to produce TSTZ has_tz = has_tz || format.HasFormatSpecifier(StrTimeSpecifier::TZ_NAME); + has_tz = has_tz || format.HasFormatSpecifier(StrTimeSpecifier::UTC_OFFSET); formats.emplace_back(format); } if (has_tz) { diff --git a/extension/icu/third_party/icu/common/putil.cpp b/extension/icu/third_party/icu/common/putil.cpp index c79811499441..0c3fd93765e0 100644 --- a/extension/icu/third_party/icu/common/putil.cpp +++ b/extension/icu/third_party/icu/common/putil.cpp @@ -1090,9 +1090,15 @@ uprv_tzname(int n) if (tzid[0] == ':') { tzid++; } - /* This might be a good Olson ID. */ - skipZoneIDPrefix(&tzid); - return tzid; +#if defined(TZDEFAULT) + if (uprv_strcmp(tzid, TZDEFAULT) != 0) { +#endif + /* This might be a good Olson ID. */ + skipZoneIDPrefix(&tzid); + return tzid; +#if defined(TZDEFAULT) + } +#endif } /* else U_TZNAME will give a better result. */ #endif diff --git a/extension/json/include/json_common.hpp b/extension/json/include/json_common.hpp index 577af8a8ef8b..81bbd6868e5e 100644 --- a/extension/json/include/json_common.hpp +++ b/extension/json/include/json_common.hpp @@ -13,6 +13,7 @@ #include "duckdb/common/operator/string_cast.hpp" #include "duckdb/planner/expression/bound_function_expression.hpp" #include "yyjson.hpp" +#include "duckdb/common/types/blob.hpp" using namespace duckdb_yyjson; // NOLINT @@ -228,11 +229,8 @@ struct JSONCommon { static string FormatParseError(const char *data, idx_t length, yyjson_read_err &error, const string &extra = "") { D_ASSERT(error.code != YYJSON_READ_SUCCESS); - // Go to blob so we can have a better error message for weird strings - auto blob = Value::BLOB(string(data, length)); // Truncate, so we don't print megabytes worth of JSON - string input = blob.ToString(); - input = input.length() > 50 ? string(input.c_str(), 47) + "..." : input; + auto input = length > 50 ? string(data, 47) + "..." : string(data, length); // Have to replace \r, otherwise output is unreadable input = StringUtil::Replace(input, "\r", "\\r"); return StringUtil::Format("Malformed JSON at byte %lld of input: %s. %s Input: \"%s\"", error.pos, error.msg, @@ -353,11 +351,19 @@ struct JSONCommon { template <> inline char *JSONCommon::WriteVal(yyjson_val *val, yyjson_alc *alc, idx_t &len) { - return yyjson_val_write_opts(val, JSONCommon::WRITE_FLAG, alc, reinterpret_cast(&len), nullptr); + size_t len_size_t; + // yyjson_val_write_opts must not throw + auto ret = yyjson_val_write_opts(val, JSONCommon::WRITE_FLAG, alc, &len_size_t, nullptr); + len = len_size_t; + return ret; } template <> inline char *JSONCommon::WriteVal(yyjson_mut_val *val, yyjson_alc *alc, idx_t &len) { - return yyjson_mut_val_write_opts(val, JSONCommon::WRITE_FLAG, alc, reinterpret_cast(&len), nullptr); + size_t len_size_t; + // yyjson_mut_val_write_opts must not throw + auto ret = yyjson_mut_val_write_opts(val, JSONCommon::WRITE_FLAG, alc, &len_size_t, nullptr); + len = len_size_t; + return ret; } struct yyjson_doc_deleter { diff --git a/extension/json/include/json_serializer.hpp b/extension/json/include/json_serializer.hpp index aa17f3ffd16b..e856bff798fb 100644 --- a/extension/json/include/json_serializer.hpp +++ b/extension/json/include/json_serializer.hpp @@ -39,6 +39,18 @@ struct JsonSerializer : Serializer { return serializer.GetRootObject(); } + template + static string SerializeToString(T &value) { + auto doc = yyjson_mut_doc_new(nullptr); + JsonSerializer serializer(doc, false, false, false); + value.Serialize(serializer); + auto result_obj = serializer.GetRootObject(); + idx_t len = 0; + auto data = yyjson_mut_val_write_opts(result_obj, JSONCommon::WRITE_PRETTY_FLAG, nullptr, + reinterpret_cast(&len), nullptr); + return string(data, len); + } + yyjson_mut_val *GetRootObject() { D_ASSERT(stack.size() == 1); // or we forgot to pop somewhere return stack.front(); diff --git a/extension/json/json_functions.cpp b/extension/json/json_functions.cpp index 2d09828c33d7..2d0ef11f5772 100644 --- a/extension/json/json_functions.cpp +++ b/extension/json/json_functions.cpp @@ -394,7 +394,11 @@ void JSONFunctions::RegisterSimpleCastFunctions(ExtensionLoader &loader) { loader.RegisterCastFunction(LogicalType::LIST(LogicalType::JSON()), LogicalTypeId::VARCHAR, CastJSONListToVarchar, json_list_to_varchar_cost); - // VARCHAR to JSON[] (also needs a special case otherwise get a VARCHAR -> VARCHAR[] cast first) + // JSON[] to JSON is allowed implicitly + loader.RegisterCastFunction(LogicalType::LIST(LogicalType::JSON()), LogicalType::JSON(), CastJSONListToVarchar, + 100); + + // VARCHAR to JSON[] (also needs a special case otherwise we get a VARCHAR -> VARCHAR[] cast first) const auto varchar_to_json_list_cost = CastFunctionSet::ImplicitCastCost(db, LogicalType::VARCHAR, LogicalType::LIST(LogicalType::JSON())) - 1; BoundCastInfo varchar_to_json_list_info(CastVarcharToJSONList, nullptr, JSONFunctionLocalState::InitCastLocalState); diff --git a/extension/json/json_functions/copy_json.cpp b/extension/json/json_functions/copy_json.cpp index 564e4aeaa3c8..17630e466e4d 100644 --- a/extension/json/json_functions/copy_json.cpp +++ b/extension/json/json_functions/copy_json.cpp @@ -18,6 +18,13 @@ static void ThrowJSONCopyParameterException(const string &loption) { } static BoundStatement CopyToJSONPlan(Binder &binder, CopyStatement &stmt) { + static const unordered_set SUPPORTED_BASE_OPTIONS { + "compression", "encoding", "use_tmp_file", "overwrite_or_ignore", "overwrite", "append", "filename_pattern", + "file_extension", "per_thread_output", "file_size_bytes", + // "partition_by", unsupported + "return_files", "preserve_order", "return_stats", "write_partition_columns", "write_empty_file", + "hive_file_pattern"}; + auto stmt_copy = stmt.Copy(); auto © = stmt_copy->Cast(); auto &copied_info = *copy.info; @@ -48,9 +55,7 @@ static BoundStatement CopyToJSONPlan(Binder &binder, CopyStatement &stmt) { csv_copy_options["suffix"] = {"\n]\n"}; csv_copy_options["new_line"] = {",\n\t"}; } - } else if (loption == "compression" || loption == "encoding" || loption == "per_thread_output" || - loption == "file_size_bytes" || loption == "use_tmp_file" || loption == "overwrite_or_ignore" || - loption == "filename_pattern" || loption == "file_extension") { + } else if (SUPPORTED_BASE_OPTIONS.find(loption) != SUPPORTED_BASE_OPTIONS.end()) { // We support these base options csv_copy_options.insert(kv); } else { diff --git a/extension/json/json_functions/json_create.cpp b/extension/json/json_functions/json_create.cpp index 4cd00249cc4f..36eacfd1870f 100644 --- a/extension/json/json_functions/json_create.cpp +++ b/extension/json/json_functions/json_create.cpp @@ -111,11 +111,11 @@ static unique_ptr JSONCreateBindParams(ScalarFunction &bound_funct auto &type = arguments[i]->return_type; if (arguments[i]->HasParameter()) { throw ParameterNotResolvedException(); - } else if (type == LogicalTypeId::SQLNULL) { - // This is needed for macro's - bound_function.arguments.push_back(type); } else if (object && i % 2 == 0) { - // Key, must be varchar + if (type != LogicalType::VARCHAR) { + throw BinderException("json_object() keys must be VARCHAR, add an explicit cast to argument \"%s\"", + arguments[i]->GetName()); + } bound_function.arguments.push_back(LogicalType::VARCHAR); } else { // Value, cast to types that we can put in JSON @@ -128,7 +128,7 @@ static unique_ptr JSONCreateBindParams(ScalarFunction &bound_funct static unique_ptr JSONObjectBind(ClientContext &context, ScalarFunction &bound_function, vector> &arguments) { if (arguments.size() % 2 != 0) { - throw InvalidInputException("json_object() requires an even number of arguments"); + throw BinderException("json_object() requires an even number of arguments"); } return JSONCreateBindParams(bound_function, arguments, true); } @@ -141,7 +141,7 @@ static unique_ptr JSONArrayBind(ClientContext &context, ScalarFunc static unique_ptr ToJSONBind(ClientContext &context, ScalarFunction &bound_function, vector> &arguments) { if (arguments.size() != 1) { - throw InvalidInputException("to_json() takes exactly one argument"); + throw BinderException("to_json() takes exactly one argument"); } return JSONCreateBindParams(bound_function, arguments, false); } @@ -149,14 +149,14 @@ static unique_ptr ToJSONBind(ClientContext &context, ScalarFunctio static unique_ptr ArrayToJSONBind(ClientContext &context, ScalarFunction &bound_function, vector> &arguments) { if (arguments.size() != 1) { - throw InvalidInputException("array_to_json() takes exactly one argument"); + throw BinderException("array_to_json() takes exactly one argument"); } auto arg_id = arguments[0]->return_type.id(); if (arguments[0]->HasParameter()) { throw ParameterNotResolvedException(); } if (arg_id != LogicalTypeId::LIST && arg_id != LogicalTypeId::SQLNULL) { - throw InvalidInputException("array_to_json() argument type must be LIST"); + throw BinderException("array_to_json() argument type must be LIST"); } return JSONCreateBindParams(bound_function, arguments, false); } @@ -164,14 +164,14 @@ static unique_ptr ArrayToJSONBind(ClientContext &context, ScalarFu static unique_ptr RowToJSONBind(ClientContext &context, ScalarFunction &bound_function, vector> &arguments) { if (arguments.size() != 1) { - throw InvalidInputException("row_to_json() takes exactly one argument"); + throw BinderException("row_to_json() takes exactly one argument"); } auto arg_id = arguments[0]->return_type.id(); if (arguments[0]->HasParameter()) { throw ParameterNotResolvedException(); } if (arguments[0]->return_type.id() != LogicalTypeId::STRUCT && arg_id != LogicalTypeId::SQLNULL) { - throw InvalidInputException("row_to_json() argument type must be STRUCT"); + throw BinderException("row_to_json() argument type must be STRUCT"); } return JSONCreateBindParams(bound_function, arguments, false); } @@ -616,6 +616,7 @@ static void CreateValues(const StructNames &names, yyjson_mut_doc *doc, yyjson_m case LogicalTypeId::VALIDITY: case LogicalTypeId::TABLE: case LogicalTypeId::LAMBDA: + case LogicalTypeId::GEOMETRY: // TODO! Add support for GEOMETRY throw InternalException("Unsupported type arrived at JSON create function"); } } diff --git a/extension/json/json_functions/json_pretty.cpp b/extension/json/json_functions/json_pretty.cpp index 1fb96081d6b6..66616feb6842 100644 --- a/extension/json/json_functions/json_pretty.cpp +++ b/extension/json/json_functions/json_pretty.cpp @@ -5,9 +5,9 @@ namespace duckdb { //! Pretty Print a given JSON Document string_t PrettyPrint(yyjson_val *val, yyjson_alc *alc, Vector &, ValidityMask &, idx_t) { D_ASSERT(alc); - idx_t len; - auto data = - yyjson_val_write_opts(val, JSONCommon::WRITE_PRETTY_FLAG, alc, reinterpret_cast(&len), nullptr); + size_t len_size_t; + auto data = yyjson_val_write_opts(val, JSONCommon::WRITE_PRETTY_FLAG, alc, &len_size_t, nullptr); + idx_t len = len_size_t; return string_t(data, len); } diff --git a/extension/json/json_functions/json_serialize_plan.cpp b/extension/json/json_functions/json_serialize_plan.cpp index aa4f65e63e90..ed30a9cc202c 100644 --- a/extension/json/json_functions/json_serialize_plan.cpp +++ b/extension/json/json_functions/json_serialize_plan.cpp @@ -162,10 +162,11 @@ static void JsonSerializePlanFunction(DataChunk &args, ExpressionState &state, V yyjson_mut_obj_add_false(doc, result_obj, "error"); yyjson_mut_obj_add_val(doc, result_obj, "plans", plans_arr); - idx_t len; + size_t len_size_t; auto data = yyjson_mut_val_write_opts(result_obj, info.format ? JSONCommon::WRITE_PRETTY_FLAG : JSONCommon::WRITE_FLAG, - alc, reinterpret_cast(&len), nullptr); + alc, &len_size_t, nullptr); + idx_t len = len_size_t; if (data == nullptr) { throw SerializationException( "Failed to serialize json, perhaps the query contains invalid utf8 characters?"); @@ -185,10 +186,11 @@ static void JsonSerializePlanFunction(DataChunk &args, ExpressionState &state, V yyjson_mut_obj_add_strcpy(doc, result_obj, entry.first.c_str(), entry.second.c_str()); } - idx_t len; + size_t len_size_t; auto data = yyjson_mut_val_write_opts(result_obj, info.format ? JSONCommon::WRITE_PRETTY_FLAG : JSONCommon::WRITE_FLAG, - alc, reinterpret_cast(&len), nullptr); + alc, &len_size_t, nullptr); + idx_t len = len_size_t; return StringVector::AddString(result, data, len); } }); diff --git a/extension/json/json_functions/json_serialize_sql.cpp b/extension/json/json_functions/json_serialize_sql.cpp index cb998c757b48..97117c1ce3ef 100644 --- a/extension/json/json_functions/json_serialize_sql.cpp +++ b/extension/json/json_functions/json_serialize_sql.cpp @@ -114,10 +114,11 @@ static void JsonSerializeFunction(DataChunk &args, ExpressionState &state, Vecto yyjson_mut_obj_add_false(doc, result_obj, "error"); yyjson_mut_obj_add_val(doc, result_obj, "statements", statements_arr); - idx_t len; + size_t len_size_t; auto data = yyjson_mut_val_write_opts(result_obj, info.format ? JSONCommon::WRITE_PRETTY_FLAG : JSONCommon::WRITE_FLAG, - alc, reinterpret_cast(&len), nullptr); + alc, &len_size_t, nullptr); + idx_t len = len_size_t; if (data == nullptr) { throw SerializationException( "Failed to serialize json, perhaps the query contains invalid utf8 characters?"); @@ -135,10 +136,11 @@ static void JsonSerializeFunction(DataChunk &args, ExpressionState &state, Vecto yyjson_mut_obj_add_strcpy(doc, result_obj, entry.first.c_str(), entry.second.c_str()); } - idx_t len; + size_t len_size_t; auto data = yyjson_mut_val_write_opts(result_obj, info.format ? JSONCommon::WRITE_PRETTY_FLAG : JSONCommon::WRITE_FLAG, - alc, reinterpret_cast(&len), nullptr); + alc, &len_size_t, nullptr); + idx_t len = len_size_t; return StringVector::AddString(result, data, len); } }); diff --git a/extension/json/json_functions/json_table_in_out.cpp b/extension/json/json_functions/json_table_in_out.cpp index b51e78502943..787e393b957e 100644 --- a/extension/json/json_functions/json_table_in_out.cpp +++ b/extension/json/json_functions/json_table_in_out.cpp @@ -244,7 +244,11 @@ static void InitializeLocalState(JSONTableInOutLocalState &lstate, DataChunk &in // Parse path, default to root if not given Value path_value("$"); if (input.data.size() > 1) { - path_value = ConstantVector::GetData(input.data[1])[0]; + auto &path_vector = input.data[1]; + if (ConstantVector::IsNull(path_vector)) { + return; + } + path_value = ConstantVector::GetData(path_vector)[0]; } if (JSONReadFunctionData::CheckPath(path_value, lstate.path, lstate.len) == JSONCommon::JSONPathType::WILDCARD) { diff --git a/extension/json/json_reader.cpp b/extension/json/json_reader.cpp index b52026a4e840..ad61da7d23c6 100644 --- a/extension/json/json_reader.cpp +++ b/extension/json/json_reader.cpp @@ -184,8 +184,7 @@ void JSONReader::OpenJSONFile() { if (!IsOpen()) { auto &fs = FileSystem::GetFileSystem(context); auto regular_file_handle = fs.OpenFile(file, FileFlags::FILE_FLAGS_READ | options.compression); - file_handle = make_uniq(QueryContext(context), std::move(regular_file_handle), - BufferAllocator::Get(context)); + file_handle = make_uniq(context, std::move(regular_file_handle), BufferAllocator::Get(context)); } Reset(); } diff --git a/extension/parquet/CMakeLists.txt b/extension/parquet/CMakeLists.txt index a8335d89e8c7..23b3ebbdb070 100644 --- a/extension/parquet/CMakeLists.txt +++ b/extension/parquet/CMakeLists.txt @@ -21,9 +21,11 @@ set(PARQUET_EXTENSION_FILES parquet_multi_file_info.cpp parquet_metadata.cpp parquet_reader.cpp + parquet_field_id.cpp parquet_statistics.cpp parquet_timestamp.cpp parquet_writer.cpp + parquet_shredding.cpp serialize_parquet.cpp zstd_file_system.cpp geo_parquet.cpp) diff --git a/extension/parquet/column_writer.cpp b/extension/parquet/column_writer.cpp index ae3ee4f51f46..0220d8b24864 100644 --- a/extension/parquet/column_writer.cpp +++ b/extension/parquet/column_writer.cpp @@ -13,6 +13,7 @@ #include "writer/list_column_writer.hpp" #include "writer/primitive_column_writer.hpp" #include "writer/struct_column_writer.hpp" +#include "writer/variant_column_writer.hpp" #include "writer/templated_column_writer.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/operator/comparison_operators.hpp" @@ -92,6 +93,18 @@ bool ColumnWriterStatistics::MaxIsExact() { return true; } +bool ColumnWriterStatistics::HasGeoStats() { + return false; +} + +optional_ptr ColumnWriterStatistics::GetGeoStats() { + return nullptr; +} + +void ColumnWriterStatistics::WriteGeoStats(duckdb_parquet::GeospatialStatistics &stats) { + D_ASSERT(false); // this should never be called +} + //===--------------------------------------------------------------------===// // ColumnWriter //===--------------------------------------------------------------------===// @@ -169,15 +182,17 @@ void ColumnWriter::CompressPage(MemoryStream &temp_writer, size_t &compressed_si } } -void ColumnWriter::HandleRepeatLevels(ColumnWriterState &state, ColumnWriterState *parent, idx_t count, - idx_t max_repeat) const { +void ColumnWriter::HandleRepeatLevels(ColumnWriterState &state, ColumnWriterState *parent, idx_t count) const { if (!parent) { // no repeat levels without a parent node return; } - while (state.repetition_levels.size() < parent->repetition_levels.size()) { - state.repetition_levels.push_back(parent->repetition_levels[state.repetition_levels.size()]); + if (state.repetition_levels.size() >= parent->repetition_levels.size()) { + return; } + state.repetition_levels.insert(state.repetition_levels.end(), + parent->repetition_levels.begin() + state.repetition_levels.size(), + parent->repetition_levels.end()); } void ColumnWriter::HandleDefineLevels(ColumnWriterState &state, ColumnWriterState *parent, const ValidityMask &validity, @@ -188,98 +203,51 @@ void ColumnWriter::HandleDefineLevels(ColumnWriterState &state, ColumnWriterStat while (state.definition_levels.size() < parent->definition_levels.size()) { idx_t current_index = state.definition_levels.size(); if (parent->definition_levels[current_index] != PARQUET_DEFINE_VALID) { + //! Inherit nulls from parent state.definition_levels.push_back(parent->definition_levels[current_index]); state.parent_null_count++; } else if (validity.RowIsValid(vector_index)) { + //! Produce a non-null define state.definition_levels.push_back(define_value); } else { + //! Produce a null define if (!can_have_nulls) { throw IOException("Parquet writer: map key column is not allowed to contain NULL values"); } state.null_count++; state.definition_levels.push_back(null_value); } + D_ASSERT(parent->is_empty.empty() || current_index < parent->is_empty.size()); if (parent->is_empty.empty() || !parent->is_empty[current_index]) { vector_index++; } } - } else { - // no parent: set definition levels only from this validity mask - if (validity.AllValid()) { - state.definition_levels.insert(state.definition_levels.end(), count, define_value); - } else { - for (idx_t i = 0; i < count; i++) { - const auto is_null = !validity.RowIsValid(i); - state.definition_levels.emplace_back(is_null ? null_value : define_value); - state.null_count += is_null; - } - } - if (!can_have_nulls && state.null_count != 0) { - throw IOException("Parquet writer: map key column is not allowed to contain NULL values"); - } - } -} - -//===--------------------------------------------------------------------===// -// WKB Column Writer -//===--------------------------------------------------------------------===// -// Used to store the metadata for a WKB-encoded geometry column when writing -// GeoParquet files. -class WKBColumnWriterState final : public StandardColumnWriterState { -public: - WKBColumnWriterState(ParquetWriter &writer, duckdb_parquet::RowGroup &row_group, idx_t col_idx) - : StandardColumnWriterState(writer, row_group, col_idx), geo_data(), geo_data_writer(writer.GetContext()) { - } - - GeoParquetColumnMetadata geo_data; - GeoParquetColumnMetadataWriter geo_data_writer; -}; - -class WKBColumnWriter final : public StandardColumnWriter { -public: - WKBColumnWriter(ParquetWriter &writer, const ParquetColumnSchema &column_schema, vector schema_path_p, - bool can_have_nulls, string name) - : StandardColumnWriter(writer, column_schema, std::move(schema_path_p), can_have_nulls), - column_name(std::move(name)) { - - this->writer.GetGeoParquetData().RegisterGeometryColumn(column_name); - } - - unique_ptr InitializeWriteState(duckdb_parquet::RowGroup &row_group) override { - auto result = make_uniq(writer, row_group, row_group.columns.size()); - result->encoding = Encoding::RLE_DICTIONARY; - RegisterToRowGroup(row_group); - return std::move(result); + return; } - void Write(ColumnWriterState &state, Vector &vector, idx_t count) override { - StandardColumnWriter::Write(state, vector, count); - - auto &geo_state = state.Cast(); - geo_state.geo_data_writer.Update(geo_state.geo_data, vector, count); + // no parent: set definition levels only from this validity mask + if (validity.AllValid()) { + state.definition_levels.insert(state.definition_levels.end(), count, define_value); + } else { + for (idx_t i = 0; i < count; i++) { + const auto is_null = !validity.RowIsValid(i); + state.definition_levels.emplace_back(is_null ? null_value : define_value); + state.null_count += is_null; + } } - - void FinalizeWrite(ColumnWriterState &state) override { - StandardColumnWriter::FinalizeWrite(state); - - // Add the geodata object to the writer - const auto &geo_state = state.Cast(); - - // Merge this state's geo column data with the writer's geo column data - writer.GetGeoParquetData().FlushColumnMeta(column_name, geo_state.geo_data); + if (!can_have_nulls && state.null_count != 0) { + throw IOException("Parquet writer: map key column is not allowed to contain NULL values"); } - -private: - string column_name; -}; +} //===--------------------------------------------------------------------===// // Create Column Writer //===--------------------------------------------------------------------===// ParquetColumnSchema ColumnWriter::FillParquetSchema(vector &schemas, - const LogicalType &type, const string &name, - optional_ptr field_ids, idx_t max_repeat, + const LogicalType &type, const string &name, bool allow_geometry, + optional_ptr field_ids, + optional_ptr shredding_types, idx_t max_repeat, idx_t max_define, bool can_have_nulls) { auto null_type = can_have_nulls ? FieldRepetitionType::OPTIONAL : FieldRepetitionType::REQUIRED; if (!can_have_nulls) { @@ -296,6 +264,70 @@ ParquetColumnSchema ColumnWriter::FillParquetSchema(vectorchild_field_ids; } } + optional_ptr shredding_type; + if (shredding_types) { + shredding_type = shredding_types->GetChild(name); + } + + if (type.id() == LogicalTypeId::STRUCT && type.GetAlias() == "PARQUET_VARIANT") { + // variant type + // variants are stored as follows: + // group VARIANT { + // metadata BYTE_ARRAY, + // value BYTE_ARRAY, + // [] + // } + + const bool is_shredded = shredding_type != nullptr; + + child_list_t child_types; + child_types.emplace_back("metadata", LogicalType::BLOB); + child_types.emplace_back("value", LogicalType::BLOB); + if (is_shredded) { + auto &typed_value_type = shredding_type->type; + if (typed_value_type.id() != LogicalTypeId::ANY) { + child_types.emplace_back("typed_value", + VariantColumnWriter::TransformTypedValueRecursive(typed_value_type)); + } + } + + // variant group + duckdb_parquet::SchemaElement top_element; + top_element.repetition_type = null_type; + top_element.num_children = child_types.size(); + top_element.logicalType.__isset.VARIANT = true; + top_element.logicalType.VARIANT.__isset.specification_version = true; + top_element.logicalType.VARIANT.specification_version = 1; + top_element.__isset.logicalType = true; + top_element.__isset.num_children = true; + top_element.__isset.repetition_type = true; + top_element.name = name; + schemas.push_back(std::move(top_element)); + + ParquetColumnSchema variant_column(name, type, max_define, max_repeat, schema_idx, 0); + variant_column.children.reserve(child_types.size()); + for (auto &child_type : child_types) { + auto &child_name = child_type.first; + bool is_optional; + if (child_name == "metadata") { + is_optional = false; + } else if (child_name == "value") { + if (is_shredded) { + //! When shredding the variant, the 'value' becomes optional + is_optional = true; + } else { + is_optional = false; + } + } else { + D_ASSERT(child_name == "typed_value"); + is_optional = true; + } + variant_column.children.emplace_back(FillParquetSchema(schemas, child_type.second, child_type.first, + allow_geometry, child_field_ids, shredding_type, + max_repeat, max_define + 1, is_optional)); + } + return variant_column; + } if (type.id() == LogicalTypeId::STRUCT || type.id() == LogicalTypeId::UNION) { auto &child_types = StructType::GetChildTypes(type); @@ -318,7 +350,8 @@ ParquetColumnSchema ColumnWriter::FillParquetSchema(vectorfield_id; } - ParquetWriter::SetSchemaProperties(type, schema_element); + ParquetWriter::SetSchemaProperties(type, schema_element, allow_geometry); schemas.push_back(std::move(schema_element)); return ParquetColumnSchema(name, type, max_define, max_repeat, schema_idx, 0); } @@ -432,6 +467,17 @@ ColumnWriter::CreateWriterRecursive(ClientContext &context, ParquetWriter &write auto &type = schema.type; auto can_have_nulls = parquet_schemas[schema.schema_index].repetition_type == FieldRepetitionType::OPTIONAL; path_in_schema.push_back(schema.name); + + if (type.id() == LogicalTypeId::STRUCT && type.GetAlias() == "PARQUET_VARIANT") { + vector> child_writers; + child_writers.reserve(schema.children.size()); + for (idx_t i = 0; i < schema.children.size(); i++) { + child_writers.push_back( + CreateWriterRecursive(context, writer, parquet_schemas, schema.children[i], path_in_schema)); + } + return make_uniq(writer, schema, path_in_schema, std::move(child_writers), can_have_nulls); + } + if (type.id() == LogicalTypeId::STRUCT || type.id() == LogicalTypeId::UNION) { // construct the child writers recursively vector> child_writers; @@ -470,9 +516,10 @@ ColumnWriter::CreateWriterRecursive(ClientContext &context, ParquetWriter &write make_uniq(writer, schema, path_in_schema, std::move(child_writers), can_have_nulls); return make_uniq(writer, schema, path_in_schema, std::move(struct_writer), can_have_nulls); } - if (type.id() == LogicalTypeId::BLOB && type.GetAlias() == "WKB_BLOB" && - GeoParquetFileMetadata::IsGeoParquetConversionEnabled(context)) { - return make_uniq(writer, schema, std::move(path_in_schema), can_have_nulls, schema.name); + + if (type.id() == LogicalTypeId::BLOB && type.GetAlias() == "WKB_BLOB") { + return make_uniq>( + writer, schema, std::move(path_in_schema), can_have_nulls); } switch (type.id()) { diff --git a/extension/parquet/decoder/dictionary_decoder.cpp b/extension/parquet/decoder/dictionary_decoder.cpp index dfce2343b66f..7ec8bed74492 100644 --- a/extension/parquet/decoder/dictionary_decoder.cpp +++ b/extension/parquet/decoder/dictionary_decoder.cpp @@ -14,39 +14,36 @@ DictionaryDecoder::DictionaryDecoder(ColumnReader &reader) void DictionaryDecoder::InitializeDictionary(idx_t new_dictionary_size, optional_ptr filter, optional_ptr filter_state, bool has_defines) { - auto old_dict_size = dictionary_size; dictionary_size = new_dictionary_size; filter_result.reset(); filter_count = 0; can_have_nulls = has_defines; - // we use the first value in the dictionary to keep a NULL - if (!dictionary) { - dictionary = make_uniq(reader.Type(), dictionary_size + 1); - } else if (dictionary_size > old_dict_size) { - dictionary->Resize(old_dict_size, dictionary_size + 1); - } - dictionary_id = - reader.reader.GetFileName() + "_" + reader.Schema().name + "_" + std::to_string(reader.chunk_read_offset); + // we use the last entry as a NULL, dictionary vectors don't have a separate validity mask - auto &dict_validity = FlatVector::Validity(*dictionary); - dict_validity.Reset(dictionary_size + 1); + const auto duckdb_dictionary_size = dictionary_size + can_have_nulls; + dictionary = DictionaryVector::CreateReusableDictionary(reader.Type(), duckdb_dictionary_size); + auto &dict_validity = FlatVector::Validity(dictionary->data); + dict_validity.Reset(duckdb_dictionary_size); if (can_have_nulls) { dict_validity.SetInvalid(dictionary_size); } - reader.Plain(reader.block, nullptr, dictionary_size, 0, *dictionary); + // now read the non-NULL values from Parquet + reader.Plain(reader.block, nullptr, dictionary_size, 0, dictionary->data); + + // immediately filter the dictionary, if applicable if (filter && CanFilter(*filter, *filter_state)) { // no filter result yet - apply filter to the dictionary // initialize the filter result - setting everything to false - filter_result = make_unsafe_uniq_array(dictionary_size); + filter_result = make_unsafe_uniq_array(duckdb_dictionary_size); // apply the filter UnifiedVectorFormat vdata; - dictionary->ToUnifiedFormat(dictionary_size, vdata); + dictionary->data.ToUnifiedFormat(duckdb_dictionary_size, vdata); SelectionVector dict_sel; - filter_count = dictionary_size; - ColumnSegment::FilterSelection(dict_sel, *dictionary, vdata, *filter, *filter_state, dictionary_size, - filter_count); + filter_count = duckdb_dictionary_size; + ColumnSegment::FilterSelection(dict_sel, dictionary->data, vdata, *filter, *filter_state, + duckdb_dictionary_size, filter_count); // now set all matching tuples to true for (idx_t i = 0; i < filter_count; i++) { @@ -97,7 +94,8 @@ idx_t DictionaryDecoder::Read(uint8_t *defines, idx_t read_count, Vector &result idx_t valid_count = GetValidValues(defines, read_count, result_offset); if (valid_count == read_count) { // all values are valid - we can directly decompress the offsets into the selection vector - dict_decoder->GetBatch(data_ptr_cast(dictionary_selection_vector.data()), valid_count); + dict_decoder->GetBatch(data_ptr_cast(dictionary_selection_vector.data()), + NumericCast(valid_count)); // we do still need to verify the offsets though uint32_t max_index = 0; for (idx_t idx = 0; idx < valid_count; idx++) { @@ -109,19 +107,18 @@ idx_t DictionaryDecoder::Read(uint8_t *defines, idx_t read_count, Vector &result } else if (valid_count > 0) { // for the valid entries - decode the offsets offset_buffer.resize(reader.reader.allocator, sizeof(uint32_t) * valid_count); - dict_decoder->GetBatch(offset_buffer.ptr, valid_count); + dict_decoder->GetBatch(offset_buffer.ptr, NumericCast(valid_count)); ConvertDictToSelVec(reinterpret_cast(offset_buffer.ptr), valid_sel, valid_count); } #ifdef DEBUG dictionary_selection_vector.Verify(read_count, dictionary_size + can_have_nulls); #endif if (result_offset == 0) { - result.Dictionary(*dictionary, dictionary_size + can_have_nulls, dictionary_selection_vector, read_count); - DictionaryVector::SetDictionaryId(result, dictionary_id); + result.Dictionary(dictionary, dictionary_selection_vector); D_ASSERT(result.GetVectorType() == VectorType::DICTIONARY_VECTOR); } else { D_ASSERT(result.GetVectorType() == VectorType::FLAT_VECTOR); - VectorOperations::Copy(*dictionary, result, dictionary_selection_vector, read_count, 0, result_offset); + VectorOperations::Copy(dictionary->data, result, dictionary_selection_vector, read_count, 0, result_offset); } return valid_count; } @@ -132,7 +129,7 @@ void DictionaryDecoder::Skip(uint8_t *defines, idx_t skip_count) { } idx_t valid_count = reader.GetValidCount(defines, skip_count); // skip past the valid offsets - dict_decoder->Skip(valid_count); + dict_decoder->Skip(NumericCast(valid_count)); } bool DictionaryDecoder::DictionarySupportsFilter(const TableFilter &filter, TableFilterState &filter_state) { diff --git a/extension/parquet/geo_parquet.cpp b/extension/parquet/geo_parquet.cpp index 54fab45b38ae..a2cc2a82168d 100644 --- a/extension/parquet/geo_parquet.cpp +++ b/extension/parquet/geo_parquet.cpp @@ -16,158 +16,6 @@ namespace duckdb { using namespace duckdb_yyjson; // NOLINT -const char *WKBGeometryTypes::ToString(WKBGeometryType type) { - switch (type) { - case WKBGeometryType::POINT: - return "Point"; - case WKBGeometryType::LINESTRING: - return "LineString"; - case WKBGeometryType::POLYGON: - return "Polygon"; - case WKBGeometryType::MULTIPOINT: - return "MultiPoint"; - case WKBGeometryType::MULTILINESTRING: - return "MultiLineString"; - case WKBGeometryType::MULTIPOLYGON: - return "MultiPolygon"; - case WKBGeometryType::GEOMETRYCOLLECTION: - return "GeometryCollection"; - case WKBGeometryType::POINT_Z: - return "Point Z"; - case WKBGeometryType::LINESTRING_Z: - return "LineString Z"; - case WKBGeometryType::POLYGON_Z: - return "Polygon Z"; - case WKBGeometryType::MULTIPOINT_Z: - return "MultiPoint Z"; - case WKBGeometryType::MULTILINESTRING_Z: - return "MultiLineString Z"; - case WKBGeometryType::MULTIPOLYGON_Z: - return "MultiPolygon Z"; - case WKBGeometryType::GEOMETRYCOLLECTION_Z: - return "GeometryCollection Z"; - default: - throw NotImplementedException("Unsupported geometry type"); - } -} - -//------------------------------------------------------------------------------ -// GeoParquetColumnMetadataWriter -//------------------------------------------------------------------------------ -GeoParquetColumnMetadataWriter::GeoParquetColumnMetadataWriter(ClientContext &context) { - executor = make_uniq(context); - - auto &catalog = Catalog::GetSystemCatalog(context); - - // These functions are required to extract the geometry type, ZM flag and bounding box from a WKB blob - auto &type_func_set = catalog.GetEntry(context, DEFAULT_SCHEMA, "st_geometrytype"); - auto &flag_func_set = catalog.GetEntry(context, DEFAULT_SCHEMA, "st_zmflag"); - auto &bbox_func_set = catalog.GetEntry(context, DEFAULT_SCHEMA, "st_extent"); - - auto wkb_type = LogicalType(LogicalTypeId::BLOB); - wkb_type.SetAlias("WKB_BLOB"); - - auto type_func = type_func_set.functions.GetFunctionByArguments(context, {wkb_type}); - auto flag_func = flag_func_set.functions.GetFunctionByArguments(context, {wkb_type}); - auto bbox_func = bbox_func_set.functions.GetFunctionByArguments(context, {wkb_type}); - - auto type_type = LogicalType::UTINYINT; - auto flag_type = flag_func.return_type; - auto bbox_type = bbox_func.return_type; - - vector> type_args; - type_args.push_back(make_uniq(wkb_type, 0)); - - vector> flag_args; - flag_args.push_back(make_uniq(wkb_type, 0)); - - vector> bbox_args; - bbox_args.push_back(make_uniq(wkb_type, 0)); - - type_expr = make_uniq(type_type, type_func, std::move(type_args), nullptr); - flag_expr = make_uniq(flag_type, flag_func, std::move(flag_args), nullptr); - bbox_expr = make_uniq(bbox_type, bbox_func, std::move(bbox_args), nullptr); - - // Add the expressions to the executor - executor->AddExpression(*type_expr); - executor->AddExpression(*flag_expr); - executor->AddExpression(*bbox_expr); - - // Initialize the input and result chunks - // The input chunk should be empty, as we always reference the input vector - input_chunk.InitializeEmpty({wkb_type}); - result_chunk.Initialize(BufferAllocator::Get(context), {type_type, flag_type, bbox_type}); -} - -void GeoParquetColumnMetadataWriter::Update(GeoParquetColumnMetadata &meta, Vector &vector, idx_t count) { - input_chunk.Reset(); - result_chunk.Reset(); - - // Reference the vector - input_chunk.data[0].Reference(vector); - input_chunk.SetCardinality(count); - - // Execute the expression - executor->Execute(input_chunk, result_chunk); - - // The first column is the geometry type - // The second column is the zm flag - // The third column is the bounding box - - UnifiedVectorFormat type_format; - UnifiedVectorFormat flag_format; - UnifiedVectorFormat bbox_format; - - result_chunk.data[0].ToUnifiedFormat(count, type_format); - result_chunk.data[1].ToUnifiedFormat(count, flag_format); - result_chunk.data[2].ToUnifiedFormat(count, bbox_format); - - const auto &bbox_components = StructVector::GetEntries(result_chunk.data[2]); - D_ASSERT(bbox_components.size() == 4); - - UnifiedVectorFormat xmin_format; - UnifiedVectorFormat ymin_format; - UnifiedVectorFormat xmax_format; - UnifiedVectorFormat ymax_format; - - bbox_components[0]->ToUnifiedFormat(count, xmin_format); - bbox_components[1]->ToUnifiedFormat(count, ymin_format); - bbox_components[2]->ToUnifiedFormat(count, xmax_format); - bbox_components[3]->ToUnifiedFormat(count, ymax_format); - - for (idx_t in_idx = 0; in_idx < count; in_idx++) { - const auto type_idx = type_format.sel->get_index(in_idx); - const auto flag_idx = flag_format.sel->get_index(in_idx); - const auto bbox_idx = bbox_format.sel->get_index(in_idx); - - const auto type_valid = type_format.validity.RowIsValid(type_idx); - const auto flag_valid = flag_format.validity.RowIsValid(flag_idx); - const auto bbox_valid = bbox_format.validity.RowIsValid(bbox_idx); - - if (!type_valid || !flag_valid || !bbox_valid) { - continue; - } - - // Update the geometry type - const auto flag = UnifiedVectorFormat::GetData(flag_format)[flag_idx]; - const auto type = UnifiedVectorFormat::GetData(type_format)[type_idx]; - if (flag == 1 || flag == 3) { - // M or ZM - throw InvalidInputException("Geoparquet does not support geometries with M coordinates"); - } - const auto has_z = flag == 2; - auto wkb_type = static_cast((type + 1) + (has_z ? 1000 : 0)); - meta.geometry_types.insert(wkb_type); - - // Update the bounding box - const auto min_x = UnifiedVectorFormat::GetData(xmin_format)[bbox_idx]; - const auto min_y = UnifiedVectorFormat::GetData(ymin_format)[bbox_idx]; - const auto max_x = UnifiedVectorFormat::GetData(xmax_format)[bbox_idx]; - const auto max_y = UnifiedVectorFormat::GetData(ymax_format)[bbox_idx]; - meta.bbox.Combine(min_x, max_x, min_y, max_y); - } -} - //------------------------------------------------------------------------------ // GeoParquetFileMetadata //------------------------------------------------------------------------------ @@ -195,25 +43,20 @@ unique_ptr GeoParquetFileMetadata::TryRead(const duckdb_ throw InvalidInputException("Geoparquet metadata is not an object"); } - auto result = make_uniq(); + // We dont actually care about the version for now, as we only support V1+native + auto result = make_uniq(GeoParquetVersion::BOTH); // Check and parse the version const auto version_val = yyjson_obj_get(root, "version"); if (!yyjson_is_str(version_val)) { throw InvalidInputException("Geoparquet metadata does not have a version"); } - result->version = yyjson_get_str(version_val); - if (StringUtil::StartsWith(result->version, "2")) { - // Guard against a breaking future 2.0 version - throw InvalidInputException("Geoparquet version %s is not supported", result->version); - } - // Check and parse the primary geometry column - const auto primary_geometry_column_val = yyjson_obj_get(root, "primary_column"); - if (!yyjson_is_str(primary_geometry_column_val)) { - throw InvalidInputException("Geoparquet metadata does not have a primary column"); + auto version = yyjson_get_str(version_val); + if (StringUtil::StartsWith(version, "3")) { + // Guard against a breaking future 3.0 version + throw InvalidInputException("Geoparquet version %s is not supported", version); } - result->primary_geometry_column = yyjson_get_str(primary_geometry_column_val); // Check and parse the geometry columns const auto columns_val = yyjson_obj_get(root, "columns"); @@ -285,25 +128,71 @@ unique_ptr GeoParquetFileMetadata::TryRead(const duckdb_ return nullptr; } -void GeoParquetFileMetadata::FlushColumnMeta(const string &column_name, const GeoParquetColumnMetadata &meta) { +void GeoParquetFileMetadata::AddGeoParquetStats(const string &column_name, const LogicalType &type, + const GeometryStatsData &stats) { + // Lock the metadata lock_guard glock(write_lock); - auto &column = geometry_columns[column_name]; + auto it = geometry_columns.find(column_name); + if (it == geometry_columns.end()) { + auto &column = geometry_columns[column_name]; - // Combine the metadata - column.geometry_types.insert(meta.geometry_types.begin(), meta.geometry_types.end()); - column.bbox.Combine(meta.bbox); + column.stats.Merge(stats); + column.insertion_index = geometry_columns.size() - 1; + } else { + it->second.stats.Merge(stats); + } } -void GeoParquetFileMetadata::Write(duckdb_parquet::FileMetaData &file_meta_data) const { +void GeoParquetFileMetadata::Write(duckdb_parquet::FileMetaData &file_meta_data) { + + // GeoParquet does not support M or ZM coordinates. So remove any columns that have them. + unordered_set invalid_columns; + for (auto &column : geometry_columns) { + if (column.second.stats.extent.HasM()) { + invalid_columns.insert(column.first); + } + } + for (auto &col_name : invalid_columns) { + geometry_columns.erase(col_name); + } + // No columns remaining, nothing to write + if (geometry_columns.empty()) { + return; + } + + // Find the primary geometry column + const auto &random_first_column = *geometry_columns.begin(); + auto primary_geometry_column = random_first_column.first; + auto primary_insertion_index = random_first_column.second.insertion_index; + + for (auto &column : geometry_columns) { + if (column.second.insertion_index < primary_insertion_index) { + primary_insertion_index = column.second.insertion_index; + primary_geometry_column = column.first; + } + } yyjson_mut_doc *doc = yyjson_mut_doc_new(nullptr); yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); // Add the version - yyjson_mut_obj_add_strncpy(doc, root, "version", version.c_str(), version.size()); + switch (version) { + case GeoParquetVersion::V1: + case GeoParquetVersion::BOTH: + yyjson_mut_obj_add_strcpy(doc, root, "version", "1.0.0"); + break; + case GeoParquetVersion::V2: + yyjson_mut_obj_add_strcpy(doc, root, "version", "2.0.0"); + break; + case GeoParquetVersion::NONE: + default: + // Should never happen, we should not be writing anything + yyjson_mut_doc_free(doc); + throw InternalException("GeoParquetVersion::NONE should not write metadata"); + } // Add the primary column yyjson_mut_obj_add_strncpy(doc, root, "primary_column", primary_geometry_column.c_str(), @@ -313,18 +202,35 @@ void GeoParquetFileMetadata::Write(duckdb_parquet::FileMetaData &file_meta_data) const auto json_columns = yyjson_mut_obj_add_obj(doc, root, "columns"); for (auto &column : geometry_columns) { + const auto column_json = yyjson_mut_obj_add_obj(doc, json_columns, column.first.c_str()); yyjson_mut_obj_add_str(doc, column_json, "encoding", "WKB"); const auto geometry_types = yyjson_mut_obj_add_arr(doc, column_json, "geometry_types"); - for (auto &geometry_type : column.second.geometry_types) { - const auto type_name = WKBGeometryTypes::ToString(geometry_type); - yyjson_mut_arr_add_str(doc, geometry_types, type_name); + + for (auto &type_name : column.second.stats.types.ToString(false)) { + yyjson_mut_arr_add_strcpy(doc, geometry_types, type_name.c_str()); + } + + const auto &bbox = column.second.stats.extent; + + if (bbox.HasXY()) { + + const auto bbox_arr = yyjson_mut_obj_add_arr(doc, column_json, "bbox"); + + if (!column.second.stats.extent.HasZ()) { + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.x_min); + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.y_min); + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.x_max); + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.y_max); + } else { + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.x_min); + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.y_min); + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.z_min); + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.x_max); + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.y_max); + yyjson_mut_arr_add_real(doc, bbox_arr, bbox.z_max); + } } - const auto bbox = yyjson_mut_obj_add_arr(doc, column_json, "bbox"); - yyjson_mut_arr_add_real(doc, bbox, column.second.bbox.min_x); - yyjson_mut_arr_add_real(doc, bbox, column.second.bbox.min_y); - yyjson_mut_arr_add_real(doc, bbox, column.second.bbox.max_x); - yyjson_mut_arr_add_real(doc, bbox, column.second.bbox.max_y); // If the CRS is present, add it if (!column.second.projjson.empty()) { @@ -366,14 +272,6 @@ bool GeoParquetFileMetadata::IsGeometryColumn(const string &column_name) const { return geometry_columns.find(column_name) != geometry_columns.end(); } -void GeoParquetFileMetadata::RegisterGeometryColumn(const string &column_name) { - lock_guard glock(write_lock); - if (primary_geometry_column.empty()) { - primary_geometry_column = column_name; - } - geometry_columns[column_name] = GeoParquetColumnMetadata(); -} - bool GeoParquetFileMetadata::IsGeoParquetConversionEnabled(const ClientContext &context) { Value geoparquet_enabled; if (!context.TryGetCurrentSetting("enable_geoparquet_conversion", geoparquet_enabled)) { @@ -396,20 +294,19 @@ LogicalType GeoParquetFileMetadata::GeometryType() { return blob_type; } +const unordered_map &GeoParquetFileMetadata::GetColumnMeta() const { + return geometry_columns; +} + unique_ptr GeoParquetFileMetadata::CreateColumnReader(ParquetReader &reader, const ParquetColumnSchema &schema, ClientContext &context) { - D_ASSERT(IsGeometryColumn(schema.name)); - - const auto &column = geometry_columns[schema.name]; - // Get the catalog auto &catalog = Catalog::GetSystemCatalog(context); // WKB encoding - if (schema.children[0].type.id() == LogicalTypeId::BLOB && - column.geometry_encoding == GeoParquetColumnEncoding::WKB) { + if (schema.children[0].type.id() == LogicalTypeId::BLOB) { // Look for a conversion function in the catalog auto &conversion_func_set = catalog.GetEntry(context, DEFAULT_SCHEMA, "st_geomfromwkb"); diff --git a/extension/parquet/include/column_writer.hpp b/extension/parquet/include/column_writer.hpp index 6452af8c18b0..929e94a112bd 100644 --- a/extension/parquet/include/column_writer.hpp +++ b/extension/parquet/include/column_writer.hpp @@ -18,6 +18,7 @@ class ParquetWriter; class ColumnWriterPageState; class PrimitiveColumnWriterState; struct ChildFieldIDs; +struct ShreddingType; class ResizeableBuffer; class ParquetBloomFilter; @@ -27,7 +28,7 @@ class ColumnWriterState { unsafe_vector definition_levels; unsafe_vector repetition_levels; - vector is_empty; + unsafe_vector is_empty; idx_t parent_null_count = 0; idx_t null_count = 0; @@ -71,11 +72,6 @@ class ColumnWriter { bool can_have_nulls); virtual ~ColumnWriter(); - ParquetWriter &writer; - const ParquetColumnSchema &column_schema; - vector schema_path; - bool can_have_nulls; - public: const LogicalType &Type() const { return column_schema.type; @@ -94,9 +90,11 @@ class ColumnWriter { } static ParquetColumnSchema FillParquetSchema(vector &schemas, - const LogicalType &type, const string &name, - optional_ptr field_ids, idx_t max_repeat = 0, - idx_t max_define = 1, bool can_have_nulls = true); + const LogicalType &type, const string &name, bool allow_geometry, + optional_ptr field_ids, + optional_ptr shredding_types, + idx_t max_repeat = 0, idx_t max_define = 1, + bool can_have_nulls = true); //! Create the column writer for a specific type recursively static unique_ptr CreateWriterRecursive(ClientContext &context, ParquetWriter &writer, const vector &parquet_schemas, @@ -129,10 +127,19 @@ class ColumnWriter { protected: void HandleDefineLevels(ColumnWriterState &state, ColumnWriterState *parent, const ValidityMask &validity, const idx_t count, const uint16_t define_value, const uint16_t null_value) const; - void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count, idx_t max_repeat) const; + void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count) const; void CompressPage(MemoryStream &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data, AllocatedData &compressed_buf); + +public: + ParquetWriter &writer; + const ParquetColumnSchema &column_schema; + vector schema_path; + bool can_have_nulls; + +protected: + vector> child_writers; }; } // namespace duckdb diff --git a/extension/parquet/include/decoder/dictionary_decoder.hpp b/extension/parquet/include/decoder/dictionary_decoder.hpp index c012a82dca89..de75b045eea9 100644 --- a/extension/parquet/include/decoder/dictionary_decoder.hpp +++ b/extension/parquet/include/decoder/dictionary_decoder.hpp @@ -47,11 +47,10 @@ class DictionaryDecoder { SelectionVector valid_sel; SelectionVector dictionary_selection_vector; idx_t dictionary_size; - unique_ptr dictionary; + buffer_ptr dictionary; unsafe_unique_array filter_result; idx_t filter_count; bool can_have_nulls; - string dictionary_id; }; } // namespace duckdb diff --git a/extension/parquet/include/geo_parquet.hpp b/extension/parquet/include/geo_parquet.hpp index 5980fff342e0..0e236c73aa24 100644 --- a/extension/parquet/include/geo_parquet.hpp +++ b/extension/parquet/include/geo_parquet.hpp @@ -16,63 +16,8 @@ #include "parquet_types.h" namespace duckdb { -struct ParquetColumnSchema; - -enum class WKBGeometryType : uint16_t { - POINT = 1, - LINESTRING = 2, - POLYGON = 3, - MULTIPOINT = 4, - MULTILINESTRING = 5, - MULTIPOLYGON = 6, - GEOMETRYCOLLECTION = 7, - - POINT_Z = 1001, - LINESTRING_Z = 1002, - POLYGON_Z = 1003, - MULTIPOINT_Z = 1004, - MULTILINESTRING_Z = 1005, - MULTIPOLYGON_Z = 1006, - GEOMETRYCOLLECTION_Z = 1007, -}; - -struct WKBGeometryTypes { - static const char *ToString(WKBGeometryType type); -}; - -struct GeometryBounds { - double min_x = NumericLimits::Maximum(); - double max_x = NumericLimits::Minimum(); - double min_y = NumericLimits::Maximum(); - double max_y = NumericLimits::Minimum(); - - GeometryBounds() = default; - - void Combine(const GeometryBounds &other) { - min_x = std::min(min_x, other.min_x); - max_x = std::max(max_x, other.max_x); - min_y = std::min(min_y, other.min_y); - max_y = std::max(max_y, other.max_y); - } - - void Combine(const double &x, const double &y) { - min_x = std::min(min_x, x); - max_x = std::max(max_x, x); - min_y = std::min(min_y, y); - max_y = std::max(max_y, y); - } - - void Combine(const double &min_x, const double &max_x, const double &min_y, const double &max_y) { - this->min_x = std::min(this->min_x, min_x); - this->max_x = std::max(this->max_x, max_x); - this->min_y = std::min(this->min_y, min_y); - this->max_y = std::max(this->max_y, max_y); - } -}; -//------------------------------------------------------------------------------ -// GeoParquetMetadata -//------------------------------------------------------------------------------ +struct ParquetColumnSchema; class ParquetReader; class ColumnReader; class ClientContext; @@ -88,60 +33,70 @@ enum class GeoParquetColumnEncoding : uint8_t { MULTIPOLYGON, }; +enum class GeoParquetVersion : uint8_t { + // Write GeoParquet 1.0 metadata + // GeoParquet 1.0 has the widest support among readers and writers + V1, + + // Write GeoParquet 2.0 + // The GeoParquet 2.0 options is identical to GeoParquet 1.0 except the underlying storage + // of spatial columns is Parquet native geometry, where the Parquet writer will include + // native statistics according to the underlying Parquet options. Compared to 'BOTH', this will + // actually write the metadata as containing GeoParquet version 2.0.0 + // However, V2 isnt standardized yet, so this option is still a bit experimental + V2, + + // Write GeoParquet 1.0 metadata, with native Parquet geometry types + // This is a bit of a hold-over option for compatibility with systems that + // reject GeoParquet 2.0 metadata, but can read Parquet native geometry types as they simply ignore the extra + // logical type. DuckDB v1.4.0 falls into this category. + BOTH, + + // Do not write GeoParquet metadata + // This option suppresses GeoParquet metadata; however, spatial types will be written as + // Parquet native Geometry/Geography. + NONE, +}; + struct GeoParquetColumnMetadata { // The encoding of the geometry column GeoParquetColumnEncoding geometry_encoding; - // The geometry types that are present in the column - set geometry_types; - - // The bounds of the geometry column - GeometryBounds bbox; + // The statistics of the geometry column + GeometryStatsData stats; // The crs of the geometry column (if any) in PROJJSON format string projjson; -}; - -class GeoParquetColumnMetadataWriter { - unique_ptr executor; - DataChunk input_chunk; - DataChunk result_chunk; - unique_ptr type_expr; - unique_ptr flag_expr; - unique_ptr bbox_expr; - -public: - explicit GeoParquetColumnMetadataWriter(ClientContext &context); - void Update(GeoParquetColumnMetadata &meta, Vector &vector, idx_t count); + // Used to track the "primary" geometry column (if any) + idx_t insertion_index = 0; }; class GeoParquetFileMetadata { public: + explicit GeoParquetFileMetadata(GeoParquetVersion geo_parquet_version) : version(geo_parquet_version) { + } + void AddGeoParquetStats(const string &column_name, const LogicalType &type, const GeometryStatsData &stats); + void Write(duckdb_parquet::FileMetaData &file_meta_data); + // Try to read GeoParquet metadata. Returns nullptr if not found, invalid or the required spatial extension is not // available. - static unique_ptr TryRead(const duckdb_parquet::FileMetaData &file_meta_data, const ClientContext &context); - void Write(duckdb_parquet::FileMetaData &file_meta_data) const; - - void FlushColumnMeta(const string &column_name, const GeoParquetColumnMetadata &meta); const unordered_map &GetColumnMeta() const; - unique_ptr CreateColumnReader(ParquetReader &reader, const ParquetColumnSchema &schema, - ClientContext &context); + static unique_ptr CreateColumnReader(ParquetReader &reader, const ParquetColumnSchema &schema, + ClientContext &context); bool IsGeometryColumn(const string &column_name) const; - void RegisterGeometryColumn(const string &column_name); static bool IsGeoParquetConversionEnabled(const ClientContext &context); static LogicalType GeometryType(); private: mutex write_lock; - string version = "1.1.0"; - string primary_geometry_column; unordered_map geometry_columns; + GeoParquetVersion version; }; } // namespace duckdb diff --git a/extension/parquet/include/parquet.json b/extension/parquet/include/parquet.json index d4f6a77c69e8..67ffecb784a7 100644 --- a/extension/parquet/include/parquet.json +++ b/extension/parquet/include/parquet.json @@ -116,7 +116,7 @@ { "class": "FieldID", "includes": [ - "parquet_writer.hpp" + "parquet_field_id.hpp" ], "members": [ { @@ -140,7 +140,7 @@ { "class": "ChildFieldIDs", "includes": [ - "parquet_writer.hpp" + "parquet_field_id.hpp" ], "members": [ { @@ -152,5 +152,45 @@ } ], "pointer_type": "none" + }, + { + "class": "ShreddingType", + "includes": [ + "parquet_shredding.hpp" + ], + "members": [ + { + "id": 100, + "name": "set", + "type": "bool" + }, + { + "id": 101, + "name": "type", + "type": "LogicalType" + }, + { + "id": 102, + "name": "children", + "type": "ChildShreddingTypes" + } + ], + "pointer_type": "none" + }, + { + "class": "ChildShreddingTypes", + "includes": [ + "parquet_shredding.hpp" + ], + "members": [ + { + "id": 100, + "name": "types", + "type": "case_insensitive_map_t", + "serialize_property": "types.operator*()", + "deserialize_property": "types.operator*()" + } + ], + "pointer_type": "none" } ] \ No newline at end of file diff --git a/extension/parquet/include/parquet_dbp_decoder.hpp b/extension/parquet/include/parquet_dbp_decoder.hpp index 31fb26cc9c28..775160215c87 100644 --- a/extension/parquet/include/parquet_dbp_decoder.hpp +++ b/extension/parquet/include/parquet_dbp_decoder.hpp @@ -18,7 +18,7 @@ class DbpDecoder { : buffer_(buffer, buffer_len), // block_size_in_values(ParquetDecodeUtils::VarintDecode(buffer_)), - number_of_miniblocks_per_block(ParquetDecodeUtils::VarintDecode(buffer_)), + number_of_miniblocks_per_block(DecodeNumberOfMiniblocksPerBlock(buffer_)), number_of_values_in_a_miniblock(block_size_in_values / number_of_miniblocks_per_block), total_value_count(ParquetDecodeUtils::VarintDecode(buffer_)), previous_value(ParquetDecodeUtils::ZigzagToInt(ParquetDecodeUtils::VarintDecode(buffer_))), @@ -31,7 +31,7 @@ class DbpDecoder { number_of_values_in_a_miniblock % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE == 0)) { throw InvalidInputException("Parquet file has invalid block sizes for DELTA_BINARY_PACKED"); } - }; + } ByteBuffer BufferPtr() const { return buffer_; @@ -68,6 +68,15 @@ class DbpDecoder { } private: + static idx_t DecodeNumberOfMiniblocksPerBlock(ByteBuffer &buffer) { + auto res = ParquetDecodeUtils::VarintDecode(buffer); + if (res == 0) { + throw InvalidInputException( + "Parquet file has invalid number of miniblocks per block for DELTA_BINARY_PACKED"); + } + return res; + } + template void GetBatchInternal(const data_ptr_t target_values_ptr, const idx_t batch_size) { if (batch_size == 0) { diff --git a/extension/parquet/include/parquet_field_id.hpp b/extension/parquet/include/parquet_field_id.hpp new file mode 100644 index 000000000000..9d5dd754c0ba --- /dev/null +++ b/extension/parquet/include/parquet_field_id.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include "duckdb/common/serializer/buffered_file_writer.hpp" +#include "duckdb/common/case_insensitive_map.hpp" + +namespace duckdb { + +struct FieldID; +struct ChildFieldIDs { + ChildFieldIDs(); + ChildFieldIDs Copy() const; + unique_ptr> ids; + + void Serialize(Serializer &serializer) const; + static ChildFieldIDs Deserialize(Deserializer &source); +}; + +struct FieldID { +public: + static constexpr const auto DUCKDB_FIELD_ID = "__duckdb_field_id"; + FieldID(); + explicit FieldID(int32_t field_id); + FieldID Copy() const; + bool set; + int32_t field_id; + ChildFieldIDs child_field_ids; + + void Serialize(Serializer &serializer) const; + static FieldID Deserialize(Deserializer &source); + +public: + static void GenerateFieldIDs(ChildFieldIDs &field_ids, idx_t &field_id, const vector &names, + const vector &sql_types); + static void GetFieldIDs(const Value &field_ids_value, ChildFieldIDs &field_ids, + unordered_set &unique_field_ids, + const case_insensitive_map_t &name_to_type_map); +}; + +} // namespace duckdb diff --git a/extension/parquet/include/parquet_reader.hpp b/extension/parquet/include/parquet_reader.hpp index 76b8158e6cce..6c910fb28cde 100644 --- a/extension/parquet/include/parquet_reader.hpp +++ b/extension/parquet/include/parquet_reader.hpp @@ -105,7 +105,6 @@ struct ParquetOptions { explicit ParquetOptions(ClientContext &context); bool binary_as_string = false; - bool variant_legacy_encoding = false; bool file_row_number = false; shared_ptr encryption_config; bool debug_use_openssl = true; @@ -211,9 +210,9 @@ class ParquetReader : public BaseFileReader { void InitializeSchema(ClientContext &context); bool ScanInternal(ClientContext &context, ParquetReaderScanState &state, DataChunk &output); //! Parse the schema of the file - unique_ptr ParseSchema(); + unique_ptr ParseSchema(ClientContext &context); ParquetColumnSchema ParseSchemaRecursive(idx_t depth, idx_t max_define, idx_t max_repeat, idx_t &next_schema_idx, - idx_t &next_file_idx); + idx_t &next_file_idx, ClientContext &context); unique_ptr CreateReader(ClientContext &context); diff --git a/extension/parquet/include/parquet_shredding.hpp b/extension/parquet/include/parquet_shredding.hpp new file mode 100644 index 000000000000..f43cbc42c819 --- /dev/null +++ b/extension/parquet/include/parquet_shredding.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include "duckdb/common/serializer/buffered_file_writer.hpp" +#include "duckdb/common/case_insensitive_map.hpp" +#include "duckdb/common/types/variant.hpp" + +namespace duckdb { + +struct ShreddingType; + +struct ChildShreddingTypes { +public: + ChildShreddingTypes(); + +public: + ChildShreddingTypes Copy() const; + +public: + void Serialize(Serializer &serializer) const; + static ChildShreddingTypes Deserialize(Deserializer &source); + +public: + unique_ptr> types; +}; + +struct ShreddingType { +public: + ShreddingType(); + explicit ShreddingType(const LogicalType &type); + +public: + ShreddingType Copy() const; + +public: + void Serialize(Serializer &serializer) const; + static ShreddingType Deserialize(Deserializer &source); + +public: + static ShreddingType GetShreddingTypes(const Value &val); + void AddChild(const string &name, ShreddingType &&child); + optional_ptr GetChild(const string &name) const; + +public: + bool set = false; + LogicalType type; + ChildShreddingTypes children; +}; + +} // namespace duckdb diff --git a/extension/parquet/include/parquet_writer.hpp b/extension/parquet/include/parquet_writer.hpp index a2bfc3a80e76..46661215ffc9 100644 --- a/extension/parquet/include/parquet_writer.hpp +++ b/extension/parquet/include/parquet_writer.hpp @@ -21,6 +21,8 @@ #include "parquet_statistics.hpp" #include "column_writer.hpp" +#include "parquet_field_id.hpp" +#include "parquet_shredding.hpp" #include "parquet_types.h" #include "geo_parquet.hpp" #include "writer/parquet_write_stats.hpp" @@ -43,29 +45,6 @@ struct PreparedRowGroup { vector> states; }; -struct FieldID; -struct ChildFieldIDs { - ChildFieldIDs(); - ChildFieldIDs Copy() const; - unique_ptr> ids; - - void Serialize(Serializer &serializer) const; - static ChildFieldIDs Deserialize(Deserializer &source); -}; - -struct FieldID { - static constexpr const auto DUCKDB_FIELD_ID = "__duckdb_field_id"; - FieldID(); - explicit FieldID(int32_t field_id); - FieldID Copy() const; - bool set; - int32_t field_id; - ChildFieldIDs child_field_ids; - - void Serialize(Serializer &serializer) const; - static FieldID Deserialize(Deserializer &source); -}; - struct ParquetBloomFilterEntry { unique_ptr bloom_filter; idx_t row_group_idx; @@ -81,11 +60,11 @@ class ParquetWriter { public: ParquetWriter(ClientContext &context, FileSystem &fs, string file_name, vector types, vector names, duckdb_parquet::CompressionCodec::type codec, ChildFieldIDs field_ids, - const vector> &kv_metadata, + ShreddingType shredding_types, const vector> &kv_metadata, shared_ptr encryption_config, optional_idx dictionary_size_limit, idx_t string_dictionary_page_size_limit, bool enable_bloom_filters, double bloom_filter_false_positive_ratio, int64_t compression_level, bool debug_use_openssl, - ParquetVersion parquet_version); + ParquetVersion parquet_version, GeoParquetVersion geoparquet_version); ~ParquetWriter(); public: @@ -95,7 +74,8 @@ class ParquetWriter { void Finalize(); static duckdb_parquet::Type::type DuckDBTypeToParquetType(const LogicalType &duckdb_type); - static void SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::SchemaElement &schema_ele); + static void SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::SchemaElement &schema_ele, + bool allow_geometry); ClientContext &GetContext() { return context; @@ -139,6 +119,9 @@ class ParquetWriter { ParquetVersion GetParquetVersion() const { return parquet_version; } + GeoParquetVersion GetGeoParquetVersion() const { + return geoparquet_version; + } const string &GetFileName() const { return file_name; } @@ -166,6 +149,7 @@ class ParquetWriter { vector column_names; duckdb_parquet::CompressionCodec::type codec; ChildFieldIDs field_ids; + ShreddingType shredding_types; shared_ptr encryption_config; optional_idx dictionary_size_limit; idx_t string_dictionary_page_size_limit; @@ -175,6 +159,7 @@ class ParquetWriter { bool debug_use_openssl; shared_ptr encryption_util; ParquetVersion parquet_version; + GeoParquetVersion geoparquet_version; vector column_schemas; unique_ptr writer; diff --git a/extension/parquet/include/reader/string_column_reader.hpp b/extension/parquet/include/reader/string_column_reader.hpp index 4bc19516a9c9..bfc0692af873 100644 --- a/extension/parquet/include/reader/string_column_reader.hpp +++ b/extension/parquet/include/reader/string_column_reader.hpp @@ -14,12 +14,25 @@ namespace duckdb { class StringColumnReader : public ColumnReader { + enum class StringColumnType : uint8_t { VARCHAR, JSON, OTHER }; + + static StringColumnType GetStringColumnType(const LogicalType &type) { + if (type.IsJSONType()) { + return StringColumnType::JSON; + } + if (type.id() == LogicalTypeId::VARCHAR) { + return StringColumnType::VARCHAR; + } + return StringColumnType::OTHER; + } + public: static constexpr const PhysicalType TYPE = PhysicalType::VARCHAR; public: StringColumnReader(ParquetReader &reader, const ParquetColumnSchema &schema); idx_t fixed_width_string_length; + const StringColumnType string_column_type; public: static void VerifyString(const char *str_data, uint32_t str_len, const bool isVarchar); diff --git a/extension/parquet/include/reader/variant/variant_binary_decoder.hpp b/extension/parquet/include/reader/variant/variant_binary_decoder.hpp index a7c7177099c0..17efcd46e075 100644 --- a/extension/parquet/include/reader/variant/variant_binary_decoder.hpp +++ b/extension/parquet/include/reader/variant/variant_binary_decoder.hpp @@ -137,10 +137,8 @@ class VariantBinaryDecoder { static VariantValue Decode(const VariantMetadata &metadata, const_data_ptr_t data); public: - static VariantValue PrimitiveTypeDecode(const VariantMetadata &metadata, const VariantValueMetadata &value_metadata, - const_data_ptr_t data); - static VariantValue ShortStringDecode(const VariantMetadata &metadata, const VariantValueMetadata &value_metadata, - const_data_ptr_t data); + static VariantValue PrimitiveTypeDecode(const VariantValueMetadata &value_metadata, const_data_ptr_t data); + static VariantValue ShortStringDecode(const VariantValueMetadata &value_metadata, const_data_ptr_t data); static VariantValue ObjectDecode(const VariantMetadata &metadata, const VariantValueMetadata &value_metadata, const_data_ptr_t data); static VariantValue ArrayDecode(const VariantMetadata &metadata, const VariantValueMetadata &value_metadata, diff --git a/extension/parquet/include/reader/variant/variant_shredded_conversion.hpp b/extension/parquet/include/reader/variant/variant_shredded_conversion.hpp index 27ece7d70683..bbcf71792db5 100644 --- a/extension/parquet/include/reader/variant/variant_shredded_conversion.hpp +++ b/extension/parquet/include/reader/variant/variant_shredded_conversion.hpp @@ -11,13 +11,14 @@ class VariantShreddedConversion { public: static vector Convert(Vector &metadata, Vector &group, idx_t offset, idx_t length, idx_t total_size, - bool is_field = false); + bool is_field); static vector ConvertShreddedLeaf(Vector &metadata, Vector &value, Vector &typed_value, idx_t offset, - idx_t length, idx_t total_size); + idx_t length, idx_t total_size, const bool is_field); static vector ConvertShreddedArray(Vector &metadata, Vector &value, Vector &typed_value, idx_t offset, - idx_t length, idx_t total_size); + idx_t length, idx_t total_size, const bool is_field); static vector ConvertShreddedObject(Vector &metadata, Vector &value, Vector &typed_value, - idx_t offset, idx_t length, idx_t total_size); + idx_t offset, idx_t length, idx_t total_size, + const bool is_field); }; } // namespace duckdb diff --git a/extension/parquet/include/reader/variant/variant_value.hpp b/extension/parquet/include/reader/variant/variant_value.hpp index a4c38ede77d7..9d3f502c324b 100644 --- a/extension/parquet/include/reader/variant/variant_value.hpp +++ b/extension/parquet/include/reader/variant/variant_value.hpp @@ -42,6 +42,7 @@ struct VariantValue { public: yyjson_mut_val *ToJSON(ClientContext &context, yyjson_mut_doc *doc) const; + static void ToVARIANT(vector &input, Vector &result); public: VariantValueType value_type; diff --git a/extension/parquet/include/reader/variant_column_reader.hpp b/extension/parquet/include/reader/variant_column_reader.hpp index 78670b14a065..69b429626a6f 100644 --- a/extension/parquet/include/reader/variant_column_reader.hpp +++ b/extension/parquet/include/reader/variant_column_reader.hpp @@ -15,7 +15,7 @@ namespace duckdb { class VariantColumnReader : public ColumnReader { public: - static constexpr const PhysicalType TYPE = PhysicalType::VARCHAR; + static constexpr const PhysicalType TYPE = PhysicalType::STRUCT; public: VariantColumnReader(ClientContext &context, ParquetReader &reader, const ParquetColumnSchema &schema, diff --git a/extension/parquet/include/writer/array_column_writer.hpp b/extension/parquet/include/writer/array_column_writer.hpp index 630bfd17f106..1ebb16c04036 100644 --- a/extension/parquet/include/writer/array_column_writer.hpp +++ b/extension/parquet/include/writer/array_column_writer.hpp @@ -25,6 +25,10 @@ class ArrayColumnWriter : public ListColumnWriter { void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count, bool vector_can_span_multiple_pages) override; void Write(ColumnWriterState &state, Vector &vector, idx_t count) override; + +protected: + void WriteArrayState(ListColumnWriterState &state, idx_t array_size, uint16_t first_repeat_level, + idx_t define_value, const bool is_empty = false); }; } // namespace duckdb diff --git a/extension/parquet/include/writer/list_column_writer.hpp b/extension/parquet/include/writer/list_column_writer.hpp index f1070b0f1975..902d3001ce89 100644 --- a/extension/parquet/include/writer/list_column_writer.hpp +++ b/extension/parquet/include/writer/list_column_writer.hpp @@ -28,13 +28,11 @@ class ListColumnWriter : public ColumnWriter { public: ListColumnWriter(ParquetWriter &writer, const ParquetColumnSchema &column_schema, vector schema_path_p, unique_ptr child_writer_p, bool can_have_nulls) - : ColumnWriter(writer, column_schema, std::move(schema_path_p), can_have_nulls), - child_writer(std::move(child_writer_p)) { + : ColumnWriter(writer, column_schema, std::move(schema_path_p), can_have_nulls) { + child_writers.push_back(std::move(child_writer_p)); } ~ListColumnWriter() override = default; - unique_ptr child_writer; - public: unique_ptr InitializeWriteState(duckdb_parquet::RowGroup &row_group) override; bool HasAnalyze() override; @@ -46,6 +44,9 @@ class ListColumnWriter : public ColumnWriter { void BeginWrite(ColumnWriterState &state) override; void Write(ColumnWriterState &state, Vector &vector, idx_t count) override; void FinalizeWrite(ColumnWriterState &state) override; + +protected: + ColumnWriter &GetChildWriter(); }; } // namespace duckdb diff --git a/extension/parquet/include/writer/parquet_write_operators.hpp b/extension/parquet/include/writer/parquet_write_operators.hpp index 1e60ddc1eda0..95bf21e2a7a0 100644 --- a/extension/parquet/include/writer/parquet_write_operators.hpp +++ b/extension/parquet/include/writer/parquet_write_operators.hpp @@ -109,6 +109,50 @@ struct ParquetTimestampSOperator : public ParquetCastOperator { } }; +// We will need a different operator for GEOGRAPHY later, so we define a base geo operator +struct ParquetBaseGeoOperator : public BaseParquetOperator { + template + static TGT Operation(SRC input) { + return input; + } + + template + static void HandleStats(ColumnWriterStatistics *stats, TGT target_value) { + auto &geo_stats = stats->Cast(); + geo_stats.Update(target_value); + } + + template + static void WriteToStream(const TGT &target_value, WriteStream &ser) { + ser.Write(target_value.GetSize()); + ser.WriteData(const_data_ptr_cast(target_value.GetData()), target_value.GetSize()); + } + + template + static idx_t WriteSize(const TGT &target_value) { + return sizeof(uint32_t) + target_value.GetSize(); + } + + template + static uint64_t XXHash64(const TGT &target_value) { + return duckdb_zstd::XXH64(target_value.GetData(), target_value.GetSize(), 0); + } + + template + static idx_t GetRowSize(const Vector &vector, idx_t index) { + // This needs to add the 4 bytes (just like WriteSize) otherwise we underestimate and we have to realloc + // This seriously harms performance, mostly by making it very inconsistent (see internal issue #4990) + return sizeof(uint32_t) + FlatVector::GetData(vector)[index].GetSize(); + } +}; + +struct ParquetGeometryOperator : public ParquetBaseGeoOperator { + template + static unique_ptr InitializeStats() { + return make_uniq(); + } +}; + struct ParquetBaseStringOperator : public BaseParquetOperator { template static TGT Operation(SRC input) { diff --git a/extension/parquet/include/writer/parquet_write_stats.hpp b/extension/parquet/include/writer/parquet_write_stats.hpp index d0635ff7780b..a74845ad2742 100644 --- a/extension/parquet/include/writer/parquet_write_stats.hpp +++ b/extension/parquet/include/writer/parquet_write_stats.hpp @@ -9,7 +9,7 @@ #pragma once #include "column_writer.hpp" -#include "writer/parquet_write_stats.hpp" +#include "geo_parquet.hpp" namespace duckdb { @@ -27,6 +27,10 @@ class ColumnWriterStatistics { virtual bool MinIsExact(); virtual bool MaxIsExact(); + virtual bool HasGeoStats(); + virtual optional_ptr GetGeoStats(); + virtual void WriteGeoStats(duckdb_parquet::GeospatialStatistics &stats); + public: template TARGET &Cast() { @@ -248,4 +252,54 @@ class UUIDStatisticsState : public ColumnWriterStatistics { } }; +class GeoStatisticsState final : public ColumnWriterStatistics { +public: + explicit GeoStatisticsState() : has_stats(false) { + geo_stats.SetEmpty(); + } + + bool has_stats; + GeometryStatsData geo_stats; + +public: + void Update(const string_t &val) { + geo_stats.Update(val); + has_stats = true; + } + bool HasGeoStats() override { + return has_stats; + } + optional_ptr GetGeoStats() override { + return geo_stats; + } + void WriteGeoStats(duckdb_parquet::GeospatialStatistics &stats) override { + const auto &types = geo_stats.types; + const auto &bbox = geo_stats.extent; + + if (bbox.HasXY()) { + stats.__isset.bbox = true; + stats.bbox.xmin = bbox.x_min; + stats.bbox.xmax = bbox.x_max; + stats.bbox.ymin = bbox.y_min; + stats.bbox.ymax = bbox.y_max; + + if (bbox.HasZ()) { + stats.bbox.__isset.zmin = true; + stats.bbox.__isset.zmax = true; + stats.bbox.zmin = bbox.z_min; + stats.bbox.zmax = bbox.z_max; + } + if (bbox.HasM()) { + stats.bbox.__isset.mmin = true; + stats.bbox.__isset.mmax = true; + stats.bbox.mmin = bbox.m_min; + stats.bbox.mmax = bbox.m_max; + } + } + + stats.__isset.geospatial_types = true; + stats.geospatial_types = types.ToWKBList(); + } +}; + } // namespace duckdb diff --git a/extension/parquet/include/writer/struct_column_writer.hpp b/extension/parquet/include/writer/struct_column_writer.hpp index 8927c391b447..bbb6cd06b61f 100644 --- a/extension/parquet/include/writer/struct_column_writer.hpp +++ b/extension/parquet/include/writer/struct_column_writer.hpp @@ -16,13 +16,11 @@ class StructColumnWriter : public ColumnWriter { public: StructColumnWriter(ParquetWriter &writer, const ParquetColumnSchema &column_schema, vector schema_path_p, vector> child_writers_p, bool can_have_nulls) - : ColumnWriter(writer, column_schema, std::move(schema_path_p), can_have_nulls), - child_writers(std::move(child_writers_p)) { + : ColumnWriter(writer, column_schema, std::move(schema_path_p), can_have_nulls) { + child_writers = std::move(child_writers_p); } ~StructColumnWriter() override = default; - vector> child_writers; - public: unique_ptr InitializeWriteState(duckdb_parquet::RowGroup &row_group) override; bool HasAnalyze() override; diff --git a/extension/parquet/include/writer/templated_column_writer.hpp b/extension/parquet/include/writer/templated_column_writer.hpp index c035bba43a0c..4c9f1d8aa6c8 100644 --- a/extension/parquet/include/writer/templated_column_writer.hpp +++ b/extension/parquet/include/writer/templated_column_writer.hpp @@ -197,6 +197,7 @@ class StandardColumnWriter : public PrimitiveColumnWriter { const bool check_parent_empty = parent && !parent->is_empty.empty(); const idx_t parent_index = state.definition_levels.size(); + D_ASSERT(!check_parent_empty || parent_index < parent->is_empty.size()); const idx_t vcount = check_parent_empty ? parent->definition_levels.size() - state.definition_levels.size() : count; diff --git a/extension/parquet/include/writer/variant_column_writer.hpp b/extension/parquet/include/writer/variant_column_writer.hpp new file mode 100644 index 000000000000..74fdda60872d --- /dev/null +++ b/extension/parquet/include/writer/variant_column_writer.hpp @@ -0,0 +1,30 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// writer/variant_column_writer.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "struct_column_writer.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" + +namespace duckdb { + +class VariantColumnWriter : public StructColumnWriter { +public: + VariantColumnWriter(ParquetWriter &writer, const ParquetColumnSchema &column_schema, vector schema_path_p, + vector> child_writers_p, bool can_have_nulls) + : StructColumnWriter(writer, column_schema, std::move(schema_path_p), std::move(child_writers_p), + can_have_nulls) { + } + ~VariantColumnWriter() override = default; + +public: + static ScalarFunction GetTransformFunction(); + static LogicalType TransformTypedValueRecursive(const LogicalType &type); +}; + +} // namespace duckdb diff --git a/extension/parquet/parquet_crypto.cpp b/extension/parquet/parquet_crypto.cpp index 07321e6ac186..b60c01155959 100644 --- a/extension/parquet/parquet_crypto.cpp +++ b/extension/parquet/parquet_crypto.cpp @@ -90,7 +90,7 @@ class EncryptionTransport : public TTransport { public: EncryptionTransport(TProtocol &prot_p, const string &key, const EncryptionUtil &encryption_util_p) : prot(prot_p), trans(*prot.getTransport()), - aes(encryption_util_p.CreateEncryptionState(reinterpret_cast(key.data()), key.size())), + aes(encryption_util_p.CreateEncryptionState(EncryptionTypes::GCM, key.size())), allocator(Allocator::DefaultAllocator(), ParquetCrypto::CRYPTO_BLOCK_SIZE) { Initialize(key); } @@ -173,8 +173,8 @@ class DecryptionTransport : public TTransport { public: DecryptionTransport(TProtocol &prot_p, const string &key, const EncryptionUtil &encryption_util_p) : prot(prot_p), trans(*prot.getTransport()), - aes(encryption_util_p.CreateEncryptionState(reinterpret_cast(key.data()), key.size())), - read_buffer_size(0), read_buffer_offset(0) { + aes(encryption_util_p.CreateEncryptionState(EncryptionTypes::GCM, key.size())), read_buffer_size(0), + read_buffer_offset(0) { Initialize(key); } uint32_t read_virt(uint8_t *buf, uint32_t len) override { @@ -207,9 +207,7 @@ class DecryptionTransport : public TTransport { data_t computed_tag[ParquetCrypto::TAG_BYTES]; transport_remaining -= trans.read(computed_tag, ParquetCrypto::TAG_BYTES); - if (aes->Finalize(read_buffer, 0, computed_tag, ParquetCrypto::TAG_BYTES) != 0) { - throw InternalException("DecryptionTransport::Finalize was called with bytes remaining in AES context out"); - } + aes->Finalize(read_buffer, 0, computed_tag, ParquetCrypto::TAG_BYTES); if (transport_remaining != 0) { throw InvalidInputException("Encoded ciphertext length differs from actual ciphertext length"); diff --git a/extension/parquet/parquet_extension.cpp b/extension/parquet/parquet_extension.cpp index 37e6cd0b7a92..5958d839c7b2 100644 --- a/extension/parquet/parquet_extension.cpp +++ b/extension/parquet/parquet_extension.cpp @@ -12,9 +12,11 @@ #include "parquet_metadata.hpp" #include "parquet_reader.hpp" #include "parquet_writer.hpp" +#include "parquet_shredding.hpp" #include "reader/struct_column_reader.hpp" #include "zstd_file_system.hpp" #include "writer/primitive_column_writer.hpp" +#include "writer/variant_column_writer.hpp" #include #include @@ -43,6 +45,9 @@ #include "duckdb/parser/parsed_data/create_table_function_info.hpp" #include "duckdb/parser/tableref/table_function_ref.hpp" #include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/storage/statistics/base_statistics.hpp" #include "duckdb/storage/table/row_group.hpp" @@ -54,156 +59,6 @@ namespace duckdb { -static case_insensitive_map_t GetChildNameToTypeMap(const LogicalType &type) { - case_insensitive_map_t name_to_type_map; - switch (type.id()) { - case LogicalTypeId::LIST: - name_to_type_map.emplace("element", ListType::GetChildType(type)); - break; - case LogicalTypeId::MAP: - name_to_type_map.emplace("key", MapType::KeyType(type)); - name_to_type_map.emplace("value", MapType::ValueType(type)); - break; - case LogicalTypeId::STRUCT: - for (auto &child_type : StructType::GetChildTypes(type)) { - if (child_type.first == FieldID::DUCKDB_FIELD_ID) { - throw BinderException("Cannot have column named \"%s\" with FIELD_IDS", FieldID::DUCKDB_FIELD_ID); - } - name_to_type_map.emplace(child_type); - } - break; - default: // LCOV_EXCL_START - throw InternalException("Unexpected type in GetChildNameToTypeMap"); - } // LCOV_EXCL_STOP - return name_to_type_map; -} - -static void GetChildNamesAndTypes(const LogicalType &type, vector &child_names, - vector &child_types) { - switch (type.id()) { - case LogicalTypeId::LIST: - child_names.emplace_back("element"); - child_types.emplace_back(ListType::GetChildType(type)); - break; - case LogicalTypeId::MAP: - child_names.emplace_back("key"); - child_names.emplace_back("value"); - child_types.emplace_back(MapType::KeyType(type)); - child_types.emplace_back(MapType::ValueType(type)); - break; - case LogicalTypeId::STRUCT: - for (auto &child_type : StructType::GetChildTypes(type)) { - child_names.emplace_back(child_type.first); - child_types.emplace_back(child_type.second); - } - break; - default: // LCOV_EXCL_START - throw InternalException("Unexpected type in GetChildNamesAndTypes"); - } // LCOV_EXCL_STOP -} - -static void GenerateFieldIDs(ChildFieldIDs &field_ids, idx_t &field_id, const vector &names, - const vector &sql_types) { - D_ASSERT(names.size() == sql_types.size()); - for (idx_t col_idx = 0; col_idx < names.size(); col_idx++) { - const auto &col_name = names[col_idx]; - auto inserted = field_ids.ids->insert(make_pair(col_name, FieldID(UnsafeNumericCast(field_id++)))); - D_ASSERT(inserted.second); - - const auto &col_type = sql_types[col_idx]; - if (col_type.id() != LogicalTypeId::LIST && col_type.id() != LogicalTypeId::MAP && - col_type.id() != LogicalTypeId::STRUCT) { - continue; - } - - // Cannot use GetChildNameToTypeMap here because we lose order, and we want to generate depth-first - vector child_names; - vector child_types; - GetChildNamesAndTypes(col_type, child_names, child_types); - - GenerateFieldIDs(inserted.first->second.child_field_ids, field_id, child_names, child_types); - } -} - -static void GetFieldIDs(const Value &field_ids_value, ChildFieldIDs &field_ids, - unordered_set &unique_field_ids, - const case_insensitive_map_t &name_to_type_map) { - const auto &struct_type = field_ids_value.type(); - if (struct_type.id() != LogicalTypeId::STRUCT) { - throw BinderException( - "Expected FIELD_IDS to be a STRUCT, e.g., {col1: 42, col2: {%s: 43, nested_col: 44}, col3: 44}", - FieldID::DUCKDB_FIELD_ID); - } - const auto &struct_children = StructValue::GetChildren(field_ids_value); - D_ASSERT(StructType::GetChildTypes(struct_type).size() == struct_children.size()); - for (idx_t i = 0; i < struct_children.size(); i++) { - const auto &col_name = StringUtil::Lower(StructType::GetChildName(struct_type, i)); - if (col_name == FieldID::DUCKDB_FIELD_ID) { - continue; - } - - auto it = name_to_type_map.find(col_name); - if (it == name_to_type_map.end()) { - string names; - for (const auto &name : name_to_type_map) { - if (!names.empty()) { - names += ", "; - } - names += name.first; - } - throw BinderException( - "Column name \"%s\" specified in FIELD_IDS not found. Consider using WRITE_PARTITION_COLUMNS if this " - "column is a partition column. Available column names: [%s]", - col_name, names); - } - D_ASSERT(field_ids.ids->find(col_name) == field_ids.ids->end()); // Caught by STRUCT - deduplicates keys - - const auto &child_value = struct_children[i]; - const auto &child_type = child_value.type(); - optional_ptr field_id_value; - optional_ptr child_field_ids_value; - - if (child_type.id() == LogicalTypeId::STRUCT) { - const auto &nested_children = StructValue::GetChildren(child_value); - D_ASSERT(StructType::GetChildTypes(child_type).size() == nested_children.size()); - for (idx_t nested_i = 0; nested_i < nested_children.size(); nested_i++) { - const auto &field_id_or_nested_col = StructType::GetChildName(child_type, nested_i); - if (field_id_or_nested_col == FieldID::DUCKDB_FIELD_ID) { - field_id_value = &nested_children[nested_i]; - } else { - child_field_ids_value = &child_value; - } - } - } else { - field_id_value = &child_value; - } - - FieldID field_id; - if (field_id_value) { - Value field_id_integer_value = field_id_value->DefaultCastAs(LogicalType::INTEGER); - const uint32_t field_id_int = IntegerValue::Get(field_id_integer_value); - if (!unique_field_ids.insert(field_id_int).second) { - throw BinderException("Duplicate field_id %s found in FIELD_IDS", field_id_integer_value.ToString()); - } - field_id = FieldID(UnsafeNumericCast(field_id_int)); - } - auto inserted = field_ids.ids->insert(make_pair(col_name, std::move(field_id))); - D_ASSERT(inserted.second); - - if (child_field_ids_value) { - const auto &col_type = it->second; - if (col_type.id() != LogicalTypeId::LIST && col_type.id() != LogicalTypeId::MAP && - col_type.id() != LogicalTypeId::STRUCT) { - throw BinderException("Column \"%s\" with type \"%s\" cannot have a nested FIELD_IDS specification", - col_name, LogicalTypeIdToString(col_type.id())); - } - - GetFieldIDs(*child_field_ids_value, inserted.first->second.child_field_ids, unique_field_ids, - GetChildNameToTypeMap(col_type)); - } - } -} - struct ParquetWriteBindData : public TableFunctionData { vector sql_types; vector column_names; @@ -233,11 +88,15 @@ struct ParquetWriteBindData : public TableFunctionData { optional_idx row_groups_per_file; ChildFieldIDs field_ids; + ShreddingType shredding_types; //! The compression level, higher value is more int64_t compression_level = ZStdFileSystem::DefaultCompressionLevel(); //! Which encodings to include when writing ParquetVersion parquet_version = ParquetVersion::V1; + + //! Which geo-parquet version to use when writing + GeoParquetVersion geoparquet_version = GeoParquetVersion::V1; }; struct ParquetWriteGlobalState : public GlobalFunctionData { @@ -291,6 +150,8 @@ static void ParquetListCopyOptions(ClientContext &context, CopyOptionsInput &inp copy_options["binary_as_string"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY); copy_options["file_row_number"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY); copy_options["can_have_nan"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY); + copy_options["geoparquet_version"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::WRITE_ONLY); + copy_options["shredding"] = CopyOption(LogicalType::ANY, CopyOptionMode::WRITE_ONLY); } static unique_ptr ParquetWriteBind(ClientContext &context, CopyFunctionBindInput &input, @@ -342,7 +203,7 @@ static unique_ptr ParquetWriteBind(ClientContext &context, CopyFun if (option.second[0].type().id() == LogicalTypeId::VARCHAR && StringUtil::Lower(StringValue::Get(option.second[0])) == "auto") { idx_t field_id = 0; - GenerateFieldIDs(bind_data->field_ids, field_id, names, sql_types); + FieldID::GenerateFieldIDs(bind_data->field_ids, field_id, names, sql_types); } else { unordered_set unique_field_ids; case_insensitive_map_t name_to_type_map; @@ -353,7 +214,57 @@ static unique_ptr ParquetWriteBind(ClientContext &context, CopyFun } name_to_type_map.emplace(names[col_idx], sql_types[col_idx]); } - GetFieldIDs(option.second[0], bind_data->field_ids, unique_field_ids, name_to_type_map); + FieldID::GetFieldIDs(option.second[0], bind_data->field_ids, unique_field_ids, name_to_type_map); + } + } else if (loption == "shredding") { + if (option.second[0].type().id() == LogicalTypeId::VARCHAR && + StringUtil::Lower(StringValue::Get(option.second[0])) == "auto") { + throw NotImplementedException("The 'auto' option is not yet implemented for 'shredding'"); + } else { + case_insensitive_set_t variant_names; + for (idx_t col_idx = 0; col_idx < names.size(); col_idx++) { + if (sql_types[col_idx].id() != LogicalTypeId::STRUCT) { + continue; + } + if (sql_types[col_idx].GetAlias() != "PARQUET_VARIANT") { + continue; + } + variant_names.emplace(names[col_idx]); + } + auto &shredding_types_value = option.second[0]; + if (shredding_types_value.type().id() != LogicalTypeId::STRUCT) { + BinderException("SHREDDING value should be a STRUCT of column names to types, i.e: {col1: " + "'INTEGER[]', col2: 'BOOLEAN'}"); + } + const auto &struct_type = shredding_types_value.type(); + const auto &struct_children = StructValue::GetChildren(shredding_types_value); + D_ASSERT(StructType::GetChildTypes(struct_type).size() == struct_children.size()); + for (idx_t i = 0; i < struct_children.size(); i++) { + const auto &col_name = StringUtil::Lower(StructType::GetChildName(struct_type, i)); + auto it = variant_names.find(col_name); + if (it == variant_names.end()) { + string names; + for (const auto &entry : variant_names) { + if (!names.empty()) { + names += ", "; + } + names += entry; + } + if (names.empty()) { + throw BinderException("VARIANT by name \"%s\" specified in SHREDDING not found. There are " + "no VARIANT columns present.", + col_name); + } else { + throw BinderException( + "VARIANT by name \"%s\" specified in SHREDDING not found. Consider using " + "WRITE_PARTITION_COLUMNS if this " + "column is a partition column. Available names of VARIANT columns: [%s]", + col_name, names); + } + } + const auto &child_value = struct_children[i]; + bind_data->shredding_types.AddChild(col_name, ShreddingType::GetShreddingTypes(child_value)); + } } } else if (loption == "kv_metadata") { auto &kv_struct = option.second[0]; @@ -426,6 +337,19 @@ static unique_ptr ParquetWriteBind(ClientContext &context, CopyFun } else { throw BinderException("Expected parquet_version 'V1' or 'V2'"); } + } else if (loption == "geoparquet_version") { + const auto roption = StringUtil::Upper(option.second[0].ToString()); + if (roption == "NONE") { + bind_data->geoparquet_version = GeoParquetVersion::NONE; + } else if (roption == "V1") { + bind_data->geoparquet_version = GeoParquetVersion::V1; + } else if (roption == "V2") { + bind_data->geoparquet_version = GeoParquetVersion::V2; + } else if (roption == "BOTH") { + bind_data->geoparquet_version = GeoParquetVersion::BOTH; + } else { + throw BinderException("Expected geoparquet_version 'NONE', 'V1' or 'BOTH'"); + } } else { throw InternalException("Unrecognized option for PARQUET: %s", option.first.c_str()); } @@ -454,10 +378,11 @@ static unique_ptr ParquetWriteInitializeGlobal(ClientContext auto &fs = FileSystem::GetFileSystem(context); global_state->writer = make_uniq( context, fs, file_path, parquet_bind.sql_types, parquet_bind.column_names, parquet_bind.codec, - parquet_bind.field_ids.Copy(), parquet_bind.kv_metadata, parquet_bind.encryption_config, - parquet_bind.dictionary_size_limit, parquet_bind.string_dictionary_page_size_limit, - parquet_bind.enable_bloom_filters, parquet_bind.bloom_filter_false_positive_ratio, - parquet_bind.compression_level, parquet_bind.debug_use_openssl, parquet_bind.parquet_version); + parquet_bind.field_ids.Copy(), parquet_bind.shredding_types.Copy(), parquet_bind.kv_metadata, + parquet_bind.encryption_config, parquet_bind.dictionary_size_limit, + parquet_bind.string_dictionary_page_size_limit, parquet_bind.enable_bloom_filters, + parquet_bind.bloom_filter_false_positive_ratio, parquet_bind.compression_level, parquet_bind.debug_use_openssl, + parquet_bind.parquet_version, parquet_bind.geoparquet_version); return std::move(global_state); } @@ -626,6 +551,39 @@ ParquetVersion EnumUtil::FromString(const char *value) { throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value)); } +template <> +const char *EnumUtil::ToChars(GeoParquetVersion value) { + switch (value) { + case GeoParquetVersion::NONE: + return "NONE"; + case GeoParquetVersion::V1: + return "V1"; + case GeoParquetVersion::V2: + return "V2"; + case GeoParquetVersion::BOTH: + return "BOTH"; + default: + throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value)); + } +} + +template <> +GeoParquetVersion EnumUtil::FromString(const char *value) { + if (StringUtil::Equals(value, "NONE")) { + return GeoParquetVersion::NONE; + } + if (StringUtil::Equals(value, "V1")) { + return GeoParquetVersion::V1; + } + if (StringUtil::Equals(value, "V2")) { + return GeoParquetVersion::V2; + } + if (StringUtil::Equals(value, "BOTH")) { + return GeoParquetVersion::BOTH; + } + throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value)); +} + static optional_idx SerializeCompressionLevel(const int64_t compression_level) { return compression_level < 0 ? NumericLimits::Maximum() - NumericCast(AbsValue(compression_level)) : NumericCast(compression_level); @@ -679,6 +637,9 @@ static void ParquetCopySerialize(Serializer &serializer, const FunctionData &bin serializer.WritePropertyWithDefault(115, "string_dictionary_page_size_limit", bind_data.string_dictionary_page_size_limit, default_value.string_dictionary_page_size_limit); + serializer.WritePropertyWithDefault(116, "geoparquet_version", bind_data.geoparquet_version, + default_value.geoparquet_version); + serializer.WriteProperty(117, "shredding_types", bind_data.shredding_types); } static unique_ptr ParquetCopyDeserialize(Deserializer &deserializer, CopyFunction &function) { @@ -711,6 +672,9 @@ static unique_ptr ParquetCopyDeserialize(Deserializer &deserialize deserializer.ReadPropertyWithExplicitDefault(114, "parquet_version", default_value.parquet_version); data->string_dictionary_page_size_limit = deserializer.ReadPropertyWithExplicitDefault( 115, "string_dictionary_page_size_limit", default_value.string_dictionary_page_size_limit); + data->geoparquet_version = + deserializer.ReadPropertyWithExplicitDefault(116, "geoparquet_version", default_value.geoparquet_version); + data->shredding_types = deserializer.ReadProperty(117, "shredding_types"); return std::move(data); } @@ -828,6 +792,51 @@ static bool IsTypeLossy(const LogicalType &type) { return type.id() == LogicalTypeId::HUGEINT || type.id() == LogicalTypeId::UHUGEINT; } +static bool IsGeometryType(const LogicalType &type, ClientContext &context) { + if (type.id() != LogicalTypeId::BLOB) { + return false; + } + if (!type.HasAlias()) { + return false; + } + if (type.GetAlias() != "GEOMETRY") { + return false; + } + return GeoParquetFileMetadata::IsGeoParquetConversionEnabled(context); +} + +static string GetShredding(case_insensitive_map_t> &options, const string &col_name) { + //! At this point, the options haven't been parsed yet, so we have to parse them ourselves. + auto it = options.find("shredding"); + if (it == options.end()) { + return string(); + } + auto &shredding = it->second; + if (shredding.empty()) { + return string(); + } + + auto &shredding_val = shredding[0]; + if (shredding_val.type().id() != LogicalTypeId::STRUCT) { + return string(); + } + + auto &shredded_variants = StructType::GetChildTypes(shredding_val.type()); + auto &values = StructValue::GetChildren(shredding_val); + for (idx_t i = 0; i < shredded_variants.size(); i++) { + auto &shredded_variant = shredded_variants[i]; + if (shredded_variant.first != col_name) { + continue; + } + auto &shredded_val = values[i]; + if (shredded_val.type().id() != LogicalTypeId::VARCHAR) { + return string(); + } + return shredded_val.GetValue(); + } + return string(); +} + static vector> ParquetWriteSelect(CopyToSelectInput &input) { auto &context = input.context; @@ -843,8 +852,7 @@ static vector> ParquetWriteSelect(CopyToSelectInput &inpu // Spatial types need to be encoded into WKB when writing GeoParquet. // But dont perform this conversion if this is a EXPORT DATABASE statement - if (input.copy_to_type == CopyToType::COPY_TO_FILE && type.id() == LogicalTypeId::BLOB && type.HasAlias() && - type.GetAlias() == "GEOMETRY" && GeoParquetFileMetadata::IsGeoParquetConversionEnabled(context)) { + if (input.copy_to_type == CopyToType::COPY_TO_FILE && IsGeometryType(type, context)) { LogicalType wkb_blob_type(LogicalTypeId::BLOB); wkb_blob_type.SetAlias("WKB_BLOB"); @@ -853,6 +861,23 @@ static vector> ParquetWriteSelect(CopyToSelectInput &inpu cast_expr->SetAlias(name); result.push_back(std::move(cast_expr)); any_change = true; + } else if (input.copy_to_type == CopyToType::COPY_TO_FILE && type.id() == LogicalTypeId::VARIANT) { + vector> arguments; + arguments.push_back(std::move(expr)); + + auto shredded_type_str = GetShredding(input.options, name); + if (!shredded_type_str.empty()) { + arguments.push_back(make_uniq(Value(shredded_type_str))); + } + + auto transform_func = VariantColumnWriter::GetTransformFunction(); + transform_func.bind(context, transform_func, arguments); + + auto func_expr = make_uniq(transform_func.return_type, transform_func, + std::move(arguments), nullptr, false); + func_expr->SetAlias(name); + result.push_back(std::move(func_expr)); + any_change = true; } // If this is an EXPORT DATABASE statement, we dont want to write "lossy" types, instead cast them to VARCHAR else if (input.copy_to_type == CopyToType::EXPORT_DATABASE && TypeVisitor::Contains(type, IsTypeLossy)) { @@ -924,6 +949,9 @@ static void LoadInternal(ExtensionLoader &loader) { ParquetBloomProbeFunction bloom_probe_fun; loader.RegisterFunction(MultiFileReader::CreateFunctionSet(bloom_probe_fun)); + // variant_to_parquet_variant + loader.RegisterFunction(VariantColumnWriter::GetTransformFunction()); + CopyFunction function("parquet"); function.copy_to_select = ParquetWriteSelect; function.copy_to_bind = ParquetWriteBind; @@ -970,9 +998,6 @@ static void LoadInternal(ExtensionLoader &loader) { "enable_geoparquet_conversion", "Attempt to decode/encode geometry data in/as GeoParquet files if the spatial extension is present.", LogicalType::BOOLEAN, Value::BOOLEAN(true)); - config.AddExtensionOption("variant_legacy_encoding", - "Enables the Parquet reader to identify a Variant structurally.", LogicalType::BOOLEAN, - Value::BOOLEAN(false)); } void ParquetExtension::Load(ExtensionLoader &loader) { diff --git a/extension/parquet/parquet_field_id.cpp b/extension/parquet/parquet_field_id.cpp new file mode 100644 index 000000000000..642fc26c76ae --- /dev/null +++ b/extension/parquet/parquet_field_id.cpp @@ -0,0 +1,180 @@ +#include "parquet_field_id.hpp" +#include "duckdb/common/exception/binder_exception.hpp" + +namespace duckdb { + +constexpr const char *FieldID::DUCKDB_FIELD_ID; + +ChildFieldIDs::ChildFieldIDs() : ids(make_uniq>()) { +} + +ChildFieldIDs ChildFieldIDs::Copy() const { + ChildFieldIDs result; + for (const auto &id : *ids) { + result.ids->emplace(id.first, id.second.Copy()); + } + return result; +} + +FieldID::FieldID() : set(false) { +} + +FieldID::FieldID(int32_t field_id_p) : set(true), field_id(field_id_p) { +} + +FieldID FieldID::Copy() const { + auto result = set ? FieldID(field_id) : FieldID(); + result.child_field_ids = child_field_ids.Copy(); + return result; +} + +static case_insensitive_map_t GetChildNameToTypeMap(const LogicalType &type) { + case_insensitive_map_t name_to_type_map; + switch (type.id()) { + case LogicalTypeId::LIST: + name_to_type_map.emplace("element", ListType::GetChildType(type)); + break; + case LogicalTypeId::MAP: + name_to_type_map.emplace("key", MapType::KeyType(type)); + name_to_type_map.emplace("value", MapType::ValueType(type)); + break; + case LogicalTypeId::STRUCT: + for (auto &child_type : StructType::GetChildTypes(type)) { + if (child_type.first == FieldID::DUCKDB_FIELD_ID) { + throw BinderException("Cannot have column named \"%s\" with FIELD_IDS", FieldID::DUCKDB_FIELD_ID); + } + name_to_type_map.emplace(child_type); + } + break; + default: // LCOV_EXCL_START + throw InternalException("Unexpected type in GetChildNameToTypeMap"); + } // LCOV_EXCL_STOP + return name_to_type_map; +} + +static void GetChildNamesAndTypes(const LogicalType &type, vector &child_names, + vector &child_types) { + switch (type.id()) { + case LogicalTypeId::LIST: + child_names.emplace_back("element"); + child_types.emplace_back(ListType::GetChildType(type)); + break; + case LogicalTypeId::MAP: + child_names.emplace_back("key"); + child_names.emplace_back("value"); + child_types.emplace_back(MapType::KeyType(type)); + child_types.emplace_back(MapType::ValueType(type)); + break; + case LogicalTypeId::STRUCT: + for (auto &child_type : StructType::GetChildTypes(type)) { + child_names.emplace_back(child_type.first); + child_types.emplace_back(child_type.second); + } + break; + default: // LCOV_EXCL_START + throw InternalException("Unexpected type in GetChildNamesAndTypes"); + } // LCOV_EXCL_STOP +} + +void FieldID::GenerateFieldIDs(ChildFieldIDs &field_ids, idx_t &field_id, const vector &names, + const vector &sql_types) { + D_ASSERT(names.size() == sql_types.size()); + for (idx_t col_idx = 0; col_idx < names.size(); col_idx++) { + const auto &col_name = names[col_idx]; + auto inserted = field_ids.ids->insert(make_pair(col_name, FieldID(UnsafeNumericCast(field_id++)))); + D_ASSERT(inserted.second); + + const auto &col_type = sql_types[col_idx]; + if (col_type.id() != LogicalTypeId::LIST && col_type.id() != LogicalTypeId::MAP && + col_type.id() != LogicalTypeId::STRUCT) { + continue; + } + + // Cannot use GetChildNameToTypeMap here because we lose order, and we want to generate depth-first + vector child_names; + vector child_types; + GetChildNamesAndTypes(col_type, child_names, child_types); + GenerateFieldIDs(inserted.first->second.child_field_ids, field_id, child_names, child_types); + } +} + +void FieldID::GetFieldIDs(const Value &field_ids_value, ChildFieldIDs &field_ids, + unordered_set &unique_field_ids, + const case_insensitive_map_t &name_to_type_map) { + const auto &struct_type = field_ids_value.type(); + if (struct_type.id() != LogicalTypeId::STRUCT) { + throw BinderException( + "Expected FIELD_IDS to be a STRUCT, e.g., {col1: 42, col2: {%s: 43, nested_col: 44}, col3: 44}", + FieldID::DUCKDB_FIELD_ID); + } + const auto &struct_children = StructValue::GetChildren(field_ids_value); + D_ASSERT(StructType::GetChildTypes(struct_type).size() == struct_children.size()); + for (idx_t i = 0; i < struct_children.size(); i++) { + const auto &col_name = StringUtil::Lower(StructType::GetChildName(struct_type, i)); + if (col_name == FieldID::DUCKDB_FIELD_ID) { + continue; + } + + auto it = name_to_type_map.find(col_name); + if (it == name_to_type_map.end()) { + string names; + for (const auto &name : name_to_type_map) { + if (!names.empty()) { + names += ", "; + } + names += name.first; + } + throw BinderException( + "Column name \"%s\" specified in FIELD_IDS not found. Consider using WRITE_PARTITION_COLUMNS if this " + "column is a partition column. Available column names: [%s]", + col_name, names); + } + D_ASSERT(field_ids.ids->find(col_name) == field_ids.ids->end()); // Caught by STRUCT - deduplicates keys + + const auto &child_value = struct_children[i]; + const auto &child_type = child_value.type(); + optional_ptr field_id_value; + optional_ptr child_field_ids_value; + + if (child_type.id() == LogicalTypeId::STRUCT) { + const auto &nested_children = StructValue::GetChildren(child_value); + D_ASSERT(StructType::GetChildTypes(child_type).size() == nested_children.size()); + for (idx_t nested_i = 0; nested_i < nested_children.size(); nested_i++) { + const auto &field_id_or_nested_col = StructType::GetChildName(child_type, nested_i); + if (field_id_or_nested_col == FieldID::DUCKDB_FIELD_ID) { + field_id_value = &nested_children[nested_i]; + } else { + child_field_ids_value = &child_value; + } + } + } else { + field_id_value = &child_value; + } + + FieldID field_id; + if (field_id_value) { + Value field_id_integer_value = field_id_value->DefaultCastAs(LogicalType::INTEGER); + const uint32_t field_id_int = IntegerValue::Get(field_id_integer_value); + if (!unique_field_ids.insert(field_id_int).second) { + throw BinderException("Duplicate field_id %s found in FIELD_IDS", field_id_integer_value.ToString()); + } + field_id = FieldID(UnsafeNumericCast(field_id_int)); + } + auto inserted = field_ids.ids->insert(make_pair(col_name, std::move(field_id))); + D_ASSERT(inserted.second); + + if (child_field_ids_value) { + const auto &col_type = it->second; + if (col_type.id() != LogicalTypeId::LIST && col_type.id() != LogicalTypeId::MAP && + col_type.id() != LogicalTypeId::STRUCT) { + throw BinderException("Column \"%s\" with type \"%s\" cannot have a nested FIELD_IDS specification", + col_name, LogicalTypeIdToString(col_type.id())); + } + + GetFieldIDs(*child_field_ids_value, inserted.first->second.child_field_ids, unique_field_ids, + GetChildNameToTypeMap(col_type)); + } + } +} + +} // namespace duckdb diff --git a/extension/parquet/parquet_metadata.cpp b/extension/parquet/parquet_metadata.cpp index d160d71979c5..af2c7f534803 100644 --- a/extension/parquet/parquet_metadata.cpp +++ b/extension/parquet/parquet_metadata.cpp @@ -6,16 +6,34 @@ #include "duckdb/common/multi_file/multi_file_reader.hpp" #include "duckdb/common/types/blob.hpp" -#include "duckdb/common/types/column/column_data_collection.hpp" #include "duckdb/planner/filter/constant_filter.hpp" #include "duckdb/main/config.hpp" +#include "duckdb/common/multi_file/multi_file_list.hpp" +#include "parquet_reader.hpp" +#include "duckdb/common/numeric_utils.hpp" namespace duckdb { -struct ParquetMetaDataBindData : public TableFunctionData { - vector return_types; +struct ParquetMetadataFilePaths { + MultiFileListScanData scan_data; shared_ptr file_list; - unique_ptr multi_file_reader; + mutex file_lock; + + bool NextFile(OpenFileInfo &result) { + D_ASSERT(file_list); + unique_lock lock(file_lock); + return file_list->Scan(scan_data, result); + } + + FileExpandResult GetExpandResult() { + D_ASSERT(file_list); + unique_lock lock(file_lock); + return file_list->GetExpandResult(); + } +}; + +struct ParquetMetaDataBindData : public TableFunctionData { + unique_ptr file_paths; }; struct ParquetBloomProbeBindData : public ParquetMetaDataBindData { @@ -31,32 +49,76 @@ enum class ParquetMetadataOperatorType : uint8_t { BLOOM_PROBE }; -struct ParquetMetaDataOperatorData : public GlobalTableFunctionState { - explicit ParquetMetaDataOperatorData(ClientContext &context, const vector &types) - : collection(context, types) { - } - - ColumnDataCollection collection; - ColumnDataScanState scan_state; +class ParquetMetadataFileProcessor { +public: + ParquetMetadataFileProcessor() = default; + virtual ~ParquetMetadataFileProcessor() = default; + void Initialize(ClientContext &context, OpenFileInfo &file_info) { + ParquetOptions parquet_options(context); + reader = make_uniq(context, file_info, parquet_options); + } + virtual void InitializeInternal(ClientContext &context) {}; + virtual idx_t TotalRowCount() = 0; + virtual void ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) = 0; + +protected: + unique_ptr reader; +}; - MultiFileListScanData file_list_scan; - OpenFileInfo current_file; +struct ParquetMetaDataBindData; +class ParquetMetaDataOperator { public: - static void BindMetaData(vector &return_types, vector &names); + template + static unique_ptr Bind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names); + static unique_ptr InitGlobal(ClientContext &context, TableFunctionInitInput &input); + template + static unique_ptr InitLocal(ExecutionContext &context, TableFunctionInitInput &input, + GlobalTableFunctionState *global_state); + template + static void Function(ClientContext &context, TableFunctionInput &data_p, DataChunk &output); + static double Progress(ClientContext &context, const FunctionData *bind_data_p, + const GlobalTableFunctionState *global_state); + + template static void BindSchema(vector &return_types, vector &names); - static void BindKeyValueMetaData(vector &return_types, vector &names); - static void BindFileMetaData(vector &return_types, vector &names); - static void BindBloomProbe(vector &return_types, vector &names); - - void LoadRowGroupMetadata(ClientContext &context, const vector &return_types, - const OpenFileInfo &file); - void LoadSchemaData(ClientContext &context, const vector &return_types, const OpenFileInfo &file); - void LoadKeyValueMetaData(ClientContext &context, const vector &return_types, - const OpenFileInfo &file); - void LoadFileMetaData(ClientContext &context, const vector &return_types, const OpenFileInfo &file); - void ExecuteBloomProbe(ClientContext &context, const vector &return_types, const OpenFileInfo &file, - const string &column_name, const Value &probe); +}; + +struct ParquetMetadataGlobalState : public GlobalTableFunctionState { + ParquetMetadataGlobalState(unique_ptr file_paths_p, ClientContext &context) + : file_paths(std::move(file_paths_p)) { + auto expand_result = file_paths->GetExpandResult(); + if (expand_result == FileExpandResult::MULTIPLE_FILES) { + max_threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); + } else { + max_threads = 1; + } + } + + idx_t MaxThreads() const override { + return max_threads; + } + + bool NextFile(ClientContext &context, OpenFileInfo &result) { + return file_paths->NextFile(result); + } + + double GetProgress() const { + // Not the most accurate, instantly assumes all files are done and equal + unique_lock lock(file_paths->file_lock); + return static_cast(file_paths->scan_data.current_file_idx) / file_paths->file_list->GetTotalFileCount(); + } + + unique_ptr file_paths; + idx_t max_threads; +}; + +struct ParquetMetadataLocalState : public LocalTableFunctionState { + unique_ptr processor; + bool file_exhausted = true; + idx_t row_idx = 0; + idx_t total_rows = 0; }; template @@ -114,7 +176,20 @@ static Value ParquetElementBoolean(bool value, bool is_iset) { //===--------------------------------------------------------------------===// // Row Group Meta Data //===--------------------------------------------------------------------===// -void ParquetMetaDataOperatorData::BindMetaData(vector &return_types, vector &names) { + +class ParquetRowGroupMetadataProcessor : public ParquetMetadataFileProcessor { +public: + void InitializeInternal(ClientContext &context) override; + idx_t TotalRowCount() override; + void ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) override; + +private: + vector column_schemas; +}; + +template <> +void ParquetMetaDataOperator::BindSchema(vector &return_types, + vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -201,6 +276,21 @@ void ParquetMetaDataOperatorData::BindMetaData(vector &return_types names.emplace_back("row_group_compressed_bytes"); return_types.emplace_back(LogicalType::BIGINT); + + names.emplace_back("geo_bbox"); + return_types.emplace_back(LogicalType::STRUCT({ + {"xmin", LogicalType::DOUBLE}, + {"xmax", LogicalType::DOUBLE}, + {"ymin", LogicalType::DOUBLE}, + {"ymax", LogicalType::DOUBLE}, + {"zmin", LogicalType::DOUBLE}, + {"zmax", LogicalType::DOUBLE}, + {"mmin", LogicalType::DOUBLE}, + {"mmax", LogicalType::DOUBLE}, + })); + + names.emplace_back("geo_types"); + return_types.emplace_back(LogicalType::LIST(LogicalType::VARCHAR)); } static Value ConvertParquetStats(const LogicalType &type, const ParquetColumnSchema &schema_ele, bool stats_is_set, @@ -211,171 +301,186 @@ static Value ConvertParquetStats(const LogicalType &type, const ParquetColumnSch return ParquetStatisticsUtils::ConvertValue(type, schema_ele, stats).DefaultCastAs(LogicalType::VARCHAR); } -void ParquetMetaDataOperatorData::LoadRowGroupMetadata(ClientContext &context, const vector &return_types, - const OpenFileInfo &file) { - collection.Reset(); - ParquetOptions parquet_options(context); - ParquetReader reader(context, file.path, parquet_options); - idx_t count = 0; - DataChunk current_chunk; - current_chunk.Initialize(context, return_types); - auto meta_data = reader.GetFileMetadata(); - vector column_schemas; +static Value ConvertParquetGeoStatsBBOX(const duckdb_parquet::GeospatialStatistics &stats) { + if (!stats.__isset.bbox) { + return Value(LogicalType::STRUCT({ + {"xmin", LogicalType::DOUBLE}, + {"xmax", LogicalType::DOUBLE}, + {"ymin", LogicalType::DOUBLE}, + {"ymax", LogicalType::DOUBLE}, + {"zmin", LogicalType::DOUBLE}, + {"zmax", LogicalType::DOUBLE}, + {"mmin", LogicalType::DOUBLE}, + {"mmax", LogicalType::DOUBLE}, + })); + } + + return Value::STRUCT({ + {"xmin", Value::DOUBLE(stats.bbox.xmin)}, + {"xmax", Value::DOUBLE(stats.bbox.xmax)}, + {"ymin", Value::DOUBLE(stats.bbox.ymin)}, + {"ymax", Value::DOUBLE(stats.bbox.ymax)}, + {"zmin", stats.bbox.__isset.zmin ? Value::DOUBLE(stats.bbox.zmin) : Value(LogicalTypeId::DOUBLE)}, + {"zmax", stats.bbox.__isset.zmax ? Value::DOUBLE(stats.bbox.zmax) : Value(LogicalTypeId::DOUBLE)}, + {"mmin", stats.bbox.__isset.mmin ? Value::DOUBLE(stats.bbox.mmin) : Value(LogicalTypeId::DOUBLE)}, + {"mmax", stats.bbox.__isset.mmax ? Value::DOUBLE(stats.bbox.mmax) : Value(LogicalTypeId::DOUBLE)}, + }); +} + +static Value ConvertParquetGeoStatsTypes(const duckdb_parquet::GeospatialStatistics &stats) { + if (!stats.__isset.geospatial_types) { + return Value(LogicalType::LIST(LogicalType::VARCHAR)); + } + vector types; + types.reserve(stats.geospatial_types.size()); + + GeometryTypeSet type_set; + for (auto &type : stats.geospatial_types) { + const auto geom_type = (type % 1000); + const auto vert_type = (type / 1000); + if (geom_type < 1 || geom_type > 7) { + throw InvalidInputException("Unsupported geometry type in Parquet geo metadata"); + } + if (vert_type < 0 || vert_type > 3) { + throw InvalidInputException("Unsupported geometry vertex type in Parquet geo metadata"); + } + type_set.Add(static_cast(geom_type), static_cast(vert_type)); + } + + for (auto &type_name : type_set.ToString(true)) { + types.push_back(Value(type_name)); + } + return Value::LIST(LogicalType::VARCHAR, types); +} + +void ParquetRowGroupMetadataProcessor::InitializeInternal(ClientContext &context) { + auto meta_data = reader->GetFileMetadata(); + column_schemas.clear(); for (idx_t schema_idx = 0; schema_idx < meta_data->schema.size(); schema_idx++) { auto &schema_element = meta_data->schema[schema_idx]; if (schema_element.num_children > 0) { continue; } ParquetColumnSchema column_schema; - column_schema.type = reader.DeriveLogicalType(schema_element, column_schema); + column_schema.type = reader->DeriveLogicalType(schema_element, column_schema); column_schemas.push_back(std::move(column_schema)); } +} - for (idx_t row_group_idx = 0; row_group_idx < meta_data->row_groups.size(); row_group_idx++) { - auto &row_group = meta_data->row_groups[row_group_idx]; - - if (row_group.columns.size() > column_schemas.size()) { - throw InternalException("Too many column in row group: corrupt file?"); - } - for (idx_t col_idx = 0; col_idx < row_group.columns.size(); col_idx++) { - auto &column = row_group.columns[col_idx]; - auto &column_schema = column_schemas[col_idx]; - auto &col_meta = column.meta_data; - auto &stats = col_meta.statistics; - auto &column_type = column_schema.type; - - // file_name, LogicalType::VARCHAR - current_chunk.SetValue(0, count, file.path); - - // row_group_id, LogicalType::BIGINT - current_chunk.SetValue(1, count, Value::BIGINT(UnsafeNumericCast(row_group_idx))); - - // row_group_num_rows, LogicalType::BIGINT - current_chunk.SetValue(2, count, Value::BIGINT(row_group.num_rows)); - - // row_group_num_columns, LogicalType::BIGINT - current_chunk.SetValue(3, count, Value::BIGINT(UnsafeNumericCast(row_group.columns.size()))); - - // row_group_bytes, LogicalType::BIGINT - current_chunk.SetValue(4, count, Value::BIGINT(row_group.total_byte_size)); - - // column_id, LogicalType::BIGINT - current_chunk.SetValue(5, count, Value::BIGINT(UnsafeNumericCast(col_idx))); - - // file_offset, LogicalType::BIGINT - current_chunk.SetValue(6, count, ParquetElementBigint(column.file_offset, row_group.__isset.file_offset)); - - // num_values, LogicalType::BIGINT - current_chunk.SetValue(7, count, Value::BIGINT(col_meta.num_values)); - - // path_in_schema, LogicalType::VARCHAR - current_chunk.SetValue(8, count, StringUtil::Join(col_meta.path_in_schema, ", ")); - - // type, LogicalType::VARCHAR - current_chunk.SetValue(9, count, ConvertParquetElementToString(col_meta.type)); - - // stats_min, LogicalType::VARCHAR - current_chunk.SetValue(10, count, - ConvertParquetStats(column_type, column_schema, stats.__isset.min, stats.min)); - - // stats_max, LogicalType::VARCHAR - current_chunk.SetValue(11, count, - ConvertParquetStats(column_type, column_schema, stats.__isset.max, stats.max)); - - // stats_null_count, LogicalType::BIGINT - current_chunk.SetValue(12, count, ParquetElementBigint(stats.null_count, stats.__isset.null_count)); - - // stats_distinct_count, LogicalType::BIGINT - current_chunk.SetValue(13, count, ParquetElementBigint(stats.distinct_count, stats.__isset.distinct_count)); - - // stats_min_value, LogicalType::VARCHAR - current_chunk.SetValue( - 14, count, ConvertParquetStats(column_type, column_schema, stats.__isset.min_value, stats.min_value)); - - // stats_max_value, LogicalType::VARCHAR - current_chunk.SetValue( - 15, count, ConvertParquetStats(column_type, column_schema, stats.__isset.max_value, stats.max_value)); - - // compression, LogicalType::VARCHAR - current_chunk.SetValue(16, count, ConvertParquetElementToString(col_meta.codec)); - - // encodings, LogicalType::VARCHAR - vector encoding_string; - encoding_string.reserve(col_meta.encodings.size()); - for (auto &encoding : col_meta.encodings) { - encoding_string.push_back(ConvertParquetElementToString(encoding)); - } - current_chunk.SetValue(17, count, Value(StringUtil::Join(encoding_string, ", "))); - - // index_page_offset, LogicalType::BIGINT - current_chunk.SetValue( - 18, count, ParquetElementBigint(col_meta.index_page_offset, col_meta.__isset.index_page_offset)); - - // dictionary_page_offset, LogicalType::BIGINT - current_chunk.SetValue( - 19, count, - ParquetElementBigint(col_meta.dictionary_page_offset, col_meta.__isset.dictionary_page_offset)); - - // data_page_offset, LogicalType::BIGINT - current_chunk.SetValue(20, count, Value::BIGINT(col_meta.data_page_offset)); - - // total_compressed_size, LogicalType::BIGINT - current_chunk.SetValue(21, count, Value::BIGINT(col_meta.total_compressed_size)); - - // total_uncompressed_size, LogicalType::BIGINT - current_chunk.SetValue(22, count, Value::BIGINT(col_meta.total_uncompressed_size)); - - // key_value_metadata, LogicalType::MAP(LogicalType::BLOB, LogicalType::BLOB) - vector map_keys, map_values; - for (auto &entry : col_meta.key_value_metadata) { - map_keys.push_back(Value::BLOB_RAW(entry.key)); - map_values.push_back(Value::BLOB_RAW(entry.value)); - } - current_chunk.SetValue( - 23, count, - Value::MAP(LogicalType::BLOB, LogicalType::BLOB, std::move(map_keys), std::move(map_values))); - - // bloom_filter_offset, LogicalType::BIGINT - current_chunk.SetValue( - 24, count, ParquetElementBigint(col_meta.bloom_filter_offset, col_meta.__isset.bloom_filter_offset)); - - // bloom_filter_length, LogicalType::BIGINT - current_chunk.SetValue( - 25, count, ParquetElementBigint(col_meta.bloom_filter_length, col_meta.__isset.bloom_filter_length)); - - // min_is_exact, LogicalType::BOOLEAN - current_chunk.SetValue(26, count, - ParquetElementBoolean(stats.is_min_value_exact, stats.__isset.is_min_value_exact)); - - // max_is_exact, LogicalType::BOOLEAN - current_chunk.SetValue(27, count, - ParquetElementBoolean(stats.is_max_value_exact, stats.__isset.is_max_value_exact)); - - // row_group_compressed_bytes - current_chunk.SetValue( - 28, count, - ParquetElementBigint(row_group.__isset.total_compressed_size, row_group.__isset.total_compressed_size)); - - count++; - if (count >= STANDARD_VECTOR_SIZE) { - current_chunk.SetCardinality(count); - collection.Append(current_chunk); - - count = 0; - current_chunk.Reset(); - } - } - } - current_chunk.SetCardinality(count); - collection.Append(current_chunk); +idx_t ParquetRowGroupMetadataProcessor::TotalRowCount() { + auto meta_data = reader->GetFileMetadata(); + return meta_data->row_groups.size() * column_schemas.size(); +} - collection.InitializeScan(scan_state); +void ParquetRowGroupMetadataProcessor::ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) { + auto meta_data = reader->GetFileMetadata(); + idx_t col_idx = row_idx % column_schemas.size(); + idx_t row_group_idx = row_idx / column_schemas.size(); + + auto &row_group = meta_data->row_groups[row_group_idx]; + + auto &column = row_group.columns[col_idx]; + auto &column_schema = column_schemas[col_idx]; + auto &col_meta = column.meta_data; + auto &stats = col_meta.statistics; + auto &column_type = column_schema.type; + + // file_name + output.SetValue(0, output_idx, reader->file.path); + // row_group_id + output.SetValue(1, output_idx, Value::BIGINT(UnsafeNumericCast(row_group_idx))); + // row_group_num_rows + output.SetValue(2, output_idx, Value::BIGINT(row_group.num_rows)); + // row_group_num_columns + output.SetValue(3, output_idx, Value::BIGINT(UnsafeNumericCast(row_group.columns.size()))); + // row_group_bytes + output.SetValue(4, output_idx, Value::BIGINT(row_group.total_byte_size)); + // column_id + output.SetValue(5, output_idx, Value::BIGINT(UnsafeNumericCast(col_idx))); + // file_offset + output.SetValue(6, output_idx, ParquetElementBigint(column.file_offset, row_group.__isset.file_offset)); + // num_values + output.SetValue(7, output_idx, Value::BIGINT(col_meta.num_values)); + // path_in_schema + output.SetValue(8, output_idx, StringUtil::Join(col_meta.path_in_schema, ", ")); + // type + output.SetValue(9, output_idx, ConvertParquetElementToString(col_meta.type)); + // stats_min + output.SetValue(10, output_idx, ConvertParquetStats(column_type, column_schema, stats.__isset.min, stats.min)); + // stats_max + output.SetValue(11, output_idx, ConvertParquetStats(column_type, column_schema, stats.__isset.max, stats.max)); + // stats_null_count + output.SetValue(12, output_idx, ParquetElementBigint(stats.null_count, stats.__isset.null_count)); + // stats_distinct_count + output.SetValue(13, output_idx, ParquetElementBigint(stats.distinct_count, stats.__isset.distinct_count)); + // stats_min_value + output.SetValue(14, output_idx, + ConvertParquetStats(column_type, column_schema, stats.__isset.min_value, stats.min_value)); + // stats_max_value + output.SetValue(15, output_idx, + ConvertParquetStats(column_type, column_schema, stats.__isset.max_value, stats.max_value)); + // compression + output.SetValue(16, output_idx, ConvertParquetElementToString(col_meta.codec)); + // encodings + vector encoding_string; + encoding_string.reserve(col_meta.encodings.size()); + for (auto &encoding : col_meta.encodings) { + encoding_string.push_back(ConvertParquetElementToString(encoding)); + } + output.SetValue(17, output_idx, Value(StringUtil::Join(encoding_string, ", "))); + // index_page_offset + output.SetValue(18, output_idx, + ParquetElementBigint(col_meta.index_page_offset, col_meta.__isset.index_page_offset)); + // dictionary_page_offset + output.SetValue(19, output_idx, + ParquetElementBigint(col_meta.dictionary_page_offset, col_meta.__isset.dictionary_page_offset)); + // data_page_offset + output.SetValue(20, output_idx, Value::BIGINT(col_meta.data_page_offset)); + // total_compressed_size + output.SetValue(21, output_idx, Value::BIGINT(col_meta.total_compressed_size)); + // total_uncompressed_size + output.SetValue(22, output_idx, Value::BIGINT(col_meta.total_uncompressed_size)); + // key_value_metadata + vector map_keys, map_values; + for (auto &entry : col_meta.key_value_metadata) { + map_keys.push_back(Value::BLOB_RAW(entry.key)); + map_values.push_back(Value::BLOB_RAW(entry.value)); + } + output.SetValue(23, output_idx, + Value::MAP(LogicalType::BLOB, LogicalType::BLOB, std::move(map_keys), std::move(map_values))); + // bloom_filter_offset + output.SetValue(24, output_idx, + ParquetElementBigint(col_meta.bloom_filter_offset, col_meta.__isset.bloom_filter_offset)); + // bloom_filter_length + output.SetValue(25, output_idx, + ParquetElementBigint(col_meta.bloom_filter_length, col_meta.__isset.bloom_filter_length)); + // min_is_exact + output.SetValue(26, output_idx, ParquetElementBoolean(stats.is_min_value_exact, stats.__isset.is_min_value_exact)); + // max_is_exact + output.SetValue(27, output_idx, ParquetElementBoolean(stats.is_max_value_exact, stats.__isset.is_max_value_exact)); + // row_group_compressed_bytes + output.SetValue(28, output_idx, + ParquetElementBigint(row_group.total_compressed_size, row_group.__isset.total_compressed_size)); + // geo_stats_bbox, LogicalType::STRUCT(...) + output.SetValue(29, output_idx, ConvertParquetGeoStatsBBOX(col_meta.geospatial_statistics)); + + // geo_stats_types, LogicalType::LIST(LogicalType::VARCHAR) + output.SetValue(30, output_idx, ConvertParquetGeoStatsTypes(col_meta.geospatial_statistics)); } //===--------------------------------------------------------------------===// // Schema Data //===--------------------------------------------------------------------===// -void ParquetMetaDataOperatorData::BindSchema(vector &return_types, vector &names) { + +class ParquetSchemaProcessor : public ParquetMetadataFileProcessor { +public: + idx_t TotalRowCount() override; + void ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) override; +}; + +template <> +void ParquetMetaDataOperator::BindSchema(vector &return_types, + vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -411,6 +516,9 @@ void ParquetMetaDataOperatorData::BindSchema(vector &return_types, names.emplace_back("duckdb_type"); return_types.emplace_back(LogicalType::VARCHAR); + + names.emplace_back("column_id"); + return_types.emplace_back(LogicalType::BIGINT); } static Value ParquetLogicalTypeToString(const duckdb_parquet::LogicalType &type, bool is_set) { @@ -459,81 +567,69 @@ static Value ParquetLogicalTypeToString(const duckdb_parquet::LogicalType &type, if (type.__isset.FLOAT16) { return Value(PrintParquetElementToString(type.FLOAT16)); } + if (type.__isset.GEOMETRY) { + return Value(PrintParquetElementToString(type.GEOMETRY)); + } + if (type.__isset.GEOGRAPHY) { + return Value(PrintParquetElementToString(type.GEOGRAPHY)); + } return Value(); } -void ParquetMetaDataOperatorData::LoadSchemaData(ClientContext &context, const vector &return_types, - const OpenFileInfo &file) { - collection.Reset(); - ParquetOptions parquet_options(context); - auto reader = make_uniq(context, file.path, parquet_options); - idx_t count = 0; - DataChunk current_chunk; - current_chunk.Initialize(context, return_types); - auto meta_data = reader->GetFileMetadata(); - for (idx_t col_idx = 0; col_idx < meta_data->schema.size(); col_idx++) { - auto &column = meta_data->schema[col_idx]; - - // file_name, LogicalType::VARCHAR - current_chunk.SetValue(0, count, file.path); - - // name, LogicalType::VARCHAR - current_chunk.SetValue(1, count, column.name); - - // type, LogicalType::VARCHAR - current_chunk.SetValue(2, count, ParquetElementString(column.type, column.__isset.type)); - - // type_length, LogicalType::INTEGER - current_chunk.SetValue(3, count, ParquetElementInteger(column.type_length, column.__isset.type_length)); - - // repetition_type, LogicalType::VARCHAR - current_chunk.SetValue(4, count, ParquetElementString(column.repetition_type, column.__isset.repetition_type)); - - // num_children, LogicalType::BIGINT - current_chunk.SetValue(5, count, ParquetElementBigint(column.num_children, column.__isset.num_children)); - - // converted_type, LogicalType::VARCHAR - current_chunk.SetValue(6, count, ParquetElementString(column.converted_type, column.__isset.converted_type)); - - // scale, LogicalType::BIGINT - current_chunk.SetValue(7, count, ParquetElementBigint(column.scale, column.__isset.scale)); - - // precision, LogicalType::BIGINT - current_chunk.SetValue(8, count, ParquetElementBigint(column.precision, column.__isset.precision)); - - // field_id, LogicalType::BIGINT - current_chunk.SetValue(9, count, ParquetElementBigint(column.field_id, column.__isset.field_id)); - - // logical_type, LogicalType::VARCHAR - current_chunk.SetValue(10, count, ParquetLogicalTypeToString(column.logicalType, column.__isset.logicalType)); - - // duckdb_type, LogicalType::VARCHAR - ParquetColumnSchema column_schema; - Value duckdb_type; - if (column.__isset.type) { - duckdb_type = reader->DeriveLogicalType(column, column_schema).ToString(); - } - current_chunk.SetValue(11, count, duckdb_type); - - count++; - if (count >= STANDARD_VECTOR_SIZE) { - current_chunk.SetCardinality(count); - collection.Append(current_chunk); - - count = 0; - current_chunk.Reset(); - } - } - current_chunk.SetCardinality(count); - collection.Append(current_chunk); +idx_t ParquetSchemaProcessor::TotalRowCount() { + return reader->GetFileMetadata()->schema.size(); +} - collection.InitializeScan(scan_state); +void ParquetSchemaProcessor::ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) { + auto meta_data = reader->GetFileMetadata(); + const auto &column = meta_data->schema[row_idx]; + + // file_name + output.SetValue(0, output_idx, reader->file.path); + // name + output.SetValue(1, output_idx, column.name); + // type + output.SetValue(2, output_idx, ParquetElementString(column.type, column.__isset.type)); + // type_length + output.SetValue(3, output_idx, ParquetElementInteger(column.type_length, column.__isset.type_length)); + // repetition_type + output.SetValue(4, output_idx, ParquetElementString(column.repetition_type, column.__isset.repetition_type)); + // num_children + output.SetValue(5, output_idx, ParquetElementBigint(column.num_children, column.__isset.num_children)); + // converted_type + output.SetValue(6, output_idx, ParquetElementString(column.converted_type, column.__isset.converted_type)); + // scale + output.SetValue(7, output_idx, ParquetElementBigint(column.scale, column.__isset.scale)); + // precision + output.SetValue(8, output_idx, ParquetElementBigint(column.precision, column.__isset.precision)); + // field_id + output.SetValue(9, output_idx, ParquetElementBigint(column.field_id, column.__isset.field_id)); + // logical_type + output.SetValue(10, output_idx, ParquetLogicalTypeToString(column.logicalType, column.__isset.logicalType)); + // duckdb_type + ParquetColumnSchema column_schema; + Value duckdb_type; + if (column.__isset.type) { + duckdb_type = reader->DeriveLogicalType(column, column_schema).ToString(); + } + output.SetValue(11, output_idx, duckdb_type); + // column_id + output.SetValue(12, output_idx, Value::BIGINT(UnsafeNumericCast(row_idx))); } //===--------------------------------------------------------------------===// // KV Meta Data //===--------------------------------------------------------------------===// -void ParquetMetaDataOperatorData::BindKeyValueMetaData(vector &return_types, vector &names) { + +class ParquetKeyValueMetadataProcessor : public ParquetMetadataFileProcessor { +public: + idx_t TotalRowCount() override; + void ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) override; +}; + +template <> +void ParquetMetaDataOperator::BindSchema( + vector &return_types, vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -544,41 +640,32 @@ void ParquetMetaDataOperatorData::BindKeyValueMetaData(vector &retu return_types.emplace_back(LogicalType::BLOB); } -void ParquetMetaDataOperatorData::LoadKeyValueMetaData(ClientContext &context, const vector &return_types, - const OpenFileInfo &file) { - collection.Reset(); - ParquetOptions parquet_options(context); - auto reader = make_uniq(context, file.path, parquet_options); - idx_t count = 0; - DataChunk current_chunk; - current_chunk.Initialize(context, return_types); - auto meta_data = reader->GetFileMetadata(); - - for (idx_t col_idx = 0; col_idx < meta_data->key_value_metadata.size(); col_idx++) { - auto &entry = meta_data->key_value_metadata[col_idx]; - - current_chunk.SetValue(0, count, Value(file.path)); - current_chunk.SetValue(1, count, Value::BLOB_RAW(entry.key)); - current_chunk.SetValue(2, count, Value::BLOB_RAW(entry.value)); +idx_t ParquetKeyValueMetadataProcessor::TotalRowCount() { + return reader->GetFileMetadata()->key_value_metadata.size(); +} - count++; - if (count >= STANDARD_VECTOR_SIZE) { - current_chunk.SetCardinality(count); - collection.Append(current_chunk); +void ParquetKeyValueMetadataProcessor::ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) { + auto meta_data = reader->GetFileMetadata(); + auto &entry = meta_data->key_value_metadata[row_idx]; - count = 0; - current_chunk.Reset(); - } - } - current_chunk.SetCardinality(count); - collection.Append(current_chunk); - collection.InitializeScan(scan_state); + output.SetValue(0, output_idx, Value(reader->file.path)); + output.SetValue(1, output_idx, Value::BLOB_RAW(entry.key)); + output.SetValue(2, output_idx, Value::BLOB_RAW(entry.value)); } //===--------------------------------------------------------------------===// // File Meta Data //===--------------------------------------------------------------------===// -void ParquetMetaDataOperatorData::BindFileMetaData(vector &return_types, vector &names) { + +class ParquetFileMetadataProcessor : public ParquetMetadataFileProcessor { +public: + idx_t TotalRowCount() override; + void ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) override; +}; + +template <> +void ParquetMetaDataOperator::BindSchema(vector &return_types, + vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -607,46 +694,61 @@ void ParquetMetaDataOperatorData::BindFileMetaData(vector &return_t return_types.emplace_back(LogicalType::UBIGINT); } -void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const vector &return_types, - const OpenFileInfo &file) { - collection.Reset(); - ParquetOptions parquet_options(context); - auto reader = make_uniq(context, file.path, parquet_options); - DataChunk current_chunk; - current_chunk.Initialize(context, return_types); +idx_t ParquetFileMetadataProcessor::TotalRowCount() { + return 1; +} + +void ParquetFileMetadataProcessor::ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) { auto meta_data = reader->GetFileMetadata(); - // file_name - current_chunk.SetValue(0, 0, Value(file.path)); - // created_by - current_chunk.SetValue(1, 0, ParquetElementStringVal(meta_data->created_by, meta_data->__isset.created_by)); - // num_rows - current_chunk.SetValue(2, 0, Value::BIGINT(meta_data->num_rows)); - // num_row_groups - current_chunk.SetValue(3, 0, Value::BIGINT(UnsafeNumericCast(meta_data->row_groups.size()))); - // format_version - current_chunk.SetValue(4, 0, Value::BIGINT(meta_data->version)); - // encryption_algorithm - current_chunk.SetValue( - 5, 0, ParquetElementString(meta_data->encryption_algorithm, meta_data->__isset.encryption_algorithm)); - // footer_signing_key_metadata - current_chunk.SetValue(6, 0, - ParquetElementStringVal(meta_data->footer_signing_key_metadata, - meta_data->__isset.footer_signing_key_metadata)); - // file_size_bytes - current_chunk.SetValue(7, 0, Value::UBIGINT(reader->GetHandle().GetFileSize())); - // footer_size - current_chunk.SetValue(8, 0, Value::UBIGINT(reader->metadata->footer_size)); - - current_chunk.SetCardinality(1); - collection.Append(current_chunk); - collection.InitializeScan(scan_state); + // file_name + output.SetValue(0, output_idx, Value(reader->file.path)); + // created_by + output.SetValue(1, output_idx, ParquetElementStringVal(meta_data->created_by, meta_data->__isset.created_by)); + // num_rows + output.SetValue(2, output_idx, Value::BIGINT(meta_data->num_rows)); + // num_row_groups + output.SetValue(3, output_idx, Value::BIGINT(UnsafeNumericCast(meta_data->row_groups.size()))); + // format_version + output.SetValue(4, output_idx, Value::BIGINT(meta_data->version)); + // encryption_algorithm + output.SetValue(5, output_idx, + ParquetElementString(meta_data->encryption_algorithm, meta_data->__isset.encryption_algorithm)); + // footer_signing_key_metadata + output.SetValue(6, output_idx, + ParquetElementStringVal(meta_data->footer_signing_key_metadata, + meta_data->__isset.footer_signing_key_metadata)); + // file_size_bytes + output.SetValue(7, output_idx, Value::UBIGINT(reader->GetHandle().GetFileSize())); + // footer_size + output.SetValue(8, output_idx, Value::UBIGINT(reader->metadata->footer_size)); } //===--------------------------------------------------------------------===// // Bloom Probe //===--------------------------------------------------------------------===// -void ParquetMetaDataOperatorData::BindBloomProbe(vector &return_types, vector &names) { + +class ParquetBloomProbeProcessor : public ParquetMetadataFileProcessor { +public: + ParquetBloomProbeProcessor(const string &probe_column, const Value &probe_value); + + void InitializeInternal(ClientContext &context) override; + idx_t TotalRowCount() override; + void ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) override; + +private: + string probe_column_name; + Value probe_constant; + optional_idx probe_column_idx; + + unique_ptr> protocol; + optional_ptr allocator; + unique_ptr filter; +}; + +template <> +void ParquetMetaDataOperator::BindSchema(vector &return_types, + vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -657,84 +759,64 @@ void ParquetMetaDataOperatorData::BindBloomProbe(vector &return_typ return_types.emplace_back(LogicalType::BOOLEAN); } -void ParquetMetaDataOperatorData::ExecuteBloomProbe(ClientContext &context, const vector &return_types, - const OpenFileInfo &file, const string &column_name, - const Value &probe) { - collection.Reset(); - ParquetOptions parquet_options(context); - auto reader = make_uniq(context, file.path, parquet_options); - idx_t count = 0; - DataChunk current_chunk; - current_chunk.Initialize(context, return_types); - auto meta_data = reader->GetFileMetadata(); +ParquetBloomProbeProcessor::ParquetBloomProbeProcessor(const string &probe_column, const Value &probe_value) + : probe_column_name(probe_column), probe_constant(probe_value) { +} + +void ParquetBloomProbeProcessor::InitializeInternal(ClientContext &context) { + probe_column_idx = optional_idx::Invalid(); - optional_idx probe_column_idx; for (idx_t column_idx = 0; column_idx < reader->columns.size(); column_idx++) { - if (reader->columns[column_idx].name == column_name) { + if (reader->columns[column_idx].name == probe_column_name) { probe_column_idx = column_idx; + break; } } if (!probe_column_idx.IsValid()) { - throw InvalidInputException("Column %s not found in %s", column_name, file.path); + throw InvalidInputException("Column %s not found in %s", probe_column_name, reader->file.path); } - auto &allocator = BufferAllocator::Get(context); auto transport = duckdb_base_std::make_shared(reader->GetHandle(), false); - auto protocol = - make_uniq>(std::move(transport)); - - D_ASSERT(!probe.IsNull()); - ConstantFilter filter(ExpressionType::COMPARE_EQUAL, - probe.CastAs(context, reader->GetColumns()[probe_column_idx.GetIndex()].type)); - - for (idx_t row_group_idx = 0; row_group_idx < meta_data->row_groups.size(); row_group_idx++) { - auto &row_group = meta_data->row_groups[row_group_idx]; - auto &column = row_group.columns[probe_column_idx.GetIndex()]; - - auto bloom_excludes = - ParquetStatisticsUtils::BloomFilterExcludes(filter, column.meta_data, *protocol, allocator); - current_chunk.SetValue(0, count, Value(file.path)); - current_chunk.SetValue(1, count, Value::BIGINT(NumericCast(row_group_idx))); - current_chunk.SetValue(2, count, Value::BOOLEAN(bloom_excludes)); - - count++; - if (count >= STANDARD_VECTOR_SIZE) { - current_chunk.SetCardinality(count); - collection.Append(current_chunk); - - count = 0; - current_chunk.Reset(); - } - } + protocol = make_uniq>(std::move(transport)); + allocator = &BufferAllocator::Get(context); + filter = make_uniq( + ExpressionType::COMPARE_EQUAL, + probe_constant.CastAs(context, reader->GetColumns()[probe_column_idx.GetIndex()].type)); +} + +idx_t ParquetBloomProbeProcessor::TotalRowCount() { + return reader->GetFileMetadata()->row_groups.size(); +} + +void ParquetBloomProbeProcessor::ReadRow(DataChunk &output, idx_t output_idx, idx_t row_idx) { + auto meta_data = reader->GetFileMetadata(); + auto &row_group = meta_data->row_groups[row_idx]; + auto &column = row_group.columns[probe_column_idx.GetIndex()]; - current_chunk.SetCardinality(count); - collection.Append(current_chunk); - collection.InitializeScan(scan_state); + D_ASSERT(!probe_constant.IsNull()); + + auto bloom_excludes = ParquetStatisticsUtils::BloomFilterExcludes(*filter, column.meta_data, *protocol, *allocator); + + output.SetValue(0, output_idx, Value(reader->file.path)); + output.SetValue(1, output_idx, Value::BIGINT(NumericCast(row_idx))); + output.SetValue(2, output_idx, Value::BOOLEAN(bloom_excludes)); } //===--------------------------------------------------------------------===// -// Bind +// Template Function Implementation //===--------------------------------------------------------------------===// -template -static unique_ptr ParquetMetaDataBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto result = make_uniq(); - switch (TYPE) { - case ParquetMetadataOperatorType::SCHEMA: - ParquetMetaDataOperatorData::BindSchema(return_types, names); - break; - case ParquetMetadataOperatorType::META_DATA: - ParquetMetaDataOperatorData::BindMetaData(return_types, names); - break; - case ParquetMetadataOperatorType::KEY_VALUE_META_DATA: - ParquetMetaDataOperatorData::BindKeyValueMetaData(return_types, names); - break; - case ParquetMetadataOperatorType::FILE_META_DATA: - ParquetMetaDataOperatorData::BindFileMetaData(return_types, names); - break; - case ParquetMetadataOperatorType::BLOOM_PROBE: { +template +unique_ptr ParquetMetaDataOperator::Bind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + // Extract file paths from input using MultiFileReader (handles both single files and arrays) + auto multi_file_reader = MultiFileReader::CreateDefault("ParquetMetadata"); + auto glob_input = FileGlobInput(FileGlobOptions::FALLBACK_GLOB, "parquet"); + + auto result = make_uniq(); + // Bind schema based on operation type + if (OP_TYPE == ParquetMetadataOperatorType::BLOOM_PROBE) { auto probe_bind_data = make_uniq(); D_ASSERT(input.inputs.size() == 3); if (input.inputs[1].IsNull() || input.inputs[2].IsNull()) { @@ -743,133 +825,145 @@ static unique_ptr ParquetMetaDataBind(ClientContext &context, Tabl probe_bind_data->probe_column_name = input.inputs[1].CastAs(context, LogicalType::VARCHAR).GetValue(); probe_bind_data->probe_constant = input.inputs[2]; result = std::move(probe_bind_data); - ParquetMetaDataOperatorData::BindBloomProbe(return_types, names); - break; - } - default: - throw InternalException("Unsupported ParquetMetadataOperatorType"); } - result->return_types = return_types; - result->multi_file_reader = MultiFileReader::Create(input.table_function); - result->file_list = result->multi_file_reader->CreateFileList(context, input.inputs[0]); + result->file_paths = make_uniq(); + result->file_paths->file_list = multi_file_reader->CreateFileList(context, input.inputs[0], glob_input); + D_ASSERT(!result->file_paths->file_list->IsEmpty()); + result->file_paths->file_list->InitializeScan(result->file_paths->scan_data); + + BindSchema(return_types, names); return std::move(result); } -template -static unique_ptr ParquetMetaDataInit(ClientContext &context, TableFunctionInitInput &input) { - auto &bind_data = input.bind_data->Cast(); - - auto result = make_uniq(context, bind_data.return_types); - - bind_data.file_list->InitializeScan(result->file_list_scan); - bind_data.file_list->Scan(result->file_list_scan, result->current_file); - - D_ASSERT(!bind_data.file_list->IsEmpty()); +unique_ptr ParquetMetaDataOperator::InitGlobal(ClientContext &context, + TableFunctionInitInput &input) { + auto &bind_data = input.bind_data->CastNoConst(); + return make_uniq(std::move(bind_data.file_paths), context); +} - switch (TYPE) { - case ParquetMetadataOperatorType::SCHEMA: - result->LoadSchemaData(context, bind_data.return_types, bind_data.file_list->GetFirstFile()); - break; +template +unique_ptr ParquetMetaDataOperator::InitLocal(ExecutionContext &context, + TableFunctionInitInput &input, + GlobalTableFunctionState *global_state) { + auto &bind_data = input.bind_data->Cast(); + auto res = make_uniq(); + switch (OP_TYPE) { case ParquetMetadataOperatorType::META_DATA: - result->LoadRowGroupMetadata(context, bind_data.return_types, bind_data.file_list->GetFirstFile()); + res->processor = make_uniq(); + break; + case ParquetMetadataOperatorType::SCHEMA: + res->processor = make_uniq(); break; case ParquetMetadataOperatorType::KEY_VALUE_META_DATA: - result->LoadKeyValueMetaData(context, bind_data.return_types, bind_data.file_list->GetFirstFile()); + res->processor = make_uniq(); break; case ParquetMetadataOperatorType::FILE_META_DATA: - result->LoadFileMetaData(context, bind_data.return_types, bind_data.file_list->GetFirstFile()); + res->processor = make_uniq(); break; case ParquetMetadataOperatorType::BLOOM_PROBE: { - auto &bloom_probe_bind_data = input.bind_data->Cast(); - result->ExecuteBloomProbe(context, bind_data.return_types, bind_data.file_list->GetFirstFile(), - bloom_probe_bind_data.probe_column_name, bloom_probe_bind_data.probe_constant); + const auto &probe_bind_data = static_cast(bind_data); + res->processor = + make_uniq(probe_bind_data.probe_column_name, probe_bind_data.probe_constant); break; } default: throw InternalException("Unsupported ParquetMetadataOperatorType"); } - - return std::move(result); + return unique_ptr_cast(std::move(res)); } -template -static void ParquetMetaDataImplementation(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = data_p.global_state->Cast(); - auto &bind_data = data_p.bind_data->Cast(); +template +void ParquetMetaDataOperator::Function(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &global_state = data_p.global_state->Cast(); + auto &local_state = data_p.local_state->Cast(); - while (true) { - if (!data.collection.Scan(data.scan_state, output)) { + idx_t output_count = 0; - // Try get next file - if (!bind_data.file_list->Scan(data.file_list_scan, data.current_file)) { - return; + while (output_count < STANDARD_VECTOR_SIZE) { + // Check if we need a new file + if (local_state.file_exhausted) { + OpenFileInfo next_file; + if (!global_state.file_paths->NextFile(next_file)) { + break; // No more files to process } - switch (TYPE) { - case ParquetMetadataOperatorType::SCHEMA: - data.LoadSchemaData(context, bind_data.return_types, data.current_file); - break; - case ParquetMetadataOperatorType::META_DATA: - data.LoadRowGroupMetadata(context, bind_data.return_types, data.current_file); - break; - case ParquetMetadataOperatorType::KEY_VALUE_META_DATA: - data.LoadKeyValueMetaData(context, bind_data.return_types, data.current_file); - break; - case ParquetMetadataOperatorType::FILE_META_DATA: - data.LoadFileMetaData(context, bind_data.return_types, data.current_file); - break; - case ParquetMetadataOperatorType::BLOOM_PROBE: { - auto &bloom_probe_bind_data = data_p.bind_data->Cast(); - data.ExecuteBloomProbe(context, bind_data.return_types, data.current_file, - bloom_probe_bind_data.probe_column_name, bloom_probe_bind_data.probe_constant); - break; - } - default: - throw InternalException("Unsupported ParquetMetadataOperatorType"); - } - continue; + local_state.processor->Initialize(context, next_file); + local_state.processor->InitializeInternal(context); + local_state.file_exhausted = false; + local_state.row_idx = 0; + local_state.total_rows = local_state.processor->TotalRowCount(); } - if (output.size() != 0) { - return; + + idx_t left_in_vector = STANDARD_VECTOR_SIZE - output_count; + idx_t left_in_file = local_state.total_rows - local_state.row_idx; + idx_t rows_to_output = 0; + if (left_in_file <= left_in_vector) { + local_state.file_exhausted = true; + rows_to_output = left_in_file; + } else { + rows_to_output = left_in_vector; } + + for (idx_t i = 0; i < rows_to_output; ++i) { + local_state.processor->ReadRow(output, output_count + i, local_state.row_idx + i); + } + output_count += rows_to_output; + local_state.row_idx += rows_to_output; } + + output.SetCardinality(output_count); +} + +double ParquetMetaDataOperator::Progress(ClientContext &context, const FunctionData *bind_data_p, + const GlobalTableFunctionState *global_state) { + auto &global_data = global_state->Cast(); + return global_data.GetProgress() * 100.0; } ParquetMetaDataFunction::ParquetMetaDataFunction() : TableFunction("parquet_metadata", {LogicalType::VARCHAR}, - ParquetMetaDataImplementation, - ParquetMetaDataBind, - ParquetMetaDataInit) { + ParquetMetaDataOperator::Function, + ParquetMetaDataOperator::Bind, + ParquetMetaDataOperator::InitGlobal, + ParquetMetaDataOperator::InitLocal) { + table_scan_progress = ParquetMetaDataOperator::Progress; } ParquetSchemaFunction::ParquetSchemaFunction() : TableFunction("parquet_schema", {LogicalType::VARCHAR}, - ParquetMetaDataImplementation, - ParquetMetaDataBind, - ParquetMetaDataInit) { + ParquetMetaDataOperator::Function, + ParquetMetaDataOperator::Bind, + ParquetMetaDataOperator::InitGlobal, + ParquetMetaDataOperator::InitLocal) { + table_scan_progress = ParquetMetaDataOperator::Progress; } ParquetKeyValueMetadataFunction::ParquetKeyValueMetadataFunction() : TableFunction("parquet_kv_metadata", {LogicalType::VARCHAR}, - ParquetMetaDataImplementation, - ParquetMetaDataBind, - ParquetMetaDataInit) { + ParquetMetaDataOperator::Function, + ParquetMetaDataOperator::Bind, + ParquetMetaDataOperator::InitGlobal, + ParquetMetaDataOperator::InitLocal) { + table_scan_progress = ParquetMetaDataOperator::Progress; } ParquetFileMetadataFunction::ParquetFileMetadataFunction() : TableFunction("parquet_file_metadata", {LogicalType::VARCHAR}, - ParquetMetaDataImplementation, - ParquetMetaDataBind, - ParquetMetaDataInit) { + ParquetMetaDataOperator::Function, + ParquetMetaDataOperator::Bind, + ParquetMetaDataOperator::InitGlobal, + ParquetMetaDataOperator::InitLocal) { + table_scan_progress = ParquetMetaDataOperator::Progress; } ParquetBloomProbeFunction::ParquetBloomProbeFunction() : TableFunction("parquet_bloom_probe", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::ANY}, - ParquetMetaDataImplementation, - ParquetMetaDataBind, - ParquetMetaDataInit) { + ParquetMetaDataOperator::Function, + ParquetMetaDataOperator::Bind, + ParquetMetaDataOperator::InitGlobal, + ParquetMetaDataOperator::InitLocal) { + table_scan_progress = ParquetMetaDataOperator::Progress; } - } // namespace duckdb diff --git a/extension/parquet/parquet_multi_file_info.cpp b/extension/parquet/parquet_multi_file_info.cpp index 9617f0c83c54..839a94f5fe65 100644 --- a/extension/parquet/parquet_multi_file_info.cpp +++ b/extension/parquet/parquet_multi_file_info.cpp @@ -397,10 +397,6 @@ bool ParquetMultiFileInfo::ParseOption(ClientContext &context, const string &ori options.binary_as_string = BooleanValue::Get(val); return true; } - if (key == "variant_legacy_encoding") { - options.variant_legacy_encoding = BooleanValue::Get(val); - return true; - } if (key == "file_row_number") { options.file_row_number = BooleanValue::Get(val); return true; diff --git a/extension/parquet/parquet_reader.cpp b/extension/parquet/parquet_reader.cpp index efe183aab6ce..81eab1b3a811 100644 --- a/extension/parquet/parquet_reader.cpp +++ b/extension/parquet/parquet_reader.cpp @@ -92,7 +92,7 @@ static shared_ptr LoadMetadata(ClientContext &context, Allocator &allocator, CachingFileHandle &file_handle, const shared_ptr &encryption_config, const EncryptionUtil &encryption_util, optional_idx footer_size) { - auto file_proto = CreateThriftFileProtocol(QueryContext(context), file_handle, false); + auto file_proto = CreateThriftFileProtocol(context, file_handle, false); auto &transport = reinterpret_cast(*file_proto->getTransport()); auto file_size = transport.GetSize(); if (file_size < 12) { @@ -225,6 +225,10 @@ LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, Parquet return LogicalType::TIME_TZ; } return LogicalType::TIME; + } else if (s_ele.logicalType.__isset.GEOMETRY) { + return LogicalType::BLOB; + } else if (s_ele.logicalType.__isset.GEOGRAPHY) { + return LogicalType::BLOB; } } if (s_ele.__isset.converted_type) { @@ -403,7 +407,7 @@ unique_ptr ParquetReader::CreateReaderRecursive(ClientContext &con const ParquetColumnSchema &schema) { switch (schema.schema_type) { case ParquetColumnSchemaType::GEOMETRY: - return metadata->geo_metadata->CreateColumnReader(*this, schema, context); + return GeoParquetFileMetadata::CreateColumnReader(*this, schema, context); case ParquetColumnSchemaType::FILE_ROW_NUMBER: return make_uniq(*this, schema); case ParquetColumnSchemaType::COLUMN: { @@ -514,58 +518,16 @@ unique_ptr ParquetColumnSchema::Stats(const FileMetaData &file_m return ParquetStatisticsUtils::TransformColumnStatistics(*this, columns, parquet_options.can_have_nan); } -static bool IsVariantType(const SchemaElement &root, const vector &children) { - if (children.size() < 2) { - return false; - } - auto &child0 = children[0]; - auto &child1 = children[1]; - - ParquetColumnSchema const *metadata; - ParquetColumnSchema const *value; - - if (child0.name == "metadata" && child1.name == "value") { - metadata = &child0; - value = &child1; - } else if (child1.name == "metadata" && child0.name == "value") { - metadata = &child1; - value = &child0; - } else { - return false; - } - - //! Verify names - if (metadata->name != "metadata") { - return false; - } - if (value->name != "value") { - return false; - } - - //! Verify types - if (metadata->parquet_type != duckdb_parquet::Type::BYTE_ARRAY) { - return false; - } - if (value->parquet_type != duckdb_parquet::Type::BYTE_ARRAY) { - return false; - } - if (children.size() == 3) { - auto &typed_value = children[2]; - if (typed_value.name != "typed_value") { - return false; - } - } else if (children.size() != 2) { - return false; - } - return true; -} - ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_define, idx_t max_repeat, - idx_t &next_schema_idx, idx_t &next_file_idx) { + idx_t &next_schema_idx, idx_t &next_file_idx, + ClientContext &context) { auto file_meta_data = GetFileMetadata(); D_ASSERT(file_meta_data); - D_ASSERT(next_schema_idx < file_meta_data->schema.size()); + if (next_schema_idx >= file_meta_data->schema.size()) { + throw InvalidInputException("Malformed Parquet schema in file \"%s\": invalid schema index %d", file.path, + next_schema_idx); + } auto &s_ele = file_meta_data->schema[next_schema_idx]; auto this_idx = next_schema_idx; @@ -583,7 +545,7 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d // Check for geoparquet spatial types if (depth == 1) { // geoparquet types have to be at the root of the schema, and have to be present in the kv metadata. - // geoarrow types, although geometry columns, are structs and have chilren and are handled below. + // geoarrow types, although geometry columns, are structs and have children and are handled below. if (metadata->geo_metadata && metadata->geo_metadata->IsGeometryColumn(s_ele.name) && s_ele.num_children == 0) { auto root_schema = ParseColumnSchema(s_ele, max_define, max_repeat, this_idx, next_file_idx++); return ParquetColumnSchema(std::move(root_schema), GeoParquetFileMetadata::GeometryType(), @@ -598,7 +560,8 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d while (c_idx < NumericCast(s_ele.num_children)) { next_schema_idx++; - auto child_schema = ParseSchemaRecursive(depth + 1, max_define, max_repeat, next_schema_idx, next_file_idx); + auto child_schema = + ParseSchemaRecursive(depth + 1, max_define, max_repeat, next_schema_idx, next_file_idx, context); child_schemas.push_back(std::move(child_schema)); c_idx++; } @@ -621,9 +584,6 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d const bool is_map = s_ele.__isset.converted_type && s_ele.converted_type == ConvertedType::MAP; bool is_map_kv = s_ele.__isset.converted_type && s_ele.converted_type == ConvertedType::MAP_KEY_VALUE; bool is_variant = s_ele.__isset.logicalType && s_ele.logicalType.__isset.VARIANT == true; - if (!is_variant) { - is_variant = parquet_options.variant_legacy_encoding && IsVariantType(s_ele, child_schemas); - } if (!is_map_kv && this_idx > 0) { // check if the parent node of this is a map @@ -659,7 +619,7 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d LogicalType result_type; if (is_variant) { - result_type = LogicalType::JSON(); + result_type = LogicalType::VARIANT(); } else { result_type = LogicalType::STRUCT(std::move(struct_types)); } @@ -698,6 +658,14 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d list_schema.children.push_back(std::move(result)); return list_schema; } + + // Convert to geometry type if possible + if (s_ele.__isset.logicalType && (s_ele.logicalType.__isset.GEOMETRY || s_ele.logicalType.__isset.GEOGRAPHY) && + GeoParquetFileMetadata::IsGeoParquetConversionEnabled(context)) { + return ParquetColumnSchema(std::move(result), GeoParquetFileMetadata::GeometryType(), + ParquetColumnSchemaType::GEOMETRY); + } + return result; } } @@ -707,29 +675,34 @@ static ParquetColumnSchema FileRowNumberSchema() { ParquetColumnSchemaType::FILE_ROW_NUMBER); } -unique_ptr ParquetReader::ParseSchema() { +unique_ptr ParquetReader::ParseSchema(ClientContext &context) { auto file_meta_data = GetFileMetadata(); idx_t next_schema_idx = 0; idx_t next_file_idx = 0; if (file_meta_data->schema.empty()) { - throw IOException("Parquet reader: no schema elements found"); + throw IOException("Failed to read Parquet file \"%s\": no schema elements found", file.path); } if (file_meta_data->schema[0].num_children == 0) { - throw IOException("Parquet reader: root schema element has no children"); + throw IOException("Failed to read Parquet file \"%s\": root schema element has no children", file.path); } - auto root = ParseSchemaRecursive(0, 0, 0, next_schema_idx, next_file_idx); + auto root = ParseSchemaRecursive(0, 0, 0, next_schema_idx, next_file_idx, context); if (root.type.id() != LogicalTypeId::STRUCT) { - throw InvalidInputException("Root element of Parquet file must be a struct"); + throw InvalidInputException("Failed to read Parquet file \"%s\": Root element of Parquet file must be a struct", + file.path); } D_ASSERT(next_schema_idx == file_meta_data->schema.size() - 1); - D_ASSERT(file_meta_data->row_groups.empty() || next_file_idx == file_meta_data->row_groups[0].columns.size()); + if (!file_meta_data->row_groups.empty() && next_file_idx != file_meta_data->row_groups[0].columns.size()) { + throw InvalidInputException("Failed to read Parquet file \"%s\": row group does not have enough columns", + file.path); + } if (parquet_options.file_row_number) { for (auto &column : root.children) { auto &name = column.name; if (StringUtil::CIEquals(name, "file_row_number")) { - throw BinderException( - "Using file_row_number option on file with column named file_row_number is not supported"); + throw BinderException("Failed to read Parquet file \"%s\": Using file_row_number option on file with " + "column named file_row_number is not supported", + file.path); } } root.children.push_back(FileRowNumberSchema()); @@ -774,7 +747,7 @@ void ParquetReader::InitializeSchema(ClientContext &context) { throw InvalidInputException("Failed to read Parquet file '%s': Need at least one non-root column in the file", GetFileName()); } - root_schema = ParseSchema(); + root_schema = ParseSchema(context); for (idx_t i = 0; i < root_schema->children.size(); i++) { auto &element = root_schema->children[i]; columns.push_back(ParseColumnDefinition(*file_meta_data, element)); @@ -794,9 +767,6 @@ ParquetOptions::ParquetOptions(ClientContext &context) { if (context.TryGetCurrentSetting("binary_as_string", lookup_value)) { binary_as_string = lookup_value.GetValue(); } - if (context.TryGetCurrentSetting("variant_legacy_encoding", lookup_value)) { - variant_legacy_encoding = lookup_value.GetValue(); - } } ParquetColumnDefinition ParquetColumnDefinition::FromSchemaValue(ClientContext &context, const Value &column_value) { @@ -823,7 +793,7 @@ ParquetReader::ParquetReader(ClientContext &context_p, OpenFileInfo file_p, Parq shared_ptr metadata_p) : BaseFileReader(std::move(file_p)), fs(CachingFileSystem::Get(context_p)), allocator(BufferAllocator::Get(context_p)), parquet_options(std::move(parquet_options_p)) { - file_handle = fs.OpenFile(QueryContext(context_p), file, FileFlags::FILE_FLAGS_READ); + file_handle = fs.OpenFile(context_p, file, FileFlags::FILE_FLAGS_READ); if (!file_handle->CanSeek()) { throw NotImplementedException( "Reading parquet files from a FIFO stream is not supported and cannot be efficiently supported since " @@ -1222,7 +1192,7 @@ void ParquetReader::InitializeScan(ClientContext &context, ParquetReaderScanStat state.prefetch_mode = false; } - state.file_handle = fs.OpenFile(QueryContext(context), file, flags); + state.file_handle = fs.OpenFile(context, file, flags); } state.adaptive_filter.reset(); state.scan_filters.clear(); @@ -1233,7 +1203,7 @@ void ParquetReader::InitializeScan(ClientContext &context, ParquetReaderScanStat } } - state.thrift_file_proto = CreateThriftFileProtocol(QueryContext(context), *state.file_handle, state.prefetch_mode); + state.thrift_file_proto = CreateThriftFileProtocol(context, *state.file_handle, state.prefetch_mode); state.root_reader = CreateReader(context); state.define_buf.resize(allocator, STANDARD_VECTOR_SIZE); state.repeat_buf.resize(allocator, STANDARD_VECTOR_SIZE); diff --git a/extension/parquet/parquet_shredding.cpp b/extension/parquet/parquet_shredding.cpp new file mode 100644 index 000000000000..b7ed673a88ce --- /dev/null +++ b/extension/parquet/parquet_shredding.cpp @@ -0,0 +1,81 @@ +#include "parquet_shredding.hpp" +#include "duckdb/common/exception/binder_exception.hpp" +#include "duckdb/common/type_visitor.hpp" + +namespace duckdb { + +ChildShreddingTypes::ChildShreddingTypes() : types(make_uniq>()) { +} + +ChildShreddingTypes ChildShreddingTypes::Copy() const { + ChildShreddingTypes result; + for (const auto &type : *types) { + result.types->emplace(type.first, type.second.Copy()); + } + return result; +} + +ShreddingType::ShreddingType() : set(false) { +} + +ShreddingType::ShreddingType(const LogicalType &type) : set(true), type(type) { +} + +ShreddingType ShreddingType::Copy() const { + auto result = set ? ShreddingType(type) : ShreddingType(); + result.children = children.Copy(); + return result; +} + +static ShreddingType ConvertShreddingTypeRecursive(const LogicalType &type) { + if (type.id() == LogicalTypeId::VARIANT) { + return ShreddingType(LogicalType(LogicalTypeId::ANY)); + } + if (!type.IsNested()) { + return ShreddingType(type); + } + + switch (type.id()) { + case LogicalTypeId::STRUCT: { + ShreddingType res(type); + auto &children = StructType::GetChildTypes(type); + for (auto &entry : children) { + res.AddChild(entry.first, ConvertShreddingTypeRecursive(entry.second)); + } + return res; + } + case LogicalTypeId::LIST: { + ShreddingType res(type); + const auto &child = ListType::GetChildType(type); + res.AddChild("element", ConvertShreddingTypeRecursive(child)); + return res; + } + default: + break; + } + throw BinderException("VARIANT can only be shredded on LIST/STRUCT/ANY/non-nested type, not %s", type.ToString()); +} + +void ShreddingType::AddChild(const string &name, ShreddingType &&child) { + children.types->emplace(name, std::move(child)); +} + +optional_ptr ShreddingType::GetChild(const string &name) const { + auto it = children.types->find(name); + if (it == children.types->end()) { + return nullptr; + } + return it->second; +} + +ShreddingType ShreddingType::GetShreddingTypes(const Value &val) { + if (val.type().id() != LogicalTypeId::VARCHAR) { + throw BinderException("SHREDDING value should be of type VARCHAR, a stringified type to use for the column"); + } + auto type_str = val.GetValue(); + auto logical_type = TransformStringToLogicalType(type_str); + + return ConvertShreddingTypeRecursive(logical_type); +} + +} // namespace duckdb diff --git a/extension/parquet/parquet_statistics.cpp b/extension/parquet/parquet_statistics.cpp index 5f7d93718ba6..a22613271cb7 100644 --- a/extension/parquet/parquet_statistics.cpp +++ b/extension/parquet/parquet_statistics.cpp @@ -395,23 +395,21 @@ unique_ptr ParquetStatisticsUtils::TransformColumnStatistics(con } break; case LogicalTypeId::VARCHAR: { - auto string_stats = StringStats::CreateEmpty(type); + auto string_stats = StringStats::CreateUnknown(type); if (parquet_stats.__isset.min_value) { StringColumnReader::VerifyString(parquet_stats.min_value.c_str(), parquet_stats.min_value.size(), true); - StringStats::Update(string_stats, parquet_stats.min_value); + StringStats::SetMin(string_stats, parquet_stats.min_value); } else if (parquet_stats.__isset.min) { StringColumnReader::VerifyString(parquet_stats.min.c_str(), parquet_stats.min.size(), true); - StringStats::Update(string_stats, parquet_stats.min); + StringStats::SetMin(string_stats, parquet_stats.min); } if (parquet_stats.__isset.max_value) { StringColumnReader::VerifyString(parquet_stats.max_value.c_str(), parquet_stats.max_value.size(), true); - StringStats::Update(string_stats, parquet_stats.max_value); + StringStats::SetMax(string_stats, parquet_stats.max_value); } else if (parquet_stats.__isset.max) { StringColumnReader::VerifyString(parquet_stats.max.c_str(), parquet_stats.max.size(), true); - StringStats::Update(string_stats, parquet_stats.max); + StringStats::SetMax(string_stats, parquet_stats.max); } - StringStats::SetContainsUnicode(string_stats); - StringStats::ResetMaxStringLength(string_stats); row_group_stats = string_stats.ToUnique(); break; } diff --git a/extension/parquet/parquet_writer.cpp b/extension/parquet/parquet_writer.cpp index 4819a2274ca5..82cb6d2767dd 100644 --- a/extension/parquet/parquet_writer.cpp +++ b/extension/parquet/parquet_writer.cpp @@ -3,6 +3,7 @@ #include "duckdb.hpp" #include "mbedtls_wrapper.hpp" #include "parquet_crypto.hpp" +#include "parquet_shredding.hpp" #include "parquet_timestamp.hpp" #include "resizable_buffer.hpp" #include "duckdb/common/file_system.hpp" @@ -35,29 +36,6 @@ using duckdb_parquet::PageType; using ParquetRowGroup = duckdb_parquet::RowGroup; using duckdb_parquet::Type; -ChildFieldIDs::ChildFieldIDs() : ids(make_uniq>()) { -} - -ChildFieldIDs ChildFieldIDs::Copy() const { - ChildFieldIDs result; - for (const auto &id : *ids) { - result.ids->emplace(id.first, id.second.Copy()); - } - return result; -} - -FieldID::FieldID() : set(false) { -} - -FieldID::FieldID(int32_t field_id_p) : set(true), field_id(field_id_p) { -} - -FieldID FieldID::Copy() const { - auto result = set ? FieldID(field_id) : FieldID(); - result.child_field_ids = child_field_ids.Copy(); - return result; -} - class MyTransport : public TTransport { public: explicit MyTransport(WriteStream &serializer) : serializer(serializer) { @@ -166,7 +144,8 @@ Type::type ParquetWriter::DuckDBTypeToParquetType(const LogicalType &duckdb_type throw NotImplementedException("Unimplemented type for Parquet \"%s\"", duckdb_type.ToString()); } -void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::SchemaElement &schema_ele) { +void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::SchemaElement &schema_ele, + bool allow_geometry) { if (duckdb_type.IsJSONType()) { schema_ele.converted_type = ConvertedType::JSON; schema_ele.__isset.converted_type = true; @@ -174,6 +153,13 @@ void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type, duckdb_p schema_ele.logicalType.__set_JSON(duckdb_parquet::JsonType()); return; } + if (duckdb_type.GetAlias() == "WKB_BLOB" && allow_geometry) { + schema_ele.__isset.logicalType = true; + schema_ele.logicalType.__isset.GEOMETRY = true; + // TODO: Set CRS in the future + schema_ele.logicalType.GEOMETRY.__isset.crs = false; + return; + } switch (duckdb_type.id()) { case LogicalTypeId::TINYINT: schema_ele.converted_type = ConvertedType::INT_8; @@ -329,6 +315,11 @@ struct ColumnStatsUnifier { bool can_have_nan = false; bool has_nan = false; + unique_ptr geo_stats; + + virtual void UnifyGeoStats(const GeometryStatsData &other) { + } + virtual void UnifyMinMax(const string &new_min, const string &new_max) = 0; virtual string StatsToString(const string &stats) = 0; }; @@ -340,18 +331,21 @@ class ParquetStatsAccumulator { ParquetWriter::ParquetWriter(ClientContext &context, FileSystem &fs, string file_name_p, vector types_p, vector names_p, CompressionCodec::type codec, ChildFieldIDs field_ids_p, - const vector> &kv_metadata, + ShreddingType shredding_types_p, const vector> &kv_metadata, shared_ptr encryption_config_p, optional_idx dictionary_size_limit_p, idx_t string_dictionary_page_size_limit_p, bool enable_bloom_filters_p, double bloom_filter_false_positive_ratio_p, - int64_t compression_level_p, bool debug_use_openssl_p, ParquetVersion parquet_version) + int64_t compression_level_p, bool debug_use_openssl_p, ParquetVersion parquet_version, + GeoParquetVersion geoparquet_version) : context(context), file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec), field_ids(std::move(field_ids_p)), - encryption_config(std::move(encryption_config_p)), dictionary_size_limit(dictionary_size_limit_p), + shredding_types(std::move(shredding_types_p)), encryption_config(std::move(encryption_config_p)), + dictionary_size_limit(dictionary_size_limit_p), string_dictionary_page_size_limit(string_dictionary_page_size_limit_p), enable_bloom_filters(enable_bloom_filters_p), bloom_filter_false_positive_ratio(bloom_filter_false_positive_ratio_p), compression_level(compression_level_p), - debug_use_openssl(debug_use_openssl_p), parquet_version(parquet_version), total_written(0), num_row_groups(0) { + debug_use_openssl(debug_use_openssl_p), parquet_version(parquet_version), geoparquet_version(geoparquet_version), + total_written(0), num_row_groups(0) { // initialize the file writer writer = make_uniq(fs, file_name.c_str(), @@ -404,10 +398,13 @@ ParquetWriter::ParquetWriter(ClientContext &context, FileSystem &fs, string file auto &unique_names = column_names; VerifyUniqueNames(unique_names); + // V1 GeoParquet stores geometries as blobs, no logical type + auto allow_geometry = geoparquet_version != GeoParquetVersion::V1; + // construct the child schemas for (idx_t i = 0; i < sql_types.size(); i++) { - auto child_schema = - ColumnWriter::FillParquetSchema(file_meta_data.schema, sql_types[i], unique_names[i], &field_ids); + auto child_schema = ColumnWriter::FillParquetSchema(file_meta_data.schema, sql_types[i], unique_names[i], + allow_geometry, &field_ids, &shredding_types); column_schemas.push_back(std::move(child_schema)); } // now construct the writers based on the schemas @@ -447,7 +444,7 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro write_states.emplace_back(col_writers.back().get().InitializeWriteState(row_group)); } - for (auto &chunk : buffer.Chunks({column_ids})) { + for (auto &chunk : buffer.Chunks(column_ids)) { for (idx_t i = 0; i < next; i++) { if (col_writers[i].get().HasAnalyze()) { col_writers[i].get().Analyze(*write_states[i], nullptr, chunk.data[i], chunk.size()); @@ -672,6 +669,49 @@ struct BlobStatsUnifier : public BaseStringStatsUnifier { } }; +struct GeoStatsUnifier : public ColumnStatsUnifier { + + void UnifyGeoStats(const GeometryStatsData &other) override { + if (geo_stats) { + geo_stats->Merge(other); + } else { + // Make copy + geo_stats = make_uniq(); + geo_stats->extent = other.extent; + geo_stats->types = other.types; + } + } + + void UnifyMinMax(const string &new_min, const string &new_max) override { + // Do nothing + } + + string StatsToString(const string &stats) override { + if (!geo_stats) { + return string(); + } + + const auto &bbox = geo_stats->extent; + const auto &types = geo_stats->types; + + const auto bbox_value = Value::STRUCT({{"xmin", bbox.x_min}, + {"xmax", bbox.x_max}, + {"ymin", bbox.y_min}, + {"ymax", bbox.y_max}, + {"zmin", bbox.z_min}, + {"zmax", bbox.z_max}, + {"mmin", bbox.m_min}, + {"mmax", bbox.m_max}}); + + vector type_strings; + for (const auto &type : types.ToString(true)) { + type_strings.push_back(Value(StringUtil::Lower(type))); + } + + return Value::STRUCT({{"bbox", bbox_value}, {"types", Value::LIST(type_strings)}}).ToString(); + } +}; + struct UUIDStatsUnifier : public BaseStringStatsUnifier { string StatsToString(const string &stats) override { if (stats.size() != 16) { @@ -754,7 +794,11 @@ static unique_ptr GetBaseStatsUnifier(const LogicalType &typ } } case LogicalTypeId::BLOB: - return make_uniq(); + if (type.GetAlias() == "WKB_BLOB") { + return make_uniq(); + } else { + return make_uniq(); + } case LogicalTypeId::VARCHAR: return make_uniq(); case LogicalTypeId::UUID: @@ -811,6 +855,9 @@ void ParquetWriter::FlushColumnStats(idx_t col_idx, duckdb_parquet::ColumnChunk } else { stats_unifier->all_nulls_set = false; } + if (writer_stats && writer_stats->HasGeoStats()) { + stats_unifier->UnifyGeoStats(*writer_stats->GetGeoStats()); + } stats_unifier->column_size_bytes += column.meta_data.total_compressed_size; } } @@ -839,6 +886,36 @@ void ParquetWriter::GatherWrittenStatistics() { if (stats_unifier->can_have_nan) { column_stats["has_nan"] = Value::BOOLEAN(stats_unifier->has_nan); } + if (stats_unifier->geo_stats) { + const auto &bbox = stats_unifier->geo_stats->extent; + const auto &types = stats_unifier->geo_stats->types; + + if (bbox.HasXY()) { + + column_stats["bbox_xmin"] = Value::DOUBLE(bbox.x_min); + column_stats["bbox_xmax"] = Value::DOUBLE(bbox.x_max); + column_stats["bbox_ymin"] = Value::DOUBLE(bbox.y_min); + column_stats["bbox_ymax"] = Value::DOUBLE(bbox.y_max); + + if (bbox.HasZ()) { + column_stats["bbox_zmin"] = Value::DOUBLE(bbox.z_min); + column_stats["bbox_zmax"] = Value::DOUBLE(bbox.z_max); + } + + if (bbox.HasM()) { + column_stats["bbox_mmin"] = Value::DOUBLE(bbox.m_min); + column_stats["bbox_mmax"] = Value::DOUBLE(bbox.m_max); + } + } + + if (!types.IsEmpty()) { + vector type_strings; + for (const auto &type : types.ToString(true)) { + type_strings.push_back(Value(StringUtil::Lower(type))); + } + column_stats["geo_types"] = Value::LIST(type_strings); + } + } written_stats->column_statistics.insert(make_pair(stats_unifier->column_name, std::move(column_stats))); } } @@ -885,7 +962,8 @@ void ParquetWriter::Finalize() { } // Add geoparquet metadata to the file metadata - if (geoparquet_data) { + if (geoparquet_data && GeoParquetFileMetadata::IsGeoParquetConversionEnabled(context) && + geoparquet_version != GeoParquetVersion::NONE) { geoparquet_data->Write(file_meta_data); } @@ -915,7 +993,7 @@ void ParquetWriter::Finalize() { GeoParquetFileMetadata &ParquetWriter::GetGeoParquetData() { if (!geoparquet_data) { - geoparquet_data = make_uniq(); + geoparquet_data = make_uniq(geoparquet_version); } return *geoparquet_data; } diff --git a/extension/parquet/reader/string_column_reader.cpp b/extension/parquet/reader/string_column_reader.cpp index 6b2a3db6d239..867dbb4d890c 100644 --- a/extension/parquet/reader/string_column_reader.cpp +++ b/extension/parquet/reader/string_column_reader.cpp @@ -9,7 +9,7 @@ namespace duckdb { // String Column Reader //===--------------------------------------------------------------------===// StringColumnReader::StringColumnReader(ParquetReader &reader, const ParquetColumnSchema &schema) - : ColumnReader(reader, schema) { + : ColumnReader(reader, schema), string_column_type(GetStringColumnType(Type())) { fixed_width_string_length = 0; if (schema.parquet_type == Type::FIXED_LEN_BYTE_ARRAY) { fixed_width_string_length = schema.type_length; @@ -26,13 +26,26 @@ void StringColumnReader::VerifyString(const char *str_data, uint32_t str_len, co size_t pos; auto utf_type = Utf8Proc::Analyze(str_data, str_len, &reason, &pos); if (utf_type == UnicodeType::INVALID) { - throw InvalidInputException("Invalid string encoding found in Parquet file: value \"" + - Blob::ToString(string_t(str_data, str_len)) + "\" is not valid UTF8!"); + throw InvalidInputException("Invalid string encoding found in Parquet file: value \"%s\" is not valid UTF8!", + Blob::ToString(string_t(str_data, str_len))); } } void StringColumnReader::VerifyString(const char *str_data, uint32_t str_len) { - VerifyString(str_data, str_len, Type().id() == LogicalTypeId::VARCHAR); + switch (string_column_type) { + case StringColumnType::VARCHAR: + VerifyString(str_data, str_len, true); + break; + case StringColumnType::JSON: { + const auto error = StringUtil::ValidateJSON(str_data, str_len); + if (!error.empty()) { + throw InvalidInputException("Invalid JSON found in Parquet file: %s", error); + } + break; + } + default: + break; + } } class ParquetStringVectorBuffer : public VectorBuffer { diff --git a/extension/parquet/reader/variant/variant_binary_decoder.cpp b/extension/parquet/reader/variant/variant_binary_decoder.cpp index eacff5501be1..0388da0b332a 100644 --- a/extension/parquet/reader/variant/variant_binary_decoder.cpp +++ b/extension/parquet/reader/variant/variant_binary_decoder.cpp @@ -15,7 +15,7 @@ static constexpr uint8_t VERSION_MASK = 0xF; static constexpr uint8_t SORTED_STRINGS_MASK = 0x1; static constexpr uint8_t SORTED_STRINGS_SHIFT = 4; static constexpr uint8_t OFFSET_SIZE_MINUS_ONE_MASK = 0x3; -static constexpr uint8_t OFFSET_SIZE_MINUS_ONE_SHIFT = 5; +static constexpr uint8_t OFFSET_SIZE_MINUS_ONE_SHIFT = 6; static constexpr uint8_t BASIC_TYPE_MASK = 0x3; static constexpr uint8_t VALUE_HEADER_SHIFT = 2; @@ -74,8 +74,8 @@ VariantMetadata::VariantMetadata(const string_t &metadata) : metadata(metadata) const_data_ptr_t ptr = reinterpret_cast(metadata_data + sizeof(uint8_t)); idx_t dictionary_size = ReadVariableLengthLittleEndian(header.offset_size, ptr); - offsets = ptr; - bytes = offsets + ((dictionary_size + 1) * header.offset_size); + auto offsets = ptr; + auto bytes = offsets + ((dictionary_size + 1) * header.offset_size); idx_t last_offset = ReadVariableLengthLittleEndian(header.offset_size, ptr); for (idx_t i = 0; i < dictionary_size; i++) { auto next_offset = ReadVariableLengthLittleEndian(header.offset_size, ptr); @@ -140,8 +140,7 @@ hugeint_t DecodeDecimal(const_data_ptr_t data, uint8_t &scale, uint8_t &width) { return result; } -VariantValue VariantBinaryDecoder::PrimitiveTypeDecode(const VariantMetadata &metadata, - const VariantValueMetadata &value_metadata, +VariantValue VariantBinaryDecoder::PrimitiveTypeDecode(const VariantValueMetadata &value_metadata, const_data_ptr_t data) { switch (value_metadata.primitive_type) { case VariantPrimitiveType::NULL_TYPE: { @@ -267,8 +266,7 @@ VariantValue VariantBinaryDecoder::PrimitiveTypeDecode(const VariantMetadata &me } } -VariantValue VariantBinaryDecoder::ShortStringDecode(const VariantMetadata &metadata, - const VariantValueMetadata &value_metadata, +VariantValue VariantBinaryDecoder::ShortStringDecode(const VariantValueMetadata &value_metadata, const_data_ptr_t data) { D_ASSERT(value_metadata.string_size < 64); auto string_data = reinterpret_cast(data); @@ -348,10 +346,10 @@ VariantValue VariantBinaryDecoder::Decode(const VariantMetadata &variant_metadat data++; switch (value_metadata.basic_type) { case VariantBasicType::PRIMITIVE: { - return PrimitiveTypeDecode(variant_metadata, value_metadata, data); + return PrimitiveTypeDecode(value_metadata, data); } case VariantBasicType::SHORT_STRING: { - return ShortStringDecode(variant_metadata, value_metadata, data); + return ShortStringDecode(value_metadata, data); } case VariantBasicType::OBJECT: { return ObjectDecode(variant_metadata, value_metadata, data); diff --git a/extension/parquet/reader/variant/variant_shredded_conversion.cpp b/extension/parquet/reader/variant/variant_shredded_conversion.cpp index 8278eb740c2b..b96304d98ee0 100644 --- a/extension/parquet/reader/variant/variant_shredded_conversion.cpp +++ b/extension/parquet/reader/variant/variant_shredded_conversion.cpp @@ -124,7 +124,7 @@ VariantValue ConvertShreddedValue::Convert(hugeint_t val) { template vector ConvertTypedValues(Vector &vec, Vector &metadata, Vector &blob, idx_t offset, idx_t length, - idx_t total_size) { + idx_t total_size, const bool is_field) { UnifiedVectorFormat metadata_format; metadata.ToUnifiedFormat(length, metadata_format); auto metadata_data = metadata_format.GetData(metadata_format); @@ -174,7 +174,12 @@ vector ConvertTypedValues(Vector &vec, Vector &metadata, Vector &b } else { ret[i] = OP::Convert(data[typed_index]); } - } else if (value_validity.RowIsValid(value_index)) { + } else { + if (is_field && !value_validity.RowIsValid(value_index)) { + //! Value is missing for this field + continue; + } + D_ASSERT(value_validity.RowIsValid(value_index)); auto metadata_value = metadata_data[metadata_format.sel->get_index(i)]; VariantMetadata variant_metadata(metadata_value); ret[i] = VariantBinaryDecoder::Decode(variant_metadata, @@ -187,7 +192,7 @@ vector ConvertTypedValues(Vector &vec, Vector &metadata, Vector &b vector VariantShreddedConversion::ConvertShreddedLeaf(Vector &metadata, Vector &value, Vector &typed_value, idx_t offset, idx_t length, - idx_t total_size) { + idx_t total_size, const bool is_field) { D_ASSERT(!typed_value.GetType().IsNested()); vector result; @@ -196,37 +201,37 @@ vector VariantShreddedConversion::ConvertShreddedLeaf(Vector &meta //! boolean case LogicalTypeId::BOOLEAN: { return ConvertTypedValues, LogicalTypeId::BOOLEAN>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! int8 case LogicalTypeId::TINYINT: { return ConvertTypedValues, LogicalTypeId::TINYINT>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! int16 case LogicalTypeId::SMALLINT: { return ConvertTypedValues, LogicalTypeId::SMALLINT>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! int32 case LogicalTypeId::INTEGER: { return ConvertTypedValues, LogicalTypeId::INTEGER>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! int64 case LogicalTypeId::BIGINT: { return ConvertTypedValues, LogicalTypeId::BIGINT>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! float case LogicalTypeId::FLOAT: { return ConvertTypedValues, LogicalTypeId::FLOAT>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! double case LogicalTypeId::DOUBLE: { return ConvertTypedValues, LogicalTypeId::DOUBLE>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! decimal4/decimal8/decimal16 case LogicalTypeId::DECIMAL: { @@ -234,15 +239,15 @@ vector VariantShreddedConversion::ConvertShreddedLeaf(Vector &meta switch (physical_type) { case PhysicalType::INT32: { return ConvertTypedValues, LogicalTypeId::DECIMAL>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } case PhysicalType::INT64: { return ConvertTypedValues, LogicalTypeId::DECIMAL>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } case PhysicalType::INT128: { return ConvertTypedValues, LogicalTypeId::DECIMAL>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } default: throw NotImplementedException("Decimal with PhysicalType (%s) not implemented for shredded Variant", @@ -252,42 +257,42 @@ vector VariantShreddedConversion::ConvertShreddedLeaf(Vector &meta //! date case LogicalTypeId::DATE: { return ConvertTypedValues, LogicalTypeId::DATE>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! time case LogicalTypeId::TIME: { return ConvertTypedValues, LogicalTypeId::TIME>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! timestamptz(6) (timestamptz(9) not implemented in DuckDB) case LogicalTypeId::TIMESTAMP_TZ: { return ConvertTypedValues, LogicalTypeId::TIMESTAMP_TZ>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! timestampntz(6) case LogicalTypeId::TIMESTAMP: { return ConvertTypedValues, LogicalTypeId::TIMESTAMP>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! timestampntz(9) case LogicalTypeId::TIMESTAMP_NS: { return ConvertTypedValues, LogicalTypeId::TIMESTAMP_NS>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! binary case LogicalTypeId::BLOB: { return ConvertTypedValues, LogicalTypeId::BLOB>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! string case LogicalTypeId::VARCHAR: { return ConvertTypedValues, LogicalTypeId::VARCHAR>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } //! uuid case LogicalTypeId::UUID: { return ConvertTypedValues, LogicalTypeId::UUID>( - typed_value, metadata, value, offset, length, total_size); + typed_value, metadata, value, offset, length, total_size, is_field); } default: throw NotImplementedException("Variant shredding on type: '%s' is not implemented", type.ToString()); @@ -395,7 +400,7 @@ static VariantValue ConvertPartiallyShreddedObject(vector vector VariantShreddedConversion::ConvertShreddedObject(Vector &metadata, Vector &value, Vector &typed_value, idx_t offset, idx_t length, - idx_t total_size) { + idx_t total_size, const bool is_field) { auto &type = typed_value.GetType(); D_ASSERT(type.id() == LogicalTypeId::STRUCT); auto &fields = StructType::GetChildTypes(type); @@ -407,6 +412,7 @@ vector VariantShreddedConversion::ConvertShreddedObject(Vector &me value.ToUnifiedFormat(total_size, value_format); auto value_data = value_format.GetData(value_format); auto &validity = value_format.validity; + (void)validity; //! 'metadata' UnifiedVectorFormat metadata_format; @@ -444,7 +450,10 @@ vector VariantShreddedConversion::ConvertShreddedObject(Vector &me if (typed_validity.RowIsValid(typed_index)) { ret[i] = ConvertPartiallyShreddedObject(shredded_fields, metadata_format, value_format, i, offset); } else { - //! The value on this row is not an object, and guaranteed to be present + if (is_field && !validity.RowIsValid(value_index)) { + //! This object is a field in the parent object, the value is missing, skip it + continue; + } D_ASSERT(validity.RowIsValid(value_index)); auto &metadata_value = metadata_data[metadata_format.sel->get_index(i)]; VariantMetadata variant_metadata(metadata_value); @@ -462,7 +471,7 @@ vector VariantShreddedConversion::ConvertShreddedObject(Vector &me vector VariantShreddedConversion::ConvertShreddedArray(Vector &metadata, Vector &value, Vector &typed_value, idx_t offset, idx_t length, - idx_t total_size) { + idx_t total_size, const bool is_field) { auto &child = ListVector::GetEntry(typed_value); auto list_size = ListVector::GetListSize(typed_value); @@ -488,23 +497,26 @@ vector VariantShreddedConversion::ConvertShreddedArray(Vector &met //! We can be sure that none of the values are binary encoded for (idx_t i = 0; i < length; i++) { auto typed_index = list_format.sel->get_index(i + offset); - //! FIXME: next 4 lines duplicated below auto entry = list_data[typed_index]; Vector child_metadata(metadata.GetValue(i)); ret[i] = VariantValue(VariantValueType::ARRAY); - ret[i].array_items = Convert(child_metadata, child, entry.offset, entry.length, list_size); + ret[i].array_items = Convert(child_metadata, child, entry.offset, entry.length, list_size, false); } } else { for (idx_t i = 0; i < length; i++) { auto typed_index = list_format.sel->get_index(i + offset); auto value_index = value_format.sel->get_index(i + offset); if (validity.RowIsValid(typed_index)) { - //! FIXME: next 4 lines duplicate auto entry = list_data[typed_index]; Vector child_metadata(metadata.GetValue(i)); ret[i] = VariantValue(VariantValueType::ARRAY); - ret[i].array_items = Convert(child_metadata, child, entry.offset, entry.length, list_size); - } else if (value_validity.RowIsValid(value_index)) { + ret[i].array_items = Convert(child_metadata, child, entry.offset, entry.length, list_size, false); + } else { + if (is_field && !value_validity.RowIsValid(value_index)) { + //! Value is missing for this field + continue; + } + D_ASSERT(value_validity.RowIsValid(value_index)); auto metadata_value = metadata_data[metadata_format.sel->get_index(i)]; VariantMetadata variant_metadata(metadata_value); ret[i] = VariantBinaryDecoder::Decode(variant_metadata, @@ -546,11 +558,11 @@ vector VariantShreddedConversion::Convert(Vector &metadata, Vector auto &type = typed_value->GetType(); vector ret; if (type.id() == LogicalTypeId::STRUCT) { - return ConvertShreddedObject(metadata, *value, *typed_value, offset, length, total_size); + return ConvertShreddedObject(metadata, *value, *typed_value, offset, length, total_size, is_field); } else if (type.id() == LogicalTypeId::LIST) { - return ConvertShreddedArray(metadata, *value, *typed_value, offset, length, total_size); + return ConvertShreddedArray(metadata, *value, *typed_value, offset, length, total_size, is_field); } else { - return ConvertShreddedLeaf(metadata, *value, *typed_value, offset, length, total_size); + return ConvertShreddedLeaf(metadata, *value, *typed_value, offset, length, total_size, is_field); } } else { if (is_field) { diff --git a/extension/parquet/reader/variant/variant_value.cpp b/extension/parquet/reader/variant/variant_value.cpp index 0ac213469433..6b3d290f42cf 100644 --- a/extension/parquet/reader/variant/variant_value.cpp +++ b/extension/parquet/reader/variant/variant_value.cpp @@ -1,4 +1,18 @@ #include "reader/variant/variant_value.hpp" +#include "duckdb/common/serializer/varint.hpp" +#include "duckdb/common/enum_util.hpp" +#include "duckdb/common/types/time.hpp" +#include "duckdb/common/types/datetime.hpp" +#include "duckdb/common/types/timestamp.hpp" +#include "duckdb/common/types/date.hpp" +#include "duckdb/common/types/interval.hpp" +#include "duckdb/common/types/decimal.hpp" +#include "duckdb/common/types/variant.hpp" +#include "duckdb/common/hugeint.hpp" +#include "duckdb/function/scalar/variant_utils.hpp" +#include "duckdb/common/string_map_set.hpp" +#include "duckdb/common/helper.hpp" +#include "duckdb/function/cast/variant/to_variant_fwd.hpp" namespace duckdb { @@ -12,6 +26,560 @@ void VariantValue::AddItem(VariantValue &&val) { array_items.push_back(std::move(val)); } +static void InitializeOffsets(DataChunk &offsets, idx_t count) { + auto keys = variant::OffsetData::GetKeys(offsets); + auto children = variant::OffsetData::GetChildren(offsets); + auto values = variant::OffsetData::GetValues(offsets); + auto blob = variant::OffsetData::GetBlob(offsets); + for (idx_t i = 0; i < count; i++) { + keys[i] = 0; + children[i] = 0; + values[i] = 0; + blob[i] = 0; + } +} + +static void AnalyzeValue(const VariantValue &value, idx_t row, DataChunk &offsets) { + auto &keys_offset = variant::OffsetData::GetKeys(offsets)[row]; + auto &children_offset = variant::OffsetData::GetChildren(offsets)[row]; + auto &values_offset = variant::OffsetData::GetValues(offsets)[row]; + auto &data_offset = variant::OffsetData::GetBlob(offsets)[row]; + + values_offset++; + switch (value.value_type) { + case VariantValueType::OBJECT: { + //! Write the count of the children + auto &children = value.object_children; + data_offset += GetVarintSize(children.size()); + if (!children.empty()) { + //! Write the children offset + data_offset += GetVarintSize(children_offset); + children_offset += children.size(); + keys_offset += children.size(); + for (auto &child : children) { + auto &child_value = child.second; + AnalyzeValue(child_value, row, offsets); + } + } + break; + } + case VariantValueType::ARRAY: { + //! Write the count of the children + auto &children = value.array_items; + data_offset += GetVarintSize(children.size()); + if (!children.empty()) { + //! Write the children offset + data_offset += GetVarintSize(children_offset); + children_offset += children.size(); + for (auto &child : children) { + AnalyzeValue(child, row, offsets); + } + } + break; + } + case VariantValueType::PRIMITIVE: { + auto &primitive = value.primitive_value; + auto type_id = primitive.type().id(); + switch (type_id) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::SQLNULL: { + break; + } + case LogicalTypeId::TINYINT: { + data_offset += sizeof(int8_t); + break; + } + case LogicalTypeId::SMALLINT: { + data_offset += sizeof(int16_t); + break; + } + case LogicalTypeId::INTEGER: { + data_offset += sizeof(int32_t); + break; + } + case LogicalTypeId::BIGINT: { + data_offset += sizeof(int64_t); + break; + } + case LogicalTypeId::DOUBLE: { + data_offset += sizeof(double); + break; + } + case LogicalTypeId::FLOAT: { + data_offset += sizeof(float); + break; + } + case LogicalTypeId::DATE: { + data_offset += sizeof(date_t); + break; + } + case LogicalTypeId::TIMESTAMP_TZ: { + data_offset += sizeof(timestamp_tz_t); + break; + } + case LogicalTypeId::TIMESTAMP: { + data_offset += sizeof(timestamp_t); + break; + } + case LogicalTypeId::TIME: { + data_offset += sizeof(dtime_t); + break; + } + case LogicalTypeId::TIMESTAMP_NS: { + data_offset += sizeof(timestamp_ns_t); + break; + } + case LogicalTypeId::UUID: { + data_offset += sizeof(hugeint_t); + break; + } + case LogicalTypeId::DECIMAL: { + auto &type = primitive.type(); + uint8_t width; + uint8_t scale; + type.GetDecimalProperties(width, scale); + + auto physical_type = type.InternalType(); + data_offset += GetVarintSize(width); + data_offset += GetVarintSize(scale); + switch (physical_type) { + case PhysicalType::INT32: { + data_offset += sizeof(int32_t); + break; + } + case PhysicalType::INT64: { + data_offset += sizeof(int64_t); + break; + } + case PhysicalType::INT128: { + data_offset += sizeof(hugeint_t); + break; + } + default: + throw InternalException("Unexpected physical type for Decimal value: %s", + EnumUtil::ToString(physical_type)); + } + break; + } + case LogicalTypeId::BLOB: + case LogicalTypeId::VARCHAR: { + auto string_data = primitive.GetValueUnsafe(); + data_offset += GetVarintSize(string_data.GetSize()); + data_offset += string_data.GetSize(); + break; + } + default: + throw InternalException("Encountered unrecognized LogicalType in VariantValue::AnalyzeValue: %s", + primitive.type().ToString()); + } + break; + } + default: + throw InternalException("VariantValueType not handled"); + } +} + +uint32_t GetOrCreateIndex(OrderedOwningStringMap &dictionary, const string_t &key) { + auto unsorted_idx = dictionary.size(); + //! This will later be remapped to the sorted idx (see FinalizeVariantKeys in 'to_variant.cpp') + return dictionary.emplace(std::make_pair(key, unsorted_idx)).first->second; +} + +static void ConvertValue(const VariantValue &value, VariantVectorData &result, idx_t row, DataChunk &offsets, + SelectionVector &keys_selvec, OrderedOwningStringMap &dictionary) { + auto blob_data = data_ptr_cast(result.blob_data[row].GetDataWriteable()); + auto keys_list_offset = result.keys_data[row].offset; + auto children_list_offset = result.children_data[row].offset; + auto values_list_offset = result.values_data[row].offset; + + auto &keys_offset = variant::OffsetData::GetKeys(offsets)[row]; + auto &children_offset = variant::OffsetData::GetChildren(offsets)[row]; + auto &values_offset = variant::OffsetData::GetValues(offsets)[row]; + auto &data_offset = variant::OffsetData::GetBlob(offsets)[row]; + + switch (value.value_type) { + case VariantValueType::OBJECT: { + //! Write the count of the children + auto &children = value.object_children; + + //! values + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::OBJECT); + result.byte_offset_data[values_list_offset + values_offset] = data_offset; + values_offset++; + + //! data + VarintEncode(children.size(), blob_data + data_offset); + data_offset += GetVarintSize(children.size()); + + if (!children.empty()) { + //! Write the children offset + VarintEncode(children_offset, blob_data + data_offset); + data_offset += GetVarintSize(children_offset); + + auto start_of_children = children_offset; + children_offset += children.size(); + + auto it = children.begin(); + for (idx_t i = 0; i < children.size(); i++) { + //! children + result.keys_index_data[children_list_offset + start_of_children + i] = keys_offset; + result.values_index_data[children_list_offset + start_of_children + i] = values_offset; + + auto &child = *it; + //! keys + auto &child_key = child.first; + auto dictionary_index = GetOrCreateIndex(dictionary, child_key); + keys_selvec.set_index(keys_list_offset + keys_offset, dictionary_index); + keys_offset++; + + auto &child_value = child.second; + ConvertValue(child_value, result, row, offsets, keys_selvec, dictionary); + it++; + } + } + break; + } + case VariantValueType::ARRAY: { + //! Write the count of the children + auto &children = value.array_items; + + //! values + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::ARRAY); + result.byte_offset_data[values_list_offset + values_offset] = data_offset; + values_offset++; + + //! data + VarintEncode(children.size(), blob_data + data_offset); + data_offset += GetVarintSize(children.size()); + + if (!children.empty()) { + //! Write the children offset + VarintEncode(children_offset, blob_data + data_offset); + data_offset += GetVarintSize(children_offset); + + auto start_of_children = children_offset; + children_offset += children.size(); + + for (idx_t i = 0; i < children.size(); i++) { + //! children + result.keys_index_validity.SetInvalid(children_list_offset + start_of_children + i); + result.values_index_data[children_list_offset + start_of_children + i] = values_offset; + + auto &child_value = children[i]; + ConvertValue(child_value, result, row, offsets, keys_selvec, dictionary); + } + } + break; + } + case VariantValueType::PRIMITIVE: { + auto &primitive = value.primitive_value; + auto type_id = primitive.type().id(); + result.byte_offset_data[values_list_offset + values_offset] = data_offset; + switch (type_id) { + case LogicalTypeId::BOOLEAN: { + if (primitive.GetValue()) { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::BOOL_TRUE); + } else { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::BOOL_FALSE); + } + break; + } + case LogicalTypeId::SQLNULL: { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::VARIANT_NULL); + break; + } + case LogicalTypeId::TINYINT: { + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::INT8); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(int8_t); + break; + } + case LogicalTypeId::SMALLINT: { + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::INT16); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(int16_t); + break; + } + case LogicalTypeId::INTEGER: { + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::INT32); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(int32_t); + break; + } + case LogicalTypeId::BIGINT: { + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::INT64); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(int64_t); + break; + } + case LogicalTypeId::DOUBLE: { + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::DOUBLE); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(double); + break; + } + case LogicalTypeId::FLOAT: { + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::FLOAT); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(float); + break; + } + case LogicalTypeId::DATE: { + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::DATE); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(date_t); + break; + } + case LogicalTypeId::TIMESTAMP_TZ: { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::TIMESTAMP_MICROS_TZ); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(timestamp_tz_t); + break; + } + case LogicalTypeId::TIMESTAMP: { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::TIMESTAMP_MICROS); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(timestamp_t); + break; + } + case LogicalTypeId::TIME: { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::TIME_MICROS); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(dtime_t); + break; + } + case LogicalTypeId::TIMESTAMP_NS: { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::TIMESTAMP_NANOS); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(timestamp_ns_t); + break; + } + case LogicalTypeId::UUID: { + result.type_ids_data[values_list_offset + values_offset] = static_cast(VariantLogicalType::UUID); + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(hugeint_t); + break; + } + case LogicalTypeId::DECIMAL: { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::DECIMAL); + auto &type = primitive.type(); + uint8_t width; + uint8_t scale; + type.GetDecimalProperties(width, scale); + + auto physical_type = type.InternalType(); + VarintEncode(width, blob_data + data_offset); + data_offset += GetVarintSize(width); + VarintEncode(scale, blob_data + data_offset); + data_offset += GetVarintSize(scale); + switch (physical_type) { + case PhysicalType::INT32: { + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(int32_t); + break; + } + case PhysicalType::INT64: { + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(int64_t); + break; + } + case PhysicalType::INT128: { + Store(primitive.GetValueUnsafe(), blob_data + data_offset); + data_offset += sizeof(hugeint_t); + break; + } + default: + throw InternalException("Unexpected physical type for Decimal value: %s", + EnumUtil::ToString(physical_type)); + } + break; + } + case LogicalTypeId::BLOB: + case LogicalTypeId::VARCHAR: { + if (type_id == LogicalTypeId::BLOB) { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::BLOB); + } else { + result.type_ids_data[values_list_offset + values_offset] = + static_cast(VariantLogicalType::VARCHAR); + } + auto string_data = primitive.GetValueUnsafe(); + auto string_size = string_data.GetSize(); + VarintEncode(string_size, blob_data + data_offset); + data_offset += GetVarintSize(string_size); + memcpy(blob_data + data_offset, string_data.GetData(), string_size); + data_offset += string_size; + break; + } + default: + throw InternalException("Encountered unrecognized LogicalType in VariantValue::AnalyzeValue: %s", + primitive.type().ToString()); + } + values_offset++; + break; + } + default: + throw InternalException("VariantValueType not handled"); + } +} + +//! Copied and modified from 'to_variant.cpp' +static void InitializeVariants(DataChunk &offsets, Vector &result, SelectionVector &keys_selvec, idx_t &selvec_size) { + auto &keys = VariantVector::GetKeys(result); + auto keys_data = ListVector::GetData(keys); + + auto &children = VariantVector::GetChildren(result); + auto children_data = ListVector::GetData(children); + + auto &values = VariantVector::GetValues(result); + auto values_data = ListVector::GetData(values); + + auto &blob = VariantVector::GetData(result); + auto blob_data = FlatVector::GetData(blob); + + idx_t children_offset = 0; + idx_t values_offset = 0; + idx_t keys_offset = 0; + + auto keys_sizes = variant::OffsetData::GetKeys(offsets); + auto children_sizes = variant::OffsetData::GetChildren(offsets); + auto values_sizes = variant::OffsetData::GetValues(offsets); + auto blob_sizes = variant::OffsetData::GetBlob(offsets); + + auto count = offsets.size(); + for (idx_t i = 0; i < count; i++) { + auto &keys_entry = keys_data[i]; + auto &children_entry = children_data[i]; + auto &values_entry = values_data[i]; + + //! keys + keys_entry.length = keys_sizes[i]; + keys_entry.offset = keys_offset; + keys_offset += keys_entry.length; + + //! children + children_entry.length = children_sizes[i]; + children_entry.offset = children_offset; + children_offset += children_entry.length; + + //! values + values_entry.length = values_sizes[i]; + values_entry.offset = values_offset; + values_offset += values_entry.length; + + //! value + blob_data[i] = StringVector::EmptyString(blob, blob_sizes[i]); + } + + //! Reserve for the children of the lists + ListVector::Reserve(keys, keys_offset); + ListVector::Reserve(children, children_offset); + ListVector::Reserve(values, values_offset); + + //! Set list sizes + ListVector::SetListSize(keys, keys_offset); + ListVector::SetListSize(children, children_offset); + ListVector::SetListSize(values, values_offset); + + keys_selvec.Initialize(keys_offset); + selvec_size = keys_offset; +} + +void VariantValue::ToVARIANT(vector &input, Vector &result) { + auto count = input.size(); + if (input.empty()) { + return; + } + + //! Keep track of all the offsets for each row. + DataChunk analyze_offsets; + analyze_offsets.Initialize( + Allocator::DefaultAllocator(), + {LogicalType::UINTEGER, LogicalType::UINTEGER, LogicalType::UINTEGER, LogicalType::UINTEGER}, count); + analyze_offsets.SetCardinality(count); + InitializeOffsets(analyze_offsets, count); + + for (idx_t i = 0; i < count; i++) { + auto &value = input[i]; + if (value.IsNull()) { + continue; + } + AnalyzeValue(value, i, analyze_offsets); + } + + SelectionVector keys_selvec; + idx_t keys_selvec_size; + InitializeVariants(analyze_offsets, result, keys_selvec, keys_selvec_size); + + auto &keys = VariantVector::GetKeys(result); + auto &keys_entry = ListVector::GetEntry(keys); + OrderedOwningStringMap dictionary(StringVector::GetStringBuffer(keys_entry).GetStringAllocator()); + + DataChunk conversion_offsets; + conversion_offsets.Initialize( + Allocator::DefaultAllocator(), + {LogicalType::UINTEGER, LogicalType::UINTEGER, LogicalType::UINTEGER, LogicalType::UINTEGER}, count); + conversion_offsets.SetCardinality(count); + InitializeOffsets(conversion_offsets, count); + + VariantVectorData variant_data(result); + for (idx_t i = 0; i < count; i++) { + auto &value = input[i]; + if (value.IsNull()) { + FlatVector::SetNull(result, i, true); + continue; + } + ConvertValue(value, variant_data, i, conversion_offsets, keys_selvec, dictionary); + } + +#ifdef DEBUG + { + auto conversion_keys_offset = variant::OffsetData::GetKeys(conversion_offsets); + auto conversion_children_offset = variant::OffsetData::GetChildren(conversion_offsets); + auto conversion_values_offset = variant::OffsetData::GetValues(conversion_offsets); + auto conversion_data_offset = variant::OffsetData::GetBlob(conversion_offsets); + + auto analyze_keys_offset = variant::OffsetData::GetKeys(analyze_offsets); + auto analyze_children_offset = variant::OffsetData::GetChildren(analyze_offsets); + auto analyze_values_offset = variant::OffsetData::GetValues(analyze_offsets); + auto analyze_data_offset = variant::OffsetData::GetBlob(analyze_offsets); + + for (idx_t i = 0; i < count; i++) { + D_ASSERT(conversion_keys_offset[i] == analyze_keys_offset[i]); + D_ASSERT(conversion_children_offset[i] == analyze_children_offset[i]); + D_ASSERT(conversion_values_offset[i] == analyze_values_offset[i]); + D_ASSERT(conversion_data_offset[i] == analyze_data_offset[i]); + } + } + +#endif + + //! Finalize the 'data' column of the VARIANT + auto conversion_data_offsets = variant::OffsetData::GetBlob(conversion_offsets); + for (idx_t i = 0; i < count; i++) { + auto &data = variant_data.blob_data[i]; + data.SetSizeAndFinalize(conversion_data_offsets[i], conversion_data_offsets[i]); + } + + VariantUtils::FinalizeVariantKeys(result, dictionary, keys_selvec, keys_selvec_size); + + keys_entry.Slice(keys_selvec, keys_selvec_size); + keys_entry.Flatten(keys_selvec_size); + + if (input.size() == 1) { + result.SetVectorType(VectorType::CONSTANT_VECTOR); + } + result.Verify(count); +} + yyjson_mut_val *VariantValue::ToJSON(ClientContext &context, yyjson_mut_doc *doc) const { switch (value_type) { case VariantValueType::PRIMITIVE: { diff --git a/extension/parquet/reader/variant_column_reader.cpp b/extension/parquet/reader/variant_column_reader.cpp index 402bcbb07934..635bfbbb5253 100644 --- a/extension/parquet/reader/variant_column_reader.cpp +++ b/extension/parquet/reader/variant_column_reader.cpp @@ -11,7 +11,7 @@ VariantColumnReader::VariantColumnReader(ClientContext &context, ParquetReader & const ParquetColumnSchema &schema, vector> child_readers_p) : ColumnReader(reader, schema), context(context), child_readers(std::move(child_readers_p)) { - D_ASSERT(Type().InternalType() == PhysicalType::VARCHAR); + D_ASSERT(Type().InternalType() == PhysicalType::STRUCT); if (child_readers[0]->Schema().name == "metadata" && child_readers[1]->Schema().name == "value") { metadata_reader_idx = 0; @@ -80,10 +80,7 @@ idx_t VariantColumnReader::Read(uint64_t num_values, data_ptr_t define_out, data "The Variant column did not contain the same amount of values for 'metadata' and 'value'"); } - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); - - vector conversion_result; + vector intermediate; if (typed_value_reader) { auto typed_values = typed_value_reader->Read(num_values, define_out, repeat_out, *group_entries[1]); if (typed_values != value_values) { @@ -91,29 +88,9 @@ idx_t VariantColumnReader::Read(uint64_t num_values, data_ptr_t define_out, data "The shredded Variant column did not contain the same amount of values for 'typed_value' and 'value'"); } } - conversion_result = - VariantShreddedConversion::Convert(metadata_intermediate, intermediate_group, 0, num_values, num_values); - - for (idx_t i = 0; i < conversion_result.size(); i++) { - auto &variant = conversion_result[i]; - if (variant.IsNull()) { - result_validity.SetInvalid(i); - continue; - } - - //! Write the result to a string - VariantDecodeResult decode_result; - decode_result.doc = yyjson_mut_doc_new(nullptr); - auto json_val = variant.ToJSON(context, decode_result.doc); - - size_t len; - decode_result.data = - yyjson_mut_val_write_opts(json_val, YYJSON_WRITE_ALLOW_INF_AND_NAN, nullptr, &len, nullptr); - if (!decode_result.data) { - throw InvalidInputException("Could not serialize the JSON to string, yyjson failed"); - } - result_data[i] = StringVector::AddString(result, decode_result.data, static_cast(len)); - } + intermediate = + VariantShreddedConversion::Convert(metadata_intermediate, intermediate_group, 0, num_values, num_values, false); + VariantValue::ToVARIANT(intermediate, result); read_count = value_values; return read_count.GetIndex(); diff --git a/extension/parquet/serialize_parquet.cpp b/extension/parquet/serialize_parquet.cpp index aa5632077ee4..6f12d5d89267 100644 --- a/extension/parquet/serialize_parquet.cpp +++ b/extension/parquet/serialize_parquet.cpp @@ -7,7 +7,8 @@ #include "duckdb/common/serializer/deserializer.hpp" #include "parquet_reader.hpp" #include "parquet_crypto.hpp" -#include "parquet_writer.hpp" +#include "parquet_field_id.hpp" +#include "parquet_shredding.hpp" namespace duckdb { @@ -21,6 +22,16 @@ ChildFieldIDs ChildFieldIDs::Deserialize(Deserializer &deserializer) { return result; } +void ChildShreddingTypes::Serialize(Serializer &serializer) const { + serializer.WritePropertyWithDefault>(100, "types", types.operator*()); +} + +ChildShreddingTypes ChildShreddingTypes::Deserialize(Deserializer &deserializer) { + ChildShreddingTypes result; + deserializer.ReadPropertyWithDefault>(100, "types", result.types.operator*()); + return result; +} + void FieldID::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(100, "set", set); serializer.WritePropertyWithDefault(101, "field_id", field_id); @@ -89,4 +100,18 @@ ParquetOptionsSerialization ParquetOptionsSerialization::Deserialize(Deserialize return result; } +void ShreddingType::Serialize(Serializer &serializer) const { + serializer.WritePropertyWithDefault(100, "set", set); + serializer.WriteProperty(101, "type", type); + serializer.WriteProperty(102, "children", children); +} + +ShreddingType ShreddingType::Deserialize(Deserializer &deserializer) { + ShreddingType result; + deserializer.ReadPropertyWithDefault(100, "set", result.set); + deserializer.ReadProperty(101, "type", result.type); + deserializer.ReadProperty(102, "children", result.children); + return result; +} + } // namespace duckdb diff --git a/extension/parquet/writer/CMakeLists.txt b/extension/parquet/writer/CMakeLists.txt index 98881a6737fb..d7d7d3b711a3 100644 --- a/extension/parquet/writer/CMakeLists.txt +++ b/extension/parquet/writer/CMakeLists.txt @@ -8,6 +8,9 @@ add_library_unity( list_column_writer.cpp primitive_column_writer.cpp struct_column_writer.cpp) + +add_subdirectory(variant) + set(PARQUET_EXTENSION_FILES ${PARQUET_EXTENSION_FILES} $ PARENT_SCOPE) diff --git a/extension/parquet/writer/array_column_writer.cpp b/extension/parquet/writer/array_column_writer.cpp index 024dbe819ac0..2a9c9a9d57cb 100644 --- a/extension/parquet/writer/array_column_writer.cpp +++ b/extension/parquet/writer/array_column_writer.cpp @@ -6,7 +6,23 @@ void ArrayColumnWriter::Analyze(ColumnWriterState &state_p, ColumnWriterState *p auto &state = state_p.Cast(); auto &array_child = ArrayVector::GetEntry(vector); auto array_size = ArrayType::GetSize(vector.GetType()); - child_writer->Analyze(*state.child_state, &state_p, array_child, array_size * count); + GetChildWriter().Analyze(*state.child_state, &state_p, array_child, array_size * count); +} + +void ArrayColumnWriter::WriteArrayState(ListColumnWriterState &state, idx_t array_size, uint16_t first_repeat_level, + idx_t define_value, const bool is_empty) { + state.definition_levels.push_back(define_value); + state.repetition_levels.push_back(first_repeat_level); + state.is_empty.push_back(is_empty); + + if (is_empty) { + return; + } + for (idx_t k = 1; k < array_size; k++) { + state.repetition_levels.push_back(MaxRepeat() + 1); + state.definition_levels.push_back(define_value); + state.is_empty.push_back(false); + } } void ArrayColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *parent, Vector &vector, idx_t count, @@ -19,48 +35,25 @@ void ArrayColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *p // write definition levels and repeats // the main difference between this and ListColumnWriter::Prepare is that we need to make sure to write out // repetition levels and definitions for the child elements of the array even if the array itself is NULL. - idx_t start = 0; idx_t vcount = parent ? parent->definition_levels.size() - state.parent_index : count; idx_t vector_index = 0; - for (idx_t i = start; i < vcount; i++) { + for (idx_t i = 0; i < vcount; i++) { idx_t parent_index = state.parent_index + i; if (parent && !parent->is_empty.empty() && parent->is_empty[parent_index]) { - state.definition_levels.push_back(parent->definition_levels[parent_index]); - state.repetition_levels.push_back(parent->repetition_levels[parent_index]); - state.is_empty.push_back(true); + WriteArrayState(state, array_size, parent->repetition_levels[parent_index], + parent->definition_levels[parent_index], true); continue; } auto first_repeat_level = parent && !parent->repetition_levels.empty() ? parent->repetition_levels[parent_index] : MaxRepeat(); if (parent && parent->definition_levels[parent_index] != PARQUET_DEFINE_VALID) { - state.definition_levels.push_back(parent->definition_levels[parent_index]); - state.repetition_levels.push_back(first_repeat_level); - state.is_empty.push_back(false); - for (idx_t k = 1; k < array_size; k++) { - state.repetition_levels.push_back(MaxRepeat() + 1); - state.definition_levels.push_back(parent->definition_levels[parent_index]); - state.is_empty.push_back(false); - } + WriteArrayState(state, array_size, first_repeat_level, parent->definition_levels[parent_index]); } else if (validity.RowIsValid(vector_index)) { // push the repetition levels - state.definition_levels.push_back(PARQUET_DEFINE_VALID); - state.is_empty.push_back(false); - - state.repetition_levels.push_back(first_repeat_level); - for (idx_t k = 1; k < array_size; k++) { - state.repetition_levels.push_back(MaxRepeat() + 1); - state.definition_levels.push_back(PARQUET_DEFINE_VALID); - state.is_empty.push_back(false); - } + WriteArrayState(state, array_size, first_repeat_level, PARQUET_DEFINE_VALID); } else { - state.definition_levels.push_back(MaxDefine() - 1); - state.repetition_levels.push_back(first_repeat_level); - state.is_empty.push_back(false); - for (idx_t k = 1; k < array_size; k++) { - state.repetition_levels.push_back(MaxRepeat() + 1); - state.definition_levels.push_back(MaxDefine() - 1); - state.is_empty.push_back(false); - } + //! Produce a null + WriteArrayState(state, array_size, first_repeat_level, MaxDefine() - 1); } vector_index++; } @@ -69,14 +62,14 @@ void ArrayColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *p auto &array_child = ArrayVector::GetEntry(vector); // The elements of a single array should not span multiple Parquet pages // So, we force the entire vector to fit on a single page by setting "vector_can_span_multiple_pages=false" - child_writer->Prepare(*state.child_state, &state_p, array_child, count * array_size, false); + GetChildWriter().Prepare(*state.child_state, &state_p, array_child, count * array_size, false); } void ArrayColumnWriter::Write(ColumnWriterState &state_p, Vector &vector, idx_t count) { auto &state = state_p.Cast(); auto array_size = ArrayType::GetSize(vector.GetType()); auto &array_child = ArrayVector::GetEntry(vector); - child_writer->Write(*state.child_state, array_child, count * array_size); + GetChildWriter().Write(*state.child_state, array_child, count * array_size); } } // namespace duckdb diff --git a/extension/parquet/writer/list_column_writer.cpp b/extension/parquet/writer/list_column_writer.cpp index 8fba00c232c8..b043a94bcb37 100644 --- a/extension/parquet/writer/list_column_writer.cpp +++ b/extension/parquet/writer/list_column_writer.cpp @@ -4,23 +4,23 @@ namespace duckdb { unique_ptr ListColumnWriter::InitializeWriteState(duckdb_parquet::RowGroup &row_group) { auto result = make_uniq(row_group, row_group.columns.size()); - result->child_state = child_writer->InitializeWriteState(row_group); + result->child_state = GetChildWriter().InitializeWriteState(row_group); return std::move(result); } bool ListColumnWriter::HasAnalyze() { - return child_writer->HasAnalyze(); + return GetChildWriter().HasAnalyze(); } void ListColumnWriter::Analyze(ColumnWriterState &state_p, ColumnWriterState *parent, Vector &vector, idx_t count) { auto &state = state_p.Cast(); auto &list_child = ListVector::GetEntry(vector); auto list_count = ListVector::GetListSize(vector); - child_writer->Analyze(*state.child_state, &state_p, list_child, list_count); + GetChildWriter().Analyze(*state.child_state, &state_p, list_child, list_count); } void ListColumnWriter::FinalizeAnalyze(ColumnWriterState &state_p) { auto &state = state_p.Cast(); - child_writer->FinalizeAnalyze(*state.child_state); + GetChildWriter().FinalizeAnalyze(*state.child_state); } static idx_t GetConsecutiveChildList(Vector &list, Vector &result, idx_t offset, idx_t count) { @@ -114,12 +114,12 @@ void ListColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *pa auto child_length = GetConsecutiveChildList(vector, child_list, 0, count); // The elements of a single list should not span multiple Parquet pages // So, we force the entire vector to fit on a single page by setting "vector_can_span_multiple_pages=false" - child_writer->Prepare(*state.child_state, &state_p, child_list, child_length, false); + GetChildWriter().Prepare(*state.child_state, &state_p, child_list, child_length, false); } void ListColumnWriter::BeginWrite(ColumnWriterState &state_p) { auto &state = state_p.Cast(); - child_writer->BeginWrite(*state.child_state); + GetChildWriter().BeginWrite(*state.child_state); } void ListColumnWriter::Write(ColumnWriterState &state_p, Vector &vector, idx_t count) { @@ -128,12 +128,17 @@ void ListColumnWriter::Write(ColumnWriterState &state_p, Vector &vector, idx_t c auto &list_child = ListVector::GetEntry(vector); Vector child_list(list_child); auto child_length = GetConsecutiveChildList(vector, child_list, 0, count); - child_writer->Write(*state.child_state, child_list, child_length); + GetChildWriter().Write(*state.child_state, child_list, child_length); } void ListColumnWriter::FinalizeWrite(ColumnWriterState &state_p) { auto &state = state_p.Cast(); - child_writer->FinalizeWrite(*state.child_state); + GetChildWriter().FinalizeWrite(*state.child_state); +} + +ColumnWriter &ListColumnWriter::GetChildWriter() { + D_ASSERT(child_writers.size() == 1); + return *child_writers[0]; } } // namespace duckdb diff --git a/extension/parquet/writer/primitive_column_writer.cpp b/extension/parquet/writer/primitive_column_writer.cpp index d4087fd3d4d7..d8af437f302c 100644 --- a/extension/parquet/writer/primitive_column_writer.cpp +++ b/extension/parquet/writer/primitive_column_writer.cpp @@ -7,6 +7,9 @@ namespace duckdb { using duckdb_parquet::Encoding; using duckdb_parquet::PageType; +constexpr const idx_t PrimitiveColumnWriter::MAX_UNCOMPRESSED_PAGE_SIZE; +constexpr const idx_t PrimitiveColumnWriter::MAX_UNCOMPRESSED_DICT_PAGE_SIZE; + PrimitiveColumnWriter::PrimitiveColumnWriter(ParquetWriter &writer, const ParquetColumnSchema &column_schema, vector schema_path, bool can_have_nulls) : ColumnWriter(writer, column_schema, std::move(schema_path), can_have_nulls) { @@ -44,7 +47,7 @@ void PrimitiveColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterStat idx_t vcount = parent ? parent->definition_levels.size() - state.definition_levels.size() : count; idx_t parent_index = state.definition_levels.size(); auto &validity = FlatVector::Validity(vector); - HandleRepeatLevels(state, parent, count, MaxRepeat()); + HandleRepeatLevels(state, parent, count); HandleDefineLevels(state, parent, validity, count, MaxDefine(), MaxDefine() - 1); idx_t vector_index = 0; @@ -111,7 +114,7 @@ void PrimitiveColumnWriter::BeginWrite(ColumnWriterState &state_p) { hdr.type = PageType::DATA_PAGE; hdr.__isset.data_page_header = true; - hdr.data_page_header.num_values = UnsafeNumericCast(page_info.row_count); + hdr.data_page_header.num_values = NumericCast(page_info.row_count); hdr.data_page_header.encoding = GetEncoding(state); hdr.data_page_header.definition_level_encoding = Encoding::RLE; hdr.data_page_header.repetition_level_encoding = Encoding::RLE; @@ -302,6 +305,28 @@ void PrimitiveColumnWriter::SetParquetStatistics(PrimitiveColumnWriterState &sta column_chunk.meta_data.statistics.__isset.distinct_count = true; column_chunk.meta_data.__isset.statistics = true; } + + if (state.stats_state->HasGeoStats()) { + + auto gpq_version = writer.GetGeoParquetVersion(); + + const auto has_real_stats = gpq_version == GeoParquetVersion::NONE || gpq_version == GeoParquetVersion::BOTH || + gpq_version == GeoParquetVersion::V2; + const auto has_json_stats = gpq_version == GeoParquetVersion::V1 || gpq_version == GeoParquetVersion::BOTH || + gpq_version == GeoParquetVersion::V2; + + if (has_real_stats) { + // Write the parquet native geospatial statistics + column_chunk.meta_data.__isset.geospatial_statistics = true; + state.stats_state->WriteGeoStats(column_chunk.meta_data.geospatial_statistics); + } + if (has_json_stats) { + // Add the geospatial statistics to the extra GeoParquet metadata + writer.GetGeoParquetData().AddGeoParquetStats(column_schema.name, column_schema.type, + *state.stats_state->GetGeoStats()); + } + } + for (const auto &write_info : state.write_info) { // only care about data page encodings, data_page_header.encoding is meaningless for dict if (write_info.page_header.type != PageType::DATA_PAGE && diff --git a/extension/parquet/writer/struct_column_writer.cpp b/extension/parquet/writer/struct_column_writer.cpp index c70c35ba2e56..c9b6bcf9d7c2 100644 --- a/extension/parquet/writer/struct_column_writer.cpp +++ b/extension/parquet/writer/struct_column_writer.cpp @@ -62,11 +62,12 @@ void StructColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState * auto &validity = FlatVector::Validity(vector); if (parent) { // propagate empty entries from the parent - while (state.is_empty.size() < parent->is_empty.size()) { - state.is_empty.push_back(parent->is_empty[state.is_empty.size()]); + if (state.is_empty.size() < parent->is_empty.size()) { + state.is_empty.insert(state.is_empty.end(), parent->is_empty.begin() + state.is_empty.size(), + parent->is_empty.end()); } } - HandleRepeatLevels(state_p, parent, count, MaxRepeat()); + HandleRepeatLevels(state_p, parent, count); HandleDefineLevels(state_p, parent, validity, count, PARQUET_DEFINE_VALID, MaxDefine() - 1); auto &child_vectors = StructVector::GetEntries(vector); for (idx_t child_idx = 0; child_idx < child_writers.size(); child_idx++) { diff --git a/extension/parquet/writer/variant/CMakeLists.txt b/extension/parquet/writer/variant/CMakeLists.txt new file mode 100644 index 000000000000..955efb46c2a6 --- /dev/null +++ b/extension/parquet/writer/variant/CMakeLists.txt @@ -0,0 +1,5 @@ +add_library_unity(duckdb_parquet_writer_variant OBJECT convert_variant.cpp) + +set(PARQUET_EXTENSION_FILES + ${PARQUET_EXTENSION_FILES} $ + PARENT_SCOPE) diff --git a/extension/parquet/writer/variant/convert_variant.cpp b/extension/parquet/writer/variant/convert_variant.cpp new file mode 100644 index 000000000000..836229d19c3b --- /dev/null +++ b/extension/parquet/writer/variant/convert_variant.cpp @@ -0,0 +1,1208 @@ +#include "writer/variant_column_writer.hpp" +#include "duckdb/common/types/variant.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/function/scalar/variant_utils.hpp" +#include "reader/variant/variant_binary_decoder.hpp" +#include "parquet_shredding.hpp" +#include "duckdb/common/types/decimal.hpp" +#include "duckdb/common/types/uuid.hpp" + +namespace duckdb { + +static idx_t CalculateByteLength(idx_t value) { + if (value == 0) { + return 1; + } + auto value_data = reinterpret_cast(&value); + idx_t irrelevant_bytes = 0; + //! Check how many of the most significant bytes are 0 + for (idx_t i = sizeof(idx_t); i > 0 && value_data[i - 1] == 0; i--) { + irrelevant_bytes++; + } + return sizeof(idx_t) - irrelevant_bytes; +} + +static uint8_t EncodeMetadataHeader(idx_t byte_length) { + D_ASSERT(byte_length <= 4); + + uint8_t header_byte = 0; + //! Set 'version' to 1 + header_byte |= static_cast(1); + //! Set 'sorted_strings' to 1 + header_byte |= static_cast(1) << 4; + //! Set 'offset_size_minus_one' to byte_length-1 + header_byte |= (static_cast(byte_length) - 1) << 6; + +#ifdef DEBUG + auto decoded_header = VariantMetadataHeader::FromHeaderByte(header_byte); + D_ASSERT(decoded_header.offset_size == byte_length); +#endif + + return header_byte; +} + +static void CreateMetadata(UnifiedVariantVectorData &variant, Vector &metadata, idx_t count) { + auto &keys = variant.keys; + auto keys_data = variant.keys_data; + + //! NOTE: the parquet variant is limited to a max dictionary size of NumericLimits::Maximum() + //! Whereas we can have NumericLimits::Maximum() *per* string in DuckDB + auto metadata_data = FlatVector::GetData(metadata); + for (idx_t row = 0; row < count; row++) { + uint64_t dictionary_count = 0; + if (variant.RowIsValid(row)) { + auto list_entry = keys_data[keys.sel->get_index(row)]; + dictionary_count = list_entry.length; + } + idx_t dictionary_size = 0; + for (idx_t i = 0; i < dictionary_count; i++) { + auto &key = variant.GetKey(row, i); + dictionary_size += key.GetSize(); + } + if (dictionary_size >= NumericLimits::Maximum()) { + throw InvalidInputException("The total length of the dictionary exceeds a 4 byte value (uint32_t), failed " + "to export VARIANT to Parquet"); + } + + auto byte_length = CalculateByteLength(dictionary_size); + auto total_length = 1 + (byte_length * (dictionary_count + 2)) + dictionary_size; + + metadata_data[row] = StringVector::EmptyString(metadata, total_length); + auto &metadata_blob = metadata_data[row]; + auto metadata_blob_data = metadata_blob.GetDataWriteable(); + + metadata_blob_data[0] = EncodeMetadataHeader(byte_length); + memcpy(metadata_blob_data + 1, reinterpret_cast(&dictionary_count), byte_length); + + auto offset_ptr = metadata_blob_data + 1 + byte_length; + auto string_ptr = metadata_blob_data + 1 + byte_length + ((dictionary_count + 1) * byte_length); + idx_t total_offset = 0; + for (idx_t i = 0; i < dictionary_count; i++) { + memcpy(offset_ptr + (i * byte_length), reinterpret_cast(&total_offset), byte_length); + auto &key = variant.GetKey(row, i); + + memcpy(string_ptr + total_offset, key.GetData(), key.GetSize()); + total_offset += key.GetSize(); + } + memcpy(offset_ptr + (dictionary_count * byte_length), reinterpret_cast(&total_offset), byte_length); + D_ASSERT(offset_ptr + ((dictionary_count + 1) * byte_length) == string_ptr); + D_ASSERT(string_ptr + total_offset == metadata_blob_data + total_length); + metadata_blob.SetSizeAndFinalize(total_length, total_length); + +#ifdef DEBUG + auto decoded_metadata = VariantMetadata(metadata_blob); + D_ASSERT(decoded_metadata.strings.size() == dictionary_count); + for (idx_t i = 0; i < dictionary_count; i++) { + D_ASSERT(decoded_metadata.strings[i] == variant.GetKey(row, i).GetString()); + } +#endif + } +} + +namespace { + +static unordered_set GetVariantType(const LogicalType &type) { + if (type.id() == LogicalTypeId::ANY) { + return {}; + } + switch (type.id()) { + case LogicalTypeId::STRUCT: + return {VariantLogicalType::OBJECT}; + case LogicalTypeId::LIST: + return {VariantLogicalType::ARRAY}; + case LogicalTypeId::BOOLEAN: + return {VariantLogicalType::BOOL_TRUE, VariantLogicalType::BOOL_FALSE}; + case LogicalTypeId::TINYINT: + return {VariantLogicalType::INT8}; + case LogicalTypeId::SMALLINT: + return {VariantLogicalType::INT16}; + case LogicalTypeId::INTEGER: + return {VariantLogicalType::INT32}; + case LogicalTypeId::BIGINT: + return {VariantLogicalType::INT64}; + case LogicalTypeId::FLOAT: + return {VariantLogicalType::FLOAT}; + case LogicalTypeId::DOUBLE: + return {VariantLogicalType::DOUBLE}; + case LogicalTypeId::DECIMAL: + return {VariantLogicalType::DECIMAL}; + case LogicalTypeId::DATE: + return {VariantLogicalType::DATE}; + case LogicalTypeId::TIME: + return {VariantLogicalType::TIME_MICROS}; + case LogicalTypeId::TIMESTAMP_TZ: + return {VariantLogicalType::TIMESTAMP_MICROS_TZ}; + case LogicalTypeId::TIMESTAMP: + return {VariantLogicalType::TIMESTAMP_MICROS}; + case LogicalTypeId::TIMESTAMP_NS: + return {VariantLogicalType::TIMESTAMP_NANOS}; + case LogicalTypeId::BLOB: + return {VariantLogicalType::BLOB}; + case LogicalTypeId::VARCHAR: + return {VariantLogicalType::VARCHAR}; + case LogicalTypeId::UUID: + return {VariantLogicalType::UUID}; + default: + throw BinderException("Type '%s' can't be translated to a VARIANT type", type.ToString()); + } +} + +struct ShreddingState { +public: + explicit ShreddingState(const LogicalType &type, idx_t total_count) + : type(type), shredded_sel(total_count), values_index_sel(total_count), result_sel(total_count) { + variant_types = GetVariantType(type); + } + +public: + bool ValueIsShredded(UnifiedVariantVectorData &variant, idx_t row, idx_t values_index) { + auto type_id = variant.GetTypeId(row, values_index); + if (!variant_types.count(type_id)) { + return false; + } + if (type_id == VariantLogicalType::DECIMAL) { + auto physical_type = type.InternalType(); + auto decimal_data = VariantUtils::DecodeDecimalData(variant, row, values_index); + auto decimal_physical_type = decimal_data.GetPhysicalType(); + return physical_type == decimal_physical_type; + } + return true; + } + void SetShredded(idx_t row, idx_t values_index, idx_t result_idx) { + shredded_sel[count] = row; + values_index_sel[count] = values_index; + result_sel[count] = result_idx; + count++; + } + case_insensitive_string_set_t ObjectFields() { + D_ASSERT(type.id() == LogicalTypeId::STRUCT); + case_insensitive_string_set_t res; + auto &child_types = StructType::GetChildTypes(type); + for (auto &entry : child_types) { + auto &type = entry.first; + res.emplace(string_t(type.c_str(), type.size())); + } + return res; + } + +public: + //! The type the field is shredded on + const LogicalType &type; + unordered_set variant_types; + //! row that is shredded + SelectionVector shredded_sel; + //! 'values_index' of the shredded value + SelectionVector values_index_sel; + //! result row of the shredded value + SelectionVector result_sel; + //! The amount of rows that are shredded on + idx_t count = 0; +}; + +} // namespace + +vector GetChildIndices(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &nested_data, + optional_ptr shredding_state) { + vector child_indices; + if (!shredding_state || shredding_state->type.id() != LogicalTypeId::STRUCT) { + for (idx_t i = 0; i < nested_data.child_count; i++) { + child_indices.push_back(i); + } + return child_indices; + } + //! FIXME: The variant spec says that field names should be case-sensitive, not insensitive + case_insensitive_string_set_t shredded_fields = shredding_state->ObjectFields(); + + for (idx_t i = 0; i < nested_data.child_count; i++) { + auto keys_index = variant.GetKeysIndex(row, i + nested_data.children_idx); + auto &key = variant.GetKey(row, keys_index); + + if (shredded_fields.count(key)) { + //! This field is shredded on, omit it from the value + continue; + } + child_indices.push_back(i); + } + return child_indices; +} + +static idx_t AnalyzeValueData(const UnifiedVariantVectorData &variant, idx_t row, uint32_t values_index, + vector &offsets, optional_ptr shredding_state) { + idx_t total_size = 0; + //! Every value has at least a value header + total_size++; + + idx_t offset_size = offsets.size(); + VariantLogicalType type_id = VariantLogicalType::VARIANT_NULL; + if (variant.RowIsValid(row)) { + type_id = variant.GetTypeId(row, values_index); + } + switch (type_id) { + case VariantLogicalType::OBJECT: { + auto nested_data = VariantUtils::DecodeNestedData(variant, row, values_index); + + //! Calculate value and key offsets for all children + idx_t total_offset = 0; + uint32_t highest_keys_index = 0; + + auto child_indices = GetChildIndices(variant, row, nested_data, shredding_state); + if (nested_data.child_count && child_indices.empty()) { + //! All fields of the object are shredded, omit the object entirely + return 0; + } + + auto num_elements = child_indices.size(); + offsets.resize(offset_size + num_elements + 1); + + for (idx_t entry = 0; entry < child_indices.size(); entry++) { + auto i = child_indices[entry]; + auto keys_index = variant.GetKeysIndex(row, i + nested_data.children_idx); + auto values_index = variant.GetValuesIndex(row, i + nested_data.children_idx); + offsets[offset_size + entry] = total_offset; + + total_offset += AnalyzeValueData(variant, row, values_index, offsets, nullptr); + highest_keys_index = MaxValue(highest_keys_index, keys_index); + } + offsets[offset_size + num_elements] = total_offset; + + //! Calculate the sizes for the objects value data + auto field_id_size = CalculateByteLength(highest_keys_index); + auto field_offset_size = CalculateByteLength(total_offset); + const bool is_large = num_elements > NumericLimits::Maximum(); + + //! Now add the sizes for the objects value data + if (is_large) { + total_size += sizeof(uint32_t); + } else { + total_size += sizeof(uint8_t); + } + total_size += num_elements * field_id_size; + total_size += (num_elements + 1) * field_offset_size; + total_size += total_offset; + break; + } + case VariantLogicalType::ARRAY: { + auto nested_data = VariantUtils::DecodeNestedData(variant, row, values_index); + + idx_t total_offset = 0; + offsets.resize(offset_size + nested_data.child_count + 1); + for (idx_t i = 0; i < nested_data.child_count; i++) { + auto values_index = variant.GetValuesIndex(row, i + nested_data.children_idx); + offsets[offset_size + i] = total_offset; + + total_offset += AnalyzeValueData(variant, row, values_index, offsets, nullptr); + } + offsets[offset_size + nested_data.child_count] = total_offset; + + auto field_offset_size = CalculateByteLength(total_offset); + auto num_elements = nested_data.child_count; + const bool is_large = num_elements > NumericLimits::Maximum(); + + if (is_large) { + total_size += sizeof(uint32_t); + } else { + total_size += sizeof(uint8_t); + } + total_size += (num_elements + 1) * field_offset_size; + total_size += total_offset; + break; + } + case VariantLogicalType::BLOB: + case VariantLogicalType::VARCHAR: { + auto string_value = VariantUtils::DecodeStringData(variant, row, values_index); + total_size += string_value.GetSize(); + if (type_id == VariantLogicalType::BLOB || string_value.GetSize() > 64) { + //! Save as regular string value + total_size += sizeof(uint32_t); + } + break; + } + case VariantLogicalType::VARIANT_NULL: + case VariantLogicalType::BOOL_TRUE: + case VariantLogicalType::BOOL_FALSE: + break; + case VariantLogicalType::INT8: + total_size += sizeof(uint8_t); + break; + case VariantLogicalType::INT16: + total_size += sizeof(uint16_t); + break; + case VariantLogicalType::INT32: + total_size += sizeof(uint32_t); + break; + case VariantLogicalType::INT64: + total_size += sizeof(uint64_t); + break; + case VariantLogicalType::FLOAT: + total_size += sizeof(float); + break; + case VariantLogicalType::DOUBLE: + total_size += sizeof(double); + break; + case VariantLogicalType::DECIMAL: { + auto decimal_data = VariantUtils::DecodeDecimalData(variant, row, values_index); + total_size += 1; + if (decimal_data.width <= 9) { + total_size += sizeof(int32_t); + } else if (decimal_data.width <= 18) { + total_size += sizeof(int64_t); + } else if (decimal_data.width <= 38) { + total_size += sizeof(uhugeint_t); + } else { + throw InvalidInputException("Can't convert VARIANT DECIMAL(%d, %d) to Parquet VARIANT", decimal_data.width, + decimal_data.scale); + } + break; + } + case VariantLogicalType::UUID: + total_size += sizeof(uhugeint_t); + break; + case VariantLogicalType::DATE: + total_size += sizeof(uint32_t); + break; + case VariantLogicalType::TIME_MICROS: + case VariantLogicalType::TIMESTAMP_MICROS: + case VariantLogicalType::TIMESTAMP_NANOS: + case VariantLogicalType::TIMESTAMP_MICROS_TZ: + total_size += sizeof(uint64_t); + break; + case VariantLogicalType::INTERVAL: + case VariantLogicalType::BIGNUM: + case VariantLogicalType::BITSTRING: + case VariantLogicalType::TIMESTAMP_MILIS: + case VariantLogicalType::TIMESTAMP_SEC: + case VariantLogicalType::TIME_MICROS_TZ: + case VariantLogicalType::TIME_NANOS: + case VariantLogicalType::UINT8: + case VariantLogicalType::UINT16: + case VariantLogicalType::UINT32: + case VariantLogicalType::UINT64: + case VariantLogicalType::UINT128: + case VariantLogicalType::INT128: + default: + throw InvalidInputException("Can't convert VARIANT of type '%s' to Parquet VARIANT", + EnumUtil::ToString(type_id)); + } + + return total_size; +} + +template +void WritePrimitiveTypeHeader(data_ptr_t &value_data) { + uint8_t value_header = 0; + value_header |= static_cast(VariantBasicType::PRIMITIVE); + value_header |= static_cast(TYPE_ID) << 2; + + *value_data = value_header; + value_data++; +} + +template +void CopySimplePrimitiveData(const UnifiedVariantVectorData &variant, data_ptr_t &value_data, idx_t row, + uint32_t values_index) { + auto byte_offset = variant.GetByteOffset(row, values_index); + auto data = const_data_ptr_cast(variant.GetData(row).GetData()); + auto ptr = data + byte_offset; + memcpy(value_data, ptr, sizeof(T)); + value_data += sizeof(T); +} + +void CopyUUIDData(const UnifiedVariantVectorData &variant, data_ptr_t &value_data, idx_t row, uint32_t values_index) { + + auto byte_offset = variant.GetByteOffset(row, values_index); + auto data = const_data_ptr_cast(variant.GetData(row).GetData()); + auto ptr = data + byte_offset; + + auto uuid = Load(ptr); + BaseUUID::ToBlob(uuid, value_data); + value_data += sizeof(uhugeint_t); +} + +static void WritePrimitiveValueData(const UnifiedVariantVectorData &variant, idx_t row, uint32_t values_index, + data_ptr_t &value_data, const vector &offsets, idx_t &offset_index) { + VariantLogicalType type_id = VariantLogicalType::VARIANT_NULL; + if (variant.RowIsValid(row)) { + type_id = variant.GetTypeId(row, values_index); + } + + D_ASSERT(type_id != VariantLogicalType::OBJECT && type_id != VariantLogicalType::ARRAY); + switch (type_id) { + case VariantLogicalType::BLOB: + case VariantLogicalType::VARCHAR: { + auto string_value = VariantUtils::DecodeStringData(variant, row, values_index); + auto string_size = string_value.GetSize(); + if (type_id == VariantLogicalType::BLOB || string_size > 64) { + if (type_id == VariantLogicalType::BLOB) { + WritePrimitiveTypeHeader(value_data); + } else { + WritePrimitiveTypeHeader(value_data); + } + Store(string_size, value_data); + value_data += sizeof(uint32_t); + } else { + uint8_t value_header = 0; + value_header |= static_cast(VariantBasicType::SHORT_STRING); + value_header |= static_cast(string_size) << 2; + + *value_data = value_header; + value_data++; + } + memcpy(value_data, reinterpret_cast(string_value.GetData()), string_size); + value_data += string_size; + break; + } + case VariantLogicalType::VARIANT_NULL: + WritePrimitiveTypeHeader(value_data); + break; + case VariantLogicalType::BOOL_TRUE: + WritePrimitiveTypeHeader(value_data); + break; + case VariantLogicalType::BOOL_FALSE: + WritePrimitiveTypeHeader(value_data); + break; + case VariantLogicalType::INT8: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::INT16: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::INT32: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::INT64: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::FLOAT: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::DOUBLE: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::UUID: + WritePrimitiveTypeHeader(value_data); + CopyUUIDData(variant, value_data, row, values_index); + break; + case VariantLogicalType::DATE: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::TIME_MICROS: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::TIMESTAMP_MICROS: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::TIMESTAMP_NANOS: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::TIMESTAMP_MICROS_TZ: + WritePrimitiveTypeHeader(value_data); + CopySimplePrimitiveData(variant, value_data, row, values_index); + break; + case VariantLogicalType::DECIMAL: { + auto decimal_data = VariantUtils::DecodeDecimalData(variant, row, values_index); + + if (decimal_data.width <= 4 || decimal_data.width > 38) { + throw InvalidInputException("Can't convert VARIANT DECIMAL(%d, %d) to Parquet VARIANT", decimal_data.width, + decimal_data.scale); + } else if (decimal_data.width <= 9) { + WritePrimitiveTypeHeader(value_data); + Store(decimal_data.scale, value_data); + value_data++; + memcpy(value_data, decimal_data.value_ptr, sizeof(int32_t)); + value_data += sizeof(int32_t); + } else if (decimal_data.width <= 18) { + WritePrimitiveTypeHeader(value_data); + Store(decimal_data.scale, value_data); + value_data++; + memcpy(value_data, decimal_data.value_ptr, sizeof(int64_t)); + value_data += sizeof(int64_t); + } else if (decimal_data.width <= 38) { + WritePrimitiveTypeHeader(value_data); + Store(decimal_data.scale, value_data); + value_data++; + memcpy(value_data, decimal_data.value_ptr, sizeof(hugeint_t)); + value_data += sizeof(hugeint_t); + } else { + throw InternalException( + "Uncovered VARIANT(DECIMAL) -> Parquet VARIANT conversion for type 'DECIMAL(%d, %d)'", + decimal_data.width, decimal_data.scale); + } + break; + } + case VariantLogicalType::INTERVAL: + case VariantLogicalType::BIGNUM: + case VariantLogicalType::BITSTRING: + case VariantLogicalType::TIMESTAMP_MILIS: + case VariantLogicalType::TIMESTAMP_SEC: + case VariantLogicalType::TIME_MICROS_TZ: + case VariantLogicalType::TIME_NANOS: + case VariantLogicalType::UINT8: + case VariantLogicalType::UINT16: + case VariantLogicalType::UINT32: + case VariantLogicalType::UINT64: + case VariantLogicalType::UINT128: + case VariantLogicalType::INT128: + default: + throw InvalidInputException("Can't convert VARIANT of type '%s' to Parquet VARIANT", + EnumUtil::ToString(type_id)); + } +} + +static void WriteValueData(const UnifiedVariantVectorData &variant, idx_t row, uint32_t values_index, + data_ptr_t &value_data, const vector &offsets, idx_t &offset_index, + optional_ptr shredding_state) { + + VariantLogicalType type_id = VariantLogicalType::VARIANT_NULL; + if (variant.RowIsValid(row)) { + type_id = variant.GetTypeId(row, values_index); + } + if (type_id == VariantLogicalType::OBJECT) { + auto nested_data = VariantUtils::DecodeNestedData(variant, row, values_index); + + //! -- Object value header -- + + auto child_indices = GetChildIndices(variant, row, nested_data, shredding_state); + if (nested_data.child_count && child_indices.empty()) { + throw InternalException( + "The entire should be omitted, should have been handled by the Analyze step already"); + } + auto num_elements = child_indices.size(); + + //! Determine the 'field_id_size' + uint32_t highest_keys_index = 0; + for (auto &i : child_indices) { + auto keys_index = variant.GetKeysIndex(row, i + nested_data.children_idx); + highest_keys_index = MaxValue(highest_keys_index, keys_index); + } + auto field_id_size = CalculateByteLength(highest_keys_index); + + uint32_t last_offset = 0; + if (num_elements) { + last_offset = offsets[offset_index + num_elements]; + } + offset_index += num_elements + 1; + auto field_offset_size = CalculateByteLength(last_offset); + + const bool is_large = num_elements > NumericLimits::Maximum(); + + uint8_t value_header = 0; + value_header |= static_cast(VariantBasicType::OBJECT); + value_header |= static_cast(is_large) << 6; + value_header |= (static_cast(field_id_size) - 1) << 4; + value_header |= (static_cast(field_offset_size) - 1) << 2; + +#ifdef DEBUG + auto object_value_header = VariantValueMetadata::FromHeaderByte(value_header); + D_ASSERT(object_value_header.basic_type == VariantBasicType::OBJECT); + D_ASSERT(object_value_header.is_large == is_large); + D_ASSERT(object_value_header.field_offset_size == field_offset_size); + D_ASSERT(object_value_header.field_id_size == field_id_size); +#endif + + *value_data = value_header; + value_data++; + + //! Write the 'num_elements' + if (is_large) { + Store(static_cast(num_elements), value_data); + value_data += sizeof(uint32_t); + } else { + Store(static_cast(num_elements), value_data); + value_data += sizeof(uint8_t); + } + + //! Write the 'field_id' entries + for (auto &i : child_indices) { + auto keys_index = variant.GetKeysIndex(row, i + nested_data.children_idx); + memcpy(value_data, reinterpret_cast(&keys_index), field_id_size); + value_data += field_id_size; + } + + //! Write the 'field_offset' entries and the child 'value's + auto children_ptr = value_data + ((num_elements + 1) * field_offset_size); + idx_t total_offset = 0; + for (auto &i : child_indices) { + auto values_index = variant.GetValuesIndex(row, i + nested_data.children_idx); + + memcpy(value_data, reinterpret_cast(&total_offset), field_offset_size); + value_data += field_offset_size; + auto start_ptr = children_ptr; + WriteValueData(variant, row, values_index, children_ptr, offsets, offset_index, nullptr); + total_offset += (children_ptr - start_ptr); + } + memcpy(value_data, reinterpret_cast(&total_offset), field_offset_size); + value_data += field_offset_size; + D_ASSERT(children_ptr - total_offset == value_data); + value_data = children_ptr; + } else if (type_id == VariantLogicalType::ARRAY) { + auto nested_data = VariantUtils::DecodeNestedData(variant, row, values_index); + + //! -- Array value header -- + + uint32_t last_offset = 0; + if (nested_data.child_count) { + last_offset = offsets[offset_index + nested_data.child_count]; + } + offset_index += nested_data.child_count + 1; + auto field_offset_size = CalculateByteLength(last_offset); + + auto num_elements = nested_data.child_count; + const bool is_large = num_elements > NumericLimits::Maximum(); + + uint8_t value_header = 0; + value_header |= static_cast(VariantBasicType::ARRAY); + value_header |= static_cast(is_large) << 4; + value_header |= (static_cast(field_offset_size) - 1) << 2; + +#ifdef DEBUG + auto array_value_header = VariantValueMetadata::FromHeaderByte(value_header); + D_ASSERT(array_value_header.basic_type == VariantBasicType::ARRAY); + D_ASSERT(array_value_header.is_large == is_large); + D_ASSERT(array_value_header.field_offset_size == field_offset_size); +#endif + + *value_data = value_header; + value_data++; + + //! Write the 'num_elements' + if (is_large) { + Store(static_cast(num_elements), value_data); + value_data += sizeof(uint32_t); + } else { + Store(static_cast(num_elements), value_data); + value_data += sizeof(uint8_t); + } + + //! Write the 'field_offset' entries and the child 'value's + auto children_ptr = value_data + ((num_elements + 1) * field_offset_size); + idx_t total_offset = 0; + for (idx_t i = 0; i < nested_data.child_count; i++) { + auto values_index = variant.GetValuesIndex(row, i + nested_data.children_idx); + + memcpy(value_data, reinterpret_cast(&total_offset), field_offset_size); + value_data += field_offset_size; + auto start_ptr = children_ptr; + WriteValueData(variant, row, values_index, children_ptr, offsets, offset_index, nullptr); + total_offset += (children_ptr - start_ptr); + } + memcpy(value_data, reinterpret_cast(&total_offset), field_offset_size); + value_data += field_offset_size; + D_ASSERT(children_ptr - total_offset == value_data); + value_data = children_ptr; + } else { + WritePrimitiveValueData(variant, row, values_index, value_data, offsets, offset_index); + } +} + +static void CreateValues(UnifiedVariantVectorData &variant, Vector &value, optional_ptr sel, + optional_ptr value_index_sel, + optional_ptr result_sel, optional_ptr shredding_state, + idx_t count) { + auto &validity = FlatVector::Validity(value); + auto value_data = FlatVector::GetData(value); + + for (idx_t i = 0; i < count; i++) { + idx_t value_index = 0; + if (value_index_sel) { + value_index = value_index_sel->get_index(i); + } + + idx_t row = i; + if (sel) { + row = sel->get_index(i); + } + + idx_t result_index = i; + if (result_sel) { + result_index = result_sel->get_index(i); + } + + bool is_shredded = false; + if (variant.RowIsValid(row) && shredding_state && shredding_state->ValueIsShredded(variant, row, value_index)) { + shredding_state->SetShredded(row, value_index, result_index); + is_shredded = true; + if (shredding_state->type.id() != LogicalTypeId::STRUCT) { + //! Value is shredded, directly write a NULL to the 'value' if the type is not an OBJECT + //! When the type is OBJECT, all excess fields would still need to be written to the 'value' + validity.SetInvalid(result_index); + continue; + } + } + + //! The (relative) offsets for each value, in the case of nesting + vector offsets; + //! Determine the size of this 'value' blob + idx_t blob_length = AnalyzeValueData(variant, row, value_index, offsets, shredding_state); + if (!blob_length) { + //! This is only allowed to happen for a shredded OBJECT, where there are no excess fields to write for the + //! OBJECT + (void)is_shredded; + D_ASSERT(is_shredded); + validity.SetInvalid(result_index); + continue; + } + value_data[result_index] = StringVector::EmptyString(value, blob_length); + auto &value_blob = value_data[result_index]; + auto value_blob_data = reinterpret_cast(value_blob.GetDataWriteable()); + + idx_t offset_index = 0; + WriteValueData(variant, row, value_index, value_blob_data, offsets, offset_index, shredding_state); + D_ASSERT(data_ptr_cast(value_blob.GetDataWriteable() + blob_length) == value_blob_data); + value_blob.SetSizeAndFinalize(blob_length, blob_length); + } +} + +//! fwd-declare static method +static void WriteVariantValues(UnifiedVariantVectorData &variant, Vector &result, + optional_ptr sel, + optional_ptr value_index_sel, + optional_ptr result_sel, idx_t count); + +static void WriteTypedObjectValues(UnifiedVariantVectorData &variant, Vector &result, const SelectionVector &sel, + const SelectionVector &value_index_sel, const SelectionVector &result_sel, + idx_t count) { + auto &type = result.GetType(); + D_ASSERT(type.id() == LogicalTypeId::STRUCT); + + auto &validity = FlatVector::Validity(result); + (void)validity; + + //! Collect the nested data for the objects + auto nested_data = make_unsafe_uniq_array_uninitialized(count); + for (idx_t i = 0; i < count; i++) { + auto row = sel[i]; + //! When we're shredding an object, the top-level struct of it should always be valid + D_ASSERT(validity.RowIsValid(result_sel[i])); + auto value_index = value_index_sel[i]; + D_ASSERT(variant.GetTypeId(row, value_index) == VariantLogicalType::OBJECT); + nested_data[i] = VariantUtils::DecodeNestedData(variant, row, value_index); + } + + auto &shredded_types = StructType::GetChildTypes(type); + auto &shredded_fields = StructVector::GetEntries(result); + D_ASSERT(shredded_types.size() == shredded_fields.size()); + + SelectionVector child_values_indexes; + SelectionVector child_row_sel; + SelectionVector child_result_sel; + child_values_indexes.Initialize(count); + child_row_sel.Initialize(count); + child_result_sel.Initialize(count); + + for (idx_t child_idx = 0; child_idx < shredded_types.size(); child_idx++) { + auto &child_vec = *shredded_fields[child_idx]; + D_ASSERT(child_vec.GetType() == shredded_types[child_idx].second); + + //! Prepare the path component to perform the lookup for + auto &key = shredded_types[child_idx].first; + VariantPathComponent path_component; + path_component.lookup_mode = VariantChildLookupMode::BY_KEY; + path_component.key = key; + + ValidityMask lookup_validity(count); + VariantUtils::FindChildValues(variant, path_component, sel, child_values_indexes, lookup_validity, + nested_data.get(), count); + + if (!lookup_validity.AllValid()) { + auto &child_variant_vectors = StructVector::GetEntries(child_vec); + + //! For some of the rows the field is missing, adjust the selection vector to exclude these rows. + idx_t child_count = 0; + for (idx_t i = 0; i < count; i++) { + if (!lookup_validity.RowIsValid(i)) { + //! The field is missing, set it to null + FlatVector::SetNull(*child_variant_vectors[0], result_sel[i], true); + if (child_variant_vectors.size() >= 2) { + FlatVector::SetNull(*child_variant_vectors[1], result_sel[i], true); + } + continue; + } + + child_row_sel[child_count] = sel[i]; + child_values_indexes[child_count] = child_values_indexes[i]; + child_result_sel[child_count] = result_sel[i]; + child_count++; + } + + if (child_count) { + //! If not all rows are missing this field, write the values for it + WriteVariantValues(variant, child_vec, child_row_sel, child_values_indexes, child_result_sel, + child_count); + } + } else { + WriteVariantValues(variant, child_vec, &sel, child_values_indexes, result_sel, count); + } + } +} + +static void WriteTypedArrayValues(UnifiedVariantVectorData &variant, Vector &result, const SelectionVector &sel, + const SelectionVector &value_index_sel, const SelectionVector &result_sel, + idx_t count) { + auto list_data = FlatVector::GetData(result); + + auto nested_data = make_unsafe_uniq_array_uninitialized(count); + + idx_t total_offset = 0; + for (idx_t i = 0; i < count; i++) { + auto row = sel[i]; + auto value_index = value_index_sel[i]; + auto result_row = result_sel[i]; + + D_ASSERT(variant.GetTypeId(row, value_index) == VariantLogicalType::ARRAY); + nested_data[i] = VariantUtils::DecodeNestedData(variant, row, value_index); + + list_entry_t list_entry; + list_entry.length = nested_data[i].child_count; + list_entry.offset = total_offset; + list_data[result_row] = list_entry; + + total_offset += nested_data[i].child_count; + } + ListVector::Reserve(result, total_offset); + ListVector::SetListSize(result, total_offset); + + SelectionVector child_sel; + child_sel.Initialize(total_offset); + + SelectionVector child_value_index_sel; + child_value_index_sel.Initialize(total_offset); + + SelectionVector child_result_sel; + child_result_sel.Initialize(total_offset); + + for (idx_t i = 0; i < count; i++) { + auto row = sel[i]; + auto result_row = result_sel[i]; + + auto &array_data = nested_data[i]; + auto &entry = list_data[result_row]; + for (idx_t j = 0; j < entry.length; j++) { + auto offset = entry.offset + j; + child_sel[offset] = row; + child_value_index_sel[offset] = variant.GetValuesIndex(row, array_data.children_idx + j); + child_result_sel[offset] = offset; + } + } + + auto &child_vector = ListVector::GetEntry(result); + WriteVariantValues(variant, child_vector, child_sel, child_value_index_sel, child_result_sel, total_offset); +} + +//! TODO: introduce a third selection vector, because we also need one to map to the result row to write +//! This becomes necessary when we introduce LISTs into the equation because lists are stored on the same VARIANT row, +//! but we're now going to write the flattened child vector +static void WriteShreddedPrimitive(UnifiedVariantVectorData &variant, Vector &result, const SelectionVector &sel, + const SelectionVector &value_index_sel, const SelectionVector &result_sel, + idx_t count, idx_t type_size) { + auto result_data = FlatVector::GetData(result); + for (idx_t i = 0; i < count; i++) { + auto row = sel[i]; + auto result_row = result_sel[i]; + auto value_index = value_index_sel[i]; + D_ASSERT(variant.RowIsValid(row)); + + auto byte_offset = variant.GetByteOffset(row, value_index); + auto &data = variant.GetData(row); + auto value_ptr = data.GetData(); + auto result_offset = type_size * result_row; + memcpy(result_data + result_offset, value_ptr + byte_offset, type_size); + } +} + +template +static void WriteShreddedDecimal(UnifiedVariantVectorData &variant, Vector &result, const SelectionVector &sel, + const SelectionVector &value_index_sel, const SelectionVector &result_sel, + idx_t count) { + auto result_data = FlatVector::GetData(result); + for (idx_t i = 0; i < count; i++) { + auto row = sel[i]; + auto result_row = result_sel[i]; + auto value_index = value_index_sel[i]; + D_ASSERT(variant.RowIsValid(row) && variant.GetTypeId(row, value_index) == VariantLogicalType::DECIMAL); + + auto decimal_data = VariantUtils::DecodeDecimalData(variant, row, value_index); + D_ASSERT(decimal_data.width <= DecimalWidth::max); + auto result_offset = sizeof(T) * result_row; + memcpy(result_data + result_offset, decimal_data.value_ptr, sizeof(T)); + } +} + +static void WriteShreddedString(UnifiedVariantVectorData &variant, Vector &result, const SelectionVector &sel, + const SelectionVector &value_index_sel, const SelectionVector &result_sel, + idx_t count) { + auto result_data = FlatVector::GetData(result); + for (idx_t i = 0; i < count; i++) { + auto row = sel[i]; + auto result_row = result_sel[i]; + auto value_index = value_index_sel[i]; + D_ASSERT(variant.RowIsValid(row) && (variant.GetTypeId(row, value_index) == VariantLogicalType::VARCHAR || + variant.GetTypeId(row, value_index) == VariantLogicalType::BLOB)); + + auto string_data = VariantUtils::DecodeStringData(variant, row, value_index); + result_data[result_row] = StringVector::AddStringOrBlob(result, string_data); + } +} + +static void WriteShreddedBoolean(UnifiedVariantVectorData &variant, Vector &result, const SelectionVector &sel, + const SelectionVector &value_index_sel, const SelectionVector &result_sel, + idx_t count) { + auto result_data = FlatVector::GetData(result); + for (idx_t i = 0; i < count; i++) { + auto row = sel[i]; + auto result_row = result_sel[i]; + auto value_index = value_index_sel[i]; + D_ASSERT(variant.RowIsValid(row)); + auto type_id = variant.GetTypeId(row, value_index); + D_ASSERT(type_id == VariantLogicalType::BOOL_FALSE || type_id == VariantLogicalType::BOOL_TRUE); + + result_data[result_row] = type_id == VariantLogicalType::BOOL_TRUE; + } +} + +static void WriteTypedPrimitiveValues(UnifiedVariantVectorData &variant, Vector &result, const SelectionVector &sel, + const SelectionVector &value_index_sel, const SelectionVector &result_sel, + idx_t count) { + auto &type = result.GetType(); + D_ASSERT(!type.IsNested()); + switch (type.id()) { + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::DATE: + case LogicalTypeId::TIME: + case LogicalTypeId::TIMESTAMP_TZ: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_NS: + case LogicalTypeId::UUID: { + const auto physical_type = type.InternalType(); + WriteShreddedPrimitive(variant, result, sel, value_index_sel, result_sel, count, GetTypeIdSize(physical_type)); + break; + } + case LogicalTypeId::DECIMAL: { + const auto physical_type = type.InternalType(); + switch (physical_type) { + //! DECIMAL4 + case PhysicalType::INT32: + WriteShreddedDecimal(variant, result, sel, value_index_sel, result_sel, count); + break; + //! DECIMAL8 + case PhysicalType::INT64: + WriteShreddedDecimal(variant, result, sel, value_index_sel, result_sel, count); + break; + //! DECIMAL16 + case PhysicalType::INT128: + WriteShreddedDecimal(variant, result, sel, value_index_sel, result_sel, count); + break; + default: + throw InvalidInputException("Can't shred on column of type '%s'", type.ToString()); + } + break; + } + case LogicalTypeId::BLOB: + case LogicalTypeId::VARCHAR: { + WriteShreddedString(variant, result, sel, value_index_sel, result_sel, count); + break; + } + case LogicalTypeId::BOOLEAN: + WriteShreddedBoolean(variant, result, sel, value_index_sel, result_sel, count); + break; + default: + throw InvalidInputException("Can't shred on type: %s", type.ToString()); + } +} + +static void WriteTypedValues(UnifiedVariantVectorData &variant, Vector &result, const SelectionVector &sel, + const SelectionVector &value_index_sel, const SelectionVector &result_sel, idx_t count) { + auto &type = result.GetType(); + + if (type.id() == LogicalTypeId::STRUCT) { + //! Shredded OBJECT + WriteTypedObjectValues(variant, result, sel, value_index_sel, result_sel, count); + } else if (type.id() == LogicalTypeId::LIST) { + //! Shredded ARRAY + WriteTypedArrayValues(variant, result, sel, value_index_sel, result_sel, count); + } else { + //! Primitive types + WriteTypedPrimitiveValues(variant, result, sel, value_index_sel, result_sel, count); + } +} + +static void WriteVariantValues(UnifiedVariantVectorData &variant, Vector &result, + optional_ptr sel, + optional_ptr value_index_sel, + optional_ptr result_sel, idx_t count) { + optional_ptr value; + optional_ptr typed_value; + + auto &result_type = result.GetType(); + D_ASSERT(result_type.id() == LogicalTypeId::STRUCT); + auto &child_types = StructType::GetChildTypes(result_type); + auto &child_vectors = StructVector::GetEntries(result); + D_ASSERT(child_types.size() == child_vectors.size()); + for (idx_t i = 0; i < child_types.size(); i++) { + auto &name = child_types[i].first; + if (name == "value") { + value = child_vectors[i].get(); + } else if (name == "typed_value") { + typed_value = child_vectors[i].get(); + } + } + + if (typed_value) { + ShreddingState shredding_state(typed_value->GetType(), count); + CreateValues(variant, *value, sel, value_index_sel, result_sel, &shredding_state, count); + + SelectionVector null_values; + if (shredding_state.count) { + WriteTypedValues(variant, *typed_value, shredding_state.shredded_sel, shredding_state.values_index_sel, + shredding_state.result_sel, shredding_state.count); + //! 'shredding_state.result_sel' will always be a subset of 'result_sel', set the rows not in the subset to + //! NULL + idx_t sel_idx = 0; + for (idx_t i = 0; i < count; i++) { + auto original_index = result_sel ? result_sel->get_index(i) : i; + if (sel_idx < shredding_state.count && shredding_state.result_sel[sel_idx] == original_index) { + sel_idx++; + continue; + } + FlatVector::SetNull(*typed_value, original_index, true); + } + } else { + //! Set all rows of the typed_value to NULL, nothing is shredded on + for (idx_t i = 0; i < count; i++) { + FlatVector::SetNull(*typed_value, result_sel ? result_sel->get_index(i) : i, true); + } + } + } else { + CreateValues(variant, *value, sel, value_index_sel, result_sel, nullptr, count); + } +} + +static void ToParquetVariant(DataChunk &input, ExpressionState &state, Vector &result) { + // DuckDB Variant: + // - keys = VARCHAR[] + // - children = STRUCT(keys_index UINTEGER, values_index UINTEGER)[] + // - values = STRUCT(type_id UTINYINT, byte_offset UINTEGER)[] + // - data = BLOB + + // Parquet VARIANT: + // - metadata = BLOB + // - value = BLOB + + auto &variant_vec = input.data[0]; + auto count = input.size(); + + RecursiveUnifiedVectorFormat recursive_format; + Vector::RecursiveToUnifiedFormat(variant_vec, count, recursive_format); + UnifiedVariantVectorData variant(recursive_format); + + auto &result_vectors = StructVector::GetEntries(result); + auto &metadata = *result_vectors[0]; + CreateMetadata(variant, metadata, count); + WriteVariantValues(variant, result, nullptr, nullptr, nullptr, count); + + if (input.AllConstant()) { + result.SetVectorType(VectorType::CONSTANT_VECTOR); + } +} + +LogicalType VariantColumnWriter::TransformTypedValueRecursive(const LogicalType &type) { + switch (type.id()) { + case LogicalTypeId::STRUCT: { + //! Wrap all fields of the struct in a struct with 'value' and 'typed_value' fields + auto &child_types = StructType::GetChildTypes(type); + child_list_t replaced_types; + for (auto &entry : child_types) { + child_list_t child_children; + child_children.emplace_back("value", LogicalType::BLOB); + if (entry.second.id() != LogicalTypeId::VARIANT) { + child_children.emplace_back("typed_value", TransformTypedValueRecursive(entry.second)); + } + replaced_types.emplace_back(entry.first, LogicalType::STRUCT(child_children)); + } + return LogicalType::STRUCT(replaced_types); + } + case LogicalTypeId::LIST: { + auto &child_type = ListType::GetChildType(type); + child_list_t replaced_types; + replaced_types.emplace_back("value", LogicalType::BLOB); + if (child_type.id() != LogicalTypeId::VARIANT) { + replaced_types.emplace_back("typed_value", TransformTypedValueRecursive(child_type)); + } + return LogicalType::LIST(LogicalType::STRUCT(replaced_types)); + } + case LogicalTypeId::UNION: + case LogicalTypeId::MAP: + case LogicalTypeId::VARIANT: + case LogicalTypeId::ARRAY: + throw BinderException("'%s' can't appear inside the a 'typed_value' shredded type!", type.ToString()); + default: + return type; + } +} + +static LogicalType GetParquetVariantType(optional_ptr shredding = nullptr) { + child_list_t children; + children.emplace_back("metadata", LogicalType::BLOB); + children.emplace_back("value", LogicalType::BLOB); + if (shredding) { + children.emplace_back("typed_value", VariantColumnWriter::TransformTypedValueRecursive(*shredding)); + } + auto res = LogicalType::STRUCT(std::move(children)); + res.SetAlias("PARQUET_VARIANT"); + return res; +} + +static unique_ptr BindTransform(ClientContext &context, ScalarFunction &bound_function, + vector> &arguments) { + if (arguments.empty()) { + return nullptr; + } + auto type = ExpressionBinder::GetExpressionReturnType(*arguments[0]); + + if (arguments.size() == 2) { + auto &shredding = *arguments[1]; + auto expr_return_type = ExpressionBinder::GetExpressionReturnType(shredding); + expr_return_type = LogicalType::NormalizeType(expr_return_type); + if (expr_return_type.id() != LogicalTypeId::VARCHAR) { + throw BinderException("Optional second argument 'shredding' has to be of type VARCHAR, i.e: " + "'STRUCT(my_field BOOLEAN)', found type: '%s' instead", + expr_return_type); + } + if (!shredding.IsFoldable()) { + throw BinderException("Optional second argument 'shredding' has to be a constant expression"); + } + Value type_str = ExpressionExecutor::EvaluateScalar(context, shredding); + if (type_str.IsNull()) { + throw BinderException("Optional second argument 'shredding' can not be NULL"); + } + auto shredded_type = TransformStringToLogicalType(type_str.GetValue()); + bound_function.return_type = GetParquetVariantType(shredded_type); + } else { + bound_function.return_type = GetParquetVariantType(); + } + + return nullptr; +} + +ScalarFunction VariantColumnWriter::GetTransformFunction() { + ScalarFunction transform("variant_to_parquet_variant", {LogicalType::VARIANT()}, LogicalType::ANY, ToParquetVariant, + BindTransform); + transform.null_handling = FunctionNullHandling::SPECIAL_HANDLING; + return transform; +} + +} // namespace duckdb diff --git a/scripts/extension-upload-all.sh b/scripts/extension-upload-all.sh index 0194dc63dfd6..d087862d3616 100755 --- a/scripts/extension-upload-all.sh +++ b/scripts/extension-upload-all.sh @@ -47,5 +47,5 @@ do echo "found extension: '$ext_name'" # args: [] - $script_dir/extension-upload-single.sh $ext_name "" "$2" "$1" "duckdb-extensions" true false "$(dirname "$f")" + $script_dir/extension-upload-single.sh $ext_name "" "$2" "$1" "duckdb-core-extensions" true false "$(dirname "$f")" done diff --git a/scripts/extension-upload-from-nightly.sh b/scripts/extension-upload-from-nightly.sh index 5308e4ff0bbc..48c16a72a4b2 100755 --- a/scripts/extension-upload-from-nightly.sh +++ b/scripts/extension-upload-from-nightly.sh @@ -23,7 +23,7 @@ fi # CONFIG FROM_BUCKET=duckdb-extensions-nightly -TO_BUCKET=duckdb-extensions +TO_BUCKET=duckdb-core-extensions CLOUDFRONT_DISTRIBUTION_ID=E2Z28NDMI4PVXP ### COPY THE FILES diff --git a/scripts/extension-upload-repository.sh b/scripts/extension-upload-repository.sh index 3583bbd658df..aa90338d361e 100755 --- a/scripts/extension-upload-repository.sh +++ b/scripts/extension-upload-repository.sh @@ -48,7 +48,7 @@ for version_dir in $BASE_DIR/*; do echo "Processing extension: $ext_name (architecture: $architecture, version: $duckdb_version, path: $f)" # args: [] - $script_dir/extension-upload-single.sh $ext_name "" "$duckdb_version" "$architecture" "duckdb-extensions" true false "$(dirname "$f")" + $script_dir/extension-upload-single.sh $ext_name "" "$duckdb_version" "$architecture" "duckdb-core-extensions" true false "$(dirname "$f")" done echo "" done diff --git a/scripts/extension-upload-wasm.sh b/scripts/extension-upload-wasm.sh index 54b1537b7dab..377c273a6bd2 100644 --- a/scripts/extension-upload-wasm.sh +++ b/scripts/extension-upload-wasm.sh @@ -49,9 +49,9 @@ do # upload compressed extension binary to S3 if [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; then #AWS_SECRET_ACCESS_KEY is empty -> dry run - aws s3 cp $f.brotli s3://duckdb-extensions/$2/$1/$ext.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" --dryrun + aws s3 cp $f.brotli s3://duckdb-core-extensions/$2/$1/$ext.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" --dryrun else - aws s3 cp $f.brotli s3://duckdb-extensions/$2/$1/$ext.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" + aws s3 cp $f.brotli s3://duckdb-core-extensions/$2/$1/$ext.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" fi done diff --git a/scripts/generate_c_api.py b/scripts/generate_c_api.py index 4f51e1863e6c..3ae660714385 100644 --- a/scripts/generate_c_api.py +++ b/scripts/generate_c_api.py @@ -81,6 +81,7 @@ 'streaming_result_interface', 'cast_functions', 'expression_interface', + 'file_system_interface', ] # The file that forms the base for the header generation diff --git a/scripts/generate_metric_enums.py b/scripts/generate_metric_enums.py index fb05478d8cd6..69b9ffd7d19c 100644 --- a/scripts/generate_metric_enums.py +++ b/scripts/generate_metric_enums.py @@ -14,41 +14,49 @@ optimizer_file = os.path.join("..", "src", "include", "duckdb", "common", "enums", "optimizer_type.hpp") metrics = [ - "QUERY_NAME", + "ATTACH_LOAD_STORAGE_LATENCY", + "ATTACH_REPLAY_WAL_LATENCY", "BLOCKED_THREAD_TIME", + "CHECKPOINT_LATENCY", "CPU_TIME", - "EXTRA_INFO", "CUMULATIVE_CARDINALITY", - "OPERATOR_TYPE", - "OPERATOR_CARDINALITY", "CUMULATIVE_ROWS_SCANNED", + "EXTRA_INFO", + "LATENCY", + "OPERATOR_CARDINALITY", + "OPERATOR_NAME", "OPERATOR_ROWS_SCANNED", "OPERATOR_TIMING", + "OPERATOR_TYPE", + "QUERY_NAME", "RESULT_SET_SIZE", - "LATENCY", "ROWS_RETURNED", - "OPERATOR_NAME", "SYSTEM_PEAK_BUFFER_MEMORY", "SYSTEM_PEAK_TEMP_DIR_SIZE", "TOTAL_BYTES_READ", "TOTAL_BYTES_WRITTEN", + "WAITING_TO_ATTACH_LATENCY", ] phase_timing_metrics = [ "ALL_OPTIMIZERS", "CUMULATIVE_OPTIMIZER_TIMING", - "PLANNER", - "PLANNER_BINDING", "PHYSICAL_PLANNER", "PHYSICAL_PLANNER_COLUMN_BINDING", - "PHYSICAL_PLANNER_RESOLVE_TYPES", "PHYSICAL_PLANNER_CREATE_PLAN", + "PHYSICAL_PLANNER_RESOLVE_TYPES", + "PLANNER", + "PLANNER_BINDING", ] query_global_metrics = [ + "ATTACH_LOAD_STORAGE_LATENCY", + "ATTACH_REPLAY_WAL_LATENCY", "BLOCKED_THREAD_TIME", + "CHECKPOINT_LATENCY", "SYSTEM_PEAK_BUFFER_MEMORY", "SYSTEM_PEAK_TEMP_DIR_SIZE", + "WAITING_TO_ATTACH_LATENCY", ] optimizer_types = [] diff --git a/scripts/generate_serialization.py b/scripts/generate_serialization.py index 9e90b9d3db46..cc468dcded47 100644 --- a/scripts/generate_serialization.py +++ b/scripts/generate_serialization.py @@ -53,7 +53,7 @@ def get_file_list(): scripts_dir = os.path.dirname(os.path.abspath(__file__)) -version_map_path = scripts_dir + '/../src/storage/version_map.json' +version_map_path = os.path.join(scripts_dir, '..', 'src', 'storage', 'version_map.json') version_map_file = file = open(version_map_path) version_map = json.load(version_map_file) @@ -90,17 +90,16 @@ def lookup_serialization_version(version: str): f"Specified version ({current_version}) could not be found in the version_map.json, and it is lower than the last defined version ({last_registered_version})!" ) exit(1) - - if hasattr(lookup_serialization_version, 'latest_version'): + if hasattr(versions, 'latest'): # We have already mapped a version to 'latest', check that the versions match - latest_version = getattr(lookup_serialization_version, 'latest_version') + latest_version = getattr(versions, 'latest') if current_version != latest_version: print( - f"Found more than one version that is not present in the version_map.json!: {current_version}, {latest_version}" + f"Found more than one version that is not present in the version_map.json!: Current: {current_version}, Latest: {latest_version}" ) exit(1) else: - setattr(lookup_serialization_version, 'latest_version', current_version) + setattr(lookup_serialization_version, 'latest', current_version) return versions['latest'] return versions[version] @@ -490,8 +489,10 @@ def get_serialize_element(self, entry: MemberVariable): assignment = '.' if self.pointer_type == 'none' else '->' default_argument = '' if default_value is None else f', {get_default_argument(default_value)}' + storage_version = lookup_serialization_version(entry.version) + conditional_serialization = storage_version != 1 template = SERIALIZE_ELEMENT_FORMAT - if entry.status != MemberVariableStatus.EXISTING: + if entry.status != MemberVariableStatus.EXISTING and not conditional_serialization: template = "\t/* [Deleted] ({property_type}) \"{property_name}\" */\n" elif entry.has_default: template = template.replace('WriteProperty', 'WritePropertyWithDefault') @@ -504,10 +505,14 @@ def get_serialize_element(self, entry: MemberVariable): assignment=assignment, ) - storage_version = lookup_serialization_version(entry.version) - if storage_version != 1: + if conditional_serialization: code = [] - code.append(f'\tif (serializer.ShouldSerialize({storage_version})) {{') + if entry.status != MemberVariableStatus.EXISTING: + # conditional delete + code.append(f'\tif (!serializer.ShouldSerialize({storage_version})) {{') + else: + # conditional serialization + code.append(f'\tif (serializer.ShouldSerialize({storage_version})) {{') code.append('\t' + serialization_code) result = '\n'.join(code) + '\t}\n' diff --git a/scripts/generate_storage_info.py b/scripts/generate_storage_info.py index 80bc380b0d70..670b24339b23 100644 --- a/scripts/generate_storage_info.py +++ b/scripts/generate_storage_info.py @@ -36,7 +36,17 @@ def main(): with open(STORAGE_INFO_PATH, "r") as cpp_file: content = cpp_file.read() + for key in version_map['serialization']['values'].keys(): + if key in ['latest']: + continue + if key not in version_map['storage']['values'].keys(): + print(f'Key {key} found in serialization version but not in storage version') + exit(1) + types = ['storage', 'serialization'] for type in version_map: + if type not in types: + print("Unexpected key {type}") + exit(1) capitalized_type = type.capitalize() upper_type = type.upper() array_code = generate_version_info_array( diff --git a/scripts/install_s3_test_server.sh b/scripts/install_s3_test_server.sh deleted file mode 100755 index 33f0e7185207..000000000000 --- a/scripts/install_s3_test_server.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# Note: needs sudo - -unamestr=$(uname) -if [[ "$unamestr" == 'Linux' ]]; then - apt-get install -y docker.io -fi - -docker --version -echo '127.0.0.1 duckdb-minio.com' >> /etc/hosts -echo '127.0.0.1 test-bucket.duckdb-minio.com' >> /etc/hosts -echo '127.0.0.1 test-bucket-2.duckdb-minio.com' >> /etc/hosts -echo '127.0.0.1 test-bucket-public.duckdb-minio.com' >> /etc/hosts \ No newline at end of file diff --git a/scripts/minio_s3.yml b/scripts/minio_s3.yml deleted file mode 100644 index 473b8630a9f0..000000000000 --- a/scripts/minio_s3.yml +++ /dev/null @@ -1,80 +0,0 @@ -services: - minio: - image: minio/minio:RELEASE.2021-11-03T03-36-36Z - hostname: duckdb-minio.com - ports: - - "9000:9000" - - "9001:9001" - volumes: - - /tmp/minio_test_data:/data - - /tmp/minio_root_data:/root/.minio - environment: - - MINIO_ROOT_USER=duckdb_minio_admin - - MINIO_ROOT_PASSWORD=duckdb_minio_admin_password - - MINIO_REGION_NAME=eu-west-1 - - MINIO_DOMAIN=duckdb-minio.com - - MINIO_ACCESS_KEY=duckdb_minio_admin - - MINIO_SECRET_KEY=duckdb_minio_admin_password - command: server /data --console-address ":9001" - - minio_setup: - image: minio/mc:RELEASE.2021-11-05T10-05-06Z - depends_on: - - minio - links: - - minio - volumes: - - ${PWD}/data:/duckdb/data - - entrypoint: > - /bin/sh -c " - until ( - /usr/bin/mc config host add myminio http://duckdb-minio.com:9000 duckdb_minio_admin duckdb_minio_admin_password - ) do - echo '...waiting...' && sleep 1; - done; - - /usr/bin/mc admin user add myminio minio_duckdb_user minio_duckdb_user_password - /usr/bin/mc admin user list myminio - /usr/bin/mc admin user info myminio minio_duckdb_user - /usr/bin/mc admin policy set myminio readwrite user=minio_duckdb_user - - /usr/bin/mc admin user add myminio minio_duckdb_user_2 minio_duckdb_user_2_password - /usr/bin/mc admin user list myminio - /usr/bin/mc admin user info myminio minio_duckdb_user_2 - /usr/bin/mc admin policy set myminio readwrite user=minio_duckdb_user_2 - - /usr/bin/mc rb --force myminio/test-bucket - /usr/bin/mc mb myminio/test-bucket - /usr/bin/mc policy get myminio/test-bucket - - /usr/bin/mc rb --force myminio/test-bucket-2 - /usr/bin/mc mb myminio/test-bucket-2 - /usr/bin/mc policy get myminio/test-bucket-2 - - /usr/bin/mc rb --force myminio/test-bucket-public - /usr/bin/mc mb myminio/test-bucket-public - /usr/bin/mc policy set download myminio/test-bucket-public - /usr/bin/mc policy get myminio/test-bucket-public - - # This is for the test of presigned URLs - # !!! When missing, be sure that you have ran 'scripts/generate_presigned_url.sh' !!! - - # small file upload - /usr/bin/mc cp /duckdb/data/csv/phonenumbers.csv myminio/test-bucket/presigned/phonenumbers.csv - /usr/bin/mc cp /duckdb/data/parquet-testing/glob/t1.parquet myminio/test-bucket/presigned/t1.parquet - - # large file upload - /usr/bin/mc cp /duckdb/data/parquet-testing/presigned/presigned-url-lineitem.parquet myminio/test-bucket/presigned/lineitem_large.parquet - - # Upload the db for the attach - /usr/bin/mc cp /duckdb/data/attach_test/attach.db myminio/test-bucket/presigned/attach.db - /usr/bin/mc cp /duckdb/data/attach_test/lineitem_sf1.db myminio/test-bucket/presigned/lineitem_sf1.db - - /usr/bin/mc share download myminio/test-bucket/presigned/phonenumbers.csv - /usr/bin/mc share download myminio/test-bucket/presigned/t1.parquet - /usr/bin/mc share download myminio/test-bucket/presigned/lineitem_large.parquet - /usr/bin/mc share download myminio/test-bucket/presigned/attach.db - - exit 0; - " \ No newline at end of file diff --git a/scripts/run_s3_test_server.sh b/scripts/run_s3_test_server.sh deleted file mode 100755 index 8809292f3c35..000000000000 --- a/scripts/run_s3_test_server.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -#Note: DONT run as root - -if [ ! -f data/attach_test/attach.db ]; then - echo "File data/attach_test/attach.db not found, run ./scripts/generate_presigned_url.sh to generate" - exit 1 -fi - -rm -rf /tmp/minio_test_data -rm -rf /tmp/minio_root_data -mkdir -p /tmp/minio_test_data -mkdir -p /tmp/minio_root_data -docker compose -f scripts/minio_s3.yml -p duckdb-minio up -d - -# for testing presigned url -sleep 10 -container_name=$(docker ps -a --format '{{.Names}}' | grep -m 1 "duckdb-minio") -echo $container_name - -export S3_SMALL_CSV_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*phonenumbers\.csv' | grep -o 'http[s]\?://[^ ]\+') -echo $S3_SMALL_CSV_PRESIGNED_URL - -export S3_SMALL_PARQUET_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*t1\.parquet' | grep -o 'http[s]\?://[^ ]\+') -echo $S3_SMALL_PARQUET_PRESIGNED_URL - -export S3_LARGE_PARQUET_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*lineitem_large\.parquet' | grep -o 'http[s]\?://[^ ]\+') -echo $S3_LARGE_PARQUET_PRESIGNED_URL - -export S3_ATTACH_DB_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*attach\.db' | grep -o 'http[s]\?://[^ ]\+') -echo $S3_ATTACH_DB_PRESIGNED_URL - -export S3_ATTACH_DB="s3://test-bucket/presigned/attach.db" \ No newline at end of file diff --git a/scripts/run_squid.sh b/scripts/run_squid.sh deleted file mode 100755 index 7b05e6a33e61..000000000000 --- a/scripts/run_squid.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -help() { - echo "Usage: ${0} [port] [auth]" - echo " port Port number for squid to lisen to (by default 3128)" - echo " auth Optional string ('auth') to force user basic authentification (autherwise no authentification is required)" - exit 0 -} - -port='3128' -auth='false' -log_dir="squid_logs" -conf_file="squid.conf" -pid_file='${service_name}.pid' - -while [[ $# -gt 0 ]]; do - case "${1}" in - -h|--help) - help - ;; - -p|--port) - port="${2}" - shift # past argument - shift # past value - ;; - --auth) - auth='true' - conf_file="squid_auth.conf" - pid_file='${service_name}_auth.pid' - shift # past argument - ;; - --log_dir) - log_dir="${2}" - shift # past argument - shift # past value - ;; - *) - echo "Unknown option ${1}" - exit 1 - ;; - esac -done - -mkdir "${log_dir}" -touch "${log_dir}/daemon.log" -chmod -R 777 "${log_dir}" - -echo "http_port 127.0.0.1:${port}" >"${conf_file}" -echo "pid_filename ${pid_file}" >>"${conf_file}" - -echo 'logfile_rotate 0' >>"${conf_file}" -echo "logfile_daemon ${log_dir}/daemon.log" >>"${conf_file}" -echo "access_log ${log_dir}/access.log" >>"${conf_file}" -echo "cache_log ${log_dir}/cache.log" >>"${conf_file}" -echo "cache_store_log ${log_dir}/cache_store.log" >>"${conf_file}" - - -if [[ "${auth}" == "true" ]]; then - # User 'john' with password 'doe' - echo 'john:$apr1$dalj9e7s$AhqY28Hvl3EcNblNJMiXa0' >squid_users - - squid_version="$(squid -v | head -n1 | grep -o 'Version [^ ]*' | cut -d ' ' -f 2)" - if [[ "$(uname)" == "Darwin" ]]; then - auth_basic_program="/opt/homebrew/Cellar/squid/${squid_version}/libexec/basic_ncsa_auth" - else - if [[ -e '/usr/lib64/squid/basic_ncsa_auth' ]]; then - auth_basic_program="/usr/lib64/squid/basic_ncsa_auth" - else - auth_basic_program="/usr/lib/squid/basic_ncsa_auth" - fi - fi - - echo '# Add authentification options' >>"${conf_file}" - echo "auth_param basic program ${auth_basic_program} squid_users" >>"${conf_file}" - echo 'auth_param basic children 3' >>"${conf_file}" - echo 'auth_param basic realm Squid BA' >>"${conf_file}" - echo 'acl auth_users proxy_auth REQUIRED' >>"${conf_file}" - echo 'http_access allow auth_users' >>"${conf_file}" - echo 'http_access deny all' >>"${conf_file}" -else - echo 'http_access allow localhost' >>"${conf_file}" -fi - -exec squid -N -f "${conf_file}" diff --git a/scripts/set_s3_test_server_variables.sh b/scripts/set_s3_test_server_variables.sh deleted file mode 100644 index 860e58f6a3a1..000000000000 --- a/scripts/set_s3_test_server_variables.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -# Run this script with 'source' or the shorthand: '.': -# i.e: source scripts/set_s3_test_server_variables.sh - -# Enable the S3 tests to run -export S3_TEST_SERVER_AVAILABLE=1 - -export AWS_DEFAULT_REGION=eu-west-1 -export AWS_ACCESS_KEY_ID=minio_duckdb_user -export AWS_SECRET_ACCESS_KEY=minio_duckdb_user_password -export DUCKDB_S3_ENDPOINT=duckdb-minio.com:9000 -export DUCKDB_S3_USE_SSL=false diff --git a/scripts/settings_scripts/update_autogenerated_functions.py b/scripts/settings_scripts/update_autogenerated_functions.py index ada2de444a46..7bd199d8a98b 100644 --- a/scripts/settings_scripts/update_autogenerated_functions.py +++ b/scripts/settings_scripts/update_autogenerated_functions.py @@ -34,7 +34,7 @@ def add_autogenerated_global_functions(setting): if setting.on_reset: cpp_code += f"\tif (!OnGlobalReset(db, config)) {{\n" cpp_code += f"\t\treturn;\n\t}}\n" - cpp_code += f"\tconfig.options.{setting.internal_setting} = DBConfig().options.{setting.internal_setting};\n" + cpp_code += f"\tconfig.options.{setting.internal_setting} = DBConfigOptions().{setting.internal_setting};\n" cpp_code += f"}}\n\n" if 'get' not in setting.custom_implementation: cpp_code += f"Value {setting.struct_name}::GetSetting(const ClientContext &context) {{\n" diff --git a/scripts/sqllogictest/__init__.py b/scripts/sqllogictest/__init__.py deleted file mode 100644 index 6053ce836ec1..000000000000 --- a/scripts/sqllogictest/__init__.py +++ /dev/null @@ -1,60 +0,0 @@ -from .token import TokenType, Token -from .base_statement import BaseStatement -from .test import SQLLogicTest -from .base_decorator import BaseDecorator -from .statement import ( - Statement, - Require, - Mode, - Halt, - Load, - Set, - Query, - HashThreshold, - Loop, - Foreach, - Endloop, - RequireEnv, - Restart, - Reconnect, - Sleep, - SleepUnit, - Skip, - Unzip, - Unskip, -) -from .decorator import SkipIf, OnlyIf -from .expected_result import ExpectedResult -from .parser import SQLLogicParser, SQLParserException - -__all__ = [ - TokenType, - Token, - BaseStatement, - SQLLogicTest, - BaseDecorator, - Statement, - ExpectedResult, - Require, - Mode, - Halt, - Load, - Set, - Query, - HashThreshold, - Loop, - Foreach, - Endloop, - RequireEnv, - Restart, - Reconnect, - Sleep, - SleepUnit, - Skip, - Unzip, - Unskip, - SkipIf, - OnlyIf, - SQLLogicParser, - SQLParserException, -] diff --git a/scripts/sqllogictest/base_decorator.py b/scripts/sqllogictest/base_decorator.py deleted file mode 100644 index 93222ce43289..000000000000 --- a/scripts/sqllogictest/base_decorator.py +++ /dev/null @@ -1,6 +0,0 @@ -from sqllogictest.token import Token - - -class BaseDecorator: - def __init__(self, token: Token): - self.token: Token = token diff --git a/scripts/sqllogictest/base_statement.py b/scripts/sqllogictest/base_statement.py deleted file mode 100644 index ed80dbf94c84..000000000000 --- a/scripts/sqllogictest/base_statement.py +++ /dev/null @@ -1,25 +0,0 @@ -from sqllogictest.token import Token, TokenType -from sqllogictest.base_decorator import BaseDecorator -from typing import List - - -class BaseStatement: - def __init__(self, header: Token, line: int): - self.header: Token = header - self.query_line: int = line - self.decorators: List[BaseDecorator] = [] - - def add_decorators(self, decorators: List[BaseDecorator]): - self.decorators = decorators - - def get_decorators(self) -> List[BaseDecorator]: - return self.decorators - - def get_query_line(self) -> int: - return self.query_line - - def get_type(self) -> TokenType: - return self.header.type - - def get_parameters(self) -> List[str]: - return self.header.parameters diff --git a/scripts/sqllogictest/decorator/__init__.py b/scripts/sqllogictest/decorator/__init__.py deleted file mode 100644 index 4457291e1da1..000000000000 --- a/scripts/sqllogictest/decorator/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .skip_if import SkipIf -from .only_if import OnlyIf - -__all__ = [SkipIf, OnlyIf] diff --git a/scripts/sqllogictest/decorator/only_if.py b/scripts/sqllogictest/decorator/only_if.py deleted file mode 100644 index 9f1d8ced499b..000000000000 --- a/scripts/sqllogictest/decorator/only_if.py +++ /dev/null @@ -1,7 +0,0 @@ -from sqllogictest.base_decorator import BaseDecorator -from sqllogictest.token import Token - - -class OnlyIf(BaseDecorator): - def __init__(self, token: Token): - super().__init__(token) diff --git a/scripts/sqllogictest/decorator/skip_if.py b/scripts/sqllogictest/decorator/skip_if.py deleted file mode 100644 index 2ad62288e381..000000000000 --- a/scripts/sqllogictest/decorator/skip_if.py +++ /dev/null @@ -1,7 +0,0 @@ -from sqllogictest.base_decorator import BaseDecorator -from sqllogictest.token import Token - - -class SkipIf(BaseDecorator): - def __init__(self, token: Token): - super().__init__(token) diff --git a/scripts/sqllogictest/expected_result.py b/scripts/sqllogictest/expected_result.py deleted file mode 100644 index ecfd5826903f..000000000000 --- a/scripts/sqllogictest/expected_result.py +++ /dev/null @@ -1,23 +0,0 @@ -from enum import Enum, auto -from typing import Optional, List - - -class ExpectedResult: - class Type(Enum): - SUCCESS = auto() - ERROR = auto() - UNKNOWN = auto() - - def __init__(self, type: "ExpectedResult.Type"): - self.type = type - self.lines: Optional[List[str]] = None - self.column_count: Optional[int] = None - - def add_lines(self, lines: List[str]): - self.lines = lines - - def set_expected_column_count(self, column_count: int): - self.column_count = column_count - - def get_expected_column_count(self) -> Optional[int]: - return self.column_count diff --git a/scripts/sqllogictest/logger.py b/scripts/sqllogictest/logger.py deleted file mode 100644 index aa83897b68eb..000000000000 --- a/scripts/sqllogictest/logger.py +++ /dev/null @@ -1,187 +0,0 @@ -import logging -import termcolor -from typing import Union -from duckdb import tokenize, token_type -from .statement import Query, Statement - - -class SQLLogicTestLogger: - def __init__(self, context, command: Union[Query, Statement], file_name: str): - self.file_name = file_name - self.context = context - self.query_line = command.query_line - self.sql_query = '\n'.join(command.lines) - - def log(self, message): - logging.error(message) - - def print_expected_result(self, values, columns, row_wise): - if row_wise: - for value in values: - print(value) - else: - c = 0 - for value in values: - if c != 0: - print("\t", end="") - print(value, end="") - c += 1 - if c >= columns: - c = 0 - print() - - def print_line_sep(self): - line_sep = "=" * 80 - print(termcolor.colored(line_sep, 'grey')) - - def print_header(self, header): - print(termcolor.colored(header, 'white', attrs=['bold'])) - - def print_file_header(self): - self.print_header(f"File {self.file_name}:{self.query_line})") - - def print_sql(self): - query = self.sql_query.strip() - if not query.endswith(";"): - query += ";" - query = self.context.replace_keywords(query) - print(query) - - def print_sql_formatted(self): - print(termcolor.colored("SQL Query", attrs=['bold'])) - query = self.context.replace_keywords(self.sql_query) - tokens = tokenize(query) - for i, token in enumerate(tokens): - next_token_start = tokens[i + 1].start if i + 1 < len(tokens) else len(query) - token_text = query[token.start : next_token_start] - # Apply highlighting based on token type - if token.type in [token_type.identifier, token_type.numeric_const, token_type.string_const]: - print(termcolor.colored(token_text, 'yellow'), end="") - elif token.type == token_type.keyword: - print(termcolor.colored(token_text, 'green', attrs=['bold']), end="") - else: - print(token_text, end="") - print() - - def print_error_header(self, description): - self.print_line_sep() - print(termcolor.colored(description, 'red', attrs=['bold']), end=" ") - print(termcolor.colored(f"({self.file_name}:{self.query_line})!", attrs=['bold'])) - - def print_result_error(self, result_values, values, expected_column_count, row_wise): - self.print_header("Expected result:") - self.print_line_sep() - self.print_expected_result(values, expected_column_count, row_wise) - self.print_line_sep() - self.print_header("Actual result:") - self.print_line_sep() - self.print_expected_result(result_values, expected_column_count, False) - - def unexpected_failure(self, result): - self.print_line_sep() - print(f"Query unexpectedly failed ({self.file_name}:{self.query_line})\n") - self.print_line_sep() - self.print_sql() - self.print_line_sep() - print(result) # FIXME - - def output_result(self, result, result_values_string): - for column_name in result.names: - print(column_name, end="\t") - print() - for column_type in result.types: - print(column_type.to_string(), end="\t") - print() - self.print_line_sep() - for r in range(result.row_count): - for c in range(result.column_count): - print(result_values_string[r * result.column_count + c], end="\t") - print() - - def output_hash(self, hash_value): - self.print_line_sep() - self.print_sql() - self.print_line_sep() - print(hash_value) - self.print_line_sep() - - def column_count_mismatch(self, result, result_values_string, expected_column_count, row_wise): - self.print_error_header("Wrong column count in query!") - print( - f"Expected {termcolor.colored(expected_column_count, 'white', attrs=['bold'])} columns, but got {termcolor.colored(result.column_count, 'white', attrs=['bold'])} columns" - ) - self.print_line_sep() - self.print_sql() - self.print_line_sep() - self.print_result_error(result_values_string, result._result, expected_column_count, row_wise) - - def not_cleanly_divisible(self, expected_column_count, actual_column_count): - self.print_error_header("Error in test!") - print(f"Expected {expected_column_count} columns, but {actual_column_count} values were supplied") - print("This is not cleanly divisible (i.e. the last row does not have enough values)") - - def wrong_row_count(self, expected_rows, result_values_string, comparison_values, expected_column_count, row_wise): - self.print_error_header("Wrong row count in query!") - row_count = len(result_values_string) - print( - f"Expected {termcolor.colored(int(expected_rows), 'white', attrs=['bold'])} rows, but got {termcolor.colored(row_count, 'white', attrs=['bold'])} rows" - ) - self.print_line_sep() - self.print_sql() - self.print_line_sep() - self.print_result_error(result_values_string, comparison_values, expected_column_count, row_wise) - - def column_count_mismatch_correct_result(self, original_expected_columns, expected_column_count, result): - self.print_line_sep() - self.print_error_header("Wrong column count in query!") - print( - f"Expected {termcolor.colored(original_expected_columns, 'white', attrs=['bold'])} columns, but got {termcolor.colored(expected_column_count, 'white', attrs=['bold'])} columns" - ) - self.print_line_sep() - self.print_sql() - print(f"The expected result {termcolor.colored('matched', 'white', attrs=['bold'])} the query result.") - print( - f"Suggested fix: modify header to \"{termcolor.colored('query', 'green')} {'I' * result.column_count}{termcolor.colored('', 'white')}\"" - ) - self.print_line_sep() - - def split_mismatch(self, row_number, expected_column_count, split_count): - self.print_line_sep() - self.print_error_header(f"Error in test! Column count mismatch after splitting on tab on row {row_number}!") - print( - f"Expected {termcolor.colored(int(expected_column_count), 'white', attrs=['bold'])} columns, but got {termcolor.colored(split_count, 'white', attrs=['bold'])} columns" - ) - print("Does the result contain tab values? In that case, place every value on a single row.") - self.print_line_sep() - - def wrong_result_hash(self, expected_hash_value, hash_value): - self.print_error_header("Wrong result hash when comparing to previous query result!") - self.print_line_sep() - self.print_sql() - self.print_line_sep() - self.print_header("Expected hash value:") - self.print_line_sep() - print(expected_hash_value) - self.print_line_sep() - self.print_header("Actual hash value:") - self.print_line_sep() - print(hash_value) - self.print_line_sep() - - def unexpected_statement(self, expect_ok, result): - description = "Query unexpectedly succeeded!" if not expect_ok else "Query unexpectedly failed!" - self.print_error_header(description) - self.print_line_sep() - self.print_sql() - self.print_line_sep() - result.print() - - def expected_error_mismatch(self, expected_error, result): - self.print_error_header( - f"Query failed, but error message did not match expected error message: {expected_error}" - ) - self.print_line_sep() - self.print_sql() - self.print_header("Actual result:") - self.print_line_sep() - result.print() diff --git a/scripts/sqllogictest/parser/__init__.py b/scripts/sqllogictest/parser/__init__.py deleted file mode 100644 index 90c98254ec73..000000000000 --- a/scripts/sqllogictest/parser/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .parser import SQLLogicParser, SQLParserException, SQLLogicTest - -__all__ = [SQLLogicParser, SQLParserException, SQLLogicTest] diff --git a/scripts/sqllogictest/parser/parser.py b/scripts/sqllogictest/parser/parser.py deleted file mode 100644 index 021693de3898..000000000000 --- a/scripts/sqllogictest/parser/parser.py +++ /dev/null @@ -1,635 +0,0 @@ -import os - -from typing import List, Optional - -from ..token import Token, TokenType - -from ..expected_result import ExpectedResult - -from ..statement import ( - Statement, - Require, - Mode, - Halt, - Set, - Load, - Query, - HashThreshold, - Loop, - Foreach, - Endloop, - RequireEnv, - Restart, - Reconnect, - Sleep, - Skip, - Unzip, - Unskip, - SortStyle, -) -from ..statement.sleep import get_sleep_unit, SleepUnit - -from ..decorator import SkipIf, OnlyIf - -from ..base_decorator import BaseDecorator -from ..base_statement import BaseStatement -from ..test import SQLLogicTest - - -def create_formatted_list(items) -> str: - res = '' - for i, option in enumerate(items): - if i + 1 == len(items): - spacer = ' or ' - elif i != 0: - spacer = ', ' - else: - spacer = '' - res += f"{spacer}'{option}'" - return res - - -def is_space(char: str): - return char == ' ' or char == '\t' or char == '\n' or char == '\v' or char == '\f' or char == '\r' - - -### -------- PARSER ---------- -class SQLParserException(Exception): - def __init__(self, message): - self.message = "Parser Error: " + message - super().__init__(self.message) - - -class SQLLogicParser: - def reset(self): - self.current_line = 0 - self.seen_statement = False - self.lines = [] - self.current_test = None - - def __init__(self): - self.reset() - self.STATEMENTS = { - TokenType.SQLLOGIC_STATEMENT: self.statement_statement, - TokenType.SQLLOGIC_QUERY: self.statement_query, - TokenType.SQLLOGIC_REQUIRE: self.statement_require, - TokenType.SQLLOGIC_HASH_THRESHOLD: self.statement_hash_threshold, - TokenType.SQLLOGIC_HALT: self.statement_halt, - TokenType.SQLLOGIC_MODE: self.statement_mode, - TokenType.SQLLOGIC_SET: self.statement_set, - TokenType.SQLLOGIC_LOOP: self.statement_loop, - TokenType.SQLLOGIC_CONCURRENT_LOOP: self.statement_loop, - TokenType.SQLLOGIC_FOREACH: self.statement_foreach, - TokenType.SQLLOGIC_CONCURRENT_FOREACH: self.statement_foreach, - TokenType.SQLLOGIC_ENDLOOP: self.statement_endloop, - TokenType.SQLLOGIC_REQUIRE_ENV: self.statement_require_env, - TokenType.SQLLOGIC_LOAD: self.statement_load, - TokenType.SQLLOGIC_RESTART: self.statement_restart, - TokenType.SQLLOGIC_RECONNECT: self.statement_reconnect, - TokenType.SQLLOGIC_SLEEP: self.statement_sleep, - TokenType.SQLLOGIC_UNZIP: self.statement_unzip, - TokenType.SQLLOGIC_INVALID: None, - } - self.DECORATORS = { - TokenType.SQLLOGIC_SKIP_IF: self.decorator_skipif, - TokenType.SQLLOGIC_ONLY_IF: self.decorator_onlyif, - } - self.FOREACH_COLLECTIONS = { - "": [ - "none", - "uncompressed", - "rle", - "bitpacking", - "dictionary", - "fsst", - "dict_fsst", - "alp", - "alprd", - ], - "": ["bool", "interval", "varchar"], - "": ["float", "double"], - "": ["tinyint", "smallint", "integer", "bigint", "hugeint"], - "": ["tinyint", "smallint", "integer", "bigint", "hugeint"], - "": ["utinyint", "usmallint", "uinteger", "ubigint", "uhugeint"], - "": [ - "bool", - "tinyint", - "smallint", - "int", - "bigint", - "hugeint", - "uhugeint", - "utinyint", - "usmallint", - "uint", - "ubigint", - "date", - "time", - "timestamp", - "timestamp_s", - "timestamp_ms", - "timestamp_ns", - "time_tz", - "timestamp_tz", - "float", - "double", - "dec_4_1", - "dec_9_4", - "dec_18_6", - "dec38_10", - "uuid", - "interval", - "varchar", - "blob", - "bit", - "small_enum", - "medium_enum", - "large_enum", - "int_array", - "double_array", - "date_array", - "timestamp_array", - "timestamptz_array", - "varchar_array", - "nested_int_array", - "struct", - "struct_of_arrays", - "array_of_structs", - "map", - "union", - "fixed_int_array", - "fixed_varchar_array", - "fixed_nested_int_array", - "fixed_nested_varchar_array", - "fixed_struct_array", - "struct_of_fixed_array", - "fixed_array_of_int_list", - "list_of_fixed_int_array", - ], - } - - def peek(self): - return self.peek_no_strip().strip() - - def peek_no_strip(self): - if self.current_line >= len(self.lines): - raise SQLParserException("File already fully consumed") - return self.lines[self.current_line] - - def consume(self): - if self.current_line >= len(self.lines): - raise SQLParserException("File already fully consumed") - self.current_line += 1 - - def fail(self, message): - file_path = self.current_test.path - error_message = f"{file_path}:{self.current_line + 1}: {message}" - raise SQLParserException(error_message) - - def get_expected_result(self, statement_type: str) -> ExpectedResult: - type_map = { - 'ok': ExpectedResult.Type.SUCCESS, - 'error': ExpectedResult.Type.ERROR, - 'maybe': ExpectedResult.Type.UNKNOWN, - } - if statement_type not in type_map: - error = 'statement argument should be ' + create_formatted_list(type_map.keys()) - self.fail(error) - return ExpectedResult(type_map[statement_type]) - - def extract_expected_lines(self) -> Optional[List[str]]: - end_of_file = self.current_line >= len(self.lines) - if end_of_file or self.peek() != "----": - return None - - self.consume() - result = [] - while self.current_line < len(self.lines) and self.peek_no_strip().strip('\n'): - result.append(self.peek_no_strip().strip('\n')) - self.consume() - return result - - def statement_statement(self, header: Token) -> Optional[BaseStatement]: - options = ['ok', 'error', 'maybe'] - if len(header.parameters) < 1: - self.fail(f"statement requires at least one parameter ({create_formatted_list(options)})") - expected_result = self.get_expected_result(header.parameters[0]) - - statement = Statement(header, self.current_line + 1) - statement.file_name = self.current_test.path - - self.next_line() - statement_text = self.extract_statement() - if statement_text == []: - self.fail("Unexpected empty statement text") - statement.add_lines(statement_text) - - expected_lines: Optional[List[str]] = self.extract_expected_lines() - if expected_result.type == ExpectedResult.Type.SUCCESS: - if expected_lines != None: - if len(expected_lines) != 0: - self.fail( - "Failed to parse statement: only statement error can have an expected error message, not statement ok" - ) - expected_result.add_lines(expected_lines) - elif expected_result.type == ExpectedResult.Type.ERROR or expected_result.type == ExpectedResult.Type.UNKNOWN: - if expected_lines != None: - expected_result.add_lines(expected_lines) - elif not self.current_test.is_sqlite_test(): - print(statement) - self.fail('Failed to parse statement: statement error needs to have an expected error message') - else: - self.fail(f"Unexpected ExpectedResult Type: {expected_result.type.name}") - - statement.expected_result = expected_result - if len(header.parameters) >= 2: - statement.set_connection(header.parameters[1]) - return statement - - def statement_query(self, header: Token) -> BaseStatement: - if len(header.parameters) < 1: - self.fail("query requires at least one parameter (query III)") - query = Query(header, self.current_line + 1) - - # parse the expected column count - query.expected_column_count = 0 - column_text = header.parameters[0] - accepted_chars = ['T', 'I', 'R'] - if not all(x in accepted_chars for x in column_text): - self.fail(f"Found unknown character in {column_text}, expected {create_formatted_list(accepted_chars)}") - expected_column_count = len(column_text) - - query.expected_column_count = expected_column_count - if query.expected_column_count == 0: - self.fail("Query requires at least a single column in the result") - - query.file_name = self.current_test.path - query.query_line = self.current_line + 1 - # extract the SQL statement - self.next_line() - statement_text = self.extract_statement() - query.add_lines(statement_text) - - # extract the expected result - expected_result = self.get_expected_result('ok') - expected_lines: Optional[List[str]] = self.extract_expected_lines() - if expected_lines != None: - expected_result.add_lines(expected_lines) - expected_result.set_expected_column_count(expected_column_count) - query.expected_result = expected_result - - def get_sort_style(parameters: List[str]) -> SortStyle: - sort_style = SortStyle.NO_SORT - if len(parameters) > 1: - sort_style = parameters[1] - if sort_style == "nosort": - # Do no sorting - sort_style = SortStyle.NO_SORT - elif sort_style == "rowsort" or sort_style == "sort": - # Row-oriented sorting - sort_style = SortStyle.ROW_SORT - elif sort_style == "valuesort": - # Sort all values independently - sort_style = SortStyle.VALUE_SORT - else: - sort_style = SortStyle.UNKNOWN - return sort_style - - # figure out the sort style - sort_style = get_sort_style(header.parameters) - if sort_style == SortStyle.UNKNOWN: - sort_style = SortStyle.NO_SORT - query.set_connection(header.parameters[1]) - query.set_sortstyle(sort_style) - - # check the label of the query - if len(header.parameters) > 2: - query.set_label(header.parameters[2]) - return query - - def statement_hash_threshold(self, header: Token) -> Optional[BaseStatement]: - if len(header.parameters) != 1: - self.fail("hash-threshold requires a parameter") - threshold = int(header.parameters[0]) - return HashThreshold(header, self.current_line + 1, threshold) - - def statement_halt(self, header: Token) -> Optional[BaseStatement]: - return Halt(header, self.current_line + 1) - - def statement_mode(self, header: Token) -> Optional[BaseStatement]: - if len(header.parameters) != 1: - self.fail("mode requires one parameter") - parameter = header.parameters[0] - if parameter == "skip": - return Skip(header, self.current_line + 1) - elif parameter == "unskip": - return Unskip(header, self.current_line + 1) - else: - return Mode(header, self.current_line + 1, parameter) - - def statement_require(self, header: Token) -> Optional[BaseStatement]: - if len(header.parameters) < 1: - self.fail("require requires a single parameter") - return Require(header, self.current_line + 1) - - def statement_set(self, header: Token) -> Optional[BaseStatement]: - parameters = header.parameters - if len(parameters) < 1: - self.fail("set requires at least 1 parameter (e.g. set ignore_error_messages HTTP Error)") - accepted_options = ['ignore_error_messages', 'always_fail_error_messages', 'seed'] - if parameters[0] in accepted_options: - error_messages = [] - # Parse the parameter list as a comma separated list of strings that can contain spaces - # e.g. `set ignore_error_messages This is an error message, This_is_another, and another` - tmp = [[y.strip() for y in x.split(',') if y.strip() != ''] for x in parameters[1:]] - for x in tmp: - error_messages.extend(x) - statement = Set(header, self.current_line + 1) - statement.add_error_messages(error_messages) - return statement - else: - self.fail( - f"unrecognized set parameter: {parameters[0]}, expected {create_formatted_list(accepted_options)}" - ) - - def statement_load(self, header: Token) -> Optional[BaseStatement]: - statement = Load(header, self.current_line + 1) - if len(header.parameters) > 1 and header.parameters[1] == "readonly": - statement.set_readonly() - if len(header.parameters) > 2: - statement.set_version(header.parameters[2]) - return statement - - def statement_loop(self, header: Token) -> Optional[BaseStatement]: - if len(header.parameters) != 3: - self.fail("Expected loop [iterator_name] [start] [end] (e.g. loop i 1 300)") - is_parallel = header.type == TokenType.SQLLOGIC_CONCURRENT_LOOP - statement = Loop(header, self.current_line + 1, is_parallel) - statement.set_name(header.parameters[0]) - statement.set_start(int(header.parameters[1])) - statement.set_end(int(header.parameters[2])) - return statement - - def statement_foreach(self, header: Token) -> Optional[BaseStatement]: - if len(header.parameters) < 2: - self.fail( - "Expected foreach [iterator_name] [m1] [m2] [etc...] (e.g. foreach type integer " "smallint float)" - ) - is_parallel = header.type == TokenType.SQLLOGIC_CONCURRENT_FOREACH - statement = Foreach(header, self.current_line + 1, is_parallel) - statement.set_name(header.parameters[0]) - raw_values = header.parameters[1:] - - def add_tokens(result, param): - token_name = param.lower().strip() - - if token_name in self.FOREACH_COLLECTIONS: - result.extend(self.FOREACH_COLLECTIONS[token_name]) - else: - result.append(param) - - foreach_tokens = [] - for value in raw_values: - add_tokens(foreach_tokens, value) - - statement.set_values(foreach_tokens) - return statement - - def statement_endloop(self, header: Token) -> Optional[BaseStatement]: - return Endloop(header, self.current_line + 1) - - def statement_require_env(self, header: Token) -> Optional[BaseStatement]: - if len(header.parameters) != 1 and len(header.parameters) != 2: - self.fail("require-env requires 1 argument: [optional: ]") - return RequireEnv(header, self.current_line + 1) - - def statement_restart(self, header: Token) -> Optional[BaseStatement]: - return Restart(header, self.current_line + 1) - - def statement_reconnect(self, header: Token) -> Optional[BaseStatement]: - return Reconnect(header, self.current_line + 1) - - def statement_sleep(self, header: Token) -> Optional[BaseStatement]: - if len(header.parameters) != 2: - self.fail("sleep requires two parameter (e.g. sleep 1 second)") - sleep_duration = int(header.parameters[0]) - sleep_unit = get_sleep_unit(header.parameters[1]) - if sleep_unit == SleepUnit.UNKNOWN: - options = ['second', 'millisecond', 'microsecond', 'nanosecond'] - self.fail(f"Unrecognized sleep mode - expected {create_formatted_list(options)}") - return Sleep(header, self.current_line + 1, sleep_duration, sleep_unit) - - def statement_unzip(self, header: Token) -> Optional[BaseStatement]: - params = header.parameters - if len(params) != 1 and len(params) != 2: - docs = """ - unzip requires 1 parameter, the path to a (g)zipped file. - Optionally a destination location can be provided, defaulting to '__TEST_DIR__/' - """ - self.fail(docs) - - source = params[0] - - accepted_filetypes = {'.gz'} - - basename = os.path.basename(source) - stem, extension = os.path.splitext(basename) - if extension not in accepted_filetypes: - accepted_options = ", ".join(list(accepted_filetypes)) - self.fail( - f"unzip: input does not end in a valid file extension ({extension}), accepted options are: {accepted_options}" - ) - destination = params[1] if len(params) == 2 else f'__TEST_DIR__/{stem}' - return Unzip(header, self.current_line + 1, source, destination) - - # Decorators - - def decorator_skipif(self, token: Token) -> Optional[BaseDecorator]: - return SkipIf(token) - - def decorator_onlyif(self, token: Token) -> Optional[BaseDecorator]: - return OnlyIf(token) - - def parse(self, file_path: str) -> Optional[SQLLogicTest]: - if not self.open_file(file_path): - return None - - while self.next_statement(): - token = self.tokenize() - - # throw explicit error on single line statements that are not separated by a comment or newline - if self.is_single_line_statement(token) and not self.next_line_empty_or_comment(): - self.fail("All test statements need to be separated by an empty line") - - # Parse any number of decorators first - parse_method = self.DECORATORS.get(token.type) - decorators: List[BaseDecorator] = [] - while parse_method != None: - decorator = parse_method(token) - if not decorator: - self.fail(f"Parser did not produce a decorator for {token.type.name}") - decorators.append(decorator) - self.next_line() - token = self.tokenize() - parse_method = self.DECORATORS.get(token.type) - - # Then parse the statement - parse_method = self.STATEMENTS.get(token.type) - if parse_method: - statement = parse_method(token) - else: - self.fail(f"Unexpected token type: {token.type.name}") - if not statement: - self.fail(f"Parser did not produce a statement for {token.type.name}") - statement.add_decorators(decorators) - self.current_test.add_statement(statement) - return self.current_test - - def open_file(self, path): - self.reset() - self.current_test = SQLLogicTest(path) - try: - with open(path, 'r') as infile: - self.lines = [line.replace("\r", "") for line in infile.readlines()] - return True - except IOError: - return False - except UnicodeDecodeError: - return False - - def empty_or_comment(self, line): - return not line.strip('\n') or line.startswith("#") - - def next_line_empty_or_comment(self): - if self.current_line + 1 >= len(self.lines): - return True - else: - return self.empty_or_comment(self.lines[self.current_line + 1]) - - def eof(self): - return self.current_line >= len(self.lines) - - def next_statement(self): - if self.seen_statement: - while not self.eof() and not self.empty_or_comment(self.peek()): - self.consume() - self.seen_statement = True - - while not self.eof() and self.empty_or_comment(self.peek()): - self.consume() - - return not self.eof() - - def next_line(self): - self.consume() - - def extract_statement(self): - statement = [] - - while not self.eof() and not self.empty_or_comment(self.peek_no_strip()): - line = self.peek_no_strip() - if line.strip('\n') == "----": - break - statement.append(line.strip('\n')) - self.consume() - return statement - - def tokenize(self): - result = Token() - if self.current_line >= len(self.lines): - result.type = TokenType.SQLLOGIC_INVALID - return result - - line = self.peek_no_strip() - argument_list = line.split() - argument_list = [x for x in line.strip('\n').split() if not is_space(x)] - - if not argument_list: - self.fail("Empty line!?") - - result.type = self.command_to_token(argument_list[0]) - result.parameters.extend(argument_list[1:]) - return result - - def is_single_line_statement(self, token): - single_line_statements = [ - TokenType.SQLLOGIC_HASH_THRESHOLD, - TokenType.SQLLOGIC_HALT, - TokenType.SQLLOGIC_MODE, - TokenType.SQLLOGIC_SET, - TokenType.SQLLOGIC_LOOP, - TokenType.SQLLOGIC_FOREACH, - TokenType.SQLLOGIC_CONCURRENT_LOOP, - TokenType.SQLLOGIC_CONCURRENT_FOREACH, - TokenType.SQLLOGIC_ENDLOOP, - TokenType.SQLLOGIC_REQUIRE, - TokenType.SQLLOGIC_REQUIRE_ENV, - TokenType.SQLLOGIC_LOAD, - TokenType.SQLLOGIC_RESTART, - TokenType.SQLLOGIC_RECONNECT, - TokenType.SQLLOGIC_SLEEP, - TokenType.SQLLOGIC_UNZIP, - ] - - if token.type in single_line_statements: - return True - elif token.type in [ - TokenType.SQLLOGIC_SKIP_IF, - TokenType.SQLLOGIC_ONLY_IF, - TokenType.SQLLOGIC_INVALID, - TokenType.SQLLOGIC_STATEMENT, - TokenType.SQLLOGIC_QUERY, - ]: - return False - else: - raise RuntimeError("Unknown SQLLogic token found!") - - def command_to_token(self, token): - token_map = { - "skipif": TokenType.SQLLOGIC_SKIP_IF, - "onlyif": TokenType.SQLLOGIC_ONLY_IF, - "statement": TokenType.SQLLOGIC_STATEMENT, - "query": TokenType.SQLLOGIC_QUERY, - "hash-threshold": TokenType.SQLLOGIC_HASH_THRESHOLD, - "halt": TokenType.SQLLOGIC_HALT, - "mode": TokenType.SQLLOGIC_MODE, - "set": TokenType.SQLLOGIC_SET, - "loop": TokenType.SQLLOGIC_LOOP, - "concurrentloop": TokenType.SQLLOGIC_CONCURRENT_LOOP, - "foreach": TokenType.SQLLOGIC_FOREACH, - "concurrentforeach": TokenType.SQLLOGIC_CONCURRENT_FOREACH, - "endloop": TokenType.SQLLOGIC_ENDLOOP, - "require": TokenType.SQLLOGIC_REQUIRE, - "require-env": TokenType.SQLLOGIC_REQUIRE_ENV, - "load": TokenType.SQLLOGIC_LOAD, - "restart": TokenType.SQLLOGIC_RESTART, - "reconnect": TokenType.SQLLOGIC_RECONNECT, - "unzip": TokenType.SQLLOGIC_UNZIP, - "sleep": TokenType.SQLLOGIC_SLEEP, - } - - if token in token_map: - return token_map[token] - else: - self.fail(f"Unrecognized parameter {token}") - return TokenType.SQLLOGIC_INVALID - - -import argparse - - -def main(): - parser = argparse.ArgumentParser(description="SQL Logic Parser") - parser.add_argument("filename", type=str, help="Path to the SQL logic file") - args = parser.parse_args() - - filename = args.filename - - parser = SQLLogicParser() - out: Optional[SQLLogicTest] = parser.parse(filename) - if not out: - raise SQLParserException(f"Test {filename} could not be parsed") - - -if __name__ == "__main__": - main() diff --git a/scripts/sqllogictest/result.py b/scripts/sqllogictest/result.py deleted file mode 100644 index a631860e51b1..000000000000 --- a/scripts/sqllogictest/result.py +++ /dev/null @@ -1,1365 +0,0 @@ -from hashlib import md5 -import gc - -from .base_statement import BaseStatement -from .test import SQLLogicTest -from .statement import ( - Statement, - Require, - Mode, - Halt, - Set, - Load, - Query, - HashThreshold, - Loop, - Foreach, - Endloop, - RequireEnv, - Restart, - Reconnect, - Sleep, - SleepUnit, - Skip, - Unzip, - SortStyle, - Unskip, -) - -from .expected_result import ExpectedResult -from typing import Optional, Any, Tuple, List, Dict, Generator -import typing - -from .logger import SQLLogicTestLogger -import duckdb -import os -import math -import time -import threading - -import re -from functools import cmp_to_key -from enum import Enum - -### Helper structs - - -class RequireResult(Enum): - MISSING = 0 - PRESENT = 1 - - -class ExecuteResult: - class Type(Enum): - SUCCESS = 0 - ERROR = 1 - SKIPPED = 2 - - def __init__(self, type: "ExecuteResult.Type"): - self.type = type - - -### Exceptions - -BUILTIN_EXTENSIONS = [ - 'json', - 'parquet', - 'icu', -] - -from duckdb import DuckDBPyConnection - -# def patch_execute(method): -# def patched_execute(self, *args, **kwargs): -# print(*args) -# return method(self, *args, **kwargs) -# return patched_execute - -# patched_execute = patch_execute(getattr(DuckDBPyConnection, "execute")) -# setattr(DuckDBPyConnection, "execute", patched_execute) - - -class SQLLogicStatementData: - # Context information about a statement - def __init__(self, test: SQLLogicTest, statement: BaseStatement): - self.test = test - self.statement = statement - - def __str__(self) -> str: - return f'{self.test.path}:{self.statement.get_query_line()}' - - __repr__ = __str__ - - -class TestException(Exception): - __test__ = False - __slots__ = ['data', 'message', 'result'] - - def __init__(self, data: SQLLogicStatementData, message: str, result: ExecuteResult): - self.message = message - super().__init__(self.message) - self.data = data - self.result = result - - def handle_result(self) -> ExecuteResult: - return self.result - - -class SkipException(TestException): - def __init__(self, data: SQLLogicStatementData, message: str): - super().__init__(data, message, ExecuteResult(ExecuteResult.Type.SKIPPED)) - - -class FailException(TestException): - def __init__(self, data: SQLLogicStatementData, message: str): - super().__init__(data, message, ExecuteResult(ExecuteResult.Type.ERROR)) - - -### Result primitive - - -class QueryResult: - def __init__(self, result: List[Tuple[Any]], types: List[str], error: Optional[Exception] = None): - self._result = result - self.types = types - self.error = error - if not error: - self._column_count = len(self.types) - self._row_count = len(result) - if self._row_count > 0: - assert self._column_count == len(self._result[0]) - - def get_value(self, column, row): - return self._result[row][column] - - def row_count(self) -> int: - return self._row_count - - @property - def column_count(self) -> int: - assert self._column_count != 0 - return self._column_count - - def has_error(self) -> bool: - return self.error != None - - def get_error(self) -> Optional[Exception]: - return self.error - - def check(self, context, query: Query) -> None: - expected_column_count = query.expected_result.get_expected_column_count() - values = query.expected_result.lines - sort_style = query.get_sortstyle() - query_label = query.get_label() - query_has_label = query_label != None - runner = context.runner - - logger = SQLLogicTestLogger(context, query, runner.test.path) - - # If the result has an error, log it - if self.has_error(): - logger.unexpected_failure() - if runner.skip_error_message(self.get_error()): - runner.finished_processing_file = True - return - context.fail(self.get_error()) - - row_count = self.row_count() - column_count = self.column_count - total_value_count = row_count * column_count - - if len(values) == 1 and result_is_hash(values[0]): - compare_hash = True - is_hash = True - else: - compare_hash = query_has_label or (runner.hash_threshold > 0 and total_value_count > runner.hash_threshold) - is_hash = False - - result_values_string = duck_db_convert_result(self, runner.original_sqlite_test) - - if runner.output_result_mode: - logger.output_result(self, result_values_string) - - if sort_style == SortStyle.ROW_SORT: - ncols = self.column_count - nrows = int(total_value_count / ncols) - rows = [result_values_string[i * ncols : (i + 1) * ncols] for i in range(nrows)] - - # Define the comparison function - def compare_rows(a, b): - for col_idx, val in enumerate(a): - a_val = val - b_val = b[col_idx] - if a_val != b_val: - return -1 if a_val < b_val else 1 - return 0 - - # Sort the individual rows based on element comparison - sorted_rows = sorted(rows, key=cmp_to_key(compare_rows)) - rows = sorted_rows - - for row_idx, row in enumerate(rows): - for col_idx, val in enumerate(row): - result_values_string[row_idx * ncols + col_idx] = val - elif sort_style == SortStyle.VALUE_SORT: - result_values_string.sort() - - comparison_values = [] - if len(values) == 1 and result_is_file(values[0]): - fname = context.replace_keywords(values[0]) - try: - comparison_values = load_result_from_file(fname, self) - # FIXME this is kind of dumb - # We concatenate it with tabs just so we can split it again later - for x in range(len(comparison_values)): - comparison_values[x] = "\t".join(list(comparison_values[x])) - except duckdb.Error as e: - logger.print_error_header(str(e)) - context.fail(f"Failed to load result from {fname}") - else: - comparison_values = values - - hash_value = "" - if runner.output_hash_mode or compare_hash: - hash_context = md5() - for val in result_values_string: - hash_context.update(str(val).encode()) - hash_context.update("\n".encode()) - digest = hash_context.hexdigest() - hash_value = f"{total_value_count} values hashing to {digest}" - if runner.output_hash_mode: - logger.output_hash(hash_value) - return - - if not compare_hash: - original_expected_columns = expected_column_count - column_count_mismatch = False - - if expected_column_count != self.column_count: - expected_column_count = self.column_count - column_count_mismatch = True - - expected_rows = len(comparison_values) / expected_column_count - row_wise = expected_column_count > 1 and len(comparison_values) == self.row_count() - - if not row_wise: - all_tabs = all("\t" in val for val in comparison_values) - row_wise = all_tabs - - if row_wise: - expected_rows = len(comparison_values) - row_wise = True - elif len(comparison_values) % expected_column_count != 0: - if column_count_mismatch: - logger.column_count_mismatch(self, values, original_expected_columns, row_wise) - else: - logger.not_cleanly_divisible(expected_column_count, len(comparison_values)) - # FIXME: the logger should just create the strings to send to self.fail()/self.skip() - context.fail("") - - if expected_rows != self.row_count(): - if column_count_mismatch: - logger.column_count_mismatch(self, values, original_expected_columns, row_wise) - else: - logger.wrong_row_count( - expected_rows, result_values_string, comparison_values, expected_column_count, row_wise - ) - context.fail("") - - if row_wise: - current_row = 0 - for i, val in enumerate(comparison_values): - splits = [x for x in val.split("\t") if x != ''] - if len(splits) != expected_column_count: - if column_count_mismatch: - logger.column_count_mismatch(self, values, original_expected_columns, row_wise) - logger.split_mismatch(i + 1, expected_column_count, len(splits)) - context.fail("") - for c, split_val in enumerate(splits): - lvalue_str = result_values_string[current_row * expected_column_count + c] - rvalue_str = split_val - success = compare_values(self, lvalue_str, split_val, c) - if not success: - logger.print_error_header("Wrong result in query!") - logger.print_line_sep() - logger.print_sql() - logger.print_line_sep() - print(f"Mismatch on row {current_row + 1}, column {c + 1}") - print(f"{lvalue_str} <> {rvalue_str}") - logger.print_line_sep() - logger.print_result_error(result_values_string, values, expected_column_count, row_wise) - context.fail("") - # Increment the assertion counter - assert success - current_row += 1 - else: - current_row, current_column = 0, 0 - for i, val in enumerate(comparison_values): - lvalue_str = result_values_string[current_row * expected_column_count + current_column] - rvalue_str = val - success = compare_values(self, lvalue_str, rvalue_str, current_column) - if not success: - logger.print_error_header("Wrong result in query!") - logger.print_line_sep() - logger.print_sql() - logger.print_line_sep() - print(f"Mismatch on row {current_row + 1}, column {current_column + 1}") - print(f"{lvalue_str} <> {rvalue_str}") - logger.print_line_sep() - logger.print_result_error(result_values_string, values, expected_column_count, row_wise) - context.fail("") - # Increment the assertion counter - assert success - - current_column += 1 - if current_column == expected_column_count: - current_row += 1 - current_column = 0 - - if column_count_mismatch: - logger.column_count_mismatch_correct_result(original_expected_columns, expected_column_count, self) - context.fail("") - else: - hash_compare_error = False - expected_hash_value = None - if query_has_label: - expected_hash_value = runner.hash_label_map.get(query_label) - if expected_hash_value is None: - runner.hash_label_map[query_label] = hash_value - runner.result_label_map[query_label] = self - else: - hash_compare_error = expected_hash_value != hash_value - - if is_hash and not hash_compare_error: - expected_hash_value = values[0] - hash_compare_error = values[0] != hash_value - - if hash_compare_error: - expected_result = runner.result_label_map.get(query_label) - logger.wrong_result_hash(expected_hash_value, hash_value) - - if expected_result: - logger.print_result_error( - result_values_string, - duck_db_convert_result(expected_result, runner.original_sqlite_test), - expected_result.column_count, - False, - ) - context.fail("") - - assert not hash_compare_error - - -class SQLLogicConnectionPool: - __slots__ = [ - 'connection', - 'cursors', - ] - - def __init__(self, con: duckdb.DuckDBPyConnection): - assert con - self.cursors = {} - self.connection = con - - def initialize_connection(self, context: "SQLLogicContext", con: duckdb.DuckDBPyConnection): - runner = context.runner - if runner.test.is_sqlite_test(): - con.execute("SET integer_division=true") - try: - con.execute("SET timezone='UTC'") - except duckdb.Error: - pass - env_var = os.getenv("LOCAL_EXTENSION_REPO") - if env_var: - con.execute("SET autoload_known_extensions=True") - con.execute(f"SET autoinstall_extension_repository='{env_var}'") - - def get_connection(self, name: Optional[str] = None) -> duckdb.DuckDBPyConnection: - """ - Either fetch the 'self.connection' object if name is None - Or get-or-create the cursor identified by name - """ - assert self.connection - if name is None: - return self.connection - - if name not in self.cursors: - # TODO: do we need to run any set up on a new named connection ?? - self.cursors[name] = self.connection.cursor() - return self.cursors[name] - - -class SQLLogicDatabase: - __slots__ = ['path', 'database', 'config'] - - def __init__( - self, path: str, context: Optional["SQLLogicContext"] = None, additional_config: Optional[Dict[str, str]] = None - ): - """ - Connection Hierarchy: - - database - └── connection - └── cursor1 - └── cursor2 - └── cursor3 - - 'connection' is a cursor of 'database'. - Every entry of 'cursors' is a cursor created from 'connection'. - - This is important to understand how ClientConfig settings affect each cursor. - """ - self.reset() - if additional_config: - self.config.update(additional_config) - self.path = path - - # Now re-open the current database - read_only = 'access_mode' in self.config and self.config['access_mode'] == 'read_only' - if 'access_mode' not in self.config: - self.config['access_mode'] = 'automatic' - self.database = duckdb.connect(path, read_only, self.config) - - # Load any previously loaded extensions again - if context: - for extension in context.runner.extensions: - self.load_extension(context, extension) - - def reset(self): - self.database: Optional[duckdb.DuckDBPyConnection] = None - self.config: Dict[str, Any] = { - 'allow_unsigned_extensions': True, - 'allow_unredacted_secrets': True, - } - self.path = '' - - def load_extension(self, context: "SQLLogicContext", extension: str): - if extension in BUILTIN_EXTENSIONS: - # No need to load - return - path = context.get_extension_path(extension) - # Serialize it as a POSIX compliant path - query = f"LOAD '{path}'" - self.database.execute(query) - - def connect(self) -> SQLLogicConnectionPool: - return SQLLogicConnectionPool(self.database.cursor()) - - -def is_regex(input: str) -> bool: - return input.startswith(":") or input.startswith(":") - - -def matches_regex(input: str, actual_str: str) -> bool: - if input.startswith(":"): - should_match = True - regex_str = input.replace(":", "") - else: - should_match = False - regex_str = input.replace(":", "") - # The exact match will never be the same, allow leading and trailing messages - if regex_str[:2] != '.*': - regex_str = ".*" + regex_str - if regex_str[-2:] != '.*': - regex_str = regex_str + '.*' - - re_options = re.DOTALL - re_pattern = re.compile(regex_str, re_options) - regex_matches = bool(re_pattern.fullmatch(actual_str)) - return regex_matches == should_match - - -def has_external_access(conn): - # this is required for the python tester to work, as we make use of replacement scans - try: - res = conn.sql("select current_setting('enable_external_access')").fetchone()[0] - return res - except duckdb.TransactionException: - return True - except duckdb.BinderException: - return True - except duckdb.InvalidInputException: - return True - - -def compare_values(result: QueryResult, actual_str, expected_str, current_column): - error = False - - if actual_str == expected_str: - return True - - if is_regex(expected_str): - return matches_regex(expected_str, actual_str) - - sql_type = result.types[current_column] - - def is_numeric(type) -> bool: - NUMERIC_TYPES = [ - "TINYINT", - "SMALLINT", - "INTEGER", - "BIGINT", - "HUGEINT", - "FLOAT", - "DOUBLE", - "DECIMAL", - "UTINYINT", - "USMALLINT", - "UINTEGER", - "UBIGINT", - "UHUGEINT", - ] - if str(type) in NUMERIC_TYPES: - return True - return 'DECIMAL' in str(type) - - if is_numeric(sql_type): - if sql_type in [duckdb.typing.FLOAT, duckdb.typing.DOUBLE]: - # ApproxEqual - expected = convert_value(expected_str, sql_type) - actual = convert_value(actual_str, sql_type) - if expected == actual: - return True - if math.isnan(expected) and math.isnan(actual): - return True - epsilon = abs(actual) * 0.01 + 0.00000001 - if abs(expected - actual) <= epsilon: - return True - return False - expected = convert_value(expected_str, sql_type) - actual = convert_value(actual_str, sql_type) - return expected == actual - - if sql_type == duckdb.typing.BOOLEAN or sql_type.id == 'timestamp with time zone': - expected = convert_value(expected_str, sql_type) - actual = convert_value(actual_str, sql_type) - return expected == actual - expected = sql_logic_test_convert_value(expected_str, sql_type, False) - actual = actual_str - error = actual != expected - - if error: - return False - return True - - -def result_is_hash(result): - parts = result.split() - if len(parts) != 5: - return False - if not parts[0].isdigit(): - return False - if parts[1] != "values" or parts[2] != "hashing" or len(parts[4]) != 32: - return False - return all([x.islower() or x.isnumeric() for x in parts[4]]) - - -def result_is_file(result: str): - return result.startswith(':') - - -def load_result_from_file(fname, result: QueryResult): - con = duckdb.connect() - con.execute(f"PRAGMA threads={os.cpu_count()}") - column_count = result.column_count - - fname = fname.replace(":", "") - - struct_definition = "STRUCT_PACK(" - for i in range(column_count): - if i > 0: - struct_definition += ", " - struct_definition += f"c{i} := VARCHAR" - struct_definition += ")" - - csv_result = con.execute( - f""" - SELECT * FROM read_csv( - '{fname}', - header=1, - sep='|', - columns={struct_definition}, - auto_detect=false, - all_varchar=true - ) - """ - ) - - return csv_result.fetchall() - - -def convert_value(value, type: str): - if value is None or value == 'NULL': - return 'NULL' - query = f'select $1::{type}' - return duckdb.execute(query, [value]).fetchone()[0] - - -def sql_logic_test_convert_value(value, sql_type, is_sqlite_test: bool) -> str: - if value is None or value == 'NULL': - return 'NULL' - if is_sqlite_test: - if sql_type in [ - duckdb.typing.BOOLEAN, - duckdb.typing.DOUBLE, - duckdb.typing.FLOAT, - ] or any([type_str in str(sql_type) for type_str in ['DECIMAL', 'HUGEINT']]): - return convert_value(value, 'BIGINT::VARCHAR') - if sql_type == duckdb.typing.BOOLEAN: - return "1" if convert_value(value, sql_type) else "0" - else: - res = convert_value(value, 'VARCHAR') - if len(res) == 0: - res = "(empty)" - else: - res = res.replace("\0", "\\0") - return res - - -def duck_db_convert_result(result: QueryResult, is_sqlite_test: bool) -> List[str]: - out_result = [] - row_count = result.row_count() - column_count = result.column_count - - for r in range(row_count): - for c in range(column_count): - value = result.get_value(c, r) - converted_value = sql_logic_test_convert_value(value, result.types[c], is_sqlite_test) - out_result.append(converted_value) - - return out_result - - -class SQLLogicRunner: - __slots__ = [ - 'skipped', - 'error', - 'skip_level', - 'loaded_databases', - 'database', - 'extensions', - 'environment_variables', - 'test', - 'hash_threshold', - 'hash_label_map', - 'result_label_map', - 'required_requires', - 'output_hash_mode', - 'output_result_mode', - 'debug_mode', - 'finished_processing_file', - 'ignore_error_messages', - 'always_fail_error_messages', - 'original_sqlite_test', - 'build_directory', - 'skip_reload', # <-- used for 'force_reload' and 'force_storage', unused for now - ] - - def reset(self): - self.skip_level: int = 0 - - # The set of databases that have been loaded by this runner at any point - # Used for cleanup - self.loaded_databases: typing.Set[str] = set() - self.database: Optional[SQLLogicDatabase] = None - self.extensions = set(BUILTIN_EXTENSIONS) - self.environment_variables: Dict[str, str] = {} - self.test: Optional[SQLLogicTest] = None - - self.hash_threshold: int = 0 - self.hash_label_map: Dict[str, str] = {} - self.result_label_map: Dict[str, Any] = {} - - # FIXME: create a CLI argument for this - self.required_requires: set = set() - self.output_hash_mode = False - self.output_result_mode = False - self.debug_mode = False - - self.finished_processing_file = False - # If these error messages occur in a test, the test will abort but still count as passed - self.ignore_error_messages = {"HTTP", "Unable to connect"} - # If these error messages occur in a statement that is expected to fail, the test will fail - self.always_fail_error_messages = {"differs from original result!", "INTERNAL"} - - self.original_sqlite_test = False - - def skip_error_message(self, message): - for error_message in self.ignore_error_messages: - if error_message in str(message): - return True - return False - - def __init__(self, build_directory: Optional[str] = None): - self.reset() - self.build_directory = build_directory - - def skip(self): - self.skip_level += 1 - - def unskip(self): - self.skip_level -= 1 - - def skip_active(self) -> bool: - return self.skip_level > 0 - - def is_required(self, param): - return param in self.required_requires - - -class SQLLogicContext: - __slots__ = [ - 'iterator', - 'runner', - 'generator', - 'STATEMENTS', - 'pool', - 'statements', - 'current_statement', - 'keywords', - 'error', - 'is_loop', - 'is_parallel', - 'build_directory', - 'cached_config_settings', - ] - - def reset(self): - self.iterator = 0 - - def replace_keywords(self, input: str): - # Apply a replacement for every registered keyword - if '__BUILD_DIRECTORY__' in input: - self.skiptest("Test contains __BUILD_DIRECTORY__ which isnt supported") - for key, value in self.keywords.items().__reversed__(): - input = input.replace(key, value) - return input - - def get_extension_path(self, extension: str): - if self.runner.build_directory is None: - self.skiptest("Tried to load an extension, but --build-dir was not set!") - root = self.runner.build_directory - path = os.path.join(root, "extension", extension, f"{extension}.duckdb_extension") - return path - - def __init__( - self, - pool: SQLLogicConnectionPool, - runner: SQLLogicRunner, - statements: List[BaseStatement], - keywords: Dict[str, str], - iteration_generator, - ): - self.statements = statements - self.runner = runner - self.is_loop = True - self.is_parallel = False - self.error: Optional[TestException] = None - self.generator: Generator[Any] = iteration_generator - self.keywords = keywords - self.cached_config_settings: List[Tuple[str, str]] = [] - self.current_statement: Optional[SQLLogicStatementData] = None - self.pool: Optional[SQLLogicConnectionPool] = pool - self.STATEMENTS = { - Query: self.execute_query, - Statement: self.execute_statement, - RequireEnv: self.execute_require_env, - Require: self.execute_require, - Load: self.execute_load, - Skip: self.execute_skip, - Unskip: self.execute_unskip, - Mode: self.execute_mode, - Sleep: self.execute_sleep, - Reconnect: self.execute_reconnect, - Halt: self.execute_halt, - Restart: self.execute_restart, - HashThreshold: self.execute_hash_threshold, - Set: self.execute_set, - Unzip: self.execute_unzip, - Loop: self.execute_loop, - Foreach: self.execute_foreach, - Endloop: None, # <-- should never be encountered outside of Loop/Foreach - } - - def add_keyword(self, key, value): - # Make sure that loop names can't silently collide - key = f'${{{key}}}' - assert key not in self.keywords - self.keywords[key] = str(value) - - def remove_keyword(self, key): - key = f'${{{key}}}' - assert key in self.keywords - self.keywords.pop(key) - - def fail(self, message: str): - self.error = FailException(self.current_statement, message) - raise self.error - - def skiptest(self, message: str): - self.error = SkipException(self.current_statement, message) - raise self.error - - def in_loop(self) -> bool: - return self.is_loop - - def get_connection(self, name: Optional[str] = None) -> duckdb.DuckDBPyConnection: - return self.pool.get_connection(name) - - def execute_load(self, load: Load): - if self.in_loop(): - # FIXME: should add support for this, the CPP tester supports this - self.skiptest("load cannot be called in a loop") - # self.fail("load cannot be called in a loop") - - readonly = load.readonly - - if load.header.parameters: - dbpath = load.header.parameters[0] - dbpath = self.replace_keywords(dbpath) - if not readonly: - # delete the target database file, if it exists - self.runner.delete_database(dbpath) - else: - dbpath = "" - self.runner.loaded_databases.add(dbpath) - - # set up the config file - additional_config = {} - if readonly: - additional_config['temp_directory'] = "" - additional_config['access_mode'] = 'read_only' - else: - additional_config['access_mode'] = 'automatic' - - if load.version: - additional_config['storage_compatibility_version'] = str(load.version) - - self.pool = None - self.runner.database = None - self.runner.database = SQLLogicDatabase(dbpath, self, additional_config) - self.pool = self.runner.database.connect() - - def execute_query(self, query: Query): - assert isinstance(query, Query) - conn = self.get_connection(query.connection_name) - if not has_external_access(conn): - self.skiptest("enable_external_access is explicitly disabled by the test") - sql_query = '\n'.join(query.lines) - sql_query = self.replace_keywords(sql_query) - - expected_result = query.expected_result - assert expected_result.type == ExpectedResult.Type.SUCCESS - - try: - statements = conn.extract_statements(sql_query) - statement = statements[-1] - if 'pivot' in sql_query and len(statements) != 1: - self.skiptest("Can not deal properly with a PIVOT statement") - - def returns_changed_rows(sql_query, statement) -> bool: - if duckdb.ExpectedResultType.CHANGED_ROWS not in statement.expected_result_type: - return False - if statement.type in [ - duckdb.StatementType.DELETE, - duckdb.StatementType.UPDATE, - duckdb.StatementType.INSERT, - duckdb.StatementType.MERGE_INTO, - ]: - if 'returning' in sql_query.lower(): - return False - return True - if statement.type in [duckdb.StatementType.COPY]: - if 'return_files' in sql_query.lower(): - return False - if 'return_stats' in sql_query.lower(): - return False - return True - return len(statement.expected_result_type) == 1 - - if returns_changed_rows(sql_query, statement): - conn.execute(sql_query) - result = conn.fetchall() - query_result = QueryResult(result, [duckdb.typing.BIGINT]) - elif duckdb.ExpectedResultType.QUERY_RESULT in statement.expected_result_type: - original_rel = conn.query(sql_query) - if original_rel is None: - query_result = QueryResult([(0,)], ['BIGINT']) - else: - original_types = original_rel.types - # We create new names for the columns, because they might be duplicated - aliased_columns = [f'c{i}' for i in range(len(original_types))] - - expressions = [f'"{name}"::VARCHAR' for name, sql_type in zip(aliased_columns, original_types)] - aliased_table = ", ".join(aliased_columns) - expression_list = ", ".join(expressions) - try: - # Select from the result, converting the Values to the right type for comparison - transformed_query = ( - f"select {expression_list} from original_rel unnamed_subquery_blabla({aliased_table})" - ) - stringified_rel = conn.query(transformed_query) - except duckdb.Error as e: - self.fail(f"Could not select from the ValueRelation: {str(e)}") - result = stringified_rel.fetchall() - query_result = QueryResult(result, original_types) - else: - conn.execute(sql_query) - result = conn.fetchall() - query_result = QueryResult(result, []) - if expected_result.lines == None: - return - except duckdb.Error as e: - print(e) - query_result = QueryResult([], [], e) - - query_result.check(self, query) - - def execute_skip(self, statement: Skip): - self.runner.skip() - - def execute_unzip(self, statement: Unzip): - import gzip - import shutil - - source = self.replace_keywords(statement.source) - destination = self.replace_keywords(statement.destination) - - with gzip.open(source, 'rb') as f_in: - with open(destination, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - print(f"Extracted to '{destination}'") - - def execute_unskip(self, statement: Unskip): - self.runner.unskip() - - def execute_halt(self, statement: Halt): - self.skiptest("HALT was encountered in file") - - def execute_restart(self, statement: Restart): - if self.is_parallel: - self.fail("Cannot restart database in parallel") - - old_settings = self.cached_config_settings - - path = self.runner.database.path - self.pool = None - self.runner.database = None - gc.collect() - self.runner.database = SQLLogicDatabase(path, self) - self.pool = self.runner.database.connect() - con = self.pool.get_connection() - - for setting in old_settings: - name, value = setting - if name in [ - 'access_mode', - 'enable_external_access', - 'allow_unsigned_extensions', - 'allow_unredacted_secrets', - 'duckdb_api', - ]: - # Cannot be set after initialization - continue - - # If enable_profiling is NULL, skip setting custom_profiling_settings to not - # accidentally enable profiling. - # In that case, custom_profiling_settings is set to the default value anyway. - if name == "custom_profiling_settings" and "enable_profiling" not in old_settings: - continue - - query = f"SET {name}='{value}'" - con.execute(query) - - def execute_set(self, statement: Set): - option = statement.header.parameters[0] - if option == 'ignore_error_messages': - string_set = ( - self.runner.ignore_error_messages - if option == "ignore_error_messages" - else self.runner.always_fail_error_messages - ) - string_set.clear() - string_set = statement.error_messages - elif option == 'seed': - con = self.get_connection() - con.execute(f"SELECT SETSEED({statement.header.parameters[1]})") - self.runner.skip_reload = True - else: - self.skiptest(f"SET '{option}' is not implemented!") - - def execute_hash_threshold(self, statement: HashThreshold): - self.runner.hash_threshold = statement.threshold - - def execute_reconnect(self, statement: Reconnect): - if self.is_parallel: - self.fail("reconnect can not be used inside a parallel loop") - self.pool = None - self.pool = self.runner.database.connect() - con = self.pool.get_connection() - self.pool.initialize_connection(self, con) - - def execute_sleep(self, statement: Sleep): - def calculate_sleep_time(duration: float, unit: SleepUnit) -> float: - if unit == SleepUnit.SECOND: - return duration - elif unit == SleepUnit.MILLISECOND: - return duration / 1000 - elif unit == SleepUnit.MICROSECOND: - return duration / 1000000 - elif unit == SleepUnit.NANOSECOND: - return duration / 1000000000 - else: - raise ValueError("Unknown sleep unit") - - unit = statement.get_unit() - duration = statement.get_duration() - - time_to_sleep = calculate_sleep_time(duration, unit) - time.sleep(time_to_sleep) - - def execute_mode(self, statement: Mode): - parameter = statement.header.parameters[0] - if parameter == "output_hash": - self.runner.output_hash_mode = True - elif parameter == "output_result": - self.runner.output_result_mode = True - elif parameter == "no_output": - self.runner.output_hash_mode = False - self.runner.output_result_mode = False - elif parameter == "debug": - self.runner.debug_mode = True - else: - raise RuntimeError("unrecognized mode: " + parameter) - - def execute_statement(self, statement: Statement): - assert isinstance(statement, Statement) - conn = self.get_connection(statement.connection_name) - if not has_external_access(conn): - self.skiptest("enable_external_access is explicitly disabled by the test") - - sql_query = '\n'.join(statement.lines) - sql_query = self.replace_keywords(sql_query) - - expected_result = statement.expected_result - try: - conn.execute(sql_query) - result = conn.fetchall() - if expected_result.type == ExpectedResult.Type.ERROR: - self.fail(f"Query unexpectedly succeeded") - if expected_result.type != ExpectedResult.Type.UNKNOWN: - assert expected_result.lines == None - except duckdb.Error as e: - if expected_result.type == ExpectedResult.Type.SUCCESS: - self.fail(f"Query unexpectedly failed: {str(e)}") - if expected_result.lines == None: - return - expected = '\n'.join(expected_result.lines) - if is_regex(expected): - if not matches_regex(expected, str(e)): - self.fail( - f"Query failed, but did not produce the right error: {expected}\nInstead it produced: {str(e)}" - ) - else: - # Sanitize the expected error - if expected.startswith('Dependency Error: '): - expected = expected.split('Dependency Error: ')[1] - if expected not in str(e): - self.fail( - f"Query failed, but did not produce the right error: {expected}\nInstead it produced: {str(e)}" - ) - - def check_require(self, statement: Require) -> RequireResult: - not_an_extension = [ - "notmingw", - "mingw", - "notwindows", - "windows", - "longdouble", - "64bit", - "noforcestorage", - "nothreadsan", - "strinline", - "vector_size", - "exact_vector_size", - "block_size", - "skip_reload", - "no_alternative_verify", - ] - param = statement.header.parameters[0].lower() - if param in not_an_extension: - if param == 'vector_size': - required_vector_size = int(statement.header.parameters[1]) - if duckdb.__standard_vector_size__ < required_vector_size: - return RequireResult.MISSING - return RequireResult.PRESENT - if param == 'exact_vector_size': - required_vector_size = int(statement.header.parameters[1]) - if duckdb.__standard_vector_size__ == required_vector_size: - return RequireResult.PRESENT - return RequireResult.MISSING - if param == 'skip_reload': - self.runner.skip_reload = True - return RequireResult.PRESENT - return RequireResult.MISSING - - # Already loaded - if param in self.runner.extensions: - return RequireResult.PRESENT - - if self.runner.build_directory is None: - return RequireResult.MISSING - - connection = self.pool.get_connection() - autoload_known_extensions = connection.execute( - "select value::BOOLEAN from duckdb_settings() where name == 'autoload_known_extensions'" - ).fetchone()[0] - if param == "no_extension_autoloading": - if autoload_known_extensions: - # If autoloading is on, we skip this test - return RequireResult.MISSING - return RequireResult.PRESENT - - allow_unsigned_extensions = connection.execute( - "select value::BOOLEAN from duckdb_settings() where name == 'allow_unsigned_extensions'" - ).fetchone()[0] - if param == "allow_unsigned_extensions": - if allow_unsigned_extensions == False: - # If extension validation is turned on (that is allow_unsigned_extensions=False), skip test - return RequireResult.MISSING - return RequireResult.PRESENT - - excluded_from_autoloading = True - for ext in self.runner.AUTOLOADABLE_EXTENSIONS: - if ext == param: - excluded_from_autoloading = False - break - - if autoload_known_extensions == False: - try: - self.runner.database.load_extension(self, param) - self.runner.extensions.add(param) - except duckdb.Error: - return RequireResult.MISSING - elif excluded_from_autoloading: - return RequireResult.MISSING - - return RequireResult.PRESENT - - def execute_require(self, statement: Require): - require_result = self.check_require(statement) - if require_result != RequireResult.MISSING: - return - param = statement.header.parameters[0].lower() - if self.runner.is_required(param): - # This extension / setting was explicitly required - self.fail("require {}: FAILED".format(param)) - self.skiptest(f"require {param}: Missing") - - def execute_require_env(self, statement: RequireEnv): - key = statement.header.parameters[0] - res = os.getenv(key) - if self.in_loop(): - # FIXME: we can just remove the keyword at the end of the loop - # I think we should support this - # ... actually the way we set up keywords here, this is already the behavior - # inside the python sqllogic runner, since contexts are created and destroyed at loop start and end - self.skiptest(f"require-env can not be called in a loop") - if res is None: - self.skiptest(f"require-env {key} failed, not set") - if len(statement.header.parameters) != 1: - expected = statement.header.parameters[1] - if res != expected: - self.skiptest(f"require-env {key} failed, expected '{expected}', but found '{res}'") - self.add_keyword(key, res) - - def get_loop_statements(self): - saved_iterator = self.iterator - # Loop until EndLoop is found - statement = None - depth = 0 - while self.iterator < len(self.statements): - statement = self.next_statement() - if statement.__class__ in [Foreach, Loop]: - depth += 1 - if statement.__class__ == Endloop: - if depth == 0: - break - depth -= 1 - if not statement or statement.__class__ != Endloop: - raise Exception("no corresponding 'endloop' found before the end of the file!") - statements = self.statements[saved_iterator : self.iterator - 1] - return statements - - def execute_parallel(self, context: "SQLLogicContext", key, value): - context.is_parallel = True - try: - # For some reason the lambda won't capture the 'value' when created outside of 'execute_parallel' - def update_value(context: SQLLogicContext) -> Generator[Any, Any, Any]: - context.add_keyword(key, value) - yield None - context.remove_keyword(key) - - context.generator = update_value - context.execute() - except TestException: - assert context.error is not None - - def execute_loop(self, loop: Loop): - statements = self.get_loop_statements() - - if not loop.parallel: - # Every iteration the 'value' of the loop key needs to change - def update_value(context: SQLLogicContext) -> Generator[Any, Any, Any]: - key = loop.name - for val in range(loop.start, loop.end): - context.add_keyword(key, val) - yield None - context.remove_keyword(key) - - loop_context = SQLLogicContext(self.pool, self.runner, statements, self.keywords.copy(), update_value) - try: - loop_context.execute() - except TestException: - self.error = loop_context.error - else: - contexts: Dict[Tuple[str, int], Any] = {} - for val in range(loop.start, loop.end): - # FIXME: these connections are expected to have the same settings - # So we need to apply the cached settings to them - contexts[(loop.name, val)] = SQLLogicContext( - self.runner.database.connect(), - self.runner, - statements, - self.keywords.copy(), - None, # generator, can't be created yet - ) - - threads = [] - for keyval, context in contexts.items(): - key, value = keyval - t = threading.Thread(target=self.execute_parallel, args=(context, key, value)) - threads.append(t) - t.start() - - for thread in threads: - thread.join() - - for _, context in contexts.items(): - if context.error is not None: - # Propagate the exception - self.error = context.error - raise self.error - - def execute_foreach(self, foreach: Foreach): - statements = self.get_loop_statements() - - if not foreach.parallel: - # Every iteration the 'value' of the loop key needs to change - def update_value(context: SQLLogicContext) -> Generator[Any, Any, Any]: - loop_keys = foreach.name.split(',') - - for val in foreach.values: - if len(loop_keys) != 1: - values = val.split(',') - else: - values = [val] - assert len(values) == len(loop_keys) - for i, key in enumerate(loop_keys): - context.add_keyword(key, values[i]) - yield None - for key in loop_keys: - context.remove_keyword(key) - - loop_context = SQLLogicContext(self.pool, self.runner, statements, self.keywords.copy(), update_value) - loop_context.execute() - else: - # parallel loop: launch threads - contexts: List[Tuple[str, int, Any]] = [] - loop_keys = foreach.name.split(',') - for val in foreach.values: - if len(loop_keys) != 1: - values = val.split(',') - else: - values = [val] - - assert len(values) == len(loop_keys) - for i, key in enumerate(loop_keys): - contexts.append( - ( - foreach.name, - values[i], - SQLLogicContext( - self.runner.database.connect(), - self.runner, - statements, - self.keywords.copy(), - None, # generator, can't be created yet - ), - ) - ) - - threads = [] - for x in contexts: - key, value, context = x - t = threading.Thread(target=self.execute_parallel, args=(context, key, value)) - threads.append(t) - t.start() - - for thread in threads: - thread.join() - - for x in contexts: - _, _, context = x - if context.error is not None: - self.error = context.error - raise self.error - - def next_statement(self): - if self.iterator >= len(self.statements): - raise Exception("'next_statement' out of range, statements already consumed") - statement = self.statements[self.iterator] - self.iterator += 1 - return statement - - def verify_statements(self) -> None: - unsupported_statements = [ - statement for statement in self.statements if statement.__class__ not in self.STATEMENTS.keys() - ] - if unsupported_statements == []: - return - types = set([x.__class__ for x in unsupported_statements]) - error = f'skipped because the following statement types are not supported: {str(list([x for x in types]))}' - self.skiptest(error) - - def update_settings(self): - # Because we need to fire a query to get the settings required for 'restart' - # we do this preemptively before executing a statement - con = self.pool.get_connection() - try: - self.cached_config_settings = con.execute( - "select name, value from duckdb_settings() where value != 'NULL' and value != ''" - ).fetchall() - except duckdb.Error: - pass - - def execute(self): - try: - for _ in self.generator(self): - self.reset() - while self.iterator < len(self.statements): - statement = self.next_statement() - self.current_statement = SQLLogicStatementData(self.runner.test, statement) - if self.runner.skip_active() and statement.__class__ != Unskip: - # Keep skipping until Unskip is found - continue - if statement.get_decorators() != []: - self.skiptest("Decorators are not supported yet") - method = self.STATEMENTS.get(statement.__class__) - if not method: - self.skiptest("Not supported by the runner") - self.update_settings() - method(statement) - except TestException as e: - raise (e) - return ExecuteResult(ExecuteResult.Type.SUCCESS) diff --git a/scripts/sqllogictest/statement/__init__.py b/scripts/sqllogictest/statement/__init__.py deleted file mode 100644 index 2600fe110963..000000000000 --- a/scripts/sqllogictest/statement/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -from .statement import Statement -from .require import Require -from .mode import Mode -from .halt import Halt -from .load import Load -from .set import Set -from .load import Load -from .query import Query, SortStyle -from .hash_threshold import HashThreshold -from .loop import Loop -from .foreach import Foreach -from .endloop import Endloop -from .require_env import RequireEnv -from .restart import Restart -from .reconnect import Reconnect -from .sleep import Sleep, SleepUnit -from .unzip import Unzip - -from .skip import Skip, Unskip - -__all__ = [ - Statement, - Require, - Mode, - Halt, - Load, - Set, - Query, - HashThreshold, - Loop, - Foreach, - Endloop, - RequireEnv, - Restart, - Reconnect, - Sleep, - SleepUnit, - Skip, - Unzip, - Unskip, - SortStyle, -] diff --git a/scripts/sqllogictest/statement/endloop.py b/scripts/sqllogictest/statement/endloop.py deleted file mode 100644 index 89b415791124..000000000000 --- a/scripts/sqllogictest/statement/endloop.py +++ /dev/null @@ -1,7 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class Endloop(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) diff --git a/scripts/sqllogictest/statement/foreach.py b/scripts/sqllogictest/statement/foreach.py deleted file mode 100644 index 7f52beea2a1a..000000000000 --- a/scripts/sqllogictest/statement/foreach.py +++ /dev/null @@ -1,17 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token -from typing import Optional, List - - -class Foreach(BaseStatement): - def __init__(self, header: Token, line: int, parallel: bool): - super().__init__(header, line) - self.parallel = parallel - self.values: List[str] = [] - self.name: Optional[str] = None - - def set_name(self, name: str): - self.name = name - - def set_values(self, values: List[str]): - self.values = values diff --git a/scripts/sqllogictest/statement/halt.py b/scripts/sqllogictest/statement/halt.py deleted file mode 100644 index 8be9e46e29a7..000000000000 --- a/scripts/sqllogictest/statement/halt.py +++ /dev/null @@ -1,7 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class Halt(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) diff --git a/scripts/sqllogictest/statement/hash_threshold.py b/scripts/sqllogictest/statement/hash_threshold.py deleted file mode 100644 index 0134b6c8158c..000000000000 --- a/scripts/sqllogictest/statement/hash_threshold.py +++ /dev/null @@ -1,8 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class HashThreshold(BaseStatement): - def __init__(self, header: Token, line: int, threshold: int): - super().__init__(header, line) - self.threshold = threshold diff --git a/scripts/sqllogictest/statement/load.py b/scripts/sqllogictest/statement/load.py deleted file mode 100644 index 98ac08439836..000000000000 --- a/scripts/sqllogictest/statement/load.py +++ /dev/null @@ -1,15 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class Load(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) - self.readonly: bool = False - self.version: Optional[int] = None - - def set_readonly(self): - self.readonly = True - - def set_version(self, version: str): - self.version = version diff --git a/scripts/sqllogictest/statement/loop.py b/scripts/sqllogictest/statement/loop.py deleted file mode 100644 index a0a2a696963c..000000000000 --- a/scripts/sqllogictest/statement/loop.py +++ /dev/null @@ -1,21 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token -from typing import Optional, List - - -class Loop(BaseStatement): - def __init__(self, header: Token, line: int, parallel: bool): - super().__init__(header, line) - self.parallel = parallel - self.name: Optional[str] = None - self.start: Optional[int] = None - self.end: Optional[int] = None - - def set_name(self, name: str): - self.name = name - - def set_start(self, start: List[str]): - self.start = start - - def set_end(self, end: List[str]): - self.end = end diff --git a/scripts/sqllogictest/statement/mode.py b/scripts/sqllogictest/statement/mode.py deleted file mode 100644 index be4155883632..000000000000 --- a/scripts/sqllogictest/statement/mode.py +++ /dev/null @@ -1,8 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class Mode(BaseStatement): - def __init__(self, header: Token, line: int, parameter: str): - super().__init__(header, line) - self.parameter = parameter diff --git a/scripts/sqllogictest/statement/query.py b/scripts/sqllogictest/statement/query.py deleted file mode 100644 index 29ac003a94a3..000000000000 --- a/scripts/sqllogictest/statement/query.py +++ /dev/null @@ -1,44 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.expected_result import ExpectedResult -from sqllogictest.token import Token -from typing import Optional, List -from enum import Enum - - -class SortStyle(Enum): - NO_SORT = 0 - ROW_SORT = 1 - VALUE_SORT = 2 - UNKNOWN = 3 - - -class Query(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) - self.label: Optional[str] = None - self.lines: List[str] = [] - self.expected_result: Optional[ExpectedResult] = None - self.connection_name: Optional[str] = None - self.sortstyle: Optional[SortStyle] = None - self.label: Optional[str] = None - - def add_lines(self, lines: List[str]): - self.lines.extend(lines) - - def set_connection(self, connection: str): - self.connection_name = connection - - def set_expected_result(self, expected_result: ExpectedResult): - self.expected_result = expected_result - - def set_sortstyle(self, sortstyle: SortStyle): - self.sortstyle = sortstyle - - def get_sortstyle(self) -> Optional[SortStyle]: - return self.sortstyle - - def set_label(self, label: str): - self.label = label - - def get_label(self) -> Optional[str]: - return self.label diff --git a/scripts/sqllogictest/statement/reconnect.py b/scripts/sqllogictest/statement/reconnect.py deleted file mode 100644 index 558e4c17c35b..000000000000 --- a/scripts/sqllogictest/statement/reconnect.py +++ /dev/null @@ -1,7 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class Reconnect(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) diff --git a/scripts/sqllogictest/statement/require.py b/scripts/sqllogictest/statement/require.py deleted file mode 100644 index 25ab0dc66e70..000000000000 --- a/scripts/sqllogictest/statement/require.py +++ /dev/null @@ -1,7 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class Require(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) diff --git a/scripts/sqllogictest/statement/require_env.py b/scripts/sqllogictest/statement/require_env.py deleted file mode 100644 index 712ac8a707f3..000000000000 --- a/scripts/sqllogictest/statement/require_env.py +++ /dev/null @@ -1,7 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class RequireEnv(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) diff --git a/scripts/sqllogictest/statement/restart.py b/scripts/sqllogictest/statement/restart.py deleted file mode 100644 index 5e271180271b..000000000000 --- a/scripts/sqllogictest/statement/restart.py +++ /dev/null @@ -1,7 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class Restart(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) diff --git a/scripts/sqllogictest/statement/set.py b/scripts/sqllogictest/statement/set.py deleted file mode 100644 index e19488a0cd7b..000000000000 --- a/scripts/sqllogictest/statement/set.py +++ /dev/null @@ -1,12 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token -from typing import List - - -class Set(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) - self.error_messages = [] - - def add_error_messages(self, error_messages: List[str]): - self.error_messages.extend(error_messages) diff --git a/scripts/sqllogictest/statement/skip.py b/scripts/sqllogictest/statement/skip.py deleted file mode 100644 index 88c4e930cc83..000000000000 --- a/scripts/sqllogictest/statement/skip.py +++ /dev/null @@ -1,12 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class Skip(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) - - -class Unskip(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) diff --git a/scripts/sqllogictest/statement/sleep.py b/scripts/sqllogictest/statement/sleep.py deleted file mode 100644 index dae4a4b43de0..000000000000 --- a/scripts/sqllogictest/statement/sleep.py +++ /dev/null @@ -1,41 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token -from enum import Enum, auto - - -class SleepUnit(Enum): - SECOND = auto() - MILLISECOND = auto() - MICROSECOND = auto() - NANOSECOND = auto() - UNKNOWN = auto() - - -def get_sleep_unit(unit): - seconds = ["second", "seconds", "sec"] - milliseconds = ["millisecond", "milliseconds", "milli"] - microseconds = ["microsecond", "microseconds", "micro"] - nanoseconds = ["nanosecond", "nanoseconds", "nano"] - if unit in seconds: - return SleepUnit.SECOND - elif unit in milliseconds: - return SleepUnit.MILLISECOND - elif unit in microseconds: - return SleepUnit.MICROSECOND - elif unit in nanoseconds: - return SleepUnit.NANOSECOND - else: - return SleepUnit.UNKNOWN - - -class Sleep(BaseStatement): - def __init__(self, header: Token, line: int, duration: int, unit: SleepUnit): - super().__init__(header, line) - self.duration = duration - self.unit = unit - - def get_duration(self) -> int: - return self.duration - - def get_unit(self) -> SleepUnit: - return self.unit diff --git a/scripts/sqllogictest/statement/statement.py b/scripts/sqllogictest/statement/statement.py deleted file mode 100644 index db1027067acf..000000000000 --- a/scripts/sqllogictest/statement/statement.py +++ /dev/null @@ -1,21 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.expected_result import ExpectedResult -from sqllogictest.token import Token -from typing import List, Optional - - -class Statement(BaseStatement): - def __init__(self, header: Token, line: int): - super().__init__(header, line) - self.lines: List[str] = [] - self.expected_result: Optional[ExpectedResult] = None - self.connection_name: Optional[str] = None - - def add_lines(self, lines: List[str]): - self.lines.extend(lines) - - def set_connection(self, connection: str): - self.connection_name = connection - - def set_expected_result(self, expected_result: ExpectedResult): - self.expected_result = expected_result diff --git a/scripts/sqllogictest/statement/unzip.py b/scripts/sqllogictest/statement/unzip.py deleted file mode 100644 index 030e1b7b4e70..000000000000 --- a/scripts/sqllogictest/statement/unzip.py +++ /dev/null @@ -1,9 +0,0 @@ -from sqllogictest.base_statement import BaseStatement -from sqllogictest.token import Token - - -class Unzip(BaseStatement): - def __init__(self, header: Token, line: int, source: str, destination: str): - super().__init__(header, line) - self.source = source - self.destination = destination diff --git a/scripts/sqllogictest/test.py b/scripts/sqllogictest/test.py deleted file mode 100644 index 21d08d7a12ea..000000000000 --- a/scripts/sqllogictest/test.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import List -from .base_statement import BaseStatement - - -class SQLLogicTest: - __slots__ = ['path', 'statements'] - - def __init__(self, path: str): - self.path: str = path - self.statements: List[BaseStatement] = [] - - def add_statement(self, statement: BaseStatement): - self.statements.append(statement) - - def is_sqlite_test(self): - return 'test/sqlite/select' in self.path or 'third_party/sqllogictest' in self.path diff --git a/scripts/sqllogictest/token.py b/scripts/sqllogictest/token.py deleted file mode 100644 index 88d42d156d38..000000000000 --- a/scripts/sqllogictest/token.py +++ /dev/null @@ -1,31 +0,0 @@ -from enum import Enum, auto - - -class TokenType(Enum): - SQLLOGIC_INVALID = auto() - SQLLOGIC_SKIP_IF = auto() - SQLLOGIC_ONLY_IF = auto() - SQLLOGIC_STATEMENT = auto() - SQLLOGIC_QUERY = auto() - SQLLOGIC_HASH_THRESHOLD = auto() - SQLLOGIC_HALT = auto() - SQLLOGIC_MODE = auto() - SQLLOGIC_SET = auto() - SQLLOGIC_LOOP = auto() - SQLLOGIC_CONCURRENT_LOOP = auto() - SQLLOGIC_FOREACH = auto() - SQLLOGIC_CONCURRENT_FOREACH = auto() - SQLLOGIC_ENDLOOP = auto() - SQLLOGIC_REQUIRE = auto() - SQLLOGIC_REQUIRE_ENV = auto() - SQLLOGIC_LOAD = auto() - SQLLOGIC_RESTART = auto() - SQLLOGIC_RECONNECT = auto() - SQLLOGIC_SLEEP = auto() - SQLLOGIC_UNZIP = auto() - - -class Token: - def __init__(self): - self.type = TokenType.SQLLOGIC_INVALID - self.parameters = [] diff --git a/scripts/test_peg_parser.py b/scripts/test_peg_parser.py index 178b9edfe17e..9fbf85e85f31 100644 --- a/scripts/test_peg_parser.py +++ b/scripts/test_peg_parser.py @@ -143,7 +143,6 @@ def parse_test_file(filename): for line in stmt.expected_result.lines ): continue - continue query = ' '.join(stmt.lines) statements.append(query) return statements @@ -188,6 +187,8 @@ def run_test_case(args_tuple): 'test/sql/peg_parser', # Fail for some reason 'test/sql/prepared/parameter_variants.test', # PostgreSQL parser bug with ?1 'test/sql/copy/s3/download_config.test', # Unknown why this passes in SQLLogicTest + 'test/sql/function/list/lambdas/arrow/lambda_scope_deprecated.test', # Error in the tokenization of *+* + 'test/sql/catalog/function/test_simple_macro.test', # Bug when mixing named parameters and non-named } if args.all_tests: # run all tests diff --git a/scripts/test_storage_compatibility.py b/scripts/test_storage_compatibility.py new file mode 100644 index 000000000000..86e4cf0bc9dc --- /dev/null +++ b/scripts/test_storage_compatibility.py @@ -0,0 +1,229 @@ +import argparse +import os +import subprocess +import re +import csv +from pathlib import Path + +parser = argparse.ArgumentParser(description='Run a full benchmark using the CLI and report the results.') +group = parser.add_mutually_exclusive_group(required=True) +group.add_argument('--old-cli', action='store', help='Path to the CLI of the old DuckDB version to test') +group.add_argument('--versions', type=str, action='store', help='DuckDB versions to test') +parser.add_argument('--new-unittest', action='store', help='Path to the new unittester to run', required=True) +parser.add_argument('--new-cli', action='store', help='Path to the new unittester to run', default=None) +parser.add_argument('--compatibility', action='store', help='Storage compatibility version', default='v1.0.0') +parser.add_argument( + '--test-config', action='store', help='Test config script to run', default='test/configs/storage_compatibility.json' +) +parser.add_argument('--db-name', action='store', help='Database name to write to', default='bwc_storage_test.db') +parser.add_argument('--abort-on-failure', action='store_true', help='Abort on first failure', default=False) +parser.add_argument('--start-offset', type=int, action='store', help='Test start offset', default=None) +parser.add_argument('--end-offset', type=int, action='store', help='Test end offset', default=None) +parser.add_argument('--no-summarize-failures', action='store_true', help='Skip failure summary', default=False) +parser.add_argument('--list-versions', action='store_true', help='Only list versions to test', default=False) +parser.add_argument( + '--run-empty-tests', + action='store_true', + help='Run tests that don' 't have a CREATE TABLE or CREATE VIEW statement', + default=False, +) + +args, extra_args = parser.parse_known_args() + +programs_to_test = [] +if args.versions is not None: + version_splits = args.versions.split('|') + for version in version_splits: + cli_path = os.path.join(Path.home(), '.duckdb', 'cli', version, 'duckdb') + if not os.path.isfile(cli_path): + os.system(f'curl https://install.duckdb.org | DUCKDB_VERSION={version} sh') + programs_to_test.append(cli_path) +else: + programs_to_test.append(args.old_cli) + +unittest_program = args.new_unittest +db_name = args.db_name +new_cli = args.new_unittest.replace('test/unittest', 'duckdb') if args.new_cli is None else args.new_cli +summarize_failures = not args.no_summarize_failures + +# Use the '-l' parameter to output the list of tests to run +proc = subprocess.run( + [unittest_program, '--test-config', args.test_config, '-l'] + extra_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, +) +stdout = proc.stdout.decode('utf8').strip() +stderr = proc.stderr.decode('utf8').strip() +if len(stderr) > 0: + print("Failed to run program " + unittest_program) + print("Returncode:", proc.returncode) + print(stdout) + print(stderr) + exit(1) + + +# The output is in the format of 'PATH\tGROUP', we're only interested in the PATH portion +test_cases = [] +first_line = True +for line in stdout.splitlines(): + if first_line: + first_line = False + continue + if len(line.strip()) == 0: + continue + splits = line.rsplit('\t', 1) + test_cases.append(splits[0]) + +test_cases.sort() +if args.compatibility != 'v1.0.0': + raise Exception("Only v1.0.0 is supported for now (FIXME)") + + +def escape_cmd_arg(arg): + if '"' in arg or '\'' in arg or ' ' in arg or '\\' in arg: + arg = arg.replace('\\', '\\\\') + arg = arg.replace('"', '\\"') + arg = arg.replace("'", "\\'") + return f'"{arg}"' + return arg + + +error_container = [] + + +def handle_failure(test, cmd, msg, stdout, stderr, returncode): + print(f"==============FAILURE============") + print(test) + print(f"==============MESSAGE============") + print(msg) + print(f"==============COMMAND============") + cmd_str = '' + for entry in cmd: + cmd_str += escape_cmd_arg(entry) + ' ' + print(cmd_str.strip()) + print(f"==============RETURNCODE=========") + print(str(returncode)) + print(f"==============STDOUT=============") + print(stdout) + print(f"==============STDERR=============") + print(stderr) + print(f"=================================") + if args.abort_on_failure: + exit(1) + else: + error_container.append({'test': test, 'stderr': stderr}) + + +def run_program(cmd, description): + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout = proc.stdout.decode('utf8').strip() + stderr = proc.stderr.decode('utf8').strip() + if proc.returncode != 0: + return { + 'test': test, + 'cmd': cmd, + 'msg': f'Failed to {description}', + 'stdout': stdout, + 'stderr': stderr, + 'returncode': proc.returncode, + } + return None + + +def try_run_program(cmd, description): + result = run_program(cmd, description) + if result is None: + return True + handle_failure(**result) + return False + + +index = 0 +start = 0 if args.start_offset is None else args.start_offset +end = len(test_cases) if args.end_offset is None else args.end_offset +for i in range(start, end): + test = test_cases[i] + skipped = '' + if not args.run_empty_tests: + with open(test, 'r') as f: + test_contents = f.read().lower() + if 'create table' not in test_contents and 'create view' not in test_contents: + skipped = ' (SKIPPED)' + + print(f'[{i}/{len(test_cases)}]: {test}{skipped}') + if skipped != '': + continue + # remove the old db + try: + os.remove(db_name) + except: + pass + cmd = [unittest_program, '--test-config', args.test_config, test] + if not try_run_program(cmd, 'Run Test'): + continue + + if not os.path.isfile(db_name): + # db not created + continue + + cmd = [ + programs_to_test[-1], + db_name, + '-c', + '.headers off', + '-csv', + '-c', + '.output table_list.csv', + '-c', + 'SHOW ALL TABLES', + ] + if not try_run_program(cmd, 'List Tables'): + continue + + tables = [] + with open('table_list.csv', newline='') as f: + reader = csv.reader(f) + for row in reader: + tables.append((row[1], row[2])) + # no tables / views + if len(tables) == 0: + continue + + # read all tables / views + failures = [] + for cli in programs_to_test: + cmd = [cli, db_name] + for table in tables: + schema_name = table[0].replace('"', '""') + table_name = table[1].replace('"', '""') + cmd += ['-c', f'FROM "{schema_name}"."{table_name}"'] + failure = run_program(cmd, 'Query Tables') + if failure is not None: + failures.append(failure) + if len(failures) > 0: + # we failed to query the tables + # this MIGHT be expected - e.g. we might have views that reference stale state (e.g. files that are deleted) + # try to run it with the new CLI - if this succeeds we have a problem + new_cmd = [new_cli] + cmd[1:] + new_failure = run_program(new_cmd, 'Query Tables (New)') + if new_failure is None: + # we succeeded with the new CLI - report the failure + for failure in failures: + handle_failure(**failure) + continue + +if len(error_container) == 0: + exit(0) + +if summarize_failures: + print( + '''\n\n==================================================== +================ FAILURES SUMMARY ================ +====================================================\n +''' + ) + for i, error in enumerate(error_container, start=1): + print(f"\n{i}:", error["test"], "\n") + print(error["stderr"]) + +exit(1) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d7593a23db68..72e996107f64 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,7 +24,7 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") set(EXIT_TIME_DESTRUCTORS_WARNING TRUE) set(CMAKE_CXX_FLAGS_DEBUG - "${CMAKE_CXX_FLAGS_DEBUG} -Wexit-time-destructors -Wimplicit-int-conversion -Wshorten-64-to-32 -Wnarrowing -Wsign-conversion -Wsign-compare -Wconversion" + "${CMAKE_CXX_FLAGS_DEBUG} -Wexit-time-destructors -Wimplicit-int-conversion -Wshorten-64-to-32 -Wnarrowing -Wsign-conversion -Wsign-compare -Wconversion -Wtype-limits" ) endif() @@ -60,9 +60,12 @@ endfunction() if(AMALGAMATION_BUILD) add_library(duckdb SHARED "${PROJECT_SOURCE_DIR}/src/amalgamation/duckdb.cpp") - set_target_properties( - duckdb PROPERTIES VERSION ${DUCKDB_VERSION_NUMBER} - SOVERSION ${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}) + if(SET_DUCKDB_LIBRARY_VERSION) + set_target_properties( + duckdb + PROPERTIES VERSION ${DUCKDB_VERSION_NUMBER} + SOVERSION ${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}) + endif() target_link_libraries(duckdb ${DUCKDB_SYSTEM_LIBS}) link_threads(duckdb PUBLIC) link_extension_libraries(duckdb PRIVATE) @@ -113,9 +116,12 @@ else() duckdb_zstd) add_library(duckdb SHARED ${ALL_OBJECT_FILES}) - set_target_properties( - duckdb PROPERTIES VERSION ${DUCKDB_VERSION_NUMBER} - SOVERSION ${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}) + if(SET_DUCKDB_LIBRARY_VERSION) + set_target_properties( + duckdb + PROPERTIES VERSION ${DUCKDB_VERSION_NUMBER} + SOVERSION ${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}) + endif() if(WIN32 AND NOT MINGW) ensure_variable_is_number(DUCKDB_MAJOR_VERSION RC_MAJOR_VERSION) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index f56db1857c4e..08e27f28f710 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -657,7 +657,7 @@ CatalogException Catalog::CreateMissingEntryException(CatalogEntryRetriever &ret auto &db_manager = DatabaseManager::Get(context); auto databases = db_manager.GetDatabases(context, max_schema_count); - for (auto database : databases) { + for (const auto &database : databases) { if (unseen_schemas.size() >= max_schema_count) { break; } @@ -1135,6 +1135,9 @@ vector> Catalog::GetAllSchemas(ClientContext &cont auto &db_manager = DatabaseManager::Get(context); auto databases = db_manager.GetDatabases(context); for (auto &database : databases) { + if (database->GetVisibility() == AttachVisibility::HIDDEN) { + continue; + } auto &catalog = database->GetCatalog(); auto new_schemas = catalog.GetSchemas(context); result.insert(result.end(), new_schemas.begin(), new_schemas.end()); @@ -1235,4 +1238,9 @@ void Catalog::FinalizeLoad(optional_ptr context) { void Catalog::OnDetach(ClientContext &context) { } +bool Catalog::HasConflictingAttachOptions(const string &path, const AttachOptions &options) { + auto const db_type = options.db_type.empty() ? "duckdb" : options.db_type; + return GetDBPath() != path || GetCatalogType() != db_type; +} + } // namespace duckdb diff --git a/src/catalog/catalog_entry/copy_function_catalog_entry.cpp b/src/catalog/catalog_entry/copy_function_catalog_entry.cpp index 25544a343987..6d639bef695e 100644 --- a/src/catalog/catalog_entry/copy_function_catalog_entry.cpp +++ b/src/catalog/catalog_entry/copy_function_catalog_entry.cpp @@ -3,6 +3,8 @@ namespace duckdb { +constexpr const char *CopyFunctionCatalogEntry::Name; + CopyFunctionCatalogEntry::CopyFunctionCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateCopyFunctionInfo &info) : StandardEntry(CatalogType::COPY_FUNCTION_ENTRY, schema, catalog, info.name), function(info.function) { diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index a1f23aa7fd0a..6202e1030878 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -23,6 +23,7 @@ #include "duckdb/parser/expression/function_expression.hpp" #include "duckdb/storage/storage_manager.hpp" #include "duckdb/storage/table_storage_info.hpp" +#include "duckdb/common/type_visitor.hpp" namespace duckdb { @@ -54,6 +55,9 @@ DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, Bou // create the physical storage vector column_defs; for (auto &col_def : columns.Physical()) { + if (TypeVisitor::Contains(col_def.Type(), LogicalTypeId::VARIANT)) { + throw NotImplementedException("A table cannot be created from a VARIANT column yet"); + } column_defs.push_back(col_def.Copy()); } storage = make_shared_ptr(catalog.GetAttached(), StorageManager::Get(catalog).GetTableIOManager(&info), @@ -1281,8 +1285,8 @@ TableFunction DuckTableEntry::GetScanFunction(ClientContext &context, unique_ptr return TableScanFunction::GetFunction(); } -vector DuckTableEntry::GetColumnSegmentInfo() { - return storage->GetColumnSegmentInfo(); +vector DuckTableEntry::GetColumnSegmentInfo(const QueryContext &context) { + return storage->GetColumnSegmentInfo(context); } TableStorageInfo DuckTableEntry::GetStorageInfo(ClientContext &context) { diff --git a/src/catalog/catalog_entry/pragma_function_catalog_entry.cpp b/src/catalog/catalog_entry/pragma_function_catalog_entry.cpp index ff247dcb07d2..9d9789192367 100644 --- a/src/catalog/catalog_entry/pragma_function_catalog_entry.cpp +++ b/src/catalog/catalog_entry/pragma_function_catalog_entry.cpp @@ -2,6 +2,7 @@ #include "duckdb/parser/parsed_data/create_pragma_function_info.hpp" namespace duckdb { +constexpr const char *PragmaFunctionCatalogEntry::Name; PragmaFunctionCatalogEntry::PragmaFunctionCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreatePragmaFunctionInfo &info) diff --git a/src/catalog/catalog_entry/scalar_function_catalog_entry.cpp b/src/catalog/catalog_entry/scalar_function_catalog_entry.cpp index 49b20f677aa6..e5778ad4ce2d 100644 --- a/src/catalog/catalog_entry/scalar_function_catalog_entry.cpp +++ b/src/catalog/catalog_entry/scalar_function_catalog_entry.cpp @@ -5,6 +5,8 @@ namespace duckdb { +constexpr const char *ScalarFunctionCatalogEntry::Name; + ScalarFunctionCatalogEntry::ScalarFunctionCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateScalarFunctionInfo &info) : FunctionEntry(CatalogType::SCALAR_FUNCTION_ENTRY, catalog, schema, info), functions(info.functions) { diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index 6153a8e8ad7a..d6a548a26765 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -13,6 +13,8 @@ namespace duckdb { +constexpr const char *SequenceCatalogEntry::Name; + SequenceData::SequenceData(CreateSequenceInfo &info) : usage_count(info.usage_count), counter(info.start_value), last_value(info.start_value), increment(info.increment), start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), cycle(info.cycle) { diff --git a/src/catalog/catalog_entry/table_catalog_entry.cpp b/src/catalog/catalog_entry/table_catalog_entry.cpp index 22a173fd8723..8582fa93c6ac 100644 --- a/src/catalog/catalog_entry/table_catalog_entry.cpp +++ b/src/catalog/catalog_entry/table_catalog_entry.cpp @@ -19,6 +19,8 @@ namespace duckdb { +constexpr const char *TableCatalogEntry::Name; + TableCatalogEntry::TableCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) : StandardEntry(CatalogType::TABLE_ENTRY, schema, catalog, info.table), columns(std::move(info.columns)), constraints(std::move(info.constraints)) { @@ -266,7 +268,7 @@ void LogicalUpdate::BindExtraColumns(TableCatalogEntry &table, LogicalGet &get, } } -vector TableCatalogEntry::GetColumnSegmentInfo() { +vector TableCatalogEntry::GetColumnSegmentInfo(const QueryContext &context) { return {}; } diff --git a/src/catalog/catalog_entry/table_function_catalog_entry.cpp b/src/catalog/catalog_entry/table_function_catalog_entry.cpp index a6a41ff6197e..f06ef164eddc 100644 --- a/src/catalog/catalog_entry/table_function_catalog_entry.cpp +++ b/src/catalog/catalog_entry/table_function_catalog_entry.cpp @@ -4,6 +4,8 @@ namespace duckdb { +constexpr const char *TableFunctionCatalogEntry::Name; + TableFunctionCatalogEntry::TableFunctionCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableFunctionInfo &info) : FunctionEntry(CatalogType::TABLE_FUNCTION_ENTRY, catalog, schema, info), functions(std::move(info.functions)) { diff --git a/src/catalog/catalog_entry/type_catalog_entry.cpp b/src/catalog/catalog_entry/type_catalog_entry.cpp index 0bb4a3f3af71..324413b7c280 100644 --- a/src/catalog/catalog_entry/type_catalog_entry.cpp +++ b/src/catalog/catalog_entry/type_catalog_entry.cpp @@ -9,6 +9,8 @@ namespace duckdb { +constexpr const char *TypeCatalogEntry::Name; + TypeCatalogEntry::TypeCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTypeInfo &info) : StandardEntry(CatalogType::TYPE_ENTRY, schema, catalog, info.name), user_type(info.type), bind_function(info.bind_function) { diff --git a/src/catalog/catalog_search_path.cpp b/src/catalog/catalog_search_path.cpp index 6af56c22d010..6388b91346c0 100644 --- a/src/catalog/catalog_search_path.cpp +++ b/src/catalog/catalog_search_path.cpp @@ -24,8 +24,8 @@ string CatalogSearchEntry::ToString() const { string CatalogSearchEntry::WriteOptionallyQuoted(const string &input) { for (idx_t i = 0; i < input.size(); i++) { - if (input[i] == '.' || input[i] == ',') { - return "\"" + input + "\""; + if (input[i] == '.' || input[i] == ',' || input[i] == '"') { + return "\"" + StringUtil::Replace(input, "\"", "\"\"") + "\""; } } return input; diff --git a/src/catalog/catalog_set.cpp b/src/catalog/catalog_set.cpp index deff8daae445..d374f6999d70 100644 --- a/src/catalog/catalog_set.cpp +++ b/src/catalog/catalog_set.cpp @@ -401,8 +401,6 @@ bool CatalogSet::DropEntryInternal(CatalogTransaction transaction, const string throw CatalogException("Cannot drop entry \"%s\" because it is an internal system entry", entry->name); } - entry->OnDrop(); - // create a new tombstone entry and replace the currently stored one // set the timestamp to the timestamp of the current transaction // and point it at the tombstone node @@ -454,6 +452,7 @@ void CatalogSet::VerifyExistenceOfDependency(transaction_t commit_id, CatalogEnt void CatalogSet::CommitDrop(transaction_t commit_id, transaction_t start_time, CatalogEntry &entry) { auto &duck_catalog = GetCatalog(); + entry.OnDrop(); // Make sure that we don't see any uncommitted changes auto transaction_id = MAX_TRANSACTION_ID; // This will allow us to see all committed changes made before this COMMIT happened diff --git a/src/catalog/duck_catalog.cpp b/src/catalog/duck_catalog.cpp index 20bbf6895604..4d1f3bb6d9c3 100644 --- a/src/catalog/duck_catalog.cpp +++ b/src/catalog/duck_catalog.cpp @@ -9,6 +9,7 @@ #include "duckdb/main/attached_database.hpp" #include "duckdb/transaction/duck_transaction_manager.hpp" #include "duckdb/function/function_list.hpp" +#include "duckdb/common/encryption_state.hpp" namespace duckdb { @@ -159,6 +160,14 @@ string DuckCatalog::GetDBPath() { return db.GetStorageManager().GetDBPath(); } +bool DuckCatalog::IsEncrypted() const { + return IsSystemCatalog() ? false : db.GetStorageManager().IsEncrypted(); +} + +string DuckCatalog::GetEncryptionCipher() const { + return IsSystemCatalog() ? string() : EncryptionTypes::CipherToString(db.GetStorageManager().GetCipher()); +} + void DuckCatalog::Verify() { #ifdef DEBUG Catalog::Verify(); diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 539ed2bd3178..03da02062631 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -13,7 +13,6 @@ add_subdirectory(tree_renderer) add_subdirectory(row_operations) add_subdirectory(serializer) add_subdirectory(sort) -add_subdirectory(sorting) add_subdirectory(types) add_subdirectory(value_operations) add_subdirectory(vector_operations) diff --git a/src/common/adbc/adbc.cpp b/src/common/adbc/adbc.cpp index 054eaaf0f58f..0ed31eb39081 100644 --- a/src/common/adbc/adbc.cpp +++ b/src/common/adbc/adbc.cpp @@ -659,12 +659,12 @@ AdbcStatusCode Ingest(duckdb_connection connection, const char *table_name, cons std::ostringstream create_table; create_table << "CREATE TABLE "; if (schema) { - create_table << schema << "."; + create_table << duckdb::KeywordHelper::WriteOptionallyQuoted(schema) << "."; } - create_table << table_name << " ("; + create_table << duckdb::KeywordHelper::WriteOptionallyQuoted(table_name) << " ("; for (idx_t i = 0; i < types.size(); i++) { - create_table << names[i] << " "; - create_table << types[i].ToString(); + create_table << duckdb::KeywordHelper::WriteOptionallyQuoted(names[i]); + create_table << " " << types[i].ToString(); if (i + 1 < types.size()) { create_table << ", "; } diff --git a/src/common/allocator.cpp b/src/common/allocator.cpp index 977087939852..609558785272 100644 --- a/src/common/allocator.cpp +++ b/src/common/allocator.cpp @@ -35,6 +35,8 @@ namespace duckdb { +constexpr const idx_t Allocator::MAXIMUM_ALLOC_SIZE; + AllocatedData::AllocatedData() : allocator(nullptr), pointer(nullptr), allocated_size(0) { } diff --git a/src/common/bignum.cpp b/src/common/bignum.cpp index 2bf228586317..fb3613e88d6e 100644 --- a/src/common/bignum.cpp +++ b/src/common/bignum.cpp @@ -322,7 +322,7 @@ string_t BignumIntermediate::Add(Vector &result_vector, const BignumIntermediate } Trim(reinterpret_cast(target_data + Bignum::BIGNUM_HEADER_SIZE), result_size_data, is_result_negative); Bignum::SetHeader(target_data, result_size_data, is_result_negative); - target.SetSizeAndFinalize(result_size_data + Bignum::BIGNUM_HEADER_SIZE); + target.SetSizeAndFinalize(result_size_data + Bignum::BIGNUM_HEADER_SIZE, result_size); return target; } void BignumIntermediate::AddInPlace(ArenaAllocator &allocator, const BignumIntermediate &rhs) { diff --git a/src/common/box_renderer.cpp b/src/common/box_renderer.cpp index 98f3b092d6e9..999392df1f83 100644 --- a/src/common/box_renderer.cpp +++ b/src/common/box_renderer.cpp @@ -980,7 +980,7 @@ void BoxRenderer::RenderRowCount(string &row_count_str, string &readable_rows_st // we still need to render the readable rows/shown strings // check if we can merge the two onto one row idx_t combined_shown_length = readable_rows_str.size() + shown_str.size() + 4; - if (combined_shown_length <= total_length) { + if (!readable_rows_str.empty() && !shown_str.empty() && combined_shown_length <= total_length) { // we can! merge them ss << config.VERTICAL; ss << " "; diff --git a/src/common/csv_writer.cpp b/src/common/csv_writer.cpp index bb9ff81d216d..0f3126691a46 100644 --- a/src/common/csv_writer.cpp +++ b/src/common/csv_writer.cpp @@ -16,7 +16,7 @@ CSVWriterState::CSVWriterState() } CSVWriterState::CSVWriterState(ClientContext &context, idx_t flush_size_p) - : flush_size(flush_size_p), stream(make_uniq(Allocator::Get(context))) { + : flush_size(flush_size_p), stream(make_uniq(Allocator::Get(context), flush_size)) { } CSVWriterState::CSVWriterState(DatabaseInstance &db, idx_t flush_size_p) @@ -198,18 +198,6 @@ void CSVWriter::ResetInternal(optional_ptr local_state) { bytes_written = 0; } -unique_ptr CSVWriter::InitializeLocalWriteState(ClientContext &context, idx_t flush_size) { - auto res = make_uniq(context, flush_size); - res->stream = make_uniq(); - return res; -} - -unique_ptr CSVWriter::InitializeLocalWriteState(DatabaseInstance &db, idx_t flush_size) { - auto res = make_uniq(db, flush_size); - res->stream = make_uniq(); - return res; -} - idx_t CSVWriter::BytesWritten() { if (shared) { lock_guard flock(lock); diff --git a/src/common/encryption_functions.cpp b/src/common/encryption_functions.cpp index 5dcfeef13ef8..1ecf1abeb537 100644 --- a/src/common/encryption_functions.cpp +++ b/src/common/encryption_functions.cpp @@ -3,9 +3,33 @@ #include "duckdb/common/encryption_functions.hpp" #include "duckdb/main/attached_database.hpp" #include "mbedtls_wrapper.hpp" +#include "duckdb/storage/storage_manager.hpp" +#include "duckdb/storage/storage_info.hpp" namespace duckdb { +EncryptionTag::EncryptionTag() : tag(new data_t[MainHeader::AES_TAG_LEN]) { +} + +data_ptr_t EncryptionTag::data() { + return tag.get(); +} + +idx_t EncryptionTag::size() const { + return MainHeader::AES_TAG_LEN; +} + +EncryptionNonce::EncryptionNonce() : nonce(new data_t[MainHeader::AES_NONCE_LEN]) { +} + +data_ptr_t EncryptionNonce::data() { + return nonce.get(); +} + +idx_t EncryptionNonce::size() const { + return MainHeader::AES_NONCE_LEN; +} + EncryptionEngine::EncryptionEngine() { } @@ -48,19 +72,21 @@ void EncryptionEngine::AddTempKeyToCache(DatabaseInstance &db) { const auto length = MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH; data_t temp_key[length]; - auto encryption_state = db.GetEncryptionUtil()->CreateEncryptionState(temp_key, length); + auto encryption_state = db.GetEncryptionUtil()->CreateEncryptionState( + /* only for random generator */ EncryptionTypes::GCM, length); encryption_state->GenerateRandomData(temp_key, length); string key_id = "temp_key"; AddKeyToCache(db, temp_key, key_id); } -void EncryptionEngine::EncryptBlock(DatabaseInstance &db, const string &key_id, FileBuffer &block, +void EncryptionEngine::EncryptBlock(AttachedDatabase &attached_db, const string &key_id, FileBuffer &block, FileBuffer &temp_buffer_manager, uint64_t delta) { + auto &db = attached_db.GetDatabase(); data_ptr_t block_offset_internal = temp_buffer_manager.InternalBuffer(); auto encrypt_key = GetKeyFromCache(db, key_id); - auto encryption_state = - db.GetEncryptionUtil()->CreateEncryptionState(encrypt_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); + auto encryption_state = db.GetEncryptionUtil()->CreateEncryptionState(attached_db.GetStorageManager().GetCipher(), + MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); EncryptionTag tag; EncryptionNonce nonce; @@ -79,22 +105,24 @@ void EncryptionEngine::EncryptBlock(DatabaseInstance &db, const string &key_id, auto aes_res = encryption_state->Process(checksum_offset, size, encryption_checksum_offset, size); if (aes_res != size) { - throw IOException("Encryption failure: in- and output size differ"); + throw IOException("Block encryption failure: in- and output size differ (%llu/%llu)", size, aes_res); } //! Finalize and extract the tag - aes_res = encryption_state->Finalize(block.InternalBuffer() + delta, 0, tag.data(), tag.size()); + encryption_state->Finalize(block.InternalBuffer() + delta, 0, tag.data(), tag.size()); - //! store the generated tag after consequetively the nonce + //! store the generated tag *behind* the nonce (but still at the beginning of the block) memcpy(block_offset_internal + nonce.size(), tag.data(), tag.size()); } -void EncryptionEngine::DecryptBlock(DatabaseInstance &db, const string &key_id, data_ptr_t internal_buffer, +void EncryptionEngine::DecryptBlock(AttachedDatabase &attached_db, const string &key_id, data_ptr_t internal_buffer, uint64_t block_size, uint64_t delta) { //! initialize encryption state + auto &db = attached_db.GetDatabase(); + auto decrypt_key = GetKeyFromCache(db, key_id); - auto encryption_state = - db.GetEncryptionUtil()->CreateEncryptionState(decrypt_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); + auto encryption_state = db.GetEncryptionUtil()->CreateEncryptionState(attached_db.GetStorageManager().GetCipher(), + MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); //! load the stored nonce and tag EncryptionTag tag; @@ -113,11 +141,11 @@ void EncryptionEngine::DecryptBlock(DatabaseInstance &db, const string &key_id, auto aes_res = encryption_state->Process(checksum_offset, size, checksum_offset, size); if (aes_res != block_size + Storage::DEFAULT_BLOCK_HEADER_SIZE) { - throw IOException("Encryption failure: in- and output size differ"); + throw IOException("Block decryption failure: in- and output size differ (%llu/%llu)", size, aes_res); } //! check the tag - aes_res = encryption_state->Finalize(internal_buffer + delta, 0, tag.data(), tag.size()); + encryption_state->Finalize(internal_buffer + delta, 0, tag.data(), tag.size()); } void EncryptionEngine::EncryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t buffer, idx_t buffer_size, @@ -129,7 +157,9 @@ void EncryptionEngine::EncryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t b auto temp_key = GetKeyFromCache(db, "temp_key"); auto encryption_util = db.GetEncryptionUtil(); - auto encryption_state = encryption_util->CreateEncryptionState(temp_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); + // we hard-code GCM here for now, it's the safest and we don't know what is configured here + auto encryption_state = + encryption_util->CreateEncryptionState(EncryptionTypes::GCM, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); // zero-out the metadata buffer memset(metadata, 0, DEFAULT_ENCRYPTED_BUFFER_HEADER_SIZE); @@ -148,7 +178,8 @@ void EncryptionEngine::EncryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t b auto aes_res = encryption_state->Process(buffer, buffer_size, buffer, buffer_size); if (aes_res != buffer_size) { - throw IOException("Encryption failure: in- and output size differ"); + throw IOException("Temporary buffer encryption failure: in- and output size differ (%llu/%llu)", buffer_size, + aes_res); } //! Finalize and extract the tag @@ -161,8 +192,8 @@ void EncryptionEngine::EncryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t b D_ASSERT(memcmp(tag.data(), metadata + nonce.size(), tag.size()) == 0); } -void EncryptionEngine::DecryptBuffer(EncryptionState &encryption_state, const_data_ptr_t temp_key, data_ptr_t buffer, - idx_t buffer_size, data_ptr_t metadata) { +static void DecryptBuffer(EncryptionState &encryption_state, const_data_ptr_t temp_key, data_ptr_t buffer, + idx_t buffer_size, data_ptr_t metadata) { //! load the stored nonce and tag EncryptionTag tag; EncryptionNonce nonce; @@ -176,7 +207,7 @@ void EncryptionEngine::DecryptBuffer(EncryptionState &encryption_state, const_da auto aes_res = encryption_state.Process(buffer, buffer_size, buffer, buffer_size); if (aes_res != buffer_size) { - throw IOException("Encryption failure: in- and output size differ"); + throw IOException("Buffer decryption failure: in- and output size differ (%llu/%llu)", buffer_size, aes_res); } //! check the tag @@ -188,7 +219,8 @@ void EncryptionEngine::DecryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t b //! initialize encryption state auto encryption_util = db.GetEncryptionUtil(); auto temp_key = GetKeyFromCache(db, "temp_key"); - auto encryption_state = encryption_util->CreateEncryptionState(temp_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); + auto encryption_state = + encryption_util->CreateEncryptionState(EncryptionTypes::GCM, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); DecryptBuffer(*encryption_state, temp_key, buffer, buffer_size, metadata); } diff --git a/src/common/encryption_state.cpp b/src/common/encryption_state.cpp index c19f8c02c305..0763bc5a6b02 100644 --- a/src/common/encryption_state.cpp +++ b/src/common/encryption_state.cpp @@ -2,33 +2,80 @@ namespace duckdb { -EncryptionState::EncryptionState(const_data_ptr_t key, idx_t key_len) { - // abstract class, no implementation needed +EncryptionState::EncryptionState(EncryptionTypes::CipherType cipher_p, idx_t key_len_p) + : cipher(cipher_p), key_len(key_len_p) { } EncryptionState::~EncryptionState() { } -void EncryptionState::InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len, - const_data_ptr_t aad, idx_t aad_len) { +void EncryptionState::InitializeEncryption(const_data_ptr_t, idx_t, const_data_ptr_t, idx_t, const_data_ptr_t, idx_t) { throw NotImplementedException("EncryptionState Abstract Class is called"); } -void EncryptionState::InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len, - const_data_ptr_t aad, idx_t aad_len) { +void EncryptionState::InitializeDecryption(const_data_ptr_t, idx_t, const_data_ptr_t, idx_t, const_data_ptr_t, idx_t) { throw NotImplementedException("EncryptionState Abstract Class is called"); } -size_t EncryptionState::Process(const_data_ptr_t in, idx_t in_len, data_ptr_t out, idx_t out_len) { +size_t EncryptionState::Process(const_data_ptr_t, idx_t, data_ptr_t, idx_t) { throw NotImplementedException("EncryptionState Abstract Class is called"); } -size_t EncryptionState::Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, idx_t tag_len) { +size_t EncryptionState::Finalize(data_ptr_t, idx_t, data_ptr_t, idx_t) { throw NotImplementedException("EncryptionState Abstract Class is called"); } -void EncryptionState::GenerateRandomData(data_ptr_t data, idx_t len) { +void EncryptionState::GenerateRandomData(data_ptr_t, idx_t) { throw NotImplementedException("EncryptionState Abstract Class is called"); } +string EncryptionTypes::CipherToString(CipherType cipher_p) { + switch (cipher_p) { + case GCM: + return "GCM"; + case CTR: + return "CTR"; + case CBC: + return "CBC"; + default: + return "INVALID"; + } +} + +EncryptionTypes::CipherType EncryptionTypes::StringToCipher(const string &encryption_cipher_p) { + auto encryption_cipher = StringUtil::Upper(encryption_cipher_p); + if (encryption_cipher == "GCM") { + return GCM; + } + if (encryption_cipher == "CTR") { + return CTR; + } + if (encryption_cipher == "CBC") { + throw NotImplementedException("CBC encryption is disabled"); + } + return INVALID; +} + +string EncryptionTypes::KDFToString(KeyDerivationFunction kdf_p) { + switch (kdf_p) { + case SHA256: + return "SHA256"; + case PBKDF2: + return "PBKDF2"; + default: + return "DEFAULT"; + } +} + +EncryptionTypes::KeyDerivationFunction EncryptionTypes::StringToKDF(const string &key_derivation_function_p) { + auto key_derivation_function = StringUtil::Upper(key_derivation_function_p); + if (key_derivation_function == "SHA256") { + return SHA256; + } + if (key_derivation_function == "PBKDF2") { + return PBKDF2; + } + return DEFAULT; +} + } // namespace duckdb diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index da03abf6d53a..5003cc5815e6 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -60,6 +60,7 @@ #include "duckdb/common/enums/set_scope.hpp" #include "duckdb/common/enums/set_type.hpp" #include "duckdb/common/enums/statement_type.hpp" +#include "duckdb/common/enums/storage_block_prefetch.hpp" #include "duckdb/common/enums/stream_execution_result.hpp" #include "duckdb/common/enums/subquery_type.hpp" #include "duckdb/common/enums/tableref_type.hpp" @@ -81,13 +82,13 @@ #include "duckdb/common/multi_file/multi_file_options.hpp" #include "duckdb/common/operator/decimal_cast_operators.hpp" #include "duckdb/common/printer.hpp" -#include "duckdb/common/sort/partition_state.hpp" #include "duckdb/common/sorting/sort_key.hpp" #include "duckdb/common/types.hpp" #include "duckdb/common/types/column/column_data_scan_states.hpp" #include "duckdb/common/types/column/partitioned_column_data.hpp" #include "duckdb/common/types/conflict_manager.hpp" #include "duckdb/common/types/date.hpp" +#include "duckdb/common/types/geometry.hpp" #include "duckdb/common/types/hyperloglog.hpp" #include "duckdb/common/types/row/block_iterator.hpp" #include "duckdb/common/types/row/partitioned_tuple_data.hpp" @@ -117,6 +118,7 @@ #include "duckdb/function/table/arrow/enum/arrow_variable_size_type.hpp" #include "duckdb/function/table_function.hpp" #include "duckdb/function/window/window_merge_sort_tree.hpp" +#include "duckdb/logging/log_storage.hpp" #include "duckdb/logging/logging.hpp" #include "duckdb/main/appender.hpp" #include "duckdb/main/capi/capi_internal.hpp" @@ -136,6 +138,7 @@ #include "duckdb/parser/expression/parameter_expression.hpp" #include "duckdb/parser/expression/star_expression.hpp" #include "duckdb/parser/expression/window_expression.hpp" +#include "duckdb/parser/parsed_data/alter_database_info.hpp" #include "duckdb/parser/parsed_data/alter_info.hpp" #include "duckdb/parser/parsed_data/alter_scalar_function_info.hpp" #include "duckdb/parser/parsed_data/alter_table_function_info.hpp" @@ -338,6 +341,23 @@ AggregateType EnumUtil::FromString(const char *value) { return static_cast(StringUtil::StringToEnum(GetAggregateTypeValues(), 2, "AggregateType", value)); } +const StringUtil::EnumStringLiteral *GetAlterDatabaseTypeValues() { + static constexpr StringUtil::EnumStringLiteral values[] { + { static_cast(AlterDatabaseType::RENAME_DATABASE), "RENAME_DATABASE" } + }; + return values; +} + +template<> +const char* EnumUtil::ToChars(AlterDatabaseType value) { + return StringUtil::EnumToString(GetAlterDatabaseTypeValues(), 1, "AlterDatabaseType", static_cast(value)); +} + +template<> +AlterDatabaseType EnumUtil::FromString(const char *value) { + return static_cast(StringUtil::StringToEnum(GetAlterDatabaseTypeValues(), 1, "AlterDatabaseType", value)); +} + const StringUtil::EnumStringLiteral *GetAlterForeignKeyTypeValues() { static constexpr StringUtil::EnumStringLiteral values[] { { static_cast(AlterForeignKeyType::AFT_ADD), "AFT_ADD" }, @@ -435,19 +455,20 @@ const StringUtil::EnumStringLiteral *GetAlterTypeValues() { { static_cast(AlterType::ALTER_SCALAR_FUNCTION), "ALTER_SCALAR_FUNCTION" }, { static_cast(AlterType::ALTER_TABLE_FUNCTION), "ALTER_TABLE_FUNCTION" }, { static_cast(AlterType::SET_COMMENT), "SET_COMMENT" }, - { static_cast(AlterType::SET_COLUMN_COMMENT), "SET_COLUMN_COMMENT" } + { static_cast(AlterType::SET_COLUMN_COMMENT), "SET_COLUMN_COMMENT" }, + { static_cast(AlterType::ALTER_DATABASE), "ALTER_DATABASE" } }; return values; } template<> const char* EnumUtil::ToChars(AlterType value) { - return StringUtil::EnumToString(GetAlterTypeValues(), 9, "AlterType", static_cast(value)); + return StringUtil::EnumToString(GetAlterTypeValues(), 10, "AlterType", static_cast(value)); } template<> AlterType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetAlterTypeValues(), 9, "AlterType", value)); + return static_cast(StringUtil::StringToEnum(GetAlterTypeValues(), 10, "AlterType", value)); } const StringUtil::EnumStringLiteral *GetAlterViewTypeValues() { @@ -1774,19 +1795,20 @@ const StringUtil::EnumStringLiteral *GetExtraTypeInfoTypeValues() { { static_cast(ExtraTypeInfoType::ARRAY_TYPE_INFO), "ARRAY_TYPE_INFO" }, { static_cast(ExtraTypeInfoType::ANY_TYPE_INFO), "ANY_TYPE_INFO" }, { static_cast(ExtraTypeInfoType::INTEGER_LITERAL_TYPE_INFO), "INTEGER_LITERAL_TYPE_INFO" }, - { static_cast(ExtraTypeInfoType::TEMPLATE_TYPE_INFO), "TEMPLATE_TYPE_INFO" } + { static_cast(ExtraTypeInfoType::TEMPLATE_TYPE_INFO), "TEMPLATE_TYPE_INFO" }, + { static_cast(ExtraTypeInfoType::GEO_TYPE_INFO), "GEO_TYPE_INFO" } }; return values; } template<> const char* EnumUtil::ToChars(ExtraTypeInfoType value) { - return StringUtil::EnumToString(GetExtraTypeInfoTypeValues(), 13, "ExtraTypeInfoType", static_cast(value)); + return StringUtil::EnumToString(GetExtraTypeInfoTypeValues(), 14, "ExtraTypeInfoType", static_cast(value)); } template<> ExtraTypeInfoType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetExtraTypeInfoTypeValues(), 13, "ExtraTypeInfoType", value)); + return static_cast(StringUtil::StringToEnum(GetExtraTypeInfoTypeValues(), 14, "ExtraTypeInfoType", value)); } const StringUtil::EnumStringLiteral *GetFileBufferTypeValues() { @@ -2038,6 +2060,30 @@ GateStatus EnumUtil::FromString(const char *value) { return static_cast(StringUtil::StringToEnum(GetGateStatusValues(), 2, "GateStatus", value)); } +const StringUtil::EnumStringLiteral *GetGeometryTypeValues() { + static constexpr StringUtil::EnumStringLiteral values[] { + { static_cast(GeometryType::INVALID), "INVALID" }, + { static_cast(GeometryType::POINT), "POINT" }, + { static_cast(GeometryType::LINESTRING), "LINESTRING" }, + { static_cast(GeometryType::POLYGON), "POLYGON" }, + { static_cast(GeometryType::MULTIPOINT), "MULTIPOINT" }, + { static_cast(GeometryType::MULTILINESTRING), "MULTILINESTRING" }, + { static_cast(GeometryType::MULTIPOLYGON), "MULTIPOLYGON" }, + { static_cast(GeometryType::GEOMETRYCOLLECTION), "GEOMETRYCOLLECTION" } + }; + return values; +} + +template<> +const char* EnumUtil::ToChars(GeometryType value) { + return StringUtil::EnumToString(GetGeometryTypeValues(), 8, "GeometryType", static_cast(value)); +} + +template<> +GeometryType EnumUtil::FromString(const char *value) { + return static_cast(StringUtil::StringToEnum(GetGeometryTypeValues(), 8, "GeometryType", value)); +} + const StringUtil::EnumStringLiteral *GetHLLStorageTypeValues() { static constexpr StringUtil::EnumStringLiteral values[] { { static_cast(HLLStorageType::HLL_V1), "HLL_V1" }, @@ -2439,6 +2485,25 @@ LogMode EnumUtil::FromString(const char *value) { return static_cast(StringUtil::StringToEnum(GetLogModeValues(), 3, "LogMode", value)); } +const StringUtil::EnumStringLiteral *GetLoggingTargetTableValues() { + static constexpr StringUtil::EnumStringLiteral values[] { + { static_cast(LoggingTargetTable::ALL_LOGS), "ALL_LOGS" }, + { static_cast(LoggingTargetTable::LOG_ENTRIES), "LOG_ENTRIES" }, + { static_cast(LoggingTargetTable::LOG_CONTEXTS), "LOG_CONTEXTS" } + }; + return values; +} + +template<> +const char* EnumUtil::ToChars(LoggingTargetTable value) { + return StringUtil::EnumToString(GetLoggingTargetTableValues(), 3, "LoggingTargetTable", static_cast(value)); +} + +template<> +LoggingTargetTable EnumUtil::FromString(const char *value) { + return static_cast(StringUtil::StringToEnum(GetLoggingTargetTableValues(), 3, "LoggingTargetTable", value)); +} + const StringUtil::EnumStringLiteral *GetLogicalOperatorTypeValues() { static constexpr StringUtil::EnumStringLiteral values[] { { static_cast(LogicalOperatorType::LOGICAL_INVALID), "LOGICAL_INVALID" }, @@ -2559,6 +2624,7 @@ const StringUtil::EnumStringLiteral *GetLogicalTypeIdValues() { { static_cast(LogicalTypeId::POINTER), "POINTER" }, { static_cast(LogicalTypeId::VALIDITY), "VALIDITY" }, { static_cast(LogicalTypeId::UUID), "UUID" }, + { static_cast(LogicalTypeId::GEOMETRY), "GEOMETRY" }, { static_cast(LogicalTypeId::STRUCT), "STRUCT" }, { static_cast(LogicalTypeId::LIST), "LIST" }, { static_cast(LogicalTypeId::MAP), "MAP" }, @@ -2575,12 +2641,12 @@ const StringUtil::EnumStringLiteral *GetLogicalTypeIdValues() { template<> const char* EnumUtil::ToChars(LogicalTypeId value) { - return StringUtil::EnumToString(GetLogicalTypeIdValues(), 50, "LogicalTypeId", static_cast(value)); + return StringUtil::EnumToString(GetLogicalTypeIdValues(), 51, "LogicalTypeId", static_cast(value)); } template<> LogicalTypeId EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetLogicalTypeIdValues(), 50, "LogicalTypeId", value)); + return static_cast(StringUtil::StringToEnum(GetLogicalTypeIdValues(), 51, "LogicalTypeId", value)); } const StringUtil::EnumStringLiteral *GetLookupResultTypeValues() { @@ -2732,32 +2798,36 @@ MetaPipelineType EnumUtil::FromString(const char *value) { const StringUtil::EnumStringLiteral *GetMetricsTypeValues() { static constexpr StringUtil::EnumStringLiteral values[] { - { static_cast(MetricsType::QUERY_NAME), "QUERY_NAME" }, + { static_cast(MetricsType::ATTACH_LOAD_STORAGE_LATENCY), "ATTACH_LOAD_STORAGE_LATENCY" }, + { static_cast(MetricsType::ATTACH_REPLAY_WAL_LATENCY), "ATTACH_REPLAY_WAL_LATENCY" }, { static_cast(MetricsType::BLOCKED_THREAD_TIME), "BLOCKED_THREAD_TIME" }, + { static_cast(MetricsType::CHECKPOINT_LATENCY), "CHECKPOINT_LATENCY" }, { static_cast(MetricsType::CPU_TIME), "CPU_TIME" }, - { static_cast(MetricsType::EXTRA_INFO), "EXTRA_INFO" }, { static_cast(MetricsType::CUMULATIVE_CARDINALITY), "CUMULATIVE_CARDINALITY" }, - { static_cast(MetricsType::OPERATOR_TYPE), "OPERATOR_TYPE" }, - { static_cast(MetricsType::OPERATOR_CARDINALITY), "OPERATOR_CARDINALITY" }, { static_cast(MetricsType::CUMULATIVE_ROWS_SCANNED), "CUMULATIVE_ROWS_SCANNED" }, + { static_cast(MetricsType::EXTRA_INFO), "EXTRA_INFO" }, + { static_cast(MetricsType::LATENCY), "LATENCY" }, + { static_cast(MetricsType::OPERATOR_CARDINALITY), "OPERATOR_CARDINALITY" }, + { static_cast(MetricsType::OPERATOR_NAME), "OPERATOR_NAME" }, { static_cast(MetricsType::OPERATOR_ROWS_SCANNED), "OPERATOR_ROWS_SCANNED" }, { static_cast(MetricsType::OPERATOR_TIMING), "OPERATOR_TIMING" }, + { static_cast(MetricsType::OPERATOR_TYPE), "OPERATOR_TYPE" }, + { static_cast(MetricsType::QUERY_NAME), "QUERY_NAME" }, { static_cast(MetricsType::RESULT_SET_SIZE), "RESULT_SET_SIZE" }, - { static_cast(MetricsType::LATENCY), "LATENCY" }, { static_cast(MetricsType::ROWS_RETURNED), "ROWS_RETURNED" }, - { static_cast(MetricsType::OPERATOR_NAME), "OPERATOR_NAME" }, { static_cast(MetricsType::SYSTEM_PEAK_BUFFER_MEMORY), "SYSTEM_PEAK_BUFFER_MEMORY" }, { static_cast(MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE), "SYSTEM_PEAK_TEMP_DIR_SIZE" }, { static_cast(MetricsType::TOTAL_BYTES_READ), "TOTAL_BYTES_READ" }, { static_cast(MetricsType::TOTAL_BYTES_WRITTEN), "TOTAL_BYTES_WRITTEN" }, + { static_cast(MetricsType::WAITING_TO_ATTACH_LATENCY), "WAITING_TO_ATTACH_LATENCY" }, { static_cast(MetricsType::ALL_OPTIMIZERS), "ALL_OPTIMIZERS" }, { static_cast(MetricsType::CUMULATIVE_OPTIMIZER_TIMING), "CUMULATIVE_OPTIMIZER_TIMING" }, - { static_cast(MetricsType::PLANNER), "PLANNER" }, - { static_cast(MetricsType::PLANNER_BINDING), "PLANNER_BINDING" }, { static_cast(MetricsType::PHYSICAL_PLANNER), "PHYSICAL_PLANNER" }, { static_cast(MetricsType::PHYSICAL_PLANNER_COLUMN_BINDING), "PHYSICAL_PLANNER_COLUMN_BINDING" }, - { static_cast(MetricsType::PHYSICAL_PLANNER_RESOLVE_TYPES), "PHYSICAL_PLANNER_RESOLVE_TYPES" }, { static_cast(MetricsType::PHYSICAL_PLANNER_CREATE_PLAN), "PHYSICAL_PLANNER_CREATE_PLAN" }, + { static_cast(MetricsType::PHYSICAL_PLANNER_RESOLVE_TYPES), "PHYSICAL_PLANNER_RESOLVE_TYPES" }, + { static_cast(MetricsType::PLANNER), "PLANNER" }, + { static_cast(MetricsType::PLANNER_BINDING), "PLANNER_BINDING" }, { static_cast(MetricsType::OPTIMIZER_EXPRESSION_REWRITER), "OPTIMIZER_EXPRESSION_REWRITER" }, { static_cast(MetricsType::OPTIMIZER_FILTER_PULLUP), "OPTIMIZER_FILTER_PULLUP" }, { static_cast(MetricsType::OPTIMIZER_FILTER_PUSHDOWN), "OPTIMIZER_FILTER_PUSHDOWN" }, @@ -2776,6 +2846,7 @@ const StringUtil::EnumStringLiteral *GetMetricsTypeValues() { { static_cast(MetricsType::OPTIMIZER_BUILD_SIDE_PROBE_SIDE), "OPTIMIZER_BUILD_SIDE_PROBE_SIDE" }, { static_cast(MetricsType::OPTIMIZER_LIMIT_PUSHDOWN), "OPTIMIZER_LIMIT_PUSHDOWN" }, { static_cast(MetricsType::OPTIMIZER_TOP_N), "OPTIMIZER_TOP_N" }, + { static_cast(MetricsType::OPTIMIZER_TOP_N_WINDOW_ELIMINATION), "OPTIMIZER_TOP_N_WINDOW_ELIMINATION" }, { static_cast(MetricsType::OPTIMIZER_COMPRESSED_MATERIALIZATION), "OPTIMIZER_COMPRESSED_MATERIALIZATION" }, { static_cast(MetricsType::OPTIMIZER_DUPLICATE_GROUPS), "OPTIMIZER_DUPLICATE_GROUPS" }, { static_cast(MetricsType::OPTIMIZER_REORDER_FILTER), "OPTIMIZER_REORDER_FILTER" }, @@ -2785,19 +2856,20 @@ const StringUtil::EnumStringLiteral *GetMetricsTypeValues() { { static_cast(MetricsType::OPTIMIZER_MATERIALIZED_CTE), "OPTIMIZER_MATERIALIZED_CTE" }, { static_cast(MetricsType::OPTIMIZER_SUM_REWRITER), "OPTIMIZER_SUM_REWRITER" }, { static_cast(MetricsType::OPTIMIZER_LATE_MATERIALIZATION), "OPTIMIZER_LATE_MATERIALIZATION" }, - { static_cast(MetricsType::OPTIMIZER_CTE_INLINING), "OPTIMIZER_CTE_INLINING" } + { static_cast(MetricsType::OPTIMIZER_CTE_INLINING), "OPTIMIZER_CTE_INLINING" }, + { static_cast(MetricsType::OPTIMIZER_COMMON_SUBPLAN), "OPTIMIZER_COMMON_SUBPLAN" } }; return values; } template<> const char* EnumUtil::ToChars(MetricsType value) { - return StringUtil::EnumToString(GetMetricsTypeValues(), 54, "MetricsType", static_cast(value)); + return StringUtil::EnumToString(GetMetricsTypeValues(), 60, "MetricsType", static_cast(value)); } template<> MetricsType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetMetricsTypeValues(), 54, "MetricsType", value)); + return static_cast(StringUtil::StringToEnum(GetMetricsTypeValues(), 60, "MetricsType", value)); } const StringUtil::EnumStringLiteral *GetMultiFileColumnMappingModeValues() { @@ -3020,6 +3092,7 @@ const StringUtil::EnumStringLiteral *GetOptimizerTypeValues() { { static_cast(OptimizerType::BUILD_SIDE_PROBE_SIDE), "BUILD_SIDE_PROBE_SIDE" }, { static_cast(OptimizerType::LIMIT_PUSHDOWN), "LIMIT_PUSHDOWN" }, { static_cast(OptimizerType::TOP_N), "TOP_N" }, + { static_cast(OptimizerType::TOP_N_WINDOW_ELIMINATION), "TOP_N_WINDOW_ELIMINATION" }, { static_cast(OptimizerType::COMPRESSED_MATERIALIZATION), "COMPRESSED_MATERIALIZATION" }, { static_cast(OptimizerType::DUPLICATE_GROUPS), "DUPLICATE_GROUPS" }, { static_cast(OptimizerType::REORDER_FILTER), "REORDER_FILTER" }, @@ -3029,19 +3102,20 @@ const StringUtil::EnumStringLiteral *GetOptimizerTypeValues() { { static_cast(OptimizerType::MATERIALIZED_CTE), "MATERIALIZED_CTE" }, { static_cast(OptimizerType::SUM_REWRITER), "SUM_REWRITER" }, { static_cast(OptimizerType::LATE_MATERIALIZATION), "LATE_MATERIALIZATION" }, - { static_cast(OptimizerType::CTE_INLINING), "CTE_INLINING" } + { static_cast(OptimizerType::CTE_INLINING), "CTE_INLINING" }, + { static_cast(OptimizerType::COMMON_SUBPLAN), "COMMON_SUBPLAN" } }; return values; } template<> const char* EnumUtil::ToChars(OptimizerType value) { - return StringUtil::EnumToString(GetOptimizerTypeValues(), 29, "OptimizerType", static_cast(value)); + return StringUtil::EnumToString(GetOptimizerTypeValues(), 31, "OptimizerType", static_cast(value)); } template<> OptimizerType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetOptimizerTypeValues(), 29, "OptimizerType", value)); + return static_cast(StringUtil::StringToEnum(GetOptimizerTypeValues(), 31, "OptimizerType", value)); } const StringUtil::EnumStringLiteral *GetOrderByNullTypeValues() { @@ -3197,28 +3271,6 @@ ParserExtensionResultType EnumUtil::FromString(const return static_cast(StringUtil::StringToEnum(GetParserExtensionResultTypeValues(), 3, "ParserExtensionResultType", value)); } -const StringUtil::EnumStringLiteral *GetPartitionSortStageValues() { - static constexpr StringUtil::EnumStringLiteral values[] { - { static_cast(PartitionSortStage::INIT), "INIT" }, - { static_cast(PartitionSortStage::SCAN), "SCAN" }, - { static_cast(PartitionSortStage::PREPARE), "PREPARE" }, - { static_cast(PartitionSortStage::MERGE), "MERGE" }, - { static_cast(PartitionSortStage::SORTED), "SORTED" }, - { static_cast(PartitionSortStage::FINISHED), "FINISHED" } - }; - return values; -} - -template<> -const char* EnumUtil::ToChars(PartitionSortStage value) { - return StringUtil::EnumToString(GetPartitionSortStageValues(), 6, "PartitionSortStage", static_cast(value)); -} - -template<> -PartitionSortStage EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetPartitionSortStageValues(), 6, "PartitionSortStage", value)); -} - const StringUtil::EnumStringLiteral *GetPartitionedColumnDataTypeValues() { static constexpr StringUtil::EnumStringLiteral values[] { { static_cast(PartitionedColumnDataType::INVALID), "INVALID" }, @@ -3555,19 +3607,20 @@ const StringUtil::EnumStringLiteral *GetQueryNodeTypeValues() { { static_cast(QueryNodeType::SET_OPERATION_NODE), "SET_OPERATION_NODE" }, { static_cast(QueryNodeType::BOUND_SUBQUERY_NODE), "BOUND_SUBQUERY_NODE" }, { static_cast(QueryNodeType::RECURSIVE_CTE_NODE), "RECURSIVE_CTE_NODE" }, - { static_cast(QueryNodeType::CTE_NODE), "CTE_NODE" } + { static_cast(QueryNodeType::CTE_NODE), "CTE_NODE" }, + { static_cast(QueryNodeType::STATEMENT_NODE), "STATEMENT_NODE" } }; return values; } template<> const char* EnumUtil::ToChars(QueryNodeType value) { - return StringUtil::EnumToString(GetQueryNodeTypeValues(), 5, "QueryNodeType", static_cast(value)); + return StringUtil::EnumToString(GetQueryNodeTypeValues(), 6, "QueryNodeType", static_cast(value)); } template<> QueryNodeType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetQueryNodeTypeValues(), 5, "QueryNodeType", value)); + return static_cast(StringUtil::StringToEnum(GetQueryNodeTypeValues(), 6, "QueryNodeType", value)); } const StringUtil::EnumStringLiteral *GetQueryResultTypeValues() { @@ -4180,19 +4233,20 @@ const StringUtil::EnumStringLiteral *GetStatisticsTypeValues() { { static_cast(StatisticsType::LIST_STATS), "LIST_STATS" }, { static_cast(StatisticsType::STRUCT_STATS), "STRUCT_STATS" }, { static_cast(StatisticsType::BASE_STATS), "BASE_STATS" }, - { static_cast(StatisticsType::ARRAY_STATS), "ARRAY_STATS" } + { static_cast(StatisticsType::ARRAY_STATS), "ARRAY_STATS" }, + { static_cast(StatisticsType::GEOMETRY_STATS), "GEOMETRY_STATS" } }; return values; } template<> const char* EnumUtil::ToChars(StatisticsType value) { - return StringUtil::EnumToString(GetStatisticsTypeValues(), 6, "StatisticsType", static_cast(value)); + return StringUtil::EnumToString(GetStatisticsTypeValues(), 7, "StatisticsType", static_cast(value)); } template<> StatisticsType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetStatisticsTypeValues(), 6, "StatisticsType", value)); + return static_cast(StringUtil::StringToEnum(GetStatisticsTypeValues(), 7, "StatisticsType", value)); } const StringUtil::EnumStringLiteral *GetStatsInfoValues() { @@ -4216,6 +4270,26 @@ StatsInfo EnumUtil::FromString(const char *value) { return static_cast(StringUtil::StringToEnum(GetStatsInfoValues(), 5, "StatsInfo", value)); } +const StringUtil::EnumStringLiteral *GetStorageBlockPrefetchValues() { + static constexpr StringUtil::EnumStringLiteral values[] { + { static_cast(StorageBlockPrefetch::REMOTE_ONLY), "REMOTE_ONLY" }, + { static_cast(StorageBlockPrefetch::NEVER), "NEVER" }, + { static_cast(StorageBlockPrefetch::ALWAYS_PREFETCH), "ALWAYS_PREFETCH" }, + { static_cast(StorageBlockPrefetch::DEBUG_FORCE_ALWAYS), "DEBUG_FORCE_ALWAYS" } + }; + return values; +} + +template<> +const char* EnumUtil::ToChars(StorageBlockPrefetch value) { + return StringUtil::EnumToString(GetStorageBlockPrefetchValues(), 4, "StorageBlockPrefetch", static_cast(value)); +} + +template<> +StorageBlockPrefetch EnumUtil::FromString(const char *value) { + return static_cast(StringUtil::StringToEnum(GetStorageBlockPrefetchValues(), 4, "StorageBlockPrefetch", value)); +} + const StringUtil::EnumStringLiteral *GetStrTimeSpecifierValues() { static constexpr StringUtil::EnumStringLiteral values[] { { static_cast(StrTimeSpecifier::ABBREVIATED_WEEKDAY_NAME), "ABBREVIATED_WEEKDAY_NAME" }, @@ -4746,6 +4820,7 @@ const StringUtil::EnumStringLiteral *GetVariantLogicalTypeValues() { { static_cast(VariantLogicalType::ARRAY), "ARRAY" }, { static_cast(VariantLogicalType::BIGNUM), "BIGNUM" }, { static_cast(VariantLogicalType::BITSTRING), "BITSTRING" }, + { static_cast(VariantLogicalType::GEOMETRY), "GEOMETRY" }, { static_cast(VariantLogicalType::ENUM_SIZE), "ENUM_SIZE" } }; return values; @@ -4753,12 +4828,12 @@ const StringUtil::EnumStringLiteral *GetVariantLogicalTypeValues() { template<> const char* EnumUtil::ToChars(VariantLogicalType value) { - return StringUtil::EnumToString(GetVariantLogicalTypeValues(), 34, "VariantLogicalType", static_cast(value)); + return StringUtil::EnumToString(GetVariantLogicalTypeValues(), 35, "VariantLogicalType", static_cast(value)); } template<> VariantLogicalType EnumUtil::FromString(const char *value) { - return static_cast(StringUtil::StringToEnum(GetVariantLogicalTypeValues(), 34, "VariantLogicalType", value)); + return static_cast(StringUtil::StringToEnum(GetVariantLogicalTypeValues(), 35, "VariantLogicalType", value)); } const StringUtil::EnumStringLiteral *GetVectorAuxiliaryDataTypeValues() { @@ -4871,6 +4946,26 @@ VerifyExistenceType EnumUtil::FromString(const char *value) return static_cast(StringUtil::StringToEnum(GetVerifyExistenceTypeValues(), 3, "VerifyExistenceType", value)); } +const StringUtil::EnumStringLiteral *GetVertexTypeValues() { + static constexpr StringUtil::EnumStringLiteral values[] { + { static_cast(VertexType::XY), "XY" }, + { static_cast(VertexType::XYZ), "XYZ" }, + { static_cast(VertexType::XYM), "XYM" }, + { static_cast(VertexType::XYZM), "XYZM" } + }; + return values; +} + +template<> +const char* EnumUtil::ToChars(VertexType value) { + return StringUtil::EnumToString(GetVertexTypeValues(), 4, "VertexType", static_cast(value)); +} + +template<> +VertexType EnumUtil::FromString(const char *value) { + return static_cast(StringUtil::StringToEnum(GetVertexTypeValues(), 4, "VertexType", value)); +} + const StringUtil::EnumStringLiteral *GetWALTypeValues() { static constexpr StringUtil::EnumStringLiteral values[] { { static_cast(WALType::INVALID), "INVALID" }, diff --git a/src/common/enums/metric_type.cpp b/src/common/enums/metric_type.cpp index 866049251d2d..84b552037d39 100644 --- a/src/common/enums/metric_type.cpp +++ b/src/common/enums/metric_type.cpp @@ -31,6 +31,7 @@ profiler_settings_t MetricsUtils::GetOptimizerMetrics() { MetricsType::OPTIMIZER_BUILD_SIDE_PROBE_SIDE, MetricsType::OPTIMIZER_LIMIT_PUSHDOWN, MetricsType::OPTIMIZER_TOP_N, + MetricsType::OPTIMIZER_TOP_N_WINDOW_ELIMINATION, MetricsType::OPTIMIZER_COMPRESSED_MATERIALIZATION, MetricsType::OPTIMIZER_DUPLICATE_GROUPS, MetricsType::OPTIMIZER_REORDER_FILTER, @@ -41,6 +42,7 @@ profiler_settings_t MetricsUtils::GetOptimizerMetrics() { MetricsType::OPTIMIZER_SUM_REWRITER, MetricsType::OPTIMIZER_LATE_MATERIALIZATION, MetricsType::OPTIMIZER_CTE_INLINING, + MetricsType::OPTIMIZER_COMMON_SUBPLAN, }; } @@ -48,12 +50,12 @@ profiler_settings_t MetricsUtils::GetPhaseTimingMetrics() { return { MetricsType::ALL_OPTIMIZERS, MetricsType::CUMULATIVE_OPTIMIZER_TIMING, - MetricsType::PLANNER, - MetricsType::PLANNER_BINDING, MetricsType::PHYSICAL_PLANNER, MetricsType::PHYSICAL_PLANNER_COLUMN_BINDING, - MetricsType::PHYSICAL_PLANNER_RESOLVE_TYPES, MetricsType::PHYSICAL_PLANNER_CREATE_PLAN, + MetricsType::PHYSICAL_PLANNER_RESOLVE_TYPES, + MetricsType::PLANNER, + MetricsType::PLANNER_BINDING, }; } @@ -95,6 +97,8 @@ MetricsType MetricsUtils::GetOptimizerMetricByType(OptimizerType type) { return MetricsType::OPTIMIZER_LIMIT_PUSHDOWN; case OptimizerType::TOP_N: return MetricsType::OPTIMIZER_TOP_N; + case OptimizerType::TOP_N_WINDOW_ELIMINATION: + return MetricsType::OPTIMIZER_TOP_N_WINDOW_ELIMINATION; case OptimizerType::COMPRESSED_MATERIALIZATION: return MetricsType::OPTIMIZER_COMPRESSED_MATERIALIZATION; case OptimizerType::DUPLICATE_GROUPS: @@ -115,6 +119,8 @@ MetricsType MetricsUtils::GetOptimizerMetricByType(OptimizerType type) { return MetricsType::OPTIMIZER_LATE_MATERIALIZATION; case OptimizerType::CTE_INLINING: return MetricsType::OPTIMIZER_CTE_INLINING; + case OptimizerType::COMMON_SUBPLAN: + return MetricsType::OPTIMIZER_COMMON_SUBPLAN; default: throw InternalException("OptimizerType %s cannot be converted to a MetricsType", EnumUtil::ToString(type)); }; @@ -158,6 +164,8 @@ OptimizerType MetricsUtils::GetOptimizerTypeByMetric(MetricsType type) { return OptimizerType::LIMIT_PUSHDOWN; case MetricsType::OPTIMIZER_TOP_N: return OptimizerType::TOP_N; + case MetricsType::OPTIMIZER_TOP_N_WINDOW_ELIMINATION: + return OptimizerType::TOP_N_WINDOW_ELIMINATION; case MetricsType::OPTIMIZER_COMPRESSED_MATERIALIZATION: return OptimizerType::COMPRESSED_MATERIALIZATION; case MetricsType::OPTIMIZER_DUPLICATE_GROUPS: @@ -178,6 +186,8 @@ OptimizerType MetricsUtils::GetOptimizerTypeByMetric(MetricsType type) { return OptimizerType::LATE_MATERIALIZATION; case MetricsType::OPTIMIZER_CTE_INLINING: return OptimizerType::CTE_INLINING; + case MetricsType::OPTIMIZER_COMMON_SUBPLAN: + return OptimizerType::COMMON_SUBPLAN; default: return OptimizerType::INVALID; }; @@ -203,6 +213,7 @@ bool MetricsUtils::IsOptimizerMetric(MetricsType type) { case MetricsType::OPTIMIZER_BUILD_SIDE_PROBE_SIDE: case MetricsType::OPTIMIZER_LIMIT_PUSHDOWN: case MetricsType::OPTIMIZER_TOP_N: + case MetricsType::OPTIMIZER_TOP_N_WINDOW_ELIMINATION: case MetricsType::OPTIMIZER_COMPRESSED_MATERIALIZATION: case MetricsType::OPTIMIZER_DUPLICATE_GROUPS: case MetricsType::OPTIMIZER_REORDER_FILTER: @@ -213,6 +224,7 @@ bool MetricsUtils::IsOptimizerMetric(MetricsType type) { case MetricsType::OPTIMIZER_SUM_REWRITER: case MetricsType::OPTIMIZER_LATE_MATERIALIZATION: case MetricsType::OPTIMIZER_CTE_INLINING: + case MetricsType::OPTIMIZER_COMMON_SUBPLAN: return true; default: return false; @@ -223,12 +235,12 @@ bool MetricsUtils::IsPhaseTimingMetric(MetricsType type) { switch(type) { case MetricsType::ALL_OPTIMIZERS: case MetricsType::CUMULATIVE_OPTIMIZER_TIMING: - case MetricsType::PLANNER: - case MetricsType::PLANNER_BINDING: case MetricsType::PHYSICAL_PLANNER: case MetricsType::PHYSICAL_PLANNER_COLUMN_BINDING: - case MetricsType::PHYSICAL_PLANNER_RESOLVE_TYPES: case MetricsType::PHYSICAL_PLANNER_CREATE_PLAN: + case MetricsType::PHYSICAL_PLANNER_RESOLVE_TYPES: + case MetricsType::PLANNER: + case MetricsType::PLANNER_BINDING: return true; default: return false; @@ -237,9 +249,13 @@ bool MetricsUtils::IsPhaseTimingMetric(MetricsType type) { bool MetricsUtils::IsQueryGlobalMetric(MetricsType type) { switch(type) { + case MetricsType::ATTACH_LOAD_STORAGE_LATENCY: + case MetricsType::ATTACH_REPLAY_WAL_LATENCY: case MetricsType::BLOCKED_THREAD_TIME: + case MetricsType::CHECKPOINT_LATENCY: case MetricsType::SYSTEM_PEAK_BUFFER_MEMORY: case MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE: + case MetricsType::WAITING_TO_ATTACH_LATENCY: return true; default: return false; diff --git a/src/common/enums/optimizer_type.cpp b/src/common/enums/optimizer_type.cpp index b0d669500105..c7441a0fa3d2 100644 --- a/src/common/enums/optimizer_type.cpp +++ b/src/common/enums/optimizer_type.cpp @@ -3,6 +3,7 @@ #include "duckdb/common/exception.hpp" #include "duckdb/common/exception/parser_exception.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/optimizer/optimizer.hpp" namespace duckdb { @@ -29,6 +30,7 @@ static const DefaultOptimizerType internal_optimizer_types[] = { {"column_lifetime", OptimizerType::COLUMN_LIFETIME}, {"limit_pushdown", OptimizerType::LIMIT_PUSHDOWN}, {"top_n", OptimizerType::TOP_N}, + {"top_n_window_elimination", OptimizerType::TOP_N_WINDOW_ELIMINATION}, {"build_side_probe_side", OptimizerType::BUILD_SIDE_PROBE_SIDE}, {"compressed_materialization", OptimizerType::COMPRESSED_MATERIALIZATION}, {"duplicate_groups", OptimizerType::DUPLICATE_GROUPS}, @@ -40,6 +42,7 @@ static const DefaultOptimizerType internal_optimizer_types[] = { {"sum_rewriter", OptimizerType::SUM_REWRITER}, {"late_materialization", OptimizerType::LATE_MATERIALIZATION}, {"cte_inlining", OptimizerType::CTE_INLINING}, + {"common_subplan", OptimizerType::COMMON_SUBPLAN}, {nullptr, OptimizerType::INVALID}}; string OptimizerTypeToString(OptimizerType type) { diff --git a/src/common/error_data.cpp b/src/common/error_data.cpp index 2ddf94af69aa..07b039ea83f2 100644 --- a/src/common/error_data.cpp +++ b/src/common/error_data.cpp @@ -80,9 +80,9 @@ void ErrorData::Throw(const string &prepended_message) const { D_ASSERT(initialized); if (!prepended_message.empty()) { string new_message = prepended_message + raw_message; - throw Exception(type, new_message, extra_info); + throw Exception(extra_info, type, new_message); } else { - throw Exception(type, raw_message, extra_info); + throw Exception(extra_info, type, raw_message); } } diff --git a/src/common/exception.cpp b/src/common/exception.cpp index 2012c1fcce2e..1d7345cf4476 100644 --- a/src/common/exception.cpp +++ b/src/common/exception.cpp @@ -19,17 +19,17 @@ Exception::Exception(ExceptionType exception_type, const string &message) : std::runtime_error(ToJSON(exception_type, message)) { } -Exception::Exception(ExceptionType exception_type, const string &message, - const unordered_map &extra_info) - : std::runtime_error(ToJSON(exception_type, message, extra_info)) { +Exception::Exception(const unordered_map &extra_info, ExceptionType exception_type, + const string &message) + : std::runtime_error(ToJSON(extra_info, exception_type, message)) { } string Exception::ToJSON(ExceptionType type, const string &message) { unordered_map extra_info; - return ToJSON(type, message, extra_info); + return ToJSON(extra_info, type, message); } -string Exception::ToJSON(ExceptionType type, const string &message, const unordered_map &extra_info) { +string Exception::ToJSON(const unordered_map &extra_info, ExceptionType type, const string &message) { #ifndef DUCKDB_DEBUG_STACKTRACE // by default we only enable stack traces for internal exceptions if (type == ExceptionType::INTERNAL || type == ExceptionType::FATAL) @@ -240,9 +240,8 @@ TypeMismatchException::TypeMismatchException(const LogicalType &type_1, const Lo TypeMismatchException::TypeMismatchException(optional_idx error_location, const LogicalType &type_1, const LogicalType &type_2, const string &msg) - : Exception(ExceptionType::MISMATCH_TYPE, - "Type " + type_1.ToString() + " does not match with " + type_2.ToString() + ". " + msg, - Exception::InitializeExtraInfo(error_location)) { + : Exception(Exception::InitializeExtraInfo(error_location), ExceptionType::MISMATCH_TYPE, + "Type " + type_1.ToString() + " does not match with " + type_2.ToString() + ". " + msg) { } TypeMismatchException::TypeMismatchException(const string &msg) : Exception(ExceptionType::MISMATCH_TYPE, msg) { @@ -306,8 +305,8 @@ DependencyException::DependencyException(const string &msg) : Exception(Exceptio IOException::IOException(const string &msg) : Exception(ExceptionType::IO, msg) { } -IOException::IOException(const string &msg, const unordered_map &extra_info) - : Exception(ExceptionType::IO, msg, extra_info) { +IOException::IOException(const unordered_map &extra_info, const string &msg) + : Exception(extra_info, ExceptionType::IO, msg) { } MissingExtensionException::MissingExtensionException(const string &msg) @@ -342,17 +341,17 @@ InternalException::InternalException(const string &msg) : Exception(ExceptionTyp InvalidInputException::InvalidInputException(const string &msg) : Exception(ExceptionType::INVALID_INPUT, msg) { } -InvalidInputException::InvalidInputException(const string &msg, const unordered_map &extra_info) - : Exception(ExceptionType::INVALID_INPUT, msg, extra_info) { +InvalidInputException::InvalidInputException(const unordered_map &extra_info, const string &msg) + : Exception(extra_info, ExceptionType::INVALID_INPUT, msg) { } InvalidConfigurationException::InvalidConfigurationException(const string &msg) : Exception(ExceptionType::INVALID_CONFIGURATION, msg) { } -InvalidConfigurationException::InvalidConfigurationException(const string &msg, - const unordered_map &extra_info) - : Exception(ExceptionType::INVALID_CONFIGURATION, msg, extra_info) { +InvalidConfigurationException::InvalidConfigurationException(const unordered_map &extra_info, + const string &msg) + : Exception(extra_info, ExceptionType::INVALID_CONFIGURATION, msg) { } OutOfMemoryException::OutOfMemoryException(const string &msg) diff --git a/src/common/exception/binder_exception.cpp b/src/common/exception/binder_exception.cpp index 62dca06fb5af..70f71a52b4ab 100644 --- a/src/common/exception/binder_exception.cpp +++ b/src/common/exception/binder_exception.cpp @@ -7,8 +7,8 @@ namespace duckdb { BinderException::BinderException(const string &msg) : Exception(ExceptionType::BINDER, msg) { } -BinderException::BinderException(const string &msg, const unordered_map &extra_info) - : Exception(ExceptionType::BINDER, msg, extra_info) { +BinderException::BinderException(const unordered_map &extra_info, const string &msg) + : Exception(extra_info, ExceptionType::BINDER, msg) { } BinderException BinderException::ColumnNotFound(const string &name, const vector &similar_bindings, @@ -20,7 +20,7 @@ BinderException BinderException::ColumnNotFound(const string &name, const vector extra_info["candidates"] = StringUtil::Join(similar_bindings, ","); } return BinderException( - StringUtil::Format("Referenced column \"%s\" not found in FROM clause!%s", name, candidate_str), extra_info); + extra_info, StringUtil::Format("Referenced column \"%s\" not found in FROM clause!%s", name, candidate_str)); } BinderException BinderException::NoMatchingFunction(const string &catalog_name, const string &schema_name, @@ -45,15 +45,14 @@ BinderException BinderException::NoMatchingFunction(const string &catalog_name, extra_info["candidates"] = StringUtil::Join(candidates, ","); } return BinderException( + extra_info, StringUtil::Format("No function matches the given name and argument types '%s'. You might need to add " "explicit type casts.\n\tCandidate functions:\n%s", - call_str, candidate_str), - extra_info); + call_str, candidate_str)); } BinderException BinderException::Unsupported(ParsedExpression &expr, const string &message) { auto extra_info = Exception::InitializeExtraInfo("UNSUPPORTED", expr.GetQueryLocation()); - return BinderException(message, extra_info); + return BinderException(extra_info, message); } - } // namespace duckdb diff --git a/src/common/exception/catalog_exception.cpp b/src/common/exception/catalog_exception.cpp index 5d890f1cdb60..b1cd4caf79cf 100644 --- a/src/common/exception/catalog_exception.cpp +++ b/src/common/exception/catalog_exception.cpp @@ -9,8 +9,8 @@ namespace duckdb { CatalogException::CatalogException(const string &msg) : Exception(ExceptionType::CATALOG, msg) { } -CatalogException::CatalogException(const string &msg, const unordered_map &extra_info) - : Exception(ExceptionType::CATALOG, msg, extra_info) { +CatalogException::CatalogException(const unordered_map &extra_info, const string &msg) + : Exception(extra_info, ExceptionType::CATALOG, msg) { } CatalogException CatalogException::MissingEntry(const EntryLookupInfo &lookup_info, const string &suggestion) { @@ -35,9 +35,9 @@ CatalogException CatalogException::MissingEntry(const EntryLookupInfo &lookup_in if (!suggestion.empty()) { extra_info["candidates"] = suggestion; } - return CatalogException(StringUtil::Format("%s with name %s does not exist%s!%s", CatalogTypeToString(type), name, - version_info, did_you_mean), - extra_info); + return CatalogException(extra_info, + StringUtil::Format("%s with name %s does not exist%s!%s", CatalogTypeToString(type), name, + version_info, did_you_mean)); } CatalogException CatalogException::MissingEntry(CatalogType type, const string &name, const string &suggestion, @@ -55,17 +55,17 @@ CatalogException CatalogException::MissingEntry(const string &type, const string if (!suggestions.empty()) { extra_info["candidates"] = StringUtil::Join(suggestions, ", "); } - return CatalogException(StringUtil::Format("unrecognized %s \"%s\"\n%s", type, name, - StringUtil::CandidatesErrorMessage(suggestions, name, "Did you mean")), - extra_info); + return CatalogException(extra_info, + StringUtil::Format("unrecognized %s \"%s\"\n%s", type, name, + StringUtil::CandidatesErrorMessage(suggestions, name, "Did you mean"))); } CatalogException CatalogException::EntryAlreadyExists(CatalogType type, const string &name, QueryErrorContext context) { auto extra_info = Exception::InitializeExtraInfo("ENTRY_ALREADY_EXISTS", optional_idx()); extra_info["name"] = name; extra_info["type"] = CatalogTypeToString(type); - return CatalogException(StringUtil::Format("%s with name \"%s\" already exists!", CatalogTypeToString(type), name), - extra_info); + return CatalogException(extra_info, + StringUtil::Format("%s with name \"%s\" already exists!", CatalogTypeToString(type), name)); } } // namespace duckdb diff --git a/src/common/exception/conversion_exception.cpp b/src/common/exception/conversion_exception.cpp index 013dbdb9e5df..bf021b4eb0e6 100644 --- a/src/common/exception/conversion_exception.cpp +++ b/src/common/exception/conversion_exception.cpp @@ -17,7 +17,7 @@ ConversionException::ConversionException(const string &msg) : Exception(Exceptio } ConversionException::ConversionException(optional_idx error_location, const string &msg) - : Exception(ExceptionType::CONVERSION, msg, Exception::InitializeExtraInfo(error_location)) { + : Exception(Exception::InitializeExtraInfo(error_location), ExceptionType::CONVERSION, msg) { } } // namespace duckdb diff --git a/src/common/exception/parser_exception.cpp b/src/common/exception/parser_exception.cpp index f3875da3890d..3afb2ea3d201 100644 --- a/src/common/exception/parser_exception.cpp +++ b/src/common/exception/parser_exception.cpp @@ -7,13 +7,12 @@ namespace duckdb { ParserException::ParserException(const string &msg) : Exception(ExceptionType::PARSER, msg) { } -ParserException::ParserException(const string &msg, const unordered_map &extra_info) - : Exception(ExceptionType::PARSER, msg, extra_info) { +ParserException::ParserException(const unordered_map &extra_info, const string &msg) + : Exception(extra_info, ExceptionType::PARSER, msg) { } ParserException ParserException::SyntaxError(const string &query, const string &error_message, optional_idx error_location) { - return ParserException(error_message, Exception::InitializeExtraInfo("SYNTAX_ERROR", error_location)); + return ParserException(Exception::InitializeExtraInfo("SYNTAX_ERROR", error_location), error_message); } - } // namespace duckdb diff --git a/src/common/exception_format_value.cpp b/src/common/exception_format_value.cpp index 51e34ec0e085..27b4eb4659ce 100644 --- a/src/common/exception_format_value.cpp +++ b/src/common/exception_format_value.cpp @@ -28,65 +28,61 @@ ExceptionFormatValue::ExceptionFormatValue(uhugeint_t uhuge_val) ExceptionFormatValue::ExceptionFormatValue(string str_val) : type(ExceptionFormatValueType::FORMAT_VALUE_TYPE_STRING), str_val(std::move(str_val)) { } -ExceptionFormatValue::ExceptionFormatValue(String str_val) - : type(ExceptionFormatValueType::FORMAT_VALUE_TYPE_STRING), str_val(str_val.ToStdString()) { +ExceptionFormatValue::ExceptionFormatValue(const String &str_val) : ExceptionFormatValue(str_val.ToStdString()) { } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(PhysicalType value) { +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const PhysicalType &value) { return ExceptionFormatValue(TypeIdToString(value)); } template <> -ExceptionFormatValue -ExceptionFormatValue::CreateFormatValue(LogicalType value) { // NOLINT: templating requires us to copy value here +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const LogicalType &value) { return ExceptionFormatValue(value.ToString()); } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(float value) { - return ExceptionFormatValue(double(value)); +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const float &value) { + return ExceptionFormatValue(static_cast(value)); } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(double value) { - return ExceptionFormatValue(double(value)); +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const double &value) { + return ExceptionFormatValue(value); } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(string value) { - return ExceptionFormatValue(std::move(value)); +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const string &value) { + return ExceptionFormatValue(value); } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(String value) { - return ExceptionFormatValue(std::move(value)); +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const String &value) { + return ExceptionFormatValue(value); } template <> -ExceptionFormatValue -ExceptionFormatValue::CreateFormatValue(SQLString value) { // NOLINT: templating requires us to copy value here +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const SQLString &value) { return KeywordHelper::WriteQuoted(value.raw_string, '\''); } template <> -ExceptionFormatValue -ExceptionFormatValue::CreateFormatValue(SQLIdentifier value) { // NOLINT: templating requires us to copy value here +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const SQLIdentifier &value) { return KeywordHelper::WriteOptionallyQuoted(value.raw_string, '"'); } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const char *value) { +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const char *const &value) { return ExceptionFormatValue(string(value)); } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(char *value) { +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(char *const &value) { return ExceptionFormatValue(string(value)); } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(idx_t value) { +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const idx_t &value) { return ExceptionFormatValue(value); } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(hugeint_t value) { +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const hugeint_t &value) { return ExceptionFormatValue(value); } template <> -ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(uhugeint_t value) { +ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const uhugeint_t &value) { return ExceptionFormatValue(value); } diff --git a/src/common/extra_type_info.cpp b/src/common/extra_type_info.cpp index 1d3160814488..6218f3e7b115 100644 --- a/src/common/extra_type_info.cpp +++ b/src/common/extra_type_info.cpp @@ -507,4 +507,19 @@ shared_ptr TemplateTypeInfo::Copy() const { return make_shared_ptr(*this); } +//===--------------------------------------------------------------------===// +// Geo Type Info +//===--------------------------------------------------------------------===// +GeoTypeInfo::GeoTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::GEO_TYPE_INFO) { +} + +bool GeoTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const { + // No additional info to compare + return true; +} + +shared_ptr GeoTypeInfo::Copy() const { + return make_shared_ptr(*this); +} + } // namespace duckdb diff --git a/src/common/file_system.cpp b/src/common/file_system.cpp index 926cfb6a0576..32288e77afcf 100644 --- a/src/common/file_system.cpp +++ b/src/common/file_system.cpp @@ -628,39 +628,9 @@ bool FileSystem::CanHandleFile(const string &fpath) { throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName()); } -static string LookupExtensionForPattern(const string &pattern) { - for (const auto &entry : EXTENSION_FILE_PREFIXES) { - if (StringUtil::StartsWith(pattern, entry.name)) { - return entry.extension; - } - } - return ""; -} - vector FileSystem::GlobFiles(const string &pattern, ClientContext &context, const FileGlobInput &input) { auto result = Glob(pattern); if (result.empty()) { - string required_extension = LookupExtensionForPattern(pattern); - if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) { - auto &dbconfig = DBConfig::GetConfig(context); - if (!ExtensionHelper::CanAutoloadExtension(required_extension) || - !dbconfig.options.autoload_known_extensions) { - auto error_message = - "File " + pattern + " requires the extension " + required_extension + " to be loaded"; - error_message = - ExtensionHelper::AddExtensionInstallHintToErrorMsg(context, error_message, required_extension); - throw MissingExtensionException(error_message); - } - // an extension is required to read this file, but it is not loaded - try to load it - ExtensionHelper::AutoLoadExtension(context, required_extension); - // success! glob again - // check the extension is loaded just in case to prevent an infinite loop here - if (!context.db->ExtensionIsLoaded(required_extension)) { - throw InternalException("Extension load \"%s\" did not throw but somehow the extension was not loaded", - required_extension); - } - return GlobFiles(pattern, context, input); - } if (input.behavior == FileGlobOptions::FALLBACK_GLOB && !HasGlob(pattern)) { // if we have no glob in the pattern and we have an extension, we try to glob if (!HasGlob(pattern)) { diff --git a/src/common/local_file_system.cpp b/src/common/local_file_system.cpp index 8733e0162046..5829cb54848e 100644 --- a/src/common/local_file_system.cpp +++ b/src/common/local_file_system.cpp @@ -369,7 +369,7 @@ unique_ptr LocalFileSystem::OpenFile(const string &path_p, FileOpenF if (flags.ReturnNullIfExists() && errno == EEXIST) { return nullptr; } - throw IOException("Cannot open file \"%s\": %s", {{"errno", std::to_string(errno)}}, path, strerror(errno)); + throw IOException({{"errno", std::to_string(errno)}}, "Cannot open file \"%s\": %s", path, strerror(errno)); } #if defined(__DARWIN__) || defined(__APPLE__) @@ -436,7 +436,7 @@ unique_ptr LocalFileSystem::OpenFile(const string &path_p, FileOpenF extended_error += ". Also, failed closing file"; } extended_error += ". See also https://duckdb.org/docs/stable/connect/concurrency"; - throw IOException("Could not set lock on file \"%s\": %s", {{"errno", std::to_string(retained_errno)}}, + throw IOException({{"errno", std::to_string(retained_errno)}}, "Could not set lock on file \"%s\": %s", path, extended_error); } } @@ -454,7 +454,7 @@ void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) { int fd = handle.Cast().fd; off_t offset = lseek(fd, UnsafeNumericCast(location), SEEK_SET); if (offset == (off_t)-1) { - throw IOException("Could not seek to location %lld for file \"%s\": %s", {{"errno", std::to_string(errno)}}, + throw IOException({{"errno", std::to_string(errno)}}, "Could not seek to location %lld for file \"%s\": %s", location, handle.path, strerror(errno)); } } @@ -463,7 +463,7 @@ idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) { int fd = handle.Cast().fd; off_t position = lseek(fd, 0, SEEK_CUR); if (position == (off_t)-1) { - throw IOException("Could not get file position file \"%s\": %s", {{"errno", std::to_string(errno)}}, + throw IOException({{"errno", std::to_string(errno)}}, "Could not get file position file \"%s\": %s", handle.path, strerror(errno)); } return UnsafeNumericCast(position); @@ -477,7 +477,7 @@ void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, i int64_t bytes_read = pread(fd, read_buffer, UnsafeNumericCast(nr_bytes), UnsafeNumericCast(location)); if (bytes_read == -1) { - throw IOException("Could not read from file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, + throw IOException({{"errno", std::to_string(errno)}}, "Could not read from file \"%s\": %s", handle.path, strerror(errno)); } if (bytes_read == 0) { @@ -498,7 +498,7 @@ int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes int fd = unix_handle.fd; int64_t bytes_read = read(fd, buffer, UnsafeNumericCast(nr_bytes)); if (bytes_read == -1) { - throw IOException("Could not read from file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, + throw IOException({{"errno", std::to_string(errno)}}, "Could not read from file \"%s\": %s", handle.path, strerror(errno)); } @@ -519,12 +519,13 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, int64_t bytes_written = pwrite(fd, write_buffer, UnsafeNumericCast(bytes_to_write), UnsafeNumericCast(current_location)); if (bytes_written < 0) { - throw IOException("Could not write file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, + throw IOException({{"errno", std::to_string(errno)}}, "Could not write file \"%s\": %s", handle.path, strerror(errno)); } if (bytes_written == 0) { - throw IOException("Could not write to file \"%s\" - attempted to write 0 bytes: %s", - {{"errno", std::to_string(errno)}}, handle.path, strerror(errno)); + throw IOException({{"errno", std::to_string(errno)}}, + "Could not write to file \"%s\" - attempted to write 0 bytes: %s", handle.path, + strerror(errno)); } write_buffer += bytes_written; bytes_to_write -= bytes_written; @@ -544,7 +545,7 @@ int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_byte MinValue(idx_t(NumericLimits::Maximum()), idx_t(bytes_to_write)); int64_t current_bytes_written = write(fd, buffer, bytes_to_write_this_call); if (current_bytes_written <= 0) { - throw IOException("Could not write file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, + throw IOException({{"errno", std::to_string(errno)}}, "Could not write file \"%s\": %s", handle.path, strerror(errno)); } buffer = (void *)(data_ptr_cast(buffer) + current_bytes_written); @@ -577,7 +578,7 @@ int64_t LocalFileSystem::GetFileSize(FileHandle &handle) { int fd = handle.Cast().fd; struct stat s; if (fstat(fd, &s) == -1) { - throw IOException("Failed to get file size for file \"%s\": %s", {{"errno", std::to_string(errno)}}, + throw IOException({{"errno", std::to_string(errno)}}, "Failed to get file size for file \"%s\": %s", handle.path, strerror(errno)); } return s.st_size; @@ -587,7 +588,7 @@ timestamp_t LocalFileSystem::GetLastModifiedTime(FileHandle &handle) { int fd = handle.Cast().fd; struct stat s; if (fstat(fd, &s) == -1) { - throw IOException("Failed to get last modified time for file \"%s\": %s", {{"errno", std::to_string(errno)}}, + throw IOException({{"errno", std::to_string(errno)}}, "Failed to get last modified time for file \"%s\": %s", handle.path, strerror(errno)); } return Timestamp::FromEpochSeconds(s.st_mtime); @@ -601,7 +602,7 @@ FileType LocalFileSystem::GetFileType(FileHandle &handle) { void LocalFileSystem::Truncate(FileHandle &handle, int64_t new_size) { int fd = handle.Cast().fd; if (ftruncate(fd, new_size) != 0) { - throw IOException("Could not truncate file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, + throw IOException({{"errno", std::to_string(errno)}}, "Could not truncate file \"%s\": %s", handle.path, strerror(errno)); } } @@ -628,12 +629,12 @@ void LocalFileSystem::CreateDirectory(const string &directory, optional_ptr opener) { auto normalized_file = NormalizeLocalPath(filename); if (std::remove(normalized_file) != 0) { - throw IOException("Could not remove file \"%s\": %s", {{"errno", std::to_string(errno)}}, filename, + throw IOException({{"errno", std::to_string(errno)}}, "Could not remove file \"%s\": %s", filename, strerror(errno)); } } @@ -767,8 +768,7 @@ void LocalFileSystem::FileSync(FileHandle &handle) { } // For other types of errors, throw normal IO exception. - throw IOException("Could not fsync file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.GetPath(), - strerror(errno)); + throw IOException("Could not fsync file \"%s\": %s", handle.GetPath(), strerror(errno)); } void LocalFileSystem::MoveFile(const string &source, const string &target, optional_ptr opener) { @@ -776,7 +776,7 @@ void LocalFileSystem::MoveFile(const string &source, const string &target, optio auto normalized_target = NormalizeLocalPath(target); //! FIXME: rename does not guarantee atomicity or overwriting target file if it exists if (rename(normalized_source, normalized_target) != 0) { - throw IOException("Could not rename file!", {{"errno", std::to_string(errno)}}); + throw IOException({{"errno", std::to_string(errno)}}, "Could not rename file!"); } } @@ -1052,7 +1052,7 @@ static int64_t FSWrite(FileHandle &handle, HANDLE hFile, void *buffer, int64_t n auto bytes_to_write = MinValue(idx_t(NumericLimits::Maximum()), idx_t(nr_bytes)); DWORD current_bytes_written = FSInternalWrite(handle, hFile, buffer, bytes_to_write, location); if (current_bytes_written <= 0) { - throw IOException("Could not write file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, + throw IOException({{"errno", std::to_string(errno)}}, "Could not write file \"%s\": %s", handle.path, strerror(errno)); } bytes_written += current_bytes_written; diff --git a/src/common/operator/cast_operators.cpp b/src/common/operator/cast_operators.cpp index f26c161317a6..723897a1c322 100644 --- a/src/common/operator/cast_operators.cpp +++ b/src/common/operator/cast_operators.cpp @@ -19,6 +19,7 @@ #include "duckdb/common/types/time.hpp" #include "duckdb/common/types/timestamp.hpp" #include "duckdb/common/types/vector.hpp" +#include "duckdb/common/types/geometry.hpp" #include "duckdb/common/types.hpp" #include "fast_float/fast_float.h" #include "duckdb/common/types/bit.hpp" @@ -1560,6 +1561,14 @@ bool TryCastBlobToUUID::Operation(string_t input, hugeint_t &result, bool strict return true; } +//===--------------------------------------------------------------------===// +// Cast To Geometry +//===--------------------------------------------------------------------===// +template <> +bool TryCastToGeometry::Operation(string_t input, string_t &result, Vector &result_vector, CastParameters ¶meters) { + return Geometry::FromString(input, result, result_vector, parameters.strict); +} + //===--------------------------------------------------------------------===// // Cast To Date //===--------------------------------------------------------------------===// diff --git a/src/common/progress_bar/terminal_progress_bar_display.cpp b/src/common/progress_bar/terminal_progress_bar_display.cpp index cdd3aaf79962..417ac609a696 100644 --- a/src/common/progress_bar/terminal_progress_bar_display.cpp +++ b/src/common/progress_bar/terminal_progress_bar_display.cpp @@ -20,21 +20,17 @@ static string FormatETA(double seconds, bool elapsed = false) { // the maximum length here is "(~10.35 minutes remaining)" (26 bytes) // always pad to this amount static constexpr idx_t RENDER_SIZE = 26; - // Desired formats: - // 00:00:00.00 remaining - // unknown remaining - // 00:00:00.00 elapsed - if (!elapsed && seconds > 3600 * 99) { - // estimate larger than 99 hours remaining - string result = "(>99 hours remaining)"; - result += string(RENDER_SIZE - result.size(), ' '); - return result; - } - if (seconds < 0) { + + if (seconds < 0 || seconds == 2147483647) { // Invalid or unknown ETA, skip rendering estimate return string(RENDER_SIZE, ' '); } + if (!elapsed && seconds > 3600 * 99) { + // estimate larger than 99 hours remaining, treat this as invalid/unknown ETA + return string(RENDER_SIZE, ' '); + } + // Round to nearest centisecond as integer auto total_centiseconds = static_cast(std::llround(seconds * 100.0)); @@ -120,9 +116,19 @@ void TerminalProgressBarDisplay::Update(double percentage) { // Filters go from 0 to 1, percentage is from 0-100 const double filter_percentage = percentage / 100.0; + ukf.Update(filter_percentage, current_time); - double estimated_seconds_remaining = std::min(ukf.GetEstimatedRemainingSeconds(), 2147483647.0); + double estimated_seconds_remaining; + // If the query is mostly completed, there can be oscillation of estimated + // time to completion since the progress updates seem sparse near the very + // end of the query, so clamp time remaining to not oscillate with estimates + // that are unlikely to be correct. + if (filter_percentage > 0.99) { + estimated_seconds_remaining = 0.5; + } else { + estimated_seconds_remaining = std::min(ukf.GetEstimatedRemainingSeconds(), 2147483647.0); + } auto percentage_int = NormalizePercentage(percentage); TerminalProgressBarDisplayedProgressInfo updated_progress_info = {(idx_t)percentage_int, diff --git a/src/common/progress_bar/unscented_kalman_filter.cpp b/src/common/progress_bar/unscented_kalman_filter.cpp index 194c6be12a66..98b2a1319616 100644 --- a/src/common/progress_bar/unscented_kalman_filter.cpp +++ b/src/common/progress_bar/unscented_kalman_filter.cpp @@ -4,7 +4,8 @@ namespace duckdb { UnscentedKalmanFilter::UnscentedKalmanFilter() : x(STATE_DIM, 0.0), P(STATE_DIM, vector(STATE_DIM, 0.0)), Q(STATE_DIM, vector(STATE_DIM, 0.0)), - R(OBS_DIM, vector(OBS_DIM, 0.0)), last_time(0.0), initialized(false) { + R(OBS_DIM, vector(OBS_DIM, 0.0)), last_time(0.0), initialized(false), last_progress(-1.0), + scale_factor(1.0) { // Calculate UKF parameters lambda = ALPHA * ALPHA * (STATE_DIM + KAPPA) - STATE_DIM; @@ -21,28 +22,48 @@ UnscentedKalmanFilter::UnscentedKalmanFilter() } // Initialize covariance matrices - P[0][0] = 0.01; // progress variance - P[1][1] = 0.001; // velocity variance + P[0][0] = 0.165; // progress variance + P[1][1] = 0.0098; // velocity variance - Q[0][0] = 1e-6; // process noise for progress - Q[1][1] = 1e-4; // process noise for velocity + Q[0][0] = 0.01; // process noise for progress + Q[1][1] = 0.0; // process noise for velocity - R[0][0] = 0.05; // measurement noise for progress + R[0][0] = 0.000077; // measurement noise for progress } void UnscentedKalmanFilter::Update(double progress, double time) { + progress *= scale_factor; if (!initialized) { Initialize(progress, time); return; } Predict(time); - UpdateInternal(progress); + if (last_progress != progress) { + UpdateInternal(progress); + last_progress = progress; + } } void UnscentedKalmanFilter::Initialize(double initial_progress, double current_time) { + // If the initial progress is zero we can't yet initalize, since the filter + // wont' have a reasonable guess about query velocity, just wait until some + // progress has been made. + if (initial_progress == 0.0 || current_time == 0.0) { + return; + } + // If the initial progress value is very small, it needs to be scaled up so + // its the same relative magnitude of updates that was used to determine + // the P, Q, and R parameters of the Kalman filter. + // + // The settings for the Kalman filter make assumptions around the magnitude of + // measurement noise. If the progress updates are very small, the updates + // could be considered noise by the Kalman filter. + scale_factor = std::max(1.0, 0.1 / initial_progress); + initial_progress *= scale_factor; x[0] = initial_progress; - x[1] = current_time == 0 ? 0.01 : initial_progress / current_time; // initial velocity guess + x[1] = initial_progress / current_time; // initial velocity guess last_time = current_time; + last_progress = initial_progress; initialized = true; } @@ -233,11 +254,11 @@ void UnscentedKalmanFilter::UpdateInternal(double measured_progress) { } // Ensure progress stays in bounds - x[0] = std::max(0.0, std::min(1.0, x[0])); + x[0] = std::max(0.0, std::min(scale_factor, x[0])); } double UnscentedKalmanFilter::GetProgress() const { - return x[0]; + return x[0] / scale_factor; } double UnscentedKalmanFilter::GetVelocity() const { @@ -246,14 +267,14 @@ double UnscentedKalmanFilter::GetVelocity() const { double UnscentedKalmanFilter::GetEstimatedRemainingSeconds() const { if (!initialized) { - return -1.0; + return 2147483647.0; } if (x[1] <= 0) { // velocity is negative or zero - we estimate this will never finish return NumericLimits::Maximum(); } - double remaining_progress = 1.0 - x[0]; - return remaining_progress / x[1]; + double remaining_progress = (1.0 * scale_factor) - x[0]; + return std::max(remaining_progress / x[1], 0.0); } double UnscentedKalmanFilter::GetProgressVariance() const { diff --git a/src/common/render_tree.cpp b/src/common/render_tree.cpp index 582d5e1ad424..ee96218145c7 100644 --- a/src/common/render_tree.cpp +++ b/src/common/render_tree.cpp @@ -103,7 +103,7 @@ static unique_ptr CreateNode(const ProfilingNode &op) { auto &info = op.GetProfilingInfo(); InsertionOrderPreservingMap extra_info; if (info.Enabled(info.settings, MetricsType::EXTRA_INFO)) { - extra_info = op.GetProfilingInfo().extra_info; + extra_info = op.GetProfilingInfo().GetMetricValue>(MetricsType::EXTRA_INFO); } string node_name = "QUERY"; diff --git a/src/common/row_operations/CMakeLists.txt b/src/common/row_operations/CMakeLists.txt index b07cd84aa466..f534c5d0b147 100644 --- a/src/common/row_operations/CMakeLists.txt +++ b/src/common/row_operations/CMakeLists.txt @@ -1,14 +1,5 @@ -add_library_unity( - duckdb_row_operations - OBJECT - row_aggregate.cpp - row_scatter.cpp - row_gather.cpp - row_matcher.cpp - row_external.cpp - row_radix_scatter.cpp - row_heap_scatter.cpp - row_heap_gather.cpp) +add_library_unity(duckdb_row_operations OBJECT row_aggregate.cpp + row_matcher.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/common/row_operations/row_external.cpp b/src/common/row_operations/row_external.cpp deleted file mode 100644 index e4e3ec87d865..000000000000 --- a/src/common/row_operations/row_external.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/types/row/row_layout.hpp" - -namespace duckdb { - -using ValidityBytes = RowLayout::ValidityBytes; - -void RowOperations::SwizzleColumns(const RowLayout &layout, const data_ptr_t base_row_ptr, const idx_t count) { - const idx_t row_width = layout.GetRowWidth(); - data_ptr_t heap_row_ptrs[STANDARD_VECTOR_SIZE]; - idx_t done = 0; - while (done != count) { - const idx_t next = MinValue(count - done, STANDARD_VECTOR_SIZE); - const data_ptr_t row_ptr = base_row_ptr + done * row_width; - // Load heap row pointers - data_ptr_t heap_ptr_ptr = row_ptr + layout.GetHeapOffset(); - for (idx_t i = 0; i < next; i++) { - heap_row_ptrs[i] = Load(heap_ptr_ptr); - heap_ptr_ptr += row_width; - } - // Loop through the blob columns - for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) { - auto physical_type = layout.GetTypes()[col_idx].InternalType(); - if (TypeIsConstantSize(physical_type)) { - continue; - } - data_ptr_t col_ptr = row_ptr + layout.GetOffsets()[col_idx]; - if (physical_type == PhysicalType::VARCHAR) { - data_ptr_t string_ptr = col_ptr + string_t::HEADER_SIZE; - for (idx_t i = 0; i < next; i++) { - if (Load(col_ptr) > string_t::INLINE_LENGTH) { - // Overwrite the string pointer with the within-row offset (if not inlined) - Store(UnsafeNumericCast(Load(string_ptr) - heap_row_ptrs[i]), - string_ptr); - } - col_ptr += row_width; - string_ptr += row_width; - } - } else { - // Non-varchar blob columns - for (idx_t i = 0; i < next; i++) { - // Overwrite the column data pointer with the within-row offset - Store(UnsafeNumericCast(Load(col_ptr) - heap_row_ptrs[i]), col_ptr); - col_ptr += row_width; - } - } - } - done += next; - } -} - -void RowOperations::SwizzleHeapPointer(const RowLayout &layout, data_ptr_t row_ptr, const data_ptr_t heap_base_ptr, - const idx_t count, const idx_t base_offset) { - const idx_t row_width = layout.GetRowWidth(); - row_ptr += layout.GetHeapOffset(); - idx_t cumulative_offset = 0; - for (idx_t i = 0; i < count; i++) { - Store(base_offset + cumulative_offset, row_ptr); - cumulative_offset += Load(heap_base_ptr + cumulative_offset); - row_ptr += row_width; - } -} - -void RowOperations::CopyHeapAndSwizzle(const RowLayout &layout, data_ptr_t row_ptr, const data_ptr_t heap_base_ptr, - data_ptr_t heap_ptr, const idx_t count) { - const auto row_width = layout.GetRowWidth(); - const auto heap_offset = layout.GetHeapOffset(); - for (idx_t i = 0; i < count; i++) { - // Figure out source and size - const auto source_heap_ptr = Load(row_ptr + heap_offset); - const auto size = Load(source_heap_ptr); - D_ASSERT(size >= sizeof(uint32_t)); - - // Copy and swizzle - memcpy(heap_ptr, source_heap_ptr, size); - Store(UnsafeNumericCast(heap_ptr - heap_base_ptr), row_ptr + heap_offset); - - // Increment for next iteration - row_ptr += row_width; - heap_ptr += size; - } -} - -void RowOperations::UnswizzleHeapPointer(const RowLayout &layout, const data_ptr_t base_row_ptr, - const data_ptr_t base_heap_ptr, const idx_t count) { - const auto row_width = layout.GetRowWidth(); - data_ptr_t heap_ptr_ptr = base_row_ptr + layout.GetHeapOffset(); - for (idx_t i = 0; i < count; i++) { - Store(base_heap_ptr + Load(heap_ptr_ptr), heap_ptr_ptr); - heap_ptr_ptr += row_width; - } -} - -static inline void VerifyUnswizzledString(const RowLayout &layout, const idx_t &col_idx, const data_ptr_t &row_ptr) { -#ifdef DEBUG - if (layout.GetTypes()[col_idx].id() != LogicalTypeId::VARCHAR) { - return; - } - idx_t entry_idx; - idx_t idx_in_entry; - ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry); - - ValidityBytes row_mask(row_ptr, layout.ColumnCount()); - if (row_mask.RowIsValid(row_mask.GetValidityEntry(entry_idx), idx_in_entry)) { - auto str = Load(row_ptr + layout.GetOffsets()[col_idx]); - str.Verify(); - } -#endif -} - -void RowOperations::UnswizzlePointers(const RowLayout &layout, const data_ptr_t base_row_ptr, - const data_ptr_t base_heap_ptr, const idx_t count) { - const idx_t row_width = layout.GetRowWidth(); - data_ptr_t heap_row_ptrs[STANDARD_VECTOR_SIZE]; - idx_t done = 0; - while (done != count) { - const idx_t next = MinValue(count - done, STANDARD_VECTOR_SIZE); - const data_ptr_t row_ptr = base_row_ptr + done * row_width; - // Restore heap row pointers - data_ptr_t heap_ptr_ptr = row_ptr + layout.GetHeapOffset(); - for (idx_t i = 0; i < next; i++) { - heap_row_ptrs[i] = base_heap_ptr + Load(heap_ptr_ptr); - Store(heap_row_ptrs[i], heap_ptr_ptr); - heap_ptr_ptr += row_width; - } - // Loop through the blob columns - for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) { - auto physical_type = layout.GetTypes()[col_idx].InternalType(); - if (TypeIsConstantSize(physical_type)) { - continue; - } - data_ptr_t col_ptr = row_ptr + layout.GetOffsets()[col_idx]; - if (physical_type == PhysicalType::VARCHAR) { - data_ptr_t string_ptr = col_ptr + string_t::HEADER_SIZE; - for (idx_t i = 0; i < next; i++) { - if (Load(col_ptr) > string_t::INLINE_LENGTH) { - // Overwrite the string offset with the pointer (if not inlined) - Store(heap_row_ptrs[i] + Load(string_ptr), string_ptr); - VerifyUnswizzledString(layout, col_idx, row_ptr + i * row_width); - } - col_ptr += row_width; - string_ptr += row_width; - } - } else { - // Non-varchar blob columns - for (idx_t i = 0; i < next; i++) { - // Overwrite the column data offset with the pointer - Store(heap_row_ptrs[i] + Load(col_ptr), col_ptr); - col_ptr += row_width; - } - } - } - done += next; - } -} - -} // namespace duckdb diff --git a/src/common/row_operations/row_gather.cpp b/src/common/row_operations/row_gather.cpp deleted file mode 100644 index 8e5ed315b924..000000000000 --- a/src/common/row_operations/row_gather.cpp +++ /dev/null @@ -1,176 +0,0 @@ -#include "duckdb/common/exception.hpp" -#include "duckdb/common/operator/constant_operators.hpp" -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/types/row/row_data_collection.hpp" -#include "duckdb/common/types/row/row_layout.hpp" -#include "duckdb/common/types/row/tuple_data_layout.hpp" -#include "duckdb/common/uhugeint.hpp" - -namespace duckdb { - -using ValidityBytes = RowLayout::ValidityBytes; - -template -static void TemplatedGatherLoop(Vector &rows, const SelectionVector &row_sel, Vector &col, - const SelectionVector &col_sel, idx_t count, const RowLayout &layout, idx_t col_no, - idx_t build_size) { - // Precompute mask indexes - const auto &offsets = layout.GetOffsets(); - const auto col_offset = offsets[col_no]; - idx_t entry_idx; - idx_t idx_in_entry; - ValidityBytes::GetEntryIndex(col_no, entry_idx, idx_in_entry); - - auto ptrs = FlatVector::GetData(rows); - auto data = FlatVector::GetData(col); - auto &col_mask = FlatVector::Validity(col); - - for (idx_t i = 0; i < count; i++) { - auto row_idx = row_sel.get_index(i); - auto row = ptrs[row_idx]; - auto col_idx = col_sel.get_index(i); - data[col_idx] = Load(row + col_offset); - ValidityBytes row_mask(row, layout.ColumnCount()); - if (!row_mask.RowIsValid(row_mask.GetValidityEntry(entry_idx), idx_in_entry)) { - if (build_size > STANDARD_VECTOR_SIZE && col_mask.AllValid()) { - //! We need to initialize the mask with the vector size. - col_mask.Initialize(build_size); - } - col_mask.SetInvalid(col_idx); - } - } -} - -static void GatherVarchar(Vector &rows, const SelectionVector &row_sel, Vector &col, const SelectionVector &col_sel, - idx_t count, const RowLayout &layout, idx_t col_no, idx_t build_size, - data_ptr_t base_heap_ptr) { - // Precompute mask indexes - const auto &offsets = layout.GetOffsets(); - const auto col_offset = offsets[col_no]; - const auto heap_offset = layout.GetHeapOffset(); - idx_t entry_idx; - idx_t idx_in_entry; - ValidityBytes::GetEntryIndex(col_no, entry_idx, idx_in_entry); - - auto ptrs = FlatVector::GetData(rows); - auto data = FlatVector::GetData(col); - auto &col_mask = FlatVector::Validity(col); - - for (idx_t i = 0; i < count; i++) { - auto row_idx = row_sel.get_index(i); - auto row = ptrs[row_idx]; - auto col_idx = col_sel.get_index(i); - auto col_ptr = row + col_offset; - data[col_idx] = Load(col_ptr); - ValidityBytes row_mask(row, layout.ColumnCount()); - if (!row_mask.RowIsValid(row_mask.GetValidityEntry(entry_idx), idx_in_entry)) { - if (build_size > STANDARD_VECTOR_SIZE && col_mask.AllValid()) { - //! We need to initialize the mask with the vector size. - col_mask.Initialize(build_size); - } - col_mask.SetInvalid(col_idx); - } else if (base_heap_ptr && Load(col_ptr) > string_t::INLINE_LENGTH) { - // Not inline, so unswizzle the copied pointer the pointer - auto heap_ptr_ptr = row + heap_offset; - auto heap_row_ptr = base_heap_ptr + Load(heap_ptr_ptr); - auto string_ptr = data_ptr_t(data + col_idx) + string_t::HEADER_SIZE; - Store(heap_row_ptr + Load(string_ptr), string_ptr); -#ifdef DEBUG - data[col_idx].Verify(); -#endif - } - } -} - -static void GatherNestedVector(Vector &rows, const SelectionVector &row_sel, Vector &col, - const SelectionVector &col_sel, idx_t count, const RowLayout &layout, idx_t col_no, - data_ptr_t base_heap_ptr) { - const auto &offsets = layout.GetOffsets(); - const auto col_offset = offsets[col_no]; - const auto heap_offset = layout.GetHeapOffset(); - auto ptrs = FlatVector::GetData(rows); - - // Build the gather locations - auto data_locations = make_unsafe_uniq_array_uninitialized(count); - auto mask_locations = make_unsafe_uniq_array_uninitialized(count); - for (idx_t i = 0; i < count; i++) { - auto row_idx = row_sel.get_index(i); - auto row = ptrs[row_idx]; - mask_locations[i] = row; - auto col_ptr = ptrs[row_idx] + col_offset; - if (base_heap_ptr) { - auto heap_ptr_ptr = row + heap_offset; - auto heap_row_ptr = base_heap_ptr + Load(heap_ptr_ptr); - data_locations[i] = heap_row_ptr + Load(col_ptr); - } else { - data_locations[i] = Load(col_ptr); - } - } - - // Deserialise into the selected locations - NestedValidity parent_validity(mask_locations.get(), col_no); - RowOperations::HeapGather(col, count, col_sel, data_locations.get(), &parent_validity); -} - -void RowOperations::Gather(Vector &rows, const SelectionVector &row_sel, Vector &col, const SelectionVector &col_sel, - const idx_t count, const RowLayout &layout, const idx_t col_no, const idx_t build_size, - data_ptr_t heap_ptr) { - D_ASSERT(rows.GetVectorType() == VectorType::FLAT_VECTOR); - D_ASSERT(rows.GetType().id() == LogicalTypeId::POINTER); // "Cannot gather from non-pointer type!" - - col.SetVectorType(VectorType::FLAT_VECTOR); - switch (col.GetType().InternalType()) { - case PhysicalType::UINT8: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::UINT16: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::UINT32: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::UINT64: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::UINT128: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::BOOL: - case PhysicalType::INT8: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::INT16: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::INT32: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::INT64: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::INT128: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::FLOAT: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::DOUBLE: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::INTERVAL: - TemplatedGatherLoop(rows, row_sel, col, col_sel, count, layout, col_no, build_size); - break; - case PhysicalType::VARCHAR: - GatherVarchar(rows, row_sel, col, col_sel, count, layout, col_no, build_size, heap_ptr); - break; - case PhysicalType::LIST: - case PhysicalType::STRUCT: - case PhysicalType::ARRAY: - GatherNestedVector(rows, row_sel, col, col_sel, count, layout, col_no, heap_ptr); - break; - default: - throw InternalException("Unimplemented type for RowOperations::Gather"); - } -} - -} // namespace duckdb diff --git a/src/common/row_operations/row_heap_gather.cpp b/src/common/row_operations/row_heap_gather.cpp deleted file mode 100644 index fa433c64e120..000000000000 --- a/src/common/row_operations/row_heap_gather.cpp +++ /dev/null @@ -1,276 +0,0 @@ -#include "duckdb/common/helper.hpp" -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/types/vector.hpp" - -namespace duckdb { - -using ValidityBytes = TemplatedValidityMask; - -template -static void TemplatedHeapGather(Vector &v, const idx_t count, const SelectionVector &sel, data_ptr_t *key_locations) { - auto target = FlatVector::GetData(v); - - for (idx_t i = 0; i < count; ++i) { - const auto col_idx = sel.get_index(i); - target[col_idx] = Load(key_locations[i]); - key_locations[i] += sizeof(T); - } -} - -static void HeapGatherStringVector(Vector &v, const idx_t vcount, const SelectionVector &sel, - data_ptr_t *key_locations) { - const auto &validity = FlatVector::Validity(v); - auto target = FlatVector::GetData(v); - - for (idx_t i = 0; i < vcount; i++) { - const auto col_idx = sel.get_index(i); - if (!validity.RowIsValid(col_idx)) { - continue; - } - auto len = Load(key_locations[i]); - key_locations[i] += sizeof(uint32_t); - target[col_idx] = StringVector::AddStringOrBlob(v, string_t(const_char_ptr_cast(key_locations[i]), len)); - key_locations[i] += len; - } -} - -static void HeapGatherStructVector(Vector &v, const idx_t vcount, const SelectionVector &sel, - data_ptr_t *key_locations) { - // struct must have a validitymask for its fields - auto &child_types = StructType::GetChildTypes(v.GetType()); - const idx_t struct_validitymask_size = (child_types.size() + 7) / 8; - data_ptr_t struct_validitymask_locations[STANDARD_VECTOR_SIZE]; - for (idx_t i = 0; i < vcount; i++) { - // use key_locations as the validitymask, and create struct_key_locations - struct_validitymask_locations[i] = key_locations[i]; - key_locations[i] += struct_validitymask_size; - } - - // now deserialize into the struct vectors - auto &children = StructVector::GetEntries(v); - for (idx_t i = 0; i < child_types.size(); i++) { - NestedValidity parent_validity(struct_validitymask_locations, i); - RowOperations::HeapGather(*children[i], vcount, sel, key_locations, &parent_validity); - } -} - -static void HeapGatherListVector(Vector &v, const idx_t vcount, const SelectionVector &sel, data_ptr_t *key_locations) { - const auto &validity = FlatVector::Validity(v); - - auto child_type = ListType::GetChildType(v.GetType()); - auto list_data = ListVector::GetData(v); - data_ptr_t list_entry_locations[STANDARD_VECTOR_SIZE]; - - uint64_t entry_offset = ListVector::GetListSize(v); - for (idx_t i = 0; i < vcount; i++) { - const auto col_idx = sel.get_index(i); - if (!validity.RowIsValid(col_idx)) { - continue; - } - // read list length - auto entry_remaining = Load(key_locations[i]); - key_locations[i] += sizeof(uint64_t); - // set list entry attributes - list_data[col_idx].length = entry_remaining; - list_data[col_idx].offset = entry_offset; - // skip over the validity mask - data_ptr_t validitymask_location = key_locations[i]; - idx_t offset_in_byte = 0; - key_locations[i] += (entry_remaining + 7) / 8; - // entry sizes - data_ptr_t var_entry_size_ptr = nullptr; - if (!TypeIsConstantSize(child_type.InternalType())) { - var_entry_size_ptr = key_locations[i]; - key_locations[i] += entry_remaining * sizeof(idx_t); - } - - // now read the list data - while (entry_remaining > 0) { - auto next = MinValue(entry_remaining, (idx_t)STANDARD_VECTOR_SIZE); - - // initialize a new vector to append - Vector append_vector(v.GetType()); - append_vector.SetVectorType(v.GetVectorType()); - - auto &list_vec_to_append = ListVector::GetEntry(append_vector); - - // set validity - //! Since we are constructing the vector, this will always be a flat vector. - auto &append_validity = FlatVector::Validity(list_vec_to_append); - for (idx_t entry_idx = 0; entry_idx < next; entry_idx++) { - append_validity.Set(entry_idx, *(validitymask_location) & (1 << offset_in_byte)); - if (++offset_in_byte == 8) { - validitymask_location++; - offset_in_byte = 0; - } - } - - // compute entry sizes and set locations where the list entries are - if (TypeIsConstantSize(child_type.InternalType())) { - // constant size list entries - const idx_t type_size = GetTypeIdSize(child_type.InternalType()); - for (idx_t entry_idx = 0; entry_idx < next; entry_idx++) { - list_entry_locations[entry_idx] = key_locations[i]; - key_locations[i] += type_size; - } - } else { - // variable size list entries - for (idx_t entry_idx = 0; entry_idx < next; entry_idx++) { - list_entry_locations[entry_idx] = key_locations[i]; - key_locations[i] += Load(var_entry_size_ptr); - var_entry_size_ptr += sizeof(idx_t); - } - } - - // now deserialize and add to listvector - RowOperations::HeapGather(list_vec_to_append, next, *FlatVector::IncrementalSelectionVector(), - list_entry_locations, nullptr); - ListVector::Append(v, list_vec_to_append, next); - - // update for next iteration - entry_remaining -= next; - entry_offset += next; - } - } -} - -static void HeapGatherArrayVector(Vector &v, const idx_t vcount, const SelectionVector &sel, - data_ptr_t *key_locations) { - // Setup - auto &child_type = ArrayType::GetChildType(v.GetType()); - auto array_size = ArrayType::GetSize(v.GetType()); - auto &child_vector = ArrayVector::GetEntry(v); - auto child_type_size = GetTypeIdSize(child_type.InternalType()); - auto child_type_is_var_size = !TypeIsConstantSize(child_type.InternalType()); - - data_ptr_t array_entry_locations[STANDARD_VECTOR_SIZE]; - - // array must have a validitymask for its elements - auto array_validitymask_size = (array_size + 7) / 8; - - for (idx_t i = 0; i < vcount; i++) { - // Setup validity mask - data_ptr_t array_validitymask_location = key_locations[i]; - key_locations[i] += array_validitymask_size; - - NestedValidity parent_validity(array_validitymask_location); - - // The size of each variable size entry is stored after the validity mask - // (if the child type is variable size) - data_ptr_t var_entry_size_ptr = nullptr; - if (child_type_is_var_size) { - var_entry_size_ptr = key_locations[i]; - key_locations[i] += array_size * sizeof(idx_t); - } - - // row idx - const auto row_idx = sel.get_index(i); - - idx_t array_start = row_idx * array_size; - idx_t elem_remaining = array_size; - - while (elem_remaining > 0) { - auto chunk_size = MinValue(static_cast(STANDARD_VECTOR_SIZE), elem_remaining); - - SelectionVector array_sel(STANDARD_VECTOR_SIZE); - - if (child_type_is_var_size) { - // variable size list entries - for (idx_t elem_idx = 0; elem_idx < chunk_size; elem_idx++) { - array_entry_locations[elem_idx] = key_locations[i]; - key_locations[i] += Load(var_entry_size_ptr); - var_entry_size_ptr += sizeof(idx_t); - array_sel.set_index(elem_idx, array_start + elem_idx); - } - } else { - // constant size list entries - for (idx_t elem_idx = 0; elem_idx < chunk_size; elem_idx++) { - array_entry_locations[elem_idx] = key_locations[i]; - key_locations[i] += child_type_size; - array_sel.set_index(elem_idx, array_start + elem_idx); - } - } - - // Pass on this array's validity mask to the child vector - RowOperations::HeapGather(child_vector, chunk_size, array_sel, array_entry_locations, &parent_validity); - - elem_remaining -= chunk_size; - array_start += chunk_size; - parent_validity.OffsetListBy(chunk_size); - } - } -} - -void RowOperations::HeapGather(Vector &v, const idx_t &vcount, const SelectionVector &sel, data_ptr_t *key_locations, - optional_ptr parent_validity) { - v.SetVectorType(VectorType::FLAT_VECTOR); - - auto &validity = FlatVector::Validity(v); - if (parent_validity) { - for (idx_t i = 0; i < vcount; i++) { - const auto valid = parent_validity->IsValid(i); - const auto col_idx = sel.get_index(i); - validity.Set(col_idx, valid); - } - } - - auto type = v.GetType().InternalType(); - switch (type) { - case PhysicalType::BOOL: - case PhysicalType::INT8: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::INT16: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::INT32: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::INT64: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::UINT8: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::UINT16: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::UINT32: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::UINT64: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::INT128: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::UINT128: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::FLOAT: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::DOUBLE: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::INTERVAL: - TemplatedHeapGather(v, vcount, sel, key_locations); - break; - case PhysicalType::VARCHAR: - HeapGatherStringVector(v, vcount, sel, key_locations); - break; - case PhysicalType::STRUCT: - HeapGatherStructVector(v, vcount, sel, key_locations); - break; - case PhysicalType::LIST: - HeapGatherListVector(v, vcount, sel, key_locations); - break; - case PhysicalType::ARRAY: - HeapGatherArrayVector(v, vcount, sel, key_locations); - break; - default: - throw NotImplementedException("Unimplemented deserialize from row-format"); - } -} - -} // namespace duckdb diff --git a/src/common/row_operations/row_heap_scatter.cpp b/src/common/row_operations/row_heap_scatter.cpp deleted file mode 100644 index 01cf7b5897ea..000000000000 --- a/src/common/row_operations/row_heap_scatter.cpp +++ /dev/null @@ -1,581 +0,0 @@ -#include "duckdb/common/helper.hpp" -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/types/vector.hpp" -#include "duckdb/common/uhugeint.hpp" - -namespace duckdb { - -using ValidityBytes = TemplatedValidityMask; - -NestedValidity::NestedValidity(data_ptr_t validitymask_location) - : list_validity_location(validitymask_location), struct_validity_locations(nullptr), entry_idx(0), idx_in_entry(0), - list_validity_offset(0) { -} - -NestedValidity::NestedValidity(data_ptr_t *validitymask_locations, idx_t child_vector_index) - : list_validity_location(nullptr), struct_validity_locations(validitymask_locations), entry_idx(0), idx_in_entry(0), - list_validity_offset(0) { - ValidityBytes::GetEntryIndex(child_vector_index, entry_idx, idx_in_entry); -} - -void NestedValidity::SetInvalid(idx_t idx) { - if (list_validity_location) { - // Is List - - idx = idx + list_validity_offset; - - idx_t list_entry_idx; - idx_t list_idx_in_entry; - ValidityBytes::GetEntryIndex(idx, list_entry_idx, list_idx_in_entry); - const auto bit = ~(1UL << list_idx_in_entry); - list_validity_location[list_entry_idx] &= bit; - } else { - // Is Struct - const auto bit = ~(1UL << idx_in_entry); - *(struct_validity_locations[idx] + entry_idx) &= bit; - } -} - -void NestedValidity::OffsetListBy(idx_t offset) { - list_validity_offset += offset; -} - -bool NestedValidity::IsValid(idx_t idx) { - if (list_validity_location) { - // Is List - - idx = idx + list_validity_offset; - - idx_t list_entry_idx; - idx_t list_idx_in_entry; - ValidityBytes::GetEntryIndex(idx, list_entry_idx, list_idx_in_entry); - const auto bit = (1UL << list_idx_in_entry); - return list_validity_location[list_entry_idx] & bit; - } else { - // Is Struct - const auto bit = (1UL << idx_in_entry); - return *(struct_validity_locations[idx] + entry_idx) & bit; - } -} - -static void ComputeStringEntrySizes(UnifiedVectorFormat &vdata, idx_t entry_sizes[], const idx_t ser_count, - const SelectionVector &sel, const idx_t offset) { - auto strings = UnifiedVectorFormat::GetData(vdata); - for (idx_t i = 0; i < ser_count; i++) { - auto idx = sel.get_index(i); - auto str_idx = vdata.sel->get_index(idx + offset); - if (vdata.validity.RowIsValid(str_idx)) { - entry_sizes[i] += sizeof(uint32_t) + strings[str_idx].GetSize(); - } - } -} - -static void ComputeStructEntrySizes(Vector &v, idx_t entry_sizes[], idx_t vcount, idx_t ser_count, - const SelectionVector &sel, idx_t offset) { - // obtain child vectors - idx_t num_children; - auto &children = StructVector::GetEntries(v); - num_children = children.size(); - // add struct validitymask size - const idx_t struct_validitymask_size = (num_children + 7) / 8; - for (idx_t i = 0; i < ser_count; i++) { - entry_sizes[i] += struct_validitymask_size; - } - // compute size of child vectors - for (auto &struct_vector : children) { - RowOperations::ComputeEntrySizes(*struct_vector, entry_sizes, vcount, ser_count, sel, offset); - } -} - -static void ComputeListEntrySizes(Vector &v, UnifiedVectorFormat &vdata, idx_t entry_sizes[], idx_t ser_count, - const SelectionVector &sel, idx_t offset) { - auto list_data = ListVector::GetData(v); - auto &child_vector = ListVector::GetEntry(v); - idx_t list_entry_sizes[STANDARD_VECTOR_SIZE]; - for (idx_t i = 0; i < ser_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx + offset); - if (vdata.validity.RowIsValid(source_idx)) { - auto list_entry = list_data[source_idx]; - - // make room for list length, list validitymask - entry_sizes[i] += sizeof(list_entry.length); - entry_sizes[i] += (list_entry.length + 7) / 8; - - // serialize size of each entry (if non-constant size) - if (!TypeIsConstantSize(ListType::GetChildType(v.GetType()).InternalType())) { - entry_sizes[i] += list_entry.length * sizeof(list_entry.length); - } - - // compute size of each the elements in list_entry and sum them - auto entry_remaining = list_entry.length; - auto entry_offset = list_entry.offset; - while (entry_remaining > 0) { - // the list entry can span multiple vectors - auto next = MinValue((idx_t)STANDARD_VECTOR_SIZE, entry_remaining); - - // compute and add to the total - std::fill_n(list_entry_sizes, next, 0); - RowOperations::ComputeEntrySizes(child_vector, list_entry_sizes, next, next, - *FlatVector::IncrementalSelectionVector(), entry_offset); - for (idx_t list_idx = 0; list_idx < next; list_idx++) { - entry_sizes[i] += list_entry_sizes[list_idx]; - } - - // update for next iteration - entry_remaining -= next; - entry_offset += next; - } - } - } -} - -static void ComputeArrayEntrySizes(Vector &v, UnifiedVectorFormat &vdata, idx_t entry_sizes[], idx_t ser_count, - const SelectionVector &sel, idx_t offset) { - - auto array_size = ArrayType::GetSize(v.GetType()); - auto child_vector = ArrayVector::GetEntry(v); - - idx_t array_entry_sizes[STANDARD_VECTOR_SIZE]; - const idx_t array_validitymask_size = (array_size + 7) / 8; - - for (idx_t i = 0; i < ser_count; i++) { - - // Validity for the array elements - entry_sizes[i] += array_validitymask_size; - - // serialize size of each entry (if non-constant size) - if (!TypeIsConstantSize(ArrayType::GetChildType(v.GetType()).InternalType())) { - entry_sizes[i] += array_size * sizeof(idx_t); - } - - auto elem_idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(elem_idx + offset); - - auto array_start = source_idx * array_size; - auto elem_remaining = array_size; - - // the array could span multiple vectors, so we divide it into chunks - while (elem_remaining > 0) { - auto chunk_size = MinValue(static_cast(STANDARD_VECTOR_SIZE), elem_remaining); - - // compute and add to the total - std::fill_n(array_entry_sizes, chunk_size, 0); - RowOperations::ComputeEntrySizes(child_vector, array_entry_sizes, chunk_size, chunk_size, - *FlatVector::IncrementalSelectionVector(), array_start); - for (idx_t arr_elem_idx = 0; arr_elem_idx < chunk_size; arr_elem_idx++) { - entry_sizes[i] += array_entry_sizes[arr_elem_idx]; - } - // update for next iteration - elem_remaining -= chunk_size; - array_start += chunk_size; - } - } -} - -void RowOperations::ComputeEntrySizes(Vector &v, UnifiedVectorFormat &vdata, idx_t entry_sizes[], idx_t vcount, - idx_t ser_count, const SelectionVector &sel, idx_t offset) { - const auto physical_type = v.GetType().InternalType(); - if (TypeIsConstantSize(physical_type)) { - const auto type_size = GetTypeIdSize(physical_type); - for (idx_t i = 0; i < ser_count; i++) { - entry_sizes[i] += type_size; - } - } else { - switch (physical_type) { - case PhysicalType::VARCHAR: - ComputeStringEntrySizes(vdata, entry_sizes, ser_count, sel, offset); - break; - case PhysicalType::STRUCT: - ComputeStructEntrySizes(v, entry_sizes, vcount, ser_count, sel, offset); - break; - case PhysicalType::LIST: - ComputeListEntrySizes(v, vdata, entry_sizes, ser_count, sel, offset); - break; - case PhysicalType::ARRAY: - ComputeArrayEntrySizes(v, vdata, entry_sizes, ser_count, sel, offset); - break; - default: - // LCOV_EXCL_START - throw NotImplementedException("Column with variable size type %s cannot be serialized to row-format", - v.GetType().ToString()); - // LCOV_EXCL_STOP - } - } -} - -void RowOperations::ComputeEntrySizes(Vector &v, idx_t entry_sizes[], idx_t vcount, idx_t ser_count, - const SelectionVector &sel, idx_t offset) { - UnifiedVectorFormat vdata; - v.ToUnifiedFormat(vcount, vdata); - ComputeEntrySizes(v, vdata, entry_sizes, vcount, ser_count, sel, offset); -} - -template -static void TemplatedHeapScatter(UnifiedVectorFormat &vdata, const SelectionVector &sel, idx_t count, - data_ptr_t *key_locations, optional_ptr parent_validity, - idx_t offset) { - auto source = UnifiedVectorFormat::GetData(vdata); - if (!parent_validity) { - for (idx_t i = 0; i < count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx + offset); - - auto target = (T *)key_locations[i]; - Store(source[source_idx], data_ptr_cast(target)); - key_locations[i] += sizeof(T); - } - } else { - for (idx_t i = 0; i < count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx + offset); - - auto target = (T *)key_locations[i]; - Store(source[source_idx], data_ptr_cast(target)); - key_locations[i] += sizeof(T); - - // set the validitymask - if (!vdata.validity.RowIsValid(source_idx)) { - parent_validity->SetInvalid(i); - } - } - } -} - -static void HeapScatterStringVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, - data_ptr_t *key_locations, optional_ptr parent_validity, - idx_t offset) { - UnifiedVectorFormat vdata; - v.ToUnifiedFormat(vcount, vdata); - - auto strings = UnifiedVectorFormat::GetData(vdata); - if (!parent_validity) { - for (idx_t i = 0; i < ser_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx + offset); - if (vdata.validity.RowIsValid(source_idx)) { - auto &string_entry = strings[source_idx]; - // store string size - Store(NumericCast(string_entry.GetSize()), key_locations[i]); - key_locations[i] += sizeof(uint32_t); - // store the string - memcpy(key_locations[i], string_entry.GetData(), string_entry.GetSize()); - key_locations[i] += string_entry.GetSize(); - } - } - } else { - for (idx_t i = 0; i < ser_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx + offset); - if (vdata.validity.RowIsValid(source_idx)) { - auto &string_entry = strings[source_idx]; - // store string size - Store(NumericCast(string_entry.GetSize()), key_locations[i]); - key_locations[i] += sizeof(uint32_t); - // store the string - memcpy(key_locations[i], string_entry.GetData(), string_entry.GetSize()); - key_locations[i] += string_entry.GetSize(); - } else { - // set the validitymask - parent_validity->SetInvalid(i); - } - } - } -} - -static void HeapScatterStructVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, - data_ptr_t *key_locations, optional_ptr parent_validity, - idx_t offset) { - UnifiedVectorFormat vdata; - v.ToUnifiedFormat(vcount, vdata); - - auto &children = StructVector::GetEntries(v); - idx_t num_children = children.size(); - - // struct must have a validitymask for its fields - const idx_t struct_validitymask_size = (num_children + 7) / 8; - data_ptr_t struct_validitymask_locations[STANDARD_VECTOR_SIZE]; - for (idx_t i = 0; i < ser_count; i++) { - // initialize the struct validity mask - struct_validitymask_locations[i] = key_locations[i]; - memset(struct_validitymask_locations[i], -1, struct_validitymask_size); - key_locations[i] += struct_validitymask_size; - - // set whether the whole struct is null - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - if (parent_validity && !vdata.validity.RowIsValid(source_idx)) { - parent_validity->SetInvalid(i); - } - } - - // now serialize the struct vectors - for (idx_t i = 0; i < children.size(); i++) { - auto &struct_vector = *children[i]; - NestedValidity struct_validity(struct_validitymask_locations, i); - RowOperations::HeapScatter(struct_vector, vcount, sel, ser_count, key_locations, &struct_validity, offset); - } -} - -static void HeapScatterListVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, - data_ptr_t *key_locations, optional_ptr parent_validity, - idx_t offset) { - UnifiedVectorFormat vdata; - v.ToUnifiedFormat(vcount, vdata); - - auto list_data = ListVector::GetData(v); - auto &child_vector = ListVector::GetEntry(v); - - UnifiedVectorFormat list_vdata; - child_vector.ToUnifiedFormat(ListVector::GetListSize(v), list_vdata); - auto child_type = ListType::GetChildType(v.GetType()).InternalType(); - - idx_t list_entry_sizes[STANDARD_VECTOR_SIZE]; - data_ptr_t list_entry_locations[STANDARD_VECTOR_SIZE]; - - for (idx_t i = 0; i < ser_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx + offset); - if (!vdata.validity.RowIsValid(source_idx)) { - if (parent_validity) { - // set the row validitymask for this column to invalid - parent_validity->SetInvalid(i); - } - continue; - } - auto list_entry = list_data[source_idx]; - - // store list length - Store(list_entry.length, key_locations[i]); - key_locations[i] += sizeof(list_entry.length); - - // make room for the validitymask - data_ptr_t list_validitymask_location = key_locations[i]; - idx_t entry_offset_in_byte = 0; - idx_t validitymask_size = (list_entry.length + 7) / 8; - memset(list_validitymask_location, -1, validitymask_size); - key_locations[i] += validitymask_size; - - // serialize size of each entry (if non-constant size) - data_ptr_t var_entry_size_ptr = nullptr; - if (!TypeIsConstantSize(child_type)) { - var_entry_size_ptr = key_locations[i]; - key_locations[i] += list_entry.length * sizeof(idx_t); - } - - auto entry_remaining = list_entry.length; - auto entry_offset = list_entry.offset; - while (entry_remaining > 0) { - // the list entry can span multiple vectors - auto next = MinValue((idx_t)STANDARD_VECTOR_SIZE, entry_remaining); - - // serialize list validity - for (idx_t entry_idx = 0; entry_idx < next; entry_idx++) { - auto list_idx = list_vdata.sel->get_index(entry_idx + entry_offset); - if (!list_vdata.validity.RowIsValid(list_idx)) { - *(list_validitymask_location) &= ~(1UL << entry_offset_in_byte); - } - if (++entry_offset_in_byte == 8) { - list_validitymask_location++; - entry_offset_in_byte = 0; - } - } - - if (TypeIsConstantSize(child_type)) { - // constant size list entries: set list entry locations - const idx_t type_size = GetTypeIdSize(child_type); - for (idx_t entry_idx = 0; entry_idx < next; entry_idx++) { - list_entry_locations[entry_idx] = key_locations[i]; - key_locations[i] += type_size; - } - } else { - // variable size list entries: compute entry sizes and set list entry locations - std::fill_n(list_entry_sizes, next, 0); - RowOperations::ComputeEntrySizes(child_vector, list_entry_sizes, next, next, - *FlatVector::IncrementalSelectionVector(), entry_offset); - for (idx_t entry_idx = 0; entry_idx < next; entry_idx++) { - list_entry_locations[entry_idx] = key_locations[i]; - key_locations[i] += list_entry_sizes[entry_idx]; - Store(list_entry_sizes[entry_idx], var_entry_size_ptr); - var_entry_size_ptr += sizeof(idx_t); - } - } - - // now serialize to the locations - RowOperations::HeapScatter(child_vector, ListVector::GetListSize(v), - *FlatVector::IncrementalSelectionVector(), next, list_entry_locations, nullptr, - entry_offset); - - // update for next iteration - entry_remaining -= next; - entry_offset += next; - } - } -} - -static void HeapScatterArrayVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, - data_ptr_t *key_locations, optional_ptr parent_validity, - idx_t offset) { - - auto &child_vector = ArrayVector::GetEntry(v); - auto array_size = ArrayType::GetSize(v.GetType()); - auto child_type = ArrayType::GetChildType(v.GetType()); - auto child_type_size = GetTypeIdSize(child_type.InternalType()); - auto child_type_is_var_size = !TypeIsConstantSize(child_type.InternalType()); - - UnifiedVectorFormat vdata; - v.ToUnifiedFormat(vcount, vdata); - - UnifiedVectorFormat child_vdata; - child_vector.ToUnifiedFormat(ArrayVector::GetTotalSize(v), child_vdata); - - data_ptr_t array_entry_locations[STANDARD_VECTOR_SIZE]; - idx_t array_entry_sizes[STANDARD_VECTOR_SIZE]; - - // array must have a validitymask for its elements - auto array_validitymask_size = (array_size + 7) / 8; - - for (idx_t i = 0; i < ser_count; i++) { - // Set if the whole array itself is null in the parent entry - auto source_idx = vdata.sel->get_index(sel.get_index(i) + offset); - if (parent_validity && !vdata.validity.RowIsValid(source_idx)) { - parent_validity->SetInvalid(i); - } - - // Now we can serialize the array itself - // Every array starts with a validity mask for the children - data_ptr_t array_validitymask_location = key_locations[i]; - memset(array_validitymask_location, -1, array_validitymask_size); - key_locations[i] += array_validitymask_size; - - NestedValidity array_parent_validity(array_validitymask_location); - - // If the array contains variable size entries, we reserve spaces for them here - data_ptr_t var_entry_size_ptr = nullptr; - if (child_type_is_var_size) { - var_entry_size_ptr = key_locations[i]; - key_locations[i] += array_size * sizeof(idx_t); - } - - // Then comes the elements - auto array_start = source_idx * array_size; - auto elem_remaining = array_size; - - while (elem_remaining > 0) { - // the array elements can span multiple vectors, so we divide it into chunks - auto chunk_size = MinValue(static_cast(STANDARD_VECTOR_SIZE), elem_remaining); - - // Setup the locations for the elements - if (child_type_is_var_size) { - // The elements are variable sized - std::fill_n(array_entry_sizes, chunk_size, 0); - RowOperations::ComputeEntrySizes(child_vector, array_entry_sizes, chunk_size, chunk_size, - *FlatVector::IncrementalSelectionVector(), array_start); - for (idx_t elem_idx = 0; elem_idx < chunk_size; elem_idx++) { - array_entry_locations[elem_idx] = key_locations[i]; - key_locations[i] += array_entry_sizes[elem_idx]; - - // Now store the size of the entry - Store(array_entry_sizes[elem_idx], var_entry_size_ptr); - var_entry_size_ptr += sizeof(idx_t); - } - } else { - // The elements are constant sized - for (idx_t elem_idx = 0; elem_idx < chunk_size; elem_idx++) { - array_entry_locations[elem_idx] = key_locations[i]; - key_locations[i] += child_type_size; - } - } - - RowOperations::HeapScatter(child_vector, ArrayVector::GetTotalSize(v), - *FlatVector::IncrementalSelectionVector(), chunk_size, array_entry_locations, - &array_parent_validity, array_start); - - // update for next iteration - elem_remaining -= chunk_size; - array_start += chunk_size; - array_parent_validity.OffsetListBy(chunk_size); - } - } -} - -void RowOperations::HeapScatter(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, - data_ptr_t *key_locations, optional_ptr parent_validity, idx_t offset) { - if (TypeIsConstantSize(v.GetType().InternalType())) { - UnifiedVectorFormat vdata; - v.ToUnifiedFormat(vcount, vdata); - RowOperations::HeapScatterVData(vdata, v.GetType().InternalType(), sel, ser_count, key_locations, - parent_validity, offset); - } else { - switch (v.GetType().InternalType()) { - case PhysicalType::VARCHAR: - HeapScatterStringVector(v, vcount, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::STRUCT: - HeapScatterStructVector(v, vcount, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::LIST: - HeapScatterListVector(v, vcount, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::ARRAY: - HeapScatterArrayVector(v, vcount, sel, ser_count, key_locations, parent_validity, offset); - break; - default: - // LCOV_EXCL_START - throw NotImplementedException("Serialization of variable length vector with type %s", - v.GetType().ToString()); - // LCOV_EXCL_STOP - } - } -} - -void RowOperations::HeapScatterVData(UnifiedVectorFormat &vdata, PhysicalType type, const SelectionVector &sel, - idx_t ser_count, data_ptr_t *key_locations, - optional_ptr parent_validity, idx_t offset) { - switch (type) { - case PhysicalType::BOOL: - case PhysicalType::INT8: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::INT16: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::INT32: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::INT64: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::UINT8: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::UINT16: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::UINT32: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::UINT64: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::INT128: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::UINT128: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::FLOAT: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::DOUBLE: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - case PhysicalType::INTERVAL: - TemplatedHeapScatter(vdata, sel, ser_count, key_locations, parent_validity, offset); - break; - default: - throw NotImplementedException("FIXME: Serialize to of constant type column to row-format"); - } -} - -} // namespace duckdb diff --git a/src/common/row_operations/row_radix_scatter.cpp b/src/common/row_operations/row_radix_scatter.cpp deleted file mode 100644 index a85a7199776e..000000000000 --- a/src/common/row_operations/row_radix_scatter.cpp +++ /dev/null @@ -1,360 +0,0 @@ -#include "duckdb/common/helper.hpp" -#include "duckdb/common/radix.hpp" -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/types/vector.hpp" -#include "duckdb/common/uhugeint.hpp" - -namespace duckdb { - -template -void TemplatedRadixScatter(UnifiedVectorFormat &vdata, const SelectionVector &sel, idx_t add_count, - data_ptr_t *key_locations, const bool desc, const bool has_null, const bool nulls_first, - const idx_t offset) { - auto source = UnifiedVectorFormat::GetData(vdata); - if (has_null) { - auto &validity = vdata.validity; - const data_t valid = nulls_first ? 1 : 0; - const data_t invalid = 1 - valid; - - for (idx_t i = 0; i < add_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - // write validity and according value - if (validity.RowIsValid(source_idx)) { - key_locations[i][0] = valid; - Radix::EncodeData(key_locations[i] + 1, source[source_idx]); - // invert bits if desc - if (desc) { - for (idx_t s = 1; s < sizeof(T) + 1; s++) { - *(key_locations[i] + s) = ~*(key_locations[i] + s); - } - } - } else { - key_locations[i][0] = invalid; - memset(key_locations[i] + 1, '\0', sizeof(T)); - } - key_locations[i] += sizeof(T) + 1; - } - } else { - for (idx_t i = 0; i < add_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - // write value - Radix::EncodeData(key_locations[i], source[source_idx]); - // invert bits if desc - if (desc) { - for (idx_t s = 0; s < sizeof(T); s++) { - *(key_locations[i] + s) = ~*(key_locations[i] + s); - } - } - key_locations[i] += sizeof(T); - } - } -} - -void RadixScatterStringVector(UnifiedVectorFormat &vdata, const SelectionVector &sel, idx_t add_count, - data_ptr_t *key_locations, const bool desc, const bool has_null, const bool nulls_first, - const idx_t prefix_len, idx_t offset) { - auto source = UnifiedVectorFormat::GetData(vdata); - if (has_null) { - auto &validity = vdata.validity; - const data_t valid = nulls_first ? 1 : 0; - const data_t invalid = 1 - valid; - - for (idx_t i = 0; i < add_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - // write validity and according value - if (validity.RowIsValid(source_idx)) { - key_locations[i][0] = valid; - Radix::EncodeStringDataPrefix(key_locations[i] + 1, source[source_idx], prefix_len); - // invert bits if desc - if (desc) { - for (idx_t s = 1; s < prefix_len + 1; s++) { - *(key_locations[i] + s) = ~*(key_locations[i] + s); - } - } - } else { - key_locations[i][0] = invalid; - memset(key_locations[i] + 1, '\0', prefix_len); - } - key_locations[i] += prefix_len + 1; - } - } else { - for (idx_t i = 0; i < add_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - // write value - Radix::EncodeStringDataPrefix(key_locations[i], source[source_idx], prefix_len); - // invert bits if desc - if (desc) { - for (idx_t s = 0; s < prefix_len; s++) { - *(key_locations[i] + s) = ~*(key_locations[i] + s); - } - } - key_locations[i] += prefix_len; - } - } -} - -void RadixScatterListVector(Vector &v, UnifiedVectorFormat &vdata, const SelectionVector &sel, idx_t add_count, - data_ptr_t *key_locations, const bool desc, const bool has_null, const bool nulls_first, - const idx_t prefix_len, const idx_t width, const idx_t offset) { - auto list_data = ListVector::GetData(v); - auto &child_vector = ListVector::GetEntry(v); - auto list_size = ListVector::GetListSize(v); - child_vector.Flatten(list_size); - - // serialize null values - if (has_null) { - auto &validity = vdata.validity; - const data_t valid = nulls_first ? 1 : 0; - const data_t invalid = 1 - valid; - - for (idx_t i = 0; i < add_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - data_ptr_t &key_location = key_locations[i]; - const data_ptr_t key_location_start = key_location; - // write validity and according value - if (validity.RowIsValid(source_idx)) { - *key_location++ = valid; - auto &list_entry = list_data[source_idx]; - if (list_entry.length > 0) { - // denote that the list is not empty with a 1 - *key_location++ = 1; - RowOperations::RadixScatter(child_vector, list_size, *FlatVector::IncrementalSelectionVector(), 1, - key_locations + i, false, true, false, prefix_len, width - 2, - list_entry.offset); - } else { - // denote that the list is empty with a 0 - *key_location++ = 0; - // mark rest of bits as empty - memset(key_location, '\0', width - 2); - key_location += width - 2; - } - // invert bits if desc - if (desc) { - // skip over validity byte, handled by nulls first/last - for (key_location = key_location_start + 1; key_location < key_location_start + width; - key_location++) { - *key_location = ~*key_location; - } - } - } else { - *key_location++ = invalid; - memset(key_location, '\0', width - 1); - key_location += width - 1; - } - D_ASSERT(key_location == key_location_start + width); - } - } else { - for (idx_t i = 0; i < add_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - auto &list_entry = list_data[source_idx]; - data_ptr_t &key_location = key_locations[i]; - const data_ptr_t key_location_start = key_location; - if (list_entry.length > 0) { - // denote that the list is not empty with a 1 - *key_location++ = 1; - RowOperations::RadixScatter(child_vector, list_size, *FlatVector::IncrementalSelectionVector(), 1, - key_locations + i, false, true, false, prefix_len, width - 1, - list_entry.offset); - } else { - // denote that the list is empty with a 0 - *key_location++ = 0; - // mark rest of bits as empty - memset(key_location, '\0', width - 1); - key_location += width - 1; - } - // invert bits if desc - if (desc) { - for (key_location = key_location_start; key_location < key_location_start + width; key_location++) { - *key_location = ~*key_location; - } - } - D_ASSERT(key_location == key_location_start + width); - } - } -} - -void RadixScatterArrayVector(Vector &v, UnifiedVectorFormat &vdata, idx_t vcount, const SelectionVector &sel, - idx_t add_count, data_ptr_t *key_locations, const bool desc, const bool has_null, - const bool nulls_first, const idx_t prefix_len, idx_t width, const idx_t offset) { - auto &child_vector = ArrayVector::GetEntry(v); - auto array_size = ArrayType::GetSize(v.GetType()); - - if (has_null) { - auto &validity = vdata.validity; - const data_t valid = nulls_first ? 1 : 0; - const data_t invalid = 1 - valid; - - for (idx_t i = 0; i < add_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - data_ptr_t &key_location = key_locations[i]; - const data_ptr_t key_location_start = key_location; - - if (validity.RowIsValid(source_idx)) { - *key_location++ = valid; - - auto array_offset = source_idx * array_size; - RowOperations::RadixScatter(child_vector, array_size, *FlatVector::IncrementalSelectionVector(), 1, - key_locations + i, false, true, false, prefix_len, width - 1, array_offset); - - // invert bits if desc - if (desc) { - // skip over validity byte, handled by nulls first/last - for (key_location = key_location_start + 1; key_location < key_location_start + width; - key_location++) { - *key_location = ~*key_location; - } - } - } else { - *key_location++ = invalid; - memset(key_location, '\0', width - 1); - key_location += width - 1; - } - D_ASSERT(key_location == key_location_start + width); - } - } else { - for (idx_t i = 0; i < add_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - data_ptr_t &key_location = key_locations[i]; - const data_ptr_t key_location_start = key_location; - - auto array_offset = source_idx * array_size; - RowOperations::RadixScatter(child_vector, array_size, *FlatVector::IncrementalSelectionVector(), 1, - key_locations + i, false, true, false, prefix_len, width, array_offset); - // invert bits if desc - if (desc) { - for (key_location = key_location_start; key_location < key_location_start + width; key_location++) { - *key_location = ~*key_location; - } - } - D_ASSERT(key_location == key_location_start + width); - } - } -} - -void RadixScatterStructVector(Vector &v, UnifiedVectorFormat &vdata, idx_t vcount, const SelectionVector &sel, - idx_t add_count, data_ptr_t *key_locations, const bool desc, const bool has_null, - const bool nulls_first, const idx_t prefix_len, idx_t width, const idx_t offset) { - // serialize null values - if (has_null) { - auto &validity = vdata.validity; - const data_t valid = nulls_first ? 1 : 0; - const data_t invalid = 1 - valid; - - for (idx_t i = 0; i < add_count; i++) { - auto idx = sel.get_index(i); - auto source_idx = vdata.sel->get_index(idx) + offset; - - // write validity and according value - if (validity.RowIsValid(source_idx)) { - key_locations[i][0] = valid; - } else { - key_locations[i][0] = invalid; - memset(key_locations[i] + 1, '\0', width - 1); - } - key_locations[i]++; - } - width--; - } - // serialize the struct - auto &child_vector = *StructVector::GetEntries(v)[0]; - RowOperations::RadixScatter(child_vector, vcount, *FlatVector::IncrementalSelectionVector(), add_count, - key_locations, false, true, false, prefix_len, width, offset); - // invert bits if desc - if (desc) { - for (idx_t i = 0; i < add_count; i++) { - for (idx_t s = 0; s < width; s++) { - *(key_locations[i] - width + s) = ~*(key_locations[i] - width + s); - } - } - } -} - -void RowOperations::RadixScatter(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, - data_ptr_t *key_locations, bool desc, bool has_null, bool nulls_first, - idx_t prefix_len, idx_t width, idx_t offset) { -#ifdef DEBUG - // initialize to verify written width later - auto key_locations_copy = make_uniq_array(ser_count); - for (idx_t i = 0; i < ser_count; i++) { - key_locations_copy[i] = key_locations[i]; - } -#endif - - UnifiedVectorFormat vdata; - v.ToUnifiedFormat(vcount, vdata); - switch (v.GetType().InternalType()) { - case PhysicalType::BOOL: - case PhysicalType::INT8: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::INT16: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::INT32: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::INT64: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::UINT8: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::UINT16: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::UINT32: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::UINT64: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::INT128: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::UINT128: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::FLOAT: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::DOUBLE: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::INTERVAL: - TemplatedRadixScatter(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset); - break; - case PhysicalType::VARCHAR: - RadixScatterStringVector(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, prefix_len, offset); - break; - case PhysicalType::LIST: - RadixScatterListVector(v, vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, prefix_len, width, - offset); - break; - case PhysicalType::STRUCT: - RadixScatterStructVector(v, vdata, vcount, sel, ser_count, key_locations, desc, has_null, nulls_first, - prefix_len, width, offset); - break; - case PhysicalType::ARRAY: - RadixScatterArrayVector(v, vdata, vcount, sel, ser_count, key_locations, desc, has_null, nulls_first, - prefix_len, width, offset); - break; - default: - throw NotImplementedException("Cannot ORDER BY column with type %s", v.GetType().ToString()); - } - -#ifdef DEBUG - for (idx_t i = 0; i < ser_count; i++) { - D_ASSERT(key_locations[i] == key_locations_copy[i] + width); - } -#endif -} - -} // namespace duckdb diff --git a/src/common/row_operations/row_scatter.cpp b/src/common/row_operations/row_scatter.cpp deleted file mode 100644 index 1912d248474a..000000000000 --- a/src/common/row_operations/row_scatter.cpp +++ /dev/null @@ -1,230 +0,0 @@ -#include "duckdb/common/exception.hpp" -#include "duckdb/common/helper.hpp" -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/types/null_value.hpp" -#include "duckdb/common/types/row/row_data_collection.hpp" -#include "duckdb/common/types/row/row_layout.hpp" -#include "duckdb/common/types/selection_vector.hpp" -#include "duckdb/common/types/vector.hpp" -#include "duckdb/common/uhugeint.hpp" - -namespace duckdb { - -using ValidityBytes = RowLayout::ValidityBytes; - -template -static void TemplatedScatter(UnifiedVectorFormat &col, Vector &rows, const SelectionVector &sel, const idx_t count, - const idx_t col_offset, const idx_t col_no, const idx_t col_count) { - auto data = UnifiedVectorFormat::GetData(col); - auto ptrs = FlatVector::GetData(rows); - - if (!col.validity.AllValid()) { - for (idx_t i = 0; i < count; i++) { - auto idx = sel.get_index(i); - auto col_idx = col.sel->get_index(idx); - auto row = ptrs[idx]; - - auto isnull = !col.validity.RowIsValid(col_idx); - T store_value = isnull ? NullValue() : data[col_idx]; - Store(store_value, row + col_offset); - if (isnull) { - ValidityBytes col_mask(ptrs[idx], col_count); - col_mask.SetInvalidUnsafe(col_no); - } - } - } else { - for (idx_t i = 0; i < count; i++) { - auto idx = sel.get_index(i); - auto col_idx = col.sel->get_index(idx); - auto row = ptrs[idx]; - - Store(data[col_idx], row + col_offset); - } - } -} - -static void ComputeStringEntrySizes(const UnifiedVectorFormat &col, idx_t entry_sizes[], const SelectionVector &sel, - const idx_t count, const idx_t offset = 0) { - auto data = UnifiedVectorFormat::GetData(col); - for (idx_t i = 0; i < count; i++) { - auto idx = sel.get_index(i); - auto col_idx = col.sel->get_index(idx) + offset; - const auto &str = data[col_idx]; - if (col.validity.RowIsValid(col_idx) && !str.IsInlined()) { - entry_sizes[i] += str.GetSize(); - } - } -} - -static void ScatterStringVector(UnifiedVectorFormat &col, Vector &rows, data_ptr_t str_locations[], - const SelectionVector &sel, const idx_t count, const idx_t col_offset, - const idx_t col_no, const idx_t col_count) { - auto string_data = UnifiedVectorFormat::GetData(col); - auto ptrs = FlatVector::GetData(rows); - - // Write out zero length to avoid swizzling problems. - const string_t null(nullptr, 0); - for (idx_t i = 0; i < count; i++) { - auto idx = sel.get_index(i); - auto col_idx = col.sel->get_index(idx); - auto row = ptrs[idx]; - if (!col.validity.RowIsValid(col_idx)) { - ValidityBytes col_mask(row, col_count); - col_mask.SetInvalidUnsafe(col_no); - Store(null, row + col_offset); - } else if (string_data[col_idx].IsInlined()) { - Store(string_data[col_idx], row + col_offset); - } else { - const auto &str = string_data[col_idx]; - string_t inserted(const_char_ptr_cast(str_locations[i]), UnsafeNumericCast(str.GetSize())); - memcpy(inserted.GetDataWriteable(), str.GetData(), str.GetSize()); - str_locations[i] += str.GetSize(); - inserted.Finalize(); - Store(inserted, row + col_offset); - } - } -} - -static void ScatterNestedVector(Vector &vec, UnifiedVectorFormat &col, Vector &rows, data_ptr_t data_locations[], - const SelectionVector &sel, const idx_t count, const idx_t col_offset, - const idx_t col_no, const idx_t vcount) { - // Store pointers to the data in the row - // Do this first because SerializeVector destroys the locations - auto ptrs = FlatVector::GetData(rows); - data_ptr_t validitymask_locations[STANDARD_VECTOR_SIZE]; - for (idx_t i = 0; i < count; i++) { - auto idx = sel.get_index(i); - auto row = ptrs[idx]; - validitymask_locations[i] = row; - - Store(data_locations[i], row + col_offset); - } - - // Serialise the data - NestedValidity parent_validity(validitymask_locations, col_no); - RowOperations::HeapScatter(vec, vcount, sel, count, data_locations, &parent_validity); -} - -void RowOperations::Scatter(DataChunk &columns, UnifiedVectorFormat col_data[], const RowLayout &layout, Vector &rows, - RowDataCollection &string_heap, const SelectionVector &sel, idx_t count) { - if (count == 0) { - return; - } - - // Set the validity mask for each row before inserting data - idx_t column_count = layout.ColumnCount(); - auto ptrs = FlatVector::GetData(rows); - for (idx_t i = 0; i < count; ++i) { - auto row_idx = sel.get_index(i); - auto row = ptrs[row_idx]; - ValidityBytes(row, column_count).SetAllValid(layout.ColumnCount()); - } - - const auto vcount = columns.size(); - auto &offsets = layout.GetOffsets(); - auto &types = layout.GetTypes(); - - // Compute the entry size of the variable size columns - vector handles; - data_ptr_t data_locations[STANDARD_VECTOR_SIZE]; - if (!layout.AllConstant()) { - idx_t entry_sizes[STANDARD_VECTOR_SIZE]; - std::fill_n(entry_sizes, count, sizeof(uint32_t)); - for (idx_t col_no = 0; col_no < types.size(); col_no++) { - if (TypeIsConstantSize(types[col_no].InternalType())) { - continue; - } - - auto &vec = columns.data[col_no]; - auto &col = col_data[col_no]; - switch (types[col_no].InternalType()) { - case PhysicalType::VARCHAR: - ComputeStringEntrySizes(col, entry_sizes, sel, count); - break; - case PhysicalType::LIST: - case PhysicalType::STRUCT: - case PhysicalType::ARRAY: - RowOperations::ComputeEntrySizes(vec, col, entry_sizes, vcount, count, sel); - break; - default: - throw InternalException("Unsupported type for RowOperations::Scatter"); - } - } - - // Build out the buffer space - handles = string_heap.Build(count, data_locations, entry_sizes); - - // Serialize information that is needed for swizzling if the computation goes out-of-core - const idx_t heap_pointer_offset = layout.GetHeapOffset(); - for (idx_t i = 0; i < count; i++) { - auto row_idx = sel.get_index(i); - auto row = ptrs[row_idx]; - // Pointer to this row in the heap block - Store(data_locations[i], row + heap_pointer_offset); - // Row size is stored in the heap in front of each row - Store(NumericCast(entry_sizes[i]), data_locations[i]); - data_locations[i] += sizeof(uint32_t); - } - } - - for (idx_t col_no = 0; col_no < types.size(); col_no++) { - auto &vec = columns.data[col_no]; - auto &col = col_data[col_no]; - auto col_offset = offsets[col_no]; - - switch (types[col_no].InternalType()) { - case PhysicalType::BOOL: - case PhysicalType::INT8: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::INT16: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::INT32: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::INT64: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::UINT8: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::UINT16: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::UINT32: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::UINT64: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::INT128: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::UINT128: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::FLOAT: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::DOUBLE: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::INTERVAL: - TemplatedScatter(col, rows, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::VARCHAR: - ScatterStringVector(col, rows, data_locations, sel, count, col_offset, col_no, column_count); - break; - case PhysicalType::LIST: - case PhysicalType::STRUCT: - case PhysicalType::ARRAY: - ScatterNestedVector(vec, col, rows, data_locations, sel, count, col_offset, col_no, vcount); - break; - default: - throw InternalException("Unsupported type for RowOperations::Scatter"); - } - } -} - -} // namespace duckdb diff --git a/src/common/settings.json b/src/common/settings.json index 08f60b5b7af2..4e130c1e109a 100644 --- a/src/common/settings.json +++ b/src/common/settings.json @@ -49,6 +49,16 @@ "default_scope": "global", "default_value": "false" }, + { + "name": "allow_parser_override_extension", + "description": "Allow extensions to override the current parser", + "type": "VARCHAR", + "scope": "global", + "on_callbacks": [ + "set", + "reset" + ] + }, { "name": "allow_persistent_secrets", "description": "Allow the creation of persistent secrets, that are stored and loaded on restarts", @@ -456,6 +466,13 @@ "scope": "local", "struct": "ErrorsAsJSONSetting" }, + { + "name": "experimental_metadata_reuse", + "description": "EXPERIMENTAL: Re-use row group and table metadata when checkpointing.", + "type": "BOOLEAN", + "default_scope": "global", + "default_value": "true" + }, { "name": "explain_output", "description": "Output of EXPLAIN statements (ALL, OPTIMIZED_ONLY, PHYSICAL_ONLY)", @@ -839,6 +856,13 @@ "scope": "global", "custom_implementation": true }, + { + "name": "storage_block_prefetch", + "description": "In which scenarios to use storage block prefetching", + "type": "ENUM", + "default_scope": "global", + "default_value": "REMOTE_ONLY" + }, { "name": "storage_compatibility_version", "description": "Serialize on checkpoint with compatibility for a given duckdb version", @@ -887,6 +911,13 @@ "user" ] }, + { + "name": "write_buffer_row_group_count", + "description": "The amount of row groups to buffer in bulk ingestion prior to flushing them together. Reducing this setting can reduce memory consumption.", + "type": "UBIGINT", + "default_scope": "global", + "default_value": "5" + }, { "name": "zstd_min_string_length", "description": "The (average) length at which to enable ZSTD compression, defaults to 4096", diff --git a/src/common/sort/CMakeLists.txt b/src/common/sort/CMakeLists.txt index d2295abb498d..8d61e5c3d359 100644 --- a/src/common/sort/CMakeLists.txt +++ b/src/common/sort/CMakeLists.txt @@ -1,12 +1,5 @@ -add_library_unity( - duckdb_sort - OBJECT - comparators.cpp - merge_sorter.cpp - partition_state.cpp - radix_sort.cpp - sort_state.cpp - sorted_block.cpp) +add_library_unity(duckdb_sort OBJECT hashed_sort.cpp sort.cpp sorted_run.cpp + sorted_run_merger.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ diff --git a/src/common/sort/comparators.cpp b/src/common/sort/comparators.cpp deleted file mode 100644 index 4df4cccc4430..000000000000 --- a/src/common/sort/comparators.cpp +++ /dev/null @@ -1,507 +0,0 @@ -#include "duckdb/common/sort/comparators.hpp" - -#include "duckdb/common/fast_mem.hpp" -#include "duckdb/common/sort/sort.hpp" -#include "duckdb/common/uhugeint.hpp" - -namespace duckdb { - -bool Comparators::TieIsBreakable(const idx_t &tie_col, const data_ptr_t &row_ptr, const SortLayout &sort_layout) { - const auto &col_idx = sort_layout.sorting_to_blob_col.at(tie_col); - // Check if the blob is NULL - ValidityBytes row_mask(row_ptr, sort_layout.column_count); - idx_t entry_idx; - idx_t idx_in_entry; - ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry); - if (!row_mask.RowIsValid(row_mask.GetValidityEntry(entry_idx), idx_in_entry)) { - // Can't break a NULL tie - return false; - } - auto &row_layout = sort_layout.blob_layout; - if (row_layout.GetTypes()[col_idx].InternalType() != PhysicalType::VARCHAR) { - // Nested type, must be broken - return true; - } - const auto &tie_col_offset = row_layout.GetOffsets()[col_idx]; - auto tie_string = Load(row_ptr + tie_col_offset); - if (tie_string.GetSize() < sort_layout.prefix_lengths[tie_col] && tie_string.GetSize() > 0) { - // No need to break the tie - we already compared the full string - return false; - } - return true; -} - -int Comparators::CompareTuple(const SBScanState &left, const SBScanState &right, const data_ptr_t &l_ptr, - const data_ptr_t &r_ptr, const SortLayout &sort_layout, const bool &external_sort) { - // Compare the sorting columns one by one - int comp_res = 0; - data_ptr_t l_ptr_offset = l_ptr; - data_ptr_t r_ptr_offset = r_ptr; - for (idx_t col_idx = 0; col_idx < sort_layout.column_count; col_idx++) { - comp_res = FastMemcmp(l_ptr_offset, r_ptr_offset, sort_layout.column_sizes[col_idx]); - if (comp_res == 0 && !sort_layout.constant_size[col_idx]) { - comp_res = BreakBlobTie(col_idx, left, right, sort_layout, external_sort); - } - if (comp_res != 0) { - break; - } - l_ptr_offset += sort_layout.column_sizes[col_idx]; - r_ptr_offset += sort_layout.column_sizes[col_idx]; - } - return comp_res; -} - -int Comparators::CompareVal(const data_ptr_t l_ptr, const data_ptr_t r_ptr, const LogicalType &type) { - switch (type.InternalType()) { - case PhysicalType::VARCHAR: - return TemplatedCompareVal(l_ptr, r_ptr); - case PhysicalType::LIST: - case PhysicalType::ARRAY: - case PhysicalType::STRUCT: { - auto l_nested_ptr = Load(l_ptr); - auto r_nested_ptr = Load(r_ptr); - return CompareValAndAdvance(l_nested_ptr, r_nested_ptr, type, true); - } - default: - throw NotImplementedException("Unimplemented CompareVal for type %s", type.ToString()); - } -} - -int Comparators::BreakBlobTie(const idx_t &tie_col, const SBScanState &left, const SBScanState &right, - const SortLayout &sort_layout, const bool &external) { - data_ptr_t l_data_ptr = left.DataPtr(*left.sb->blob_sorting_data); - data_ptr_t r_data_ptr = right.DataPtr(*right.sb->blob_sorting_data); - if (!TieIsBreakable(tie_col, l_data_ptr, sort_layout) && !TieIsBreakable(tie_col, r_data_ptr, sort_layout)) { - // Quick check to see if ties can be broken - return 0; - } - // Align the pointers - const idx_t &col_idx = sort_layout.sorting_to_blob_col.at(tie_col); - const auto &tie_col_offset = sort_layout.blob_layout.GetOffsets()[col_idx]; - l_data_ptr += tie_col_offset; - r_data_ptr += tie_col_offset; - // Do the comparison - const int order = sort_layout.order_types[tie_col] == OrderType::DESCENDING ? -1 : 1; - const auto &type = sort_layout.blob_layout.GetTypes()[col_idx]; - int result; - if (external) { - // Store heap pointers - data_ptr_t l_heap_ptr = left.HeapPtr(*left.sb->blob_sorting_data); - data_ptr_t r_heap_ptr = right.HeapPtr(*right.sb->blob_sorting_data); - // Unswizzle offset to pointer - UnswizzleSingleValue(l_data_ptr, l_heap_ptr, type); - UnswizzleSingleValue(r_data_ptr, r_heap_ptr, type); - // Compare - result = CompareVal(l_data_ptr, r_data_ptr, type); - // Swizzle the pointers back to offsets - SwizzleSingleValue(l_data_ptr, l_heap_ptr, type); - SwizzleSingleValue(r_data_ptr, r_heap_ptr, type); - } else { - result = CompareVal(l_data_ptr, r_data_ptr, type); - } - return order * result; -} - -template -int Comparators::TemplatedCompareVal(const data_ptr_t &left_ptr, const data_ptr_t &right_ptr) { - const auto left_val = Load(left_ptr); - const auto right_val = Load(right_ptr); - if (Equals::Operation(left_val, right_val)) { - return 0; - } else if (LessThan::Operation(left_val, right_val)) { - return -1; - } else { - return 1; - } -} - -int Comparators::CompareValAndAdvance(data_ptr_t &l_ptr, data_ptr_t &r_ptr, const LogicalType &type, bool valid) { - switch (type.InternalType()) { - case PhysicalType::BOOL: - case PhysicalType::INT8: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::INT16: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::INT32: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::INT64: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::UINT8: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::UINT16: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::UINT32: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::UINT64: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::INT128: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::UINT128: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::FLOAT: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::DOUBLE: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::INTERVAL: - return TemplatedCompareAndAdvance(l_ptr, r_ptr); - case PhysicalType::VARCHAR: - return CompareStringAndAdvance(l_ptr, r_ptr, valid); - case PhysicalType::LIST: - return CompareListAndAdvance(l_ptr, r_ptr, ListType::GetChildType(type), valid); - case PhysicalType::STRUCT: - return CompareStructAndAdvance(l_ptr, r_ptr, StructType::GetChildTypes(type), valid); - case PhysicalType::ARRAY: - return CompareArrayAndAdvance(l_ptr, r_ptr, ArrayType::GetChildType(type), valid, ArrayType::GetSize(type)); - default: - throw NotImplementedException("Unimplemented CompareValAndAdvance for type %s", type.ToString()); - } -} - -template -int Comparators::TemplatedCompareAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr) { - auto result = TemplatedCompareVal(left_ptr, right_ptr); - left_ptr += sizeof(T); - right_ptr += sizeof(T); - return result; -} - -int Comparators::CompareStringAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, bool valid) { - if (!valid) { - return 0; - } - uint32_t left_string_size = Load(left_ptr); - uint32_t right_string_size = Load(right_ptr); - left_ptr += sizeof(uint32_t); - right_ptr += sizeof(uint32_t); - auto memcmp_res = memcmp(const_char_ptr_cast(left_ptr), const_char_ptr_cast(right_ptr), - std::min(left_string_size, right_string_size)); - - left_ptr += left_string_size; - right_ptr += right_string_size; - - if (memcmp_res != 0) { - return memcmp_res; - } - if (left_string_size == right_string_size) { - return 0; - } - if (left_string_size < right_string_size) { - return -1; - } - return 1; -} - -int Comparators::CompareStructAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, - const child_list_t &types, bool valid) { - idx_t count = types.size(); - // Load validity masks - ValidityBytes left_validity(left_ptr, types.size()); - ValidityBytes right_validity(right_ptr, types.size()); - left_ptr += (count + 7) / 8; - right_ptr += (count + 7) / 8; - // Initialize variables - bool left_valid; - bool right_valid; - idx_t entry_idx; - idx_t idx_in_entry; - // Compare - int comp_res = 0; - for (idx_t i = 0; i < count; i++) { - ValidityBytes::GetEntryIndex(i, entry_idx, idx_in_entry); - left_valid = left_validity.RowIsValid(left_validity.GetValidityEntry(entry_idx), idx_in_entry); - right_valid = right_validity.RowIsValid(right_validity.GetValidityEntry(entry_idx), idx_in_entry); - auto &type = types[i].second; - if ((left_valid == right_valid) || TypeIsConstantSize(type.InternalType())) { - comp_res = CompareValAndAdvance(left_ptr, right_ptr, types[i].second, left_valid && valid); - } - if (!left_valid && !right_valid) { - comp_res = 0; - } else if (!left_valid) { - comp_res = 1; - } else if (!right_valid) { - comp_res = -1; - } - if (comp_res != 0) { - break; - } - } - return comp_res; -} - -int Comparators::CompareArrayAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, const LogicalType &type, - bool valid, idx_t array_size) { - if (!valid) { - return 0; - } - - // Load array validity masks - ValidityBytes left_validity(left_ptr, array_size); - ValidityBytes right_validity(right_ptr, array_size); - left_ptr += (array_size + 7) / 8; - right_ptr += (array_size + 7) / 8; - - int comp_res = 0; - if (TypeIsConstantSize(type.InternalType())) { - // Templated code for fixed-size types - switch (type.InternalType()) { - case PhysicalType::BOOL: - case PhysicalType::INT8: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::INT16: - comp_res = - TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::INT32: - comp_res = - TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::INT64: - comp_res = - TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::UINT8: - comp_res = - TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::UINT16: - comp_res = - TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::UINT32: - comp_res = - TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::UINT64: - comp_res = - TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::INT128: - comp_res = - TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::FLOAT: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::DOUBLE: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - case PhysicalType::INTERVAL: - comp_res = - TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, array_size); - break; - default: - throw NotImplementedException("CompareListAndAdvance for fixed-size type %s", type.ToString()); - } - } else { - // Variable-sized array entries - bool left_valid; - bool right_valid; - idx_t entry_idx; - idx_t idx_in_entry; - // Size (in bytes) of all variable-sizes entries is stored before the entries begin, - // to make deserialization easier. We need to skip over them - left_ptr += array_size * sizeof(idx_t); - right_ptr += array_size * sizeof(idx_t); - for (idx_t i = 0; i < array_size; i++) { - ValidityBytes::GetEntryIndex(i, entry_idx, idx_in_entry); - left_valid = left_validity.RowIsValid(left_validity.GetValidityEntry(entry_idx), idx_in_entry); - right_valid = right_validity.RowIsValid(right_validity.GetValidityEntry(entry_idx), idx_in_entry); - if (left_valid && right_valid) { - switch (type.InternalType()) { - case PhysicalType::LIST: - comp_res = CompareListAndAdvance(left_ptr, right_ptr, ListType::GetChildType(type), left_valid); - break; - case PhysicalType::ARRAY: - comp_res = CompareArrayAndAdvance(left_ptr, right_ptr, ArrayType::GetChildType(type), left_valid, - ArrayType::GetSize(type)); - break; - case PhysicalType::VARCHAR: - comp_res = CompareStringAndAdvance(left_ptr, right_ptr, left_valid); - break; - case PhysicalType::STRUCT: - comp_res = - CompareStructAndAdvance(left_ptr, right_ptr, StructType::GetChildTypes(type), left_valid); - break; - default: - throw NotImplementedException("CompareArrayAndAdvance for variable-size type %s", type.ToString()); - } - } else if (!left_valid && !right_valid) { - comp_res = 0; - } else if (left_valid) { - comp_res = -1; - } else { - comp_res = 1; - } - if (comp_res != 0) { - break; - } - } - } - return comp_res; -} - -int Comparators::CompareListAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, const LogicalType &type, - bool valid) { - if (!valid) { - return 0; - } - // Load list lengths - auto left_len = Load(left_ptr); - auto right_len = Load(right_ptr); - left_ptr += sizeof(idx_t); - right_ptr += sizeof(idx_t); - // Load list validity masks - ValidityBytes left_validity(left_ptr, left_len); - ValidityBytes right_validity(right_ptr, right_len); - left_ptr += (left_len + 7) / 8; - right_ptr += (right_len + 7) / 8; - // Compare - int comp_res = 0; - idx_t count = MinValue(left_len, right_len); - if (TypeIsConstantSize(type.InternalType())) { - // Templated code for fixed-size types - switch (type.InternalType()) { - case PhysicalType::BOOL: - case PhysicalType::INT8: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::INT16: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::INT32: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::INT64: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::UINT8: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::UINT16: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::UINT32: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::UINT64: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::INT128: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::UINT128: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::FLOAT: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::DOUBLE: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - case PhysicalType::INTERVAL: - comp_res = TemplatedCompareListLoop(left_ptr, right_ptr, left_validity, right_validity, count); - break; - default: - throw NotImplementedException("CompareListAndAdvance for fixed-size type %s", type.ToString()); - } - } else { - // Variable-sized list entries - bool left_valid; - bool right_valid; - idx_t entry_idx; - idx_t idx_in_entry; - // Size (in bytes) of all variable-sizes entries is stored before the entries begin, - // to make deserialization easier. We need to skip over them - left_ptr += left_len * sizeof(idx_t); - right_ptr += right_len * sizeof(idx_t); - for (idx_t i = 0; i < count; i++) { - ValidityBytes::GetEntryIndex(i, entry_idx, idx_in_entry); - left_valid = left_validity.RowIsValid(left_validity.GetValidityEntry(entry_idx), idx_in_entry); - right_valid = right_validity.RowIsValid(right_validity.GetValidityEntry(entry_idx), idx_in_entry); - if (left_valid && right_valid) { - switch (type.InternalType()) { - case PhysicalType::LIST: - comp_res = CompareListAndAdvance(left_ptr, right_ptr, ListType::GetChildType(type), left_valid); - break; - case PhysicalType::ARRAY: - comp_res = CompareArrayAndAdvance(left_ptr, right_ptr, ArrayType::GetChildType(type), left_valid, - ArrayType::GetSize(type)); - break; - case PhysicalType::VARCHAR: - comp_res = CompareStringAndAdvance(left_ptr, right_ptr, left_valid); - break; - case PhysicalType::STRUCT: - comp_res = - CompareStructAndAdvance(left_ptr, right_ptr, StructType::GetChildTypes(type), left_valid); - break; - default: - throw NotImplementedException("CompareListAndAdvance for variable-size type %s", type.ToString()); - } - } else if (!left_valid && !right_valid) { - comp_res = 0; - } else if (left_valid) { - comp_res = -1; - } else { - comp_res = 1; - } - if (comp_res != 0) { - break; - } - } - } - // All values that we looped over were equal - if (comp_res == 0 && left_len != right_len) { - // Smaller lists first - if (left_len < right_len) { - comp_res = -1; - } else { - comp_res = 1; - } - } - return comp_res; -} - -template -int Comparators::TemplatedCompareListLoop(data_ptr_t &left_ptr, data_ptr_t &right_ptr, - const ValidityBytes &left_validity, const ValidityBytes &right_validity, - const idx_t &count) { - int comp_res = 0; - bool left_valid; - bool right_valid; - idx_t entry_idx; - idx_t idx_in_entry; - for (idx_t i = 0; i < count; i++) { - ValidityBytes::GetEntryIndex(i, entry_idx, idx_in_entry); - left_valid = left_validity.RowIsValid(left_validity.GetValidityEntry(entry_idx), idx_in_entry); - right_valid = right_validity.RowIsValid(right_validity.GetValidityEntry(entry_idx), idx_in_entry); - comp_res = TemplatedCompareAndAdvance(left_ptr, right_ptr); - if (!left_valid && !right_valid) { - comp_res = 0; - } else if (!left_valid) { - comp_res = 1; - } else if (!right_valid) { - comp_res = -1; - } - if (comp_res != 0) { - break; - } - } - return comp_res; -} - -void Comparators::UnswizzleSingleValue(data_ptr_t data_ptr, const data_ptr_t &heap_ptr, const LogicalType &type) { - if (type.InternalType() == PhysicalType::VARCHAR) { - data_ptr += string_t::HEADER_SIZE; - } - Store(heap_ptr + Load(data_ptr), data_ptr); -} - -void Comparators::SwizzleSingleValue(data_ptr_t data_ptr, const data_ptr_t &heap_ptr, const LogicalType &type) { - if (type.InternalType() == PhysicalType::VARCHAR) { - data_ptr += string_t::HEADER_SIZE; - } - Store(UnsafeNumericCast(Load(data_ptr) - heap_ptr), data_ptr); -} - -} // namespace duckdb diff --git a/src/common/sorting/hashed_sort.cpp b/src/common/sort/hashed_sort.cpp similarity index 81% rename from src/common/sorting/hashed_sort.cpp rename to src/common/sort/hashed_sort.cpp index e40f8380e39e..cdf5dde5be66 100644 --- a/src/common/sorting/hashed_sort.cpp +++ b/src/common/sort/hashed_sort.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/sorting/hashed_sort.hpp" +#include "duckdb/common/sorting/sorted_run.hpp" #include "duckdb/common/radix_partitioning.hpp" #include "duckdb/parallel/base_pipeline_event.hpp" #include "duckdb/parallel/thread_context.hpp" @@ -26,7 +27,9 @@ class HashedSortGroup { // Source atomic tasks_completed; unique_ptr sort_source; - unique_ptr sorted; + + unique_ptr columns; + unique_ptr run; }; HashedSortGroup::HashedSortGroup(ClientContext &client, optional_ptr sort, idx_t group_idx) @@ -51,19 +54,18 @@ class HashedSortGlobalSinkState : public GlobalSinkState { HashedSortGlobalSinkState(ClientContext &client, const HashedSort &hashed_sort); - bool HasMergeTasks() const; - // OVER(PARTITION BY...) (hash grouping) unique_ptr CreatePartition(idx_t new_bits) const; + void SyncPartitioning(const HashedSortGlobalSinkState &other); void UpdateLocalPartition(GroupingPartition &local_partition, GroupingAppend &partition_append); void CombineLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append); - void Finalize(ClientContext &context, InterruptState &interrupt_state); + ProgressData GetSinkProgress(ClientContext &context, const ProgressData source_progress) const; //! System and query state const HashedSort &hashed_sort; BufferManager &buffer_manager; Allocator &allocator; - mutex lock; + mutable mutex lock; // OVER(PARTITION BY...) (hash grouping) GroupingPartition grouping_data; @@ -174,6 +176,15 @@ void HashedSortGlobalSinkState::UpdateLocalPartition(GroupingPartition &local_pa SyncLocalPartition(local_partition, partition_append); } +void HashedSortGlobalSinkState::SyncPartitioning(const HashedSortGlobalSinkState &other) { + fixed_bits = other.grouping_data ? other.grouping_data->GetRadixBits() : 0; + + const auto old_bits = grouping_data ? grouping_data->GetRadixBits() : 0; + if (fixed_bits != old_bits) { + grouping_data = CreatePartition(fixed_bits); + } +} + void HashedSortGlobalSinkState::CombineLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) { if (!local_partition) { @@ -207,6 +218,30 @@ void HashedSortGlobalSinkState::CombineLocalPartition(GroupingPartition &local_p } } +ProgressData HashedSortGlobalSinkState::GetSinkProgress(ClientContext &client, const ProgressData source) const { + ProgressData result; + result.done = source.done / 2; + result.total = source.total; + result.invalid = source.invalid; + + // Sort::GetSinkProgress assumes that there is only 1 sort. + // So we just use it to figure out how many rows have been sorted. + const ProgressData zero_progress; + lock_guard guard(lock); + const auto &sort = hashed_sort.sort; + for (auto &hash_group : hash_groups) { + if (!hash_group || !hash_group->sort_global) { + continue; + } + + const auto group_progress = sort->GetSinkProgress(client, *hash_group->sort_global, zero_progress); + result.done += group_progress.done; + result.invalid = result.invalid || group_progress.invalid; + } + + return result; +} + SinkFinalizeType HashedSort::Finalize(ClientContext &client, OperatorSinkFinalizeInput &finalize) const { auto &gsink = finalize.global_state.Cast(); @@ -233,8 +268,10 @@ SinkFinalizeType HashedSort::Finalize(ClientContext &client, OperatorSinkFinaliz return SinkFinalizeType::READY; } -bool HashedSortGlobalSinkState::HasMergeTasks() const { - return (!hash_groups.empty()); +ProgressData HashedSort::GetSinkProgress(ClientContext &client, GlobalSinkState &gstate, + const ProgressData source) const { + auto &gsink = gstate.Cast(); + return gsink.GetSinkProgress(client, source); } //===--------------------------------------------------------------------===// @@ -317,9 +354,18 @@ HashedSortLocalSinkState::HashedSortLocalSinkState(ExecutionContext &context, co } // OVER(...) payload_chunk.Initialize(allocator, payload_types); + } else { + unsorted = make_uniq(context.client, hashed_sort.payload_types); + unsorted->InitializeAppend(unsorted_append); } } +void HashedSort::Synchronize(const GlobalSinkState &source, GlobalSinkState &target) const { + auto &src = source.Cast(); + auto &tgt = target.Cast(); + tgt.SyncPartitioning(src); +} + void HashedSortLocalSinkState::Hash(DataChunk &input_chunk, Vector &hash_vector) { const auto count = input_chunk.size(); D_ASSERT(group_chunk.ColumnCount() > 0); @@ -345,10 +391,6 @@ SinkResultType HashedSort::Sink(ExecutionContext &context, DataChunk &input_chun // OVER() if (gstate.hashed_sort.sort_col_count == 0) { - if (!lstate.unsorted) { - lstate.unsorted = make_uniq(context.client, payload_types); - lstate.unsorted->InitializeAppend(lstate.unsorted_append); - } lstate.unsorted->Append(lstate.unsorted_append, input_chunk); return SinkResultType::NEED_MORE_INPUT; } @@ -370,6 +412,15 @@ SinkResultType HashedSort::Sink(ExecutionContext &context, DataChunk &input_chun payload_chunk.data[input_chunk.ColumnCount() + i].Reference(sort_chunk.data[i]); } } + + // Append a forced payload column + if (force_payload) { + auto &vec = payload_chunk.data[input_chunk.ColumnCount() + sort_chunk.ColumnCount()]; + D_ASSERT(vec.GetType().id() == LogicalTypeId::BOOLEAN); + vec.SetVectorType(VectorType::CONSTANT_VECTOR); + ConstantVector::SetNull(vec, true); + } + payload_chunk.SetCardinality(input_chunk); // OVER(ORDER BY...) @@ -412,14 +463,14 @@ SinkCombineResultType HashedSort::Combine(ExecutionContext &context, OperatorSin auto &hash_groups = gstate.hash_groups; if (!hash_groups.empty()) { D_ASSERT(hash_groups.size() == 1); - auto &unsorted = *hash_groups[0]->sorted; + auto &unsorted = *hash_groups[0]->columns; if (lstate.unsorted) { unsorted.Combine(*lstate.unsorted); lstate.unsorted.reset(); } } else { auto new_group = make_uniq(context.client, sort, idx_t(0)); - new_group->sorted = std::move(lstate.unsorted); + new_group->columns = std::move(lstate.unsorted); hash_groups.emplace_back(std::move(new_group)); } return SinkCombineResultType::FINISHED; @@ -485,7 +536,7 @@ SinkCombineResultType HashedSort::Combine(ExecutionContext &context, OperatorSin class HashedSortMaterializeTask : public ExecutorTask { public: HashedSortMaterializeTask(Pipeline &pipeline, shared_ptr event, const PhysicalOperator &op, - HashedSortGroup &hash_group, idx_t tasks_scheduled); + HashedSortGroup &hash_group, idx_t tasks_scheduled, bool build_runs); TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override; @@ -497,13 +548,14 @@ class HashedSortMaterializeTask : public ExecutorTask { Pipeline &pipeline; HashedSortGroup &hash_group; const idx_t tasks_scheduled; + const bool build_runs; }; HashedSortMaterializeTask::HashedSortMaterializeTask(Pipeline &pipeline, shared_ptr event, const PhysicalOperator &op, HashedSortGroup &hash_group, - idx_t tasks_scheduled) + idx_t tasks_scheduled, bool build_runs) : ExecutorTask(pipeline.GetClientContext(), std::move(event), op), pipeline(pipeline), hash_group(hash_group), - tasks_scheduled(tasks_scheduled) { + tasks_scheduled(tasks_scheduled), build_runs(build_runs) { } TaskExecutionResult HashedSortMaterializeTask::ExecuteTask(TaskExecutionMode mode) { @@ -513,9 +565,20 @@ TaskExecutionResult HashedSortMaterializeTask::ExecuteTask(TaskExecutionMode mod auto sort_local = sort.GetLocalSourceState(execution, sort_global); InterruptState interrupt((weak_ptr(shared_from_this()))); OperatorSourceInput input {sort_global, *sort_local, interrupt}; - sort.MaterializeColumnData(execution, input); + if (build_runs) { + sort.MaterializeSortedRun(execution, input); + } else { + sort.MaterializeColumnData(execution, input); + } if (++hash_group.tasks_completed == tasks_scheduled) { - hash_group.sorted = sort.GetColumnData(input); + if (build_runs) { + hash_group.run = sort.GetSortedRun(sort_global); + if (!hash_group.run) { + hash_group.run = make_uniq(execution.client, sort, false); + } + } else { + hash_group.columns = sort.GetColumnData(input); + } } event->FinishTask(); @@ -528,18 +591,20 @@ TaskExecutionResult HashedSortMaterializeTask::ExecuteTask(TaskExecutionMode mod // Formerly PartitionMergeEvent class HashedSortMaterializeEvent : public BasePipelineEvent { public: - HashedSortMaterializeEvent(HashedSortGlobalSinkState &gstate, Pipeline &pipeline, const PhysicalOperator &op); + HashedSortMaterializeEvent(HashedSortGlobalSinkState &gstate, Pipeline &pipeline, const PhysicalOperator &op, + bool build_runs); HashedSortGlobalSinkState &gstate; const PhysicalOperator &op; + const bool build_runs; public: void Schedule() override; }; HashedSortMaterializeEvent::HashedSortMaterializeEvent(HashedSortGlobalSinkState &gstate, Pipeline &pipeline, - const PhysicalOperator &op) - : BasePipelineEvent(pipeline), gstate(gstate), op(op) { + const PhysicalOperator &op, bool build_runs) + : BasePipelineEvent(pipeline), gstate(gstate), op(op), build_runs(build_runs) { } void HashedSortMaterializeEvent::Schedule() { @@ -550,7 +615,7 @@ void HashedSortMaterializeEvent::Schedule() { const auto num_threads = NumericCast(ts.NumberOfThreads()); auto &sort = *gstate.hashed_sort.sort; - vector> merge_tasks; + vector> tasks; for (auto &hash_group : gstate.hash_groups) { if (!hash_group) { continue; @@ -559,12 +624,12 @@ void HashedSortMaterializeEvent::Schedule() { hash_group->sort_source = sort.GetGlobalSourceState(client, global_sink); const auto tasks_scheduled = MinValue(num_threads, hash_group->sort_source->MaxThreads()); for (idx_t t = 0; t < tasks_scheduled; ++t) { - merge_tasks.emplace_back( - make_uniq(*pipeline, shared_from_this(), op, *hash_group, tasks_scheduled)); + tasks.emplace_back(make_uniq(*pipeline, shared_from_this(), op, *hash_group, + tasks_scheduled, build_runs)); } } - SetTasks(std::move(merge_tasks)); + SetTasks(std::move(tasks)); } //===--------------------------------------------------------------------===// @@ -573,22 +638,26 @@ void HashedSortMaterializeEvent::Schedule() { class HashedSortGlobalSourceState : public GlobalSourceState { public: using HashGroupPtr = unique_ptr; + using SortedRunPtr = unique_ptr; HashedSortGlobalSourceState(ClientContext &client, HashedSortGlobalSinkState &gsink) { if (!gsink.count) { return; } - hash_groups.resize(gsink.hash_groups.size()); + columns.resize(gsink.hash_groups.size()); + runs.resize(gsink.hash_groups.size()); for (auto &hash_group : gsink.hash_groups) { if (!hash_group) { continue; } const auto group_idx = hash_group->group_idx; - hash_groups[group_idx] = std::move(hash_group->sorted); + columns[group_idx] = std::move(hash_group->columns); + runs[group_idx] = std::move(hash_group->run); } } - vector hash_groups; + vector columns; + vector runs; }; //===--------------------------------------------------------------------===// @@ -619,7 +688,8 @@ void HashedSort::GenerateOrderings(Orders &partitions, Orders &orders, HashedSort::HashedSort(ClientContext &client, const vector> &partition_bys, const vector &order_bys, const Types &input_types, - const vector> &partition_stats, idx_t estimated_cardinality) + const vector> &partition_stats, idx_t estimated_cardinality, + bool require_payload) : client(client), estimated_cardinality(estimated_cardinality), payload_types(input_types) { GenerateOrderings(partitions, orders, partition_bys, order_bys, partition_stats); @@ -650,6 +720,15 @@ HashedSort::HashedSort(ClientContext &client, const vector sort_set(sort_ids.begin(), sort_ids.end()); + force_payload = (sort_set.size() >= payload_types.size()); + if (force_payload) { + payload_types.emplace_back(LogicalType::BOOLEAN); + } + } vector projection_map; sort = make_uniq(client, orders, payload_types, projection_map); } @@ -674,7 +753,7 @@ unique_ptr HashedSort::GetLocalSourceState(ExecutionContext &c vector &HashedSort::GetHashGroups(GlobalSourceState &gstate) const { auto &gsource = gstate.Cast(); - return gsource.hash_groups; + return gsource.columns; } SinkFinalizeType HashedSort::MaterializeHashGroups(Pipeline &pipeline, Event &event, const PhysicalOperator &op, @@ -684,7 +763,28 @@ SinkFinalizeType HashedSort::MaterializeHashGroups(Pipeline &pipeline, Event &ev // OVER() if (sort_col_count == 0) { auto &hash_group = *gsink.hash_groups[0]; - auto &unsorted = *hash_group.sorted; + auto &unsorted = *hash_group.columns; + if (!unsorted.Count()) { + return SinkFinalizeType::NO_OUTPUT_POSSIBLE; + } + return SinkFinalizeType::READY; + } + + // Schedule all the sorts for maximum thread utilisation + auto sort_event = make_shared_ptr(gsink, pipeline, op, false); + event.InsertEvent(std::move(sort_event)); + + return SinkFinalizeType::READY; +} + +SinkFinalizeType HashedSort::MaterializeSortedRuns(Pipeline &pipeline, Event &event, const PhysicalOperator &op, + OperatorSinkFinalizeInput &finalize) const { + auto &gsink = finalize.global_state.Cast(); + + // OVER() + if (sort_col_count == 0) { + auto &hash_group = *gsink.hash_groups[0]; + auto &unsorted = *hash_group.columns; if (!unsorted.Count()) { return SinkFinalizeType::NO_OUTPUT_POSSIBLE; } @@ -692,10 +792,15 @@ SinkFinalizeType HashedSort::MaterializeHashGroups(Pipeline &pipeline, Event &ev } // Schedule all the sorts for maximum thread utilisation - auto sort_event = make_shared_ptr(gsink, pipeline, op); + auto sort_event = make_shared_ptr(gsink, pipeline, op, true); event.InsertEvent(std::move(sort_event)); return SinkFinalizeType::READY; } +vector &HashedSort::GetSortedRuns(GlobalSourceState &gstate) const { + auto &gsource = gstate.Cast(); + return gsource.runs; +} + } // namespace duckdb diff --git a/src/common/sort/merge_sorter.cpp b/src/common/sort/merge_sorter.cpp deleted file mode 100644 index c670fd574381..000000000000 --- a/src/common/sort/merge_sorter.cpp +++ /dev/null @@ -1,667 +0,0 @@ -#include "duckdb/common/fast_mem.hpp" -#include "duckdb/common/sort/comparators.hpp" -#include "duckdb/common/sort/sort.hpp" - -namespace duckdb { - -MergeSorter::MergeSorter(GlobalSortState &state, BufferManager &buffer_manager) - : state(state), buffer_manager(buffer_manager), sort_layout(state.sort_layout) { -} - -void MergeSorter::PerformInMergeRound() { - while (true) { - // Check for interrupts after merging a partition - if (state.context.interrupted) { - throw InterruptException(); - } - { - lock_guard pair_guard(state.lock); - if (state.pair_idx == state.num_pairs) { - break; - } - GetNextPartition(); - } - MergePartition(); - } -} - -void MergeSorter::MergePartition() { - auto &left_block = *left->sb; - auto &right_block = *right->sb; -#ifdef DEBUG - D_ASSERT(left_block.radix_sorting_data.size() == left_block.payload_data->data_blocks.size()); - D_ASSERT(right_block.radix_sorting_data.size() == right_block.payload_data->data_blocks.size()); - if (!state.payload_layout.AllConstant() && state.external) { - D_ASSERT(left_block.payload_data->data_blocks.size() == left_block.payload_data->heap_blocks.size()); - D_ASSERT(right_block.payload_data->data_blocks.size() == right_block.payload_data->heap_blocks.size()); - } - if (!sort_layout.all_constant) { - D_ASSERT(left_block.radix_sorting_data.size() == left_block.blob_sorting_data->data_blocks.size()); - D_ASSERT(right_block.radix_sorting_data.size() == right_block.blob_sorting_data->data_blocks.size()); - if (state.external) { - D_ASSERT(left_block.blob_sorting_data->data_blocks.size() == - left_block.blob_sorting_data->heap_blocks.size()); - D_ASSERT(right_block.blob_sorting_data->data_blocks.size() == - right_block.blob_sorting_data->heap_blocks.size()); - } - } -#endif - // Set up the write block - // Each merge task produces a SortedBlock with exactly state.block_capacity rows or less - result->InitializeWrite(); - // Initialize arrays to store merge data - bool left_smaller[STANDARD_VECTOR_SIZE]; - idx_t next_entry_sizes[STANDARD_VECTOR_SIZE]; - // Merge loop -#ifdef DEBUG - auto l_count = left->Remaining(); - auto r_count = right->Remaining(); -#endif - while (true) { - auto l_remaining = left->Remaining(); - auto r_remaining = right->Remaining(); - if (l_remaining + r_remaining == 0) { - // Done - break; - } - const idx_t next = MinValue(l_remaining + r_remaining, (idx_t)STANDARD_VECTOR_SIZE); - if (l_remaining != 0 && r_remaining != 0) { - // Compute the merge (not needed if one side is exhausted) - ComputeMerge(next, left_smaller); - } - // Actually merge the data (radix, blob, and payload) - MergeRadix(next, left_smaller); - if (!sort_layout.all_constant) { - MergeData(*result->blob_sorting_data, *left_block.blob_sorting_data, *right_block.blob_sorting_data, next, - left_smaller, next_entry_sizes, true); - D_ASSERT(result->radix_sorting_data.size() == result->blob_sorting_data->data_blocks.size()); - } - MergeData(*result->payload_data, *left_block.payload_data, *right_block.payload_data, next, left_smaller, - next_entry_sizes, false); - D_ASSERT(result->radix_sorting_data.size() == result->payload_data->data_blocks.size()); - } -#ifdef DEBUG - D_ASSERT(result->Count() == l_count + r_count); -#endif -} - -void MergeSorter::GetNextPartition() { - // Create result block - state.sorted_blocks_temp[state.pair_idx].push_back(make_uniq(buffer_manager, state)); - result = state.sorted_blocks_temp[state.pair_idx].back().get(); - // Determine which blocks must be merged - auto &left_block = *state.sorted_blocks[state.pair_idx * 2]; - auto &right_block = *state.sorted_blocks[state.pair_idx * 2 + 1]; - const idx_t l_count = left_block.Count(); - const idx_t r_count = right_block.Count(); - // Initialize left and right reader - left = make_uniq(buffer_manager, state); - right = make_uniq(buffer_manager, state); - // Compute the work that this thread must do using Merge Path - idx_t l_end; - idx_t r_end; - if (state.l_start + state.r_start + state.block_capacity < l_count + r_count) { - left->sb = state.sorted_blocks[state.pair_idx * 2].get(); - right->sb = state.sorted_blocks[state.pair_idx * 2 + 1].get(); - const idx_t intersection = state.l_start + state.r_start + state.block_capacity; - GetIntersection(intersection, l_end, r_end); - D_ASSERT(l_end <= l_count); - D_ASSERT(r_end <= r_count); - D_ASSERT(intersection == l_end + r_end); - } else { - l_end = l_count; - r_end = r_count; - } - // Create slices of the data that this thread must merge - left->SetIndices(0, 0); - right->SetIndices(0, 0); - left_input = left_block.CreateSlice(state.l_start, l_end, left->entry_idx); - right_input = right_block.CreateSlice(state.r_start, r_end, right->entry_idx); - left->sb = left_input.get(); - right->sb = right_input.get(); - state.l_start = l_end; - state.r_start = r_end; - D_ASSERT(left->Remaining() + right->Remaining() == state.block_capacity || (l_end == l_count && r_end == r_count)); - // Update global state - if (state.l_start == l_count && state.r_start == r_count) { - // Delete references to previous pair - state.sorted_blocks[state.pair_idx * 2] = nullptr; - state.sorted_blocks[state.pair_idx * 2 + 1] = nullptr; - // Advance pair - state.pair_idx++; - state.l_start = 0; - state.r_start = 0; - } -} - -int MergeSorter::CompareUsingGlobalIndex(SBScanState &l, SBScanState &r, const idx_t l_idx, const idx_t r_idx) { - D_ASSERT(l_idx < l.sb->Count()); - D_ASSERT(r_idx < r.sb->Count()); - - // Easy comparison using the previous result (intersections must increase monotonically) - if (l_idx < state.l_start) { - return -1; - } - if (r_idx < state.r_start) { - return 1; - } - - l.sb->GlobalToLocalIndex(l_idx, l.block_idx, l.entry_idx); - r.sb->GlobalToLocalIndex(r_idx, r.block_idx, r.entry_idx); - - l.PinRadix(l.block_idx); - r.PinRadix(r.block_idx); - data_ptr_t l_ptr = l.radix_handle.Ptr() + l.entry_idx * sort_layout.entry_size; - data_ptr_t r_ptr = r.radix_handle.Ptr() + r.entry_idx * sort_layout.entry_size; - - int comp_res; - if (sort_layout.all_constant) { - comp_res = FastMemcmp(l_ptr, r_ptr, sort_layout.comparison_size); - } else { - l.PinData(*l.sb->blob_sorting_data); - r.PinData(*r.sb->blob_sorting_data); - comp_res = Comparators::CompareTuple(l, r, l_ptr, r_ptr, sort_layout, state.external); - } - return comp_res; -} - -void MergeSorter::GetIntersection(const idx_t diagonal, idx_t &l_idx, idx_t &r_idx) { - const idx_t l_count = left->sb->Count(); - const idx_t r_count = right->sb->Count(); - // Cover some edge cases - // Code coverage off because these edge cases cannot happen unless other code changes - // Edge cases have been tested extensively while developing Merge Path in a script - // LCOV_EXCL_START - if (diagonal >= l_count + r_count) { - l_idx = l_count; - r_idx = r_count; - return; - } else if (diagonal == 0) { - l_idx = 0; - r_idx = 0; - return; - } else if (l_count == 0) { - l_idx = 0; - r_idx = diagonal; - return; - } else if (r_count == 0) { - r_idx = 0; - l_idx = diagonal; - return; - } - // LCOV_EXCL_STOP - // Determine offsets for the binary search - const idx_t l_offset = MinValue(l_count, diagonal); - const idx_t r_offset = diagonal > l_count ? diagonal - l_count : 0; - D_ASSERT(l_offset + r_offset == diagonal); - const idx_t search_space = diagonal > MaxValue(l_count, r_count) ? l_count + r_count - diagonal - : MinValue(diagonal, MinValue(l_count, r_count)); - // Double binary search - idx_t li = 0; - idx_t ri = search_space - 1; - idx_t middle; - int comp_res; - while (li <= ri) { - middle = (li + ri) / 2; - l_idx = l_offset - middle; - r_idx = r_offset + middle; - if (l_idx == l_count || r_idx == 0) { - comp_res = CompareUsingGlobalIndex(*left, *right, l_idx - 1, r_idx); - if (comp_res > 0) { - l_idx--; - r_idx++; - } else { - return; - } - if (l_idx == 0 || r_idx == r_count) { - // This case is incredibly difficult to cover as it is dependent on parallelism randomness - // But it has been tested extensively during development in a script - // LCOV_EXCL_START - return; - // LCOV_EXCL_STOP - } else { - break; - } - } - comp_res = CompareUsingGlobalIndex(*left, *right, l_idx, r_idx); - if (comp_res > 0) { - li = middle + 1; - } else { - ri = middle - 1; - } - } - int l_r_min1 = CompareUsingGlobalIndex(*left, *right, l_idx, r_idx - 1); - int l_min1_r = CompareUsingGlobalIndex(*left, *right, l_idx - 1, r_idx); - if (l_r_min1 > 0 && l_min1_r < 0) { - return; - } else if (l_r_min1 > 0) { - l_idx--; - r_idx++; - } else if (l_min1_r < 0) { - l_idx++; - r_idx--; - } -} - -void MergeSorter::ComputeMerge(const idx_t &count, bool left_smaller[]) { - auto &l = *left; - auto &r = *right; - auto &l_sorted_block = *l.sb; - auto &r_sorted_block = *r.sb; - // Save indices to restore afterwards - idx_t l_block_idx_before = l.block_idx; - idx_t l_entry_idx_before = l.entry_idx; - idx_t r_block_idx_before = r.block_idx; - idx_t r_entry_idx_before = r.entry_idx; - // Data pointers for both sides - data_ptr_t l_radix_ptr; - data_ptr_t r_radix_ptr; - // Compute the merge of the next 'count' tuples - idx_t compared = 0; - while (compared < count) { - // Move to the next block (if needed) - if (l.block_idx < l_sorted_block.radix_sorting_data.size() && - l.entry_idx == l_sorted_block.radix_sorting_data[l.block_idx]->count) { - l.block_idx++; - l.entry_idx = 0; - } - if (r.block_idx < r_sorted_block.radix_sorting_data.size() && - r.entry_idx == r_sorted_block.radix_sorting_data[r.block_idx]->count) { - r.block_idx++; - r.entry_idx = 0; - } - const bool l_done = l.block_idx == l_sorted_block.radix_sorting_data.size(); - const bool r_done = r.block_idx == r_sorted_block.radix_sorting_data.size(); - if (l_done || r_done) { - // One of the sides is exhausted, no need to compare - break; - } - // Pin the radix sorting data - left->PinRadix(l.block_idx); - l_radix_ptr = left->RadixPtr(); - right->PinRadix(r.block_idx); - r_radix_ptr = right->RadixPtr(); - - const idx_t l_count = l_sorted_block.radix_sorting_data[l.block_idx]->count; - const idx_t r_count = r_sorted_block.radix_sorting_data[r.block_idx]->count; - // Compute the merge - if (sort_layout.all_constant) { - // All sorting columns are constant size - for (; compared < count && l.entry_idx < l_count && r.entry_idx < r_count; compared++) { - left_smaller[compared] = FastMemcmp(l_radix_ptr, r_radix_ptr, sort_layout.comparison_size) < 0; - const bool &l_smaller = left_smaller[compared]; - const bool r_smaller = !l_smaller; - // Use comparison bool (0 or 1) to increment entries and pointers - l.entry_idx += l_smaller; - r.entry_idx += r_smaller; - l_radix_ptr += l_smaller * sort_layout.entry_size; - r_radix_ptr += r_smaller * sort_layout.entry_size; - } - } else { - // Pin the blob data - left->PinData(*l_sorted_block.blob_sorting_data); - right->PinData(*r_sorted_block.blob_sorting_data); - // Merge with variable size sorting columns - for (; compared < count && l.entry_idx < l_count && r.entry_idx < r_count; compared++) { - left_smaller[compared] = - Comparators::CompareTuple(*left, *right, l_radix_ptr, r_radix_ptr, sort_layout, state.external) < 0; - const bool &l_smaller = left_smaller[compared]; - const bool r_smaller = !l_smaller; - // Use comparison bool (0 or 1) to increment entries and pointers - l.entry_idx += l_smaller; - r.entry_idx += r_smaller; - l_radix_ptr += l_smaller * sort_layout.entry_size; - r_radix_ptr += r_smaller * sort_layout.entry_size; - } - } - } - // Reset block indices - left->SetIndices(l_block_idx_before, l_entry_idx_before); - right->SetIndices(r_block_idx_before, r_entry_idx_before); -} - -void MergeSorter::MergeRadix(const idx_t &count, const bool left_smaller[]) { - auto &l = *left; - auto &r = *right; - // Save indices to restore afterwards - idx_t l_block_idx_before = l.block_idx; - idx_t l_entry_idx_before = l.entry_idx; - idx_t r_block_idx_before = r.block_idx; - idx_t r_entry_idx_before = r.entry_idx; - - auto &l_blocks = l.sb->radix_sorting_data; - auto &r_blocks = r.sb->radix_sorting_data; - RowDataBlock *l_block = nullptr; - RowDataBlock *r_block = nullptr; - - data_ptr_t l_ptr; - data_ptr_t r_ptr; - - RowDataBlock *result_block = result->radix_sorting_data.back().get(); - auto result_handle = buffer_manager.Pin(result_block->block); - data_ptr_t result_ptr = result_handle.Ptr() + result_block->count * sort_layout.entry_size; - - idx_t copied = 0; - while (copied < count) { - // Move to the next block (if needed) - if (l.block_idx < l_blocks.size() && l.entry_idx == l_blocks[l.block_idx]->count) { - // Delete reference to previous block - l_blocks[l.block_idx]->block = nullptr; - // Advance block - l.block_idx++; - l.entry_idx = 0; - } - if (r.block_idx < r_blocks.size() && r.entry_idx == r_blocks[r.block_idx]->count) { - // Delete reference to previous block - r_blocks[r.block_idx]->block = nullptr; - // Advance block - r.block_idx++; - r.entry_idx = 0; - } - const bool l_done = l.block_idx == l_blocks.size(); - const bool r_done = r.block_idx == r_blocks.size(); - // Pin the radix sortable blocks - idx_t l_count; - if (!l_done) { - l_block = l_blocks[l.block_idx].get(); - left->PinRadix(l.block_idx); - l_ptr = l.RadixPtr(); - l_count = l_block->count; - } else { - l_count = 0; - } - idx_t r_count; - if (!r_done) { - r_block = r_blocks[r.block_idx].get(); - r.PinRadix(r.block_idx); - r_ptr = r.RadixPtr(); - r_count = r_block->count; - } else { - r_count = 0; - } - // Copy using computed merge - if (!l_done && !r_done) { - // Both sides have data - merge - MergeRows(l_ptr, l.entry_idx, l_count, r_ptr, r.entry_idx, r_count, *result_block, result_ptr, - sort_layout.entry_size, left_smaller, copied, count); - } else if (r_done) { - // Right side is exhausted - FlushRows(l_ptr, l.entry_idx, l_count, *result_block, result_ptr, sort_layout.entry_size, copied, count); - } else { - // Left side is exhausted - FlushRows(r_ptr, r.entry_idx, r_count, *result_block, result_ptr, sort_layout.entry_size, copied, count); - } - } - // Reset block indices - left->SetIndices(l_block_idx_before, l_entry_idx_before); - right->SetIndices(r_block_idx_before, r_entry_idx_before); -} - -void MergeSorter::MergeData(SortedData &result_data, SortedData &l_data, SortedData &r_data, const idx_t &count, - const bool left_smaller[], idx_t next_entry_sizes[], bool reset_indices) { - auto &l = *left; - auto &r = *right; - // Save indices to restore afterwards - idx_t l_block_idx_before = l.block_idx; - idx_t l_entry_idx_before = l.entry_idx; - idx_t r_block_idx_before = r.block_idx; - idx_t r_entry_idx_before = r.entry_idx; - - const auto &layout = result_data.layout; - const idx_t row_width = layout.GetRowWidth(); - const idx_t heap_pointer_offset = layout.GetHeapOffset(); - - // Left and right row data to merge - data_ptr_t l_ptr; - data_ptr_t r_ptr; - // Accompanying left and right heap data (if needed) - data_ptr_t l_heap_ptr; - data_ptr_t r_heap_ptr; - - // Result rows to write to - RowDataBlock *result_data_block = result_data.data_blocks.back().get(); - auto result_data_handle = buffer_manager.Pin(result_data_block->block); - data_ptr_t result_data_ptr = result_data_handle.Ptr() + result_data_block->count * row_width; - // Result heap to write to (if needed) - RowDataBlock *result_heap_block = nullptr; - BufferHandle result_heap_handle; - data_ptr_t result_heap_ptr; - if (!layout.AllConstant() && state.external) { - result_heap_block = result_data.heap_blocks.back().get(); - result_heap_handle = buffer_manager.Pin(result_heap_block->block); - result_heap_ptr = result_heap_handle.Ptr() + result_heap_block->byte_offset; - } - - idx_t copied = 0; - while (copied < count) { - // Move to new data blocks (if needed) - if (l.block_idx < l_data.data_blocks.size() && l.entry_idx == l_data.data_blocks[l.block_idx]->count) { - // Delete reference to previous block - l_data.data_blocks[l.block_idx]->block = nullptr; - if (!layout.AllConstant() && state.external) { - l_data.heap_blocks[l.block_idx]->block = nullptr; - } - // Advance block - l.block_idx++; - l.entry_idx = 0; - } - if (r.block_idx < r_data.data_blocks.size() && r.entry_idx == r_data.data_blocks[r.block_idx]->count) { - // Delete reference to previous block - r_data.data_blocks[r.block_idx]->block = nullptr; - if (!layout.AllConstant() && state.external) { - r_data.heap_blocks[r.block_idx]->block = nullptr; - } - // Advance block - r.block_idx++; - r.entry_idx = 0; - } - const bool l_done = l.block_idx == l_data.data_blocks.size(); - const bool r_done = r.block_idx == r_data.data_blocks.size(); - // Pin the row data blocks - if (!l_done) { - l.PinData(l_data); - l_ptr = l.DataPtr(l_data); - } - if (!r_done) { - r.PinData(r_data); - r_ptr = r.DataPtr(r_data); - } - const idx_t &l_count = !l_done ? l_data.data_blocks[l.block_idx]->count : 0; - const idx_t &r_count = !r_done ? r_data.data_blocks[r.block_idx]->count : 0; - // Perform the merge - if (layout.AllConstant() || !state.external) { - // If all constant size, or if we are doing an in-memory sort, we do not need to touch the heap - if (!l_done && !r_done) { - // Both sides have data - merge - MergeRows(l_ptr, l.entry_idx, l_count, r_ptr, r.entry_idx, r_count, *result_data_block, result_data_ptr, - row_width, left_smaller, copied, count); - } else if (r_done) { - // Right side is exhausted - FlushRows(l_ptr, l.entry_idx, l_count, *result_data_block, result_data_ptr, row_width, copied, count); - } else { - // Left side is exhausted - FlushRows(r_ptr, r.entry_idx, r_count, *result_data_block, result_data_ptr, row_width, copied, count); - } - } else { - // External sorting with variable size data. Pin the heap blocks too - if (!l_done) { - l_heap_ptr = l.BaseHeapPtr(l_data) + Load(l_ptr + heap_pointer_offset); - D_ASSERT(l_heap_ptr - l.BaseHeapPtr(l_data) >= 0); - D_ASSERT((idx_t)(l_heap_ptr - l.BaseHeapPtr(l_data)) < l_data.heap_blocks[l.block_idx]->byte_offset); - } - if (!r_done) { - r_heap_ptr = r.BaseHeapPtr(r_data) + Load(r_ptr + heap_pointer_offset); - D_ASSERT(r_heap_ptr - r.BaseHeapPtr(r_data) >= 0); - D_ASSERT((idx_t)(r_heap_ptr - r.BaseHeapPtr(r_data)) < r_data.heap_blocks[r.block_idx]->byte_offset); - } - // Both the row and heap data need to be dealt with - if (!l_done && !r_done) { - // Both sides have data - merge - idx_t l_idx_copy = l.entry_idx; - idx_t r_idx_copy = r.entry_idx; - data_ptr_t result_data_ptr_copy = result_data_ptr; - idx_t copied_copy = copied; - // Merge row data - MergeRows(l_ptr, l_idx_copy, l_count, r_ptr, r_idx_copy, r_count, *result_data_block, - result_data_ptr_copy, row_width, left_smaller, copied_copy, count); - const idx_t merged = copied_copy - copied; - // Compute the entry sizes and number of heap bytes that will be copied - idx_t copy_bytes = 0; - data_ptr_t l_heap_ptr_copy = l_heap_ptr; - data_ptr_t r_heap_ptr_copy = r_heap_ptr; - for (idx_t i = 0; i < merged; i++) { - // Store base heap offset in the row data - Store(result_heap_block->byte_offset + copy_bytes, result_data_ptr + heap_pointer_offset); - result_data_ptr += row_width; - // Compute entry size and add to total - const bool &l_smaller = left_smaller[copied + i]; - const bool r_smaller = !l_smaller; - auto &entry_size = next_entry_sizes[copied + i]; - entry_size = - l_smaller * Load(l_heap_ptr_copy) + r_smaller * Load(r_heap_ptr_copy); - D_ASSERT(entry_size >= sizeof(uint32_t)); - D_ASSERT(NumericCast(l_heap_ptr_copy - l.BaseHeapPtr(l_data)) + l_smaller * entry_size <= - l_data.heap_blocks[l.block_idx]->byte_offset); - D_ASSERT(NumericCast(r_heap_ptr_copy - r.BaseHeapPtr(r_data)) + r_smaller * entry_size <= - r_data.heap_blocks[r.block_idx]->byte_offset); - l_heap_ptr_copy += l_smaller * entry_size; - r_heap_ptr_copy += r_smaller * entry_size; - copy_bytes += entry_size; - } - // Reallocate result heap block size (if needed) - if (result_heap_block->byte_offset + copy_bytes > result_heap_block->capacity) { - idx_t new_capacity = result_heap_block->byte_offset + copy_bytes; - buffer_manager.ReAllocate(result_heap_block->block, new_capacity); - result_heap_block->capacity = new_capacity; - result_heap_ptr = result_heap_handle.Ptr() + result_heap_block->byte_offset; - } - D_ASSERT(result_heap_block->byte_offset + copy_bytes <= result_heap_block->capacity); - // Now copy the heap data - for (idx_t i = 0; i < merged; i++) { - const bool &l_smaller = left_smaller[copied + i]; - const bool r_smaller = !l_smaller; - const auto &entry_size = next_entry_sizes[copied + i]; - memcpy(result_heap_ptr, - reinterpret_cast(l_smaller * CastPointerToValue(l_heap_ptr) + - r_smaller * CastPointerToValue(r_heap_ptr)), - entry_size); - D_ASSERT(Load(result_heap_ptr) == entry_size); - result_heap_ptr += entry_size; - l_heap_ptr += l_smaller * entry_size; - r_heap_ptr += r_smaller * entry_size; - l.entry_idx += l_smaller; - r.entry_idx += r_smaller; - } - // Update result indices and pointers - result_heap_block->count += merged; - result_heap_block->byte_offset += copy_bytes; - copied += merged; - } else if (r_done) { - // Right side is exhausted - flush left - FlushBlobs(layout, l_count, l_ptr, l.entry_idx, l_heap_ptr, *result_data_block, result_data_ptr, - *result_heap_block, result_heap_handle, result_heap_ptr, copied, count); - } else { - // Left side is exhausted - flush right - FlushBlobs(layout, r_count, r_ptr, r.entry_idx, r_heap_ptr, *result_data_block, result_data_ptr, - *result_heap_block, result_heap_handle, result_heap_ptr, copied, count); - } - D_ASSERT(result_data_block->count == result_heap_block->count); - } - } - if (reset_indices) { - left->SetIndices(l_block_idx_before, l_entry_idx_before); - right->SetIndices(r_block_idx_before, r_entry_idx_before); - } -} - -void MergeSorter::MergeRows(data_ptr_t &l_ptr, idx_t &l_entry_idx, const idx_t &l_count, data_ptr_t &r_ptr, - idx_t &r_entry_idx, const idx_t &r_count, RowDataBlock &target_block, - data_ptr_t &target_ptr, const idx_t &entry_size, const bool left_smaller[], idx_t &copied, - const idx_t &count) { - const idx_t next = MinValue(count - copied, target_block.capacity - target_block.count); - idx_t i; - for (i = 0; i < next && l_entry_idx < l_count && r_entry_idx < r_count; i++) { - const bool &l_smaller = left_smaller[copied + i]; - const bool r_smaller = !l_smaller; - // Use comparison bool (0 or 1) to copy an entry from either side - FastMemcpy( - target_ptr, - reinterpret_cast(l_smaller * CastPointerToValue(l_ptr) + r_smaller * CastPointerToValue(r_ptr)), - entry_size); - target_ptr += entry_size; - // Use the comparison bool to increment entries and pointers - l_entry_idx += l_smaller; - r_entry_idx += r_smaller; - l_ptr += l_smaller * entry_size; - r_ptr += r_smaller * entry_size; - } - // Update counts - target_block.count += i; - copied += i; -} - -void MergeSorter::FlushRows(data_ptr_t &source_ptr, idx_t &source_entry_idx, const idx_t &source_count, - RowDataBlock &target_block, data_ptr_t &target_ptr, const idx_t &entry_size, idx_t &copied, - const idx_t &count) { - // Compute how many entries we can fit - idx_t next = MinValue(count - copied, target_block.capacity - target_block.count); - next = MinValue(next, source_count - source_entry_idx); - // Copy them all in a single memcpy - const idx_t copy_bytes = next * entry_size; - memcpy(target_ptr, source_ptr, copy_bytes); - target_ptr += copy_bytes; - source_ptr += copy_bytes; - // Update counts - source_entry_idx += next; - target_block.count += next; - copied += next; -} - -void MergeSorter::FlushBlobs(const RowLayout &layout, const idx_t &source_count, data_ptr_t &source_data_ptr, - idx_t &source_entry_idx, data_ptr_t &source_heap_ptr, RowDataBlock &target_data_block, - data_ptr_t &target_data_ptr, RowDataBlock &target_heap_block, - BufferHandle &target_heap_handle, data_ptr_t &target_heap_ptr, idx_t &copied, - const idx_t &count) { - const idx_t row_width = layout.GetRowWidth(); - const idx_t heap_pointer_offset = layout.GetHeapOffset(); - idx_t source_entry_idx_copy = source_entry_idx; - data_ptr_t target_data_ptr_copy = target_data_ptr; - idx_t copied_copy = copied; - // Flush row data - FlushRows(source_data_ptr, source_entry_idx_copy, source_count, target_data_block, target_data_ptr_copy, row_width, - copied_copy, count); - const idx_t flushed = copied_copy - copied; - // Compute the entry sizes and number of heap bytes that will be copied - idx_t copy_bytes = 0; - data_ptr_t source_heap_ptr_copy = source_heap_ptr; - for (idx_t i = 0; i < flushed; i++) { - // Store base heap offset in the row data - Store(target_heap_block.byte_offset + copy_bytes, target_data_ptr + heap_pointer_offset); - target_data_ptr += row_width; - // Compute entry size and add to total - auto entry_size = Load(source_heap_ptr_copy); - D_ASSERT(entry_size >= sizeof(uint32_t)); - source_heap_ptr_copy += entry_size; - copy_bytes += entry_size; - } - // Reallocate result heap block size (if needed) - if (target_heap_block.byte_offset + copy_bytes > target_heap_block.capacity) { - idx_t new_capacity = target_heap_block.byte_offset + copy_bytes; - buffer_manager.ReAllocate(target_heap_block.block, new_capacity); - target_heap_block.capacity = new_capacity; - target_heap_ptr = target_heap_handle.Ptr() + target_heap_block.byte_offset; - } - D_ASSERT(target_heap_block.byte_offset + copy_bytes <= target_heap_block.capacity); - // Copy the heap data in one go - memcpy(target_heap_ptr, source_heap_ptr, copy_bytes); - target_heap_ptr += copy_bytes; - source_heap_ptr += copy_bytes; - source_entry_idx += flushed; - copied += flushed; - // Update result indices and pointers - target_heap_block.count += flushed; - target_heap_block.byte_offset += copy_bytes; - D_ASSERT(target_heap_block.byte_offset <= target_heap_block.capacity); -} - -} // namespace duckdb diff --git a/src/common/sort/partition_state.cpp b/src/common/sort/partition_state.cpp deleted file mode 100644 index 2a0a658950e2..000000000000 --- a/src/common/sort/partition_state.cpp +++ /dev/null @@ -1,671 +0,0 @@ -#include "duckdb/common/sort/partition_state.hpp" - -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/main/config.hpp" -#include "duckdb/parallel/executor_task.hpp" - -namespace duckdb { - -PartitionGlobalHashGroup::PartitionGlobalHashGroup(ClientContext &context, const Orders &partitions, - const Orders &orders, const Types &payload_types, bool external) - : count(0) { - - RowLayout payload_layout; - payload_layout.Initialize(payload_types); - global_sort = make_uniq(context, orders, payload_layout); - global_sort->external = external; - - // Set up a comparator for the partition subset - partition_layout = global_sort->sort_layout.GetPrefixComparisonLayout(partitions.size()); -} - -void PartitionGlobalHashGroup::ComputeMasks(ValidityMask &partition_mask, OrderMasks &order_masks) { - D_ASSERT(count > 0); - - SBIterator prev(*global_sort, ExpressionType::COMPARE_LESSTHAN); - SBIterator curr(*global_sort, ExpressionType::COMPARE_LESSTHAN); - - partition_mask.SetValidUnsafe(0); - unordered_map prefixes; - for (auto &order_mask : order_masks) { - order_mask.second.SetValidUnsafe(0); - D_ASSERT(order_mask.first >= partition_layout.column_count); - prefixes[order_mask.first] = global_sort->sort_layout.GetPrefixComparisonLayout(order_mask.first); - } - - for (++curr; curr.GetIndex() < count; ++curr) { - // Compare the partition subset first because if that differs, then so does the full ordering - const auto part_cmp = ComparePartitions(prev, curr); - - if (part_cmp) { - partition_mask.SetValidUnsafe(curr.GetIndex()); - for (auto &order_mask : order_masks) { - order_mask.second.SetValidUnsafe(curr.GetIndex()); - } - } else { - for (auto &order_mask : order_masks) { - if (prev.Compare(curr, prefixes[order_mask.first])) { - order_mask.second.SetValidUnsafe(curr.GetIndex()); - } - } - } - ++prev; - } -} - -void PartitionGlobalSinkState::GenerateOrderings(Orders &partitions, Orders &orders, - const vector> &partition_bys, - const Orders &order_bys, - const vector> &partition_stats) { - - // we sort by both 1) partition by expression list and 2) order by expressions - const auto partition_cols = partition_bys.size(); - for (idx_t prt_idx = 0; prt_idx < partition_cols; prt_idx++) { - auto &pexpr = partition_bys[prt_idx]; - - if (partition_stats.empty() || !partition_stats[prt_idx]) { - orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, pexpr->Copy(), nullptr); - } else { - orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, pexpr->Copy(), - partition_stats[prt_idx]->ToUnique()); - } - partitions.emplace_back(orders.back().Copy()); - } - - for (const auto &order : order_bys) { - orders.emplace_back(order.Copy()); - } -} - -PartitionGlobalSinkState::PartitionGlobalSinkState(ClientContext &context, - const vector> &partition_bys, - const vector &order_bys, - const Types &payload_types, - const vector> &partition_stats, - idx_t estimated_cardinality) - : context(context), buffer_manager(BufferManager::GetBufferManager(context)), allocator(Allocator::Get(context)), - fixed_bits(0), payload_types(payload_types), memory_per_thread(0), max_bits(1), count(0) { - - GenerateOrderings(partitions, orders, partition_bys, order_bys, partition_stats); - - memory_per_thread = PhysicalOperator::GetMaxThreadMemory(context); - external = ClientConfig::GetConfig(context).force_external; - - const auto thread_pages = PreviousPowerOfTwo(memory_per_thread / (4 * buffer_manager.GetBlockAllocSize())); - while (max_bits < 10 && (thread_pages >> max_bits) > 1) { - ++max_bits; - } - - grouping_types_ptr = make_shared_ptr(); - if (!orders.empty()) { - if (partitions.empty()) { - // Sort early into a dedicated hash group if we only sort. - grouping_types_ptr->Initialize(payload_types, TupleDataValidityType::CAN_HAVE_NULL_VALUES); - auto new_group = make_uniq(context, partitions, orders, payload_types, external); - hash_groups.emplace_back(std::move(new_group)); - } else { - auto types = payload_types; - types.push_back(LogicalType::HASH); - grouping_types_ptr->Initialize(types, TupleDataValidityType::CAN_HAVE_NULL_VALUES); - ResizeGroupingData(estimated_cardinality); - } - } -} - -bool PartitionGlobalSinkState::HasMergeTasks() const { - if (grouping_data) { - auto &groups = grouping_data->GetPartitions(); - return !groups.empty(); - } else if (!hash_groups.empty()) { - D_ASSERT(hash_groups.size() == 1); - return hash_groups[0]->count > 0; - } else { - return false; - } -} - -void PartitionGlobalSinkState::SyncPartitioning(const PartitionGlobalSinkState &other) { - fixed_bits = other.grouping_data ? other.grouping_data->GetRadixBits() : 0; - - const auto old_bits = grouping_data ? grouping_data->GetRadixBits() : 0; - if (fixed_bits != old_bits) { - const auto hash_col_idx = payload_types.size(); - grouping_data = - make_uniq(buffer_manager, grouping_types_ptr, fixed_bits, hash_col_idx); - } -} - -unique_ptr PartitionGlobalSinkState::CreatePartition(idx_t new_bits) const { - const auto hash_col_idx = payload_types.size(); - return make_uniq(buffer_manager, grouping_types_ptr, new_bits, hash_col_idx); -} - -void PartitionGlobalSinkState::ResizeGroupingData(idx_t cardinality) { - // Have we started to combine? Then just live with it. - if (fixed_bits || (grouping_data && !grouping_data->GetPartitions().empty())) { - return; - } - // Is the average partition size too large? - const idx_t partition_size = DEFAULT_ROW_GROUP_SIZE; - const auto bits = grouping_data ? grouping_data->GetRadixBits() : 0; - auto new_bits = bits ? bits : 4; - while (new_bits < max_bits && (cardinality / RadixPartitioning::NumberOfPartitions(new_bits)) > partition_size) { - ++new_bits; - } - - // Repartition the grouping data - if (new_bits != bits) { - grouping_data = CreatePartition(new_bits); - } -} - -void PartitionGlobalSinkState::SyncLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) { - // We are done if the local_partition is right sized. - auto &local_radix = local_partition->Cast(); - const auto new_bits = grouping_data->GetRadixBits(); - if (local_radix.GetRadixBits() == new_bits) { - return; - } - - // If the local partition is now too small, flush it and reallocate - auto new_partition = CreatePartition(new_bits); - local_partition->FlushAppendState(*local_append); - local_partition->Repartition(context, *new_partition); - - local_partition = std::move(new_partition); - local_append = make_uniq(); - local_partition->InitializeAppendState(*local_append); -} - -void PartitionGlobalSinkState::UpdateLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) { - // Make sure grouping_data doesn't change under us. - lock_guard guard(lock); - - if (!local_partition) { - local_partition = CreatePartition(grouping_data->GetRadixBits()); - local_append = make_uniq(); - local_partition->InitializeAppendState(*local_append); - return; - } - - // Grow the groups if they are too big - ResizeGroupingData(count); - - // Sync local partition to have the same bit count - SyncLocalPartition(local_partition, local_append); -} - -void PartitionGlobalSinkState::CombineLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) { - if (!local_partition) { - return; - } - local_partition->FlushAppendState(*local_append); - - // Make sure grouping_data doesn't change under us. - // Combine has an internal mutex, so this is single-threaded anyway. - lock_guard guard(lock); - SyncLocalPartition(local_partition, local_append); - grouping_data->Combine(*local_partition); -} - -PartitionLocalMergeState::PartitionLocalMergeState(PartitionGlobalSinkState &gstate) - : merge_state(nullptr), stage(PartitionSortStage::INIT), finished(true), executor(gstate.context) { - - // Set up the sort expression computation. - vector sort_types; - for (auto &order : gstate.orders) { - auto &oexpr = order.expression; - sort_types.emplace_back(oexpr->return_type); - executor.AddExpression(*oexpr); - } - sort_chunk.Initialize(gstate.allocator, sort_types); - payload_chunk.Initialize(gstate.allocator, gstate.payload_types); -} - -void PartitionLocalMergeState::Scan() { - if (!merge_state->group_data) { - // OVER(ORDER BY...) - // Already sorted - return; - } - - auto &group_data = *merge_state->group_data; - auto &hash_group = *merge_state->hash_group; - auto &chunk_state = merge_state->chunk_state; - // Copy the data from the group into the sort code. - auto &global_sort = *hash_group.global_sort; - LocalSortState local_sort; - local_sort.Initialize(global_sort, global_sort.buffer_manager); - - TupleDataScanState local_scan; - group_data.InitializeScan(local_scan, merge_state->column_ids); - while (group_data.Scan(chunk_state, local_scan, payload_chunk)) { - sort_chunk.Reset(); - executor.Execute(payload_chunk, sort_chunk); - - local_sort.SinkChunk(sort_chunk, payload_chunk); - if (local_sort.SizeInBytes() > merge_state->memory_per_thread) { - local_sort.Sort(global_sort, true); - } - hash_group.count += payload_chunk.size(); - } - - global_sort.AddLocalState(local_sort); -} - -// Per-thread sink state -PartitionLocalSinkState::PartitionLocalSinkState(ClientContext &context, PartitionGlobalSinkState &gstate_p) - : gstate(gstate_p), allocator(Allocator::Get(context)), executor(context) { - - vector group_types; - for (idx_t prt_idx = 0; prt_idx < gstate.partitions.size(); prt_idx++) { - auto &pexpr = *gstate.partitions[prt_idx].expression.get(); - group_types.push_back(pexpr.return_type); - executor.AddExpression(pexpr); - } - sort_cols = gstate.orders.size() + group_types.size(); - - if (sort_cols) { - auto payload_types = gstate.payload_types; - if (!group_types.empty()) { - // OVER(PARTITION BY...) - group_chunk.Initialize(allocator, group_types); - payload_types.emplace_back(LogicalType::HASH); - } else { - // OVER(ORDER BY...) - for (idx_t ord_idx = 0; ord_idx < gstate.orders.size(); ord_idx++) { - auto &pexpr = *gstate.orders[ord_idx].expression.get(); - group_types.push_back(pexpr.return_type); - executor.AddExpression(pexpr); - } - group_chunk.Initialize(allocator, group_types); - - // Single partition - auto &global_sort = *gstate.hash_groups[0]->global_sort; - local_sort = make_uniq(); - local_sort->Initialize(global_sort, global_sort.buffer_manager); - } - // OVER(...) - payload_chunk.Initialize(allocator, payload_types); - } else { - // OVER() - payload_layout.Initialize(gstate.payload_types); - } -} - -void PartitionLocalSinkState::Hash(DataChunk &input_chunk, Vector &hash_vector) { - const auto count = input_chunk.size(); - D_ASSERT(group_chunk.ColumnCount() > 0); - - // OVER(PARTITION BY...) (hash grouping) - group_chunk.Reset(); - executor.Execute(input_chunk, group_chunk); - VectorOperations::Hash(group_chunk.data[0], hash_vector, count); - for (idx_t prt_idx = 1; prt_idx < group_chunk.ColumnCount(); ++prt_idx) { - VectorOperations::CombineHash(hash_vector, group_chunk.data[prt_idx], count); - } -} - -void PartitionLocalSinkState::Sink(DataChunk &input_chunk) { - gstate.count += input_chunk.size(); - - // OVER() - if (sort_cols == 0) { - // No sorts, so build paged row chunks - if (!rows) { - const auto entry_size = payload_layout.GetRowWidth(); - const auto block_size = gstate.buffer_manager.GetBlockSize(); - const auto capacity = MaxValue(STANDARD_VECTOR_SIZE, block_size / entry_size + 1); - rows = make_uniq(gstate.buffer_manager, capacity, entry_size); - strings = make_uniq(gstate.buffer_manager, block_size, 1U, true); - } - const auto row_count = input_chunk.size(); - const auto row_sel = FlatVector::IncrementalSelectionVector(); - Vector addresses(LogicalType::POINTER); - auto key_locations = FlatVector::GetData(addresses); - const auto prev_rows_blocks = rows->blocks.size(); - auto handles = rows->Build(row_count, key_locations, nullptr, row_sel); - auto input_data = input_chunk.ToUnifiedFormat(); - RowOperations::Scatter(input_chunk, input_data.get(), payload_layout, addresses, *strings, *row_sel, row_count); - // Mark that row blocks contain pointers (heap blocks are pinned) - if (!payload_layout.AllConstant()) { - D_ASSERT(strings->keep_pinned); - for (size_t i = prev_rows_blocks; i < rows->blocks.size(); ++i) { - rows->blocks[i]->block->SetSwizzling("PartitionLocalSinkState::Sink"); - } - } - return; - } - - if (local_sort) { - // OVER(ORDER BY...) - group_chunk.Reset(); - executor.Execute(input_chunk, group_chunk); - local_sort->SinkChunk(group_chunk, input_chunk); - - auto &hash_group = *gstate.hash_groups[0]; - hash_group.count += input_chunk.size(); - - if (local_sort->SizeInBytes() > gstate.memory_per_thread) { - auto &global_sort = *hash_group.global_sort; - local_sort->Sort(global_sort, true); - } - return; - } - - // OVER(...) - payload_chunk.Reset(); - auto &hash_vector = payload_chunk.data.back(); - Hash(input_chunk, hash_vector); - for (idx_t col_idx = 0; col_idx < input_chunk.ColumnCount(); ++col_idx) { - payload_chunk.data[col_idx].Reference(input_chunk.data[col_idx]); - } - payload_chunk.SetCardinality(input_chunk); - - gstate.UpdateLocalPartition(local_partition, local_append); - local_partition->Append(*local_append, payload_chunk); -} - -void PartitionLocalSinkState::Combine() { - // OVER() - if (sort_cols == 0) { - // Only one partition again, so need a global lock. - lock_guard glock(gstate.lock); - if (gstate.rows) { - if (rows) { - gstate.rows->Merge(*rows); - gstate.strings->Merge(*strings); - rows.reset(); - strings.reset(); - } - } else { - gstate.rows = std::move(rows); - gstate.strings = std::move(strings); - } - return; - } - - if (local_sort) { - // OVER(ORDER BY...) - auto &hash_group = *gstate.hash_groups[0]; - auto &global_sort = *hash_group.global_sort; - global_sort.AddLocalState(*local_sort); - local_sort.reset(); - return; - } - - // OVER(...) - gstate.CombineLocalPartition(local_partition, local_append); -} - -PartitionGlobalMergeState::PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data_p, - hash_t hash_bin) - : sink(sink), group_data(std::move(group_data_p)), group_idx(sink.hash_groups.size()), - memory_per_thread(sink.memory_per_thread), - num_threads(NumericCast(TaskScheduler::GetScheduler(sink.context).NumberOfThreads())), - stage(PartitionSortStage::INIT), total_tasks(0), tasks_assigned(0), tasks_completed(0) { - - auto new_group = make_uniq(sink.context, sink.partitions, sink.orders, sink.payload_types, - sink.external); - sink.hash_groups.emplace_back(std::move(new_group)); - - hash_group = sink.hash_groups[group_idx].get(); - global_sort = sink.hash_groups[group_idx]->global_sort.get(); - - sink.bin_groups[hash_bin] = group_idx; - - column_ids.reserve(sink.payload_types.size()); - for (column_t i = 0; i < sink.payload_types.size(); ++i) { - column_ids.emplace_back(i); - } - group_data->InitializeScan(chunk_state, column_ids); -} - -PartitionGlobalMergeState::PartitionGlobalMergeState(PartitionGlobalSinkState &sink) - : sink(sink), group_idx(0), memory_per_thread(sink.memory_per_thread), - num_threads(NumericCast(TaskScheduler::GetScheduler(sink.context).NumberOfThreads())), - stage(PartitionSortStage::INIT), total_tasks(0), tasks_assigned(0), tasks_completed(0) { - - const hash_t hash_bin = 0; - hash_group = sink.hash_groups[group_idx].get(); - global_sort = sink.hash_groups[group_idx]->global_sort.get(); - - sink.bin_groups[hash_bin] = group_idx; -} - -void PartitionLocalMergeState::Prepare() { - merge_state->group_data.reset(); - - auto &global_sort = *merge_state->global_sort; - global_sort.PrepareMergePhase(); -} - -void PartitionLocalMergeState::Merge() { - auto &global_sort = *merge_state->global_sort; - MergeSorter merge_sorter(global_sort, global_sort.buffer_manager); - merge_sorter.PerformInMergeRound(); -} - -void PartitionLocalMergeState::Sorted() { - merge_state->sink.OnSortedPartition(merge_state->group_idx); -} - -void PartitionLocalMergeState::ExecuteTask() { - switch (stage) { - case PartitionSortStage::SCAN: - Scan(); - break; - case PartitionSortStage::PREPARE: - Prepare(); - break; - case PartitionSortStage::MERGE: - Merge(); - break; - case PartitionSortStage::SORTED: - Sorted(); - break; - default: - throw InternalException("Unexpected PartitionSortStage in ExecuteTask!"); - } - - merge_state->CompleteTask(); - finished = true; -} - -bool PartitionGlobalMergeState::AssignTask(PartitionLocalMergeState &local_state) { - lock_guard guard(lock); - - if (tasks_assigned >= total_tasks && !TryPrepareNextStage()) { - return false; - } - - local_state.merge_state = this; - local_state.stage = stage; - local_state.finished = false; - tasks_assigned++; - - return true; -} - -void PartitionGlobalMergeState::CompleteTask() { - lock_guard guard(lock); - - ++tasks_completed; -} - -bool PartitionGlobalMergeState::TryPrepareNextStage() { - if (tasks_completed < total_tasks) { - return false; - } - - tasks_assigned = tasks_completed = 0; - - switch (stage.load()) { - case PartitionSortStage::INIT: - // If the partitions are unordered, don't scan in parallel - // because it produces non-deterministic orderings. - // This can theoretically happen with ORDER BY, - // but that is something the query should be explicit about. - total_tasks = sink.orders.size() > sink.partitions.size() ? num_threads : 1; - stage = PartitionSortStage::SCAN; - return true; - - case PartitionSortStage::SCAN: - total_tasks = 1; - stage = PartitionSortStage::PREPARE; - return true; - - case PartitionSortStage::PREPARE: - if (!(global_sort->sorted_blocks.size() / 2)) { - break; - } - stage = PartitionSortStage::MERGE; - global_sort->InitializeMergeRound(); - total_tasks = num_threads; - return true; - - case PartitionSortStage::MERGE: - global_sort->CompleteMergeRound(true); - if (!(global_sort->sorted_blocks.size() / 2)) { - break; - } - global_sort->InitializeMergeRound(); - total_tasks = num_threads; - return true; - - case PartitionSortStage::SORTED: - stage = PartitionSortStage::FINISHED; - total_tasks = 0; - return false; - - case PartitionSortStage::FINISHED: - return false; - } - - stage = PartitionSortStage::SORTED; - total_tasks = 1; - - return true; -} - -PartitionGlobalMergeStates::PartitionGlobalMergeStates(PartitionGlobalSinkState &sink) { - // Schedule all the sorts for maximum thread utilisation - if (sink.grouping_data) { - auto &partitions = sink.grouping_data->GetPartitions(); - sink.bin_groups.resize(partitions.size(), partitions.size()); - for (hash_t hash_bin = 0; hash_bin < partitions.size(); ++hash_bin) { - auto &group_data = partitions[hash_bin]; - // Prepare for merge sort phase - if (group_data->Count()) { - auto state = make_uniq(sink, std::move(group_data), hash_bin); - states.emplace_back(std::move(state)); - } - } - } else { - // OVER(ORDER BY...) - // Already sunk into the single global sort, so set up single merge with no data - sink.bin_groups.resize(1, 1); - auto state = make_uniq(sink); - states.emplace_back(std::move(state)); - } - - sink.OnBeginMerge(); -} - -class PartitionMergeTask : public ExecutorTask { -public: - PartitionMergeTask(shared_ptr event_p, ClientContext &context_p, PartitionGlobalMergeStates &hash_groups_p, - PartitionGlobalSinkState &gstate, const PhysicalOperator &op) - : ExecutorTask(context_p, std::move(event_p), op), local_state(gstate), hash_groups(hash_groups_p) { - } - - TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override; - - string TaskType() const override { - return "PartitionMergeTask"; - } - -private: - struct ExecutorCallback : public PartitionGlobalMergeStates::Callback { - explicit ExecutorCallback(Executor &executor) : executor(executor) { - } - - bool HasError() const override { - return executor.HasError(); - } - - Executor &executor; - }; - - PartitionLocalMergeState local_state; - PartitionGlobalMergeStates &hash_groups; -}; - -bool PartitionGlobalMergeStates::ExecuteTask(PartitionLocalMergeState &local_state, Callback &callback) { - // Loop until all hash groups are done - size_t sorted = 0; - while (sorted < states.size()) { - // First check if there is an unfinished task for this thread - if (callback.HasError()) { - return false; - } - if (!local_state.TaskFinished()) { - local_state.ExecuteTask(); - continue; - } - - // Thread is done with its assigned task, try to fetch new work - for (auto group = sorted; group < states.size(); ++group) { - auto &global_state = states[group]; - if (global_state->IsFinished()) { - // This hash group is done - // Update the high water mark of densely completed groups - if (sorted == group) { - ++sorted; - } - continue; - } - - // Try to assign work for this hash group to this thread - if (global_state->AssignTask(local_state)) { - // We assigned a task to this thread! - // Break out of this loop to re-enter the top-level loop and execute the task - break; - } - - // We were able to prepare the next merge round, - // but we were not able to assign a task for it to this thread - // The tasks were assigned to other threads while this thread waited for the lock - // Go to the next iteration to see if another hash group has a task - } - } - - return true; -} - -TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) { - ExecutorCallback callback(executor); - - if (!hash_groups.ExecuteTask(local_state, callback)) { - return TaskExecutionResult::TASK_ERROR; - } - - event->FinishTask(); - return TaskExecutionResult::TASK_FINISHED; -} - -void PartitionMergeEvent::Schedule() { - auto &context = pipeline->GetClientContext(); - - // Schedule tasks equal to the number of threads, which will each merge multiple partitions - auto &ts = TaskScheduler::GetScheduler(context); - auto num_threads = NumericCast(ts.NumberOfThreads()); - - vector> merge_tasks; - for (idx_t tnum = 0; tnum < num_threads; tnum++) { - merge_tasks.emplace_back(make_uniq(shared_from_this(), context, merge_states, gstate, op)); - } - SetTasks(std::move(merge_tasks)); -} - -} // namespace duckdb diff --git a/src/common/sort/radix_sort.cpp b/src/common/sort/radix_sort.cpp deleted file mode 100644 index b193cee619df..000000000000 --- a/src/common/sort/radix_sort.cpp +++ /dev/null @@ -1,352 +0,0 @@ -#include "duckdb/common/fast_mem.hpp" -#include "duckdb/common/sort/comparators.hpp" -#include "duckdb/common/sort/duckdb_pdqsort.hpp" -#include "duckdb/common/sort/sort.hpp" - -namespace duckdb { - -//! Calls std::sort on strings that are tied by their prefix after the radix sort -static void SortTiedBlobs(BufferManager &buffer_manager, const data_ptr_t dataptr, const idx_t &start, const idx_t &end, - const idx_t &tie_col, bool *ties, const data_ptr_t blob_ptr, const SortLayout &sort_layout) { - const auto row_width = sort_layout.blob_layout.GetRowWidth(); - // Locate the first blob row in question - data_ptr_t row_ptr = dataptr + start * sort_layout.entry_size; - data_ptr_t blob_row_ptr = blob_ptr + Load(row_ptr + sort_layout.comparison_size) * row_width; - if (!Comparators::TieIsBreakable(tie_col, blob_row_ptr, sort_layout)) { - // Quick check to see if ties can be broken - return; - } - // Fill pointer array for sorting - auto ptr_block = make_unsafe_uniq_array_uninitialized(end - start); - auto entry_ptrs = (data_ptr_t *)ptr_block.get(); - for (idx_t i = start; i < end; i++) { - entry_ptrs[i - start] = row_ptr; - row_ptr += sort_layout.entry_size; - } - // Slow pointer-based sorting - const int order = sort_layout.order_types[tie_col] == OrderType::DESCENDING ? -1 : 1; - const idx_t &col_idx = sort_layout.sorting_to_blob_col.at(tie_col); - const auto &tie_col_offset = sort_layout.blob_layout.GetOffsets()[col_idx]; - auto logical_type = sort_layout.blob_layout.GetTypes()[col_idx]; - std::sort(entry_ptrs, entry_ptrs + end - start, - [&blob_ptr, &order, &sort_layout, &tie_col_offset, &row_width, &logical_type](const data_ptr_t l, - const data_ptr_t r) { - idx_t left_idx = Load(l + sort_layout.comparison_size); - idx_t right_idx = Load(r + sort_layout.comparison_size); - data_ptr_t left_ptr = blob_ptr + left_idx * row_width + tie_col_offset; - data_ptr_t right_ptr = blob_ptr + right_idx * row_width + tie_col_offset; - return order * Comparators::CompareVal(left_ptr, right_ptr, logical_type) < 0; - }); - // Re-order - auto temp_block = buffer_manager.GetBufferAllocator().Allocate((end - start) * sort_layout.entry_size); - data_ptr_t temp_ptr = temp_block.get(); - for (idx_t i = 0; i < end - start; i++) { - FastMemcpy(temp_ptr, entry_ptrs[i], sort_layout.entry_size); - temp_ptr += sort_layout.entry_size; - } - memcpy(dataptr + start * sort_layout.entry_size, temp_block.get(), (end - start) * sort_layout.entry_size); - // Determine if there are still ties (if this is not the last column) - if (tie_col < sort_layout.column_count - 1) { - data_ptr_t idx_ptr = dataptr + start * sort_layout.entry_size + sort_layout.comparison_size; - // Load current entry - data_ptr_t current_ptr = blob_ptr + Load(idx_ptr) * row_width + tie_col_offset; - for (idx_t i = 0; i < end - start - 1; i++) { - // Load next entry and compare - idx_ptr += sort_layout.entry_size; - data_ptr_t next_ptr = blob_ptr + Load(idx_ptr) * row_width + tie_col_offset; - ties[start + i] = Comparators::CompareVal(current_ptr, next_ptr, logical_type) == 0; - current_ptr = next_ptr; - } - } -} - -//! Identifies sequences of rows that are tied by the prefix of a blob column, and sorts them -static void SortTiedBlobs(BufferManager &buffer_manager, SortedBlock &sb, bool *ties, data_ptr_t dataptr, - const idx_t &count, const idx_t &tie_col, const SortLayout &sort_layout) { - D_ASSERT(!ties[count - 1]); - auto &blob_block = *sb.blob_sorting_data->data_blocks.back(); - auto blob_handle = buffer_manager.Pin(blob_block.block); - const data_ptr_t blob_ptr = blob_handle.Ptr(); - - for (idx_t i = 0; i < count; i++) { - if (!ties[i]) { - continue; - } - idx_t j; - for (j = i + 1; j < count; j++) { - if (!ties[j]) { - break; - } - } - SortTiedBlobs(buffer_manager, dataptr, i, j + 1, tie_col, ties, blob_ptr, sort_layout); - i = j; - } -} - -//! Returns whether there are any 'true' values in the ties[] array -static bool AnyTies(bool ties[], const idx_t &count) { - D_ASSERT(!ties[count - 1]); - bool any_ties = false; - for (idx_t i = 0; i < count - 1; i++) { - any_ties = any_ties || ties[i]; - } - return any_ties; -} - -//! Compares subsequent rows to check for ties -static void ComputeTies(data_ptr_t dataptr, const idx_t &count, const idx_t &col_offset, const idx_t &tie_size, - bool ties[], const SortLayout &sort_layout) { - D_ASSERT(!ties[count - 1]); - D_ASSERT(col_offset + tie_size <= sort_layout.comparison_size); - // Align dataptr - dataptr += col_offset; - for (idx_t i = 0; i < count - 1; i++) { - ties[i] = ties[i] && FastMemcmp(dataptr, dataptr + sort_layout.entry_size, tie_size) == 0; - dataptr += sort_layout.entry_size; - } -} - -//! Textbook LSD radix sort -void RadixSortLSD(BufferManager &buffer_manager, const data_ptr_t &dataptr, const idx_t &count, const idx_t &col_offset, - const idx_t &row_width, const idx_t &sorting_size) { - auto temp_block = buffer_manager.GetBufferAllocator().Allocate(count * row_width); - bool swap = false; - - idx_t counts[SortConstants::VALUES_PER_RADIX]; - for (idx_t r = 1; r <= sorting_size; r++) { - // Init counts to 0 - memset(counts, 0, sizeof(counts)); - // Const some values for convenience - const data_ptr_t source_ptr = swap ? temp_block.get() : dataptr; - const data_ptr_t target_ptr = swap ? dataptr : temp_block.get(); - const idx_t offset = col_offset + sorting_size - r; - // Collect counts - data_ptr_t offset_ptr = source_ptr + offset; - for (idx_t i = 0; i < count; i++) { - counts[*offset_ptr]++; - offset_ptr += row_width; - } - // Compute offsets from counts - idx_t max_count = counts[0]; - for (idx_t val = 1; val < SortConstants::VALUES_PER_RADIX; val++) { - max_count = MaxValue(max_count, counts[val]); - counts[val] = counts[val] + counts[val - 1]; - } - if (max_count == count) { - continue; - } - // Re-order the data in temporary array - data_ptr_t row_ptr = source_ptr + (count - 1) * row_width; - for (idx_t i = 0; i < count; i++) { - idx_t &radix_offset = --counts[*(row_ptr + offset)]; - FastMemcpy(target_ptr + radix_offset * row_width, row_ptr, row_width); - row_ptr -= row_width; - } - swap = !swap; - } - // Move data back to original buffer (if it was swapped) - if (swap) { - memcpy(dataptr, temp_block.get(), count * row_width); - } -} - -//! Insertion sort, used when count of values is low -inline void InsertionSort(const data_ptr_t orig_ptr, const data_ptr_t temp_ptr, const idx_t &count, - const idx_t &col_offset, const idx_t &row_width, const idx_t &total_comp_width, - const idx_t &offset, bool swap) { - const data_ptr_t source_ptr = swap ? temp_ptr : orig_ptr; - const data_ptr_t target_ptr = swap ? orig_ptr : temp_ptr; - if (count > 1) { - const idx_t total_offset = col_offset + offset; - auto temp_val = make_unsafe_uniq_array_uninitialized(row_width); - const data_ptr_t val = temp_val.get(); - const auto comp_width = total_comp_width - offset; - for (idx_t i = 1; i < count; i++) { - FastMemcpy(val, source_ptr + i * row_width, row_width); - idx_t j = i; - while (j > 0 && - FastMemcmp(source_ptr + (j - 1) * row_width + total_offset, val + total_offset, comp_width) > 0) { - FastMemcpy(source_ptr + j * row_width, source_ptr + (j - 1) * row_width, row_width); - j--; - } - FastMemcpy(source_ptr + j * row_width, val, row_width); - } - } - if (swap) { - memcpy(target_ptr, source_ptr, count * row_width); - } -} - -//! MSD radix sort that switches to insertion sort with low bucket sizes -void RadixSortMSD(const data_ptr_t orig_ptr, const data_ptr_t temp_ptr, const idx_t &count, const idx_t &col_offset, - const idx_t &row_width, const idx_t &comp_width, const idx_t &offset, idx_t locations[], bool swap) { - const data_ptr_t source_ptr = swap ? temp_ptr : orig_ptr; - const data_ptr_t target_ptr = swap ? orig_ptr : temp_ptr; - // Init counts to 0 - memset(locations, 0, SortConstants::MSD_RADIX_LOCATIONS * sizeof(idx_t)); - idx_t *counts = locations + 1; - // Collect counts - const idx_t total_offset = col_offset + offset; - data_ptr_t offset_ptr = source_ptr + total_offset; - for (idx_t i = 0; i < count; i++) { - counts[*offset_ptr]++; - offset_ptr += row_width; - } - // Compute locations from counts - idx_t max_count = 0; - for (idx_t radix = 0; radix < SortConstants::VALUES_PER_RADIX; radix++) { - max_count = MaxValue(max_count, counts[radix]); - counts[radix] += locations[radix]; - } - if (max_count != count) { - // Re-order the data in temporary array - data_ptr_t row_ptr = source_ptr; - for (idx_t i = 0; i < count; i++) { - const idx_t &radix_offset = locations[*(row_ptr + total_offset)]++; - FastMemcpy(target_ptr + radix_offset * row_width, row_ptr, row_width); - row_ptr += row_width; - } - swap = !swap; - } - // Check if done - if (offset == comp_width - 1) { - if (swap) { - memcpy(orig_ptr, temp_ptr, count * row_width); - } - return; - } - if (max_count == count) { - RadixSortMSD(orig_ptr, temp_ptr, count, col_offset, row_width, comp_width, offset + 1, - locations + SortConstants::MSD_RADIX_LOCATIONS, swap); - return; - } - // Recurse - idx_t radix_count = locations[0]; - for (idx_t radix = 0; radix < SortConstants::VALUES_PER_RADIX; radix++) { - const idx_t loc = (locations[radix] - radix_count) * row_width; - if (radix_count > SortConstants::INSERTION_SORT_THRESHOLD) { - RadixSortMSD(orig_ptr + loc, temp_ptr + loc, radix_count, col_offset, row_width, comp_width, offset + 1, - locations + SortConstants::MSD_RADIX_LOCATIONS, swap); - } else if (radix_count != 0) { - InsertionSort(orig_ptr + loc, temp_ptr + loc, radix_count, col_offset, row_width, comp_width, offset + 1, - swap); - } - radix_count = locations[radix + 1] - locations[radix]; - } -} - -//! Calls different sort functions, depending on the count and sorting sizes -void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr, const idx_t &count, const idx_t &col_offset, - const idx_t &sorting_size, const SortLayout &sort_layout, bool contains_string) { - - if (contains_string) { - auto begin = duckdb_pdqsort::PDQIterator(dataptr, sort_layout.entry_size); - auto end = begin + count; - duckdb_pdqsort::PDQConstants constants(sort_layout.entry_size, col_offset, sorting_size, *end); - return duckdb_pdqsort::pdqsort_branchless(begin, begin + count, constants); - } - - if (count <= SortConstants::INSERTION_SORT_THRESHOLD) { - return InsertionSort(dataptr, nullptr, count, col_offset, sort_layout.entry_size, sorting_size, 0, false); - } - - if (sorting_size <= SortConstants::MSD_RADIX_SORT_SIZE_THRESHOLD) { - return RadixSortLSD(buffer_manager, dataptr, count, col_offset, sort_layout.entry_size, sorting_size); - } - - const auto block_size = buffer_manager.GetBlockSize(); - auto temp_block = - buffer_manager.Allocate(MemoryTag::ORDER_BY, MaxValue(count * sort_layout.entry_size, block_size)); - auto pre_allocated_array = - make_unsafe_uniq_array_uninitialized(sorting_size * SortConstants::MSD_RADIX_LOCATIONS); - RadixSortMSD(dataptr, temp_block.Ptr(), count, col_offset, sort_layout.entry_size, sorting_size, 0, - pre_allocated_array.get(), false); -} - -//! Identifies sequences of rows that are tied, and calls radix sort on these -static void SubSortTiedTuples(BufferManager &buffer_manager, const data_ptr_t dataptr, const idx_t &count, - const idx_t &col_offset, const idx_t &sorting_size, bool ties[], - const SortLayout &sort_layout, bool contains_string) { - D_ASSERT(!ties[count - 1]); - for (idx_t i = 0; i < count; i++) { - if (!ties[i]) { - continue; - } - idx_t j; - for (j = i + 1; j < count; j++) { - if (!ties[j]) { - break; - } - } - RadixSort(buffer_manager, dataptr + i * sort_layout.entry_size, j - i + 1, col_offset, sorting_size, - sort_layout, contains_string); - i = j; - } -} - -void LocalSortState::SortInMemory() { - auto &sb = *sorted_blocks.back(); - auto &block = *sb.radix_sorting_data.back(); - const auto &count = block.count; - auto handle = buffer_manager->Pin(block.block); - const auto dataptr = handle.Ptr(); - // Assign an index to each row - data_ptr_t idx_dataptr = dataptr + sort_layout->comparison_size; - for (uint32_t i = 0; i < count; i++) { - Store(i, idx_dataptr); - idx_dataptr += sort_layout->entry_size; - } - // Radix sort and break ties until no more ties, or until all columns are sorted - idx_t sorting_size = 0; - idx_t col_offset = 0; - unsafe_unique_array ties_ptr; - bool *ties = nullptr; - bool contains_string = false; - for (idx_t i = 0; i < sort_layout->column_count; i++) { - sorting_size += sort_layout->column_sizes[i]; - contains_string = contains_string || sort_layout->logical_types[i].InternalType() == PhysicalType::VARCHAR; - if (sort_layout->constant_size[i] && i < sort_layout->column_count - 1) { - // Add columns to the sorting size until we reach a variable size column, or the last column - continue; - } - - if (!ties) { - // This is the first sort - RadixSort(*buffer_manager, dataptr, count, col_offset, sorting_size, *sort_layout, contains_string); - ties_ptr = make_unsafe_uniq_array_uninitialized(count); - ties = ties_ptr.get(); - std::fill_n(ties, count - 1, true); - ties[count - 1] = false; - } else { - // For subsequent sorts, we only have to subsort the tied tuples - SubSortTiedTuples(*buffer_manager, dataptr, count, col_offset, sorting_size, ties, *sort_layout, - contains_string); - } - - contains_string = false; - - if (sort_layout->constant_size[i] && i == sort_layout->column_count - 1) { - // All columns are sorted, no ties to break because last column is constant size - break; - } - - ComputeTies(dataptr, count, col_offset, sorting_size, ties, *sort_layout); - if (!AnyTies(ties, count)) { - // No ties, stop sorting - break; - } - - if (!sort_layout->constant_size[i]) { - SortTiedBlobs(*buffer_manager, sb, ties, dataptr, count, i, *sort_layout); - if (!AnyTies(ties, count)) { - // No more ties after tie-breaking, stop - break; - } - } - - col_offset += sorting_size; - sorting_size = 0; - } -} - -} // namespace duckdb diff --git a/src/common/sorting/sort.cpp b/src/common/sort/sort.cpp similarity index 95% rename from src/common/sorting/sort.cpp rename to src/common/sort/sort.cpp index 2159878ff5b2..8f2a1e6e7b5c 100644 --- a/src/common/sorting/sort.cpp +++ b/src/common/sort/sort.cpp @@ -141,7 +141,7 @@ class SortLocalSinkState : public LocalSinkState { D_ASSERT(!sorted_run); // TODO: we want to pass "sort.is_index_sort" instead of just "false" here // so that we can do an approximate sort, but that causes issues in the ART - sorted_run = make_uniq(context, sort.key_layout, sort.payload_layout, false); + sorted_run = make_uniq(context, sort, false); } public: @@ -366,8 +366,7 @@ ProgressData Sort::GetSinkProgress(ClientContext &context, GlobalSinkState &gsta class SortGlobalSourceState : public GlobalSourceState { public: SortGlobalSourceState(const Sort &sort, ClientContext &context, SortGlobalSinkState &sink_p) - : sink(sink_p), merger(*sort.decode_sort_key, sort.key_layout, std::move(sink.sorted_runs), - sort.output_projection_columns, sink.partition_size, sink.external, false), + : sink(sink_p), merger(sort, std::move(sink.sorted_runs), sink.partition_size, sink.external, false), merger_global_state(merger.total_count == 0 ? nullptr : merger.GetGlobalSourceState(context)) { // TODO: we want to pass "sort.is_index_sort" instead of just "false" here // so that we can do an approximate sort, but that causes issues in the ART @@ -378,6 +377,15 @@ class SortGlobalSourceState : public GlobalSourceState { return merger_global_state ? merger_global_state->MaxThreads() : 1; } + void Destroy() { + if (!merger_global_state) { + return; + } + auto guard = merger_global_state->Lock(); + merger.sorted_runs.clear(); + sink.temporary_memory_state.reset(); + } + public: //! The global sink state SortGlobalSinkState &sink; @@ -477,16 +485,26 @@ SourceResultType Sort::MaterializeColumnData(ExecutionContext &context, Operator } // Merge into global output collection - auto guard = gstate.Lock(); - if (!gstate.column_data) { - gstate.column_data = std::move(local_column_data); - } else { - gstate.column_data->Merge(*local_column_data); + { + auto guard = gstate.Lock(); + if (!gstate.column_data) { + gstate.column_data = std::move(local_column_data); + } else { + gstate.column_data->Merge(*local_column_data); + } } + // Destroy local state before returning + input.local_state.Cast().merger_local_state.reset(); + // Return type indicates whether materialization is done const auto progress_data = GetProgress(context.client, input.global_state); - return progress_data.done == progress_data.total ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT; + if (progress_data.done == progress_data.total) { + // Destroy global state before returning + gstate.Destroy(); + return SourceResultType::FINISHED; + } + return SourceResultType::HAVE_MORE_OUTPUT; } unique_ptr Sort::GetColumnData(OperatorSourceInput &input) const { @@ -502,12 +520,15 @@ SourceResultType Sort::MaterializeSortedRun(ExecutionContext &context, OperatorS } auto &lstate = input.local_state.Cast(); OperatorSourceInput merger_input {*gstate.merger_global_state, *lstate.merger_local_state, input.interrupt_state}; - return gstate.merger.MaterializeMerge(context, merger_input); + return gstate.merger.MaterializeSortedRun(context, merger_input); } unique_ptr Sort::GetSortedRun(GlobalSourceState &global_state) { auto &gstate = global_state.Cast(); - return gstate.merger.GetMaterialized(gstate); + if (gstate.merger.total_count == 0) { + return nullptr; + } + return gstate.merger.GetSortedRun(*gstate.merger_global_state); } } // namespace duckdb diff --git a/src/common/sort/sort_state.cpp b/src/common/sort/sort_state.cpp deleted file mode 100644 index 369f032f197c..000000000000 --- a/src/common/sort/sort_state.cpp +++ /dev/null @@ -1,487 +0,0 @@ -#include "duckdb/common/fast_mem.hpp" -#include "duckdb/common/radix.hpp" -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/sort/sort.hpp" -#include "duckdb/common/sort/sorted_block.hpp" -#include "duckdb/storage/buffer/buffer_pool.hpp" - -#include -#include - -namespace duckdb { - -idx_t GetNestedSortingColSize(idx_t &col_size, const LogicalType &type) { - auto physical_type = type.InternalType(); - if (TypeIsConstantSize(physical_type)) { - col_size += GetTypeIdSize(physical_type); - return 0; - } else { - switch (physical_type) { - case PhysicalType::VARCHAR: { - // Nested strings are between 4 and 11 chars long for alignment - auto size_before_str = col_size; - col_size += 11; - col_size -= (col_size - 12) % 8; - return col_size - size_before_str; - } - case PhysicalType::LIST: - // Lists get 2 bytes (null and empty list) - col_size += 2; - return GetNestedSortingColSize(col_size, ListType::GetChildType(type)); - case PhysicalType::STRUCT: - // Structs get 1 bytes (null) - col_size++; - return GetNestedSortingColSize(col_size, StructType::GetChildType(type, 0)); - case PhysicalType::ARRAY: - // Arrays get 1 bytes (null) - col_size++; - return GetNestedSortingColSize(col_size, ArrayType::GetChildType(type)); - default: - throw NotImplementedException("Unable to order column with type %s", type.ToString()); - } - } -} - -SortLayout::SortLayout(const vector &orders) - : column_count(orders.size()), all_constant(true), comparison_size(0), entry_size(0) { - vector blob_layout_types; - for (idx_t i = 0; i < column_count; i++) { - const auto &order = orders[i]; - - order_types.push_back(order.type); - order_by_null_types.push_back(order.null_order); - auto &expr = *order.expression; - logical_types.push_back(expr.return_type); - - auto physical_type = expr.return_type.InternalType(); - constant_size.push_back(TypeIsConstantSize(physical_type)); - - if (order.stats) { - stats.push_back(order.stats.get()); - has_null.push_back(stats.back()->CanHaveNull()); - } else { - stats.push_back(nullptr); - has_null.push_back(true); - } - - idx_t col_size = has_null.back() ? 1 : 0; - prefix_lengths.push_back(0); - if (!TypeIsConstantSize(physical_type) && physical_type != PhysicalType::VARCHAR) { - prefix_lengths.back() = GetNestedSortingColSize(col_size, expr.return_type); - } else if (physical_type == PhysicalType::VARCHAR) { - idx_t size_before = col_size; - if (stats.back() && StringStats::HasMaxStringLength(*stats.back())) { - col_size += StringStats::MaxStringLength(*stats.back()); - if (col_size > 12) { - col_size = 12; - } else { - constant_size.back() = true; - } - } else { - col_size = 12; - } - prefix_lengths.back() = col_size - size_before; - } else { - col_size += GetTypeIdSize(physical_type); - } - - comparison_size += col_size; - column_sizes.push_back(col_size); - } - entry_size = comparison_size + sizeof(uint32_t); - - // 8-byte alignment - if (entry_size % 8 != 0) { - // First assign more bytes to strings instead of aligning - idx_t bytes_to_fill = 8 - (entry_size % 8); - for (idx_t col_idx = 0; col_idx < column_count; col_idx++) { - if (bytes_to_fill == 0) { - break; - } - if (logical_types[col_idx].InternalType() == PhysicalType::VARCHAR && stats[col_idx] && - StringStats::HasMaxStringLength(*stats[col_idx])) { - idx_t diff = StringStats::MaxStringLength(*stats[col_idx]) - prefix_lengths[col_idx]; - if (diff > 0) { - // Increase all sizes accordingly - idx_t increase = MinValue(bytes_to_fill, diff); - column_sizes[col_idx] += increase; - prefix_lengths[col_idx] += increase; - constant_size[col_idx] = increase == diff; - comparison_size += increase; - entry_size += increase; - bytes_to_fill -= increase; - } - } - } - entry_size = AlignValue(entry_size); - } - - for (idx_t col_idx = 0; col_idx < column_count; col_idx++) { - all_constant = all_constant && constant_size[col_idx]; - if (!constant_size[col_idx]) { - sorting_to_blob_col[col_idx] = blob_layout_types.size(); - blob_layout_types.push_back(logical_types[col_idx]); - } - } - - blob_layout.Initialize(blob_layout_types); -} - -SortLayout SortLayout::GetPrefixComparisonLayout(idx_t num_prefix_cols) const { - SortLayout result; - result.column_count = num_prefix_cols; - result.all_constant = true; - result.comparison_size = 0; - for (idx_t col_idx = 0; col_idx < num_prefix_cols; col_idx++) { - result.order_types.push_back(order_types[col_idx]); - result.order_by_null_types.push_back(order_by_null_types[col_idx]); - result.logical_types.push_back(logical_types[col_idx]); - - result.all_constant = result.all_constant && constant_size[col_idx]; - result.constant_size.push_back(constant_size[col_idx]); - - result.comparison_size += column_sizes[col_idx]; - result.column_sizes.push_back(column_sizes[col_idx]); - - result.prefix_lengths.push_back(prefix_lengths[col_idx]); - result.stats.push_back(stats[col_idx]); - result.has_null.push_back(has_null[col_idx]); - } - result.entry_size = entry_size; - result.blob_layout = blob_layout; - result.sorting_to_blob_col = sorting_to_blob_col; - return result; -} - -LocalSortState::LocalSortState() : initialized(false) { - if (!Radix::IsLittleEndian()) { - throw NotImplementedException("Sorting is not supported on big endian architectures"); - } -} - -void LocalSortState::Initialize(GlobalSortState &global_sort_state, BufferManager &buffer_manager_p) { - sort_layout = &global_sort_state.sort_layout; - payload_layout = &global_sort_state.payload_layout; - buffer_manager = &buffer_manager_p; - const auto block_size = buffer_manager->GetBlockSize(); - - // Radix sorting data - auto entries_per_block = RowDataCollection::EntriesPerBlock(sort_layout->entry_size, block_size); - radix_sorting_data = make_uniq(*buffer_manager, entries_per_block, sort_layout->entry_size); - - // Blob sorting data - if (!sort_layout->all_constant) { - auto blob_row_width = sort_layout->blob_layout.GetRowWidth(); - entries_per_block = RowDataCollection::EntriesPerBlock(blob_row_width, block_size); - blob_sorting_data = make_uniq(*buffer_manager, entries_per_block, blob_row_width); - blob_sorting_heap = make_uniq(*buffer_manager, block_size, 1U, true); - } - - // Payload data - auto payload_row_width = payload_layout->GetRowWidth(); - entries_per_block = RowDataCollection::EntriesPerBlock(payload_row_width, block_size); - payload_data = make_uniq(*buffer_manager, entries_per_block, payload_row_width); - payload_heap = make_uniq(*buffer_manager, block_size, 1U, true); - initialized = true; -} - -void LocalSortState::SinkChunk(DataChunk &sort, DataChunk &payload) { - D_ASSERT(sort.size() == payload.size()); - // Build and serialize sorting data to radix sortable rows - auto data_pointers = FlatVector::GetData(addresses); - auto handles = radix_sorting_data->Build(sort.size(), data_pointers, nullptr); - for (idx_t sort_col = 0; sort_col < sort.ColumnCount(); sort_col++) { - bool has_null = sort_layout->has_null[sort_col]; - bool nulls_first = sort_layout->order_by_null_types[sort_col] == OrderByNullType::NULLS_FIRST; - bool desc = sort_layout->order_types[sort_col] == OrderType::DESCENDING; - RowOperations::RadixScatter(sort.data[sort_col], sort.size(), sel_ptr, sort.size(), data_pointers, desc, - has_null, nulls_first, sort_layout->prefix_lengths[sort_col], - sort_layout->column_sizes[sort_col]); - } - - // Also fully serialize blob sorting columns (to be able to break ties - if (!sort_layout->all_constant) { - DataChunk blob_chunk; - blob_chunk.SetCardinality(sort.size()); - for (idx_t sort_col = 0; sort_col < sort.ColumnCount(); sort_col++) { - if (!sort_layout->constant_size[sort_col]) { - blob_chunk.data.emplace_back(sort.data[sort_col]); - } - } - handles = blob_sorting_data->Build(blob_chunk.size(), data_pointers, nullptr); - auto blob_data = blob_chunk.ToUnifiedFormat(); - RowOperations::Scatter(blob_chunk, blob_data.get(), sort_layout->blob_layout, addresses, *blob_sorting_heap, - sel_ptr, blob_chunk.size()); - D_ASSERT(blob_sorting_heap->keep_pinned); - } - - // Finally, serialize payload data - handles = payload_data->Build(payload.size(), data_pointers, nullptr); - auto input_data = payload.ToUnifiedFormat(); - RowOperations::Scatter(payload, input_data.get(), *payload_layout, addresses, *payload_heap, sel_ptr, - payload.size()); - D_ASSERT(payload_heap->keep_pinned); -} - -idx_t LocalSortState::SizeInBytes() const { - idx_t size_in_bytes = radix_sorting_data->SizeInBytes() + payload_data->SizeInBytes(); - if (!sort_layout->all_constant) { - size_in_bytes += blob_sorting_data->SizeInBytes() + blob_sorting_heap->SizeInBytes(); - } - if (!payload_layout->AllConstant()) { - size_in_bytes += payload_heap->SizeInBytes(); - } - return size_in_bytes; -} - -void LocalSortState::Sort(GlobalSortState &global_sort_state, bool reorder_heap) { - D_ASSERT(radix_sorting_data->count == payload_data->count); - if (radix_sorting_data->count == 0) { - return; - } - // Move all data to a single SortedBlock - sorted_blocks.emplace_back(make_uniq(*buffer_manager, global_sort_state)); - auto &sb = *sorted_blocks.back(); - // Fixed-size sorting data - auto sorting_block = ConcatenateBlocks(*radix_sorting_data); - sb.radix_sorting_data.push_back(std::move(sorting_block)); - // Variable-size sorting data - if (!sort_layout->all_constant) { - auto &blob_data = *blob_sorting_data; - auto new_block = ConcatenateBlocks(blob_data); - sb.blob_sorting_data->data_blocks.push_back(std::move(new_block)); - } - // Payload data - auto payload_block = ConcatenateBlocks(*payload_data); - sb.payload_data->data_blocks.push_back(std::move(payload_block)); - // Now perform the actual sort - SortInMemory(); - // Re-order before the merge sort - ReOrder(global_sort_state, reorder_heap); -} - -unique_ptr LocalSortState::ConcatenateBlocks(RowDataCollection &row_data) { - // Don't copy and delete if there is only one block. - if (row_data.blocks.size() == 1) { - auto new_block = std::move(row_data.blocks[0]); - row_data.blocks.clear(); - row_data.count = 0; - return new_block; - } - // Create block with the correct capacity - auto &buffer_manager = row_data.buffer_manager; - const idx_t &entry_size = row_data.entry_size; - idx_t capacity = MaxValue((buffer_manager.GetBlockSize() + entry_size - 1) / entry_size, row_data.count); - auto new_block = make_uniq(MemoryTag::ORDER_BY, buffer_manager, capacity, entry_size); - new_block->count = row_data.count; - auto new_block_handle = buffer_manager.Pin(new_block->block); - data_ptr_t new_block_ptr = new_block_handle.Ptr(); - // Copy the data of the blocks into a single block - for (idx_t i = 0; i < row_data.blocks.size(); i++) { - auto &block = row_data.blocks[i]; - auto block_handle = buffer_manager.Pin(block->block); - memcpy(new_block_ptr, block_handle.Ptr(), block->count * entry_size); - new_block_ptr += block->count * entry_size; - block.reset(); - } - row_data.blocks.clear(); - row_data.count = 0; - return new_block; -} - -void LocalSortState::ReOrder(SortedData &sd, data_ptr_t sorting_ptr, RowDataCollection &heap, GlobalSortState &gstate, - bool reorder_heap) { - sd.swizzled = reorder_heap; - auto &unordered_data_block = sd.data_blocks.back(); - const idx_t count = unordered_data_block->count; - auto unordered_data_handle = buffer_manager->Pin(unordered_data_block->block); - const data_ptr_t unordered_data_ptr = unordered_data_handle.Ptr(); - // Create new block that will hold re-ordered row data - auto ordered_data_block = make_uniq(MemoryTag::ORDER_BY, *buffer_manager, - unordered_data_block->capacity, unordered_data_block->entry_size); - ordered_data_block->count = count; - auto ordered_data_handle = buffer_manager->Pin(ordered_data_block->block); - data_ptr_t ordered_data_ptr = ordered_data_handle.Ptr(); - // Re-order fixed-size row layout - const idx_t row_width = sd.layout.GetRowWidth(); - const idx_t sorting_entry_size = gstate.sort_layout.entry_size; - for (idx_t i = 0; i < count; i++) { - auto index = Load(sorting_ptr); - FastMemcpy(ordered_data_ptr, unordered_data_ptr + index * row_width, row_width); - ordered_data_ptr += row_width; - sorting_ptr += sorting_entry_size; - } - ordered_data_block->block->SetSwizzling( - sd.layout.AllConstant() || !sd.swizzled ? nullptr : "LocalSortState::ReOrder.ordered_data"); - // Replace the unordered data block with the re-ordered data block - sd.data_blocks.clear(); - sd.data_blocks.push_back(std::move(ordered_data_block)); - // Deal with the heap (if necessary) - if (!sd.layout.AllConstant() && reorder_heap) { - // Swizzle the column pointers to offsets - RowOperations::SwizzleColumns(sd.layout, ordered_data_handle.Ptr(), count); - sd.data_blocks.back()->block->SetSwizzling(nullptr); - // Create a single heap block to store the ordered heap - idx_t total_byte_offset = - std::accumulate(heap.blocks.begin(), heap.blocks.end(), (idx_t)0, - [](idx_t a, const unique_ptr &b) { return a + b->byte_offset; }); - idx_t heap_block_size = MaxValue(total_byte_offset, buffer_manager->GetBlockSize()); - auto ordered_heap_block = make_uniq(MemoryTag::ORDER_BY, *buffer_manager, heap_block_size, 1U); - ordered_heap_block->count = count; - ordered_heap_block->byte_offset = total_byte_offset; - auto ordered_heap_handle = buffer_manager->Pin(ordered_heap_block->block); - data_ptr_t ordered_heap_ptr = ordered_heap_handle.Ptr(); - // Fill the heap in order - ordered_data_ptr = ordered_data_handle.Ptr(); - const idx_t heap_pointer_offset = sd.layout.GetHeapOffset(); - for (idx_t i = 0; i < count; i++) { - auto heap_row_ptr = Load(ordered_data_ptr + heap_pointer_offset); - auto heap_row_size = Load(heap_row_ptr); - memcpy(ordered_heap_ptr, heap_row_ptr, heap_row_size); - ordered_heap_ptr += heap_row_size; - ordered_data_ptr += row_width; - } - // Swizzle the base pointer to the offset of each row in the heap - RowOperations::SwizzleHeapPointer(sd.layout, ordered_data_handle.Ptr(), ordered_heap_handle.Ptr(), count); - // Move the re-ordered heap to the SortedData, and clear the local heap - sd.heap_blocks.push_back(std::move(ordered_heap_block)); - heap.pinned_blocks.clear(); - heap.blocks.clear(); - heap.count = 0; - } -} - -void LocalSortState::ReOrder(GlobalSortState &gstate, bool reorder_heap) { - auto &sb = *sorted_blocks.back(); - auto sorting_handle = buffer_manager->Pin(sb.radix_sorting_data.back()->block); - const data_ptr_t sorting_ptr = sorting_handle.Ptr() + gstate.sort_layout.comparison_size; - // Re-order variable size sorting columns - if (!gstate.sort_layout.all_constant) { - ReOrder(*sb.blob_sorting_data, sorting_ptr, *blob_sorting_heap, gstate, reorder_heap); - } - // And the payload - ReOrder(*sb.payload_data, sorting_ptr, *payload_heap, gstate, reorder_heap); -} - -GlobalSortState::GlobalSortState(ClientContext &context_p, const vector &orders, - RowLayout &payload_layout) - : context(context_p), buffer_manager(BufferManager::GetBufferManager(context)), sort_layout(SortLayout(orders)), - payload_layout(payload_layout), block_capacity(0), external(false) { -} - -void GlobalSortState::AddLocalState(LocalSortState &local_sort_state) { - if (!local_sort_state.radix_sorting_data) { - return; - } - - // Sort accumulated data - // we only re-order the heap when the data is expected to not fit in memory - // re-ordering the heap avoids random access when reading/merging but incurs a significant cost of shuffling data - // when data fits in memory, doing random access on reads is cheaper than re-shuffling - local_sort_state.Sort(*this, external || !local_sort_state.sorted_blocks.empty()); - - // Append local state sorted data to this global state - lock_guard append_guard(lock); - for (auto &sb : local_sort_state.sorted_blocks) { - sorted_blocks.push_back(std::move(sb)); - } - auto &payload_heap = local_sort_state.payload_heap; - for (idx_t i = 0; i < payload_heap->blocks.size(); i++) { - heap_blocks.push_back(std::move(payload_heap->blocks[i])); - pinned_blocks.push_back(std::move(payload_heap->pinned_blocks[i])); - } - if (!sort_layout.all_constant) { - auto &blob_heap = local_sort_state.blob_sorting_heap; - for (idx_t i = 0; i < blob_heap->blocks.size(); i++) { - heap_blocks.push_back(std::move(blob_heap->blocks[i])); - pinned_blocks.push_back(std::move(blob_heap->pinned_blocks[i])); - } - } -} - -void GlobalSortState::PrepareMergePhase() { - // Determine if we need to use do an external sort - idx_t total_heap_size = - std::accumulate(sorted_blocks.begin(), sorted_blocks.end(), (idx_t)0, - [](idx_t a, const unique_ptr &b) { return a + b->HeapSize(); }); - if (external || (pinned_blocks.empty() && total_heap_size * 4 > buffer_manager.GetQueryMaxMemory())) { - external = true; - } - // Use the data that we have to determine which partition size to use during the merge - if (external && total_heap_size > 0) { - // If we have variable size data we need to be conservative, as there might be skew - idx_t max_block_size = 0; - for (auto &sb : sorted_blocks) { - idx_t size_in_bytes = sb->SizeInBytes(); - if (size_in_bytes > max_block_size) { - max_block_size = size_in_bytes; - block_capacity = sb->Count(); - } - } - } else { - for (auto &sb : sorted_blocks) { - block_capacity = MaxValue(block_capacity, sb->Count()); - } - } - // Unswizzle and pin heap blocks if we can fit everything in memory - if (!external) { - for (auto &sb : sorted_blocks) { - sb->blob_sorting_data->Unswizzle(); - sb->payload_data->Unswizzle(); - } - } -} - -void GlobalSortState::InitializeMergeRound() { - D_ASSERT(sorted_blocks_temp.empty()); - // If we reverse this list, the blocks that were merged last will be merged first in the next round - // These are still in memory, therefore this reduces the amount of read/write to disk! - std::reverse(sorted_blocks.begin(), sorted_blocks.end()); - // Uneven number of blocks - keep one on the side - if (sorted_blocks.size() % 2 == 1) { - odd_one_out = std::move(sorted_blocks.back()); - sorted_blocks.pop_back(); - } - // Init merge path path indices - pair_idx = 0; - num_pairs = sorted_blocks.size() / 2; - l_start = 0; - r_start = 0; - // Allocate room for merge results - for (idx_t p_idx = 0; p_idx < num_pairs; p_idx++) { - sorted_blocks_temp.emplace_back(); - } -} - -void GlobalSortState::CompleteMergeRound(bool keep_radix_data) { - sorted_blocks.clear(); - for (auto &sorted_block_vector : sorted_blocks_temp) { - sorted_blocks.push_back(make_uniq(buffer_manager, *this)); - sorted_blocks.back()->AppendSortedBlocks(sorted_block_vector); - } - sorted_blocks_temp.clear(); - if (odd_one_out) { - sorted_blocks.push_back(std::move(odd_one_out)); - odd_one_out = nullptr; - } - // Only one block left: Done! - if (sorted_blocks.size() == 1 && !keep_radix_data) { - sorted_blocks[0]->radix_sorting_data.clear(); - sorted_blocks[0]->blob_sorting_data = nullptr; - } -} -void GlobalSortState::Print() { - PayloadScanner scanner(*this, false); - DataChunk chunk; - chunk.Initialize(Allocator::DefaultAllocator(), scanner.GetPayloadTypes()); - for (;;) { - scanner.Scan(chunk); - const auto count = chunk.size(); - if (!count) { - break; - } - chunk.Print(); - } -} - -} // namespace duckdb diff --git a/src/common/sort/sorted_block.cpp b/src/common/sort/sorted_block.cpp deleted file mode 100644 index c4766c956b40..000000000000 --- a/src/common/sort/sorted_block.cpp +++ /dev/null @@ -1,387 +0,0 @@ -#include "duckdb/common/sort/sorted_block.hpp" - -#include "duckdb/common/constants.hpp" -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/sort/sort.hpp" -#include "duckdb/common/types/row/row_data_collection.hpp" - -#include - -namespace duckdb { - -SortedData::SortedData(SortedDataType type, const RowLayout &layout, BufferManager &buffer_manager, - GlobalSortState &state) - : type(type), layout(layout), swizzled(state.external), buffer_manager(buffer_manager), state(state) { -} - -idx_t SortedData::Count() { - idx_t count = std::accumulate(data_blocks.begin(), data_blocks.end(), (idx_t)0, - [](idx_t a, const unique_ptr &b) { return a + b->count; }); - if (!layout.AllConstant() && state.external) { - D_ASSERT(count == std::accumulate(heap_blocks.begin(), heap_blocks.end(), (idx_t)0, - [](idx_t a, const unique_ptr &b) { return a + b->count; })); - } - return count; -} - -void SortedData::CreateBlock() { - const auto block_size = buffer_manager.GetBlockSize(); - auto capacity = MaxValue((block_size + layout.GetRowWidth() - 1) / layout.GetRowWidth(), state.block_capacity); - data_blocks.push_back(make_uniq(MemoryTag::ORDER_BY, buffer_manager, capacity, layout.GetRowWidth())); - if (!layout.AllConstant() && state.external) { - heap_blocks.push_back(make_uniq(MemoryTag::ORDER_BY, buffer_manager, block_size, 1U)); - D_ASSERT(data_blocks.size() == heap_blocks.size()); - } -} - -unique_ptr SortedData::CreateSlice(idx_t start_block_index, idx_t end_block_index, idx_t end_entry_index) { - // Add the corresponding blocks to the result - auto result = make_uniq(type, layout, buffer_manager, state); - for (idx_t i = start_block_index; i <= end_block_index; i++) { - result->data_blocks.push_back(data_blocks[i]->Copy()); - if (!layout.AllConstant() && state.external) { - result->heap_blocks.push_back(heap_blocks[i]->Copy()); - } - } - // All of the blocks that come before block with idx = start_block_idx can be reset (other references exist) - for (idx_t i = 0; i < start_block_index; i++) { - data_blocks[i]->block = nullptr; - if (!layout.AllConstant() && state.external) { - heap_blocks[i]->block = nullptr; - } - } - // Use start and end entry indices to set the boundaries - D_ASSERT(end_entry_index <= result->data_blocks.back()->count); - result->data_blocks.back()->count = end_entry_index; - if (!layout.AllConstant() && state.external) { - result->heap_blocks.back()->count = end_entry_index; - } - return result; -} - -void SortedData::Unswizzle() { - if (layout.AllConstant() || !swizzled) { - return; - } - for (idx_t i = 0; i < data_blocks.size(); i++) { - auto &data_block = data_blocks[i]; - auto &heap_block = heap_blocks[i]; - D_ASSERT(data_block->block->IsSwizzled()); - auto data_handle_p = buffer_manager.Pin(data_block->block); - auto heap_handle_p = buffer_manager.Pin(heap_block->block); - RowOperations::UnswizzlePointers(layout, data_handle_p.Ptr(), heap_handle_p.Ptr(), data_block->count); - state.heap_blocks.push_back(std::move(heap_block)); - state.pinned_blocks.push_back(std::move(heap_handle_p)); - } - swizzled = false; - heap_blocks.clear(); -} - -SortedBlock::SortedBlock(BufferManager &buffer_manager, GlobalSortState &state) - : buffer_manager(buffer_manager), state(state), sort_layout(state.sort_layout), - payload_layout(state.payload_layout) { - blob_sorting_data = make_uniq(SortedDataType::BLOB, sort_layout.blob_layout, buffer_manager, state); - payload_data = make_uniq(SortedDataType::PAYLOAD, payload_layout, buffer_manager, state); -} - -idx_t SortedBlock::Count() const { - idx_t count = std::accumulate(radix_sorting_data.begin(), radix_sorting_data.end(), (idx_t)0, - [](idx_t a, const unique_ptr &b) { return a + b->count; }); - if (!sort_layout.all_constant) { - D_ASSERT(count == blob_sorting_data->Count()); - } - D_ASSERT(count == payload_data->Count()); - return count; -} - -void SortedBlock::InitializeWrite() { - CreateBlock(); - if (!sort_layout.all_constant) { - blob_sorting_data->CreateBlock(); - } - payload_data->CreateBlock(); -} - -void SortedBlock::CreateBlock() { - const auto block_size = buffer_manager.GetBlockSize(); - auto capacity = MaxValue((block_size + sort_layout.entry_size - 1) / sort_layout.entry_size, state.block_capacity); - radix_sorting_data.push_back( - make_uniq(MemoryTag::ORDER_BY, buffer_manager, capacity, sort_layout.entry_size)); -} - -void SortedBlock::AppendSortedBlocks(vector> &sorted_blocks) { - D_ASSERT(Count() == 0); - for (auto &sb : sorted_blocks) { - for (auto &radix_block : sb->radix_sorting_data) { - radix_sorting_data.push_back(std::move(radix_block)); - } - if (!sort_layout.all_constant) { - for (auto &blob_block : sb->blob_sorting_data->data_blocks) { - blob_sorting_data->data_blocks.push_back(std::move(blob_block)); - } - for (auto &heap_block : sb->blob_sorting_data->heap_blocks) { - blob_sorting_data->heap_blocks.push_back(std::move(heap_block)); - } - } - for (auto &payload_data_block : sb->payload_data->data_blocks) { - payload_data->data_blocks.push_back(std::move(payload_data_block)); - } - if (!payload_data->layout.AllConstant()) { - for (auto &payload_heap_block : sb->payload_data->heap_blocks) { - payload_data->heap_blocks.push_back(std::move(payload_heap_block)); - } - } - } -} - -void SortedBlock::GlobalToLocalIndex(const idx_t &global_idx, idx_t &local_block_index, idx_t &local_entry_index) { - if (global_idx == Count()) { - local_block_index = radix_sorting_data.size() - 1; - local_entry_index = radix_sorting_data.back()->count; - return; - } - D_ASSERT(global_idx < Count()); - local_entry_index = global_idx; - for (local_block_index = 0; local_block_index < radix_sorting_data.size(); local_block_index++) { - const idx_t &block_count = radix_sorting_data[local_block_index]->count; - if (local_entry_index >= block_count) { - local_entry_index -= block_count; - } else { - break; - } - } - D_ASSERT(local_entry_index < radix_sorting_data[local_block_index]->count); -} - -unique_ptr SortedBlock::CreateSlice(const idx_t start, const idx_t end, idx_t &entry_idx) { - // Identify blocks/entry indices of this slice - idx_t start_block_index; - idx_t start_entry_index; - GlobalToLocalIndex(start, start_block_index, start_entry_index); - idx_t end_block_index; - idx_t end_entry_index; - GlobalToLocalIndex(end, end_block_index, end_entry_index); - // Add the corresponding blocks to the result - auto result = make_uniq(buffer_manager, state); - for (idx_t i = start_block_index; i <= end_block_index; i++) { - result->radix_sorting_data.push_back(radix_sorting_data[i]->Copy()); - } - // Reset all blocks that come before block with idx = start_block_idx (slice holds new reference) - for (idx_t i = 0; i < start_block_index; i++) { - radix_sorting_data[i]->block = nullptr; - } - // Use start and end entry indices to set the boundaries - entry_idx = start_entry_index; - D_ASSERT(end_entry_index <= result->radix_sorting_data.back()->count); - result->radix_sorting_data.back()->count = end_entry_index; - // Same for the var size sorting data - if (!sort_layout.all_constant) { - result->blob_sorting_data = blob_sorting_data->CreateSlice(start_block_index, end_block_index, end_entry_index); - } - // And the payload data - result->payload_data = payload_data->CreateSlice(start_block_index, end_block_index, end_entry_index); - return result; -} - -idx_t SortedBlock::HeapSize() const { - idx_t result = 0; - if (!sort_layout.all_constant) { - for (auto &block : blob_sorting_data->heap_blocks) { - result += block->capacity; - } - } - if (!payload_layout.AllConstant()) { - for (auto &block : payload_data->heap_blocks) { - result += block->capacity; - } - } - return result; -} - -idx_t SortedBlock::SizeInBytes() const { - idx_t bytes = 0; - for (idx_t i = 0; i < radix_sorting_data.size(); i++) { - bytes += radix_sorting_data[i]->capacity * sort_layout.entry_size; - if (!sort_layout.all_constant) { - bytes += blob_sorting_data->data_blocks[i]->capacity * sort_layout.blob_layout.GetRowWidth(); - bytes += blob_sorting_data->heap_blocks[i]->capacity; - } - bytes += payload_data->data_blocks[i]->capacity * payload_layout.GetRowWidth(); - if (!payload_layout.AllConstant()) { - bytes += payload_data->heap_blocks[i]->capacity; - } - } - return bytes; -} - -SBScanState::SBScanState(BufferManager &buffer_manager, GlobalSortState &state) - : buffer_manager(buffer_manager), sort_layout(state.sort_layout), state(state), block_idx(0), entry_idx(0) { -} - -void SBScanState::PinRadix(idx_t block_idx_to) { - auto &radix_sorting_data = sb->radix_sorting_data; - D_ASSERT(block_idx_to < radix_sorting_data.size()); - auto &block = radix_sorting_data[block_idx_to]; - if (!radix_handle.IsValid() || radix_handle.GetBlockHandle() != block->block) { - radix_handle = buffer_manager.Pin(block->block); - } -} - -void SBScanState::PinData(SortedData &sd) { - D_ASSERT(block_idx < sd.data_blocks.size()); - auto &data_handle = sd.type == SortedDataType::BLOB ? blob_sorting_data_handle : payload_data_handle; - auto &heap_handle = sd.type == SortedDataType::BLOB ? blob_sorting_heap_handle : payload_heap_handle; - - auto &data_block = sd.data_blocks[block_idx]; - if (!data_handle.IsValid() || data_handle.GetBlockHandle() != data_block->block) { - data_handle = buffer_manager.Pin(data_block->block); - } - if (sd.layout.AllConstant() || !state.external) { - return; - } - auto &heap_block = sd.heap_blocks[block_idx]; - if (!heap_handle.IsValid() || heap_handle.GetBlockHandle() != heap_block->block) { - heap_handle = buffer_manager.Pin(heap_block->block); - } -} - -data_ptr_t SBScanState::RadixPtr() const { - return radix_handle.Ptr() + entry_idx * sort_layout.entry_size; -} - -data_ptr_t SBScanState::DataPtr(SortedData &sd) const { - auto &data_handle = sd.type == SortedDataType::BLOB ? blob_sorting_data_handle : payload_data_handle; - D_ASSERT(sd.data_blocks[block_idx]->block->Readers() != 0 && - data_handle.GetBlockHandle() == sd.data_blocks[block_idx]->block); - return data_handle.Ptr() + entry_idx * sd.layout.GetRowWidth(); -} - -data_ptr_t SBScanState::HeapPtr(SortedData &sd) const { - return BaseHeapPtr(sd) + Load(DataPtr(sd) + sd.layout.GetHeapOffset()); -} - -data_ptr_t SBScanState::BaseHeapPtr(SortedData &sd) const { - auto &heap_handle = sd.type == SortedDataType::BLOB ? blob_sorting_heap_handle : payload_heap_handle; - D_ASSERT(!sd.layout.AllConstant() && state.external); - D_ASSERT(sd.heap_blocks[block_idx]->block->Readers() != 0 && - heap_handle.GetBlockHandle() == sd.heap_blocks[block_idx]->block); - return heap_handle.Ptr(); -} - -idx_t SBScanState::Remaining() const { - const auto &blocks = sb->radix_sorting_data; - idx_t remaining = 0; - if (block_idx < blocks.size()) { - remaining += blocks[block_idx]->count - entry_idx; - for (idx_t i = block_idx + 1; i < blocks.size(); i++) { - remaining += blocks[i]->count; - } - } - return remaining; -} - -void SBScanState::SetIndices(idx_t block_idx_to, idx_t entry_idx_to) { - block_idx = block_idx_to; - entry_idx = entry_idx_to; -} - -PayloadScanner::PayloadScanner(SortedData &sorted_data, GlobalSortState &global_sort_state, bool flush_p) { - auto count = sorted_data.Count(); - auto &layout = sorted_data.layout; - const auto block_size = global_sort_state.buffer_manager.GetBlockSize(); - - // Create collections to put the data into so we can use RowDataCollectionScanner - rows = make_uniq(global_sort_state.buffer_manager, block_size, 1U); - rows->count = count; - - heap = make_uniq(global_sort_state.buffer_manager, block_size, 1U); - if (!sorted_data.layout.AllConstant()) { - heap->count = count; - } - - if (flush_p) { - // If we are flushing, we can just move the data - rows->blocks = std::move(sorted_data.data_blocks); - if (!layout.AllConstant()) { - heap->blocks = std::move(sorted_data.heap_blocks); - } - } else { - // Not flushing, create references to the blocks - for (auto &block : sorted_data.data_blocks) { - rows->blocks.emplace_back(block->Copy()); - } - if (!layout.AllConstant()) { - for (auto &block : sorted_data.heap_blocks) { - heap->blocks.emplace_back(block->Copy()); - } - } - } - - scanner = make_uniq(*rows, *heap, layout, global_sort_state.external, flush_p); -} - -PayloadScanner::PayloadScanner(GlobalSortState &global_sort_state, bool flush_p) - : PayloadScanner(*global_sort_state.sorted_blocks[0]->payload_data, global_sort_state, flush_p) { -} - -PayloadScanner::PayloadScanner(GlobalSortState &global_sort_state, idx_t block_idx, bool flush_p) { - auto &sorted_data = *global_sort_state.sorted_blocks[0]->payload_data; - auto count = sorted_data.data_blocks[block_idx]->count; - auto &layout = sorted_data.layout; - const auto block_size = global_sort_state.buffer_manager.GetBlockSize(); - - // Create collections to put the data into so we can use RowDataCollectionScanner - rows = make_uniq(global_sort_state.buffer_manager, block_size, 1U); - if (flush_p) { - rows->blocks.emplace_back(std::move(sorted_data.data_blocks[block_idx])); - } else { - rows->blocks.emplace_back(sorted_data.data_blocks[block_idx]->Copy()); - } - rows->count = count; - - heap = make_uniq(global_sort_state.buffer_manager, block_size, 1U); - if (!sorted_data.layout.AllConstant() && sorted_data.swizzled) { - if (flush_p) { - heap->blocks.emplace_back(std::move(sorted_data.heap_blocks[block_idx])); - } else { - heap->blocks.emplace_back(sorted_data.heap_blocks[block_idx]->Copy()); - } - heap->count = count; - } - - scanner = make_uniq(*rows, *heap, layout, global_sort_state.external, flush_p); -} - -void PayloadScanner::Scan(DataChunk &chunk) { - scanner->Scan(chunk); -} - -int SBIterator::ComparisonValue(ExpressionType comparison) { - switch (comparison) { - case ExpressionType::COMPARE_LESSTHAN: - case ExpressionType::COMPARE_GREATERTHAN: - return -1; - case ExpressionType::COMPARE_LESSTHANOREQUALTO: - case ExpressionType::COMPARE_GREATERTHANOREQUALTO: - return 0; - default: - throw InternalException("Unimplemented comparison type for IEJoin!"); - } -} - -static idx_t GetBlockCountWithEmptyCheck(const GlobalSortState &gss) { - D_ASSERT(!gss.sorted_blocks.empty()); - return gss.sorted_blocks[0]->radix_sorting_data.size(); -} - -SBIterator::SBIterator(GlobalSortState &gss, ExpressionType comparison, idx_t entry_idx_p) - : sort_layout(gss.sort_layout), block_count(GetBlockCountWithEmptyCheck(gss)), block_capacity(gss.block_capacity), - entry_size(sort_layout.entry_size), all_constant(sort_layout.all_constant), external(gss.external), - cmp(ComparisonValue(comparison)), scan(gss.buffer_manager, gss), block_ptr(nullptr), entry_ptr(nullptr) { - - scan.sb = gss.sorted_blocks[0].get(); - scan.block_idx = block_count; - SetIndex(entry_idx_p); -} - -} // namespace duckdb diff --git a/src/common/sorting/sorted_run.cpp b/src/common/sort/sorted_run.cpp similarity index 65% rename from src/common/sorting/sorted_run.cpp rename to src/common/sort/sorted_run.cpp index 0554990fa4ee..c72c554ec924 100644 --- a/src/common/sorting/sorted_run.cpp +++ b/src/common/sort/sorted_run.cpp @@ -1,6 +1,7 @@ #include "duckdb/common/sorting/sorted_run.hpp" #include "duckdb/common/types/row/tuple_data_collection.hpp" +#include "duckdb/common/sorting/sort.hpp" #include "duckdb/common/sorting/sort_key.hpp" #include "duckdb/common/types/row/block_iterator.hpp" @@ -9,14 +10,161 @@ namespace duckdb { -SortedRun::SortedRun(ClientContext &context_p, shared_ptr key_layout, - shared_ptr payload_layout, bool is_index_sort_p) - : context(context_p), - key_data(make_uniq(BufferManager::GetBufferManager(context), std::move(key_layout))), - payload_data( - payload_layout && payload_layout->ColumnCount() != 0 - ? make_uniq(BufferManager::GetBufferManager(context), std::move(payload_layout)) - : nullptr), +//===--------------------------------------------------------------------===// +// SortedRunScanState +//===--------------------------------------------------------------------===// +SortedRunScanState::SortedRunScanState(ClientContext &context, const Sort &sort_p) + : sort(sort_p), key_executor(context, *sort.decode_sort_key) { + key.Initialize(context, {sort.key_layout->GetTypes()[0]}); + decoded_key.Initialize(context, {sort.decode_sort_key->return_type}); +} + +void SortedRunScanState::Scan(const SortedRun &sorted_run, const Vector &sort_key_pointers, const idx_t &count, + DataChunk &chunk) { + const auto sort_key_type = sort.key_layout->GetSortKeyType(); + switch (sort_key_type) { + case SortKeyType::NO_PAYLOAD_FIXED_8: + return TemplatedScan(sorted_run, sort_key_pointers, count, chunk); + case SortKeyType::NO_PAYLOAD_FIXED_16: + return TemplatedScan(sorted_run, sort_key_pointers, count, chunk); + case SortKeyType::NO_PAYLOAD_FIXED_24: + return TemplatedScan(sorted_run, sort_key_pointers, count, chunk); + case SortKeyType::NO_PAYLOAD_FIXED_32: + return TemplatedScan(sorted_run, sort_key_pointers, count, chunk); + case SortKeyType::NO_PAYLOAD_VARIABLE_32: + return TemplatedScan(sorted_run, sort_key_pointers, count, chunk); + case SortKeyType::PAYLOAD_FIXED_16: + return TemplatedScan(sorted_run, sort_key_pointers, count, chunk); + case SortKeyType::PAYLOAD_FIXED_24: + return TemplatedScan(sorted_run, sort_key_pointers, count, chunk); + case SortKeyType::PAYLOAD_FIXED_32: + return TemplatedScan(sorted_run, sort_key_pointers, count, chunk); + case SortKeyType::PAYLOAD_VARIABLE_32: + return TemplatedScan(sorted_run, sort_key_pointers, count, chunk); + default: + throw NotImplementedException("SortedRunMergerLocalState::ScanPartition for %s", + EnumUtil::ToString(sort_key_type)); + } +} + +template +void TemplatedGetKeyAndPayload(SORT_KEY *const *const sort_keys, const idx_t &count, DataChunk &key, + data_ptr_t *const payload_ptrs) { + const auto key_data = FlatVector::GetData(key.data[0]); + for (idx_t i = 0; i < count; i++) { + auto &sort_key = *sort_keys[i]; + sort_key.Deconstruct(key_data[i]); + if (SORT_KEY::HAS_PAYLOAD) { + payload_ptrs[i] = sort_key.GetPayload(); + } + } + key.SetCardinality(count); +} + +template +void GetKeyAndPayload(SORT_KEY *const *const sort_keys, const idx_t &count, DataChunk &key, + data_ptr_t *const payload_ptrs) { + const auto type_id = key.data[0].GetType().id(); + switch (type_id) { + case LogicalTypeId::BLOB: + return TemplatedGetKeyAndPayload(sort_keys, count, key, payload_ptrs); + case LogicalTypeId::BIGINT: + return TemplatedGetKeyAndPayload(sort_keys, count, key, payload_ptrs); + default: + throw NotImplementedException("GetKeyAndPayload for %s", EnumUtil::ToString(type_id)); + } +} + +template +void TemplatedReconstructSortKey(SORT_KEY *const *const sort_keys, const idx_t &count) { + for (idx_t i = 0; i < count; i++) { + sort_keys[i]->ByteSwap(); + } +} + +template +void ReconstructSortKey(SORT_KEY *const *const sort_keys, const idx_t &count, const LogicalType &type) { + switch (type.id()) { + case LogicalTypeId::BLOB: + return TemplatedReconstructSortKey(sort_keys, count); + case LogicalTypeId::BIGINT: + break; // NOP + default: + throw NotImplementedException("ReconstructSortKey for %s", EnumUtil::ToString(type.id())); + } +} + +template +void SortedRunScanState::TemplatedScan(const SortedRun &sorted_run, const Vector &sort_key_pointers, const idx_t &count, + DataChunk &chunk) { + using SORT_KEY = SortKey; + + const auto &output_projection_columns = sort.output_projection_columns; + idx_t opc_idx = 0; + + const auto sort_keys = FlatVector::GetData(sort_key_pointers); + const auto payload_ptrs = FlatVector::GetData(payload_state.chunk_state.row_locations); + bool gathered_payload = false; + + // Decode from key + if (!output_projection_columns[0].is_payload) { + key.Reset(); + GetKeyAndPayload(sort_keys, count, key, payload_ptrs); + + decoded_key.Reset(); + key_executor.Execute(key, decoded_key); + ReconstructSortKey(sort_keys, count, key.data[0].GetType()); + + const auto &decoded_key_entries = StructVector::GetEntries(decoded_key.data[0]); + for (; opc_idx < output_projection_columns.size(); opc_idx++) { + const auto &opc = output_projection_columns[opc_idx]; + if (opc.is_payload) { + break; + } + chunk.data[opc.output_col_idx].Reference(*decoded_key_entries[opc.layout_col_idx]); + } + + gathered_payload = true; + } + + // If there are no payload columns, we're done here + if (opc_idx != output_projection_columns.size()) { + if (!gathered_payload) { + // Gather row pointers from keys + for (idx_t i = 0; i < count; i++) { + payload_ptrs[i] = sort_keys[i]->GetPayload(); + } + } + + // Init scan state + auto &payload_data = *sorted_run.payload_data; + if (payload_state.pin_state.properties == TupleDataPinProperties::INVALID) { + payload_data.InitializeScan(payload_state, TupleDataPinProperties::ALREADY_PINNED); + } + TupleDataCollection::ResetCachedCastVectors(payload_state.chunk_state, payload_state.chunk_state.column_ids); + + // Now gather from payload + for (; opc_idx < output_projection_columns.size(); opc_idx++) { + const auto &opc = output_projection_columns[opc_idx]; + D_ASSERT(opc.is_payload); + payload_data.Gather(payload_state.chunk_state.row_locations, *FlatVector::IncrementalSelectionVector(), + count, opc.layout_col_idx, chunk.data[opc.output_col_idx], + *FlatVector::IncrementalSelectionVector(), + payload_state.chunk_state.cached_cast_vectors[opc.layout_col_idx]); + } + } + + chunk.SetCardinality(count); +} + +//===--------------------------------------------------------------------===// +// SortedRun +//===--------------------------------------------------------------------===// +SortedRun::SortedRun(ClientContext &context_p, const Sort &sort_p, bool is_index_sort_p) + : context(context_p), sort(sort_p), key_data(make_uniq(context, sort.key_layout)), + payload_data(sort.payload_layout && sort.payload_layout->ColumnCount() != 0 + ? make_uniq(context, sort.payload_layout) + : nullptr), is_index_sort(is_index_sort_p), finalized(false) { key_data->InitializeAppend(key_append_state, TupleDataPinProperties::KEEP_EVERYTHING_PINNED); if (payload_data) { @@ -25,8 +173,7 @@ SortedRun::SortedRun(ClientContext &context_p, shared_ptr key_l } unique_ptr SortedRun::CreateRunForMaterialization() const { - auto res = make_uniq(context, key_data->GetLayoutPtr(), - payload_data ? payload_data->GetLayoutPtr() : nullptr, is_index_sort); + auto res = make_uniq(context, sort, is_index_sort); res->key_append_state.pin_state.properties = TupleDataPinProperties::UNPIN_AFTER_DONE; res->payload_append_state.pin_state.properties = TupleDataPinProperties::UNPIN_AFTER_DONE; res->finalized = true; @@ -304,6 +451,9 @@ void SortedRun::Finalize(bool external) { const auto sort_key_type = key_data->GetLayout().GetSortKeyType(); if (!SortKeyUtils::IsConstantSize(sort_key_type) || SortKeyUtils::HasPayload(sort_key_type)) { Reorder(context, key_data, payload_data); + } else { + // This ensures keys are unpinned even if they are constant size and have no payload + key_data->Unpin(); } } diff --git a/src/common/sorting/sorted_run_merger.cpp b/src/common/sort/sorted_run_merger.cpp similarity index 90% rename from src/common/sorting/sorted_run_merger.cpp rename to src/common/sort/sorted_run_merger.cpp index eb879edc51ac..874a7fc0415d 100644 --- a/src/common/sorting/sorted_run_merger.cpp +++ b/src/common/sort/sorted_run_merger.cpp @@ -1,5 +1,6 @@ #include "duckdb/common/sorting/sorted_run_merger.hpp" +#include "duckdb/common/sorting/sort.hpp" #include "duckdb/common/sorting/sorted_run.hpp" #include "duckdb/common/sorting/sort_key.hpp" #include "duckdb/common/types/row/block_iterator.hpp" @@ -100,7 +101,7 @@ class SortedRunMergerLocalState : public LocalSourceState { //! Whether this thread has finished the work it has been assigned bool TaskFinished() const; //! Do the work this thread has been assigned - void ExecuteTask(SortedRunMergerGlobalState &gstate, optional_ptr chunk); + SourceResultType ExecuteTask(SortedRunMergerGlobalState &gstate, optional_ptr chunk); private: //! Computes upper partition boundaries using K-way Merge Path @@ -154,12 +155,10 @@ class SortedRunMergerLocalState : public LocalSourceState { //! Variables for scanning idx_t merged_partition_count; idx_t merged_partition_index; - TupleDataScanState payload_state; - //! For decoding sort keys - ExpressionExecutor key_executor; - DataChunk key; - DataChunk decoded_key; + //! For scanning + Vector sort_key_pointers; + SortedRunScanState sorted_run_scan_state; }; //===--------------------------------------------------------------------===// @@ -172,7 +171,7 @@ class SortedRunMergerGlobalState : public GlobalSourceState { merger(merger_p), num_runs(merger.sorted_runs.size()), num_partitions((merger.total_count + (merger.partition_size - 1)) / merger.partition_size), iterator_state_type(GetBlockIteratorStateType(merger.external)), - sort_key_type(merger.key_layout->GetSortKeyType()), next_partition_idx(0), total_scanned(0), + sort_key_type(merger.sort.key_layout->GetSortKeyType()), next_partition_idx(0), total_scanned(0), destroy_partition_idx(0) { // Initialize partitions partitions.resize(num_partitions); @@ -292,7 +291,7 @@ SortedRunMergerLocalState::SortedRunMergerLocalState(SortedRunMergerGlobalState : iterator_state_type(gstate.iterator_state_type), sort_key_type(gstate.sort_key_type), task(SortedRunMergerTask::FINISHED), run_boundaries(gstate.num_runs), merged_partition_count(DConstants::INVALID_INDEX), merged_partition_index(DConstants::INVALID_INDEX), - key_executor(gstate.context, gstate.merger.decode_sort_key) { + sorted_run_scan_state(gstate.context, gstate.merger.sort), sort_key_pointers(LogicalType::POINTER) { for (const auto &run : gstate.merger.sorted_runs) { auto &key_data = *run->key_data; switch (iterator_state_type) { @@ -308,8 +307,6 @@ SortedRunMergerLocalState::SortedRunMergerLocalState(SortedRunMergerGlobalState EnumUtil::ToString(iterator_state_type)); } } - key.Initialize(gstate.context, {gstate.merger.key_layout->GetTypes()[0]}); - decoded_key.Initialize(gstate.context, {gstate.merger.decode_sort_key.return_type}); } bool SortedRunMergerLocalState::TaskFinished() const { @@ -328,7 +325,8 @@ bool SortedRunMergerLocalState::TaskFinished() const { } } -void SortedRunMergerLocalState::ExecuteTask(SortedRunMergerGlobalState &gstate, optional_ptr chunk) { +SourceResultType SortedRunMergerLocalState::ExecuteTask(SortedRunMergerGlobalState &gstate, + optional_ptr chunk) { D_ASSERT(task != SortedRunMergerTask::FINISHED); switch (task) { case SortedRunMergerTask::COMPUTE_BOUNDARIES: @@ -352,14 +350,18 @@ void SortedRunMergerLocalState::ExecuteTask(SortedRunMergerGlobalState &gstate, if (!chunk || chunk->size() == 0) { gstate.DestroyScannedData(); gstate.partitions[partition_idx.GetIndex()]->scanned = true; - gstate.total_scanned += merged_partition_count; + const auto scan_count_after_adding = gstate.total_scanned.fetch_add(merged_partition_count); partition_idx = optional_idx::Invalid(); task = SortedRunMergerTask::FINISHED; + if (scan_count_after_adding == gstate.merger.total_count) { + return SourceResultType::FINISHED; + } } break; default: throw NotImplementedException("SortedRunMergerLocalState::ExecuteTask for task"); } + return SourceResultType::HAVE_MORE_OUTPUT; } void SortedRunMergerLocalState::ComputePartitionBoundaries(SortedRunMergerGlobalState &gstate, @@ -718,61 +720,16 @@ void SortedRunMergerLocalState::TemplatedScanPartition(SortedRunMergerGlobalStat using SORT_KEY = SortKey; const auto count = MinValue(merged_partition_count - merged_partition_index, STANDARD_VECTOR_SIZE); - const auto &output_projection_columns = gstate.merger.output_projection_columns; - idx_t opc_idx = 0; - + // Grab pointers to sort keys const auto merged_partition_keys = reinterpret_cast(merged_partition.get()) + merged_partition_index; - const auto payload_ptrs = FlatVector::GetData(payload_state.chunk_state.row_locations); - bool gathered_payload = false; - - // Decode from key - if (!output_projection_columns[0].is_payload) { - key.Reset(); - GetKeyAndPayload(merged_partition_keys, count, key, payload_ptrs); - - decoded_key.Reset(); - key_executor.Execute(key, decoded_key); - - const auto &decoded_key_entries = StructVector::GetEntries(decoded_key.data[0]); - for (; opc_idx < output_projection_columns.size(); opc_idx++) { - const auto &opc = output_projection_columns[opc_idx]; - if (opc.is_payload) { - break; - } - chunk.data[opc.output_col_idx].Reference(*decoded_key_entries[opc.layout_col_idx]); - } - gathered_payload = true; - } - - // If there are no payload columns, we're done here - if (opc_idx != output_projection_columns.size()) { - if (!gathered_payload) { - // Gather row pointers from keys - for (idx_t i = 0; i < count; i++) { - payload_ptrs[i] = merged_partition_keys[i].GetPayload(); - } - } - - // Init scan state - auto &payload_data = *gstate.merger.sorted_runs.back()->payload_data; - if (payload_state.pin_state.properties == TupleDataPinProperties::INVALID) { - payload_data.InitializeScan(payload_state, TupleDataPinProperties::ALREADY_PINNED); - } - TupleDataCollection::ResetCachedCastVectors(payload_state.chunk_state, payload_state.chunk_state.column_ids); - - // Now gather from payload - for (; opc_idx < output_projection_columns.size(); opc_idx++) { - const auto &opc = output_projection_columns[opc_idx]; - D_ASSERT(opc.is_payload); - payload_data.Gather(payload_state.chunk_state.row_locations, *FlatVector::IncrementalSelectionVector(), - count, opc.layout_col_idx, chunk.data[opc.output_col_idx], - *FlatVector::IncrementalSelectionVector(), - payload_state.chunk_state.cached_cast_vectors[opc.layout_col_idx]); - } + const auto sort_keys = FlatVector::GetData(sort_key_pointers); + for (idx_t i = 0; i < count; i++) { + sort_keys[i] = &merged_partition_keys[i]; } - merged_partition_index += count; - chunk.SetCardinality(count); + + // Scan + sorted_run_scan_state.Scan(*gstate.merger.sorted_runs[0], sort_key_pointers, count, chunk); } void SortedRunMergerLocalState::MaterializePartition(SortedRunMergerGlobalState &gstate) { @@ -812,7 +769,9 @@ void SortedRunMergerLocalState::MaterializePartition(SortedRunMergerGlobalState // Add to global state lock_guard guard(gstate.materialized_partition_lock); - gstate.materialized_partitions.resize(partition_idx.GetIndex()); + if (gstate.materialized_partitions.size() < partition_idx.GetIndex() + 1) { + gstate.materialized_partitions.resize(partition_idx.GetIndex() + 1); + } gstate.materialized_partitions[partition_idx.GetIndex()] = std::move(sorted_run); } @@ -833,7 +792,7 @@ unique_ptr SortedRunMergerLocalState::TemplatedMaterializePartition(S while (merged_partition_index < merged_partition_count) { const auto count = MinValue(merged_partition_count - merged_partition_index, STANDARD_VECTOR_SIZE); - for (idx_t i = 0; i < count + count; i++) { + for (idx_t i = 0; i < count; i++) { auto &key = merged_partition_keys[merged_partition_index + i]; key_locations[i] = data_ptr_cast(&key); if (!SORT_KEY::CONSTANT_SIZE) { @@ -855,7 +814,7 @@ unique_ptr SortedRunMergerLocalState::TemplatedMaterializePartition(S if (!sorted_run->payload_data->GetLayout().AllConstant()) { sorted_run->payload_data->FindHeapPointers(payload_data_input, count); } - sorted_run->payload_append_state.chunk_state.heap_sizes.Reference(key_data_input.heap_sizes); + sorted_run->payload_append_state.chunk_state.heap_sizes.Reference(payload_data_input.heap_sizes); sorted_run->payload_data->Build(sorted_run->payload_append_state.pin_state, sorted_run->payload_append_state.chunk_state, 0, count); sorted_run->payload_data->CopyRows(sorted_run->payload_append_state.chunk_state, payload_data_input, @@ -876,18 +835,16 @@ unique_ptr SortedRunMergerLocalState::TemplatedMaterializePartition(S //===--------------------------------------------------------------------===// // Sorted Run Merger //===--------------------------------------------------------------------===// -SortedRunMerger::SortedRunMerger(const Expression &decode_sort_key_p, shared_ptr key_layout_p, - vector> &&sorted_runs_p, - const vector &output_projection_columns_p, +SortedRunMerger::SortedRunMerger(const Sort &sort_p, vector> &&sorted_runs_p, idx_t partition_size_p, bool external_p, bool is_index_sort_p) - : decode_sort_key(decode_sort_key_p), key_layout(std::move(key_layout_p)), sorted_runs(std::move(sorted_runs_p)), - output_projection_columns(output_projection_columns_p), total_count(SortedRunsTotalCount(sorted_runs)), + : sort(sort_p), sorted_runs(std::move(sorted_runs_p)), total_count(SortedRunsTotalCount(sorted_runs)), partition_size(partition_size_p), external(external_p), is_index_sort(is_index_sort_p) { } unique_ptr SortedRunMerger::GetLocalSourceState(ExecutionContext &, GlobalSourceState &gstate_p) const { auto &gstate = gstate_p.Cast(); + auto guard = gstate.Lock(); return make_uniq(gstate); } @@ -929,30 +886,28 @@ ProgressData SortedRunMerger::GetProgress(ClientContext &, GlobalSourceState &gs //===--------------------------------------------------------------------===// // Non-Standard Interface //===--------------------------------------------------------------------===// -SourceResultType SortedRunMerger::MaterializeMerge(ExecutionContext &, OperatorSourceInput &input) const { +SourceResultType SortedRunMerger::MaterializeSortedRun(ExecutionContext &, OperatorSourceInput &input) const { auto &gstate = input.global_state.Cast(); auto &lstate = input.local_state.Cast(); + SourceResultType res = SourceResultType::HAVE_MORE_OUTPUT; while (true) { if (!lstate.TaskFinished() || gstate.AssignTask(lstate)) { - lstate.ExecuteTask(gstate, nullptr); + res = lstate.ExecuteTask(gstate, nullptr); } else { break; } } - if (gstate.total_scanned == total_count) { - // This signals that the data has been fully materialized - return SourceResultType::FINISHED; - } - // This signals that no more tasks are left, but that the data has not yet been fully materialized - return SourceResultType::HAVE_MORE_OUTPUT; + // The thread that completes the materialization returns FINISHED, all other threads return HAVE_MORE_OUTPUT + return res; } -unique_ptr SortedRunMerger::GetMaterialized(GlobalSourceState &global_state) { +unique_ptr SortedRunMerger::GetSortedRun(GlobalSourceState &global_state) { auto &gstate = global_state.Cast(); + D_ASSERT(total_count != 0); + lock_guard guard(gstate.materialized_partition_lock); if (gstate.materialized_partitions.empty()) { - D_ASSERT(total_count == 0); return nullptr; } auto &target = *gstate.materialized_partitions[0]; @@ -963,7 +918,9 @@ unique_ptr SortedRunMerger::GetMaterialized(GlobalSourceState &global target.payload_data->Combine(*source.payload_data); } } - return std::move(gstate.materialized_partitions[0]); + auto res = std::move(gstate.materialized_partitions[0]); + gstate.materialized_partitions.clear(); + return res; } } // namespace duckdb diff --git a/src/common/sorting/CMakeLists.txt b/src/common/sorting/CMakeLists.txt deleted file mode 100644 index 09366bb04bf0..000000000000 --- a/src/common/sorting/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -add_library_unity(duckdb_sorting OBJECT hashed_sort.cpp sort.cpp sorted_run.cpp - sorted_run_merger.cpp) - -set(ALL_OBJECT_FILES - ${ALL_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/common/string_util.cpp b/src/common/string_util.cpp index 71e865880c82..d3931d0bfa24 100644 --- a/src/common/string_util.cpp +++ b/src/common/string_util.cpp @@ -287,9 +287,13 @@ bool StringUtil::IsUpper(const string &str) { // Jenkins hash function: https://en.wikipedia.org/wiki/Jenkins_hash_function uint64_t StringUtil::CIHash(const string &str) { + return StringUtil::CIHash(str.c_str(), str.size()); +} + +uint64_t StringUtil::CIHash(const char *str, idx_t size) { uint32_t hash = 0; - for (auto c : str) { - hash += static_cast(StringUtil::CharacterToLower(static_cast(c))); + for (idx_t i = 0; i < size; i++) { + hash += static_cast(StringUtil::CharacterToLower(static_cast(str[i]))); hash += hash << 10; hash ^= hash >> 6; } @@ -572,6 +576,15 @@ static unique_ptr ParseJSON(const string &json, yyjson_doc *doc, yy const bool bool_val = yyjson_get_bool(root); return make_uniq(bool_val ? "true" : "false"); } + case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_UINT: + return make_uniq(to_string(unsafe_yyjson_get_uint(root))); + case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_SINT: + return make_uniq(to_string(unsafe_yyjson_get_sint(root))); + case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL: + case YYJSON_TYPE_RAW | YYJSON_SUBTYPE_NONE: + return make_uniq(to_string(unsafe_yyjson_get_real(root))); + case YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE: + return make_uniq("null"); default: yyjson_doc_free(doc); throw SerializationException("Failed to parse JSON string: %s", json); @@ -693,6 +706,21 @@ string StringUtil::ToComplexJSONMap(const ComplexJSON &complex_json) { return ComplexJSON::GetValueRecursive(complex_json); } +string StringUtil::ValidateJSON(const char *data, const idx_t &len) { + // Same flags as in JSON extension + static constexpr auto READ_FLAG = + YYJSON_READ_ALLOW_INF_AND_NAN | YYJSON_READ_ALLOW_TRAILING_COMMAS | YYJSON_READ_BIGNUM_AS_RAW; + yyjson_read_err error; + yyjson_doc *doc = yyjson_read_opts((char *)data, len, READ_FLAG, nullptr, &error); // NOLINT: for yyjson + if (error.code != YYJSON_READ_SUCCESS) { + return StringUtil::Format("Malformed JSON at byte %lld of input: %s. Input: \"%s\"", error.pos, error.msg, + string(data, len)); + } + + yyjson_doc_free(doc); + return string(); +} + string StringUtil::ExceptionToJSONMap(ExceptionType type, const string &message, const unordered_map &map) { D_ASSERT(map.find("exception_type") == map.end()); diff --git a/src/common/symbols.cpp b/src/common/symbols.cpp index 626c2fe9754d..9eb7206b5f69 100644 --- a/src/common/symbols.cpp +++ b/src/common/symbols.cpp @@ -111,7 +111,6 @@ template class unique_ptr; template class unique_ptr; template class unique_ptr; template class unique_ptr; -template class unique_ptr; template class unique_ptr; template class unique_ptr; diff --git a/src/common/types.cpp b/src/common/types.cpp index 0c48f75e473d..6542fafc2772 100644 --- a/src/common/types.cpp +++ b/src/common/types.cpp @@ -31,6 +31,9 @@ namespace duckdb { +constexpr idx_t ArrayType::MAX_ARRAY_SIZE; +const idx_t UnionType::MAX_UNION_MEMBERS; + LogicalType::LogicalType() : LogicalType(LogicalTypeId::INVALID) { } @@ -159,6 +162,8 @@ PhysicalType LogicalType::GetInternalType() { return PhysicalType::UNKNOWN; case LogicalTypeId::AGGREGATE_STATE: return PhysicalType::VARCHAR; + case LogicalTypeId::GEOMETRY: + return PhysicalType::VARCHAR; default: throw InternalException("Invalid LogicalType %s", ToString()); } @@ -624,6 +629,10 @@ LogicalType GetUserTypeRecursive(const LogicalType &type, ClientContext &context if (type.id() == LogicalTypeId::LIST) { return LogicalType::LIST(GetUserTypeRecursive(ListType::GetChildType(type), context)); } + if (type.id() == LogicalTypeId::ARRAY) { + return LogicalType::ARRAY(GetUserTypeRecursive(ArrayType::GetChildType(type), context), + ArrayType::GetSize(type)); + } if (type.id() == LogicalTypeId::MAP) { return LogicalType::MAP(GetUserTypeRecursive(MapType::KeyType(type), context), GetUserTypeRecursive(MapType::ValueType(type), context)); @@ -1340,6 +1349,8 @@ static idx_t GetLogicalTypeScore(const LogicalType &type) { return 102; case LogicalTypeId::BIGNUM: return 103; + case LogicalTypeId::GEOMETRY: + return 104; // nested types case LogicalTypeId::STRUCT: return 125; @@ -2010,6 +2021,15 @@ LogicalType LogicalType::VARIANT() { return LogicalType(LogicalTypeId::VARIANT, std::move(info)); } +//===--------------------------------------------------------------------===// +// Spatial Types +//===--------------------------------------------------------------------===// + +LogicalType LogicalType::GEOMETRY() { + auto info = make_shared_ptr(); + return LogicalType(LogicalTypeId::GEOMETRY, std::move(info)); +} + //===--------------------------------------------------------------------===// // Logical Type //===--------------------------------------------------------------------===// diff --git a/src/common/types/CMakeLists.txt b/src/common/types/CMakeLists.txt index c3b15f036cd2..5fb744ad2a64 100644 --- a/src/common/types/CMakeLists.txt +++ b/src/common/types/CMakeLists.txt @@ -36,7 +36,8 @@ add_library_unity( vector_buffer.cpp vector.cpp vector_cache.cpp - vector_constants.cpp) + vector_constants.cpp + geometry.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/common/types/column/column_data_allocator.cpp b/src/common/types/column/column_data_allocator.cpp index 976a993de39e..b4f3f4d74c19 100644 --- a/src/common/types/column/column_data_allocator.cpp +++ b/src/common/types/column/column_data_allocator.cpp @@ -37,10 +37,10 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) { switch (type) { case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR: case ColumnDataAllocatorType::HYBRID: - alloc.allocator = other.alloc.allocator; + alloc.buffer_manager = other.alloc.buffer_manager; break; case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR: - alloc.buffer_manager = other.alloc.buffer_manager; + alloc.allocator = other.alloc.allocator; break; default: throw InternalException("Unrecognized column data allocator type"); diff --git a/src/common/types/column/column_data_collection.cpp b/src/common/types/column/column_data_collection.cpp index f30af0b937bc..ad478d515182 100644 --- a/src/common/types/column/column_data_collection.cpp +++ b/src/common/types/column/column_data_collection.cpp @@ -842,7 +842,8 @@ void ColumnDataCopyArray(ColumnDataMetaData &meta_data, const UnifiedVectorForma child_vector.ToUnifiedFormat(copy_count * array_size, child_vector_data); // Broadcast and sync the validity of the array vector to the child vector - + // This requires creating a copy of the validity mask: we cannot modify the input validity + child_vector_data.validity = ValidityMask(child_vector_data.validity, child_vector_data.validity.Capacity()); if (source_data.validity.IsMaskSet()) { for (idx_t i = 0; i < copy_count; i++) { auto source_idx = source_data.sel->get_index(offset + i); @@ -1284,11 +1285,11 @@ struct ValueResultEquals { bool ColumnDataCollection::ResultEquals(const ColumnDataCollection &left, const ColumnDataCollection &right, string &error_message, bool ordered) { if (left.ColumnCount() != right.ColumnCount()) { - error_message = "Column count mismatch"; + error_message = StringUtil::Format("Column count mismatch (%d vs %d)", left.Count(), right.Count()); return false; } if (left.Count() != right.Count()) { - error_message = "Row count mismatch"; + error_message = StringUtil::Format("Row count mismatch (%d vs %d)", left.Count(), right.Count()); return false; } auto left_rows = left.GetRows(); diff --git a/src/common/types/decimal.cpp b/src/common/types/decimal.cpp index 5ecb39a0a818..8fa2264551c8 100644 --- a/src/common/types/decimal.cpp +++ b/src/common/types/decimal.cpp @@ -3,6 +3,7 @@ #include "duckdb/common/types/cast_helpers.hpp" namespace duckdb { +constexpr uint8_t Decimal::MAX_WIDTH_DECIMAL; template string TemplatedDecimalToString(SIGNED value, uint8_t width, uint8_t scale) { diff --git a/src/common/types/geometry.cpp b/src/common/types/geometry.cpp new file mode 100644 index 000000000000..5d66bed291f9 --- /dev/null +++ b/src/common/types/geometry.cpp @@ -0,0 +1,887 @@ +#include "duckdb/common/types/geometry.hpp" +#include "duckdb/common/types/string_type.hpp" +#include "duckdb/common/types/vector.hpp" +#include "fast_float/fast_float.h" +#include "fmt/format.h" + +//---------------------------------------------------------------------------------------------------------------------- +// Internals +//---------------------------------------------------------------------------------------------------------------------- +namespace duckdb { + +namespace { + +class BlobWriter { +public: + template + void Write(const T &value) { + auto ptr = reinterpret_cast(&value); + buffer.insert(buffer.end(), ptr, ptr + sizeof(T)); + } + + template + struct Reserved { + size_t offset; + T value; + }; + + template + Reserved Reserve() { + auto offset = buffer.size(); + buffer.resize(buffer.size() + sizeof(T)); + return {offset, T()}; + } + + template + void Write(const Reserved &reserved) { + if (reserved.offset + sizeof(T) > buffer.size()) { + throw InternalException("Write out of bounds in BinaryWriter"); + } + auto ptr = reinterpret_cast(&reserved.value); + // We've reserved 0 bytes, so we can safely memcpy + memcpy(buffer.data() + reserved.offset, ptr, sizeof(T)); + } + + void Write(const char *data, size_t size) { + D_ASSERT(data != nullptr); + buffer.insert(buffer.end(), data, data + size); + } + + const vector &GetBuffer() const { + return buffer; + } + +private: + vector buffer; +}; + +class BlobReader { +public: + BlobReader(const char *data, uint32_t size) : beg(data), pos(data), end(data + size) { + } + + template + T Read() { + if (pos + sizeof(T) > end) { + throw InvalidInputException("Unexpected end of binary data at position %zu", pos - beg); + } + T value; + if (LE) { + memcpy(&value, pos, sizeof(T)); + pos += sizeof(T); + } else { + char temp[sizeof(T)]; + for (size_t i = 0; i < sizeof(T); ++i) { + temp[i] = pos[sizeof(T) - 1 - i]; + } + memcpy(&value, temp, sizeof(T)); + pos += sizeof(T); + } + return value; + } + + void Skip(size_t size) { + if (pos + size > end) { + throw InvalidInputException("Skipping beyond end of binary data at position %zu", pos - beg); + } + pos += size; + } + + const char *Reserve(size_t size) { + if (pos + size > end) { + throw InvalidInputException("Reserving beyond end of binary data at position %zu", pos - beg); + } + auto current_pos = pos; + pos += size; + return current_pos; + } + + size_t GetPosition() const { + return static_cast(pos - beg); + } + + bool IsAtEnd() const { + return pos >= end; + } + +private: + const char *beg; + const char *pos; + const char *end; +}; + +class TextWriter { +public: + void Write(const char *str) { + buffer.insert(buffer.end(), str, str + strlen(str)); + } + void Write(char c) { + buffer.push_back(c); + } + void Write(double value) { + duckdb_fmt::format_to(std::back_inserter(buffer), "{}", value); + // Remove trailing zero + if (buffer.back() == '0') { + buffer.pop_back(); + if (buffer.back() == '.') { + buffer.pop_back(); + } + } + } + const vector &GetBuffer() const { + return buffer; + } + +private: + vector buffer; +}; + +class TextReader { +public: + TextReader(const char *text, const uint32_t size) : beg(text), pos(text), end(text + size) { + } + + bool TryMatch(const char *str) { + auto ptr = pos; + while (*str && pos < end && tolower(*pos) == tolower(*str)) { + pos++; + str++; + } + if (*str == '\0') { + SkipWhitespace(); // remove trailing whitespace + return true; // matched + } else { + pos = ptr; // reset position + return false; // not matched + } + } + + bool TryMatch(char c) { + if (pos < end && tolower(*pos) == tolower(c)) { + pos++; + SkipWhitespace(); // remove trailing whitespace + return true; // matched + } + return false; // not matched + } + + void Match(const char *str) { + if (!TryMatch(str)) { + throw InvalidInputException("Expected '%s' but got '%c' at position %zu", str, *pos, pos - beg); + } + } + + void Match(char c) { + if (!TryMatch(c)) { + throw InvalidInputException("Expected '%c' but got '%c' at position %zu", c, *pos, pos - beg); + } + } + + double MatchNumber() { + // Now use fast_float to parse the number + double num; + const auto res = duckdb_fast_float::from_chars(pos, end, num); + if (res.ec != std::errc()) { + throw InvalidInputException("Expected number at position %zu", pos - beg); + } + + pos = res.ptr; // update position to the end of the parsed number + + SkipWhitespace(); // remove trailing whitespace + return num; // return the parsed number + } + + idx_t GetPosition() const { + return static_cast(pos - beg); + } + + void Reset() { + pos = beg; + } + +private: + void SkipWhitespace() { + while (pos < end && isspace(*pos)) { + pos++; + } + } + + const char *beg; + const char *pos; + const char *end; +}; + +void FromStringRecursive(TextReader &reader, BlobWriter &writer, uint32_t depth, bool parent_has_z, bool parent_has_m) { + + if (depth == Geometry::MAX_RECURSION_DEPTH) { + throw InvalidInputException("Geometry string exceeds maximum recursion depth of %d", + Geometry::MAX_RECURSION_DEPTH); + } + + GeometryType type; + + if (reader.TryMatch("point")) { + type = GeometryType::POINT; + } else if (reader.TryMatch("linestring")) { + type = GeometryType::LINESTRING; + } else if (reader.TryMatch("polygon")) { + type = GeometryType::POLYGON; + } else if (reader.TryMatch("multipoint")) { + type = GeometryType::MULTIPOINT; + } else if (reader.TryMatch("multilinestring")) { + type = GeometryType::MULTILINESTRING; + } else if (reader.TryMatch("multipolygon")) { + type = GeometryType::MULTIPOLYGON; + } else if (reader.TryMatch("geometrycollection")) { + type = GeometryType::GEOMETRYCOLLECTION; + } else { + throw InvalidInputException("Unknown geometry type at position %zu", reader.GetPosition()); + } + + const auto has_z = reader.TryMatch("z"); + const auto has_m = reader.TryMatch("m"); + + const auto is_empty = reader.TryMatch("empty"); + + if ((depth != 0) && ((parent_has_z != has_z) || (parent_has_m != has_m))) { + throw InvalidInputException("Geometry has inconsistent Z/M dimensions, starting at position %zu", + reader.GetPosition()); + } + + // How many dimensions does this geometry have? + const uint32_t dims = 2 + (has_z ? 1 : 0) + (has_m ? 1 : 0); + + // WKB type + const auto meta = static_cast(type) + (has_z ? 1000 : 0) + (has_m ? 2000 : 0); + // Write the geometry type and vertex type + writer.Write(1); // LE Byte Order + writer.Write(meta); + + switch (type) { + case GeometryType::POINT: { + if (is_empty) { + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + // Write NaN for each dimension, if point is empty + writer.Write(std::numeric_limits::quiet_NaN()); + } + } else { + reader.Match('('); + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + auto value = reader.MatchNumber(); + writer.Write(value); + } + reader.Match(')'); + } + } break; + case GeometryType::LINESTRING: { + if (is_empty) { + writer.Write(0); // No vertices in empty linestring + break; + } + auto vert_count = writer.Reserve(); + reader.Match('('); + do { + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + auto value = reader.MatchNumber(); + writer.Write(value); + } + vert_count.value++; + } while (reader.TryMatch(',')); + reader.Match(')'); + writer.Write(vert_count); + } break; + case GeometryType::POLYGON: { + if (is_empty) { + writer.Write(0); + break; // No rings in empty polygon + } + auto ring_count = writer.Reserve(); + reader.Match('('); + do { + auto vert_count = writer.Reserve(); + reader.Match('('); + do { + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + auto value = reader.MatchNumber(); + writer.Write(value); + } + vert_count.value++; + } while (reader.TryMatch(',')); + reader.Match(')'); + writer.Write(vert_count); + ring_count.value++; + } while (reader.TryMatch(',')); + reader.Match(')'); + writer.Write(ring_count); + } break; + case GeometryType::MULTIPOINT: { + if (is_empty) { + writer.Write(0); // No points in empty multipoint + break; + } + auto part_count = writer.Reserve(); + reader.Match('('); + do { + bool has_paren = reader.TryMatch('('); + + const auto part_meta = static_cast(GeometryType::POINT) + (has_z ? 1000 : 0) + (has_m ? 2000 : 0); + writer.Write(1); + writer.Write(part_meta); + + if (reader.TryMatch("EMPTY")) { + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + // Write NaN for each dimension, if point is empty + writer.Write(std::numeric_limits::quiet_NaN()); + } + } else { + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + auto value = reader.MatchNumber(); + writer.Write(value); + } + } + if (has_paren) { + reader.Match(')'); // Match the closing parenthesis if it was opened + } + part_count.value++; + } while (reader.TryMatch(',')); + writer.Write(part_count); + } break; + case GeometryType::MULTILINESTRING: { + if (is_empty) { + writer.Write(0); + return; // No linestrings in empty multilinestring + } + auto part_count = writer.Reserve(); + reader.Match('('); + do { + + const auto part_meta = + static_cast(GeometryType::LINESTRING) + (has_z ? 1000 : 0) + (has_m ? 2000 : 0); + writer.Write(1); + writer.Write(part_meta); + + auto vert_count = writer.Reserve(); + reader.Match('('); + do { + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + auto value = reader.MatchNumber(); + writer.Write(value); + } + vert_count.value++; + } while (reader.TryMatch(',')); + reader.Match(')'); + writer.Write(vert_count); + part_count.value++; + } while (reader.TryMatch(',')); + reader.Match(')'); + writer.Write(part_count); + } break; + case GeometryType::MULTIPOLYGON: { + if (is_empty) { + writer.Write(0); // No polygons in empty multipolygon + break; + } + auto part_count = writer.Reserve(); + reader.Match('('); + do { + + const auto part_meta = + static_cast(GeometryType::POLYGON) + (has_z ? 1000 : 0) + (has_m ? 2000 : 0); + writer.Write(1); + writer.Write(part_meta); + + auto ring_count = writer.Reserve(); + reader.Match('('); + do { + auto vert_count = writer.Reserve(); + reader.Match('('); + do { + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + auto value = reader.MatchNumber(); + writer.Write(value); + } + vert_count.value++; + } while (reader.TryMatch(',')); + reader.Match(')'); + writer.Write(vert_count); + ring_count.value++; + } while (reader.TryMatch(',')); + reader.Match(')'); + writer.Write(ring_count); + part_count.value++; + } while (reader.TryMatch(',')); + reader.Match(')'); + writer.Write(part_count); + } break; + case GeometryType::GEOMETRYCOLLECTION: { + if (is_empty) { + writer.Write(0); // No geometries in empty geometry collection + break; + } + auto part_count = writer.Reserve(); + reader.Match('('); + do { + // Recursively parse the geometry inside the collection + FromStringRecursive(reader, writer, depth + 1, has_z, has_m); + part_count.value++; + } while (reader.TryMatch(',')); + reader.Match(')'); + writer.Write(part_count); + } break; + default: + throw InvalidInputException("Unknown geometry type %d at position %zu", static_cast(type), + reader.GetPosition()); + } +} + +void ToStringRecursive(BlobReader &reader, TextWriter &writer, idx_t depth, bool parent_has_z, bool parent_has_m) { + if (depth == Geometry::MAX_RECURSION_DEPTH) { + throw InvalidInputException("Geometry exceeds maximum recursion depth of %d", Geometry::MAX_RECURSION_DEPTH); + } + + // Read the byte order (should always be 1 for little-endian) + auto byte_order = reader.Read(); + if (byte_order != 1) { + throw InvalidInputException("Unsupported byte order %d in WKB", byte_order); + } + + const auto meta = reader.Read(); + const auto type = static_cast(meta % 1000); + const auto flag = meta / 1000; + const auto has_z = (flag & 0x01) != 0; + const auto has_m = (flag & 0x02) != 0; + + if ((depth != 0) && ((parent_has_z != has_z) || (parent_has_m != has_m))) { + throw InvalidInputException("Geometry has inconsistent Z/M dimensions, starting at position %zu", + reader.GetPosition()); + } + + const uint32_t dims = 2 + (has_z ? 1 : 0) + (has_m ? 1 : 0); + const auto flag_str = has_z ? (has_m ? " ZM " : " Z ") : (has_m ? " M " : " "); + + switch (type) { + case GeometryType::POINT: { + writer.Write("POINT"); + writer.Write(flag_str); + + double vert[4] = {0, 0, 0, 0}; + auto all_nan = true; + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + vert[d_idx] = reader.Read(); + all_nan &= std::isnan(vert[d_idx]); + } + if (all_nan) { + writer.Write("EMPTY"); + return; + } + writer.Write('('); + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + if (d_idx > 0) { + writer.Write(' '); + } + writer.Write(vert[d_idx]); + } + writer.Write(')'); + } break; + case GeometryType::LINESTRING: { + writer.Write("LINESTRING"); + ; + writer.Write(flag_str); + const auto vert_count = reader.Read(); + if (vert_count == 0) { + writer.Write("EMPTY"); + return; + } + writer.Write('('); + for (uint32_t vert_idx = 0; vert_idx < vert_count; vert_idx++) { + if (vert_idx > 0) { + writer.Write(", "); + } + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + if (d_idx > 0) { + writer.Write(' '); + } + auto value = reader.Read(); + writer.Write(value); + } + } + writer.Write(')'); + } break; + case GeometryType::POLYGON: { + writer.Write("POLYGON"); + writer.Write(flag_str); + const auto ring_count = reader.Read(); + if (ring_count == 0) { + writer.Write("EMPTY"); + return; + } + writer.Write('('); + for (uint32_t ring_idx = 0; ring_idx < ring_count; ring_idx++) { + if (ring_idx > 0) { + writer.Write(", "); + } + const auto vert_count = reader.Read(); + if (vert_count == 0) { + writer.Write("EMPTY"); + continue; + } + writer.Write('('); + for (uint32_t vert_idx = 0; vert_idx < vert_count; vert_idx++) { + if (vert_idx > 0) { + writer.Write(", "); + } + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + if (d_idx > 0) { + writer.Write(' '); + } + auto value = reader.Read(); + writer.Write(value); + } + } + writer.Write(')'); + } + writer.Write(')'); + } break; + case GeometryType::MULTIPOINT: { + writer.Write("MULTIPOINT"); + writer.Write(flag_str); + const auto part_count = reader.Read(); + if (part_count == 0) { + writer.Write("EMPTY"); + return; + } + writer.Write('('); + for (uint32_t part_idx = 0; part_idx < part_count; part_idx++) { + const auto part_byte_order = reader.Read(); + if (part_byte_order != 1) { + throw InvalidInputException("Unsupported byte order %d in WKB", part_byte_order); + } + const auto part_meta = reader.Read(); + const auto part_type = static_cast(part_meta % 1000); + const auto part_flag = part_meta / 1000; + const auto part_has_z = (part_flag & 0x01) != 0; + const auto part_has_m = (part_flag & 0x02) != 0; + + if (part_type != GeometryType::POINT) { + throw InvalidInputException("Expected POINT in MULTIPOINT but got %d", static_cast(part_type)); + } + + if ((has_z != part_has_z) || (has_m != part_has_m)) { + throw InvalidInputException( + "Geometry has inconsistent Z/M dimensions in MULTIPOINT, starting at position %zu", + reader.GetPosition()); + } + if (part_idx > 0) { + writer.Write(", "); + } + double vert[4] = {0, 0, 0, 0}; + auto all_nan = true; + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + vert[d_idx] = reader.Read(); + all_nan &= std::isnan(vert[d_idx]); + } + if (all_nan) { + writer.Write("EMPTY"); + continue; + } + // writer.Write('('); + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + if (d_idx > 0) { + writer.Write(' '); + } + writer.Write(vert[d_idx]); + } + // writer.Write(')'); + } + writer.Write(')'); + + } break; + case GeometryType::MULTILINESTRING: { + writer.Write("MULTILINESTRING"); + writer.Write(flag_str); + const auto part_count = reader.Read(); + if (part_count == 0) { + writer.Write("EMPTY"); + return; + } + writer.Write('('); + for (uint32_t part_idx = 0; part_idx < part_count; part_idx++) { + const auto part_byte_order = reader.Read(); + if (part_byte_order != 1) { + throw InvalidInputException("Unsupported byte order %d in WKB", part_byte_order); + } + const auto part_meta = reader.Read(); + const auto part_type = static_cast(part_meta % 1000); + const auto part_flag = part_meta / 1000; + const auto part_has_z = (part_flag & 0x01) != 0; + const auto part_has_m = (part_flag & 0x02) != 0; + + if (part_type != GeometryType::LINESTRING) { + throw InvalidInputException("Expected LINESTRING in MULTILINESTRING but got %d", + static_cast(part_type)); + } + if ((has_z != part_has_z) || (has_m != part_has_m)) { + throw InvalidInputException( + "Geometry has inconsistent Z/M dimensions in MULTILINESTRING, starting at position %zu", + reader.GetPosition()); + } + if (part_idx > 0) { + writer.Write(", "); + } + const auto vert_count = reader.Read(); + if (vert_count == 0) { + writer.Write("EMPTY"); + continue; + } + writer.Write('('); + for (uint32_t vert_idx = 0; vert_idx < vert_count; vert_idx++) { + if (vert_idx > 0) { + writer.Write(", "); + } + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + if (d_idx > 0) { + writer.Write(' '); + } + auto value = reader.Read(); + writer.Write(value); + } + } + writer.Write(')'); + } + writer.Write(')'); + } break; + case GeometryType::MULTIPOLYGON: { + writer.Write("MULTIPOLYGON"); + writer.Write(flag_str); + const auto part_count = reader.Read(); + if (part_count == 0) { + writer.Write("EMPTY"); + return; + } + writer.Write('('); + for (uint32_t part_idx = 0; part_idx < part_count; part_idx++) { + if (part_idx > 0) { + writer.Write(", "); + } + + const auto part_byte_order = reader.Read(); + if (part_byte_order != 1) { + throw InvalidInputException("Unsupported byte order %d in WKB", part_byte_order); + } + const auto part_meta = reader.Read(); + const auto part_type = static_cast(part_meta % 1000); + const auto part_flag = part_meta / 1000; + const auto part_has_z = (part_flag & 0x01) != 0; + const auto part_has_m = (part_flag & 0x02) != 0; + if (part_type != GeometryType::POLYGON) { + throw InvalidInputException("Expected POLYGON in MULTIPOLYGON but got %d", static_cast(part_type)); + } + if ((has_z != part_has_z) || (has_m != part_has_m)) { + throw InvalidInputException( + "Geometry has inconsistent Z/M dimensions in MULTIPOLYGON, starting at position %zu", + reader.GetPosition()); + } + + const auto ring_count = reader.Read(); + if (ring_count == 0) { + writer.Write("EMPTY"); + continue; + } + writer.Write('('); + for (uint32_t ring_idx = 0; ring_idx < ring_count; ring_idx++) { + if (ring_idx > 0) { + writer.Write(", "); + } + const auto vert_count = reader.Read(); + if (vert_count == 0) { + writer.Write("EMPTY"); + continue; + } + writer.Write('('); + for (uint32_t vert_idx = 0; vert_idx < vert_count; vert_idx++) { + if (vert_idx > 0) { + writer.Write(", "); + } + for (uint32_t d_idx = 0; d_idx < dims; d_idx++) { + if (d_idx > 0) { + writer.Write(' '); + } + auto value = reader.Read(); + writer.Write(value); + } + } + writer.Write(')'); + } + writer.Write(')'); + } + writer.Write(')'); + } break; + case GeometryType::GEOMETRYCOLLECTION: { + writer.Write("GEOMETRYCOLLECTION"); + writer.Write(flag_str); + const auto part_count = reader.Read(); + if (part_count == 0) { + writer.Write("EMPTY"); + return; + } + writer.Write('('); + for (uint32_t part_idx = 0; part_idx < part_count; part_idx++) { + if (part_idx > 0) { + writer.Write(", "); + } + // Recursively parse the geometry inside the collection + ToStringRecursive(reader, writer, depth + 1, has_z, has_m); + } + writer.Write(')'); + } break; + default: + throw InvalidInputException("Unsupported geometry type %d in WKB", static_cast(type)); + } +} + +} // namespace + +} // namespace duckdb + +//---------------------------------------------------------------------------------------------------------------------- +// Public interface +//---------------------------------------------------------------------------------------------------------------------- +namespace duckdb { + +constexpr const idx_t Geometry::MAX_RECURSION_DEPTH; + +bool Geometry::FromString(const string_t &wkt_text, string_t &result, Vector &result_vector, bool strict) { + TextReader reader(wkt_text.GetData(), static_cast(wkt_text.GetSize())); + BlobWriter writer; + + FromStringRecursive(reader, writer, 0, false, false); + + const auto &buffer = writer.GetBuffer(); + result = StringVector::AddStringOrBlob(result_vector, buffer.data(), buffer.size()); + return true; +} + +string_t Geometry::ToString(Vector &result, const string_t &geom) { + BlobReader reader(geom.GetData(), static_cast(geom.GetSize())); + TextWriter writer; + + ToStringRecursive(reader, writer, 0, false, false); + + // Convert the buffer to string_t + const auto &buffer = writer.GetBuffer(); + return StringVector::AddString(result, buffer.data(), buffer.size()); +} + +pair Geometry::GetType(const string_t &wkb) { + BlobReader reader(wkb.GetData(), static_cast(wkb.GetSize())); + + // Read the byte order (should always be 1 for little-endian) + const auto byte_order = reader.Read(); + if (byte_order != 1) { + throw InvalidInputException("Unsupported byte order %d in WKB", byte_order); + } + + const auto meta = reader.Read(); + const auto type_id = meta % 1000; + const auto flag_id = meta / 1000; + + if (type_id < 1 || type_id > 7) { + throw InvalidInputException("Unsupported geometry type %d in WKB", type_id); + } + if (flag_id > 3) { + throw InvalidInputException("Unsupported geometry flag %d in WKB", flag_id); + } + + const auto geom_type = static_cast(type_id); + const auto vert_type = static_cast(flag_id); + + return {geom_type, vert_type}; +} + +template +static uint32_t ParseVerticesInternal(BlobReader &reader, GeometryExtent &extent, uint32_t vert_count, bool check_nan) { + uint32_t count = 0; + + // Issue a single .Reserve() for all vertices, to minimize bounds checking overhead + const auto ptr = const_data_ptr_cast(reader.Reserve(vert_count * sizeof(VERTEX_TYPE))); + + for (uint32_t vert_idx = 0; vert_idx < vert_count; vert_idx++) { + VERTEX_TYPE vertex = Load(ptr + vert_idx * sizeof(VERTEX_TYPE)); + if (check_nan && vertex.AllNan()) { + continue; + } + + extent.Extend(vertex); + count++; + } + return count; +} + +static uint32_t ParseVertices(BlobReader &reader, GeometryExtent &extent, uint32_t vert_count, VertexType type, + bool check_nan) { + switch (type) { + case VertexType::XY: + return ParseVerticesInternal(reader, extent, vert_count, check_nan); + case VertexType::XYZ: + return ParseVerticesInternal(reader, extent, vert_count, check_nan); + case VertexType::XYM: + return ParseVerticesInternal(reader, extent, vert_count, check_nan); + case VertexType::XYZM: + return ParseVerticesInternal(reader, extent, vert_count, check_nan); + default: + throw InvalidInputException("Unsupported vertex type %d in WKB", static_cast(type)); + } +} + +uint32_t Geometry::GetExtent(const string_t &wkb, GeometryExtent &extent) { + BlobReader reader(wkb.GetData(), static_cast(wkb.GetSize())); + + uint32_t vertex_count = 0; + + while (!reader.IsAtEnd()) { + const auto byte_order = reader.Read(); + if (byte_order != 1) { + throw InvalidInputException("Unsupported byte order %d in WKB", byte_order); + } + const auto meta = reader.Read(); + const auto type_id = meta % 1000; + const auto flag_id = meta / 1000; + if (type_id < 1 || type_id > 7) { + throw InvalidInputException("Unsupported geometry type %d in WKB", type_id); + } + if (flag_id > 3) { + throw InvalidInputException("Unsupported geometry flag %d in WKB", flag_id); + } + const auto geom_type = static_cast(type_id); + const auto vert_type = static_cast(flag_id); + + switch (geom_type) { + case GeometryType::POINT: { + vertex_count += ParseVertices(reader, extent, 1, vert_type, true); + } break; + case GeometryType::LINESTRING: { + const auto vert_count = reader.Read(); + vertex_count += ParseVertices(reader, extent, vert_count, vert_type, false); + } break; + case GeometryType::POLYGON: { + const auto ring_count = reader.Read(); + for (uint32_t ring_idx = 0; ring_idx < ring_count; ring_idx++) { + const auto vert_count = reader.Read(); + vertex_count += ParseVertices(reader, extent, vert_count, vert_type, false); + } + } break; + case GeometryType::MULTIPOINT: + case GeometryType::MULTILINESTRING: + case GeometryType::MULTIPOLYGON: + case GeometryType::GEOMETRYCOLLECTION: { + // Skip count. We don't need it for extent calculation. + reader.Skip(sizeof(uint32_t)); + } break; + default: + throw InvalidInputException("Unsupported geometry type %d in WKB", static_cast(geom_type)); + } + } + return vertex_count; +} + +} // namespace duckdb diff --git a/src/common/types/row/CMakeLists.txt b/src/common/types/row/CMakeLists.txt index 384de7fd9bcc..d10285942c9f 100644 --- a/src/common/types/row/CMakeLists.txt +++ b/src/common/types/row/CMakeLists.txt @@ -8,9 +8,6 @@ add_library_unity( OBJECT block_iterator.cpp partitioned_tuple_data.cpp - row_data_collection.cpp - row_data_collection_scanner.cpp - row_layout.cpp tuple_data_allocator.cpp tuple_data_collection.cpp tuple_data_iterator.cpp diff --git a/src/common/types/row/row_data_collection.cpp b/src/common/types/row/row_data_collection.cpp deleted file mode 100644 index b178b7fb50bd..000000000000 --- a/src/common/types/row/row_data_collection.cpp +++ /dev/null @@ -1,141 +0,0 @@ -#include "duckdb/common/types/row/row_data_collection.hpp" - -namespace duckdb { - -RowDataCollection::RowDataCollection(BufferManager &buffer_manager, idx_t block_capacity, idx_t entry_size, - bool keep_pinned) - : buffer_manager(buffer_manager), count(0), block_capacity(block_capacity), entry_size(entry_size), - keep_pinned(keep_pinned) { - D_ASSERT(block_capacity * entry_size + entry_size > buffer_manager.GetBlockSize()); -} - -idx_t RowDataCollection::AppendToBlock(RowDataBlock &block, BufferHandle &handle, - vector &append_entries, idx_t remaining, idx_t entry_sizes[]) { - idx_t append_count = 0; - data_ptr_t dataptr; - if (entry_sizes) { - D_ASSERT(entry_size == 1); - // compute how many entries fit if entry size is variable - dataptr = handle.Ptr() + block.byte_offset; - for (idx_t i = 0; i < remaining; i++) { - if (block.byte_offset + entry_sizes[i] > block.capacity) { - if (block.count == 0 && append_count == 0 && entry_sizes[i] > block.capacity) { - // special case: single entry is bigger than block capacity - // resize current block to fit the entry, append it, and move to the next block - block.capacity = entry_sizes[i]; - buffer_manager.ReAllocate(block.block, block.capacity); - dataptr = handle.Ptr(); - append_count++; - block.byte_offset += entry_sizes[i]; - } - break; - } - append_count++; - block.byte_offset += entry_sizes[i]; - } - } else { - append_count = MinValue(remaining, block.capacity - block.count); - dataptr = handle.Ptr() + block.count * entry_size; - } - append_entries.emplace_back(dataptr, append_count); - block.count += append_count; - return append_count; -} - -RowDataBlock &RowDataCollection::CreateBlock() { - blocks.push_back(make_uniq(MemoryTag::ORDER_BY, buffer_manager, block_capacity, entry_size)); - return *blocks.back(); -} - -vector RowDataCollection::Build(idx_t added_count, data_ptr_t key_locations[], idx_t entry_sizes[], - const SelectionVector *sel) { - vector handles; - vector append_entries; - - // first allocate space of where to serialize the keys and payload columns - idx_t remaining = added_count; - { - // first append to the last block (if any) - lock_guard append_lock(rdc_lock); - count += added_count; - - if (!blocks.empty()) { - auto &last_block = *blocks.back(); - if (last_block.count < last_block.capacity) { - // last block has space: pin the buffer of this block - auto handle = buffer_manager.Pin(last_block.block); - // now append to the block - idx_t append_count = AppendToBlock(last_block, handle, append_entries, remaining, entry_sizes); - remaining -= append_count; - handles.push_back(std::move(handle)); - } - } - while (remaining > 0) { - // now for the remaining data, allocate new buffers to store the data and append there - auto &new_block = CreateBlock(); - auto handle = buffer_manager.Pin(new_block.block); - - // offset the entry sizes array if we have added entries already - idx_t *offset_entry_sizes = entry_sizes ? entry_sizes + added_count - remaining : nullptr; - - idx_t append_count = AppendToBlock(new_block, handle, append_entries, remaining, offset_entry_sizes); - D_ASSERT(new_block.count > 0); - remaining -= append_count; - - if (keep_pinned) { - pinned_blocks.push_back(std::move(handle)); - } else { - handles.push_back(std::move(handle)); - } - } - } - // now set up the key_locations based on the append entries - idx_t append_idx = 0; - for (auto &append_entry : append_entries) { - idx_t next = append_idx + append_entry.count; - if (entry_sizes) { - for (; append_idx < next; append_idx++) { - key_locations[append_idx] = append_entry.baseptr; - append_entry.baseptr += entry_sizes[append_idx]; - } - } else { - for (; append_idx < next; append_idx++) { - auto idx = sel->get_index(append_idx); - key_locations[idx] = append_entry.baseptr; - append_entry.baseptr += entry_size; - } - } - } - // return the unique pointers to the handles because they must stay pinned - return handles; -} - -void RowDataCollection::Merge(RowDataCollection &other) { - if (other.count == 0) { - return; - } - RowDataCollection temp(buffer_manager, buffer_manager.GetBlockSize(), 1); - { - // One lock at a time to avoid deadlocks - lock_guard read_lock(other.rdc_lock); - temp.count = other.count; - temp.block_capacity = other.block_capacity; - temp.entry_size = other.entry_size; - temp.blocks = std::move(other.blocks); - temp.pinned_blocks = std::move(other.pinned_blocks); - } - other.Clear(); - - lock_guard write_lock(rdc_lock); - count += temp.count; - block_capacity = MaxValue(block_capacity, temp.block_capacity); - entry_size = MaxValue(entry_size, temp.entry_size); - for (auto &block : temp.blocks) { - blocks.emplace_back(std::move(block)); - } - for (auto &handle : temp.pinned_blocks) { - pinned_blocks.emplace_back(std::move(handle)); - } -} - -} // namespace duckdb diff --git a/src/common/types/row/row_data_collection_scanner.cpp b/src/common/types/row/row_data_collection_scanner.cpp deleted file mode 100644 index 9b3a4be06efc..000000000000 --- a/src/common/types/row/row_data_collection_scanner.cpp +++ /dev/null @@ -1,330 +0,0 @@ -#include "duckdb/common/types/row/row_data_collection_scanner.hpp" - -#include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/types/row/row_data_collection.hpp" -#include "duckdb/storage/buffer_manager.hpp" - -#include - -namespace duckdb { - -void RowDataCollectionScanner::AlignHeapBlocks(RowDataCollection &swizzled_block_collection, - RowDataCollection &swizzled_string_heap, - RowDataCollection &block_collection, RowDataCollection &string_heap, - const RowLayout &layout) { - if (block_collection.count == 0) { - return; - } - - if (layout.AllConstant()) { - // No heap blocks! Just merge fixed-size data - swizzled_block_collection.Merge(block_collection); - return; - } - - // We create one heap block per data block and swizzle the pointers - D_ASSERT(string_heap.keep_pinned == swizzled_string_heap.keep_pinned); - auto &buffer_manager = block_collection.buffer_manager; - auto &heap_blocks = string_heap.blocks; - idx_t heap_block_idx = 0; - idx_t heap_block_remaining = heap_blocks[heap_block_idx]->count; - for (auto &data_block : block_collection.blocks) { - if (heap_block_remaining == 0) { - heap_block_remaining = heap_blocks[++heap_block_idx]->count; - } - - // Pin the data block and swizzle the pointers within the rows - auto data_handle = buffer_manager.Pin(data_block->block); - auto data_ptr = data_handle.Ptr(); - if (!string_heap.keep_pinned) { - D_ASSERT(!data_block->block->IsSwizzled()); - RowOperations::SwizzleColumns(layout, data_ptr, data_block->count); - data_block->block->SetSwizzling(nullptr); - } - // At this point the data block is pinned and the heap pointer is valid - // so we can copy heap data as needed - - // We want to copy as little of the heap data as possible, check how the data and heap blocks line up - if (heap_block_remaining >= data_block->count) { - // Easy: current heap block contains all strings for this data block, just copy (reference) the block - swizzled_string_heap.blocks.emplace_back(heap_blocks[heap_block_idx]->Copy()); - swizzled_string_heap.blocks.back()->count = data_block->count; - - // Swizzle the heap pointer if we are not pinning the heap - auto &heap_block = swizzled_string_heap.blocks.back()->block; - auto heap_handle = buffer_manager.Pin(heap_block); - if (!swizzled_string_heap.keep_pinned) { - auto heap_ptr = Load(data_ptr + layout.GetHeapOffset()); - auto heap_offset = heap_ptr - heap_handle.Ptr(); - RowOperations::SwizzleHeapPointer(layout, data_ptr, heap_ptr, data_block->count, - NumericCast(heap_offset)); - } else { - swizzled_string_heap.pinned_blocks.emplace_back(std::move(heap_handle)); - } - - // Update counter - heap_block_remaining -= data_block->count; - } else { - // Strings for this data block are spread over the current heap block and the next (and possibly more) - if (string_heap.keep_pinned) { - // The heap is changing underneath the data block, - // so swizzle the string pointers to make them portable. - RowOperations::SwizzleColumns(layout, data_ptr, data_block->count); - } - idx_t data_block_remaining = data_block->count; - vector> ptrs_and_sizes; - idx_t total_size = 0; - const auto base_row_ptr = data_ptr; - while (data_block_remaining > 0) { - if (heap_block_remaining == 0) { - heap_block_remaining = heap_blocks[++heap_block_idx]->count; - } - auto next = MinValue(data_block_remaining, heap_block_remaining); - - // Figure out where to start copying strings, and how many bytes we need to copy - auto heap_start_ptr = Load(data_ptr + layout.GetHeapOffset()); - auto heap_end_ptr = - Load(data_ptr + layout.GetHeapOffset() + (next - 1) * layout.GetRowWidth()); - auto size = NumericCast(heap_end_ptr - heap_start_ptr + Load(heap_end_ptr)); - ptrs_and_sizes.emplace_back(heap_start_ptr, size); - D_ASSERT(size <= heap_blocks[heap_block_idx]->byte_offset); - - // Swizzle the heap pointer - RowOperations::SwizzleHeapPointer(layout, data_ptr, heap_start_ptr, next, total_size); - total_size += size; - - // Update where we are in the data and heap blocks - data_ptr += next * layout.GetRowWidth(); - data_block_remaining -= next; - heap_block_remaining -= next; - } - - // Finally, we allocate a new heap block and copy data to it - swizzled_string_heap.blocks.emplace_back(make_uniq( - MemoryTag::ORDER_BY, buffer_manager, MaxValue(total_size, buffer_manager.GetBlockSize()), 1U)); - auto new_heap_handle = buffer_manager.Pin(swizzled_string_heap.blocks.back()->block); - auto new_heap_ptr = new_heap_handle.Ptr(); - for (auto &ptr_and_size : ptrs_and_sizes) { - memcpy(new_heap_ptr, ptr_and_size.first, ptr_and_size.second); - new_heap_ptr += ptr_and_size.second; - } - new_heap_ptr = new_heap_handle.Ptr(); - if (swizzled_string_heap.keep_pinned) { - // Since the heap blocks are pinned, we can unswizzle the data again. - swizzled_string_heap.pinned_blocks.emplace_back(std::move(new_heap_handle)); - RowOperations::UnswizzlePointers(layout, base_row_ptr, new_heap_ptr, data_block->count); - RowOperations::UnswizzleHeapPointer(layout, base_row_ptr, new_heap_ptr, data_block->count); - } - } - } - - // We're done with variable-sized data, now just merge the fixed-size data - swizzled_block_collection.Merge(block_collection); - D_ASSERT(swizzled_block_collection.blocks.size() == swizzled_string_heap.blocks.size()); - - // Update counts and cleanup - swizzled_string_heap.count = string_heap.count; - string_heap.Clear(); -} - -void RowDataCollectionScanner::ScanState::PinData() { - auto &rows = scanner.rows; - D_ASSERT(block_idx < rows.blocks.size()); - auto &data_block = rows.blocks[block_idx]; - if (!data_handle.IsValid() || data_handle.GetBlockHandle() != data_block->block) { - data_handle = rows.buffer_manager.Pin(data_block->block); - } - if (scanner.layout.AllConstant() || !scanner.external) { - return; - } - - auto &heap = scanner.heap; - D_ASSERT(block_idx < heap.blocks.size()); - auto &heap_block = heap.blocks[block_idx]; - if (!heap_handle.IsValid() || heap_handle.GetBlockHandle() != heap_block->block) { - heap_handle = heap.buffer_manager.Pin(heap_block->block); - } -} - -RowDataCollectionScanner::RowDataCollectionScanner(RowDataCollection &rows_p, RowDataCollection &heap_p, - const RowLayout &layout_p, bool external_p, bool flush_p) - : rows(rows_p), heap(heap_p), layout(layout_p), read_state(*this), total_count(rows.count), total_scanned(0), - external(external_p), flush(flush_p), unswizzling(!layout.AllConstant() && external && !heap.keep_pinned) { - - if (unswizzling) { - D_ASSERT(rows.blocks.size() == heap.blocks.size()); - } - - ValidateUnscannedBlock(); -} - -RowDataCollectionScanner::RowDataCollectionScanner(RowDataCollection &rows_p, RowDataCollection &heap_p, - const RowLayout &layout_p, bool external_p, idx_t block_idx, - bool flush_p) - : rows(rows_p), heap(heap_p), layout(layout_p), read_state(*this), total_count(rows.count), total_scanned(0), - external(external_p), flush(flush_p), unswizzling(!layout.AllConstant() && external && !heap.keep_pinned) { - - if (unswizzling) { - D_ASSERT(rows.blocks.size() == heap.blocks.size()); - } - - D_ASSERT(block_idx < rows.blocks.size()); - read_state.block_idx = block_idx; - read_state.entry_idx = 0; - - // Pretend that we have scanned up to the start block - // and will stop at the end - auto begin = rows.blocks.begin(); - auto end = begin + NumericCast(block_idx); - total_scanned = - std::accumulate(begin, end, idx_t(0), [&](idx_t c, const unique_ptr &b) { return c + b->count; }); - total_count = total_scanned + (*end)->count; - - ValidateUnscannedBlock(); -} - -void RowDataCollectionScanner::SwizzleBlockInternal(RowDataBlock &data_block, RowDataBlock &heap_block) { - // Pin the data block and swizzle the pointers within the rows - D_ASSERT(!data_block.block->IsSwizzled()); - auto data_handle = rows.buffer_manager.Pin(data_block.block); - auto data_ptr = data_handle.Ptr(); - RowOperations::SwizzleColumns(layout, data_ptr, data_block.count); - data_block.block->SetSwizzling(nullptr); - - // Swizzle the heap pointers - auto heap_handle = heap.buffer_manager.Pin(heap_block.block); - auto heap_ptr = Load(data_ptr + layout.GetHeapOffset()); - auto heap_offset = heap_ptr - heap_handle.Ptr(); - RowOperations::SwizzleHeapPointer(layout, data_ptr, heap_ptr, data_block.count, NumericCast(heap_offset)); -} - -void RowDataCollectionScanner::SwizzleBlock(idx_t block_idx) { - if (rows.count == 0) { - return; - } - - if (!unswizzling) { - // No swizzled blocks! - return; - } - - auto &data_block = rows.blocks[block_idx]; - if (data_block->block && !data_block->block->IsSwizzled()) { - SwizzleBlockInternal(*data_block, *heap.blocks[block_idx]); - } -} - -void RowDataCollectionScanner::ReSwizzle() { - if (rows.count == 0) { - return; - } - - if (!unswizzling) { - // No swizzled blocks! - return; - } - - D_ASSERT(rows.blocks.size() == heap.blocks.size()); - for (idx_t i = 0; i < rows.blocks.size(); ++i) { - auto &data_block = rows.blocks[i]; - if (data_block->block && !data_block->block->IsSwizzled()) { - SwizzleBlockInternal(*data_block, *heap.blocks[i]); - } - } -} - -void RowDataCollectionScanner::ValidateUnscannedBlock() const { - if (unswizzling && read_state.block_idx < rows.blocks.size() && Remaining()) { - D_ASSERT(rows.blocks[read_state.block_idx]->block->IsSwizzled()); - } -} - -void RowDataCollectionScanner::Scan(DataChunk &chunk) { - auto count = MinValue((idx_t)STANDARD_VECTOR_SIZE, total_count - total_scanned); - if (count == 0) { - chunk.SetCardinality(count); - return; - } - - // Only flush blocks we processed. - const auto flush_block_idx = read_state.block_idx; - - const idx_t &row_width = layout.GetRowWidth(); - // Set up a batch of pointers to scan data from - idx_t scanned = 0; - auto data_pointers = FlatVector::GetData(addresses); - - // We must pin ALL blocks we are going to gather from - vector pinned_blocks; - while (scanned < count) { - read_state.PinData(); - auto &data_block = rows.blocks[read_state.block_idx]; - idx_t next = MinValue(data_block->count - read_state.entry_idx, count - scanned); - const data_ptr_t data_ptr = read_state.data_handle.Ptr() + read_state.entry_idx * row_width; - // Set up the next pointers - data_ptr_t row_ptr = data_ptr; - for (idx_t i = 0; i < next; i++) { - data_pointers[scanned + i] = row_ptr; - row_ptr += row_width; - } - // Unswizzle the offsets back to pointers (if needed) - if (unswizzling) { - RowOperations::UnswizzlePointers(layout, data_ptr, read_state.heap_handle.Ptr(), next); - rows.blocks[read_state.block_idx]->block->SetSwizzling("RowDataCollectionScanner::Scan"); - } - // Update state indices - read_state.entry_idx += next; - scanned += next; - total_scanned += next; - if (read_state.entry_idx == data_block->count) { - // Pin completed blocks so we don't lose them - pinned_blocks.emplace_back(rows.buffer_manager.Pin(data_block->block)); - if (unswizzling) { - auto &heap_block = heap.blocks[read_state.block_idx]; - pinned_blocks.emplace_back(heap.buffer_manager.Pin(heap_block->block)); - } - read_state.block_idx++; - read_state.entry_idx = 0; - ValidateUnscannedBlock(); - } - } - D_ASSERT(scanned == count); - // Deserialize the payload data - for (idx_t col_no = 0; col_no < layout.ColumnCount(); col_no++) { - RowOperations::Gather(addresses, *FlatVector::IncrementalSelectionVector(), chunk.data[col_no], - *FlatVector::IncrementalSelectionVector(), count, layout, col_no); - } - chunk.SetCardinality(count); - chunk.Verify(); - - // Switch to a new set of pinned blocks - read_state.pinned_blocks.swap(pinned_blocks); - - if (flush) { - // Release blocks we have passed. - for (idx_t i = flush_block_idx; i < read_state.block_idx; ++i) { - rows.blocks[i]->block = nullptr; - if (unswizzling) { - heap.blocks[i]->block = nullptr; - } - } - } else if (unswizzling) { - // Reswizzle blocks we have passed so they can be flushed safely. - for (idx_t i = flush_block_idx; i < read_state.block_idx; ++i) { - auto &data_block = rows.blocks[i]; - if (data_block->block && !data_block->block->IsSwizzled()) { - SwizzleBlockInternal(*data_block, *heap.blocks[i]); - } - } - } -} - -void RowDataCollectionScanner::Reset(bool flush_p) { - flush = flush_p; - total_scanned = 0; - - read_state.block_idx = 0; - read_state.entry_idx = 0; -} - -} // namespace duckdb diff --git a/src/common/types/row/row_layout.cpp b/src/common/types/row/row_layout.cpp deleted file mode 100644 index 3add8e425d98..000000000000 --- a/src/common/types/row/row_layout.cpp +++ /dev/null @@ -1,62 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/common/types/row_layout.cpp -// -// -//===----------------------------------------------------------------------===// - -#include "duckdb/common/types/row/row_layout.hpp" - -#include "duckdb/planner/expression/bound_aggregate_expression.hpp" - -namespace duckdb { - -RowLayout::RowLayout() : flag_width(0), data_width(0), row_width(0), all_constant(true), heap_pointer_offset(0) { -} - -void RowLayout::Initialize(vector types_p, bool align) { - offsets.clear(); - types = std::move(types_p); - - // Null mask at the front - 1 bit per value. - flag_width = ValidityBytes::ValidityMaskSize(types.size()); - row_width = flag_width; - - // Whether all columns are constant size. - for (const auto &type : types) { - all_constant = all_constant && TypeIsConstantSize(type.InternalType()); - } - - // This enables pointer swizzling for out-of-core computation. - if (!all_constant) { - // When unswizzled, the pointer lives here. - // When swizzled, the pointer is replaced by an offset. - heap_pointer_offset = row_width; - // The 8 byte pointer will be replaced with an 8 byte idx_t when swizzled. - // However, this cannot be sizeof(data_ptr_t), since 32 bit builds use 4 byte pointers. - row_width += sizeof(idx_t); - } - - // Data columns. No alignment required. - for (const auto &type : types) { - offsets.push_back(row_width); - const auto internal_type = type.InternalType(); - if (TypeIsConstantSize(internal_type) || internal_type == PhysicalType::VARCHAR) { - row_width += GetTypeIdSize(type.InternalType()); - } else { - // Variable size types use pointers to the actual data (can be swizzled). - // Again, we would use sizeof(data_ptr_t), but this is not guaranteed to be equal to sizeof(idx_t). - row_width += sizeof(idx_t); - } - } - - data_width = row_width - flag_width; - - // Alignment padding for the next row - if (align) { - row_width = AlignValue(row_width); - } -} - -} // namespace duckdb diff --git a/src/common/types/row/tuple_data_allocator.cpp b/src/common/types/row/tuple_data_allocator.cpp index f7e0eaa57229..7c5fcd32bfc8 100644 --- a/src/common/types/row/tuple_data_allocator.cpp +++ b/src/common/types/row/tuple_data_allocator.cpp @@ -241,8 +241,14 @@ TupleDataChunkPart TupleDataAllocator::BuildChunkPart(TupleDataPinState &pin_sta if (total_heap_size == 0) { result.SetHeapEmpty(); } else { - const auto heap_remaining = MaxValue( - heap_blocks.empty() ? block_size : heap_blocks.back().RemainingCapacity(), heap_sizes[append_offset]); + idx_t heap_remaining; + if (!heap_blocks.empty() && heap_blocks.back().RemainingCapacity() >= heap_sizes[append_offset]) { + // We have enough room for the current entry + heap_remaining = heap_blocks.back().RemainingCapacity(); + } else { + // We need to allocate a new block + heap_remaining = MaxValue(block_size, heap_sizes[append_offset]); + } if (total_heap_size <= heap_remaining) { // Everything fits diff --git a/src/common/types/row/tuple_data_collection.cpp b/src/common/types/row/tuple_data_collection.cpp index ffd4a2b4cecc..9329dbb5856d 100644 --- a/src/common/types/row/tuple_data_collection.cpp +++ b/src/common/types/row/tuple_data_collection.cpp @@ -18,6 +18,10 @@ TupleDataCollection::TupleDataCollection(BufferManager &buffer_manager, shared_p Initialize(); } +TupleDataCollection::TupleDataCollection(ClientContext &context, shared_ptr layout_ptr) + : TupleDataCollection(BufferManager::GetBufferManager(context), std::move(layout_ptr)) { +} + TupleDataCollection::~TupleDataCollection() { } @@ -562,11 +566,16 @@ void TupleDataCollection::InitializeScan(TupleDataParallelScanState &state, vect InitializeScan(state.scan_state, std::move(column_ids), properties); } -idx_t TupleDataCollection::FetchChunk(TupleDataScanState &state, const idx_t segment_idx, const idx_t chunk_idx, - const bool init_heap) { - auto &segment = *segments[segment_idx]; - allocator->InitializeChunkState(segment, state.pin_state, state.chunk_state, chunk_idx, init_heap); - return segment.chunks[chunk_idx].count; +idx_t TupleDataCollection::FetchChunk(TupleDataScanState &state, idx_t chunk_idx, bool init_heap) { + for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) { + auto &segment = *segments[segment_idx]; + if (chunk_idx < segment.ChunkCount()) { + segment.allocator->InitializeChunkState(segment, state.pin_state, state.chunk_state, chunk_idx, init_heap); + return segment.chunks[chunk_idx].count; + } + chunk_idx -= segment.ChunkCount(); + } + throw InternalException("Chunk index out of in TupleDataCollection::FetchChunk"); } bool TupleDataCollection::Scan(TupleDataScanState &state, DataChunk &result) { diff --git a/src/common/types/selection_vector.cpp b/src/common/types/selection_vector.cpp index 145b6bfa1bc8..a1232340c0a5 100644 --- a/src/common/types/selection_vector.cpp +++ b/src/common/types/selection_vector.cpp @@ -50,6 +50,14 @@ buffer_ptr SelectionVector::Slice(const SelectionVector &sel, idx return data; } +idx_t SelectionVector::SliceInPlace(const SelectionVector &source, idx_t count) { + for (idx_t i = 0; i < count; ++i) { + set_index(i, get_index(source.get_index(i))); + } + + return count; +} + void SelectionVector::Verify(idx_t count, idx_t vector_size) const { #ifdef DEBUG D_ASSERT(vector_size >= 1); diff --git a/src/common/types/string_type.cpp b/src/common/types/string_type.cpp index f5a236557760..bea85327a5c8 100644 --- a/src/common/types/string_type.cpp +++ b/src/common/types/string_type.cpp @@ -6,6 +6,8 @@ #include "utf8proc_wrapper.hpp" namespace duckdb { +constexpr idx_t string_t::MAX_STRING_SIZE; +constexpr idx_t string_t::INLINE_LENGTH; void string_t::Verify() const { #ifdef DEBUG diff --git a/src/common/types/value.cpp b/src/common/types/value.cpp index 2bef3a82daa0..a232904e12e3 100644 --- a/src/common/types/value.cpp +++ b/src/common/types/value.cpp @@ -919,6 +919,13 @@ Value Value::BIGNUM(const string &data) { return result; } +Value Value::GEOMETRY(const_data_ptr_t data, idx_t len) { + Value result(LogicalTypeId::GEOMETRY); + result.is_null = false; + result.value_info_ = make_shared_ptr(string(const_char_ptr_cast(data), len)); + return result; +} + Value Value::BLOB(const string &data) { Value result(LogicalType::BLOB); result.is_null = false; diff --git a/src/common/types/vector.cpp b/src/common/types/vector.cpp index ad27b162dae4..27ce2b078d3f 100644 --- a/src/common/types/vector.cpp +++ b/src/common/types/vector.cpp @@ -96,8 +96,7 @@ Vector::Vector(const Value &value) : type(value.type()) { Vector::Vector(Vector &&other) noexcept : vector_type(other.vector_type), type(std::move(other.type)), data(other.data), - validity(std::move(other.validity)), buffer(std::move(other.buffer)), auxiliary(std::move(other.auxiliary)), - cached_hashes(std::move(other.cached_hashes)) { + validity(std::move(other.validity)), buffer(std::move(other.buffer)), auxiliary(std::move(other.auxiliary)) { } void Vector::Reference(const Value &value) { @@ -171,7 +170,6 @@ void Vector::Reinterpret(const Vector &other) { auxiliary = make_shared_ptr(std::move(new_vector)); } else { AssignSharedPointer(auxiliary, other.auxiliary); - AssignSharedPointer(cached_hashes, other.cached_hashes); } data = other.data; validity = other.validity; @@ -235,6 +233,9 @@ void Vector::Slice(const Vector &other, const SelectionVector &sel, idx_t count) } void Vector::Slice(const SelectionVector &sel, idx_t count) { + if (!sel.IsSet() || count == 0) { + return; // Nothing to do here + } if (GetVectorType() == VectorType::CONSTANT_VECTOR) { // dictionary on a constant is just a constant return; @@ -276,7 +277,6 @@ void Vector::Slice(const SelectionVector &sel, idx_t count) { vector_type = VectorType::DICTIONARY_VECTOR; buffer = std::move(dict_buffer); auxiliary = std::move(child_ref); - cached_hashes.reset(); } void Vector::Dictionary(idx_t dictionary_size, const SelectionVector &sel, idx_t count) { @@ -287,15 +287,25 @@ void Vector::Dictionary(idx_t dictionary_size, const SelectionVector &sel, idx_t } void Vector::Dictionary(Vector &dict, idx_t dictionary_size, const SelectionVector &sel, idx_t count) { - if (DictionaryVector::CanCacheHashes(dict.GetType()) && !dict.cached_hashes) { - // Create an empty hash vector for this dictionary, potentially to be used for caching hashes later - // This needs to happen here, as we need to add "cached_hashes" to the original input Vector "dict" - dict.cached_hashes = make_buffer(Vector(LogicalType::HASH, false, false, 0)); - } Reference(dict); Dictionary(dictionary_size, sel, count); } +void Vector::Dictionary(buffer_ptr reusable_dict, const SelectionVector &sel) { + D_ASSERT(type.InternalType() != PhysicalType::STRUCT); + D_ASSERT(type == reusable_dict->data.GetType()); + vector_type = VectorType::DICTIONARY_VECTOR; + data = reusable_dict->data.data; + validity.Reset(); + + auto dict_buffer = make_buffer(sel); + dict_buffer->SetDictionarySize(reusable_dict->size.GetIndex()); + dict_buffer->SetDictionaryId(reusable_dict->id); + buffer = std::move(dict_buffer); + + auxiliary = std::move(reusable_dict); +} + void Vector::Slice(const SelectionVector &sel, idx_t count, SelCache &cache) { if (GetVectorType() == VectorType::DICTIONARY_VECTOR && GetType().InternalType() != PhysicalType::STRUCT) { // dictionary vector: need to merge dictionaries @@ -724,6 +734,10 @@ Value Vector::GetValueInternal(const Vector &v_p, idx_t index_p) { auto str = reinterpret_cast(data)[index]; return Value::BIGNUM(const_data_ptr_cast(str.data.GetData()), str.data.GetSize()); } + case LogicalTypeId::GEOMETRY: { + auto str = reinterpret_cast(data)[index]; + return Value::GEOMETRY(const_data_ptr_cast(str.GetData()), str.GetSize()); + } case LogicalTypeId::AGGREGATE_STATE: { auto str = reinterpret_cast(data)[index]; return Value::AGGREGATE_STATE(vector->GetType(), const_data_ptr_cast(str.GetData()), str.GetSize()); @@ -1846,7 +1860,9 @@ void Vector::DebugTransformToDictionary(Vector &vector, idx_t count) { inverted_sel.set_index(offset++, current_index); inverted_sel.set_index(offset++, current_index); } - Vector inverted_vector(vector, inverted_sel, verify_count); + auto reusable_dict = DictionaryVector::CreateReusableDictionary(vector.type, verify_count); + auto &inverted_vector = reusable_dict->data; + inverted_vector.Slice(vector, inverted_sel, verify_count); inverted_vector.Flatten(verify_count); // now insert the NULL values at every other position for (idx_t i = 0; i < count; i++) { @@ -1860,8 +1876,13 @@ void Vector::DebugTransformToDictionary(Vector &vector, idx_t count) { original_sel.set_index(offset++, verify_count - 1 - i * 2); } // now slice the inverted vector with the inverted selection vector - vector.Dictionary(inverted_vector, verify_count, original_sel, count); - DictionaryVector::SetDictionaryId(vector, UUID::ToString(UUID::GenerateRandomUUID())); + if (vector.GetType().InternalType() == PhysicalType::STRUCT) { + // Reusable dictionary API does not work for STRUCT + vector.Dictionary(inverted_vector, verify_count, original_sel, count); + vector.buffer->Cast().SetDictionaryId(reusable_dict->id); + } else { + vector.Dictionary(reusable_dict, original_sel); + } vector.Verify(count); } @@ -1922,17 +1943,27 @@ void Vector::DebugShuffleNestedVector(Vector &vector, idx_t count) { //===--------------------------------------------------------------------===// // DictionaryVector //===--------------------------------------------------------------------===// +buffer_ptr DictionaryVector::CreateReusableDictionary(const LogicalType &type, const idx_t &size) { + auto res = make_buffer(Vector(type, size)); + res->size = size; + res->id = UUID::ToString(UUID::GenerateRandomUUID()); + return res; +} + const Vector &DictionaryVector::GetCachedHashes(Vector &input) { D_ASSERT(CanCacheHashes(input)); - auto &dictionary = Child(input); - auto &dictionary_hashes = dictionary.cached_hashes->Cast().data; - if (!dictionary_hashes.data) { + + auto &child = input.auxiliary->Cast(); + lock_guard guard(child.cached_hashes_lock); + + if (!child.cached_hashes.data) { // Uninitialized: hash the dictionary - const auto dictionary_count = DictionarySize(input).GetIndex(); - dictionary_hashes.Initialize(false, dictionary_count); - VectorOperations::Hash(dictionary, dictionary_hashes, dictionary_count); + const auto dictionary_size = DictionarySize(input).GetIndex(); + D_ASSERT(!child.size.IsValid() || child.size.GetIndex() == dictionary_size); + child.cached_hashes.Initialize(false, dictionary_size); + VectorOperations::Hash(child.data, child.cached_hashes, dictionary_size); } - return dictionary.cached_hashes->Cast().data; + return child.cached_hashes; } //===--------------------------------------------------------------------===// diff --git a/src/common/value_operations/comparison_operations.cpp b/src/common/value_operations/comparison_operations.cpp index ac4c88c01652..775757c59fc5 100644 --- a/src/common/value_operations/comparison_operations.cpp +++ b/src/common/value_operations/comparison_operations.cpp @@ -141,6 +141,11 @@ static bool TemplatedBooleanOperation(const Value &left, const Value &right) { auto &right_children = StructValue::GetChildren(right); // this should be enforced by the type D_ASSERT(left_children.size() == right_children.size()); + if (left_children.empty()) { + const auto const_true = Value::BOOLEAN(true); + return ValuePositionComparator::Final(const_true, const_true); + } + idx_t i = 0; for (; i < left_children.size() - 1; ++i) { if (ValuePositionComparator::Definite(left_children[i], right_children[i])) { diff --git a/src/common/vector_operations/is_distinct_from.cpp b/src/common/vector_operations/is_distinct_from.cpp index e57f9738d1e1..22b61dbc03c8 100644 --- a/src/common/vector_operations/is_distinct_from.cpp +++ b/src/common/vector_operations/is_distinct_from.cpp @@ -289,6 +289,7 @@ template idx_t DistinctSelect(Vector &left, Vector &right, const SelectionVector *sel, idx_t count, SelectionVector *true_sel, SelectionVector *false_sel, optional_ptr null_mask) { if (!sel) { + D_ASSERT(count <= STANDARD_VECTOR_SIZE); sel = FlatVector::IncrementalSelectionVector(); } @@ -478,21 +479,22 @@ void ExtractNestedSelection(const SelectionVector &slice_sel, const idx_t count, } void ExtractNestedMask(const SelectionVector &slice_sel, const idx_t count, const SelectionVector &sel, - ValidityMask *child_mask, optional_ptr null_mask) { + ValidityMask *child_mask_p, optional_ptr null_mask) { - if (!child_mask) { + if (!child_mask_p) { return; } + auto &child_mask = *child_mask_p; for (idx_t i = 0; i < count; ++i) { const auto slice_idx = slice_sel.get_index(i); const auto result_idx = sel.get_index(slice_idx); - if (child_mask && !child_mask->RowIsValid(slice_idx)) { + if (!child_mask.RowIsValid(slice_idx)) { null_mask->SetInvalid(result_idx); } } - child_mask->Reset(null_mask->Capacity()); + child_mask.Reset(null_mask->Capacity()); } void DensifyNestedSelection(const SelectionVector &dense_sel, const idx_t count, SelectionVector &slice_sel) { @@ -890,6 +892,7 @@ idx_t DistinctSelectNested(Vector &left, Vector &right, optional_ptr(l_not_null, r_not_null, count, match_count, *sel, maybe_vec, true_opt, false_opt, null_mask); - switch (left.GetType().InternalType()) { + auto &left_type = left.GetType(); + switch (left_type.InternalType()) { case PhysicalType::LIST: match_count += DistinctSelectList(l_not_null, r_not_null, unknown, maybe_vec, true_opt, false_opt, null_mask); diff --git a/src/common/vector_operations/vector_hash.cpp b/src/common/vector_operations/vector_hash.cpp index 062ee1719679..95551ecdbbaf 100644 --- a/src/common/vector_operations/vector_hash.cpp +++ b/src/common/vector_operations/vector_hash.cpp @@ -483,8 +483,8 @@ void CombineHashTypeSwitch(Vector &hashes, Vector &input, const SelectionVector void VectorOperations::Hash(Vector &input, Vector &result, idx_t count) { if (input.GetVectorType() == VectorType::DICTIONARY_VECTOR && DictionaryVector::CanCacheHashes(input)) { - VectorOperations::Copy(DictionaryVector::GetCachedHashes(input), result, DictionaryVector::SelVector(input), - count, 0, 0); + Vector input_hashes(DictionaryVector::GetCachedHashes(input), DictionaryVector::SelVector(input), count); + TemplatedLoopHash(input_hashes, result, nullptr, count); } else { HashTypeSwitch(input, result, nullptr, count); } diff --git a/src/common/virtual_file_system.cpp b/src/common/virtual_file_system.cpp index 757ebef16fb5..dfb542ec718a 100644 --- a/src/common/virtual_file_system.cpp +++ b/src/common/virtual_file_system.cpp @@ -17,6 +17,7 @@ VirtualFileSystem::VirtualFileSystem(unique_ptr &&inner) : default_f unique_ptr VirtualFileSystem::OpenFileExtended(const OpenFileInfo &file, FileOpenFlags flags, optional_ptr opener) { + auto compression = flags.Compression(); if (compression == FileCompressionType::AUTO_DETECT) { // auto-detect compression settings based on file name @@ -34,8 +35,9 @@ unique_ptr VirtualFileSystem::OpenFileExtended(const OpenFileInfo &f } } // open the base file handle in UNCOMPRESSED mode + flags.SetCompression(FileCompressionType::UNCOMPRESSED); - auto file_handle = FindFileSystem(file.path).OpenFile(file, flags, opener); + auto file_handle = FindFileSystem(file.path, opener).OpenFile(file, flags, opener); if (!file_handle) { return nullptr; } @@ -111,7 +113,7 @@ void VirtualFileSystem::RemoveDirectory(const string &directory, optional_ptr &callback, optional_ptr opener) { - return FindFileSystem(directory).ListFiles(directory, callback, opener); + return FindFileSystem(directory, opener).ListFiles(directory, callback, opener); } void VirtualFileSystem::MoveFile(const string &source, const string &target, optional_ptr opener) { @@ -119,7 +121,7 @@ void VirtualFileSystem::MoveFile(const string &source, const string &target, opt } bool VirtualFileSystem::FileExists(const string &filename, optional_ptr opener) { - return FindFileSystem(filename).FileExists(filename, opener); + return FindFileSystem(filename, opener).FileExists(filename, opener); } bool VirtualFileSystem::IsPipe(const string &filename, optional_ptr opener) { @@ -139,10 +141,18 @@ string VirtualFileSystem::PathSeparator(const string &path) { } vector VirtualFileSystem::Glob(const string &path, FileOpener *opener) { - return FindFileSystem(path).Glob(path, opener); + return FindFileSystem(path, opener).Glob(path, opener); } void VirtualFileSystem::RegisterSubSystem(unique_ptr fs) { + // Sub-filesystem number is not expected to be huge, also filesystem registration should be called infrequently. + const auto &name = fs->GetName(); + for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) { + if (sub_system->get()->GetName() == name) { + throw InvalidInputException("Filesystem with name %s has already been registered, cannot re-register!", + name); + } + } sub_systems.push_back(std::move(fs)); } @@ -216,16 +226,61 @@ bool VirtualFileSystem::SubSystemIsDisabled(const string &name) { return disabled_file_systems.find(name) != disabled_file_systems.end(); } +FileSystem &VirtualFileSystem::FindFileSystem(const string &path, optional_ptr opener) { + return FindFileSystem(path, FileOpener::TryGetDatabase(opener)); +} + +FileSystem &VirtualFileSystem::FindFileSystem(const string &path, optional_ptr db_instance) { + auto fs = FindFileSystemInternal(path); + + if (!fs && db_instance) { + string required_extension; + + for (const auto &entry : EXTENSION_FILE_PREFIXES) { + if (StringUtil::StartsWith(path, entry.name)) { + required_extension = entry.extension; + } + } + if (!required_extension.empty() && db_instance && !db_instance->ExtensionIsLoaded(required_extension)) { + auto &dbconfig = DBConfig::GetConfig(*db_instance); + if (!ExtensionHelper::CanAutoloadExtension(required_extension) || + !dbconfig.options.autoload_known_extensions) { + auto error_message = "File " + path + " requires the extension " + required_extension + " to be loaded"; + error_message = + ExtensionHelper::AddExtensionInstallHintToErrorMsg(*db_instance, error_message, required_extension); + throw MissingExtensionException(error_message); + } + // an extension is required to read this file, but it is not loaded - try to load it + ExtensionHelper::AutoLoadExtension(*db_instance, required_extension); + } + + // Retry after having autoloaded + fs = FindFileSystem(path); + } + + if (!fs) { + fs = default_fs; + } + if (!disabled_file_systems.empty() && disabled_file_systems.find(fs->GetName()) != disabled_file_systems.end()) { + throw PermissionException("File system %s has been disabled by configuration", fs->GetName()); + } + return *fs; +} + FileSystem &VirtualFileSystem::FindFileSystem(const string &path) { - auto &fs = FindFileSystemInternal(path); - if (!disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) { - throw PermissionException("File system %s has been disabled by configuration", fs.GetName()); + auto fs = FindFileSystemInternal(path); + if (!fs) { + fs = default_fs; + } + if (!disabled_file_systems.empty() && disabled_file_systems.find(fs->GetName()) != disabled_file_systems.end()) { + throw PermissionException("File system %s has been disabled by configuration", fs->GetName()); } - return fs; + return *fs; } -FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) { +optional_ptr VirtualFileSystem::FindFileSystemInternal(const string &path) { FileSystem *fs = nullptr; + for (auto &sub_system : sub_systems) { if (sub_system->CanHandleFile(path)) { if (sub_system->IsManuallySet()) { @@ -237,7 +292,9 @@ FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) { if (fs) { return *fs; } - return *default_fs; + + // We could use default_fs, that's on the caller + return nullptr; } } // namespace duckdb diff --git a/src/execution/expression_executor.cpp b/src/execution/expression_executor.cpp index ec11c128905d..8ea2bba0909a 100644 --- a/src/execution/expression_executor.cpp +++ b/src/execution/expression_executor.cpp @@ -181,6 +181,8 @@ void ExpressionExecutor::Verify(const Expression &expr, Vector &vector, idx_t co } else { VectorOperations::DefaultCast(vector, intermediate, count, true); } + intermediate.Verify(count); + //! FIXME: this is probably also where we want to test 'variant_normalize' Vector result(vector.GetType(), true, false, count); //! Then cast back into the original type @@ -190,6 +192,7 @@ void ExpressionExecutor::Verify(const Expression &expr, Vector &vector, idx_t co VectorOperations::DefaultCast(intermediate, result, count, true); } vector.Reference(result); + vector.Verify(count); } } diff --git a/src/execution/expression_executor/execute_function.cpp b/src/execution/expression_executor/execute_function.cpp index a7e99287b769..a7c3b026bd94 100644 --- a/src/execution/expression_executor/execute_function.cpp +++ b/src/execution/expression_executor/execute_function.cpp @@ -71,7 +71,7 @@ bool ExecuteFunctionState::TryExecuteDictionaryExpression(const BoundFunctionExp return false; // Dictionary is too large, bail } - if (input_dictionary_id != current_input_dictionary_id) { + if (!output_dictionary || current_input_dictionary_id != input_dictionary_id) { // We haven't seen this dictionary before const auto chunk_fill_ratio = static_cast(args.size()) / STANDARD_VECTOR_SIZE; if (input_dictionary_size > STANDARD_VECTOR_SIZE && chunk_fill_ratio <= CHUNK_FILL_RATIO_THRESHOLD) { @@ -82,9 +82,8 @@ bool ExecuteFunctionState::TryExecuteDictionaryExpression(const BoundFunctionExp } // We can do dictionary optimization! Re-initialize + output_dictionary = DictionaryVector::CreateReusableDictionary(result.GetType(), input_dictionary_size); current_input_dictionary_id = input_dictionary_id; - output_dictionary = make_uniq(result.GetType(), input_dictionary_size); - output_dictionary_id = UUID::ToString(UUID::GenerateRandomUUID()); // Set up the input chunk DataChunk input_chunk; @@ -105,16 +104,14 @@ bool ExecuteFunctionState::TryExecuteDictionaryExpression(const BoundFunctionExp input_chunk.SetCardinality(count); // Execute, storing the result in an intermediate vector, and copying it to the output dictionary - Vector output_intermediate(output_dictionary->GetType()); + Vector output_intermediate(result.GetType()); expr.function.function(input_chunk, state, output_intermediate); - VectorOperations::Copy(output_intermediate, *output_dictionary, count, 0, offset); + VectorOperations::Copy(output_intermediate, output_dictionary->data, count, 0, offset); } } - // Create a dictionary result vector and give it an ID - const auto &input_sel_vector = DictionaryVector::SelVector(unary_input); - result.Dictionary(*output_dictionary, input_dictionary_size, input_sel_vector, args.size()); - DictionaryVector::SetDictionaryId(result, output_dictionary_id); + // Result references the dictionary + result.Dictionary(output_dictionary, DictionaryVector::SelVector(unary_input)); return true; } diff --git a/src/execution/index/art/art_merger.cpp b/src/execution/index/art/art_merger.cpp index 70781cbfbbe5..61d2ec317700 100644 --- a/src/execution/index/art/art_merger.cpp +++ b/src/execution/index/art/art_merger.cpp @@ -217,9 +217,6 @@ void ARTMerger::MergeNodeAndPrefix(Node &node, Node &prefix, const GateStatus pa auto child = node.GetChildMutable(art, byte); // Reduce the prefix to the bytes after pos. - // We always reduce by at least one byte, - // thus, if the prefix was a gate, it no longer is. - prefix.SetGateStatus(GateStatus::GATE_NOT_SET); Prefix::Reduce(art, prefix, pos); if (child) { diff --git a/src/execution/index/art/base_leaf.cpp b/src/execution/index/art/base_leaf.cpp index a694ca3b593d..4a9332fc9d17 100644 --- a/src/execution/index/art/base_leaf.cpp +++ b/src/execution/index/art/base_leaf.cpp @@ -30,8 +30,10 @@ void BaseLeaf::InsertByteInternal(BaseLeaf &n, const uint8_t byt } template -BaseLeaf &BaseLeaf::DeleteByteInternal(ART &art, Node &node, const uint8_t byte) { - auto &n = Node::Ref(art, node, node.GetType()); +NodeHandle> BaseLeaf::DeleteByteInternal(ART &art, Node &node, + const uint8_t byte) { + NodeHandle> handle(art, node); + auto &n = handle.Get(); uint8_t child_pos = 0; for (; child_pos < n.count; child_pos++) { @@ -45,7 +47,7 @@ BaseLeaf &BaseLeaf::DeleteByteInternal(ART &art, for (uint8_t i = child_pos; i < n.count; i++) { n.key[i] = n.key[i + 1]; } - return n; + return handle; } //===--------------------------------------------------------------------===// @@ -53,27 +55,36 @@ BaseLeaf &BaseLeaf::DeleteByteInternal(ART &art, //===--------------------------------------------------------------------===// void Node7Leaf::InsertByte(ART &art, Node &node, const uint8_t byte) { - // The node is full. Grow to Node15. - auto &n7 = Node::Ref(art, node, NODE_7_LEAF); - if (n7.count == CAPACITY) { - auto node7 = node; - Node15Leaf::GrowNode7Leaf(art, node, node7); - Node15Leaf::InsertByte(art, node, byte); - return; - } + { + NodeHandle handle(art, node); + auto &n7 = handle.Get(); - InsertByteInternal(n7, byte); + if (n7.count != CAPACITY) { + InsertByteInternal(n7, byte); + return; + } + } + // The node is full. Grow to Node15. + auto node7 = node; + Node15Leaf::GrowNode7Leaf(art, node, node7); + Node15Leaf::InsertByte(art, node, byte); } void Node7Leaf::DeleteByte(ART &art, Node &node, Node &prefix, const uint8_t byte, const ARTKey &row_id) { - auto &n7 = DeleteByteInternal(art, node, byte); + idx_t remainder; + { + auto n7_handle = DeleteByteInternal(art, node, byte); + auto &n7 = n7_handle.Get(); + + if (n7.count != 1) { + return; + } - // Compress one-way nodes. - if (n7.count == 1) { + // Compress one-way nodes. D_ASSERT(node.GetGateStatus() == GateStatus::GATE_NOT_SET); // Get the remaining row ID. - auto remainder = UnsafeNumericCast(row_id.GetRowId()) & AND_LAST_BYTE; + remainder = UnsafeNumericCast(row_id.GetRowId()) & AND_LAST_BYTE; remainder |= UnsafeNumericCast(n7.key[0]); // Free the prefix (nodes) and inline the remainder. @@ -82,23 +93,27 @@ void Node7Leaf::DeleteByte(ART &art, Node &node, Node &prefix, const uint8_t byt Leaf::New(prefix, UnsafeNumericCast(remainder)); return; } - - // Free the Node7Leaf and inline the remainder. - Node::FreeNode(art, node); - Leaf::New(node, UnsafeNumericCast(remainder)); } + // Free the Node7Leaf and inline the remainder. + Node::FreeNode(art, node); + Leaf::New(node, UnsafeNumericCast(remainder)); } void Node7Leaf::ShrinkNode15Leaf(ART &art, Node &node7_leaf, Node &node15_leaf) { - auto &n7 = New(art, node7_leaf); - auto &n15 = Node::Ref(art, node15_leaf, NType::NODE_15_LEAF); - node7_leaf.SetGateStatus(node15_leaf.GetGateStatus()); + { + auto n7_handle = New(art, node7_leaf); + auto &n7 = n7_handle.Get(); - n7.count = n15.count; - for (uint8_t i = 0; i < n15.count; i++) { - n7.key[i] = n15.key[i]; - } + NodeHandle n15_handle(art, node15_leaf); + auto &n15 = n15_handle.Get(); + node7_leaf.SetGateStatus(node15_leaf.GetGateStatus()); + + n7.count = n15.count; + for (uint8_t i = 0; i < n15.count; i++) { + n7.key[i] = n15.key[i]; + } + } Node::FreeNode(art, node15_leaf); } @@ -107,54 +122,66 @@ void Node7Leaf::ShrinkNode15Leaf(ART &art, Node &node7_leaf, Node &node15_leaf) //===--------------------------------------------------------------------===// void Node15Leaf::InsertByte(ART &art, Node &node, const uint8_t byte) { - // The node is full. Grow to Node256Leaf. - auto &n15 = Node::Ref(art, node, NODE_15_LEAF); - if (n15.count == CAPACITY) { - auto node15 = node; - Node256Leaf::GrowNode15Leaf(art, node, node15); - Node256Leaf::InsertByte(art, node, byte); - return; + { + NodeHandle n15_handle(art, node); + auto &n15 = n15_handle.Get(); + if (n15.count != CAPACITY) { + InsertByteInternal(n15, byte); + return; + } } - - InsertByteInternal(n15, byte); + auto node15 = node; + Node256Leaf::GrowNode15Leaf(art, node, node15); + Node256Leaf::InsertByte(art, node, byte); } void Node15Leaf::DeleteByte(ART &art, Node &node, const uint8_t byte) { - auto &n15 = DeleteByteInternal(art, node, byte); - - // Shrink node to Node7. - if (n15.count < Node7Leaf::CAPACITY) { - auto node15 = node; - Node7Leaf::ShrinkNode15Leaf(art, node, node15); + { + auto n15_handle = DeleteByteInternal(art, node, byte); + auto &n15 = n15_handle.Get(); + if (n15.count >= Node7Leaf::CAPACITY) { + return; + } } + auto node15 = node; + Node7Leaf::ShrinkNode15Leaf(art, node, node15); } void Node15Leaf::GrowNode7Leaf(ART &art, Node &node15_leaf, Node &node7_leaf) { - auto &n7 = Node::Ref(art, node7_leaf, NType::NODE_7_LEAF); - auto &n15 = New(art, node15_leaf); - node15_leaf.SetGateStatus(node7_leaf.GetGateStatus()); + { + NodeHandle n7_handle(art, node7_leaf); + auto &n7 = n7_handle.Get(); - n15.count = n7.count; - for (uint8_t i = 0; i < n7.count; i++) { - n15.key[i] = n7.key[i]; - } + auto n15_handle = New(art, node15_leaf); + auto &n15 = n15_handle.Get(); + node15_leaf.SetGateStatus(node7_leaf.GetGateStatus()); + n15.count = n7.count; + for (uint8_t i = 0; i < n7.count; i++) { + n15.key[i] = n7.key[i]; + } + } Node::FreeNode(art, node7_leaf); } void Node15Leaf::ShrinkNode256Leaf(ART &art, Node &node15_leaf, Node &node256_leaf) { - auto &n15 = New(art, node15_leaf); - auto &n256 = Node::Ref(art, node256_leaf, NType::NODE_256_LEAF); - node15_leaf.SetGateStatus(node256_leaf.GetGateStatus()); - - ValidityMask mask(&n256.mask[0], Node256::CAPACITY); - for (uint16_t i = 0; i < Node256::CAPACITY; i++) { - if (mask.RowIsValid(i)) { - n15.key[n15.count] = UnsafeNumericCast(i); - n15.count++; + { + auto n15_handle = New(art, node15_leaf); + auto &n15 = n15_handle.Get(); + + NodeHandle n256_handle(art, node256_leaf); + auto &n256 = n256_handle.Get(); + + node15_leaf.SetGateStatus(node256_leaf.GetGateStatus()); + + ValidityMask mask(&n256.mask[0], Node256::CAPACITY); + for (uint16_t i = 0; i < Node256::CAPACITY; i++) { + if (mask.RowIsValid(i)) { + n15.key[n15.count] = UnsafeNumericCast(i); + n15.count++; + } } } - Node::FreeNode(art, node256_leaf); } diff --git a/src/execution/index/art/prefix.cpp b/src/execution/index/art/prefix.cpp index 00e94967abf7..1d7861135d91 100644 --- a/src/execution/index/art/prefix.cpp +++ b/src/execution/index/art/prefix.cpp @@ -100,6 +100,10 @@ void Prefix::Reduce(ART &art, Node &node, const idx_t pos) { D_ASSERT(node.HasMetadata()); D_ASSERT(pos < Count(art)); + // We always reduce by at least one byte, + // thus, if the prefix was a gate, it no longer is. + node.SetGateStatus(GateStatus::GATE_NOT_SET); + Prefix prefix(art, node); if (pos == idx_t(prefix.data[Count(art)] - 1)) { auto next = *prefix.ptr; diff --git a/src/execution/index/fixed_size_allocator.cpp b/src/execution/index/fixed_size_allocator.cpp index dd4758bb971a..cffd0b61c7dc 100644 --- a/src/execution/index/fixed_size_allocator.cpp +++ b/src/execution/index/fixed_size_allocator.cpp @@ -4,9 +4,9 @@ namespace duckdb { -FixedSizeAllocator::FixedSizeAllocator(const idx_t segment_size, BlockManager &block_manager) - : block_manager(block_manager), buffer_manager(block_manager.buffer_manager), segment_size(segment_size), - total_segment_count(0) { +FixedSizeAllocator::FixedSizeAllocator(const idx_t segment_size, BlockManager &block_manager, MemoryTag memory_tag) + : block_manager(block_manager), buffer_manager(block_manager.buffer_manager), memory_tag(memory_tag), + segment_size(segment_size), total_segment_count(0) { if (segment_size > block_manager.GetBlockSize() - sizeof(validity_t)) { throw InternalException("The maximum segment size of fixed-size allocators is " + @@ -48,7 +48,7 @@ IndexPointer FixedSizeAllocator::New() { if (!buffer_with_free_space.IsValid()) { // Add a new buffer. auto buffer_id = GetAvailableBufferId(); - buffers[buffer_id] = make_uniq(block_manager); + buffers[buffer_id] = make_uniq(block_manager, memory_tag); buffers_with_free_space.insert(buffer_id); buffer_with_free_space = buffer_id; diff --git a/src/execution/index/fixed_size_buffer.cpp b/src/execution/index/fixed_size_buffer.cpp index 82bbccac2783..26e56cd5ce03 100644 --- a/src/execution/index/fixed_size_buffer.cpp +++ b/src/execution/index/fixed_size_buffer.cpp @@ -35,12 +35,12 @@ void PartialBlockForIndex::Clear() { constexpr idx_t FixedSizeBuffer::BASE[]; constexpr uint8_t FixedSizeBuffer::SHIFT[]; -FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager) +FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager, MemoryTag memory_tag) : block_manager(block_manager), readers(0), segment_count(0), allocation_size(0), dirty(false), vacuum(false), loaded(false), block_pointer(), block_handle(nullptr) { auto &buffer_manager = block_manager.buffer_manager; - buffer_handle = buffer_manager.Allocate(MemoryTag::ART_INDEX, &block_manager, false); + buffer_handle = buffer_manager.Allocate(memory_tag, &block_manager, false); block_handle = buffer_handle.GetBlockHandle(); // Zero-initialize the buffer as it might get serialized to storage. diff --git a/src/execution/join_hashtable.cpp b/src/execution/join_hashtable.cpp index cfa845a88863..f991ead7ef4b 100644 --- a/src/execution/join_hashtable.cpp +++ b/src/execution/join_hashtable.cpp @@ -114,6 +114,11 @@ JoinHashTable::JoinHashTable(ClientContext &context_p, const PhysicalOperator &o single_join_error_on_multiple_rows = DBConfig::GetSetting(context); } + if (conditions.size() == 1 && + (join_type == JoinType::SEMI || join_type == JoinType::ANTI || join_type == JoinType::MARK)) { + insert_duplicate_keys = false; + } + InitializePartitionMasks(); } @@ -562,16 +567,18 @@ static inline void InsertMatchesAndIncrementMisses(atomic entries[], } // Insert the rows that match - for (idx_t i = 0; i < key_match_count; i++) { - const auto need_compare_idx = state.key_match_sel.get_index(i); - const auto entry_index = state.keys_to_compare_sel.get_index(need_compare_idx); + if (ht.insert_duplicate_keys) { + for (idx_t i = 0; i < key_match_count; i++) { + const auto need_compare_idx = state.key_match_sel.get_index(i); + const auto entry_index = state.keys_to_compare_sel.get_index(need_compare_idx); - const auto &ht_offset = ht_offsets[entry_index]; - auto &entry = entries[ht_offset]; - const auto row_ptr_to_insert = lhs_row_locations[entry_index]; + const auto &ht_offset = ht_offsets[entry_index]; + auto &entry = entries[ht_offset]; + const auto row_ptr_to_insert = lhs_row_locations[entry_index]; - const auto salt = hash_salts[entry_index]; - InsertRowToEntry(entry, row_ptr_to_insert, salt, ht.pointer_offset); + const auto salt = hash_salts[entry_index]; + InsertRowToEntry(entry, row_ptr_to_insert, salt, ht.pointer_offset); + } } // Linear probing: each of the entries that do not match move to the next entry in the HT diff --git a/src/execution/operator/aggregate/physical_window.cpp b/src/execution/operator/aggregate/physical_window.cpp index 265dbff3a422..102f491f0e15 100644 --- a/src/execution/operator/aggregate/physical_window.cpp +++ b/src/execution/operator/aggregate/physical_window.cpp @@ -334,6 +334,12 @@ SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, Clie return global_partition.MaterializeHashGroups(pipeline, event, *this, hfinalize); } +ProgressData PhysicalWindow::GetSinkProgress(ClientContext &context, GlobalSinkState &gstate, + const ProgressData source_progress) const { + auto &gsink = gstate.Cast(); + return gsink.global_partition->GetSinkProgress(context, *gsink.hashed_sink, source_progress); +} + //===--------------------------------------------------------------------===// // Source //===--------------------------------------------------------------------===// @@ -381,8 +387,8 @@ class WindowGlobalSourceState : public GlobalSourceState { atomic finished; //! Stop producing tasks atomic stopped; - //! The number of rows returned - atomic returned; + //! The number of tasks completed. This will combine both build and evaluate. + atomic completed; public: idx_t MaxThreads() override { @@ -397,7 +403,7 @@ class WindowGlobalSourceState : public GlobalSourceState { }; WindowGlobalSourceState::WindowGlobalSourceState(ClientContext &client, WindowGlobalSinkState &gsink_p) - : client(client), gsink(gsink_p), next_group(0), locals(0), started(0), finished(0), stopped(false), returned(0) { + : client(client), gsink(gsink_p), next_group(0), locals(0), started(0), finished(0), stopped(false), completed(0) { auto &global_partition = *gsink.global_partition; auto hashed_source = global_partition.GetGlobalSourceState(client, *gsink.hashed_sink); @@ -888,6 +894,9 @@ void WindowGlobalSourceState::FinishTask(TaskPtr task) { auto &v = active_groups; v.erase(std::remove(v.begin(), v.end(), group_idx), v.end()); } + + // Count the global tasks completed. + ++completed; } bool WindowLocalSourceState::TryAssignTask() { @@ -1020,17 +1029,20 @@ OrderPreservationType PhysicalWindow::SourceOrder() const { ProgressData PhysicalWindow::GetProgress(ClientContext &client, GlobalSourceState &gsource_p) const { auto &gsource = gsource_p.Cast(); - const auto returned = gsource.returned.load(); - auto &gsink = gsource.gsink; const auto count = gsink.count.load(); + const auto completed = gsource.completed.load(); + ProgressData res; if (count) { - res.done = double(returned); - res.total = double(count); + res.done = double(completed); + res.total = double(gsource.total_tasks); + // Convert to tuples. + res.Normalize(double(count)); } else { res.SetInvalid(); } + return res; } @@ -1071,8 +1083,6 @@ SourceResultType PhysicalWindow::GetData(ExecutionContext &context, DataChunk &c } } - gsource.returned += chunk.size(); - if (chunk.size() == 0) { return SourceResultType::FINISHED; } diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index f390819a69c2..5ed14a992a37 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -797,7 +797,7 @@ void StringValueResult::NullPaddingQuotedNewlineCheck() const { // If we have null_padding set, we found a quoted new line, we are scanning the file in parallel; We error. LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::NullPaddingFail(state_machine.options, lines_per_batch, path); - error_handler.Error(csv_error, try_row); + error_handler.Error(csv_error, true); } } diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index 7fd64d889ec6..07558c6bcf53 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -275,7 +275,7 @@ CSVError::CSVError(string error_message_p, CSVErrorType type_p, LinesPerBoundary CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx_p, string csv_row_p, LinesPerBoundary error_info_p, idx_t row_byte_position, optional_idx byte_position_p, - const CSVReaderOptions &reader_options, const string &fixes, const string ¤t_path) + const CSVReaderOptions &reader_options, const string &fixes, const String ¤t_path) : error_message(std::move(error_message_p)), type(type_p), column_idx(column_idx_p), csv_row(std::move(csv_row_p)), error_info(error_info_p), row_byte_position(row_byte_position), byte_position(byte_position_p) { // What were the options @@ -319,7 +319,7 @@ void CSVError::RemoveNewLine(string &error) { CSVError CSVError::CastError(const CSVReaderOptions &options, const string &column_name, string &cast_error, idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position, - optional_idx byte_position, LogicalTypeId type, const string ¤t_path) { + optional_idx byte_position, LogicalTypeId type, const String ¤t_path) { std::ostringstream error; // Which column error << "Error when converting column \"" << column_name << "\". "; @@ -350,7 +350,7 @@ CSVError CSVError::CastError(const CSVReaderOptions &options, const string &colu } CSVError CSVError::LineSizeError(const CSVReaderOptions &options, LinesPerBoundary error_info, string &csv_row, - idx_t byte_position, const string ¤t_path) { + idx_t byte_position, const String ¤t_path) { std::ostringstream error; error << "Maximum line size of " << options.maximum_line_size.GetValue() << " bytes exceeded. "; error << "Actual Size:" << csv_row.size() << " bytes." << '\n'; @@ -365,7 +365,7 @@ CSVError CSVError::LineSizeError(const CSVReaderOptions &options, LinesPerBounda CSVError CSVError::InvalidState(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, optional_idx byte_position, - const string ¤t_path) { + const String ¤t_path) { std::ostringstream error; error << "The CSV Parser state machine reached an invalid state.\nThis can happen when is not possible to parse " "your CSV File with the given options, or the CSV File is not RFC 4180 compliant "; @@ -521,7 +521,7 @@ CSVError CSVError::SniffingError(const CSVReaderOptions &options, const string & } CSVError CSVError::NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info, - const string ¤t_path) { + const String ¤t_path) { std::ostringstream error; error << " The parallel scanner does not support null_padding in conjunction with quoted new lines. Please " "disable the parallel csv reader with parallel=false" @@ -533,7 +533,7 @@ CSVError CSVError::NullPaddingFail(const CSVReaderOptions &options, LinesPerBoun CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, - optional_idx byte_position, const string ¤t_path) { + optional_idx byte_position, const String ¤t_path) { std::ostringstream error; error << "Value with unterminated quote found." << '\n'; std::ostringstream how_to_fix_it; @@ -551,7 +551,7 @@ CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, - optional_idx byte_position, const string ¤t_path) { + optional_idx byte_position, const String ¤t_path) { std::ostringstream error; // We don't have a fix for this std::ostringstream how_to_fix_it; @@ -581,7 +581,7 @@ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, i CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, optional_idx byte_position, - const string ¤t_path) { + const String ¤t_path) { std::ostringstream error; // How many columns were expected and how many were found error << "Invalid unicode (byte sequence mismatch) detected. This file is not " << options.encoding << " encoded." diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index 5801e99b0636..d963df00c2f1 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -465,7 +465,7 @@ bool CSVReaderOptions::WasTypeManuallySet(idx_t i) const { return was_type_manually_set[i]; } -string CSVReaderOptions::ToString(const string ¤t_file_path) const { +string CSVReaderOptions::ToString(const String ¤t_file_path) const { auto &delimiter = dialect_options.state_machine_options.delimiter; auto "e = dialect_options.state_machine_options.quote; auto &escape = dialect_options.state_machine_options.escape; @@ -475,7 +475,7 @@ string CSVReaderOptions::ToString(const string ¤t_file_path) const { auto &skip_rows = dialect_options.skip_rows; auto &header = dialect_options.header; - string error = " file = " + current_file_path + "\n "; + string error = " file = " + current_file_path.ToStdString() + "\n "; // Let's first print options that can either be set by the user or by the sniffer // delimiter error += FormatOptionLine("delimiter", delimiter); diff --git a/src/execution/operator/helper/physical_buffered_batch_collector.cpp b/src/execution/operator/helper/physical_buffered_batch_collector.cpp index 404d143431b5..d08332a72ac6 100644 --- a/src/execution/operator/helper/physical_buffered_batch_collector.cpp +++ b/src/execution/operator/helper/physical_buffered_batch_collector.cpp @@ -94,7 +94,7 @@ unique_ptr PhysicalBufferedBatchCollector::GetLocalSinkState(Exe unique_ptr PhysicalBufferedBatchCollector::GetGlobalSinkState(ClientContext &context) const { auto state = make_uniq(); state->context = context.shared_from_this(); - state->buffered_data = make_shared_ptr(state->context); + state->buffered_data = make_shared_ptr(context); return std::move(state); } diff --git a/src/execution/operator/helper/physical_buffered_collector.cpp b/src/execution/operator/helper/physical_buffered_collector.cpp index 7795230dc514..f0bdea11cb75 100644 --- a/src/execution/operator/helper/physical_buffered_collector.cpp +++ b/src/execution/operator/helper/physical_buffered_collector.cpp @@ -48,7 +48,7 @@ SinkCombineResultType PhysicalBufferedCollector::Combine(ExecutionContext &conte unique_ptr PhysicalBufferedCollector::GetGlobalSinkState(ClientContext &context) const { auto state = make_uniq(); state->context = context.shared_from_this(); - state->buffered_data = make_shared_ptr(state->context); + state->buffered_data = make_shared_ptr(context); return std::move(state); } diff --git a/src/execution/operator/helper/physical_limit.cpp b/src/execution/operator/helper/physical_limit.cpp index 5a4339c63052..3963ff36e601 100644 --- a/src/execution/operator/helper/physical_limit.cpp +++ b/src/execution/operator/helper/physical_limit.cpp @@ -8,6 +8,8 @@ namespace duckdb { +constexpr const idx_t PhysicalLimit::MAX_LIMIT_VALUE; + PhysicalLimit::PhysicalLimit(PhysicalPlan &physical_plan, vector types, BoundLimitNode limit_val_p, BoundLimitNode offset_val_p, idx_t estimated_cardinality) : PhysicalOperator(physical_plan, PhysicalOperatorType::LIMIT, std::move(types), estimated_cardinality), diff --git a/src/execution/operator/helper/physical_prepare.cpp b/src/execution/operator/helper/physical_prepare.cpp index 784d6ada4135..1d150574323e 100644 --- a/src/execution/operator/helper/physical_prepare.cpp +++ b/src/execution/operator/helper/physical_prepare.cpp @@ -8,7 +8,7 @@ SourceResultType PhysicalPrepare::GetData(ExecutionContext &context, DataChunk & auto &client = context.client; // store the prepared statement in the context - ClientData::Get(client).prepared_statements[name] = prepared; + ClientData::Get(client).prepared_statements[name.ToStdString()] = prepared; return SourceResultType::FINISHED; } diff --git a/src/execution/operator/helper/physical_reset.cpp b/src/execution/operator/helper/physical_reset.cpp index 711cc1b3bb82..9476b4e23c48 100644 --- a/src/execution/operator/helper/physical_reset.cpp +++ b/src/execution/operator/helper/physical_reset.cpp @@ -36,8 +36,7 @@ SourceResultType PhysicalReset::GetData(ExecutionContext &context, DataChunk &ch auto extension_name = Catalog::AutoloadExtensionByConfigName(context.client, name); entry = config.extension_parameters.find(name.ToStdString()); if (entry == config.extension_parameters.end()) { - throw InvalidInputException("Extension parameter %s was not found after autoloading", - name.ToStdString()); + throw InvalidInputException("Extension parameter %s was not found after autoloading", name); } } ResetExtensionVariable(context, config, entry->second); @@ -65,9 +64,9 @@ SourceResultType PhysicalReset::GetData(ExecutionContext &context, DataChunk &ch } if (variable_scope == SetScope::SESSION) { auto &client_config = ClientConfig::GetConfig(context.client); - client_config.set_variables.erase(name.ToStdString()); + client_config.set_variables.erase(option->name); } else { - config.ResetGenericOption(name); + config.ResetGenericOption(option->name); } return SourceResultType::FINISHED; } diff --git a/src/execution/operator/helper/physical_set.cpp b/src/execution/operator/helper/physical_set.cpp index e8362ad9ce83..7c28d925eb42 100644 --- a/src/execution/operator/helper/physical_set.cpp +++ b/src/execution/operator/helper/physical_set.cpp @@ -6,17 +6,17 @@ namespace duckdb { -void PhysicalSet::SetGenericVariable(ClientContext &context, const string &name, SetScope scope, Value target_value) { +void PhysicalSet::SetGenericVariable(ClientContext &context, const String &name, SetScope scope, Value target_value) { if (scope == SetScope::GLOBAL) { auto &config = DBConfig::GetConfig(context); config.SetOption(name, std::move(target_value)); } else { auto &client_config = ClientConfig::GetConfig(context); - client_config.set_variables[name] = std::move(target_value); + client_config.set_variables[name.ToStdString()] = std::move(target_value); } } -void PhysicalSet::SetExtensionVariable(ClientContext &context, ExtensionOption &extension_option, const string &name, +void PhysicalSet::SetExtensionVariable(ClientContext &context, ExtensionOption &extension_option, const String &name, SetScope scope, const Value &value) { auto &target_type = extension_option.type; Value target_value = value.CastAs(context, target_type); @@ -36,10 +36,10 @@ SourceResultType PhysicalSet::GetData(ExecutionContext &context, DataChunk &chun auto option = DBConfig::GetOptionByName(name); if (!option) { // check if this is an extra extension variable - auto entry = config.extension_parameters.find(name); + auto entry = config.extension_parameters.find(name.ToStdString()); if (entry == config.extension_parameters.end()) { auto extension_name = Catalog::AutoloadExtensionByConfigName(context.client, name); - entry = config.extension_parameters.find(name); + entry = config.extension_parameters.find(name.ToStdString()); if (entry == config.extension_parameters.end()) { throw InvalidInputException("Extension parameter %s was not found after autoloading", name); } diff --git a/src/execution/operator/helper/physical_set_variable.cpp b/src/execution/operator/helper/physical_set_variable.cpp index 430e7055e552..67f1a1615b38 100644 --- a/src/execution/operator/helper/physical_set_variable.cpp +++ b/src/execution/operator/helper/physical_set_variable.cpp @@ -1,12 +1,13 @@ #include "duckdb/execution/operator/helper/physical_set_variable.hpp" #include "duckdb/main/client_config.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" namespace duckdb { -PhysicalSetVariable::PhysicalSetVariable(PhysicalPlan &physical_plan, string name_p, idx_t estimated_cardinality) +PhysicalSetVariable::PhysicalSetVariable(PhysicalPlan &physical_plan, const string &name_p, idx_t estimated_cardinality) : PhysicalOperator(physical_plan, PhysicalOperatorType::SET_VARIABLE, {LogicalType::BOOLEAN}, estimated_cardinality), - name(std::move(name_p)) { + name(physical_plan.ArenaRef().MakeString(name_p)) { } SourceResultType PhysicalSetVariable::GetData(ExecutionContext &context, DataChunk &chunk, diff --git a/src/execution/operator/join/perfect_hash_join_executor.cpp b/src/execution/operator/join/perfect_hash_join_executor.cpp index a48eaee4fe1e..d9db21dbd614 100644 --- a/src/execution/operator/join/perfect_hash_join_executor.cpp +++ b/src/execution/operator/join/perfect_hash_join_executor.cpp @@ -125,14 +125,14 @@ bool PerfectHashJoinExecutor::CanDoPerfectHashJoin(const PhysicalHashJoin &op, c //===--------------------------------------------------------------------===// bool PerfectHashJoinExecutor::BuildPerfectHashTable(LogicalType &key_type) { // First, allocate memory for each build column - auto build_size = perfect_join_statistics.build_range + 1; + const auto build_size = perfect_join_statistics.build_range + 1; for (const auto &type : join.rhs_output_columns.col_types) { - perfect_hash_table.emplace_back(type, build_size); + perfect_hash_table.emplace_back(DictionaryVector::CreateReusableDictionary(type, build_size)); } // and for duplicate_checking - bitmap_build_idx = make_unsafe_uniq_array_uninitialized(build_size); - memset(bitmap_build_idx.get(), 0, sizeof(bool) * build_size); // set false + bitmap_build_idx.Initialize(build_size); + bitmap_build_idx.SetAllInvalid(build_size); // Now fill columns with build data return FullScanHashTable(key_type); @@ -168,22 +168,25 @@ bool PerfectHashJoinExecutor::FullScanHashTable(LogicalType &key_type) { if (!success) { return false; } - if (unique_keys == perfect_join_statistics.build_range + 1 && !ht.has_null) { + + const auto build_size = perfect_join_statistics.build_range + 1; + if (unique_keys == build_size && !ht.has_null) { perfect_join_statistics.is_build_dense = true; + bitmap_build_idx.Reset(build_size); // All valid } key_count = unique_keys; // do not consider keys out of the range // Full scan the remaining build columns and fill the perfect hash table - const auto build_size = perfect_join_statistics.build_range + 1; + for (idx_t i = 0; i < join.rhs_output_columns.col_types.size(); i++) { - auto &vector = perfect_hash_table[i]; + auto &vector = perfect_hash_table[i]->data; const auto output_col_idx = ht.output_columns[i]; D_ASSERT(vector.GetType() == ht.layout_ptr->GetTypes()[output_col_idx]); - if (build_size > STANDARD_VECTOR_SIZE) { - auto &col_mask = FlatVector::Validity(vector); - col_mask.Initialize(build_size); - } + auto &col_mask = FlatVector::Validity(vector); + col_mask.Reset(build_size); data_collection.Gather(tuples_addresses, sel_tuples, key_count, output_col_idx, vector, sel_build, nullptr); + // This ensures the empty entries are set to NULL, so that the emitted dictionary vectors make sense + col_mask.Combine(bitmap_build_idx, build_size); } return true; @@ -227,19 +230,19 @@ bool PerfectHashJoinExecutor::TemplatedFillSelectionVectorBuild(Vector &source, auto max_value = perfect_join_statistics.build_max.GetValueUnsafe(); UnifiedVectorFormat vector_data; source.ToUnifiedFormat(count, vector_data); - auto data = reinterpret_cast(vector_data.data); + const auto data = vector_data.GetData(); // generate the selection vector for (idx_t i = 0, sel_idx = 0; i < count; ++i) { auto data_idx = vector_data.sel->get_index(i); auto input_value = data[data_idx]; // add index to selection vector if value in the range if (min_value <= input_value && input_value <= max_value) { - auto idx = (idx_t)(input_value - min_value); // subtract min value to get the idx position + auto idx = UnsafeNumericCast(input_value - min_value); // subtract min value to get the idx position sel_vec.set_index(sel_idx, idx); - if (bitmap_build_idx[idx]) { + if (bitmap_build_idx.RowIsValidUnsafe(idx)) { return false; } else { - bitmap_build_idx[idx] = true; + bitmap_build_idx.SetValidUnsafe(idx); unique_keys++; } seq_sel_vec.set_index(sel_idx++, i); @@ -302,9 +305,7 @@ OperatorResultType PerfectHashJoinExecutor::ProbePerfectHashTable(ExecutionConte for (idx_t i = 0; i < join.rhs_output_columns.col_types.size(); i++) { auto &result_vector = result.data[lhs_output_columns.ColumnCount() + i]; D_ASSERT(result_vector.GetType() == ht.layout_ptr->GetTypes()[ht.output_columns[i]]); - auto &build_vec = perfect_hash_table[i]; - result_vector.Reference(build_vec); - result_vector.Slice(state.build_sel_vec, probe_sel_count); + result_vector.Dictionary(perfect_hash_table[i], state.build_sel_vec); } return OperatorResultType::NEED_MORE_INPUT; } @@ -367,9 +368,9 @@ void PerfectHashJoinExecutor::TemplatedFillSelectionVectorProbe(Vector &source, auto input_value = data[data_idx]; // add index to selection vector if value in the range if (min_value <= input_value && input_value <= max_value) { - auto idx = (idx_t)(input_value - min_value); // subtract min value to get the idx position - // check for matches in the build - if (bitmap_build_idx[idx]) { + auto idx = UnsafeNumericCast(input_value - min_value); // subtract min value to get the idx + // position check for matches in the build + if (bitmap_build_idx.RowIsValid(idx)) { build_sel_vec.set_index(sel_idx, idx); probe_sel_vec.set_index(sel_idx++, i); probe_sel_count++; @@ -386,9 +387,9 @@ void PerfectHashJoinExecutor::TemplatedFillSelectionVectorProbe(Vector &source, auto input_value = data[data_idx]; // add index to selection vector if value in the range if (min_value <= input_value && input_value <= max_value) { - auto idx = (idx_t)(input_value - min_value); // subtract min value to get the idx position - // check for matches in the build - if (bitmap_build_idx[idx]) { + auto idx = UnsafeNumericCast(input_value - min_value); // subtract min value to get the idx + // position check for matches in the build + if (bitmap_build_idx.RowIsValid(idx)) { build_sel_vec.set_index(sel_idx, idx); probe_sel_vec.set_index(sel_idx++, i); probe_sel_count++; diff --git a/src/execution/operator/join/physical_asof_join.cpp b/src/execution/operator/join/physical_asof_join.cpp index 719781992551..08d2ed320b39 100644 --- a/src/execution/operator/join/physical_asof_join.cpp +++ b/src/execution/operator/join/physical_asof_join.cpp @@ -1,13 +1,17 @@ #include "duckdb/execution/operator/join/physical_asof_join.hpp" #include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/sort/partition_state.hpp" -#include "duckdb/common/sort/sort.hpp" +#include "duckdb/common/sorting/hashed_sort.hpp" +#include "duckdb/common/sorting/sort_key.hpp" +#include "duckdb/common/sorting/sorted_run.hpp" +#include "duckdb/common/types/row/block_iterator.hpp" #include "duckdb/common/vector_operations/vector_operations.hpp" #include "duckdb/execution/expression_executor.hpp" #include "duckdb/execution/operator/join/outer_join_marker.hpp" +#include "duckdb/execution/operator/join/physical_range_join.hpp" #include "duckdb/main/client_context.hpp" #include "duckdb/parallel/event.hpp" +#include "duckdb/parallel/meta_pipeline.hpp" #include "duckdb/parallel/thread_context.hpp" namespace duckdb { @@ -16,9 +20,10 @@ PhysicalAsOfJoin::PhysicalAsOfJoin(PhysicalPlan &physical_plan, LogicalCompariso PhysicalOperator &right) : PhysicalComparisonJoin(physical_plan, op, PhysicalOperatorType::ASOF_JOIN, std::move(op.conditions), op.join_type, op.estimated_cardinality), - comparison_type(ExpressionType::INVALID), predicate(std::move(op.predicate)) { + comparison_type(ExpressionType::INVALID) { // Convert the conditions partitions and sorts + D_ASSERT(!op.predicate.get()); for (auto &cond : conditions) { D_ASSERT(cond.left->return_type == cond.right->return_type); join_key_types.push_back(cond.left->return_type); @@ -74,51 +79,44 @@ PhysicalAsOfJoin::PhysicalAsOfJoin(PhysicalPlan &physical_plan, LogicalCompariso //===--------------------------------------------------------------------===// class AsOfGlobalSinkState : public GlobalSinkState { public: - AsOfGlobalSinkState(ClientContext &context, const PhysicalAsOfJoin &op) - : rhs_sink(context, op.rhs_partitions, op.rhs_orders, op.children[1].get().GetTypes(), {}, - op.estimated_cardinality), - is_outer(IsRightOuterJoin(op.join_type)), has_null(false) { + using HashedSortPtr = unique_ptr; + using HashedSinkPtr = unique_ptr; + using PartitionMarkers = vector; + + AsOfGlobalSinkState(ClientContext &client, const PhysicalAsOfJoin &op) { + // Set up partitions for both sides + hashed_sorts.reserve(2); + hashed_sinks.reserve(2); + const vector> partitions_stats; + auto &lhs = op.children[0].get(); + auto sort = make_uniq(client, op.lhs_partitions, op.lhs_orders, lhs.GetTypes(), partitions_stats, + lhs.estimated_cardinality, true); + hashed_sinks.emplace_back(sort->GetGlobalSinkState(client)); + hashed_sorts.emplace_back(std::move(sort)); + + auto &rhs = op.children[1].get(); + sort = make_uniq(client, op.rhs_partitions, op.rhs_orders, rhs.GetTypes(), partitions_stats, + rhs.estimated_cardinality, true); + hashed_sinks.emplace_back(sort->GetGlobalSinkState(client)); + hashed_sorts.emplace_back(std::move(sort)); } - idx_t Count() const { - return rhs_sink.count; - } - - PartitionLocalSinkState *RegisterBuffer(ClientContext &context) { - lock_guard guard(lock); - lhs_buffers.emplace_back(make_uniq(context, *lhs_sink)); - return lhs_buffers.back().get(); - } - - PartitionGlobalSinkState rhs_sink; - - // One per partition - const bool is_outer; - vector right_outers; - bool has_null; - - // Left side buffering - unique_ptr lhs_sink; - - mutex lock; - vector> lhs_buffers; + //! The child that is being materialised (right/1 then left/0) + size_t child = 1; + //! The child's partitioning description + vector hashed_sorts; + //! The child's partitioning buffer + vector hashed_sinks; }; class AsOfLocalSinkState : public LocalSinkState { public: - explicit AsOfLocalSinkState(ClientContext &context, PartitionGlobalSinkState &gstate_p) - : local_partition(context, gstate_p) { + AsOfLocalSinkState(ExecutionContext &context, AsOfGlobalSinkState &gsink) { + auto &hashed_sort = *gsink.hashed_sorts[gsink.child]; + local_partition = hashed_sort.GetLocalSinkState(context); } - void Sink(DataChunk &input_chunk) { - local_partition.Sink(input_chunk); - } - - void Combine() { - local_partition.Combine(); - } - - PartitionLocalSinkState local_partition; + unique_ptr local_partition; }; unique_ptr PhysicalAsOfJoin::GetGlobalSinkState(ClientContext &context) const { @@ -126,219 +124,285 @@ unique_ptr PhysicalAsOfJoin::GetGlobalSinkState(ClientContext & } unique_ptr PhysicalAsOfJoin::GetLocalSinkState(ExecutionContext &context) const { - // We only sink the RHS auto &gsink = sink_state->Cast(); - return make_uniq(context.client, gsink.rhs_sink); + return make_uniq(context, gsink); } -SinkResultType PhysicalAsOfJoin::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { - auto &lstate = input.local_state.Cast(); +SinkResultType PhysicalAsOfJoin::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &sink) const { + auto &gstate = sink.global_state.Cast(); + auto &lstate = sink.local_state.Cast(); - lstate.Sink(chunk); + auto &hashed_sort = *gstate.hashed_sorts[gstate.child]; + auto &gsink = *gstate.hashed_sinks[gstate.child]; + auto &lsink = *lstate.local_partition; - return SinkResultType::NEED_MORE_INPUT; + OperatorSinkInput hsink {gsink, lsink, sink.interrupt_state}; + return hashed_sort.Sink(context, chunk, hsink); } -SinkCombineResultType PhysicalAsOfJoin::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const { - auto &lstate = input.local_state.Cast(); - lstate.Combine(); - return SinkCombineResultType::FINISHED; +SinkCombineResultType PhysicalAsOfJoin::Combine(ExecutionContext &context, OperatorSinkCombineInput &combine) const { + auto &gstate = combine.global_state.Cast(); + auto &lstate = combine.local_state.Cast(); + + auto &hashed_sort = *gstate.hashed_sorts[gstate.child]; + auto &gsink = *gstate.hashed_sinks[gstate.child]; + auto &lsink = *lstate.local_partition; + + OperatorSinkCombineInput hcombine {gsink, lsink, combine.interrupt_state}; + return hashed_sort.Combine(context, hcombine); } //===--------------------------------------------------------------------===// // Finalize //===--------------------------------------------------------------------===// -SinkFinalizeType PhysicalAsOfJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const { - auto &gstate = input.global_state.Cast(); - - // The data is all in so we can initialise the left partitioning. - const vector> partitions_stats; - gstate.lhs_sink = make_uniq(context, lhs_partitions, lhs_orders, - children[0].get().GetTypes(), partitions_stats, 0U); - gstate.lhs_sink->SyncPartitioning(gstate.rhs_sink); - - // Find the first group to sort - if (!gstate.rhs_sink.HasMergeTasks() && EmptyResultIfRHSIsEmpty()) { - // Empty input! - return SinkFinalizeType::NO_OUTPUT_POSSIBLE; +SinkFinalizeType PhysicalAsOfJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &client, + OperatorSinkFinalizeInput &finalize) const { + auto &gstate = finalize.global_state.Cast(); + + // The data is all in so we can synchronise the left partitioning. + auto &hashed_sort = *gstate.hashed_sorts[gstate.child]; + auto &hashed_sink = *gstate.hashed_sinks[gstate.child]; + OperatorSinkFinalizeInput hfinalize {hashed_sink, finalize.interrupt_state}; + if (gstate.child == 1) { + auto &lhs_groups = *gstate.hashed_sinks[1 - gstate.child]; + auto &rhs_groups = hashed_sink; + hashed_sort.Synchronize(rhs_groups, lhs_groups); + + auto result = hashed_sort.Finalize(client, hfinalize); + if (result != SinkFinalizeType::READY && EmptyResultIfRHSIsEmpty()) { + // Empty input! + gstate.child = 1 - gstate.child; + return result; + } + } else { + hashed_sort.Finalize(client, hfinalize); } + // Switch sides + gstate.child = 1 - gstate.child; + // Schedule all the sorts for maximum thread utilisation - auto new_event = make_shared_ptr(gstate.rhs_sink, pipeline, *this); - event.InsertEvent(std::move(new_event)); + return hashed_sort.MaterializeSortedRuns(pipeline, event, *this, hfinalize); +} - return SinkFinalizeType::READY; +OperatorResultType PhysicalAsOfJoin::ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk, + GlobalOperatorState &gstate, OperatorState &lstate_p) const { + return OperatorResultType::FINISHED; } //===--------------------------------------------------------------------===// -// Operator +// Source //===--------------------------------------------------------------------===// -class AsOfGlobalState : public GlobalOperatorState { -public: - explicit AsOfGlobalState(AsOfGlobalSinkState &gsink) { - // for FULL/RIGHT OUTER JOIN, initialize right_outers to false for every tuple - auto &rhs_partition = gsink.rhs_sink; - auto &right_outers = gsink.right_outers; - right_outers.reserve(rhs_partition.hash_groups.size()); - for (const auto &hash_group : rhs_partition.hash_groups) { - right_outers.emplace_back(OuterJoinMarker(gsink.is_outer)); - right_outers.back().Initialize(hash_group->count); - } - } -}; +enum class AsOfJoinSourceStage : uint8_t { INNER, RIGHT, DONE }; -unique_ptr PhysicalAsOfJoin::GetGlobalOperatorState(ClientContext &context) const { - auto &gsink = sink_state->Cast(); - return make_uniq(gsink); -} - -class AsOfLocalState : public CachingOperatorState { +class AsOfPayloadScanner { public: - AsOfLocalState(ClientContext &context, const PhysicalAsOfJoin &op) - : context(context), allocator(Allocator::Get(context)), op(op), lhs_executor(context), - left_outer(IsLeftOuterJoin(op.join_type)), fetch_next_left(true) { - lhs_keys.Initialize(allocator, op.join_key_types); - for (const auto &cond : op.conditions) { - lhs_executor.AddExpression(*cond.left); - } - - lhs_payload.Initialize(allocator, op.children[0].get().GetTypes()); - lhs_sel.Initialize(); - left_outer.Initialize(STANDARD_VECTOR_SIZE); + using Types = vector; + using Columns = vector; - auto &gsink = op.sink_state->Cast(); - lhs_partition_sink = gsink.RegisterBuffer(context); + AsOfPayloadScanner(const SortedRun &sorted_run, const HashedSort &hashed_sort); + idx_t Base() const { + return base; + } + idx_t Scanned() const { + return scanned; + } + idx_t Remaining() const { + return count - scanned; + } + bool Scan(DataChunk &chunk) { + // Free the previous blocks + block_state.SetKeepPinned(true); + block_state.SetPinPayload(true); + + base = scanned; + const auto result = (this->*scan_func)(); + chunk.ReferenceColumns(scan_chunk, scan_ids); + scanned += scan_chunk.size(); + ++chunk_idx; + return result; } - bool Sink(DataChunk &input); - OperatorResultType ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk); - - ClientContext &context; - Allocator &allocator; - const PhysicalAsOfJoin &op; - - ExpressionExecutor lhs_executor; - DataChunk lhs_keys; - ValidityMask lhs_valid_mask; - SelectionVector lhs_sel; - DataChunk lhs_payload; +private: + template + bool TemplatedScan() { + using SORT_KEY = SortKey; + using BLOCK_ITERATOR = block_iterator_t; + BLOCK_ITERATOR itr(block_state, chunk_idx, 0); + + const auto sort_keys = FlatVector::GetData(sort_key_pointers); + const auto result_count = MinValue(Remaining(), STANDARD_VECTOR_SIZE); + for (idx_t i = 0; i < result_count; ++i) { + const auto idx = block_state.GetIndex(chunk_idx, i); + sort_keys[i] = &itr[idx]; + } - OuterJoinMarker left_outer; - bool fetch_next_left; + // Scan + scan_chunk.Reset(); + scan_state.Scan(sorted_run, sort_key_pointers, result_count, scan_chunk); + return scan_chunk.size() > 0; + } - optional_ptr lhs_partition_sink; + // Only figure out the scan function once. + using scan_t = bool (duckdb::AsOfPayloadScanner::*)(); + scan_t scan_func; + + const SortedRun &sorted_run; + ExternalBlockIteratorState block_state; + Vector sort_key_pointers = Vector(LogicalType::POINTER); + SortedRunScanState scan_state; + const Columns scan_ids; + DataChunk scan_chunk; + const idx_t count; + idx_t base = 0; + idx_t scanned = 0; + idx_t chunk_idx = 0; }; -bool AsOfLocalState::Sink(DataChunk &input) { - // Compute the join keys - lhs_keys.Reset(); - lhs_executor.Execute(input, lhs_keys); - lhs_keys.Flatten(); +AsOfPayloadScanner::AsOfPayloadScanner(const SortedRun &sorted_run, const HashedSort &hashed_sort) + : sorted_run(sorted_run), block_state(*sorted_run.key_data, sorted_run.payload_data.get()), + scan_state(sorted_run.context, sorted_run.sort), scan_ids(hashed_sort.scan_ids), count(sorted_run.Count()) { - // Combine the NULLs - const auto count = input.size(); - lhs_valid_mask.Reset(); - for (auto col_idx : op.null_sensitive) { - auto &col = lhs_keys.data[col_idx]; - UnifiedVectorFormat unified; - col.ToUnifiedFormat(count, unified); - lhs_valid_mask.Combine(unified.validity, count); - } - - // Convert the mask to a selection vector - // and mark all the rows that cannot match for early return. - idx_t lhs_valid = 0; - const auto entry_count = lhs_valid_mask.EntryCount(count); - idx_t base_idx = 0; - left_outer.Reset(); - for (idx_t entry_idx = 0; entry_idx < entry_count;) { - const auto validity_entry = lhs_valid_mask.GetValidityEntry(entry_idx++); - const auto next = MinValue(base_idx + ValidityMask::BITS_PER_VALUE, count); - if (ValidityMask::AllValid(validity_entry)) { - for (; base_idx < next; ++base_idx) { - lhs_sel.set_index(lhs_valid++, base_idx); - left_outer.SetMatch(base_idx); - } - } else if (ValidityMask::NoneValid(validity_entry)) { - base_idx = next; - } else { - const auto start = base_idx; - for (; base_idx < next; ++base_idx) { - if (ValidityMask::RowIsValid(validity_entry, base_idx - start)) { - lhs_sel.set_index(lhs_valid++, base_idx); - left_outer.SetMatch(base_idx); - } - } - } + scan_chunk.Initialize(sorted_run.context, hashed_sort.payload_types); + const auto sort_key_type = sorted_run.key_data->GetLayout().GetSortKeyType(); + switch (sort_key_type) { + case SortKeyType::NO_PAYLOAD_FIXED_8: + scan_func = &AsOfPayloadScanner::TemplatedScan; + break; + case SortKeyType::NO_PAYLOAD_FIXED_16: + scan_func = &AsOfPayloadScanner::TemplatedScan; + break; + case SortKeyType::NO_PAYLOAD_FIXED_24: + scan_func = &AsOfPayloadScanner::TemplatedScan; + break; + case SortKeyType::NO_PAYLOAD_FIXED_32: + scan_func = &AsOfPayloadScanner::TemplatedScan; + break; + case SortKeyType::NO_PAYLOAD_VARIABLE_32: + scan_func = &AsOfPayloadScanner::TemplatedScan; + break; + case SortKeyType::PAYLOAD_FIXED_16: + scan_func = &AsOfPayloadScanner::TemplatedScan; + break; + case SortKeyType::PAYLOAD_FIXED_24: + scan_func = &AsOfPayloadScanner::TemplatedScan; + break; + case SortKeyType::PAYLOAD_FIXED_32: + scan_func = &AsOfPayloadScanner::TemplatedScan; + break; + case SortKeyType::PAYLOAD_VARIABLE_32: + scan_func = &AsOfPayloadScanner::TemplatedScan; + break; + default: + throw NotImplementedException("AsOfPayloadScanner for %s", EnumUtil::ToString(sort_key_type)); } +} - // Slice the keys to the ones we can match - lhs_payload.Reset(); - if (lhs_valid == count) { - lhs_payload.Reference(input); - lhs_payload.SetCardinality(input); - } else { - lhs_payload.Slice(input, lhs_sel, lhs_valid); - lhs_payload.SetCardinality(lhs_valid); +class AsOfLocalSourceState; - // Flush the ones that can't match - fetch_next_left = false; - } +class AsOfGlobalSourceState : public GlobalSourceState { +public: + using HashGroupPtr = unique_ptr; + using HashGroups = vector; - lhs_partition_sink->Sink(lhs_payload); + AsOfGlobalSourceState(ClientContext &client, const PhysicalAsOfJoin &op); - return false; -} + //! Assign a new task to the local state + bool AssignTask(AsOfLocalSourceState &lsource); + //! Can we shift to the next stage? + bool TryPrepareNextStage(); -OperatorResultType AsOfLocalState::ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk) { - input.Verify(); - Sink(input); + //! The parent operator + const PhysicalAsOfJoin &op; + //! For synchronizing the external hash join + atomic stage; + //! The child's hash groups + vector hashed_groups; + //! Whether the right side is outer + const bool is_right_outer; + //! The right outer join markers (one per partition) + vector right_outers; + //! The next buffer to flush + atomic next_left; + //! The number of flushed buffers + atomic flushed_left; + //! The right outer output read position. + atomic next_right; + //! The right outer output read position. + atomic flushed_right; - // If there were any unmatchable rows, return them now so we can forget about them. - if (!fetch_next_left) { - fetch_next_left = true; - left_outer.ConstructLeftJoinResult(input, chunk); - left_outer.Reset(); +public: + idx_t MaxThreads() override { + return hashed_groups[1].size(); } +}; - // Just keep asking for data and buffering it - return OperatorResultType::NEED_MORE_INPUT; -} - -OperatorResultType PhysicalAsOfJoin::ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk, - GlobalOperatorState &gstate, OperatorState &lstate_p) const { - auto &gsink = sink_state->Cast(); - auto &lstate = lstate_p.Cast(); +AsOfGlobalSourceState::AsOfGlobalSourceState(ClientContext &client, const PhysicalAsOfJoin &op) + : op(op), stage(AsOfJoinSourceStage::INNER), is_right_outer(IsRightOuterJoin(op.join_type)), next_left(0), + flushed_left(0), next_right(0), flushed_right(0) { - if (gsink.rhs_sink.count == 0) { - // empty RHS - if (!EmptyResultIfRHSIsEmpty()) { - ConstructEmptyJoinResult(join_type, gsink.has_null, input, chunk); - return OperatorResultType::NEED_MORE_INPUT; - } else { - return OperatorResultType::FINISHED; + // Take ownership of the hash groups + auto &gsink = op.sink_state->Cast(); + hashed_groups.resize(2); + for (idx_t child = 0; child < 2; ++child) { + auto &hashed_sort = *gsink.hashed_sorts[child]; + auto &hashed_sink = *gsink.hashed_sinks[child]; + auto hashed_source = hashed_sort.GetGlobalSourceState(client, hashed_sink); + auto &sorted_runs = hashed_sort.GetSortedRuns(*hashed_source); + auto &hash_groups = hashed_groups[child]; + hash_groups.resize(sorted_runs.size()); + + for (idx_t group_idx = 0; group_idx < sorted_runs.size(); ++group_idx) { + hash_groups[group_idx] = std::move(sorted_runs[group_idx]); } } - return lstate.ExecuteInternal(context, input, chunk); + // for FULL/RIGHT OUTER JOIN, initialize right_outers to false for every tuple + auto &rhs_groups = hashed_groups[1]; + right_outers.reserve(rhs_groups.size()); + for (const auto &hash_group : rhs_groups) { + right_outers.emplace_back(OuterJoinMarker(is_right_outer)); + right_outers.back().Initialize(hash_group ? hash_group->Count() : 0); + } } -//===--------------------------------------------------------------------===// -// Source -//===--------------------------------------------------------------------===// class AsOfProbeBuffer { public: using Orders = vector; - static bool IsExternal(ClientContext &context) { - return ClientConfig::GetConfig(context).force_external; + AsOfProbeBuffer(ClientContext &client, const PhysicalAsOfJoin &op, AsOfGlobalSourceState &gsource); + +public: + // Comparison utilities + static bool IsStrictComparison(ExpressionType comparison) { + switch (comparison) { + case ExpressionType::COMPARE_LESSTHAN: + case ExpressionType::COMPARE_GREATERTHAN: + return true; + case ExpressionType::COMPARE_LESSTHANOREQUALTO: + case ExpressionType::COMPARE_GREATERTHANOREQUALTO: + return false; + default: + throw NotImplementedException("Unsupported comparison type for ASOF join"); + } } - AsOfProbeBuffer(ClientContext &context, const PhysicalAsOfJoin &op); + //! Is left cmp right? + template + static inline bool Compare(const T &lhs, const T &rhs, const bool strict) { + const bool less_than = lhs < rhs; + if (!less_than && !strict) { + return !(rhs < lhs); + } + return less_than; + } + + template + void ResolveJoin(bool *found_matches, idx_t *matches); + + using resolve_join_t = void (duckdb::AsOfProbeBuffer::*)(bool *, idx_t *); + resolve_join_t resolve_join_func; -public: - void ResolveJoin(bool *found_matches, idx_t *matches = nullptr); bool Scanning() const { return lhs_scanner.get(); } @@ -346,6 +410,20 @@ class AsOfProbeBuffer { bool NextLeft(); void EndLeftScan(); + //! Create a new iterator for the sorted run + static unique_ptr CreateIteratorState(SortedRun &sorted) { + auto state = make_uniq(*sorted.key_data, sorted.payload_data.get()); + + // Unless we do this, we will only get values from the first chunk + Repin(*state); + + return state; + } + //! Reset the pins for an iterator so we release memory in a timely manner + static void Repin(ExternalBlockIteratorState &iter) { + // Don't pin the payload because we are not using it here. + iter.SetKeepPinned(true); + } // resolve joins that output max N elements (SEMI, ANTI, MARK) void ResolveSimpleJoin(ExecutionContext &context, DataChunk &chunk); // resolve joins that can potentially output N*M elements (INNER, LEFT, FULL) @@ -356,181 +434,264 @@ class AsOfProbeBuffer { return !fetch_next_left || (lhs_scanner && lhs_scanner->Remaining()); } - ClientContext &context; - Allocator &allocator; + ClientContext &client; const PhysicalAsOfJoin &op; - BufferManager &buffer_manager; - const bool force_external; - const idx_t memory_per_thread; - Orders lhs_orders; + //! The source state + AsOfGlobalSourceState &gsource; + //! Is the inequality strict? + const bool strict; // LHS scanning - SelectionVector lhs_sel; - optional_ptr left_hash; + SelectionVector lhs_scan_sel; + optional_ptr left_group; OuterJoinMarker left_outer; - unique_ptr left_itr; - unique_ptr lhs_scanner; + unique_ptr left_itr; + unique_ptr lhs_scanner; + DataChunk lhs_scanned; DataChunk lhs_payload; - idx_t left_group = 0; + ExpressionExecutor lhs_executor; + DataChunk lhs_keys; + ValidityMask lhs_valid_mask; + idx_t left_bin = 0; + SelectionVector lhs_match_sel; // RHS scanning - optional_ptr right_hash; + optional_ptr right_group; optional_ptr right_outer; - unique_ptr right_itr; - unique_ptr rhs_scanner; + unique_ptr right_itr; + idx_t right_pos; // ExternalBlockIteratorState doesn't know this... + unique_ptr rhs_scanner; DataChunk rhs_payload; - idx_t right_group = 0; + ExpressionExecutor rhs_executor; + DataChunk rhs_input; + DataChunk rhs_keys; + idx_t right_bin = 0; // Predicate evaluation - SelectionVector filter_sel; - ExpressionExecutor filterer; + SelectionVector tail_sel; idx_t lhs_match_count; bool fetch_next_left; }; -AsOfProbeBuffer::AsOfProbeBuffer(ClientContext &context, const PhysicalAsOfJoin &op) - : context(context), allocator(Allocator::Get(context)), op(op), - buffer_manager(BufferManager::GetBufferManager(context)), force_external(IsExternal(context)), - memory_per_thread(op.GetMaxThreadMemory(context)), left_outer(IsLeftOuterJoin(op.join_type)), filterer(context), - fetch_next_left(true) { - vector> partition_stats; - Orders partitions; // Not used. - PartitionGlobalSinkState::GenerateOrderings(partitions, lhs_orders, op.lhs_partitions, op.lhs_orders, - partition_stats); - - // We sort the row numbers of the incoming block, not the rows - lhs_payload.Initialize(allocator, op.children[0].get().GetTypes()); - rhs_payload.Initialize(allocator, op.children[1].get().GetTypes()); - - lhs_sel.Initialize(); +AsOfProbeBuffer::AsOfProbeBuffer(ClientContext &client, const PhysicalAsOfJoin &op, AsOfGlobalSourceState &gsource) + : client(client), op(op), gsource(gsource), strict(IsStrictComparison(op.comparison_type)), + left_outer(IsLeftOuterJoin(op.join_type)), lhs_executor(client), rhs_executor(client), fetch_next_left(true) { + + lhs_keys.Initialize(client, op.join_key_types); + for (const auto &cond : op.conditions) { + lhs_executor.AddExpression(*cond.left); + } + + lhs_scanned.Initialize(client, op.children[0].get().GetTypes()); + lhs_payload.Initialize(client, op.children[0].get().GetTypes()); + rhs_payload.Initialize(client, op.children[1].get().GetTypes()); + rhs_input.Initialize(client, op.children[1].get().GetTypes()); + + lhs_scan_sel.Initialize(); + lhs_match_sel.Initialize(); left_outer.Initialize(STANDARD_VECTOR_SIZE); - if (op.predicate) { - filter_sel.Initialize(); - filterer.AddExpression(*op.predicate); + // If we have equality predicates, we need some more buffers. + if (op.conditions.size() > 1) { + tail_sel.Initialize(); + rhs_keys.Initialize(client, op.join_key_types); + for (const auto &cond : op.conditions) { + rhs_executor.AddExpression(*cond.right); + } } } void AsOfProbeBuffer::BeginLeftScan(hash_t scan_bin) { auto &gsink = op.sink_state->Cast(); - auto &lhs_sink = *gsink.lhs_sink; - left_group = lhs_sink.bin_groups[scan_bin]; + // Always set right_bin too for memory management + auto &rhs_groups = gsource.hashed_groups[1]; + if (scan_bin < rhs_groups.size()) { + right_bin = scan_bin; + } else { + right_bin = rhs_groups.size(); + } - // Always set right_group too for memory management - auto &rhs_sink = gsink.rhs_sink; - if (scan_bin < rhs_sink.bin_groups.size()) { - right_group = rhs_sink.bin_groups[scan_bin]; + auto &lhs_groups = gsource.hashed_groups[0]; + if (scan_bin < lhs_groups.size()) { + left_bin = scan_bin; } else { - right_group = rhs_sink.bin_groups.size(); + left_bin = lhs_groups.size(); + } + + if (left_bin >= lhs_groups.size()) { + return; } - if (left_group >= lhs_sink.bin_groups.size()) { + left_group = lhs_groups[left_bin].get(); + if (!left_group || !left_group->Count()) { return; } - auto iterator_comp = ExpressionType::INVALID; - switch (op.comparison_type) { - case ExpressionType::COMPARE_GREATERTHANOREQUALTO: - iterator_comp = ExpressionType::COMPARE_LESSTHANOREQUALTO; + // Set up function pointer for sort type + const auto sort_key_type = left_group->key_data->GetLayout().GetSortKeyType(); + switch (sort_key_type) { + case SortKeyType::NO_PAYLOAD_FIXED_8: + resolve_join_func = &AsOfProbeBuffer::ResolveJoin; + break; + case SortKeyType::NO_PAYLOAD_FIXED_16: + resolve_join_func = &AsOfProbeBuffer::ResolveJoin; + break; + case SortKeyType::NO_PAYLOAD_FIXED_24: + resolve_join_func = &AsOfProbeBuffer::ResolveJoin; + break; + case SortKeyType::NO_PAYLOAD_FIXED_32: + resolve_join_func = &AsOfProbeBuffer::ResolveJoin; break; - case ExpressionType::COMPARE_GREATERTHAN: - iterator_comp = ExpressionType::COMPARE_LESSTHAN; + case SortKeyType::NO_PAYLOAD_VARIABLE_32: + resolve_join_func = &AsOfProbeBuffer::ResolveJoin; break; - case ExpressionType::COMPARE_LESSTHANOREQUALTO: - iterator_comp = ExpressionType::COMPARE_GREATERTHANOREQUALTO; + case SortKeyType::PAYLOAD_FIXED_16: + resolve_join_func = &AsOfProbeBuffer::ResolveJoin; break; - case ExpressionType::COMPARE_LESSTHAN: - iterator_comp = ExpressionType::COMPARE_GREATERTHAN; + case SortKeyType::PAYLOAD_FIXED_24: + resolve_join_func = &AsOfProbeBuffer::ResolveJoin; + break; + case SortKeyType::PAYLOAD_FIXED_32: + resolve_join_func = &AsOfProbeBuffer::ResolveJoin; + break; + case SortKeyType::PAYLOAD_VARIABLE_32: + resolve_join_func = &AsOfProbeBuffer::ResolveJoin; break; default: throw NotImplementedException("Unsupported comparison type for ASOF join"); } - left_hash = lhs_sink.hash_groups[left_group].get(); - auto &left_sort = *(left_hash->global_sort); - if (left_sort.sorted_blocks.empty()) { - return; - } - lhs_scanner = make_uniq(left_sort, false); - left_itr = make_uniq(left_sort, iterator_comp); + lhs_scanner = make_uniq(*left_group, *gsink.hashed_sorts[0]); + left_itr = CreateIteratorState(*left_group); // We are only probing the corresponding right side bin, which may be empty - // If they are empty, we leave the iterator as null so we can emit left matches - if (right_group < rhs_sink.bin_groups.size()) { - right_hash = rhs_sink.hash_groups[right_group].get(); - right_outer = gsink.right_outers.data() + right_group; - auto &right_sort = *(right_hash->global_sort); - right_itr = make_uniq(right_sort, iterator_comp); - rhs_scanner = make_uniq(right_sort, false); + // If it is empty, we leave the iterator as null so we can emit left matches + right_pos = 0; + if (right_bin < rhs_groups.size()) { + right_group = rhs_groups[right_bin].get(); + right_outer = gsource.right_outers.data() + right_bin; + if (right_group && right_group->Count()) { + right_itr = CreateIteratorState(*right_group); + rhs_scanner = make_uniq(*right_group, *gsink.hashed_sorts[1]); + } } } bool AsOfProbeBuffer::NextLeft() { - if (!HasMoreData()) { + // Scan the next sorted chunk + lhs_scanned.Reset(); + if (!lhs_scanner || !lhs_scanner->Scan(lhs_scanned)) { return false; } - // Scan the next sorted chunk - lhs_payload.Reset(); - left_itr->SetIndex(lhs_scanner->Scanned()); - lhs_scanner->Scan(lhs_payload); + // Compute the join keys + lhs_keys.Reset(); + lhs_executor.Execute(lhs_scanned, lhs_keys); + lhs_keys.Flatten(); + + // Combine the NULLs + const auto count = lhs_scanned.size(); + lhs_valid_mask.Reset(); + for (auto col_idx : op.null_sensitive) { + auto &col = lhs_keys.data[col_idx]; + UnifiedVectorFormat unified; + col.ToUnifiedFormat(count, unified); + lhs_valid_mask.Combine(unified.validity, count); + } + + // Convert the mask to a selection vector + // and mark all the rows that cannot match for early return. + idx_t lhs_valid = 0; + const auto entry_count = lhs_valid_mask.EntryCount(count); + idx_t base_idx = 0; + for (idx_t entry_idx = 0; entry_idx < entry_count;) { + const auto validity_entry = lhs_valid_mask.GetValidityEntry(entry_idx++); + const auto next = MinValue(base_idx + ValidityMask::BITS_PER_VALUE, count); + if (ValidityMask::AllValid(validity_entry)) { + for (; base_idx < next; ++base_idx) { + lhs_scan_sel.set_index(lhs_valid++, base_idx); + } + } else if (ValidityMask::NoneValid(validity_entry)) { + base_idx = next; + } else { + const auto start = base_idx; + for (; base_idx < next; ++base_idx) { + if (ValidityMask::RowIsValid(validity_entry, base_idx - start)) { + lhs_scan_sel.set_index(lhs_valid++, base_idx); + } + } + } + } + + // Slice the keys to the ones we can match + if (lhs_valid < count) { + lhs_payload.Slice(lhs_scanned, lhs_scan_sel, lhs_valid); + } else { + lhs_payload.Reference(lhs_scanned); + } return true; } void AsOfProbeBuffer::EndLeftScan() { - auto &gsink = op.sink_state->Cast(); - - right_hash = nullptr; + right_group = nullptr; right_itr.reset(); rhs_scanner.reset(); right_outer = nullptr; - auto &rhs_sink = gsink.rhs_sink; - if (!gsink.is_outer && right_group < rhs_sink.bin_groups.size()) { - rhs_sink.hash_groups[right_group].reset(); + auto &rhs_groups = gsource.hashed_groups[1]; + if (!gsource.is_right_outer && right_bin < rhs_groups.size()) { + rhs_groups[right_bin].reset(); } - left_hash = nullptr; + left_group = nullptr; left_itr.reset(); lhs_scanner.reset(); - auto &lhs_sink = *gsink.lhs_sink; - if (left_group < lhs_sink.bin_groups.size()) { - lhs_sink.hash_groups[left_group].reset(); + auto &lhs_groups = gsource.hashed_groups[0]; + if (left_bin < lhs_groups.size()) { + lhs_groups[left_bin].reset(); } } +template void AsOfProbeBuffer::ResolveJoin(bool *found_match, idx_t *matches) { + using SORT_KEY = SortKey; + using BLOCKS_ITERATOR = block_iterator_t; + // If there was no right partition, there are no matches lhs_match_count = 0; if (!right_itr) { return; } + Repin(*left_itr); + BLOCKS_ITERATOR left_key(*left_itr); + + Repin(*right_itr); + BLOCKS_ITERATOR right_key(*right_itr); + const auto count = lhs_payload.size(); - const auto left_base = left_itr->GetIndex(); + const auto left_base = lhs_scanner->Base(); + auto lhs_sel = (count == lhs_scanned.size()) ? FlatVector::IncrementalSelectionVector() : &lhs_scan_sel; // Searching for right <= left for (idx_t i = 0; i < count; ++i) { - left_itr->SetIndex(left_base + i); - // If right > left, then there is no match - if (!right_itr->Compare(*left_itr)) { + const auto left_pos = left_base + lhs_sel->get_index(i); + if (!Compare(right_key[right_pos], left_key[left_pos], strict)) { continue; } // Exponential search forward for a non-matching value using radix iterators // (We use exponential search to avoid thrashing the block manager on large probes) idx_t bound = 1; - idx_t begin = right_itr->GetIndex(); - right_itr->SetIndex(begin + bound); - while (right_itr->GetIndex() < right_hash->count) { - if (right_itr->Compare(*left_itr)) { + idx_t begin = right_pos; + while (begin + bound < right_group->Count()) { + if (Compare(right_key[begin + bound], left_key[left_pos], strict)) { // If right <= left, jump ahead bound *= 2; - right_itr->SetIndex(begin + bound); } else { break; } @@ -539,23 +700,22 @@ void AsOfProbeBuffer::ResolveJoin(bool *found_match, idx_t *matches) { // Binary search for the first non-matching value using radix iterators // The previous value (which we know exists) is the match auto first = begin + bound / 2; - auto last = MinValue(begin + bound, right_hash->count); + auto last = MinValue(begin + bound, right_group->Count()); while (first < last) { const auto mid = first + (last - first) / 2; - right_itr->SetIndex(mid); - if (right_itr->Compare(*left_itr)) { + if (Compare(right_key[mid], left_key[left_pos], strict)) { // If right <= left, new lower bound first = mid + 1; } else { last = mid; } } - right_itr->SetIndex(--first); + right_pos = --first; - // Check partitions for strict equality - if (right_hash->ComparePartitions(*left_itr, *right_itr)) { - continue; - } + // TODO: Check partitions for strict equality + // if (right_group->ComparePartitions(*left_itr, *right_itr)) { + // continue; + // } // Emit match data if (found_match) { @@ -564,18 +724,14 @@ void AsOfProbeBuffer::ResolveJoin(bool *found_match, idx_t *matches) { if (matches) { matches[i] = first; } - lhs_sel.set_index(lhs_match_count++, i); + lhs_match_sel.set_index(lhs_match_count++, i); } } -unique_ptr PhysicalAsOfJoin::GetOperatorState(ExecutionContext &context) const { - return make_uniq(context.client, *this); -} - void AsOfProbeBuffer::ResolveSimpleJoin(ExecutionContext &context, DataChunk &chunk) { // perform the actual join bool found_match[STANDARD_VECTOR_SIZE] = {false}; - ResolveJoin(found_match); + (this->*resolve_join_func)(found_match, nullptr); // now construct the result based on the join result switch (op.join_type) { @@ -593,10 +749,12 @@ void AsOfProbeBuffer::ResolveSimpleJoin(ExecutionContext &context, DataChunk &ch void AsOfProbeBuffer::ResolveComplexJoin(ExecutionContext &context, DataChunk &chunk) { // perform the actual join idx_t matches[STANDARD_VECTOR_SIZE]; - ResolveJoin(nullptr, matches); + (this->*resolve_join_func)(nullptr, matches); + // Extract the rhs input columns from the match + rhs_input.Reset(); for (idx_t i = 0; i < lhs_match_count; ++i) { - const auto idx = lhs_sel[i]; + const auto idx = lhs_match_sel[i]; const auto match_pos = matches[idx]; // Skip to the range containing the match while (match_pos >= rhs_scanner->Scanned()) { @@ -606,30 +764,71 @@ void AsOfProbeBuffer::ResolveComplexJoin(ExecutionContext &context, DataChunk &c // Append the individual values // TODO: Batch the copies const auto source_offset = match_pos - (rhs_scanner->Scanned() - rhs_payload.size()); - for (column_t col_idx = 0; col_idx < op.right_projection_map.size(); ++col_idx) { - const auto rhs_idx = op.right_projection_map[col_idx]; - auto &source = rhs_payload.data[rhs_idx]; - auto &target = chunk.data[lhs_payload.ColumnCount() + col_idx]; + for (column_t col_idx = 0; col_idx < rhs_payload.data.size(); ++col_idx) { + auto &source = rhs_payload.data[col_idx]; + auto &target = rhs_input.data[col_idx]; VectorOperations::Copy(source, target, source_offset + 1, source_offset, i); } } + rhs_input.SetCardinality(lhs_match_count); // Slice the left payload into the result for (column_t i = 0; i < lhs_payload.ColumnCount(); ++i) { - chunk.data[i].Slice(lhs_payload.data[i], lhs_sel, lhs_match_count); + chunk.data[i].Slice(lhs_payload.data[i], lhs_match_sel, lhs_match_count); + } + + // Reference the projected right payload into the result + for (column_t col_idx = 0; col_idx < op.right_projection_map.size(); ++col_idx) { + const auto rhs_idx = op.right_projection_map[col_idx]; + auto &source = rhs_input.data[rhs_idx]; + auto &target = chunk.data[lhs_payload.ColumnCount() + col_idx]; + target.Reference(source); } chunk.SetCardinality(lhs_match_count); - auto match_sel = &lhs_sel; - if (filterer.expressions.size() == 1) { - lhs_match_count = filterer.SelectExpression(chunk, filter_sel); - chunk.Slice(filter_sel, lhs_match_count); - match_sel = &filter_sel; + + // Filter out partition mismatches + const auto equal_cols = op.conditions.size() - 1; + if (equal_cols) { + // Prepare the lhs keys + if (lhs_match_count < lhs_keys.size()) { + lhs_keys.Slice(lhs_match_sel, lhs_match_count); + } + + rhs_keys.Reset(); + rhs_executor.Execute(rhs_input, rhs_keys); + + auto sel = FlatVector::IncrementalSelectionVector(); + auto tail_count = lhs_match_count; + for (size_t cmp_idx = 0; cmp_idx < equal_cols; ++cmp_idx) { + auto &left = lhs_keys.data[cmp_idx]; + auto &right = rhs_keys.data[cmp_idx]; + if (tail_count < rhs_keys.size()) { + left.Slice(*sel, tail_count); + right.Slice(*sel, tail_count); + } + tail_count = PhysicalRangeJoin::SelectJoinTail(op.conditions[cmp_idx].comparison, left, right, sel, + tail_count, &tail_sel); + sel = &tail_sel; + } + + // Did anything get filtered out? + if (tail_count < lhs_match_count) { + if (tail_count == 0) { + // Need to reset here otherwise we may use the non-flat chunk when constructing LEFT/OUTER + chunk.Reset(); + lhs_match_count = tail_count; + } else { + chunk.Slice(*sel, tail_count); + // Slice lhs_match_sel to the remaining lhs rows + lhs_match_count = lhs_match_sel.SliceInPlace(*sel, tail_count); + } + } } // Update the match masks for the rows we ended up with left_outer.Reset(); for (idx_t i = 0; i < lhs_match_count; ++i) { - const auto idx = match_sel->get_index(i); + const auto idx = lhs_match_sel.get_index(i); left_outer.SetMatch(idx); const auto first = matches[idx]; right_outer->SetMatch(first); @@ -646,7 +845,7 @@ void AsOfProbeBuffer::GetData(ExecutionContext &context, DataChunk &chunk) { if (left_outer.Enabled()) { // left join: before we move to the next chunk, see if we need to output any vectors that didn't // have a match found - left_outer.ConstructLeftJoinResult(lhs_payload, chunk); + left_outer.ConstructLeftJoinResult(lhs_scanned, chunk); left_outer.Reset(); } return; @@ -675,118 +874,80 @@ void AsOfProbeBuffer::GetData(ExecutionContext &context, DataChunk &chunk) { } } -class AsOfGlobalSourceState : public GlobalSourceState { -public: - explicit AsOfGlobalSourceState(AsOfGlobalSinkState &gsink_p) - : gsink(gsink_p), next_combine(0), combined(0), merged(0), mergers(0), next_left(0), flushed(0), next_right(0) { - } - - PartitionGlobalMergeStates &GetMergeStates() { - lock_guard guard(lock); - if (!merge_states) { - merge_states = make_uniq(*gsink.lhs_sink); - } - return *merge_states; - } - - AsOfGlobalSinkState &gsink; - //! The next buffer to combine - atomic next_combine; - //! The number of combined buffers - atomic combined; - //! The number of combined buffers - atomic merged; - //! The number of combined buffers - atomic mergers; - //! The next buffer to flush - atomic next_left; - //! The number of flushed buffers - atomic flushed; - //! The right outer output read position. - atomic next_right; - //! The merge handler - mutex lock; - unique_ptr merge_states; - -public: - idx_t MaxThreads() override { - return gsink.lhs_buffers.size(); - } -}; - -unique_ptr PhysicalAsOfJoin::GetGlobalSourceState(ClientContext &context) const { - auto &gsink = sink_state->Cast(); - return make_uniq(gsink); +unique_ptr PhysicalAsOfJoin::GetGlobalSourceState(ClientContext &client) const { + return make_uniq(client, *this); } class AsOfLocalSourceState : public LocalSourceState { public: - using HashGroupPtr = unique_ptr; + using HashGroupPtr = unique_ptr; + + AsOfLocalSourceState(ExecutionContext &context, AsOfGlobalSourceState &gsource, const PhysicalAsOfJoin &op); - AsOfLocalSourceState(AsOfGlobalSourceState &gsource, const PhysicalAsOfJoin &op, ClientContext &client_p); + //! Task management + bool TaskFinished() const { + if (hash_group) { + return !scanner.get(); + } else { + return !probe_buffer.Scanning(); + } + } - // Return true if we were not interrupted (another thread died) - bool CombineLeftPartitions(); - bool MergeLeftPartitions(); + void ExecuteInnerTask(DataChunk &chunk); + void ExecuteOuterTask(DataChunk &chunk); + + void ExecuteTask(DataChunk &chunk) { + if (hash_group) { + ExecuteOuterTask(chunk); + } else { + ExecuteInnerTask(chunk); + } + } idx_t BeginRightScan(const idx_t hash_bin); AsOfGlobalSourceState &gsource; - ClientContext &client; + ExecutionContext &context; //! The left side partition being probed AsOfProbeBuffer probe_buffer; - //! The read partition + //! The rhs group idx_t hash_bin; HashGroupPtr hash_group; //! The read cursor - unique_ptr scanner; - //! Pointer to the matches - const bool *found_match = {}; + unique_ptr scanner; + //! The right outer buffer + DataChunk rhs_chunk; + //! The right outer slicer + SelectionVector rsel; + //! Pointer to the right marker + const bool *rhs_matches = {}; }; -AsOfLocalSourceState::AsOfLocalSourceState(AsOfGlobalSourceState &gsource, const PhysicalAsOfJoin &op, - ClientContext &client_p) - : gsource(gsource), client(client_p), probe_buffer(gsource.gsink.lhs_sink->context, op) { - gsource.mergers++; -} - -bool AsOfLocalSourceState::CombineLeftPartitions() { - const auto buffer_count = gsource.gsink.lhs_buffers.size(); - while (gsource.combined < buffer_count && !client.interrupted) { - const auto next_combine = gsource.next_combine++; - if (next_combine < buffer_count) { - gsource.gsink.lhs_buffers[next_combine]->Combine(); - ++gsource.combined; - } else { - TaskScheduler::GetScheduler(client).YieldThread(); - } - } +AsOfLocalSourceState::AsOfLocalSourceState(ExecutionContext &context, AsOfGlobalSourceState &gsource, + const PhysicalAsOfJoin &op) + : gsource(gsource), context(context), probe_buffer(context.client, op, gsource), rsel(STANDARD_VECTOR_SIZE) { - return !client.interrupted; -} - -bool AsOfLocalSourceState::MergeLeftPartitions() { - PartitionGlobalMergeStates::Callback local_callback; - PartitionLocalMergeState local_merge(*gsource.gsink.lhs_sink); - gsource.GetMergeStates().ExecuteTask(local_merge, local_callback); - gsource.merged++; - while (gsource.merged < gsource.mergers && !client.interrupted) { - TaskScheduler::GetScheduler(client).YieldThread(); - } - return !client.interrupted; + rhs_chunk.Initialize(context.client, op.children[1].get().GetTypes()); } idx_t AsOfLocalSourceState::BeginRightScan(const idx_t hash_bin_p) { hash_bin = hash_bin_p; - hash_group = std::move(gsource.gsink.rhs_sink.hash_groups[hash_bin]); - if (hash_group->global_sort->sorted_blocks.empty()) { + auto &rhs_groups = gsource.hashed_groups[1]; + if (hash_bin >= rhs_groups.size()) { + return 0; + } + + hash_group = std::move(rhs_groups[hash_bin]); + if (!hash_group || !hash_group->Count()) { return 0; } - scanner = make_uniq(*hash_group->global_sort); - found_match = gsource.gsink.right_outers[hash_bin].GetMatches(); + auto &gsink = gsource.op.sink_state->Cast(); + scanner = make_uniq(*hash_group, *gsink.hashed_sorts[1]); + + rhs_matches = gsource.right_outers[hash_bin].GetMatches(); return scanner->Remaining(); } @@ -794,122 +955,164 @@ idx_t AsOfLocalSourceState::BeginRightScan(const idx_t hash_bin_p) { unique_ptr PhysicalAsOfJoin::GetLocalSourceState(ExecutionContext &context, GlobalSourceState &gstate) const { auto &gsource = gstate.Cast(); - return make_uniq(gsource, *this, context.client); + return make_uniq(context, gsource, *this); } -SourceResultType PhysicalAsOfJoin::GetData(ExecutionContext &context, DataChunk &chunk, - OperatorSourceInput &input) const { - auto &gsource = input.global_state.Cast(); - auto &lsource = input.local_state.Cast(); - auto &rhs_sink = gsource.gsink.rhs_sink; - auto &client = context.client; - - // Step 1: Combine the partitions - if (!lsource.CombineLeftPartitions()) { - return SourceResultType::FINISHED; - } - - // Step 2: Sort on all threads - if (!lsource.MergeLeftPartitions()) { - return SourceResultType::FINISHED; - } - - // Step 3: Join the partitions - auto &lhs_sink = *gsource.gsink.lhs_sink; - const auto left_bins = lhs_sink.grouping_data ? lhs_sink.grouping_data->GetPartitions().size() : 1; - while (gsource.flushed < left_bins) { - // Make sure we have something to flush - if (!lsource.probe_buffer.Scanning()) { - const auto left_bin = gsource.next_left++; - if (left_bin < left_bins) { - // More to flush - lsource.probe_buffer.BeginLeftScan(left_bin); - } else if (!IsRightOuterJoin(join_type) || client.interrupted) { - return SourceResultType::FINISHED; +bool AsOfGlobalSourceState::TryPrepareNextStage() { + // Inside the lock. + auto &lhs_groups = hashed_groups[0]; + auto &rhs_groups = hashed_groups[1]; + switch (stage.load()) { + case AsOfJoinSourceStage::INNER: + if (flushed_left >= lhs_groups.size()) { + stage = IsRightOuterJoin(op.join_type) ? AsOfJoinSourceStage::RIGHT : AsOfJoinSourceStage::DONE; + return true; + } + break; + case AsOfJoinSourceStage::RIGHT: + if (flushed_right >= rhs_groups.size()) { + stage = AsOfJoinSourceStage::DONE; + return true; + } + break; + default: + break; + } + return false; +} + +bool AsOfGlobalSourceState::AssignTask(AsOfLocalSourceState &lsource) { + auto guard = Lock(); + + auto &lhs_groups = hashed_groups[0]; + auto &rhs_groups = hashed_groups[1]; + + switch (stage.load()) { + case AsOfJoinSourceStage::INNER: + while (next_left < lhs_groups.size()) { + // More to flush + const auto left_bin = next_left++; + lsource.probe_buffer.BeginLeftScan(left_bin); + if (!lsource.TaskFinished()) { + return true; } else { - // Wait for all threads to finish - // TODO: How to implement a spin wait correctly? - // Returning BLOCKED seems to hang the system. - TaskScheduler::GetScheduler(client).YieldThread(); - continue; + ++flushed_left; } } - - lsource.probe_buffer.GetData(context, chunk); - if (chunk.size()) { - return SourceResultType::HAVE_MORE_OUTPUT; - } else if (lsource.probe_buffer.HasMoreData()) { - // Join the next partition - continue; - } else { - lsource.probe_buffer.EndLeftScan(); - gsource.flushed++; + break; + case AsOfJoinSourceStage::RIGHT: + while (next_right < rhs_groups.size()) { + const auto right_bin = next_right++; + lsource.BeginRightScan(right_bin); + if (!lsource.TaskFinished()) { + return true; + } else { + ++flushed_right; + } } + break; + default: + break; } - // Step 4: Emit right join matches - if (!IsRightOuterJoin(join_type)) { - return SourceResultType::FINISHED; - } + return false; +} - auto &hash_groups = rhs_sink.hash_groups; - const auto right_groups = hash_groups.size(); +void AsOfLocalSourceState::ExecuteInnerTask(DataChunk &chunk) { + while (probe_buffer.HasMoreData()) { + probe_buffer.GetData(context, chunk); + if (chunk.size()) { + return; + } + } + probe_buffer.EndLeftScan(); + gsource.flushed_left++; +} - DataChunk rhs_chunk; - rhs_chunk.Initialize(Allocator::Get(context.client), rhs_sink.payload_types); - SelectionVector rsel(STANDARD_VECTOR_SIZE); - - while (chunk.size() == 0) { - // Move to the next bin if we are done. - while (!lsource.scanner || !lsource.scanner->Remaining()) { - lsource.scanner.reset(); - lsource.hash_group.reset(); - auto hash_bin = gsource.next_right++; - if (hash_bin >= right_groups) { - return SourceResultType::FINISHED; - } +SourceResultType PhysicalAsOfJoin::GetData(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const { + auto &gsource = input.global_state.Cast(); + auto &lsource = input.local_state.Cast(); - for (; hash_bin < hash_groups.size(); hash_bin = gsource.next_right++) { - if (hash_groups[hash_bin]) { - break; - } + // Any call to GetData must produce tuples, otherwise the pipeline executor thinks that we're done + // Therefore, we loop until we've produced tuples, or until the operator is actually done + while (gsource.stage != AsOfJoinSourceStage::DONE && chunk.size() == 0) { + if (!lsource.TaskFinished() || gsource.AssignTask(lsource)) { + lsource.ExecuteTask(chunk); + } else { + auto guard = gsource.Lock(); + if (gsource.TryPrepareNextStage() || gsource.stage == AsOfJoinSourceStage::DONE) { + gsource.UnblockTasks(guard); + } else { + return gsource.BlockSource(guard, input.interrupt_state); } - lsource.BeginRightScan(hash_bin); } - const auto rhs_position = lsource.scanner->Scanned(); - lsource.scanner->Scan(rhs_chunk); + } + + return chunk.size() > 0 ? SourceResultType::HAVE_MORE_OUTPUT : SourceResultType::FINISHED; +} + +void AsOfLocalSourceState::ExecuteOuterTask(DataChunk &chunk) { + idx_t result_count = 0; + while (!result_count) { + const auto rhs_position = scanner->Scanned(); + scanner->Scan(rhs_chunk); const auto count = rhs_chunk.size(); if (count == 0) { - return SourceResultType::FINISHED; + scanner.reset(); + ++gsource.flushed_right; + return; } // figure out which tuples didn't find a match in the RHS - auto found_match = lsource.found_match; - idx_t result_count = 0; + result_count = 0; for (idx_t i = 0; i < count; i++) { - if (!found_match[rhs_position + i]) { + if (!rhs_matches[rhs_position + i]) { rsel.set_index(result_count++, i); } } + } - if (result_count > 0) { - // if there were any tuples that didn't find a match, output them - const idx_t left_column_count = children[0].get().GetTypes().size(); - for (idx_t col_idx = 0; col_idx < left_column_count; ++col_idx) { - chunk.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR); - ConstantVector::SetNull(chunk.data[col_idx], true); - } - for (idx_t col_idx = 0; col_idx < right_projection_map.size(); ++col_idx) { - const auto rhs_idx = right_projection_map[col_idx]; - chunk.data[left_column_count + col_idx].Slice(rhs_chunk.data[rhs_idx], rsel, result_count); - } - chunk.SetCardinality(result_count); - break; - } + // if there were any tuples that didn't find a match, output them + const auto &op = gsource.op; + const idx_t left_column_count = op.children[0].get().GetTypes().size(); + for (idx_t col_idx = 0; col_idx < left_column_count; ++col_idx) { + chunk.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR); + ConstantVector::SetNull(chunk.data[col_idx], true); + } + for (idx_t col_idx = 0; col_idx < op.right_projection_map.size(); ++col_idx) { + const auto rhs_idx = op.right_projection_map[col_idx]; + chunk.data[left_column_count + col_idx].Slice(rhs_chunk.data[rhs_idx], rsel, result_count); } + chunk.SetCardinality(result_count); +} - return chunk.size() > 0 ? SourceResultType::HAVE_MORE_OUTPUT : SourceResultType::FINISHED; +//===--------------------------------------------------------------------===// +// Pipeline Construction +//===--------------------------------------------------------------------===// +void PhysicalAsOfJoin::BuildPipelines(Pipeline ¤t, MetaPipeline &meta_pipeline) { + D_ASSERT(children.size() == 2); + if (meta_pipeline.HasRecursiveCTE()) { + throw NotImplementedException("AsOf joins are not supported in recursive CTEs yet"); + } + + // becomes a source after both children fully sink their data + meta_pipeline.GetState().SetPipelineSource(current, *this); + + // Create one child meta pipeline that will hold the LHS and RHS pipelines + auto &child_meta_pipeline = meta_pipeline.CreateChildMetaPipeline(current, *this); + + // Build out RHS first because that is the order the join planner expects. + auto rhs_pipeline = child_meta_pipeline.GetBasePipeline(); + children[1].get().BuildPipelines(*rhs_pipeline, child_meta_pipeline); + + // Build out LHS + auto &lhs_pipeline = child_meta_pipeline.CreatePipeline(); + children[0].get().BuildPipelines(lhs_pipeline, child_meta_pipeline); + + // Despite having the same sink, LHS and everything created after it need their own (same) PipelineFinishEvent + child_meta_pipeline.AddFinishEvent(lhs_pipeline); } } // namespace duckdb diff --git a/src/execution/operator/join/physical_iejoin.cpp b/src/execution/operator/join/physical_iejoin.cpp index 90aba4722240..d4ea9f38f5e2 100644 --- a/src/execution/operator/join/physical_iejoin.cpp +++ b/src/execution/operator/join/physical_iejoin.cpp @@ -1,15 +1,8 @@ -#include - #include "duckdb/execution/operator/join/physical_iejoin.hpp" #include "duckdb/common/atomic.hpp" -#include "duckdb/common/operator/comparison_operators.hpp" #include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/sort/sort.hpp" -#include "duckdb/common/sort/sorted_block.hpp" -#include "duckdb/common/thread.hpp" -#include "duckdb/common/vector_operations/vector_operations.hpp" -#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/common/sorting/sort_key.hpp" #include "duckdb/main/client_context.hpp" #include "duckdb/parallel/event.hpp" #include "duckdb/parallel/meta_pipeline.hpp" @@ -17,6 +10,8 @@ #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" +#include + namespace duckdb { PhysicalIEJoin::PhysicalIEJoin(PhysicalPlan &physical_plan, LogicalComparisonJoin &op, PhysicalOperator &left, @@ -82,17 +77,15 @@ class IEJoinGlobalState : public GlobalSinkState { public: IEJoinGlobalState(ClientContext &context, const PhysicalIEJoin &op) : child(1) { tables.resize(2); - RowLayout lhs_layout; - lhs_layout.Initialize(op.children[0].get().GetTypes()); + const auto &lhs_types = op.children[0].get().GetTypes(); vector lhs_order; lhs_order.emplace_back(op.lhs_orders[0].Copy()); - tables[0] = make_uniq(context, lhs_order, lhs_layout, op); + tables[0] = make_uniq(context, lhs_order, lhs_types, op); - RowLayout rhs_layout; - rhs_layout.Initialize(op.children[1].get().GetTypes()); + const auto &rhs_types = op.children[1].get().GetTypes(); vector rhs_order; rhs_order.emplace_back(op.rhs_orders[0].Copy()); - tables[1] = make_uniq(context, rhs_order, rhs_layout, op); + tables[1] = make_uniq(context, rhs_order, rhs_types, op); if (op.filter_pushdown) { skip_filter_pushdown = op.filter_pushdown->probe_info.empty(); @@ -100,11 +93,18 @@ class IEJoinGlobalState : public GlobalSinkState { } } - void Sink(DataChunk &input, IEJoinLocalState &lstate); - void Finalize(Pipeline &pipeline, Event &event) { + void Sink(ExecutionContext &context, DataChunk &input, IEJoinLocalState &lstate); + + void Finalize(ClientContext &client, InterruptState &interrupt) { // Sort the current input child D_ASSERT(child < tables.size()); - tables[child]->Finalize(pipeline, event); + tables[child]->Finalize(client, interrupt); + }; + + void Materialize(Pipeline &pipeline, Event &event) { + // Sort the current input child + D_ASSERT(child < tables.size()); + tables[child]->Materialize(pipeline, event); child = child ? 0 : 2; skip_filter_pushdown = true; }; @@ -123,8 +123,8 @@ class IEJoinLocalState : public LocalSinkState { public: using LocalSortedTable = PhysicalRangeJoin::LocalSortedTable; - IEJoinLocalState(ClientContext &context, const PhysicalRangeJoin &op, IEJoinGlobalState &gstate) - : table(context, op, gstate.child) { + IEJoinLocalState(ExecutionContext &context, const PhysicalRangeJoin &op, IEJoinGlobalState &gstate) + : table(context, *gstate.tables[gstate.child], gstate.child) { if (op.filter_pushdown) { local_filter_state = op.filter_pushdown->GetLocalState(*gstate.global_filter_state); @@ -144,32 +144,23 @@ unique_ptr PhysicalIEJoin::GetGlobalSinkState(ClientContext &co unique_ptr PhysicalIEJoin::GetLocalSinkState(ExecutionContext &context) const { auto &ie_sink = sink_state->Cast(); - return make_uniq(context.client, *this, ie_sink); + return make_uniq(context, *this, ie_sink); } -void IEJoinGlobalState::Sink(DataChunk &input, IEJoinLocalState &lstate) { - auto &table = *tables[child]; - auto &global_sort_state = table.global_sort_state; - auto &local_sort_state = lstate.table.local_sort_state; - +void IEJoinGlobalState::Sink(ExecutionContext &context, DataChunk &input, IEJoinLocalState &lstate) { // Sink the data into the local sort state - lstate.table.Sink(input, global_sort_state); - - // When sorting data reaches a certain size, we sort it - if (local_sort_state.SizeInBytes() >= table.memory_per_thread) { - local_sort_state.Sort(global_sort_state, true); - } + lstate.table.Sink(context, input); } SinkResultType PhysicalIEJoin::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { auto &gstate = input.global_state.Cast(); auto &lstate = input.local_state.Cast(); - if (gstate.child == 0 && gstate.tables[1]->global_sort_state.sorted_blocks.empty() && EmptyResultIfRHSIsEmpty()) { + if (gstate.child == 0 && gstate.tables[1]->Count() == 0 && EmptyResultIfRHSIsEmpty()) { return SinkResultType::FINISHED; } - gstate.Sink(chunk, lstate); + gstate.Sink(context, chunk, lstate); if (filter_pushdown && !gstate.skip_filter_pushdown) { filter_pushdown->Sink(lstate.table.keys, *lstate.local_filter_state); @@ -181,7 +172,7 @@ SinkResultType PhysicalIEJoin::Sink(ExecutionContext &context, DataChunk &chunk, SinkCombineResultType PhysicalIEJoin::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const { auto &gstate = input.global_state.Cast(); auto &lstate = input.local_state.Cast(); - gstate.tables[gstate.child]->Combine(lstate.table); + gstate.tables[gstate.child]->Combine(context, lstate.table); auto &client_profiler = QueryProfiler::Get(context.client); context.thread.profiler.Flush(*this); @@ -197,14 +188,13 @@ SinkCombineResultType PhysicalIEJoin::Combine(ExecutionContext &context, Operato //===--------------------------------------------------------------------===// // Finalize //===--------------------------------------------------------------------===// -SinkFinalizeType PhysicalIEJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, +SinkFinalizeType PhysicalIEJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &client, OperatorSinkFinalizeInput &input) const { auto &gstate = input.global_state.Cast(); if (filter_pushdown && !gstate.skip_filter_pushdown) { - (void)filter_pushdown->Finalize(context, nullptr, *gstate.global_filter_state, *this); + (void)filter_pushdown->Finalize(client, nullptr, *gstate.global_filter_state, *this); } auto &table = *gstate.tables[gstate.child]; - auto &global_sort_state = table.global_sort_state; if ((gstate.child == 1 && PropagatesBuildSide(join_type)) || (gstate.child == 0 && IsLeftOuterJoin(join_type))) { // for FULL/LEFT/RIGHT OUTER JOIN, initialize found_match to false for every tuple @@ -212,15 +202,18 @@ SinkFinalizeType PhysicalIEJoin::Finalize(Pipeline &pipeline, Event &event, Clie } SinkFinalizeType res; - if (gstate.child == 1 && global_sort_state.sorted_blocks.empty() && EmptyResultIfRHSIsEmpty()) { + if (gstate.child == 1 && table.Count() == 0 && EmptyResultIfRHSIsEmpty()) { // Empty input! res = SinkFinalizeType::NO_OUTPUT_POSSIBLE; } else { res = SinkFinalizeType::READY; } + // Clean up the current table + gstate.Finalize(client, input.interrupt_state); + // Move to the next input child - gstate.Finalize(pipeline, event); + gstate.Materialize(pipeline, event); return res; } @@ -236,21 +229,52 @@ OperatorResultType PhysicalIEJoin::ExecuteInternal(ExecutionContext &context, Da //===--------------------------------------------------------------------===// // Source //===--------------------------------------------------------------------===// +enum class IEJoinSourceStage : uint8_t { INIT, INNER, OUTER, DONE }; + struct IEJoinUnion { using SortedTable = PhysicalRangeJoin::GlobalSortedTable; + using ChunkRange = std::pair; + + // Comparison utilities + static bool IsStrictComparison(ExpressionType comparison) { + switch (comparison) { + case ExpressionType::COMPARE_LESSTHAN: + case ExpressionType::COMPARE_GREATERTHAN: + return true; + case ExpressionType::COMPARE_LESSTHANOREQUALTO: + case ExpressionType::COMPARE_GREATERTHANOREQUALTO: + return false; + default: + throw InternalException("Unimplemented comparison type for IEJoin!"); + } + } - static idx_t AppendKey(SortedTable &table, ExpressionExecutor &executor, SortedTable &marked, int64_t increment, - int64_t base, const idx_t block_idx); - - static void Sort(SortedTable &table) { - auto &global_sort_state = table.global_sort_state; - global_sort_state.PrepareMergePhase(); - while (global_sort_state.sorted_blocks.size() > 1) { - global_sort_state.InitializeMergeRound(); - MergeSorter merge_sorter(global_sort_state, global_sort_state.buffer_manager); - merge_sorter.PerformInMergeRound(); - global_sort_state.CompleteMergeRound(true); + template + static inline bool Compare(const T &lhs, const T &rhs, const bool strict) { + const bool less_than = lhs < rhs; + if (!less_than && !strict) { + return !(rhs < lhs); } + return less_than; + } + + template + static bool TemplatedCompareKeys(ExternalBlockIteratorState &state1, const idx_t pos1, + ExternalBlockIteratorState &state2, const idx_t pos2, bool strict); + + static bool CompareKeys(ExternalBlockIteratorState &state1, const idx_t pos1, ExternalBlockIteratorState &state2, + const idx_t pos2, bool strict, const SortKeyType &sort_key_type); + + static bool CompareBounds(SortedTable &t1, const ChunkRange &b1, SortedTable &t2, const ChunkRange &b2, + bool strict); + + static idx_t AppendKey(ExecutionContext &context, InterruptState &interrupt, SortedTable &table, + ExpressionExecutor &executor, SortedTable &marked, int64_t increment, int64_t rid, + const ChunkRange &range); + + static void Sort(ExecutionContext &context, InterruptState &interrupt, SortedTable &table) { + table.Finalize(context.client, interrupt); + table.Materialize(context, interrupt); } template @@ -258,21 +282,17 @@ struct IEJoinUnion { vector result; result.reserve(table.count); - auto &gstate = table.global_sort_state; - auto &blocks = *gstate.sorted_blocks[0]->payload_data; - PayloadScanner scanner(blocks, gstate, false); + auto &collection = *table.sorted->payload_data; + vector scan_ids(1, col_idx); + TupleDataScanState state; + collection.InitializeScan(state, scan_ids); DataChunk payload; - payload.Initialize(Allocator::DefaultAllocator(), gstate.payload_layout.GetTypes()); - for (;;) { - payload.Reset(); - scanner.Scan(payload); - const auto count = payload.size(); - if (!count) { - break; - } + collection.InitializeScanChunk(state, payload); - const auto data_ptr = FlatVector::GetData(payload.data[col_idx]); + while (collection.Scan(state, payload)) { + const auto count = payload.size(); + const auto data_ptr = FlatVector::GetData(payload.data[0]); for (idx_t i = 0; i < count; i++) { result.push_back(UnsafeNumericCast(data_ptr[i])); } @@ -281,12 +301,40 @@ struct IEJoinUnion { return result; } - IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, SortedTable &t1, const idx_t b1, SortedTable &t2, - const idx_t b2); + class UnionIterator { + public: + UnionIterator(SortedTable &table, bool strict) : state(table.CreateIteratorState()), strict(strict) { + } + + inline idx_t GetIndex() const { + return index; + } + + inline void SetIndex(idx_t i) { + index = i; + } + + UnionIterator &operator++() { + ++index; + return *this; + } + + unique_ptr state; + idx_t index = 0; + const bool strict; + }; + + IEJoinUnion(ExecutionContext &context, const PhysicalIEJoin &op, SortedTable &t1, const ChunkRange &b1, + SortedTable &t2, const ChunkRange &b2); idx_t SearchL1(idx_t pos); + + template bool NextRow(); + using next_row_t = bool (duckdb::IEJoinUnion::*)(); + next_row_t next_row_func; + //! Inverted loop idx_t JoinComplexBlocks(SelectionVector &lsel, SelectionVector &rsel); @@ -314,49 +362,64 @@ struct IEJoinUnion { idx_t n; idx_t i; idx_t j; - unique_ptr op1; - unique_ptr off1; - unique_ptr op2; - unique_ptr off2; + unique_ptr op1; + unique_ptr off1; + unique_ptr op2; + unique_ptr off2; int64_t lrid; }; -idx_t IEJoinUnion::AppendKey(SortedTable &table, ExpressionExecutor &executor, SortedTable &marked, int64_t increment, - int64_t base, const idx_t block_idx) { - LocalSortState local_sort_state; - local_sort_state.Initialize(marked.global_sort_state, marked.global_sort_state.buffer_manager); +idx_t IEJoinUnion::AppendKey(ExecutionContext &context, InterruptState &interrupt, SortedTable &table, + ExpressionExecutor &executor, SortedTable &marked, int64_t increment, int64_t rid, + const ChunkRange &chunk_range) { + const auto chunk_begin = chunk_range.first; + const auto chunk_end = chunk_range.second; // Reading const auto valid = table.count - table.has_null; - auto &gstate = table.global_sort_state; - PayloadScanner scanner(gstate, block_idx); - auto table_idx = block_idx * gstate.block_capacity; + auto &source = *table.sorted->payload_data; + TupleDataScanState scanner; + source.InitializeScan(scanner); DataChunk scanned; - scanned.Initialize(Allocator::DefaultAllocator(), scanner.GetPayloadTypes()); + source.InitializeScanChunk(scanner, scanned); - // Writing - auto types = local_sort_state.sort_layout->logical_types; - const idx_t payload_idx = types.size(); + // TODO: Random access into TupleDataCollection (NextScanIndex is private...) + idx_t table_idx = 0; + for (idx_t i = 0; i < chunk_begin; ++i) { + source.Scan(scanner, scanned); + table_idx += scanned.size(); + } - const auto &payload_types = local_sort_state.payload_layout->GetTypes(); - types.insert(types.end(), payload_types.begin(), payload_types.end()); - const idx_t rid_idx = types.size() - 1; + // Writing + auto &sort = *marked.sort; + auto local_sort_state = sort.GetLocalSinkState(context); + vector types; + for (const auto &expr : executor.expressions) { + types.emplace_back(expr->return_type); + } + const idx_t rid_idx = types.size(); + types.emplace_back(LogicalType::BIGINT); DataChunk keys; DataChunk payload; keys.Initialize(Allocator::DefaultAllocator(), types); + OperatorSinkInput sink {*marked.global_sink, *local_sort_state, interrupt}; idx_t inserted = 0; - for (auto rid = base; table_idx < valid;) { - scanned.Reset(); - scanner.Scan(scanned); + for (auto chunk_idx = chunk_begin; chunk_idx < chunk_end; ++chunk_idx) { + source.Scan(scanner, scanned); // NULLs are at the end, so stop when we reach them auto scan_count = scanned.size(); if (table_idx + scan_count > valid) { - scan_count = valid - table_idx; - scanned.SetCardinality(scan_count); + if (table_idx >= valid) { + scan_count = 0; + ; + } else { + scan_count = valid - table_idx; + scanned.SetCardinality(scan_count); + } } if (scan_count == 0) { break; @@ -375,43 +438,89 @@ idx_t IEJoinUnion::AppendKey(SortedTable &table, ExpressionExecutor &executor, S rid += increment * UnsafeNumericCast(scan_count); // Sort on the sort columns (which will no longer be needed) - keys.Split(payload, payload_idx); - local_sort_state.SinkChunk(keys, payload); + sort.Sink(context, keys, sink); inserted += scan_count; - keys.Fuse(payload); - - // Flush when we have enough data - if (local_sort_state.SizeInBytes() >= marked.memory_per_thread) { - local_sort_state.Sort(marked.global_sort_state, true); - } } - marked.global_sort_state.AddLocalState(local_sort_state); + OperatorSinkCombineInput combine {*marked.global_sink, *local_sort_state, interrupt}; + sort.Combine(context, combine); marked.count += inserted; return inserted; } -IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, SortedTable &t1, const idx_t b1, - SortedTable &t2, const idx_t b2) +// TODO: Function pointers? +template +bool IEJoinUnion::TemplatedCompareKeys(ExternalBlockIteratorState &state1, const idx_t pos1, + ExternalBlockIteratorState &state2, const idx_t pos2, bool strict) { + using SORT_KEY = SortKey; + using BLOCKS_ITERATOR = block_iterator_t; + + BLOCKS_ITERATOR bounds1(state1, pos1); + BLOCKS_ITERATOR bounds2(state2, pos2); + + return Compare(*bounds1, *bounds2, strict); +} + +bool IEJoinUnion::CompareKeys(ExternalBlockIteratorState &state1, const idx_t pos1, ExternalBlockIteratorState &state2, + const idx_t pos2, bool strict, const SortKeyType &sort_key_type) { + + switch (sort_key_type) { + case SortKeyType::NO_PAYLOAD_FIXED_8: + return TemplatedCompareKeys(state1, pos1, state2, pos2, strict); + case SortKeyType::NO_PAYLOAD_FIXED_16: + return TemplatedCompareKeys(state1, pos1, state2, pos2, strict); + case SortKeyType::NO_PAYLOAD_FIXED_24: + return TemplatedCompareKeys(state1, pos1, state2, pos2, strict); + case SortKeyType::NO_PAYLOAD_FIXED_32: + return TemplatedCompareKeys(state1, pos1, state2, pos2, strict); + case SortKeyType::NO_PAYLOAD_VARIABLE_32: + return TemplatedCompareKeys(state1, pos1, state2, pos2, strict); + case SortKeyType::PAYLOAD_FIXED_16: + return TemplatedCompareKeys(state1, pos1, state2, pos2, strict); + case SortKeyType::PAYLOAD_FIXED_24: + return TemplatedCompareKeys(state1, pos1, state2, pos2, strict); + case SortKeyType::PAYLOAD_FIXED_32: + return TemplatedCompareKeys(state1, pos1, state2, pos2, strict); + case SortKeyType::PAYLOAD_VARIABLE_32: + return TemplatedCompareKeys(state1, pos1, state2, pos2, strict); + default: + throw NotImplementedException("IEJoinUnion::CompareKeys for %s", EnumUtil::ToString(sort_key_type)); + } +} + +bool IEJoinUnion::CompareBounds(SortedTable &t1, const ChunkRange &b1, SortedTable &t2, const ChunkRange &b2, + bool strict) { + auto &keys1 = *t1.sorted->key_data; + ExternalBlockIteratorState state1(keys1, nullptr); + const idx_t pos1 = t1.BlockStart(b1.first); + + auto &keys2 = *t2.sorted->key_data; + ExternalBlockIteratorState state2(keys2, nullptr); + const idx_t pos2 = t2.BlockEnd(b2.second - 1); + + const auto sort_key_type = t1.GetSortKeyType(); + D_ASSERT(sort_key_type == t2.GetSortKeyType()); + return CompareKeys(state1, pos1, state2, pos2, strict, sort_key_type); +} + +IEJoinUnion::IEJoinUnion(ExecutionContext &context, const PhysicalIEJoin &op, SortedTable &t1, const ChunkRange &b1, + SortedTable &t2, const ChunkRange &b2) : n(0), i(0) { // input : query Q with 2 join predicates t1.X op1 t2.X' and t1.Y op2 t2.Y', tables T, T' of sizes m and n resp. // output: a list of tuple pairs (ti , tj) // Note that T/T' are already sorted on X/X' and contain the payload data // We only join the two block numbers and use the sizes of the blocks as the counts + InterruptState interrupt; + // 0. Filter out tables with no overlap - if (!t1.BlockSize(b1) || !t2.BlockSize(b2)) { + if (t1.sorted->key_data->ChunkCount() <= b1.first || t2.sorted->key_data->ChunkCount() <= b2.first) { return; } - const auto &cmp1 = op.conditions[0].comparison; - SBIterator bounds1(t1.global_sort_state, cmp1); - SBIterator bounds2(t2.global_sort_state, cmp1); - // t1.X[0] op1 t2.X'[-1] - bounds1.SetIndex(bounds1.block_capacity * b1); - bounds2.SetIndex(bounds2.block_capacity * b2 + t2.BlockSize(b2) - 1); - if (!bounds1.Compare(bounds2)) { + const auto strict1 = IsStrictComparison(op.conditions[0].comparison); + if (!CompareBounds(t1, b1, t2, b2, strict1)) { return; } @@ -428,8 +537,6 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte vector types; types.emplace_back(order2.expression->return_type); types.emplace_back(LogicalType::BIGINT); - RowLayout payload_layout; - payload_layout.Initialize(types); // Sort on the first expression auto ref = make_uniq(order1.expression->return_type, 0U); @@ -451,37 +558,37 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte // Using this OrderType, if i < j then value[i] (from left table) and value[j] (from right table) match // the condition (t1.time <= t2.time or t1.time < t2.time), then from_left will force them into the correct order. auto from_left = make_uniq(Value::BOOLEAN(true)); - orders.emplace_back(SBIterator::ComparisonValue(cmp1) == 0 ? OrderType::DESCENDING : OrderType::ASCENDING, - OrderByNullType::ORDER_DEFAULT, std::move(from_left)); + orders.emplace_back(!strict1 ? OrderType::DESCENDING : OrderType::ASCENDING, OrderByNullType::ORDER_DEFAULT, + std::move(from_left)); - l1 = make_uniq(context, orders, payload_layout, op); + l1 = make_uniq(context.client, orders, types, op); // LHS has positive rids - ExpressionExecutor l_executor(context); + ExpressionExecutor l_executor(context.client); l_executor.AddExpression(*order1.expression); // add const column true auto left_const = make_uniq(Value::BOOLEAN(true)); l_executor.AddExpression(*left_const); l_executor.AddExpression(*order2.expression); - AppendKey(t1, l_executor, *l1, 1, 1, b1); + AppendKey(context, interrupt, t1, l_executor, *l1, 1, 1, b1); // RHS has negative rids - ExpressionExecutor r_executor(context); + ExpressionExecutor r_executor(context.client); r_executor.AddExpression(*op.rhs_orders[0].expression); // add const column flase auto right_const = make_uniq(Value::BOOLEAN(false)); r_executor.AddExpression(*right_const); r_executor.AddExpression(*op.rhs_orders[1].expression); - AppendKey(t2, r_executor, *l1, -1, -1, b2); + AppendKey(context, interrupt, t2, r_executor, *l1, -1, -1, b2); - if (l1->global_sort_state.sorted_blocks.empty()) { + if (!l1->Count()) { return; } - Sort(*l1); + Sort(context, interrupt, *l1); - op1 = make_uniq(l1->global_sort_state, cmp1); - off1 = make_uniq(l1->global_sort_state, cmp1); + op1 = make_uniq(*l1, strict1); + off1 = make_uniq(*l1, strict1); // We don't actually need the L1 column, just its sort key, which is in the sort blocks li = ExtractColumn(*l1, types.size() - 1); @@ -493,22 +600,19 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte // For this we just need a two-column table of Y, P types.clear(); types.emplace_back(LogicalType::BIGINT); - payload_layout.Initialize(types); // Sort on the first expression orders.clear(); ref = make_uniq(order2.expression->return_type, 0U); orders.emplace_back(order2.type, order2.null_order, std::move(ref)); - ExpressionExecutor executor(context); + ExpressionExecutor executor(context.client); executor.AddExpression(*orders[0].expression); - l2 = make_uniq(context, orders, payload_layout, op); - for (idx_t base = 0, block_idx = 0; block_idx < l1->BlockCount(); ++block_idx) { - base += AppendKey(*l1, executor, *l2, 1, NumericCast(base), block_idx); - } + l2 = make_uniq(context.client, orders, types, op); + AppendKey(context, interrupt, *l1, executor, *l2, 1, 0, {0, l1->BlockCount()}); - Sort(*l2); + Sort(context, interrupt, *l2); // We don't actually need the L2 column, just its sort key, which is in the sort blocks @@ -526,15 +630,57 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte bloom_filter.Initialize(bloom_array.data(), bloom_count); // 11. for(i←1 to n) do - const auto &cmp2 = op.conditions[1].comparison; - op2 = make_uniq(l2->global_sort_state, cmp2); - off2 = make_uniq(l2->global_sort_state, cmp2); + const auto strict2 = IsStrictComparison(op.conditions[1].comparison); + op2 = make_uniq(*l2, strict2); + off2 = make_uniq(*l2, strict2); i = 0; j = 0; - (void)NextRow(); + + const auto sort_key_type = l2->GetSortKeyType(); + switch (sort_key_type) { + case SortKeyType::NO_PAYLOAD_FIXED_8: + next_row_func = &IEJoinUnion::NextRow; + break; + case SortKeyType::NO_PAYLOAD_FIXED_16: + next_row_func = &IEJoinUnion::NextRow; + break; + case SortKeyType::NO_PAYLOAD_FIXED_24: + next_row_func = &IEJoinUnion::NextRow; + break; + case SortKeyType::NO_PAYLOAD_FIXED_32: + next_row_func = &IEJoinUnion::NextRow; + break; + case SortKeyType::NO_PAYLOAD_VARIABLE_32: + next_row_func = &IEJoinUnion::NextRow; + break; + case SortKeyType::PAYLOAD_FIXED_16: + next_row_func = &IEJoinUnion::NextRow; + break; + case SortKeyType::PAYLOAD_FIXED_24: + next_row_func = &IEJoinUnion::NextRow; + break; + case SortKeyType::PAYLOAD_FIXED_32: + next_row_func = &IEJoinUnion::NextRow; + break; + case SortKeyType::PAYLOAD_VARIABLE_32: + next_row_func = &IEJoinUnion::NextRow; + break; + default: + throw NotImplementedException("IEJoinUnion for %s", EnumUtil::ToString(sort_key_type)); + } + + (this->*next_row_func)(); } +template bool IEJoinUnion::NextRow() { + using SORT_KEY = SortKey; + using BLOCKS_ITERATOR = block_iterator_t; + + BLOCKS_ITERATOR off2_itr(*off2->state); + BLOCKS_ITERATOR op2_itr(*op2->state); + const auto strict = off2->strict; + for (; i < n; ++i) { // 12. pos ← P[i] auto pos = p[i]; @@ -546,7 +692,7 @@ bool IEJoinUnion::NextRow() { // 16. B[pos] ← 1 op2->SetIndex(i); for (; off2->GetIndex() < n; ++(*off2)) { - if (!off2->Compare(*op2)) { + if (!Compare(off2_itr[off2->GetIndex()], op2_itr[op2->GetIndex()], strict)) { break; } const auto p2 = p[off2->GetIndex()]; @@ -652,7 +798,7 @@ idx_t IEJoinUnion::JoinComplexBlocks(SelectionVector &lsel, SelectionVector &rse } ++i; - if (!NextRow()) { + if (!(this->*next_row_func)()) { break; } } @@ -660,13 +806,83 @@ idx_t IEJoinUnion::JoinComplexBlocks(SelectionVector &lsel, SelectionVector &rse return result_count; } +class IEJoinLocalSourceState; + +class IEJoinGlobalSourceState : public GlobalSourceState { +public: + IEJoinGlobalSourceState(const PhysicalIEJoin &op, IEJoinGlobalState &gsink) + : op(op), gsink(gsink), stage(IEJoinSourceStage::INIT), next_pair(0), completed(0), left_outers(0), + next_left(0), right_outers(0), next_right(0) { + auto &left_table = *gsink.tables[0]; + auto &right_table = *gsink.tables[1]; + + left_blocks = left_table.BlockCount(); + left_ranges = (left_blocks + left_per_thread - 1) / left_per_thread; + + right_blocks = right_table.BlockCount(); + right_ranges = (right_blocks + right_per_thread - 1) / right_per_thread; + + pair_count = left_ranges * right_ranges; + } + + void Initialize(); + bool TryPrepareNextStage(); + bool AssignTask(ExecutionContext &context, IEJoinLocalSourceState &lstate); + +public: + idx_t MaxThreads() override; + + ProgressData GetProgress() const; + + const PhysicalIEJoin &op; + IEJoinGlobalState &gsink; + + atomic stage; + + // Join queue state + idx_t left_blocks = 0; + idx_t left_ranges = 0; + const idx_t left_per_thread = 1024; + idx_t right_blocks = 0; + idx_t right_ranges = 0; + const idx_t right_per_thread = 1024; + idx_t pair_count; + atomic next_pair; + atomic completed; + + // Outer joins + atomic left_outers; + atomic next_left; + + atomic right_outers; + atomic next_right; +}; + class IEJoinLocalSourceState : public LocalSourceState { public: - explicit IEJoinLocalSourceState(ClientContext &context, const PhysicalIEJoin &op) - : op(op), true_sel(STANDARD_VECTOR_SIZE), left_executor(context), right_executor(context), - left_matches(nullptr), right_matches(nullptr) { - auto &allocator = Allocator::Get(context); - unprojected.Initialize(allocator, op.unprojected_types); + IEJoinLocalSourceState(ClientContext &client, IEJoinGlobalSourceState &gsource) + : gsource(gsource), lsel(STANDARD_VECTOR_SIZE), rsel(STANDARD_VECTOR_SIZE), true_sel(STANDARD_VECTOR_SIZE), + left_executor(client), right_executor(client), left_matches(nullptr), right_matches(nullptr) + + { + auto &op = gsource.op; + auto &allocator = Allocator::Get(client); + unprojected.InitializeEmpty(op.unprojected_types); + lpayload.Initialize(allocator, op.children[0].get().GetTypes()); + rpayload.Initialize(allocator, op.children[1].get().GetTypes()); + + auto &ie_sink = op.sink_state->Cast(); + auto &left_table = *ie_sink.tables[0]; + auto &right_table = *ie_sink.tables[1]; + + left_iterator = left_table.CreateIteratorState(); + right_iterator = right_table.CreateIteratorState(); + + left_table.InitializePayloadState(left_chunk_state); + right_table.InitializePayloadState(right_chunk_state); + + left_scan_state = left_table.CreateScanState(client); + right_scan_state = right_table.CreateScanState(client); if (op.conditions.size() < 3) { return; @@ -703,16 +919,40 @@ class IEJoinLocalSourceState : public LocalSourceState { return count; } - const PhysicalIEJoin &op; + // Are we executing a task? + bool TaskFinished() const { + return !joiner && !left_matches && !right_matches; + } + + // resolve joins that can potentially output N*M elements (INNER, LEFT, FULL) + void ResolveComplexJoin(ExecutionContext &context, DataChunk &result); + // Resolve left join results + void ExecuteLeftTask(ExecutionContext &context, DataChunk &result); + // Resolve right join results + void ExecuteRightTask(ExecutionContext &context, DataChunk &result); + // Execute the current task + void ExecuteTask(ExecutionContext &context, DataChunk &result); + + IEJoinGlobalSourceState &gsource; // Joining unique_ptr joiner; idx_t left_base; idx_t left_block_index; + unique_ptr left_iterator; + TupleDataChunkState left_chunk_state; + SelectionVector lsel; + DataChunk lpayload; + unique_ptr left_scan_state; idx_t right_base; idx_t right_block_index; + unique_ptr right_iterator; + TupleDataChunkState right_chunk_state; + SelectionVector rsel; + DataChunk rpayload; + unique_ptr right_scan_state; // Trailing predicates SelectionVector true_sel; @@ -732,254 +972,246 @@ class IEJoinLocalSourceState : public LocalSourceState { bool *right_matches; }; -void PhysicalIEJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &result, LocalSourceState &state_p) const { - auto &state = state_p.Cast(); - auto &ie_sink = sink_state->Cast(); +void IEJoinLocalSourceState::ExecuteTask(ExecutionContext &context, DataChunk &result) { + if (joiner) { + ResolveComplexJoin(context, result); + } else if (left_matches != nullptr) { + ExecuteLeftTask(context, result); + } else if (right_matches != nullptr) { + ExecuteRightTask(context, result); + } +} + +void IEJoinLocalSourceState::ResolveComplexJoin(ExecutionContext &context, DataChunk &result) { + auto &op = gsource.op; + auto &ie_sink = op.sink_state->Cast(); + const auto &conditions = op.conditions; + + auto &chunk = unprojected; + auto &left_table = *ie_sink.tables[0]; + const auto left_cols = op.children[0].get().GetTypes().size(); + auto &right_table = *ie_sink.tables[1]; - const auto left_cols = children[0].get().GetTypes().size(); - auto &chunk = state.unprojected; do { - SelectionVector lsel(STANDARD_VECTOR_SIZE); - SelectionVector rsel(STANDARD_VECTOR_SIZE); - auto result_count = state.joiner->JoinComplexBlocks(lsel, rsel); + auto result_count = joiner->JoinComplexBlocks(lsel, rsel); if (result_count == 0) { // exhausted this pair + joiner.reset(); + ++gsource.completed; return; } // found matches: extract them - chunk.Reset(); - SliceSortedPayload(chunk, left_table.global_sort_state, state.left_block_index, lsel, result_count, 0); - SliceSortedPayload(chunk, right_table.global_sort_state, state.right_block_index, rsel, result_count, - left_cols); - chunk.SetCardinality(result_count); + left_table.Repin(*left_iterator); + right_table.Repin(*right_iterator); + + op.SliceSortedPayload(lpayload, left_table, *left_iterator, left_chunk_state, left_block_index, lsel, + result_count, *left_scan_state); + op.SliceSortedPayload(rpayload, right_table, *right_iterator, right_chunk_state, right_block_index, rsel, + result_count, *right_scan_state); auto sel = FlatVector::IncrementalSelectionVector(); if (conditions.size() > 2) { // If there are more expressions to compute, - // split the result chunk into the left and right halves - // so we can compute the values for comparison. + // use the left and right payloads + // to we can compute the values for comparison. const auto tail_cols = conditions.size() - 2; - DataChunk right_chunk; - chunk.Split(right_chunk, left_cols); - state.left_executor.SetChunk(chunk); - state.right_executor.SetChunk(right_chunk); + left_executor.SetChunk(lpayload); + right_executor.SetChunk(rpayload); auto tail_count = result_count; - auto true_sel = &state.true_sel; + auto match_sel = &true_sel; for (size_t cmp_idx = 0; cmp_idx < tail_cols; ++cmp_idx) { - auto &left = state.left_keys.data[cmp_idx]; - state.left_executor.ExecuteExpression(cmp_idx, left); + auto &left = left_keys.data[cmp_idx]; + left_executor.ExecuteExpression(cmp_idx, left); - auto &right = state.right_keys.data[cmp_idx]; - state.right_executor.ExecuteExpression(cmp_idx, right); + auto &right = right_keys.data[cmp_idx]; + right_executor.ExecuteExpression(cmp_idx, right); if (tail_count < result_count) { left.Slice(*sel, tail_count); right.Slice(*sel, tail_count); } - tail_count = SelectJoinTail(conditions[cmp_idx + 2].comparison, left, right, sel, tail_count, true_sel); - sel = true_sel; + tail_count = + op.SelectJoinTail(conditions[cmp_idx + 2].comparison, left, right, sel, tail_count, match_sel); + sel = match_sel; } - chunk.Fuse(right_chunk); if (tail_count < result_count) { result_count = tail_count; - chunk.Slice(*sel, result_count); + lpayload.Slice(*sel, result_count); + rpayload.Slice(*sel, result_count); } } + // Merge the payloads + chunk.Reset(); + for (column_t col_idx = 0; col_idx < chunk.ColumnCount(); ++col_idx) { + if (col_idx < left_cols) { + chunk.data[col_idx].Reference(lpayload.data[col_idx]); + } else { + chunk.data[col_idx].Reference(rpayload.data[col_idx - left_cols]); + } + } + chunk.SetCardinality(result_count); + // We need all of the data to compute other predicates, // but we only return what is in the projection map - ProjectResult(chunk, result); + op.ProjectResult(chunk, result); // found matches: mark the found matches if required if (left_table.found_match) { for (idx_t i = 0; i < result_count; i++) { - left_table.found_match[state.left_base + lsel[sel->get_index(i)]] = true; + left_table.found_match[left_base + lsel[sel->get_index(i)]] = true; } } if (right_table.found_match) { for (idx_t i = 0; i < result_count; i++) { - right_table.found_match[state.right_base + rsel[sel->get_index(i)]] = true; + right_table.found_match[right_base + rsel[sel->get_index(i)]] = true; } } result.Verify(); } while (result.size() == 0); } -class IEJoinGlobalSourceState : public GlobalSourceState { -public: - explicit IEJoinGlobalSourceState(const PhysicalIEJoin &op, IEJoinGlobalState &gsink) - : op(op), gsink(gsink), initialized(false), next_pair(0), completed(0), left_outers(0), next_left(0), - right_outers(0), next_right(0) { +void IEJoinGlobalSourceState::Initialize() { + auto guard = Lock(); + if (stage != IEJoinSourceStage::INIT) { + return; } - void Initialize() { - auto guard = Lock(); - if (initialized) { - return; - } + // Compute the starting row for each block + auto &left_table = *gsink.tables[0]; + const auto left_blocks = left_table.BlockCount(); - // Compute the starting row for reach block - // (In theory these are all the same size, but you never know...) - auto &left_table = *gsink.tables[0]; - const auto left_blocks = left_table.BlockCount(); - idx_t left_base = 0; + auto &right_table = *gsink.tables[1]; + const auto right_blocks = right_table.BlockCount(); - for (size_t lhs = 0; lhs < left_blocks; ++lhs) { - left_bases.emplace_back(left_base); - left_base += left_table.BlockSize(lhs); - } + // Outer join block counts + if (left_table.found_match) { + left_outers = left_blocks; + } - auto &right_table = *gsink.tables[1]; - const auto right_blocks = right_table.BlockCount(); - idx_t right_base = 0; - for (size_t rhs = 0; rhs < right_blocks; ++rhs) { - right_bases.emplace_back(right_base); - right_base += right_table.BlockSize(rhs); - } + if (right_table.found_match) { + right_outers = right_blocks; + } - // Outer join block counts - if (left_table.found_match) { - left_outers = left_blocks; + // Ready for action + stage = IEJoinSourceStage::INNER; +} +bool IEJoinGlobalSourceState::TryPrepareNextStage() { + // Inside lock + switch (stage.load()) { + case IEJoinSourceStage::INNER: + if (completed >= pair_count) { + stage = IEJoinSourceStage::OUTER; + return true; } - - if (right_table.found_match) { - right_outers = right_blocks; + break; + case IEJoinSourceStage::OUTER: + if (next_left >= left_outers && next_right >= right_outers) { + stage = IEJoinSourceStage::DONE; + return true; } - - // Ready for action - initialized = true; - } - -public: - idx_t MaxThreads() override { - // We can't leverage any more threads than block pairs. - const auto &sink_state = (op.sink_state->Cast()); - return sink_state.tables[0]->BlockCount() * sink_state.tables[1]->BlockCount(); + break; + default: + break; } - void GetNextPair(ClientContext &client, IEJoinLocalSourceState &lstate) { - auto &left_table = *gsink.tables[0]; - auto &right_table = *gsink.tables[1]; + return false; +} - const auto left_blocks = left_table.BlockCount(); - const auto right_blocks = right_table.BlockCount(); - const auto pair_count = left_blocks * right_blocks; +idx_t IEJoinGlobalSourceState::MaxThreads() { + // We can't leverage any more threads than block pairs. + const auto &sink_state = (op.sink_state->Cast()); + return sink_state.tables[0]->BlockCount() * sink_state.tables[1]->BlockCount(); +} - // Regular block - const auto i = next_pair++; - if (i < pair_count) { - const auto b1 = i / right_blocks; - const auto b2 = i % right_blocks; +bool IEJoinGlobalSourceState::AssignTask(ExecutionContext &context, IEJoinLocalSourceState &lstate) { + auto guard = Lock(); - lstate.left_block_index = b1; - lstate.left_base = left_bases[b1]; + using ChunkRange = IEJoinUnion::ChunkRange; + auto &left_table = *gsink.tables[0]; + auto &right_table = *gsink.tables[1]; - lstate.right_block_index = b2; - lstate.right_base = right_bases[b2]; + // Regular block + switch (stage.load()) { + case IEJoinSourceStage::INNER: + if (next_pair < pair_count) { + const auto i = next_pair++; + const auto b1 = (i / right_ranges) * left_per_thread; + const auto b2 = (i % right_ranges) * right_per_thread; - lstate.joiner = make_uniq(client, op, left_table, b1, right_table, b2); - return; - } + ChunkRange l_range {b1, MinValue(left_blocks, b1 + left_per_thread)}; + lstate.left_block_index = l_range.first; + lstate.left_base = left_table.BlockStart(l_range.first); - // Outer joins - if (!left_outers && !right_outers) { - return; - } + ChunkRange r_range {b2, MinValue(right_blocks, b2 + right_per_thread)}; + lstate.right_block_index = r_range.first; + lstate.right_base = right_table.BlockStart(r_range.first); - // Spin wait for regular blocks to finish(!) - while (completed < pair_count) { - std::this_thread::yield(); + lstate.joiner = make_uniq(context, op, left_table, l_range, right_table, r_range); + return true; } - + break; + case IEJoinSourceStage::OUTER: // Left outer blocks - const auto l = next_left++; - if (l < left_outers) { + if (next_left < left_outers) { + const auto l = next_left++; lstate.joiner = nullptr; lstate.left_block_index = l; - lstate.left_base = left_bases[l]; + lstate.left_base = left_table.BlockStart(l); lstate.left_matches = left_table.found_match.get() + lstate.left_base; lstate.outer_idx = 0; lstate.outer_count = left_table.BlockSize(l); - return; + return true; } else { lstate.left_matches = nullptr; } - // Right outer block - const auto r = next_right++; - if (r < right_outers) { + // Right outer blocks + if (next_right < right_outers) { + const auto r = next_right++; lstate.joiner = nullptr; lstate.right_block_index = r; - lstate.right_base = right_bases[r]; + lstate.right_base = right_table.BlockStart(r); lstate.right_matches = right_table.found_match.get() + lstate.right_base; lstate.outer_idx = 0; lstate.outer_count = right_table.BlockSize(r); - return; + return true; } else { lstate.right_matches = nullptr; } + break; + default: + break; } - void PairCompleted(ClientContext &client, IEJoinLocalSourceState &lstate) { - lstate.joiner.reset(); - ++completed; - GetNextPair(client, lstate); - } - - ProgressData GetProgress() const { - auto &left_table = *gsink.tables[0]; - auto &right_table = *gsink.tables[1]; - - const auto left_blocks = left_table.BlockCount(); - const auto right_blocks = right_table.BlockCount(); - const auto pair_count = left_blocks * right_blocks; + return false; +} - const auto count = pair_count + left_outers + right_outers; +ProgressData IEJoinGlobalSourceState::GetProgress() const { + const auto count = pair_count + left_outers + right_outers; - const auto l = MinValue(next_left.load(), left_outers.load()); - const auto r = MinValue(next_right.load(), right_outers.load()); - const auto returned = completed.load() + l + r; + const auto l = MinValue(next_left.load(), left_outers.load()); + const auto r = MinValue(next_right.load(), right_outers.load()); + const auto returned = completed.load() + l + r; - ProgressData res; - if (count) { - res.done = double(returned); - res.total = double(count); - } else { - res.SetInvalid(); - } - return res; + ProgressData res; + if (count) { + res.done = double(returned); + res.total = double(count); + } else { + res.SetInvalid(); } - - const PhysicalIEJoin &op; - IEJoinGlobalState &gsink; - - bool initialized; - - // Join queue state - atomic next_pair; - atomic completed; - - // Block base row number - vector left_bases; - vector right_bases; - - // Outer joins - atomic left_outers; - atomic next_left; - - atomic right_outers; - atomic next_right; -}; - + return res; +} unique_ptr PhysicalIEJoin::GetGlobalSourceState(ClientContext &context) const { auto &gsink = sink_state->Cast(); return make_uniq(*this, gsink); @@ -987,7 +1219,8 @@ unique_ptr PhysicalIEJoin::GetGlobalSourceState(ClientContext unique_ptr PhysicalIEJoin::GetLocalSourceState(ExecutionContext &context, GlobalSourceState &gstate) const { - return make_uniq(context.client, *this); + auto &gsource = gstate.Cast(); + return make_uniq(context.client, gsource); } ProgressData PhysicalIEJoin::GetProgress(ClientContext &context, GlobalSourceState &gsource_p) const { @@ -997,80 +1230,97 @@ ProgressData PhysicalIEJoin::GetProgress(ClientContext &context, GlobalSourceSta SourceResultType PhysicalIEJoin::GetData(ExecutionContext &context, DataChunk &result, OperatorSourceInput &input) const { - auto &ie_sink = sink_state->Cast(); - auto &ie_gstate = input.global_state.Cast(); - auto &ie_lstate = input.local_state.Cast(); + auto &gsource = input.global_state.Cast(); + auto &lsource = input.local_state.Cast(); - ie_gstate.Initialize(); + gsource.Initialize(); - if (!ie_lstate.joiner && !ie_lstate.left_matches && !ie_lstate.right_matches) { - ie_gstate.GetNextPair(context.client, ie_lstate); + // Any call to GetData must produce tuples, otherwise the pipeline executor thinks that we're done + // Therefore, we loop until we've produced tuples, or until the operator is actually done + while (gsource.stage != IEJoinSourceStage::DONE && result.size() == 0) { + if (!lsource.TaskFinished() || gsource.AssignTask(context, lsource)) { + lsource.ExecuteTask(context, result); + } else { + auto guard = gsource.Lock(); + if (gsource.TryPrepareNextStage() || gsource.stage == IEJoinSourceStage::DONE) { + gsource.UnblockTasks(guard); + } else { + return gsource.BlockSource(guard, input.interrupt_state); + } + } } + return result.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT; +} - // Process INNER results - while (ie_lstate.joiner) { - ResolveComplexJoin(context, result, ie_lstate); +void IEJoinLocalSourceState::ExecuteLeftTask(ExecutionContext &context, DataChunk &result) { + auto &op = gsource.op; + auto &ie_sink = op.sink_state->Cast(); - if (result.size()) { - return SourceResultType::HAVE_MORE_OUTPUT; - } + const auto left_cols = op.children[0].get().GetTypes().size(); + auto &chunk = unprojected; - ie_gstate.PairCompleted(context.client, ie_lstate); + const idx_t count = SelectOuterRows(left_matches); + if (!count) { + left_matches = nullptr; + return; } - // Process LEFT OUTER results - const auto left_cols = children[0].get().GetTypes().size(); - while (ie_lstate.left_matches) { - const idx_t count = ie_lstate.SelectOuterRows(ie_lstate.left_matches); - if (!count) { - ie_gstate.GetNextPair(context.client, ie_lstate); - continue; - } - auto &chunk = ie_lstate.unprojected; - chunk.Reset(); - SliceSortedPayload(chunk, ie_sink.tables[0]->global_sort_state, ie_lstate.left_block_index, ie_lstate.true_sel, - count); + auto &left_table = *ie_sink.tables[0]; + + left_table.Repin(*left_iterator); + op.SliceSortedPayload(lpayload, left_table, *left_iterator, left_chunk_state, left_block_index, true_sel, count, + *left_scan_state); - // Fill in NULLs to the right - for (auto col_idx = left_cols; col_idx < chunk.ColumnCount(); ++col_idx) { + // Fill in NULLs to the right + chunk.Reset(); + for (column_t col_idx = 0; col_idx < chunk.ColumnCount(); ++col_idx) { + if (col_idx < left_cols) { + chunk.data[col_idx].Reference(lpayload.data[col_idx]); + } else { chunk.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR); ConstantVector::SetNull(chunk.data[col_idx], true); } + } - ProjectResult(chunk, result); - result.SetCardinality(count); - result.Verify(); + op.ProjectResult(chunk, result); + result.SetCardinality(count); + result.Verify(); +} + +void IEJoinLocalSourceState::ExecuteRightTask(ExecutionContext &context, DataChunk &result) { + auto &op = gsource.op; + auto &ie_sink = op.sink_state->Cast(); + const auto left_cols = op.children[0].get().GetTypes().size(); - return result.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT; + auto &chunk = unprojected; + + const idx_t count = SelectOuterRows(right_matches); + if (!count) { + right_matches = nullptr; + return; } - // Process RIGHT OUTER results - while (ie_lstate.right_matches) { - const idx_t count = ie_lstate.SelectOuterRows(ie_lstate.right_matches); - if (!count) { - ie_gstate.GetNextPair(context.client, ie_lstate); - continue; - } + auto &right_table = *ie_sink.tables[1]; + auto &rsel = true_sel; - auto &chunk = ie_lstate.unprojected; - chunk.Reset(); - SliceSortedPayload(chunk, ie_sink.tables[1]->global_sort_state, ie_lstate.right_block_index, ie_lstate.true_sel, - count, left_cols); + right_table.Repin(*right_iterator); + op.SliceSortedPayload(rpayload, right_table, *right_iterator, right_chunk_state, right_block_index, rsel, count, + *right_scan_state); - // Fill in NULLs to the left - for (idx_t col_idx = 0; col_idx < left_cols; ++col_idx) { + // Fill in NULLs to the left + chunk.Reset(); + for (column_t col_idx = 0; col_idx < chunk.ColumnCount(); ++col_idx) { + if (col_idx < left_cols) { chunk.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR); ConstantVector::SetNull(chunk.data[col_idx], true); + } else { + chunk.data[col_idx].Reference(rpayload.data[col_idx - left_cols]); } - - ProjectResult(chunk, result); - result.SetCardinality(count); - result.Verify(); - - break; } - return result.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT; + op.ProjectResult(chunk, result); + result.SetCardinality(count); + result.Verify(); } //===--------------------------------------------------------------------===// diff --git a/src/execution/operator/join/physical_nested_loop_join.cpp b/src/execution/operator/join/physical_nested_loop_join.cpp index d96cda05d321..b449eee2f380 100644 --- a/src/execution/operator/join/physical_nested_loop_join.cpp +++ b/src/execution/operator/join/physical_nested_loop_join.cpp @@ -1,7 +1,5 @@ #include "duckdb/execution/operator/join/physical_nested_loop_join.hpp" #include "duckdb/parallel/thread_context.hpp" -#include "duckdb/common/operator/comparison_operators.hpp" -#include "duckdb/common/vector_operations/vector_operations.hpp" #include "duckdb/execution/expression_executor.hpp" #include "duckdb/execution/nested_loop_join.hpp" #include "duckdb/main/client_context.hpp" @@ -9,20 +7,23 @@ namespace duckdb { -PhysicalNestedLoopJoin::PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalOperator &op, PhysicalOperator &left, - PhysicalOperator &right, vector cond, JoinType join_type, +PhysicalNestedLoopJoin::PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalComparisonJoin &op, + PhysicalOperator &left, PhysicalOperator &right, + vector cond, JoinType join_type, idx_t estimated_cardinality, unique_ptr pushdown_info_p) : PhysicalComparisonJoin(physical_plan, op, PhysicalOperatorType::NESTED_LOOP_JOIN, std::move(cond), join_type, - estimated_cardinality) { + estimated_cardinality), + predicate(std::move(op.predicate)) { filter_pushdown = std::move(pushdown_info_p); children.push_back(left); children.push_back(right); } -PhysicalNestedLoopJoin::PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalOperator &op, PhysicalOperator &left, - PhysicalOperator &right, vector cond, JoinType join_type, +PhysicalNestedLoopJoin::PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalComparisonJoin &op, + PhysicalOperator &left, PhysicalOperator &right, + vector cond, JoinType join_type, idx_t estimated_cardinality) : PhysicalNestedLoopJoin(physical_plan, op, left, right, std::move(cond), join_type, estimated_cardinality, nullptr) { @@ -273,7 +274,7 @@ class PhysicalNestedLoopJoinState : public CachingOperatorState { PhysicalNestedLoopJoinState(ClientContext &context, const PhysicalNestedLoopJoin &op, const vector &conditions) : fetch_next_left(true), fetch_next_right(false), lhs_executor(context), left_tuple(0), right_tuple(0), - left_outer(IsLeftOuterJoin(op.join_type)) { + left_outer(IsLeftOuterJoin(op.join_type)), pred_executor(context) { vector condition_types; for (auto &cond : conditions) { lhs_executor.AddExpression(*cond.left); @@ -284,6 +285,11 @@ class PhysicalNestedLoopJoinState : public CachingOperatorState { right_condition.Initialize(allocator, condition_types); right_payload.Initialize(allocator, op.children[1].get().GetTypes()); left_outer.Initialize(STANDARD_VECTOR_SIZE); + + if (op.predicate) { + pred_executor.AddExpression(*op.predicate); + pred_matches.Initialize(); + } } bool fetch_next_left; @@ -302,6 +308,10 @@ class PhysicalNestedLoopJoinState : public CachingOperatorState { OuterJoinMarker left_outer; + //! Predicate + ExpressionExecutor pred_executor; + SelectionVector pred_matches; + public: void Finalize(const PhysicalOperator &op, ExecutionContext &context) override { context.thread.profiler.Flush(op); @@ -438,11 +448,20 @@ OperatorResultType PhysicalNestedLoopJoin::ResolveComplexJoin(ExecutionContext & if (match_count > 0) { // we have matching tuples! // construct the result - state.left_outer.SetMatches(lvector, match_count); - gstate.right_outer.SetMatches(rvector, match_count, state.condition_scan_state.current_row_index); - chunk.Slice(input, lvector, match_count); chunk.Slice(right_payload, rvector, match_count, input.ColumnCount()); + + // If we have a predicate, apply it to the result + if (predicate) { + auto &sel = state.pred_matches; + match_count = state.pred_executor.SelectExpression(chunk, sel); + chunk.Slice(sel, match_count); + lvector.SliceInPlace(sel, match_count); + rvector.SliceInPlace(sel, match_count); + } + + state.left_outer.SetMatches(lvector, match_count); + gstate.right_outer.SetMatches(rvector, match_count, state.condition_scan_state.current_row_index); } // check if we exhausted the RHS, if we did we need to move to the next right chunk in the next iteration diff --git a/src/execution/operator/join/physical_piecewise_merge_join.cpp b/src/execution/operator/join/physical_piecewise_merge_join.cpp index 1bd48ab62b14..c29f58a680f4 100644 --- a/src/execution/operator/join/physical_piecewise_merge_join.cpp +++ b/src/execution/operator/join/physical_piecewise_merge_join.cpp @@ -1,11 +1,8 @@ #include "duckdb/execution/operator/join/physical_piecewise_merge_join.hpp" -#include "duckdb/common/fast_mem.hpp" -#include "duckdb/common/operator/comparison_operators.hpp" #include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/sort/comparators.hpp" -#include "duckdb/common/sort/sort.hpp" -#include "duckdb/common/vector_operations/vector_operations.hpp" +#include "duckdb/common/sorting/sort_key.hpp" +#include "duckdb/common/types/row/block_iterator.hpp" #include "duckdb/execution/expression_executor.hpp" #include "duckdb/execution/operator/join/outer_join_marker.hpp" #include "duckdb/main/client_context.hpp" @@ -65,15 +62,14 @@ class MergeJoinGlobalState : public GlobalSinkState { using GlobalSortedTable = PhysicalRangeJoin::GlobalSortedTable; public: - MergeJoinGlobalState(ClientContext &context, const PhysicalPiecewiseMergeJoin &op) { - RowLayout rhs_layout; - rhs_layout.Initialize(op.children[1].get().GetTypes()); + MergeJoinGlobalState(ClientContext &client, const PhysicalPiecewiseMergeJoin &op) { + const auto &rhs_types = op.children[1].get().GetTypes(); vector rhs_order; rhs_order.emplace_back(op.rhs_orders[0].Copy()); - table = make_uniq(context, rhs_order, rhs_layout, op); + table = make_uniq(client, rhs_order, rhs_types, op); if (op.filter_pushdown) { skip_filter_pushdown = op.filter_pushdown->probe_info.empty(); - global_filter_state = op.filter_pushdown->GetGlobalState(context, op); + global_filter_state = op.filter_pushdown->GetGlobalState(client, op); } } @@ -81,8 +77,9 @@ class MergeJoinGlobalState : public GlobalSinkState { return table->count; } - void Sink(DataChunk &input, MergeJoinLocalState &lstate); + void Sink(ExecutionContext &context, DataChunk &input, MergeJoinLocalState &lstate); + //! The sorted table unique_ptr table; //! Should we not bother pushing down filters? bool skip_filter_pushdown = false; @@ -92,16 +89,19 @@ class MergeJoinGlobalState : public GlobalSinkState { class MergeJoinLocalState : public LocalSinkState { public: - explicit MergeJoinLocalState(ClientContext &context, const PhysicalRangeJoin &op, MergeJoinGlobalState &gstate, - const idx_t child) - : table(context, op, child) { + using GlobalSortedTable = PhysicalRangeJoin::GlobalSortedTable; + using LocalSortedTable = PhysicalRangeJoin::LocalSortedTable; + + MergeJoinLocalState(ExecutionContext &context, MergeJoinGlobalState &gstate, const idx_t child) + : table(context, *gstate.table, child) { + auto &op = gstate.table->op; if (op.filter_pushdown) { local_filter_state = op.filter_pushdown->GetLocalState(*gstate.global_filter_state); } } //! The local sort state - PhysicalRangeJoin::LocalSortedTable table; + LocalSortedTable table; //! Local state for accumulating filter statistics unique_ptr local_filter_state; }; @@ -113,20 +113,12 @@ unique_ptr PhysicalPiecewiseMergeJoin::GetGlobalSinkState(Clien unique_ptr PhysicalPiecewiseMergeJoin::GetLocalSinkState(ExecutionContext &context) const { // We only sink the RHS auto &gstate = sink_state->Cast(); - return make_uniq(context.client, *this, gstate, 1U); + return make_uniq(context, gstate, 1U); } -void MergeJoinGlobalState::Sink(DataChunk &input, MergeJoinLocalState &lstate) { - auto &global_sort_state = table->global_sort_state; - auto &local_sort_state = lstate.table.local_sort_state; - +void MergeJoinGlobalState::Sink(ExecutionContext &context, DataChunk &input, MergeJoinLocalState &lstate) { // Sink the data into the local sort state - lstate.table.Sink(input, global_sort_state); - - // When sorting data reaches a certain size, we sort it - if (local_sort_state.SizeInBytes() >= table->memory_per_thread) { - local_sort_state.Sort(global_sort_state, true); - } + lstate.table.Sink(context, input); } SinkResultType PhysicalPiecewiseMergeJoin::Sink(ExecutionContext &context, DataChunk &chunk, @@ -134,7 +126,7 @@ SinkResultType PhysicalPiecewiseMergeJoin::Sink(ExecutionContext &context, DataC auto &gstate = input.global_state.Cast(); auto &lstate = input.local_state.Cast(); - gstate.Sink(chunk, lstate); + gstate.Sink(context, chunk, lstate); if (filter_pushdown && !gstate.skip_filter_pushdown) { filter_pushdown->Sink(lstate.table.keys, *lstate.local_filter_state); @@ -147,7 +139,7 @@ SinkCombineResultType PhysicalPiecewiseMergeJoin::Combine(ExecutionContext &cont OperatorSinkCombineInput &input) const { auto &gstate = input.global_state.Cast(); auto &lstate = input.local_state.Cast(); - gstate.table->Combine(lstate.table); + gstate.table->Combine(context, lstate.table); auto &client_profiler = QueryProfiler::Get(context.client); context.thread.profiler.Flush(*this); @@ -162,25 +154,28 @@ SinkCombineResultType PhysicalPiecewiseMergeJoin::Combine(ExecutionContext &cont //===--------------------------------------------------------------------===// // Finalize //===--------------------------------------------------------------------===// -SinkFinalizeType PhysicalPiecewiseMergeJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, +SinkFinalizeType PhysicalPiecewiseMergeJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &client, OperatorSinkFinalizeInput &input) const { auto &gstate = input.global_state.Cast(); if (filter_pushdown && !gstate.skip_filter_pushdown) { - (void)filter_pushdown->Finalize(context, nullptr, *gstate.global_filter_state, *this); + (void)filter_pushdown->Finalize(client, nullptr, *gstate.global_filter_state, *this); } - auto &global_sort_state = gstate.table->global_sort_state; + + gstate.table->Finalize(client, input.interrupt_state); if (PropagatesBuildSide(join_type)) { // for FULL/RIGHT OUTER JOIN, initialize found_match to false for every tuple gstate.table->IntializeMatches(); } - if (global_sort_state.sorted_blocks.empty() && EmptyResultIfRHSIsEmpty()) { + + if (gstate.table->Count() == 0 && EmptyResultIfRHSIsEmpty()) { // Empty input! + gstate.table->MaterializeEmpty(client); return SinkFinalizeType::NO_OUTPUT_POSSIBLE; } // Sort the current input child - gstate.table->Finalize(pipeline, event); + gstate.table->Materialize(pipeline, event); return SinkFinalizeType::READY; } @@ -191,46 +186,50 @@ SinkFinalizeType PhysicalPiecewiseMergeJoin::Finalize(Pipeline &pipeline, Event class PiecewiseMergeJoinState : public CachingOperatorState { public: using LocalSortedTable = PhysicalRangeJoin::LocalSortedTable; + using GlobalSortedTable = PhysicalRangeJoin::GlobalSortedTable; - PiecewiseMergeJoinState(ClientContext &context, const PhysicalPiecewiseMergeJoin &op, bool force_external) - : context(context), allocator(Allocator::Get(context)), op(op), - buffer_manager(BufferManager::GetBufferManager(context)), force_external(force_external), - left_outer(IsLeftOuterJoin(op.join_type)), left_position(0), first_fetch(true), finished(true), - right_position(0), right_chunk_index(0), rhs_executor(context) { - vector condition_types; - for (auto &order : op.lhs_orders) { - condition_types.push_back(order.expression->return_type); - } + PiecewiseMergeJoinState(ClientContext &client, const PhysicalPiecewiseMergeJoin &op) + : client(client), allocator(Allocator::Get(client)), op(op), left_outer(IsLeftOuterJoin(op.join_type)), + left_position(0), first_fetch(true), finished(true), right_position(0), right_chunk_index(0), + rhs_executor(client) { left_outer.Initialize(STANDARD_VECTOR_SIZE); - lhs_layout.Initialize(op.children[0].get().GetTypes()); - lhs_payload.Initialize(allocator, op.children[0].get().GetTypes()); + lhs_payload.Initialize(client, op.children[0].get().GetTypes()); + // Sort on the first column lhs_order.emplace_back(op.lhs_orders[0].Copy()); // Set up shared data for multiple predicates sel.Initialize(STANDARD_VECTOR_SIZE); - condition_types.clear(); + vector condition_types; for (auto &order : op.rhs_orders) { rhs_executor.AddExpression(*order.expression); condition_types.push_back(order.expression->return_type); } - rhs_keys.Initialize(allocator, condition_types); + rhs_keys.Initialize(client, condition_types); + rhs_input.Initialize(client, op.children[1].get().GetTypes()); + + auto &gsink = op.sink_state->Cast(); + auto &rhs_table = *gsink.table; + rhs_iterator = rhs_table.CreateIteratorState(); + rhs_table.InitializePayloadState(rhs_chunk_state); + rhs_scan_state = rhs_table.CreateScanState(client); + + // Since we have now materialized the payload, the keys will not have payloads? + sort_key_type = rhs_table.GetSortKeyType(); } - ClientContext &context; + ClientContext &client; Allocator &allocator; const PhysicalPiecewiseMergeJoin &op; - BufferManager &buffer_manager; - bool force_external; // Block sorting DataChunk lhs_payload; OuterJoinMarker left_outer; vector lhs_order; - RowLayout lhs_layout; + unique_ptr lhs_global_table; unique_ptr lhs_local_table; - unique_ptr lhs_global_state; - unique_ptr scanner; + SortKeyType sort_key_type; + TupleDataScanState lhs_scan; // Simple scans idx_t left_position; @@ -238,178 +237,127 @@ class PiecewiseMergeJoinState : public CachingOperatorState { // Complex scans bool first_fetch; bool finished; + unique_ptr lhs_iterator; + unique_ptr rhs_iterator; idx_t right_position; idx_t right_chunk_index; idx_t right_base; idx_t prev_left_index; + TupleDataChunkState rhs_chunk_state; + unique_ptr rhs_scan_state; // Secondary predicate shared data SelectionVector sel; DataChunk rhs_keys; DataChunk rhs_input; ExpressionExecutor rhs_executor; - vector payload_heap_handles; public: - void ResolveJoinKeys(DataChunk &input) { + void ResolveJoinKeys(ExecutionContext &context, DataChunk &input) { // sort by join key - lhs_global_state = make_uniq(context, lhs_order, lhs_layout); - lhs_local_table = make_uniq(context, op, 0U); - lhs_local_table->Sink(input, *lhs_global_state); - - // Set external (can be forced with the PRAGMA) - lhs_global_state->external = force_external; - lhs_global_state->AddLocalState(lhs_local_table->local_sort_state); - lhs_global_state->PrepareMergePhase(); - while (lhs_global_state->sorted_blocks.size() > 1) { - MergeSorter merge_sorter(*lhs_global_state, buffer_manager); - merge_sorter.PerformInMergeRound(); - lhs_global_state->CompleteMergeRound(); - } - - // Scan the sorted payload - D_ASSERT(lhs_global_state->sorted_blocks.size() == 1); - - scanner = make_uniq(*lhs_global_state->sorted_blocks[0]->payload_data, *lhs_global_state); - lhs_payload.Reset(); - scanner->Scan(lhs_payload); + const auto &lhs_types = lhs_payload.GetTypes(); + lhs_global_table = make_uniq(context.client, lhs_order, lhs_types, op); + lhs_local_table = make_uniq(context, *lhs_global_table, 0U); + lhs_local_table->Sink(context, input); + lhs_global_table->Combine(context, *lhs_local_table); + + InterruptState interrupt; + lhs_global_table->Finalize(context.client, interrupt); + lhs_global_table->Materialize(context, interrupt); + + // Scan the sorted payload (minus the primary sort column) + auto &lhs_table = *lhs_global_table; + auto &lhs_payload_data = *lhs_table.sorted->payload_data; + lhs_payload_data.InitializeScan(lhs_scan); + lhs_payload_data.Scan(lhs_scan, lhs_payload); // Recompute the sorted keys from the sorted input - lhs_local_table->keys.Reset(); - lhs_local_table->executor.Execute(lhs_payload, lhs_local_table->keys); - } + auto &lhs_keys = lhs_local_table->keys; + lhs_keys.Reset(); + lhs_local_table->executor.Execute(lhs_payload, lhs_keys); - void Finalize(const PhysicalOperator &op, ExecutionContext &context) override { - if (lhs_local_table) { - context.thread.profiler.Flush(op); - } + lhs_iterator = lhs_table.CreateIteratorState(); } }; unique_ptr PhysicalPiecewiseMergeJoin::GetOperatorState(ExecutionContext &context) const { - bool force_external = ClientConfig::GetConfig(context.client).force_external; - return make_uniq(context.client, *this, force_external); + return make_uniq(context.client, *this); } -static inline idx_t SortedBlockNotNull(const idx_t base, const idx_t count, const idx_t not_null) { - return MinValue(base + count, MaxValue(base, not_null)) - base; +static inline idx_t SortedChunkNotNull(const idx_t chunk_idx, const idx_t count, const idx_t has_null) { + const auto chunk_begin = chunk_idx * STANDARD_VECTOR_SIZE; + const auto chunk_end = MinValue(chunk_begin + STANDARD_VECTOR_SIZE, count); + const auto not_null = count - has_null; + return MinValue(chunk_end, MaxValue(chunk_begin, not_null)) - chunk_begin; } -static int MergeJoinComparisonValue(ExpressionType comparison) { +static bool MergeJoinStrictComparison(ExpressionType comparison) { switch (comparison) { case ExpressionType::COMPARE_LESSTHAN: case ExpressionType::COMPARE_GREATERTHAN: - return -1; + return true; case ExpressionType::COMPARE_LESSTHANOREQUALTO: case ExpressionType::COMPARE_GREATERTHANOREQUALTO: - return 0; + return false; default: throw InternalException("Unimplemented comparison type for merge join!"); } } -struct BlockMergeInfo { - GlobalSortState &state; - //! The block being scanned - const idx_t block_idx; - //! The number of not-NULL values in the block (they are at the end) - const idx_t not_null; - //! The current offset in the block - idx_t &entry_idx; - SelectionVector result; - - BlockMergeInfo(GlobalSortState &state, idx_t block_idx, idx_t &entry_idx, idx_t not_null) - : state(state), block_idx(block_idx), not_null(not_null), entry_idx(entry_idx), result(STANDARD_VECTOR_SIZE) { - } -}; - -static void MergeJoinPinSortingBlock(SBScanState &scan, const idx_t block_idx) { - scan.SetIndices(block_idx, 0); - scan.PinRadix(block_idx); - - auto &sd = *scan.sb->blob_sorting_data; - if (block_idx < sd.data_blocks.size()) { - scan.PinData(sd); +// Compare using +bool MergeJoinBefore(const T &lhs, const T &rhs, const bool strict) { + const bool less_than = lhs < rhs; + if (!less_than && !strict) { + return !(rhs < lhs); } + return less_than; } -static data_ptr_t MergeJoinRadixPtr(SBScanState &scan, const idx_t entry_idx) { - scan.entry_idx = entry_idx; - return scan.RadixPtr(); -} +template +static idx_t TemplatedMergeJoinSimpleBlocks(PiecewiseMergeJoinState &lstate, MergeJoinGlobalState &gstate, + bool *found_match, const bool strict) { + using SORT_KEY = SortKey; + using BLOCK_ITERATOR = block_iterator_t; -static idx_t MergeJoinSimpleBlocks(PiecewiseMergeJoinState &lstate, MergeJoinGlobalState &rstate, bool *found_match, - const ExpressionType comparison) { - const auto cmp = MergeJoinComparisonValue(comparison); - - // The sort parameters should all be the same - auto &lsort = *lstate.lhs_global_state; - auto &rsort = rstate.table->global_sort_state; - D_ASSERT(lsort.sort_layout.all_constant == rsort.sort_layout.all_constant); - const auto all_constant = lsort.sort_layout.all_constant; - D_ASSERT(lsort.external == rsort.external); - const auto external = lsort.external; - - // There should only be one sorted block if they have been sorted - D_ASSERT(lsort.sorted_blocks.size() == 1); - SBScanState lread(lsort.buffer_manager, lsort); - lread.sb = lsort.sorted_blocks[0].get(); - - const idx_t l_block_idx = 0; - idx_t l_entry_idx = 0; - const auto lhs_not_null = lstate.lhs_local_table->count - lstate.lhs_local_table->has_null; - MergeJoinPinSortingBlock(lread, l_block_idx); - auto l_ptr = MergeJoinRadixPtr(lread, l_entry_idx); - - D_ASSERT(rsort.sorted_blocks.size() == 1); - SBScanState rread(rsort.buffer_manager, rsort); - rread.sb = rsort.sorted_blocks[0].get(); + // We only need the keys because we are extracting the row numbers + auto &lhs_table = *lstate.lhs_global_table; + D_ASSERT(SORT_KEY_TYPE == lhs_table.GetSortKeyType()); + auto &lhs_iterator = *lstate.lhs_iterator; + const auto lhs_not_null = lhs_table.count - lhs_table.has_null; - const auto cmp_size = lsort.sort_layout.comparison_size; - const auto entry_size = lsort.sort_layout.entry_size; + auto &rhs_table = *gstate.table; + auto &rhs_iterator = *lstate.rhs_iterator; + const auto rhs_not_null = rhs_table.count - rhs_table.has_null; - idx_t right_base = 0; - for (idx_t r_block_idx = 0; r_block_idx < rread.sb->radix_sorting_data.size(); r_block_idx++) { - // we only care about the BIGGEST value in each of the RHS data blocks + idx_t l_entry_idx = 0; + BLOCK_ITERATOR lhs_itr(lhs_iterator); + BLOCK_ITERATOR rhs_itr(rhs_iterator); + for (idx_t r_idx = 0; r_idx < rhs_not_null; r_idx += STANDARD_VECTOR_SIZE) { + // Repin the RHS to release memory + // This is safe because we only return the LHS values + // Note we only do this for the RHS because the LHS is only one chunk. + rhs_table.Repin(rhs_iterator); + + // we only care about the BIGGEST value in the RHS // because we want to figure out if the LHS values are less than [or equal] to ANY value - // get the biggest value from the RHS chunk - MergeJoinPinSortingBlock(rread, r_block_idx); - - auto &rblock = *rread.sb->radix_sorting_data[r_block_idx]; - const auto r_not_null = - SortedBlockNotNull(right_base, rblock.count, rstate.table->count - rstate.table->has_null); - if (r_not_null == 0) { - break; - } - const auto r_entry_idx = r_not_null - 1; - right_base += rblock.count; - - auto r_ptr = MergeJoinRadixPtr(rread, r_entry_idx); + const auto r_entry_idx = MinValue(r_idx + STANDARD_VECTOR_SIZE, rhs_not_null) - 1; // now we start from the current lpos value and check if we found a new value that is [<= OR <] the max RHS // value while (true) { - int comp_res; - if (all_constant) { - comp_res = FastMemcmp(l_ptr, r_ptr, cmp_size); - } else { - lread.entry_idx = l_entry_idx; - rread.entry_idx = r_entry_idx; - comp_res = Comparators::CompareTuple(lread, rread, l_ptr, r_ptr, lsort.sort_layout, external); - } - - if (comp_res <= cmp) { + // Note that both subscripts here are table indices, not chunk indices. + if (MergeJoinBefore(lhs_itr[l_entry_idx], rhs_itr[r_entry_idx], strict)) { // found a match for lpos, set it in the found_match vector found_match[l_entry_idx] = true; l_entry_idx++; - l_ptr += entry_size; if (l_entry_idx >= lhs_not_null) { // early out: we exhausted the entire LHS and they all match return 0; } } else { // we found no match: any subsequent value from the LHS we scan now will be bigger and thus also not - // match move to the next RHS chunk + // match. Move to the next RHS chunk break; } } @@ -417,13 +365,42 @@ static idx_t MergeJoinSimpleBlocks(PiecewiseMergeJoinState &lstate, MergeJoinGlo return 0; } +static idx_t MergeJoinSimpleBlocks(PiecewiseMergeJoinState &lstate, MergeJoinGlobalState &gstate, bool *match, + const ExpressionType comparison) { + const auto strict = MergeJoinStrictComparison(comparison); + + switch (lstate.sort_key_type) { + case SortKeyType::NO_PAYLOAD_FIXED_8: + return TemplatedMergeJoinSimpleBlocks(lstate, gstate, match, strict); + case SortKeyType::NO_PAYLOAD_FIXED_16: + return TemplatedMergeJoinSimpleBlocks(lstate, gstate, match, strict); + case SortKeyType::NO_PAYLOAD_FIXED_24: + return TemplatedMergeJoinSimpleBlocks(lstate, gstate, match, strict); + case SortKeyType::NO_PAYLOAD_FIXED_32: + return TemplatedMergeJoinSimpleBlocks(lstate, gstate, match, strict); + case SortKeyType::NO_PAYLOAD_VARIABLE_32: + return TemplatedMergeJoinSimpleBlocks(lstate, gstate, match, strict); + case SortKeyType::PAYLOAD_FIXED_16: + return TemplatedMergeJoinSimpleBlocks(lstate, gstate, match, strict); + case SortKeyType::PAYLOAD_FIXED_24: + return TemplatedMergeJoinSimpleBlocks(lstate, gstate, match, strict); + case SortKeyType::PAYLOAD_FIXED_32: + return TemplatedMergeJoinSimpleBlocks(lstate, gstate, match, strict); + case SortKeyType::PAYLOAD_VARIABLE_32: + return TemplatedMergeJoinSimpleBlocks(lstate, gstate, match, strict); + default: + throw NotImplementedException("MergeJoinSimpleBlocks for %s", EnumUtil::ToString(lstate.sort_key_type)); + } +} + void PhysicalPiecewiseMergeJoin::ResolveSimpleJoin(ExecutionContext &context, DataChunk &input, DataChunk &chunk, OperatorState &state_p) const { auto &state = state_p.Cast(); auto &gstate = sink_state->Cast(); - state.ResolveJoinKeys(input); - auto &lhs_table = *state.lhs_local_table; + state.ResolveJoinKeys(context, input); + auto &lhs_table = *state.lhs_global_table; + auto &lhs_keys = state.lhs_local_table->keys; // perform the actual join bool found_match[STANDARD_VECTOR_SIZE]; @@ -439,8 +416,8 @@ void PhysicalPiecewiseMergeJoin::ResolveSimpleJoin(ExecutionContext &context, Da case JoinType::MARK: { // The only part of the join keys that is actually used is the validity mask. // Since the payload is sorted, we can just set the tail end of the validity masks to invalid. - for (auto &key : lhs_table.keys.data) { - key.Flatten(lhs_table.keys.size()); + for (auto &key : lhs_keys.data) { + key.Flatten(lhs_keys.size()); auto &mask = FlatVector::Validity(key); if (mask.AllValid()) { continue; @@ -451,7 +428,7 @@ void PhysicalPiecewiseMergeJoin::ResolveSimpleJoin(ExecutionContext &context, Da } } // So we make a set of keys that have the validity mask set for the - PhysicalJoin::ConstructMarkJoinResult(lhs_table.keys, payload, chunk, found_match, gstate.table->has_null); + PhysicalJoin::ConstructMarkJoinResult(lhs_keys, payload, chunk, found_match, gstate.table->has_null); break; } case JoinType::SEMI: @@ -465,41 +442,42 @@ void PhysicalPiecewiseMergeJoin::ResolveSimpleJoin(ExecutionContext &context, Da } } -static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const ExpressionType comparison, - idx_t &prev_left_index) { - const auto cmp = MergeJoinComparisonValue(comparison); - - // The sort parameters should all be the same - D_ASSERT(l.state.sort_layout.all_constant == r.state.sort_layout.all_constant); - const auto all_constant = r.state.sort_layout.all_constant; - D_ASSERT(l.state.external == r.state.external); - const auto external = l.state.external; - - // There should only be one sorted block if they have been sorted - D_ASSERT(l.state.sorted_blocks.size() == 1); - SBScanState lread(l.state.buffer_manager, l.state); - lread.sb = l.state.sorted_blocks[0].get(); - D_ASSERT(lread.sb->radix_sorting_data.size() == 1); - MergeJoinPinSortingBlock(lread, l.block_idx); - auto l_start = MergeJoinRadixPtr(lread, 0); - auto l_ptr = MergeJoinRadixPtr(lread, l.entry_idx); - - D_ASSERT(r.state.sorted_blocks.size() == 1); - SBScanState rread(r.state.buffer_manager, r.state); - rread.sb = r.state.sorted_blocks[0].get(); +struct ChunkMergeInfo { + //! The iteration state + ExternalBlockIteratorState &state; + //! The block being scanned + const idx_t block_idx; + //! The number of not-NULL values in the chunk (they are at the end) + const idx_t not_null; + //! The current offset in the chunk + idx_t &entry_idx; + //! The offsets that match + SelectionVector result; - if (r.entry_idx >= r.not_null) { - return 0; + ChunkMergeInfo(ExternalBlockIteratorState &state, idx_t block_idx, idx_t &entry_idx, idx_t not_null) + : state(state), block_idx(block_idx), not_null(not_null), entry_idx(entry_idx), result(STANDARD_VECTOR_SIZE) { } - MergeJoinPinSortingBlock(rread, r.block_idx); - auto r_ptr = MergeJoinRadixPtr(rread, r.entry_idx); + idx_t GetIndex() const { + return state.GetIndex(block_idx, entry_idx); + } +}; + +template +static idx_t TemplatedMergeJoinComplexBlocks(ChunkMergeInfo &l, ChunkMergeInfo &r, const bool strict, + idx_t &prev_left_index) { + using SORT_KEY = SortKey; + using BLOCK_ITERATOR = block_iterator_t; - const auto cmp_size = l.state.sort_layout.comparison_size; - const auto entry_size = l.state.sort_layout.entry_size; + if (r.entry_idx >= r.not_null) { + return 0; + } idx_t result_count = 0; + BLOCK_ITERATOR l_ptr(l.state); + BLOCK_ITERATOR r_ptr(r.state); while (true) { + if (l.entry_idx < prev_left_index) { // left side smaller: found match l.result.set_index(result_count, sel_t(l.entry_idx)); @@ -507,7 +485,7 @@ static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const result_count++; // move left side forward l.entry_idx++; - l_ptr += entry_size; + ++l_ptr; if (result_count == STANDARD_VECTOR_SIZE) { // out of space! break; @@ -515,22 +493,14 @@ static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const continue; } if (l.entry_idx < l.not_null) { - int comp_res; - if (all_constant) { - comp_res = FastMemcmp(l_ptr, r_ptr, cmp_size); - } else { - lread.entry_idx = l.entry_idx; - rread.entry_idx = r.entry_idx; - comp_res = Comparators::CompareTuple(lread, rread, l_ptr, r_ptr, l.state.sort_layout, external); - } - if (comp_res <= cmp) { + if (MergeJoinBefore(l_ptr[l.GetIndex()], r_ptr[r.GetIndex()], strict)) { // left side smaller: found match l.result.set_index(result_count, sel_t(l.entry_idx)); r.result.set_index(result_count, sel_t(r.entry_idx)); result_count++; // move left side forward l.entry_idx++; - l_ptr += entry_size; + ++l_ptr; if (result_count == STANDARD_VECTOR_SIZE) { // out of space! break; @@ -546,27 +516,53 @@ static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const if (r.entry_idx >= r.not_null) { break; } - r_ptr += entry_size; + ++r_ptr; - l_ptr = l_start; l.entry_idx = 0; } return result_count; } +static idx_t MergeJoinComplexBlocks(const SortKeyType &sort_key_type, ChunkMergeInfo &l, ChunkMergeInfo &r, + const ExpressionType comparison, idx_t &prev_left_index) { + const auto strict = MergeJoinStrictComparison(comparison); + + switch (sort_key_type) { + case SortKeyType::NO_PAYLOAD_FIXED_8: + return TemplatedMergeJoinComplexBlocks(l, r, strict, prev_left_index); + case SortKeyType::NO_PAYLOAD_FIXED_16: + return TemplatedMergeJoinComplexBlocks(l, r, strict, prev_left_index); + case SortKeyType::NO_PAYLOAD_FIXED_24: + return TemplatedMergeJoinComplexBlocks(l, r, strict, prev_left_index); + case SortKeyType::NO_PAYLOAD_FIXED_32: + return TemplatedMergeJoinComplexBlocks(l, r, strict, prev_left_index); + case SortKeyType::NO_PAYLOAD_VARIABLE_32: + return TemplatedMergeJoinComplexBlocks(l, r, strict, prev_left_index); + case SortKeyType::PAYLOAD_FIXED_16: + return TemplatedMergeJoinComplexBlocks(l, r, strict, prev_left_index); + case SortKeyType::PAYLOAD_FIXED_24: + return TemplatedMergeJoinComplexBlocks(l, r, strict, prev_left_index); + case SortKeyType::PAYLOAD_FIXED_32: + return TemplatedMergeJoinComplexBlocks(l, r, strict, prev_left_index); + case SortKeyType::PAYLOAD_VARIABLE_32: + return TemplatedMergeJoinComplexBlocks(l, r, strict, prev_left_index); + default: + throw NotImplementedException("MergeJoinSimpleBlocks for %s", EnumUtil::ToString(sort_key_type)); + } +} + OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &input, DataChunk &chunk, OperatorState &state_p) const { auto &state = state_p.Cast(); auto &gstate = sink_state->Cast(); - auto &rsorted = *gstate.table->global_sort_state.sorted_blocks[0]; const auto left_cols = input.ColumnCount(); const auto tail_cols = conditions.size() - 1; - state.payload_heap_handles.clear(); do { if (state.first_fetch) { - state.ResolveJoinKeys(input); + state.ResolveJoinKeys(context, input); + state.lhs_payload.Verify(); state.right_chunk_index = 0; state.right_base = 0; @@ -588,36 +584,44 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionConte return OperatorResultType::NEED_MORE_INPUT; } - auto &lhs_table = *state.lhs_local_table; + auto &lhs_table = *state.lhs_global_table; const auto lhs_not_null = lhs_table.count - lhs_table.has_null; - BlockMergeInfo left_info(*state.lhs_global_state, 0, state.left_position, lhs_not_null); + ChunkMergeInfo left_info(*state.lhs_iterator, 0, state.left_position, lhs_not_null); + + auto &rhs_table = *gstate.table; + auto &rhs_iterator = *state.rhs_iterator; + const auto rhs_not_null = SortedChunkNotNull(state.right_chunk_index, rhs_table.count, rhs_table.has_null); + ChunkMergeInfo right_info(rhs_iterator, state.right_chunk_index, state.right_position, rhs_not_null); - const auto &rblock = *rsorted.radix_sorting_data[state.right_chunk_index]; - const auto rhs_not_null = - SortedBlockNotNull(state.right_base, rblock.count, gstate.table->count - gstate.table->has_null); - BlockMergeInfo right_info(gstate.table->global_sort_state, state.right_chunk_index, state.right_position, - rhs_not_null); + // Repin so we don't hang on to data after we have scanned it + // Note we only do this for the RHS because the LHS is only one chunk. + rhs_table.Repin(rhs_iterator); - idx_t result_count = - MergeJoinComplexBlocks(left_info, right_info, conditions[0].comparison, state.prev_left_index); + idx_t result_count = MergeJoinComplexBlocks(state.sort_key_type, left_info, right_info, + conditions[0].comparison, state.prev_left_index); if (result_count == 0) { // exhausted this chunk on the right side // move to the next right chunk state.left_position = 0; state.right_position = 0; - state.right_base += rsorted.radix_sorting_data[state.right_chunk_index]->count; + state.right_base += STANDARD_VECTOR_SIZE; state.right_chunk_index++; - if (state.right_chunk_index >= rsorted.radix_sorting_data.size()) { + if (state.right_chunk_index >= rhs_table.BlockCount()) { state.finished = true; } } else { // found matches: extract them + SliceSortedPayload(state.rhs_input, rhs_table, rhs_iterator, state.rhs_chunk_state, right_info.block_idx, + right_info.result, result_count, *state.rhs_scan_state); + chunk.Reset(); - for (idx_t c = 0; c < state.lhs_payload.ColumnCount(); ++c) { - chunk.data[c].Slice(state.lhs_payload.data[c], left_info.result, result_count); + for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); ++col_idx) { + if (col_idx < left_cols) { + chunk.data[col_idx].Slice(state.lhs_payload.data[col_idx], left_info.result, result_count); + } else { + chunk.data[col_idx].Reference(state.rhs_input.data[col_idx - left_cols]); + } } - state.payload_heap_handles.push_back(SliceSortedPayload(chunk, right_info.state, right_info.block_idx, - right_info.result, result_count, left_cols)); chunk.SetCardinality(result_count); auto sel = FlatVector::IncrementalSelectionVector(); @@ -625,13 +629,12 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionConte // If there are more expressions to compute, // split the result chunk into the left and right halves // so we can compute the values for comparison. - chunk.Split(state.rhs_input, left_cols); state.rhs_executor.SetChunk(state.rhs_input); state.rhs_keys.Reset(); auto tail_count = result_count; for (size_t cmp_idx = 1; cmp_idx < conditions.size(); ++cmp_idx) { - Vector left(lhs_table.keys.data[cmp_idx]); + Vector left(state.lhs_local_table->keys.data[cmp_idx]); left.Slice(left_info.result, result_count); auto &right = state.rhs_keys.data[cmp_idx]; @@ -645,7 +648,6 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionConte SelectJoinTail(conditions[cmp_idx].comparison, left, right, sel, tail_count, &state.sel); sel = &state.sel; } - chunk.Fuse(state.rhs_input); if (tail_count < result_count) { result_count = tail_count; @@ -713,54 +715,78 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ExecuteInternal(ExecutionContext //===--------------------------------------------------------------------===// // Source //===--------------------------------------------------------------------===// -class PiecewiseJoinScanState : public GlobalSourceState { +class PiecewiseJoinGlobalScanState : public GlobalSourceState { +public: + explicit PiecewiseJoinGlobalScanState(TupleDataCollection &payload) : payload(payload), right_outer_position(0) { + payload.InitializeScan(parallel_scan); + } + + idx_t Scan(TupleDataLocalScanState &local_scan, DataChunk &chunk) { + lock_guard guard(lock); + const auto result = right_outer_position; + payload.Scan(parallel_scan, local_scan, chunk); + right_outer_position += chunk.size(); + return result; + } + + TupleDataCollection &payload; + public: - explicit PiecewiseJoinScanState(const PhysicalPiecewiseMergeJoin &op) : op(op), right_outer_position(0) { + idx_t MaxThreads() override { + return payload.ChunkCount(); } +private: mutex lock; - const PhysicalPiecewiseMergeJoin &op; - unique_ptr scanner; + TupleDataParallelScanState parallel_scan; idx_t right_outer_position; +}; +class PiecewiseJoinLocalScanState : public LocalSourceState { public: - idx_t MaxThreads() override { - auto &sink = op.sink_state->Cast(); - return sink.Count() / (STANDARD_VECTOR_SIZE * idx_t(10)); + explicit PiecewiseJoinLocalScanState(PiecewiseJoinGlobalScanState &gstate) : rsel(STANDARD_VECTOR_SIZE) { + gstate.payload.InitializeScan(scanner); + gstate.payload.InitializeChunk(rhs_chunk); } + + TupleDataLocalScanState scanner; + DataChunk rhs_chunk; + SelectionVector rsel; }; unique_ptr PhysicalPiecewiseMergeJoin::GetGlobalSourceState(ClientContext &context) const { - return make_uniq(*this); + auto &gsink = sink_state->Cast(); + return make_uniq(*gsink.table->sorted->payload_data); +} + +unique_ptr PhysicalPiecewiseMergeJoin::GetLocalSourceState(ExecutionContext &context, + GlobalSourceState &gstate) const { + return make_uniq(gstate.Cast()); } SourceResultType PhysicalPiecewiseMergeJoin::GetData(ExecutionContext &context, DataChunk &result, - OperatorSourceInput &input) const { + OperatorSourceInput &source) const { D_ASSERT(PropagatesBuildSide(join_type)); // check if we need to scan any unmatched tuples from the RHS for the full/right outer join - auto &sink = sink_state->Cast(); - auto &state = input.global_state.Cast(); - - lock_guard l(state.lock); - if (!state.scanner) { - // Initialize scanner (if not yet initialized) - auto &sort_state = sink.table->global_sort_state; - if (sort_state.sorted_blocks.empty()) { - return SourceResultType::FINISHED; - } - state.scanner = make_uniq(*sort_state.sorted_blocks[0]->payload_data, sort_state); + auto &gsink = sink_state->Cast(); + auto &gsource = source.global_state.Cast(); + + // RHS was empty, so nothing to do? + if (!gsink.table->count) { + return SourceResultType::FINISHED; } // if the LHS is exhausted in a FULL/RIGHT OUTER JOIN, we scan the found_match for any chunks we // still need to output - const auto found_match = sink.table->found_match.get(); + const auto found_match = gsink.table->found_match.get(); - DataChunk rhs_chunk; - rhs_chunk.Initialize(Allocator::Get(context.client), sink.table->global_sort_state.payload_layout.GetTypes()); - SelectionVector rsel(STANDARD_VECTOR_SIZE); + auto &lsource = source.local_state.Cast(); + auto &rhs_chunk = lsource.rhs_chunk; + auto &rsel = lsource.rsel; for (;;) { // Read the next sorted chunk - state.scanner->Scan(rhs_chunk); + rhs_chunk.Reset(); + const auto rhs_pos = gsource.Scan(lsource.scanner, rhs_chunk); const auto count = rhs_chunk.size(); if (count == 0) { @@ -770,11 +796,10 @@ SourceResultType PhysicalPiecewiseMergeJoin::GetData(ExecutionContext &context, idx_t result_count = 0; // figure out which tuples didn't find a match in the RHS for (idx_t i = 0; i < count; i++) { - if (!found_match[state.right_outer_position + i]) { + if (!found_match[rhs_pos + i]) { rsel.set_index(result_count++, i); } } - state.right_outer_position += count; if (result_count > 0) { // if there were any tuples that didn't find a match, output them diff --git a/src/execution/operator/join/physical_range_join.cpp b/src/execution/operator/join/physical_range_join.cpp index 4fefafbd4ea6..f92a2bb889ab 100644 --- a/src/execution/operator/join/physical_range_join.cpp +++ b/src/execution/operator/join/physical_range_join.cpp @@ -1,10 +1,7 @@ #include "duckdb/execution/operator/join/physical_range_join.hpp" -#include "duckdb/common/fast_mem.hpp" -#include "duckdb/common/operator/comparison_operators.hpp" #include "duckdb/common/row_operations/row_operations.hpp" -#include "duckdb/common/sort/comparators.hpp" -#include "duckdb/common/sort/sort.hpp" +#include "duckdb/common/sorting/sort_key.hpp" #include "duckdb/common/types/validity_mask.hpp" #include "duckdb/common/types/vector.hpp" #include "duckdb/common/unordered_map.hpp" @@ -14,15 +11,15 @@ #include "duckdb/parallel/base_pipeline_event.hpp" #include "duckdb/parallel/thread_context.hpp" #include "duckdb/parallel/executor_task.hpp" - -#include +#include "duckdb/planner/expression/bound_reference_expression.hpp" namespace duckdb { -PhysicalRangeJoin::LocalSortedTable::LocalSortedTable(ClientContext &context, const PhysicalRangeJoin &op, +PhysicalRangeJoin::LocalSortedTable::LocalSortedTable(ExecutionContext &context, GlobalSortedTable &global_table, const idx_t child) - : op(op), executor(context), has_null(0), count(0) { + : global_table(global_table), executor(context.client), has_null(0), count(0) { // Initialize order clause expression executor and key DataChunk + const auto &op = global_table.op; vector types; for (const auto &cond : op.conditions) { const auto &expr = child ? cond.right : cond.left; @@ -30,15 +27,19 @@ PhysicalRangeJoin::LocalSortedTable::LocalSortedTable(ClientContext &context, co types.push_back(expr->return_type); } - auto &allocator = Allocator::Get(context); + auto &allocator = Allocator::Get(context.client); keys.Initialize(allocator, types); + + local_sink = global_table.sort->GetLocalSinkState(context); + + // Only sort the primary key + types.resize(1); + const auto &payload_types = op.children[child].get().types; + types.insert(types.end(), payload_types.begin(), payload_types.end()); + sort_chunk.InitializeEmpty(types); } -void PhysicalRangeJoin::LocalSortedTable::Sink(DataChunk &input, GlobalSortState &global_sort_state) { - // Initialize local state (if necessary) - if (!local_sort_state.initialized) { - local_sort_state.Initialize(global_sort_state, global_sort_state.buffer_manager); - } +void PhysicalRangeJoin::LocalSortedTable::Sink(ExecutionContext &context, DataChunk &input) { // Obtain sorting columns keys.Reset(); @@ -47,121 +48,180 @@ void PhysicalRangeJoin::LocalSortedTable::Sink(DataChunk &input, GlobalSortState // Do not operate on primary key directly to avoid modifying the input chunk Vector primary = keys.data[0]; // Count the NULLs so we can exclude them later - has_null += MergeNulls(primary, op.conditions); + has_null += MergeNulls(primary, global_table.op.conditions); count += keys.size(); // Only sort the primary key - DataChunk join_head; - join_head.data.emplace_back(primary); - join_head.SetCardinality(keys.size()); + sort_chunk.data[0].Reference(primary); + for (column_t col_idx = 0; col_idx < input.ColumnCount(); ++col_idx) { + sort_chunk.data[col_idx + 1].Reference(input.data[col_idx]); + } + sort_chunk.SetCardinality(input); // Sink the data into the local sort state - local_sort_state.SinkChunk(join_head, input); + InterruptState interrupt; + OperatorSinkInput sink {*global_table.global_sink, *local_sink, interrupt}; + global_table.sort->Sink(context, sort_chunk, sink); } -PhysicalRangeJoin::GlobalSortedTable::GlobalSortedTable(ClientContext &context, const vector &orders, - RowLayout &payload_layout, const PhysicalOperator &op_p) - : op(op_p), global_sort_state(context, orders, payload_layout), has_null(0), count(0), memory_per_thread(0) { +PhysicalRangeJoin::GlobalSortedTable::GlobalSortedTable(ClientContext &client, + const vector &order_bys, + const vector &payload_types, + const PhysicalRangeJoin &op) + : op(op), has_null(0), count(0), tasks_completed(0) { + + // Set up the sort. We will materialize keys ourselves, so just set up references. + vector orders; + vector input_types; + for (const auto &order_by : order_bys) { + auto order = order_by.Copy(); + const auto type = order.expression->return_type; + input_types.emplace_back(type); + order.expression = make_uniq(type, orders.size()); + orders.emplace_back(std::move(order)); + } + + vector projection_map; + for (const auto &type : payload_types) { + projection_map.emplace_back(input_types.size()); + input_types.emplace_back(type); + } + + sort = make_uniq(client, orders, input_types, projection_map); - // Set external (can be forced with the PRAGMA) - global_sort_state.external = ClientConfig::GetConfig(context).force_external; - memory_per_thread = PhysicalRangeJoin::GetMaxThreadMemory(context); + global_sink = sort->GetGlobalSinkState(client); } -void PhysicalRangeJoin::GlobalSortedTable::Combine(LocalSortedTable <able) { - global_sort_state.AddLocalState(ltable.local_sort_state); +void PhysicalRangeJoin::GlobalSortedTable::Combine(ExecutionContext &context, LocalSortedTable <able) { + InterruptState interrupt; + OperatorSinkCombineInput combine {*global_sink, *ltable.local_sink, interrupt}; + sort->Combine(context, combine); has_null += ltable.has_null; count += ltable.count; } +void PhysicalRangeJoin::GlobalSortedTable::Finalize(ClientContext &client, InterruptState &interrupt) { + OperatorSinkFinalizeInput finalize {*global_sink, interrupt}; + sort->Finalize(client, finalize); +} + void PhysicalRangeJoin::GlobalSortedTable::IntializeMatches() { found_match = make_unsafe_uniq_array_uninitialized(Count()); memset(found_match.get(), 0, sizeof(bool) * Count()); } +void PhysicalRangeJoin::GlobalSortedTable::MaterializeEmpty(ClientContext &client) { + D_ASSERT(!sorted); + sorted = make_uniq(client, *sort, false); +} + void PhysicalRangeJoin::GlobalSortedTable::Print() { - global_sort_state.Print(); + D_ASSERT(sorted); + auto &collection = *sorted->payload_data; + TupleDataScanState scanner; + collection.InitializeScan(scanner); + + DataChunk payload; + collection.InitializeScanChunk(scanner, payload); + + while (collection.Scan(scanner, payload)) { + payload.Print(); + } } -class RangeJoinMergeTask : public ExecutorTask { +//===--------------------------------------------------------------------===// +// RangeJoinMaterializeTask +//===--------------------------------------------------------------------===// +class RangeJoinMaterializeTask : public ExecutorTask { public: using GlobalSortedTable = PhysicalRangeJoin::GlobalSortedTable; public: - RangeJoinMergeTask(shared_ptr event_p, ClientContext &context, GlobalSortedTable &table) - : ExecutorTask(context, std::move(event_p), table.op), context(context), table(table) { + RangeJoinMaterializeTask(Pipeline &pipeline, shared_ptr event, ClientContext &client, + GlobalSortedTable &table, idx_t tasks_scheduled) + : ExecutorTask(client, std::move(event), table.op), pipeline(pipeline), table(table), + tasks_scheduled(tasks_scheduled) { } TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override { - // Initialize iejoin sorted and iterate until done - auto &global_sort_state = table.global_sort_state; - MergeSorter merge_sorter(global_sort_state, BufferManager::GetBufferManager(context)); - merge_sorter.PerformInMergeRound(); - event->FinishTask(); + ExecutionContext execution(pipeline.GetClientContext(), *thread_context, &pipeline); + auto &sort = *table.sort; + auto &sort_global = *table.global_source; + auto sort_local = sort.GetLocalSourceState(execution, sort_global); + InterruptState interrupt((weak_ptr(shared_from_this()))); + OperatorSourceInput input {sort_global, *sort_local, interrupt}; + sort.MaterializeSortedRun(execution, input); + if (++table.tasks_completed == tasks_scheduled) { + table.sorted = sort.GetSortedRun(sort_global); + if (!table.sorted) { + table.MaterializeEmpty(execution.client); + } + } + event->FinishTask(); return TaskExecutionResult::TASK_FINISHED; } string TaskType() const override { - return "RangeJoinMergeTask"; + return "RangeJoinMaterializeTask"; } private: - ClientContext &context; + Pipeline &pipeline; GlobalSortedTable &table; + const idx_t tasks_scheduled; }; -class RangeJoinMergeEvent : public BasePipelineEvent { +//===--------------------------------------------------------------------===// +// RangeJoinMaterializeEvent +//===--------------------------------------------------------------------===// +class RangeJoinMaterializeEvent : public BasePipelineEvent { public: using GlobalSortedTable = PhysicalRangeJoin::GlobalSortedTable; public: - RangeJoinMergeEvent(GlobalSortedTable &table_p, Pipeline &pipeline_p) - : BasePipelineEvent(pipeline_p), table(table_p) { + RangeJoinMaterializeEvent(GlobalSortedTable &table, Pipeline &pipeline) + : BasePipelineEvent(pipeline), table(table) { } GlobalSortedTable &table; public: void Schedule() override { - auto &context = pipeline->GetClientContext(); + auto &client = pipeline->GetClientContext(); - // Schedule tasks equal to the number of threads, which will each merge multiple partitions - auto &ts = TaskScheduler::GetScheduler(context); + // Schedule as many tasks as the sort will allow + auto &ts = TaskScheduler::GetScheduler(client); auto num_threads = NumericCast(ts.NumberOfThreads()); - - vector> iejoin_tasks; - for (idx_t tnum = 0; tnum < num_threads; tnum++) { - iejoin_tasks.push_back(make_uniq(shared_from_this(), context, table)); + vector> tasks; + + auto &sort = *table.sort; + auto &global_sink = *table.global_sink; + table.global_source = sort.GetGlobalSourceState(client, global_sink); + const auto tasks_scheduled = MinValue(num_threads, table.global_source->MaxThreads()); + for (idx_t tnum = 0; tnum < tasks_scheduled; ++tnum) { + tasks.push_back( + make_uniq(*pipeline, shared_from_this(), client, table, tasks_scheduled)); } - SetTasks(std::move(iejoin_tasks)); - } - - void FinishEvent() override { - auto &global_sort_state = table.global_sort_state; - global_sort_state.CompleteMergeRound(true); - if (global_sort_state.sorted_blocks.size() > 1) { - // Multiple blocks remaining: Schedule the next round - table.ScheduleMergeTasks(*pipeline, *this); - } + SetTasks(std::move(tasks)); } }; -void PhysicalRangeJoin::GlobalSortedTable::ScheduleMergeTasks(Pipeline &pipeline, Event &event) { - // Initialize global sort state for a round of merging - global_sort_state.InitializeMergeRound(); - auto new_event = make_shared_ptr(*this, pipeline); - event.InsertEvent(std::move(new_event)); +void PhysicalRangeJoin::GlobalSortedTable::Materialize(Pipeline &pipeline, Event &event) { + // Schedule all the sorts for maximum thread utilisation + auto sort_event = make_shared_ptr(*this, pipeline); + event.InsertEvent(std::move(sort_event)); } -void PhysicalRangeJoin::GlobalSortedTable::Finalize(Pipeline &pipeline, Event &event) { - // Prepare for merge sort phase - global_sort_state.PrepareMergePhase(); - - // Start the merge phase or finish if a merge is not necessary - if (global_sort_state.sorted_blocks.size() > 1) { - ScheduleMergeTasks(pipeline, event); +void PhysicalRangeJoin::GlobalSortedTable::Materialize(ExecutionContext &context, InterruptState &interrupt) { + global_source = sort->GetGlobalSourceState(context.client, *global_sink); + auto local_source = sort->GetLocalSourceState(context, *global_source); + OperatorSourceInput source {*global_source, *local_source, interrupt}; + sort->MaterializeSortedRun(context, source); + sorted = sort->GetSortedRun(*global_source); + if (!sorted) { + MaterializeEmpty(context.client); } } @@ -336,56 +396,75 @@ void PhysicalRangeJoin::ProjectResult(DataChunk &chunk, DataChunk &result) const result.SetCardinality(chunk); } -BufferHandle PhysicalRangeJoin::SliceSortedPayload(DataChunk &payload, GlobalSortState &state, const idx_t block_idx, - const SelectionVector &result, const idx_t result_count, - const idx_t left_cols) { - // There should only be one sorted block if they have been sorted - D_ASSERT(state.sorted_blocks.size() == 1); - SBScanState read_state(state.buffer_manager, state); - read_state.sb = state.sorted_blocks[0].get(); - auto &sorted_data = *read_state.sb->payload_data; - - read_state.SetIndices(block_idx, 0); - read_state.PinData(sorted_data); - const auto data_ptr = read_state.DataPtr(sorted_data); - data_ptr_t heap_ptr = nullptr; - - // Set up a batch of pointers to scan data from - Vector addresses(LogicalType::POINTER, result_count); - auto data_pointers = FlatVector::GetData(addresses); - - // Set up the data pointers for the values that are actually referenced - const idx_t &row_width = sorted_data.layout.GetRowWidth(); - - auto prev_idx = result.get_index(0); - SelectionVector gsel(result_count); - idx_t addr_count = 0; - gsel.set_index(0, addr_count); - data_pointers[addr_count] = data_ptr + prev_idx * row_width; - for (idx_t i = 1; i < result_count; ++i) { - const auto row_idx = result.get_index(i); - if (row_idx != prev_idx) { - data_pointers[++addr_count] = data_ptr + row_idx * row_width; - prev_idx = row_idx; - } - gsel.set_index(i, addr_count); +template +static void TemplatedSliceSortedPayload(DataChunk &chunk, const SortedRun &sorted_run, + ExternalBlockIteratorState &state, Vector &sort_key_pointers, + SortedRunScanState &scan_state, const idx_t chunk_idx, SelectionVector &result, + const idx_t result_count) { + using SORT_KEY = SortKey; + using BLOCK_ITERATOR = block_iterator_t; + BLOCK_ITERATOR itr(state, chunk_idx, 0); + + const auto sort_keys = FlatVector::GetData(sort_key_pointers); + for (idx_t i = 0; i < result_count; ++i) { + const auto idx = state.GetIndex(chunk_idx, result.get_index(i)); + sort_keys[i] = &itr[idx]; } - ++addr_count; - // Unswizzle the offsets back to pointers (if needed) - if (!sorted_data.layout.AllConstant() && state.external) { - heap_ptr = read_state.payload_heap_handle.Ptr(); - } + // Scan + chunk.Reset(); + scan_state.Scan(sorted_run, sort_key_pointers, result_count, chunk); +} - // Deserialize the payload data - auto sel = FlatVector::IncrementalSelectionVector(); - for (idx_t col_no = 0; col_no < sorted_data.layout.ColumnCount(); col_no++) { - auto &col = payload.data[left_cols + col_no]; - RowOperations::Gather(addresses, *sel, col, *sel, addr_count, sorted_data.layout, col_no, 0, heap_ptr); - col.Slice(gsel, result_count); +void PhysicalRangeJoin::SliceSortedPayload(DataChunk &chunk, GlobalSortedTable &table, + ExternalBlockIteratorState &state, TupleDataChunkState &chunk_state, + const idx_t chunk_idx, SelectionVector &result, const idx_t result_count, + SortedRunScanState &scan_state) { + + auto &sorted = *table.sorted; + auto &sort_keys = chunk_state.row_locations; + const auto sort_key_type = table.GetSortKeyType(); + + switch (sort_key_type) { + case SortKeyType::NO_PAYLOAD_FIXED_8: + TemplatedSliceSortedPayload(chunk, sorted, state, sort_keys, scan_state, + chunk_idx, result, result_count); + break; + case SortKeyType::NO_PAYLOAD_FIXED_16: + TemplatedSliceSortedPayload(chunk, sorted, state, sort_keys, scan_state, + chunk_idx, result, result_count); + break; + case SortKeyType::NO_PAYLOAD_FIXED_24: + TemplatedSliceSortedPayload(chunk, sorted, state, sort_keys, scan_state, + chunk_idx, result, result_count); + break; + case SortKeyType::NO_PAYLOAD_FIXED_32: + TemplatedSliceSortedPayload(chunk, sorted, state, sort_keys, scan_state, + chunk_idx, result, result_count); + break; + case SortKeyType::NO_PAYLOAD_VARIABLE_32: + TemplatedSliceSortedPayload(chunk, sorted, state, sort_keys, scan_state, + chunk_idx, result, result_count); + break; + case SortKeyType::PAYLOAD_FIXED_16: + TemplatedSliceSortedPayload(chunk, sorted, state, sort_keys, scan_state, + chunk_idx, result, result_count); + break; + case SortKeyType::PAYLOAD_FIXED_24: + TemplatedSliceSortedPayload(chunk, sorted, state, sort_keys, scan_state, + chunk_idx, result, result_count); + break; + case SortKeyType::PAYLOAD_FIXED_32: + TemplatedSliceSortedPayload(chunk, sorted, state, sort_keys, scan_state, + chunk_idx, result, result_count); + break; + case SortKeyType::PAYLOAD_VARIABLE_32: + TemplatedSliceSortedPayload(chunk, sorted, state, sort_keys, scan_state, + chunk_idx, result, result_count); + break; + default: + throw NotImplementedException("MergeJoinSimpleBlocks for %s", EnumUtil::ToString(sort_key_type)); } - - return std::move(read_state.payload_heap_handle); } idx_t PhysicalRangeJoin::SelectJoinTail(const ExpressionType &condition, Vector &left, Vector &right, diff --git a/src/execution/operator/order/physical_top_n.cpp b/src/execution/operator/order/physical_top_n.cpp index ec082601c304..f1c5ab8e4794 100644 --- a/src/execution/operator/order/physical_top_n.cpp +++ b/src/execution/operator/order/physical_top_n.cpp @@ -1,6 +1,7 @@ #include "duckdb/execution/operator/order/physical_top_n.hpp" #include "duckdb/common/assert.hpp" +#include "duckdb/common/arena_containers.hpp" #include "duckdb/execution/expression_executor.hpp" #include "duckdb/function/create_sort_key.hpp" #include "duckdb/storage/data_table.hpp" @@ -85,7 +86,8 @@ class TopNHeap { Allocator &allocator; BufferManager &buffer_manager; - unsafe_vector heap; + ArenaAllocator arena_allocator; + unsafe_arena_vector heap; const vector &payload_types; const vector &orders; vector modifiers; @@ -162,10 +164,11 @@ class TopNHeap { //===--------------------------------------------------------------------===// TopNHeap::TopNHeap(ClientContext &context, Allocator &allocator, const vector &payload_types_p, const vector &orders_p, idx_t limit, idx_t offset) - : allocator(allocator), buffer_manager(BufferManager::GetBufferManager(context)), payload_types(payload_types_p), - orders(orders_p), limit(limit), offset(offset), heap_size(limit + offset), executor(context), - sort_key_heap(allocator), matching_sel(STANDARD_VECTOR_SIZE), final_sel(STANDARD_VECTOR_SIZE), - true_sel(STANDARD_VECTOR_SIZE), false_sel(STANDARD_VECTOR_SIZE), new_remaining_sel(STANDARD_VECTOR_SIZE) { + : allocator(allocator), buffer_manager(BufferManager::GetBufferManager(context)), arena_allocator(allocator), + heap(arena_allocator), payload_types(payload_types_p), orders(orders_p), limit(limit), offset(offset), + heap_size(limit + offset), executor(context), sort_key_heap(allocator), matching_sel(STANDARD_VECTOR_SIZE), + final_sel(STANDARD_VECTOR_SIZE), true_sel(STANDARD_VECTOR_SIZE), false_sel(STANDARD_VECTOR_SIZE), + new_remaining_sel(STANDARD_VECTOR_SIZE) { // initialize the executor and the sort_chunk vector sort_types; for (auto &order : orders) { diff --git a/src/execution/operator/persistent/physical_batch_insert.cpp b/src/execution/operator/persistent/physical_batch_insert.cpp index d3fa408bb925..025af5ba1db1 100644 --- a/src/execution/operator/persistent/physical_batch_insert.cpp +++ b/src/execution/operator/persistent/physical_batch_insert.cpp @@ -70,7 +70,8 @@ class CollectionMerger { } auto result_collection_index = collection_indexes[0]; - auto &result_collection = data_table.GetOptimisticCollection(context, result_collection_index); + auto &optimistic_collection = data_table.GetOptimisticCollection(context, result_collection_index); + auto &result_collection = *optimistic_collection.collection; if (collection_indexes.size() > 1) { // Merge all collections into one result collection. @@ -89,7 +90,7 @@ class CollectionMerger { auto &collection = data_table.GetOptimisticCollection(context, collection_indexes[i]); TableScanState scan_state; scan_state.Initialize(column_ids); - collection.InitializeScan(scan_state.local_state, column_ids, nullptr); + collection.collection->InitializeScan(context, scan_state.local_state, column_ids, nullptr); while (true) { scan_chunk.Reset(); @@ -99,15 +100,15 @@ class CollectionMerger { } auto new_row_group = result_collection.Append(scan_chunk, append_state); if (new_row_group) { - writer.WriteNewRowGroup(result_collection); + writer.WriteNewRowGroup(optimistic_collection); } } data_table.ResetOptimisticCollection(context, collection_indexes[i]); } result_collection.FinalizeAppend(TransactionData(0, 0), append_state); - writer.WriteLastRowGroup(result_collection); + writer.WriteLastRowGroup(optimistic_collection); } else if (batch_type == RowGroupBatchType::NOT_FLUSHED) { - writer.WriteLastRowGroup(result_collection); + writer.WriteLastRowGroup(optimistic_collection); } collection_indexes.clear(); @@ -116,12 +117,12 @@ class CollectionMerger { }; struct RowGroupBatchEntry { - RowGroupBatchEntry(RowGroupCollection &collection, const idx_t batch_idx, const PhysicalIndex collection_index, - const RowGroupBatchType type) - : batch_idx(batch_idx), total_rows(collection.GetTotalRows()), unflushed_memory(0), + RowGroupBatchEntry(OptimisticWriteCollection &collection, const idx_t batch_idx, + const PhysicalIndex collection_index, const RowGroupBatchType type) + : batch_idx(batch_idx), total_rows(collection.collection->GetTotalRows()), unflushed_memory(0), collection_index(collection_index), type(type) { if (type == RowGroupBatchType::NOT_FLUSHED) { - unflushed_memory = collection.GetAllocationSize(); + unflushed_memory = collection.collection->GetAllocationSize(); } } @@ -193,14 +194,13 @@ class BatchInsertLocalState : public LocalSinkState { void CreateNewCollection(ClientContext &context, DuckTableEntry &table_entry, const vector &insert_types) { - auto table_info = table_entry.GetStorage().GetDataTableInfo(); - auto &io_manager = TableIOManager::Get(table_entry.GetStorage()); - - // Create the local row group collection. - auto max_row_id = NumericCast(MAX_ROW_ID); - auto collection = make_uniq(std::move(table_info), io_manager, insert_types, max_row_id); - collection->InitializeEmpty(); - collection->InitializeAppend(current_append_state); + if (!optimistic_writer) { + optimistic_writer = make_uniq(context, table_entry.GetStorage()); + } + auto collection = optimistic_writer->CreateCollection(table_entry.GetStorage(), insert_types); + auto &row_collection = *collection->collection; + row_collection.InitializeEmpty(); + row_collection.InitializeAppend(current_append_state); auto &data_table = table_entry.GetStorage(); collection_index = data_table.CreateOptimisticCollection(context, std::move(collection)); @@ -375,16 +375,17 @@ void BatchInsertGlobalState::AddCollection(ClientContext &context, const idx_t b throw InternalException("Batch index of the added collection (%llu) is smaller than the min batch index (%llu)", batch_index, min_batch_index); } - auto &collection = table.GetStorage().GetOptimisticCollection(context, collection_index); + auto &optimistic_collection = table.GetStorage().GetOptimisticCollection(context, collection_index); + auto &collection = *optimistic_collection.collection; auto new_count = collection.GetTotalRows(); auto batch_type = new_count < row_group_size ? RowGroupBatchType::NOT_FLUSHED : RowGroupBatchType::FLUSHED; if (batch_type == RowGroupBatchType::FLUSHED && writer) { - writer->WriteLastRowGroup(collection); + writer->WriteLastRowGroup(optimistic_collection); } lock_guard l(lock); insert_count += new_count; // add the collection to the batch index - RowGroupBatchEntry new_entry(collection, batch_index, collection_index, batch_type); + RowGroupBatchEntry new_entry(optimistic_collection, batch_index, collection_index, batch_type); if (batch_type == RowGroupBatchType::NOT_FLUSHED) { memory_manager.IncreaseUnflushedMemory(new_entry.unflushed_memory); } @@ -466,7 +467,9 @@ SinkNextBatchType PhysicalBatchInsert::NextBatch(ExecutionContext &context, Oper } // batch index has changed: move the old collection to the global state and create a new collection TransactionData tdata(0, 0); - auto &collection = gstate.table.GetStorage().GetOptimisticCollection(context.client, lstate.collection_index); + auto &optimistic_collection = + gstate.table.GetStorage().GetOptimisticCollection(context.client, lstate.collection_index); + auto &collection = *optimistic_collection.collection; collection.FinalizeAppend(tdata, lstate.current_append_state); gstate.AddCollection(context.client, lstate.current_index, lstate.partition_info.min_batch_index.GetIndex(), lstate.collection_index, lstate.optimistic_writer); @@ -526,9 +529,6 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &i lock_guard l(gstate.lock); // no collection yet: create a new one lstate.CreateNewCollection(context.client, table, insert_types); - if (!lstate.optimistic_writer) { - lstate.optimistic_writer = make_uniq(context.client, table.GetStorage()); - } } if (lstate.current_index != batch_index) { @@ -545,11 +545,12 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &i storage.VerifyAppendConstraints(*lstate.constraint_state, context.client, insert_chunk, local_table_storage, nullptr); - auto &collection = table.GetStorage().GetOptimisticCollection(context.client, lstate.collection_index); + auto &optimistic_collection = table.GetStorage().GetOptimisticCollection(context.client, lstate.collection_index); + auto &collection = *optimistic_collection.collection; auto new_row_group = collection.Append(insert_chunk, lstate.current_append_state); if (new_row_group) { // we have already written to disk - flush the next row group as well - lstate.optimistic_writer->WriteNewRowGroup(collection); + lstate.optimistic_writer->WriteNewRowGroup(optimistic_collection); } return SinkResultType::NEED_MORE_INPUT; } @@ -569,7 +570,9 @@ SinkCombineResultType PhysicalBatchInsert::Combine(ExecutionContext &context, Op if (lstate.collection_index.IsValid()) { TransactionData tdata(0, 0); - auto &collection = gstate.table.GetStorage().GetOptimisticCollection(context.client, lstate.collection_index); + auto &optimistic_collection = + gstate.table.GetStorage().GetOptimisticCollection(context.client, lstate.collection_index); + auto &collection = *optimistic_collection.collection; collection.FinalizeAppend(tdata, lstate.current_append_state); if (collection.GetTotalRows() > 0) { auto batch_index = lstate.partition_info.min_batch_index.GetIndex(); @@ -667,7 +670,8 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event, } memory_manager.ReduceUnflushedMemory(entry.unflushed_memory); - auto &collection = data_table.GetOptimisticCollection(context, entry.collection_index); + auto &optimistic_collection = data_table.GetOptimisticCollection(context, entry.collection_index); + auto &collection = *optimistic_collection.collection; collection.Scan(transaction, [&](DataChunk &insert_chunk) { data_table.LocalAppend(append_state, context, insert_chunk, false); return true; diff --git a/src/execution/operator/persistent/physical_insert.cpp b/src/execution/operator/persistent/physical_insert.cpp index 1875d86c1290..8c152c2b40cb 100644 --- a/src/execution/operator/persistent/physical_insert.cpp +++ b/src/execution/operator/persistent/physical_insert.cpp @@ -651,27 +651,26 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &insert D_ASSERT(!return_chunk); auto &data_table = gstate.table.GetStorage(); if (!lstate.collection_index.IsValid()) { - auto table_info = storage.GetDataTableInfo(); - auto &io_manager = TableIOManager::Get(table.GetStorage()); - - // Create the local row group collection. - auto max_row_id = NumericCast(MAX_ROW_ID); - auto collection = make_uniq(std::move(table_info), io_manager, insert_types, max_row_id); - collection->InitializeEmpty(); - collection->InitializeAppend(lstate.local_append_state); - lock_guard l(gstate.lock); lstate.optimistic_writer = make_uniq(context.client, data_table); - lstate.collection_index = data_table.CreateOptimisticCollection(context.client, std::move(collection)); + // Create the local row group collection. + auto optimistic_collection = lstate.optimistic_writer->CreateCollection(storage, insert_types); + auto &collection = *optimistic_collection->collection; + collection.InitializeEmpty(); + collection.InitializeAppend(lstate.local_append_state); + + lstate.collection_index = + data_table.CreateOptimisticCollection(context.client, std::move(optimistic_collection)); } OnConflictHandling(table, context, gstate, lstate, insert_chunk); D_ASSERT(action_type != OnConflictAction::UPDATE); - auto &collection = data_table.GetOptimisticCollection(context.client, lstate.collection_index); + auto &optimistic_collection = data_table.GetOptimisticCollection(context.client, lstate.collection_index); + auto &collection = *optimistic_collection.collection; auto new_row_group = collection.Append(insert_chunk, lstate.local_append_state); if (new_row_group) { - lstate.optimistic_writer->WriteNewRowGroup(collection); + lstate.optimistic_writer->WriteNewRowGroup(optimistic_collection); } return SinkResultType::NEED_MORE_INPUT; } @@ -694,7 +693,8 @@ SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, Operato // parallel append: finalize the append TransactionData tdata(0, 0); auto &data_table = gstate.table.GetStorage(); - auto &collection = data_table.GetOptimisticCollection(context.client, lstate.collection_index); + auto &optimistic_collection = data_table.GetOptimisticCollection(context.client, lstate.collection_index); + auto &collection = *optimistic_collection.collection; collection.FinalizeAppend(tdata, lstate.local_append_state); auto append_count = collection.GetTotalRows(); @@ -713,9 +713,9 @@ SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, Operato storage.FinalizeLocalAppend(append_state); } else { // we have written rows to disk optimistically - merge directly into the transaction-local storage - lstate.optimistic_writer->WriteLastRowGroup(collection); + lstate.optimistic_writer->WriteLastRowGroup(optimistic_collection); lstate.optimistic_writer->FinalFlush(); - gstate.table.GetStorage().LocalMerge(context.client, collection); + gstate.table.GetStorage().LocalMerge(context.client, optimistic_collection); auto &optimistic_writer = gstate.table.GetStorage().GetOptimisticWriter(context.client); optimistic_writer.Merge(*lstate.optimistic_writer); } diff --git a/src/execution/operator/schema/physical_alter.cpp b/src/execution/operator/schema/physical_alter.cpp index 0d463e5ad22e..957a762c4628 100644 --- a/src/execution/operator/schema/physical_alter.cpp +++ b/src/execution/operator/schema/physical_alter.cpp @@ -1,5 +1,6 @@ #include "duckdb/execution/operator/schema/physical_alter.hpp" #include "duckdb/parser/parsed_data/alter_table_info.hpp" +#include "duckdb/parser/parsed_data/alter_database_info.hpp" #include "duckdb/main/database_manager.hpp" #include "duckdb/catalog/catalog.hpp" @@ -9,8 +10,14 @@ namespace duckdb { // Source //===--------------------------------------------------------------------===// SourceResultType PhysicalAlter::GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const { - auto &catalog = Catalog::GetCatalog(context.client, info->catalog); - catalog.Alter(context.client, *info); + if (info->type == AlterType::ALTER_DATABASE) { + auto &db_info = info->Cast(); + auto &db_manager = DatabaseManager::Get(context.client); + db_manager.Alter(context.client, db_info); + } else { + auto &catalog = Catalog::GetCatalog(context.client, info->catalog); + catalog.Alter(context.client, *info); + } return SourceResultType::FINISHED; } diff --git a/src/execution/operator/schema/physical_attach.cpp b/src/execution/operator/schema/physical_attach.cpp index d76cd1165181..48e687703b16 100644 --- a/src/execution/operator/schema/physical_attach.cpp +++ b/src/execution/operator/schema/physical_attach.cpp @@ -36,7 +36,7 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c if (info->on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT || info->on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { // constant-time lookup in the catalog for the db name - auto existing_db = db_manager.GetDatabase(context.client, name); + auto existing_db = db_manager.GetDatabase(name); if (existing_db) { if ((existing_db->IsReadOnly() && options.access_mode == AccessMode::READ_WRITE) || (!existing_db->IsReadOnly() && options.access_mode == AccessMode::READ_ONLY)) { @@ -51,10 +51,8 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c existing_db->GetCatalog().SetDefaultTable(options.default_table.schema, options.default_table.name); } if (info->on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { - // same path, name and type, DB does not need replacing - auto const db_type = options.db_type.empty() ? "duckdb" : options.db_type; - if (existing_db->GetCatalog().GetDBPath() == path && - existing_db->GetCatalog().GetCatalogType() == db_type) { + // allow custom catalogs to override this behavior + if (!existing_db->GetCatalog().HasConflictingAttachOptions(path, options)) { return SourceResultType::FINISHED; } } else { @@ -63,18 +61,7 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c } } - // Get the database type and attach the database. - db_manager.GetDatabaseType(context.client, *info, config, options); - auto attached_db = db_manager.AttachDatabase(context.client, *info, options); - - //! Initialize the database. - attached_db->Initialize(context.client); - if (!options.default_table.name.empty()) { - attached_db->GetCatalog().SetDefaultTable(options.default_table.schema, options.default_table.name); - } - attached_db->FinalizeLoad(context.client); - - db_manager.FinalizeAttach(context.client, *info, std::move(attached_db)); + db_manager.AttachDatabase(context.client, *info, options); return SourceResultType::FINISHED; } diff --git a/src/execution/operator/set/physical_union.cpp b/src/execution/operator/set/physical_union.cpp index a0846ef333f2..e43117c929e1 100644 --- a/src/execution/operator/set/physical_union.cpp +++ b/src/execution/operator/set/physical_union.cpp @@ -6,12 +6,14 @@ namespace duckdb { -PhysicalUnion::PhysicalUnion(PhysicalPlan &physical_plan, vector types, PhysicalOperator &top, - PhysicalOperator &bottom, idx_t estimated_cardinality, bool allow_out_of_order) - : PhysicalOperator(physical_plan, PhysicalOperatorType::UNION, std::move(types), estimated_cardinality), +PhysicalUnion::PhysicalUnion(PhysicalPlan &physical_plan, vector types_p, + const ArenaLinkedList> &children_p, + idx_t estimated_cardinality, bool allow_out_of_order) + : PhysicalOperator(physical_plan, PhysicalOperatorType::UNION, std::move(types_p), estimated_cardinality), allow_out_of_order(allow_out_of_order) { - children.push_back(top); - children.push_back(bottom); + for (auto &child : children_p) { + children.push_back(child); + } } //===--------------------------------------------------------------------===// @@ -56,41 +58,46 @@ void PhysicalUnion::BuildPipelines(Pipeline ¤t, MetaPipeline &meta_pipelin } } - // create a union pipeline that has identical dependencies to 'current' - auto &union_pipeline = meta_pipeline.CreateUnionPipeline(current, order_matters); - + // create union pipelines that has identical dependencies to 'current' + vector> union_pipelines; + for (idx_t i = 0; i + 1 < children.size(); i++) { + auto &union_pipeline = meta_pipeline.CreateUnionPipeline(current, order_matters); + union_pipelines.push_back(union_pipeline); + } // continue with the current pipeline children[0].get().BuildPipelines(current, meta_pipeline); - - vector> dependencies; - optional_ptr last_child_ptr; - // users commonly UNION ALL together a bunch of cheap scan pipelines (e.g., instead of a multi file list) - // in these cases, we don't want to avoid breadth-first plan evaluation, - // as it doesn't pose a threat to memory usage (it's just a bunch of straight scans) - const auto can_saturate_threads = - ContainsSink(children[0]) && children[0].get().CanSaturateThreads(current.GetClientContext()); - if (order_matters || can_saturate_threads) { - // we add dependencies if order matters: union_pipeline comes after all pipelines created by building current - dependencies = meta_pipeline.AddDependenciesFrom(union_pipeline, union_pipeline, false); - // we also add dependencies if the LHS child can saturate all available threads - // in that case, we recursively make all RHS children depend on the LHS. - // This prevents breadth-first plan evaluation - if (can_saturate_threads) { - last_child_ptr = meta_pipeline.GetLastChild(); + bool can_saturate_threads = + ContainsSink(children[0].get()) && children[0].get().CanSaturateThreads(current.GetClientContext()); + for (idx_t i = 1; i < children.size(); i++) { + auto &union_pipeline = union_pipelines[children.size() - i - 1].get(); + vector> dependencies; + optional_ptr last_child_ptr; + if (ContainsSink(children[i - 1].get()) && + children[i - 1].get().CanSaturateThreads(current.GetClientContext())) { + can_saturate_threads = true; } - } - - // build the union pipeline - children[1].get().BuildPipelines(union_pipeline, meta_pipeline); + if (order_matters || can_saturate_threads) { + // we add dependencies if order matters: union_pipeline comes after all pipelines created by building + // current + dependencies = meta_pipeline.AddDependenciesFrom(union_pipeline, union_pipeline, false); + // we also add dependencies if the LHS child can saturate all available threads + // in that case, we recursively make all RHS children depend on the LHS. + // This prevents breadth-first plan evaluation + if (can_saturate_threads) { + last_child_ptr = meta_pipeline.GetLastChild(); + } + } + // build the union pipeline + children[i].get().BuildPipelines(union_pipeline, meta_pipeline); - if (last_child_ptr) { - // the pointer was set, set up the dependencies - meta_pipeline.AddRecursiveDependencies(dependencies, *last_child_ptr); + if (last_child_ptr) { + // the pointer was set, set up the dependencies + meta_pipeline.AddRecursiveDependencies(dependencies, *last_child_ptr); + } + // Assign proper batch index to the union pipeline + // This needs to happen after the pipelines have been built because unions can be nested + meta_pipeline.AssignNextBatchIndex(union_pipeline); } - - // Assign proper batch index to the union pipeline - // This needs to happen after the pipelines have been built because unions can be nested - meta_pipeline.AssignNextBatchIndex(union_pipeline); } vector> PhysicalUnion::GetSources() const { diff --git a/src/execution/physical_plan/plan_asof_join.cpp b/src/execution/physical_plan/plan_asof_join.cpp index 5759583c58d9..11ac71492716 100644 --- a/src/execution/physical_plan/plan_asof_join.cpp +++ b/src/execution/physical_plan/plan_asof_join.cpp @@ -13,6 +13,7 @@ #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" #include "duckdb/planner/expression/bound_window_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" #include "duckdb/main/settings.hpp" namespace duckdb { @@ -27,7 +28,7 @@ PhysicalPlanGenerator::PlanAsOfLoopJoin(LogicalComparisonJoin &op, PhysicalOpera // // ∏ * \ pk // | - // Γ pk;first(P),arg_xxx(B,inequality) + // Γ pk;first(P),arg_xxx_null(B,inequality) // | // ∏ *,inequality // | @@ -43,10 +44,9 @@ PhysicalPlanGenerator::PlanAsOfLoopJoin(LogicalComparisonJoin &op, PhysicalOpera const auto &probe_types = op.children[0]->types; join_op.types.insert(join_op.types.end(), probe_types.begin(), probe_types.end()); - // TODO: We can't handle predicates right now because we would have to remap column references. - if (op.predicate) { - return nullptr; - } + // Project pk + LogicalType pk_type = LogicalType::BIGINT; + join_op.types.emplace_back(pk_type); // Fill in the projection maps to simplify the code below // Since NLJ doesn't support projection, but ASOF does, @@ -65,9 +65,25 @@ PhysicalPlanGenerator::PlanAsOfLoopJoin(LogicalComparisonJoin &op, PhysicalOpera } } - // Project pk - LogicalType pk_type = LogicalType::BIGINT; - join_op.types.emplace_back(pk_type); + // Remap predicate column references. + if (op.predicate) { + vector swap_projection_map; + const auto rhs_width = op.children[1]->types.size(); + for (const auto &l : join_op.right_projection_map) { + swap_projection_map.emplace_back(l + rhs_width); + } + for (const auto &r : join_op.left_projection_map) { + swap_projection_map.emplace_back(r); + } + join_op.predicate = op.predicate->Copy(); + ExpressionIterator::EnumerateExpression(join_op.predicate, [&](Expression &child) { + if (child.GetExpressionClass() == ExpressionClass::BOUND_REF) { + auto &col_idx = child.Cast().index; + const auto new_idx = swap_projection_map[col_idx]; + col_idx = new_idx; + } + }); + } auto binder = Binder::CreateBinder(context); FunctionBinder function_binder(*binder); @@ -88,13 +104,13 @@ PhysicalPlanGenerator::PlanAsOfLoopJoin(LogicalComparisonJoin &op, PhysicalOpera case ExpressionType::COMPARE_GREATERTHAN: D_ASSERT(asof_idx == op.conditions.size()); asof_idx = i; - arg_min_max = "arg_max"; + arg_min_max = "arg_max_null"; break; case ExpressionType::COMPARE_LESSTHANOREQUALTO: case ExpressionType::COMPARE_LESSTHAN: D_ASSERT(asof_idx == op.conditions.size()); asof_idx = i; - arg_min_max = "arg_min"; + arg_min_max = "arg_min_null"; break; case ExpressionType::COMPARE_EQUAL: case ExpressionType::COMPARE_NOTEQUAL: @@ -208,7 +224,7 @@ PhysicalPlanGenerator::PlanAsOfLoopJoin(LogicalComparisonJoin &op, PhysicalOpera auto window_types = probe.GetTypes(); window_types.emplace_back(pk_type); - idx_t probe_cardinality = op.children[0]->EstimateCardinality(context); + const auto probe_cardinality = op.EstimateCardinality(context); auto &window = Make(window_types, std::move(window_select), probe_cardinality); window.children.emplace_back(probe); @@ -275,10 +291,12 @@ PhysicalOperator &PhysicalPlanGenerator::PlanAsOfJoin(LogicalComparisonJoin &op) } D_ASSERT(asof_idx < op.conditions.size()); - bool force_asof_join = DBConfig::GetSetting(context); - if (!force_asof_join) { - idx_t asof_join_threshold = DBConfig::GetSetting(context); - if (op.children[0]->has_estimated_cardinality && lhs_cardinality < asof_join_threshold) { + // If there is a non-comparison predicate, we have to use NLJ. + const bool has_predicate = op.predicate.get(); + const bool force_asof_join = DBConfig::GetSetting(context); + if (!force_asof_join || has_predicate) { + const idx_t asof_join_threshold = DBConfig::GetSetting(context); + if (has_predicate || (op.children[0]->has_estimated_cardinality && lhs_cardinality < asof_join_threshold)) { auto result = PlanAsOfLoopJoin(op, left, right); if (result) { return *result; diff --git a/src/execution/physical_plan/plan_set_operation.cpp b/src/execution/physical_plan/plan_set_operation.cpp index aa274f16de12..2d7427861c6d 100644 --- a/src/execution/physical_plan/plan_set_operation.cpp +++ b/src/execution/physical_plan/plan_set_operation.cpp @@ -32,23 +32,29 @@ static JoinCondition CreateNotDistinctComparison(const LogicalType &type, idx_t } PhysicalOperator &PhysicalPlanGenerator::CreatePlan(LogicalSetOperation &op) { - D_ASSERT(op.children.size() == 2); - - reference left = CreatePlan(*op.children[0]); - reference right = CreatePlan(*op.children[1]); - - if (left.get().GetTypes() != right.get().GetTypes()) { - throw InvalidInputException("Type mismatch for SET OPERATION"); + ArenaLinkedList> children(physical_plan->ArenaRef()); + for (auto &child : op.children) { + children.push_back(CreatePlan(*child)); + } + for (idx_t i = 1; i < children.size(); i++) { + if (children[i].get().GetTypes() != children[0].get().GetTypes()) { + throw InvalidInputException("Type mismatch for SET OPERATION"); + } } optional_ptr result; switch (op.type) { case LogicalOperatorType::LOGICAL_UNION: // UNION - result = Make(op.types, left, right, op.estimated_cardinality, op.allow_out_of_order); + result = Make(op.types, std::move(children), op.estimated_cardinality, op.allow_out_of_order); break; case LogicalOperatorType::LOGICAL_EXCEPT: case LogicalOperatorType::LOGICAL_INTERSECT: { + if (children.size() != 2) { + throw InternalException("EXCEPT / INTERSECT must have exactly two children"); + } + auto &left = children[0]; + auto &right = children[1]; auto &types = left.get().GetTypes(); vector conditions; // create equality condition for all columns diff --git a/src/function/CMakeLists.txt b/src/function/CMakeLists.txt index f9ee106420cb..2837e5c97c08 100644 --- a/src/function/CMakeLists.txt +++ b/src/function/CMakeLists.txt @@ -29,7 +29,8 @@ add_library_unity( table_macro_function.cpp scalar_function.cpp table_function.cpp - udf_function.cpp) + udf_function.cpp + copy_blob.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/function/aggregate/distributive/minmax.cpp b/src/function/aggregate/distributive/minmax.cpp index ce5ef12afaec..1d6ccfe7919a 100644 --- a/src/function/aggregate/distributive/minmax.cpp +++ b/src/function/aggregate/distributive/minmax.cpp @@ -368,7 +368,7 @@ unique_ptr BindMinMax(ClientContext &context, AggregateFunction &f // Bind function like arg_min/arg_max. function.arguments[0] = arguments[0]->return_type; function.return_type = arguments[0]->return_type; - return nullptr; + return make_uniq(); } } @@ -431,7 +431,6 @@ class MinMaxNState { template void MinMaxNUpdate(Vector inputs[], AggregateInputData &aggr_input, idx_t input_count, Vector &state_vector, idx_t count) { - auto &val_vector = inputs[0]; auto &n_vector = inputs[1]; @@ -441,7 +440,7 @@ void MinMaxNUpdate(Vector inputs[], AggregateInputData &aggr_input, idx_t input_ auto val_extra_state = STATE::VAL_TYPE::CreateExtraState(val_vector, count); - STATE::VAL_TYPE::PrepareData(val_vector, count, val_extra_state, val_format); + STATE::VAL_TYPE::PrepareData(val_vector, count, val_extra_state, val_format, true); n_vector.ToUnifiedFormat(count, n_format); state_vector.ToUnifiedFormat(count, state_format); diff --git a/src/function/aggregate/sorted_aggregate_function.cpp b/src/function/aggregate/sorted_aggregate_function.cpp index bde4c1479ba8..64135df059d8 100644 --- a/src/function/aggregate/sorted_aggregate_function.cpp +++ b/src/function/aggregate/sorted_aggregate_function.cpp @@ -10,6 +10,7 @@ #include "duckdb/planner/expression/bound_reference_expression.hpp" #include "duckdb/parser/expression_map.hpp" #include "duckdb/parallel/thread_context.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/main/settings.hpp" namespace duckdb { diff --git a/src/function/cast/CMakeLists.txt b/src/function/cast/CMakeLists.txt index 1ae51a0d56c6..1f672981613c 100644 --- a/src/function/cast/CMakeLists.txt +++ b/src/function/cast/CMakeLists.txt @@ -11,6 +11,7 @@ add_library_unity( decimal_cast.cpp default_casts.cpp enum_casts.cpp + geo_casts.cpp list_casts.cpp map_cast.cpp numeric_casts.cpp diff --git a/src/function/cast/default_casts.cpp b/src/function/cast/default_casts.cpp index 0c0c1c0588a5..558329f7077c 100644 --- a/src/function/cast/default_casts.cpp +++ b/src/function/cast/default_casts.cpp @@ -162,6 +162,8 @@ BoundCastInfo DefaultCasts::GetDefaultCastFunction(BindCastInput &input, const L return EnumCastSwitch(input, source, target); case LogicalTypeId::ARRAY: return ArrayCastSwitch(input, source, target); + case LogicalTypeId::GEOMETRY: + return GeoCastSwitch(input, source, target); case LogicalTypeId::BIGNUM: return BignumCastSwitch(input, source, target); case LogicalTypeId::AGGREGATE_STATE: diff --git a/src/function/cast/geo_casts.cpp b/src/function/cast/geo_casts.cpp new file mode 100644 index 000000000000..b57b4cb9e987 --- /dev/null +++ b/src/function/cast/geo_casts.cpp @@ -0,0 +1,24 @@ +#include "duckdb/common/types/geometry.hpp" +#include "duckdb/function/cast/default_casts.hpp" +#include "duckdb/function/cast/vector_cast_helpers.hpp" + +namespace duckdb { + +static bool GeometryToVarcharCast(Vector &source, Vector &result, idx_t count, CastParameters ¶meters) { + UnaryExecutor::Execute( + source, result, count, [&](const string_t &input) -> string_t { return Geometry::ToString(result, input); }); + return true; +} + +BoundCastInfo DefaultCasts::GeoCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target) { + + // now switch on the result type + switch (target.id()) { + case LogicalTypeId::VARCHAR: + return GeometryToVarcharCast; + default: + return TryVectorNullCast; + } +} + +} // namespace duckdb diff --git a/src/function/cast/string_cast.cpp b/src/function/cast/string_cast.cpp index 511d09a86128..930231808f97 100644 --- a/src/function/cast/string_cast.cpp +++ b/src/function/cast/string_cast.cpp @@ -490,6 +490,8 @@ BoundCastInfo DefaultCasts::StringCastSwitch(BindCastInput &input, const Logical return BoundCastInfo(&VectorCastHelpers::TryCastStringLoop); case LogicalTypeId::UUID: return BoundCastInfo(&VectorCastHelpers::TryCastStringLoop); + case LogicalTypeId::GEOMETRY: + return BoundCastInfo(&VectorCastHelpers::TryCastStringLoop); case LogicalTypeId::SQLNULL: return &DefaultCasts::TryVectorNullCast; case LogicalTypeId::VARCHAR: diff --git a/src/function/cast/struct_cast.cpp b/src/function/cast/struct_cast.cpp index 97a9354d1051..a7a6ecc9a1b2 100644 --- a/src/function/cast/struct_cast.cpp +++ b/src/function/cast/struct_cast.cpp @@ -12,8 +12,8 @@ unique_ptr StructBoundCastData::BindStructToStructCast(BindCastIn auto &source_children = StructType::GetChildTypes(source); auto &target_children = StructType::GetChildTypes(target); - auto target_is_unnamed = StructType::IsUnnamed(target); - auto source_is_unnamed = StructType::IsUnnamed(source); + auto target_is_unnamed = target_children.empty() || StructType::IsUnnamed(target); + auto source_is_unnamed = source_children.empty() || StructType::IsUnnamed(source); auto is_unnamed = target_is_unnamed || source_is_unnamed; if (is_unnamed && source_children.size() != target_children.size()) { diff --git a/src/function/cast/variant/from_variant.cpp b/src/function/cast/variant/from_variant.cpp index ca377b326985..f29db2b85988 100644 --- a/src/function/cast/variant/from_variant.cpp +++ b/src/function/cast/variant/from_variant.cpp @@ -1,3 +1,4 @@ +#include "yyjson_utils.hpp" #include "duckdb/function/cast/default_casts.hpp" #include "duckdb/common/types/variant.hpp" #include "duckdb/function/scalar/variant_utils.hpp" @@ -49,22 +50,6 @@ struct DecimalConversionPayloadFromVariant { idx_t scale; }; -struct ConvertedJSONHolder { -public: - ~ConvertedJSONHolder() { - if (doc) { - yyjson_mut_doc_free(doc); - } - if (stringified_json) { - free(stringified_json); - } - } - -public: - yyjson_mut_doc *doc = nullptr; - char *stringified_json = nullptr; -}; - } // namespace //===--------------------------------------------------------------------===// @@ -364,6 +349,14 @@ static bool ConvertVariantToStruct(FromVariantConversionData &conversion_data, V SelectionVector child_values_sel; child_values_sel.Initialize(count); + SelectionVector row_sel(0, count); + if (row.IsValid()) { + auto row_index = row.GetIndex(); + for (idx_t i = 0; i < count; i++) { + row_sel[i] = static_cast(row_index); + } + } + for (idx_t child_idx = 0; child_idx < child_types.size(); child_idx++) { auto &child_name = child_types[child_idx].first; @@ -372,14 +365,21 @@ static bool ConvertVariantToStruct(FromVariantConversionData &conversion_data, V VariantPathComponent component; component.key = child_name; component.lookup_mode = VariantChildLookupMode::BY_KEY; - auto collection_result = - VariantUtils::FindChildValues(conversion_data.variant, component, row, child_values_sel, child_data, count); - if (!collection_result.Success()) { - D_ASSERT(collection_result.type == VariantChildDataCollectionResult::Type::COMPONENT_NOT_FOUND); - auto nested_index = collection_result.nested_data_index; - auto row_index = row.IsValid() ? row.GetIndex() : nested_index; + ValidityMask lookup_validity(count); + VariantUtils::FindChildValues(conversion_data.variant, component, row_sel, child_values_sel, lookup_validity, + child_data, count); + if (!lookup_validity.AllValid()) { + optional_idx nested_index; + for (idx_t i = 0; i < count; i++) { + if (!lookup_validity.RowIsValid(i)) { + nested_index = i; + break; + } + } + D_ASSERT(nested_index.IsValid()); + auto row_index = row.IsValid() ? row.GetIndex() : nested_index.GetIndex(); auto object_keys = - VariantUtils::GetObjectKeys(conversion_data.variant, row_index, child_data[nested_index]); + VariantUtils::GetObjectKeys(conversion_data.variant, row_index, child_data[nested_index.GetIndex()]); conversion_data.error = StringUtil::Format("VARIANT(OBJECT(%s)) is missing key '%s'", StringUtil::Join(object_keys, ","), component.key); return false; @@ -550,6 +550,11 @@ static bool CastVariant(FromVariantConversionData &conversion_data, Vector &resu return CastVariantToPrimitive>( conversion_data, result, sel, offset, count, row, string_payload); } + case LogicalTypeId::GEOMETRY: { + StringConversionPayload string_payload(result); + return CastVariantToPrimitive>( + conversion_data, result, sel, offset, count, row, string_payload); + } case LogicalTypeId::VARCHAR: { if (target_type.IsJSONType()) { return CastVariantToJSON(conversion_data, result, sel, offset, count, row); @@ -686,6 +691,8 @@ BoundCastInfo DefaultCasts::VariantCastSwitch(BindCastInput &input, const Logica case LogicalTypeId::UUID: case LogicalTypeId::ARRAY: return BoundCastInfo(CastFromVARIANT); + case LogicalTypeId::GEOMETRY: + return BoundCastInfo(CastFromVARIANT); case LogicalTypeId::VARCHAR: { return BoundCastInfo(CastFromVARIANT); } diff --git a/src/function/cast/variant/to_json.cpp b/src/function/cast/variant/to_json.cpp index 9d35c142cd11..482fa90c26c6 100644 --- a/src/function/cast/variant/to_json.cpp +++ b/src/function/cast/variant/to_json.cpp @@ -10,6 +10,7 @@ #include "duckdb/common/types/decimal.hpp" #include "duckdb/common/types/time.hpp" #include "duckdb/common/types/timestamp.hpp" +#include "duckdb/common/types/variant_visitor.hpp" using namespace duckdb_yyjson; // NOLINT @@ -17,256 +18,211 @@ namespace duckdb { //! ------------ Variant -> JSON ------------ -yyjson_mut_val *VariantCasts::ConvertVariantToJSON(yyjson_mut_doc *doc, const RecursiveUnifiedVectorFormat &source, - idx_t row, uint32_t values_idx) { - auto index = source.unified.sel->get_index(row); - if (!source.unified.validity.RowIsValid(index)) { - return yyjson_mut_null(doc); - } +namespace { + +struct JSONConverter { + using result_type = yyjson_mut_val *; - //! values - auto &values = UnifiedVariantVector::GetValues(source); - auto values_data = values.GetData(values); - - //! type_ids - auto &type_ids = UnifiedVariantVector::GetValuesTypeId(source); - auto type_ids_data = type_ids.GetData(type_ids); - - //! byte_offsets - auto &byte_offsets = UnifiedVariantVector::GetValuesByteOffset(source); - auto byte_offsets_data = byte_offsets.GetData(byte_offsets); - - //! children - auto &children = UnifiedVariantVector::GetChildren(source); - auto children_data = children.GetData(children); - - //! values_index - auto &values_index = UnifiedVariantVector::GetChildrenValuesIndex(source); - auto values_index_data = values_index.GetData(values_index); - - //! keys_index - auto &keys_index = UnifiedVariantVector::GetChildrenKeysIndex(source); - auto keys_index_data = keys_index.GetData(keys_index); - - //! keys - auto &keys = UnifiedVariantVector::GetKeys(source); - auto keys_data = keys.GetData(keys); - auto &keys_entry = UnifiedVariantVector::GetKeysEntry(source); - auto keys_entry_data = keys_entry.GetData(keys_entry); - - //! list entries - auto keys_list_entry = keys_data[keys.sel->get_index(row)]; - auto children_list_entry = children_data[children.sel->get_index(row)]; - auto values_list_entry = values_data[values.sel->get_index(row)]; - - //! The 'values' data of the value we're currently converting - values_idx += values_list_entry.offset; - auto type_id = static_cast(type_ids_data[type_ids.sel->get_index(values_idx)]); - auto byte_offset = byte_offsets_data[byte_offsets.sel->get_index(values_idx)]; - - //! The blob data of the Variant, accessed by byte offset retrieved above ^ - auto &value = UnifiedVariantVector::GetData(source); - auto value_data = value.GetData(value); - auto &blob = value_data[value.sel->get_index(row)]; - auto blob_data = const_data_ptr_cast(blob.GetData()); - - auto ptr = const_data_ptr_cast(blob_data + byte_offset); - switch (type_id) { - case VariantLogicalType::VARIANT_NULL: + static yyjson_mut_val *VisitNull(yyjson_mut_doc *doc) { return yyjson_mut_null(doc); - case VariantLogicalType::BOOL_TRUE: - return yyjson_mut_true(doc); - case VariantLogicalType::BOOL_FALSE: - return yyjson_mut_false(doc); - case VariantLogicalType::INT8: { - auto val = Load(ptr); - return yyjson_mut_sint(doc, val); - } - case VariantLogicalType::INT16: { - auto val = Load(ptr); - return yyjson_mut_sint(doc, val); } - case VariantLogicalType::INT32: { - auto val = Load(ptr); - return yyjson_mut_sint(doc, val); + + static yyjson_mut_val *VisitBoolean(bool val, yyjson_mut_doc *doc) { + return val ? yyjson_mut_true(doc) : yyjson_mut_false(doc); } - case VariantLogicalType::INT64: { - auto val = Load(ptr); - return yyjson_mut_sint(doc, val); + + template + static yyjson_mut_val *VisitInteger(T val, yyjson_mut_doc *doc) { + throw InternalException("JSONConverter::VisitInteger not implemented!"); } - case VariantLogicalType::INT128: { - auto val = Load(ptr); - auto val_str = val.ToString(); - return yyjson_mut_rawncpy(doc, val_str.c_str(), val_str.size()); + + static yyjson_mut_val *VisitTime(dtime_t val, yyjson_mut_doc *doc) { + auto val_str = Time::ToString(val); + return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::UINT8: { - auto val = Load(ptr); - return yyjson_mut_sint(doc, val); + + static yyjson_mut_val *VisitTimeNanos(dtime_ns_t val, yyjson_mut_doc *doc) { + auto val_str = Value::TIME_NS(val).ToString(); + return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::UINT16: { - auto val = Load(ptr); - return yyjson_mut_sint(doc, val); + + static yyjson_mut_val *VisitTimeTZ(dtime_tz_t val, yyjson_mut_doc *doc) { + auto val_str = Value::TIMETZ(val).ToString(); + return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::UINT32: { - auto val = Load(ptr); - return yyjson_mut_sint(doc, val); + + static yyjson_mut_val *VisitTimestampSec(timestamp_sec_t val, yyjson_mut_doc *doc) { + auto val_str = Value::TIMESTAMPSEC(val).ToString(); + return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::UINT64: { - auto val = Load(ptr); - return yyjson_mut_uint(doc, val); + + static yyjson_mut_val *VisitTimestampMs(timestamp_ms_t val, yyjson_mut_doc *doc) { + auto val_str = Value::TIMESTAMPMS(val).ToString(); + return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::UINT128: { - auto val = Load(ptr); - auto val_str = val.ToString(); - return yyjson_mut_rawncpy(doc, val_str.c_str(), val_str.size()); + + static yyjson_mut_val *VisitTimestamp(timestamp_t val, yyjson_mut_doc *doc) { + auto val_str = Timestamp::ToString(val); + return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::UUID: { - auto val = Value::UUID(Load(ptr)); - auto val_str = val.ToString(); + + static yyjson_mut_val *VisitTimestampNanos(timestamp_ns_t val, yyjson_mut_doc *doc) { + auto val_str = Value::TIMESTAMPNS(val).ToString(); return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::INTERVAL: { - auto val = Value::INTERVAL(Load(ptr)); - auto val_str = val.ToString(); + + static yyjson_mut_val *VisitTimestampTZ(timestamp_tz_t val, yyjson_mut_doc *doc) { + auto val_str = Value::TIMESTAMPTZ(val).ToString(); return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::FLOAT: { - auto val = Load(ptr); + + static yyjson_mut_val *VisitFloat(float val, yyjson_mut_doc *doc) { return yyjson_mut_real(doc, val); } - case VariantLogicalType::DOUBLE: { - auto val = Load(ptr); + + static yyjson_mut_val *VisitDouble(double val, yyjson_mut_doc *doc) { return yyjson_mut_real(doc, val); } - case VariantLogicalType::DATE: { - auto val = Load(ptr); - auto val_str = Date::ToString(date_t(val)); + + static yyjson_mut_val *VisitUUID(hugeint_t val, yyjson_mut_doc *doc) { + auto val_str = Value::UUID(val).ToString(); return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::BLOB: { - auto string_length = VarintDecode(ptr); - auto string_data = reinterpret_cast(ptr); - auto val_str = Value::BLOB(const_data_ptr_cast(string_data), string_length).ToString(); + + static yyjson_mut_val *VisitDate(date_t val, yyjson_mut_doc *doc) { + auto val_str = Date::ToString(val); return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::VARCHAR: { - auto string_length = VarintDecode(ptr); - auto string_data = reinterpret_cast(ptr); - return yyjson_mut_strncpy(doc, string_data, static_cast(string_length)); - } - case VariantLogicalType::DECIMAL: { - auto width = NumericCast(VarintDecode(ptr)); - auto scale = NumericCast(VarintDecode(ptr)); - string val_str; - if (width > DecimalWidth::max) { - val_str = Decimal::ToString(Load(ptr), width, scale); - } else if (width > DecimalWidth::max) { - val_str = Decimal::ToString(Load(ptr), width, scale); - } else if (width > DecimalWidth::max) { - val_str = Decimal::ToString(Load(ptr), width, scale); - } else { - val_str = Decimal::ToString(Load(ptr), width, scale); - } - return yyjson_mut_rawncpy(doc, val_str.c_str(), val_str.size()); - } - case VariantLogicalType::TIME_MICROS: { - auto val = Load(ptr); - auto val_str = Time::ToString(val); + static yyjson_mut_val *VisitInterval(interval_t val, yyjson_mut_doc *doc) { + auto val_str = Value::INTERVAL(val).ToString(); return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::TIME_MICROS_TZ: { - auto val = Value::TIMETZ(Load(ptr)); - auto val_str = val.ToString(); - return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); + + static yyjson_mut_val *VisitString(const string_t &str, yyjson_mut_doc *doc) { + return yyjson_mut_strncpy(doc, str.GetData(), str.GetSize()); } - case VariantLogicalType::TIMESTAMP_MICROS: { - auto val = Load(ptr); - auto val_str = Timestamp::ToString(val); + + static yyjson_mut_val *VisitBlob(const string_t &str, yyjson_mut_doc *doc) { + auto val_str = Value::BLOB(const_data_ptr_cast(str.GetData()), str.GetSize()).ToString(); return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::TIMESTAMP_SEC: { - auto val = Value::TIMESTAMPSEC(Load(ptr)); - auto val_str = val.ToString(); - return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); + + static yyjson_mut_val *VisitBignum(const string_t &str, yyjson_mut_doc *doc) { + auto val_str = Value::BIGNUM(const_data_ptr_cast(str.GetData()), str.GetSize()).ToString(); + return yyjson_mut_rawncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::TIMESTAMP_NANOS: { - auto val = Value::TIMESTAMPNS(Load(ptr)); - auto val_str = val.ToString(); + + static yyjson_mut_val *VisitGeometry(const string_t &str, yyjson_mut_doc *doc) { + auto val_str = Value::GEOMETRY(const_data_ptr_cast(str.GetData()), str.GetSize()).ToString(); return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::TIMESTAMP_MILIS: { - auto val = Value::TIMESTAMPMS(Load(ptr)); - auto val_str = val.ToString(); + + static yyjson_mut_val *VisitBitstring(const string_t &str, yyjson_mut_doc *doc) { + auto val_str = Value::BIT(const_data_ptr_cast(str.GetData()), str.GetSize()).ToString(); return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::TIMESTAMP_MICROS_TZ: { - auto val = Value::TIMESTAMPTZ(Load(ptr)); - auto val_str = val.ToString(); - return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); + + template + static yyjson_mut_val *VisitDecimal(T val, uint32_t width, uint32_t scale, yyjson_mut_doc *doc) { + string val_str; + if (std::is_same::value) { + val_str = Decimal::ToString(val, static_cast(width), static_cast(scale)); + } else if (std::is_same::value) { + val_str = Decimal::ToString(val, static_cast(width), static_cast(scale)); + } else if (std::is_same::value) { + val_str = Decimal::ToString(val, static_cast(width), static_cast(scale)); + } else if (std::is_same::value) { + val_str = Decimal::ToString(val, static_cast(width), static_cast(scale)); + } else { + throw InternalException("Unhandled decimal type"); + } + return yyjson_mut_rawncpy(doc, val_str.c_str(), val_str.size()); } - case VariantLogicalType::ARRAY: { - auto count = VarintDecode(ptr); + + static yyjson_mut_val *VisitArray(const UnifiedVariantVectorData &variant, idx_t row, + const VariantNestedData &nested_data, yyjson_mut_doc *doc) { auto arr = yyjson_mut_arr(doc); - if (!count) { - return arr; - } - auto child_index_start = VarintDecode(ptr); - for (idx_t i = 0; i < count; i++) { - auto index = values_index.sel->get_index(children_list_entry.offset + child_index_start + i); - auto child_index = values_index_data[index]; -#ifdef DEBUG - auto key_id_index = keys_index.sel->get_index(children_list_entry.offset + child_index_start + i); - D_ASSERT(!keys_index.validity.RowIsValid(key_id_index)); -#endif - auto val = ConvertVariantToJSON(doc, source, row, child_index); - if (!val) { - return nullptr; - } - yyjson_mut_arr_add_val(arr, val); + auto array_items = VariantVisitor::VisitArrayItems(variant, row, nested_data, doc); + for (auto &entry : array_items) { + yyjson_mut_arr_add_val(arr, entry); } return arr; } - case VariantLogicalType::OBJECT: { - auto count = VarintDecode(ptr); + + static yyjson_mut_val *VisitObject(const UnifiedVariantVectorData &variant, idx_t row, + const VariantNestedData &nested_data, yyjson_mut_doc *doc) { auto obj = yyjson_mut_obj(doc); - if (!count) { - return obj; - } - auto child_index_start = VarintDecode(ptr); - - for (idx_t i = 0; i < count; i++) { - auto children_index = values_index.sel->get_index(children_list_entry.offset + child_index_start + i); - auto child_value_idx = values_index_data[children_index]; - auto val = ConvertVariantToJSON(doc, source, row, child_value_idx); - if (!val) { - return nullptr; - } - auto keys_index_index = keys_index.sel->get_index(children_list_entry.offset + child_index_start + i); - D_ASSERT(keys_index.validity.RowIsValid(keys_index_index)); - auto child_key_id = keys_index_data[keys_index_index]; - auto &key = keys_entry_data[keys_entry.sel->get_index(keys_list_entry.offset + child_key_id)]; - yyjson_mut_obj_put(obj, yyjson_mut_strncpy(doc, key.GetData(), key.GetSize()), val); + auto object_items = VariantVisitor::VisitObjectItems(variant, row, nested_data, doc); + for (auto &entry : object_items) { + yyjson_mut_obj_put(obj, yyjson_mut_strncpy(doc, entry.first.c_str(), entry.first.size()), entry.second); } return obj; } - case VariantLogicalType::BITSTRING: { - auto string_length = VarintDecode(ptr); - auto string_data = reinterpret_cast(ptr); - auto val_str = Value::BIT(const_data_ptr_cast(string_data), string_length).ToString(); - return yyjson_mut_strncpy(doc, val_str.c_str(), val_str.size()); - } - case VariantLogicalType::BIGNUM: { - auto string_length = VarintDecode(ptr); - auto string_data = reinterpret_cast(ptr); - auto val_str = Value::BIGNUM(const_data_ptr_cast(string_data), string_length).ToString(); - return yyjson_mut_rawncpy(doc, val_str.c_str(), val_str.size()); - } - default: - throw InternalException("VariantLogicalType(%d) not handled", static_cast(type_id)); + + static yyjson_mut_val *VisitDefault(VariantLogicalType type_id, const_data_ptr_t, yyjson_mut_doc *) { + throw InternalException("VariantLogicalType(%s) not handled", EnumUtil::ToString(type_id)); } +}; + +template <> +yyjson_mut_val *JSONConverter::VisitInteger(int8_t val, yyjson_mut_doc *doc) { + return yyjson_mut_sint(doc, val); +} - return nullptr; +template <> +yyjson_mut_val *JSONConverter::VisitInteger(int16_t val, yyjson_mut_doc *doc) { + return yyjson_mut_sint(doc, val); +} + +template <> +yyjson_mut_val *JSONConverter::VisitInteger(int32_t val, yyjson_mut_doc *doc) { + return yyjson_mut_sint(doc, val); +} + +template <> +yyjson_mut_val *JSONConverter::VisitInteger(int64_t val, yyjson_mut_doc *doc) { + return yyjson_mut_sint(doc, val); +} + +template <> +yyjson_mut_val *JSONConverter::VisitInteger(hugeint_t val, yyjson_mut_doc *doc) { + auto val_str = val.ToString(); + return yyjson_mut_rawncpy(doc, val_str.c_str(), val_str.size()); +} + +template <> +yyjson_mut_val *JSONConverter::VisitInteger(uint8_t val, yyjson_mut_doc *doc) { + return yyjson_mut_sint(doc, val); +} + +template <> +yyjson_mut_val *JSONConverter::VisitInteger(uint16_t val, yyjson_mut_doc *doc) { + return yyjson_mut_sint(doc, val); +} + +template <> +yyjson_mut_val *JSONConverter::VisitInteger(uint32_t val, yyjson_mut_doc *doc) { + return yyjson_mut_sint(doc, val); +} + +template <> +yyjson_mut_val *JSONConverter::VisitInteger(uint64_t val, yyjson_mut_doc *doc) { + return yyjson_mut_uint(doc, val); +} + +template <> +yyjson_mut_val *JSONConverter::VisitInteger(uhugeint_t val, yyjson_mut_doc *doc) { + auto val_str = val.ToString(); + return yyjson_mut_rawncpy(doc, val_str.c_str(), val_str.size()); +} + +} // namespace + +yyjson_mut_val *VariantCasts::ConvertVariantToJSON(yyjson_mut_doc *doc, const RecursiveUnifiedVectorFormat &source, + idx_t row, uint32_t values_idx) { + UnifiedVariantVectorData variant(source); + return VariantVisitor::Visit(variant, row, values_idx, doc); } } // namespace duckdb diff --git a/src/function/cast/variant/to_variant.cpp b/src/function/cast/variant/to_variant.cpp index 4a15fecff87a..4cfb290f4048 100644 --- a/src/function/cast/variant/to_variant.cpp +++ b/src/function/cast/variant/to_variant.cpp @@ -8,7 +8,6 @@ #include "duckdb/function/cast/variant/to_variant.hpp" namespace duckdb { - namespace variant { static void InitializeOffsets(DataChunk &offsets, idx_t count) { @@ -84,38 +83,6 @@ static void InitializeVariants(DataChunk &offsets, Vector &result, SelectionVect selvec_size = keys_offset; } -static void FinalizeVariantKeys(Vector &variant, OrderedOwningStringMap &dictionary, SelectionVector &sel, - idx_t sel_size) { - auto &keys = VariantVector::GetKeys(variant); - auto &keys_entry = ListVector::GetEntry(keys); - auto keys_entry_data = FlatVector::GetData(keys_entry); - - bool already_sorted = true; - - vector unsorted_to_sorted(dictionary.size()); - auto it = dictionary.begin(); - for (uint32_t sorted_idx = 0; sorted_idx < dictionary.size(); sorted_idx++) { - auto unsorted_idx = it->second; - if (unsorted_idx != sorted_idx) { - already_sorted = false; - } - unsorted_to_sorted[unsorted_idx] = sorted_idx; - D_ASSERT(sorted_idx < ListVector::GetListSize(keys)); - keys_entry_data[sorted_idx] = it->first; - keys_entry_data[sorted_idx].SetSizeAndFinalize(static_cast(keys_entry_data[sorted_idx].GetSize())); - it++; - } - - if (!already_sorted) { - //! Adjust the selection vector to point to the right dictionary index - for (idx_t i = 0; i < sel_size; i++) { - auto &entry = sel[i]; - auto sorted_idx = unsorted_to_sorted[entry]; - entry = sorted_idx; - } - } -} - static bool GatherOffsetsAndSizes(ToVariantSourceData &source, ToVariantGlobalResultData &result, idx_t count) { InitializeOffsets(result.offsets, count); //! First pass - collect sizes/offsets @@ -129,6 +96,9 @@ static bool WriteVariantResultData(ToVariantSourceData &source, ToVariantGlobalR } static bool CastToVARIANT(Vector &source, Vector &result, idx_t count, CastParameters ¶meters) { + if (!count) { + return true; + } DataChunk offsets; offsets.Initialize(Allocator::DefaultAllocator(), {LogicalType::UINTEGER, LogicalType::UINTEGER, LogicalType::UINTEGER, LogicalType::UINTEGER}, @@ -140,8 +110,8 @@ static bool CastToVARIANT(Vector &source, Vector &result, idx_t count, CastParam //! Initialize the dictionary OrderedOwningStringMap dictionary(StringVector::GetStringBuffer(keys_entry).GetStringAllocator()); SelectionVector keys_selvec; - ToVariantSourceData source_data(source, count); + { VariantVectorData variant_data(result); ToVariantGlobalResultData result_data(variant_data, offsets, dictionary, keys_selvec); @@ -162,12 +132,14 @@ static bool CastToVARIANT(Vector &source, Vector &result, idx_t count, CastParam } } - FinalizeVariantKeys(result, dictionary, keys_selvec, keys_selvec_size); + VariantUtils::FinalizeVariantKeys(result, dictionary, keys_selvec, keys_selvec_size); //! Finalize the 'data' auto &blob = VariantVector::GetData(result); auto blob_data = FlatVector::GetData(blob); + auto blob_offsets = OffsetData::GetBlob(offsets); for (idx_t i = 0; i < count; i++) { - blob_data[i].SetSizeAndFinalize(static_cast(blob_data[i].GetSize())); + auto size = blob_offsets[i]; + blob_data[i].SetSizeAndFinalize(size, size); } keys_entry.Slice(keys_selvec, keys_selvec_size); diff --git a/src/function/compression_config.cpp b/src/function/compression_config.cpp index c150608ab96e..dcd6068c49be 100644 --- a/src/function/compression_config.cpp +++ b/src/function/compression_config.cpp @@ -34,80 +34,191 @@ static const DefaultCompressionMethod internal_compression_methods[] = { DictFSSTCompressionFun::TypeIsSupported}, {CompressionType::COMPRESSION_AUTO, nullptr, nullptr}}; -static optional_ptr FindCompressionFunction(CompressionFunctionSet &set, CompressionType type, - const PhysicalType physical_type) { - auto &functions = set.functions; - auto comp_entry = functions.find(type); - if (comp_entry != functions.end()) { - auto &type_functions = comp_entry->second; - auto type_entry = type_functions.find(physical_type); - if (type_entry != type_functions.end()) { - return &type_entry->second; +idx_t CompressionFunctionSet::GetCompressionIndex(PhysicalType physical_type) { + switch (physical_type) { + case PhysicalType::BOOL: + return 0; + case PhysicalType::UINT8: + return 1; + case PhysicalType::INT8: + return 2; + case PhysicalType::UINT16: + return 3; + case PhysicalType::INT16: + return 4; + case PhysicalType::UINT32: + return 5; + case PhysicalType::INT32: + return 6; + case PhysicalType::UINT64: + return 7; + case PhysicalType::INT64: + return 8; + case PhysicalType::FLOAT: + return 9; + case PhysicalType::DOUBLE: + return 10; + case PhysicalType::INTERVAL: + return 11; + case PhysicalType::LIST: + return 12; + case PhysicalType::STRUCT: + return 13; + case PhysicalType::ARRAY: + return 14; + case PhysicalType::VARCHAR: + return 15; + case PhysicalType::UINT128: + return 16; + case PhysicalType::INT128: + return 17; + case PhysicalType::BIT: + return 18; + default: + throw InternalException("Unsupported physical type for compression index"); + } +} + +idx_t CompressionFunctionSet::GetCompressionIndex(CompressionType type) { + return static_cast(type); +} + +CompressionFunctionSet::CompressionFunctionSet() { + for (idx_t i = 0; i < PHYSICAL_TYPE_COUNT; i++) { + is_loaded[i] = false; + } + ResetDisabledMethods(); + functions.resize(PHYSICAL_TYPE_COUNT); +} + +bool EmitCompressionFunction(CompressionType type) { + switch (type) { + case CompressionType::COMPRESSION_UNCOMPRESSED: + case CompressionType::COMPRESSION_RLE: + case CompressionType::COMPRESSION_BITPACKING: + case CompressionType::COMPRESSION_DICTIONARY: + case CompressionType::COMPRESSION_CHIMP: + case CompressionType::COMPRESSION_PATAS: + case CompressionType::COMPRESSION_ALP: + case CompressionType::COMPRESSION_ALPRD: + case CompressionType::COMPRESSION_FSST: + case CompressionType::COMPRESSION_ZSTD: + case CompressionType::COMPRESSION_ROARING: + case CompressionType::COMPRESSION_DICT_FSST: + return true; + default: + return false; + } +} + +vector> CompressionFunctionSet::GetCompressionFunctions(PhysicalType physical_type) { + LoadCompressionFunctions(physical_type); + auto index = GetCompressionIndex(physical_type); + auto &function_list = functions[index]; + vector> result; + for (auto &entry : function_list) { + auto compression_index = GetCompressionIndex(entry.type); + if (is_disabled[compression_index]) { + // explicitly disabled + continue; } + if (!EmitCompressionFunction(entry.type)) { + continue; + } + result.push_back(entry); } - return nullptr; + return result; +} + +void CompressionFunctionSet::LoadCompressionFunctions(PhysicalType physical_type) { + auto index = GetCompressionIndex(physical_type); + auto &function_list = functions[index]; + if (is_loaded[index]) { + return; + } + // not loaded - try to load it + lock_guard guard(lock); + // verify nobody loaded it in the mean-time + if (is_loaded[index]) { + return; + } + // actually perform the load + for (idx_t i = 0; internal_compression_methods[i].get_function; i++) { + TryLoadCompression(internal_compression_methods[i].type, physical_type, function_list); + } + is_loaded[index] = true; } -static optional_ptr LoadCompressionFunction(CompressionFunctionSet &set, CompressionType type, - const PhysicalType physical_type) { +void CompressionFunctionSet::TryLoadCompression(CompressionType type, PhysicalType physical_type, + vector &result) { for (idx_t i = 0; internal_compression_methods[i].get_function; i++) { const auto &method = internal_compression_methods[i]; if (method.type == type) { if (!method.supports_type(physical_type)) { - return nullptr; + // not supported for this type + return; } // The type is supported. We create the function and insert it into the set of available functions. - auto function = method.get_function(physical_type); - set.functions[type].insert(make_pair(physical_type, function)); - return FindCompressionFunction(set, type, physical_type); + result.push_back(method.get_function(physical_type)); + return; } } throw InternalException("Unsupported compression function type"); } -static void TryLoadCompression(DBConfig &config, vector> &result, CompressionType type, - const PhysicalType physical_type) { - if (config.options.disabled_compression_methods.find(type) != config.options.disabled_compression_methods.end()) { - // explicitly disabled - return; +optional_ptr CompressionFunctionSet::GetCompressionFunction(CompressionType type, + const PhysicalType physical_type) { + LoadCompressionFunctions(physical_type); + + auto index = GetCompressionIndex(physical_type); + auto &function_list = functions[index]; + for (auto &function : function_list) { + if (function.type == type) { + return function; + } } - auto function = config.GetCompressionFunction(type, physical_type); - if (!function) { - return; + return nullptr; +} + +void CompressionFunctionSet::SetDisabledCompressionMethods(const vector &methods) { + ResetDisabledMethods(); + for (auto &method : methods) { + auto idx = GetCompressionIndex(method); + is_disabled[idx] = true; } - result.push_back(*function); } -vector> DBConfig::GetCompressionFunctions(const PhysicalType physical_type) { - vector> result; - TryLoadCompression(*this, result, CompressionType::COMPRESSION_UNCOMPRESSED, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_RLE, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_BITPACKING, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_DICTIONARY, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_CHIMP, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_PATAS, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_ALP, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_ALPRD, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_FSST, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_ZSTD, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_ROARING, physical_type); - TryLoadCompression(*this, result, CompressionType::COMPRESSION_DICT_FSST, physical_type); +void DBConfig::SetDisabledCompressionMethods(const vector &methods) { + compression_functions->SetDisabledCompressionMethods(methods); +} + +vector CompressionFunctionSet::GetDisabledCompressionMethods() const { + vector result; + for (idx_t i = 0; i < COMPRESSION_TYPE_COUNT; i++) { + if (is_disabled[i]) { + result.push_back(static_cast(i)); + } + } return result; } -optional_ptr DBConfig::GetCompressionFunction(CompressionType type, - const PhysicalType physical_type) { - lock_guard l(compression_functions->lock); +vector DBConfig::GetDisabledCompressionMethods() const { + return compression_functions->GetDisabledCompressionMethods(); +} - // Check if the function is already loaded into the global compression functions. - auto function = FindCompressionFunction(*compression_functions, type, physical_type); - if (function) { - return function; +void CompressionFunctionSet::ResetDisabledMethods() { + for (idx_t i = 0; i < COMPRESSION_TYPE_COUNT; i++) { + is_disabled[i] = false; } +} - // We could not find the function in the global compression functions, - // so we attempt loading it. - return LoadCompressionFunction(*compression_functions, type, physical_type); +vector> DBConfig::GetCompressionFunctions(const PhysicalType physical_type) { + return compression_functions->GetCompressionFunctions(physical_type); +} + +optional_ptr DBConfig::GetCompressionFunction(CompressionType type, + const PhysicalType physical_type) { + return compression_functions->GetCompressionFunction(type, physical_type); } } // namespace duckdb diff --git a/src/function/copy_blob.cpp b/src/function/copy_blob.cpp new file mode 100644 index 000000000000..2af12a8c3b73 --- /dev/null +++ b/src/function/copy_blob.cpp @@ -0,0 +1,157 @@ +#include "duckdb/common/file_system.hpp" +#include "duckdb/common/vector_operations/unary_executor.hpp" +#include "duckdb/function/built_in_functions.hpp" +#include "duckdb/function/copy_function.hpp" + +namespace duckdb { + +//---------------------------------------------------------------------------------------------------------------------- +// Bind +//---------------------------------------------------------------------------------------------------------------------- +namespace { + +struct WriteBlobBindData final : public TableFunctionData { + FileCompressionType compression_type = FileCompressionType::AUTO_DETECT; +}; + +string ParseStringOption(const Value &value, const string &loption) { + if (value.IsNull()) { + return string(); + } + if (value.type().id() == LogicalTypeId::LIST) { + auto &children = ListValue::GetChildren(value); + if (children.size() != 1) { + throw BinderException("\"%s\" expects a single argument as a string value", loption); + } + return ParseStringOption(children[0], loption); + } + if (value.type().id() != LogicalTypeId::VARCHAR) { + throw BinderException("\"%s\" expects a string argument!", loption); + } + return value.GetValue(); +} + +unique_ptr WriteBlobBind(ClientContext &context, CopyFunctionBindInput &input, + const vector &names, const vector &sql_types) { + if (sql_types.size() != 1 || sql_types.back().id() != LogicalTypeId::BLOB) { + throw BinderException("\"COPY (FORMAT BLOB)\" only supports a single BLOB column"); + } + + auto result = make_uniq(); + + for (auto &lopt : input.info.options) { + if (StringUtil::CIEquals(lopt.first, "compression")) { + auto compression_str = ParseStringOption(lopt.second[0], lopt.first); + result->compression_type = FileCompressionTypeFromString(compression_str); + } else { + throw BinderException("Unrecognized option for COPY (FORMAT BLOB): \"%s\"", lopt.first); + } + } + + return std::move(result); +} + +//---------------------------------------------------------------------------------------------------------------------- +// Global State +//---------------------------------------------------------------------------------------------------------------------- +struct WriteBlobGlobalState final : public GlobalFunctionData { + unique_ptr handle; + mutex lock; +}; + +unique_ptr WriteBlobInitializeGlobal(ClientContext &context, FunctionData &bind_data, + const string &file_path) { + + auto &bdata = bind_data.Cast(); + auto &fs = FileSystem::GetFileSystem(context); + + auto flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW | bdata.compression_type; + auto handle = fs.OpenFile(file_path, flags); + + auto result = make_uniq(); + result->handle = std::move(handle); + + return std::move(result); +} + +//---------------------------------------------------------------------------------------------------------------------- +// Local State +//---------------------------------------------------------------------------------------------------------------------- +struct WriteBlobLocalState final : public LocalFunctionData {}; + +unique_ptr WriteBlobInitializeLocal(ExecutionContext &context, FunctionData &bind_data) { + return make_uniq_base(); +} + +//---------------------------------------------------------------------------------------------------------------------- +// Sink +//---------------------------------------------------------------------------------------------------------------------- +void WriteBlobSink(ExecutionContext &context, FunctionData &bind_data, GlobalFunctionData &gstate, + LocalFunctionData &lstate, DataChunk &input) { + D_ASSERT(input.ColumnCount() == 1); + + auto &state = gstate.Cast(); + lock_guard glock(state.lock); + + auto &handle = state.handle; + + UnifiedVectorFormat vdata; + input.data[0].ToUnifiedFormat(input.size(), vdata); + const auto blobs = UnifiedVectorFormat::GetData(vdata); + + for (idx_t row_idx = 0; row_idx < input.size(); row_idx++) { + const auto out_idx = vdata.sel->get_index(row_idx); + if (vdata.validity.RowIsValid(out_idx)) { + + auto &blob = blobs[out_idx]; + auto blob_len = blob.GetSize(); + auto blob_ptr = blob.GetDataWriteable(); + auto blob_end = blob_ptr + blob_len; + + while (blob_ptr < blob_end) { + auto written = handle->Write(blob_ptr, blob_len); + if (written <= 0) { + throw IOException("Failed to write to file!"); + } + blob_ptr += written; + } + } + } +} + +//---------------------------------------------------------------------------------------------------------------------- +// Combine +//---------------------------------------------------------------------------------------------------------------------- +void WriteBlobCombine(ExecutionContext &context, FunctionData &bind_data, GlobalFunctionData &gstate, + LocalFunctionData &lstate) { +} + +//---------------------------------------------------------------------------------------------------------------------- +// Finalize +//---------------------------------------------------------------------------------------------------------------------- +void WriteBlobFinalize(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate) { + auto &state = gstate.Cast(); + lock_guard glock(state.lock); + + state.handle->Close(); +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- +// Register +//---------------------------------------------------------------------------------------------------------------------- +void BuiltinFunctions::RegisterCopyFunctions() { + CopyFunction info("blob"); + info.copy_to_bind = WriteBlobBind; + info.copy_to_initialize_local = WriteBlobInitializeLocal; + info.copy_to_initialize_global = WriteBlobInitializeGlobal; + info.copy_to_sink = WriteBlobSink; + info.copy_to_combine = WriteBlobCombine; + info.copy_to_finalize = WriteBlobFinalize; + info.extension = "blob"; + + AddFunction(info); +} + +} // namespace duckdb diff --git a/src/function/function.cpp b/src/function/function.cpp index 7c9ea0cd7e74..a74c206d8467 100644 --- a/src/function/function.cpp +++ b/src/function/function.cpp @@ -101,6 +101,8 @@ void BuiltinFunctions::Initialize() { RegisterPragmaFunctions(); + RegisterCopyFunctions(); + // initialize collations AddCollation("nocase", LowerFun::GetFunction(), true); AddCollation("noaccent", StripAccentsFun::GetFunction(), true); diff --git a/src/function/function_list.cpp b/src/function/function_list.cpp index d73467d3ac67..74dbb347b4d2 100644 --- a/src/function/function_list.cpp +++ b/src/function/function_list.cpp @@ -177,6 +177,7 @@ static const StaticFunctionDefinition function[] = { DUCKDB_SCALAR_FUNCTION_ALIAS(UcaseFun), DUCKDB_SCALAR_FUNCTION(UpperFun), DUCKDB_SCALAR_FUNCTION_SET(VariantExtractFun), + DUCKDB_SCALAR_FUNCTION(VariantNormalizeFun), DUCKDB_SCALAR_FUNCTION(VariantTypeofFun), DUCKDB_SCALAR_FUNCTION_SET(WriteLogFun), DUCKDB_SCALAR_FUNCTION(ConcatOperatorFun), diff --git a/src/function/macro_function.cpp b/src/function/macro_function.cpp index 2f407c0256a2..66e36181b5be 100644 --- a/src/function/macro_function.cpp +++ b/src/function/macro_function.cpp @@ -47,7 +47,7 @@ MacroBindResult MacroFunction::BindMacroFunction( InsertionOrderPreservingMap> &named_arguments, idx_t depth) { ExpressionBinder expr_binder(binder, binder.context); - + expr_binder.lambda_bindings = binder.lambda_bindings; // Find argument types and separate positional and default arguments vector positional_arg_types; InsertionOrderPreservingMap named_arg_types; diff --git a/src/function/scalar/compressed_materialization/compress_string.cpp b/src/function/scalar/compressed_materialization/compress_string.cpp index 39821858db5b..e643eccbf9b4 100644 --- a/src/function/scalar/compressed_materialization/compress_string.cpp +++ b/src/function/scalar/compressed_materialization/compress_string.cpp @@ -198,7 +198,7 @@ scalar_function_t GetStringDecompressFunctionSwitch(const LogicalType &input_typ case LogicalTypeId::UHUGEINT: return GetStringDecompressFunction(input_type); case LogicalTypeId::HUGEINT: - return GetStringCompressFunction(input_type); + return GetStringDecompressFunction(input_type); default: throw InternalException("Unexpected type in GetStringDecompressFunctionSwitch"); } diff --git a/src/function/scalar/create_sort_key.cpp b/src/function/scalar/create_sort_key.cpp index 2f5463e3f3ca..d9127d359b1c 100644 --- a/src/function/scalar/create_sort_key.cpp +++ b/src/function/scalar/create_sort_key.cpp @@ -696,13 +696,15 @@ void PrepareSortData(Vector &result, idx_t size, SortKeyLengthInfo &key_lengths, } } -void FinalizeSortData(Vector &result, idx_t size) { +void FinalizeSortData(Vector &result, idx_t size, const SortKeyLengthInfo &key_lengths, + const unsafe_vector &offsets) { switch (result.GetType().id()) { case LogicalTypeId::BLOB: { auto result_data = FlatVector::GetData(result); // call Finalize on the result for (idx_t r = 0; r < size; r++) { - result_data[r].Finalize(); + result_data[r].SetSizeAndFinalize(NumericCast(offsets[r]), + key_lengths.variable_lengths[r] + key_lengths.constant_length); } break; } @@ -739,7 +741,7 @@ void CreateSortKeyInternal(vector> &sort_key_data, SortKeyConstructInfo info(modifiers[c], offsets, data_pointers.get()); ConstructSortKey(*sort_key_data[c], info); } - FinalizeSortData(result, row_count); + FinalizeSortData(result, row_count, key_lengths, offsets); } } // namespace diff --git a/src/function/scalar/nested_functions.cpp b/src/function/scalar/nested_functions.cpp index 2d5359c4e859..b09f04275787 100644 --- a/src/function/scalar/nested_functions.cpp +++ b/src/function/scalar/nested_functions.cpp @@ -3,21 +3,22 @@ namespace duckdb { void MapUtil::ReinterpretMap(Vector &result, Vector &input, idx_t count) { + // Copy the list size + const auto list_size = ListVector::GetListSize(input); + ListVector::SetListSize(result, list_size); + UnifiedVectorFormat input_data; input.ToUnifiedFormat(count, input_data); + // Copy the list validity FlatVector::SetValidity(result, input_data.validity); // Copy the struct validity UnifiedVectorFormat input_struct_data; - ListVector::GetEntry(input).ToUnifiedFormat(count, input_struct_data); + ListVector::GetEntry(input).ToUnifiedFormat(list_size, input_struct_data); auto &result_struct = ListVector::GetEntry(result); FlatVector::SetValidity(result_struct, input_struct_data.validity); - // Copy the list size - auto list_size = ListVector::GetListSize(input); - ListVector::SetListSize(result, list_size); - // Copy the list buffer (the list_entry_t data) result.CopyBuffer(input); diff --git a/src/function/scalar/operator/arithmetic.cpp b/src/function/scalar/operator/arithmetic.cpp index 82cd9b5b7051..1dde43871c4e 100644 --- a/src/function/scalar/operator/arithmetic.cpp +++ b/src/function/scalar/operator/arithmetic.cpp @@ -1220,7 +1220,7 @@ hugeint_t InterpolateOperator::Operation(const hugeint_t &lo, const double d, co template <> uhugeint_t InterpolateOperator::Operation(const uhugeint_t &lo, const double d, const uhugeint_t &hi) { - return Hugeint::Convert(Operation(Uhugeint::Cast(lo), d, Uhugeint::Cast(hi))); + return Uhugeint::Convert(Operation(Uhugeint::Cast(lo), d, Uhugeint::Cast(hi))); } static interval_t MultiplyByDouble(const interval_t &i, const double &d) { // NOLINT diff --git a/src/function/scalar/struct/remap_struct.cpp b/src/function/scalar/struct/remap_struct.cpp index 136a89165073..e58c29d88811 100644 --- a/src/function/scalar/struct/remap_struct.cpp +++ b/src/function/scalar/struct/remap_struct.cpp @@ -11,6 +11,10 @@ namespace duckdb { namespace { +static bool IsRemappable(const LogicalType &type) { + return type.IsNested() && type.id() != LogicalTypeId::VARIANT; +} + struct RemapColumnInfo { optional_idx index; optional_idx default_index; @@ -230,7 +234,7 @@ void RemapStruct(Vector &input, Vector &default_vector, Vector &result, idx_t re void RemapNested(Vector &input, Vector &default_vector, Vector &result, idx_t result_size, const vector &remap_info) { auto &source_type = input.GetType(); - D_ASSERT(source_type.IsNested()); + D_ASSERT(IsRemappable(source_type)); switch (source_type.id()) { case LogicalTypeId::STRUCT: return RemapStruct(input, default_vector, result, result_size, remap_info); @@ -293,7 +297,7 @@ struct RemapIndex { RemapIndex index; index.index = idx; index.type = type; - if (type.IsNested()) { + if (IsRemappable(type)) { index.child_map = make_uniq>(GetMap(type)); } return index; @@ -344,8 +348,8 @@ struct RemapEntry { auto &source_type = entry->second.type; auto &target_type = target_entry->second.type; - bool source_is_nested = source_type.IsNested(); - bool target_is_nested = target_type.IsNested(); + bool source_is_nested = IsRemappable(source_type); + bool target_is_nested = IsRemappable(target_type); RemapEntry remap; remap.index = entry->second.index; remap.target_type = target_entry->second.type; @@ -387,7 +391,7 @@ struct RemapEntry { remap.default_index = default_idx; if (default_type.id() == LogicalTypeId::STRUCT) { // nested remap - recurse - if (!target_type.IsNested()) { + if (!IsRemappable(target_type)) { throw BinderException("Default value is a struct - target value should be a nested type, is '%s'", target_type.ToString()); } @@ -436,7 +440,7 @@ struct RemapEntry { RemapColumnInfo info; info.index = entry->second.index; info.default_index = entry->second.default_index; - if (child_type.IsNested() && entry->second.child_remaps) { + if (IsRemappable(child_type) && entry->second.child_remaps) { // type is nested and a mapping for it is given - recurse info.child_remap_info = ConstructMap(child_type, *entry->second.child_remaps); } @@ -447,7 +451,7 @@ struct RemapEntry { static vector ConstructMap(const LogicalType &type, const case_insensitive_map_t &remap_map) { - D_ASSERT(type.IsNested()); + D_ASSERT(IsRemappable(type)); switch (type.id()) { case LogicalTypeId::STRUCT: { auto &target_children = StructType::GetChildTypes(type); @@ -484,7 +488,7 @@ struct RemapEntry { auto remap_entry = remap_map.find(entry->second); D_ASSERT(remap_entry != remap_map.end()); // this entry is remapped - fetch the target type - if (child_type.IsNested() && remap_entry->second.child_remaps) { + if (IsRemappable(child_type) && remap_entry->second.child_remaps) { // type is nested and a mapping for it is given - recurse new_source_children.emplace_back(child_name, RemapCast(child_type, *remap_entry->second.child_remaps)); @@ -552,7 +556,7 @@ unique_ptr RemapStructBind(ClientContext &context, ScalarFunction // remap target can be NULL continue; } - if (!arg->return_type.IsNested()) { + if (!IsRemappable(arg->return_type)) { throw BinderException("Struct remap can only remap nested types, not '%s'", arg->return_type.ToString()); } else if (arg->return_type.id() == LogicalTypeId::STRUCT && StructType::IsUnnamed(arg->return_type)) { throw BinderException("Struct remap can only remap named structs"); @@ -569,7 +573,7 @@ unique_ptr RemapStructBind(ClientContext &context, ScalarFunction throw BinderException("The defaults have to be either NULL or a named STRUCT, not an unnamed struct"); } - if ((from_type.IsNested() || to_type.IsNested()) && from_type.id() != to_type.id()) { + if ((IsRemappable(from_type) || IsRemappable(to_type)) && from_type.id() != to_type.id()) { throw BinderException("Can't change source type (%s) to target type (%s), type conversion not allowed", from_type.ToString(), to_type.ToString()); } diff --git a/src/function/scalar/variant/CMakeLists.txt b/src/function/scalar/variant/CMakeLists.txt index 76336e4faf70..a29c46d67dfb 100644 --- a/src/function/scalar/variant/CMakeLists.txt +++ b/src/function/scalar/variant/CMakeLists.txt @@ -1,5 +1,5 @@ add_library_unity(duckdb_func_variant_main OBJECT variant_utils.cpp - variant_extract.cpp variant_typeof.cpp) + variant_extract.cpp variant_typeof.cpp variant_normalize.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/function/scalar/variant/functions.json b/src/function/scalar/variant/functions.json index 3597c01b4e0e..a92790571019 100644 --- a/src/function/scalar/variant/functions.json +++ b/src/function/scalar/variant/functions.json @@ -39,11 +39,21 @@ ], "type": "scalar_function_set" }, + { + "name": "variant_normalize", + "parameters": "input_variant", + "description": "Normalizes the `input_variant` to a canonical representation.", + "example": "variant_normalize({'b': [1,2,3], 'a': 42})::VARIANT)", + "categories": [ + "variant" + ], + "type": "scalar_function" + }, { "name": "variant_typeof", "parameters": "input_variant", "description": "Returns the internal type of the `input_variant`.", - "example": "variant_typeof({'a': 42, 'b': [1,2,3])::VARIANT)", + "example": "variant_typeof({'a': 42, 'b': [1,2,3]})::VARIANT)", "categories": [ "variant" ], diff --git a/src/function/scalar/variant/variant_extract.cpp b/src/function/scalar/variant/variant_extract.cpp index 4b3c4a815d79..76d5c84cd32b 100644 --- a/src/function/scalar/variant/variant_extract.cpp +++ b/src/function/scalar/variant/variant_extract.cpp @@ -12,6 +12,7 @@ struct BindData : public FunctionData { public: explicit BindData(const string &str); explicit BindData(uint32_t index); + BindData(const BindData &other) = default; public: unique_ptr Copy() const override; @@ -28,15 +29,15 @@ BindData::BindData(const string &str) : FunctionData() { component.key = str; } BindData::BindData(uint32_t index) : FunctionData() { + if (index == 0) { + throw BinderException("Extracting index 0 from VARIANT(ARRAY) is invalid, indexes are 1-based"); + } component.lookup_mode = VariantChildLookupMode::BY_INDEX; - component.index = index; + component.index = index - 1; } unique_ptr BindData::Copy() const { - if (component.lookup_mode == VariantChildLookupMode::BY_INDEX) { - return make_uniq(component.index); - } - return make_uniq(component.key); + return make_uniq(*this); } bool BindData::Equals(const FunctionData &other) const { @@ -101,6 +102,7 @@ static void VariantExtractFunction(DataChunk &input, ExpressionState &state, Vec auto &path = input.data[1]; D_ASSERT(path.GetVectorType() == VectorType::CONSTANT_VECTOR); + (void)path; auto &func_expr = state.expr.Cast(); auto &info = func_expr.bind_info->Cast(); @@ -141,22 +143,26 @@ static void VariantExtractFunction(DataChunk &input, ExpressionState &state, Vec } //! Look up the value_index of the child we're extracting - auto child_collection_result = - VariantUtils::FindChildValues(variant, component, optional_idx(), new_value_index_sel, nested_data, count); - if (!child_collection_result.Success()) { - if (child_collection_result.type == VariantChildDataCollectionResult::Type::INDEX_ZERO) { - throw InvalidInputException("Extracting index 0 from VARIANT(ARRAY) is invalid, indexes are 1-based"); + ValidityMask lookup_validity(count); + VariantUtils::FindChildValues(variant, component, nullptr, new_value_index_sel, lookup_validity, nested_data, + count); + if (!lookup_validity.AllValid()) { + optional_idx index; + for (idx_t i = 0; i < count; i++) { + if (!lookup_validity.RowIsValid(i)) { + index = i; + break; + } } + D_ASSERT(index.IsValid()); switch (component.lookup_mode) { case VariantChildLookupMode::BY_INDEX: { - D_ASSERT(child_collection_result.type == VariantChildDataCollectionResult::Type::COMPONENT_NOT_FOUND); - auto nested_index = child_collection_result.nested_data_index; + auto nested_index = index.GetIndex(); throw InvalidInputException("VARIANT(ARRAY(%d)) is missing index %d", nested_data[nested_index].child_count, component.index); } case VariantChildLookupMode::BY_KEY: { - D_ASSERT(child_collection_result.type == VariantChildDataCollectionResult::Type::COMPONENT_NOT_FOUND); - auto nested_index = child_collection_result.nested_data_index; + auto nested_index = index.GetIndex(); auto row_index = nested_index; auto object_keys = VariantUtils::GetObjectKeys(variant, row_index, nested_data[nested_index]); throw InvalidInputException("VARIANT(OBJECT(%s)) is missing key '%s'", StringUtil::Join(object_keys, ","), diff --git a/src/function/scalar/variant/variant_normalize.cpp b/src/function/scalar/variant/variant_normalize.cpp new file mode 100644 index 000000000000..ef79e38e3708 --- /dev/null +++ b/src/function/scalar/variant/variant_normalize.cpp @@ -0,0 +1,311 @@ +#include "duckdb/function/scalar/variant_utils.hpp" +#include "duckdb/function/scalar/variant_functions.hpp" +#include "duckdb/function/scalar/regexp.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/execution/expression_executor.hpp" + +#include "duckdb/function/cast/variant/to_variant_fwd.hpp" +#include "duckdb/common/types/variant_visitor.hpp" + +namespace duckdb { + +namespace { + +struct VariantNormalizerState { +public: + VariantNormalizerState(idx_t result_row, VariantVectorData &source, OrderedOwningStringMap &dictionary, + SelectionVector &keys_selvec) + : source(source), dictionary(dictionary), keys_selvec(keys_selvec), + keys_index_validity(source.keys_index_validity) { + auto keys_list_entry = source.keys_data[result_row]; + auto values_list_entry = source.values_data[result_row]; + auto children_list_entry = source.children_data[result_row]; + + keys_offset = keys_list_entry.offset; + children_offset = children_list_entry.offset; + + blob_data = data_ptr_cast(source.blob_data[result_row].GetDataWriteable()); + type_ids = source.type_ids_data + values_list_entry.offset; + byte_offsets = source.byte_offset_data + values_list_entry.offset; + values_indexes = source.values_index_data + children_list_entry.offset; + keys_indexes = source.keys_index_data + children_list_entry.offset; + } + +public: + data_ptr_t GetDestination() { + return blob_data + blob_size; + } + uint32_t GetOrCreateIndex(const string_t &key) { + auto unsorted_idx = dictionary.size(); + //! This will later be remapped to the sorted idx (see FinalizeVariantKeys in 'to_variant.cpp') + return dictionary.emplace(std::make_pair(key, unsorted_idx)).first->second; + } + +public: + uint32_t keys_size = 0; + uint32_t children_size = 0; + uint32_t values_size = 0; + uint32_t blob_size = 0; + + VariantVectorData &source; + OrderedOwningStringMap &dictionary; + SelectionVector &keys_selvec; + + uint64_t keys_offset; + uint64_t children_offset; + ValidityMask &keys_index_validity; + + data_ptr_t blob_data; + uint8_t *type_ids; + uint32_t *byte_offsets; + uint32_t *values_indexes; + uint32_t *keys_indexes; +}; + +struct VariantNormalizer { + using result_type = void; + + static void VisitNull(VariantNormalizerState &state) { + return; + } + static void VisitBoolean(bool val, VariantNormalizerState &state) { + return; + } + + static void VisitMetadata(VariantLogicalType type_id, VariantNormalizerState &state) { + state.type_ids[state.values_size] = static_cast(type_id); + state.byte_offsets[state.values_size] = state.blob_size; + state.values_size++; + } + + template + static void VisitInteger(T val, VariantNormalizerState &state) { + Store(val, state.GetDestination()); + state.blob_size += sizeof(T); + } + static void VisitFloat(float val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitDouble(double val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitUUID(hugeint_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitDate(date_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitInterval(interval_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitTime(dtime_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitTimeNanos(dtime_ns_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitTimeTZ(dtime_tz_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitTimestampSec(timestamp_sec_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitTimestampMs(timestamp_ms_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitTimestamp(timestamp_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitTimestampNanos(timestamp_ns_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + static void VisitTimestampTZ(timestamp_tz_t val, VariantNormalizerState &state) { + VisitInteger(val, state); + } + + static void WriteStringInternal(const string_t &str, VariantNormalizerState &state) { + } + + static void VisitString(const string_t &str, VariantNormalizerState &state) { + auto length = str.GetSize(); + state.blob_size += VarintEncode(length, state.GetDestination()); + memcpy(state.GetDestination(), str.GetData(), length); + state.blob_size += length; + } + static void VisitBlob(const string_t &blob, VariantNormalizerState &state) { + return VisitString(blob, state); + } + static void VisitBignum(const string_t &bignum, VariantNormalizerState &state) { + return VisitString(bignum, state); + } + static void VisitGeometry(const string_t &geom, VariantNormalizerState &state) { + return VisitString(geom, state); + } + static void VisitBitstring(const string_t &bits, VariantNormalizerState &state) { + return VisitString(bits, state); + } + + template + static void VisitDecimal(T val, uint32_t width, uint32_t scale, VariantNormalizerState &state) { + state.blob_size += VarintEncode(width, state.GetDestination()); + state.blob_size += VarintEncode(scale, state.GetDestination()); + Store(val, state.GetDestination()); + state.blob_size += sizeof(T); + } + + static void VisitArray(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &nested_data, + VariantNormalizerState &state) { + state.blob_size += VarintEncode(nested_data.child_count, state.GetDestination()); + if (!nested_data.child_count) { + return; + } + idx_t result_children_idx = state.children_size; + state.blob_size += VarintEncode(result_children_idx, state.GetDestination()); + state.children_size += nested_data.child_count; + + for (idx_t i = 0; i < nested_data.child_count; i++) { + auto source_children_idx = nested_data.children_idx + i; + auto values_index = variant.GetValuesIndex(row, source_children_idx); + + //! Set the 'values_index' for the child, and set the 'keys_index' to NULL + state.values_indexes[result_children_idx] = state.values_size; + state.keys_index_validity.SetInvalid(state.children_offset + result_children_idx); + result_children_idx++; + + //! Visit the child value + VariantVisitor::Visit(variant, row, values_index, state); + } + } + + static void VisitObject(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &nested_data, + VariantNormalizerState &state) { + state.blob_size += VarintEncode(nested_data.child_count, state.GetDestination()); + if (!nested_data.child_count) { + return; + } + uint32_t children_idx = state.children_size; + uint32_t keys_idx = state.keys_size; + state.blob_size += VarintEncode(children_idx, state.GetDestination()); + state.children_size += nested_data.child_count; + state.keys_size += nested_data.child_count; + + //! First iterate through all fields to populate the map of key -> field + map sorted_fields; + for (idx_t i = 0; i < nested_data.child_count; i++) { + auto keys_index = variant.GetKeysIndex(row, nested_data.children_idx + i); + auto &key = variant.GetKey(row, keys_index); + sorted_fields.emplace(key, i); + } + + //! Then visit the fields in sorted order + for (auto &entry : sorted_fields) { + auto source_children_idx = nested_data.children_idx + entry.second; + + //! Add the key of the field to the result + auto keys_index = variant.GetKeysIndex(row, source_children_idx); + auto &key = variant.GetKey(row, keys_index); + auto dict_index = state.GetOrCreateIndex(key); + state.keys_selvec.set_index(state.keys_offset + keys_idx, dict_index); + + //! Visit the child value + auto values_index = variant.GetValuesIndex(row, source_children_idx); + state.values_indexes[children_idx] = state.values_size; + state.keys_indexes[children_idx] = keys_idx; + children_idx++; + keys_idx++; + VariantVisitor::Visit(variant, row, values_index, state); + } + } + + static void VisitDefault(VariantLogicalType type_id, const_data_ptr_t, VariantNormalizerState &state) { + throw InternalException("VariantLogicalType(%s) not handled", EnumUtil::ToString(type_id)); + } +}; + +} // namespace + +static void VariantNormalizeFunction(DataChunk &input, ExpressionState &state, Vector &result) { + auto count = input.size(); + + D_ASSERT(input.ColumnCount() == 1); + auto &variant_vec = input.data[0]; + D_ASSERT(variant_vec.GetType() == LogicalType::VARIANT()); + + //! Set up the access helper for the source VARIANT + RecursiveUnifiedVectorFormat source_format; + Vector::RecursiveToUnifiedFormat(variant_vec, count, source_format); + UnifiedVariantVectorData variant(source_format); + + //! Take the original sizes of the lists, the result will be similar size, never bigger + auto original_keys_size = ListVector::GetListSize(VariantVector::GetKeys(variant_vec)); + auto original_children_size = ListVector::GetListSize(VariantVector::GetChildren(variant_vec)); + auto original_values_size = ListVector::GetListSize(VariantVector::GetValues(variant_vec)); + + auto &keys = VariantVector::GetKeys(result); + auto &children = VariantVector::GetChildren(result); + auto &values = VariantVector::GetValues(result); + auto &data = VariantVector::GetData(result); + + ListVector::Reserve(keys, original_keys_size); + ListVector::SetListSize(keys, 0); + ListVector::Reserve(children, original_children_size); + ListVector::SetListSize(children, 0); + ListVector::Reserve(values, original_values_size); + ListVector::SetListSize(values, 0); + + //! Initialize the dictionary + auto &keys_entry = ListVector::GetEntry(keys); + OrderedOwningStringMap dictionary(StringVector::GetStringBuffer(keys_entry).GetStringAllocator()); + + VariantVectorData variant_data(result); + SelectionVector keys_selvec; + keys_selvec.Initialize(original_keys_size); + + for (idx_t i = 0; i < count; i++) { + if (!variant.RowIsValid(i)) { + FlatVector::SetNull(result, i, true); + continue; + } + //! Allocate for the new data, use the same size as source + auto &blob_data = variant_data.blob_data[i]; + auto original_data = variant.GetData(i); + blob_data = StringVector::EmptyString(data, original_data.GetSize()); + + auto &keys_list_entry = variant_data.keys_data[i]; + keys_list_entry.offset = ListVector::GetListSize(keys); + + auto &children_list_entry = variant_data.children_data[i]; + children_list_entry.offset = ListVector::GetListSize(children); + + auto &values_list_entry = variant_data.values_data[i]; + values_list_entry.offset = ListVector::GetListSize(values); + + //! Visit the source to populate the result + VariantNormalizerState visitor_state(i, variant_data, dictionary, keys_selvec); + VariantVisitor::Visit(variant, i, 0, visitor_state); + + blob_data.SetSizeAndFinalize(visitor_state.blob_size, original_data.GetSize()); + keys_list_entry.length = visitor_state.keys_size; + children_list_entry.length = visitor_state.children_size; + values_list_entry.length = visitor_state.values_size; + + ListVector::SetListSize(keys, ListVector::GetListSize(keys) + visitor_state.keys_size); + ListVector::SetListSize(children, ListVector::GetListSize(children) + visitor_state.children_size); + ListVector::SetListSize(values, ListVector::GetListSize(values) + visitor_state.values_size); + } + + VariantUtils::FinalizeVariantKeys(result, dictionary, keys_selvec, ListVector::GetListSize(keys)); + keys_entry.Slice(keys_selvec, ListVector::GetListSize(keys)); + + if (input.AllConstant()) { + result.SetVectorType(VectorType::CONSTANT_VECTOR); + } + result.Verify(count); +} + +ScalarFunction VariantNormalizeFun::GetFunction() { + auto variant_type = LogicalType::VARIANT(); + return ScalarFunction("variant_normalize", {variant_type}, variant_type, VariantNormalizeFunction); +} + +} // namespace duckdb diff --git a/src/function/scalar/variant/variant_utils.cpp b/src/function/scalar/variant/variant_utils.cpp index e2ea445480d9..63295b67ca84 100644 --- a/src/function/scalar/variant/variant_utils.cpp +++ b/src/function/scalar/variant/variant_utils.cpp @@ -4,9 +4,22 @@ #include "duckdb/common/types/string_type.hpp" #include "duckdb/common/types/decimal.hpp" #include "duckdb/common/serializer/varint.hpp" +#include "duckdb/common/types/variant_visitor.hpp" namespace duckdb { +PhysicalType VariantDecimalData::GetPhysicalType() const { + if (width > DecimalWidth::max) { + return PhysicalType::INT128; + } else if (width > DecimalWidth::max) { + return PhysicalType::INT64; + } else if (width > DecimalWidth::max) { + return PhysicalType::INT32; + } else { + return PhysicalType::INT16; + } +} + bool VariantUtils::IsNestedType(const UnifiedVariantVectorData &variant, idx_t row, uint32_t value_index) { auto type_id = variant.GetTypeId(row, value_index); return type_id == VariantLogicalType::ARRAY || type_id == VariantLogicalType::OBJECT; @@ -19,10 +32,19 @@ VariantDecimalData VariantUtils::DecodeDecimalData(const UnifiedVariantVectorDat auto data = const_data_ptr_cast(variant.GetData(row).GetData()); auto ptr = data + byte_offset; - VariantDecimalData result; - result.width = VarintDecode(ptr); - result.scale = VarintDecode(ptr); - return result; + auto width = VarintDecode(ptr); + auto scale = VarintDecode(ptr); + auto value_ptr = ptr; + return VariantDecimalData(width, scale, value_ptr); +} + +string_t VariantUtils::DecodeStringData(const UnifiedVariantVectorData &variant, idx_t row, uint32_t value_index) { + auto byte_offset = variant.GetByteOffset(row, value_index); + auto data = const_data_ptr_cast(variant.GetData(row).GetData()); + auto ptr = data + byte_offset; + + auto length = VarintDecode(ptr); + return string_t(reinterpret_cast(ptr), length); } VariantNestedData VariantUtils::DecodeNestedData(const UnifiedVariantVectorData &variant, idx_t row, @@ -53,13 +75,13 @@ vector VariantUtils::GetObjectKeys(const UnifiedVariantVectorData &varia return object_keys; } -VariantChildDataCollectionResult VariantUtils::FindChildValues(const UnifiedVariantVectorData &variant, - const VariantPathComponent &component, optional_idx row, - SelectionVector &res, VariantNestedData *nested_data, - idx_t count) { +//! FIXME: this shouldn't return a "result", it should populate a validity mask instead. +void VariantUtils::FindChildValues(const UnifiedVariantVectorData &variant, const VariantPathComponent &component, + optional_ptr sel, SelectionVector &res, + ValidityMask &res_validity, VariantNestedData *nested_data, idx_t count) { for (idx_t i = 0; i < count; i++) { - auto row_index = row.IsValid() ? row.GetIndex() : i; + auto row_index = sel ? sel->get_index(i) : i; auto &nested_data_entry = nested_data[i]; if (nested_data_entry.is_null) { @@ -67,13 +89,10 @@ VariantChildDataCollectionResult VariantUtils::FindChildValues(const UnifiedVari } if (component.lookup_mode == VariantChildLookupMode::BY_INDEX) { auto child_idx = component.index; - if (child_idx == 0) { - return VariantChildDataCollectionResult::IndexZero(); - } - child_idx--; if (child_idx >= nested_data_entry.child_count) { //! The list is too small to contain this index - return VariantChildDataCollectionResult::NotFound(i); + res_validity.SetInvalid(i); + continue; } auto value_id = variant.GetValuesIndex(row_index, nested_data_entry.children_idx + child_idx); res[i] = static_cast(value_id); @@ -93,10 +112,9 @@ VariantChildDataCollectionResult VariantUtils::FindChildValues(const UnifiedVari } } if (!found_child) { - return VariantChildDataCollectionResult::NotFound(i); + res_validity.SetInvalid(i); } } - return VariantChildDataCollectionResult(); } vector VariantUtils::ValueIsNull(const UnifiedVariantVectorData &variant, const SelectionVector &sel, @@ -146,133 +164,204 @@ VariantUtils::CollectNestedData(const UnifiedVariantVectorData &variant, Variant return VariantNestedDataCollectionResult(); } -Value VariantUtils::ConvertVariantToValue(const UnifiedVariantVectorData &variant, idx_t row, idx_t values_idx) { - if (!variant.RowIsValid(row)) { - return Value(LogicalTypeId::SQLNULL); +namespace { + +struct ValueConverter { + using result_type = Value; + + static Value VisitNull() { + return Value(LogicalType::SQLNULL); + } + + static Value VisitBoolean(bool val) { + return Value::BOOLEAN(val); } - //! The 'values' data of the value we're currently converting - auto type_id = variant.GetTypeId(row, values_idx); - auto byte_offset = variant.GetByteOffset(row, values_idx); + template + static Value VisitInteger(T val) { + throw InternalException("ValueConverter::VisitInteger not implemented!"); + } - //! The blob data of the Variant, accessed by byte offset retrieved above ^ - auto blob_data = const_data_ptr_cast(variant.GetData(row).GetData()); + static Value VisitTime(dtime_t val) { + return Value::TIME(val); + } - auto ptr = const_data_ptr_cast(blob_data + byte_offset); - switch (type_id) { - case VariantLogicalType::VARIANT_NULL: - return Value(LogicalType::SQLNULL); - case VariantLogicalType::BOOL_TRUE: - return Value::BOOLEAN(true); - case VariantLogicalType::BOOL_FALSE: - return Value::BOOLEAN(false); - case VariantLogicalType::INT8: - return Value::TINYINT(Load(ptr)); - case VariantLogicalType::INT16: - return Value::SMALLINT(Load(ptr)); - case VariantLogicalType::INT32: - return Value::INTEGER(Load(ptr)); - case VariantLogicalType::INT64: - return Value::BIGINT(Load(ptr)); - case VariantLogicalType::INT128: - return Value::HUGEINT(Load(ptr)); - case VariantLogicalType::UINT8: - return Value::UTINYINT(Load(ptr)); - case VariantLogicalType::UINT16: - return Value::USMALLINT(Load(ptr)); - case VariantLogicalType::UINT32: - return Value::UINTEGER(Load(ptr)); - case VariantLogicalType::UINT64: - return Value::UBIGINT(Load(ptr)); - case VariantLogicalType::UINT128: - return Value::UHUGEINT(Load(ptr)); - case VariantLogicalType::UUID: - return Value::UUID(Load(ptr)); - case VariantLogicalType::INTERVAL: - return Value::INTERVAL(Load(ptr)); - case VariantLogicalType::FLOAT: - return Value::FLOAT(Load(ptr)); - case VariantLogicalType::DOUBLE: - return Value::DOUBLE(Load(ptr)); - case VariantLogicalType::DATE: - return Value::DATE(date_t(Load(ptr))); - case VariantLogicalType::BLOB: { - auto string_length = VarintDecode(ptr); - auto string_data = reinterpret_cast(ptr); - return Value::BLOB(const_data_ptr_cast(string_data), string_length); - } - case VariantLogicalType::VARCHAR: { - auto string_length = VarintDecode(ptr); - auto string_data = reinterpret_cast(ptr); - return Value(string_t(string_data, string_length)); - } - case VariantLogicalType::DECIMAL: { - auto width = NumericCast(VarintDecode(ptr)); - auto scale = NumericCast(VarintDecode(ptr)); - - if (width > DecimalWidth::max) { - return Value::DECIMAL(Load(ptr), width, scale); - } else if (width > DecimalWidth::max) { - return Value::DECIMAL(Load(ptr), width, scale); - } else if (width > DecimalWidth::max) { - return Value::DECIMAL(Load(ptr), width, scale); + static Value VisitTimeNanos(dtime_ns_t val) { + return Value::TIME_NS(val); + } + + static Value VisitTimeTZ(dtime_tz_t val) { + return Value::TIMETZ(val); + } + + static Value VisitTimestampSec(timestamp_sec_t val) { + return Value::TIMESTAMPSEC(val); + } + + static Value VisitTimestampMs(timestamp_ms_t val) { + return Value::TIMESTAMPMS(val); + } + + static Value VisitTimestamp(timestamp_t val) { + return Value::TIMESTAMP(val); + } + + static Value VisitTimestampNanos(timestamp_ns_t val) { + return Value::TIMESTAMPNS(val); + } + + static Value VisitTimestampTZ(timestamp_tz_t val) { + return Value::TIMESTAMPTZ(val); + } + + static Value VisitFloat(float val) { + return Value::FLOAT(val); + } + static Value VisitDouble(double val) { + return Value::DOUBLE(val); + } + static Value VisitUUID(hugeint_t val) { + return Value::UUID(val); + } + static Value VisitDate(date_t val) { + return Value::DATE(val); + } + static Value VisitInterval(interval_t val) { + return Value::INTERVAL(val); + } + + static Value VisitString(const string_t &str) { + return Value(str); + } + static Value VisitBlob(const string_t &str) { + return Value::BLOB(const_data_ptr_cast(str.GetData()), str.GetSize()); + } + static Value VisitBignum(const string_t &str) { + return Value::BIGNUM(const_data_ptr_cast(str.GetData()), str.GetSize()); + } + static Value VisitGeometry(const string_t &str) { + return Value::GEOMETRY(const_data_ptr_cast(str.GetData()), str.GetSize()); + } + static Value VisitBitstring(const string_t &str) { + return Value::BIT(const_data_ptr_cast(str.GetData()), str.GetSize()); + } + + template + static Value VisitDecimal(T val, uint32_t width, uint32_t scale) { + if (std::is_same::value) { + return Value::DECIMAL(val, static_cast(width), static_cast(scale)); + } else if (std::is_same::value) { + return Value::DECIMAL(val, static_cast(width), static_cast(scale)); + } else if (std::is_same::value) { + return Value::DECIMAL(val, static_cast(width), static_cast(scale)); + } else if (std::is_same::value) { + return Value::DECIMAL(val, static_cast(width), static_cast(scale)); } else { - return Value::DECIMAL(Load(ptr), width, scale); + throw InternalException("Unhandled decimal type"); } } - case VariantLogicalType::TIME_MICROS: - return Value::TIME(Load(ptr)); - case VariantLogicalType::TIME_MICROS_TZ: - return Value::TIMETZ(Load(ptr)); - case VariantLogicalType::TIMESTAMP_MICROS: - return Value::TIMESTAMP(Load(ptr)); - case VariantLogicalType::TIMESTAMP_SEC: - return Value::TIMESTAMPSEC(Load(ptr)); - case VariantLogicalType::TIMESTAMP_NANOS: - return Value::TIMESTAMPNS(Load(ptr)); - case VariantLogicalType::TIMESTAMP_MILIS: - return Value::TIMESTAMPMS(Load(ptr)); - case VariantLogicalType::TIMESTAMP_MICROS_TZ: - return Value::TIMESTAMPTZ(Load(ptr)); - case VariantLogicalType::ARRAY: { - auto count = VarintDecode(ptr); - vector array_items; - if (count) { - auto child_index_start = VarintDecode(ptr); - for (idx_t i = 0; i < count; i++) { - auto child_index = variant.GetValuesIndex(row, child_index_start + i); - array_items.emplace_back(ConvertVariantToValue(variant, row, child_index)); - } - } + + static Value VisitArray(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &nested_data) { + auto array_items = VariantVisitor::VisitArrayItems(variant, row, nested_data); return Value::LIST(LogicalType::VARIANT(), std::move(array_items)); } - case VariantLogicalType::OBJECT: { - auto count = VarintDecode(ptr); - child_list_t object_children; - if (count) { - auto child_index_start = VarintDecode(ptr); - for (idx_t i = 0; i < count; i++) { - auto child_value_idx = variant.GetValuesIndex(row, child_index_start + i); - auto val = ConvertVariantToValue(variant, row, child_value_idx); - - auto child_key_id = variant.GetKeysIndex(row, child_index_start + i); - auto &key = variant.GetKey(row, child_key_id); - object_children.emplace_back(key.GetString(), std::move(val)); - } - } + static Value VisitObject(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &nested_data) { + auto object_children = VariantVisitor::VisitObjectItems(variant, row, nested_data); return Value::STRUCT(std::move(object_children)); } - case VariantLogicalType::BITSTRING: { - auto string_length = VarintDecode(ptr); - return Value::BIT(ptr, string_length); + + static Value VisitDefault(VariantLogicalType type_id, const_data_ptr_t) { + throw InternalException("VariantLogicalType(%s) not handled", EnumUtil::ToString(type_id)); } - case VariantLogicalType::BIGNUM: { - auto string_length = VarintDecode(ptr); - return Value::BIGNUM(ptr, string_length); +}; + +template <> +Value ValueConverter::VisitInteger(int8_t val) { + return Value::TINYINT(val); +} + +template <> +Value ValueConverter::VisitInteger(int16_t val) { + return Value::SMALLINT(val); +} + +template <> +Value ValueConverter::VisitInteger(int32_t val) { + return Value::INTEGER(val); +} + +template <> +Value ValueConverter::VisitInteger(int64_t val) { + return Value::BIGINT(val); +} + +template <> +Value ValueConverter::VisitInteger(hugeint_t val) { + return Value::HUGEINT(val); +} + +template <> +Value ValueConverter::VisitInteger(uint8_t val) { + return Value::UTINYINT(val); +} + +template <> +Value ValueConverter::VisitInteger(uint16_t val) { + return Value::USMALLINT(val); +} + +template <> +Value ValueConverter::VisitInteger(uint32_t val) { + return Value::UINTEGER(val); +} + +template <> +Value ValueConverter::VisitInteger(uint64_t val) { + return Value::UBIGINT(val); +} + +template <> +Value ValueConverter::VisitInteger(uhugeint_t val) { + return Value::UHUGEINT(val); +} + +} // namespace + +Value VariantUtils::ConvertVariantToValue(const UnifiedVariantVectorData &variant, idx_t row, uint32_t values_idx) { + return VariantVisitor::Visit(variant, row, values_idx); +} + +void VariantUtils::FinalizeVariantKeys(Vector &variant, OrderedOwningStringMap &dictionary, + SelectionVector &sel, idx_t sel_size) { + auto &keys = VariantVector::GetKeys(variant); + auto &keys_entry = ListVector::GetEntry(keys); + auto keys_entry_data = FlatVector::GetData(keys_entry); + + bool already_sorted = true; + + vector unsorted_to_sorted(dictionary.size()); + auto it = dictionary.begin(); + for (uint32_t sorted_idx = 0; sorted_idx < dictionary.size(); sorted_idx++) { + auto unsorted_idx = it->second; + if (unsorted_idx != sorted_idx) { + already_sorted = false; + } + unsorted_to_sorted[unsorted_idx] = sorted_idx; + D_ASSERT(sorted_idx < ListVector::GetListSize(keys)); + keys_entry_data[sorted_idx] = it->first; + auto size = static_cast(keys_entry_data[sorted_idx].GetSize()); + keys_entry_data[sorted_idx].SetSizeAndFinalize(size, size); + it++; } - default: - throw InternalException("VariantLogicalType(%d) not handled", static_cast(type_id)); + + if (!already_sorted) { + //! Adjust the selection vector to point to the right dictionary index + for (idx_t i = 0; i < sel_size; i++) { + auto &entry = sel[i]; + auto sorted_idx = unsorted_to_sorted[entry]; + entry = sorted_idx; + } } } @@ -352,12 +441,14 @@ bool VariantUtils::Verify(Vector &variant, const SelectionVector &sel_p, idx_t c if (key_id.validity.RowIsValid(key_id_index)) { auto children_key_id = key_id_data[key_id_index]; D_ASSERT(children_key_id < keys_list_entry.length); + (void)children_key_id; } auto value_id_index = value_id.sel->get_index(j + children_list_entry.offset); D_ASSERT(value_id.validity.RowIsValid(value_id_index)); auto children_value_id = value_id_data[value_id_index]; D_ASSERT(children_value_id < values_list_entry.length); + (void)children_value_id; } //! verify values @@ -397,6 +488,7 @@ bool VariantUtils::Verify(Vector &variant, const SelectionVector &sel_p, idx_t c } else { D_ASSERT(!key_id.validity.RowIsValid(child_key_id_index)); } + (void)child_key_id_index; } break; } diff --git a/src/function/table/CMakeLists.txt b/src/function/table/CMakeLists.txt index e9c2a32d9ef6..e67a9fd7d40b 100644 --- a/src/function/table/CMakeLists.txt +++ b/src/function/table/CMakeLists.txt @@ -15,6 +15,7 @@ add_library_unity( repeat_row.cpp copy_csv.cpp read_csv.cpp + read_duckdb.cpp sniff_csv.cpp read_file.cpp system_functions.cpp diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index 600e50f49436..1ffd6e7ee4d5 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -280,7 +280,31 @@ struct GlobalWriteCSVData : public GlobalFunctionData { return writer.FileSize(); } + unique_ptr GetLocalState(ClientContext &context, const idx_t flush_size) { + { + lock_guard guard(local_state_lock); + if (!local_states.empty()) { + auto result = std::move(local_states.back()); + local_states.pop_back(); + return result; + } + } + auto result = make_uniq(context, flush_size); + result->require_manual_flush = true; + return result; + } + + void StoreLocalState(unique_ptr lstate) { + lock_guard guard(local_state_lock); + lstate->Reset(); + local_states.push_back(std::move(lstate)); + } + CSVWriter writer; + +private: + mutex local_state_lock; + vector> local_states; }; static unique_ptr WriteCSVInitializeLocal(ExecutionContext &context, FunctionData &bind_data) { @@ -371,9 +395,7 @@ CopyFunctionExecutionMode WriteCSVExecutionMode(bool preserve_insertion_order, b // Prepare Batch //===--------------------------------------------------------------------===// struct WriteCSVBatchData : public PreparedBatchData { - explicit WriteCSVBatchData(ClientContext &context, const idx_t flush_size) - : writer_local_state(make_uniq(context, flush_size)) { - writer_local_state->require_manual_flush = true; + explicit WriteCSVBatchData(unique_ptr writer_state) : writer_local_state(std::move(writer_state)) { } //! The thread-local buffer to write data into @@ -397,7 +419,8 @@ unique_ptr WriteCSVPrepareBatch(ClientContext &context, Funct auto &global_state = gstate.Cast(); // write CSV chunks to the batch data - auto batch = make_uniq(context, NextPowerOfTwo(collection->SizeInBytes())); + auto local_writer_state = global_state.GetLocalState(context, NextPowerOfTwo(collection->SizeInBytes())); + auto batch = make_uniq(std::move(local_writer_state)); for (auto &chunk : collection->Chunks()) { WriteCSVChunkInternal(global_state.writer, *batch->writer_local_state, cast_chunk, chunk, executor); } @@ -412,6 +435,7 @@ void WriteCSVFlushBatch(ClientContext &context, FunctionData &bind_data, GlobalF auto &csv_batch = batch.Cast(); auto &global_state = gstate.Cast(); global_state.writer.Flush(*csv_batch.writer_local_state); + global_state.StoreLocalState(std::move(csv_batch.writer_local_state)); } //===--------------------------------------------------------------------===// diff --git a/src/function/table/direct_file_reader.cpp b/src/function/table/direct_file_reader.cpp index 8aa6aba352e1..eacfe1de141d 100644 --- a/src/function/table/direct_file_reader.cpp +++ b/src/function/table/direct_file_reader.cpp @@ -64,7 +64,7 @@ void DirectFileReader::Scan(ClientContext &context, GlobalTableFunctionState &gl if (FileSystem::IsRemoteFile(file.path)) { flags |= FileFlags::FILE_FLAGS_DIRECT_IO; } - file_handle = fs.OpenFile(QueryContext(context), file, flags); + file_handle = fs.OpenFile(context, file, flags); } for (idx_t col_idx = 0; col_idx < state.column_ids.size(); col_idx++) { diff --git a/src/function/table/query_function.cpp b/src/function/table/query_function.cpp index c44b1919c041..4da6599b1009 100644 --- a/src/function/table/query_function.cpp +++ b/src/function/table/query_function.cpp @@ -3,17 +3,32 @@ #include "duckdb/function/table/range.hpp" #include "duckdb/function/function_set.hpp" #include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/parser/statement/multi_statement.hpp" +#include "duckdb/parser/statement/select_statement.hpp" namespace duckdb { static unique_ptr ParseSubquery(const string &query, const ParserOptions &options, const string &err_msg) { Parser parser(options); parser.ParseQuery(query); - if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { + if (parser.statements.size() != 1) { + throw ParserException(err_msg); + } + + auto &stmt = parser.statements[0]; + + if (stmt->type == StatementType::SELECT_STATEMENT) { + // Regular SELECT statement + auto select_stmt = unique_ptr_cast(std::move(stmt)); + return duckdb::make_uniq(std::move(select_stmt)); + } else if (stmt->type == StatementType::MULTI_STATEMENT) { + // MultiStatement (e.g., from PIVOT statements that create enum types) + throw ParserException( + "PIVOT statements without explicit IN clauses are not supported in query() function. " + "Please specify the pivot values explicitly, e.g.: PIVOT ... ON col IN (val1, val2, ...)"); + } else { throw ParserException(err_msg); } - auto select_stmt = unique_ptr_cast(std::move(parser.statements[0])); - return duckdb::make_uniq(std::move(select_stmt)); } static string UnionTablesQuery(TableFunctionBindInput &input) { diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 415bb32375cb..28a58afa076c 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -1,4 +1,5 @@ #include "duckdb/function/table/read_csv.hpp" +#include "duckdb/function/table/read_duckdb.hpp" #include "duckdb/common/enum_util.hpp" #include "duckdb/common/multi_file/multi_file_reader.hpp" @@ -192,8 +193,10 @@ unique_ptr ReadCSVReplacement(ClientContext &context, ReplacementScanI void BuiltinFunctions::RegisterReadFunctions() { CSVCopyFunction::RegisterFunction(*this); ReadCSVTableFunction::RegisterFunction(*this); + AddFunction(MultiFileReader::CreateFunctionSet(ReadDuckDBTableFunction::GetFunction())); auto &config = DBConfig::GetConfig(*transaction.db); config.replacement_scans.emplace_back(ReadCSVReplacement); + config.replacement_scans.emplace_back(ReadDuckDBTableFunction::ReplacementScan); } } // namespace duckdb diff --git a/src/function/table/read_duckdb.cpp b/src/function/table/read_duckdb.cpp new file mode 100644 index 000000000000..c68f1c32e461 --- /dev/null +++ b/src/function/table/read_duckdb.cpp @@ -0,0 +1,530 @@ +#include "duckdb/function/table/read_duckdb.hpp" +#include "duckdb/common/multi_file/multi_file_reader.hpp" +#include "duckdb/common/multi_file/multi_file_function.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/main/attached_database.hpp" +#include "duckdb/function/replacement_scan.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" + +namespace duckdb { + +struct DuckDBMultiFileInfo : MultiFileReaderInterface { + static unique_ptr CreateInterface(ClientContext &context); + + unique_ptr InitializeOptions(ClientContext &context, + optional_ptr info) override; + bool ParseCopyOption(ClientContext &context, const string &key, const vector &values, + BaseFileReaderOptions &options, vector &expected_names, + vector &expected_types) override; + bool ParseOption(ClientContext &context, const string &key, const Value &val, MultiFileOptions &file_options, + BaseFileReaderOptions &options) override; + void FinalizeCopyBind(ClientContext &context, BaseFileReaderOptions &options, const vector &expected_names, + const vector &expected_types) override; + void FinalizeBindData(MultiFileBindData &multi_file_data) override; + + unique_ptr InitializeBindData(MultiFileBindData &multi_file_data, + unique_ptr options) override; + void BindReader(ClientContext &context, vector &return_types, vector &names, + MultiFileBindData &bind_data) override; + unique_ptr InitializeGlobalState(ClientContext &context, MultiFileBindData &bind_data, + MultiFileGlobalState &global_state) override; + unique_ptr InitializeLocalState(ExecutionContext &, GlobalTableFunctionState &) override; + shared_ptr CreateReader(ClientContext &context, GlobalTableFunctionState &gstate, + BaseUnionData &union_data, const MultiFileBindData &bind_data_p) override; + shared_ptr CreateReader(ClientContext &context, GlobalTableFunctionState &gstate, + const OpenFileInfo &file, idx_t file_idx, + const MultiFileBindData &bind_data) override; + shared_ptr CreateReader(ClientContext &context, const OpenFileInfo &file, + BaseFileReaderOptions &options, + const MultiFileOptions &file_options) override; + void FinishReading(ClientContext &context, GlobalTableFunctionState &global_state, + LocalTableFunctionState &local_state) override; + unique_ptr GetCardinality(const MultiFileBindData &bind_data, idx_t file_count) override; + void GetVirtualColumns(ClientContext &, MultiFileBindData &, virtual_column_map_t &result) override; + unique_ptr Copy() override; + FileGlobInput GetGlobInput() override; +}; + +class DuckDBFileReaderOptions : public BaseFileReaderOptions { +public: + string schema_name; + string table_name; + + bool Matches(TableCatalogEntry &table) const; + bool HasSelection() const; + string PrintOptions() const; + string GetCandidates(const vector> &tables) const; +}; + +struct DuckDBReadBindData : TableFunctionData { + unique_ptr options; + optional_idx initial_file_cardinality; + + unique_ptr Copy() const override { + auto result = make_uniq(); + result->options = make_uniq(*options); + result->initial_file_cardinality = initial_file_cardinality; + return std::move(result); + } +}; + +struct AttachedDatabaseWrapper { + AttachedDatabaseWrapper(ClientContext &context, shared_ptr attached_database_p); + ~AttachedDatabaseWrapper(); + + ClientContext &context; + shared_ptr attached_database; + optional_ptr table_entry; +}; + +class DuckDBReader : public BaseFileReader { +public: + DuckDBReader(ClientContext &context, OpenFileInfo file, const DuckDBFileReaderOptions &options); + ~DuckDBReader() override; + +public: + bool TryInitializeScan(ClientContext &context, GlobalTableFunctionState &gstate, + LocalTableFunctionState &lstate) override; + void Scan(ClientContext &context, GlobalTableFunctionState &global_state, LocalTableFunctionState &local_state, + DataChunk &chunk) override; + shared_ptr GetUnionData(idx_t file_idx) override; + void FinishFile(ClientContext &context, GlobalTableFunctionState &gstate) override; + double GetProgressInFile(ClientContext &context) override; + unique_ptr GetStatistics(ClientContext &context, const string &name) override; + void AddVirtualColumn(column_t virtual_column_id) override; + string GetReaderType() const override { + return "duckdb"; + } + optional_idx NumRows(); + AttachedDatabase &GetAttachedDatabase(); + TableCatalogEntry &GetTableEntry(); + +private: + ClientContext &context; + shared_ptr db_wrapper; + TableFunction scan_function; + unique_ptr bind_data; + unique_ptr global_state; + atomic finished; + idx_t column_count; + string schema_name; + string table_name; +}; + +struct DuckDBReadGlobalState : GlobalTableFunctionState {}; + +struct DuckDBReadLocalState : LocalTableFunctionState { + unique_ptr local_state; + shared_ptr attached_database; +}; + +string DuckDBFileReaderOptions::GetCandidates(const vector> &tables) const { + if (tables.empty()) { + return string(); + } + case_insensitive_map_t table_names; + for (auto &table : tables) { + table_names[table.get().name]++; + } + vector candidate_list; + for (auto &table_ref : tables) { + auto &table = table_ref.get(); + if (table_names[table.name] > 1) { + // name conflicts across schemas - add the schema name + auto &schema = table.ParentSchema(); + candidate_list.push_back(schema.name + "." + table.name); + } else { + candidate_list.push_back(table.name); + } + } + string search_term = schema_name; + if (!search_term.empty()) { + search_term += "."; + } + search_term += table_name; + return StringUtil::CandidatesErrorMessage(candidate_list, search_term, "Candidates"); +} + +bool DuckDBFileReaderOptions::HasSelection() const { + if (!table_name.empty()) { + return true; + } + if (!schema_name.empty()) { + return true; + } + return false; +} + +string DuckDBFileReaderOptions::PrintOptions() const { + string options; + if (!schema_name.empty()) { + options += "schema_name=\"" + schema_name + "\""; + } + if (!options.empty()) { + options += ", "; + } + if (!table_name.empty()) { + options += "table_name=\"" + table_name + "\""; + } + return options; +} + +bool DuckDBFileReaderOptions::Matches(TableCatalogEntry &table) const { + if (!schema_name.empty() && !StringUtil::CIEquals(table.ParentSchema().name, schema_name)) { + return false; + } + if (!table_name.empty() && !StringUtil::CIEquals(table.name, table_name)) { + return false; + } + return true; +} + +AttachedDatabaseWrapper::AttachedDatabaseWrapper(ClientContext &context, + shared_ptr attached_database_p) + : context(context), attached_database(std::move(attached_database_p)) { +} + +AttachedDatabaseWrapper::~AttachedDatabaseWrapper() { + if (attached_database) { + auto &db_manager = DatabaseManager::Get(context); + db_manager.DetachDatabase(context, attached_database->GetName(), OnEntryNotFound::RETURN_NULL); + attached_database.reset(); + } +} +DuckDBReader::DuckDBReader(ClientContext &context_p, OpenFileInfo file_p, const DuckDBFileReaderOptions &options) + : BaseFileReader(std::move(file_p)), context(context_p), finished(false) { + auto &attached = GetAttachedDatabase(); + auto &catalog = attached.GetCatalog(); + vector> tables; + vector> candidate_tables; + catalog.ScanSchemas(context, [&](SchemaCatalogEntry &schema) { + schema.Scan(CatalogType::TABLE_ENTRY, [&](CatalogEntry &entry) { + if (entry.type != CatalogType::TABLE_ENTRY) { + return; + } + auto &table = entry.Cast(); + if (options.Matches(table)) { + tables.push_back(table); + } + candidate_tables.push_back(table); + }); + }); + if (tables.size() != 1) { + string error_msg = tables.empty() ? "does not have any tables" : "has multiple tables"; + string extra_info; + if (!options.HasSelection()) { + extra_info = "\nSelect a table using `table_name=''"; + } else { + extra_info = " matching " + options.PrintOptions(); + } + string candidate_str = options.GetCandidates(candidate_tables); + throw BinderException("Database \"%s\" %s%s%s", file.path, error_msg, extra_info, candidate_str); + } + auto &table = tables[0].get(); + for (auto &col : table.GetColumns().Logical()) { + columns.emplace_back(col.Name(), col.Type()); + } + column_count = columns.size(); + schema_name = table.ParentSchema().name; + table_name = table.name; + db_wrapper->table_entry = table; +} + +DuckDBReader::~DuckDBReader() { +} + +AttachedDatabase &DuckDBReader::GetAttachedDatabase() { + if (!db_wrapper) { + auto &db_manager = DatabaseManager::Get(context); + AttachInfo info; + info.path = file.path; + // use invalid UTF-8 so that a conflicting database name cannot be attached by a user + info.name = "\x80__duckdb_reader_" + info.path; + + info.on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT; + unordered_map attach_kv; + AttachOptions attach_options(attach_kv, AccessMode::READ_ONLY); + attach_options.visibility = AttachVisibility::HIDDEN; + + auto attached = db_manager.AttachDatabase(context, info, attach_options); + db_wrapper = make_shared_ptr(context, std::move(attached)); + } + return *db_wrapper->attached_database; +} + +TableCatalogEntry &DuckDBReader::GetTableEntry() { + auto &attached = GetAttachedDatabase(); + if (!db_wrapper->table_entry) { + auto &catalog = attached.GetCatalog(); + db_wrapper->table_entry = + catalog.GetEntry(context, schema_name, table_name, OnEntryNotFound::THROW_EXCEPTION); + } + return *db_wrapper->table_entry; +} + +bool DuckDBReader::TryInitializeScan(ClientContext &context, GlobalTableFunctionState &gstate, + LocalTableFunctionState &lstate_p) { + auto &lstate = lstate_p.Cast(); + if (finished) { + lstate.attached_database.reset(); + return false; + } + if (!global_state) { + lstate.attached_database.reset(); + auto &table_entry = GetTableEntry(); + scan_function = table_entry.GetScanFunction(context, bind_data); + for (auto &col : column_indexes) { + if (col.GetPrimaryIndex() >= column_count) { + col = ColumnIndex(COLUMN_IDENTIFIER_ROW_ID); + } else { + auto &column = table_entry.GetColumn(LogicalIndex(col.GetPrimaryIndex())); + if (column.Generated()) { + throw NotImplementedException("Unsupported: read_duckdb cannot read generated column %s", + column.Name()); + } + } + } + + // initialize the scan over this table + TableFunctionInitInput input(bind_data.get(), column_indexes, vector(), filters.get()); + global_state = scan_function.init_global(context, input); + } + AssignSharedPointer(lstate.attached_database, db_wrapper); + // initialize the local scan + ThreadContext thread(context); + ExecutionContext exec_context(context, thread, nullptr); + TableFunctionInitInput input(bind_data.get(), column_indexes, vector(), filters.get()); + lstate.local_state = scan_function.init_local(exec_context, input, global_state.get()); + return true; +} + +void DuckDBReader::Scan(ClientContext &context, GlobalTableFunctionState &gstate_p, LocalTableFunctionState &lstate_p, + DataChunk &chunk) { + chunk.Reset(); + auto &lstate = lstate_p.Cast(); + TableFunctionInput input(bind_data.get(), lstate.local_state, global_state); + scan_function.function(context, input, chunk); + if (chunk.size() == 0) { + finished = true; + } +} + +void DuckDBReader::FinishFile(ClientContext &context, GlobalTableFunctionState &gstate) { + db_wrapper.reset(); +} + +optional_idx DuckDBReader::NumRows() { + auto &table_entry = GetTableEntry(); + return table_entry.GetStorage().GetTotalRows(); +} + +unique_ptr DuckDBReader::GetStatistics(ClientContext &context, const string &name) { + if (!scan_function.statistics) { + return BaseFileReader::GetStatistics(context, name); + } + auto &table_entry = GetTableEntry(); + if (!table_entry.ColumnExists(name)) { + return nullptr; + } + return scan_function.statistics(context, bind_data.get(), table_entry.GetColumn(name).Logical().index); +} + +double DuckDBReader::GetProgressInFile(ClientContext &context) { + if (!scan_function.table_scan_progress || !global_state) { + return BaseFileReader::GetProgressInFile(context); + } + return scan_function.table_scan_progress(context, bind_data.get(), global_state.get()); +} + +void DuckDBReader::AddVirtualColumn(column_t virtual_column_id) { + if (virtual_column_id != COLUMN_IDENTIFIER_ROW_ID) { + throw InternalException("Unsupported virtual column id %d for duckdb reader", virtual_column_id); + } +} + +unique_ptr DuckDBMultiFileInfo::CreateInterface(ClientContext &context) { + return make_uniq(); +} + +unique_ptr DuckDBMultiFileInfo::InitializeOptions(ClientContext &context, + optional_ptr info) { + return make_uniq(); +} + +bool DuckDBMultiFileInfo::ParseCopyOption(ClientContext &context, const string &key, const vector &values, + BaseFileReaderOptions &options, vector &expected_names, + vector &expected_types) { + return false; +} + +bool DuckDBMultiFileInfo::ParseOption(ClientContext &context, const string &key, const Value &val, + MultiFileOptions &file_options, BaseFileReaderOptions &options_p) { + auto &options = options_p.Cast(); + if (key == "schema_name") { + options.schema_name = StringValue::Get(val); + return true; + } + if (key == "table_name") { + options.table_name = StringValue::Get(val); + return true; + } + return false; +} + +void DuckDBMultiFileInfo::FinalizeCopyBind(ClientContext &context, BaseFileReaderOptions &options, + const vector &expected_names, + const vector &expected_types) { + throw InternalException("Unimplemented method in DuckDBMultiFileInfo"); +} + +void DuckDBMultiFileInfo::FinalizeBindData(MultiFileBindData &multi_file_data) { + auto &bind_data = multi_file_data.bind_data->Cast(); + if (multi_file_data.initial_reader) { + auto &initial_reader = multi_file_data.initial_reader->Cast(); + bind_data.initial_file_cardinality = initial_reader.NumRows(); + } +} + +unique_ptr DuckDBMultiFileInfo::InitializeBindData(MultiFileBindData &multi_file_data, + unique_ptr options_p) { + auto result = make_uniq(); + result->options = unique_ptr_cast(std::move(options_p)); + return std::move(result); +} + +void DuckDBMultiFileInfo::BindReader(ClientContext &context, vector &return_types, vector &names, + MultiFileBindData &bind_data) { + auto &duckdb_bind_data = bind_data.bind_data->Cast(); + bind_data.reader_bind = + bind_data.multi_file_reader->BindReader(context, return_types, names, *bind_data.file_list, bind_data, + *duckdb_bind_data.options, bind_data.file_options); +} + +unique_ptr DuckDBMultiFileInfo::InitializeGlobalState(ClientContext &context, + MultiFileBindData &bind_data, + MultiFileGlobalState &global_state) { + return make_uniq(); +} + +unique_ptr DuckDBMultiFileInfo::InitializeLocalState(ExecutionContext &, + GlobalTableFunctionState &) { + return make_uniq(); +} + +struct DuckDBReaderUnionData : BaseUnionData { + explicit DuckDBReaderUnionData(OpenFileInfo file_p) : BaseUnionData(std::move(file_p)) { + } +}; + +shared_ptr DuckDBMultiFileInfo::CreateReader(ClientContext &context, GlobalTableFunctionState &gstate, + BaseUnionData &union_data_p, + const MultiFileBindData &multi_bind_data) { + auto &union_data = union_data_p.Cast(); + auto &bind_data = multi_bind_data.bind_data->Cast(); + return make_shared_ptr(context, union_data.file, *bind_data.options); +} + +shared_ptr DuckDBMultiFileInfo::CreateReader(ClientContext &context, GlobalTableFunctionState &gstate, + const OpenFileInfo &file, idx_t file_idx, + const MultiFileBindData &multi_bind_data) { + auto &bind_data = multi_bind_data.bind_data->Cast(); + return make_shared_ptr(context, file, *bind_data.options); +} + +shared_ptr DuckDBMultiFileInfo::CreateReader(ClientContext &context, const OpenFileInfo &file, + BaseFileReaderOptions &options, + const MultiFileOptions &file_options) { + return make_shared_ptr(context, file, options.Cast()); +} + +shared_ptr DuckDBReader::GetUnionData(idx_t file_idx) { + auto result = make_uniq(file); + for (auto &column : columns) { + result->names.push_back(column.name); + result->types.push_back(column.type); + } + result->reader = shared_from_this(); + return std::move(result); +} + +void DuckDBMultiFileInfo::FinishReading(ClientContext &context, GlobalTableFunctionState &global_state, + LocalTableFunctionState &lstate_p) { + auto &lstate = lstate_p.Cast(); + lstate.attached_database.reset(); +} + +unique_ptr DuckDBMultiFileInfo::GetCardinality(const MultiFileBindData &bind_data_p, idx_t file_count) { + auto &bind_data = bind_data_p.bind_data->Cast(); + idx_t estimated_cardinality = file_count; + if (bind_data.initial_file_cardinality.IsValid()) { + estimated_cardinality = file_count * bind_data.initial_file_cardinality.GetIndex(); + } + return make_uniq(estimated_cardinality); +} + +unique_ptr DuckDBMultiFileInfo::Copy() { + return make_uniq(); +} + +FileGlobInput DuckDBMultiFileInfo::GetGlobInput() { + return FileGlobInput(FileGlobOptions::FALLBACK_GLOB, "db"); +} + +void DuckDBMultiFileInfo::GetVirtualColumns(ClientContext &, MultiFileBindData &, virtual_column_map_t &result) { + result.insert(make_pair(COLUMN_IDENTIFIER_ROW_ID, TableColumn("rowid", LogicalType::BIGINT))); +} + +void ReadDuckDBAddNamedParameters(TableFunction &table_function) { + table_function.named_parameters["schema_name"] = LogicalType::VARCHAR; + table_function.named_parameters["table_name"] = LogicalType::VARCHAR; + + MultiFileReader::AddParameters(table_function); +} + +static vector DuckDBGetRowIdColumns(ClientContext &, optional_ptr) { + vector result; + result.emplace_back(MultiFileReader::COLUMN_IDENTIFIER_FILE_INDEX); + result.emplace_back(COLUMN_IDENTIFIER_ROW_ID); + return result; +} + +static bool DuckDBScanPushdownExpression(ClientContext &context, const LogicalGet &get, Expression &expr) { + return true; +} + +TableFunction ReadDuckDBTableFunction::GetFunction() { + MultiFileFunction read_duckdb("read_duckdb"); + read_duckdb.statistics = MultiFileFunction::MultiFileScanStats; + read_duckdb.get_row_id_columns = DuckDBGetRowIdColumns; + read_duckdb.pushdown_expression = DuckDBScanPushdownExpression; + read_duckdb.filter_pushdown = true; + read_duckdb.filter_prune = true; + read_duckdb.late_materialization = true; + ReadDuckDBAddNamedParameters(read_duckdb); + return static_cast(read_duckdb); +} + +unique_ptr ReadDuckDBTableFunction::ReplacementScan(ClientContext &context, ReplacementScanInput &input, + optional_ptr) { + auto table_name = ReplacementScan::GetFullPath(input); + auto lower_name = StringUtil::Lower(table_name); + if (!StringUtil::EndsWith(lower_name, ".db") && !StringUtil::Contains(lower_name, ".db?") && + !StringUtil::EndsWith(lower_name, ".ddb") && !StringUtil::Contains(lower_name, ".ddb?") && + !StringUtil::EndsWith(lower_name, ".duckdb") && !StringUtil::Contains(lower_name, ".duckdb?")) { + return nullptr; + } + auto table_function = make_uniq(); + vector> children; + children.push_back(make_uniq(Value(table_name))); + table_function->function = make_uniq("read_duckdb", std::move(children)); + + if (!FileSystem::HasGlob(table_name)) { + auto &fs = FileSystem::GetFileSystem(context); + table_function->alias = fs.ExtractBaseName(table_name); + } + return std::move(table_function); +} + +} // namespace duckdb diff --git a/src/function/table/system/CMakeLists.txt b/src/function/table/system/CMakeLists.txt index 965837223ea7..d622ae6db82f 100644 --- a/src/function/table/system/CMakeLists.txt +++ b/src/function/table/system/CMakeLists.txt @@ -1,6 +1,7 @@ add_library_unity( duckdb_table_func_system OBJECT + duckdb_connection_count.cpp duckdb_approx_database_count.cpp duckdb_columns.cpp duckdb_constraints.cpp diff --git a/src/function/table/system/duckdb_connection_count.cpp b/src/function/table/system/duckdb_connection_count.cpp new file mode 100644 index 000000000000..ce7857f3bffa --- /dev/null +++ b/src/function/table/system/duckdb_connection_count.cpp @@ -0,0 +1,45 @@ +#include "duckdb/function/table/system_functions.hpp" + +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/database.hpp" +#include "duckdb/main/connection_manager.hpp" + +namespace duckdb { + +struct DuckDBConnectionCountData : public GlobalTableFunctionState { + DuckDBConnectionCountData() : count(0), finished(false) { + } + idx_t count; + bool finished; +}; + +static unique_ptr DuckDBConnectionCountBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + names.emplace_back("count"); + return_types.emplace_back(LogicalType::UBIGINT); + return nullptr; +} + +unique_ptr DuckDBConnectionCountInit(ClientContext &context, TableFunctionInitInput &input) { + auto result = make_uniq(); + auto &conn_manager = context.db->GetConnectionManager(); + result->count = conn_manager.GetConnectionCount(); + return std::move(result); +} + +void DuckDBConnectionCountFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = data_p.global_state->Cast(); + if (data.finished) { + return; + } + output.SetValue(0, 0, Value::UBIGINT(data.count)); + output.SetCardinality(1); + data.finished = true; +} + +void DuckDBConnectionCountFun::RegisterFunction(BuiltinFunctions &set) { + set.AddFunction(TableFunction("duckdb_connection_count", {}, DuckDBConnectionCountFunction, + DuckDBConnectionCountBind, DuckDBConnectionCountInit)); +} + +} // namespace duckdb diff --git a/src/function/table/system/duckdb_databases.cpp b/src/function/table/system/duckdb_databases.cpp index 0e2031f794ae..0e705eb692ad 100644 --- a/src/function/table/system/duckdb_databases.cpp +++ b/src/function/table/system/duckdb_databases.cpp @@ -1,6 +1,7 @@ #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database_manager.hpp" #include "duckdb/main/attached_database.hpp" +#include "duckdb/storage/storage_manager.hpp" namespace duckdb { @@ -38,6 +39,11 @@ static unique_ptr DuckDBDatabasesBind(ClientContext &context, Tabl names.emplace_back("readonly"); return_types.emplace_back(LogicalType::BOOLEAN); + names.emplace_back("encrypted"); + return_types.emplace_back(LogicalType::BOOLEAN); + + names.emplace_back("cipher"); + return_types.emplace_back(LogicalType::VARCHAR); return nullptr; } @@ -61,8 +67,11 @@ void DuckDBDatabasesFunction(ClientContext &context, TableFunctionInput &data_p, idx_t count = 0; while (data.offset < data.entries.size() && count < STANDARD_VECTOR_SIZE) { auto &entry = data.entries[data.offset++]; - auto &attached = *entry; + auto &catalog = attached.GetCatalog(); + if (attached.GetVisibility() == AttachVisibility::HIDDEN) { + continue; + } // return values: idx_t col = 0; @@ -72,12 +81,16 @@ void DuckDBDatabasesFunction(ClientContext &context, TableFunctionInput &data_p, output.SetValue(col++, count, Value::BIGINT(NumericCast(attached.oid))); bool is_internal = attached.IsSystem() || attached.IsTemporary(); bool is_readonly = attached.IsReadOnly(); + string cipher_str; // path, VARCHAR Value db_path; if (!is_internal) { - bool in_memory = attached.GetCatalog().InMemory(); + bool in_memory = catalog.InMemory(); if (!in_memory) { - db_path = Value(attached.GetCatalog().GetDBPath()); + db_path = Value(catalog.GetDBPath()); + } + if (catalog.IsEncrypted()) { + cipher_str = catalog.GetEncryptionCipher(); } } output.SetValue(col++, count, db_path); @@ -88,9 +101,13 @@ void DuckDBDatabasesFunction(ClientContext &context, TableFunctionInput &data_p, // internal, BOOLEAN output.SetValue(col++, count, Value::BOOLEAN(is_internal)); // type, VARCHAR - output.SetValue(col++, count, Value(attached.GetCatalog().GetCatalogType())); + output.SetValue(col++, count, Value(catalog.GetCatalogType())); // readonly, BOOLEAN output.SetValue(col++, count, Value::BOOLEAN(is_readonly)); + // encrypted, BOOLEAN + output.SetValue(col++, count, Value::BOOLEAN(catalog.IsEncrypted())); + // cipher, VARCHAR + output.SetValue(col++, count, cipher_str.empty() ? Value() : Value(cipher_str)); count++; } diff --git a/src/function/table/system/duckdb_functions.cpp b/src/function/table/system/duckdb_functions.cpp index b0c7656fe0bb..044a9c8e665f 100644 --- a/src/function/table/system/duckdb_functions.cpp +++ b/src/function/table/system/duckdb_functions.cpp @@ -17,6 +17,7 @@ #include "duckdb/main/client_data.hpp" namespace duckdb { +constexpr const char *AggregateFunctionCatalogEntry::Name; struct DuckDBFunctionsData : public GlobalTableFunctionState { DuckDBFunctionsData() : offset(0), offset_in_entry(0) { diff --git a/src/function/table/system/duckdb_settings.cpp b/src/function/table/system/duckdb_settings.cpp index 3ec34d908343..9908854bddc7 100644 --- a/src/function/table/system/duckdb_settings.cpp +++ b/src/function/table/system/duckdb_settings.cpp @@ -12,6 +12,10 @@ struct DuckDBSettingValue { string input_type; string scope; vector aliases; + + inline bool operator<(const DuckDBSettingValue &rhs) const { + return name < rhs.name; + }; }; struct DuckDBSettingsData : public GlobalTableFunctionState { @@ -79,7 +83,12 @@ unique_ptr DuckDBSettingsInit(ClientContext &context, if (entry != aliases.end()) { value.aliases = std::move(entry->second); } - + for (auto &alias : value.aliases) { + DuckDBSettingValue alias_value = value; + alias_value.name = StringValue::Get(alias); + alias_value.aliases.clear(); + result->settings.push_back(std::move(alias_value)); + } result->settings.push_back(std::move(value)); } for (auto &ext_param : config.extension_parameters) { @@ -98,6 +107,7 @@ unique_ptr DuckDBSettingsInit(ClientContext &context, result->settings.push_back(std::move(value)); } + std::sort(result->settings.begin(), result->settings.end()); return std::move(result); } diff --git a/src/function/table/system/pragma_storage_info.cpp b/src/function/table/system/pragma_storage_info.cpp index 5500c1c5d522..7ba6cfd692a0 100644 --- a/src/function/table/system/pragma_storage_info.cpp +++ b/src/function/table/system/pragma_storage_info.cpp @@ -88,7 +88,7 @@ static unique_ptr PragmaStorageInfoBind(ClientContext &context, Ta Binder::BindSchemaOrCatalog(context, qname.catalog, qname.schema); auto &table_entry = Catalog::GetEntry(context, qname.catalog, qname.schema, qname.name); auto result = make_uniq(table_entry); - result->column_segments_info = table_entry.GetColumnSegmentInfo(); + result->column_segments_info = table_entry.GetColumnSegmentInfo(context); return std::move(result); } diff --git a/src/function/table/system_functions.cpp b/src/function/table/system_functions.cpp index d10ec5d317f3..0a6a035073a0 100644 --- a/src/function/table/system_functions.cpp +++ b/src/function/table/system_functions.cpp @@ -18,6 +18,7 @@ void BuiltinFunctions::RegisterSQLiteFunctions() { PragmaDatabaseSize::RegisterFunction(*this); PragmaUserAgent::RegisterFunction(*this); + DuckDBConnectionCountFun::RegisterFunction(*this); DuckDBApproxDatabaseCountFun::RegisterFunction(*this); DuckDBColumnsFun::RegisterFunction(*this); DuckDBConstraintsFun::RegisterFunction(*this); diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 9e2755e7e1c7..99a9bcf7976d 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -265,6 +265,9 @@ class DuckTableScanState : public TableScanGlobalState { l_state.scan_state.options.force_fetch_row = ClientConfig::GetConfig(context).force_fetch_row; do { + if (context.interrupted) { + throw InterruptException(); + } if (bind_data.is_create_index) { storage.CreateIndexScan(l_state.scan_state, output, TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED); diff --git a/src/function/window/window_merge_sort_tree.cpp b/src/function/window/window_merge_sort_tree.cpp index 6af3d0e5bd5e..5943e6228627 100644 --- a/src/function/window/window_merge_sort_tree.cpp +++ b/src/function/window/window_merge_sort_tree.cpp @@ -1,5 +1,8 @@ #include "duckdb/function/window/window_merge_sort_tree.hpp" +#include "duckdb/main/client_config.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/common/types/column/column_data_collection.hpp" #include #include diff --git a/src/function/window/window_rank_function.cpp b/src/function/window/window_rank_function.cpp index af70521a06e6..e9898762a702 100644 --- a/src/function/window/window_rank_function.cpp +++ b/src/function/window/window_rank_function.cpp @@ -1,6 +1,7 @@ #include "duckdb/function/window/window_rank_function.hpp" #include "duckdb/function/window/window_shared_expressions.hpp" #include "duckdb/function/window/window_token_tree.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/planner/expression/bound_window_expression.hpp" namespace duckdb { diff --git a/src/function/window/window_rownumber_function.cpp b/src/function/window/window_rownumber_function.cpp index f0929d6423fc..8b5401c5d33c 100644 --- a/src/function/window/window_rownumber_function.cpp +++ b/src/function/window/window_rownumber_function.cpp @@ -1,6 +1,7 @@ #include "duckdb/function/window/window_rownumber_function.hpp" #include "duckdb/function/window/window_shared_expressions.hpp" #include "duckdb/function/window/window_token_tree.hpp" +#include "duckdb/main/client_context.hpp" #include "duckdb/planner/expression/bound_window_expression.hpp" namespace duckdb { diff --git a/src/include/duckdb.h b/src/include/duckdb.h index 2f7c23ef3b52..1e8fdab705b7 100644 --- a/src/include/duckdb.h +++ b/src/include/duckdb.h @@ -35,11 +35,7 @@ #ifdef DUCKDB_STATIC_BUILD #define DUCKDB_EXTENSION_API #else -#ifdef DUCKDB_BUILD_LOADABLE_EXTENSION #define DUCKDB_EXTENSION_API __declspec(dllexport) -#else -#define DUCKDB_EXTENSION_API -#endif #endif #else #define DUCKDB_EXTENSION_API __attribute__((visibility("default"))) @@ -245,6 +241,20 @@ typedef enum duckdb_error_type { //! An enum over DuckDB's different cast modes. typedef enum duckdb_cast_mode { DUCKDB_CAST_NORMAL = 0, DUCKDB_CAST_TRY = 1 } duckdb_cast_mode; +typedef enum duckdb_file_flag { + DUCKDB_FILE_FLAG_INVALID = 0, + // Open the file with "read" capabilities. + DUCKDB_FILE_FLAG_READ = 1, + // Open the file with "write" capabilities. + DUCKDB_FILE_FLAG_WRITE = 2, + // Create a new file, or open if it already exists. + DUCKDB_FILE_FLAG_CREATE = 3, + // Create a new file, or fail if it already exists. + DUCKDB_FILE_FLAG_CREATE_NEW = 4, + // Open the file in "append" mode. + DUCKDB_FILE_FLAG_APPEND = 5, +} duckdb_file_flag; + //===--------------------------------------------------------------------===// // General type definitions //===--------------------------------------------------------------------===// @@ -760,6 +770,22 @@ typedef struct _duckdb_arrow_options { void *internal_ptr; } * duckdb_arrow_options; +//===--------------------------------------------------------------------===// +// Virtual File System Access +//===--------------------------------------------------------------------===// + +typedef struct _duckdb_file_open_options { + void *internal_ptr; +} * duckdb_file_open_options; + +typedef struct _duckdb_file_system { + void *internal_ptr; +} * duckdb_file_system; + +typedef struct _duckdb_file_handle { + void *internal_ptr; +} * duckdb_file_handle; + //===--------------------------------------------------------------------===// // DuckDB extension access //===--------------------------------------------------------------------===// @@ -870,10 +896,10 @@ Interrupt running query DUCKDB_C_API void duckdb_interrupt(duckdb_connection connection); /*! -Get progress of the running query +Get the progress of the running query. -* @param connection The working connection -* @return -1 if no progress or a percentage of the progress +* @param connection The connection running the query. +* @return The query progress type containing progress information. */ DUCKDB_C_API duckdb_query_progress_type duckdb_query_progress(duckdb_connection connection); @@ -1288,10 +1314,10 @@ DUCKDB_C_API duckdb_result_type duckdb_result_return_type(duckdb_result result); // Safe Fetch Functions //===--------------------------------------------------------------------===// -// These functions will perform conversions if necessary. -// On failure (e.g. if conversion cannot be performed or if the value is NULL) a default value is returned. -// Note that these functions are slow since they perform bounds checking and conversion -// For fast access of values prefer using `duckdb_result_get_chunk` +// This function group is deprecated. +// To access the values in a result, use `duckdb_fetch_chunk` repeatedly. +// For each chunk, use the `duckdb_data_chunk` interface to access any columns and their values. + #ifndef DUCKDB_API_NO_DEPRECATED /*! **DEPRECATION NOTICE**: This method is scheduled for removal in a future release. @@ -1420,8 +1446,7 @@ DUCKDB_C_API duckdb_timestamp duckdb_value_timestamp(duckdb_result *result, idx_ DUCKDB_C_API duckdb_interval duckdb_value_interval(duckdb_result *result, idx_t col, idx_t row); /*! -**DEPRECATED**: Use duckdb_value_string instead. This function does not work correctly if the string contains null -bytes. +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. * @return The text value at the specified location as a null-terminated string, or nullptr if the value cannot be converted. The result must be freed with `duckdb_free`. @@ -1431,16 +1456,12 @@ DUCKDB_C_API char *duckdb_value_varchar(duckdb_result *result, idx_t col, idx_t /*! **DEPRECATION NOTICE**: This method is scheduled for removal in a future release. -No support for nested types, and for other complex types. -The resulting field "string.data" must be freed with `duckdb_free.` - * @return The string value at the specified location. Attempts to cast the result value to string. */ DUCKDB_C_API duckdb_string duckdb_value_string(duckdb_result *result, idx_t col, idx_t row); /*! -**DEPRECATED**: Use duckdb_value_string_internal instead. This function does not work correctly if the string contains -null bytes. +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. * @return The char* value at the specified location. ONLY works on VARCHAR columns and does not auto-cast. If the column is NOT a VARCHAR column this function will return NULL. @@ -1450,8 +1471,8 @@ The result must NOT be freed. DUCKDB_C_API char *duckdb_value_varchar_internal(duckdb_result *result, idx_t col, idx_t row); /*! -**DEPRECATED**: Use duckdb_value_string_internal instead. This function does not work correctly if the string contains -null bytes. +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * @return The char* value at the specified location. ONLY works on VARCHAR columns and does not auto-cast. If the column is NOT a VARCHAR column this function will return NULL. @@ -1808,6 +1829,53 @@ Returns the statement type of the statement to be executed */ DUCKDB_C_API duckdb_statement_type duckdb_prepared_statement_type(duckdb_prepared_statement statement); +/*! +Returns the number of columns present in a the result of the prepared statement. If any of the column types are invalid, +the result will be 1. + +* @param prepared_statement The prepared statement. +* @return The number of columns present in the result of the prepared statement. +*/ +DUCKDB_C_API idx_t duckdb_prepared_statement_column_count(duckdb_prepared_statement prepared_statement); + +/*! +Returns the name of the specified column of the result of the prepared_statement. +The returned string should be freed using `duckdb_free`. + +Returns `nullptr` if the column is out of range. + +* @param prepared_statement The prepared statement. +* @param col_idx The column index. +* @return The column name of the specified column. +*/ +DUCKDB_C_API const char *duckdb_prepared_statement_column_name(duckdb_prepared_statement prepared_statement, + idx_t col_idx); + +/*! +Returns the column type of the specified column of the result of the prepared_statement. + +Returns `DUCKDB_TYPE_INVALID` if the column is out of range. +The return type of this call should be destroyed with `duckdb_destroy_logical_type`. + +* @param prepared_statement The prepared statement to fetch the column type from. +* @param col_idx The column index. +* @return The logical type of the specified column. +*/ +DUCKDB_C_API duckdb_logical_type +duckdb_prepared_statement_column_logical_type(duckdb_prepared_statement prepared_statement, idx_t col_idx); + +/*! +Returns the column type of the specified column of the result of the prepared_statement. + +Returns `DUCKDB_TYPE_INVALID` if the column is out of range. + +* @param prepared_statement The prepared statement to fetch the column type from. +* @param col_idx The column index. +* @return The type of the specified column. +*/ +DUCKDB_C_API duckdb_type duckdb_prepared_statement_column_type(duckdb_prepared_statement prepared_statement, + idx_t col_idx); + //===--------------------------------------------------------------------===// // Bind Values to Prepared Statements //===--------------------------------------------------------------------===// @@ -3263,23 +3331,26 @@ Returns the size of the child vector of the list. DUCKDB_C_API idx_t duckdb_list_vector_get_size(duckdb_vector vector); /*! -Sets the total size of the underlying child-vector of a list vector. +Sets the size of the underlying child-vector of a list vector. +Note that this does NOT reserve the memory in the child buffer, +and that it is possible to set a size exceeding the capacity. +To set the capacity, use `duckdb_list_vector_reserve`. * @param vector The list vector. * @param size The size of the child list. -* @return The duckdb state. Returns DuckDBError if the vector is nullptr. +* @return The duckdb state. Returns DuckDBError, if the vector is nullptr. */ DUCKDB_C_API duckdb_state duckdb_list_vector_set_size(duckdb_vector vector, idx_t size); /*! -Sets the total capacity of the underlying child-vector of a list. - -After calling this method, you must call `duckdb_vector_get_validity` and `duckdb_vector_get_data` to obtain current -data and validity pointers +Sets the capacity of the underlying child-vector of a list vector. +We increment to the next power of two, based on the required capacity. +Thus, the capacity might not match the size of the list (capacity >= size), +which is set via `duckdb_list_vector_set_size`. * @param vector The list vector. -* @param required_capacity the total capacity to reserve. -* @return The duckdb state. Returns DuckDBError if the vector is nullptr. +* @param required_capacity The child buffer capacity to reserve. +* @return The duckdb state. Returns DuckDBError, if the vector is nullptr. */ DUCKDB_C_API duckdb_state duckdb_list_vector_reserve(duckdb_vector vector, idx_t required_capacity); @@ -4602,6 +4673,14 @@ Check if the column at 'index' index of the table has a DEFAULT expression. */ DUCKDB_C_API duckdb_state duckdb_column_has_default(duckdb_table_description table_description, idx_t index, bool *out); +/*! +Return the number of columns of the described table. + +* @param table_description The table_description to query. +* @return The column count. +*/ +DUCKDB_C_API idx_t duckdb_table_description_get_column_count(duckdb_table_description table_description); + /*! Obtain the column name at 'index'. The out result must be destroyed with `duckdb_free`. @@ -4612,6 +4691,17 @@ The out result must be destroyed with `duckdb_free`. */ DUCKDB_C_API char *duckdb_table_description_get_column_name(duckdb_table_description table_description, idx_t index); +/*! +Obtain the column type at 'index'. +The return value must be destroyed with `duckdb_destroy_logical_type`. + +* @param table_description The table_description to query. +* @param index The index of the column to query. +* @return The column type. +*/ +DUCKDB_C_API duckdb_logical_type duckdb_table_description_get_column_type(duckdb_table_description table_description, + idx_t index); + //===--------------------------------------------------------------------===// // Arrow Interface //===--------------------------------------------------------------------===// @@ -5118,6 +5208,150 @@ Folds an expression creating a folded value. DUCKDB_C_API duckdb_error_data duckdb_expression_fold(duckdb_client_context context, duckdb_expression expr, duckdb_value *out_value); +//===--------------------------------------------------------------------===// +// File System Interface +//===--------------------------------------------------------------------===// + +/*! +Get a file system instance associated with the given client context. + +* @param context The client context. +* @return The resulting file system instance. Must be destroyed with `duckdb_destroy_file_system`. +*/ +DUCKDB_C_API duckdb_file_system duckdb_client_context_get_file_system(duckdb_client_context context); + +/*! +Destroys the given file system instance. +* @param file_system The file system instance to destroy. +*/ +DUCKDB_C_API void duckdb_destroy_file_system(duckdb_file_system *file_system); + +/*! +Retrieves the last error that occurred on the given file system instance. + +* @param file_system The file system instance. +* @return The error data. +*/ +DUCKDB_C_API duckdb_error_data duckdb_file_system_error_data(duckdb_file_system file_system); + +/*! +Opens a file at the given path with the specified options. + +* @param file_system The file system instance. +* @param path The path to the file. +* @param options The file open options specifying how to open the file. +* @param out_file The resulting file handle instance, or `nullptr` if the open failed. Must be destroyed with +`duckdb_destroy_file_handle`. +* @return Whether the operation was successful. If not, the error data can be retrieved using +`duckdb_file_system_error_data`. +*/ +DUCKDB_C_API duckdb_state duckdb_file_system_open(duckdb_file_system file_system, const char *path, + duckdb_file_open_options options, duckdb_file_handle *out_file); + +/*! +Creates a new file open options instance with blank settings. + +* @return The new file open options instance. Must be destroyed with `duckdb_destroy_file_open_options`. +*/ +DUCKDB_C_API duckdb_file_open_options duckdb_create_file_open_options(); + +/*! +Sets a specific flag in the file open options. + +* @param options The file open options instance. +* @param flag The flag to set (e.g., read, write). +* @param value If the flag is enabled or disabled. +* @return `DuckDBSuccess` on success or `DuckDBError` if the flag is unrecognized or unsupported by this version of +DuckDB. +*/ +DUCKDB_C_API duckdb_state duckdb_file_open_options_set_flag(duckdb_file_open_options options, duckdb_file_flag flag, + bool value); + +/*! +Destroys the given file open options instance. +* @param options The file open options instance to destroy. +*/ +DUCKDB_C_API void duckdb_destroy_file_open_options(duckdb_file_open_options *options); + +/*! +Destroys the given file handle and deallocates all associated resources. +This will also close the file if it is still open. + +* @param file_handle The file handle to destroy. +*/ +DUCKDB_C_API void duckdb_destroy_file_handle(duckdb_file_handle *file_handle); + +/*! +Retrieves the last error that occurred on the given file handle. + +* @param file_handle The file handle. +* @return The error data. Must be destroyed with `duckdb_destroy_error_data` +*/ +DUCKDB_C_API duckdb_error_data duckdb_file_handle_error_data(duckdb_file_handle file_handle); + +/*! +Reads data from the file into the buffer. + +* @param file_handle The file handle to read from. +* @param buffer The buffer to read data into. +* @param size The number of bytes to read. +* @return The number of bytes actually read, or negative on error. +*/ +DUCKDB_C_API int64_t duckdb_file_handle_read(duckdb_file_handle file_handle, void *buffer, int64_t size); + +/*! +Writes data from the buffer to the file. + +* @param file_handle The file handle to write to. +* @param buffer The buffer containing data to write. +* @param size The number of bytes to write. +* @return The number of bytes actually written, or negative on error. +*/ +DUCKDB_C_API int64_t duckdb_file_handle_write(duckdb_file_handle file_handle, const void *buffer, int64_t size); + +/*! +Tells the current position in the file. + +* @param file_handle The file handle to tell the position of. +* @return The current position in the file, or negative on error. +*/ +DUCKDB_C_API int64_t duckdb_file_handle_tell(duckdb_file_handle file_handle); + +/*! +Gets the size of the file. + +* @param file_handle The file handle to get the size of. +* @return The size of the file in bytes, or negative on error. +*/ +DUCKDB_C_API int64_t duckdb_file_handle_size(duckdb_file_handle file_handle); + +/*! +Seeks to a specific position in the file. + +* @param file_handle The file handle to seek in. +* @return `DuckDBSuccess` on success or `DuckDBError` on failure. If unsuccessful, the error data can be retrieved using +`duckdb_file_handle_error_data`. +*/ +DUCKDB_C_API duckdb_state duckdb_file_handle_seek(duckdb_file_handle file_handle, int64_t position); + +/*! +Synchronizes the file's state with the underlying storage. + +* @param file_handle The file handle to synchronize. +* @return `DuckDBSuccess` on success or `DuckDBError` on failure. If unsuccessful, the error data can be retrieved using +`duckdb_file_handle_error_data`. +*/ +DUCKDB_C_API duckdb_state duckdb_file_handle_sync(duckdb_file_handle file_handle); + +/*! +Closes the given file handle. + +* @param file_handle The file handle to close. +* @return `DuckDBSuccess` on success or `DuckDBError` on failure. If unsuccessful, the error data can be retrieved using +`duckdb_file_handle_error_data`. +*/ +DUCKDB_C_API duckdb_state duckdb_file_handle_close(duckdb_file_handle file_handle); + #endif #ifdef __cplusplus diff --git a/src/include/duckdb/catalog/catalog.hpp b/src/include/duckdb/catalog/catalog.hpp index ac733fb6b013..9817f6c03066 100644 --- a/src/include/duckdb/catalog/catalog.hpp +++ b/src/include/duckdb/catalog/catalog.hpp @@ -26,6 +26,7 @@ #include namespace duckdb { +struct AttachOptions; struct CreateSchemaInfo; struct DropInfo; struct BoundCreateTableInfo; @@ -321,6 +322,12 @@ class Catalog { virtual bool SupportsTimeTravel() const { return false; } + virtual bool IsEncrypted() const { + return false; + } + virtual string GetEncryptionCipher() const { + return string(); + } //! Whether or not this catalog should search a specific type with the standard priority DUCKDB_API virtual CatalogLookupBehavior CatalogTypeLookupRule(CatalogType type) const { @@ -340,6 +347,9 @@ class Catalog { //! Returns the dependency manager of this catalog - if the catalog has anye virtual optional_ptr GetDependencyManager(); + //! Whether attaching a catalog with the given path and attach options would be considered a conflict + virtual bool HasConflictingAttachOptions(const string &path, const AttachOptions &options); + public: template static optional_ptr GetEntry(ClientContext &context, const string &catalog_name, const string &schema_name, diff --git a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp index 0cf71fa734a3..7f206a43d792 100644 --- a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp @@ -47,7 +47,7 @@ class DuckTableEntry : public TableCatalogEntry { TableFunction GetScanFunction(ClientContext &context, unique_ptr &bind_data) override; - vector GetColumnSegmentInfo() override; + vector GetColumnSegmentInfo(const QueryContext &context) override; TableStorageInfo GetStorageInfo(ClientContext &context) override; diff --git a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp index 5cab72c59bb6..1e40319cf270 100644 --- a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp @@ -111,7 +111,7 @@ class TableCatalogEntry : public StandardEntry { static string ColumnNamesToSQL(const ColumnList &columns); //! Returns a list of segment information for this table, if exists - virtual vector GetColumnSegmentInfo(); + virtual vector GetColumnSegmentInfo(const QueryContext &context); //! Returns the storage info of this table virtual TableStorageInfo GetStorageInfo(ClientContext &context) = 0; diff --git a/src/include/duckdb/catalog/default/builtin_types/types.hpp b/src/include/duckdb/catalog/default/builtin_types/types.hpp index e2265c8c7fe4..f3a71b594807 100644 --- a/src/include/duckdb/catalog/default/builtin_types/types.hpp +++ b/src/include/duckdb/catalog/default/builtin_types/types.hpp @@ -19,7 +19,7 @@ struct DefaultType { LogicalTypeId type; }; -using builtin_type_array = std::array; +using builtin_type_array = std::array; static constexpr const builtin_type_array BUILTIN_TYPES{{ {"decimal", LogicalTypeId::DECIMAL}, @@ -97,7 +97,8 @@ static constexpr const builtin_type_array BUILTIN_TYPES{{ {"real", LogicalTypeId::FLOAT}, {"float4", LogicalTypeId::FLOAT}, {"double", LogicalTypeId::DOUBLE}, - {"float8", LogicalTypeId::DOUBLE} + {"float8", LogicalTypeId::DOUBLE}, + {"geometry", LogicalTypeId::GEOMETRY} }}; } // namespace duckdb diff --git a/src/include/duckdb/catalog/default/builtin_types/types.json b/src/include/duckdb/catalog/default/builtin_types/types.json index f40f0d5106f0..52ae689f44b9 100644 --- a/src/include/duckdb/catalog/default/builtin_types/types.json +++ b/src/include/duckdb/catalog/default/builtin_types/types.json @@ -290,5 +290,12 @@ "float8" ], "description": "double precision floating-point number (8 bytes)" + }, + { + "id": "GEOMETRY", + "names": [ + "geometry" + ], + "description": "A data type that represents geometric objects such as points, lines, and polygons." } ] diff --git a/src/include/duckdb/catalog/duck_catalog.hpp b/src/include/duckdb/catalog/duck_catalog.hpp index d683ffd01cff..38156a6eb57a 100644 --- a/src/include/duckdb/catalog/duck_catalog.hpp +++ b/src/include/duckdb/catalog/duck_catalog.hpp @@ -70,6 +70,8 @@ class DuckCatalog : public Catalog { DUCKDB_API bool InMemory() override; DUCKDB_API string GetDBPath() override; + DUCKDB_API bool IsEncrypted() const override; + DUCKDB_API string GetEncryptionCipher() const override; DUCKDB_API optional_idx GetCatalogVersion(ClientContext &context) override; diff --git a/src/include/duckdb/common/arena_containers.hpp b/src/include/duckdb/common/arena_containers.hpp new file mode 100644 index 000000000000..7b91587eac85 --- /dev/null +++ b/src/include/duckdb/common/arena_containers.hpp @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/common/arena_containers.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/arena_stl_allocator.hpp" + +namespace duckdb { + +template +using arena_vector = vector>; + +template +using unsafe_arena_vector = vector>; + +} // namespace duckdb diff --git a/src/include/duckdb/common/arena_stl_allocator.hpp b/src/include/duckdb/common/arena_stl_allocator.hpp new file mode 100644 index 000000000000..46f6bc6a823d --- /dev/null +++ b/src/include/duckdb/common/arena_stl_allocator.hpp @@ -0,0 +1,88 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/common/arena_stl_allocator.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/storage/arena_allocator.hpp" + +namespace duckdb { + +template +class arena_stl_allocator { // NOLINT: match stl case +public: + //! Typedefs + typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef value_type &reference; + typedef value_type const &const_reference; + typedef value_type *pointer; + typedef value_type const *const_pointer; + + //! Propagation traits + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + using is_always_equal = std::false_type; + + //! Rebind + template + struct rebind { + using other = arena_stl_allocator; + }; + +public: + arena_stl_allocator(ArenaAllocator &arena_allocator_p) noexcept // NOLINT: allow implicit conversion + : arena_allocator(arena_allocator_p) { + } + template + arena_stl_allocator(const arena_stl_allocator &other) noexcept // NOLINT: allow implicit conversion + : arena_allocator(other.arena_allocator) { + } + +public: + pointer allocate(size_type n) { // NOLINT: match stl case + arena_allocator.get().AlignNext(); + return reinterpret_cast(arena_allocator.get().Allocate(n * sizeof(T))); + } + + void deallocate(pointer p, size_type n) noexcept { // NOLINT: match stl case + } + + template + void construct(U *p, Args &&...args) { // NOLINT: match stl case + ::new (p) U(std::forward(args)...); + } + + template + void destroy(U *p) noexcept { // NOLINT: match stl case + p->~U(); + } + + pointer address(reference x) const { // NOLINT: match stl case + return &x; + } + + const_pointer address(const_reference x) const { // NOLINT: match stl case + return &x; + } + +public: + bool operator==(const arena_stl_allocator &other) const noexcept { + return RefersToSameObject(arena_allocator, other.arena_allocator); + } + bool operator!=(const arena_stl_allocator &other) const noexcept { + return !(*this == other); + } + +private: + //! Need to use std::reference_wrapper because "reference" is already a typedef + std::reference_wrapper arena_allocator; +}; + +} // namespace duckdb diff --git a/src/include/duckdb/common/assert.hpp b/src/include/duckdb/common/assert.hpp index dbf0744e74c2..4bf4b90e54b7 100644 --- a/src/include/duckdb/common/assert.hpp +++ b/src/include/duckdb/common/assert.hpp @@ -38,3 +38,6 @@ DUCKDB_API void DuckDBAssertInternal(bool condition, const char *condition_name, #define D_ASSERT_IS_ENABLED #endif + +//! Force assertion implementation, which always asserts whatever build type is used. +#define ALWAYS_ASSERT(condition) duckdb::DuckDBAssertInternal(bool(condition), #condition, __FILE__, __LINE__) diff --git a/src/include/duckdb/common/csv_writer.hpp b/src/include/duckdb/common/csv_writer.hpp index b2d0e066e1eb..188d1de7f12a 100644 --- a/src/include/duckdb/common/csv_writer.hpp +++ b/src/include/duckdb/common/csv_writer.hpp @@ -90,9 +90,6 @@ class CSVWriter { //! Closes the writer, optionally writes a postfix void Close(); - unique_ptr InitializeLocalWriteState(ClientContext &context, idx_t flush_size); - unique_ptr InitializeLocalWriteState(DatabaseInstance &db, idx_t flush_size); - vector> string_casts; idx_t BytesWritten(); diff --git a/src/include/duckdb/common/encryption_functions.hpp b/src/include/duckdb/common/encryption_functions.hpp index 8106eee7fb57..07f50b98c7da 100644 --- a/src/include/duckdb/common/encryption_functions.hpp +++ b/src/include/duckdb/common/encryption_functions.hpp @@ -1,41 +1,29 @@ #pragma once #include "duckdb/common/helper.hpp" -#include "duckdb/common/types/value.hpp" -#include "duckdb/common/encryption_state.hpp" -#include "duckdb/common/encryption_key_manager.hpp" -#include "duckdb/storage/object_cache.hpp" namespace duckdb { -struct EncryptionTag { - EncryptionTag() = default; - - data_ptr_t data() { - return tag; - } +class DatabaseInstance; +class AttachedDatabase; +class FileBuffer; - idx_t size() const { - return MainHeader::AES_TAG_LEN; - } +struct EncryptionTag { + EncryptionTag(); + data_ptr_t data(); + idx_t size() const; private: - data_t tag[MainHeader::AES_TAG_LEN]; + unique_ptr tag; }; struct EncryptionNonce { - EncryptionNonce() = default; - - data_ptr_t data() { - return nonce; - } - - idx_t size() const { - return MainHeader::AES_NONCE_LEN; - } + EncryptionNonce(); + data_ptr_t data(); + idx_t size() const; private: - data_t nonce[MainHeader::AES_NONCE_LEN]; + unique_ptr nonce; }; class EncryptionEngine { @@ -53,67 +41,14 @@ class EncryptionEngine { static void AddTempKeyToCache(DatabaseInstance &db); //! Encryption Functions - static void EncryptBlock(DatabaseInstance &db, const string &key_id, FileBuffer &block, + static void EncryptBlock(AttachedDatabase &attached_db, const string &key_id, FileBuffer &block, FileBuffer &temp_buffer_manager, uint64_t delta); - static void DecryptBlock(DatabaseInstance &db, const string &key_id, data_ptr_t internal_buffer, + static void DecryptBlock(AttachedDatabase &attached_db, const string &key_id, data_ptr_t internal_buffer, uint64_t block_size, uint64_t delta); static void EncryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t buffer, idx_t buffer_size, data_ptr_t metadata); - static void DecryptBuffer(EncryptionState &encryption_state, const_data_ptr_t temp_key, data_ptr_t buffer, - idx_t buffer_size, data_ptr_t metadata); - static void DecryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t buffer, idx_t buffer_size, data_ptr_t metadata); -}; -class EncryptionTypes { - -public: - enum CipherType : uint8_t { UNKNOWN = 0, GCM = 1, CTR = 2, CBC = 3 }; - enum KeyDerivationFunction : uint8_t { DEFAULT = 0, SHA256 = 1, PBKDF2 = 2 }; - - string CipherToString(CipherType cipher_p) const { - switch (cipher_p) { - case GCM: - return "gcm"; - case CTR: - return "ctr"; - case CBC: - return "cbc"; - default: - return "unknown"; - } - } - - static CipherType StringToCipher(const string &encryption_cipher) { - if (encryption_cipher == "gcm") { - return CipherType::GCM; - } else if (encryption_cipher == "ctr") { - return CipherType::CTR; - } else if (encryption_cipher == "cbc") { - return CipherType::CBC; - } - return CipherType::UNKNOWN; - } - - string KDFToString(KeyDerivationFunction kdf_p) const { - switch (kdf_p) { - case SHA256: - return "sha256"; - case PBKDF2: - return "pbkdf2"; - default: - return "default"; - } - } - - KeyDerivationFunction StringToKDF(const string &key_derivation_function) const { - if (key_derivation_function == "sha256") { - return KeyDerivationFunction::SHA256; - } else if (key_derivation_function == "pbkdf2") { - return KeyDerivationFunction::PBKDF2; - } else { - return KeyDerivationFunction::DEFAULT; - } - } + static void DecryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t buffer, idx_t buffer_size, data_ptr_t metadata); }; } // namespace duckdb diff --git a/src/include/duckdb/common/encryption_state.hpp b/src/include/duckdb/common/encryption_state.hpp index d87a257bb59c..32c0597a9c49 100644 --- a/src/include/duckdb/common/encryption_state.hpp +++ b/src/include/duckdb/common/encryption_state.hpp @@ -9,14 +9,27 @@ #pragma once #include "duckdb/common/helper.hpp" -#include "duckdb/common/types/value.hpp" +#include "duckdb/common/string_util.hpp" namespace duckdb { +class EncryptionTypes { + +public: + enum CipherType : uint8_t { INVALID = 0, GCM = 1, CTR = 2, CBC = 3 }; + enum KeyDerivationFunction : uint8_t { DEFAULT = 0, SHA256 = 1, PBKDF2 = 2 }; + enum Mode { ENCRYPT, DECRYPT }; + + static string CipherToString(CipherType cipher_p); + static CipherType StringToCipher(const string &encryption_cipher_p); + static string KDFToString(KeyDerivationFunction kdf_p); + static KeyDerivationFunction StringToKDF(const string &key_derivation_function_p); +}; + class EncryptionState { public: - DUCKDB_API explicit EncryptionState(const_data_ptr_t key = nullptr, idx_t key_len = 0); + DUCKDB_API explicit EncryptionState(EncryptionTypes::CipherType cipher_p, idx_t key_len); DUCKDB_API virtual ~EncryptionState(); public: @@ -28,9 +41,9 @@ class EncryptionState { DUCKDB_API virtual size_t Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, idx_t tag_len); DUCKDB_API virtual void GenerateRandomData(data_ptr_t data, idx_t len); -public: - enum Mode { ENCRYPT, DECRYPT }; - enum Cipher { GCM, CTR }; +protected: + EncryptionTypes::CipherType cipher; + idx_t key_len; }; class EncryptionUtil { @@ -39,8 +52,9 @@ class EncryptionUtil { DUCKDB_API explicit EncryptionUtil() {}; public: - virtual shared_ptr CreateEncryptionState(const_data_ptr_t key = nullptr, idx_t key_len = 0) const { - return make_shared_ptr(); + virtual shared_ptr CreateEncryptionState(EncryptionTypes::CipherType cipher_p, + idx_t key_len = 0) const { + return make_shared_ptr(cipher_p, key_len); } virtual ~EncryptionUtil() { diff --git a/src/include/duckdb/common/enum_util.hpp b/src/include/duckdb/common/enum_util.hpp index 415f86a2f47a..9854e49f43b8 100644 --- a/src/include/duckdb/common/enum_util.hpp +++ b/src/include/duckdb/common/enum_util.hpp @@ -50,6 +50,8 @@ enum class AggregateOrderDependent : uint8_t; enum class AggregateType : uint8_t; +enum class AlterDatabaseType : uint8_t; + enum class AlterForeignKeyType : uint8_t; enum class AlterScalarFunctionType : uint8_t; @@ -200,6 +202,8 @@ enum class FunctionStability : uint8_t; enum class GateStatus : uint8_t; +enum class GeometryType : uint8_t; + enum class HLLStorageType : uint8_t; enum class HTTPStatusCode : uint16_t; @@ -234,6 +238,8 @@ enum class LogLevel : uint8_t; enum class LogMode : uint8_t; +enum class LoggingTargetTable : uint8_t; + enum class LogicalOperatorType : uint8_t; enum class LogicalTypeId : uint8_t; @@ -290,8 +296,6 @@ enum class ParseInfoType : uint8_t; enum class ParserExtensionResultType : uint8_t; -enum class PartitionSortStage : uint8_t; - enum class PartitionedColumnDataType : uint8_t; enum class PartitionedTupleDataType : uint8_t; @@ -378,6 +382,8 @@ enum class StatisticsType : uint8_t; enum class StatsInfo : uint8_t; +enum class StorageBlockPrefetch : uint8_t; + enum class StrTimeSpecifier : uint8_t; enum class StreamExecutionResult : uint8_t; @@ -434,6 +440,8 @@ enum class VerificationType : uint8_t; enum class VerifyExistenceType : uint8_t; +enum class VertexType : uint8_t; + enum class WALType : uint8_t; enum class WindowAggregationMode : uint32_t; @@ -472,6 +480,9 @@ const char* EnumUtil::ToChars(AggregateOrderDependent v template<> const char* EnumUtil::ToChars(AggregateType value); +template<> +const char* EnumUtil::ToChars(AlterDatabaseType value); + template<> const char* EnumUtil::ToChars(AlterForeignKeyType value); @@ -697,6 +708,9 @@ const char* EnumUtil::ToChars(FunctionStability value); template<> const char* EnumUtil::ToChars(GateStatus value); +template<> +const char* EnumUtil::ToChars(GeometryType value); + template<> const char* EnumUtil::ToChars(HLLStorageType value); @@ -748,6 +762,9 @@ const char* EnumUtil::ToChars(LogLevel value); template<> const char* EnumUtil::ToChars(LogMode value); +template<> +const char* EnumUtil::ToChars(LoggingTargetTable value); + template<> const char* EnumUtil::ToChars(LogicalOperatorType value); @@ -832,9 +849,6 @@ const char* EnumUtil::ToChars(ParseInfoType value); template<> const char* EnumUtil::ToChars(ParserExtensionResultType value); -template<> -const char* EnumUtil::ToChars(PartitionSortStage value); - template<> const char* EnumUtil::ToChars(PartitionedColumnDataType value); @@ -964,6 +978,9 @@ const char* EnumUtil::ToChars(StatisticsType value); template<> const char* EnumUtil::ToChars(StatsInfo value); +template<> +const char* EnumUtil::ToChars(StorageBlockPrefetch value); + template<> const char* EnumUtil::ToChars(StrTimeSpecifier value); @@ -1048,6 +1065,9 @@ const char* EnumUtil::ToChars(VerificationType value); template<> const char* EnumUtil::ToChars(VerifyExistenceType value); +template<> +const char* EnumUtil::ToChars(VertexType value); + template<> const char* EnumUtil::ToChars(WALType value); @@ -1091,6 +1111,9 @@ AggregateOrderDependent EnumUtil::FromString(const char template<> AggregateType EnumUtil::FromString(const char *value); +template<> +AlterDatabaseType EnumUtil::FromString(const char *value); + template<> AlterForeignKeyType EnumUtil::FromString(const char *value); @@ -1316,6 +1339,9 @@ FunctionStability EnumUtil::FromString(const char *value); template<> GateStatus EnumUtil::FromString(const char *value); +template<> +GeometryType EnumUtil::FromString(const char *value); + template<> HLLStorageType EnumUtil::FromString(const char *value); @@ -1367,6 +1393,9 @@ LogLevel EnumUtil::FromString(const char *value); template<> LogMode EnumUtil::FromString(const char *value); +template<> +LoggingTargetTable EnumUtil::FromString(const char *value); + template<> LogicalOperatorType EnumUtil::FromString(const char *value); @@ -1451,9 +1480,6 @@ ParseInfoType EnumUtil::FromString(const char *value); template<> ParserExtensionResultType EnumUtil::FromString(const char *value); -template<> -PartitionSortStage EnumUtil::FromString(const char *value); - template<> PartitionedColumnDataType EnumUtil::FromString(const char *value); @@ -1583,6 +1609,9 @@ StatisticsType EnumUtil::FromString(const char *value); template<> StatsInfo EnumUtil::FromString(const char *value); +template<> +StorageBlockPrefetch EnumUtil::FromString(const char *value); + template<> StrTimeSpecifier EnumUtil::FromString(const char *value); @@ -1667,6 +1696,9 @@ VerificationType EnumUtil::FromString(const char *value); template<> VerifyExistenceType EnumUtil::FromString(const char *value); +template<> +VertexType EnumUtil::FromString(const char *value); + template<> WALType EnumUtil::FromString(const char *value); diff --git a/src/include/duckdb/common/enums/metric_type.hpp b/src/include/duckdb/common/enums/metric_type.hpp index 8fd2790ab115..110c9d53f030 100644 --- a/src/include/duckdb/common/enums/metric_type.hpp +++ b/src/include/duckdb/common/enums/metric_type.hpp @@ -20,32 +20,36 @@ namespace duckdb { enum class MetricsType : uint8_t { - QUERY_NAME, + ATTACH_LOAD_STORAGE_LATENCY, + ATTACH_REPLAY_WAL_LATENCY, BLOCKED_THREAD_TIME, + CHECKPOINT_LATENCY, CPU_TIME, - EXTRA_INFO, CUMULATIVE_CARDINALITY, - OPERATOR_TYPE, - OPERATOR_CARDINALITY, CUMULATIVE_ROWS_SCANNED, + EXTRA_INFO, + LATENCY, + OPERATOR_CARDINALITY, + OPERATOR_NAME, OPERATOR_ROWS_SCANNED, OPERATOR_TIMING, + OPERATOR_TYPE, + QUERY_NAME, RESULT_SET_SIZE, - LATENCY, ROWS_RETURNED, - OPERATOR_NAME, SYSTEM_PEAK_BUFFER_MEMORY, SYSTEM_PEAK_TEMP_DIR_SIZE, TOTAL_BYTES_READ, TOTAL_BYTES_WRITTEN, + WAITING_TO_ATTACH_LATENCY, ALL_OPTIMIZERS, CUMULATIVE_OPTIMIZER_TIMING, - PLANNER, - PLANNER_BINDING, PHYSICAL_PLANNER, PHYSICAL_PLANNER_COLUMN_BINDING, - PHYSICAL_PLANNER_RESOLVE_TYPES, PHYSICAL_PLANNER_CREATE_PLAN, + PHYSICAL_PLANNER_RESOLVE_TYPES, + PLANNER, + PLANNER_BINDING, OPTIMIZER_EXPRESSION_REWRITER, OPTIMIZER_FILTER_PULLUP, OPTIMIZER_FILTER_PUSHDOWN, @@ -64,6 +68,7 @@ enum class MetricsType : uint8_t { OPTIMIZER_BUILD_SIDE_PROBE_SIDE, OPTIMIZER_LIMIT_PUSHDOWN, OPTIMIZER_TOP_N, + OPTIMIZER_TOP_N_WINDOW_ELIMINATION, OPTIMIZER_COMPRESSED_MATERIALIZATION, OPTIMIZER_DUPLICATE_GROUPS, OPTIMIZER_REORDER_FILTER, @@ -74,6 +79,7 @@ enum class MetricsType : uint8_t { OPTIMIZER_SUM_REWRITER, OPTIMIZER_LATE_MATERIALIZATION, OPTIMIZER_CTE_INLINING, + OPTIMIZER_COMMON_SUBPLAN, }; struct MetricsTypeHashFunction { diff --git a/src/include/duckdb/common/enums/optimizer_type.hpp b/src/include/duckdb/common/enums/optimizer_type.hpp index b57823028749..82675c7d56f7 100644 --- a/src/include/duckdb/common/enums/optimizer_type.hpp +++ b/src/include/duckdb/common/enums/optimizer_type.hpp @@ -33,6 +33,7 @@ enum class OptimizerType : uint32_t { BUILD_SIDE_PROBE_SIDE, LIMIT_PUSHDOWN, TOP_N, + TOP_N_WINDOW_ELIMINATION, COMPRESSED_MATERIALIZATION, DUPLICATE_GROUPS, REORDER_FILTER, @@ -42,7 +43,8 @@ enum class OptimizerType : uint32_t { MATERIALIZED_CTE, SUM_REWRITER, LATE_MATERIALIZATION, - CTE_INLINING + CTE_INLINING, + COMMON_SUBPLAN, }; string OptimizerTypeToString(OptimizerType type); diff --git a/src/include/duckdb/common/enums/storage_block_prefetch.hpp b/src/include/duckdb/common/enums/storage_block_prefetch.hpp new file mode 100644 index 000000000000..273d3fffa8ef --- /dev/null +++ b/src/include/duckdb/common/enums/storage_block_prefetch.hpp @@ -0,0 +1,17 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/common/enums/storage_block_prefetch.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/constants.hpp" + +namespace duckdb { + +enum class StorageBlockPrefetch : uint8_t { REMOTE_ONLY = 0, NEVER = 1, ALWAYS_PREFETCH = 2, DEBUG_FORCE_ALWAYS = 3 }; + +} // namespace duckdb diff --git a/src/include/duckdb/common/exception.hpp b/src/include/duckdb/common/exception.hpp index 480dd23853df..13fa9bc12bb6 100644 --- a/src/include/duckdb/common/exception.hpp +++ b/src/include/duckdb/common/exception.hpp @@ -94,15 +94,16 @@ enum class ExceptionType : uint8_t { class Exception : public std::runtime_error { public: DUCKDB_API Exception(ExceptionType exception_type, const string &message); - DUCKDB_API Exception(ExceptionType exception_type, const string &message, - const unordered_map &extra_info); + + DUCKDB_API Exception(const unordered_map &extra_info, ExceptionType exception_type, + const string &message); public: DUCKDB_API static string ExceptionTypeToString(ExceptionType type); DUCKDB_API static ExceptionType StringToExceptionType(const string &type); template - static string ConstructMessage(const string &msg, ARGS... params) { + static string ConstructMessage(const string &msg, ARGS const &...params) { const std::size_t num_args = sizeof...(ARGS); if (num_args == 0) { return msg; @@ -122,8 +123,9 @@ class Exception : public std::runtime_error { //! Whether this exception type can occur during execution of a query DUCKDB_API static bool IsExecutionError(ExceptionType type); DUCKDB_API static string ToJSON(ExceptionType type, const string &message); - DUCKDB_API static string ToJSON(ExceptionType type, const string &message, - const unordered_map &extra_info); + + DUCKDB_API static string ToJSON(const unordered_map &extra_info, ExceptionType type, + const string &message); DUCKDB_API static bool InvalidatesTransaction(ExceptionType exception_type); DUCKDB_API static bool InvalidatesDatabase(ExceptionType exception_type); @@ -131,8 +133,8 @@ class Exception : public std::runtime_error { DUCKDB_API static string ConstructMessageRecursive(const string &msg, std::vector &values); template - static string ConstructMessageRecursive(const string &msg, std::vector &values, T param, - ARGS... params) { + static string ConstructMessageRecursive(const string &msg, std::vector &values, + const T ¶m, ARGS &&...params) { values.push_back(ExceptionFormatValue::CreateFormatValue(param)); return ConstructMessageRecursive(msg, values, params...); } @@ -155,8 +157,8 @@ class ConnectionException : public Exception { DUCKDB_API explicit ConnectionException(const string &msg); template - explicit ConnectionException(const string &msg, ARGS... params) - : ConnectionException(ConstructMessage(msg, params...)) { + explicit ConnectionException(const string &msg, ARGS &&...params) + : ConnectionException(ConstructMessage(msg, std::forward(params)...)) { } }; @@ -165,8 +167,8 @@ class PermissionException : public Exception { DUCKDB_API explicit PermissionException(const string &msg); template - explicit PermissionException(const string &msg, ARGS... params) - : PermissionException(ConstructMessage(msg, params...)) { + explicit PermissionException(const string &msg, ARGS &&...params) + : PermissionException(ConstructMessage(msg, std::forward(params)...)) { } }; @@ -175,8 +177,8 @@ class OutOfRangeException : public Exception { DUCKDB_API explicit OutOfRangeException(const string &msg); template - explicit OutOfRangeException(const string &msg, ARGS... params) - : OutOfRangeException(ConstructMessage(msg, params...)) { + explicit OutOfRangeException(const string &msg, ARGS &&...params) + : OutOfRangeException(ConstructMessage(msg, std::forward(params)...)) { } DUCKDB_API OutOfRangeException(const int64_t value, const PhysicalType orig_type, const PhysicalType new_type); DUCKDB_API OutOfRangeException(const hugeint_t value, const PhysicalType orig_type, const PhysicalType new_type); @@ -189,8 +191,8 @@ class OutOfMemoryException : public Exception { DUCKDB_API explicit OutOfMemoryException(const string &msg); template - explicit OutOfMemoryException(const string &msg, ARGS... params) - : OutOfMemoryException(ConstructMessage(msg, params...)) { + explicit OutOfMemoryException(const string &msg, ARGS &&...params) + : OutOfMemoryException(ConstructMessage(msg, std::forward(params)...)) { } private: @@ -202,7 +204,8 @@ class SyntaxException : public Exception { DUCKDB_API explicit SyntaxException(const string &msg); template - explicit SyntaxException(const string &msg, ARGS... params) : SyntaxException(ConstructMessage(msg, params...)) { + explicit SyntaxException(const string &msg, ARGS &&...params) + : SyntaxException(ConstructMessage(msg, std::forward(params)...)) { } }; @@ -211,8 +214,8 @@ class ConstraintException : public Exception { DUCKDB_API explicit ConstraintException(const string &msg); template - explicit ConstraintException(const string &msg, ARGS... params) - : ConstraintException(ConstructMessage(msg, params...)) { + explicit ConstraintException(const string &msg, ARGS &&...params) + : ConstraintException(ConstructMessage(msg, std::forward(params)...)) { } }; @@ -221,25 +224,27 @@ class DependencyException : public Exception { DUCKDB_API explicit DependencyException(const string &msg); template - explicit DependencyException(const string &msg, ARGS... params) - : DependencyException(ConstructMessage(msg, params...)) { + explicit DependencyException(const string &msg, ARGS &&...params) + : DependencyException(ConstructMessage(msg, std::forward(params)...)) { } }; class IOException : public Exception { public: DUCKDB_API explicit IOException(const string &msg); - DUCKDB_API explicit IOException(const string &msg, const unordered_map &extra_info); + + DUCKDB_API explicit IOException(const unordered_map &extra_info, const string &msg); explicit IOException(ExceptionType exception_type, const string &msg) : Exception(exception_type, msg) { } template - explicit IOException(const string &msg, ARGS... params) : IOException(ConstructMessage(msg, params...)) { + explicit IOException(const string &msg, ARGS &&...params) + : IOException(ConstructMessage(msg, std::forward(params)...)) { } template - explicit IOException(const string &msg, const unordered_map &extra_info, ARGS... params) - : IOException(ConstructMessage(msg, params...), extra_info) { + explicit IOException(const unordered_map &extra_info, const string &msg, ARGS &&...params) + : IOException(extra_info, ConstructMessage(msg, std::forward(params)...)) { } }; @@ -248,8 +253,8 @@ class MissingExtensionException : public Exception { DUCKDB_API explicit MissingExtensionException(const string &msg); template - explicit MissingExtensionException(const string &msg, ARGS... params) - : MissingExtensionException(ConstructMessage(msg, params...)) { + explicit MissingExtensionException(const string &msg, ARGS &&...params) + : MissingExtensionException(ConstructMessage(msg, std::forward(params)...)) { } }; @@ -258,8 +263,8 @@ class NotImplementedException : public Exception { DUCKDB_API explicit NotImplementedException(const string &msg); template - explicit NotImplementedException(const string &msg, ARGS... params) - : NotImplementedException(ConstructMessage(msg, params...)) { + explicit NotImplementedException(const string &msg, ARGS &&...params) + : NotImplementedException(ConstructMessage(msg, std::forward(params)...)) { } }; @@ -273,8 +278,8 @@ class SerializationException : public Exception { DUCKDB_API explicit SerializationException(const string &msg); template - explicit SerializationException(const string &msg, ARGS... params) - : SerializationException(ConstructMessage(msg, params...)) { + explicit SerializationException(const string &msg, ARGS &&...params) + : SerializationException(ConstructMessage(msg, std::forward(params)...)) { } }; @@ -283,8 +288,8 @@ class SequenceException : public Exception { DUCKDB_API explicit SequenceException(const string &msg); template - explicit SequenceException(const string &msg, ARGS... params) - : SequenceException(ConstructMessage(msg, params...)) { + explicit SequenceException(const string &msg, ARGS &&...params) + : SequenceException(ConstructMessage(msg, std::forward(params)...)) { } }; @@ -298,14 +303,15 @@ class FatalException : public Exception { explicit FatalException(const string &msg) : FatalException(ExceptionType::FATAL, msg) { } template - explicit FatalException(const string &msg, ARGS... params) : FatalException(ConstructMessage(msg, params...)) { + explicit FatalException(const string &msg, ARGS &&...params) + : FatalException(ConstructMessage(msg, std::forward(params)...)) { } protected: DUCKDB_API explicit FatalException(ExceptionType type, const string &msg); template - explicit FatalException(ExceptionType type, const string &msg, ARGS... params) - : FatalException(type, ConstructMessage(msg, params...)) { + explicit FatalException(ExceptionType type, const string &msg, ARGS &&...params) + : FatalException(type, ConstructMessage(msg, std::forward(params)...)) { } }; @@ -314,23 +320,25 @@ class InternalException : public Exception { DUCKDB_API explicit InternalException(const string &msg); template - explicit InternalException(const string &msg, ARGS... params) - : InternalException(ConstructMessage(msg, params...)) { + explicit InternalException(const string &msg, ARGS &&...params) + : InternalException(ConstructMessage(msg, std::forward(params)...)) { } }; class InvalidInputException : public Exception { public: DUCKDB_API explicit InvalidInputException(const string &msg); - DUCKDB_API explicit InvalidInputException(const string &msg, const unordered_map &extra_info); + DUCKDB_API explicit InvalidInputException(const unordered_map &extra_info, const string &msg); template - explicit InvalidInputException(const string &msg, ARGS... params) - : InvalidInputException(ConstructMessage(msg, params...)) { + explicit InvalidInputException(const string &msg, ARGS &&...params) + : InvalidInputException(ConstructMessage(msg, std::forward(params)...)) { } + template - explicit InvalidInputException(const Expression &expr, const string &msg, ARGS... params) - : InvalidInputException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(expr)) { + explicit InvalidInputException(const Expression &expr, const string &msg, ARGS &&...params) + : InvalidInputException(Exception::InitializeExtraInfo(expr), + ConstructMessage(msg, std::forward(params)...)) { } }; @@ -339,24 +347,26 @@ class ExecutorException : public Exception { DUCKDB_API explicit ExecutorException(const string &msg); template - explicit ExecutorException(const string &msg, ARGS... params) - : ExecutorException(ConstructMessage(msg, params...)) { + explicit ExecutorException(const string &msg, ARGS &&...params) + : ExecutorException(ConstructMessage(msg, std::forward(params)...)) { } }; class InvalidConfigurationException : public Exception { public: DUCKDB_API explicit InvalidConfigurationException(const string &msg); - DUCKDB_API explicit InvalidConfigurationException(const string &msg, - const unordered_map &extra_info); + + DUCKDB_API explicit InvalidConfigurationException(const unordered_map &extra_info, + const string &msg); template - explicit InvalidConfigurationException(const string &msg, ARGS... params) - : InvalidConfigurationException(ConstructMessage(msg, params...)) { + explicit InvalidConfigurationException(const string &msg, ARGS &&...params) + : InvalidConfigurationException(ConstructMessage(msg, std::forward(params)...)) { } template - explicit InvalidConfigurationException(const Expression &expr, const string &msg, ARGS... params) - : InvalidConfigurationException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(expr)) { + explicit InvalidConfigurationException(const Expression &expr, const string &msg, ARGS &&...params) + : InvalidConfigurationException(ConstructMessage(msg, std::forward(params)...), + Exception::InitializeExtraInfo(expr)) { } }; @@ -381,8 +391,8 @@ class ParameterNotAllowedException : public Exception { DUCKDB_API explicit ParameterNotAllowedException(const string &msg); template - explicit ParameterNotAllowedException(const string &msg, ARGS... params) - : ParameterNotAllowedException(ConstructMessage(msg, params...)) { + explicit ParameterNotAllowedException(const string &msg, ARGS &&...params) + : ParameterNotAllowedException(ConstructMessage(msg, std::forward(params)...)) { } }; diff --git a/src/include/duckdb/common/exception/binder_exception.hpp b/src/include/duckdb/common/exception/binder_exception.hpp index 2590cb094a7f..fd7158f87336 100644 --- a/src/include/duckdb/common/exception/binder_exception.hpp +++ b/src/include/duckdb/common/exception/binder_exception.hpp @@ -15,31 +15,39 @@ namespace duckdb { class BinderException : public Exception { public: - DUCKDB_API explicit BinderException(const string &msg, const unordered_map &extra_info); DUCKDB_API explicit BinderException(const string &msg); + DUCKDB_API explicit BinderException(const unordered_map &extra_info, const string &msg); + template - explicit BinderException(const string &msg, ARGS... params) : BinderException(ConstructMessage(msg, params...)) { + explicit BinderException(const string &msg, ARGS &&...params) + : BinderException(ConstructMessage(msg, std::forward(params)...)) { } + template - explicit BinderException(const TableRef &ref, const string &msg, ARGS... params) - : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(ref)) { + explicit BinderException(const TableRef &ref, const string &msg, ARGS &&...params) + : BinderException(Exception::InitializeExtraInfo(ref), ConstructMessage(msg, std::forward(params)...)) { } template - explicit BinderException(const ParsedExpression &expr, const string &msg, ARGS... params) - : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(expr)) { + explicit BinderException(const ParsedExpression &expr, const string &msg, ARGS &&...params) + : BinderException(Exception::InitializeExtraInfo(expr), ConstructMessage(msg, std::forward(params)...)) { } + template - explicit BinderException(const Expression &expr, const string &msg, ARGS... params) - : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(expr)) { + explicit BinderException(const Expression &expr, const string &msg, ARGS &&...params) + : BinderException(Exception::InitializeExtraInfo(expr), ConstructMessage(msg, std::forward(params)...)) { } + template - explicit BinderException(QueryErrorContext error_context, const string &msg, ARGS... params) - : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(error_context)) { + explicit BinderException(QueryErrorContext error_context, const string &msg, ARGS &&...params) + : BinderException(Exception::InitializeExtraInfo(error_context), + ConstructMessage(msg, std::forward(params)...)) { } + template - explicit BinderException(optional_idx error_location, const string &msg, ARGS... params) - : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(error_location)) { + explicit BinderException(optional_idx error_location, const string &msg, ARGS &&...params) + : BinderException(Exception::InitializeExtraInfo(error_location), + ConstructMessage(msg, std::forward(params)...)) { } static BinderException ColumnNotFound(const string &name, const vector &similar_bindings, diff --git a/src/include/duckdb/common/exception/catalog_exception.hpp b/src/include/duckdb/common/exception/catalog_exception.hpp index 498fafd19b73..1095531d00f6 100644 --- a/src/include/duckdb/common/exception/catalog_exception.hpp +++ b/src/include/duckdb/common/exception/catalog_exception.hpp @@ -19,14 +19,18 @@ struct EntryLookupInfo; class CatalogException : public Exception { public: DUCKDB_API explicit CatalogException(const string &msg); - DUCKDB_API explicit CatalogException(const string &msg, const unordered_map &extra_info); + + DUCKDB_API explicit CatalogException(const unordered_map &extra_info, const string &msg); template - explicit CatalogException(const string &msg, ARGS... params) : CatalogException(ConstructMessage(msg, params...)) { + explicit CatalogException(const string &msg, ARGS &&...params) + : CatalogException(ConstructMessage(msg, std::forward(params)...)) { } + template - explicit CatalogException(QueryErrorContext error_context, const string &msg, ARGS... params) - : CatalogException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(error_context)) { + explicit CatalogException(QueryErrorContext error_context, const string &msg, ARGS &&...params) + : CatalogException(Exception::InitializeExtraInfo(error_context), + ConstructMessage(msg, std::forward(params)...)) { } static CatalogException MissingEntry(const EntryLookupInfo &lookup_info, const string &suggestion); diff --git a/src/include/duckdb/common/exception/conversion_exception.hpp b/src/include/duckdb/common/exception/conversion_exception.hpp index 5330f46e66b6..9252d07907c9 100644 --- a/src/include/duckdb/common/exception/conversion_exception.hpp +++ b/src/include/duckdb/common/exception/conversion_exception.hpp @@ -12,22 +12,24 @@ #include "duckdb/common/optional_idx.hpp" namespace duckdb { - class ConversionException : public Exception { public: DUCKDB_API explicit ConversionException(const string &msg); + DUCKDB_API explicit ConversionException(optional_idx error_location, const string &msg); + DUCKDB_API ConversionException(const PhysicalType orig_type, const PhysicalType new_type); + DUCKDB_API ConversionException(const LogicalType &orig_type, const LogicalType &new_type); template - explicit ConversionException(const string &msg, ARGS... params) - : ConversionException(ConstructMessage(msg, params...)) { + explicit ConversionException(const string &msg, ARGS &&...params) + : ConversionException(ConstructMessage(msg, std::forward(params)...)) { } + template - explicit ConversionException(optional_idx error_location, const string &msg, ARGS... params) - : ConversionException(error_location, ConstructMessage(msg, params...)) { + explicit ConversionException(optional_idx error_location, const string &msg, ARGS &&...params) + : ConversionException(error_location, ConstructMessage(msg, std::forward(params)...)) { } }; - } // namespace duckdb diff --git a/src/include/duckdb/common/exception/http_exception.hpp b/src/include/duckdb/common/exception/http_exception.hpp index aff00d23de68..b0d0e9c2d70d 100644 --- a/src/include/duckdb/common/exception/http_exception.hpp +++ b/src/include/duckdb/common/exception/http_exception.hpp @@ -24,9 +24,9 @@ class HTTPException : public Exception { } template ::status = 0, typename... ARGS> - explicit HTTPException(RESPONSE &response, const string &msg, ARGS... params) + explicit HTTPException(RESPONSE &response, const string &msg, ARGS &&...params) : HTTPException(static_cast(response.status), response.body, response.headers, response.reason, msg, - params...) { + std::forward(params)...) { } template @@ -35,16 +35,16 @@ class HTTPException : public Exception { }; template ::code = 0, typename... ARGS> - explicit HTTPException(RESPONSE &response, const string &msg, ARGS... params) + explicit HTTPException(RESPONSE &response, const string &msg, ARGS &&...params) : HTTPException(static_cast(response.code), response.body, response.headers, response.error, msg, - params...) { + std::forward(params)...) { } template explicit HTTPException(int status_code, const string &response_body, const HEADERS &headers, const string &reason, - const string &msg, ARGS... params) - : Exception(ExceptionType::HTTP, ConstructMessage(msg, params...), - HTTPExtraInfo(status_code, response_body, headers, reason)) { + const string &msg, ARGS &&...params) + : Exception(HTTPExtraInfo(status_code, response_body, headers, reason), ExceptionType::HTTP, + ConstructMessage(msg, std::forward(params)...)) { } template diff --git a/src/include/duckdb/common/exception/parser_exception.hpp b/src/include/duckdb/common/exception/parser_exception.hpp index 363a34457901..26ce6c585cb6 100644 --- a/src/include/duckdb/common/exception/parser_exception.hpp +++ b/src/include/duckdb/common/exception/parser_exception.hpp @@ -17,18 +17,21 @@ namespace duckdb { class ParserException : public Exception { public: DUCKDB_API explicit ParserException(const string &msg); - DUCKDB_API explicit ParserException(const string &msg, const unordered_map &extra_info); + + DUCKDB_API explicit ParserException(const unordered_map &extra_info, const string &msg); template - explicit ParserException(const string &msg, ARGS... params) : ParserException(ConstructMessage(msg, params...)) { + explicit ParserException(const string &msg, ARGS &&...params) + : ParserException(ConstructMessage(msg, std::forward(params)...)) { } template - explicit ParserException(optional_idx error_location, const string &msg, ARGS... params) - : ParserException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(error_location)) { + explicit ParserException(optional_idx error_location, const string &msg, ARGS &&...params) + : ParserException(Exception::InitializeExtraInfo(error_location), + ConstructMessage(msg, std::forward(params)...)) { } template - explicit ParserException(const ParsedExpression &expr, const string &msg, ARGS... params) - : ParserException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(expr)) { + explicit ParserException(const ParsedExpression &expr, const string &msg, ARGS &&...params) + : ParserException(Exception::InitializeExtraInfo(expr), ConstructMessage(msg, std::forward(params)...)) { } static ParserException SyntaxError(const string &query, const string &error_message, optional_idx error_location); diff --git a/src/include/duckdb/common/exception/transaction_exception.hpp b/src/include/duckdb/common/exception/transaction_exception.hpp index f0164df696f2..5ca0be62bb97 100644 --- a/src/include/duckdb/common/exception/transaction_exception.hpp +++ b/src/include/duckdb/common/exception/transaction_exception.hpp @@ -11,15 +11,13 @@ #include "duckdb/common/exception.hpp" namespace duckdb { - class TransactionException : public Exception { public: DUCKDB_API explicit TransactionException(const string &msg); template - explicit TransactionException(const string &msg, ARGS... params) - : TransactionException(ConstructMessage(msg, params...)) { + explicit TransactionException(const string &msg, ARGS &&...params) + : TransactionException(ConstructMessage(msg, std::forward(params)...)) { } }; - } // namespace duckdb diff --git a/src/include/duckdb/common/exception_format_value.hpp b/src/include/duckdb/common/exception_format_value.hpp index 3693db54ceca..7beeead6e733 100644 --- a/src/include/duckdb/common/exception_format_value.hpp +++ b/src/include/duckdb/common/exception_format_value.hpp @@ -49,13 +49,13 @@ enum class ExceptionFormatValueType : uint8_t { }; struct ExceptionFormatValue { - DUCKDB_API ExceptionFormatValue(double dbl_val); // NOLINT - DUCKDB_API ExceptionFormatValue(int64_t int_val); // NOLINT - DUCKDB_API ExceptionFormatValue(idx_t uint_val); // NOLINT - DUCKDB_API ExceptionFormatValue(string str_val); // NOLINT - DUCKDB_API ExceptionFormatValue(String str_val); // NOLINT - DUCKDB_API ExceptionFormatValue(hugeint_t hg_val); // NOLINT - DUCKDB_API ExceptionFormatValue(uhugeint_t uhg_val); // NOLINT + DUCKDB_API ExceptionFormatValue(double dbl_val); // NOLINT + DUCKDB_API ExceptionFormatValue(int64_t int_val); // NOLINT + DUCKDB_API ExceptionFormatValue(idx_t uint_val); // NOLINT + DUCKDB_API ExceptionFormatValue(string str_val); // NOLINT + DUCKDB_API ExceptionFormatValue(const String &str_val); // NOLINT + DUCKDB_API ExceptionFormatValue(hugeint_t hg_val); // NOLINT + DUCKDB_API ExceptionFormatValue(uhugeint_t uhg_val); // NOLINT ExceptionFormatValueType type; @@ -65,37 +65,37 @@ struct ExceptionFormatValue { public: template - static ExceptionFormatValue CreateFormatValue(T value) { + static ExceptionFormatValue CreateFormatValue(const T &value) { return int64_t(value); } static string Format(const string &msg, std::vector &values); }; template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(PhysicalType value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const PhysicalType &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(SQLString value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const SQLString &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(SQLIdentifier value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const SQLIdentifier &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(LogicalType value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const LogicalType &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(float value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const float &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(double value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const double &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(string value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const string &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(String value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const String &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const char *value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const char *const &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(char *value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(char *const &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(idx_t value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const idx_t &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(hugeint_t value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const hugeint_t &value); template <> -DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(uhugeint_t value); +DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const uhugeint_t &value); } // namespace duckdb diff --git a/src/include/duckdb/common/extra_type_info.hpp b/src/include/duckdb/common/extra_type_info.hpp index d5e35ee96889..60d84686df10 100644 --- a/src/include/duckdb/common/extra_type_info.hpp +++ b/src/include/duckdb/common/extra_type_info.hpp @@ -28,7 +28,8 @@ enum class ExtraTypeInfoType : uint8_t { ARRAY_TYPE_INFO = 9, ANY_TYPE_INFO = 10, INTEGER_LITERAL_TYPE_INFO = 11, - TEMPLATE_TYPE_INFO = 12 + TEMPLATE_TYPE_INFO = 12, + GEO_TYPE_INFO = 13 }; struct ExtraTypeInfo { @@ -278,4 +279,16 @@ struct TemplateTypeInfo : public ExtraTypeInfo { TemplateTypeInfo(); }; +struct GeoTypeInfo : public ExtraTypeInfo { +public: + GeoTypeInfo(); + + void Serialize(Serializer &serializer) const override; + static shared_ptr Deserialize(Deserializer &source); + shared_ptr Copy() const override; + +protected: + bool EqualsInternal(ExtraTypeInfo *other_p) const override; +}; + } // namespace duckdb diff --git a/src/include/duckdb/common/fsst.hpp b/src/include/duckdb/common/fsst.hpp index 9b6ef0911b52..4c2e1a479fa7 100644 --- a/src/include/duckdb/common/fsst.hpp +++ b/src/include/duckdb/common/fsst.hpp @@ -57,7 +57,7 @@ class FSSTPrimitives { decompressed_string_size, string_t::INLINE_LENGTH); } D_ASSERT(decompressed_string_size <= string_t::INLINE_LENGTH); - result.str.SetSizeAndFinalize(UnsafeNumericCast(decompressed_string_size)); + result.str.SetSizeAndFinalize(UnsafeNumericCast(decompressed_string_size), string_t::INLINE_LENGTH); return result.str; } static string DecompressValue(void *duckdb_fsst_decoder, const char *compressed_string, diff --git a/src/include/duckdb/common/hugeint.hpp b/src/include/duckdb/common/hugeint.hpp index c9b54bd95053..acdc4fb4b8a2 100644 --- a/src/include/duckdb/common/hugeint.hpp +++ b/src/include/duckdb/common/hugeint.hpp @@ -76,7 +76,7 @@ struct hugeint_t { // NOLINT: use numeric casing DUCKDB_API explicit operator int16_t() const; DUCKDB_API explicit operator int32_t() const; DUCKDB_API explicit operator int64_t() const; - DUCKDB_API operator uhugeint_t() const; // NOLINT: Allow implicit conversion from `hugeint_t` + DUCKDB_API explicit operator uhugeint_t() const; }; } // namespace duckdb diff --git a/src/include/duckdb/common/insertion_order_preserving_map.hpp b/src/include/duckdb/common/insertion_order_preserving_map.hpp index 7240d62d8d32..5b6f11730408 100644 --- a/src/include/duckdb/common/insertion_order_preserving_map.hpp +++ b/src/include/duckdb/common/insertion_order_preserving_map.hpp @@ -95,6 +95,10 @@ class InsertionOrderPreservingMap { map.resize(nz); } + void clear() { // NOLINT: match stl API + map.clear(); + } + void insert(const string &key, V &&value) { // NOLINT: match stl API if (contains(key)) { return; diff --git a/src/include/duckdb/common/multi_file/multi_file_data.hpp b/src/include/duckdb/common/multi_file/multi_file_data.hpp index fd6380a7e31a..523084d6ee0a 100644 --- a/src/include/duckdb/common/multi_file/multi_file_data.hpp +++ b/src/include/duckdb/common/multi_file/multi_file_data.hpp @@ -139,7 +139,7 @@ struct MultiFileLocalColumnId { } public: - operator idx_t() { // NOLINT: allow implicit conversion + operator idx_t() const { // NOLINT: allow implicit conversion return column_id; } idx_t GetId() const { @@ -170,7 +170,7 @@ struct MultiFileLocalIndex { } public: - operator idx_t() { // NOLINT: allow implicit conversion + operator idx_t() const { // NOLINT: allow implicit conversion return index; } idx_t GetIndex() const { diff --git a/src/include/duckdb/common/multi_file/multi_file_function.hpp b/src/include/duckdb/common/multi_file/multi_file_function.hpp index ab17f7b8739a..1ed169568b10 100644 --- a/src/include/duckdb/common/multi_file/multi_file_function.hpp +++ b/src/include/duckdb/common/multi_file/multi_file_function.hpp @@ -199,6 +199,7 @@ class MultiFileFunction : public TableFunction { auto options = interface->InitializeOptions(context, nullptr); MultiFileOptions file_options; + file_options.auto_detect_hive_partitioning = false; for (auto &option : input.info.options) { auto loption = StringUtil::Lower(option.first); diff --git a/src/include/duckdb/common/operator/cast_operators.hpp b/src/include/duckdb/common/operator/cast_operators.hpp index ac55e5a69d86..e495e9760a1c 100644 --- a/src/include/duckdb/common/operator/cast_operators.hpp +++ b/src/include/duckdb/common/operator/cast_operators.hpp @@ -1070,6 +1070,19 @@ bool TryCastBlobToUUID::Operation(string_t input, hugeint_t &result, Vector &res template <> bool TryCastBlobToUUID::Operation(string_t input, hugeint_t &result, bool strict); +//===--------------------------------------------------------------------===// +// GEOMETRY +//===--------------------------------------------------------------------===// +struct TryCastToGeometry { + template + static inline bool Operation(SRC input, DST &result, Vector &result_vector, CastParameters ¶meters) { + throw InternalException("Unsupported type for try cast to geometry"); + } +}; + +template <> +bool TryCastToGeometry::Operation(string_t input, string_t &result, Vector &result_vector, CastParameters ¶meters); + //===--------------------------------------------------------------------===// // Pointers //===--------------------------------------------------------------------===// diff --git a/src/include/duckdb/common/operator/comparison_operators.hpp b/src/include/duckdb/common/operator/comparison_operators.hpp index f1a6f6eb33c5..a847e217bbfb 100644 --- a/src/include/duckdb/common/operator/comparison_operators.hpp +++ b/src/include/duckdb/common/operator/comparison_operators.hpp @@ -210,15 +210,4 @@ inline bool GreaterThan::Operation(const interval_t &left, const interval_t &rig return Interval::GreaterThan(left, right); } -//===--------------------------------------------------------------------===// -// Specialized Hugeint Comparison Operators -//===--------------------------------------------------------------------===// -template <> -inline bool Equals::Operation(const hugeint_t &left, const hugeint_t &right) { - return Hugeint::Equals(left, right); -} -template <> -inline bool GreaterThan::Operation(const hugeint_t &left, const hugeint_t &right) { - return Hugeint::GreaterThan(left, right); -} } // namespace duckdb diff --git a/src/include/duckdb/common/profiler.hpp b/src/include/duckdb/common/profiler.hpp index 5fb65337af1a..3a5cb402edb1 100644 --- a/src/include/duckdb/common/profiler.hpp +++ b/src/include/duckdb/common/profiler.hpp @@ -13,36 +13,52 @@ namespace duckdb { -//! The profiler can be used to measure elapsed time +//! Profiler class to measure the elapsed time. template class BaseProfiler { public: - //! Starts the timer + //! Start the timer. void Start() { finished = false; + ran = true; start = Tick(); } - //! Finishes timing + //! End the timer. void End() { end = Tick(); finished = true; } + //! Reset the timer. + void Reset() { + finished = false; + ran = false; + } - //! Returns the elapsed time in seconds. If End() has been called, returns - //! the total elapsed time. Otherwise returns how far along the timer is - //! right now. + //! Returns the elapsed time in seconds. + //! If ran is false, it returns 0. + //! If End() has been called, it returns the total elapsed time, + //! otherwise, returns how far along the timer is right now. double Elapsed() const { + if (!ran) { + return 0; + } auto measured_end = finished ? end : Tick(); return std::chrono::duration_cast>(measured_end - start).count(); } private: + //! Current time point. time_point Tick() const { return T::now(); } + //! Start time point. time_point start; + //! End time point. time_point end; + //! True, if end End() been called. bool finished = false; + //! True, if the timer was ran. + bool ran = false; }; using Profiler = BaseProfiler; diff --git a/src/include/duckdb/common/progress_bar/unscented_kalman_filter.hpp b/src/include/duckdb/common/progress_bar/unscented_kalman_filter.hpp index c4eea9784e00..d53aba6ca7ac 100644 --- a/src/include/duckdb/common/progress_bar/unscented_kalman_filter.hpp +++ b/src/include/duckdb/common/progress_bar/unscented_kalman_filter.hpp @@ -20,7 +20,7 @@ class UnscentedKalmanFilter { static constexpr size_t SIGMA_POINTS = 2 * STATE_DIM + 1; // UKF parameters - static constexpr double ALPHA = 0.1; + static constexpr double ALPHA = 0.044; static constexpr double BETA = 1.0; static constexpr double KAPPA = 0.0; @@ -35,6 +35,8 @@ class UnscentedKalmanFilter { double last_time; bool initialized; + double last_progress; + double scale_factor; // Helper functions vector> MatrixSqrt(const vector> &mat); diff --git a/src/include/duckdb/common/row_operations/row_operations.hpp b/src/include/duckdb/common/row_operations/row_operations.hpp index 557e9cd5b10e..ee1d11afbb32 100644 --- a/src/include/duckdb/common/row_operations/row_operations.hpp +++ b/src/include/duckdb/common/row_operations/row_operations.hpp @@ -25,22 +25,6 @@ struct SelectionVector; class StringHeap; struct UnifiedVectorFormat; -// The NestedValidity class help to set/get the validity from inside nested vectors -class NestedValidity { - data_ptr_t list_validity_location; - data_ptr_t *struct_validity_locations; - idx_t entry_idx; - idx_t idx_in_entry; - idx_t list_validity_offset; - -public: - explicit NestedValidity(data_ptr_t validitymask_location); - NestedValidity(data_ptr_t *validitymask_locations, idx_t child_vector_index); - void SetInvalid(idx_t idx); - bool IsValid(idx_t idx); - void OffsetListBy(idx_t offset); -}; - struct RowOperationsState { explicit RowOperationsState(ArenaAllocator &allocator) : allocator(allocator) { } @@ -49,7 +33,7 @@ struct RowOperationsState { unique_ptr addresses; // Re-usable vector for row_aggregate.cpp }; -// RowOperations contains a set of operations that operate on data using a RowLayout +// RowOperations contains a set of operations that operate on data using a TupleDataLayout struct RowOperations { //===--------------------------------------------------------------------===// // Aggregation Operators @@ -70,66 +54,6 @@ struct RowOperations { //! finalize - unaligned addresses, updated static void FinalizeStates(RowOperationsState &state, TupleDataLayout &layout, Vector &addresses, DataChunk &result, idx_t aggr_idx); - - //===--------------------------------------------------------------------===// - // Read/Write Operators - //===--------------------------------------------------------------------===// - //! Scatter group data to the rows. Initialises the ValidityMask. - static void Scatter(DataChunk &columns, UnifiedVectorFormat col_data[], const RowLayout &layout, Vector &rows, - RowDataCollection &string_heap, const SelectionVector &sel, idx_t count); - //! Gather a single column. - //! If heap_ptr is not null, then the data is assumed to contain swizzled pointers, - //! which will be unswizzled in memory. - static void Gather(Vector &rows, const SelectionVector &row_sel, Vector &col, const SelectionVector &col_sel, - const idx_t count, const RowLayout &layout, const idx_t col_no, const idx_t build_size = 0, - data_ptr_t heap_ptr = nullptr); - - //===--------------------------------------------------------------------===// - // Heap Operators - //===--------------------------------------------------------------------===// - //! Compute the entry sizes of a vector with variable size type (used before building heap buffer space). - static void ComputeEntrySizes(Vector &v, idx_t entry_sizes[], idx_t vcount, idx_t ser_count, - const SelectionVector &sel, idx_t offset = 0); - //! Compute the entry sizes of vector data with variable size type (used before building heap buffer space). - static void ComputeEntrySizes(Vector &v, UnifiedVectorFormat &vdata, idx_t entry_sizes[], idx_t vcount, - idx_t ser_count, const SelectionVector &sel, idx_t offset = 0); - //! Scatter vector with variable size type to the heap. - static void HeapScatter(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, - data_ptr_t *key_locations, optional_ptr parent_validity, idx_t offset = 0); - //! Scatter vector data with variable size type to the heap. - static void HeapScatterVData(UnifiedVectorFormat &vdata, PhysicalType type, const SelectionVector &sel, - idx_t ser_count, data_ptr_t *key_locations, - optional_ptr parent_validity, idx_t offset = 0); - //! Gather a single column with variable size type from the heap. - static void HeapGather(Vector &v, const idx_t &vcount, const SelectionVector &sel, data_ptr_t key_locations[], - optional_ptr parent_validity); - - //===--------------------------------------------------------------------===// - // Sorting Operators - //===--------------------------------------------------------------------===// - //! Scatter vector data to the rows in radix-sortable format. - static void RadixScatter(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, - data_ptr_t key_locations[], bool desc, bool has_null, bool nulls_first, idx_t prefix_len, - idx_t width, idx_t offset = 0); - - //===--------------------------------------------------------------------===// - // Out-of-Core Operators - //===--------------------------------------------------------------------===// - //! Swizzles blob pointers to offset within heap row - static void SwizzleColumns(const RowLayout &layout, const data_ptr_t base_row_ptr, const idx_t count); - //! Swizzles the base pointer of each row to offset within heap block - static void SwizzleHeapPointer(const RowLayout &layout, data_ptr_t row_ptr, const data_ptr_t heap_base_ptr, - const idx_t count, const idx_t base_offset = 0); - //! Copies 'count' heap rows that are pointed to by the rows at 'row_ptr' to 'heap_ptr' and swizzles the pointers - static void CopyHeapAndSwizzle(const RowLayout &layout, data_ptr_t row_ptr, const data_ptr_t heap_base_ptr, - data_ptr_t heap_ptr, const idx_t count); - - //! Unswizzles the base offset within heap block the rows to pointers - static void UnswizzleHeapPointer(const RowLayout &layout, const data_ptr_t base_row_ptr, - const data_ptr_t base_heap_ptr, const idx_t count); - //! Unswizzles all offsets back to pointers - static void UnswizzlePointers(const RowLayout &layout, const data_ptr_t base_row_ptr, - const data_ptr_t base_heap_ptr, const idx_t count); }; } // namespace duckdb diff --git a/src/include/duckdb/common/serializer/deserializer.hpp b/src/include/duckdb/common/serializer/deserializer.hpp index 03f30544fae0..633e74fa1caf 100644 --- a/src/include/duckdb/common/serializer/deserializer.hpp +++ b/src/include/duckdb/common/serializer/deserializer.hpp @@ -179,17 +179,32 @@ class Deserializer { } template - void ReadList(const field_id_t field_id, const char *tag, FUNC func) { - OnPropertyBegin(field_id, tag); + void ReadListInternal(FUNC func) { auto size = OnListBegin(); List list {*this}; for (idx_t i = 0; i < size; i++) { func(list, i); } OnListEnd(); + } + + template + void ReadList(const field_id_t field_id, const char *tag, FUNC func) { + OnPropertyBegin(field_id, tag); + ReadListInternal(func); OnPropertyEnd(); } + template + void ReadOptionalList(const field_id_t field_id, const char *tag, FUNC func) { + if (!OnOptionalPropertyBegin(field_id, tag)) { + OnOptionalPropertyEnd(false); + return; + } + ReadListInternal(func); + OnOptionalPropertyEnd(true); + } + template void ReadObject(const field_id_t field_id, const char *tag, FUNC func) { OnPropertyBegin(field_id, tag); diff --git a/src/include/duckdb/common/serializer/varint.hpp b/src/include/duckdb/common/serializer/varint.hpp index 8d0316a32e29..8cccd6f56c0b 100644 --- a/src/include/duckdb/common/serializer/varint.hpp +++ b/src/include/duckdb/common/serializer/varint.hpp @@ -35,7 +35,8 @@ uint8_t GetVarintSize(T val) { } template -void VarintEncode(T val, data_ptr_t ptr) { +idx_t VarintEncode(T val, data_ptr_t ptr) { + idx_t size = 0; do { uint8_t byte = val & 127; val >>= 7; @@ -44,11 +45,14 @@ void VarintEncode(T val, data_ptr_t ptr) { } *ptr = byte; ptr++; + size++; } while (val != 0); + return size; } template -void VarintEncode(T val, MemoryStream &ser) { +idx_t VarintEncode(T val, MemoryStream &ser) { + idx_t size = 0; do { uint8_t byte = val & 127; val >>= 7; @@ -56,7 +60,9 @@ void VarintEncode(T val, MemoryStream &ser) { byte |= 128; } ser.WriteData(&byte, sizeof(uint8_t)); + size++; } while (val != 0); + return size; } } // namespace duckdb diff --git a/src/include/duckdb/common/sort/comparators.hpp b/src/include/duckdb/common/sort/comparators.hpp deleted file mode 100644 index 5f3cd38071a3..000000000000 --- a/src/include/duckdb/common/sort/comparators.hpp +++ /dev/null @@ -1,65 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/common/sort/comparators.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/common/types.hpp" -#include "duckdb/common/types/row/row_layout.hpp" - -namespace duckdb { - -struct SortLayout; -struct SBScanState; - -using ValidityBytes = RowLayout::ValidityBytes; - -struct Comparators { -public: - //! Whether a tie between two blobs can be broken - static bool TieIsBreakable(const idx_t &col_idx, const data_ptr_t &row_ptr, const SortLayout &sort_layout); - //! Compares the tuples that a being read from in the 'left' and 'right blocks during merge sort - //! (only in case we cannot simply 'memcmp' - if there are blob columns) - static int CompareTuple(const SBScanState &left, const SBScanState &right, const data_ptr_t &l_ptr, - const data_ptr_t &r_ptr, const SortLayout &sort_layout, const bool &external_sort); - //! Compare two blob values - static int CompareVal(const data_ptr_t l_ptr, const data_ptr_t r_ptr, const LogicalType &type); - -private: - //! Compares two blob values that were initially tied by their prefix - static int BreakBlobTie(const idx_t &tie_col, const SBScanState &left, const SBScanState &right, - const SortLayout &sort_layout, const bool &external); - //! Compare two fixed-size values - template - static int TemplatedCompareVal(const data_ptr_t &left_ptr, const data_ptr_t &right_ptr); - - //! Compare two values at the pointers (can be recursive if nested type) - static int CompareValAndAdvance(data_ptr_t &l_ptr, data_ptr_t &r_ptr, const LogicalType &type, bool valid); - //! Compares two fixed-size values at the given pointers - template - static int TemplatedCompareAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr); - //! Compares two string values at the given pointers - static int CompareStringAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, bool valid); - //! Compares two struct values at the given pointers (recursive) - static int CompareStructAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, - const child_list_t &types, bool valid); - static int CompareArrayAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, const LogicalType &type, bool valid, - idx_t array_size); - //! Compare two list values at the pointers (can be recursive if nested type) - static int CompareListAndAdvance(data_ptr_t &left_ptr, data_ptr_t &right_ptr, const LogicalType &type, bool valid); - //! Compares a list of fixed-size values - template - static int TemplatedCompareListLoop(data_ptr_t &left_ptr, data_ptr_t &right_ptr, const ValidityBytes &left_validity, - const ValidityBytes &right_validity, const idx_t &count); - - //! Unwizzles an offset into a pointer - static void UnswizzleSingleValue(data_ptr_t data_ptr, const data_ptr_t &heap_ptr, const LogicalType &type); - //! Swizzles a pointer into an offset - static void SwizzleSingleValue(data_ptr_t data_ptr, const data_ptr_t &heap_ptr, const LogicalType &type); -}; - -} // namespace duckdb diff --git a/src/include/duckdb/common/sort/duckdb_pdqsort.hpp b/src/include/duckdb/common/sort/duckdb_pdqsort.hpp deleted file mode 100644 index c935a713aa47..000000000000 --- a/src/include/duckdb/common/sort/duckdb_pdqsort.hpp +++ /dev/null @@ -1,710 +0,0 @@ -/* -pdqsort.h - Pattern-defeating quicksort. - -Copyright (c) 2021 Orson Peters - -This software is provided 'as-is', without any express or implied warranty. In no event will the -authors be held liable for any damages arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, including commercial -applications, and to alter it and redistribute it freely, subject to the following restrictions: - -1. The origin of this software must not be misrepresented; you must not claim that you wrote the - original software. If you use this software in a product, an acknowledgment in the product - documentation would be appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and must not be misrepresented as - being the original software. - -3. This notice may not be removed or altered from any source distribution. -*/ - -#pragma once - -#include "duckdb/common/constants.hpp" -#include "duckdb/common/fast_mem.hpp" -#include "duckdb/common/helper.hpp" -#include "duckdb/common/types.hpp" -#include "duckdb/common/unique_ptr.hpp" - -#include -#include -#include -#include -#include - -namespace duckdb_pdqsort { - -using duckdb::data_ptr_t; -using duckdb::data_t; -using duckdb::FastMemcmp; -using duckdb::FastMemcpy; -using duckdb::idx_t; -using duckdb::make_unsafe_uniq_array_uninitialized; -using duckdb::unique_ptr; -using duckdb::unsafe_unique_array; - -// NOLINTBEGIN - -enum { - // Partitions below this size are sorted using insertion sort. - insertion_sort_threshold = 24, - - // Partitions above this size use Tukey's ninther to select the pivot. - ninther_threshold = 128, - - // When we detect an already sorted partition, attempt an insertion sort that allows this - // amount of element moves before giving up. - partial_insertion_sort_limit = 8, - - // Must be multiple of 8 due to loop unrolling, and < 256 to fit in unsigned char. - block_size = 64, - - // Cacheline size, assumes power of two. - cacheline_size = 64 - -}; - -// Returns floor(log2(n)), assumes n > 0. -template -inline int log2(T n) { - int log = 0; - while (n >>= 1) { - ++log; - } - return log; -} - -struct PDQConstants { - PDQConstants(idx_t entry_size, idx_t comp_offset, idx_t comp_size, data_ptr_t end) - : entry_size(entry_size), comp_offset(comp_offset), comp_size(comp_size), - tmp_buf_ptr(make_unsafe_uniq_array_uninitialized(entry_size)), tmp_buf(tmp_buf_ptr.get()), - iter_swap_buf_ptr(make_unsafe_uniq_array_uninitialized(entry_size)), - iter_swap_buf(iter_swap_buf_ptr.get()), - swap_offsets_buf_ptr(make_unsafe_uniq_array_uninitialized(entry_size)), - swap_offsets_buf(swap_offsets_buf_ptr.get()), end(end) { - } - - const duckdb::idx_t entry_size; - const idx_t comp_offset; - const idx_t comp_size; - - unsafe_unique_array tmp_buf_ptr; - const data_ptr_t tmp_buf; - - unsafe_unique_array iter_swap_buf_ptr; - const data_ptr_t iter_swap_buf; - - unsafe_unique_array swap_offsets_buf_ptr; - const data_ptr_t swap_offsets_buf; - - const data_ptr_t end; -}; - -struct PDQIterator { - PDQIterator(data_ptr_t ptr, const idx_t &entry_size) : ptr(ptr), entry_size(entry_size) { - } - - inline PDQIterator(const PDQIterator &other) : ptr(other.ptr), entry_size(other.entry_size) { - } - - inline const data_ptr_t &operator*() const { - return ptr; - } - - inline PDQIterator &operator++() { - ptr += entry_size; - return *this; - } - - inline PDQIterator &operator--() { - ptr -= entry_size; - return *this; - } - - inline PDQIterator operator++(int) { - auto tmp = *this; - ptr += entry_size; - return tmp; - } - - inline PDQIterator operator--(int) { - auto tmp = *this; - ptr -= entry_size; - return tmp; - } - - inline PDQIterator operator+(const idx_t &i) const { - auto result = *this; - result.ptr += i * entry_size; - return result; - } - - inline PDQIterator operator-(const idx_t &i) const { - PDQIterator result = *this; - result.ptr -= i * entry_size; - return result; - } - - inline PDQIterator &operator=(const PDQIterator &other) { - D_ASSERT(entry_size == other.entry_size); - ptr = other.ptr; - return *this; - } - - inline friend idx_t operator-(const PDQIterator &lhs, const PDQIterator &rhs) { - D_ASSERT(duckdb::NumericCast(*lhs - *rhs) % lhs.entry_size == 0); - D_ASSERT(*lhs - *rhs >= 0); - return duckdb::NumericCast(*lhs - *rhs) / lhs.entry_size; - } - - inline friend bool operator<(const PDQIterator &lhs, const PDQIterator &rhs) { - return *lhs < *rhs; - } - - inline friend bool operator>(const PDQIterator &lhs, const PDQIterator &rhs) { - return *lhs > *rhs; - } - - inline friend bool operator>=(const PDQIterator &lhs, const PDQIterator &rhs) { - return *lhs >= *rhs; - } - - inline friend bool operator<=(const PDQIterator &lhs, const PDQIterator &rhs) { - return *lhs <= *rhs; - } - - inline friend bool operator==(const PDQIterator &lhs, const PDQIterator &rhs) { - return *lhs == *rhs; - } - - inline friend bool operator!=(const PDQIterator &lhs, const PDQIterator &rhs) { - return *lhs != *rhs; - } - -private: - data_ptr_t ptr; - const idx_t &entry_size; -}; - -static inline bool comp(const data_ptr_t &l, const data_ptr_t &r, const PDQConstants &constants) { - D_ASSERT(l == constants.tmp_buf || l == constants.swap_offsets_buf || l < constants.end); - D_ASSERT(r == constants.tmp_buf || r == constants.swap_offsets_buf || r < constants.end); - return FastMemcmp(l + constants.comp_offset, r + constants.comp_offset, constants.comp_size) < 0; -} - -static inline const data_ptr_t &GET_TMP(const data_ptr_t &src, const PDQConstants &constants) { - D_ASSERT(src != constants.tmp_buf && src != constants.swap_offsets_buf && src < constants.end); - FastMemcpy(constants.tmp_buf, src, constants.entry_size); - return constants.tmp_buf; -} - -static inline const data_ptr_t &SWAP_OFFSETS_GET_TMP(const data_ptr_t &src, const PDQConstants &constants) { - D_ASSERT(src != constants.tmp_buf && src != constants.swap_offsets_buf && src < constants.end); - FastMemcpy(constants.swap_offsets_buf, src, constants.entry_size); - return constants.swap_offsets_buf; -} - -static inline void MOVE(const data_ptr_t &dest, const data_ptr_t &src, const PDQConstants &constants) { - D_ASSERT(dest == constants.tmp_buf || dest == constants.swap_offsets_buf || dest < constants.end); - D_ASSERT(src == constants.tmp_buf || src == constants.swap_offsets_buf || src < constants.end); - FastMemcpy(dest, src, constants.entry_size); -} - -static inline void iter_swap(const PDQIterator &lhs, const PDQIterator &rhs, const PDQConstants &constants) { - D_ASSERT(*lhs < constants.end); - D_ASSERT(*rhs < constants.end); - FastMemcpy(constants.iter_swap_buf, *lhs, constants.entry_size); - FastMemcpy(*lhs, *rhs, constants.entry_size); - FastMemcpy(*rhs, constants.iter_swap_buf, constants.entry_size); -} - -// Sorts [begin, end) using insertion sort with the given comparison function. -inline void insertion_sort(const PDQIterator &begin, const PDQIterator &end, const PDQConstants &constants) { - if (begin == end) { - return; - } - - for (PDQIterator cur = begin + 1; cur != end; ++cur) { - PDQIterator sift = cur; - PDQIterator sift_1 = cur - 1; - - // Compare first so we can avoid 2 moves for an element already positioned correctly. - if (comp(*sift, *sift_1, constants)) { - const auto &tmp = GET_TMP(*sift, constants); - - do { - MOVE(*sift--, *sift_1, constants); - } while (sift != begin && comp(tmp, *--sift_1, constants)); - - MOVE(*sift, tmp, constants); - } - } -} - -// Sorts [begin, end) using insertion sort with the given comparison function. Assumes -// *(begin - 1) is an element smaller than or equal to any element in [begin, end). -inline void unguarded_insertion_sort(const PDQIterator &begin, const PDQIterator &end, const PDQConstants &constants) { - if (begin == end) { - return; - } - - for (PDQIterator cur = begin + 1; cur != end; ++cur) { - PDQIterator sift = cur; - PDQIterator sift_1 = cur - 1; - - // Compare first so we can avoid 2 moves for an element already positioned correctly. - if (comp(*sift, *sift_1, constants)) { - const auto &tmp = GET_TMP(*sift, constants); - - do { - MOVE(*sift--, *sift_1, constants); - } while (comp(tmp, *--sift_1, constants)); - - MOVE(*sift, tmp, constants); - } - } -} - -// Attempts to use insertion sort on [begin, end). Will return false if more than -// partial_insertion_sort_limit elements were moved, and abort sorting. Otherwise it will -// successfully sort and return true. -inline bool partial_insertion_sort(const PDQIterator &begin, const PDQIterator &end, const PDQConstants &constants) { - if (begin == end) { - return true; - } - - std::size_t limit = 0; - for (PDQIterator cur = begin + 1; cur != end; ++cur) { - PDQIterator sift = cur; - PDQIterator sift_1 = cur - 1; - - // Compare first so we can avoid 2 moves for an element already positioned correctly. - if (comp(*sift, *sift_1, constants)) { - const auto &tmp = GET_TMP(*sift, constants); - - do { - MOVE(*sift--, *sift_1, constants); - } while (sift != begin && comp(tmp, *--sift_1, constants)); - - MOVE(*sift, tmp, constants); - limit += cur - sift; - } - - if (limit > partial_insertion_sort_limit) { - return false; - } - } - - return true; -} - -inline void sort2(const PDQIterator &a, const PDQIterator &b, const PDQConstants &constants) { - if (comp(*b, *a, constants)) { - iter_swap(a, b, constants); - } -} - -// Sorts the elements *a, *b and *c using comparison function comp. -inline void sort3(const PDQIterator &a, const PDQIterator &b, const PDQIterator &c, const PDQConstants &constants) { - sort2(a, b, constants); - sort2(b, c, constants); - sort2(a, b, constants); -} - -template -inline T *align_cacheline(T *p) { -#if defined(UINTPTR_MAX) && __cplusplus >= 201103L - std::uintptr_t ip = reinterpret_cast(p); -#else - std::size_t ip = reinterpret_cast(p); -#endif - ip = (ip + cacheline_size - 1) & -duckdb::UnsafeNumericCast(cacheline_size); - return reinterpret_cast(ip); -} - -inline void swap_offsets(const PDQIterator &first, const PDQIterator &last, unsigned char *offsets_l, - unsigned char *offsets_r, size_t num, bool use_swaps, const PDQConstants &constants) { - if (use_swaps) { - // This case is needed for the descending distribution, where we need - // to have proper swapping for pdqsort to remain O(n). - for (size_t i = 0; i < num; ++i) { - iter_swap(first + offsets_l[i], last - offsets_r[i], constants); - } - } else if (num > 0) { - PDQIterator l = first + offsets_l[0]; - PDQIterator r = last - offsets_r[0]; - const auto &tmp = SWAP_OFFSETS_GET_TMP(*l, constants); - MOVE(*l, *r, constants); - for (size_t i = 1; i < num; ++i) { - l = first + offsets_l[i]; - MOVE(*r, *l, constants); - r = last - offsets_r[i]; - MOVE(*l, *r, constants); - } - MOVE(*r, tmp, constants); - } -} - -// Partitions [begin, end) around pivot *begin using comparison function comp. Elements equal -// to the pivot are put in the right-hand partition. Returns the position of the pivot after -// partitioning and whether the passed sequence already was correctly partitioned. Assumes the -// pivot is a median of at least 3 elements and that [begin, end) is at least -// insertion_sort_threshold long. Uses branchless partitioning. -inline std::pair partition_right_branchless(const PDQIterator &begin, const PDQIterator &end, - const PDQConstants &constants) { - // Move pivot into local for speed. - const auto &pivot = GET_TMP(*begin, constants); - PDQIterator first = begin; - PDQIterator last = end; - - // Find the first element greater than or equal than the pivot (the median of 3 guarantees - // this exists). - while (comp(*++first, pivot, constants)) { - } - - // Find the first element strictly smaller than the pivot. We have to guard this search if - // there was no element before *first. - if (first - 1 == begin) { - while (first < last && !comp(*--last, pivot, constants)) { - } - } else { - while (!comp(*--last, pivot, constants)) { - } - } - - // If the first pair of elements that should be swapped to partition are the same element, - // the passed in sequence already was correctly partitioned. - bool already_partitioned = first >= last; - if (!already_partitioned) { - iter_swap(first, last, constants); - ++first; - - // The following branchless partitioning is derived from "BlockQuicksort: How Branch - // Mispredictions don’t affect Quicksort" by Stefan Edelkamp and Armin Weiss, but - // heavily micro-optimized. - unsigned char offsets_l_storage[block_size + cacheline_size]; - unsigned char offsets_r_storage[block_size + cacheline_size]; - unsigned char *offsets_l = align_cacheline(offsets_l_storage); - unsigned char *offsets_r = align_cacheline(offsets_r_storage); - - PDQIterator offsets_l_base = first; - PDQIterator offsets_r_base = last; - size_t num_l, num_r, start_l, start_r; - num_l = num_r = start_l = start_r = 0; - - while (first < last) { - // Fill up offset blocks with elements that are on the wrong side. - // First we determine how much elements are considered for each offset block. - size_t num_unknown = last - first; - size_t left_split = num_l == 0 ? (num_r == 0 ? num_unknown / 2 : num_unknown) : 0; - size_t right_split = num_r == 0 ? (num_unknown - left_split) : 0; - - // Fill the offset blocks. - if (left_split >= block_size) { - for (unsigned char i = 0; i < block_size;) { - offsets_l[num_l] = i++; - num_l += !comp(*first, pivot, constants); - ++first; - offsets_l[num_l] = i++; - num_l += !comp(*first, pivot, constants); - ++first; - offsets_l[num_l] = i++; - num_l += !comp(*first, pivot, constants); - ++first; - offsets_l[num_l] = i++; - num_l += !comp(*first, pivot, constants); - ++first; - offsets_l[num_l] = i++; - num_l += !comp(*first, pivot, constants); - ++first; - offsets_l[num_l] = i++; - num_l += !comp(*first, pivot, constants); - ++first; - offsets_l[num_l] = i++; - num_l += !comp(*first, pivot, constants); - ++first; - offsets_l[num_l] = i++; - num_l += !comp(*first, pivot, constants); - ++first; - } - } else { - for (unsigned char i = 0; i < left_split;) { - offsets_l[num_l] = i++; - num_l += !comp(*first, pivot, constants); - ++first; - } - } - - if (right_split >= block_size) { - for (unsigned char i = 0; i < block_size;) { - offsets_r[num_r] = ++i; - num_r += comp(*--last, pivot, constants); - offsets_r[num_r] = ++i; - num_r += comp(*--last, pivot, constants); - offsets_r[num_r] = ++i; - num_r += comp(*--last, pivot, constants); - offsets_r[num_r] = ++i; - num_r += comp(*--last, pivot, constants); - offsets_r[num_r] = ++i; - num_r += comp(*--last, pivot, constants); - offsets_r[num_r] = ++i; - num_r += comp(*--last, pivot, constants); - offsets_r[num_r] = ++i; - num_r += comp(*--last, pivot, constants); - offsets_r[num_r] = ++i; - num_r += comp(*--last, pivot, constants); - } - } else { - for (unsigned char i = 0; i < right_split;) { - offsets_r[num_r] = ++i; - num_r += comp(*--last, pivot, constants); - } - } - - // Swap elements and update block sizes and first/last boundaries. - size_t num = std::min(num_l, num_r); - swap_offsets(offsets_l_base, offsets_r_base, offsets_l + start_l, offsets_r + start_r, num, num_l == num_r, - constants); - num_l -= num; - num_r -= num; - start_l += num; - start_r += num; - - if (num_l == 0) { - start_l = 0; - offsets_l_base = first; - } - - if (num_r == 0) { - start_r = 0; - offsets_r_base = last; - } - } - - // We have now fully identified [first, last)'s proper position. Swap the last elements. - if (num_l) { - offsets_l += start_l; - while (num_l--) { - iter_swap(offsets_l_base + offsets_l[num_l], --last, constants); - } - first = last; - } - if (num_r) { - offsets_r += start_r; - while (num_r--) { - iter_swap(offsets_r_base - offsets_r[num_r], first, constants), ++first; - } - last = first; - } - } - - // Put the pivot in the right place. - PDQIterator pivot_pos = first - 1; - MOVE(*begin, *pivot_pos, constants); - MOVE(*pivot_pos, pivot, constants); - - return std::make_pair(pivot_pos, already_partitioned); -} - -// Partitions [begin, end) around pivot *begin using comparison function comp. Elements equal -// to the pivot are put in the right-hand partition. Returns the position of the pivot after -// partitioning and whether the passed sequence already was correctly partitioned. Assumes the -// pivot is a median of at least 3 elements and that [begin, end) is at least -// insertion_sort_threshold long. -inline std::pair partition_right(const PDQIterator &begin, const PDQIterator &end, - const PDQConstants &constants) { - // Move pivot into local for speed. - const auto &pivot = GET_TMP(*begin, constants); - - PDQIterator first = begin; - PDQIterator last = end; - - // Find the first element greater than or equal than the pivot (the median of 3 guarantees - // this exists). - while (comp(*++first, pivot, constants)) { - } - - // Find the first element strictly smaller than the pivot. We have to guard this search if - // there was no element before *first. - if (first - 1 == begin) { - while (first < last && !comp(*--last, pivot, constants)) { - } - } else { - while (!comp(*--last, pivot, constants)) { - } - } - - // If the first pair of elements that should be swapped to partition are the same element, - // the passed in sequence already was correctly partitioned. - bool already_partitioned = first >= last; - - // Keep swapping pairs of elements that are on the wrong side of the pivot. Previously - // swapped pairs guard the searches, which is why the first iteration is special-cased - // above. - while (first < last) { - iter_swap(first, last, constants); - while (comp(*++first, pivot, constants)) { - } - while (!comp(*--last, pivot, constants)) { - } - } - - // Put the pivot in the right place. - PDQIterator pivot_pos = first - 1; - MOVE(*begin, *pivot_pos, constants); - MOVE(*pivot_pos, pivot, constants); - - return std::make_pair(pivot_pos, already_partitioned); -} - -// Similar function to the one above, except elements equal to the pivot are put to the left of -// the pivot and it doesn't check or return if the passed sequence already was partitioned. -// Since this is rarely used (the many equal case), and in that case pdqsort already has O(n) -// performance, no block quicksort is applied here for simplicity. -inline PDQIterator partition_left(const PDQIterator &begin, const PDQIterator &end, const PDQConstants &constants) { - const auto &pivot = GET_TMP(*begin, constants); - PDQIterator first = begin; - PDQIterator last = end; - - while (comp(pivot, *--last, constants)) { - } - - if (last + 1 == end) { - while (first < last && !comp(pivot, *++first, constants)) { - } - } else { - while (!comp(pivot, *++first, constants)) { - } - } - - while (first < last) { - iter_swap(first, last, constants); - while (comp(pivot, *--last, constants)) { - } - while (!comp(pivot, *++first, constants)) { - } - } - - PDQIterator pivot_pos = last; - MOVE(*begin, *pivot_pos, constants); - MOVE(*pivot_pos, pivot, constants); - - return pivot_pos; -} - -template -inline void pdqsort_loop(PDQIterator begin, const PDQIterator &end, const PDQConstants &constants, int bad_allowed, - bool leftmost = true) { - // Use a while loop for tail recursion elimination. - while (true) { - idx_t size = end - begin; - - // Insertion sort is faster for small arrays. - if (size < insertion_sort_threshold) { - if (leftmost) { - insertion_sort(begin, end, constants); - } else { - unguarded_insertion_sort(begin, end, constants); - } - return; - } - - // Choose pivot as median of 3 or pseudomedian of 9. - idx_t s2 = size / 2; - if (size > ninther_threshold) { - sort3(begin, begin + s2, end - 1, constants); - sort3(begin + 1, begin + (s2 - 1), end - 2, constants); - sort3(begin + 2, begin + (s2 + 1), end - 3, constants); - sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), constants); - iter_swap(begin, begin + s2, constants); - } else { - sort3(begin + s2, begin, end - 1, constants); - } - - // If *(begin - 1) is the end of the right partition of a previous partition operation - // there is no element in [begin, end) that is smaller than *(begin - 1). Then if our - // pivot compares equal to *(begin - 1) we change strategy, putting equal elements in - // the left partition, greater elements in the right partition. We do not have to - // recurse on the left partition, since it's sorted (all equal). - if (!leftmost && !comp(*(begin - 1), *begin, constants)) { - begin = partition_left(begin, end, constants) + 1; - continue; - } - - // Partition and get results. - std::pair part_result = - Branchless ? partition_right_branchless(begin, end, constants) : partition_right(begin, end, constants); - PDQIterator pivot_pos = part_result.first; - bool already_partitioned = part_result.second; - - // Check for a highly unbalanced partition. - idx_t l_size = pivot_pos - begin; - idx_t r_size = end - (pivot_pos + 1); - bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; - - // If we got a highly unbalanced partition we shuffle elements to break many patterns. - if (highly_unbalanced) { - // If we had too many bad partitions, switch to heapsort to guarantee O(n log n). - // if (--bad_allowed == 0) { - // std::make_heap(begin, end, comp); - // std::sort_heap(begin, end, comp); - // return; - // } - - if (l_size >= insertion_sort_threshold) { - iter_swap(begin, begin + l_size / 4, constants); - iter_swap(pivot_pos - 1, pivot_pos - l_size / 4, constants); - - if (l_size > ninther_threshold) { - iter_swap(begin + 1, begin + (l_size / 4 + 1), constants); - iter_swap(begin + 2, begin + (l_size / 4 + 2), constants); - iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1), constants); - iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2), constants); - } - } - - if (r_size >= insertion_sort_threshold) { - iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4), constants); - iter_swap(end - 1, end - r_size / 4, constants); - - if (r_size > ninther_threshold) { - iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4), constants); - iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4), constants); - iter_swap(end - 2, end - (1 + r_size / 4), constants); - iter_swap(end - 3, end - (2 + r_size / 4), constants); - } - } - } else { - // If we were decently balanced and we tried to sort an already partitioned - // sequence try to use insertion sort. - if (already_partitioned && partial_insertion_sort(begin, pivot_pos, constants) && - partial_insertion_sort(pivot_pos + 1, end, constants)) { - return; - } - } - - // Sort the left partition first using recursion and do tail recursion elimination for - // the right-hand partition. - pdqsort_loop(begin, pivot_pos, constants, bad_allowed, leftmost); - begin = pivot_pos + 1; - leftmost = false; - } -} - -inline void pdqsort(const PDQIterator &begin, const PDQIterator &end, const PDQConstants &constants) { - if (begin == end) { - return; - } - pdqsort_loop(begin, end, constants, log2(end - begin)); -} - -inline void pdqsort_branchless(const PDQIterator &begin, const PDQIterator &end, const PDQConstants &constants) { - if (begin == end) { - return; - } - pdqsort_loop(begin, end, constants, log2(end - begin)); -} -// NOLINTEND - -} // namespace duckdb_pdqsort diff --git a/src/include/duckdb/common/sort/partition_state.hpp b/src/include/duckdb/common/sort/partition_state.hpp deleted file mode 100644 index 8170875e86dc..000000000000 --- a/src/include/duckdb/common/sort/partition_state.hpp +++ /dev/null @@ -1,245 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/common/sort/partition_state.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/common/sort/sort.hpp" -#include "duckdb/common/radix_partitioning.hpp" -#include "duckdb/parallel/base_pipeline_event.hpp" - -namespace duckdb { - -class PartitionGlobalHashGroup { -public: - using GlobalSortStatePtr = unique_ptr; - using Orders = vector; - using Types = vector; - using OrderMasks = unordered_map; - - PartitionGlobalHashGroup(ClientContext &context, const Orders &partitions, const Orders &orders, - const Types &payload_types, bool external); - - inline int ComparePartitions(const SBIterator &left, const SBIterator &right) { - int part_cmp = 0; - if (partition_layout.all_constant) { - part_cmp = FastMemcmp(left.entry_ptr, right.entry_ptr, partition_layout.comparison_size); - } else { - part_cmp = Comparators::CompareTuple(left.scan, right.scan, left.entry_ptr, right.entry_ptr, - partition_layout, left.external); - } - return part_cmp; - } - - void ComputeMasks(ValidityMask &partition_mask, OrderMasks &order_masks); - - GlobalSortStatePtr global_sort; - atomic count; - - // Mask computation - SortLayout partition_layout; -}; - -class PartitionGlobalSinkState { -public: - using HashGroupPtr = unique_ptr; - using Orders = vector; - using Types = vector; - - using GroupingPartition = unique_ptr; - using GroupingAppend = unique_ptr; - - static void GenerateOrderings(Orders &partitions, Orders &orders, - const vector> &partition_bys, const Orders &order_bys, - const vector> &partitions_stats); - - PartitionGlobalSinkState(ClientContext &context, const vector> &partition_bys, - const vector &order_bys, const Types &payload_types, - const vector> &partitions_stats, idx_t estimated_cardinality); - virtual ~PartitionGlobalSinkState() = default; - - bool HasMergeTasks() const; - - unique_ptr CreatePartition(idx_t new_bits) const; - void SyncPartitioning(const PartitionGlobalSinkState &other); - - void UpdateLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append); - void CombineLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append); - - virtual void OnBeginMerge() {}; - virtual void OnSortedPartition(const idx_t hash_bin_p) {}; - - ClientContext &context; - BufferManager &buffer_manager; - Allocator &allocator; - mutex lock; - - // OVER(PARTITION BY...) (hash grouping) - unique_ptr grouping_data; - //! Payload plus hash column - shared_ptr grouping_types_ptr; - //! The number of radix bits if this partition is being synced with another - idx_t fixed_bits; - - // OVER(...) (sorting) - Orders partitions; - Orders orders; - const Types payload_types; - vector hash_groups; - bool external; - // Reverse lookup from hash bins to non-empty hash groups - vector bin_groups; - - // OVER() (no sorting) - unique_ptr rows; - unique_ptr strings; - - // Threading - idx_t memory_per_thread; - idx_t max_bits; - atomic count; - -private: - void ResizeGroupingData(idx_t cardinality); - void SyncLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append); -}; - -class PartitionLocalSinkState { -public: - using LocalSortStatePtr = unique_ptr; - - PartitionLocalSinkState(ClientContext &context, PartitionGlobalSinkState &gstate_p); - - // Global state - PartitionGlobalSinkState &gstate; - Allocator &allocator; - - // Shared expression evaluation - ExpressionExecutor executor; - DataChunk group_chunk; - DataChunk payload_chunk; - size_t sort_cols; - - // OVER(PARTITION BY...) (hash grouping) - unique_ptr local_partition; - unique_ptr local_append; - - // OVER(ORDER BY...) (only sorting) - LocalSortStatePtr local_sort; - - // OVER() (no sorting) - RowLayout payload_layout; - unique_ptr rows; - unique_ptr strings; - - //! Compute the hash values - void Hash(DataChunk &input_chunk, Vector &hash_vector); - //! Sink an input chunk - void Sink(DataChunk &input_chunk); - //! Merge the state into the global state. - void Combine(); -}; - -enum class PartitionSortStage : uint8_t { INIT, SCAN, PREPARE, MERGE, SORTED, FINISHED }; - -class PartitionLocalMergeState; - -class PartitionGlobalMergeState { -public: - using GroupDataPtr = unique_ptr; - - // OVER(PARTITION BY...) - PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data, hash_t hash_bin); - - // OVER(ORDER BY...) - explicit PartitionGlobalMergeState(PartitionGlobalSinkState &sink); - - bool IsFinished() const { - return stage == PartitionSortStage::FINISHED; - } - - bool AssignTask(PartitionLocalMergeState &local_state); - bool TryPrepareNextStage(); - void CompleteTask(); - - PartitionGlobalSinkState &sink; - GroupDataPtr group_data; - PartitionGlobalHashGroup *hash_group; - const idx_t group_idx; - vector column_ids; - TupleDataParallelScanState chunk_state; - GlobalSortState *global_sort; - const idx_t memory_per_thread; - const idx_t num_threads; - -private: - mutable mutex lock; - atomic stage; - idx_t total_tasks; - idx_t tasks_assigned; - idx_t tasks_completed; -}; - -class PartitionLocalMergeState { -public: - explicit PartitionLocalMergeState(PartitionGlobalSinkState &gstate); - - bool TaskFinished() { - return finished; - } - - void Prepare(); - void Scan(); - void Merge(); - void Sorted(); - - void ExecuteTask(); - - PartitionGlobalMergeState *merge_state; - PartitionSortStage stage; - atomic finished; - - // Sorting buffers - ExpressionExecutor executor; - DataChunk sort_chunk; - DataChunk payload_chunk; -}; - -class PartitionGlobalMergeStates { -public: - struct Callback { - virtual ~Callback() = default; - - virtual bool HasError() const { - return false; - } - }; - - using PartitionGlobalMergeStatePtr = unique_ptr; - - explicit PartitionGlobalMergeStates(PartitionGlobalSinkState &sink); - - bool ExecuteTask(PartitionLocalMergeState &local_state, Callback &callback); - - vector states; -}; - -class PartitionMergeEvent : public BasePipelineEvent { -public: - PartitionMergeEvent(PartitionGlobalSinkState &gstate_p, Pipeline &pipeline_p, const PhysicalOperator &op_p) - : BasePipelineEvent(pipeline_p), gstate(gstate_p), merge_states(gstate_p), op(op_p) { - } - - PartitionGlobalSinkState &gstate; - PartitionGlobalMergeStates merge_states; - const PhysicalOperator &op; - -public: - void Schedule() override; -}; - -} // namespace duckdb diff --git a/src/include/duckdb/common/sort/sort.hpp b/src/include/duckdb/common/sort/sort.hpp deleted file mode 100644 index 188ea21273fa..000000000000 --- a/src/include/duckdb/common/sort/sort.hpp +++ /dev/null @@ -1,290 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/common/sort/sort.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/common/sort/sorted_block.hpp" -#include "duckdb/common/types/row/row_data_collection.hpp" -#include "duckdb/planner/bound_query_node.hpp" - -namespace duckdb { - -class RowLayout; -struct LocalSortState; - -struct SortConstants { - static constexpr idx_t VALUES_PER_RADIX = 256; - static constexpr idx_t MSD_RADIX_LOCATIONS = VALUES_PER_RADIX + 1; - static constexpr idx_t INSERTION_SORT_THRESHOLD = 24; - static constexpr idx_t MSD_RADIX_SORT_SIZE_THRESHOLD = 4; -}; - -struct SortLayout { -public: - SortLayout() { - } - explicit SortLayout(const vector &orders); - SortLayout GetPrefixComparisonLayout(idx_t num_prefix_cols) const; - -public: - idx_t column_count; - vector order_types; - vector order_by_null_types; - vector logical_types; - - bool all_constant; - vector constant_size; - vector column_sizes; - vector prefix_lengths; - vector stats; - vector has_null; - - idx_t comparison_size; - idx_t entry_size; - - RowLayout blob_layout; - unordered_map sorting_to_blob_col; -}; - -struct GlobalSortState { -public: - GlobalSortState(ClientContext &context, const vector &orders, RowLayout &payload_layout); - - //! Add local state sorted data to this global state - void AddLocalState(LocalSortState &local_sort_state); - //! Prepares the GlobalSortState for the merge sort phase (after completing radix sort phase) - void PrepareMergePhase(); - //! Initializes the global sort state for another round of merging - void InitializeMergeRound(); - //! Completes the cascaded merge sort round. - //! Pass true if you wish to use the radix data for further comparisons. - void CompleteMergeRound(bool keep_radix_data = false); - //! Print the sorted data to the console. - void Print(); - -public: - //! The client context - ClientContext &context; - //! The lock for updating the order global state - mutex lock; - //! The buffer manager - BufferManager &buffer_manager; - - //! Sorting and payload layouts - const SortLayout sort_layout; - const RowLayout payload_layout; - - //! Sorted data - vector> sorted_blocks; - vector>> sorted_blocks_temp; - unique_ptr odd_one_out; - - //! Pinned heap data (if sorting in memory) - vector> heap_blocks; - vector pinned_blocks; - - //! Capacity (number of rows) used to initialize blocks - idx_t block_capacity; - //! Whether we are doing an external sort - bool external; - - //! Progress in merge path stage - idx_t pair_idx; - idx_t num_pairs; - idx_t l_start; - idx_t r_start; -}; - -struct LocalSortState { -public: - LocalSortState(); - - //! Initialize the layouts and RowDataCollections - void Initialize(GlobalSortState &global_sort_state, BufferManager &buffer_manager_p); - //! Sink one DataChunk into the local sort state - void SinkChunk(DataChunk &sort, DataChunk &payload); - //! Size of accumulated data in bytes - idx_t SizeInBytes() const; - //! Sort the data accumulated so far - void Sort(GlobalSortState &global_sort_state, bool reorder_heap); - //! Concatenate the blocks held by a RowDataCollection into a single block - static unique_ptr ConcatenateBlocks(RowDataCollection &row_data); - -private: - //! Sorts the data in the newly created SortedBlock - void SortInMemory(); - //! Re-order the local state after sorting - void ReOrder(GlobalSortState &gstate, bool reorder_heap); - //! Re-order a SortedData object after sorting - void ReOrder(SortedData &sd, data_ptr_t sorting_ptr, RowDataCollection &heap, GlobalSortState &gstate, - bool reorder_heap); - -public: - //! Whether this local state has been initialized - bool initialized; - //! The buffer manager - BufferManager *buffer_manager; - //! The sorting and payload layouts - const SortLayout *sort_layout; - const RowLayout *payload_layout; - //! Radix/memcmp sortable data - unique_ptr radix_sorting_data; - //! Variable sized sorting data and accompanying heap - unique_ptr blob_sorting_data; - unique_ptr blob_sorting_heap; - //! Payload data and accompanying heap - unique_ptr payload_data; - unique_ptr payload_heap; - //! Sorted data - vector> sorted_blocks; - -private: - //! Selection vector and addresses for scattering the data to rows - const SelectionVector &sel_ptr = *FlatVector::IncrementalSelectionVector(); - Vector addresses = Vector(LogicalType::POINTER); -}; - -struct MergeSorter { -public: - MergeSorter(GlobalSortState &state, BufferManager &buffer_manager); - - //! Finds and merges partitions until the current cascaded merge round is finished - void PerformInMergeRound(); - -private: - //! The global sorting state - GlobalSortState &state; - //! The sorting and payload layouts - BufferManager &buffer_manager; - const SortLayout &sort_layout; - - //! The left and right reader - unique_ptr left; - unique_ptr right; - - //! Input and output blocks - unique_ptr left_input; - unique_ptr right_input; - SortedBlock *result; - -private: - //! Computes the left and right block that will be merged next (Merge Path partition) - void GetNextPartition(); - //! Finds the boundary of the next partition using binary search - void GetIntersection(const idx_t diagonal, idx_t &l_idx, idx_t &r_idx); - //! Compare values within SortedBlocks using a global index - int CompareUsingGlobalIndex(SBScanState &l, SBScanState &r, const idx_t l_idx, const idx_t r_idx); - - //! Finds the next partition and merges it - void MergePartition(); - - //! Computes how the next 'count' tuples should be merged by setting the 'left_smaller' array - void ComputeMerge(const idx_t &count, bool left_smaller[]); - - //! Merges the radix sorting blocks according to the 'left_smaller' array - void MergeRadix(const idx_t &count, const bool left_smaller[]); - //! Merges SortedData according to the 'left_smaller' array - void MergeData(SortedData &result_data, SortedData &l_data, SortedData &r_data, const idx_t &count, - const bool left_smaller[], idx_t next_entry_sizes[], bool reset_indices); - //! Merges constant size rows according to the 'left_smaller' array - void MergeRows(data_ptr_t &l_ptr, idx_t &l_entry_idx, const idx_t &l_count, data_ptr_t &r_ptr, idx_t &r_entry_idx, - const idx_t &r_count, RowDataBlock &target_block, data_ptr_t &target_ptr, const idx_t &entry_size, - const bool left_smaller[], idx_t &copied, const idx_t &count); - //! Flushes constant size rows into the result - void FlushRows(data_ptr_t &source_ptr, idx_t &source_entry_idx, const idx_t &source_count, - RowDataBlock &target_block, data_ptr_t &target_ptr, const idx_t &entry_size, idx_t &copied, - const idx_t &count); - //! Flushes blob rows and accompanying heap - void FlushBlobs(const RowLayout &layout, const idx_t &source_count, data_ptr_t &source_data_ptr, - idx_t &source_entry_idx, data_ptr_t &source_heap_ptr, RowDataBlock &target_data_block, - data_ptr_t &target_data_ptr, RowDataBlock &target_heap_block, BufferHandle &target_heap_handle, - data_ptr_t &target_heap_ptr, idx_t &copied, const idx_t &count); -}; - -struct SBIterator { - static int ComparisonValue(ExpressionType comparison); - - SBIterator(GlobalSortState &gss, ExpressionType comparison, idx_t entry_idx_p = 0); - - inline idx_t GetIndex() const { - return entry_idx; - } - - inline void SetIndex(idx_t entry_idx_p) { - const auto new_block_idx = entry_idx_p / block_capacity; - if (new_block_idx != scan.block_idx) { - scan.SetIndices(new_block_idx, 0); - if (new_block_idx < block_count) { - scan.PinRadix(scan.block_idx); - block_ptr = scan.RadixPtr(); - if (!all_constant) { - scan.PinData(*scan.sb->blob_sorting_data); - } - } - } - - scan.entry_idx = entry_idx_p % block_capacity; - entry_ptr = block_ptr + scan.entry_idx * entry_size; - entry_idx = entry_idx_p; - } - - inline SBIterator &operator++() { - if (++scan.entry_idx < block_capacity) { - entry_ptr += entry_size; - ++entry_idx; - } else { - SetIndex(entry_idx + 1); - } - - return *this; - } - - inline SBIterator &operator--() { - if (scan.entry_idx) { - --scan.entry_idx; - --entry_idx; - entry_ptr -= entry_size; - } else { - SetIndex(entry_idx - 1); - } - - return *this; - } - - inline bool Compare(const SBIterator &other, const SortLayout &prefix) const { - int comp_res; - if (all_constant) { - comp_res = FastMemcmp(entry_ptr, other.entry_ptr, prefix.comparison_size); - } else { - comp_res = Comparators::CompareTuple(scan, other.scan, entry_ptr, other.entry_ptr, prefix, external); - } - - return comp_res <= cmp; - } - - inline bool Compare(const SBIterator &other) const { - return Compare(other, sort_layout); - } - - // Fixed comparison parameters - const SortLayout &sort_layout; - const idx_t block_count; - const idx_t block_capacity; - const size_t entry_size; - const bool all_constant; - const bool external; - const int cmp; - - // Iteration state - SBScanState scan; - idx_t entry_idx; - data_ptr_t block_ptr; - data_ptr_t entry_ptr; -}; - -} // namespace duckdb diff --git a/src/include/duckdb/common/sort/sorted_block.hpp b/src/include/duckdb/common/sort/sorted_block.hpp deleted file mode 100644 index b6941bda20d8..000000000000 --- a/src/include/duckdb/common/sort/sorted_block.hpp +++ /dev/null @@ -1,165 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/common/sort/sorted_block.hpp -// -// -//===----------------------------------------------------------------------===// -#pragma once - -#include "duckdb/common/fast_mem.hpp" -#include "duckdb/common/sort/comparators.hpp" -#include "duckdb/common/types/row/row_data_collection_scanner.hpp" -#include "duckdb/common/types/row/row_layout.hpp" -#include "duckdb/storage/buffer/buffer_handle.hpp" - -namespace duckdb { - -class BufferManager; -struct RowDataBlock; -struct SortLayout; -struct GlobalSortState; - -enum class SortedDataType { BLOB, PAYLOAD }; - -//! Object that holds sorted rows, and an accompanying heap if there are blobs -struct SortedData { -public: - SortedData(SortedDataType type, const RowLayout &layout, BufferManager &buffer_manager, GlobalSortState &state); - //! Number of rows that this object holds - idx_t Count(); - //! Initialize new block to write to - void CreateBlock(); - //! Create a slice that holds the rows between the start and end indices - unique_ptr CreateSlice(idx_t start_block_index, idx_t end_block_index, idx_t end_entry_index); - //! Unswizzles all - void Unswizzle(); - -public: - const SortedDataType type; - //! Layout of this data - const RowLayout layout; - //! Data and heap blocks - vector> data_blocks; - vector> heap_blocks; - //! Whether the pointers in this sorted data are swizzled - bool swizzled; - -private: - //! The buffer manager - BufferManager &buffer_manager; - //! The global state - GlobalSortState &state; -}; - -//! Block that holds sorted rows: radix, blob and payload data -struct SortedBlock { -public: - SortedBlock(BufferManager &buffer_manager, GlobalSortState &gstate); - //! Number of rows that this object holds - idx_t Count() const; - //! Initialize this block to write data to - void InitializeWrite(); - //! Init new block to write to - void CreateBlock(); - //! Fill this sorted block by appending the blocks held by a vector of sorted blocks - void AppendSortedBlocks(vector> &sorted_blocks); - //! Locate the block and entry index of a row in this block, - //! given an index between 0 and the total number of rows in this block - void GlobalToLocalIndex(const idx_t &global_idx, idx_t &local_block_index, idx_t &local_entry_index); - //! Create a slice that holds the rows between the start and end indices - unique_ptr CreateSlice(const idx_t start, const idx_t end, idx_t &entry_idx); - - //! Size (in bytes) of the heap of this block - idx_t HeapSize() const; - //! Total size (in bytes) of this block - idx_t SizeInBytes() const; - -public: - //! Radix/memcmp sortable data - vector> radix_sorting_data; - //! Variable sized sorting data - unique_ptr blob_sorting_data; - //! Payload data - unique_ptr payload_data; - -private: - //! Buffer manager, global state, and sorting layout constants - BufferManager &buffer_manager; - GlobalSortState &state; - const SortLayout &sort_layout; - const RowLayout &payload_layout; -}; - -//! State used to scan a SortedBlock e.g. during merge sort -struct SBScanState { -public: - SBScanState(BufferManager &buffer_manager, GlobalSortState &state); - - void PinRadix(idx_t block_idx_to); - void PinData(SortedData &sd); - - data_ptr_t RadixPtr() const; - data_ptr_t DataPtr(SortedData &sd) const; - data_ptr_t HeapPtr(SortedData &sd) const; - data_ptr_t BaseHeapPtr(SortedData &sd) const; - - idx_t Remaining() const; - - void SetIndices(idx_t block_idx_to, idx_t entry_idx_to); - -public: - BufferManager &buffer_manager; - const SortLayout &sort_layout; - GlobalSortState &state; - - SortedBlock *sb; - - idx_t block_idx; - idx_t entry_idx; - - BufferHandle radix_handle; - - BufferHandle blob_sorting_data_handle; - BufferHandle blob_sorting_heap_handle; - - BufferHandle payload_data_handle; - BufferHandle payload_heap_handle; -}; - -//! Used to scan the data into DataChunks after sorting -struct PayloadScanner { -public: - PayloadScanner(SortedData &sorted_data, GlobalSortState &global_sort_state, bool flush = true); - explicit PayloadScanner(GlobalSortState &global_sort_state, bool flush = true); - - //! Scan a single block - PayloadScanner(GlobalSortState &global_sort_state, idx_t block_idx, bool flush = false); - - //! The type layout of the payload - inline const vector &GetPayloadTypes() const { - return scanner->GetTypes(); - } - - //! The number of rows scanned so far - inline idx_t Scanned() const { - return scanner->Scanned(); - } - - //! The number of remaining rows - inline idx_t Remaining() const { - return scanner->Remaining(); - } - - //! Scans the next data chunk from the sorted data - void Scan(DataChunk &chunk); - -private: - //! The sorted data being scanned - unique_ptr rows; - unique_ptr heap; - //! The actual scanner - unique_ptr scanner; -}; - -} // namespace duckdb diff --git a/src/include/duckdb/common/sorting/hashed_sort.hpp b/src/include/duckdb/common/sorting/hashed_sort.hpp index 3f1877b1a442..022b4c6062d6 100644 --- a/src/include/duckdb/common/sorting/hashed_sort.hpp +++ b/src/include/duckdb/common/sorting/hashed_sort.hpp @@ -17,6 +17,7 @@ class HashedSort { using Orders = vector; using Types = vector; using HashGroupPtr = unique_ptr; + using SortedRunPtr = unique_ptr; static void GenerateOrderings(Orders &partitions, Orders &orders, const vector> &partition_bys, const Orders &order_bys, @@ -24,7 +25,8 @@ class HashedSort { HashedSort(ClientContext &context, const vector> &partition_bys, const vector &order_bys, const Types &payload_types, - const vector> &partitions_stats, idx_t estimated_cardinality); + const vector> &partitions_stats, idx_t estimated_cardinality, + bool require_payload = false); public: //===--------------------------------------------------------------------===// @@ -35,6 +37,9 @@ class HashedSort { SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const; SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const; SinkFinalizeType Finalize(ClientContext &client, OperatorSinkFinalizeInput &finalize) const; + ProgressData GetSinkProgress(ClientContext &context, GlobalSinkState &gstate, + const ProgressData source_progress) const; + void Synchronize(const GlobalSinkState &source, GlobalSinkState &target) const; public: //===--------------------------------------------------------------------===// @@ -51,6 +56,10 @@ class HashedSort { OperatorSinkFinalizeInput &finalize) const; vector &GetHashGroups(GlobalSourceState &global_state) const; + SinkFinalizeType MaterializeSortedRuns(Pipeline &pipeline, Event &event, const PhysicalOperator &op, + OperatorSinkFinalizeInput &finalize) const; + vector &GetSortedRuns(GlobalSourceState &global_state) const; + public: ClientContext &client; //! The host's estimated row count @@ -61,6 +70,8 @@ class HashedSort { Orders orders; idx_t sort_col_count; Types payload_types; + //! Are we creating a dummy payload column? + bool force_payload = false; // Input columns in the sorted output vector scan_ids; // Key columns in the sorted output diff --git a/src/include/duckdb/common/sorting/sort.hpp b/src/include/duckdb/common/sorting/sort.hpp index 597b8261b626..de1e33f3b476 100644 --- a/src/include/duckdb/common/sorting/sort.hpp +++ b/src/include/duckdb/common/sorting/sort.hpp @@ -8,25 +8,44 @@ #pragma once -#include "duckdb/common/sorting/sorted_run.hpp" -#include "duckdb/common/types/row/tuple_data_layout.hpp" #include "duckdb/execution/physical_operator_states.hpp" +#include "duckdb/execution/progress_data.hpp" #include "duckdb/common/sorting/sort_projection_column.hpp" +#include "duckdb/planner/bound_result_modifier.hpp" namespace duckdb { class SortLocalSinkState; class SortGlobalSinkState; + class SortLocalSourceState; class SortGlobalSourceState; +class SortedRun; +class SortedRunScanState; + +class SortedRunMerger; +class SortedRunMergerLocalState; +class SortedRunMergerGlobalState; + +class TupleDataLayout; +class ColumnDataCollection; + //! Class that sorts the data, follows the PhysicalOperator interface class Sort { friend class SortLocalSinkState; friend class SortGlobalSinkState; + friend class SortLocalSourceState; friend class SortGlobalSourceState; + friend class SortedRun; + friend class SortedRunScanState; + + friend class SortedRunMerger; + friend class SortedRunMergerLocalState; + friend class SortedRunMergerGlobalState; + public: Sort(ClientContext &context, const vector &orders, const vector &input_types, vector projection_map, bool is_index_sort = false); @@ -45,7 +64,7 @@ class Sort { vector input_projection_map; vector output_projection_columns; - //! Whether to force an external sort + //! Whether to force an approximate sort bool is_index_sort; public: diff --git a/src/include/duckdb/common/sorting/sorted_run.hpp b/src/include/duckdb/common/sorting/sorted_run.hpp index fe0d67e32128..53ce1c5c0106 100644 --- a/src/include/duckdb/common/sorting/sorted_run.hpp +++ b/src/include/duckdb/common/sorting/sorted_run.hpp @@ -9,18 +9,40 @@ #pragma once #include "duckdb/common/types/row/tuple_data_states.hpp" +#include "duckdb/execution/expression_executor.hpp" namespace duckdb { +class Sort; +class SortedRun; class BufferManager; class DataChunk; class TupleDataCollection; class TupleDataLayout; +class SortedRunScanState { +public: + SortedRunScanState(ClientContext &context, const Sort &sort); + +public: + void Scan(const SortedRun &sorted_run, const Vector &sort_key_pointers, const idx_t &count, DataChunk &chunk); + +private: + template + void TemplatedScan(const SortedRun &sorted_run, const Vector &sort_key_pointers, const idx_t &count, + DataChunk &chunk); + +private: + const Sort &sort; + ExpressionExecutor key_executor; + DataChunk key; + DataChunk decoded_key; + TupleDataScanState payload_state; +}; + class SortedRun { public: - SortedRun(ClientContext &context, shared_ptr key_layout, - shared_ptr payload_layout, bool is_index_sort); + SortedRun(ClientContext &context, const Sort &sort, bool is_index_sort); unique_ptr CreateRunForMaterialization() const; ~SortedRun(); @@ -36,8 +58,13 @@ class SortedRun { //! Size of this sorted run idx_t SizeInBytes() const; +private: + mutex merger_global_state_lock; + unique_ptr merge_global_state; + public: ClientContext &context; + const Sort &sort; //! Key and payload collections (and associated append states) unique_ptr key_data; diff --git a/src/include/duckdb/common/sorting/sorted_run_merger.hpp b/src/include/duckdb/common/sorting/sorted_run_merger.hpp index 21a56df8319c..fd894d698dee 100644 --- a/src/include/duckdb/common/sorting/sorted_run_merger.hpp +++ b/src/include/duckdb/common/sorting/sorted_run_merger.hpp @@ -9,10 +9,10 @@ #pragma once #include "duckdb/execution/physical_operator_states.hpp" -#include "duckdb/common/sorting/sort_projection_column.hpp" namespace duckdb { +class Sort; class TupleDataLayout; struct BoundOrderByNode; struct ProgressData; @@ -24,9 +24,7 @@ class SortedRunMerger { friend class SortedRunMergerGlobalState; public: - SortedRunMerger(const Expression &decode_sort_key, shared_ptr key_layout, - vector> &&sorted_runs, - const vector &output_projection_columns, idx_t partition_size, bool external, + SortedRunMerger(const Sort &sort, vector> &&sorted_runs, idx_t partition_size, bool external, bool is_index_sort); public: @@ -44,14 +42,12 @@ class SortedRunMerger { //===--------------------------------------------------------------------===// // Non-Standard Interface //===--------------------------------------------------------------------===// - SourceResultType MaterializeMerge(ExecutionContext &context, OperatorSourceInput &input) const; - unique_ptr GetMaterialized(GlobalSourceState &global_state); + SourceResultType MaterializeSortedRun(ExecutionContext &context, OperatorSourceInput &input) const; + unique_ptr GetSortedRun(GlobalSourceState &global_state); public: - const Expression &decode_sort_key; - shared_ptr key_layout; + const Sort &sort; vector> sorted_runs; - const vector &output_projection_columns; const idx_t total_count; const idx_t partition_size; diff --git a/src/include/duckdb/common/string_map_set.hpp b/src/include/duckdb/common/string_map_set.hpp index 00600c421c3c..40bd51171410 100644 --- a/src/include/duckdb/common/string_map_set.hpp +++ b/src/include/duckdb/common/string_map_set.hpp @@ -28,9 +28,26 @@ struct StringEquality { } }; +struct StringCIHash { + std::size_t operator()(const string_t &k) const { + return StringUtil::CIHash(k.GetData(), k.GetSize()); + } +}; + +struct StringCIEquality { + bool operator()(const string_t &a, const string_t &b) const { + return StringUtil::CIEquals(a.GetData(), a.GetSize(), b.GetData(), b.GetSize()); + } +}; + template using string_map_t = unordered_map; using string_set_t = unordered_set; +template +using case_insensitive_string_map_t = unordered_map; + +using case_insensitive_string_set_t = unordered_set; + } // namespace duckdb diff --git a/src/include/duckdb/common/string_util.hpp b/src/include/duckdb/common/string_util.hpp index 8c0c19bef93a..85c24a1d6005 100644 --- a/src/include/duckdb/common/string_util.hpp +++ b/src/include/duckdb/common/string_util.hpp @@ -217,6 +217,7 @@ class StringUtil { //! Case insensitive hash DUCKDB_API static uint64_t CIHash(const string &str); + DUCKDB_API static uint64_t CIHash(const char *str, idx_t size); //! Case insensitive equals DUCKDB_API static bool CIEquals(const string &l1, const string &l2); @@ -318,6 +319,8 @@ class StringUtil { //! Transforms an complex JSON to a JSON string DUCKDB_API static string ToComplexJSONMap(const ComplexJSON &complex_json); + DUCKDB_API static string ValidateJSON(const char *data, const idx_t &len); + DUCKDB_API static string GetFileName(const string &file_path); DUCKDB_API static string GetFileExtension(const string &file_name); DUCKDB_API static string GetFileStem(const string &file_name); diff --git a/src/include/duckdb/common/types.hpp b/src/include/duckdb/common/types.hpp index 0f7ddbb2de74..6d85ce2de2f5 100644 --- a/src/include/duckdb/common/types.hpp +++ b/src/include/duckdb/common/types.hpp @@ -230,6 +230,8 @@ enum class LogicalTypeId : uint8_t { VALIDITY = 53, UUID = 54, + GEOMETRY = 60, + STRUCT = 100, LIST = 101, MAP = 102, @@ -430,6 +432,7 @@ struct LogicalType { DUCKDB_API static LogicalType UNION(child_list_t members); // NOLINT DUCKDB_API static LogicalType ARRAY(const LogicalType &child, optional_idx index); // NOLINT DUCKDB_API static LogicalType ENUM(Vector &ordered_data, idx_t size); // NOLINT + DUCKDB_API static LogicalType GEOMETRY(); // NOLINT // ANY but with special rules (default is LogicalType::ANY, 5) DUCKDB_API static LogicalType ANY_PARAMS(LogicalType target, idx_t cast_score = 5); // NOLINT DUCKDB_API static LogicalType TEMPLATE(const string &name); // NOLINT diff --git a/src/include/duckdb/common/types/geometry.hpp b/src/include/duckdb/common/types/geometry.hpp new file mode 100644 index 000000000000..228729a3ed63 --- /dev/null +++ b/src/include/duckdb/common/types/geometry.hpp @@ -0,0 +1,204 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/common/types/time.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/common.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/common/pair.hpp" +#include +#include + +namespace duckdb { + +struct GeometryStatsData; + +enum class GeometryType : uint8_t { + INVALID = 0, + POINT = 1, + LINESTRING = 2, + POLYGON = 3, + MULTIPOINT = 4, + MULTILINESTRING = 5, + MULTIPOLYGON = 6, + GEOMETRYCOLLECTION = 7, +}; + +enum class VertexType : uint8_t { XY = 0, XYZ = 1, XYM = 2, XYZM = 3 }; + +struct VertexXY { + static constexpr auto TYPE = VertexType::XY; + static constexpr auto HAS_Z = false; + static constexpr auto HAS_M = false; + + double x; + double y; + + bool AllNan() const { + return std::isnan(x) && std::isnan(y); + } +}; + +struct VertexXYZ { + static constexpr auto TYPE = VertexType::XYZ; + static constexpr auto HAS_Z = true; + static constexpr auto HAS_M = false; + + double x; + double y; + double z; + + bool AllNan() const { + return std::isnan(x) && std::isnan(y) && std::isnan(z); + } +}; +struct VertexXYM { + static constexpr auto TYPE = VertexType::XYM; + static constexpr auto HAS_M = true; + static constexpr auto HAS_Z = false; + + double x; + double y; + double m; + + bool AllNan() const { + return std::isnan(x) && std::isnan(y) && std::isnan(m); + } +}; + +struct VertexXYZM { + static constexpr auto TYPE = VertexType::XYZM; + static constexpr auto HAS_Z = true; + static constexpr auto HAS_M = true; + + double x; + double y; + double z; + double m; + + bool AllNan() const { + return std::isnan(x) && std::isnan(y) && std::isnan(z) && std::isnan(m); + } +}; + +class GeometryExtent { +public: + static constexpr auto UNKNOWN_MIN = -std::numeric_limits::infinity(); + static constexpr auto UNKNOWN_MAX = +std::numeric_limits::infinity(); + + static constexpr auto EMPTY_MIN = +std::numeric_limits::infinity(); + static constexpr auto EMPTY_MAX = -std::numeric_limits::infinity(); + + // "Unknown" extent means we don't know the bounding box. + // Merging with an unknown extent results in an unknown extent. + // Everything intersects with an unknown extent. + static GeometryExtent Unknown() { + return GeometryExtent {UNKNOWN_MIN, UNKNOWN_MIN, UNKNOWN_MIN, UNKNOWN_MIN, + UNKNOWN_MAX, UNKNOWN_MAX, UNKNOWN_MAX, UNKNOWN_MAX}; + } + + // "Empty" extent means the smallest possible bounding box. + // Merging with an empty extent has no effect. + // Nothing intersects with an empty extent. + static GeometryExtent Empty() { + return GeometryExtent {EMPTY_MIN, EMPTY_MIN, EMPTY_MIN, EMPTY_MIN, EMPTY_MAX, EMPTY_MAX, EMPTY_MAX, EMPTY_MAX}; + } + + // Does this extent have any X/Y values set? + // In other words, is the range of the x/y axes not empty and not unknown? + bool HasXY() const { + return std::isfinite(x_min) && std::isfinite(y_min) && std::isfinite(x_max) && std::isfinite(y_max); + } + // Does this extent have any Z values set? + // In other words, is the range of the Z-axis not empty and not unknown? + bool HasZ() const { + return std::isfinite(z_min) && std::isfinite(z_max); + } + // Does this extent have any M values set? + // In other words, is the range of the M-axis not empty and not unknown? + bool HasM() const { + return std::isfinite(m_min) && std::isfinite(m_max); + } + + void Extend(const VertexXY &vertex) { + x_min = MinValue(x_min, vertex.x); + x_max = MaxValue(x_max, vertex.x); + y_min = MinValue(y_min, vertex.y); + y_max = MaxValue(y_max, vertex.y); + } + + void Extend(const VertexXYZ &vertex) { + x_min = MinValue(x_min, vertex.x); + x_max = MaxValue(x_max, vertex.x); + y_min = MinValue(y_min, vertex.y); + y_max = MaxValue(y_max, vertex.y); + z_min = MinValue(z_min, vertex.z); + z_max = MaxValue(z_max, vertex.z); + } + + void Extend(const VertexXYM &vertex) { + x_min = MinValue(x_min, vertex.x); + x_max = MaxValue(x_max, vertex.x); + y_min = MinValue(y_min, vertex.y); + y_max = MaxValue(y_max, vertex.y); + m_min = MinValue(m_min, vertex.m); + m_max = MaxValue(m_max, vertex.m); + } + + void Extend(const VertexXYZM &vertex) { + x_min = MinValue(x_min, vertex.x); + x_max = MaxValue(x_max, vertex.x); + y_min = MinValue(y_min, vertex.y); + y_max = MaxValue(y_max, vertex.y); + z_min = MinValue(z_min, vertex.z); + z_max = MaxValue(z_max, vertex.z); + m_min = MinValue(m_min, vertex.m); + m_max = MaxValue(m_max, vertex.m); + } + + void Merge(const GeometryExtent &other) { + x_min = MinValue(x_min, other.x_min); + y_min = MinValue(y_min, other.y_min); + z_min = MinValue(z_min, other.z_min); + m_min = MinValue(m_min, other.m_min); + + x_max = MaxValue(x_max, other.x_max); + y_max = MaxValue(y_max, other.y_max); + z_max = MaxValue(z_max, other.z_max); + m_max = MaxValue(m_max, other.m_max); + } + + double x_min; + double y_min; + double z_min; + double m_min; + + double x_max; + double y_max; + double z_max; + double m_max; +}; + +class Geometry { +public: + static constexpr idx_t MAX_RECURSION_DEPTH = 16; + + //! Convert from WKT + DUCKDB_API static bool FromString(const string_t &wkt_text, string_t &result, Vector &result_vector, bool strict); + + //! Convert to WKT + DUCKDB_API static string_t ToString(Vector &result, const string_t &geom); + + //! Get the geometry type and vertex type from the WKB + DUCKDB_API static pair GetType(const string_t &wkb); + + //! Update the bounding box, return number of vertices processed + DUCKDB_API static uint32_t GetExtent(const string_t &wkb, GeometryExtent &extent); +}; + +} // namespace duckdb diff --git a/src/include/duckdb/common/types/hugeint.hpp b/src/include/duckdb/common/types/hugeint.hpp index 3720bf844ec2..9fa5d447ba89 100644 --- a/src/include/duckdb/common/types/hugeint.hpp +++ b/src/include/duckdb/common/types/hugeint.hpp @@ -129,38 +129,38 @@ class Hugeint { static int Sign(hugeint_t n); static hugeint_t Abs(hugeint_t n); // comparison operators - static bool Equals(hugeint_t lhs, hugeint_t rhs) { + static bool Equals(const hugeint_t &lhs, const hugeint_t &rhs) { bool lower_equals = lhs.lower == rhs.lower; bool upper_equals = lhs.upper == rhs.upper; return lower_equals && upper_equals; } - static bool NotEquals(hugeint_t lhs, hugeint_t rhs) { + static bool NotEquals(const hugeint_t &lhs, const hugeint_t &rhs) { return !Equals(lhs, rhs); } - static bool GreaterThan(hugeint_t lhs, hugeint_t rhs) { + static bool GreaterThan(const hugeint_t &lhs, const hugeint_t &rhs) { bool upper_bigger = lhs.upper > rhs.upper; bool upper_equal = lhs.upper == rhs.upper; bool lower_bigger = lhs.lower > rhs.lower; return upper_bigger || (upper_equal && lower_bigger); } - static bool GreaterThanEquals(hugeint_t lhs, hugeint_t rhs) { + static bool GreaterThanEquals(const hugeint_t &lhs, const hugeint_t &rhs) { bool upper_bigger = lhs.upper > rhs.upper; bool upper_equal = lhs.upper == rhs.upper; bool lower_bigger_equals = lhs.lower >= rhs.lower; return upper_bigger || (upper_equal && lower_bigger_equals); } - static bool LessThan(hugeint_t lhs, hugeint_t rhs) { + static bool LessThan(const hugeint_t &lhs, const hugeint_t &rhs) { bool upper_smaller = lhs.upper < rhs.upper; bool upper_equal = lhs.upper == rhs.upper; bool lower_smaller = lhs.lower < rhs.lower; return upper_smaller || (upper_equal && lower_smaller); } - static bool LessThanEquals(hugeint_t lhs, hugeint_t rhs) { + static bool LessThanEquals(const hugeint_t &lhs, const hugeint_t &rhs) { bool upper_smaller = lhs.upper < rhs.upper; bool upper_equal = lhs.upper == rhs.upper; bool lower_smaller_equals = lhs.lower <= rhs.lower; diff --git a/src/include/duckdb/common/types/row/block_iterator.hpp b/src/include/duckdb/common/types/row/block_iterator.hpp index c29b094a86cf..f9ec233eba37 100644 --- a/src/include/duckdb/common/types/row/block_iterator.hpp +++ b/src/include/duckdb/common/types/row/block_iterator.hpp @@ -183,15 +183,14 @@ class ExternalBlockIteratorState { key_scan_state.pin_state.row_handles.acquire_handles(pins); key_scan_state.pin_state.heap_handles.acquire_handles(pins); } - key_data.FetchChunk(key_scan_state, 0, chunk_idx, false); + key_data.FetchChunk(key_scan_state, chunk_idx, false); if (pin_payload && payload_data) { if (keep_pinned) { payload_scan_state.pin_state.row_handles.acquire_handles(pins); payload_scan_state.pin_state.heap_handles.acquire_handles(pins); } - const auto chunk_count = payload_data->FetchChunk(payload_scan_state, 0, chunk_idx, false); + const auto chunk_count = payload_data->FetchChunk(payload_scan_state, chunk_idx, false); const auto sort_keys = reinterpret_cast(key_ptrs); - payload_data->FetchChunk(payload_scan_state, 0, chunk_idx, false); const auto payload_ptrs = FlatVector::GetData(payload_scan_state.chunk_state.row_locations); for (idx_t i = 0; i < chunk_count; i++) { sort_keys[i]->SetPayload(payload_ptrs[i]); diff --git a/src/include/duckdb/common/types/row/tuple_data_collection.hpp b/src/include/duckdb/common/types/row/tuple_data_collection.hpp index d759341ee9f8..e0f0a0fd256d 100644 --- a/src/include/duckdb/common/types/row/tuple_data_collection.hpp +++ b/src/include/duckdb/common/types/row/tuple_data_collection.hpp @@ -50,6 +50,7 @@ class TupleDataCollection { public: //! Constructs a TupleDataCollection with the specified layout TupleDataCollection(BufferManager &buffer_manager, shared_ptr layout_ptr); + TupleDataCollection(ClientContext &context, shared_ptr layout_ptr); ~TupleDataCollection(); @@ -185,8 +186,8 @@ class TupleDataCollection { //! Initialize a parallel scan over the tuple data collection over a subset of the columns void InitializeScan(TupleDataParallelScanState &gstate, vector column_ids, TupleDataPinProperties properties = TupleDataPinProperties::UNPIN_AFTER_DONE) const; - //! Grab the chunk state for the given segment and chunk index, returns the count of the chunk - idx_t FetchChunk(TupleDataScanState &state, idx_t segment_idx, idx_t chunk_idx, bool init_heap); + //! Grab the chunk state for the given chunk index, returns the count of the chunk + idx_t FetchChunk(TupleDataScanState &state, idx_t chunk_idx, bool init_heap); //! Scans a DataChunk from the TupleDataCollection bool Scan(TupleDataScanState &state, DataChunk &result); //! Scans a DataChunk from the TupleDataCollection diff --git a/src/include/duckdb/common/types/selection_vector.hpp b/src/include/duckdb/common/types/selection_vector.hpp index ceb5637ac9fc..5575e5a086ec 100644 --- a/src/include/duckdb/common/types/selection_vector.hpp +++ b/src/include/duckdb/common/types/selection_vector.hpp @@ -108,6 +108,7 @@ struct SelectionVector { return selection_data; } buffer_ptr Slice(const SelectionVector &sel, idx_t count) const; + idx_t SliceInPlace(const SelectionVector &sel, idx_t count); string ToString(idx_t count = 0) const; void Print(idx_t count = 0) const; diff --git a/src/include/duckdb/common/types/string_type.hpp b/src/include/duckdb/common/types/string_type.hpp index 1f69a53a73b4..59bb3c293033 100644 --- a/src/include/duckdb/common/types/string_type.hpp +++ b/src/include/duckdb/common/types/string_type.hpp @@ -97,8 +97,14 @@ struct string_t { return value.inlined.length; } - void SetSizeAndFinalize(uint32_t size) { + void SetSizeAndFinalize(uint32_t size, idx_t allocated_size) { value.inlined.length = size; + if (allocated_size > INLINE_LENGTH && IsInlined()) { + //! Data was written to the 'value.pointer.ptr', has to be copied to the inlined bytes + D_ASSERT(value.pointer.ptr); + auto ptr = value.pointer.ptr; + memcpy(GetDataWriteable(), ptr, size); + } Finalize(); VerifyCharacters(); } diff --git a/src/include/duckdb/common/types/value.hpp b/src/include/duckdb/common/types/value.hpp index 1993d02950aa..bba9a7297049 100644 --- a/src/include/duckdb/common/types/value.hpp +++ b/src/include/duckdb/common/types/value.hpp @@ -201,6 +201,8 @@ class Value { DUCKDB_API static Value BIGNUM(const_data_ptr_t data, idx_t len); DUCKDB_API static Value BIGNUM(const string &data); + DUCKDB_API static Value GEOMETRY(const_data_ptr_t data, idx_t len); + //! Creates an aggregate state DUCKDB_API static Value AGGREGATE_STATE(const LogicalType &type, const_data_ptr_t data, idx_t len); // NOLINT diff --git a/src/include/duckdb/common/types/variant.hpp b/src/include/duckdb/common/types/variant.hpp index cc8a9ffa6298..bef2f2353c10 100644 --- a/src/include/duckdb/common/types/variant.hpp +++ b/src/include/duckdb/common/types/variant.hpp @@ -29,8 +29,18 @@ struct VariantNestedData { }; struct VariantDecimalData { +public: + VariantDecimalData(uint32_t width, uint32_t scale, const_data_ptr_t value_ptr) + : width(width), scale(scale), value_ptr(value_ptr) { + } + +public: + PhysicalType GetPhysicalType() const; + +public: uint32_t width; uint32_t scale; + const_data_ptr_t value_ptr = nullptr; }; struct VariantVectorData { @@ -105,6 +115,7 @@ enum class VariantLogicalType : uint8_t { ARRAY = 30, BIGNUM = 31, BITSTRING = 32, + GEOMETRY = 33, ENUM_SIZE /* always kept as last item of the enum */ }; diff --git a/src/include/duckdb/common/types/variant_visitor.hpp b/src/include/duckdb/common/types/variant_visitor.hpp new file mode 100644 index 000000000000..950980aefc85 --- /dev/null +++ b/src/include/duckdb/common/types/variant_visitor.hpp @@ -0,0 +1,232 @@ +#pragma once + +#include "duckdb/common/types/variant.hpp" +#include "duckdb/function/scalar/variant_utils.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/common/vector.hpp" +#include "duckdb/common/types/decimal.hpp" +#include "duckdb/common/enum_util.hpp" + +#include + +namespace duckdb { + +template +class VariantVisitor { + // Detects if T has a static VisitMetadata with signature + // void VisitMetadata(VariantLogicalType, Args...) + template + class has_visit_metadata { + private: + template + static auto test(int) -> decltype(U::VisitMetadata(std::declval(), std::declval()...), + std::true_type {}); + + template + static std::false_type test(...); + + public: + static constexpr bool value = decltype(test(0))::value; + }; + +public: + template + static ReturnType Visit(const UnifiedVariantVectorData &variant, idx_t row, uint32_t values_idx, Args &&...args) { + if (!variant.RowIsValid(row)) { + return Visitor::VisitNull(std::forward(args)...); + } + + auto type_id = variant.GetTypeId(row, values_idx); + auto byte_offset = variant.GetByteOffset(row, values_idx); + auto blob_data = const_data_ptr_cast(variant.GetData(row).GetData()); + auto ptr = const_data_ptr_cast(blob_data + byte_offset); + + VisitMetadata(type_id, std::forward(args)...); + + switch (type_id) { + case VariantLogicalType::VARIANT_NULL: + return Visitor::VisitNull(std::forward(args)...); + case VariantLogicalType::BOOL_TRUE: + return Visitor::VisitBoolean(true, std::forward(args)...); + case VariantLogicalType::BOOL_FALSE: + return Visitor::VisitBoolean(false, std::forward(args)...); + case VariantLogicalType::INT8: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::INT16: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::INT32: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::INT64: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::INT128: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::UINT8: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::UINT16: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::UINT32: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::UINT64: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::UINT128: + return Visitor::template VisitInteger(Load(ptr), std::forward(args)...); + case VariantLogicalType::FLOAT: + return Visitor::VisitFloat(Load(ptr), std::forward(args)...); + case VariantLogicalType::DOUBLE: + return Visitor::VisitDouble(Load(ptr), std::forward(args)...); + case VariantLogicalType::UUID: + return Visitor::VisitUUID(Load(ptr), std::forward(args)...); + case VariantLogicalType::DATE: + return Visitor::VisitDate(date_t(Load(ptr)), std::forward(args)...); + case VariantLogicalType::INTERVAL: + return Visitor::VisitInterval(Load(ptr), std::forward(args)...); + case VariantLogicalType::VARCHAR: + case VariantLogicalType::BLOB: + case VariantLogicalType::BITSTRING: + case VariantLogicalType::BIGNUM: + case VariantLogicalType::GEOMETRY: + return VisitString(type_id, variant, row, values_idx, std::forward(args)...); + case VariantLogicalType::DECIMAL: + return VisitDecimal(variant, row, values_idx, std::forward(args)...); + case VariantLogicalType::ARRAY: + return VisitArray(variant, row, values_idx, std::forward(args)...); + case VariantLogicalType::OBJECT: + return VisitObject(variant, row, values_idx, std::forward(args)...); + case VariantLogicalType::TIME_MICROS: + return Visitor::VisitTime(Load(ptr), std::forward(args)...); + case VariantLogicalType::TIME_NANOS: + return Visitor::VisitTimeNanos(Load(ptr), std::forward(args)...); + case VariantLogicalType::TIME_MICROS_TZ: + return Visitor::VisitTimeTZ(Load(ptr), std::forward(args)...); + case VariantLogicalType::TIMESTAMP_SEC: + return Visitor::VisitTimestampSec(Load(ptr), std::forward(args)...); + case VariantLogicalType::TIMESTAMP_MILIS: + return Visitor::VisitTimestampMs(Load(ptr), std::forward(args)...); + case VariantLogicalType::TIMESTAMP_MICROS: + return Visitor::VisitTimestamp(Load(ptr), std::forward(args)...); + case VariantLogicalType::TIMESTAMP_NANOS: + return Visitor::VisitTimestampNanos(Load(ptr), std::forward(args)...); + case VariantLogicalType::TIMESTAMP_MICROS_TZ: + return Visitor::VisitTimestampTZ(Load(ptr), std::forward(args)...); + default: + return Visitor::VisitDefault(type_id, ptr, std::forward(args)...); + } + } + + // Non-void version + template + static typename std::enable_if::value, vector>::type + VisitArrayItems(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &array_data, + Args &&...args) { + vector array_items; + array_items.reserve(array_data.child_count); + for (idx_t i = 0; i < array_data.child_count; i++) { + auto values_index = variant.GetValuesIndex(row, array_data.children_idx + i); + array_items.emplace_back(Visit(variant, row, values_index, std::forward(args)...)); + } + return array_items; + } + + // Void version + template + static typename std::enable_if::value, void>::type + VisitArrayItems(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &array_data, + Args &&...args) { + for (idx_t i = 0; i < array_data.child_count; i++) { + auto values_index = variant.GetValuesIndex(row, array_data.children_idx + i); + Visit(variant, row, values_index, std::forward(args)...); + } + } + + template + static child_list_t VisitObjectItems(const UnifiedVariantVectorData &variant, idx_t row, + const VariantNestedData &object_data, Args &&...args) { + child_list_t object_items; + for (idx_t i = 0; i < object_data.child_count; i++) { + auto values_index = variant.GetValuesIndex(row, object_data.children_idx + i); + auto val = Visit(variant, row, values_index, std::forward(args)...); + + auto keys_index = variant.GetKeysIndex(row, object_data.children_idx + i); + auto &key = variant.GetKey(row, keys_index); + + object_items.emplace_back(key.GetString(), std::move(val)); + } + return object_items; + } + +private: + template + static typename std::enable_if::value, void>::type + VisitMetadata(VariantLogicalType type_id, Args &&...args) { + Visitor::VisitMetadata(type_id, std::forward(args)...); + } + + // Fallback if the method does not exist + template + static typename std::enable_if::value, void>::type VisitMetadata(VariantLogicalType, + Args &&...) { + // do nothing + } + + template + static ReturnType VisitArray(const UnifiedVariantVectorData &variant, idx_t row, uint32_t values_idx, + Args &&...args) { + auto decoded_nested_data = VariantUtils::DecodeNestedData(variant, row, values_idx); + return Visitor::VisitArray(variant, row, decoded_nested_data, std::forward(args)...); + } + + template + static ReturnType VisitObject(const UnifiedVariantVectorData &variant, idx_t row, uint32_t values_idx, + Args &&...args) { + auto decoded_nested_data = VariantUtils::DecodeNestedData(variant, row, values_idx); + return Visitor::VisitObject(variant, row, decoded_nested_data, std::forward(args)...); + } + + template + static ReturnType VisitString(VariantLogicalType type_id, const UnifiedVariantVectorData &variant, idx_t row, + uint32_t values_idx, Args &&...args) { + auto decoded_string = VariantUtils::DecodeStringData(variant, row, values_idx); + if (type_id == VariantLogicalType::VARCHAR) { + return Visitor::VisitString(decoded_string, std::forward(args)...); + } + if (type_id == VariantLogicalType::BLOB) { + return Visitor::VisitBlob(decoded_string, std::forward(args)...); + } + if (type_id == VariantLogicalType::BIGNUM) { + return Visitor::VisitBignum(decoded_string, std::forward(args)...); + } + if (type_id == VariantLogicalType::GEOMETRY) { + return Visitor::VisitGeometry(decoded_string, std::forward(args)...); + } + if (type_id == VariantLogicalType::BITSTRING) { + return Visitor::VisitBitstring(decoded_string, std::forward(args)...); + } + throw InternalException("String-backed variant type (%s) not handled", EnumUtil::ToString(type_id)); + } + + template + static ReturnType VisitDecimal(const UnifiedVariantVectorData &variant, idx_t row, uint32_t values_idx, + Args &&...args) { + auto decoded_decimal = VariantUtils::DecodeDecimalData(variant, row, values_idx); + auto &width = decoded_decimal.width; + auto &scale = decoded_decimal.scale; + auto &ptr = decoded_decimal.value_ptr; + if (width > DecimalWidth::max) { + throw InternalException("Can't handle decimal of width: %d", width); + } else if (width > DecimalWidth::max) { + return Visitor::template VisitDecimal(Load(ptr), width, scale, + std::forward(args)...); + } else if (width > DecimalWidth::max) { + return Visitor::template VisitDecimal(Load(ptr), width, scale, + std::forward(args)...); + } else if (width > DecimalWidth::max) { + return Visitor::template VisitDecimal(Load(ptr), width, scale, + std::forward(args)...); + } else { + return Visitor::template VisitDecimal(Load(ptr), width, scale, + std::forward(args)...); + } + } +}; + +} // namespace duckdb diff --git a/src/include/duckdb/common/types/vector.hpp b/src/include/duckdb/common/types/vector.hpp index 1ab48c05601c..9f359a488868 100644 --- a/src/include/duckdb/common/types/vector.hpp +++ b/src/include/duckdb/common/types/vector.hpp @@ -11,6 +11,7 @@ #include "duckdb/common/bitset.hpp" #include "duckdb/common/common.hpp" #include "duckdb/common/enums/vector_type.hpp" +#include "duckdb/common/mutex.hpp" #include "duckdb/common/types/selection_vector.hpp" #include "duckdb/common/types/validity_mask.hpp" #include "duckdb/common/types/value.hpp" @@ -21,6 +22,7 @@ namespace duckdb { class VectorCache; +class VectorChildBuffer; class VectorStringBuffer; class VectorStructBuffer; class VectorListBuffer; @@ -195,6 +197,8 @@ class Vector { DUCKDB_API void Dictionary(idx_t dictionary_size, const SelectionVector &sel, idx_t count); //! Creates a reference to a dictionary of the other vector DUCKDB_API void Dictionary(Vector &dict, idx_t dictionary_size, const SelectionVector &sel, idx_t count); + //! Creates a dictionary on the reusable dict + DUCKDB_API void Dictionary(buffer_ptr reusable_dict, const SelectionVector &sel); //! Creates the data of this vector with the specified type. Any data that //! is currently in the vector is destroyed. @@ -306,20 +310,24 @@ class Vector { //! The buffer holding auxiliary data of the vector //! e.g. a string vector uses this to store strings buffer_ptr auxiliary; - //! The buffer holding precomputed hashes of the data in the vector - //! used for caching hashes of string dictionaries - buffer_ptr cached_hashes; }; -//! The DictionaryBuffer holds a selection vector +//! The VectorChildBuffer holds a child Vector class VectorChildBuffer : public VectorBuffer { public: explicit VectorChildBuffer(Vector vector) - : VectorBuffer(VectorBufferType::VECTOR_CHILD_BUFFER), data(std::move(vector)) { + : VectorBuffer(VectorBufferType::VECTOR_CHILD_BUFFER), data(std::move(vector)), + cached_hashes(LogicalType::HASH, nullptr) { } public: Vector data; + //! Optional size/id to uniquely identify re-occurring dictionaries + optional_idx size; + string id; + //! For caching the hashes of a child buffer + mutex cached_hashes_lock; + Vector cached_hashes; }; struct ConstantVector { @@ -409,22 +417,27 @@ struct DictionaryVector { } static inline optional_idx DictionarySize(const Vector &vector) { VerifyDictionary(vector); + const auto &child_buffer = vector.auxiliary->Cast(); + if (child_buffer.size.IsValid()) { + return child_buffer.size; + } return vector.buffer->Cast().GetDictionarySize(); } static inline const string &DictionaryId(const Vector &vector) { VerifyDictionary(vector); + const auto &child_buffer = vector.auxiliary->Cast(); + if (!child_buffer.id.empty()) { + return child_buffer.id; + } return vector.buffer->Cast().GetDictionaryId(); } - static inline void SetDictionaryId(Vector &vector, string new_id) { - VerifyDictionary(vector); - vector.buffer->Cast().SetDictionaryId(std::move(new_id)); - } static inline bool CanCacheHashes(const LogicalType &type) { return type.InternalType() == PhysicalType::VARCHAR; } static inline bool CanCacheHashes(const Vector &vector) { return DictionarySize(vector).IsValid() && CanCacheHashes(vector.GetType()); } + static buffer_ptr CreateReusableDictionary(const LogicalType &type, const idx_t &size); static const Vector &GetCachedHashes(Vector &input); }; diff --git a/src/include/duckdb/common/vector.hpp b/src/include/duckdb/common/vector.hpp index 676adac20788..55236010d9ba 100644 --- a/src/include/duckdb/common/vector.hpp +++ b/src/include/duckdb/common/vector.hpp @@ -17,10 +17,10 @@ namespace duckdb { -template -class vector : public std::vector> { // NOLINT: matching name of std +template > +class vector : public std::vector { // NOLINT: matching name of std public: - using original = std::vector>; + using original = std::vector; using original::original; using size_type = typename original::size_type; using const_reference = typename original::const_reference; diff --git a/src/include/duckdb/common/virtual_file_system.hpp b/src/include/duckdb/common/virtual_file_system.hpp index 6a0f0346a75a..07c49b541ba5 100644 --- a/src/include/duckdb/common/virtual_file_system.hpp +++ b/src/include/duckdb/common/virtual_file_system.hpp @@ -11,6 +11,7 @@ #include "duckdb/common/file_system.hpp" #include "duckdb/common/map.hpp" #include "duckdb/common/unordered_set.hpp" +#include "duckdb/main/extension_helper.hpp" namespace duckdb { @@ -82,8 +83,10 @@ class VirtualFileSystem : public FileSystem { } private: + FileSystem &FindFileSystem(const string &path, optional_ptr file_opener); + FileSystem &FindFileSystem(const string &path, optional_ptr database_instance); FileSystem &FindFileSystem(const string &path); - FileSystem &FindFileSystemInternal(const string &path); + optional_ptr FindFileSystemInternal(const string &path); private: vector> sub_systems; diff --git a/src/include/duckdb/common/winapi.hpp b/src/include/duckdb/common/winapi.hpp index 625b60b89f13..67287565fd9a 100644 --- a/src/include/duckdb/common/winapi.hpp +++ b/src/include/duckdb/common/winapi.hpp @@ -29,11 +29,7 @@ #ifdef DUCKDB_STATIC_BUILD #define DUCKDB_EXTENSION_API #else -#ifdef DUCKDB_BUILD_LOADABLE_EXTENSION #define DUCKDB_EXTENSION_API __declspec(dllexport) -#else -#define DUCKDB_EXTENSION_API -#endif #endif #else #define DUCKDB_EXTENSION_API __attribute__((visibility("default"))) diff --git a/src/include/duckdb/execution/expression_executor_state.hpp b/src/include/duckdb/execution/expression_executor_state.hpp index 8f0b77ccf2f5..112e76109f36 100644 --- a/src/include/duckdb/execution/expression_executor_state.hpp +++ b/src/include/duckdb/execution/expression_executor_state.hpp @@ -75,12 +75,10 @@ struct ExecuteFunctionState : public ExpressionState { //! Only valid when the expression is eligible for the dictionary expression optimization //! This is the case when the input is "practically unary", i.e., only one non-const input column optional_idx input_col_idx; - //! Storage ID of the input dictionary vector - string current_input_dictionary_id; //! Vector holding the expression executed on the entire dictionary - unique_ptr output_dictionary; - //! ID of the output dictionary_vector - string output_dictionary_id; + buffer_ptr output_dictionary; + //! ID of the input dictionary Vector + string current_input_dictionary_id; }; struct ExpressionExecutorState { diff --git a/src/include/duckdb/execution/index/art/base_leaf.hpp b/src/include/duckdb/execution/index/art/base_leaf.hpp index 209d022dc034..797c18469d81 100644 --- a/src/include/duckdb/execution/index/art/base_leaf.hpp +++ b/src/include/duckdb/execution/index/art/base_leaf.hpp @@ -31,13 +31,15 @@ class BaseLeaf { public: //! Get a new BaseLeaf and initialize it. - static BaseLeaf &New(ART &art, Node &node) { + static NodeHandle New(ART &art, Node &node) { node = Node::GetAllocator(art, TYPE).New(); node.SetMetadata(static_cast(TYPE)); - auto &n = Node::Ref(art, node, TYPE); + NodeHandle handle(art, node); + auto &n = handle.Get(); + n.count = 0; - return n; + return handle; } //! Returns true, if the byte exists, else false. @@ -70,7 +72,7 @@ class BaseLeaf { private: static void InsertByteInternal(BaseLeaf &n, const uint8_t byte); - static BaseLeaf &DeleteByteInternal(ART &art, Node &node, const uint8_t byte); + static NodeHandle DeleteByteInternal(ART &art, Node &node, const uint8_t byte); }; //! Node7Leaf holds up to seven sorted bytes. diff --git a/src/include/duckdb/execution/index/fixed_size_allocator.hpp b/src/include/duckdb/execution/index/fixed_size_allocator.hpp index 691a4aac65f2..65ffd167f32d 100644 --- a/src/include/duckdb/execution/index/fixed_size_allocator.hpp +++ b/src/include/duckdb/execution/index/fixed_size_allocator.hpp @@ -30,7 +30,8 @@ class FixedSizeAllocator { public: //! Construct a new fixed-size allocator - FixedSizeAllocator(const idx_t segment_size, BlockManager &block_manager); + FixedSizeAllocator(const idx_t segment_size, BlockManager &block_manager, + MemoryTag memory_tag = MemoryTag::ART_INDEX); //! Block manager of the database instance BlockManager &block_manager; @@ -152,6 +153,8 @@ class FixedSizeAllocator { void VerifyBuffers(); private: + //! Memory tag of memory that is allocated through the allocator + MemoryTag memory_tag; //! Allocation size of one segment in a buffer //! We only need this value to calculate bitmask_count, bitmask_offset, and //! available_segments_per_buffer diff --git a/src/include/duckdb/execution/index/fixed_size_buffer.hpp b/src/include/duckdb/execution/index/fixed_size_buffer.hpp index e7c5b6aa9cd3..6ca7dc1aa35e 100644 --- a/src/include/duckdb/execution/index/fixed_size_buffer.hpp +++ b/src/include/duckdb/execution/index/fixed_size_buffer.hpp @@ -43,7 +43,7 @@ class FixedSizeBuffer { public: //! Constructor for a new in-memory buffer - explicit FixedSizeBuffer(BlockManager &block_manager); + explicit FixedSizeBuffer(BlockManager &block_manager, MemoryTag memory_tag); //! Constructor for deserializing buffer metadata from disk FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const idx_t allocation_size, const BlockPointer &block_pointer); diff --git a/src/include/duckdb/execution/join_hashtable.hpp b/src/include/duckdb/execution/join_hashtable.hpp index d2a55529a949..4d0e6ae47dc6 100644 --- a/src/include/duckdb/execution/join_hashtable.hpp +++ b/src/include/duckdb/execution/join_hashtable.hpp @@ -277,6 +277,8 @@ class JoinHashTable { uint64_t bitmask = DConstants::INVALID_INDEX; //! Whether or not we error on multiple rows found per match in a SINGLE join bool single_join_error_on_multiple_rows = true; + //! Whether or not to perform deduplication based on join_keys when building ht + bool insert_duplicate_keys = true; struct { mutex mj_lock; diff --git a/src/include/duckdb/execution/operator/aggregate/physical_window.hpp b/src/include/duckdb/execution/operator/aggregate/physical_window.hpp index f749a27ccd1a..8629c70704f8 100644 --- a/src/include/duckdb/execution/operator/aggregate/physical_window.hpp +++ b/src/include/duckdb/execution/operator/aggregate/physical_window.hpp @@ -59,6 +59,8 @@ class PhysicalWindow : public PhysicalOperator { SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override; SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, OperatorSinkFinalizeInput &input) const override; + ProgressData GetSinkProgress(ClientContext &context, GlobalSinkState &gstate, + const ProgressData source_progress) const override; unique_ptr GetLocalSinkState(ExecutionContext &context) const override; unique_ptr GetGlobalSinkState(ClientContext &context) const override; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 30ba0abc5fbc..aad90df94958 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -52,21 +52,21 @@ class CSVError { CSVError() {}; CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info, idx_t row_byte_position, optional_idx byte_position, const CSVReaderOptions &reader_options, - const string &fixes, const string ¤t_path); + const string &fixes, const String ¤t_path); CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info); //! Produces error messages for column name -> type mismatch. static CSVError ColumnTypesError(case_insensitive_map_t sql_types_per_column, const vector &names); //! Produces error messages for casting errors static CSVError CastError(const CSVReaderOptions &options, const string &column_name, string &cast_error, idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position, - optional_idx byte_position, LogicalTypeId type, const string ¤t_path); + optional_idx byte_position, LogicalTypeId type, const String ¤t_path); //! Produces error for when the line size exceeds the maximum line size option static CSVError LineSizeError(const CSVReaderOptions &options, LinesPerBoundary error_info, string &csv_row, - idx_t byte_position, const string ¤t_path); + idx_t byte_position, const String ¤t_path); //! Produces error for when the state machine reaches an invalid state static CSVError InvalidState(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, optional_idx byte_position, - const string ¤t_path); + const String ¤t_path); //! Produces an error message for a dialect sniffing error. static CSVError SniffingError(const CSVReaderOptions &options, const string &search_space, idx_t max_columns_found, SetColumns &set_columns, bool type_detection); @@ -76,17 +76,17 @@ class CSVError { //! Produces error messages for unterminated quoted values static CSVError UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, - optional_idx byte_position, const string ¤t_path); + optional_idx byte_position, const String ¤t_path); //! Produces error messages for null_padding option is set, and we have quoted new values in parallel static CSVError NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info, - const string ¤t_path); + const String ¤t_path); //! Produces error for incorrect (e.g., smaller and lower than the predefined) number of columns in a CSV Line static CSVError IncorrectColumnAmountError(const CSVReaderOptions &state_machine, idx_t actual_columns, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, - optional_idx byte_position, const string ¤t_path); + optional_idx byte_position, const String ¤t_path); static CSVError InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, optional_idx byte_position, - const string ¤t_path); + const String ¤t_path); idx_t GetBoundaryIndex() const { return error_info.boundary_idx; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index b3d4f9dd5cd3..744710bef2f0 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -196,7 +196,7 @@ struct CSVReaderOptions { //! Verify options are not conflicting void Verify(MultiFileOptions &file_options); - string ToString(const string ¤t_file_path) const; + string ToString(const String ¤t_file_path) const; //! If the type for column with idx i was manually set bool WasTypeManuallySet(idx_t i) const; diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index bacabfc4fca3..331f9d6691a0 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -41,13 +41,15 @@ class FullLinePosition { return {}; } string result; - if (end.buffer_idx == begin.buffer_idx) { - if (buffer_handles.find(end.buffer_idx) == buffer_handles.end()) { + if (end.buffer_idx == begin.buffer_idx || begin.buffer_pos == begin.buffer_size) { + idx_t buffer_idx = end.buffer_idx; + if (buffer_handles.find(buffer_idx) == buffer_handles.end()) { return {}; } - auto buffer = buffer_handles[begin.buffer_idx]->Ptr(); - first_char_nl = buffer[begin.buffer_pos] == '\n' || buffer[begin.buffer_pos] == '\r'; - for (idx_t i = begin.buffer_pos + first_char_nl; i < end.buffer_pos; i++) { + idx_t start_pos = begin.buffer_pos == begin.buffer_size ? 0 : begin.buffer_pos; + auto buffer = buffer_handles[buffer_idx]->Ptr(); + first_char_nl = buffer[start_pos] == '\n' || buffer[start_pos] == '\r'; + for (idx_t i = start_pos + first_char_nl; i < end.buffer_pos; i++) { result += buffer[i]; } } else { @@ -55,6 +57,9 @@ class FullLinePosition { buffer_handles.find(end.buffer_idx) == buffer_handles.end()) { return {}; } + if (begin.buffer_pos >= begin.buffer_size) { + throw InternalException("CSV reader: buffer pos out of range for buffer"); + } auto first_buffer = buffer_handles[begin.buffer_idx]->Ptr(); auto first_buffer_size = buffer_handles[begin.buffer_idx]->actual_size; auto second_buffer = buffer_handles[end.buffer_idx]->Ptr(); @@ -248,7 +253,7 @@ class StringValueResult : public ScannerResult { //! We store borked rows so we can generate multiple errors during flushing unordered_set borked_rows; - const string path; + String path; //! Variable used when trying to figure out where a new segment starts, we must always start from a Valid //! (i.e., non-comment) line. diff --git a/src/include/duckdb/execution/operator/helper/physical_prepare.hpp b/src/include/duckdb/execution/operator/helper/physical_prepare.hpp index 373b82e5d6de..e8d17dc3719f 100644 --- a/src/include/duckdb/execution/operator/helper/physical_prepare.hpp +++ b/src/include/duckdb/execution/operator/helper/physical_prepare.hpp @@ -11,6 +11,7 @@ #include "duckdb/execution/physical_operator.hpp" #include "duckdb/common/enums/physical_operator_type.hpp" #include "duckdb/main/prepared_statement_data.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" namespace duckdb { @@ -19,13 +20,13 @@ class PhysicalPrepare : public PhysicalOperator { static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::PREPARE; public: - PhysicalPrepare(PhysicalPlan &physical_plan, string name_p, shared_ptr prepared, + PhysicalPrepare(PhysicalPlan &physical_plan, const std::string &name_p, shared_ptr prepared, idx_t estimated_cardinality) : PhysicalOperator(physical_plan, PhysicalOperatorType::PREPARE, {LogicalType::BOOLEAN}, estimated_cardinality), - name(std::move(name_p)), prepared(std::move(prepared)) { + name(physical_plan.ArenaRef().MakeString(name_p)), prepared(std::move(prepared)) { } - string name; + String name; shared_ptr prepared; public: diff --git a/src/include/duckdb/execution/operator/helper/physical_set.hpp b/src/include/duckdb/execution/operator/helper/physical_set.hpp index c7f2fb038f7f..0f4cc2a807b4 100644 --- a/src/include/duckdb/execution/operator/helper/physical_set.hpp +++ b/src/include/duckdb/execution/operator/helper/physical_set.hpp @@ -11,6 +11,7 @@ #include "duckdb/common/enums/set_scope.hpp" #include "duckdb/execution/physical_operator.hpp" #include "duckdb/parser/parsed_data/vacuum_info.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" namespace duckdb { @@ -26,7 +27,7 @@ class PhysicalSet : public PhysicalOperator { PhysicalSet(PhysicalPlan &physical_plan, const string &name_p, Value value_p, SetScope scope_p, idx_t estimated_cardinality) : PhysicalOperator(physical_plan, PhysicalOperatorType::SET, {LogicalType::BOOLEAN}, estimated_cardinality), - name(name_p), value(std::move(value_p)), scope(scope_p) { + name(physical_plan.ArenaRef().MakeString(name_p)), value(std::move(value_p)), scope(scope_p) { } public: @@ -37,13 +38,13 @@ class PhysicalSet : public PhysicalOperator { return true; } - static void SetExtensionVariable(ClientContext &context, ExtensionOption &extension_option, const string &name, + static void SetExtensionVariable(ClientContext &context, ExtensionOption &extension_option, const String &name, SetScope scope, const Value &value); - static void SetGenericVariable(ClientContext &context, const string &name, SetScope scope, Value target_value); + static void SetGenericVariable(ClientContext &context, const String &name, SetScope scope, Value target_value); public: - const string name; + String name; const Value value; const SetScope scope; }; diff --git a/src/include/duckdb/execution/operator/helper/physical_set_variable.hpp b/src/include/duckdb/execution/operator/helper/physical_set_variable.hpp index 4574cd868ebc..7c5378855e25 100644 --- a/src/include/duckdb/execution/operator/helper/physical_set_variable.hpp +++ b/src/include/duckdb/execution/operator/helper/physical_set_variable.hpp @@ -18,7 +18,7 @@ class PhysicalSetVariable : public PhysicalOperator { static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::SET_VARIABLE; public: - PhysicalSetVariable(PhysicalPlan &physical_plan, string name, idx_t estimated_cardinality); + PhysicalSetVariable(PhysicalPlan &physical_plan, const string &name_p, idx_t estimated_cardinality); public: // Source interface @@ -37,7 +37,7 @@ class PhysicalSetVariable : public PhysicalOperator { } public: - const string name; + String name; }; } // namespace duckdb diff --git a/src/include/duckdb/execution/operator/join/perfect_hash_join_executor.hpp b/src/include/duckdb/execution/operator/join/perfect_hash_join_executor.hpp index 0affaf4cc217..9e5134481525 100644 --- a/src/include/duckdb/execution/operator/join/perfect_hash_join_executor.hpp +++ b/src/include/duckdb/execution/operator/join/perfect_hash_join_executor.hpp @@ -29,7 +29,7 @@ struct PerfectHashJoinStats { //! PhysicalHashJoin represents a hash loop join between two tables class PerfectHashJoinExecutor { - using PerfectHashTable = vector; + using PerfectHashTable = vector>; public: PerfectHashJoinExecutor(const PhysicalHashJoin &join, JoinHashTable &ht); @@ -64,7 +64,7 @@ class PerfectHashJoinExecutor { //! Build statistics PerfectHashJoinStats perfect_join_statistics; //! Stores the occurrences of each value in the build side - unsafe_unique_array bitmap_build_idx; + ValidityMask bitmap_build_idx; //! Stores the number of unique keys in the build side idx_t unique_keys = 0; }; diff --git a/src/include/duckdb/execution/operator/join/physical_asof_join.hpp b/src/include/duckdb/execution/operator/join/physical_asof_join.hpp index 6089a728b7c2..24382e8d9025 100644 --- a/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +++ b/src/include/duckdb/execution/operator/join/physical_asof_join.hpp @@ -37,18 +37,6 @@ class PhysicalAsOfJoin : public PhysicalComparisonJoin { // Projection mappings vector right_projection_map; - // Predicate (join conditions that don't reference both sides) - unique_ptr predicate; - -public: - // Operator Interface - unique_ptr GetGlobalOperatorState(ClientContext &context) const override; - unique_ptr GetOperatorState(ExecutionContext &context) const override; - - bool ParallelOperator() const override { - return true; - } - protected: // CachingOperator Interface OperatorResultType ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk, @@ -83,6 +71,9 @@ class PhysicalAsOfJoin : public PhysicalComparisonJoin { bool ParallelSink() const override { return true; } + +public: + void BuildPipelines(Pipeline ¤t, MetaPipeline &meta_pipeline) override; }; } // namespace duckdb diff --git a/src/include/duckdb/execution/operator/join/physical_iejoin.hpp b/src/include/duckdb/execution/operator/join/physical_iejoin.hpp index b57fe772d3a0..a93109a0ee3a 100644 --- a/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +++ b/src/include/duckdb/execution/operator/join/physical_iejoin.hpp @@ -70,10 +70,6 @@ class PhysicalIEJoin : public PhysicalRangeJoin { public: void BuildPipelines(Pipeline ¤t, MetaPipeline &meta_pipeline) override; - -private: - // resolve joins that can potentially output N*M elements (INNER, LEFT, FULL) - void ResolveComplexJoin(ExecutionContext &context, DataChunk &result, LocalSourceState &state) const; }; } // namespace duckdb diff --git a/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp b/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp index 25ed9ed0623e..2cdff137405c 100644 --- a/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp +++ b/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp @@ -18,13 +18,16 @@ class PhysicalNestedLoopJoin : public PhysicalComparisonJoin { static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::NESTED_LOOP_JOIN; public: - PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalOperator &op, PhysicalOperator &left, + PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalComparisonJoin &op, PhysicalOperator &left, PhysicalOperator &right, vector cond, JoinType join_type, idx_t estimated_cardinality, unique_ptr pushdown_info); - PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalOperator &op, PhysicalOperator &left, + PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalComparisonJoin &op, PhysicalOperator &left, PhysicalOperator &right, vector cond, JoinType join_type, idx_t estimated_cardinality); + // Predicate (join conditions that don't reference both sides) + unique_ptr predicate; + public: // Operator Interface unique_ptr GetOperatorState(ExecutionContext &context) const override; diff --git a/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp b/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp index 4da01aff3fe4..12d974dddbe8 100644 --- a/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +++ b/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp @@ -46,6 +46,8 @@ class PhysicalPiecewiseMergeJoin : public PhysicalRangeJoin { public: // Source interface unique_ptr GetGlobalSourceState(ClientContext &context) const override; + unique_ptr GetLocalSourceState(ExecutionContext &context, + GlobalSourceState &gstate) const override; SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override; bool IsSource() const override { diff --git a/src/include/duckdb/execution/operator/join/physical_range_join.hpp b/src/include/duckdb/execution/operator/join/physical_range_join.hpp index 4ee6ef557ee8..1edb36ed4add 100644 --- a/src/include/duckdb/execution/operator/join/physical_range_join.hpp +++ b/src/include/duckdb/execution/operator/join/physical_range_join.hpp @@ -1,43 +1,42 @@ //===----------------------------------------------------------------------===// // DuckDB // -// duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +// duckdb/execution/operator/join/physical_range_join.hpp // // //===----------------------------------------------------------------------===// #pragma once +#include "duckdb/common/types/row/block_iterator.hpp" #include "duckdb/execution/operator/join/physical_comparison_join.hpp" -#include "duckdb/planner/bound_result_modifier.hpp" -#include "duckdb/common/sort/sort.hpp" +#include "duckdb/common/sorting/sort.hpp" +#include "duckdb/common/sorting/sorted_run.hpp" namespace duckdb { -struct GlobalSortState; - //! PhysicalRangeJoin represents one or more inequality range join predicates between //! two tables class PhysicalRangeJoin : public PhysicalComparisonJoin { public: + class GlobalSortedTable; + class LocalSortedTable { public: - LocalSortedTable(ClientContext &context, const PhysicalRangeJoin &op, const idx_t child); + LocalSortedTable(ExecutionContext &context, GlobalSortedTable &global_table, const idx_t child); - void Sink(DataChunk &input, GlobalSortState &global_sort_state); + void Sink(ExecutionContext &context, DataChunk &input); - inline void Sort(GlobalSortState &global_sort_state) { - local_sort_state.Sort(global_sort_state, true); - } - - //! The hosting operator - const PhysicalRangeJoin &op; + //! The global table we are connected to + GlobalSortedTable &global_table; //! The local sort state - LocalSortState local_sort_state; + unique_ptr local_sink; //! Local copy of the sorting expression executor ExpressionExecutor executor; //! Holds a vector of incoming sorting columns DataChunk keys; + //! The sort data + DataChunk sort_chunk; //! The number of NULL values idx_t has_null; //! The total number of rows @@ -50,45 +49,89 @@ class PhysicalRangeJoin : public PhysicalComparisonJoin { class GlobalSortedTable { public: - GlobalSortedTable(ClientContext &context, const vector &orders, RowLayout &payload_layout, - const PhysicalOperator &op); + GlobalSortedTable(ClientContext &client, const vector &orders, + const vector &payload_layout, const PhysicalRangeJoin &op); inline idx_t Count() const { return count; } inline idx_t BlockCount() const { - if (global_sort_state.sorted_blocks.empty()) { - return 0; - } - D_ASSERT(global_sort_state.sorted_blocks.size() == 1); - return global_sort_state.sorted_blocks[0]->radix_sorting_data.size(); + return sorted->key_data->ChunkCount(); + } + + inline idx_t BlockStart(idx_t i) const { + return MinValue(i * STANDARD_VECTOR_SIZE, count); + } + + inline idx_t BlockEnd(idx_t i) const { + return BlockStart(i + 1) - 1; } inline idx_t BlockSize(idx_t i) const { - return global_sort_state.sorted_blocks[0]->radix_sorting_data[i]->count; + return i < BlockCount() ? MinValue(STANDARD_VECTOR_SIZE, count - BlockStart(i)) : 0; + } + + inline SortKeyType GetSortKeyType() const { + return sorted->key_data->GetLayout().GetSortKeyType(); } - void Combine(LocalSortedTable <able); void IntializeMatches(); + + //! Combine local states + void Combine(ExecutionContext &context, LocalSortedTable <able); + //! Prepare for sorting. + void Finalize(ClientContext &client, InterruptState &interrupt); + //! Schedules the materialisation process. + void Materialize(Pipeline &pipeline, Event &event); + //! Single-threaded materialisation. + void Materialize(ExecutionContext &context, InterruptState &interrupt); + //! Materialize an empty sorted run. + void MaterializeEmpty(ClientContext &client); + //! Print the table to the console void Print(); - //! Starts the sorting process. - void Finalize(Pipeline &pipeline, Event &event); - //! Schedules tasks to merge sort the current child's data during a Finalize phase - void ScheduleMergeTasks(Pipeline &pipeline, Event &event); + //! Create an iteration state + unique_ptr CreateIteratorState() { + auto state = make_uniq(*sorted->key_data, sorted->payload_data.get()); + + // Unless we do this, we will only get values from the first chunk + Repin(*state); + + return state; + } + //! Reset the pins for an iterator so we release memory in a timely manner + static void Repin(ExternalBlockIteratorState &iter) { + iter.SetKeepPinned(true); + iter.SetPinPayload(true); + } + //! Create an iteration state + unique_ptr CreateScanState(ClientContext &client) { + return make_uniq(client, *sort); + } + //! Initialize a payload scanning state + void InitializePayloadState(TupleDataChunkState &state) { + sorted->payload_data->InitializeChunkState(state); + } //! The hosting operator - const PhysicalOperator &op; - GlobalSortState global_sort_state; + const PhysicalRangeJoin &op; + //! The sort description + unique_ptr sort; + //! The shared sort state + unique_ptr global_sink; //! Whether or not the RHS has NULL values atomic has_null; //! The total number of rows in the RHS atomic count; + //! The number of materialisation tasks completed in parallel + atomic tasks_completed; + //! The shared materialisation state + unique_ptr global_source; + //! The materialized data + unique_ptr sorted; //! A bool indicating for each tuple in the RHS if they found a match (only used in FULL OUTER JOIN) unsafe_unique_array found_match; - //! Memory usage per thread - idx_t memory_per_thread; }; public: @@ -106,10 +149,9 @@ class PhysicalRangeJoin : public PhysicalComparisonJoin { public: // Gather the result values and slice the payload columns to those values. - // Returns a buffer handle to the pinned heap block (if any) - static BufferHandle SliceSortedPayload(DataChunk &payload, GlobalSortState &state, const idx_t block_idx, - const SelectionVector &result, const idx_t result_count, - const idx_t left_cols = 0); + static void SliceSortedPayload(DataChunk &chunk, GlobalSortedTable &table, ExternalBlockIteratorState &state, + TupleDataChunkState &chunk_state, const idx_t chunk_idx, SelectionVector &result, + const idx_t result_count, SortedRunScanState &scan_state); // Apply a tail condition to the current selection static idx_t SelectJoinTail(const ExpressionType &condition, Vector &left, Vector &right, const SelectionVector *sel, idx_t count, SelectionVector *true_sel); diff --git a/src/include/duckdb/execution/operator/set/physical_union.hpp b/src/include/duckdb/execution/operator/set/physical_union.hpp index 36bc3d58cf9c..8873858bdec0 100644 --- a/src/include/duckdb/execution/operator/set/physical_union.hpp +++ b/src/include/duckdb/execution/operator/set/physical_union.hpp @@ -17,8 +17,9 @@ class PhysicalUnion : public PhysicalOperator { static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::UNION; public: - PhysicalUnion(PhysicalPlan &physical_plan, vector types, PhysicalOperator &top, - PhysicalOperator &bottom, idx_t estimated_cardinality, bool allow_out_of_order); + PhysicalUnion(PhysicalPlan &physical_plan, vector types, + const ArenaLinkedList> &children_p, idx_t estimated_cardinality, + bool allow_out_of_order); bool allow_out_of_order; diff --git a/src/include/duckdb/function/aggregate/minmax_n_helpers.hpp b/src/include/duckdb/function/aggregate/minmax_n_helpers.hpp index a26772819c43..666369cc794f 100644 --- a/src/include/duckdb/function/aggregate/minmax_n_helpers.hpp +++ b/src/include/duckdb/function/aggregate/minmax_n_helpers.hpp @@ -242,6 +242,30 @@ class BinaryAggregateHeap { idx_t size; }; +enum class ArgMinMaxNullHandling { IGNORE_ANY_NULL, HANDLE_ARG_NULL, HANDLE_ANY_NULL }; + +struct ArgMinMaxFunctionData : FunctionData { + explicit ArgMinMaxFunctionData(ArgMinMaxNullHandling null_handling_p = ArgMinMaxNullHandling::IGNORE_ANY_NULL, + bool nulls_last_p = true) + : null_handling(null_handling_p), nulls_last(nulls_last_p) { + } + + unique_ptr Copy() const override { + auto copy = make_uniq(); + copy->null_handling = null_handling; + copy->nulls_last = nulls_last; + return std::move(copy); + } + + bool Equals(const FunctionData &other_p) const override { + auto &other = other_p.Cast(); + return other.null_handling == null_handling && other.nulls_last == nulls_last; + } + + ArgMinMaxNullHandling null_handling; + bool nulls_last; +}; + //------------------------------------------------------------------------------ // Specializations for fixed size types, strings, and anything else (using sortkey) //------------------------------------------------------------------------------ @@ -254,7 +278,7 @@ struct MinMaxFixedValue { return UnifiedVectorFormat::GetData(format)[idx]; } - static void Assign(Vector &vector, const idx_t idx, const TYPE &value) { + static void Assign(Vector &vector, const idx_t idx, const TYPE &value, const bool nulls_last) { FlatVector::GetData(vector)[idx] = value; } @@ -263,7 +287,8 @@ struct MinMaxFixedValue { return false; } - static void PrepareData(Vector &input, const idx_t count, EXTRA_STATE &, UnifiedVectorFormat &format) { + static void PrepareData(Vector &input, const idx_t count, EXTRA_STATE &, UnifiedVectorFormat &format, + const bool nulls_last) { input.ToUnifiedFormat(count, format); } }; @@ -276,7 +301,7 @@ struct MinMaxStringValue { return UnifiedVectorFormat::GetData(format)[idx]; } - static void Assign(Vector &vector, const idx_t idx, const TYPE &value) { + static void Assign(Vector &vector, const idx_t idx, const TYPE &value, const bool nulls_last) { FlatVector::GetData(vector)[idx] = StringVector::AddStringOrBlob(vector, value); } @@ -285,7 +310,8 @@ struct MinMaxStringValue { return false; } - static void PrepareData(Vector &input, const idx_t count, EXTRA_STATE &, UnifiedVectorFormat &format) { + static void PrepareData(Vector &input, const idx_t count, EXTRA_STATE &, UnifiedVectorFormat &format, + const bool nulls_last) { input.ToUnifiedFormat(count, format); } }; @@ -299,8 +325,9 @@ struct MinMaxFallbackValue { return UnifiedVectorFormat::GetData(format)[idx]; } - static void Assign(Vector &vector, const idx_t idx, const TYPE &value) { - OrderModifiers modifiers(OrderType::ASCENDING, OrderByNullType::NULLS_LAST); + static void Assign(Vector &vector, const idx_t idx, const TYPE &value, const bool nulls_last) { + auto order_by_null_type = nulls_last ? OrderByNullType::NULLS_LAST : OrderByNullType::NULLS_FIRST; + OrderModifiers modifiers(OrderType::ASCENDING, order_by_null_type); CreateSortKeyHelpers::DecodeSortKey(value, vector, idx, modifiers); } @@ -308,14 +335,61 @@ struct MinMaxFallbackValue { return Vector(LogicalTypeId::BLOB); } - static void PrepareData(Vector &input, const idx_t count, EXTRA_STATE &extra_state, UnifiedVectorFormat &format) { - const OrderModifiers modifiers(OrderType::ASCENDING, OrderByNullType::NULLS_LAST); + static void PrepareData(Vector &input, const idx_t count, EXTRA_STATE &extra_state, UnifiedVectorFormat &format, + const bool nulls_last) { + auto order_by_null_type = nulls_last ? OrderByNullType::NULLS_LAST : OrderByNullType::NULLS_FIRST; + const OrderModifiers modifiers(OrderType::ASCENDING, order_by_null_type); CreateSortKeyHelpers::CreateSortKeyWithValidity(input, extra_state, modifiers, count); input.Flatten(count); extra_state.ToUnifiedFormat(count, format); } }; +template +struct ValueOrNull { + T value; + bool is_valid; + + bool operator==(const ValueOrNull &other) const { + return is_valid == other.is_valid && value == other.value; + } + + bool operator>(const ValueOrNull &other) const { + if (is_valid && other.is_valid) { + return value > other.value; + } + if (!is_valid && !other.is_valid) { + return false; + } + + return is_valid ^ NULLS_LAST; + } +}; + +template +struct MinMaxFixedValueOrNull { + using TYPE = ValueOrNull; + using EXTRA_STATE = bool; + + static TYPE Create(const UnifiedVectorFormat &format, const idx_t idx) { + return TYPE {UnifiedVectorFormat::GetData(format)[idx], format.validity.RowIsValid(idx)}; + } + + static void Assign(Vector &vector, const idx_t idx, const TYPE &value, const bool nulls_last) { + FlatVector::Validity(vector).Set(idx, value.is_valid); + FlatVector::GetData(vector)[idx] = value.value; + } + + static EXTRA_STATE CreateExtraState(Vector &input, idx_t count) { + return false; + } + + static void PrepareData(Vector &input, const idx_t count, EXTRA_STATE &extra_state, UnifiedVectorFormat &format, + const bool nulls_last) { + input.ToUnifiedFormat(count, format); + } +}; + //------------------------------------------------------------------------------ // MinMaxN Operation (common for both ArgMinMaxN and MinMaxN) //------------------------------------------------------------------------------ @@ -343,7 +417,11 @@ struct MinMaxNOperation { } template - static void Finalize(Vector &state_vector, AggregateInputData &, Vector &result, idx_t count, idx_t offset) { + static void Finalize(Vector &state_vector, AggregateInputData &input_data, Vector &result, idx_t count, + idx_t offset) { + // We only expect bind data from arg_max, otherwise nulls last is the default + const bool nulls_last = + input_data.bind_data ? input_data.bind_data->Cast().nulls_last : true; UnifiedVectorFormat state_format; state_vector.ToUnifiedFormat(count, state_format); @@ -387,7 +465,7 @@ struct MinMaxNOperation { auto heap = state.heap.SortAndGetHeap(); for (idx_t slot = 0; slot < state.heap.Size(); slot++) { - STATE::VAL_TYPE::Assign(child_data, current_offset++, state.heap.GetValue(heap[slot])); + STATE::VAL_TYPE::Assign(child_data, current_offset++, state.heap.GetValue(heap[slot]), nulls_last); } } diff --git a/src/include/duckdb/function/built_in_functions.hpp b/src/include/duckdb/function/built_in_functions.hpp index fb8ef3166303..766466847e45 100644 --- a/src/include/duckdb/function/built_in_functions.hpp +++ b/src/include/duckdb/function/built_in_functions.hpp @@ -53,6 +53,7 @@ class BuiltinFunctions { void RegisterTableFunctions(); void RegisterArrowFunctions(); void RegisterSnifferFunction(); + void RegisterCopyFunctions(); void RegisterExtensionOverloads(); diff --git a/src/include/duckdb/function/cast/default_casts.hpp b/src/include/duckdb/function/cast/default_casts.hpp index d87a3a976e52..107b482b7ef7 100644 --- a/src/include/duckdb/function/cast/default_casts.hpp +++ b/src/include/duckdb/function/cast/default_casts.hpp @@ -170,6 +170,7 @@ struct DefaultCasts { static BoundCastInfo UnionCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target); static BoundCastInfo VariantCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target); static BoundCastInfo UUIDCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target); + static BoundCastInfo GeoCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target); static BoundCastInfo BignumCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target); static BoundCastInfo ImplicitToUnionCast(BindCastInput &input, const LogicalType &source, const LogicalType &target); diff --git a/src/include/duckdb/function/cast/variant/json_to_variant.hpp b/src/include/duckdb/function/cast/variant/json_to_variant.hpp index a533c4faa93c..b1f10305f678 100644 --- a/src/include/duckdb/function/cast/variant/json_to_variant.hpp +++ b/src/include/duckdb/function/cast/variant/json_to_variant.hpp @@ -147,11 +147,11 @@ static bool ConvertJSONPrimitive(yyjson_val *val, ToVariantGlobalResultData &res case YYJSON_TYPE_RAW | YYJSON_SUBTYPE_NONE: { WriteVariantMetadata(result, result_index, values_offset_data, blob_offset_data[result_index], nullptr, 0, VariantLogicalType::VARCHAR); - auto str = unsafe_yyjson_get_str(val); uint32_t length = NumericCast(unsafe_yyjson_get_len(val)); - auto str_blob_data = blob_data + blob_offset_data[result_index]; auto length_varint_size = GetVarintSize(length); if (WRITE_DATA) { + auto str = unsafe_yyjson_get_str(val); + auto str_blob_data = blob_data + blob_offset_data[result_index]; VarintEncode(length, str_blob_data); memcpy(str_blob_data + length_varint_size, const_data_ptr_cast(str), length); } @@ -169,30 +169,30 @@ static bool ConvertJSONPrimitive(yyjson_val *val, ToVariantGlobalResultData &res break; } case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_UINT: { - auto value = unsafe_yyjson_get_uint(val); WriteVariantMetadata(result, result_index, values_offset_data, blob_offset_data[result_index], nullptr, 0, VariantLogicalType::UINT64); if (WRITE_DATA) { + auto value = unsafe_yyjson_get_uint(val); memcpy(blob_data + blob_offset_data[result_index], const_data_ptr_cast(&value), sizeof(uint64_t)); } blob_offset_data[result_index] += sizeof(uint64_t); break; } case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_SINT: { - auto value = unsafe_yyjson_get_sint(val); WriteVariantMetadata(result, result_index, values_offset_data, blob_offset_data[result_index], nullptr, 0, VariantLogicalType::INT64); if (WRITE_DATA) { + auto value = unsafe_yyjson_get_sint(val); memcpy(blob_data + blob_offset_data[result_index], const_data_ptr_cast(&value), sizeof(int64_t)); } blob_offset_data[result_index] += sizeof(int64_t); break; } case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL: { - auto value = unsafe_yyjson_get_real(val); WriteVariantMetadata(result, result_index, values_offset_data, blob_offset_data[result_index], nullptr, 0, VariantLogicalType::DOUBLE); if (WRITE_DATA) { + auto value = unsafe_yyjson_get_real(val); memcpy(blob_data + blob_offset_data[result_index], const_data_ptr_cast(&value), sizeof(double)); } blob_offset_data[result_index] += sizeof(double); diff --git a/src/include/duckdb/function/cast/variant/primitive_to_variant.hpp b/src/include/duckdb/function/cast/variant/primitive_to_variant.hpp index 2e7fbf68e94a..9aa105bd39e8 100644 --- a/src/include/duckdb/function/cast/variant/primitive_to_variant.hpp +++ b/src/include/duckdb/function/cast/variant/primitive_to_variant.hpp @@ -357,6 +357,9 @@ bool ConvertPrimitiveToVariant(ToVariantSourceData &source, ToVariantGlobalResul case LogicalTypeId::CHAR: return ConvertPrimitiveTemplated( source, result, count, selvec, values_index_selvec, empty_payload, is_root); + case LogicalTypeId::GEOMETRY: + return ConvertPrimitiveTemplated( + source, result, count, selvec, values_index_selvec, empty_payload, is_root); case LogicalTypeId::BLOB: return ConvertPrimitiveTemplated( source, result, count, selvec, values_index_selvec, empty_payload, is_root); diff --git a/src/include/duckdb/function/cast/variant/struct_to_variant.hpp b/src/include/duckdb/function/cast/variant/struct_to_variant.hpp index fbda413b07a4..209598a74808 100644 --- a/src/include/duckdb/function/cast/variant/struct_to_variant.hpp +++ b/src/include/duckdb/function/cast/variant/struct_to_variant.hpp @@ -31,8 +31,6 @@ bool ConvertStructToVariant(ToVariantSourceData &source, ToVariantGlobalResultDa auto result_index = selvec ? selvec->get_index(i) : i; auto &blob_offset = blob_offset_data[result_index]; - auto &children_list_entry = variant.children_data[result_index]; - auto &keys_list_entry = variant.keys_data[result_index]; if (source_validity.RowIsValid(index)) { WriteVariantMetadata(result, result_index, values_offset_data, blob_offset, values_index_selvec, @@ -55,6 +53,9 @@ bool ConvertStructToVariant(ToVariantSourceData &source, ToVariantGlobalResultDa //! children if (WRITE_DATA) { + auto &children_list_entry = variant.children_data[result_index]; + auto &keys_list_entry = variant.keys_data[result_index]; + idx_t children_index = children_list_entry.offset + children_offset_data[result_index]; idx_t keys_offset = keys_list_entry.offset + keys_offset_data[result_index]; for (idx_t child_idx = 0; child_idx < children.size(); child_idx++) { @@ -97,7 +98,7 @@ bool ConvertStructToVariant(ToVariantSourceData &source, ToVariantGlobalResultDa } } if (WRITE_DATA) { - //! Now forward the selection to point to the next index in the children.values_index + //! Now move the selection forward to write the value_id for the next struct child, for each row for (idx_t i = 0; i < sel.count; i++) { sel.children_selection[i]++; } diff --git a/src/include/duckdb/function/cast/variant/to_variant_fwd.hpp b/src/include/duckdb/function/cast/variant/to_variant_fwd.hpp index 643dc269a660..00edc64596b2 100644 --- a/src/include/duckdb/function/cast/variant/to_variant_fwd.hpp +++ b/src/include/duckdb/function/cast/variant/to_variant_fwd.hpp @@ -92,11 +92,11 @@ struct ContainerSelectionVectors { template void WriteArrayChildren(VariantVectorData &result, uint64_t children_offset, uint32_t &children_offset_data, const list_entry_t source_entry, idx_t result_index, ContainerSelectionVectors &sel) { - idx_t children_index = children_offset + children_offset_data; for (idx_t child_idx = 0; child_idx < source_entry.length; child_idx++) { //! Set up the selection vector for the child of the list vector sel.new_selection.set_index(child_idx + sel.count, result_index); if (WRITE_DATA) { + idx_t children_index = children_offset + children_offset_data; sel.children_selection.set_index(child_idx + sel.count, children_index + child_idx); result.keys_index_validity.SetInvalid(children_index + child_idx); } diff --git a/src/include/duckdb/function/cast/variant/variant_to_variant.hpp b/src/include/duckdb/function/cast/variant/variant_to_variant.hpp index 71d7677bf3e0..d97954118c3e 100644 --- a/src/include/duckdb/function/cast/variant/variant_to_variant.hpp +++ b/src/include/duckdb/function/cast/variant/variant_to_variant.hpp @@ -1,93 +1,246 @@ #pragma once #include "duckdb/function/cast/variant/to_variant_fwd.hpp" +#include "duckdb/common/types/variant_visitor.hpp" namespace duckdb { namespace variant { -static bool VariantIsTrivialPrimitive(VariantLogicalType type) { - switch (type) { - case VariantLogicalType::INT8: - case VariantLogicalType::INT16: - case VariantLogicalType::INT32: - case VariantLogicalType::INT64: - case VariantLogicalType::INT128: - case VariantLogicalType::UINT8: - case VariantLogicalType::UINT16: - case VariantLogicalType::UINT32: - case VariantLogicalType::UINT64: - case VariantLogicalType::UINT128: - case VariantLogicalType::FLOAT: - case VariantLogicalType::DOUBLE: - case VariantLogicalType::UUID: - case VariantLogicalType::DATE: - case VariantLogicalType::TIME_MICROS: - case VariantLogicalType::TIME_NANOS: - case VariantLogicalType::TIMESTAMP_SEC: - case VariantLogicalType::TIMESTAMP_MILIS: - case VariantLogicalType::TIMESTAMP_MICROS: - case VariantLogicalType::TIMESTAMP_NANOS: - case VariantLogicalType::TIME_MICROS_TZ: - case VariantLogicalType::TIMESTAMP_MICROS_TZ: - case VariantLogicalType::INTERVAL: - return true; - default: - return false; +namespace { + +struct AnalyzeState { +public: + explicit AnalyzeState(uint32_t &children_offset) : children_offset(children_offset) { } -} -static uint32_t VariantTrivialPrimitiveSize(VariantLogicalType type) { - switch (type) { - case VariantLogicalType::INT8: - return sizeof(int8_t); - case VariantLogicalType::INT16: - return sizeof(int16_t); - case VariantLogicalType::INT32: - return sizeof(int32_t); - case VariantLogicalType::INT64: - return sizeof(int64_t); - case VariantLogicalType::INT128: - return sizeof(hugeint_t); - case VariantLogicalType::UINT8: - return sizeof(uint8_t); - case VariantLogicalType::UINT16: - return sizeof(uint16_t); - case VariantLogicalType::UINT32: - return sizeof(uint32_t); - case VariantLogicalType::UINT64: - return sizeof(uint64_t); - case VariantLogicalType::UINT128: - return sizeof(uhugeint_t); - case VariantLogicalType::FLOAT: +public: + uint32_t &children_offset; +}; + +struct WriteState { +public: + WriteState(uint32_t &keys_offset, uint32_t &children_offset, uint32_t &blob_offset, data_ptr_t blob_data, + uint32_t &blob_size) + : keys_offset(keys_offset), children_offset(children_offset), blob_offset(blob_offset), blob_data(blob_data), + blob_size(blob_size) { + } + +public: + inline data_ptr_t GetDestination() { + return blob_data + blob_offset + blob_size; + } + +public: + uint32_t &keys_offset; + uint32_t &children_offset; + uint32_t &blob_offset; + data_ptr_t blob_data; + uint32_t &blob_size; +}; + +struct VariantToVariantSizeAnalyzer { + using result_type = uint32_t; + + static uint32_t VisitNull(AnalyzeState &state) { + return 0; + } + static uint32_t VisitBoolean(bool, AnalyzeState &state) { + return 0; + } + + template + static uint32_t VisitInteger(T, AnalyzeState &state) { + return sizeof(T); + } + + static uint32_t VisitFloat(float, AnalyzeState &state) { return sizeof(float); - case VariantLogicalType::DOUBLE: + } + static uint32_t VisitDouble(double, AnalyzeState &state) { return sizeof(double); - case VariantLogicalType::UUID: + } + static uint32_t VisitUUID(hugeint_t, AnalyzeState &state) { return sizeof(hugeint_t); - case VariantLogicalType::DATE: + } + static uint32_t VisitDate(date_t, AnalyzeState &state) { return sizeof(int32_t); - case VariantLogicalType::TIME_MICROS: + } + static uint32_t VisitInterval(interval_t, AnalyzeState &state) { + return sizeof(interval_t); + } + + static uint32_t VisitTime(dtime_t, AnalyzeState &state) { return sizeof(dtime_t); - case VariantLogicalType::TIME_NANOS: + } + static uint32_t VisitTimeNanos(dtime_ns_t, AnalyzeState &state) { return sizeof(dtime_ns_t); - case VariantLogicalType::TIMESTAMP_SEC: + } + static uint32_t VisitTimeTZ(dtime_tz_t, AnalyzeState &state) { + return sizeof(dtime_tz_t); + } + static uint32_t VisitTimestampSec(timestamp_sec_t, AnalyzeState &state) { return sizeof(timestamp_sec_t); - case VariantLogicalType::TIMESTAMP_MILIS: + } + static uint32_t VisitTimestampMs(timestamp_ms_t, AnalyzeState &state) { return sizeof(timestamp_ms_t); - case VariantLogicalType::TIMESTAMP_MICROS: + } + static uint32_t VisitTimestamp(timestamp_t, AnalyzeState &state) { return sizeof(timestamp_t); - case VariantLogicalType::TIMESTAMP_NANOS: + } + static uint32_t VisitTimestampNanos(timestamp_ns_t, AnalyzeState &state) { return sizeof(timestamp_ns_t); - case VariantLogicalType::TIME_MICROS_TZ: - return sizeof(dtime_tz_t); - case VariantLogicalType::TIMESTAMP_MICROS_TZ: + } + static uint32_t VisitTimestampTZ(timestamp_tz_t, AnalyzeState &state) { return sizeof(timestamp_tz_t); - case VariantLogicalType::INTERVAL: - return sizeof(interval_t); - default: - throw InternalException("VariantLogicalType '%s' is not a trivial primitive", EnumUtil::ToString(type)); } -} + + static uint32_t VisitString(const string_t &str, AnalyzeState &state) { + auto length = static_cast(str.GetSize()); + return GetVarintSize(length) + length; + } + + static uint32_t VisitBlob(const string_t &blob, AnalyzeState &state) { + return VisitString(blob, state); + } + static uint32_t VisitBignum(const string_t &bignum, AnalyzeState &state) { + return VisitString(bignum, state); + } + static uint32_t VisitGeometry(const string_t &geom, AnalyzeState &state) { + return VisitString(geom, state); + } + static uint32_t VisitBitstring(const string_t &bits, AnalyzeState &state) { + return VisitString(bits, state); + } + + template + static uint32_t VisitDecimal(T, uint32_t width, uint32_t scale, AnalyzeState &state) { + uint32_t size = GetVarintSize(width) + GetVarintSize(scale); + size += sizeof(T); + return size; + } + + static uint32_t VisitArray(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &nested_data, + AnalyzeState &state) { + uint32_t size = GetVarintSize(nested_data.child_count); + if (nested_data.child_count) { + size += GetVarintSize(nested_data.children_idx + state.children_offset); + } + return size; + } + + static uint32_t VisitObject(const UnifiedVariantVectorData &variant, idx_t row, + const VariantNestedData &nested_data, AnalyzeState &state) { + return VisitArray(variant, row, nested_data, state); + } + + static uint32_t VisitDefault(VariantLogicalType type_id, const_data_ptr_t, AnalyzeState &) { + throw InternalException("Unrecognized VariantLogicalType: %s", EnumUtil::ToString(type_id)); + } +}; + +struct VariantToVariantDataWriter { + using result_type = void; + + static void VisitNull(WriteState &state) { + return; + } + static void VisitBoolean(bool, WriteState &state) { + return; + } + + template + static void VisitInteger(T val, WriteState &state) { + Store(val, state.GetDestination()); + state.blob_size += sizeof(T); + } + static void VisitFloat(float val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitDouble(double val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitUUID(hugeint_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitDate(date_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitInterval(interval_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitTime(dtime_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitTimeNanos(dtime_ns_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitTimeTZ(dtime_tz_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitTimestampSec(timestamp_sec_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitTimestampMs(timestamp_ms_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitTimestamp(timestamp_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitTimestampNanos(timestamp_ns_t val, WriteState &state) { + VisitInteger(val, state); + } + static void VisitTimestampTZ(timestamp_tz_t val, WriteState &state) { + VisitInteger(val, state); + } + + static void VisitString(const string_t &str, WriteState &state) { + auto length = str.GetSize(); + state.blob_size += VarintEncode(length, state.GetDestination()); + memcpy(state.GetDestination(), str.GetData(), length); + state.blob_size += length; + } + static void VisitBlob(const string_t &blob, WriteState &state) { + return VisitString(blob, state); + } + static void VisitBignum(const string_t &bignum, WriteState &state) { + return VisitString(bignum, state); + } + static void VisitGeometry(const string_t &geom, WriteState &state) { + return VisitString(geom, state); + } + static void VisitBitstring(const string_t &bits, WriteState &state) { + return VisitString(bits, state); + } + + template + static void VisitDecimal(T val, uint32_t width, uint32_t scale, WriteState &state) { + state.blob_size += VarintEncode(width, state.GetDestination()); + state.blob_size += VarintEncode(scale, state.GetDestination()); + Store(val, state.GetDestination()); + state.blob_size += sizeof(T); + } + + static void VisitArray(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &nested_data, + WriteState &state) { + state.blob_size += VarintEncode(nested_data.child_count, state.GetDestination()); + if (nested_data.child_count) { + //! NOTE: The 'child_index' stored in the OBJECT/ARRAY data could require more bits + //! That's the reason we have to rewrite the data in VARIANT->VARIANT cast + state.blob_size += VarintEncode(nested_data.children_idx + state.children_offset, state.GetDestination()); + } + } + + static void VisitObject(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &nested_data, + WriteState &state) { + return VisitArray(variant, row, nested_data, state); + } + + static void VisitDefault(VariantLogicalType type_id, const_data_ptr_t, WriteState &) { + throw InternalException("Unrecognized VariantLogicalType: %s", EnumUtil::ToString(type_id)); + } +}; + +} // namespace template bool ConvertVariantToVariant(ToVariantSourceData &source_data, ToVariantGlobalResultData &result_data, idx_t count, @@ -113,7 +266,6 @@ bool ConvertVariantToVariant(ToVariantSourceData &source_data, ToVariantGlobalRe auto &keys_offset = keys_offset_data[result_index]; auto &children_offset = children_offset_data[result_index]; - auto &values_offset = values_offset_data[result_index]; auto &blob_offset = blob_offset_data[result_index]; uint32_t keys_count = 0; @@ -126,6 +278,7 @@ bool ConvertVariantToVariant(ToVariantSourceData &source_data, ToVariantGlobalRe continue; } if (WRITE_DATA && values_index_selvec) { + auto &values_offset = values_offset_data[result_index]; //! Write the values_index for the parent of this column result.values_index_data[values_index_selvec->get_index(source_index)] = values_offset; } @@ -140,6 +293,7 @@ bool ConvertVariantToVariant(ToVariantSourceData &source_data, ToVariantGlobalRe source_children_index++) { //! values_index if (WRITE_DATA) { + auto &values_offset = values_offset_data[result_index]; auto source_value_index = source.GetValuesIndex(source_index, source_children_index); result.values_index_data[children_list_entry.offset + children_offset + source_children_index] = values_offset + source_value_index; @@ -167,99 +321,26 @@ bool ConvertVariantToVariant(ToVariantSourceData &source_data, ToVariantGlobalRe } } - auto source_blob_data = const_data_ptr_cast(source.GetData(source_index).GetData()); - - //! Then write all values auto source_values_list_entry = source.GetValuesListEntry(source_index); - for (uint32_t source_value_index = 0; source_value_index < source_values_list_entry.length; - source_value_index++) { - auto source_type_id = source.GetTypeId(source_index, source_value_index); - auto source_byte_offset = source.GetByteOffset(source_index, source_value_index); - - //! NOTE: we have to deserialize these in both passes - //! because to figure out the size of the 'data' that is added by the VARIANT, we have to traverse the - //! VARIANT solely because the 'child_index' stored in the OBJECT/ARRAY data could require more bits - WriteVariantMetadata(result_data, result_index, values_offset_data, blob_offset + blob_size, - nullptr, 0, source_type_id); - - if (source_type_id == VariantLogicalType::ARRAY || source_type_id == VariantLogicalType::OBJECT) { - auto source_nested_data = VariantUtils::DecodeNestedData(source, source_index, source_value_index); - if (WRITE_DATA) { - VarintEncode(source_nested_data.child_count, blob_data + blob_offset + blob_size); - } - blob_size += GetVarintSize(source_nested_data.child_count); - if (source_nested_data.child_count) { - auto new_child_index = source_nested_data.children_idx + children_offset; - if (WRITE_DATA) { - VarintEncode(new_child_index, blob_data + blob_offset + blob_size); - } - blob_size += GetVarintSize(new_child_index); - } - } else if (source_type_id == VariantLogicalType::VARIANT_NULL || - source_type_id == VariantLogicalType::BOOL_FALSE || - source_type_id == VariantLogicalType::BOOL_TRUE) { - // no-op - } else if (source_type_id == VariantLogicalType::DECIMAL) { - auto decimal_blob_data = source_blob_data + source_byte_offset; - auto width = static_cast(VarintDecode(decimal_blob_data)); - auto width_varint_size = GetVarintSize(width); - if (WRITE_DATA) { - memcpy(blob_data + blob_offset + blob_size, decimal_blob_data - width_varint_size, - width_varint_size); - } - blob_size += width_varint_size; - auto scale = static_cast(VarintDecode(decimal_blob_data)); - auto scale_varint_size = GetVarintSize(scale); - if (WRITE_DATA) { - memcpy(blob_data + blob_offset + blob_size, decimal_blob_data - scale_varint_size, - scale_varint_size); - } - blob_size += scale_varint_size; - - if (width > DecimalWidth::max) { - if (WRITE_DATA) { - memcpy(blob_data + blob_offset + blob_size, decimal_blob_data, sizeof(hugeint_t)); - } - blob_size += sizeof(hugeint_t); - } else if (width > DecimalWidth::max) { - if (WRITE_DATA) { - memcpy(blob_data + blob_offset + blob_size, decimal_blob_data, sizeof(int64_t)); - } - blob_size += sizeof(int64_t); - } else if (width > DecimalWidth::max) { - if (WRITE_DATA) { - memcpy(blob_data + blob_offset + blob_size, decimal_blob_data, sizeof(int32_t)); - } - blob_size += sizeof(int32_t); - } else { - if (WRITE_DATA) { - memcpy(blob_data + blob_offset + blob_size, decimal_blob_data, sizeof(int16_t)); - } - blob_size += sizeof(int16_t); - } - } else if (source_type_id == VariantLogicalType::BITSTRING || - source_type_id == VariantLogicalType::BIGNUM || source_type_id == VariantLogicalType::VARCHAR || - source_type_id == VariantLogicalType::BLOB) { - auto str_blob_data = source_blob_data + source_byte_offset; - auto str_length = VarintDecode(str_blob_data); - auto str_length_varint_size = GetVarintSize(str_length); - if (WRITE_DATA) { - memcpy(blob_data + blob_offset + blob_size, str_blob_data - str_length_varint_size, - str_length_varint_size); - } - blob_size += str_length_varint_size; - if (WRITE_DATA) { - memcpy(blob_data + blob_offset + blob_size, str_blob_data, str_length); - } - blob_size += str_length; - } else if (VariantIsTrivialPrimitive(source_type_id)) { - auto size = VariantTrivialPrimitiveSize(source_type_id); - if (WRITE_DATA) { - memcpy(blob_data + blob_offset + blob_size, source_blob_data + source_byte_offset, size); - } - blob_size += size; - } else { - throw InternalException("Unrecognized VariantLogicalType: %s", EnumUtil::ToString(source_type_id)); + + if (WRITE_DATA) { + WriteState write_state(keys_offset, children_offset, blob_offset, blob_data, blob_size); + for (uint32_t source_value_index = 0; source_value_index < source_values_list_entry.length; + source_value_index++) { + auto source_type_id = source.GetTypeId(source_index, source_value_index); + WriteVariantMetadata(result_data, result_index, values_offset_data, blob_offset + blob_size, + nullptr, 0, source_type_id); + + VariantVisitor::Visit(source, source_index, source_value_index, + write_state); + } + } else { + AnalyzeState analyze_state(children_offset); + for (uint32_t source_value_index = 0; source_value_index < source_values_list_entry.length; + source_value_index++) { + values_offset_data[result_index]++; + blob_size += VariantVisitor::Visit(source, source_index, + source_value_index, analyze_state); } } diff --git a/src/include/duckdb/function/compression_function.hpp b/src/include/duckdb/function/compression_function.hpp index b924ccba2b78..97fff72b1f89 100644 --- a/src/include/duckdb/function/compression_function.hpp +++ b/src/include/duckdb/function/compression_function.hpp @@ -17,6 +17,8 @@ #include "duckdb/storage/data_pointer.hpp" #include "duckdb/storage/storage_info.hpp" #include "duckdb/storage/block_manager.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/storage/storage_lock.hpp" namespace duckdb { class DatabaseInstance; @@ -27,7 +29,6 @@ class SegmentStatistics; class TableFilter; struct TableFilterState; struct ColumnSegmentState; - struct ColumnFetchState; struct ColumnScanState; struct PrefetchState; @@ -173,7 +174,8 @@ typedef void (*compression_compress_finalize_t)(CompressionState &state); // Uncompress / Scan //===--------------------------------------------------------------------===// typedef void (*compression_init_prefetch_t)(ColumnSegment &segment, PrefetchState &prefetch_state); -typedef unique_ptr (*compression_init_segment_scan_t)(ColumnSegment &segment); +typedef unique_ptr (*compression_init_segment_scan_t)(const QueryContext &context, + ColumnSegment &segment); //! Function prototype used for reading an entire vector (STANDARD_VECTOR_SIZE) typedef void (*compression_scan_vector_t)(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, @@ -220,7 +222,8 @@ typedef void (*compression_cleanup_state_t)(ColumnSegment &segment); // GetSegmentInfo (optional) //===--------------------------------------------------------------------===// //! Function prototype for retrieving segment information straight from the column segment -typedef InsertionOrderPreservingMap (*compression_get_segment_info_t)(ColumnSegment &segment); +typedef InsertionOrderPreservingMap (*compression_get_segment_info_t)(QueryContext context, + ColumnSegment &segment); enum class CompressionValidity : uint8_t { REQUIRES_VALIDITY, NO_VALIDITY_REQUIRED }; @@ -333,8 +336,31 @@ class CompressionFunction { //! The set of compression functions struct CompressionFunctionSet { + static constexpr idx_t COMPRESSION_TYPE_COUNT = 15; + static constexpr idx_t PHYSICAL_TYPE_COUNT = 19; + +public: + CompressionFunctionSet(); + + vector> GetCompressionFunctions(PhysicalType physical_type); + optional_ptr GetCompressionFunction(CompressionType type, PhysicalType physical_type); + void SetDisabledCompressionMethods(const vector &methods); + vector GetDisabledCompressionMethods() const; + +private: mutex lock; - map> functions; + atomic is_disabled[COMPRESSION_TYPE_COUNT]; + atomic is_loaded[PHYSICAL_TYPE_COUNT]; + vector> functions; + +private: + void LoadCompressionFunctions(PhysicalType physical_type); + + static void TryLoadCompression(CompressionType type, PhysicalType physical_type, + vector &result); + static idx_t GetCompressionIndex(PhysicalType physical_type); + static idx_t GetCompressionIndex(CompressionType type); + void ResetDisabledMethods(); }; } // namespace duckdb diff --git a/src/include/duckdb/function/scalar/variant_functions.hpp b/src/include/duckdb/function/scalar/variant_functions.hpp index 6408639ec3e5..c318a923697e 100644 --- a/src/include/duckdb/function/scalar/variant_functions.hpp +++ b/src/include/duckdb/function/scalar/variant_functions.hpp @@ -25,11 +25,21 @@ struct VariantExtractFun { static ScalarFunctionSet GetFunctions(); }; +struct VariantNormalizeFun { + static constexpr const char *Name = "variant_normalize"; + static constexpr const char *Parameters = "input_variant"; + static constexpr const char *Description = "Normalizes the `input_variant` to a canonical representation."; + static constexpr const char *Example = "variant_normalize({'b': [1,2,3], 'a': 42})::VARIANT)"; + static constexpr const char *Categories = "variant"; + + static ScalarFunction GetFunction(); +}; + struct VariantTypeofFun { static constexpr const char *Name = "variant_typeof"; static constexpr const char *Parameters = "input_variant"; static constexpr const char *Description = "Returns the internal type of the `input_variant`."; - static constexpr const char *Example = "variant_typeof({'a': 42, 'b': [1,2,3])::VARIANT)"; + static constexpr const char *Example = "variant_typeof({'a': 42, 'b': [1,2,3]})::VARIANT)"; static constexpr const char *Categories = "variant"; static ScalarFunction GetFunction(); diff --git a/src/include/duckdb/function/scalar/variant_utils.hpp b/src/include/duckdb/function/scalar/variant_utils.hpp index f0c4cb82bce5..1c20b19c0e9f 100644 --- a/src/include/duckdb/function/scalar/variant_utils.hpp +++ b/src/include/duckdb/function/scalar/variant_utils.hpp @@ -66,20 +66,25 @@ struct VariantUtils { uint32_t value_index); DUCKDB_API static VariantNestedData DecodeNestedData(const UnifiedVariantVectorData &variant, idx_t row, uint32_t value_index); + DUCKDB_API static string_t DecodeStringData(const UnifiedVariantVectorData &variant, idx_t row, + uint32_t value_index); DUCKDB_API static vector GetObjectKeys(const UnifiedVariantVectorData &variant, idx_t row, const VariantNestedData &nested_data); - DUCKDB_API static VariantChildDataCollectionResult FindChildValues(const UnifiedVariantVectorData &variant, - const VariantPathComponent &component, - optional_idx row, SelectionVector &res, - VariantNestedData *nested_data, idx_t count); + DUCKDB_API static void FindChildValues(const UnifiedVariantVectorData &variant, + const VariantPathComponent &component, + optional_ptr sel, SelectionVector &res, + ValidityMask &res_validity, VariantNestedData *nested_data, idx_t count); DUCKDB_API static VariantNestedDataCollectionResult CollectNestedData(const UnifiedVariantVectorData &variant, VariantLogicalType expected_type, const SelectionVector &sel, idx_t count, optional_idx row, idx_t offset, VariantNestedData *child_data, ValidityMask &validity); DUCKDB_API static vector ValueIsNull(const UnifiedVariantVectorData &variant, const SelectionVector &sel, idx_t count, optional_idx row); - DUCKDB_API static Value ConvertVariantToValue(const UnifiedVariantVectorData &variant, idx_t row, idx_t values_idx); + DUCKDB_API static Value ConvertVariantToValue(const UnifiedVariantVectorData &variant, idx_t row, + uint32_t values_idx); DUCKDB_API static bool Verify(Vector &variant, const SelectionVector &sel_p, idx_t count); + DUCKDB_API static void FinalizeVariantKeys(Vector &variant, OrderedOwningStringMap &dictionary, + SelectionVector &sel, idx_t sel_size); }; } // namespace duckdb diff --git a/src/include/duckdb/function/table/read_duckdb.hpp b/src/include/duckdb/function/table/read_duckdb.hpp new file mode 100644 index 000000000000..3bf2fd4e8ed3 --- /dev/null +++ b/src/include/duckdb/function/table/read_duckdb.hpp @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/function/table/read_duckdb.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/function/table_function.hpp" + +namespace duckdb { +struct ReplacementScanInput; +struct ReplacementScanData; + +struct ReadDuckDBTableFunction { + static TableFunction GetFunction(); + static unique_ptr ReplacementScan(ClientContext &context, ReplacementScanInput &input, + optional_ptr data); +}; + +} // namespace duckdb diff --git a/src/include/duckdb/function/table/system_functions.hpp b/src/include/duckdb/function/table/system_functions.hpp index e325b2f46010..49c5e794c11c 100644 --- a/src/include/duckdb/function/table/system_functions.hpp +++ b/src/include/duckdb/function/table/system_functions.hpp @@ -47,6 +47,10 @@ struct DuckDBSchemasFun { static void RegisterFunction(BuiltinFunctions &set); }; +struct DuckDBConnectionCountFun { + static void RegisterFunction(BuiltinFunctions &set); +}; + struct DuckDBApproxDatabaseCountFun { static void RegisterFunction(BuiltinFunctions &set); }; diff --git a/src/include/duckdb/logging/log_storage.hpp b/src/include/duckdb/logging/log_storage.hpp index 10c06dddce09..9a8a6740577e 100644 --- a/src/include/duckdb/logging/log_storage.hpp +++ b/src/include/duckdb/logging/log_storage.hpp @@ -33,7 +33,7 @@ struct CSVReaderOptions; //! Logging storage can store entries normalized or denormalized. This enum describes what a single table/file/etc //! contains -enum class LoggingTargetTable { +enum class LoggingTargetTable : uint8_t { ALL_LOGS, // Denormalized: log entries consisting of both the full log entry and the context LOG_ENTRIES, // Normalized: contains only the log entries and a context_id LOG_CONTEXTS, // Normalized: contains only the log contexts @@ -165,7 +165,7 @@ class BufferingLogStorage : public LogStorage { //! Debug option for testing buffering behaviour bool only_flush_on_full_buffer = false; //! The buffers used for each table - unordered_map> buffers; + map> buffers; //! This flag is set whenever a new context_is written to the entry buffer. It means that the next flush of //! LoggingTargetTable::LOG_ENTRIES also requires a flush of LoggingTargetTable::LOG_CONTEXTS bool flush_contexts_on_next_entry_flush = false; @@ -218,9 +218,9 @@ class CSVLogStorage : public BufferingLogStorage { void InitializeCastChunk(LoggingTargetTable table); //! The cast buffers used to cast from the original types to the VARCHAR types ready to write to CSV format - unordered_map> cast_buffers; + map> cast_buffers; //! The writers to be registered by child classes - unordered_map> writers; + map> writers; //! CSV Options to initialize the CSVWriters with. TODO: cleanup, this is now a little bit of a mixed bag of //! settings @@ -302,7 +302,7 @@ class FileLogStorage : public CSVLogStorage { }; //! The table info per table - unordered_map tables; + map tables; //! Base path to generate the file paths from string base_path; @@ -349,7 +349,7 @@ class InMemoryLogStorage : public BufferingLogStorage { //! Helper function to get the buffer ColumnDataCollection &GetBuffer(LoggingTargetTable table) const; - unordered_map> log_storage_buffers; + map> log_storage_buffers; }; } // namespace duckdb diff --git a/src/include/duckdb/logging/log_type.hpp b/src/include/duckdb/logging/log_type.hpp index c843fe1b0961..23d901c4ea57 100644 --- a/src/include/duckdb/logging/log_type.hpp +++ b/src/include/duckdb/logging/log_type.hpp @@ -17,6 +17,9 @@ struct FileHandle; struct BaseRequest; struct HTTPResponse; class PhysicalOperator; +class AttachedDatabase; +class RowGroup; +struct DataTableInfo; //! Log types provide some structure to the formats that the different log messages can have //! For now, this holds a type that the VARCHAR value will be auto-cast into. @@ -54,9 +57,7 @@ class QueryLogType : public LogType { QueryLogType() : LogType(NAME, LEVEL) {}; - static string ConstructLogMessage(const string &str) { - return str; - } + static string ConstructLogMessage(const string &str); }; class FileSystemLogType : public LogType { @@ -105,4 +106,26 @@ class PhysicalOperatorLogType : public LogType { const vector> &info); }; +class CheckpointLogType : public LogType { +public: + static constexpr const char *NAME = "Checkpoint"; + static constexpr LogLevel LEVEL = LogLevel::LOG_DEBUG; + + //! Construct the log type + CheckpointLogType(); + + static LogicalType GetLogType(); + + //! Vacuum + static string ConstructLogMessage(const AttachedDatabase &db, DataTableInfo &table, idx_t segment_idx, + idx_t merge_count, idx_t target_count, idx_t merge_rows, idx_t row_start); + //! Checkpoint + static string ConstructLogMessage(const AttachedDatabase &db, DataTableInfo &table, idx_t segment_idx, + RowGroup &row_group); + +private: + static string CreateLog(const AttachedDatabase &db, DataTableInfo &table, const char *op, vector map_keys, + vector map_values); +}; + } // namespace duckdb diff --git a/src/include/duckdb/main/attached_database.hpp b/src/include/duckdb/main/attached_database.hpp index b258ffdb7129..cb539e6b0e95 100644 --- a/src/include/duckdb/main/attached_database.hpp +++ b/src/include/duckdb/main/attached_database.hpp @@ -23,6 +23,7 @@ class StorageExtension; class DatabaseManager; struct AttachInfo; +struct StoredDatabasePath; enum class AttachedDatabaseType { READ_WRITE_DATABASE, @@ -31,12 +32,20 @@ enum class AttachedDatabaseType { TEMP_DATABASE, }; +enum class AttachVisibility { SHOWN, HIDDEN }; + +class DatabaseFilePathManager; + struct StoredDatabasePath { - StoredDatabasePath(DatabaseManager &manager, string path, const string &name); + StoredDatabasePath(DatabaseManager &db_manager, DatabaseFilePathManager &manager, string path, const string &name); ~StoredDatabasePath(); - DatabaseManager &manager; + DatabaseManager &db_manager; + DatabaseFilePathManager &manager; string path; + +public: + void OnDetach(); }; //! AttachOptions holds information about a database we plan to attach. These options are generalized, i.e., @@ -55,6 +64,12 @@ struct AttachOptions { unordered_map options; //! (optionally) a catalog can be provided with a default table QualifiedName default_table; + //! Whether or not this is the main database + bool is_main_database = false; + //! The visibility of the attached database + AttachVisibility visibility = AttachVisibility::SHOWN; + //! The stored database path (in the path manager) + unique_ptr stored_database_path; }; //! The AttachedDatabase represents an attached database instance. @@ -91,6 +106,9 @@ class AttachedDatabase : public CatalogEntry, public enable_shared_from_this stored_database_path; @@ -114,6 +133,7 @@ class AttachedDatabase : public CatalogEntry, public enable_shared_from_this parent_catalog; optional_ptr storage_extension; + AttachVisibility visibility = AttachVisibility::SHOWN; bool is_initial_database = false; bool is_closed = false; }; diff --git a/src/include/duckdb/main/buffered_data/batched_buffered_data.hpp b/src/include/duckdb/main/buffered_data/batched_buffered_data.hpp index c3ffadbe25b9..57ea17d6be10 100644 --- a/src/include/duckdb/main/buffered_data/batched_buffered_data.hpp +++ b/src/include/duckdb/main/buffered_data/batched_buffered_data.hpp @@ -32,7 +32,7 @@ class BatchedBufferedData : public BufferedData { static constexpr const BufferedData::Type TYPE = BufferedData::Type::BATCHED; public: - explicit BatchedBufferedData(weak_ptr context); + explicit BatchedBufferedData(ClientContext &context); public: void Append(const DataChunk &chunk, idx_t batch); diff --git a/src/include/duckdb/main/buffered_data/buffered_data.hpp b/src/include/duckdb/main/buffered_data/buffered_data.hpp index 0f32675ceac9..06a72b0f64dc 100644 --- a/src/include/duckdb/main/buffered_data/buffered_data.hpp +++ b/src/include/duckdb/main/buffered_data/buffered_data.hpp @@ -28,7 +28,7 @@ class BufferedData { enum class Type { SIMPLE, BATCHED }; public: - BufferedData(Type type, weak_ptr context_p); + BufferedData(Type type, ClientContext &context); virtual ~BufferedData(); public: diff --git a/src/include/duckdb/main/buffered_data/simple_buffered_data.hpp b/src/include/duckdb/main/buffered_data/simple_buffered_data.hpp index 967cc1ab7636..40a5a6edebac 100644 --- a/src/include/duckdb/main/buffered_data/simple_buffered_data.hpp +++ b/src/include/duckdb/main/buffered_data/simple_buffered_data.hpp @@ -24,7 +24,7 @@ class SimpleBufferedData : public BufferedData { static constexpr const BufferedData::Type TYPE = BufferedData::Type::SIMPLE; public: - explicit SimpleBufferedData(weak_ptr context); + explicit SimpleBufferedData(ClientContext &context); ~SimpleBufferedData() override; public: diff --git a/src/include/duckdb/main/capi/capi_internal.hpp b/src/include/duckdb/main/capi/capi_internal.hpp index 8307b70a35f8..3b736d88a208 100644 --- a/src/include/duckdb/main/capi/capi_internal.hpp +++ b/src/include/duckdb/main/capi/capi_internal.hpp @@ -51,6 +51,8 @@ struct PreparedStatementWrapper { //! Map of name -> values case_insensitive_map_t values; unique_ptr statement; + bool success = true; + ErrorData error_data; }; struct ExtractStatementsWrapper { diff --git a/src/include/duckdb/main/capi/extension_api.hpp b/src/include/duckdb/main/capi/extension_api.hpp index 0d3cdfcc8e89..899a331cb968 100644 --- a/src/include/duckdb/main/capi/extension_api.hpp +++ b/src/include/duckdb/main/capi/extension_api.hpp @@ -501,6 +501,26 @@ typedef struct { bool (*duckdb_expression_is_foldable)(duckdb_expression expr); duckdb_error_data (*duckdb_expression_fold)(duckdb_client_context context, duckdb_expression expr, duckdb_value *out_value); + // API to manage file system operations + + duckdb_file_system (*duckdb_client_context_get_file_system)(duckdb_client_context context); + void (*duckdb_destroy_file_system)(duckdb_file_system *file_system); + duckdb_state (*duckdb_file_system_open)(duckdb_file_system file_system, const char *path, + duckdb_file_open_options options, duckdb_file_handle *out_file); + duckdb_error_data (*duckdb_file_system_error_data)(duckdb_file_system file_system); + duckdb_file_open_options (*duckdb_create_file_open_options)(); + duckdb_state (*duckdb_file_open_options_set_flag)(duckdb_file_open_options options, duckdb_file_flag flag, + bool value); + void (*duckdb_destroy_file_open_options)(duckdb_file_open_options *options); + void (*duckdb_destroy_file_handle)(duckdb_file_handle *file_handle); + duckdb_error_data (*duckdb_file_handle_error_data)(duckdb_file_handle file_handle); + duckdb_state (*duckdb_file_handle_close)(duckdb_file_handle file_handle); + int64_t (*duckdb_file_handle_read)(duckdb_file_handle file_handle, void *buffer, int64_t size); + int64_t (*duckdb_file_handle_write)(duckdb_file_handle file_handle, const void *buffer, int64_t size); + duckdb_state (*duckdb_file_handle_seek)(duckdb_file_handle file_handle, int64_t position); + int64_t (*duckdb_file_handle_tell)(duckdb_file_handle file_handle); + duckdb_state (*duckdb_file_handle_sync)(duckdb_file_handle file_handle); + int64_t (*duckdb_file_handle_size)(duckdb_file_handle file_handle); // New functions around the client context idx_t (*duckdb_client_context_get_connection_id)(duckdb_client_context context); @@ -509,6 +529,13 @@ typedef struct { duckdb_value (*duckdb_get_table_names)(duckdb_connection connection, const char *query, bool qualified); void (*duckdb_connection_get_arrow_options)(duckdb_connection connection, duckdb_arrow_options *out_arrow_options); void (*duckdb_destroy_arrow_options)(duckdb_arrow_options *arrow_options); + // API to get information about the results of a prepared statement + + idx_t (*duckdb_prepared_statement_column_count)(duckdb_prepared_statement prepared_statement); + const char *(*duckdb_prepared_statement_column_name)(duckdb_prepared_statement prepared_statement, idx_t col_idx); + duckdb_logical_type (*duckdb_prepared_statement_column_logical_type)(duckdb_prepared_statement prepared_statement, + idx_t col_idx); + duckdb_type (*duckdb_prepared_statement_column_type)(duckdb_prepared_statement prepared_statement, idx_t col_idx); // New query execution functions duckdb_arrow_options (*duckdb_result_get_arrow_options)(duckdb_result *result); @@ -527,6 +554,11 @@ typedef struct { // New string functions that are added char *(*duckdb_value_to_string)(duckdb_value value); + // New functions around the table description + + idx_t (*duckdb_table_description_get_column_count)(duckdb_table_description table_description); + duckdb_logical_type (*duckdb_table_description_get_column_type)(duckdb_table_description table_description, + idx_t index); // New functions around table function binding void (*duckdb_table_function_get_client_context)(duckdb_bind_info info, duckdb_client_context *out_context); @@ -980,12 +1012,32 @@ inline duckdb_ext_api_v1 CreateAPIv1() { result.duckdb_expression_return_type = duckdb_expression_return_type; result.duckdb_expression_is_foldable = duckdb_expression_is_foldable; result.duckdb_expression_fold = duckdb_expression_fold; + result.duckdb_client_context_get_file_system = duckdb_client_context_get_file_system; + result.duckdb_destroy_file_system = duckdb_destroy_file_system; + result.duckdb_file_system_open = duckdb_file_system_open; + result.duckdb_file_system_error_data = duckdb_file_system_error_data; + result.duckdb_create_file_open_options = duckdb_create_file_open_options; + result.duckdb_file_open_options_set_flag = duckdb_file_open_options_set_flag; + result.duckdb_destroy_file_open_options = duckdb_destroy_file_open_options; + result.duckdb_destroy_file_handle = duckdb_destroy_file_handle; + result.duckdb_file_handle_error_data = duckdb_file_handle_error_data; + result.duckdb_file_handle_close = duckdb_file_handle_close; + result.duckdb_file_handle_read = duckdb_file_handle_read; + result.duckdb_file_handle_write = duckdb_file_handle_write; + result.duckdb_file_handle_seek = duckdb_file_handle_seek; + result.duckdb_file_handle_tell = duckdb_file_handle_tell; + result.duckdb_file_handle_sync = duckdb_file_handle_sync; + result.duckdb_file_handle_size = duckdb_file_handle_size; result.duckdb_client_context_get_connection_id = duckdb_client_context_get_connection_id; result.duckdb_destroy_client_context = duckdb_destroy_client_context; result.duckdb_connection_get_client_context = duckdb_connection_get_client_context; result.duckdb_get_table_names = duckdb_get_table_names; result.duckdb_connection_get_arrow_options = duckdb_connection_get_arrow_options; result.duckdb_destroy_arrow_options = duckdb_destroy_arrow_options; + result.duckdb_prepared_statement_column_count = duckdb_prepared_statement_column_count; + result.duckdb_prepared_statement_column_name = duckdb_prepared_statement_column_name; + result.duckdb_prepared_statement_column_logical_type = duckdb_prepared_statement_column_logical_type; + result.duckdb_prepared_statement_column_type = duckdb_prepared_statement_column_type; result.duckdb_result_get_arrow_options = duckdb_result_get_arrow_options; result.duckdb_scalar_function_set_bind = duckdb_scalar_function_set_bind; result.duckdb_scalar_function_bind_set_error = duckdb_scalar_function_bind_set_error; @@ -997,6 +1049,8 @@ inline duckdb_ext_api_v1 CreateAPIv1() { result.duckdb_scalar_function_bind_get_argument = duckdb_scalar_function_bind_get_argument; result.duckdb_scalar_function_set_bind_data_copy = duckdb_scalar_function_set_bind_data_copy; result.duckdb_value_to_string = duckdb_value_to_string; + result.duckdb_table_description_get_column_count = duckdb_table_description_get_column_count; + result.duckdb_table_description_get_column_type = duckdb_table_description_get_column_type; result.duckdb_table_function_get_client_context = duckdb_table_function_get_client_context; result.duckdb_create_map_value = duckdb_create_map_value; result.duckdb_create_union_value = duckdb_create_union_value; diff --git a/src/include/duckdb/main/capi/header_generation/README.md b/src/include/duckdb/main/capi/header_generation/README.md index 65d46e9587eb..9c5dd1d074d3 100644 --- a/src/include/duckdb/main/capi/header_generation/README.md +++ b/src/include/duckdb/main/capi/header_generation/README.md @@ -1,49 +1,89 @@ -# DuckDB C API header generation -DuckDB's C API headers are code generated from json definition files. There are multiple reasons for this: -- it makes it easy to structure the C API headers cleanly -- the definition files allows driving code-gen in DuckDB clients that are based on the C API -- it allows creating a "versioned struct of function pointer"-type API for duckdb extension +# C API + +The C API provides a stable interface based on DuckDB's internal C++ code. +Thus, applications built against the C API avoid breaking due to changes in DuckDB's fast-moving codebase. + +## Headers + +### Header generation + +DuckDB's C API headers are code generated from JSON definition files. +There are multiple reasons for this: +- easy to structure the C API headers +- code generation in DuckDB clients based on the C API without parsing a C header +- versioned function-pointer-struct API for DuckDB extensions based on the C API The C API currently consists of 2 main parts: -- the C API as defined in `duckdb.h` -- the struct-of-function pointers in `duckdb_extension.h` - -## The main header `duckdb.h` -This header contains macros, typedefs and all functions of the DuckDB C API. This API is what programs that link to DuckDB -will use. To link against DuckDB over the C API, the `duckdb.h` file of the desired DuckDB version -should be included, after which the program can link to DuckDB of that specific version. - -## The extension API header `duckdb_extension.h` -The C extension API consists of a struct-of-function-pointers that is passed to an extension on load. Because this struct -is versioned, extensions based on this API can work across a wide range of DuckDB versions without requiring a rebuild. This is -achieved by forcing the struct-of-function-pointers to never change between DuckDB versions, only adding new functions at -the end of the struct. Due to the rigid nature of the struct, not all function that are added to the C API are immediately -added to this struct. - -# How to add a function to the C API -The only way to add function to the C api is to add them to the `JSON` definition files in `src/include/duckdb/main/capi/header_generation/functions`. To add a C API function to the headers, there are two options. -Note that the one you are probably looking for is adding the function to both C API and the `dev` version of the Extension C API. - -## Adding the function to the C API only -To add a function to the C API, but not the struct-of-function-pointers, simply create an entry in one of the `src/include/duckdb/main/capi/header_generation/functions/*.json` file for it. -Next, the function should be added to the exclusion list in `src/include/duckdb/main/capi/header_generation/apis/v0/exclusion_list.json` to mark them as excluded. Please provide -a reason for each group of functions to ensure we properly document why these are excluded. Also consider merging groups of excluded functions with similar exlusion reason together to keep things clean. The final step is to -run the generation script `scripts/generate_c_api.py` or use the makefile `make generate-files`. - -## Adding the function to both the C API and Extension C API -To add a function to the C API and to the struct-of-function-pointers, again create an entry in one of the `src/include/duckdb/main/capi/header_generation/functions/*.json` files. -Then, the function should be added to `src/include/duckdb/main/capi/header_generation/apis/v1/*/*.json`. Again, run the script `scripts/generate_c_api.py` or the makefile target `make generate-files` to generate. - -### Adding a function to the `dev` version -The first way a function will be added to the Extension C API is likely in the `dev` version. This will allow testing the function from extensions, for example in CI, while not yet promising eternal stability of the function. -Most functions that are added will fall in this category. - -### Adding a function to a stable version -When a function is deemed stable, the function can be added to a versioned part of the Extension C API struct. - -To not break backwards compatibility, functions can only be added to struct-of-function-pointers by adding a new `JSON` file to the `src/include/duckdb/main/capi/header_generation/apis/v0` directory. -Note that the version tag is important here since these determine the order of the struct. - -Another thing to consider is not needlessly bumping this version for every individual C API function that is added. It is preferred to group -additions to the struct in the `dev` version first, as this struct is effectively immutable and if we make a mess out of it we will suffer for eternity. A sensible choice would -be to only merge new functions into a stable version of the struct periodically, e.g. as a new DuckDB release approaches. \ No newline at end of file +- the C API as defined in `duckdb.h` (main header) +- the function-pointer-struct in `duckdb_extension.h` (extension header) + +### The main header `duckdb.h` + +The main header contains macros, typedefs and all functions of the DuckDB C API. +Programs linking to DuckDB use this API, e.g., the C API-based clients. +To link against DuckDB via the C API, the `duckdb.h` file of the desired DuckDB version +must be included, after which the program can link to a DuckDB (bundled) library of that specific version. + +### The extension API header `duckdb_extension.h` + +The C extension API consists of a struct of function pointers that is passed to an extension on load. +Because this struct is versioned, extensions based on the extension API can work across a wide range of +DuckDB versions without requiring a rebuild. +Forcing the function-pointer-struct to never change between DuckDB versions achieves this stability. +It is append-only, meaning new functions are always added to the end of the struct. +Due to the rigid nature of the struct, not all function that are added to the C API are immediately added to this struct. + +## Adding a function to the C API + +The only way to add functions to the C api is to add them to the `JSON` definition files in `src/include/duckdb/main/capi/header_generation/functions`. +Please note that the C API is intended as a minimal wrapper around DuckDB's internal C++ functions. +As such, we strive to add as few helper-only functions to the C API as possible. +To add a C API function to the headers, there are two options. +Note that you are probably looking for option 2, which adds the function to both the unstable C API and the `dev` version of the extension API header. + +### 1. Adding a function to the C API only + +Here are the steps to add a function to the C API, but not the function-pointer-struct: + +1. Create an entry in one of the `src/include/duckdb/main/capi/header_generation/functions/*.json` files. +2. Add the function to the exclusion list in `src/include/duckdb/main/capi/header_generation/apis/v0/exclusion_list.json` to mark it as excluded. +Please provide a reason for each group of functions to ensure we properly document why these are excluded. +Also consider merging groups of excluded functions with similar exclusion reasons together to keep things tidy. +3. Run the generation script `scripts/generate_c_api.py` or the Makefile target `make generate-files`. + +### 2. Adding a function to the C API and the extension API + +Here are the steps to add a function to the C API and to the function-pointer-struct: + +1. Create an entry in one of the `src/include/duckdb/main/capi/header_generation/functions/*.json` files. +2. Add the function to `src/include/duckdb/main/capi/header_generation/apis/v1/*/*.json`. +3. Run the generation script `scripts/generate_c_api.py` or the Makefile target `make generate-files`. + +#### Adding a function to the `dev` version + +The first way a function is added to the extension API is likely via the `dev` version. +This allows testing from extensions, for example in the CI, while not yet promising stability of the function. +Most functions fall in this category. + +#### Adding a function to a stable version + +When a function is deemed stable, the function can be added to a versioned part of the extension API struct. + +To not break backwards compatibility, functions are only added to the function-pointer-struct by +adding a new `JSON` file to the `src/include/duckdb/main/capi/header_generation/apis/v0` directory. +Note that the version tag is important here since it determines the order of the struct. + +Another thing to consider is grouping additions to the stable version, thus, avoiding needlessly bumping this version for every new stable function. + +## The JSON files + +As mentioned before, DuckDB's C API headers are code generated from JSON definition files. + +- A schema to define a list of unstable functions. Describes the files in `./apis/v1/unstable`: +``` +./schemata/unstable_function_list_schema.json +``` +- A schema to define a function group. Describes the files in `./functions`: +``` +./schemata/function_group_schema.json +``` diff --git a/src/include/duckdb/main/capi/header_generation/apis/v1/unstable/file_system.json b/src/include/duckdb/main/capi/header_generation/apis/v1/unstable/file_system.json new file mode 100644 index 000000000000..96719b95dacf --- /dev/null +++ b/src/include/duckdb/main/capi/header_generation/apis/v1/unstable/file_system.json @@ -0,0 +1,24 @@ +{ + "version": "unstable_new_file_system_api", + "description": "API to manage file system operations", + "entries": [ + "duckdb_client_context_get_file_system", + "duckdb_destroy_file_system", + "duckdb_file_system_open", + "duckdb_file_system_error_data", + + "duckdb_create_file_open_options", + "duckdb_file_open_options_set_flag", + "duckdb_destroy_file_open_options", + + "duckdb_destroy_file_handle", + "duckdb_file_handle_error_data", + "duckdb_file_handle_close", + "duckdb_file_handle_read", + "duckdb_file_handle_write", + "duckdb_file_handle_seek", + "duckdb_file_handle_tell", + "duckdb_file_handle_sync", + "duckdb_file_handle_size" + ] +} diff --git a/src/include/duckdb/main/capi/header_generation/apis/v1/unstable/new_prepared_statement_functions.json b/src/include/duckdb/main/capi/header_generation/apis/v1/unstable/new_prepared_statement_functions.json new file mode 100644 index 000000000000..2be76a7391e9 --- /dev/null +++ b/src/include/duckdb/main/capi/header_generation/apis/v1/unstable/new_prepared_statement_functions.json @@ -0,0 +1,10 @@ +{ + "version": "unstable_new_prepared_statement_functions", + "description": "API to get information about the results of a prepared statement", + "entries": [ + "duckdb_prepared_statement_column_count", + "duckdb_prepared_statement_column_name", + "duckdb_prepared_statement_column_logical_type", + "duckdb_prepared_statement_column_type" + ] +} diff --git a/src/include/duckdb/main/capi/header_generation/apis/v1/unstable/new_table_description_functions.json b/src/include/duckdb/main/capi/header_generation/apis/v1/unstable/new_table_description_functions.json new file mode 100644 index 000000000000..28cd47a50bfd --- /dev/null +++ b/src/include/duckdb/main/capi/header_generation/apis/v1/unstable/new_table_description_functions.json @@ -0,0 +1,8 @@ +{ + "version": "unstable_new_table_description_functions", + "description": "New functions around the table description", + "entries": [ + "duckdb_table_description_get_column_count", + "duckdb_table_description_get_column_type" + ] +} \ No newline at end of file diff --git a/src/include/duckdb/main/capi/header_generation/functions/file_system_interface.json b/src/include/duckdb/main/capi/header_generation/functions/file_system_interface.json new file mode 100644 index 000000000000..030293284087 --- /dev/null +++ b/src/include/duckdb/main/capi/header_generation/functions/file_system_interface.json @@ -0,0 +1,318 @@ +{ + "group": "file_system_interface", + "deprecated": false, + "entries": [ + { + "name": "duckdb_client_context_get_file_system", + "return_type": "duckdb_file_system", + "params": [ + { + "type": "duckdb_client_context", + "name": "context" + } + ], + "comment": { + "description": "Get a file system instance associated with the given client context.\n\n", + "param_comments": { + "context": "The client context." + }, + "return_value": "The resulting file system instance. Must be destroyed with `duckdb_destroy_file_system`." + } + }, + { + "name": "duckdb_destroy_file_system", + "return_type": "void", + "params": [ + { + "type": "duckdb_file_system*", + "name": "file_system" + } + ], + "comment": { + "description": "Destroys the given file system instance.\n", + "param_comments": { + "file_system": "The file system instance to destroy." + } + } + }, + { + "name": "duckdb_file_system_error_data", + "return_type": "duckdb_error_data", + "params": [ + { + "type": "duckdb_file_system", + "name": "file_system" + } + ], + "comment": { + "description": "Retrieves the last error that occurred on the given file system instance.\n\n", + "param_comments": { + "file_system": "The file system instance." + }, + "return_value": "The error data." + } + }, + { + "name": "duckdb_file_system_open", + "return_type": "duckdb_state", + "params": [ + { + "type": "duckdb_file_system", + "name": "file_system" + }, + { + "type": "const char*", + "name": "path" + }, + { + "type": "duckdb_file_open_options", + "name": "options" + }, + { + "type": "duckdb_file_handle*", + "name": "out_file" + } + ], + "comment": { + "description": "Opens a file at the given path with the specified options.\n\n", + "param_comments": { + "file_system": "The file system instance.", + "path": "The path to the file.", + "options": "The file open options specifying how to open the file.", + "out_file": "The resulting file handle instance, or `nullptr` if the open failed. Must be destroyed with `duckdb_destroy_file_handle`." + }, + "return_value": "Whether the operation was successful. If not, the error data can be retrieved using `duckdb_file_system_error_data`." + } + }, + { + "name": "duckdb_create_file_open_options", + "return_type": "duckdb_file_open_options", + "params": [], + "comment": { + "description": "Creates a new file open options instance with blank settings.\n\n", + "param_comments": {}, + "return_value": "The new file open options instance. Must be destroyed with `duckdb_destroy_file_open_options`." + } + }, + { + "name": "duckdb_file_open_options_set_flag", + "return_type": "duckdb_state", + "params": [ + { + "type": "duckdb_file_open_options", + "name": "options" + }, + { + "type": "duckdb_file_flag", + "name": "flag" + }, + { + "type": "bool", + "name": "value" + } + ], + "comment": { + "description": "Sets a specific flag in the file open options.\n\n", + "param_comments": { + "options": "The file open options instance.", + "flag": "The flag to set (e.g., read, write).", + "value": "If the flag is enabled or disabled." + }, + "return_value": "`DuckDBSuccess` on success or `DuckDBError` if the flag is unrecognized or unsupported by this version of DuckDB." + } + }, + { + "name": "duckdb_destroy_file_open_options", + "return_type": "void", + "params": [ + { + "type": "duckdb_file_open_options*", + "name": "options" + } + ], + "comment": { + "description": "Destroys the given file open options instance.\n", + "param_comments": { + "options": "The file open options instance to destroy." + } + } + }, + { + "name": "duckdb_destroy_file_handle", + "return_type": "void", + "params": [ + { + "type": "duckdb_file_handle*", + "name": "file_handle" + } + ], + "comment": { + "description": "Destroys the given file handle and deallocates all associated resources.\nThis will also close the file if it is still open.\n\n", + "param_comments": { + "file_handle": "The file handle to destroy." + } + } + }, + { + "name": "duckdb_file_handle_error_data", + "return_type": "duckdb_error_data", + "params": [ + { + "type": "duckdb_file_handle", + "name": "file_handle" + } + ], + "comment": { + "description": "Retrieves the last error that occurred on the given file handle.\n\n", + "param_comments": { + "file_handle": "The file handle." + }, + "return_value": "The error data. Must be destroyed with `duckdb_destroy_error_data`" + } + }, + { + "name": "duckdb_file_handle_read", + "return_type": "int64_t", + "params": [ + { + "type": "duckdb_file_handle", + "name": "file_handle" + }, + { + "type": "void*", + "name": "buffer" + }, + { + "type": "int64_t", + "name": "size" + } + ], + "comment": { + "description": "Reads data from the file into the buffer.\n\n", + "param_comments": { + "file_handle": "The file handle to read from.", + "buffer": "The buffer to read data into.", + "size": "The number of bytes to read." + }, + "return_value": "The number of bytes actually read, or negative on error." + } + }, + { + "name": "duckdb_file_handle_write", + "return_type": "int64_t", + "params": [ + { + "type": "duckdb_file_handle", + "name": "file_handle" + }, + { + "type": "const void*", + "name": "buffer" + }, + { + "type": "int64_t", + "name": "size" + } + ], + "comment": { + "description": "Writes data from the buffer to the file.\n\n", + "param_comments": { + "file_handle": "The file handle to write to.", + "buffer": "The buffer containing data to write.", + "size": "The number of bytes to write." + }, + "return_value": "The number of bytes actually written, or negative on error." + } + }, + { + "name": "duckdb_file_handle_tell", + "return_type": "int64_t", + "params": [ + { + "type": "duckdb_file_handle", + "name": "file_handle" + } + ], + "comment": { + "description": "Tells the current position in the file.\n\n", + "param_comments": { + "file_handle": "The file handle to tell the position of." + }, + "return_value": "The current position in the file, or negative on error." + } + }, + { + "name": "duckdb_file_handle_size", + "return_type": "int64_t", + "params": [ + { + "type": "duckdb_file_handle", + "name": "file_handle" + } + ], + "comment": { + "description": "Gets the size of the file.\n\n", + "param_comments": { + "file_handle": "The file handle to get the size of." + }, + "return_value": "The size of the file in bytes, or negative on error." + } + }, + { + "name": "duckdb_file_handle_seek", + "return_type": "duckdb_state", + "params": [ + { + "type": "duckdb_file_handle", + "name": "file_handle" + }, + { + "type": "int64_t", + "name": "position" + } + ], + "comment": { + "description": "Seeks to a specific position in the file.\n\n", + "param_comments": { + "file_handle": "The file handle to seek in.", + "offset": "The position to seek to." + }, + "return_value": "`DuckDBSuccess` on success or `DuckDBError` on failure. If unsuccessful, the error data can be retrieved using `duckdb_file_handle_error_data`." + } + }, + { + "name": "duckdb_file_handle_sync", + "return_type": "duckdb_state", + "params": [ + { + "type": "duckdb_file_handle", + "name": "file_handle" + } + ], + "comment": { + "description": "Synchronizes the file's state with the underlying storage.\n\n", + "param_comments": { + "file_handle": "The file handle to synchronize." + }, + "return_value": "`DuckDBSuccess` on success or `DuckDBError` on failure. If unsuccessful, the error data can be retrieved using `duckdb_file_handle_error_data`." + } + }, + { + "name": "duckdb_file_handle_close", + "return_type": "duckdb_state", + "params": [ + { + "type": "duckdb_file_handle", + "name": "file_handle" + } + ], + "comment": { + "description": "Closes the given file handle.\n\n", + "param_comments": { + "file_handle": "The file handle to close." + }, + "return_value": "`DuckDBSuccess` on success or `DuckDBError` on failure. If unsuccessful, the error data can be retrieved using `duckdb_file_handle_error_data`." + } + } + ] +} \ No newline at end of file diff --git a/src/include/duckdb/main/capi/header_generation/functions/open_connect.json b/src/include/duckdb/main/capi/header_generation/functions/open_connect.json index c466539f3a87..cebee589264e 100644 --- a/src/include/duckdb/main/capi/header_generation/functions/open_connect.json +++ b/src/include/duckdb/main/capi/header_generation/functions/open_connect.json @@ -183,11 +183,11 @@ } ], "comment": { - "description": "Get progress of the running query\n\n", + "description": "Get the progress of the running query.\n\n", "param_comments": { - "connection": "The working connection" + "connection": "The connection running the query." }, - "return_value": "-1 if no progress or a percentage of the progress" + "return_value": "The query progress type containing progress information." } }, { diff --git a/src/include/duckdb/main/capi/header_generation/functions/prepared_statements.json b/src/include/duckdb/main/capi/header_generation/functions/prepared_statements.json index 4aeeb17a405b..a2087ad3a356 100644 --- a/src/include/duckdb/main/capi/header_generation/functions/prepared_statements.json +++ b/src/include/duckdb/main/capi/header_generation/functions/prepared_statements.json @@ -172,6 +172,89 @@ }, "return_value": "duckdb_statement_type value or DUCKDB_STATEMENT_TYPE_INVALID" } + }, + { + "name": "duckdb_prepared_statement_column_count", + "return_type": "idx_t", + "params": [ + { + "type": "duckdb_prepared_statement", + "name": "prepared_statement" + } + ], + "comment": { + "description": "Returns the number of columns present in a the result of the prepared statement. If any of the column types are invalid, the result will be 1.\n\n", + "param_comments": { + "prepared_statement": "The prepared statement." + }, + "return_value": "The number of columns present in the result of the prepared statement." + } + }, + { + "name": "duckdb_prepared_statement_column_name", + "return_type": "const char *", + "params": [ + { + "type": "duckdb_prepared_statement", + "name": "prepared_statement" + }, + { + "type": "idx_t", + "name": "col_idx" + } + ], + "comment": { + "description": "Returns the name of the specified column of the result of the prepared_statement.\nThe returned string should be freed using `duckdb_free`.\n\nReturns `nullptr` if the column is out of range.\n\n", + "param_comments": { + "prepared_statement": "The prepared statement.", + "col_idx": "The column index." + }, + "return_value": "The column name of the specified column." + } + }, + { + "name": "duckdb_prepared_statement_column_logical_type", + "return_type": "duckdb_logical_type", + "params": [ + { + "type": "duckdb_prepared_statement", + "name": "prepared_statement" + }, + { + "type": "idx_t", + "name": "col_idx" + } + ], + "comment": { + "description": "Returns the column type of the specified column of the result of the prepared_statement.\n\nReturns `DUCKDB_TYPE_INVALID` if the column is out of range.\nThe return type of this call should be destroyed with `duckdb_destroy_logical_type`.\n\n", + "param_comments": { + "prepared_statement": "The prepared statement to fetch the column type from.", + "col_idx": "The column index." + }, + "return_value": "The logical type of the specified column." + } + }, + { + "name": "duckdb_prepared_statement_column_type", + "return_type": "duckdb_type", + "params": [ + { + "type": "duckdb_prepared_statement", + "name": "prepared_statement" + }, + { + "type": "idx_t", + "name": "col_idx" + } + ], + "comment": { + "description": "Returns the column type of the specified column of the result of the prepared_statement.\n\nReturns `DUCKDB_TYPE_INVALID` if the column is out of range.\n\n", + "param_comments": { + "prepared_statement": "The prepared statement to fetch the column type from.", + "col_idx": "The column index." + }, + "return_value": "The type of the specified column." + } } ] } \ No newline at end of file diff --git a/src/include/duckdb/main/capi/header_generation/functions/safe_fetch_functions.json b/src/include/duckdb/main/capi/header_generation/functions/safe_fetch_functions.json index c05ba2c9fcbc..c4d24d209095 100644 --- a/src/include/duckdb/main/capi/header_generation/functions/safe_fetch_functions.json +++ b/src/include/duckdb/main/capi/header_generation/functions/safe_fetch_functions.json @@ -1,7 +1,7 @@ { "group": "safe_fetch_functions", "deprecated": true, - "description": "// These functions will perform conversions if necessary.\n// On failure (e.g. if conversion cannot be performed or if the value is NULL) a default value is returned.\n// Note that these functions are slow since they perform bounds checking and conversion\n// For fast access of values prefer using `duckdb_result_get_chunk`", + "description": "// This function group is deprecated.\n// To access the values in a result, use `duckdb_fetch_chunk` repeatedly.\n// For each chunk, use the `duckdb_data_chunk` interface to access any columns and their values.\n\n", "entries": [ { "name": "duckdb_value_boolean", @@ -417,7 +417,7 @@ } ], "comment": { - "description": "**DEPRECATED**: Use duckdb_value_string instead. This function does not work correctly if the string contains null bytes.\n\n", + "description": "**DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n", "return_value": "The text value at the specified location as a null-terminated string, or nullptr if the value cannot be\nconverted. The result must be freed with `duckdb_free`." } }, @@ -439,7 +439,7 @@ } ], "comment": { - "description": "**DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\nNo support for nested types, and for other complex types.\nThe resulting field \"string.data\" must be freed with `duckdb_free.`\n\n", + "description": "**DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n", "return_value": "The string value at the specified location. Attempts to cast the result value to string." } }, @@ -461,7 +461,7 @@ } ], "comment": { - "description": "**DEPRECATED**: Use duckdb_value_string_internal instead. This function does not work correctly if the string contains\nnull bytes.\n\n", + "description": "**DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n", "return_value": "The char* value at the specified location. ONLY works on VARCHAR columns and does not auto-cast.\nIf the column is NOT a VARCHAR column this function will return NULL.\n\nThe result must NOT be freed." } }, @@ -483,7 +483,7 @@ } ], "comment": { - "description": "**DEPRECATED**: Use duckdb_value_string_internal instead. This function does not work correctly if the string contains\nnull bytes.\n", + "description": "**DEPRECATION NOTICE**: This method is scheduled for removal in a future release.\n\n", "return_value": "The char* value at the specified location. ONLY works on VARCHAR columns and does not auto-cast.\nIf the column is NOT a VARCHAR column this function will return NULL.\n\nThe result must NOT be freed." } }, diff --git a/src/include/duckdb/main/capi/header_generation/functions/table_description.json b/src/include/duckdb/main/capi/header_generation/functions/table_description.json index 2a5ac5038b28..f8219458ea03 100644 --- a/src/include/duckdb/main/capi/header_generation/functions/table_description.json +++ b/src/include/duckdb/main/capi/header_generation/functions/table_description.json @@ -131,6 +131,23 @@ "return_value": "`DuckDBSuccess` on success or `DuckDBError` on failure." } }, + { + "name": "duckdb_table_description_get_column_count", + "return_type": "idx_t", + "params": [ + { + "type": "duckdb_table_description", + "name": "table_description" + } + ], + "comment": { + "description": "Return the number of columns of the described table.\n\n", + "param_comments": { + "table_description": "The table_description to query." + }, + "return_value": "The column count." + } + }, { "name": "duckdb_table_description_get_column_name", "return_type": "char *", @@ -152,6 +169,28 @@ }, "return_value": "The column name." } + }, + { + "name": "duckdb_table_description_get_column_type", + "return_type": "duckdb_logical_type", + "params": [ + { + "type": "duckdb_table_description", + "name": "table_description" + }, + { + "type": "idx_t", + "name": "index" + } + ], + "comment": { + "description": "Obtain the column type at 'index'.\nThe return value must be destroyed with `duckdb_destroy_logical_type`.\n\n", + "param_comments": { + "table_description": "The table_description to query.", + "index": "The index of the column to query." + }, + "return_value": "The column type." + } } ] } \ No newline at end of file diff --git a/src/include/duckdb/main/capi/header_generation/functions/vector_interface.json b/src/include/duckdb/main/capi/header_generation/functions/vector_interface.json index 29520006403b..57c39c3ff1c4 100644 --- a/src/include/duckdb/main/capi/header_generation/functions/vector_interface.json +++ b/src/include/duckdb/main/capi/header_generation/functions/vector_interface.json @@ -212,12 +212,12 @@ } ], "comment": { - "description": "Sets the total size of the underlying child-vector of a list vector.\n\n", + "description": "Sets the size of the underlying child-vector of a list vector.\nNote that this does NOT reserve the memory in the child buffer,\nand that it is possible to set a size exceeding the capacity.\nTo set the capacity, use `duckdb_list_vector_reserve`.\n\n", "param_comments": { "vector": "The list vector.", "size": "The size of the child list." }, - "return_value": "The duckdb state. Returns DuckDBError if the vector is nullptr." + "return_value": "The duckdb state. Returns DuckDBError, if the vector is nullptr." } }, { @@ -234,12 +234,12 @@ } ], "comment": { - "description": "Sets the total capacity of the underlying child-vector of a list.\n\nAfter calling this method, you must call `duckdb_vector_get_validity` and `duckdb_vector_get_data` to obtain current\ndata and validity pointers\n\n", + "description": "Sets the capacity of the underlying child-vector of a list vector.\nWe increment to the next power of two, based on the required capacity.\nThus, the capacity might not match the size of the list (capacity >= size),\nwhich is set via `duckdb_list_vector_set_size`.\n\n", "param_comments": { "vector": "The list vector.", - "required_capacity": "the total capacity to reserve." + "required_capacity": "The child buffer capacity to reserve." }, - "return_value": "The duckdb state. Returns DuckDBError if the vector is nullptr." + "return_value": "The duckdb state. Returns DuckDBError, if the vector is nullptr." } }, { diff --git a/src/include/duckdb/main/capi/header_generation/header_base.hpp.template b/src/include/duckdb/main/capi/header_generation/header_base.hpp.template index f74da3359a04..34386a8c187b 100644 --- a/src/include/duckdb/main/capi/header_generation/header_base.hpp.template +++ b/src/include/duckdb/main/capi/header_generation/header_base.hpp.template @@ -29,11 +29,7 @@ #ifdef DUCKDB_STATIC_BUILD #define DUCKDB_EXTENSION_API #else -#ifdef DUCKDB_BUILD_LOADABLE_EXTENSION #define DUCKDB_EXTENSION_API __declspec(dllexport) -#else -#define DUCKDB_EXTENSION_API -#endif #endif #else #define DUCKDB_EXTENSION_API __attribute__((visibility("default"))) @@ -245,6 +241,20 @@ typedef enum duckdb_cast_mode { DUCKDB_CAST_TRY = 1 } duckdb_cast_mode; +typedef enum duckdb_file_flag { + DUCKDB_FILE_FLAG_INVALID = 0, + // Open the file with "read" capabilities. + DUCKDB_FILE_FLAG_READ = 1, + // Open the file with "write" capabilities. + DUCKDB_FILE_FLAG_WRITE = 2, + // Create a new file, or open if it already exists. + DUCKDB_FILE_FLAG_CREATE = 3, + // Create a new file, or fail if it already exists. + DUCKDB_FILE_FLAG_CREATE_NEW = 4, + // Open the file in "append" mode. + DUCKDB_FILE_FLAG_APPEND = 5, +} duckdb_file_flag; + //===--------------------------------------------------------------------===// // General type definitions //===--------------------------------------------------------------------===// @@ -759,6 +769,21 @@ typedef struct _duckdb_arrow_options { void *internal_ptr; } * duckdb_arrow_options; +//===--------------------------------------------------------------------===// +// Virtual File System Access +//===--------------------------------------------------------------------===// + +typedef struct _duckdb_file_open_options { + void *internal_ptr; +} * duckdb_file_open_options; + +typedef struct _duckdb_file_system { + void *internal_ptr; +} * duckdb_file_system; + +typedef struct _duckdb_file_handle { + void *internal_ptr; +} * duckdb_file_handle; //===--------------------------------------------------------------------===// // DuckDB extension access diff --git a/src/include/duckdb/main/capi/header_generation/schemata/function_group_schema.json b/src/include/duckdb/main/capi/header_generation/schemata/function_group_schema.json new file mode 100644 index 000000000000..8e5140ddcb01 --- /dev/null +++ b/src/include/duckdb/main/capi/header_generation/schemata/function_group_schema.json @@ -0,0 +1,111 @@ +{ + "type": "object", + "title": "DuckDB C API function group definition", + "description": "The schema of a C API function group definition. Function group definitions are the main component of the C API header generation.", + "required": ["group", "deprecated", "entries"], + "properties": { + "group": { + "type": "string", + "description": "The group name of this function group." + }, + "deprecated": { + "type": "boolean", + "description": "Marks the entire function group as deprecated, if true." + }, + "use_instead": { + "type": "string", + "description": "Pointers on what functionality to use, if the entire function group is marked as deprecated." + }, + "entries": { + "type": "array", + "description": "The array of function definitions in this group.", + "items": { + "type": "object", + "required": ["name", "return_type", "params", "comment"], + "properties": { + "name": { + "type": "string", + "description": "The function name." + }, + "return_type": { + "type": "string", + "description": "The return type of the function." + }, + "params": { + "type": "array", + "description": "The array of function parameters.", + "items": { + "type": "object", + "required": ["type", "name"], + "properties": { + "type": { + "type": "string", + "description": "The parameter type." + }, + "name": { + "type": "string", + "description": "The parameter name." + } + }, + "additionalProperties": false + } + }, + "comment": { + "type": "object", + "description": "The function documentation.", + "deprecated": { + "type": "boolean", + "description": "Marks the function as deprecated, if true." + }, + "outdated": { + "type": "boolean", + "description": "Marks the function as outdated, if true." + }, + "use_instead": { + "type": "string", + "description": "Pointers on what functionality to use, if the function is marked as deprecated or outdated." + }, + "required": ["description", "param_comments"], + "properties": { + "description": { + "type": "string", + "description": "The function description." + }, + "param_comments": { + "type": "object", + "description": "A comment on each function parameter.", + "patternProperties": { + ".*": { + "type": "string" + } + }, + "additionalProperties": false + }, + "param_destroy_functions": { + "type": "object", + "description": "Which parameters to destroy, and how.", + "patternProperties": { + ".*": { + "type": "string" + } + }, + "additionalProperties": false + }, + "return_value": { + "type": "string", + "description": "A comment on the return value." + }, + "return_value_destroy_function": { + "type": "string", + "function": "The function to destroy the return value with, if applicable." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/src/include/duckdb/main/capi/header_generation/schemata/unstable_function_list_schema.json b/src/include/duckdb/main/capi/header_generation/schemata/unstable_function_list_schema.json new file mode 100644 index 000000000000..bba15f34cf39 --- /dev/null +++ b/src/include/duckdb/main/capi/header_generation/schemata/unstable_function_list_schema.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "title": "DuckDB C API unstable function list definition", + "description": "The schema of a C API unstable function list.", + "required": ["version", "description", "entries"], + "properties": { + "version": { + "type": "string", + "description": "The version name for the corresponding #ifdef section in the header files." + }, + "description": { + "type": "string", + "description": "A description of the functions in the list." + }, + "entries": { + "type": "array", + "description": "The list of functions.", + "items": { + "type": "string" + } + } + } +} \ No newline at end of file diff --git a/src/include/duckdb/main/client_config.hpp b/src/include/duckdb/main/client_config.hpp index f9e673b19d46..284e63ce9573 100644 --- a/src/include/duckdb/main/client_config.hpp +++ b/src/include/duckdb/main/client_config.hpp @@ -121,7 +121,7 @@ struct ClientConfig { bool AnyVerification() const; - void SetUserVariable(const string &name, Value value); + void SetUserVariable(const String &name, Value value); bool GetUserVariable(const string &name, Value &result); void ResetUserVariable(const String &name); diff --git a/src/include/duckdb/main/client_context.hpp b/src/include/duckdb/main/client_context.hpp index 5291481be759..21f56ccf94b0 100644 --- a/src/include/duckdb/main/client_context.hpp +++ b/src/include/duckdb/main/client_context.hpp @@ -304,6 +304,8 @@ class ClientContext : public enable_shared_from_this { CreatePreparedStatementInternal(ClientContextLock &lock, const string &query, unique_ptr statement, optional_ptr> values); + SettingLookupResult TryGetCurrentSettingInternal(const string &key, Value &result) const; + private: //! Lock on using the ClientContext in parallel mutex context_lock; @@ -335,6 +337,8 @@ class QueryContext { } QueryContext(optional_ptr context) : context(context) { // NOLINT: allow implicit construction } + QueryContext(ClientContext &context) : context(&context) { // NOLINT: allow implicit construction + } public: bool Valid() const { diff --git a/src/include/duckdb/main/config.hpp b/src/include/duckdb/main/config.hpp index 2f67b7c40fe5..c038bea16cd3 100644 --- a/src/include/duckdb/main/config.hpp +++ b/src/include/duckdb/main/config.hpp @@ -55,6 +55,7 @@ class SecretManager; class CompressionInfo; class EncryptionUtil; class HTTPUtil; +class DatabaseFilePathManager; struct CompressionFunctionSet; struct DatabaseCacheEntry; @@ -109,6 +110,8 @@ struct DBConfigOptions { #else bool autoinstall_known_extensions = false; #endif + //! Setting for the parser override registered by extensions. Allowed options: "default, "fallback", "strict" + string allow_parser_override_extension = "default"; //! Override for the default extension repository string custom_extension_repo = ""; //! Override for the default autoload extension repository @@ -160,8 +163,6 @@ struct DBConfigOptions { uint64_t zstd_min_string_length = 4096; //! Force a specific compression method to be used when checkpointing (if available) CompressionType force_compression = CompressionType::COMPRESSION_AUTO; - //! The set of disabled compression methods (default empty) - set disabled_compression_methods; //! Force a specific bitpacking mode to be used when using the bitpacking compression method BitpackingMode force_bitpacking_mode = BitpackingMode::AUTO; //! Database configuration variables as controlled by SET @@ -194,8 +195,6 @@ struct DBConfigOptions { string duckdb_api; //! Metadata from DuckDB callers string custom_user_agent; - //! By default, WAL is encrypted for encrypted databases - bool wal_encryption = true; //! Encrypt the temp files bool temp_file_encryption = false; //! The default block allocation size for new duckdb database files (new as-in, they do not yet exist). @@ -273,6 +272,8 @@ struct DBConfig { shared_ptr http_util; //! Reference to the database cache entry (if any) shared_ptr db_cache_entry; + //! Reference to the database file path manager + shared_ptr path_manager; public: DUCKDB_API static DBConfig &GetConfig(ClientContext &context); @@ -301,7 +302,7 @@ struct DBConfig { DUCKDB_API void SetOptionByName(const string &name, const Value &value); DUCKDB_API void SetOptionsByName(const case_insensitive_map_t &values); DUCKDB_API void ResetOption(optional_ptr db, const ConfigurationOption &option); - DUCKDB_API void SetOption(const string &name, Value value); + DUCKDB_API void SetOption(const String &name, Value value); DUCKDB_API void ResetOption(const String &name); DUCKDB_API void ResetGenericOption(const String &name); static LogicalType ParseLogicalType(const string &type); @@ -315,6 +316,10 @@ struct DBConfig { //! Returns the compression function matching the compression and physical type. DUCKDB_API optional_ptr GetCompressionFunction(CompressionType type, const PhysicalType physical_type); + //! Sets the disabled compression methods + DUCKDB_API void SetDisabledCompressionMethods(const vector &disabled_compression_methods); + //! Returns a list of disabled compression methods + DUCKDB_API vector GetDisabledCompressionMethods() const; //! Returns the encode function matching the encoding name. DUCKDB_API optional_ptr GetEncodeFunction(const string &name) const; diff --git a/src/include/duckdb/main/connection.hpp b/src/include/duckdb/main/connection.hpp index c27d84d21b80..72c61209c6a2 100644 --- a/src/include/duckdb/main/connection.hpp +++ b/src/include/duckdb/main/connection.hpp @@ -50,7 +50,6 @@ class Connection { DUCKDB_API ~Connection(); shared_ptr context; - warning_callback_t warning_cb; public: //! Returns query profiling information for the current query diff --git a/src/include/duckdb/main/connection_manager.hpp b/src/include/duckdb/main/connection_manager.hpp index 7fa5c66b5276..1c647ce02085 100644 --- a/src/include/duckdb/main/connection_manager.hpp +++ b/src/include/duckdb/main/connection_manager.hpp @@ -40,7 +40,6 @@ class ConnectionManager { mutex connections_lock; reference_map_t> connections; atomic connection_count; - atomic current_connection_id; }; diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp index 11936d5f74d7..2486d1e0e84d 100644 --- a/src/include/duckdb/main/database.hpp +++ b/src/include/duckdb/main/database.hpp @@ -115,14 +115,14 @@ class DuckDB { void LoadStaticExtension() { T extension; auto &manager = ExtensionManager::Get(*instance); - auto info = manager.BeginLoad(extension.Name()); - if (!info) { + auto load_info = manager.BeginLoad(extension.Name()); + if (!load_info) { // already loaded - return return; } // Instantiate a new loader - ExtensionLoader loader(*instance, extension.Name()); + ExtensionLoader loader(*load_info); // Call the Load method of the extension extension.Load(loader); @@ -133,7 +133,7 @@ class DuckDB { ExtensionInstallInfo install_info; install_info.mode = ExtensionInstallMode::STATICALLY_LINKED; install_info.version = extension.Version(); - info->FinishLoad(install_info); + load_info->FinishLoad(install_info); } DUCKDB_API FileSystem &GetFileSystem(); diff --git a/src/include/duckdb/main/database_file_path_manager.hpp b/src/include/duckdb/main/database_file_path_manager.hpp new file mode 100644 index 000000000000..3af2f1873bff --- /dev/null +++ b/src/include/duckdb/main/database_file_path_manager.hpp @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/main/database_file_path_manager.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/common.hpp" +#include "duckdb/common/mutex.hpp" +#include "duckdb/common/case_insensitive_map.hpp" +#include "duckdb/common/enums/on_create_conflict.hpp" +#include "duckdb/common/enums/access_mode.hpp" +#include "duckdb/common/reference_map.hpp" + +namespace duckdb { +struct AttachInfo; +struct AttachOptions; +class DatabaseManager; + +enum class InsertDatabasePathResult { SUCCESS, ALREADY_EXISTS }; + +struct DatabasePathInfo { + DatabasePathInfo(DatabaseManager &manager, string name_p, AccessMode access_mode); + + string name; + AccessMode access_mode; + reference_set_t attached_databases; + idx_t reference_count = 1; +}; + +//! The DatabaseFilePathManager is used to ensure we only ever open a single database file once +class DatabaseFilePathManager { +public: + idx_t ApproxDatabaseCount() const; + InsertDatabasePathResult InsertDatabasePath(DatabaseManager &manager, const string &path, const string &name, + OnCreateConflict on_conflict, AttachOptions &options); + //! Erase a database path - indicating we are done with using it + void EraseDatabasePath(const string &path); + //! Called when a database is detached, but before it is fully finished being used + void DetachDatabase(DatabaseManager &manager, const string &path); + +private: + //! The lock to add entries to the db_paths map + mutable mutex db_paths_lock; + //! A set containing all attached database path + //! This allows to attach many databases efficiently, and to avoid attaching the + //! same file path twice + case_insensitive_map_t db_paths; +}; + +} // namespace duckdb diff --git a/src/include/duckdb/main/database_manager.hpp b/src/include/duckdb/main/database_manager.hpp index 0a45e75dc5bc..b516e858f321 100644 --- a/src/include/duckdb/main/database_manager.hpp +++ b/src/include/duckdb/main/database_manager.hpp @@ -17,6 +17,7 @@ #include "duckdb/common/optional_ptr.hpp" #include "duckdb/main/config.hpp" #include "duckdb/parser/parsed_data/attach_info.hpp" +#include "duckdb/main/database_file_path_manager.hpp" namespace duckdb { class AttachedDatabase; @@ -27,6 +28,7 @@ class ClientContext; class DatabaseInstance; class TaskScheduler; struct AttachOptions; +struct AlterInfo; //! The DatabaseManager is a class that sits at the root of all attached databases class DatabaseManager { @@ -47,13 +49,14 @@ class DatabaseManager { void FinalizeStartup(); //! Get an attached database by its name optional_ptr GetDatabase(ClientContext &context, const string &name); + shared_ptr GetDatabase(const string &name); //! Attach a new database shared_ptr AttachDatabase(ClientContext &context, AttachInfo &info, AttachOptions &options); - optional_ptr FinalizeAttach(ClientContext &context, AttachInfo &info, - shared_ptr database); //! Detach an existing database void DetachDatabase(ClientContext &context, const string &name, OnEntryNotFound if_not_found); + //! Alter operation dispatcher + void Alter(ClientContext &context, AlterInfo &info); //! Rollback the attach of a database shared_ptr DetachInternal(const string &name); //! Returns a reference to the system catalog @@ -63,11 +66,7 @@ class DatabaseManager { void SetDefaultDatabase(ClientContext &context, const string &new_value); //! Inserts a path to name mapping to the database paths map - void InsertDatabasePath(const string &path, const string &name); - //! Erases a path from the database paths map - void EraseDatabasePath(const string &path); - //! Check if a path has a conflict - void CheckPathConflict(const string &path, const string &name); + InsertDatabasePathResult InsertDatabasePath(const AttachInfo &info, AttachOptions &options); //! Returns the database type. This might require checking the header of the file, in which case the file handle is //! necessary. We can only grab the file handle, if it is not yet held, even for uncommitted changes. Thus, we have @@ -80,10 +79,7 @@ class DatabaseManager { //! Scans the catalog set and returns each committed database entry vector> GetDatabases(); //! Returns the approximate count of attached databases. - idx_t ApproxDatabaseCount() { - lock_guard path_lock(db_paths_lock); - return db_paths_to_name.size(); - } + idx_t ApproxDatabaseCount(); //! Removes all databases from the catalog set. This is necessary for the database instance's destructor, //! as the database manager has to be alive when destroying the catalog set objects. void ResetDatabases(unique_ptr &scheduler); @@ -109,6 +105,12 @@ class DatabaseManager { //! Gets a list of all attached database paths vector GetAttachedDatabasePaths(); + shared_ptr GetDatabaseInternal(const lock_guard &, const string &name); + +private: + optional_ptr FinalizeAttach(ClientContext &context, AttachInfo &info, + shared_ptr database); + private: //! The system database is a special database that holds system entries (e.g. functions) shared_ptr system; @@ -124,13 +126,13 @@ class DatabaseManager { atomic current_transaction_id; //! The current default database string default_database; + //! Manager for ensuring we never open the same database file twice in the same program + shared_ptr path_manager; - //! The lock to add entries to the database path map - mutex db_paths_lock; - //! A set containing all attached database path - //! This allows to attach many databases efficiently, and to avoid attaching the - //! same file path twice - case_insensitive_map_t db_paths_to_name; +private: + //! Rename an existing database + void RenameDatabase(ClientContext &context, const string &old_name, const string &new_name, + OnEntryNotFound if_not_found); }; } // namespace duckdb diff --git a/src/include/duckdb/main/db_instance_cache.hpp b/src/include/duckdb/main/db_instance_cache.hpp index ebf00b96dac0..8fa226b2d10e 100644 --- a/src/include/duckdb/main/db_instance_cache.hpp +++ b/src/include/duckdb/main/db_instance_cache.hpp @@ -15,6 +15,7 @@ namespace duckdb { class DBInstanceCache; +class DatabaseFilePathManager; struct DatabaseCacheEntry { DatabaseCacheEntry(); @@ -27,8 +28,8 @@ struct DatabaseCacheEntry { class DBInstanceCache { public: - DBInstanceCache() { - } + DBInstanceCache(); + ~DBInstanceCache(); //! Gets a DB Instance from the cache if already exists (Fails if the configurations do not match) shared_ptr GetInstance(const string &database, const DBConfig &config_dict); @@ -42,6 +43,7 @@ class DBInstanceCache { const std::function &on_create = nullptr); private: + shared_ptr path_manager; //! A map with the cached instances unordered_map> db_instances; diff --git a/src/include/duckdb/main/error_manager.hpp b/src/include/duckdb/main/error_manager.hpp index aaedffd4b966..065f6399a9a3 100644 --- a/src/include/duckdb/main/error_manager.hpp +++ b/src/include/duckdb/main/error_manager.hpp @@ -34,38 +34,39 @@ enum class ErrorType : uint16_t { class ErrorManager { public: template - string FormatException(ErrorType error_type, ARGS... params) { + string FormatException(ErrorType error_type, ARGS &&...params) { vector values; - return FormatExceptionRecursive(error_type, values, params...); + return FormatExceptionRecursive(error_type, values, std::forward(params)...); } DUCKDB_API string FormatExceptionRecursive(ErrorType error_type, vector &values); template string FormatExceptionRecursive(ErrorType error_type, vector &values, T param, - ARGS... params) { + ARGS &&...params) { values.push_back(ExceptionFormatValue::CreateFormatValue(param)); - return FormatExceptionRecursive(error_type, values, params...); + return FormatExceptionRecursive(error_type, values, std::forward(params)...); } template - static string FormatException(ClientContext &context, ErrorType error_type, ARGS... params) { - return Get(context).FormatException(error_type, params...); + static string FormatException(ClientContext &context, ErrorType error_type, ARGS &&...params) { + return Get(context).FormatException(error_type, std::forward(params)...); } DUCKDB_API static InvalidInputException InvalidUnicodeError(const String &input, const string &context); DUCKDB_API static FatalException InvalidatedDatabase(ClientContext &context, const string &invalidated_msg); + DUCKDB_API static TransactionException InvalidatedTransaction(ClientContext &context); //! Adds a custom error for a specific error type void AddCustomError(ErrorType type, string new_error); DUCKDB_API static ErrorManager &Get(ClientContext &context); + DUCKDB_API static ErrorManager &Get(DatabaseInstance &context); private: map custom_errors; }; - } // namespace duckdb diff --git a/src/include/duckdb/main/extension_entries.hpp b/src/include/duckdb/main/extension_entries.hpp index 7885d74f1943..937217fdbf97 100644 --- a/src/include/duckdb/main/extension_entries.hpp +++ b/src/include/duckdb/main/extension_entries.hpp @@ -69,8 +69,10 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"approx_top_k", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"arg_max", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"arg_max_null", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, + {"arg_max_nulls_last", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"arg_min", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"arg_min_null", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, + {"arg_min_nulls_last", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"argmax", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"argmin", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"array_agg", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, @@ -154,11 +156,16 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"duckdb_proj_version", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"ducklake_add_data_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, {"ducklake_cleanup_old_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, + {"ducklake_current_snapshot", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, + {"ducklake_delete_orphaned_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, {"ducklake_expire_snapshots", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, {"ducklake_flush_inlined_data", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, + {"ducklake_last_committed_snapshot", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, {"ducklake_list_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, {"ducklake_merge_adjacent_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, {"ducklake_options", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, + {"ducklake_rewrite_data_files", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, + {"ducklake_set_commit_message", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, {"ducklake_set_option", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, {"ducklake_snapshots", "ducklake", CatalogType::TABLE_FUNCTION_ENTRY}, {"ducklake_table_changes", "ducklake", CatalogType::TABLE_MACRO_ENTRY}, @@ -222,6 +229,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"iceberg_metadata", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY}, {"iceberg_scan", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY}, {"iceberg_snapshots", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY}, + {"iceberg_to_ducklake", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY}, {"icu_calendar_names", "icu", CatalogType::TABLE_FUNCTION_ENTRY}, {"icu_collate_af", "icu", CatalogType::SCALAR_FUNCTION_ENTRY}, {"icu_collate_am", "icu", CatalogType::SCALAR_FUNCTION_ENTRY}, @@ -552,6 +560,8 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"st_area_spheroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_asgeojson", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_ashexwkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, + {"st_asmvt", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY}, + {"st_asmvtgeom", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_assvg", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_astext", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_aswkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, @@ -771,6 +781,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"var_pop", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"var_samp", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"variance", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY}, + {"variant_to_parquet_variant", "parquet", CatalogType::SCALAR_FUNCTION_ENTRY}, {"vector_type", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY}, {"version", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY}, {"vss_join", "vss", CatalogType::TABLE_MACRO_ENTRY}, @@ -999,6 +1010,7 @@ static constexpr ExtensionFunctionOverloadEntry EXTENSION_FUNCTION_OVERLOADS[] = }; // END_OF_EXTENSION_FUNCTION_OVERLOADS static constexpr ExtensionEntry EXTENSION_SETTINGS[] = { + {"auto_fallback_to_full_download", "httpfs"}, {"azure_account_name", "azure"}, {"azure_context_caching", "azure"}, {"azure_credential_chain", "azure"}, @@ -1019,6 +1031,7 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = { {"ducklake_max_retry_count", "ducklake"}, {"ducklake_retry_backoff", "ducklake"}, {"ducklake_retry_wait_ms", "ducklake"}, + {"enable_curl_server_cert_verification", "httpfs"}, {"enable_geoparquet_conversion", "parquet"}, {"enable_server_cert_verification", "httpfs"}, {"force_download", "httpfs"}, @@ -1034,6 +1047,8 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = { {"mysql_bit1_as_boolean", "mysql_scanner"}, {"mysql_debug_show_queries", "mysql_scanner"}, {"mysql_experimental_filter_pushdown", "mysql_scanner"}, + {"mysql_session_time_zone", "mysql_scanner"}, + {"mysql_time_as_time", "mysql_scanner"}, {"mysql_tinyint1_as_boolean", "mysql_scanner"}, {"parquet_metadata_cache", "parquet"}, {"pg_array_as_varchar", "postgres_scanner"}, @@ -1066,8 +1081,8 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = { {"ui_local_port", "ui"}, {"ui_polling_interval", "ui"}, {"ui_remote_url", "ui"}, + {"unsafe_disable_etag_checks", "httpfs"}, {"unsafe_enable_version_guessing", "iceberg"}, - {"variant_legacy_encoding", "parquet"}, }; // END_OF_EXTENSION_SETTINGS static constexpr ExtensionEntry EXTENSION_SECRET_TYPES[] = { diff --git a/src/include/duckdb/main/extension_helper.hpp b/src/include/duckdb/main/extension_helper.hpp index 1ce98e709485..e1037acea1f4 100644 --- a/src/include/duckdb/main/extension_helper.hpp +++ b/src/include/duckdb/main/extension_helper.hpp @@ -250,6 +250,8 @@ class ExtensionHelper { ExtensionInitResult &result, string &error); //! Version tags occur with and without 'v', tag in extension path is always with 'v' static const string NormalizeVersionTag(const string &version_tag); + static void LoadExternalExtensionInternal(DatabaseInstance &db, FileSystem &fs, const string &extension, + ExtensionActiveLoad &info); private: static ExtensionLoadResult LoadExtensionInternal(DuckDB &db, const std::string &extension, bool initial_load); diff --git a/src/include/duckdb/main/extension_manager.hpp b/src/include/duckdb/main/extension_manager.hpp index 19e1776a0a66..5f0ed3d3dbbf 100644 --- a/src/include/duckdb/main/extension_manager.hpp +++ b/src/include/duckdb/main/extension_manager.hpp @@ -12,6 +12,7 @@ #include "duckdb/main/extension_install_info.hpp" namespace duckdb { +class ErrorData; class ExtensionInfo { public: @@ -34,6 +35,7 @@ class ExtensionActiveLoad { public: void FinishLoad(ExtensionInstallInfo &install_info); + void LoadFail(const ErrorData &error); }; class ExtensionManager { diff --git a/src/include/duckdb/main/profiling_info.hpp b/src/include/duckdb/main/profiling_info.hpp index 904f0205dbe8..e54a13618fd5 100644 --- a/src/include/duckdb/main/profiling_info.hpp +++ b/src/include/duckdb/main/profiling_info.hpp @@ -32,9 +32,6 @@ class ProfilingInfo { profiler_settings_t expanded_settings; //! Contains all enabled metrics. profiler_metrics_t metrics; - //! Additional metrics. - // FIXME: move to metrics. - InsertionOrderPreservingMap extra_info; public: ProfilingInfo() = default; @@ -44,8 +41,8 @@ class ProfilingInfo { public: static profiler_settings_t DefaultSettings(); - static profiler_settings_t DefaultRootSettings(); - static profiler_settings_t DefaultOperatorSettings(); + static profiler_settings_t RootScopeSettings(); + static profiler_settings_t OperatorScopeSettings(); public: void ResetMetrics(); @@ -102,6 +99,7 @@ class ProfilingInfo { return MaxValue(old_value, new_value); }); } + template void MetricMax(const MetricsType type, const METRIC_TYPE &value) { auto new_value = Value::CreateValue(value); @@ -109,4 +107,19 @@ class ProfilingInfo { } }; +// Specialization for InsertionOrderPreservingMap +template <> +inline InsertionOrderPreservingMap +ProfilingInfo::GetMetricValue>(const MetricsType type) const { + auto val = metrics.at(type); + InsertionOrderPreservingMap result; + auto children = MapValue::GetChildren(val); + for (auto &child : children) { + auto struct_children = StructValue::GetChildren(child); + auto key = struct_children[0].GetValue(); + auto value = struct_children[1].GetValue(); + result.insert(key, value); + } + return result; +} } // namespace duckdb diff --git a/src/include/duckdb/main/query_profiler.hpp b/src/include/duckdb/main/query_profiler.hpp index 0f7b8812d812..44df56025c24 100644 --- a/src/include/duckdb/main/query_profiler.hpp +++ b/src/include/duckdb/main/query_profiler.hpp @@ -94,7 +94,6 @@ class OperatorProfiler { DUCKDB_API void Flush(const PhysicalOperator &phys_op); DUCKDB_API OperatorInformation &GetOperatorInfo(const PhysicalOperator &phys_op); DUCKDB_API bool OperatorInfoIsInitialized(const PhysicalOperator &phys_op); - DUCKDB_API void AddExtraInfo(InsertionOrderPreservingMap extra_info); public: ClientContext &context; @@ -117,15 +116,35 @@ class OperatorProfiler { struct QueryMetrics { QueryMetrics() : total_bytes_read(0), total_bytes_written(0) {}; + //! Reset the query metrics. + void Reset() { + query = ""; + latency.Reset(); + waiting_to_attach_latency.Reset(); + attach_load_storage_latency.Reset(); + attach_replay_wal_latency.Reset(); + checkpoint_latency.Reset(); + total_bytes_read = 0; + total_bytes_written = 0; + } + ProfilingInfo query_global_info; - //! The SQL string of the query + //! The SQL string of the query. string query; - //! The timer used to time the excution time of the entire query + //! The timer of the execution of the entire query. Profiler latency; - //! The total bytes read by the file system + //! The timer of the delay when waiting to ATTACH a file. + Profiler waiting_to_attach_latency; + //! The timer for loading from storage. + Profiler attach_load_storage_latency; + //! The timer for replaying the WAL file. + Profiler attach_replay_wal_latency; + //! The timer for running checkpoints. + Profiler checkpoint_latency; + //! The total bytes read by the file system. atomic total_bytes_read; - //! The total bytes written by the file system + //! The total bytes written by the file system. atomic total_bytes_written; }; @@ -138,9 +157,6 @@ class QueryProfiler { DUCKDB_API explicit QueryProfiler(ClientContext &context); public: - //! Propagate save_location, enabled, detailed_enabled and automatic_print_format. - void Propagate(QueryProfiler &qp); - DUCKDB_API bool IsEnabled() const; DUCKDB_API bool IsDetailedEnabled() const; DUCKDB_API ProfilerPrintFormat GetPrintFormat(ExplainFormat format = ExplainFormat::DEFAULT) const; @@ -159,6 +175,10 @@ class QueryProfiler { //! Adds nr_bytes bytes to the total bytes written. DUCKDB_API void AddBytesWritten(const idx_t nr_bytes); + //! Start/End a timer for a specific metric type. + DUCKDB_API void StartTimer(MetricsType type); + DUCKDB_API void EndTimer(MetricsType type); + DUCKDB_API void StartExplainAnalyze(); //! Adds the timings gathered by an OperatorProfiler to this query profiler @@ -180,7 +200,8 @@ class QueryProfiler { DUCKDB_API string ToString(ExplainFormat format = ExplainFormat::DEFAULT) const; DUCKDB_API string ToString(ProfilerPrintFormat format) const; - static InsertionOrderPreservingMap JSONSanitize(const InsertionOrderPreservingMap &input); + // Sanitize a Value::MAP + static Value JSONSanitize(const Value &input); static string JSONSanitize(const string &text); static string DrawPadded(const string &str, idx_t width); DUCKDB_API string ToJSON() const; diff --git a/src/include/duckdb/main/relation.hpp b/src/include/duckdb/main/relation.hpp index 37b137aae81b..bc383ffe0467 100644 --- a/src/include/duckdb/main/relation.hpp +++ b/src/include/duckdb/main/relation.hpp @@ -49,7 +49,7 @@ class RelationContextWrapper : public ClientContextWrapper { explicit RelationContextWrapper(const ClientContextWrapper &context) : ClientContextWrapper(context) {}; void TryBindRelation(Relation &relation, vector &columns) override { - GetContext()->InternalTryBindRelation(relation, columns); + GetContext()->TryBindRelation(relation, columns); } private: @@ -78,7 +78,8 @@ class Relation : public enable_shared_from_this { public: DUCKDB_API virtual const vector &Columns() = 0; - DUCKDB_API virtual unique_ptr GetQueryNode(); + DUCKDB_API virtual unique_ptr GetQueryNode() = 0; + DUCKDB_API virtual string GetQuery(); DUCKDB_API virtual BoundStatement Bind(Binder &binder); DUCKDB_API virtual string GetAlias(); diff --git a/src/include/duckdb/main/relation/create_table_relation.hpp b/src/include/duckdb/main/relation/create_table_relation.hpp index 7d5462941bfd..8df59b8d251e 100644 --- a/src/include/duckdb/main/relation/create_table_relation.hpp +++ b/src/include/duckdb/main/relation/create_table_relation.hpp @@ -26,6 +26,8 @@ class CreateTableRelation : public Relation { public: BoundStatement Bind(Binder &binder) override; + unique_ptr GetQueryNode() override; + string GetQuery() override; const vector &Columns() override; string ToString(idx_t depth) override; bool IsReadOnly() override { diff --git a/src/include/duckdb/main/relation/create_view_relation.hpp b/src/include/duckdb/main/relation/create_view_relation.hpp index cb826a86cd33..aa09b0def99e 100644 --- a/src/include/duckdb/main/relation/create_view_relation.hpp +++ b/src/include/duckdb/main/relation/create_view_relation.hpp @@ -26,6 +26,8 @@ class CreateViewRelation : public Relation { public: BoundStatement Bind(Binder &binder) override; + unique_ptr GetQueryNode() override; + string GetQuery() override; const vector &Columns() override; string ToString(idx_t depth) override; bool IsReadOnly() override { diff --git a/src/include/duckdb/main/relation/delete_relation.hpp b/src/include/duckdb/main/relation/delete_relation.hpp index c07445ba4e2f..0c25c6576401 100644 --- a/src/include/duckdb/main/relation/delete_relation.hpp +++ b/src/include/duckdb/main/relation/delete_relation.hpp @@ -26,6 +26,8 @@ class DeleteRelation : public Relation { public: BoundStatement Bind(Binder &binder) override; + unique_ptr GetQueryNode() override; + string GetQuery() override; const vector &Columns() override; string ToString(idx_t depth) override; bool IsReadOnly() override { diff --git a/src/include/duckdb/main/relation/explain_relation.hpp b/src/include/duckdb/main/relation/explain_relation.hpp index 888583b2b6e4..96be08d8ff65 100644 --- a/src/include/duckdb/main/relation/explain_relation.hpp +++ b/src/include/duckdb/main/relation/explain_relation.hpp @@ -24,6 +24,8 @@ class ExplainRelation : public Relation { public: BoundStatement Bind(Binder &binder) override; + unique_ptr GetQueryNode() override; + string GetQuery() override; const vector &Columns() override; string ToString(idx_t depth) override; bool IsReadOnly() override { diff --git a/src/include/duckdb/main/relation/insert_relation.hpp b/src/include/duckdb/main/relation/insert_relation.hpp index 3695cde7b007..fccb0ae929b3 100644 --- a/src/include/duckdb/main/relation/insert_relation.hpp +++ b/src/include/duckdb/main/relation/insert_relation.hpp @@ -23,6 +23,8 @@ class InsertRelation : public Relation { public: BoundStatement Bind(Binder &binder) override; + unique_ptr GetQueryNode() override; + string GetQuery() override; const vector &Columns() override; string ToString(idx_t depth) override; bool IsReadOnly() override { diff --git a/src/include/duckdb/main/relation/query_relation.hpp b/src/include/duckdb/main/relation/query_relation.hpp index bdb035652ccf..b1be001b9fe6 100644 --- a/src/include/duckdb/main/relation/query_relation.hpp +++ b/src/include/duckdb/main/relation/query_relation.hpp @@ -28,6 +28,7 @@ class QueryRelation : public Relation { public: static unique_ptr ParseStatement(ClientContext &context, const string &query, const string &error); unique_ptr GetQueryNode() override; + string GetQuery() override; unique_ptr GetTableRef() override; BoundStatement Bind(Binder &binder) override; diff --git a/src/include/duckdb/main/relation/update_relation.hpp b/src/include/duckdb/main/relation/update_relation.hpp index 58ad203b2ef7..91eac246e19d 100644 --- a/src/include/duckdb/main/relation/update_relation.hpp +++ b/src/include/duckdb/main/relation/update_relation.hpp @@ -29,6 +29,8 @@ class UpdateRelation : public Relation { public: BoundStatement Bind(Binder &binder) override; + unique_ptr GetQueryNode() override; + string GetQuery() override; const vector &Columns() override; string ToString(idx_t depth) override; bool IsReadOnly() override { diff --git a/src/include/duckdb/main/relation/write_csv_relation.hpp b/src/include/duckdb/main/relation/write_csv_relation.hpp index 99d2ebe8e5b0..cf0853ff3968 100644 --- a/src/include/duckdb/main/relation/write_csv_relation.hpp +++ b/src/include/duckdb/main/relation/write_csv_relation.hpp @@ -23,6 +23,8 @@ class WriteCSVRelation : public Relation { public: BoundStatement Bind(Binder &binder) override; + unique_ptr GetQueryNode() override; + string GetQuery() override; const vector &Columns() override; string ToString(idx_t depth) override; bool IsReadOnly() override { diff --git a/src/include/duckdb/main/relation/write_parquet_relation.hpp b/src/include/duckdb/main/relation/write_parquet_relation.hpp index d32089212981..138eee7c7e4b 100644 --- a/src/include/duckdb/main/relation/write_parquet_relation.hpp +++ b/src/include/duckdb/main/relation/write_parquet_relation.hpp @@ -24,6 +24,8 @@ class WriteParquetRelation : public Relation { public: BoundStatement Bind(Binder &binder) override; + unique_ptr GetQueryNode() override; + string GetQuery() override; const vector &Columns() override; string ToString(idx_t depth) override; bool IsReadOnly() override { diff --git a/src/include/duckdb/main/secret/secret.hpp b/src/include/duckdb/main/secret/secret.hpp index ed80344136c6..fd8a1b241518 100644 --- a/src/include/duckdb/main/secret/secret.hpp +++ b/src/include/duckdb/main/secret/secret.hpp @@ -296,7 +296,9 @@ class KeyValueSecretReader { Value result; auto lookup_result = TryGetSecretKeyOrSetting(secret_key, setting_name, result); if (lookup_result) { - value_out = result.GetValue(); + if (!result.IsNull()) { + value_out = result.GetValue(); + } } return lookup_result; } diff --git a/src/include/duckdb/main/settings.hpp b/src/include/duckdb/main/settings.hpp index 2627ce5935ad..6817f8e869c2 100644 --- a/src/include/duckdb/main/settings.hpp +++ b/src/include/duckdb/main/settings.hpp @@ -18,6 +18,7 @@ #include "duckdb/common/enums/output_type.hpp" #include "duckdb/common/enums/thread_pin_mode.hpp" #include "duckdb/common/enums/arrow_format_version.hpp" +#include "duckdb/common/enums/storage_block_prefetch.hpp" namespace duckdb { @@ -94,6 +95,18 @@ struct AllowExtensionsMetadataMismatchSetting { static constexpr SetScope DefaultScope = SetScope::GLOBAL; }; +struct AllowParserOverrideExtensionSetting { + using RETURN_TYPE = string; + static constexpr const char *Name = "allow_parser_override_extension"; + static constexpr const char *Description = "Allow extensions to override the current parser"; + static constexpr const char *InputType = "VARCHAR"; + static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value ¶meter); + static void ResetGlobal(DatabaseInstance *db, DBConfig &config); + static bool OnGlobalSet(DatabaseInstance *db, DBConfig &config, const Value &input); + static bool OnGlobalReset(DatabaseInstance *db, DBConfig &config); + static Value GetSetting(const ClientContext &context); +}; + struct AllowPersistentSecretsSetting { using RETURN_TYPE = bool; static constexpr const char *Name = "allow_persistent_secrets"; @@ -639,6 +652,15 @@ struct ErrorsAsJSONSetting { static Value GetSetting(const ClientContext &context); }; +struct ExperimentalMetadataReuseSetting { + using RETURN_TYPE = bool; + static constexpr const char *Name = "experimental_metadata_reuse"; + static constexpr const char *Description = "EXPERIMENTAL: Re-use row group and table metadata when checkpointing."; + static constexpr const char *InputType = "BOOLEAN"; + static constexpr const char *DefaultValue = "true"; + static constexpr SetScope DefaultScope = SetScope::GLOBAL; +}; + struct ExplainOutputSetting { using RETURN_TYPE = ExplainOutputType; static constexpr const char *Name = "explain_output"; @@ -1172,6 +1194,16 @@ struct SecretDirectorySetting { static Value GetSetting(const ClientContext &context); }; +struct StorageBlockPrefetchSetting { + using RETURN_TYPE = StorageBlockPrefetch; + static constexpr const char *Name = "storage_block_prefetch"; + static constexpr const char *Description = "In which scenarios to use storage block prefetching"; + static constexpr const char *InputType = "VARCHAR"; + static constexpr const char *DefaultValue = "REMOTE_ONLY"; + static constexpr SetScope DefaultScope = SetScope::GLOBAL; + static void OnSet(SettingCallbackInfo &info, Value &input); +}; + struct StorageCompatibilityVersionSetting { using RETURN_TYPE = string; static constexpr const char *Name = "storage_compatibility_version"; @@ -1233,6 +1265,16 @@ struct UsernameSetting { static Value GetSetting(const ClientContext &context); }; +struct WriteBufferRowGroupCountSetting { + using RETURN_TYPE = idx_t; + static constexpr const char *Name = "write_buffer_row_group_count"; + static constexpr const char *Description = "The amount of row groups to buffer in bulk ingestion prior to flushing " + "them together. Reducing this setting can reduce memory consumption."; + static constexpr const char *InputType = "UBIGINT"; + static constexpr const char *DefaultValue = "5"; + static constexpr SetScope DefaultScope = SetScope::GLOBAL; +}; + struct ZstdMinStringLengthSetting { using RETURN_TYPE = idx_t; static constexpr const char *Name = "zstd_min_string_length"; diff --git a/src/include/duckdb/optimizer/common_subplan_optimizer.hpp b/src/include/duckdb/optimizer/common_subplan_optimizer.hpp new file mode 100644 index 000000000000..8d8e35ea1815 --- /dev/null +++ b/src/include/duckdb/optimizer/common_subplan_optimizer.hpp @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/optimizer/common_subplan_optimizer.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/planner/logical_operator.hpp" + +namespace duckdb { + +class Optimizer; +class LogicalOperator; + +//! The CommonSubplanOptimizer optimizer detects common subplans, and converts them to refs of a materialized CTE +class CommonSubplanOptimizer { +public: + explicit CommonSubplanOptimizer(Optimizer &optimizer); + +public: + unique_ptr Optimize(unique_ptr op); + +private: + //! The optimizer + Optimizer &optimizer; +}; + +} // namespace duckdb diff --git a/src/include/duckdb/optimizer/cte_inlining.hpp b/src/include/duckdb/optimizer/cte_inlining.hpp index 90439b11e02d..97529f6ee4f3 100644 --- a/src/include/duckdb/optimizer/cte_inlining.hpp +++ b/src/include/duckdb/optimizer/cte_inlining.hpp @@ -25,6 +25,7 @@ class CTEInlining { public: explicit CTEInlining(Optimizer &optimizer); unique_ptr Optimize(unique_ptr op); + static bool EndsInAggregateOrDistinct(const LogicalOperator &op); private: void TryInlining(unique_ptr &op); diff --git a/src/include/duckdb/optimizer/filter_pushdown.hpp b/src/include/duckdb/optimizer/filter_pushdown.hpp index c2fc87a5249f..29c2f0ac4e6b 100644 --- a/src/include/duckdb/optimizer/filter_pushdown.hpp +++ b/src/include/duckdb/optimizer/filter_pushdown.hpp @@ -96,14 +96,17 @@ class FilterPushdown { unique_ptr FinishPushdown(unique_ptr op); //! Adds a filter to the set of filters. Returns FilterResult::UNSATISFIABLE if the subtree should be stripped, or //! FilterResult::SUCCESS otherwise + + unique_ptr PushFiltersIntoDelimJoin(unique_ptr op); FilterResult AddFilter(unique_ptr expr); //! Extract filter bindings to compare them with expressions in an operator and determine if the filter //! can be pushed down void ExtractFilterBindings(const Expression &expr, vector &bindings); //! Generate filters from the current set of filters stored in the FilterCombiner void GenerateFilters(); - //! if there are filters in this FilterPushdown node, push them into the combiner - void PushFilters(); + //! if there are filters in this FilterPushdown node, push them into the combiner. Returns + //! FilterResult::UNSATISFIABLE if the subtree should be stripped, or FilterResult::SUCCESS otherwise + FilterResult PushFilters(); }; } // namespace duckdb diff --git a/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp b/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp index 652ae47d6023..c1f2c4586157 100644 --- a/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp +++ b/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp @@ -65,7 +65,8 @@ class RelationStatisticsHelper { static RelationStats CombineStatsOfReorderableOperator(vector &bindings, vector relation_stats); //! Called after reordering a query plan with potentially 2+ relations. - static RelationStats CombineStatsOfNonReorderableOperator(LogicalOperator &op, vector child_stats); + static RelationStats CombineStatsOfNonReorderableOperator(LogicalOperator &op, + const vector &child_stats); static void CopyRelationStats(RelationStats &to, const RelationStats &from); private: diff --git a/src/include/duckdb/optimizer/statistics_propagator.hpp b/src/include/duckdb/optimizer/statistics_propagator.hpp index 6f7744a9f5f7..19feda673479 100644 --- a/src/include/duckdb/optimizer/statistics_propagator.hpp +++ b/src/include/duckdb/optimizer/statistics_propagator.hpp @@ -113,6 +113,8 @@ class StatisticsPropagator { bool ExpressionIsConstant(Expression &expr, const Value &val); bool ExpressionIsConstantOrNull(Expression &expr, const Value &val); + unique_ptr PropagateUnion(LogicalSetOperation &setop, unique_ptr &node_ptr); + private: Optimizer &optimizer; ClientContext &context; diff --git a/src/include/duckdb/optimizer/topn_window_elimination.hpp b/src/include/duckdb/optimizer/topn_window_elimination.hpp new file mode 100644 index 000000000000..c871bfc65246 --- /dev/null +++ b/src/include/duckdb/optimizer/topn_window_elimination.hpp @@ -0,0 +1,73 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/optimizer/topn_window_elimination.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/main/client_context.hpp" +#include "duckdb/optimizer/column_binding_replacer.hpp" +#include "duckdb/optimizer/remove_unused_columns.hpp" + +namespace duckdb { + +enum class TopNPayloadType { SINGLE_COLUMN, STRUCT_PACK }; + +struct TopNWindowEliminationParameters { + //! Whether the sort is ASCENDING or DESCENDING + OrderType order_type; + //! The number of values in the LIMIT clause + int64_t limit; + //! How we fetch the payload columns + TopNPayloadType payload_type; + //! Whether to include row numbers + bool include_row_number; + //! Whether the val or arg column contains null values + bool can_be_null = false; +}; + +class TopNWindowElimination : public BaseColumnPruner { +public: + explicit TopNWindowElimination(ClientContext &context, Optimizer &optimizer, + optional_ptr>> stats_p); + + unique_ptr Optimize(unique_ptr op); + +private: + bool CanOptimize(LogicalOperator &op); + unique_ptr OptimizeInternal(unique_ptr op, ColumnBindingReplacer &replacer); + + unique_ptr CreateAggregateOperator(LogicalWindow &window, vector> args, + const TopNWindowEliminationParameters ¶ms) const; + unique_ptr TryCreateUnnestOperator(unique_ptr op, + const TopNWindowEliminationParameters ¶ms) const; + unique_ptr CreateProjectionOperator(unique_ptr op, + const TopNWindowEliminationParameters ¶ms, + const map &group_idxs) const; + + vector> GenerateAggregatePayload(const vector &bindings, + const LogicalWindow &window, map &group_idxs); + vector TraverseProjectionBindings(const std::vector &old_bindings, + LogicalOperator *&op); + unique_ptr CreateAggregateExpression(vector> aggregate_params, bool requires_arg, + const TopNWindowEliminationParameters ¶ms) const; + unique_ptr CreateRowNumberGenerator(unique_ptr aggregate_column_ref) const; + void AddStructExtractExprs(vector> &exprs, const LogicalType &struct_type, + const unique_ptr &aggregate_column_ref) const; + static void UpdateTopmostBindings(idx_t window_idx, const unique_ptr &op, + const map &group_idxs, + const vector &topmost_bindings, + vector &new_bindings, ColumnBindingReplacer &replacer); + TopNWindowEliminationParameters ExtractOptimizerParameters(const LogicalWindow &window, const LogicalFilter &filter, + const vector &bindings, + vector> &aggregate_payload); + +private: + ClientContext &context; + Optimizer &optimizer; + optional_ptr>> stats; +}; +} // namespace duckdb diff --git a/src/include/duckdb/parser/common_table_expression_info.hpp b/src/include/duckdb/parser/common_table_expression_info.hpp index 55275369288b..7a784eeb1d0c 100644 --- a/src/include/duckdb/parser/common_table_expression_info.hpp +++ b/src/include/duckdb/parser/common_table_expression_info.hpp @@ -16,16 +16,20 @@ namespace duckdb { class SelectStatement; struct CommonTableExpressionInfo { + ~CommonTableExpressionInfo(); + vector aliases; vector> key_targets; unique_ptr query; CTEMaterialize materialized = CTEMaterialize::CTE_MATERIALIZE_DEFAULT; +public: void Serialize(Serializer &serializer) const; static unique_ptr Deserialize(Deserializer &deserializer); unique_ptr Copy(); - ~CommonTableExpressionInfo(); +private: + CTEMaterialize GetMaterializedForSerialization(Serializer &serializer) const; }; } // namespace duckdb diff --git a/src/include/duckdb/parser/parsed_data/alter_database_info.hpp b/src/include/duckdb/parser/parsed_data/alter_database_info.hpp new file mode 100644 index 000000000000..fc7be14d2106 --- /dev/null +++ b/src/include/duckdb/parser/parsed_data/alter_database_info.hpp @@ -0,0 +1,52 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/parser/parsed_data/alter_database_info.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/parser/parsed_data/alter_info.hpp" + +namespace duckdb { + +enum class AlterDatabaseType : uint8_t { RENAME_DATABASE = 0 }; + +struct AlterDatabaseInfo : public AlterInfo { +public: + explicit AlterDatabaseInfo(AlterDatabaseType alter_database_type); + AlterDatabaseInfo(AlterDatabaseType alter_database_type, string catalog_p, OnEntryNotFound if_not_found); + ~AlterDatabaseInfo() override; + + AlterDatabaseType alter_database_type; + +public: + CatalogType GetCatalogType() const override; + string ToString() const override = 0; + + static unique_ptr Deserialize(Deserializer &deserializer); + +protected: + void Serialize(Serializer &serializer) const override; +}; + +struct RenameDatabaseInfo : public AlterDatabaseInfo { +public: + RenameDatabaseInfo(); + RenameDatabaseInfo(string catalog_p, string new_name_p, OnEntryNotFound if_not_found); + + string new_name; + +public: + unique_ptr Copy() const override; + string ToString() const override; + + static unique_ptr Deserialize(Deserializer &deserializer); + +protected: + void Serialize(Serializer &serializer) const override; +}; + +} // namespace duckdb diff --git a/src/include/duckdb/parser/parsed_data/alter_info.hpp b/src/include/duckdb/parser/parsed_data/alter_info.hpp index 7e47a2e88c6a..7dcf8307484d 100644 --- a/src/include/duckdb/parser/parsed_data/alter_info.hpp +++ b/src/include/duckdb/parser/parsed_data/alter_info.hpp @@ -24,7 +24,8 @@ enum class AlterType : uint8_t { ALTER_SCALAR_FUNCTION = 5, ALTER_TABLE_FUNCTION = 6, SET_COMMENT = 7, - SET_COLUMN_COMMENT = 8 + SET_COLUMN_COMMENT = 8, + ALTER_DATABASE = 9 }; struct AlterEntryData { diff --git a/src/include/duckdb/parser/parsed_data/vacuum_info.hpp b/src/include/duckdb/parser/parsed_data/vacuum_info.hpp index 5540d38a2f48..12c4f77ca817 100644 --- a/src/include/duckdb/parser/parsed_data/vacuum_info.hpp +++ b/src/include/duckdb/parser/parsed_data/vacuum_info.hpp @@ -10,7 +10,6 @@ #include "duckdb/parser/parsed_data/parse_info.hpp" #include "duckdb/parser/tableref.hpp" -#include "duckdb/planner/tableref/bound_basetableref.hpp" #include "duckdb/common/unordered_map.hpp" #include "duckdb/common/optional_ptr.hpp" #include "duckdb/catalog/dependency_list.hpp" diff --git a/src/include/duckdb/parser/parser_extension.hpp b/src/include/duckdb/parser/parser_extension.hpp index 164a45c3412a..a3d2dcf64896 100644 --- a/src/include/duckdb/parser/parser_extension.hpp +++ b/src/include/duckdb/parser/parser_extension.hpp @@ -11,6 +11,7 @@ #include "duckdb/common/common.hpp" #include "duckdb/common/enums/statement_type.hpp" #include "duckdb/function/table_function.hpp" +#include "duckdb/parser/sql_statement.hpp" namespace duckdb { @@ -76,6 +77,25 @@ struct ParserExtensionPlanResult { // NOLINT: work-around bug in clang-tidy typedef ParserExtensionPlanResult (*plan_function_t)(ParserExtensionInfo *info, ClientContext &context, unique_ptr parse_data); +//===--------------------------------------------------------------------===// +// Parser override +//===--------------------------------------------------------------------===// +struct ParserOverrideResult { + explicit ParserOverrideResult() : type(ParserExtensionResultType::DISPLAY_ORIGINAL_ERROR) {}; + + explicit ParserOverrideResult(vector> statements_p) + : type(ParserExtensionResultType::PARSE_SUCCESSFUL), statements(std::move(statements_p)) {}; + + explicit ParserOverrideResult(std::exception &error_p) + : type(ParserExtensionResultType::DISPLAY_EXTENSION_ERROR), error(error_p) {}; + + ParserExtensionResultType type; + vector> statements; + ErrorData error; +}; + +typedef ParserOverrideResult (*parser_override_function_t)(ParserExtensionInfo *info, const string &query); + //===--------------------------------------------------------------------===// // ParserExtension //===--------------------------------------------------------------------===// @@ -83,11 +103,14 @@ class ParserExtension { public: //! The parse function of the parser extension. //! Takes a query string as input and returns ParserExtensionParseData (on success) or an error - parse_function_t parse_function; + parse_function_t parse_function = nullptr; //! The plan function of the parser extension //! Takes as input the result of the parse_function, and outputs various properties of the resulting plan - plan_function_t plan_function; + plan_function_t plan_function = nullptr; + + //! Override the current parser with a new parser and return a vector of SQL statements + parser_override_function_t parser_override = nullptr; //! Additional parser info passed to the parse function shared_ptr parser_info; diff --git a/src/include/duckdb/parser/parser_options.hpp b/src/include/duckdb/parser/parser_options.hpp index d388fb116951..d9a42632a83e 100644 --- a/src/include/duckdb/parser/parser_options.hpp +++ b/src/include/duckdb/parser/parser_options.hpp @@ -18,6 +18,7 @@ struct ParserOptions { bool integer_division = false; idx_t max_expression_depth = 1000; const vector *extensions = nullptr; + string parser_override_setting = "default"; }; } // namespace duckdb diff --git a/src/include/duckdb/parser/query_node.hpp b/src/include/duckdb/parser/query_node.hpp index ec03da095e4d..5c091b259ba6 100644 --- a/src/include/duckdb/parser/query_node.hpp +++ b/src/include/duckdb/parser/query_node.hpp @@ -25,7 +25,8 @@ enum class QueryNodeType : uint8_t { SET_OPERATION_NODE = 2, BOUND_SUBQUERY_NODE = 3, RECURSIVE_CTE_NODE = 4, - CTE_NODE = 5 + CTE_NODE = 5, + STATEMENT_NODE = 6 }; struct CommonTableExpressionInfo; @@ -59,8 +60,6 @@ class QueryNode { //! CTEs (used by SelectNode and SetOperationNode) CommonTableExpressionMap cte_map; - virtual const vector> &GetSelectList() const = 0; - public: //! Convert the query node to a string virtual string ToString() const = 0; diff --git a/src/include/duckdb/parser/query_node/cte_node.hpp b/src/include/duckdb/parser/query_node/cte_node.hpp index bc997a6c7740..fd2589fd2ab2 100644 --- a/src/include/duckdb/parser/query_node/cte_node.hpp +++ b/src/include/duckdb/parser/query_node/cte_node.hpp @@ -14,6 +14,7 @@ namespace duckdb { +//! DEPRECATED - CTENode is only preserved for backwards compatibility when serializing older databases class CTENode : public QueryNode { public: static constexpr const QueryNodeType TYPE = QueryNodeType::CTE_NODE; @@ -23,30 +24,18 @@ class CTENode : public QueryNode { } string ctename; - //! The query of the CTE unique_ptr query; - //! Child unique_ptr child; - //! Aliases of the CTE node vector aliases; CTEMaterialize materialized = CTEMaterialize::CTE_MATERIALIZE_DEFAULT; - const vector> &GetSelectList() const override { - return query->GetSelectList(); - } - public: - //! Convert the query node to a string string ToString() const override; bool Equals(const QueryNode *other) const override; - //! Create a copy of this SelectNode unique_ptr Copy() const override; - //! Serializes a QueryNode to a stand-alone binary blob - //! Deserializes a blob back into a QueryNode - void Serialize(Serializer &serializer) const override; static unique_ptr Deserialize(Deserializer &source); }; diff --git a/src/include/duckdb/parser/query_node/list.hpp b/src/include/duckdb/parser/query_node/list.hpp index 94bfd3438e2b..3a2894cc4c06 100644 --- a/src/include/duckdb/parser/query_node/list.hpp +++ b/src/include/duckdb/parser/query_node/list.hpp @@ -2,3 +2,4 @@ #include "duckdb/parser/query_node/cte_node.hpp" #include "duckdb/parser/query_node/select_node.hpp" #include "duckdb/parser/query_node/set_operation_node.hpp" +#include "duckdb/parser/query_node/statement_node.hpp" diff --git a/src/include/duckdb/parser/query_node/recursive_cte_node.hpp b/src/include/duckdb/parser/query_node/recursive_cte_node.hpp index 6d73fda4aca6..1f5f16ead15d 100644 --- a/src/include/duckdb/parser/query_node/recursive_cte_node.hpp +++ b/src/include/duckdb/parser/query_node/recursive_cte_node.hpp @@ -33,10 +33,6 @@ class RecursiveCTENode : public QueryNode { //! targets for key variants vector> key_targets; - const vector> &GetSelectList() const override { - return left->GetSelectList(); - } - public: //! Convert the query node to a string string ToString() const override; diff --git a/src/include/duckdb/parser/query_node/select_node.hpp b/src/include/duckdb/parser/query_node/select_node.hpp index 62aa9c0b2e64..dfc474d14a5b 100644 --- a/src/include/duckdb/parser/query_node/select_node.hpp +++ b/src/include/duckdb/parser/query_node/select_node.hpp @@ -43,10 +43,6 @@ class SelectNode : public QueryNode { //! The SAMPLE clause unique_ptr sample; - const vector> &GetSelectList() const override { - return select_list; - } - public: //! Convert the query node to a string string ToString() const override; diff --git a/src/include/duckdb/parser/query_node/set_operation_node.hpp b/src/include/duckdb/parser/query_node/set_operation_node.hpp index de36f3cad61a..3070e224534a 100644 --- a/src/include/duckdb/parser/query_node/set_operation_node.hpp +++ b/src/include/duckdb/parser/query_node/set_operation_node.hpp @@ -26,14 +26,8 @@ class SetOperationNode : public QueryNode { SetOperationType setop_type = SetOperationType::NONE; //! whether the ALL modifier was used or not bool setop_all = false; - //! The left side of the set operation - unique_ptr left; - //! The right side of the set operation - unique_ptr right; - - const vector> &GetSelectList() const override { - return left->GetSelectList(); - } + //! The children of the set operation + vector> children; public: //! Convert the query node to a string @@ -54,7 +48,8 @@ class SetOperationNode : public QueryNode { SetOperationNode(SetOperationType setop_type, unique_ptr left, unique_ptr right, vector> children, bool setop_all); - vector> SerializeChildNodes() const; + unique_ptr SerializeChildNode(Serializer &serializer, idx_t index) const; + bool SerializeChildList(Serializer &serializer) const; }; } // namespace duckdb diff --git a/src/include/duckdb/parser/query_node/statement_node.hpp b/src/include/duckdb/parser/query_node/statement_node.hpp new file mode 100644 index 000000000000..26db46a58231 --- /dev/null +++ b/src/include/duckdb/parser/query_node/statement_node.hpp @@ -0,0 +1,41 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/parser/query_node/statement_node.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/parser/parsed_expression.hpp" +#include "duckdb/parser/query_node.hpp" +#include "duckdb/parser/sql_statement.hpp" + +namespace duckdb { + +class StatementNode : public QueryNode { +public: + static constexpr const QueryNodeType TYPE = QueryNodeType::STATEMENT_NODE; + +public: + explicit StatementNode(SQLStatement &stmt_p); + + SQLStatement &stmt; + +public: + //! Convert the query node to a string + string ToString() const override; + + bool Equals(const QueryNode *other) const override; + //! Create a copy of this SelectNode + unique_ptr Copy() const override; + + //! Serializes a QueryNode to a stand-alone binary blob + //! Deserializes a blob back into a QueryNode + + void Serialize(Serializer &serializer) const override; + static unique_ptr Deserialize(Deserializer &source); +}; + +} // namespace duckdb diff --git a/src/include/duckdb/parser/tableref/bound_ref_wrapper.hpp b/src/include/duckdb/parser/tableref/bound_ref_wrapper.hpp index be997eb5d20a..c8586a5595aa 100644 --- a/src/include/duckdb/parser/tableref/bound_ref_wrapper.hpp +++ b/src/include/duckdb/parser/tableref/bound_ref_wrapper.hpp @@ -9,7 +9,6 @@ #pragma once #include "duckdb/parser/tableref.hpp" -#include "duckdb/planner/bound_tableref.hpp" #include "duckdb/planner/binder.hpp" namespace duckdb { @@ -20,10 +19,10 @@ class BoundRefWrapper : public TableRef { static constexpr const TableReferenceType TYPE = TableReferenceType::BOUND_TABLE_REF; public: - BoundRefWrapper(unique_ptr bound_ref_p, shared_ptr binder_p); + BoundRefWrapper(BoundStatement bound_ref_p, shared_ptr binder_p); //! The bound reference object - unique_ptr bound_ref; + BoundStatement bound_ref; //! The binder that was used to bind this table ref shared_ptr binder; diff --git a/src/include/duckdb/parser/tokens.hpp b/src/include/duckdb/parser/tokens.hpp index 6eeb8c5e234e..d5646739cc3e 100644 --- a/src/include/duckdb/parser/tokens.hpp +++ b/src/include/duckdb/parser/tokens.hpp @@ -53,6 +53,7 @@ class SelectNode; class SetOperationNode; class RecursiveCTENode; class CTENode; +class StatementNode; //===--------------------------------------------------------------------===// // Expressions diff --git a/src/include/duckdb/parser/transformer.hpp b/src/include/duckdb/parser/transformer.hpp index cb9ad500a8b1..1945ebc5a921 100644 --- a/src/include/duckdb/parser/transformer.hpp +++ b/src/include/duckdb/parser/transformer.hpp @@ -80,7 +80,7 @@ class Transformer { //! The set of pivot entries to create vector> pivot_entries; //! Sets of stored CTEs, if any - vector stored_cte_map; + vector> stored_cte_map; //! Whether or not we are currently binding a window definition bool in_window_definition = false; @@ -116,6 +116,8 @@ class Transformer { unique_ptr TransformSelectStmt(duckdb_libpgquery::PGNode &node, bool is_select = true); //! Transform a Postgres T_AlterStmt node into a AlterStatement unique_ptr TransformAlter(duckdb_libpgquery::PGAlterTableStmt &stmt); + //! Transform a Postgres T_AlterDatabaseStmt node into a AlterStatement + unique_ptr TransformAlterDatabase(duckdb_libpgquery::PGAlterDatabaseStmt &stmt); //! Transform a Postgres duckdb_libpgquery::T_PGRenameStmt node into a RenameStatement unique_ptr TransformRename(duckdb_libpgquery::PGRenameStmt &stmt); //! Transform a Postgres duckdb_libpgquery::T_PGCreateStmt node into a CreateStatement @@ -215,6 +217,8 @@ class Transformer { unique_ptr TransformSelectNodeInternal(duckdb_libpgquery::PGSelectStmt &select, bool is_select = true); unique_ptr TransformSelectInternal(duckdb_libpgquery::PGSelectStmt &select); void TransformModifiers(duckdb_libpgquery::PGSelectStmt &stmt, QueryNode &node); + bool SetOperationsMatch(duckdb_libpgquery::PGSelectStmt &root, duckdb_libpgquery::PGNode &node); + void TransformSetOperationChildren(duckdb_libpgquery::PGSelectStmt &stmt, SetOperationNode &result); //===--------------------------------------------------------------------===// // Expression Transform @@ -300,7 +304,6 @@ class Transformer { string TransformAlias(duckdb_libpgquery::PGAlias *root, vector &column_name_alias); vector TransformStringList(duckdb_libpgquery::PGList *list); void TransformCTE(duckdb_libpgquery::PGWithClause &de_with_clause, CommonTableExpressionMap &cte_map); - static unique_ptr TransformMaterializedCTE(unique_ptr root); unique_ptr TransformRecursiveCTE(duckdb_libpgquery::PGCommonTableExpr &node, CommonTableExpressionInfo &info); diff --git a/src/include/duckdb/planner/bind_context.hpp b/src/include/duckdb/planner/bind_context.hpp index d9c20dd1d3af..db5b52c78fc0 100644 --- a/src/include/duckdb/planner/bind_context.hpp +++ b/src/include/duckdb/planner/bind_context.hpp @@ -23,7 +23,7 @@ namespace duckdb { class Binder; class LogicalGet; -class BoundQueryNode; +struct BoundStatement; class StarExpression; @@ -43,9 +43,6 @@ class BindContext { public: explicit BindContext(Binder &binder); - //! Keep track of recursive CTE references - case_insensitive_map_t> cte_references; - public: //! Given a column name, find the matching table it belongs to. Throws an //! exception if no table has a column of the given name. @@ -57,7 +54,7 @@ class BindContext { //! matching ones vector GetSimilarBindings(const string &column_name); - optional_ptr GetCTEBinding(const string &ctename); + optional_ptr GetCTEBinding(const BindingAlias &ctename); //! Binds a column expression to the base table. Returns the bound expression //! or throws an exception if the column could not be bound. BindResult BindColumn(ColumnRefExpression &colref, idx_t depth); @@ -105,11 +102,11 @@ class BindContext { const vector &types, vector &bound_column_ids, optional_ptr entry, virtual_column_map_t virtual_columns); //! Adds a table view with a given alias to the BindContext. - void AddView(idx_t index, const string &alias, SubqueryRef &ref, BoundQueryNode &subquery, ViewCatalogEntry &view); + void AddView(idx_t index, const string &alias, SubqueryRef &ref, BoundStatement &subquery, ViewCatalogEntry &view); //! Adds a subquery with a given alias to the BindContext. - void AddSubquery(idx_t index, const string &alias, SubqueryRef &ref, BoundQueryNode &subquery); + void AddSubquery(idx_t index, const string &alias, SubqueryRef &ref, BoundStatement &subquery); //! Adds a subquery with a given alias to the BindContext. - void AddSubquery(idx_t index, const string &alias, TableFunctionRef &ref, BoundQueryNode &subquery); + void AddSubquery(idx_t index, const string &alias, TableFunctionRef &ref, BoundStatement &subquery); //! Adds a binding to a catalog entry with a given alias to the BindContext. void AddEntryBinding(idx_t index, const string &alias, const vector &names, const vector &types, StandardEntry &entry); @@ -119,10 +116,9 @@ class BindContext { //! Adds a base table with the given alias to the CTE BindContext. //! We need this to correctly bind recursive CTEs with multiple references. - void AddCTEBinding(idx_t index, const string &alias, const vector &names, const vector &types, - bool using_key = false); - - void RemoveCTEBinding(const string &alias); + void AddCTEBinding(idx_t index, BindingAlias alias, const vector &names, const vector &types, + CTEType cte_type = CTEType::CAN_BE_REFERENCED); + void AddCTEBinding(unique_ptr binding); //! Add an implicit join condition (e.g. USING (x)) void AddUsingBinding(const string &column_name, UsingColumnSet &set); @@ -146,13 +142,6 @@ class BindContext { string GetActualColumnName(const BindingAlias &binding_alias, const string &column_name); string GetActualColumnName(Binding &binding, const string &column_name); - case_insensitive_map_t> GetCTEBindings() { - return cte_bindings; - } - void SetCTEBindings(case_insensitive_map_t> bindings) { - cte_bindings = std::move(bindings); - } - //! Alias a set of column names for the specified table, using the original names if there are not enough aliases //! specified. static vector AliasColumnNames(const string &table_name, const vector &names, @@ -184,10 +173,7 @@ class BindContext { vector> bindings_list; //! The set of columns used in USING join conditions case_insensitive_map_t> using_columns; - //! Using column sets - vector> using_column_sets; - //! The set of CTE bindings - case_insensitive_map_t> cte_bindings; + vector> cte_bindings; }; } // namespace duckdb diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index bc81db9bdf51..cf01e74d6585 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -27,7 +27,6 @@ #include "duckdb/planner/joinside.hpp" #include "duckdb/planner/bound_constraint.hpp" #include "duckdb/planner/logical_operator.hpp" -#include "duckdb/planner/tableref/bound_delimgetref.hpp" #include "duckdb/common/enums/copy_option_mode.hpp" //! fwd declare @@ -69,6 +68,8 @@ struct PivotColumnEntry; struct UnpivotEntry; struct CopyInfo; struct CopyOption; +struct BoundSetOpChild; +struct BoundCTEData; template class IndexVector; @@ -100,6 +101,89 @@ struct CorrelatedColumnInfo { } }; +struct CorrelatedColumns { +private: + using container_type = vector; + +public: + CorrelatedColumns() : delim_index(1ULL << 63) { + } + + void AddColumn(container_type::value_type info) { + // Add to beginning + correlated_columns.insert(correlated_columns.begin(), std::move(info)); + delim_index++; + } + + void SetDelimIndexToZero() { + delim_index = 0; + } + + idx_t GetDelimIndex() const { + return delim_index; + } + + const container_type::value_type &operator[](const idx_t &index) const { + return correlated_columns.at(index); + } + + idx_t size() const { // NOLINT: match stl case + return correlated_columns.size(); + } + + bool empty() const { // NOLINT: match stl case + return correlated_columns.empty(); + } + + void clear() { // NOLINT: match stl case + correlated_columns.clear(); + } + + container_type::iterator begin() { // NOLINT: match stl case + return correlated_columns.begin(); + } + + container_type::iterator end() { // NOLINT: match stl case + return correlated_columns.end(); + } + + container_type::const_iterator begin() const { // NOLINT: match stl case + return correlated_columns.begin(); + } + + container_type::const_iterator end() const { // NOLINT: match stl case + return correlated_columns.end(); + } + +private: + container_type correlated_columns; + idx_t delim_index; +}; + +//! GlobalBinderState is state shared over the ENTIRE query, including subqueries, views, etc +struct GlobalBinderState { + //! The count of bound_tables + idx_t bound_tables = 0; + //! Statement properties + StatementProperties prop; + //! Binding mode + BindingMode mode = BindingMode::STANDARD_BINDING; + //! Table names extracted for BindingMode::EXTRACT_NAMES or BindingMode::EXTRACT_QUALIFIED_NAMES. + unordered_set table_names; + //! Replacement Scans extracted for BindingMode::EXTRACT_REPLACEMENT_SCANS + case_insensitive_map_t> replacement_scans; + //! Using column sets + vector> using_column_sets; + //! The set of parameter expressions bound by this binder + optional_ptr parameters; +}; + +// QueryBinderState is state shared WITHIN a query, a new query-binder state is created when binding inside e.g. a view +struct QueryBinderState { + //! The vector of active binders + vector> active_binders; +}; + //! Bind the parsed query tree to the actual columns present in the catalog. /*! The binder is responsible for binding tables and columns to actual physical @@ -116,17 +200,11 @@ class Binder : public enable_shared_from_this { //! The client context ClientContext &context; - //! A mapping of names to common table expressions - case_insensitive_map_t> CTE_bindings; // NOLINT - //! The CTEs that have already been bound - reference_set_t bound_ctes; //! The bind context BindContext bind_context; //! The set of correlated columns bound by this binder (FIXME: this should probably be an unordered_set and not a //! vector) - vector correlated_columns; - //! The set of parameter expressions bound by this binder - optional_ptr parameters; + CorrelatedColumns correlated_columns; //! The alias for the currently processing subquery, if it exists string alias; //! Macro parameter bindings (if any) @@ -173,8 +251,7 @@ class Binder : public enable_shared_from_this { QueryErrorContext &error_context, string &func_name); unique_ptr BindPragma(PragmaInfo &info, QueryErrorContext error_context); - unique_ptr Bind(TableRef &ref); - unique_ptr CreatePlan(BoundTableRef &ref); + BoundStatement Bind(TableRef &ref); //! Generates an unused index for a table idx_t GenerateTableIndex(); @@ -182,12 +259,8 @@ class Binder : public enable_shared_from_this { optional_ptr GetCatalogEntry(const string &catalog, const string &schema, const EntryLookupInfo &lookup_info, OnEntryNotFound on_entry_not_found); - //! Add a common table expression to the binder - void AddCTE(const string &name, CommonTableExpressionInfo &cte); //! Find all candidate common table expression by name; returns empty vector if none exists - vector> FindCTE(const string &name, bool skip = false); - - bool CTEIsAlreadyBound(CommonTableExpressionInfo &cte); + optional_ptr GetCTEBinding(const BindingAlias &name); //! Add the view to the set of currently bound views - used for detecting recursive view definitions void AddBoundView(ViewCatalogEntry &view); @@ -200,7 +273,7 @@ class Binder : public enable_shared_from_this { vector> &GetActiveBinders(); - void MergeCorrelatedColumns(vector &other); + void MergeCorrelatedColumns(CorrelatedColumns &other); //! Add a correlated column to this binder (if it does not exist) void AddCorrelatedColumn(const CorrelatedColumnInfo &info); @@ -230,12 +303,11 @@ class Binder : public enable_shared_from_this { void AddReplacementScan(const string &table_name, unique_ptr replacement); const unordered_set &GetTableNames(); case_insensitive_map_t> &GetReplacementScans(); - optional_ptr GetRootStatement() { - return root_statement; - } CatalogEntryRetriever &EntryRetriever() { return entry_retriever; } + optional_ptr GetParameters(); + void SetParameters(BoundParameterMap ¶meters); //! Returns a ColumnRefExpression after it was resolved (i.e. past the STAR expression/USING clauses) static optional_ptr GetResolvedColumnExpression(ParsedExpression &root_expr); @@ -252,42 +324,28 @@ class Binder : public enable_shared_from_this { private: //! The parent binder (if any) shared_ptr parent; - //! The vector of active binders - vector> active_binders; - //! The count of bound_tables - idx_t bound_tables; + //! What kind of node we are binding using this binder + BinderType binder_type = BinderType::REGULAR_BINDER; + //! Global binder state + shared_ptr global_binder_state; + //! Query binder state + shared_ptr query_binder_state; //! Whether or not the binder has any unplanned dependent joins that still need to be planned/flattened bool has_unplanned_dependent_joins = false; //! Whether or not outside dependent joins have been planned and flattened bool is_outside_flattened = true; - //! What kind of node we are binding using this binder - BinderType binder_type = BinderType::REGULAR_BINDER; //! Whether or not the binder can contain NULLs as the root of expressions bool can_contain_nulls = false; - //! The root statement of the query that is currently being parsed - optional_ptr root_statement; - //! Binding mode - BindingMode mode = BindingMode::STANDARD_BINDING; - //! Table names extracted for BindingMode::EXTRACT_NAMES or BindingMode::EXTRACT_QUALIFIED_NAMES. - unordered_set table_names; - //! Replacement Scans extracted for BindingMode::EXTRACT_REPLACEMENT_SCANS - case_insensitive_map_t> replacement_scans; //! The set of bound views reference_set_t bound_views; //! Used to retrieve CatalogEntry's CatalogEntryRetriever entry_retriever; //! Unnamed subquery index idx_t unnamed_subquery_index = 1; - //! Statement properties - StatementProperties prop; - //! Root binder - Binder &root_binder; //! Binder depth idx_t depth; private: - //! Get the root binder (binder with no parent) - Binder &GetRootBinder(); //! Determine the depth of the binder idx_t GetBinderDepth() const; //! Increase the depth of the binder @@ -305,7 +363,7 @@ class Binder : public enable_shared_from_this { void MoveCorrelatedExpressions(Binder &other); //! Tries to bind the table name with replacement scans - unique_ptr BindWithReplacementScan(ClientContext &context, BaseTableRef &ref); + BoundStatement BindWithReplacementScan(ClientContext &context, BaseTableRef &ref); template BoundStatement BindWithCTE(T &statement); @@ -346,41 +404,39 @@ class Binder : public enable_shared_from_this { unique_ptr BindTableMacro(FunctionExpression &function, TableMacroCatalogEntry ¯o_func, idx_t depth); - unique_ptr BindMaterializedCTE(CommonTableExpressionMap &cte_map); - unique_ptr BindCTE(CTENode &statement); + BoundStatement BindCTE(const string &ctename, CommonTableExpressionInfo &info); - unique_ptr BindNode(SelectNode &node); - unique_ptr BindNode(SetOperationNode &node); - unique_ptr BindNode(RecursiveCTENode &node); - unique_ptr BindNode(CTENode &node); - unique_ptr BindNode(QueryNode &node); + BoundStatement BindNode(SelectNode &node); + BoundStatement BindNode(SetOperationNode &node); + BoundStatement BindNode(RecursiveCTENode &node); + BoundStatement BindNode(QueryNode &node); + BoundStatement BindNode(StatementNode &node); unique_ptr VisitQueryNode(BoundQueryNode &node, unique_ptr root); - unique_ptr CreatePlan(BoundRecursiveCTENode &node); - unique_ptr CreatePlan(BoundCTENode &node); - unique_ptr CreatePlan(BoundCTENode &node, unique_ptr base); unique_ptr CreatePlan(BoundSelectNode &statement); unique_ptr CreatePlan(BoundSetOperationNode &node); unique_ptr CreatePlan(BoundQueryNode &node); - unique_ptr BindJoin(Binder &parent, TableRef &ref); - unique_ptr Bind(BaseTableRef &ref); - unique_ptr Bind(BoundRefWrapper &ref); - unique_ptr Bind(JoinRef &ref); - unique_ptr Bind(SubqueryRef &ref, optional_ptr cte = nullptr); - unique_ptr Bind(TableFunctionRef &ref); - unique_ptr Bind(EmptyTableRef &ref); - unique_ptr Bind(DelimGetRef &ref); - unique_ptr Bind(ExpressionListRef &ref); - unique_ptr Bind(ColumnDataRef &ref); - unique_ptr Bind(PivotRef &expr); - unique_ptr Bind(ShowRef &ref); + void BuildUnionByNameInfo(BoundSetOperationNode &result); + + BoundStatement BindJoin(Binder &parent, TableRef &ref); + BoundStatement Bind(BaseTableRef &ref); + BoundStatement Bind(BoundRefWrapper &ref); + BoundStatement Bind(JoinRef &ref); + BoundStatement Bind(SubqueryRef &ref); + BoundStatement Bind(TableFunctionRef &ref); + BoundStatement Bind(EmptyTableRef &ref); + BoundStatement Bind(DelimGetRef &ref); + BoundStatement Bind(ExpressionListRef &ref); + BoundStatement Bind(ColumnDataRef &ref); + BoundStatement Bind(PivotRef &expr); + BoundStatement Bind(ShowRef &ref); unique_ptr BindPivot(PivotRef &expr, vector> all_columns); unique_ptr BindUnpivot(Binder &child_binder, PivotRef &expr, vector> all_columns, unique_ptr &where_clause); - unique_ptr BindBoundPivot(PivotRef &expr); + BoundStatement BindBoundPivot(PivotRef &expr); void ExtractUnpivotEntries(Binder &child_binder, PivotColumnEntry &entry, vector &unpivot_entries); void ExtractUnpivotColumnName(ParsedExpression &expr, vector &result); @@ -389,26 +445,14 @@ class Binder : public enable_shared_from_this { bool BindTableFunctionParameters(TableFunctionCatalogEntry &table_function, vector> &expressions, vector &arguments, vector ¶meters, named_parameter_map_t &named_parameters, - unique_ptr &subquery, ErrorData &error); - void BindTableInTableOutFunction(vector> &expressions, - unique_ptr &subquery); - unique_ptr BindTableFunction(TableFunction &function, vector parameters); - unique_ptr BindTableFunctionInternal(TableFunction &table_function, const TableFunctionRef &ref, - vector parameters, - named_parameter_map_t named_parameters, - vector input_table_types, - vector input_table_names); - - unique_ptr CreatePlan(BoundBaseTableRef &ref); + BoundStatement &subquery, ErrorData &error); + void BindTableInTableOutFunction(vector> &expressions, BoundStatement &subquery); + BoundStatement BindTableFunction(TableFunction &function, vector parameters); + BoundStatement BindTableFunctionInternal(TableFunction &table_function, const TableFunctionRef &ref, + vector parameters, named_parameter_map_t named_parameters, + vector input_table_types, vector input_table_names); + unique_ptr CreatePlan(BoundJoinRef &ref); - unique_ptr CreatePlan(BoundSubqueryRef &ref); - unique_ptr CreatePlan(BoundTableFunction &ref); - unique_ptr CreatePlan(BoundEmptyTableRef &ref); - unique_ptr CreatePlan(BoundExpressionListRef &ref); - unique_ptr CreatePlan(BoundColumnDataRef &ref); - unique_ptr CreatePlan(BoundCTERef &ref); - unique_ptr CreatePlan(BoundPivotRef &ref); - unique_ptr CreatePlan(BoundDelimGetRef &ref); BoundStatement BindCopyTo(CopyStatement &stmt, const CopyFunction &function, CopyToType copy_to_type); BoundStatement BindCopyFrom(CopyStatement &stmt, const CopyFunction &function); @@ -428,12 +472,12 @@ class Binder : public enable_shared_from_this { void PlanSubqueries(unique_ptr &expr, unique_ptr &root); unique_ptr PlanSubquery(BoundSubqueryExpression &expr, unique_ptr &root); unique_ptr PlanLateralJoin(unique_ptr left, unique_ptr right, - vector &correlated_columns, + CorrelatedColumns &correlated_columns, JoinType join_type = JoinType::INNER, unique_ptr condition = nullptr); - unique_ptr CastLogicalOperatorToTypes(vector &source_types, - vector &target_types, + unique_ptr CastLogicalOperatorToTypes(const vector &source_types, + const vector &target_types, unique_ptr op); BindingAlias FindBinding(const string &using_column, const string &join_side); @@ -443,8 +487,6 @@ class Binder : public enable_shared_from_this { BindingAlias RetrieveUsingBinding(Binder ¤t_binder, optional_ptr current_set, const string &column_name, const string &join_side); - void AddCTEMap(CommonTableExpressionMap &cte_map); - void ExpandStarExpressions(vector> &select_list, vector> &new_select_list); void ExpandStarExpression(unique_ptr expr, vector> &new_select_list); @@ -465,14 +507,14 @@ class Binder : public enable_shared_from_this { LogicalType BindLogicalTypeInternal(const LogicalType &type, optional_ptr catalog, const string &schema); - unique_ptr BindSelectNode(SelectNode &statement, unique_ptr from_table); + BoundStatement BindSelectNode(SelectNode &statement, BoundStatement from_table); unique_ptr BindCopyDatabaseSchema(Catalog &source_catalog, const string &target_database_name); unique_ptr BindCopyDatabaseData(Catalog &source_catalog, const string &target_database_name); - unique_ptr BindShowQuery(ShowRef &ref); - unique_ptr BindShowTable(ShowRef &ref); - unique_ptr BindSummarize(ShowRef &ref); + BoundStatement BindShowQuery(ShowRef &ref); + BoundStatement BindShowTable(ShowRef &ref); + BoundStatement BindSummarize(ShowRef &ref); void BindInsertColumnList(TableCatalogEntry &table, vector &columns, bool default_values, vector &named_column_map, vector &expected_types, @@ -493,6 +535,9 @@ class Binder : public enable_shared_from_this { static void CheckInsertColumnCountMismatch(idx_t expected_columns, idx_t result_columns, bool columns_provided, const string &tname); + BoundCTEData PrepareCTE(const string &ctename, CommonTableExpressionInfo &statement); + BoundStatement FinishCTE(BoundCTEData &bound_cte, BoundStatement child_data); + private: Binder(ClientContext &context, shared_ptr parent, BinderType binder_type); }; diff --git a/src/include/duckdb/planner/bound_query_node.hpp b/src/include/duckdb/planner/bound_query_node.hpp index cd5a78b6aa94..76c461e78a4c 100644 --- a/src/include/duckdb/planner/bound_query_node.hpp +++ b/src/include/duckdb/planner/bound_query_node.hpp @@ -17,13 +17,8 @@ namespace duckdb { //! Bound equivalent of QueryNode class BoundQueryNode { public: - explicit BoundQueryNode(QueryNodeType type) : type(type) { - } - virtual ~BoundQueryNode() { - } + virtual ~BoundQueryNode() = default; - //! The type of the query node, either SetOperation or Select - QueryNodeType type; //! The result modifiers that should be applied to this query node vector> modifiers; @@ -34,23 +29,6 @@ class BoundQueryNode { public: virtual idx_t GetRootIndex() = 0; - -public: - template - TARGET &Cast() { - if (type != TARGET::TYPE) { - throw InternalException("Failed to cast bound query node to type - query node type mismatch"); - } - return reinterpret_cast(*this); - } - - template - const TARGET &Cast() const { - if (type != TARGET::TYPE) { - throw InternalException("Failed to cast bound query node to type - query node type mismatch"); - } - return reinterpret_cast(*this); - } }; } // namespace duckdb diff --git a/src/include/duckdb/planner/bound_statement.hpp b/src/include/duckdb/planner/bound_statement.hpp index bb1f7bfec82b..23fae54d6f5c 100644 --- a/src/include/duckdb/planner/bound_statement.hpp +++ b/src/include/duckdb/planner/bound_statement.hpp @@ -9,17 +9,31 @@ #pragma once #include "duckdb/common/string.hpp" +#include "duckdb/common/unique_ptr.hpp" #include "duckdb/common/vector.hpp" +#include "duckdb/common/enums/set_operation_type.hpp" +#include "duckdb/common/shared_ptr.hpp" namespace duckdb { class LogicalOperator; struct LogicalType; +struct BoundStatement; +class ParsedExpression; +class Binder; + +struct ExtraBoundInfo { + SetOperationType setop_type = SetOperationType::NONE; + vector> child_binders; + vector bound_children; + vector> original_expressions; +}; struct BoundStatement { unique_ptr plan; vector types; vector names; + ExtraBoundInfo extra_info; }; } // namespace duckdb diff --git a/src/include/duckdb/planner/bound_tableref.hpp b/src/include/duckdb/planner/bound_tableref.hpp deleted file mode 100644 index 0a831c54aeed..000000000000 --- a/src/include/duckdb/planner/bound_tableref.hpp +++ /dev/null @@ -1,46 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/bound_tableref.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/common/common.hpp" -#include "duckdb/common/enums/tableref_type.hpp" -#include "duckdb/parser/parsed_data/sample_options.hpp" - -namespace duckdb { - -class BoundTableRef { -public: - explicit BoundTableRef(TableReferenceType type) : type(type) { - } - virtual ~BoundTableRef() { - } - - //! The type of table reference - TableReferenceType type; - //! The sample options (if any) - unique_ptr sample; - -public: - template - TARGET &Cast() { - if (type != TARGET::TYPE) { - throw InternalException("Failed to cast bound table ref to type - table ref type mismatch"); - } - return reinterpret_cast(*this); - } - - template - const TARGET &Cast() const { - if (type != TARGET::TYPE) { - throw InternalException("Failed to cast bound table ref to type - table ref type mismatch"); - } - return reinterpret_cast(*this); - } -}; -} // namespace duckdb diff --git a/src/include/duckdb/planner/bound_tokens.hpp b/src/include/duckdb/planner/bound_tokens.hpp index bd75aac19370..862ef5a114a7 100644 --- a/src/include/duckdb/planner/bound_tokens.hpp +++ b/src/include/duckdb/planner/bound_tokens.hpp @@ -16,8 +16,6 @@ namespace duckdb { class BoundQueryNode; class BoundSelectNode; class BoundSetOperationNode; -class BoundRecursiveCTENode; -class BoundCTENode; //===--------------------------------------------------------------------===// // Expressions @@ -45,18 +43,7 @@ class BoundWindowExpression; //===--------------------------------------------------------------------===// // TableRefs //===--------------------------------------------------------------------===// -class BoundTableRef; - -class BoundBaseTableRef; class BoundJoinRef; -class BoundSubqueryRef; -class BoundTableFunction; -class BoundEmptyTableRef; -class BoundExpressionListRef; -class BoundColumnDataRef; -class BoundCTERef; -class BoundPivotRef; - class BoundMergeIntoAction; } // namespace duckdb diff --git a/src/include/duckdb/planner/expression/bound_subquery_expression.hpp b/src/include/duckdb/planner/expression/bound_subquery_expression.hpp index aa07a67b9305..35792c8d4684 100644 --- a/src/include/duckdb/planner/expression/bound_subquery_expression.hpp +++ b/src/include/duckdb/planner/expression/bound_subquery_expression.hpp @@ -29,7 +29,7 @@ class BoundSubqueryExpression : public Expression { //! The binder used to bind the subquery node shared_ptr binder; //! The bound subquery node - unique_ptr subquery; + BoundStatement subquery; //! The subquery type SubqueryType subquery_type; //! the child expressions to compare with (in case of IN, ANY, ALL operators) diff --git a/src/include/duckdb/planner/expression_binder.hpp b/src/include/duckdb/planner/expression_binder.hpp index 3650e74503fc..b2712f3bfe20 100644 --- a/src/include/duckdb/planner/expression_binder.hpp +++ b/src/include/duckdb/planner/expression_binder.hpp @@ -193,6 +193,8 @@ class ExpressionBinder { BindResult BindUnsupportedExpression(ParsedExpression &expr, idx_t depth, const string &message); + optional_ptr BindAndQualifyFunction(FunctionExpression &function, bool allow_throw); + protected: virtual BindResult BindGroupingFunction(OperatorExpression &op, idx_t depth); virtual BindResult BindFunction(FunctionExpression &expr, ScalarFunctionCatalogEntry &function, idx_t depth); diff --git a/src/include/duckdb/planner/expression_binder/lateral_binder.hpp b/src/include/duckdb/planner/expression_binder/lateral_binder.hpp index eb68a0cdf791..55f046cd7ec9 100644 --- a/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/lateral_binder.hpp @@ -24,7 +24,7 @@ class LateralBinder : public ExpressionBinder { return !correlated_columns.empty(); } - static void ReduceExpressionDepth(LogicalOperator &op, const vector &info); + static void ReduceExpressionDepth(LogicalOperator &op, const CorrelatedColumns &info); protected: BindResult BindExpression(unique_ptr &expr_ptr, idx_t depth, @@ -37,7 +37,7 @@ class LateralBinder : public ExpressionBinder { void ExtractCorrelatedColumns(Expression &expr); private: - vector correlated_columns; + CorrelatedColumns correlated_columns; }; } // namespace duckdb diff --git a/src/include/duckdb/planner/expression_binder/select_bind_state.hpp b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp index d11d94731eb0..956c66fabc84 100644 --- a/src/include/duckdb/planner/expression_binder/select_bind_state.hpp +++ b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp @@ -11,7 +11,6 @@ #include "duckdb/planner/bound_query_node.hpp" #include "duckdb/planner/logical_operator.hpp" #include "duckdb/parser/expression_map.hpp" -#include "duckdb/planner/bound_tableref.hpp" #include "duckdb/parser/parsed_data/sample_options.hpp" #include "duckdb/parser/group_by_node.hpp" diff --git a/src/include/duckdb/planner/expression_iterator.hpp b/src/include/duckdb/planner/expression_iterator.hpp index 5b2e2e8a81eb..52599a1c822d 100644 --- a/src/include/duckdb/planner/expression_iterator.hpp +++ b/src/include/duckdb/planner/expression_iterator.hpp @@ -14,8 +14,6 @@ #include namespace duckdb { -class BoundQueryNode; -class BoundTableRef; class ExpressionIterator { public: @@ -47,18 +45,4 @@ class ExpressionIterator { } }; -class BoundNodeVisitor { -public: - virtual ~BoundNodeVisitor() = default; - - virtual void VisitBoundQueryNode(BoundQueryNode &op); - virtual void VisitBoundTableRef(BoundTableRef &ref); - virtual void VisitExpression(unique_ptr &expression); - -protected: - // The VisitExpressionChildren method is called at the end of every call to VisitExpression to recursively visit all - // expressions in an expression tree. It can be overloaded to prevent automatically visiting the entire tree. - virtual void VisitExpressionChildren(Expression &expression); -}; - } // namespace duckdb diff --git a/src/include/duckdb/planner/extension_callback.hpp b/src/include/duckdb/planner/extension_callback.hpp index 060d29236f01..389f8f257017 100644 --- a/src/include/duckdb/planner/extension_callback.hpp +++ b/src/include/duckdb/planner/extension_callback.hpp @@ -11,7 +11,9 @@ #include "duckdb/common/common.hpp" namespace duckdb { +class ClientContext; class DatabaseInstance; +class ErrorData; class ExtensionCallback { public: @@ -24,9 +26,15 @@ class ExtensionCallback { //! Called when a connection is closed virtual void OnConnectionClosed(ClientContext &context) { } + //! Called before an extension starts loading + virtual void OnBeginExtensionLoad(DatabaseInstance &db, const string &name) { + } //! Called after an extension is finished loading virtual void OnExtensionLoaded(DatabaseInstance &db, const string &name) { } + //! Called after an extension fails to load loading + virtual void OnExtensionLoadFail(DatabaseInstance &db, const string &name, const ErrorData &error) { + } }; } // namespace duckdb diff --git a/src/include/duckdb/planner/logical_operator.hpp b/src/include/duckdb/planner/logical_operator.hpp index e7f533bdd6a7..743a9153b528 100644 --- a/src/include/duckdb/planner/logical_operator.hpp +++ b/src/include/duckdb/planner/logical_operator.hpp @@ -45,6 +45,7 @@ class LogicalOperator { public: virtual vector GetColumnBindings(); + virtual idx_t GetRootIndex(); static string ColumnBindingsToString(const vector &bindings); void PrintColumnBindings(); static vector GenerateColumnBindings(idx_t table_idx, idx_t column_count); diff --git a/src/include/duckdb/planner/operator/logical_cte.hpp b/src/include/duckdb/planner/operator/logical_cte.hpp index cd2ed3c21d8b..0548cd4e7965 100644 --- a/src/include/duckdb/planner/operator/logical_cte.hpp +++ b/src/include/duckdb/planner/operator/logical_cte.hpp @@ -35,6 +35,6 @@ class LogicalCTE : public LogicalOperator { string ctename; idx_t table_index; idx_t column_count; - vector correlated_columns; + CorrelatedColumns correlated_columns; }; } // namespace duckdb diff --git a/src/include/duckdb/planner/operator/logical_cteref.hpp b/src/include/duckdb/planner/operator/logical_cteref.hpp index a2c0b90bb801..0d2e79be4b5b 100644 --- a/src/include/duckdb/planner/operator/logical_cteref.hpp +++ b/src/include/duckdb/planner/operator/logical_cteref.hpp @@ -20,9 +20,9 @@ class LogicalCTERef : public LogicalOperator { public: LogicalCTERef(idx_t table_index, idx_t cte_index, vector types, vector colnames, - CTEMaterialize materialized_cte, bool is_recurring = false) + bool is_recurring = false) : LogicalOperator(LogicalOperatorType::LOGICAL_CTE_REF), table_index(table_index), cte_index(cte_index), - correlated_columns(0), materialized_cte(materialized_cte), is_recurring(is_recurring) { + correlated_columns(0), is_recurring(is_recurring) { D_ASSERT(!types.empty()); chunk_types = std::move(types); bound_columns = std::move(colnames); @@ -37,8 +37,6 @@ class LogicalCTERef : public LogicalOperator { vector chunk_types; //! Number of correlated columns idx_t correlated_columns; - //! Does this operator read a materialized CTE? - CTEMaterialize materialized_cte; //! Does this operator read the recurring CTE table bool is_recurring = false; diff --git a/src/include/duckdb/planner/operator/logical_dependent_join.hpp b/src/include/duckdb/planner/operator/logical_dependent_join.hpp index 724f2bc573c2..5e4c839193b4 100644 --- a/src/include/duckdb/planner/operator/logical_dependent_join.hpp +++ b/src/include/duckdb/planner/operator/logical_dependent_join.hpp @@ -27,7 +27,7 @@ class LogicalDependentJoin : public LogicalComparisonJoin { public: explicit LogicalDependentJoin(unique_ptr left, unique_ptr right, - vector correlated_columns, JoinType type, + CorrelatedColumns correlated_columns, JoinType type, unique_ptr condition); explicit LogicalDependentJoin(JoinType type); @@ -35,7 +35,7 @@ class LogicalDependentJoin : public LogicalComparisonJoin { //! The conditions of the join unique_ptr join_condition; //! The list of columns that have correlations with the right - vector correlated_columns; + CorrelatedColumns correlated_columns; SubqueryType subquery_type = SubqueryType::INVALID; bool perform_delim = true; @@ -51,7 +51,7 @@ class LogicalDependentJoin : public LogicalComparisonJoin { public: static unique_ptr Create(unique_ptr left, unique_ptr right, - vector correlated_columns, JoinType type, + CorrelatedColumns correlated_columns, JoinType type, unique_ptr condition); }; } // namespace duckdb diff --git a/src/include/duckdb/planner/operator/logical_prepare.hpp b/src/include/duckdb/planner/operator/logical_prepare.hpp index 58279c08a5f4..4a1abb40ebdc 100644 --- a/src/include/duckdb/planner/operator/logical_prepare.hpp +++ b/src/include/duckdb/planner/operator/logical_prepare.hpp @@ -46,7 +46,7 @@ class LogicalPrepare : public LogicalOperator { } bool RequireOptimizer() const override { - if (!prepared->properties.bound_all_parameters) { + if (!prepared->properties.bound_all_parameters || prepared->properties.always_require_rebind) { return false; } return children[0]->RequireOptimizer(); diff --git a/src/include/duckdb/planner/operator/logical_set_operation.hpp b/src/include/duckdb/planner/operator/logical_set_operation.hpp index 0dbfc43cd027..8900916dbad0 100644 --- a/src/include/duckdb/planner/operator/logical_set_operation.hpp +++ b/src/include/duckdb/planner/operator/logical_set_operation.hpp @@ -14,10 +14,7 @@ namespace duckdb { class LogicalSetOperation : public LogicalOperator { LogicalSetOperation(idx_t table_index, idx_t column_count, LogicalOperatorType type, bool setop_all, - bool allow_out_of_order) - : LogicalOperator(type), table_index(table_index), column_count(column_count), setop_all(setop_all), - allow_out_of_order(allow_out_of_order) { - } + bool allow_out_of_order); public: static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_INVALID; @@ -25,14 +22,9 @@ class LogicalSetOperation : public LogicalOperator { public: LogicalSetOperation(idx_t table_index, idx_t column_count, unique_ptr top, unique_ptr bottom, LogicalOperatorType type, bool setop_all, - bool allow_out_of_order = true) - : LogicalOperator(type), table_index(table_index), column_count(column_count), setop_all(setop_all), - allow_out_of_order(allow_out_of_order) { - D_ASSERT(type == LogicalOperatorType::LOGICAL_UNION || type == LogicalOperatorType::LOGICAL_EXCEPT || - type == LogicalOperatorType::LOGICAL_INTERSECT); - children.push_back(std::move(top)); - children.push_back(std::move(bottom)); - } + bool allow_out_of_order = true); + LogicalSetOperation(idx_t table_index, idx_t column_count, vector> children, + LogicalOperatorType type, bool setop_all, bool allow_out_of_order = true); idx_t table_index; idx_t column_count; diff --git a/src/include/duckdb/planner/query_node/bound_cte_node.hpp b/src/include/duckdb/planner/query_node/bound_cte_node.hpp deleted file mode 100644 index cbfdecd1f362..000000000000 --- a/src/include/duckdb/planner/query_node/bound_cte_node.hpp +++ /dev/null @@ -1,46 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/query_node/bound_cte_node.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/bound_query_node.hpp" - -namespace duckdb { - -class BoundCTENode : public BoundQueryNode { -public: - static constexpr const QueryNodeType TYPE = QueryNodeType::CTE_NODE; - -public: - BoundCTENode() : BoundQueryNode(QueryNodeType::CTE_NODE) { - } - - //! Keep track of the CTE name this node represents - string ctename; - - //! The cte node - unique_ptr query; - //! The child node - unique_ptr child; - //! Index used by the set operation - idx_t setop_index; - //! The binder used by the query side of the CTE - shared_ptr query_binder; - //! The binder used by the child side of the CTE - shared_ptr child_binder; - - CTEMaterialize materialized = CTEMaterialize::CTE_MATERIALIZE_DEFAULT; - -public: - idx_t GetRootIndex() override { - return child->GetRootIndex(); - } -}; - -} // namespace duckdb diff --git a/src/include/duckdb/planner/query_node/bound_recursive_cte_node.hpp b/src/include/duckdb/planner/query_node/bound_recursive_cte_node.hpp deleted file mode 100644 index 3da295e2ab79..000000000000 --- a/src/include/duckdb/planner/query_node/bound_recursive_cte_node.hpp +++ /dev/null @@ -1,49 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/query_node/bound_recursive_cte_node.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/bound_query_node.hpp" - -namespace duckdb { - -//! Bound equivalent of SetOperationNode -class BoundRecursiveCTENode : public BoundQueryNode { -public: - static constexpr const QueryNodeType TYPE = QueryNodeType::RECURSIVE_CTE_NODE; - -public: - BoundRecursiveCTENode() : BoundQueryNode(QueryNodeType::RECURSIVE_CTE_NODE) { - } - - //! Keep track of the CTE name this node represents - string ctename; - - bool union_all; - //! The left side of the set operation - unique_ptr left; - //! The right side of the set operation - unique_ptr right; - //! Target columns for the recursive key variant - vector> key_targets; - - //! Index used by the set operation - idx_t setop_index; - //! The binder used by the left side of the set operation - shared_ptr left_binder; - //! The binder used by the right side of the set operation - shared_ptr right_binder; - -public: - idx_t GetRootIndex() override { - return setop_index; - } -}; - -} // namespace duckdb diff --git a/src/include/duckdb/planner/query_node/bound_select_node.hpp b/src/include/duckdb/planner/query_node/bound_select_node.hpp index b3a22966a230..3fdc186e9a1a 100644 --- a/src/include/duckdb/planner/query_node/bound_select_node.hpp +++ b/src/include/duckdb/planner/query_node/bound_select_node.hpp @@ -11,7 +11,6 @@ #include "duckdb/planner/bound_query_node.hpp" #include "duckdb/planner/logical_operator.hpp" #include "duckdb/parser/expression_map.hpp" -#include "duckdb/planner/bound_tableref.hpp" #include "duckdb/parser/parsed_data/sample_options.hpp" #include "duckdb/parser/group_by_node.hpp" #include "duckdb/planner/expression_binder/select_bind_state.hpp" @@ -36,18 +35,12 @@ struct BoundUnnestNode { //! Bound equivalent of SelectNode class BoundSelectNode : public BoundQueryNode { public: - static constexpr const QueryNodeType TYPE = QueryNodeType::SELECT_NODE; - -public: - BoundSelectNode() : BoundQueryNode(QueryNodeType::SELECT_NODE) { - } - //! Bind information SelectBindState bind_state; //! The projection list vector> select_list; //! The FROM clause - unique_ptr from_table; + BoundStatement from_table; //! The WHERE clause unique_ptr where_clause; //! list of groups diff --git a/src/include/duckdb/planner/query_node/bound_set_operation_node.hpp b/src/include/duckdb/planner/query_node/bound_set_operation_node.hpp index e3ce0457e577..675007b50c1c 100644 --- a/src/include/duckdb/planner/query_node/bound_set_operation_node.hpp +++ b/src/include/duckdb/planner/query_node/bound_set_operation_node.hpp @@ -17,37 +17,17 @@ namespace duckdb { //! Bound equivalent of SetOperationNode class BoundSetOperationNode : public BoundQueryNode { public: - static constexpr const QueryNodeType TYPE = QueryNodeType::SET_OPERATION_NODE; - -public: - BoundSetOperationNode() : BoundQueryNode(QueryNodeType::SET_OPERATION_NODE) { - } - //! The type of set operation SetOperationType setop_type = SetOperationType::NONE; //! whether the ALL modifier was used or not bool setop_all = false; - //! The left side of the set operation - unique_ptr left; - //! The right side of the set operation - unique_ptr right; + //! The bound children + vector bound_children; + //! Child binders + vector> child_binders; //! Index used by the set operation idx_t setop_index; - //! The binder used by the left side of the set operation - shared_ptr left_binder; - //! The binder used by the right side of the set operation - shared_ptr right_binder; - - //! Exprs used by the UNION BY NAME opeartons to add a new projection - vector> left_reorder_exprs; - vector> right_reorder_exprs; - - //! The exprs of the child node may be rearranged(UNION BY NAME), - //! this vector records the new index of the expression after rearrangement - //! used by GatherAlias(...) function to create new reorder index - vector left_reorder_idx; - vector right_reorder_idx; public: idx_t GetRootIndex() override { diff --git a/src/include/duckdb/planner/query_node/list.hpp b/src/include/duckdb/planner/query_node/list.hpp index 5c7dbda9492b..dcac81248934 100644 --- a/src/include/duckdb/planner/query_node/list.hpp +++ b/src/include/duckdb/planner/query_node/list.hpp @@ -1,4 +1,2 @@ -#include "duckdb/planner/query_node/bound_recursive_cte_node.hpp" -#include "duckdb/planner/query_node/bound_cte_node.hpp" #include "duckdb/planner/query_node/bound_select_node.hpp" #include "duckdb/planner/query_node/bound_set_operation_node.hpp" diff --git a/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp b/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp index 2f343e901f23..14ad4510c9a2 100644 --- a/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +++ b/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp @@ -18,7 +18,7 @@ namespace duckdb { //! The FlattenDependentJoins class is responsible for pushing the dependent join down into the plan to create a //! flattened subquery struct FlattenDependentJoins { - FlattenDependentJoins(Binder &binder, const vector &correlated, bool perform_delim = true, + FlattenDependentJoins(Binder &binder, const CorrelatedColumns &correlated, bool perform_delim = true, bool any_join = false, optional_ptr parent = nullptr); static unique_ptr DecorrelateIndependent(Binder &binder, unique_ptr plan); @@ -47,7 +47,7 @@ struct FlattenDependentJoins { reference_map_t has_correlated_expressions; column_binding_map_t correlated_map; column_binding_map_t replacement_map; - const vector &correlated_columns; + const CorrelatedColumns &correlated_columns; vector delim_types; bool perform_delim; diff --git a/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp b/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp index 6b238ffccf2e..81a097b49ecb 100644 --- a/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +++ b/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp @@ -16,7 +16,7 @@ namespace duckdb { //! Helper class to recursively detect correlated expressions inside a single LogicalOperator class HasCorrelatedExpressions : public LogicalOperatorVisitor { public: - explicit HasCorrelatedExpressions(const vector &correlated, bool lateral = false, + explicit HasCorrelatedExpressions(const CorrelatedColumns &correlated, bool lateral = false, idx_t lateral_depth = 0); void VisitOperator(LogicalOperator &op) override; @@ -28,7 +28,7 @@ class HasCorrelatedExpressions : public LogicalOperatorVisitor { unique_ptr VisitReplace(BoundColumnRefExpression &expr, unique_ptr *expr_ptr) override; unique_ptr VisitReplace(BoundSubqueryExpression &expr, unique_ptr *expr_ptr) override; - const vector &correlated_columns; + const CorrelatedColumns &correlated_columns; // Tracks number of nested laterals idx_t lateral_depth; }; diff --git a/src/include/duckdb/planner/subquery/rewrite_cte_scan.hpp b/src/include/duckdb/planner/subquery/rewrite_cte_scan.hpp index e2c507e731bf..72886f80e59c 100644 --- a/src/include/duckdb/planner/subquery/rewrite_cte_scan.hpp +++ b/src/include/duckdb/planner/subquery/rewrite_cte_scan.hpp @@ -17,13 +17,13 @@ namespace duckdb { //! Helper class to rewrite correlated cte scans within a single LogicalOperator class RewriteCTEScan : public LogicalOperatorVisitor { public: - RewriteCTEScan(idx_t table_index, const vector &correlated_columns); + RewriteCTEScan(idx_t table_index, const CorrelatedColumns &correlated_columns); void VisitOperator(LogicalOperator &op) override; private: idx_t table_index; - const vector &correlated_columns; + const CorrelatedColumns &correlated_columns; }; } // namespace duckdb diff --git a/src/include/duckdb/planner/table_binding.hpp b/src/include/duckdb/planner/table_binding.hpp index 9aedc7e70058..836f52c417df 100644 --- a/src/include/duckdb/planner/table_binding.hpp +++ b/src/include/duckdb/planner/table_binding.hpp @@ -17,6 +17,7 @@ #include "duckdb/planner/binding_alias.hpp" #include "duckdb/common/column_index.hpp" #include "duckdb/common/table_column.hpp" +#include "duckdb/planner/bound_statement.hpp" namespace duckdb { class BindContext; @@ -26,30 +27,16 @@ class SubqueryRef; class LogicalGet; class TableCatalogEntry; class TableFunctionCatalogEntry; -class BoundTableFunction; class StandardEntry; struct ColumnBinding; -enum class BindingType { BASE, TABLE, DUMMY, CATALOG_ENTRY }; +enum class BindingType { BASE, TABLE, DUMMY, CATALOG_ENTRY, CTE }; //! A Binding represents a binding to a table, table-producing function or subquery with a specified table index. struct Binding { Binding(BindingType binding_type, BindingAlias alias, vector types, vector names, idx_t index); virtual ~Binding() = default; - //! The type of Binding - BindingType binding_type; - //! The alias of the binding - BindingAlias alias; - //! The table index of the binding - idx_t index; - //! The types of the bound columns - vector types; - //! Column names of the subquery - vector names; - //! Name -> index for the names - case_insensitive_map_t name_map; - public: bool TryGetBindingIndex(const string &column_name, column_t &column_index); column_t GetBindingIndex(const string &column_name); @@ -59,6 +46,14 @@ struct Binding { virtual optional_ptr GetStandardEntry(); string GetAlias() const; + BindingType GetBindingType(); + const BindingAlias &GetBindingAlias(); + idx_t GetIndex(); + const vector &GetColumnTypes(); + const vector &GetColumnNames(); + idx_t GetColumnCount(); + void SetColumnType(idx_t col_idx, LogicalType type); + static BindingAlias GetAlias(const string &explicit_alias, const StandardEntry &entry); static BindingAlias GetAlias(const string &explicit_alias, optional_ptr entry); @@ -78,6 +73,23 @@ struct Binding { } return reinterpret_cast(*this); } + +protected: + void Initialize(); + +protected: + //! The type of Binding + BindingType binding_type; + //! The alias of the binding + BindingAlias alias; + //! The table index of the binding + idx_t index; + //! The types of the bound columns + vector types; + //! Column names of the subquery + vector names; + //! Name -> index for the names + case_insensitive_map_t name_map; }; struct EntryBinding : public Binding { @@ -149,4 +161,44 @@ struct DummyBinding : public Binding { unique_ptr ParamToArg(ColumnRefExpression &col_ref); }; +enum class CTEType { CAN_BE_REFERENCED, CANNOT_BE_REFERENCED }; +struct CTEBinding; + +struct CTEBindState { + CTEBindState(Binder &parent_binder, QueryNode &cte_def, const vector &aliases); + ~CTEBindState(); + + Binder &parent_binder; + QueryNode &cte_def; + const vector &aliases; + idx_t active_binder_count; + shared_ptr query_binder; + BoundStatement query; + vector names; + vector types; + +public: + bool IsBound() const; + void Bind(CTEBinding &binding); +}; + +struct CTEBinding : public Binding { +public: + static constexpr const BindingType TYPE = BindingType::CTE; + +public: + CTEBinding(BindingAlias alias, vector types, vector names, idx_t index, CTEType type); + CTEBinding(BindingAlias alias, shared_ptr bind_state, idx_t index); + +public: + bool CanBeReferenced() const; + bool IsReferenced() const; + void Reference(); + +private: + CTEType cte_type; + idx_t reference_count; + shared_ptr bind_state; +}; + } // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/bound_basetableref.hpp b/src/include/duckdb/planner/tableref/bound_basetableref.hpp deleted file mode 100644 index b1f7f6f462c9..000000000000 --- a/src/include/duckdb/planner/tableref/bound_basetableref.hpp +++ /dev/null @@ -1,30 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/tableref/bound_basetableref.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/bound_tableref.hpp" -#include "duckdb/planner/logical_operator.hpp" - -namespace duckdb { -class TableCatalogEntry; - -//! Represents a TableReference to a base table in the schema -class BoundBaseTableRef : public BoundTableRef { -public: - static constexpr const TableReferenceType TYPE = TableReferenceType::BASE_TABLE; - -public: - BoundBaseTableRef(TableCatalogEntry &table, unique_ptr get) - : BoundTableRef(TableReferenceType::BASE_TABLE), table(table), get(std::move(get)) { - } - - TableCatalogEntry &table; - unique_ptr get; -}; -} // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/bound_column_data_ref.hpp b/src/include/duckdb/planner/tableref/bound_column_data_ref.hpp deleted file mode 100644 index 025bc47127b0..000000000000 --- a/src/include/duckdb/planner/tableref/bound_column_data_ref.hpp +++ /dev/null @@ -1,30 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/tableref/bound_column_data_ref.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/bound_tableref.hpp" -#include "duckdb/common/optionally_owned_ptr.hpp" -#include "duckdb/common/types/column/column_data_collection.hpp" - -namespace duckdb { -//! Represents a TableReference to a base table in the schema -class BoundColumnDataRef : public BoundTableRef { -public: - static constexpr const TableReferenceType TYPE = TableReferenceType::COLUMN_DATA; - -public: - explicit BoundColumnDataRef(optionally_owned_ptr collection) - : BoundTableRef(TableReferenceType::COLUMN_DATA), collection(std::move(collection)) { - } - //! The (optionally owned) materialized column data to scan - optionally_owned_ptr collection; - //! The index in the bind context - idx_t bind_index; -}; -} // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/bound_cteref.hpp b/src/include/duckdb/planner/tableref/bound_cteref.hpp deleted file mode 100644 index e79a35b64287..000000000000 --- a/src/include/duckdb/planner/tableref/bound_cteref.hpp +++ /dev/null @@ -1,43 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/tableref/bound_cteref.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/bound_tableref.hpp" -#include "duckdb/common/enums/cte_materialize.hpp" - -namespace duckdb { - -class BoundCTERef : public BoundTableRef { -public: - static constexpr const TableReferenceType TYPE = TableReferenceType::CTE; - -public: - BoundCTERef(idx_t bind_index, idx_t cte_index, CTEMaterialize materialized_cte) - : BoundTableRef(TableReferenceType::CTE), bind_index(bind_index), cte_index(cte_index), - materialized_cte(materialized_cte) { - } - - BoundCTERef(idx_t bind_index, idx_t cte_index, CTEMaterialize materialized_cte, bool is_recurring) - : BoundTableRef(TableReferenceType::CTE), bind_index(bind_index), cte_index(cte_index), - materialized_cte(materialized_cte), is_recurring(is_recurring) { - } - //! The set of columns bound to this base table reference - vector bound_columns; - //! The types of the values list - vector types; - //! The index in the bind context - idx_t bind_index; - //! The index of the cte - idx_t cte_index; - //! Is this a reference to a materialized CTE? - CTEMaterialize materialized_cte; - //! Is this a reference to the recurring table of a CTE - bool is_recurring = false; -}; -} // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/bound_delimgetref.hpp b/src/include/duckdb/planner/tableref/bound_delimgetref.hpp deleted file mode 100644 index 7b1022482eb3..000000000000 --- a/src/include/duckdb/planner/tableref/bound_delimgetref.hpp +++ /dev/null @@ -1,26 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/tableref/bound_delimgetref.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/bound_tableref.hpp" - -namespace duckdb { - -class BoundDelimGetRef : public BoundTableRef { -public: - static constexpr const TableReferenceType TYPE = TableReferenceType::DELIM_GET; - -public: - BoundDelimGetRef(idx_t bind_index, const vector &column_types_p) - : BoundTableRef(TableReferenceType::DELIM_GET), bind_index(bind_index), column_types(column_types_p) { - } - idx_t bind_index; - vector column_types; -}; -} // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/bound_dummytableref.hpp b/src/include/duckdb/planner/tableref/bound_dummytableref.hpp deleted file mode 100644 index 3a68f5166f78..000000000000 --- a/src/include/duckdb/planner/tableref/bound_dummytableref.hpp +++ /dev/null @@ -1,26 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/tableref/bound_dummytableref.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/bound_tableref.hpp" - -namespace duckdb { - -//! Represents a cross product -class BoundEmptyTableRef : public BoundTableRef { -public: - static constexpr const TableReferenceType TYPE = TableReferenceType::EMPTY_FROM; - -public: - explicit BoundEmptyTableRef(idx_t bind_index) - : BoundTableRef(TableReferenceType::EMPTY_FROM), bind_index(bind_index) { - } - idx_t bind_index; -}; -} // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/bound_expressionlistref.hpp b/src/include/duckdb/planner/tableref/bound_expressionlistref.hpp deleted file mode 100644 index 7fc563ddada5..000000000000 --- a/src/include/duckdb/planner/tableref/bound_expressionlistref.hpp +++ /dev/null @@ -1,33 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/tableref/bound_expressionlistref.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/bound_tableref.hpp" -#include "duckdb/planner/expression.hpp" - -namespace duckdb { -//! Represents a TableReference to a base table in the schema -class BoundExpressionListRef : public BoundTableRef { -public: - static constexpr const TableReferenceType TYPE = TableReferenceType::EXPRESSION_LIST; - -public: - BoundExpressionListRef() : BoundTableRef(TableReferenceType::EXPRESSION_LIST) { - } - - //! The bound VALUES list - vector>> values; - //! The generated names of the values list - vector names; - //! The types of the values list - vector types; - //! The index in the bind context - idx_t bind_index; -}; -} // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/bound_joinref.hpp b/src/include/duckdb/planner/tableref/bound_joinref.hpp index 38c83c95fbbd..87976ba3069b 100644 --- a/src/include/duckdb/planner/tableref/bound_joinref.hpp +++ b/src/include/duckdb/planner/tableref/bound_joinref.hpp @@ -11,19 +11,14 @@ #include "duckdb/planner/binder.hpp" #include "duckdb/common/enums/join_type.hpp" #include "duckdb/common/enums/joinref_type.hpp" -#include "duckdb/planner/bound_tableref.hpp" #include "duckdb/planner/expression.hpp" namespace duckdb { //! Represents a join -class BoundJoinRef : public BoundTableRef { +class BoundJoinRef { public: - static constexpr const TableReferenceType TYPE = TableReferenceType::JOIN; - -public: - explicit BoundJoinRef(JoinRefType ref_type) - : BoundTableRef(TableReferenceType::JOIN), type(JoinType::INNER), ref_type(ref_type), lateral(false) { + explicit BoundJoinRef(JoinRefType ref_type) : type(JoinType::INNER), ref_type(ref_type), lateral(false) { } //! The binder used to bind the LHS of the join @@ -31,9 +26,9 @@ class BoundJoinRef : public BoundTableRef { //! The binder used to bind the RHS of the join shared_ptr right_binder; //! The left hand side of the join - unique_ptr left; + BoundStatement left; //! The right hand side of the join - unique_ptr right; + BoundStatement right; //! The join condition unique_ptr condition; //! Duplicate Eliminated Columns (if any) @@ -47,7 +42,7 @@ class BoundJoinRef : public BoundTableRef { //! Whether or not this is a lateral join bool lateral; //! The correlated columns of the right-side with the left-side - vector correlated_columns; + CorrelatedColumns correlated_columns; //! The mark index, for mark joins generated by the relational API idx_t mark_index {}; }; diff --git a/src/include/duckdb/planner/tableref/bound_pivotref.hpp b/src/include/duckdb/planner/tableref/bound_pivotref.hpp index 3219f63072b2..5a2d68aa1967 100644 --- a/src/include/duckdb/planner/tableref/bound_pivotref.hpp +++ b/src/include/duckdb/planner/tableref/bound_pivotref.hpp @@ -9,7 +9,6 @@ #pragma once #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/bound_tableref.hpp" #include "duckdb/planner/expression.hpp" #include "duckdb/parser/tableref/pivotref.hpp" #include "duckdb/function/aggregate_function.hpp" @@ -30,19 +29,13 @@ struct BoundPivotInfo { static BoundPivotInfo Deserialize(Deserializer &deserializer); }; -class BoundPivotRef : public BoundTableRef { +class BoundPivotRef { public: - static constexpr const TableReferenceType TYPE = TableReferenceType::PIVOT; - -public: - explicit BoundPivotRef() : BoundTableRef(TableReferenceType::PIVOT) { - } - idx_t bind_index; //! The binder used to bind the child of the pivot shared_ptr child_binder; //! The child node of the pivot - unique_ptr child; + BoundStatement child; //! The bound pivot info BoundPivotInfo bound_pivot; }; diff --git a/src/include/duckdb/planner/tableref/bound_pos_join_ref.hpp b/src/include/duckdb/planner/tableref/bound_pos_join_ref.hpp deleted file mode 100644 index 4cb057e41280..000000000000 --- a/src/include/duckdb/planner/tableref/bound_pos_join_ref.hpp +++ /dev/null @@ -1,38 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/tableref/bound_pos_join_ref.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/bound_tableref.hpp" - -namespace duckdb { - -//! Represents a positional join -class BoundPositionalJoinRef : public BoundTableRef { -public: - static constexpr const TableReferenceType TYPE = TableReferenceType::POSITIONAL_JOIN; - -public: - BoundPositionalJoinRef() : BoundTableRef(TableReferenceType::POSITIONAL_JOIN), lateral(false) { - } - - //! The binder used to bind the LHS of the positional join - shared_ptr left_binder; - //! The binder used to bind the RHS of the positional join - shared_ptr right_binder; - //! The left hand side of the positional join - unique_ptr left; - //! The right hand side of the positional join - unique_ptr right; - //! Whether or not this is a lateral positional join - bool lateral; - //! The correlated columns of the right-side with the left-side - vector correlated_columns; -}; -} // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/bound_subqueryref.hpp b/src/include/duckdb/planner/tableref/bound_subqueryref.hpp deleted file mode 100644 index 2d1061c98f5a..000000000000 --- a/src/include/duckdb/planner/tableref/bound_subqueryref.hpp +++ /dev/null @@ -1,32 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/tableref/bound_subqueryref.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/bound_query_node.hpp" -#include "duckdb/planner/bound_tableref.hpp" - -namespace duckdb { - -//! Represents a cross product -class BoundSubqueryRef : public BoundTableRef { -public: - static constexpr const TableReferenceType TYPE = TableReferenceType::SUBQUERY; - -public: - BoundSubqueryRef(shared_ptr binder_p, unique_ptr subquery) - : BoundTableRef(TableReferenceType::SUBQUERY), binder(std::move(binder_p)), subquery(std::move(subquery)) { - } - - //! The binder used to bind the subquery - shared_ptr binder; - //! The bound subquery node (if any) - unique_ptr subquery; -}; -} // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/bound_table_function.hpp b/src/include/duckdb/planner/tableref/bound_table_function.hpp deleted file mode 100644 index 6aafe2b36b37..000000000000 --- a/src/include/duckdb/planner/tableref/bound_table_function.hpp +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/planner/tableref/bound_table_function.hpp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "duckdb/planner/bound_tableref.hpp" -#include "duckdb/planner/logical_operator.hpp" -#include "duckdb/planner/tableref/bound_subqueryref.hpp" - -namespace duckdb { - -//! Represents a reference to a table-producing function call -class BoundTableFunction : public BoundTableRef { -public: - static constexpr const TableReferenceType TYPE = TableReferenceType::TABLE_FUNCTION; - -public: - explicit BoundTableFunction(unique_ptr get) - : BoundTableRef(TableReferenceType::TABLE_FUNCTION), get(std::move(get)) { - } - - unique_ptr get; - unique_ptr subquery; -}; - -} // namespace duckdb diff --git a/src/include/duckdb/planner/tableref/list.hpp b/src/include/duckdb/planner/tableref/list.hpp index 79a00ce6285e..dbc8394df95d 100644 --- a/src/include/duckdb/planner/tableref/list.hpp +++ b/src/include/duckdb/planner/tableref/list.hpp @@ -1,11 +1,2 @@ -#include "duckdb/planner/tableref/bound_basetableref.hpp" -#include "duckdb/planner/tableref/bound_cteref.hpp" -#include "duckdb/planner/tableref/bound_dummytableref.hpp" -#include "duckdb/planner/tableref/bound_expressionlistref.hpp" #include "duckdb/planner/tableref/bound_joinref.hpp" -#include "duckdb/planner/tableref/bound_subqueryref.hpp" -#include "duckdb/planner/tableref/bound_column_data_ref.hpp" -#include "duckdb/planner/tableref/bound_table_function.hpp" #include "duckdb/planner/tableref/bound_pivotref.hpp" -#include "duckdb/parser/tableref/delimgetref.hpp" -#include "duckdb/planner/tableref/bound_delimgetref.hpp" diff --git a/src/include/duckdb/storage/block_manager.hpp b/src/include/duckdb/storage/block_manager.hpp index bfea18bea776..ef6f1941edc5 100644 --- a/src/include/duckdb/storage/block_manager.hpp +++ b/src/include/duckdb/storage/block_manager.hpp @@ -24,6 +24,8 @@ class ClientContext; class DatabaseInstance; class MetadataManager; +enum class ConvertToPersistentMode { DESTRUCTIVE, THREAD_SAFE }; + //! BlockManager is an abstract representation to manage blocks on DuckDB. When writing or reading blocks, the //! BlockManager creates and accesses blocks. The concrete types implement specific block storage strategies. class BlockManager { @@ -37,6 +39,9 @@ class BlockManager { BufferManager &buffer_manager; public: + BufferManager &GetBufferManager() const { + return buffer_manager; + } //! Creates a new block inside the block manager virtual unique_ptr ConvertBlock(block_id_t block_id, FileBuffer &source_buffer) = 0; virtual unique_ptr CreateBlock(block_id_t block_id, FileBuffer *source_buffer) = 0; @@ -82,6 +87,10 @@ class BlockManager { } //! Whether or not the attached database is in-memory virtual bool InMemory() = 0; + //! Whether or not to prefetch + virtual bool Prefetch() { + return false; + } //! Sync changes made to the block manager virtual void FileSync() = 0; @@ -91,10 +100,15 @@ class BlockManager { //! Register a block with the given block id in the base file shared_ptr RegisterBlock(block_id_t block_id); //! Convert an existing in-memory buffer into a persistent disk-backed block + //! If mode is set to destructive (default) - the old_block will be destroyed as part of this method + //! This can only be safely used when there is no other (lingering) usage of old_block + //! If there is concurrent usage of the block elsewhere - use the THREAD_SAFE mode which creates an extra copy shared_ptr ConvertToPersistent(QueryContext context, block_id_t block_id, - shared_ptr old_block, BufferHandle old_handle); + shared_ptr old_block, BufferHandle old_handle, + ConvertToPersistentMode mode = ConvertToPersistentMode::DESTRUCTIVE); shared_ptr ConvertToPersistent(QueryContext context, block_id_t block_id, - shared_ptr old_block); + shared_ptr old_block, + ConvertToPersistentMode mode = ConvertToPersistentMode::DESTRUCTIVE); void UnregisterBlock(BlockHandle &block); //! UnregisterBlock, only accepts non-temporary block ids diff --git a/src/include/duckdb/storage/buffer_manager.hpp b/src/include/duckdb/storage/buffer_manager.hpp index 619e89a5adeb..3d4f5e5950ce 100644 --- a/src/include/duckdb/storage/buffer_manager.hpp +++ b/src/include/duckdb/storage/buffer_manager.hpp @@ -58,6 +58,7 @@ class BufferManager { virtual void ReAllocate(shared_ptr &handle, idx_t block_size) = 0; //! Pin a block handle. virtual BufferHandle Pin(shared_ptr &handle) = 0; + virtual BufferHandle Pin(const QueryContext &context, shared_ptr &handle) = 0; //! Pre-fetch a series of blocks. //! Using this function is a performance suggestion. virtual void Prefetch(vector> &handles) = 0; @@ -100,6 +101,8 @@ class BufferManager { //! Set a new swap limit. virtual void SetSwapLimit(optional_idx limit = optional_idx()); + //! Get the block manager used for in-memory data + virtual BlockManager &GetTemporaryBlockManager() = 0; //! Get the temporary file information of each temporary file. virtual vector GetTemporaryFiles(); //! Get the path to the temporary file directory. diff --git a/src/include/duckdb/storage/caching_file_system.hpp b/src/include/duckdb/storage/caching_file_system.hpp index 99e949418964..93cec496d620 100644 --- a/src/include/duckdb/storage/caching_file_system.hpp +++ b/src/include/duckdb/storage/caching_file_system.hpp @@ -61,7 +61,8 @@ struct CachingFileHandle { //! Tries to read from the cache, filling "overlapping_ranges" with ranges that overlap with the request. //! Returns an invalid BufferHandle if it fails BufferHandle TryReadFromCache(data_ptr_t &buffer, idx_t nr_bytes, idx_t location, - vector> &overlapping_ranges); + vector> &overlapping_ranges, + optional_idx &start_location_of_next_range); //! Try to read from the specified range, return an invalid BufferHandle if it fails BufferHandle TryReadFromFileRange(const unique_ptr &guard, CachedFileRange &file_range, data_ptr_t &buffer, idx_t nr_bytes, idx_t location); diff --git a/src/include/duckdb/storage/checkpoint/row_group_writer.hpp b/src/include/duckdb/storage/checkpoint/row_group_writer.hpp index f86632c1bc98..ba1eea5b29da 100644 --- a/src/include/duckdb/storage/checkpoint/row_group_writer.hpp +++ b/src/include/duckdb/storage/checkpoint/row_group_writer.hpp @@ -40,6 +40,7 @@ class RowGroupWriter { virtual void FinishWritingColumns() { } + DatabaseInstance &GetDatabase(); PartialBlockManager &GetPartialBlockManager() { return partial_block_manager; } diff --git a/src/include/duckdb/storage/checkpoint/table_data_writer.hpp b/src/include/duckdb/storage/checkpoint/table_data_writer.hpp index 0860eb7c97ca..30dd141b3ac5 100644 --- a/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +++ b/src/include/duckdb/storage/checkpoint/table_data_writer.hpp @@ -30,7 +30,8 @@ class TableDataWriter { void WriteTableData(Serializer &metadata_serializer); virtual void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) = 0; - virtual void FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, Serializer &serializer) = 0; + virtual void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection, + Serializer &serializer) = 0; virtual unique_ptr GetRowGroupWriter(RowGroup &row_group) = 0; virtual void AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr writer); @@ -54,7 +55,8 @@ class SingleFileTableDataWriter : public TableDataWriter { public: void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) override; - void FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, Serializer &serializer) override; + void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection, + Serializer &serializer) override; unique_ptr GetRowGroupWriter(RowGroup &row_group) override; CheckpointType GetCheckpointType() const override; MetadataManager &GetMetadataManager() override; diff --git a/src/include/duckdb/storage/compression/alp/alp_analyze.hpp b/src/include/duckdb/storage/compression/alp/alp_analyze.hpp index bac590d0ec83..13eecf42b810 100644 --- a/src/include/duckdb/storage/compression/alp/alp_analyze.hpp +++ b/src/include/duckdb/storage/compression/alp/alp_analyze.hpp @@ -82,7 +82,12 @@ unique_ptr AlpInitAnalyze(ColumnData &col_data, PhysicalType type) */ template bool AlpAnalyze(AnalyzeState &state, Vector &input, idx_t count) { - auto &analyze_state = (AlpAnalyzeState &)state; + if (state.info.GetBlockSize() + state.info.GetBlockHeaderSize() < DEFAULT_BLOCK_ALLOC_SIZE) { + return false; + } + + auto &analyze_state = state.Cast>(); + bool must_skip_current_vector = alp::AlpUtils::MustSkipSamplingFromCurrentVector( analyze_state.vectors_count, analyze_state.vectors_sampled_count, count); analyze_state.vectors_count += 1; diff --git a/src/include/duckdb/storage/compression/alp/alp_scan.hpp b/src/include/duckdb/storage/compression/alp/alp_scan.hpp index 28b52b8483a0..8c7d12e673f3 100644 --- a/src/include/duckdb/storage/compression/alp/alp_scan.hpp +++ b/src/include/duckdb/storage/compression/alp/alp_scan.hpp @@ -201,7 +201,7 @@ struct AlpScanState : public SegmentScanState { }; template -unique_ptr AlpInitScan(ColumnSegment &segment) { +unique_ptr AlpInitScan(const QueryContext &context, ColumnSegment &segment) { auto result = make_uniq_base>(segment); return result; } diff --git a/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp b/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp index 25901667e0ef..da7f8bda0fcc 100644 --- a/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp +++ b/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp @@ -47,8 +47,12 @@ unique_ptr AlpRDInitAnalyze(ColumnData &col_data, PhysicalType typ */ template bool AlpRDAnalyze(AnalyzeState &state, Vector &input, idx_t count) { + if (state.info.GetBlockSize() + state.info.GetBlockHeaderSize() < DEFAULT_BLOCK_ALLOC_SIZE) { + return false; + } + using EXACT_TYPE = typename FloatingToExact::TYPE; - auto &analyze_state = (AlpRDAnalyzeState &)state; + auto &analyze_state = state.Cast>(); bool must_skip_current_vector = alp::AlpUtils::MustSkipSamplingFromCurrentVector( analyze_state.vectors_count, analyze_state.vectors_sampled_count, count); diff --git a/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp b/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp index a3feb94b529b..520d38fa2f17 100644 --- a/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp +++ b/src/include/duckdb/storage/compression/alprd/alprd_scan.hpp @@ -208,7 +208,7 @@ struct AlpRDScanState : public SegmentScanState { }; template -unique_ptr AlpRDInitScan(ColumnSegment &segment) { +unique_ptr AlpRDInitScan(const QueryContext &context, ColumnSegment &segment) { auto result = make_uniq_base>(segment); return result; } diff --git a/src/include/duckdb/storage/compression/chimp/chimp_scan.hpp b/src/include/duckdb/storage/compression/chimp/chimp_scan.hpp index de11979cb8d7..970b35f23391 100644 --- a/src/include/duckdb/storage/compression/chimp/chimp_scan.hpp +++ b/src/include/duckdb/storage/compression/chimp/chimp_scan.hpp @@ -252,7 +252,7 @@ struct ChimpScanState : public SegmentScanState { }; template -unique_ptr ChimpInitScan(ColumnSegment &segment) { +unique_ptr ChimpInitScan(const QueryContext &context, ColumnSegment &segment) { auto result = make_uniq_base>(segment); return result; } diff --git a/src/include/duckdb/storage/compression/dict_fsst/decompression.hpp b/src/include/duckdb/storage/compression/dict_fsst/decompression.hpp index 032370f8624c..1cb377baf8f7 100644 --- a/src/include/duckdb/storage/compression/dict_fsst/decompression.hpp +++ b/src/include/duckdb/storage/compression/dict_fsst/decompression.hpp @@ -59,7 +59,7 @@ struct CompressedStringScanState : public SegmentScanState { data_ptr_t dictionary_indices_ptr; data_ptr_t string_lengths_ptr; - buffer_ptr dictionary; + buffer_ptr dictionary; void *decoder = nullptr; bool all_values_inlined = false; diff --git a/src/include/duckdb/storage/compression/dictionary/decompression.hpp b/src/include/duckdb/storage/compression/dictionary/decompression.hpp index 1656ec718fd8..e7381f11a3be 100644 --- a/src/include/duckdb/storage/compression/dictionary/decompression.hpp +++ b/src/include/duckdb/storage/compression/dictionary/decompression.hpp @@ -41,7 +41,7 @@ struct CompressedStringScanState : public StringScanState { uint32_t *index_buffer_ptr; uint32_t index_buffer_count; - buffer_ptr dictionary; + buffer_ptr dictionary; idx_t dictionary_size; StringDictionaryContainer dict; idx_t block_size; diff --git a/src/include/duckdb/storage/compression/empty_validity.hpp b/src/include/duckdb/storage/compression/empty_validity.hpp index 1118f77f2467..476faf89fc1e 100644 --- a/src/include/duckdb/storage/compression/empty_validity.hpp +++ b/src/include/duckdb/storage/compression/empty_validity.hpp @@ -77,7 +77,7 @@ class EmptyValidityCompression { auto &checkpoint_state = checkpoint_data.GetCheckpointState(); checkpoint_state.FlushSegment(std::move(compressed_segment), std::move(handle), 0); } - static unique_ptr InitScan(ColumnSegment &segment) { + static unique_ptr InitScan(const QueryContext &context, ColumnSegment &segment) { return make_uniq(); } static void ScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result, diff --git a/src/include/duckdb/storage/compression/patas/patas_scan.hpp b/src/include/duckdb/storage/compression/patas/patas_scan.hpp index b523600e37e3..4261d2d23084 100644 --- a/src/include/duckdb/storage/compression/patas/patas_scan.hpp +++ b/src/include/duckdb/storage/compression/patas/patas_scan.hpp @@ -204,7 +204,7 @@ struct PatasScanState : public SegmentScanState { }; template -unique_ptr PatasInitScan(ColumnSegment &segment) { +unique_ptr PatasInitScan(const QueryContext &context, ColumnSegment &segment) { auto result = make_uniq_base>(segment); return result; } diff --git a/src/include/duckdb/storage/data_pointer.hpp b/src/include/duckdb/storage/data_pointer.hpp index 0ae44d7f3b61..54f6f239f9a8 100644 --- a/src/include/duckdb/storage/data_pointer.hpp +++ b/src/include/duckdb/storage/data_pointer.hpp @@ -19,6 +19,7 @@ namespace duckdb { class Serializer; class Deserializer; +class QueryContext; struct ColumnSegmentState { virtual ~ColumnSegmentState() { diff --git a/src/include/duckdb/storage/data_table.hpp b/src/include/duckdb/storage/data_table.hpp index 1c947e279170..bf355ed05fb7 100644 --- a/src/include/duckdb/storage/data_table.hpp +++ b/src/include/duckdb/storage/data_table.hpp @@ -42,6 +42,7 @@ struct TableDeleteState; struct ConstraintState; struct TableUpdateState; enum class VerifyExistenceType : uint8_t; +struct OptimisticWriteCollection; enum class DataTableVersion { MAIN_TABLE, // this is the newest version of the table - it has not been altered or dropped @@ -124,12 +125,12 @@ class DataTable : public enable_shared_from_this { const vector> &bound_constraints, optional_ptr> column_ids); //! Merge a row group collection into the transaction-local storage - void LocalMerge(ClientContext &context, RowGroupCollection &collection); + void LocalMerge(ClientContext &context, OptimisticWriteCollection &collection); //! Create an optimistic row group collection for this table. Used for optimistically writing parallel appends. //! Returns the index into the optimistic_collections vector for newly created collection. - PhysicalIndex CreateOptimisticCollection(ClientContext &context, unique_ptr collection); + PhysicalIndex CreateOptimisticCollection(ClientContext &context, unique_ptr collection); //! Returns the optimistic row group collection corresponding to the index. - RowGroupCollection &GetOptimisticCollection(ClientContext &context, const PhysicalIndex collection_index); + OptimisticWriteCollection &GetOptimisticCollection(ClientContext &context, const PhysicalIndex collection_index); //! Resets the optimistic row group collection corresponding to the index. void ResetOptimisticCollection(ClientContext &context, const PhysicalIndex collection_index); //! Returns the optimistic writer of the corresponding local table. @@ -195,7 +196,7 @@ class DataTable : public enable_shared_from_this { //! Remove the chunk with the specified set of row identifiers from all indexes of the table void RemoveFromIndexes(TableAppendState &state, DataChunk &chunk, Vector &row_identifiers); //! Remove the row identifiers from all the indexes of the table - void RemoveFromIndexes(Vector &row_identifiers, idx_t count); + void RemoveFromIndexes(const QueryContext &context, Vector &row_identifiers, idx_t count); void SetAsMainTable() { this->version = DataTableVersion::MAIN_TABLE; @@ -233,7 +234,7 @@ class DataTable : public enable_shared_from_this { idx_t ColumnCount() const; idx_t GetTotalRows() const; - vector GetColumnSegmentInfo(); + vector GetColumnSegmentInfo(const QueryContext &context); //! Scans the next chunk for the CREATE INDEX operator bool CreateIndexScan(TableScanState &state, DataChunk &result, TableScanType type); @@ -258,6 +259,7 @@ class DataTable : public enable_shared_from_this { void VacuumIndexes(); void VerifyIndexBuffers(); void CleanupAppend(transaction_t lowest_transaction, idx_t start, idx_t count); + void Destroy(); string GetTableName() const; void SetTableName(string new_name); diff --git a/src/include/duckdb/storage/metadata/metadata_manager.hpp b/src/include/duckdb/storage/metadata/metadata_manager.hpp index a3ff0cc580e3..88ace35be7aa 100644 --- a/src/include/duckdb/storage/metadata/metadata_manager.hpp +++ b/src/include/duckdb/storage/metadata/metadata_manager.hpp @@ -38,6 +38,9 @@ struct MetadataBlock { idx_t FreeBlocksToInteger(); void FreeBlocksFromInteger(idx_t blocks); + static vector BlocksFromInteger(idx_t free_list); + + string ToString() const; }; struct MetadataPointer { @@ -59,10 +62,14 @@ class MetadataManager { MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager); ~MetadataManager(); + BufferManager &GetBufferManager() const { + return buffer_manager; + } + MetadataHandle AllocateHandle(); MetadataHandle Pin(const MetadataPointer &pointer); - MetadataHandle Pin(QueryContext context, const MetadataPointer &pointer); + MetadataHandle Pin(const QueryContext &context, const MetadataPointer &pointer); MetaBlockPointer GetDiskPointer(const MetadataPointer &pointer, uint32_t offset = 0); MetadataPointer FromDiskPointer(MetaBlockPointer pointer); @@ -89,17 +96,19 @@ class MetadataManager { protected: BlockManager &block_manager; BufferManager &buffer_manager; + mutable mutex block_lock; unordered_map blocks; unordered_map modified_blocks; protected: - block_id_t AllocateNewBlock(); - block_id_t PeekNextBlockId(); - block_id_t GetNextBlockId(); - - void AddBlock(MetadataBlock new_block, bool if_exists = false); - void AddAndRegisterBlock(MetadataBlock block); - void ConvertToTransient(MetadataBlock &block); + block_id_t AllocateNewBlock(unique_lock &block_lock); + block_id_t PeekNextBlockId() const; + block_id_t GetNextBlockId() const; + + void AddBlock(unique_lock &block_lock, MetadataBlock new_block, bool if_exists = false); + void AddAndRegisterBlock(unique_lock &block_lock, MetadataBlock block); + void ConvertToTransient(unique_lock &block_lock, MetadataBlock &block); + MetadataPointer FromDiskPointerInternal(unique_lock &block_lock, MetaBlockPointer pointer); }; } // namespace duckdb diff --git a/src/include/duckdb/storage/optimistic_data_writer.hpp b/src/include/duckdb/storage/optimistic_data_writer.hpp index aa855e0f157c..98d37143720e 100644 --- a/src/include/duckdb/storage/optimistic_data_writer.hpp +++ b/src/include/duckdb/storage/optimistic_data_writer.hpp @@ -13,22 +13,38 @@ namespace duckdb { class PartialBlockManager; +struct OptimisticWriteCollection { + ~OptimisticWriteCollection(); + + shared_ptr collection; + idx_t last_flushed = 0; + idx_t complete_row_groups = 0; + vector> partial_block_managers; +}; + +enum class OptimisticWritePartialManagers { PER_COLUMN, GLOBAL }; + class OptimisticDataWriter { public: OptimisticDataWriter(ClientContext &context, DataTable &table); OptimisticDataWriter(DataTable &table, OptimisticDataWriter &parent); ~OptimisticDataWriter(); + //! Creates a collection to write to + unique_ptr + CreateCollection(DataTable &storage, const vector &insert_types, + OptimisticWritePartialManagers type = OptimisticWritePartialManagers::PER_COLUMN); //! Write a new row group to disk (if possible) - void WriteNewRowGroup(RowGroupCollection &row_groups); + void WriteNewRowGroup(OptimisticWriteCollection &row_groups); //! Write the last row group of a collection to disk - void WriteLastRowGroup(RowGroupCollection &row_groups); + void WriteLastRowGroup(OptimisticWriteCollection &row_groups); //! Final flush of the optimistic writer - fully flushes the partial block manager void FinalFlush(); //! Flushes a specific row group to disk - void FlushToDisk(RowGroup &row_group); + void FlushToDisk(OptimisticWriteCollection &collection, const vector> &row_groups); //! Merge the partially written blocks from one optimistic writer into another void Merge(OptimisticDataWriter &other); + void Merge(unique_ptr &other_manager); //! Rollback void Rollback(); diff --git a/src/include/duckdb/storage/serialization/logical_operator.json b/src/include/duckdb/storage/serialization/logical_operator.json index 7791419ee384..24f94e464e95 100644 --- a/src/include/duckdb/storage/serialization/logical_operator.json +++ b/src/include/duckdb/storage/serialization/logical_operator.json @@ -391,11 +391,6 @@ "name": "bound_columns", "type": "vector" }, - { - "id": 204, - "name": "materialized_cte", - "type": "CTEMaterialize" - }, { "id": 205, "name": "is_recurring", @@ -406,8 +401,7 @@ "table_index", "cte_index", "chunk_types", - "bound_columns", - "materialized_cte" + "bound_columns" ] }, { diff --git a/src/include/duckdb/storage/serialization/nodes.json b/src/include/duckdb/storage/serialization/nodes.json index e79584d8d15c..1cbece4e0d8e 100644 --- a/src/include/duckdb/storage/serialization/nodes.json +++ b/src/include/duckdb/storage/serialization/nodes.json @@ -41,7 +41,8 @@ { "id": 102, "name": "materialized", - "type": "CTEMaterialize" + "type": "CTEMaterialize", + "serialize_property": "GetMaterializedForSerialization(serializer)" }, { "id": 103, diff --git a/src/include/duckdb/storage/serialization/parse_info.json b/src/include/duckdb/storage/serialization/parse_info.json index 7c7ca5d60a03..28789ecc20ca 100644 --- a/src/include/duckdb/storage/serialization/parse_info.json +++ b/src/include/duckdb/storage/serialization/parse_info.json @@ -391,6 +391,34 @@ } ] }, + { + "class": "AlterDatabaseInfo", + "base": "AlterInfo", + "enum": "ALTER_DATABASE", + "class_type": "alter_database_type", + "includes": [ + "duckdb/parser/parsed_data/alter_database_info.hpp" + ], + "members": [ + { + "id": 300, + "name": "alter_database_type", + "type": "AlterDatabaseType" + } + ] + }, + { + "class": "RenameDatabaseInfo", + "base": "AlterDatabaseInfo", + "enum": "RENAME_DATABASE", + "members": [ + { + "id": 400, + "name": "new_name", + "type": "string" + } + ] + }, { "class": "AttachInfo", "base": "ParseInfo", diff --git a/src/include/duckdb/storage/serialization/query_node.json b/src/include/duckdb/storage/serialization/query_node.json index 5dac4d5a88a9..6242fa91d290 100644 --- a/src/include/duckdb/storage/serialization/query_node.json +++ b/src/include/duckdb/storage/serialization/query_node.json @@ -21,6 +21,11 @@ "name": "cte_map", "type": "CommonTableExpressionMap" } + ], + "finalize_deserialization": [ + "if (type == QueryNodeType::CTE_NODE) {", + "\tresult = std::move(result->Cast().child);", + "}" ] }, { @@ -90,12 +95,14 @@ { "id": 201, "name": "left", - "type": "QueryNode*" + "type": "QueryNode*", + "serialize_property": "SerializeChildNode(serializer, 0)" }, { "id": 202, "name": "right", - "type": "QueryNode*" + "type": "QueryNode*", + "serialize_property": "SerializeChildNode(serializer, 1)" }, { "id": 203, @@ -107,7 +114,7 @@ "id": 204, "name": "children", "type": "vector", - "serialize_property": "SerializeChildNodes()" + "version": "6" } ], "constructor": ["setop_type", "left", "right", "children", "setop_all"] diff --git a/src/include/duckdb/storage/serialization/storage.json b/src/include/duckdb/storage/serialization/storage.json index db2311e92cdd..f2dd8b0bfec1 100644 --- a/src/include/duckdb/storage/serialization/storage.json +++ b/src/include/duckdb/storage/serialization/storage.json @@ -110,7 +110,9 @@ { "id": 100, "name": "row_start", - "type": "uint64_t" + "type": "uint64_t", + "status": "deleted", + "version": "7" }, { "id": 101, diff --git a/src/include/duckdb/storage/serialization/types.json b/src/include/duckdb/storage/serialization/types.json index 9f622a3e27dd..6ce5c515aa5c 100644 --- a/src/include/duckdb/storage/serialization/types.json +++ b/src/include/duckdb/storage/serialization/types.json @@ -248,5 +248,12 @@ "type": "string" } ] + }, + { + "class": "GeoTypeInfo", + "base": "ExtraTypeInfo", + "enum": "GEO_TYPE_INFO", + "members": [ + ] } ] diff --git a/src/include/duckdb/storage/single_file_block_manager.hpp b/src/include/duckdb/storage/single_file_block_manager.hpp index b54a70319462..ae7d74fd3a5e 100644 --- a/src/include/duckdb/storage/single_file_block_manager.hpp +++ b/src/include/duckdb/storage/single_file_block_manager.hpp @@ -30,8 +30,8 @@ struct EncryptionOptions { bool additional_authenticated_data = false; //! derived encryption key id string derived_key_id; - //! Cipher used for encryption - EncryptionTypes::CipherType cipher = EncryptionTypes::CipherType::UNKNOWN; + // //! Cipher used for encryption + // EncryptionTypes::CipherType cipher = EncryptionTypes::CipherType::INVALID; //! key derivation function (kdf) used EncryptionTypes::KeyDerivationFunction kdf = EncryptionTypes::KeyDerivationFunction::SHA256; //! Key Length @@ -115,6 +115,8 @@ class SingleFileBlockManager : public BlockManager { idx_t FreeBlocks() override; //! Whether or not the attached database is a remote file bool IsRemote() override; + //! Whether or not to prefetch + bool Prefetch() override; //! Return the checkpoint iteration of the file. uint64_t GetCheckpointIteration() const { @@ -146,7 +148,7 @@ class SingleFileBlockManager : public BlockManager { idx_t GetBlockLocation(block_id_t block_id) const; // Encrypt, Store, Decrypt the canary - static void StoreEncryptedCanary(DatabaseInstance &db, MainHeader &main_header, const string &key_id); + static void StoreEncryptedCanary(AttachedDatabase &db, MainHeader &main_header, const string &key_id); static void StoreDBIdentifier(MainHeader &main_header, const data_ptr_t db_identifier); void StoreEncryptionMetadata(MainHeader &main_header) const; diff --git a/src/include/duckdb/storage/standard_buffer_manager.hpp b/src/include/duckdb/storage/standard_buffer_manager.hpp index d0a54c597c27..ff4ce4684525 100644 --- a/src/include/duckdb/storage/standard_buffer_manager.hpp +++ b/src/include/duckdb/storage/standard_buffer_manager.hpp @@ -71,7 +71,7 @@ class StandardBufferManager : public BufferManager { void ReAllocate(shared_ptr &handle, idx_t block_size) final; BufferHandle Pin(shared_ptr &handle) final; - BufferHandle Pin(QueryContext context, shared_ptr &handle); + BufferHandle Pin(const QueryContext &context, shared_ptr &handle) final; void Prefetch(vector> &handles) final; void Unpin(shared_ptr &handle) final; @@ -84,6 +84,8 @@ class StandardBufferManager : public BufferManager { //! Returns information about memory usage vector GetMemoryUsageInfo() const override; + BlockManager &GetTemporaryBlockManager() final; + //! Returns a list of all temporary files vector GetTemporaryFiles() final; diff --git a/src/include/duckdb/storage/statistics/base_statistics.hpp b/src/include/duckdb/storage/statistics/base_statistics.hpp index f8b59dc1e528..e37879f61b18 100644 --- a/src/include/duckdb/storage/statistics/base_statistics.hpp +++ b/src/include/duckdb/storage/statistics/base_statistics.hpp @@ -15,6 +15,7 @@ #include "duckdb/common/types/value.hpp" #include "duckdb/storage/statistics/numeric_stats.hpp" #include "duckdb/storage/statistics/string_stats.hpp" +#include "duckdb/storage/statistics/geometry_stats.hpp" namespace duckdb { struct SelectionVector; @@ -33,7 +34,15 @@ enum class StatsInfo : uint8_t { CAN_HAVE_NULL_AND_VALID_VALUES = 4 }; -enum class StatisticsType : uint8_t { NUMERIC_STATS, STRING_STATS, LIST_STATS, STRUCT_STATS, BASE_STATS, ARRAY_STATS }; +enum class StatisticsType : uint8_t { + NUMERIC_STATS, + STRING_STATS, + LIST_STATS, + STRUCT_STATS, + BASE_STATS, + ARRAY_STATS, + GEOMETRY_STATS +}; class BaseStatistics { friend struct NumericStats; @@ -41,6 +50,7 @@ class BaseStatistics { friend struct StructStats; friend struct ListStats; friend struct ArrayStats; + friend struct GeometryStats; public: DUCKDB_API ~BaseStatistics(); @@ -83,7 +93,7 @@ class BaseStatistics { inline void SetHasNullFast() { has_null = true; } - //! Set that the CURRENT level can have valiod values + //! Set that the CURRENT level can have valid values //! Note that this is not correct for nested types unless this information is propagated in a different manner //! Use Set(StatsInfo::CAN_HAVE_VALID_VALUES) in the general case inline void SetHasNoNullFast() { @@ -104,7 +114,7 @@ class BaseStatistics { static BaseStatistics Deserialize(Deserializer &deserializer); //! Verify that a vector does not violate the statistics - void Verify(Vector &vector, const SelectionVector &sel, idx_t count) const; + void Verify(Vector &vector, const SelectionVector &sel, idx_t count, bool ignore_has_null = false) const; void Verify(Vector &vector, idx_t count) const; string ToString() const; @@ -146,6 +156,8 @@ class BaseStatistics { NumericStatsData numeric_data; //! String stats data, for string stats StringStatsData string_data; + //! Geometry stats data, for geometry stats + GeometryStatsData geometry_data; } stats_union; //! Child stats (for LIST and STRUCT) unsafe_unique_array child_stats; diff --git a/src/include/duckdb/storage/statistics/geometry_stats.hpp b/src/include/duckdb/storage/statistics/geometry_stats.hpp new file mode 100644 index 000000000000..e8db7285b780 --- /dev/null +++ b/src/include/duckdb/storage/statistics/geometry_stats.hpp @@ -0,0 +1,144 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/storage/statistics/geometry_stats.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/common.hpp" +#include "duckdb/common/enums/expression_type.hpp" +#include "duckdb/common/enums/filter_propagate_result.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/common/operator/comparison_operators.hpp" +#include "duckdb/common/types/hugeint.hpp" +#include "duckdb/common/array_ptr.hpp" +#include "duckdb/common/types/geometry.hpp" + +namespace duckdb { +class BaseStatistics; +struct SelectionVector; + +class GeometryTypeSet { +public: + static constexpr auto VERT_TYPES = 4; + static constexpr auto PART_TYPES = 8; + + static GeometryTypeSet Unknown() { + GeometryTypeSet result; + for (idx_t i = 0; i < VERT_TYPES; i++) { + result.sets[i] = 0xFF; + } + return result; + } + static GeometryTypeSet Empty() { + GeometryTypeSet result; + for (idx_t i = 0; i < VERT_TYPES; i++) { + result.sets[i] = 0; + } + return result; + } + + bool IsEmpty() const { + for (idx_t i = 0; i < VERT_TYPES; i++) { + if (sets[i] != 0) { + return false; + } + } + return true; + } + + bool IsUnknown() const { + for (idx_t i = 0; i < VERT_TYPES; i++) { + if (sets[i] != 0xFF) { + return false; + } + } + return true; + } + + void Add(GeometryType geom_type, VertexType vert_type) { + const auto vert_idx = static_cast(vert_type); + const auto geom_idx = static_cast(geom_type); + D_ASSERT(vert_idx < VERT_TYPES); + D_ASSERT(geom_idx < PART_TYPES); + sets[vert_idx] |= (1 << geom_idx); + } + + void Merge(const GeometryTypeSet &other) { + for (idx_t i = 0; i < VERT_TYPES; i++) { + sets[i] |= other.sets[i]; + } + } + + vector ToWKBList() const { + vector result; + for (uint8_t vert_idx = 0; vert_idx < VERT_TYPES; vert_idx++) { + for (uint8_t geom_idx = 1; geom_idx < PART_TYPES; geom_idx++) { + if (sets[vert_idx] & (1 << geom_idx)) { + result.push_back(geom_idx + vert_idx * 1000); + } + } + } + return result; + } + + vector ToString(bool snake_case) const; + + uint8_t sets[VERT_TYPES]; +}; + +struct GeometryStatsData { + + GeometryTypeSet types; + GeometryExtent extent; + + void SetEmpty() { + types = GeometryTypeSet::Empty(); + extent = GeometryExtent::Empty(); + } + + void SetUnknown() { + types = GeometryTypeSet::Unknown(); + extent = GeometryExtent::Unknown(); + } + + void Merge(const GeometryStatsData &other) { + types.Merge(other.types); + extent.Merge(other.extent); + } + + void Update(const string_t &geom_blob) { + + // Parse type + const auto type_info = Geometry::GetType(geom_blob); + types.Add(type_info.first, type_info.second); + + // Update extent + Geometry::GetExtent(geom_blob, extent); + } +}; + +struct GeometryStats { + //! Unknown statistics + DUCKDB_API static BaseStatistics CreateUnknown(LogicalType type); + //! Empty statistics + DUCKDB_API static BaseStatistics CreateEmpty(LogicalType type); + + DUCKDB_API static void Serialize(const BaseStatistics &stats, Serializer &serializer); + DUCKDB_API static void Deserialize(Deserializer &deserializer, BaseStatistics &base); + + DUCKDB_API static string ToString(const BaseStatistics &stats); + + DUCKDB_API static void Update(BaseStatistics &stats, const string_t &value); + DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other); + DUCKDB_API static void Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count); + +private: + static GeometryStatsData &GetDataUnsafe(BaseStatistics &stats); + static const GeometryStatsData &GetDataUnsafe(const BaseStatistics &stats); +}; + +} // namespace duckdb diff --git a/src/include/duckdb/storage/statistics/string_stats.hpp b/src/include/duckdb/storage/statistics/string_stats.hpp index 0982f8905aea..6e5814a363c5 100644 --- a/src/include/duckdb/storage/statistics/string_stats.hpp +++ b/src/include/duckdb/storage/statistics/string_stats.hpp @@ -71,6 +71,8 @@ struct StringStats { ExpressionType comparison_type, const string &value); DUCKDB_API static void Update(BaseStatistics &stats, const string_t &value); + DUCKDB_API static void SetMin(BaseStatistics &stats, const string_t &value); + DUCKDB_API static void SetMax(BaseStatistics &stats, const string_t &value); DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other); DUCKDB_API static void Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count); diff --git a/src/include/duckdb/storage/storage_info.hpp b/src/include/duckdb/storage/storage_info.hpp index 0da8582925e9..804a76403956 100644 --- a/src/include/duckdb/storage/storage_info.hpp +++ b/src/include/duckdb/storage/storage_info.hpp @@ -12,6 +12,7 @@ #include "duckdb/common/limits.hpp" #include "duckdb/common/string.hpp" #include "duckdb/common/vector_size.hpp" +#include "duckdb/common/encryption_state.hpp" namespace duckdb { @@ -34,7 +35,7 @@ class QueryContext; //! The configurable block allocation size. #ifndef DUCKDB_BLOCK_HEADER_STORAGE_SIZE #define DUCKDB_BLOCK_HEADER_STORAGE_SIZE DEFAULT_BLOCK_HEADER_STORAGE_SIZE -#define DEFAULT_ENCRYPTED_BUFFER_HEADER_SIZE 28ULL +#define DEFAULT_ENCRYPTED_BUFFER_HEADER_SIZE 32ULL #endif using block_id_t = int64_t; @@ -55,8 +56,8 @@ struct Storage { constexpr static idx_t DEFAULT_BLOCK_HEADER_SIZE = sizeof(idx_t); //! The default block header size for blocks written to storage. constexpr static idx_t MAX_BLOCK_HEADER_SIZE = 128ULL; - //! Block header size for encrypted blocks (40 bytes) - constexpr static idx_t ENCRYPTED_BLOCK_HEADER_SIZE = 40ULL; + //! Block header size for encrypted blocks (64 bytes) + constexpr static idx_t ENCRYPTED_BLOCK_HEADER_SIZE = 64ULL; //! The default block size. constexpr static idx_t DEFAULT_BLOCK_SIZE = DEFAULT_BLOCK_ALLOC_SIZE - DEFAULT_BLOCK_HEADER_SIZE; @@ -104,8 +105,8 @@ class MainHeader { static constexpr idx_t ENCRYPTION_METADATA_LEN = 8; //! The canary is a known plaintext for detecting wrong keys early. static constexpr idx_t CANARY_BYTE_SIZE = 8; - //! Nonce, IV (nonce + counter) and tag length. - static constexpr uint64_t AES_NONCE_LEN = 12; + //! Nonce, IV (nonce + counter) and tag length + static constexpr uint64_t AES_NONCE_LEN = 16; static constexpr uint64_t AES_IV_LEN = 16; static constexpr uint64_t AES_TAG_LEN = 16; @@ -119,7 +120,10 @@ class MainHeader { } bool IsEncrypted() const { - return flags[0] == MainHeader::ENCRYPTED_DATABASE_FLAG; + return flags[0] & MainHeader::ENCRYPTED_DATABASE_FLAG; + } + void SetEncrypted() { + flags[0] |= MainHeader::ENCRYPTED_DATABASE_FLAG; } void SetEncryptionMetadata(data_ptr_t source) { @@ -127,6 +131,10 @@ class MainHeader { memcpy(encryption_metadata, source, ENCRYPTION_METADATA_LEN); } + EncryptionTypes::CipherType GetEncryptionCipher() { + return static_cast(encryption_metadata[2]); + } + void SetDBIdentifier(data_ptr_t source) { memset(db_identifier, 0, DB_IDENTIFIER_LEN); memcpy(db_identifier, source, DB_IDENTIFIER_LEN); diff --git a/src/include/duckdb/storage/storage_manager.hpp b/src/include/duckdb/storage/storage_manager.hpp index ea60405b5238..c96a76ff77dc 100644 --- a/src/include/duckdb/storage/storage_manager.hpp +++ b/src/include/duckdb/storage/storage_manager.hpp @@ -104,6 +104,7 @@ class StorageManager { virtual vector GetMetadataInfo() = 0; virtual shared_ptr GetTableIOManager(BoundCreateTableInfo *info) = 0; virtual BlockManager &GetBlockManager() = 0; + virtual void Destroy(); void SetStorageVersion(idx_t version) { storage_version = version; @@ -124,6 +125,19 @@ class StorageManager { bool CompressionIsEnabled() const { return storage_options.compress_in_memory == CompressInMemory::COMPRESS; } + EncryptionTypes::CipherType GetCipher() const { + return storage_options.encryption_cipher; + } + void SetCipher(EncryptionTypes::CipherType cipher_p) { + D_ASSERT(cipher_p != EncryptionTypes::INVALID); + if (cipher_p == EncryptionTypes::CBC) { + throw InvalidInputException("CBC cipher is disabled"); + } + storage_options.encryption_cipher = cipher_p; + } + bool IsEncrypted() const { + return storage_options.encryption; + } protected: virtual void LoadDatabase(QueryContext context) = 0; @@ -180,6 +194,7 @@ class SingleFileStorageManager : public StorageManager { vector GetMetadataInfo() override; shared_ptr GetTableIOManager(BoundCreateTableInfo *info) override; BlockManager &GetBlockManager() override; + void Destroy() override; protected: void LoadDatabase(QueryContext context) override; diff --git a/src/include/duckdb/storage/storage_options.hpp b/src/include/duckdb/storage/storage_options.hpp index a3ff65acc4e5..786924b2cf6a 100644 --- a/src/include/duckdb/storage/storage_options.hpp +++ b/src/include/duckdb/storage/storage_options.hpp @@ -11,6 +11,7 @@ #include "duckdb/common/common.hpp" #include "duckdb/common/optional_idx.hpp" #include "duckdb/common/types/value.hpp" +#include "duckdb/common/encryption_state.hpp" namespace duckdb { @@ -30,8 +31,8 @@ struct StorageOptions { //! Whether the database is encrypted bool encryption = false; - //! Encryption algorithm (default = GCM) - string encryption_cipher = "gcm"; + //! Encryption algorithm + EncryptionTypes::CipherType encryption_cipher = EncryptionTypes::INVALID; //! encryption key //! FIXME: change to a unique_ptr in the future shared_ptr user_key; diff --git a/src/include/duckdb/storage/string_uncompressed.hpp b/src/include/duckdb/storage/string_uncompressed.hpp index b5342829cffe..755e99339ac3 100644 --- a/src/include/duckdb/storage/string_uncompressed.hpp +++ b/src/include/duckdb/storage/string_uncompressed.hpp @@ -67,7 +67,7 @@ struct UncompressedStringStorage { static unique_ptr StringInitAnalyze(ColumnData &col_data, PhysicalType type); static bool StringAnalyze(AnalyzeState &state_p, Vector &input, idx_t count); static idx_t StringFinalAnalyze(AnalyzeState &state_p); - static unique_ptr StringInitScan(ColumnSegment &segment); + static unique_ptr StringInitScan(const QueryContext &context, ColumnSegment &segment); static void StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset); static void StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result); @@ -201,7 +201,11 @@ struct UncompressedStringStorage { public: static inline void UpdateStringStats(SegmentStatistics &stats, const string_t &new_value) { - StringStats::Update(stats.statistics, new_value); + if (stats.statistics.GetStatsType() == StatisticsType::GEOMETRY_STATS) { + GeometryStats::Update(stats.statistics, new_value); + } else { + StringStats::Update(stats.statistics, new_value); + } } static void SetDictionary(ColumnSegment &segment, BufferHandle &handle, StringDictionaryContainer dict); diff --git a/src/include/duckdb/storage/table/array_column_data.hpp b/src/include/duckdb/storage/table/array_column_data.hpp index abc9577a3b0a..c246d68b6ddc 100644 --- a/src/include/duckdb/storage/table/array_column_data.hpp +++ b/src/include/duckdb/storage/table/array_column_data.hpp @@ -48,10 +48,10 @@ class ArrayColumnData : public ColumnData { idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override; void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) override; - void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) override; - void UpdateColumn(TransactionData transaction, const vector &column_path, Vector &update_vector, - row_t *row_ids, idx_t update_count, idx_t depth) override; + void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector, + row_t *row_ids, idx_t update_count) override; + void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector &column_path, + Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override; unique_ptr GetUpdateStatistics() override; void CommitDropColumn() override; @@ -65,8 +65,8 @@ class ArrayColumnData : public ColumnData { PersistentColumnData Serialize() override; void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override; - void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector col_path, - vector &result) override; + void GetColumnSegmentInfo(const QueryContext &context, duckdb::idx_t row_group_index, + vector col_path, vector &result) override; void Verify(RowGroup &parent) override; }; diff --git a/src/include/duckdb/storage/table/chunk_info.hpp b/src/include/duckdb/storage/table/chunk_info.hpp index 44b92dd7437e..9e33b42019a3 100644 --- a/src/include/duckdb/storage/table/chunk_info.hpp +++ b/src/include/duckdb/storage/table/chunk_info.hpp @@ -11,6 +11,7 @@ #include "duckdb/common/common.hpp" #include "duckdb/common/vector_size.hpp" #include "duckdb/common/atomic.hpp" +#include "duckdb/execution/index/index_pointer.hpp" namespace duckdb { class RowGroup; @@ -20,6 +21,7 @@ struct TransactionData; struct DeleteInfo; class Serializer; class Deserializer; +class FixedSizeAllocator; enum class ChunkInfoType : uint8_t { CONSTANT_INFO, VECTOR_INFO, EMPTY_INFO }; @@ -38,19 +40,19 @@ class ChunkInfo { public: //! Gets up to max_count entries from the chunk info. If the ret is 0>ret>max_count, the selection vector is filled //! with the tuples - virtual idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) = 0; + virtual idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) const = 0; virtual idx_t GetCommittedSelVector(transaction_t min_start_id, transaction_t min_transaction_id, SelectionVector &sel_vector, idx_t max_count) = 0; //! Returns whether or not a single row in the ChunkInfo should be used or not for the given transaction virtual bool Fetch(TransactionData transaction, row_t row) = 0; virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0; - virtual idx_t GetCommittedDeletedCount(idx_t max_count) = 0; + virtual idx_t GetCommittedDeletedCount(idx_t max_count) const = 0; virtual bool Cleanup(transaction_t lowest_transaction, unique_ptr &result) const; virtual bool HasDeletes() const = 0; virtual void Write(WriteStream &writer) const; - static unique_ptr Read(ReadStream &reader); + static unique_ptr Read(FixedSizeAllocator &allocator, ReadStream &reader); public: template @@ -81,12 +83,12 @@ class ChunkConstantInfo : public ChunkInfo { transaction_t delete_id; public: - idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) override; + idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) const override; idx_t GetCommittedSelVector(transaction_t min_start_id, transaction_t min_transaction_id, SelectionVector &sel_vector, idx_t max_count) override; bool Fetch(TransactionData transaction, row_t row) override; void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override; - idx_t GetCommittedDeletedCount(idx_t max_count) override; + idx_t GetCommittedDeletedCount(idx_t max_count) const override; bool Cleanup(transaction_t lowest_transaction, unique_ptr &result) const override; bool HasDeletes() const override; @@ -105,27 +107,19 @@ class ChunkVectorInfo : public ChunkInfo { static constexpr const ChunkInfoType TYPE = ChunkInfoType::VECTOR_INFO; public: - explicit ChunkVectorInfo(idx_t start); - - //! The transaction ids of the transactions that inserted the tuples (if any) - transaction_t inserted[STANDARD_VECTOR_SIZE]; - transaction_t insert_id; - bool same_inserted_id; - - //! The transaction ids of the transactions that deleted the tuples (if any) - transaction_t deleted[STANDARD_VECTOR_SIZE]; - bool any_deleted; + explicit ChunkVectorInfo(FixedSizeAllocator &allocator, idx_t start, transaction_t insert_id = 0); + ~ChunkVectorInfo() override; public: idx_t GetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector, idx_t max_count) const; - idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) override; + idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) const override; idx_t GetCommittedSelVector(transaction_t min_start_id, transaction_t min_transaction_id, SelectionVector &sel_vector, idx_t max_count) override; bool Fetch(TransactionData transaction, row_t row) override; void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override; bool Cleanup(transaction_t lowest_transaction, unique_ptr &result) const override; - idx_t GetCommittedDeletedCount(idx_t max_count) override; + idx_t GetCommittedDeletedCount(idx_t max_count) const override; void Append(idx_t start, idx_t end, transaction_t commit_id); @@ -138,14 +132,32 @@ class ChunkVectorInfo : public ChunkInfo { void CommitDelete(transaction_t commit_id, const DeleteInfo &info); bool HasDeletes() const override; + bool AnyDeleted() const; + bool HasConstantInsertionId() const; + transaction_t ConstantInsertId() const; void Write(WriteStream &writer) const override; - static unique_ptr Read(ReadStream &reader); + static unique_ptr Read(FixedSizeAllocator &allocator, ReadStream &reader); private: template idx_t TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector, idx_t max_count) const; + + IndexPointer GetInsertedPointer() const; + IndexPointer GetDeletedPointer() const; + IndexPointer GetInitializedInsertedPointer(); + IndexPointer GetInitializedDeletedPointer(); + +private: + FixedSizeAllocator &allocator; + //! The transaction ids of the transactions that inserted the tuples (if any) + IndexPointer inserted_data; + //! The constant insert id (if there is only one) + transaction_t constant_insert_id; + + //! The transaction ids of the transactions that deleted the tuples (if any) + IndexPointer deleted_data; }; } // namespace duckdb diff --git a/src/include/duckdb/storage/table/column_data.hpp b/src/include/duckdb/storage/table/column_data.hpp index 60f99962be5b..ab8a5970e99d 100644 --- a/src/include/duckdb/storage/table/column_data.hpp +++ b/src/include/duckdb/storage/table/column_data.hpp @@ -39,14 +39,16 @@ struct PersistentColumnData; using column_segment_vector_t = vector>; struct ColumnCheckpointInfo { - ColumnCheckpointInfo(RowGroupWriteInfo &info, idx_t column_idx) : info(info), column_idx(column_idx) { - } + ColumnCheckpointInfo(RowGroupWriteInfo &info, idx_t column_idx); - RowGroupWriteInfo &info; idx_t column_idx; public: + PartialBlockManager &GetPartialBlockManager(); CompressionType GetCompressionType(); + +private: + RowGroupWriteInfo &info; }; class ColumnData { @@ -154,10 +156,10 @@ class ColumnData { virtual void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx); - virtual void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count); - virtual void UpdateColumn(TransactionData transaction, const vector &column_path, Vector &update_vector, - row_t *row_ids, idx_t update_count, idx_t depth); + virtual void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector, + row_t *row_ids, idx_t update_count); + virtual void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector &column_path, + Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth); virtual unique_ptr GetUpdateStatistics(); virtual void CommitDropColumn(); @@ -178,7 +180,8 @@ class ColumnData { static shared_ptr Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row, ReadStream &source, const LogicalType &type); - virtual void GetColumnSegmentInfo(idx_t row_group_index, vector col_path, vector &result); + virtual void GetColumnSegmentInfo(const QueryContext &context, idx_t row_group_index, vector col_path, + vector &result); virtual void Verify(RowGroup &parent); FilterPropagateResult CheckZonemap(TableFilter &filter); @@ -217,9 +220,9 @@ class ColumnData { void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count, bool allow_updates, bool scan_committed); void FetchUpdateRow(TransactionData transaction, row_t row_id, Vector &result, idx_t result_idx); - void UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count, Vector &base_vector); - idx_t FetchUpdateData(row_t *row_ids, Vector &base_vector); + void UpdateInternal(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector, + row_t *row_ids, idx_t update_count, Vector &base_vector); + idx_t FetchUpdateData(ColumnScanState &state, row_t *row_ids, Vector &base_vector); idx_t GetVectorCount(idx_t vector_index) const; diff --git a/src/include/duckdb/storage/table/column_segment.hpp b/src/include/duckdb/storage/table/column_segment.hpp index 61b2c0d4ffe3..e9966495864f 100644 --- a/src/include/duckdb/storage/table/column_segment.hpp +++ b/src/include/duckdb/storage/table/column_segment.hpp @@ -29,7 +29,6 @@ class DatabaseInstance; class TableFilter; class Transaction; class UpdateSegment; - struct ColumnAppendState; struct ColumnFetchState; struct ColumnScanState; diff --git a/src/include/duckdb/storage/table/in_memory_checkpoint.hpp b/src/include/duckdb/storage/table/in_memory_checkpoint.hpp index 68896f664c0f..e36e6169d1f0 100644 --- a/src/include/duckdb/storage/table/in_memory_checkpoint.hpp +++ b/src/include/duckdb/storage/table/in_memory_checkpoint.hpp @@ -50,7 +50,8 @@ class InMemoryTableDataWriter : public TableDataWriter { public: void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) override; - void FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, Serializer &serializer) override; + void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection, + Serializer &serializer) override; unique_ptr GetRowGroupWriter(RowGroup &row_group) override; CheckpointType GetCheckpointType() const override; MetadataManager &GetMetadataManager() override; diff --git a/src/include/duckdb/storage/table/list_column_data.hpp b/src/include/duckdb/storage/table/list_column_data.hpp index c8e75d136e6e..621ece4517fe 100644 --- a/src/include/duckdb/storage/table/list_column_data.hpp +++ b/src/include/duckdb/storage/table/list_column_data.hpp @@ -46,10 +46,10 @@ class ListColumnData : public ColumnData { idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override; void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) override; - void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) override; - void UpdateColumn(TransactionData transaction, const vector &column_path, Vector &update_vector, - row_t *row_ids, idx_t update_count, idx_t depth) override; + void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector, + row_t *row_ids, idx_t update_count) override; + void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector &column_path, + Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override; unique_ptr GetUpdateStatistics() override; void CommitDropColumn() override; @@ -63,8 +63,8 @@ class ListColumnData : public ColumnData { PersistentColumnData Serialize() override; void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override; - void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector col_path, - vector &result) override; + void GetColumnSegmentInfo(const QueryContext &context, duckdb::idx_t row_group_index, + vector col_path, vector &result) override; private: uint64_t FetchListOffset(idx_t row_idx); diff --git a/src/include/duckdb/storage/table/row_group.hpp b/src/include/duckdb/storage/table/row_group.hpp index 29eca52ae43d..62b54eeedd17 100644 --- a/src/include/duckdb/storage/table/row_group.hpp +++ b/src/include/duckdb/storage/table/row_group.hpp @@ -53,13 +53,22 @@ class StorageCommitState; struct RowGroupWriteInfo { RowGroupWriteInfo(PartialBlockManager &manager, const vector &compression_types, - CheckpointType checkpoint_type = CheckpointType::FULL_CHECKPOINT) - : manager(manager), compression_types(compression_types), checkpoint_type(checkpoint_type) { - } + CheckpointType checkpoint_type = CheckpointType::FULL_CHECKPOINT); + RowGroupWriteInfo(PartialBlockManager &manager, const vector &compression_types, + vector> &column_partial_block_managers_p); +private: PartialBlockManager &manager; + +public: const vector &compression_types; CheckpointType checkpoint_type; + +public: + PartialBlockManager &GetPartialBlockManager(idx_t column_idx); + +private: + optional_ptr>> column_partial_block_managers; }; struct RowGroupWriteData { @@ -149,6 +158,8 @@ class RowGroup : public SegmentBase { //! Delete the given set of rows in the version manager idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count); + static vector WriteToDisk(RowGroupWriteInfo &info, + const vector> &row_groups); RowGroupWriteData WriteToDisk(RowGroupWriteInfo &info); //! Returns the number of committed rows (count - committed deletes) idx_t GetCommittedRowCount(); @@ -160,19 +171,19 @@ class RowGroup : public SegmentBase { void InitializeAppend(RowGroupAppendState &append_state); void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count); - void Update(TransactionData transaction, DataChunk &updates, row_t *ids, idx_t offset, idx_t count, - const vector &column_ids); + void Update(TransactionData transaction, DataTable &data_table, DataChunk &updates, row_t *ids, idx_t offset, + idx_t count, const vector &column_ids); //! Update a single column; corresponds to DataTable::UpdateColumn //! This method should only be called from the WAL - void UpdateColumn(TransactionData transaction, DataChunk &updates, Vector &row_ids, - const vector &column_path); + void UpdateColumn(TransactionData transaction, DataTable &data_table, DataChunk &updates, Vector &row_ids, + idx_t offset, idx_t count, const vector &column_path); void MergeStatistics(idx_t column_idx, const BaseStatistics &other); void MergeIntoStatistics(idx_t column_idx, BaseStatistics &other); void MergeIntoStatistics(TableStatistics &other); unique_ptr GetStatistics(idx_t column_idx); - void GetColumnSegmentInfo(idx_t row_group_index, vector &result); + void GetColumnSegmentInfo(const QueryContext &context, idx_t row_group_index, vector &result); PartitionStatistics GetPartitionStats() const; idx_t GetAllocationSize() const { @@ -225,6 +236,7 @@ class RowGroup : public SegmentBase { atomic allocation_size; unique_ptr row_id_column_data; atomic row_id_is_loaded; + atomic has_changes; }; } // namespace duckdb diff --git a/src/include/duckdb/storage/table/row_group_collection.hpp b/src/include/duckdb/storage/table/row_group_collection.hpp index 769242606f68..80aa5266831e 100644 --- a/src/include/duckdb/storage/table/row_group_collection.hpp +++ b/src/include/duckdb/storage/table/row_group_collection.hpp @@ -36,6 +36,7 @@ struct CollectionCheckpointState; struct PersistentCollectionData; class CheckpointTask; class TableIOManager; +class DataTable; class RowGroupCollection { public: @@ -51,21 +52,24 @@ class RowGroupCollection { void Initialize(PersistentCollectionData &data); void Initialize(PersistentTableData &data); void InitializeEmpty(); + void FinalizeCheckpoint(MetaBlockPointer pointer); bool IsEmpty() const; void AppendRowGroup(SegmentLock &l, idx_t start_row); //! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc) - RowGroup *GetRowGroup(int64_t index); + optional_ptr GetRowGroup(int64_t index); void Verify(); + void Destroy(); - void InitializeScan(CollectionScanState &state, const vector &column_ids, + void InitializeScan(const QueryContext &context, CollectionScanState &state, const vector &column_ids, optional_ptr table_filters); void InitializeCreateIndexScan(CreateIndexScanState &state); - void InitializeScanWithOffset(CollectionScanState &state, const vector &column_ids, idx_t start_row, - idx_t end_row); - static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroupCollection &collection, - RowGroup &row_group, idx_t vector_index, idx_t max_row); + void InitializeScanWithOffset(const QueryContext &context, CollectionScanState &state, + const vector &column_ids, idx_t start_row, idx_t end_row); + static bool InitializeScanInRowGroup(const QueryContext &context, CollectionScanState &state, + RowGroupCollection &collection, RowGroup &row_group, idx_t vector_index, + idx_t max_row); void InitializeParallelScan(ParallelCollectionScanState &state); bool NextParallelScan(ClientContext &context, ParallelCollectionScanState &state, CollectionScanState &scan_state); @@ -95,12 +99,13 @@ class RowGroupCollection { optional_ptr commit_state); bool IsPersistent() const; - void RemoveFromIndexes(TableIndexList &indexes, Vector &row_identifiers, idx_t count); + void RemoveFromIndexes(const QueryContext &context, TableIndexList &indexes, Vector &row_identifiers, idx_t count); idx_t Delete(TransactionData transaction, DataTable &table, row_t *ids, idx_t count); - void Update(TransactionData transaction, row_t *ids, const vector &column_ids, DataChunk &updates); - void UpdateColumn(TransactionData transaction, Vector &row_ids, const vector &column_path, - DataChunk &updates); + void Update(TransactionData transaction, DataTable &table, row_t *ids, const vector &column_ids, + DataChunk &updates); + void UpdateColumn(TransactionData transaction, DataTable &table, Vector &row_ids, + const vector &column_path, DataChunk &updates); void Checkpoint(TableDataWriter &writer, TableStatistics &global_stats); @@ -114,7 +119,7 @@ class RowGroupCollection { void CommitDropTable(); vector GetPartitionStats() const; - vector GetColumnSegmentInfo(); + vector GetColumnSegmentInfo(const QueryContext &context); const vector &GetTypes() const; shared_ptr AddColumn(ClientContext &context, ColumnDefinition &new_column, @@ -122,7 +127,7 @@ class RowGroupCollection { shared_ptr RemoveColumn(idx_t col_idx); shared_ptr AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type, vector bound_columns, Expression &cast_expr); - void VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint); + void VerifyNewConstraint(const QueryContext &context, DataTable &parent, const BoundConstraint &constraint); void CopyStats(TableStatistics &stats); unique_ptr CopyStats(column_t column_id); @@ -145,10 +150,13 @@ class RowGroupCollection { idx_t GetRowGroupSize() const { return row_group_size; } + void SetAppendRequiresNewRowGroup(); private: bool IsEmpty(SegmentLock &) const; + optional_ptr NextUpdateRowGroup(row_t *ids, idx_t &pos, idx_t count) const; + private: //! BlockManager BlockManager &block_manager; @@ -169,6 +177,8 @@ class RowGroupCollection { atomic allocation_size; //! Root metadata pointer, if the collection is loaded from disk MetaBlockPointer metadata_pointer; + //! Whether or not we need to append a new row group prior to appending + bool requires_new_row_group; }; } // namespace duckdb diff --git a/src/include/duckdb/storage/table/row_id_column_data.hpp b/src/include/duckdb/storage/table/row_id_column_data.hpp index 3bc6572ae2c6..f839c6b24619 100644 --- a/src/include/duckdb/storage/table/row_id_column_data.hpp +++ b/src/include/duckdb/storage/table/row_id_column_data.hpp @@ -48,10 +48,10 @@ class RowIdColumnData : public ColumnData { void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) override; void RevertAppend(row_t start_row) override; - void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) override; - void UpdateColumn(TransactionData transaction, const vector &column_path, Vector &update_vector, - row_t *row_ids, idx_t update_count, idx_t depth) override; + void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector, + row_t *row_ids, idx_t update_count) override; + void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector &column_path, + Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override; void CommitDropColumn() override; diff --git a/src/include/duckdb/storage/table/row_version_manager.hpp b/src/include/duckdb/storage/table/row_version_manager.hpp index bb0d0056b9f0..5ba674cc6f49 100644 --- a/src/include/duckdb/storage/table/row_version_manager.hpp +++ b/src/include/duckdb/storage/table/row_version_manager.hpp @@ -12,20 +12,25 @@ #include "duckdb/storage/table/chunk_info.hpp" #include "duckdb/storage/storage_info.hpp" #include "duckdb/common/mutex.hpp" +#include "duckdb/execution/index/fixed_size_allocator.hpp" namespace duckdb { struct DeleteInfo; class MetadataManager; +class BufferManager; struct MetaBlockPointer; class RowVersionManager { public: - explicit RowVersionManager(idx_t start) noexcept; + explicit RowVersionManager(BufferManager &buffer_manager, idx_t start) noexcept; - idx_t GetStart() { + idx_t GetStart() const { return start; } + FixedSizeAllocator &GetAllocator() { + return allocator; + } void SetStart(idx_t start); idx_t GetCommittedDeletedCount(idx_t count); @@ -48,6 +53,7 @@ class RowVersionManager { private: mutex version_lock; + FixedSizeAllocator allocator; idx_t start; vector> vector_info; bool has_changes; diff --git a/src/include/duckdb/storage/table/scan_state.hpp b/src/include/duckdb/storage/table/scan_state.hpp index 97416cbf2203..8e85053ecd4d 100644 --- a/src/include/duckdb/storage/table/scan_state.hpp +++ b/src/include/duckdb/storage/table/scan_state.hpp @@ -78,6 +78,8 @@ struct IndexScanState { typedef unordered_map buffer_handle_set_t; struct ColumnScanState { + //! The query context for this scan + QueryContext context; //! The column segment that is currently being scanned ColumnSegment *current = nullptr; //! Column segment tree @@ -105,9 +107,9 @@ struct ColumnScanState { optional_ptr scan_options; public: - void Initialize(const LogicalType &type, const vector &children, + void Initialize(const QueryContext &context_p, const LogicalType &type, const vector &children, optional_ptr options); - void Initialize(const LogicalType &type, optional_ptr options); + void Initialize(const QueryContext &context_p, const LogicalType &type, optional_ptr options); //! Move the scan state forward by "count" rows (including all child states) void Next(idx_t count); //! Move ONLY this state forward by "count" rows (i.e. not the child states) @@ -115,6 +117,8 @@ struct ColumnScanState { }; struct ColumnFetchState { + //! The query context for this fetch + QueryContext context; //! The set of pinned block handles for this set of fetches buffer_handle_set_t handles; //! Any child states of the fetch @@ -202,7 +206,7 @@ class CollectionScanState { RandomEngine random; public: - void Initialize(const vector &types); + void Initialize(const QueryContext &context, const vector &types); const vector &GetColumnIds(); ScanFilterInfo &GetFilterInfo(); ScanSamplingInfo &GetSamplingInfo(); diff --git a/src/include/duckdb/storage/table/segment_lock.hpp b/src/include/duckdb/storage/table/segment_lock.hpp index 22ed6ceeca98..ce440fdb7cfc 100644 --- a/src/include/duckdb/storage/table/segment_lock.hpp +++ b/src/include/duckdb/storage/table/segment_lock.hpp @@ -31,6 +31,10 @@ struct SegmentLock { return *this; } + void Release() { + lock.unlock(); + } + private: unique_lock lock; }; diff --git a/src/include/duckdb/storage/table/segment_tree.hpp b/src/include/duckdb/storage/table/segment_tree.hpp index 16e297bae314..f427a5275f26 100644 --- a/src/include/duckdb/storage/table/segment_tree.hpp +++ b/src/include/duckdb/storage/table/segment_tree.hpp @@ -76,6 +76,9 @@ class SegmentTree { auto l = Lock(); return ReferenceSegments(l); } + vector> &ReferenceLoadedSegmentsMutable(SegmentLock &l) { + return nodes; + } const vector> &ReferenceLoadedSegments(SegmentLock &l) const { return nodes; } @@ -183,10 +186,10 @@ class SegmentTree { //! Erase all segments after a specific segment void EraseSegments(SegmentLock &l, idx_t segment_start) { LoadAllSegments(l); - if (segment_start >= nodes.size() - 1) { + if (segment_start >= nodes.size()) { return; } - nodes.erase(nodes.begin() + UnsafeNumericCast(segment_start) + 1, nodes.end()); + nodes.erase(nodes.begin() + UnsafeNumericCast(segment_start), nodes.end()); } //! Get the segment index of the column segment for the given row @@ -219,7 +222,14 @@ class SegmentTree { // binary search to find the node while (lower <= upper) { idx_t index = (lower + upper) / 2; - D_ASSERT(index < nodes.size()); + if (index >= nodes.size()) { + string segments; + for (auto &entry : nodes) { + segments += StringUtil::Format("Start %d Count %d", entry.row_start, entry.node->count.load()); + } + throw InternalException("Segment tree index not found for row number %d\nSegments:%s", row_number, + segments); + } auto &entry = nodes[index]; D_ASSERT(entry.row_start == entry.node->start); if (row_number < entry.row_start) { diff --git a/src/include/duckdb/storage/table/standard_column_data.hpp b/src/include/duckdb/storage/table/standard_column_data.hpp index 48ac6ccb79be..ec06eb30a4cf 100644 --- a/src/include/duckdb/storage/table/standard_column_data.hpp +++ b/src/include/duckdb/storage/table/standard_column_data.hpp @@ -47,10 +47,10 @@ class StandardColumnData : public ColumnData { idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override; void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) override; - void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) override; - void UpdateColumn(TransactionData transaction, const vector &column_path, Vector &update_vector, - row_t *row_ids, idx_t update_count, idx_t depth) override; + void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector, + row_t *row_ids, idx_t update_count) override; + void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector &column_path, + Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override; unique_ptr GetUpdateStatistics() override; void CommitDropColumn() override; @@ -61,8 +61,8 @@ class StandardColumnData : public ColumnData { void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count, Vector &scan_vector) override; - void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector col_path, - vector &result) override; + void GetColumnSegmentInfo(const QueryContext &context, duckdb::idx_t row_group_index, + vector col_path, vector &result) override; bool IsPersistent() override; bool HasAnyChanges() const override; diff --git a/src/include/duckdb/storage/table/struct_column_data.hpp b/src/include/duckdb/storage/table/struct_column_data.hpp index d05436bfc5af..798a21326921 100644 --- a/src/include/duckdb/storage/table/struct_column_data.hpp +++ b/src/include/duckdb/storage/table/struct_column_data.hpp @@ -46,10 +46,10 @@ class StructColumnData : public ColumnData { idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override; void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) override; - void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) override; - void UpdateColumn(TransactionData transaction, const vector &column_path, Vector &update_vector, - row_t *row_ids, idx_t update_count, idx_t depth) override; + void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector, + row_t *row_ids, idx_t update_count) override; + void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector &column_path, + Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override; unique_ptr GetUpdateStatistics() override; void CommitDropColumn() override; @@ -63,8 +63,8 @@ class StructColumnData : public ColumnData { PersistentColumnData Serialize() override; void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override; - void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector col_path, - vector &result) override; + void GetColumnSegmentInfo(const QueryContext &context, duckdb::idx_t row_group_index, + vector col_path, vector &result) override; void Verify(RowGroup &parent) override; }; diff --git a/src/include/duckdb/storage/table/update_segment.hpp b/src/include/duckdb/storage/table/update_segment.hpp index 75cf25ecfbb0..3f5b9d2119dd 100644 --- a/src/include/duckdb/storage/table/update_segment.hpp +++ b/src/include/duckdb/storage/table/update_segment.hpp @@ -38,8 +38,8 @@ class UpdateSegment { void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result); void FetchCommitted(idx_t vector_index, Vector &result); void FetchCommittedRange(idx_t start_row, idx_t count, Vector &result); - void Update(TransactionData transaction, idx_t column_index, Vector &update, row_t *ids, idx_t count, - Vector &base_data); + void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update, row_t *ids, + idx_t count, Vector &base_data); void FetchRow(TransactionData transaction, idx_t row_id, Vector &result, idx_t result_idx); void RollbackUpdate(UpdateInfo &info); diff --git a/src/include/duckdb/transaction/cleanup_state.hpp b/src/include/duckdb/transaction/cleanup_state.hpp index 0de2faabd569..21f1176745fa 100644 --- a/src/include/duckdb/transaction/cleanup_state.hpp +++ b/src/include/duckdb/transaction/cleanup_state.hpp @@ -11,6 +11,7 @@ #include "duckdb/transaction/undo_buffer.hpp" #include "duckdb/common/types/data_chunk.hpp" #include "duckdb/common/unordered_map.hpp" +#include "duckdb/main/client_context.hpp" namespace duckdb { @@ -21,7 +22,7 @@ struct UpdateInfo; class CleanupState { public: - explicit CleanupState(transaction_t lowest_active_transaction); + explicit CleanupState(const QueryContext &context, transaction_t lowest_active_transaction); ~CleanupState(); // all tables with indexes that possibly need a vacuum (after e.g. a delete) @@ -31,6 +32,7 @@ class CleanupState { void CleanupEntry(UndoFlags type, data_ptr_t data); private: + QueryContext context; //! Lowest active transaction transaction_t lowest_active_transaction; // data for index cleanup diff --git a/src/include/duckdb/transaction/duck_transaction.hpp b/src/include/duckdb/transaction/duck_transaction.hpp index 12c4d180c25f..b9080f192f71 100644 --- a/src/include/duckdb/transaction/duck_transaction.hpp +++ b/src/include/duckdb/transaction/duck_transaction.hpp @@ -35,14 +35,12 @@ class DuckTransaction : public Transaction { transaction_t transaction_id; //! The commit id of this transaction, if it has successfully been committed transaction_t commit_id; - //! Highest active query when the transaction finished, used for cleaning up - transaction_t highest_active_query; atomic catalog_version; //! Transactions undergo Cleanup, after (1) removing them directly in RemoveTransaction, - //! or (2) after they exist old_transactions. - //! Some (after rollback) enter old_transactions, but do not require Cleanup. + //! or (2) after they enter cleanup_queue. + //! Some (after rollback) enter cleanup_queue, but do not require Cleanup. bool awaiting_cleanup; public: @@ -76,7 +74,7 @@ class DuckTransaction : public Transaction { idx_t base_row); void PushSequenceUsage(SequenceCatalogEntry &entry, const SequenceData &data); void PushAppend(DataTable &table, idx_t row_start, idx_t row_count); - UndoBufferReference CreateUpdateInfo(idx_t type_size, idx_t entries); + UndoBufferReference CreateUpdateInfo(idx_t type_size, DataTable &data_table, idx_t entries); bool IsDuckTransaction() const override { return true; @@ -90,6 +88,7 @@ class DuckTransaction : public Transaction { //! Get a shared lock on a table shared_ptr SharedLockTable(DataTableInfo &info); + //! Hold an owning reference of the table, needed to safely reference it inside the transaction commit/undo logic void ModifyTable(DataTable &tbl); private: diff --git a/src/include/duckdb/transaction/duck_transaction_manager.hpp b/src/include/duckdb/transaction/duck_transaction_manager.hpp index 63531ae7d7e2..a3bf3f47a627 100644 --- a/src/include/duckdb/transaction/duck_transaction_manager.hpp +++ b/src/include/duckdb/transaction/duck_transaction_manager.hpp @@ -110,8 +110,6 @@ class DuckTransactionManager : public TransactionManager { vector> active_transactions; //! Set of recently committed transactions vector> recently_committed_transactions; - //! Transactions awaiting GC - vector> old_transactions; //! The lock used for transaction operations mutex transaction_lock; //! The checkpoint lock diff --git a/src/include/duckdb/transaction/local_storage.hpp b/src/include/duckdb/transaction/local_storage.hpp index 1e043a3b0cfe..1cab839d0e9f 100644 --- a/src/include/duckdb/transaction/local_storage.hpp +++ b/src/include/duckdb/transaction/local_storage.hpp @@ -40,11 +40,13 @@ class LocalTableStorage : public enable_shared_from_this { ExpressionExecutor &default_executor); ~LocalTableStorage(); + QueryContext context; + reference table_ref; Allocator &allocator; //! The main row group collection. - shared_ptr row_groups; + unique_ptr row_groups; //! The set of unique append indexes. TableIndexList append_indexes; //! The set of delete indexes. @@ -55,7 +57,7 @@ class LocalTableStorage : public enable_shared_from_this { idx_t deleted_rows; //! The optimistic row group collections associated with this table. - vector> optimistic_collections; + vector> optimistic_collections; //! The main optimistic data writer associated with this table. OptimisticDataWriter optimistic_writer; @@ -79,14 +81,16 @@ class LocalTableStorage : public enable_shared_from_this { //! Create an optimistic row group collection for this table. //! Returns the index into the optimistic_collections vector for newly created collection. - PhysicalIndex CreateOptimisticCollection(unique_ptr collection); + PhysicalIndex CreateOptimisticCollection(unique_ptr collection); //! Returns the optimistic row group collection corresponding to the index. - RowGroupCollection &GetOptimisticCollection(const PhysicalIndex collection_index); + OptimisticWriteCollection &GetOptimisticCollection(const PhysicalIndex collection_index); //! Resets the optimistic row group collection corresponding to the index. void ResetOptimisticCollection(const PhysicalIndex collection_index); //! Returns the optimistic writer. OptimisticDataWriter &GetOptimisticWriter(); + RowGroupCollection &GetCollection(); + private: mutex collections_lock; }; @@ -141,12 +145,12 @@ class LocalStorage { //! Finish appending to the local storage static void FinalizeAppend(LocalAppendState &state); //! Merge a row group collection into the transaction-local storage - void LocalMerge(DataTable &table, RowGroupCollection &collection); + void LocalMerge(DataTable &table, OptimisticWriteCollection &collection); //! Create an optimistic row group collection for this table. //! Returns the index into the optimistic_collections vector for newly created collection. - PhysicalIndex CreateOptimisticCollection(DataTable &table, unique_ptr collection); + PhysicalIndex CreateOptimisticCollection(DataTable &table, unique_ptr collection); //! Returns the optimistic row group collection corresponding to the index. - RowGroupCollection &GetOptimisticCollection(DataTable &table, const PhysicalIndex collection_index); + OptimisticWriteCollection &GetOptimisticCollection(DataTable &table, const PhysicalIndex collection_index); //! Resets the optimistic row group collection corresponding to the index. void ResetOptimisticCollection(DataTable &table, const PhysicalIndex collection_index); //! Returns the optimistic writer. @@ -187,6 +191,10 @@ class LocalStorage { void VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint); + ClientContext &GetClientContext() const { + return context; + } + private: ClientContext &context; DuckTransaction &transaction; diff --git a/src/include/duckdb/transaction/meta_transaction.hpp b/src/include/duckdb/transaction/meta_transaction.hpp index b8f0fef828ac..71693ee14d97 100644 --- a/src/include/duckdb/transaction/meta_transaction.hpp +++ b/src/include/duckdb/transaction/meta_transaction.hpp @@ -16,6 +16,7 @@ #include "duckdb/common/optional_ptr.hpp" #include "duckdb/common/reference_map.hpp" #include "duckdb/common/error_data.hpp" +#include "duckdb/common/case_insensitive_map.hpp" namespace duckdb { class AttachedDatabase; @@ -74,7 +75,10 @@ class MetaTransaction { const vector> &OpenedTransactions() const { return all_transactions; } + optional_ptr GetReferencedDatabase(const string &name); + shared_ptr GetReferencedDatabaseOwning(const string &name); AttachedDatabase &UseDatabase(shared_ptr &database); + void DetachDatabase(AttachedDatabase &database); private: //! Lock to prevent all_transactions and transactions from getting out of sync @@ -87,8 +91,12 @@ class MetaTransaction { optional_ptr modified_database; //! Whether or not the meta transaction is marked as read only bool is_read_only; + //! Lock for referenced_databases + mutex referenced_database_lock; //! The set of used / referenced databases reference_map_t> referenced_databases; + //! Map of name -> used database for databases that are in-use by this transaction + case_insensitive_map_t> used_databases; }; } // namespace duckdb diff --git a/src/include/duckdb/transaction/update_info.hpp b/src/include/duckdb/transaction/update_info.hpp index 7cccd923e74b..5eb1392611d2 100644 --- a/src/include/duckdb/transaction/update_info.hpp +++ b/src/include/duckdb/transaction/update_info.hpp @@ -17,6 +17,7 @@ namespace duckdb { class UpdateSegment; struct DataTableInfo; +class DataTable; //! UpdateInfo is a class that represents a set of updates applied to a single vector. //! The UpdateInfo struct contains metadata associated with the update. @@ -26,6 +27,8 @@ struct DataTableInfo; struct UpdateInfo { //! The update segment that this update info affects UpdateSegment *segment; + //! The table this was update was made on + DataTable *table; //! The column index of which column we are updating idx_t column_index; //! The version number @@ -87,7 +90,7 @@ struct UpdateInfo { //! Returns the total allocation size for an UpdateInfo entry, together with space for the tuple data static idx_t GetAllocSize(idx_t type_size); //! Initialize an UpdateInfo struct that has been allocated using GetAllocSize (i.e. has extra space after it) - static void Initialize(UpdateInfo &info, transaction_t transaction_id); + static void Initialize(UpdateInfo &info, DataTable &data_table, transaction_t transaction_id); }; } // namespace duckdb diff --git a/src/include/duckdb/transaction/wal_write_state.hpp b/src/include/duckdb/transaction/wal_write_state.hpp index aad1a672cb71..4c68da4875a6 100644 --- a/src/include/duckdb/transaction/wal_write_state.hpp +++ b/src/include/duckdb/transaction/wal_write_state.hpp @@ -31,7 +31,7 @@ class WALWriteState { void CommitEntry(UndoFlags type, data_ptr_t data); private: - void SwitchTable(DataTableInfo *table, UndoFlags new_op); + void SwitchTable(DataTableInfo &table, UndoFlags new_op); void WriteCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data); void WriteDelete(DeleteInfo &info); diff --git a/src/include/duckdb/verification/statement_verifier.hpp b/src/include/duckdb/verification/statement_verifier.hpp index a60abf18705f..77fed981540d 100644 --- a/src/include/duckdb/verification/statement_verifier.hpp +++ b/src/include/duckdb/verification/statement_verifier.hpp @@ -85,6 +85,8 @@ class StatementVerifier { private: const vector> empty_select_list = {}; + + const vector> &GetSelectList(QueryNode &node); }; } // namespace duckdb diff --git a/src/include/duckdb_extension.h b/src/include/duckdb_extension.h index 01d4c8307b63..f014be548bc8 100644 --- a/src/include/duckdb_extension.h +++ b/src/include/duckdb_extension.h @@ -578,6 +578,28 @@ typedef struct { duckdb_value *out_value); #endif +// API to manage file system operations +#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE + duckdb_file_system (*duckdb_client_context_get_file_system)(duckdb_client_context context); + void (*duckdb_destroy_file_system)(duckdb_file_system *file_system); + duckdb_state (*duckdb_file_system_open)(duckdb_file_system file_system, const char *path, + duckdb_file_open_options options, duckdb_file_handle *out_file); + duckdb_error_data (*duckdb_file_system_error_data)(duckdb_file_system file_system); + duckdb_file_open_options (*duckdb_create_file_open_options)(); + duckdb_state (*duckdb_file_open_options_set_flag)(duckdb_file_open_options options, duckdb_file_flag flag, + bool value); + void (*duckdb_destroy_file_open_options)(duckdb_file_open_options *options); + void (*duckdb_destroy_file_handle)(duckdb_file_handle *file_handle); + duckdb_error_data (*duckdb_file_handle_error_data)(duckdb_file_handle file_handle); + duckdb_state (*duckdb_file_handle_close)(duckdb_file_handle file_handle); + int64_t (*duckdb_file_handle_read)(duckdb_file_handle file_handle, void *buffer, int64_t size); + int64_t (*duckdb_file_handle_write)(duckdb_file_handle file_handle, const void *buffer, int64_t size); + duckdb_state (*duckdb_file_handle_seek)(duckdb_file_handle file_handle, int64_t position); + int64_t (*duckdb_file_handle_tell)(duckdb_file_handle file_handle); + duckdb_state (*duckdb_file_handle_sync)(duckdb_file_handle file_handle); + int64_t (*duckdb_file_handle_size)(duckdb_file_handle file_handle); +#endif + // New functions around the client context #ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE idx_t (*duckdb_client_context_get_connection_id)(duckdb_client_context context); @@ -588,6 +610,15 @@ typedef struct { void (*duckdb_destroy_arrow_options)(duckdb_arrow_options *arrow_options); #endif +// API to get information about the results of a prepared statement +#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE + idx_t (*duckdb_prepared_statement_column_count)(duckdb_prepared_statement prepared_statement); + const char *(*duckdb_prepared_statement_column_name)(duckdb_prepared_statement prepared_statement, idx_t col_idx); + duckdb_logical_type (*duckdb_prepared_statement_column_logical_type)(duckdb_prepared_statement prepared_statement, + idx_t col_idx); + duckdb_type (*duckdb_prepared_statement_column_type)(duckdb_prepared_statement prepared_statement, idx_t col_idx); +#endif + // New query execution functions #ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE duckdb_arrow_options (*duckdb_result_get_arrow_options)(duckdb_result *result); @@ -612,6 +643,13 @@ typedef struct { char *(*duckdb_value_to_string)(duckdb_value value); #endif +// New functions around the table description +#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE + idx_t (*duckdb_table_description_get_column_count)(duckdb_table_description table_description); + duckdb_logical_type (*duckdb_table_description_get_column_type)(duckdb_table_description table_description, + idx_t index); +#endif + // New functions around table function binding #ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE void (*duckdb_table_function_get_client_context)(duckdb_bind_info info, duckdb_client_context *out_context); @@ -1084,6 +1122,24 @@ typedef struct { #define duckdb_expression_is_foldable duckdb_ext_api.duckdb_expression_is_foldable #define duckdb_expression_fold duckdb_ext_api.duckdb_expression_fold +// Version unstable_new_file_system_api +#define duckdb_client_context_get_file_system duckdb_ext_api.duckdb_client_context_get_file_system +#define duckdb_destroy_file_system duckdb_ext_api.duckdb_destroy_file_system +#define duckdb_file_system_error_data duckdb_ext_api.duckdb_file_system_error_data +#define duckdb_file_system_open duckdb_ext_api.duckdb_file_system_open +#define duckdb_create_file_open_options duckdb_ext_api.duckdb_create_file_open_options +#define duckdb_file_open_options_set_flag duckdb_ext_api.duckdb_file_open_options_set_flag +#define duckdb_destroy_file_open_options duckdb_ext_api.duckdb_destroy_file_open_options +#define duckdb_destroy_file_handle duckdb_ext_api.duckdb_destroy_file_handle +#define duckdb_file_handle_error_data duckdb_ext_api.duckdb_file_handle_error_data +#define duckdb_file_handle_read duckdb_ext_api.duckdb_file_handle_read +#define duckdb_file_handle_write duckdb_ext_api.duckdb_file_handle_write +#define duckdb_file_handle_tell duckdb_ext_api.duckdb_file_handle_tell +#define duckdb_file_handle_size duckdb_ext_api.duckdb_file_handle_size +#define duckdb_file_handle_seek duckdb_ext_api.duckdb_file_handle_seek +#define duckdb_file_handle_sync duckdb_ext_api.duckdb_file_handle_sync +#define duckdb_file_handle_close duckdb_ext_api.duckdb_file_handle_close + // Version unstable_new_open_connect_functions #define duckdb_connection_get_client_context duckdb_ext_api.duckdb_connection_get_client_context #define duckdb_connection_get_arrow_options duckdb_ext_api.duckdb_connection_get_arrow_options @@ -1092,6 +1148,12 @@ typedef struct { #define duckdb_destroy_arrow_options duckdb_ext_api.duckdb_destroy_arrow_options #define duckdb_get_table_names duckdb_ext_api.duckdb_get_table_names +// Version unstable_new_prepared_statement_functions +#define duckdb_prepared_statement_column_count duckdb_ext_api.duckdb_prepared_statement_column_count +#define duckdb_prepared_statement_column_name duckdb_ext_api.duckdb_prepared_statement_column_name +#define duckdb_prepared_statement_column_logical_type duckdb_ext_api.duckdb_prepared_statement_column_logical_type +#define duckdb_prepared_statement_column_type duckdb_ext_api.duckdb_prepared_statement_column_type + // Version unstable_new_query_execution_functions #define duckdb_result_get_arrow_options duckdb_ext_api.duckdb_result_get_arrow_options @@ -1109,6 +1171,10 @@ typedef struct { // Version unstable_new_string_functions #define duckdb_value_to_string duckdb_ext_api.duckdb_value_to_string +// Version unstable_new_table_description_functions +#define duckdb_table_description_get_column_count duckdb_ext_api.duckdb_table_description_get_column_count +#define duckdb_table_description_get_column_type duckdb_ext_api.duckdb_table_description_get_column_type + // Version unstable_new_table_function_functions #define duckdb_table_function_get_client_context duckdb_ext_api.duckdb_table_function_get_client_context diff --git a/src/logging/log_types.cpp b/src/logging/log_types.cpp index 6f2d4bacd714..f78abae591b4 100644 --- a/src/logging/log_types.cpp +++ b/src/logging/log_types.cpp @@ -1,3 +1,4 @@ +#include "duckdb/main/attached_database.hpp" #include "duckdb/logging/file_system_logger.hpp" #include "duckdb/logging/log_type.hpp" #include "duckdb/common/file_opener.hpp" @@ -13,7 +14,18 @@ constexpr LogLevel FileSystemLogType::LEVEL; constexpr LogLevel QueryLogType::LEVEL; constexpr LogLevel HTTPLogType::LEVEL; constexpr LogLevel PhysicalOperatorLogType::LEVEL; +constexpr LogLevel CheckpointLogType::LEVEL; +//===--------------------------------------------------------------------===// +// QueryLogType +//===--------------------------------------------------------------------===// +string QueryLogType::ConstructLogMessage(const string &str) { + return str; +} + +//===--------------------------------------------------------------------===// +// FileSystemLogType +//===--------------------------------------------------------------------===// FileSystemLogType::FileSystemLogType() : LogType(NAME, LEVEL, GetLogType()) { } @@ -36,6 +48,9 @@ LogicalType FileSystemLogType::GetLogType() { return LogicalType::STRUCT(child_list); } +//===--------------------------------------------------------------------===// +// HTTPLogType +//===--------------------------------------------------------------------===// HTTPLogType::HTTPLogType() : LogType(NAME, LEVEL, GetLogType()) { } @@ -92,6 +107,9 @@ string HTTPLogType::ConstructLogMessage(BaseRequest &request, optional_ptr child_list = { + {"database", LogicalType::VARCHAR}, + {"schema", LogicalType::VARCHAR}, + {"table", LogicalType::VARCHAR}, + {"type", LogicalType::VARCHAR}, + {"info", LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)}, + }; + return LogicalType::STRUCT(child_list); +} + +string CheckpointLogType::CreateLog(const AttachedDatabase &db, DataTableInfo &table, const char *op_name, + vector map_keys, vector map_values) { + child_list_t child_list = { + {"database", db.name}, + {"schema", table.GetSchemaName()}, + {"table", table.GetTableName()}, + {"type", op_name}, + {"info", Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(map_keys), std::move(map_values))}, + }; + + return Value::STRUCT(std::move(child_list)).ToString(); +} + +string CheckpointLogType::ConstructLogMessage(const AttachedDatabase &db, DataTableInfo &table, idx_t segment_idx, + idx_t merge_count, idx_t target_count, idx_t merge_rows, + idx_t row_start) { + vector map_keys = {"segment_idx", "merge_count", "target_count", "merge_rows", "row_start"}; + vector map_values = {to_string(segment_idx), to_string(merge_count), to_string(target_count), + to_string(merge_rows), to_string(row_start)}; + return CreateLog(db, table, "vacuum", std::move(map_keys), std::move(map_values)); +} + +string CheckpointLogType::ConstructLogMessage(const AttachedDatabase &db, DataTableInfo &table, idx_t segment_idx, + RowGroup &row_group) { + vector map_keys = {"segment_idx", "start", "count"}; + vector map_values = {to_string(segment_idx), to_string(row_group.start), to_string(row_group.count.load())}; + return CreateLog(db, table, "checkpoint", std::move(map_keys), std::move(map_values)); +} } // namespace duckdb diff --git a/src/main/CMakeLists.txt b/src/main/CMakeLists.txt index 00727194c5ea..3095587f474a 100644 --- a/src/main/CMakeLists.txt +++ b/src/main/CMakeLists.txt @@ -29,6 +29,7 @@ add_library_unity( config.cpp connection.cpp database.cpp + database_file_path_manager.cpp database_path_and_type.cpp database_manager.cpp db_instance_cache.cpp diff --git a/src/main/attached_database.cpp b/src/main/attached_database.cpp index 3ddd3a2e60ec..70e8be932cab 100644 --- a/src/main/attached_database.cpp +++ b/src/main/attached_database.cpp @@ -10,21 +10,26 @@ #include "duckdb/storage/storage_manager.hpp" #include "duckdb/transaction/duck_transaction_manager.hpp" #include "duckdb/main/database_path_and_type.hpp" +#include "duckdb/main/valid_checker.hpp" namespace duckdb { -StoredDatabasePath::StoredDatabasePath(DatabaseManager &manager, string path_p, const string &name) - : manager(manager), path(std::move(path_p)) { - manager.InsertDatabasePath(path, name); +StoredDatabasePath::StoredDatabasePath(DatabaseManager &db_manager, DatabaseFilePathManager &manager, string path_p, + const string &name) + : db_manager(db_manager), manager(manager), path(std::move(path_p)) { } StoredDatabasePath::~StoredDatabasePath() { manager.EraseDatabasePath(path); } + +void StoredDatabasePath::OnDetach() { + manager.DetachDatabase(db_manager, path); +} + //===--------------------------------------------------------------------===// // Attach Options //===--------------------------------------------------------------------===// - AttachOptions::AttachOptions(const DBConfigOptions &options) : access_mode(options.access_mode), db_type(options.database_type) { } @@ -99,9 +104,10 @@ AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, str } else { type = AttachedDatabaseType::READ_WRITE_DATABASE; } + visibility = options.visibility; // We create the storage after the catalog to guarantee we allow extensions to instantiate the DuckCatalog. catalog = make_uniq(*this); - InsertDatabasePath(file_path_p); + stored_database_path = std::move(options.stored_database_path); storage = make_uniq(*this, std::move(file_path_p), options); transaction_manager = make_uniq(*this); internal = true; @@ -116,14 +122,15 @@ AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, Sto } else { type = AttachedDatabaseType::READ_WRITE_DATABASE; } + visibility = options.visibility; optional_ptr storage_info = storage_extension->storage_info.get(); catalog = storage_extension->attach(storage_info, context, *this, name, info, options); + stored_database_path = std::move(options.stored_database_path); if (!catalog) { throw InternalException("AttachedDatabase - attach function did not return a catalog"); } if (catalog->IsDuckCatalog()) { - InsertDatabasePath(info.path); // The attached database uses the DuckCatalog. storage = make_uniq(*this, info.path, options); } @@ -151,17 +158,17 @@ bool AttachedDatabase::IsReadOnly() const { return type == AttachedDatabaseType::READ_ONLY_DATABASE; } -void AttachedDatabase::InsertDatabasePath(const string &path) { - if (path.empty() || path == IN_MEMORY_PATH) { - return; - } - stored_database_path = make_uniq(db.GetDatabaseManager(), path, name); -} - bool AttachedDatabase::NameIsReserved(const string &name) { return name == DEFAULT_SCHEMA || name == TEMP_CATALOG || name == SYSTEM_CATALOG; } +string AttachedDatabase::StoredPath() const { + if (stored_database_path) { + return stored_database_path->path; + } + return string(); +} + static string RemoveQueryParams(const string &name) { auto vec = StringUtil::Split(name, "?"); D_ASSERT(!vec.empty()); @@ -186,7 +193,7 @@ void AttachedDatabase::Initialize(optional_ptr context) { catalog->Initialize(context, false); } if (storage) { - storage->Initialize(QueryContext(context)); + storage->Initialize(context); } } @@ -234,10 +241,12 @@ void AttachedDatabase::SetReadOnlyDatabase() { } void AttachedDatabase::OnDetach(ClientContext &context) { - if (!catalog) { - return; + if (catalog) { + catalog->OnDetach(context); + } + if (stored_database_path && visibility != AttachVisibility::HIDDEN) { + stored_database_path->OnDetach(); } - catalog->OnDetach(context); } void AttachedDatabase::Close() { @@ -249,14 +258,21 @@ void AttachedDatabase::Close() { // shutting down: attempt to checkpoint the database // but only if we are not cleaning up as part of an exception unwind - if (!Exception::UncaughtException() && storage && !storage->InMemory()) { - try { - auto &config = DBConfig::GetConfig(db); - if (config.options.checkpoint_on_shutdown) { - CheckpointOptions options; - options.wal_action = CheckpointWALAction::DELETE_WAL; - storage->CreateCheckpoint(QueryContext(), options); + if (!Exception::UncaughtException() && storage && !ValidChecker::IsInvalidated(db)) { + if (!storage->InMemory()) { + try { + auto &config = DBConfig::GetConfig(db); + if (config.options.checkpoint_on_shutdown) { + CheckpointOptions options; + options.wal_action = CheckpointWALAction::DELETE_WAL; + storage->CreateCheckpoint(QueryContext(), options); + } + } catch (...) { // NOLINT } + } + try { + // destroy the storage + storage->Destroy(); } catch (...) { // NOLINT } } diff --git a/src/main/buffered_data/batched_buffered_data.cpp b/src/main/buffered_data/batched_buffered_data.cpp index 3c593374cfa2..e9f949098d85 100644 --- a/src/main/buffered_data/batched_buffered_data.cpp +++ b/src/main/buffered_data/batched_buffered_data.cpp @@ -14,9 +14,8 @@ void BatchedBufferedData::BlockSink(const InterruptState &blocked_sink, idx_t ba blocked_sinks.emplace(batch, blocked_sink); } -BatchedBufferedData::BatchedBufferedData(weak_ptr context) - : BufferedData(BufferedData::Type::BATCHED, std::move(context)), buffer_byte_count(0), read_queue_byte_count(0), - min_batch(0) { +BatchedBufferedData::BatchedBufferedData(ClientContext &context) + : BufferedData(BufferedData::Type::BATCHED, context), buffer_byte_count(0), read_queue_byte_count(0), min_batch(0) { read_queue_capacity = (idx_t)(static_cast(total_buffer_size) * 0.6); buffer_capacity = (idx_t)(static_cast(total_buffer_size) * 0.4); } diff --git a/src/main/buffered_data/buffered_data.cpp b/src/main/buffered_data/buffered_data.cpp index 156539815e04..0e01df8dc692 100644 --- a/src/main/buffered_data/buffered_data.cpp +++ b/src/main/buffered_data/buffered_data.cpp @@ -4,9 +4,8 @@ namespace duckdb { -BufferedData::BufferedData(Type type, weak_ptr context_p) : type(type), context(std::move(context_p)) { - auto client_context = context.lock(); - auto &config = ClientConfig::GetConfig(*client_context); +BufferedData::BufferedData(Type type, ClientContext &context_p) : type(type), context(context_p.shared_from_this()) { + auto &config = ClientConfig::GetConfig(context_p); total_buffer_size = config.streaming_buffer_size; } diff --git a/src/main/buffered_data/simple_buffered_data.cpp b/src/main/buffered_data/simple_buffered_data.cpp index 4b6a3a534177..59cde1f43f10 100644 --- a/src/main/buffered_data/simple_buffered_data.cpp +++ b/src/main/buffered_data/simple_buffered_data.cpp @@ -6,8 +6,7 @@ namespace duckdb { -SimpleBufferedData::SimpleBufferedData(weak_ptr context) - : BufferedData(BufferedData::Type::SIMPLE, std::move(context)) { +SimpleBufferedData::SimpleBufferedData(ClientContext &context) : BufferedData(BufferedData::Type::SIMPLE, context) { buffered_count = 0; buffer_size = total_buffer_size; } diff --git a/src/main/capi/CMakeLists.txt b/src/main/capi/CMakeLists.txt index 88e061cd12a2..e92abe7501a3 100644 --- a/src/main/capi/CMakeLists.txt +++ b/src/main/capi/CMakeLists.txt @@ -27,7 +27,8 @@ add_library_unity( table_description-c.cpp table_function-c.cpp threading-c.cpp - value-c.cpp) + value-c.cpp + file_system-c.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ diff --git a/src/main/capi/aggregate_function-c.cpp b/src/main/capi/aggregate_function-c.cpp index 2ee31cff5e8c..4eb461123642 100644 --- a/src/main/capi/aggregate_function-c.cpp +++ b/src/main/capi/aggregate_function-c.cpp @@ -153,8 +153,13 @@ duckdb_aggregate_function duckdb_create_aggregate_function() { duckdb::CAPIAggregateStateInit, duckdb::CAPIAggregateUpdate, duckdb::CAPIAggregateCombine, duckdb::CAPIAggregateFinalize, nullptr, duckdb::CAPIAggregateBind); - function->function_info = duckdb::make_shared_ptr(); - return reinterpret_cast(function); + try { + function->function_info = duckdb::make_shared_ptr(); + return reinterpret_cast(function); + } catch (...) { + delete function; + return nullptr; + } } void duckdb_destroy_aggregate_function(duckdb_aggregate_function *function) { @@ -266,8 +271,12 @@ duckdb_aggregate_function_set duckdb_create_aggregate_function_set(const char *n if (!name || !*name) { return nullptr; } - auto function_set = new duckdb::AggregateFunctionSet(name); - return reinterpret_cast(function_set); + try { + auto function_set = new duckdb::AggregateFunctionSet(name); + return reinterpret_cast(function_set); + } catch (...) { + return nullptr; + } } void duckdb_destroy_aggregate_function_set(duckdb_aggregate_function_set *set) { @@ -318,6 +327,7 @@ duckdb_state duckdb_register_aggregate_function_set(duckdb_connection connection con->context->RunFunctionInTransaction([&]() { auto &catalog = duckdb::Catalog::GetSystemCatalog(*con->context); duckdb::CreateAggregateFunctionInfo sf_info(set); + sf_info.on_conflict = duckdb::OnCreateConflict::ALTER_ON_CONFLICT; catalog.CreateFunction(*con->context, sf_info); }); } catch (...) { diff --git a/src/main/capi/arrow-c.cpp b/src/main/capi/arrow-c.cpp index 204f092ef927..a1bc5391fdbb 100644 --- a/src/main/capi/arrow-c.cpp +++ b/src/main/capi/arrow-c.cpp @@ -159,9 +159,14 @@ void duckdb_destroy_arrow_converted_schema(duckdb_arrow_converted_schema *arrow_ duckdb_state duckdb_query_arrow(duckdb_connection connection, const char *query, duckdb_arrow *out_result) { Connection *conn = (Connection *)connection; auto wrapper = new ArrowResultWrapper(); - wrapper->result = conn->Query(query); - *out_result = (duckdb_arrow)wrapper; - return !wrapper->result->HasError() ? DuckDBSuccess : DuckDBError; + try { + wrapper->result = conn->Query(query); + *out_result = (duckdb_arrow)wrapper; + return !wrapper->result->HasError() ? DuckDBSuccess : DuckDBError; + } catch (...) { + delete wrapper; + return DuckDBError; + } } duckdb_state duckdb_query_arrow_schema(duckdb_arrow result, duckdb_arrow_schema *out_schema) { @@ -313,11 +318,16 @@ duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_st return DuckDBError; } auto arrow_wrapper = new ArrowResultWrapper(); - auto result = wrapper->statement->Execute(wrapper->values, false); - D_ASSERT(result->type == QueryResultType::MATERIALIZED_RESULT); - arrow_wrapper->result = duckdb::unique_ptr_cast(std::move(result)); - *out_result = reinterpret_cast(arrow_wrapper); - return !arrow_wrapper->result->HasError() ? DuckDBSuccess : DuckDBError; + try { + auto result = wrapper->statement->Execute(wrapper->values, false); + D_ASSERT(result->type == QueryResultType::MATERIALIZED_RESULT); + arrow_wrapper->result = duckdb::unique_ptr_cast(std::move(result)); + *out_result = reinterpret_cast(arrow_wrapper); + return !arrow_wrapper->result->HasError() ? DuckDBSuccess : DuckDBError; + } catch (...) { + delete arrow_wrapper; + return DuckDBError; + } } namespace arrow_array_stream_wrapper { @@ -463,13 +473,25 @@ duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *t private_data->array = reinterpret_cast(arrow_array); private_data->done = false; - ArrowArrayStream *stream = new ArrowArrayStream; - *out_stream = reinterpret_cast(stream); - stream->get_schema = arrow_array_stream_wrapper::GetSchema; - stream->get_next = arrow_array_stream_wrapper::GetNext; - stream->get_last_error = arrow_array_stream_wrapper::GetLastError; - stream->release = arrow_array_stream_wrapper::Release; - stream->private_data = private_data; - - return duckdb_arrow_scan(connection, table_name, reinterpret_cast(stream)); + ArrowArrayStream *stream; + try { + stream = new ArrowArrayStream; + } catch (...) { + delete private_data; + return DuckDBError; + } + try { + *out_stream = reinterpret_cast(stream); + stream->get_schema = arrow_array_stream_wrapper::GetSchema; + stream->get_next = arrow_array_stream_wrapper::GetNext; + stream->get_last_error = arrow_array_stream_wrapper::GetLastError; + stream->release = arrow_array_stream_wrapper::Release; + stream->private_data = private_data; + + return duckdb_arrow_scan(connection, table_name, reinterpret_cast(stream)); + } catch (...) { + delete private_data; + delete stream; + return DuckDBError; + } } diff --git a/src/main/capi/data_chunk-c.cpp b/src/main/capi/data_chunk-c.cpp index a7a3a86c53be..77f6482ab024 100644 --- a/src/main/capi/data_chunk-c.cpp +++ b/src/main/capi/data_chunk-c.cpp @@ -48,8 +48,12 @@ void duckdb_data_chunk_reset(duckdb_data_chunk chunk) { duckdb_vector duckdb_create_vector(duckdb_logical_type type, idx_t capacity) { auto dtype = reinterpret_cast(type); - auto vector = new duckdb::Vector(*dtype, capacity); - return reinterpret_cast(vector); + try { + auto vector = new duckdb::Vector(*dtype, capacity); + return reinterpret_cast(vector); + } catch (...) { + return nullptr; + } } void duckdb_destroy_vector(duckdb_vector *vector) { @@ -163,20 +167,20 @@ idx_t duckdb_list_vector_get_size(duckdb_vector vector) { duckdb_state duckdb_list_vector_set_size(duckdb_vector vector, idx_t size) { if (!vector) { - return duckdb_state::DuckDBError; + return DuckDBError; } auto v = reinterpret_cast(vector); duckdb::ListVector::SetListSize(*v, size); - return duckdb_state::DuckDBSuccess; + return DuckDBSuccess; } duckdb_state duckdb_list_vector_reserve(duckdb_vector vector, idx_t required_capacity) { if (!vector) { - return duckdb_state::DuckDBError; + return DuckDBError; } auto v = reinterpret_cast(vector); duckdb::ListVector::Reserve(*v, required_capacity); - return duckdb_state::DuckDBSuccess; + return DuckDBSuccess; } duckdb_vector duckdb_struct_vector_get_child(duckdb_vector vector, idx_t index) { diff --git a/src/main/capi/duckdb-c.cpp b/src/main/capi/duckdb-c.cpp index 43205cdf0ddd..344fa265df6f 100644 --- a/src/main/capi/duckdb-c.cpp +++ b/src/main/capi/duckdb-c.cpp @@ -148,8 +148,12 @@ void duckdb_connection_get_client_context(duckdb_connection connection, duckdb_c return; } Connection *conn = reinterpret_cast(connection); - auto wrapper = new CClientContextWrapper(*conn->context); - *out_context = reinterpret_cast(wrapper); + try { + auto wrapper = new CClientContextWrapper(*conn->context); + *out_context = reinterpret_cast(wrapper); + } catch (...) { + *out_context = nullptr; + } } void duckdb_connection_get_arrow_options(duckdb_connection connection, duckdb_arrow_options *out_arrow_options) { @@ -157,9 +161,13 @@ void duckdb_connection_get_arrow_options(duckdb_connection connection, duckdb_ar return; } Connection *conn = reinterpret_cast(connection); - auto client_properties = conn->context->GetClientProperties(); - auto wrapper = new CClientArrowOptionsWrapper(client_properties); - *out_arrow_options = reinterpret_cast(wrapper); + try { + auto client_properties = conn->context->GetClientProperties(); + auto wrapper = new CClientArrowOptionsWrapper(client_properties); + *out_arrow_options = reinterpret_cast(wrapper); + } catch (...) { + *out_arrow_options = nullptr; + } } idx_t duckdb_client_context_get_connection_id(duckdb_client_context context) { @@ -185,8 +193,12 @@ void duckdb_destroy_arrow_options(duckdb_arrow_options *arrow_options) { duckdb_state duckdb_query(duckdb_connection connection, const char *query, duckdb_result *out) { Connection *conn = reinterpret_cast(connection); - auto result = conn->Query(query); - return DuckDBTranslateResult(std::move(result), out); + try { + auto result = conn->Query(query); + return DuckDBTranslateResult(std::move(result), out); + } catch (...) { + return DuckDBError; + } } const char *duckdb_library_version() { @@ -195,26 +207,38 @@ const char *duckdb_library_version() { duckdb_value duckdb_get_table_names(duckdb_connection connection, const char *query, bool qualified) { Connection *conn = reinterpret_cast(connection); - auto table_names = conn->GetTableNames(query, qualified); + try { + auto table_names = conn->GetTableNames(query, qualified); - auto count = table_names.size(); - auto ptr = malloc(count * sizeof(duckdb_value)); - auto list_values = reinterpret_cast(ptr); + auto count = table_names.size(); + auto ptr = malloc(count * sizeof(duckdb_value)); + if (!ptr) { + return nullptr; + } + auto list_values = reinterpret_cast(ptr); - idx_t name_ix = 0; - for (const auto &name : table_names) { - list_values[name_ix] = duckdb_create_varchar(name.c_str()); - name_ix++; - } + try { + idx_t name_ix = 0; + for (const auto &name : table_names) { + list_values[name_ix] = duckdb_create_varchar(name.c_str()); + name_ix++; + } - auto varchar_type = duckdb_create_logical_type(DUCKDB_TYPE_VARCHAR); - auto list_value = duckdb_create_list_value(varchar_type, list_values, count); + auto varchar_type = duckdb_create_logical_type(DUCKDB_TYPE_VARCHAR); + auto list_value = duckdb_create_list_value(varchar_type, list_values, count); - for (idx_t i = 0; i < count; i++) { - duckdb_destroy_value(&list_values[i]); - } - duckdb_free(ptr); - duckdb_destroy_logical_type(&varchar_type); + for (idx_t i = 0; i < count; i++) { + duckdb_destroy_value(&list_values[i]); + } + duckdb_free(ptr); + duckdb_destroy_logical_type(&varchar_type); - return list_value; + return list_value; + } catch (...) { + duckdb_free(ptr); + return nullptr; + } + } catch (...) { + return nullptr; + } } diff --git a/src/main/capi/file_system-c.cpp b/src/main/capi/file_system-c.cpp new file mode 100644 index 000000000000..af82daa6cb08 --- /dev/null +++ b/src/main/capi/file_system-c.cpp @@ -0,0 +1,274 @@ +#include "duckdb/main/capi/capi_internal.hpp" + +namespace duckdb { +namespace { +struct CFileSystem { + + FileSystem &fs; + ErrorData error_data; + + explicit CFileSystem(FileSystem &fs_p) : fs(fs_p) { + } + + void SetError(const char *message) { + error_data = ErrorData(ExceptionType::IO, message); + } + void SetError(const std::exception &ex) { + error_data = ErrorData(ex); + } +}; + +struct CFileOpenOptions { + duckdb::FileOpenFlags flags; +}; + +struct CFileHandle { + ErrorData error_data; + unique_ptr handle; + + void SetError(const char *message) { + error_data = ErrorData(ExceptionType::IO, message); + } + void SetError(const std::exception &ex) { + error_data = ErrorData(ex); + } +}; + +} // namespace +} // namespace duckdb + +duckdb_file_system duckdb_client_context_get_file_system(duckdb_client_context context) { + if (!context) { + return nullptr; + } + auto ctx = reinterpret_cast(context); + auto wrapper = new duckdb::CFileSystem(duckdb::FileSystem::GetFileSystem(ctx->context)); + return reinterpret_cast(wrapper); +} + +void duckdb_destroy_file_system(duckdb_file_system *file_system) { + if (!file_system || !*file_system) { + return; + } + const auto fs = reinterpret_cast(*file_system); + delete fs; + *file_system = nullptr; +} + +duckdb_file_open_options duckdb_create_file_open_options() { + auto options = new duckdb::CFileOpenOptions(); + return reinterpret_cast(options); +} + +duckdb_state duckdb_file_open_options_set_flag(duckdb_file_open_options options, duckdb_file_flag flag, bool value) { + if (!options) { + return DuckDBError; + } + auto coptions = reinterpret_cast(options); + + switch (flag) { + case DUCKDB_FILE_FLAG_READ: + coptions->flags |= duckdb::FileOpenFlags::FILE_FLAGS_READ; + break; + case DUCKDB_FILE_FLAG_WRITE: + coptions->flags |= duckdb::FileOpenFlags::FILE_FLAGS_WRITE; + break; + case DUCKDB_FILE_FLAG_APPEND: + coptions->flags |= duckdb::FileOpenFlags::FILE_FLAGS_APPEND; + break; + case DUCKDB_FILE_FLAG_CREATE: + coptions->flags |= duckdb::FileOpenFlags::FILE_FLAGS_FILE_CREATE; + break; + case DUCKDB_FILE_FLAG_CREATE_NEW: + coptions->flags |= duckdb::FileOpenFlags::FILE_FLAGS_EXCLUSIVE_CREATE; + break; + default: + return DuckDBError; + } + return DuckDBSuccess; +} + +void duckdb_destroy_file_open_options(duckdb_file_open_options *options) { + if (!options || !*options) { + return; + } + auto coptions = reinterpret_cast(*options); + delete coptions; + *options = nullptr; +} + +duckdb_state duckdb_file_system_open(duckdb_file_system fs, const char *path, duckdb_file_open_options options, + duckdb_file_handle *out_file) { + if (!fs) { + *out_file = nullptr; + return DuckDBError; + } + auto cfs = reinterpret_cast(fs); + if (!path || !options || !out_file) { + cfs->SetError("Invalid input to duckdb_file_system_open"); + *out_file = nullptr; + return DuckDBError; + } + + try { + auto coptions = reinterpret_cast(options); + auto handle = cfs->fs.OpenFile(duckdb::string(path), coptions->flags); + auto wrapper = new duckdb::CFileHandle(); + wrapper->handle = std::move(handle); + *out_file = reinterpret_cast(wrapper); + return DuckDBSuccess; + } catch (const std::exception &ex) { + cfs->SetError(ex); + *out_file = nullptr; + return DuckDBError; + } catch (...) { + cfs->SetError("Unknown error occurred during file open"); + *out_file = nullptr; + return DuckDBError; + } +} + +duckdb_error_data duckdb_file_system_error_data(duckdb_file_system fs) { + auto wrapper = new duckdb::ErrorDataWrapper(); + if (!fs) { + return reinterpret_cast(wrapper); + } + auto cfs = reinterpret_cast(fs); + wrapper->error_data = cfs->error_data; + return reinterpret_cast(wrapper); +} + +void duckdb_destroy_file_handle(duckdb_file_handle *file) { + if (!file || !*file) { + return; + } + auto cfile = reinterpret_cast(*file); + cfile->handle->Close(); // Ensure the file is closed before destroying + delete cfile; + *file = nullptr; +} + +duckdb_error_data duckdb_file_handle_error_data(duckdb_file_handle file) { + auto wrapper = new duckdb::ErrorDataWrapper(); + if (!file) { + return reinterpret_cast(wrapper); + } + auto cfile = reinterpret_cast(file); + wrapper->error_data = cfile->error_data; + return reinterpret_cast(wrapper); +} + +int64_t duckdb_file_handle_read(duckdb_file_handle file, void *buffer, int64_t size) { + if (!file || !buffer || size < 0) { + return -1; + } + auto cfile = reinterpret_cast(file); + try { + return cfile->handle->Read(buffer, static_cast(size)); + } catch (std::exception &ex) { + cfile->SetError(ex); + return -1; + } catch (...) { + cfile->SetError("Unknown error occurred during file read"); + return -1; + } +} + +int64_t duckdb_file_handle_write(duckdb_file_handle file, const void *buffer, int64_t size) { + if (!file || !buffer || size < 0) { + return -1; + } + auto cfile = reinterpret_cast(file); + try { + return cfile->handle->Write(const_cast(buffer), static_cast(size)); + } catch (std::exception &ex) { + cfile->SetError(ex); + return -1; + } catch (...) { + cfile->SetError("Unknown error occurred during file write"); + return -1; + } +} + +int64_t duckdb_file_handle_tell(duckdb_file_handle file) { + if (!file) { + return -1; + } + auto cfile = reinterpret_cast(file); + try { + return static_cast(cfile->handle->SeekPosition()); + } catch (std::exception &ex) { + cfile->SetError(ex); + return -1; + } catch (...) { + cfile->SetError("Unknown error occurred when getting file position"); + return -1; + } +} + +int64_t duckdb_file_handle_size(duckdb_file_handle file) { + if (!file) { + return -1; + } + auto cfile = reinterpret_cast(file); + try { + return static_cast(cfile->handle->GetFileSize()); + } catch (std::exception &ex) { + cfile->SetError(ex); + return -1; + } catch (...) { + cfile->SetError("Unknown error occurred when getting file size"); + return -1; + } +} + +duckdb_state duckdb_file_handle_seek(duckdb_file_handle file, int64_t position) { + if (!file || position < 0) { + return DuckDBError; + } + auto cfile = reinterpret_cast(file); + try { + cfile->handle->Seek(static_cast(position)); + return DuckDBSuccess; + } catch (std::exception &ex) { + cfile->SetError(ex); + return DuckDBError; + } catch (...) { + cfile->SetError("Unknown error occurred when seeking in file"); + return DuckDBError; + } +} + +duckdb_state duckdb_file_handle_sync(duckdb_file_handle file) { + if (!file) { + return DuckDBError; + } + auto cfile = reinterpret_cast(file); + try { + cfile->handle->Sync(); + return DuckDBSuccess; + } catch (std::exception &ex) { + cfile->SetError(ex); + return DuckDBError; + } catch (...) { + cfile->SetError("Unknown error occurred when syncing file"); + return DuckDBError; + } +} + +duckdb_state duckdb_file_handle_close(duckdb_file_handle file) { + if (!file) { + return DuckDBError; + } + auto cfile = reinterpret_cast(file); + try { + cfile->handle->Close(); + return DuckDBSuccess; + } catch (std::exception &ex) { + cfile->SetError(ex); + return DuckDBError; + } catch (...) { + cfile->SetError("Unknown error occurred when closing file"); + return DuckDBError; + } +} diff --git a/src/main/capi/helper-c.cpp b/src/main/capi/helper-c.cpp index c936bab5928c..9096d7e65bf1 100644 --- a/src/main/capi/helper-c.cpp +++ b/src/main/capi/helper-c.cpp @@ -92,6 +92,8 @@ duckdb_type LogicalTypeIdToC(const LogicalTypeId type) { switch (type) { case LogicalTypeId::INVALID: return DUCKDB_TYPE_INVALID; + case LogicalTypeId::UNKNOWN: + return DUCKDB_TYPE_INVALID; case LogicalTypeId::BOOLEAN: return DUCKDB_TYPE_BOOLEAN; case LogicalTypeId::TINYINT: diff --git a/src/main/capi/logical_types-c.cpp b/src/main/capi/logical_types-c.cpp index 7198ec8e6b2c..d30a2d6db899 100644 --- a/src/main/capi/logical_types-c.cpp +++ b/src/main/capi/logical_types-c.cpp @@ -49,9 +49,13 @@ duckdb_logical_type duckdb_create_list_type(duckdb_logical_type type) { if (!type) { return nullptr; } - duckdb::LogicalType *logical_type = new duckdb::LogicalType; - *logical_type = duckdb::LogicalType::LIST(*reinterpret_cast(type)); - return reinterpret_cast(logical_type); + try { + duckdb::LogicalType *logical_type = + new duckdb::LogicalType(duckdb::LogicalType::LIST(*reinterpret_cast(type))); + return reinterpret_cast(logical_type); + } catch (...) { + return nullptr; + } } duckdb_logical_type duckdb_create_array_type(duckdb_logical_type type, idx_t array_size) { @@ -61,9 +65,13 @@ duckdb_logical_type duckdb_create_array_type(duckdb_logical_type type, idx_t arr if (array_size >= duckdb::ArrayType::MAX_ARRAY_SIZE) { return nullptr; } - duckdb::LogicalType *ltype = new duckdb::LogicalType; - *ltype = duckdb::LogicalType::ARRAY(*reinterpret_cast(type), array_size); - return reinterpret_cast(ltype); + try { + duckdb::LogicalType *ltype = new duckdb::LogicalType( + duckdb::LogicalType::ARRAY(*reinterpret_cast(type), array_size)); + return reinterpret_cast(ltype); + } catch (...) { + return nullptr; + } } duckdb_logical_type duckdb_create_union_type(duckdb_logical_type *member_types_p, const char **member_names, @@ -72,14 +80,17 @@ duckdb_logical_type duckdb_create_union_type(duckdb_logical_type *member_types_p return nullptr; } duckdb::LogicalType **member_types = reinterpret_cast(member_types_p); - duckdb::LogicalType *mtype = new duckdb::LogicalType; - duckdb::child_list_t members; + try { + duckdb::child_list_t members; - for (idx_t i = 0; i < member_count; i++) { - members.push_back(make_pair(member_names[i], *member_types[i])); + for (idx_t i = 0; i < member_count; i++) { + members.push_back(make_pair(member_names[i], *member_types[i])); + } + duckdb::LogicalType *mtype = new duckdb::LogicalType(duckdb::LogicalType::UNION(members)); + return reinterpret_cast(mtype); + } catch (...) { + return nullptr; } - *mtype = duckdb::LogicalType::UNION(members); - return reinterpret_cast(mtype); } duckdb_logical_type duckdb_create_struct_type(duckdb_logical_type *member_types_p, const char **member_names, @@ -94,14 +105,17 @@ duckdb_logical_type duckdb_create_struct_type(duckdb_logical_type *member_types_ } } - duckdb::LogicalType *mtype = new duckdb::LogicalType; - duckdb::child_list_t members; + try { + duckdb::child_list_t members; - for (idx_t i = 0; i < member_count; i++) { - members.push_back(make_pair(member_names[i], *member_types[i])); + for (idx_t i = 0; i < member_count; i++) { + members.push_back(make_pair(member_names[i], *member_types[i])); + } + duckdb::LogicalType *mtype = new duckdb::LogicalType(duckdb::LogicalType::STRUCT(members)); + return reinterpret_cast(mtype); + } catch (...) { + return nullptr; } - *mtype = duckdb::LogicalType::STRUCT(members); - return reinterpret_cast(mtype); } duckdb_logical_type duckdb_create_enum_type(const char **member_names, idx_t member_count) { @@ -118,19 +132,25 @@ duckdb_logical_type duckdb_create_enum_type(const char **member_names, idx_t mem enum_vector_ptr[i] = duckdb::StringVector::AddStringOrBlob(enum_vector, member_names[i]); } - duckdb::LogicalType *mtype = new duckdb::LogicalType; - *mtype = duckdb::LogicalType::ENUM(enum_vector, member_count); - return reinterpret_cast(mtype); + try { + duckdb::LogicalType *mtype = new duckdb::LogicalType(duckdb::LogicalType::ENUM(enum_vector, member_count)); + return reinterpret_cast(mtype); + } catch (...) { + return nullptr; + } } duckdb_logical_type duckdb_create_map_type(duckdb_logical_type key_type, duckdb_logical_type value_type) { if (!key_type || !value_type) { return nullptr; } - duckdb::LogicalType *mtype = new duckdb::LogicalType; - *mtype = duckdb::LogicalType::MAP(*reinterpret_cast(key_type), - *reinterpret_cast(value_type)); - return reinterpret_cast(mtype); + try { + duckdb::LogicalType *mtype = new duckdb::LogicalType(duckdb::LogicalType::MAP( + *reinterpret_cast(key_type), *reinterpret_cast(value_type))); + return reinterpret_cast(mtype); + } catch (...) { + return nullptr; + } } duckdb_logical_type duckdb_create_decimal_type(uint8_t width, uint8_t scale) { diff --git a/src/main/capi/prepared-c.cpp b/src/main/capi/prepared-c.cpp index e32b321d2be5..ac5b638f8311 100644 --- a/src/main/capi/prepared-c.cpp +++ b/src/main/capi/prepared-c.cpp @@ -51,10 +51,14 @@ duckdb_state duckdb_prepare_extracted_statement(duckdb_connection connection, return DuckDBError; } auto wrapper = new PreparedStatementWrapper(); - wrapper->statement = conn->Prepare(std::move(source_wrapper->statements[index])); - - *out_prepared_statement = (duckdb_prepared_statement)wrapper; - return wrapper->statement->HasError() ? DuckDBError : DuckDBSuccess; + try { + wrapper->statement = conn->Prepare(std::move(source_wrapper->statements[index])); + *out_prepared_statement = (duckdb_prepared_statement)wrapper; + return wrapper->statement->HasError() ? DuckDBError : DuckDBSuccess; + } catch (...) { + delete wrapper; + return DuckDBError; + } } const char *duckdb_extract_statements_error(duckdb_extracted_statements extracted_statements) { @@ -72,14 +76,25 @@ duckdb_state duckdb_prepare(duckdb_connection connection, const char *query, } auto wrapper = new PreparedStatementWrapper(); Connection *conn = reinterpret_cast(connection); - wrapper->statement = conn->Prepare(query); - *out_prepared_statement = reinterpret_cast(wrapper); - return !wrapper->statement->HasError() ? DuckDBSuccess : DuckDBError; + try { + wrapper->statement = conn->Prepare(query); + *out_prepared_statement = reinterpret_cast(wrapper); + return !wrapper->statement->HasError() ? DuckDBSuccess : DuckDBError; + } catch (...) { + delete wrapper; + return DuckDBError; + } } const char *duckdb_prepare_error(duckdb_prepared_statement prepared_statement) { auto wrapper = reinterpret_cast(prepared_statement); - if (!wrapper || !wrapper->statement || !wrapper->statement->HasError()) { + if (!wrapper) { + return nullptr; + } + if (!wrapper->success) { + return wrapper->error_data.Message().c_str(); + } + if (!wrapper->statement || !wrapper->statement->HasError()) { return nullptr; } return wrapper->statement->error.Message().c_str(); @@ -167,6 +182,52 @@ duckdb_state duckdb_clear_bindings(duckdb_prepared_statement prepared_statement) return DuckDBSuccess; } +idx_t duckdb_prepared_statement_column_count(duckdb_prepared_statement prepared_statement) { + auto wrapper = reinterpret_cast(prepared_statement); + if (!wrapper || !wrapper->statement || wrapper->statement->HasError()) { + return 0; + } + return wrapper->statement->ColumnCount(); +} + +const char *duckdb_prepared_statement_column_name(duckdb_prepared_statement prepared_statement, idx_t col_idx) { + auto wrapper = reinterpret_cast(prepared_statement); + if (!wrapper || !wrapper->statement || wrapper->statement->HasError()) { + return nullptr; + } + auto &names = wrapper->statement->GetNames(); + + if (col_idx >= names.size()) { + return nullptr; + } + return strdup(names[col_idx].c_str()); +} + +duckdb_logical_type duckdb_prepared_statement_column_logical_type(duckdb_prepared_statement prepared_statement, + idx_t col_idx) { + auto wrapper = reinterpret_cast(prepared_statement); + if (!wrapper || !wrapper->statement || wrapper->statement->HasError()) { + return nullptr; + } + auto types = wrapper->statement->GetTypes(); + if (col_idx >= types.size()) { + return nullptr; + } + return reinterpret_cast(new LogicalType(types[col_idx])); +} + +duckdb_type duckdb_prepared_statement_column_type(duckdb_prepared_statement prepared_statement, idx_t col_idx) { + auto logical_type = duckdb_prepared_statement_column_logical_type(prepared_statement, col_idx); + if (!logical_type) { + return DUCKDB_TYPE_INVALID; + } + + auto type = duckdb_get_type_id(logical_type); + duckdb_destroy_logical_type(&logical_type); + + return type; +} + duckdb_state duckdb_bind_value(duckdb_prepared_statement prepared_statement, idx_t param_idx, duckdb_value val) { auto value = reinterpret_cast(val); auto wrapper = reinterpret_cast(prepared_statement); @@ -174,9 +235,10 @@ duckdb_state duckdb_bind_value(duckdb_prepared_statement prepared_statement, idx return DuckDBError; } if (param_idx <= 0 || param_idx > wrapper->statement->named_param_map.size()) { - wrapper->statement->error = + wrapper->error_data = duckdb::InvalidInputException("Can not bind to parameter number %d, statement only has %d parameter(s)", param_idx, wrapper->statement->named_param_map.size()); + wrapper->success = false; return DuckDBError; } auto identifier = duckdb_parameter_name_internal(prepared_statement, param_idx); @@ -372,8 +434,12 @@ duckdb_state duckdb_execute_prepared_streaming(duckdb_prepared_statement prepare return DuckDBError; } - auto result = wrapper->statement->Execute(wrapper->values, true); - return DuckDBTranslateResult(std::move(result), out_result); + try { + auto result = wrapper->statement->Execute(wrapper->values, true); + return DuckDBTranslateResult(std::move(result), out_result); + } catch (...) { + return DuckDBError; + } } duckdb_statement_type duckdb_prepared_statement_type(duckdb_prepared_statement statement) { diff --git a/src/main/capi/scalar_function-c.cpp b/src/main/capi/scalar_function-c.cpp index b84dbfe7076a..7233b2c2064d 100644 --- a/src/main/capi/scalar_function-c.cpp +++ b/src/main/capi/scalar_function-c.cpp @@ -406,6 +406,7 @@ duckdb_state duckdb_register_scalar_function_set(duckdb_connection connection, d con->context->RunFunctionInTransaction([&]() { auto &catalog = duckdb::Catalog::GetSystemCatalog(*con->context); duckdb::CreateScalarFunctionInfo sf_info(scalar_function_set); + sf_info.on_conflict = duckdb::OnCreateConflict::ALTER_ON_CONFLICT; catalog.CreateFunction(*con->context, sf_info); }); } catch (...) { diff --git a/src/main/capi/table_description-c.cpp b/src/main/capi/table_description-c.cpp index 26624bbfc1d9..cfcd01c4350d 100644 --- a/src/main/capi/table_description-c.cpp +++ b/src/main/capi/table_description-c.cpp @@ -1,5 +1,5 @@ -#include "duckdb/main/capi/capi_internal.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/main/capi/capi_internal.hpp" using duckdb::Connection; using duckdb::ErrorData; @@ -68,14 +68,14 @@ const char *duckdb_table_description_error(duckdb_table_description table) { return wrapper->error.c_str(); } -duckdb_state GetTableDescription(TableDescriptionWrapper *wrapper, idx_t index) { +duckdb_state GetTableDescription(TableDescriptionWrapper *wrapper, duckdb::optional_idx index) { if (!wrapper) { return DuckDBError; } auto &table = wrapper->description; - if (index >= table->columns.size()) { - wrapper->error = duckdb::StringUtil::Format("Column index %d is out of range, table only has %d columns", index, - table->columns.size()); + if (index.IsValid() && index.GetIndex() >= table->columns.size()) { + wrapper->error = duckdb::StringUtil::Format("Column index %d is out of range, table only has %d columns", + index.GetIndex(), table->columns.size()); return DuckDBError; } return DuckDBSuccess; @@ -97,6 +97,16 @@ duckdb_state duckdb_column_has_default(duckdb_table_description table_descriptio return DuckDBSuccess; } +idx_t duckdb_table_description_get_column_count(duckdb_table_description table_description) { + auto wrapper = reinterpret_cast(table_description); + if (GetTableDescription(wrapper, duckdb::optional_idx()) == DuckDBError) { + return 0; + } + + auto &table = wrapper->description; + return table->columns.size(); +} + char *duckdb_table_description_get_column_name(duckdb_table_description table_description, idx_t index) { auto wrapper = reinterpret_cast(table_description); if (GetTableDescription(wrapper, index) == DuckDBError) { @@ -113,3 +123,16 @@ char *duckdb_table_description_get_column_name(duckdb_table_description table_de return result; } + +duckdb_logical_type duckdb_table_description_get_column_type(duckdb_table_description table_description, idx_t index) { + auto wrapper = reinterpret_cast(table_description); + if (GetTableDescription(wrapper, index) == DuckDBError) { + return nullptr; + } + + auto &table = wrapper->description; + auto &column = table->columns[index]; + + auto logical_type = new duckdb::LogicalType(column.Type()); + return reinterpret_cast(logical_type); +} diff --git a/src/main/capi/table_function-c.cpp b/src/main/capi/table_function-c.cpp index 268d9a04a042..deb382ebd7df 100644 --- a/src/main/capi/table_function-c.cpp +++ b/src/main/capi/table_function-c.cpp @@ -347,6 +347,7 @@ duckdb_state duckdb_register_table_function(duckdb_connection connection, duckdb con->context->RunFunctionInTransaction([&]() { auto &catalog = duckdb::Catalog::GetSystemCatalog(*con->context); duckdb::CreateTableFunctionInfo tf_info(tf); + tf_info.on_conflict = duckdb::OnCreateConflict::ALTER_ON_CONFLICT; catalog.CreateTableFunction(*con->context, tf_info); }); } catch (...) { // LCOV_EXCL_START diff --git a/src/main/client_config.cpp b/src/main/client_config.cpp index 868c80730bc6..e8e7d8d864b5 100644 --- a/src/main/client_config.cpp +++ b/src/main/client_config.cpp @@ -8,8 +8,8 @@ bool ClientConfig::AnyVerification() const { return query_verification_enabled || verify_external || verify_serializer || verify_fetch_row; } -void ClientConfig::SetUserVariable(const string &name, Value value) { - user_variables[name] = std::move(value); +void ClientConfig::SetUserVariable(const String &name, Value value) { + user_variables[name.ToStdString()] = std::move(value); } bool ClientConfig::GetUserVariable(const string &name, Value &result) { diff --git a/src/main/client_context.cpp b/src/main/client_context.cpp index 46bf97b828a0..831f823704e8 100644 --- a/src/main/client_context.cpp +++ b/src/main/client_context.cpp @@ -217,15 +217,15 @@ void ClientContext::BeginQueryInternal(ClientContextLock &lock, const string &qu state->QueryBegin(*this); } - // Flush the old Logger + // Flush the old logger. logger->Flush(); - // Refresh the logger to ensure we are in sync with global log settings - LoggingContext context(LogContextScope::CONNECTION); - context.connection_id = connection_id; - context.transaction_id = transaction.ActiveTransaction().global_transaction_id; - context.query_id = transaction.GetActiveQuery(); - logger = db->GetLogManager().CreateLogger(context, true); + // Refresh the logger to ensure we are in sync with the global log settings. + LoggingContext logging_context(LogContextScope::CONNECTION); + logging_context.connection_id = connection_id; + logging_context.transaction_id = transaction.ActiveTransaction().global_transaction_id; + logging_context.query_id = transaction.GetActiveQuery(); + logger = db->GetLogManager().CreateLogger(logging_context, true); DUCKDB_LOG(*this, QueryLogType, query); } @@ -889,6 +889,10 @@ unique_ptr ClientContext::PendingStatementOrPreparedStatemen shared_ptr &prepared, const PendingQueryParameters ¶meters) { unique_ptr pending; + // Start the profiler. + auto &profiler = QueryProfiler::Get(*this); + profiler.StartQuery(query, IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get())); + try { BeginQueryInternal(lock, query); } catch (std::exception &ex) { @@ -900,9 +904,6 @@ unique_ptr ClientContext::PendingStatementOrPreparedStatemen } return ErrorResult(std::move(error), query); } - // start the profiler - auto &profiler = QueryProfiler::Get(*this); - profiler.StartQuery(query, IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get())); bool invalidate_query = true; try { @@ -1406,15 +1407,7 @@ unique_ptr ClientContext::Execute(const shared_ptr &relat return ErrorResult(ErrorData(err_str)); } -SettingLookupResult ClientContext::TryGetCurrentSetting(const std::string &key, Value &result) const { - // first check the built-in settings - auto &db_config = DBConfig::GetConfig(*this); - auto option = db_config.GetOptionByName(key); - if (option && option->get_setting) { - result = option->get_setting(*this); - return SettingLookupResult(SettingScope::LOCAL); - } - +SettingLookupResult ClientContext::TryGetCurrentSettingInternal(const string &key, Value &result) const { // check the client session values const auto &session_config_map = config.set_variables; @@ -1428,6 +1421,21 @@ SettingLookupResult ClientContext::TryGetCurrentSetting(const std::string &key, return db->TryGetCurrentSetting(key, result); } +SettingLookupResult ClientContext::TryGetCurrentSetting(const string &key, Value &result) const { + // first check the built-in settings + auto &db_config = DBConfig::GetConfig(*this); + auto option = db_config.GetOptionByName(key); + if (option) { + if (option->get_setting) { + result = option->get_setting(*this); + return SettingLookupResult(SettingScope::LOCAL); + } + // alias - search for the default key + return TryGetCurrentSettingInternal(option->name, result); + } + return TryGetCurrentSettingInternal(key, result); +} + ParserOptions ClientContext::GetParserOptions() const { auto &client_config = ClientConfig::GetConfig(*this); ParserOptions options; @@ -1435,6 +1443,7 @@ ParserOptions ClientContext::GetParserOptions() const { options.integer_division = DBConfig::GetSetting(*this); options.max_expression_depth = client_config.max_expression_depth; options.extensions = &DBConfig::GetConfig(*this).parser_extensions; + options.parser_override_setting = DBConfig::GetConfig(*this).options.allow_parser_override_extension; return options; } diff --git a/src/main/client_data.cpp b/src/main/client_data.cpp index 1348c0b09a33..0c63cca163b4 100644 --- a/src/main/client_data.cpp +++ b/src/main/client_data.cpp @@ -56,6 +56,9 @@ class ClientBufferManager : public BufferManager { return buffer_manager.ReAllocate(handle, block_size); } BufferHandle Pin(shared_ptr &handle) override { + return Pin(QueryContext(), handle); + } + BufferHandle Pin(const QueryContext &context, shared_ptr &handle) override { return buffer_manager.Pin(handle); } void Prefetch(vector> &handles) override { @@ -116,6 +119,9 @@ class ClientBufferManager : public BufferManager { return buffer_manager.SetSwapLimit(limit); } + BlockManager &GetTemporaryBlockManager() override { + return buffer_manager.GetTemporaryBlockManager(); + } vector GetTemporaryFiles() override { return buffer_manager.GetTemporaryFiles(); } diff --git a/src/main/config.cpp b/src/main/config.cpp index f98ac7c17989..b3baf6fcf9d3 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -63,6 +63,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_GLOBAL(AllocatorFlushThresholdSetting), DUCKDB_GLOBAL(AllowCommunityExtensionsSetting), DUCKDB_SETTING(AllowExtensionsMetadataMismatchSetting), + DUCKDB_GLOBAL(AllowParserOverrideExtensionSetting), DUCKDB_GLOBAL(AllowPersistentSecretsSetting), DUCKDB_GLOBAL(AllowUnredactedSecretsSetting), DUCKDB_GLOBAL(AllowUnsignedExtensionsSetting), @@ -115,6 +116,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(EnableViewDependenciesSetting), DUCKDB_GLOBAL(EnabledLogTypes), DUCKDB_LOCAL(ErrorsAsJSONSetting), + DUCKDB_SETTING(ExperimentalMetadataReuseSetting), DUCKDB_LOCAL(ExplainOutputSetting), DUCKDB_GLOBAL(ExtensionDirectorySetting), DUCKDB_GLOBAL(ExternalThreadsSetting), @@ -167,21 +169,23 @@ static const ConfigurationOption internal_options[] = { DUCKDB_LOCAL(SchemaSetting), DUCKDB_LOCAL(SearchPathSetting), DUCKDB_GLOBAL(SecretDirectorySetting), + DUCKDB_SETTING_CALLBACK(StorageBlockPrefetchSetting), DUCKDB_GLOBAL(StorageCompatibilityVersionSetting), DUCKDB_LOCAL(StreamingBufferSizeSetting), DUCKDB_GLOBAL(TempDirectorySetting), DUCKDB_GLOBAL(TempFileEncryptionSetting), DUCKDB_GLOBAL(ThreadsSetting), DUCKDB_GLOBAL(UsernameSetting), + DUCKDB_SETTING(WriteBufferRowGroupCountSetting), DUCKDB_GLOBAL(ZstdMinStringLengthSetting), FINAL_SETTING}; -static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 82), - DUCKDB_SETTING_ALIAS("null_order", 33), - DUCKDB_SETTING_ALIAS("profiling_output", 101), - DUCKDB_SETTING_ALIAS("user", 115), - DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 20), - DUCKDB_SETTING_ALIAS("worker_threads", 114), +static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 84), + DUCKDB_SETTING_ALIAS("null_order", 34), + DUCKDB_SETTING_ALIAS("profiling_output", 103), + DUCKDB_SETTING_ALIAS("user", 118), + DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 21), + DUCKDB_SETTING_ALIAS("worker_threads", 117), FINAL_ALIAS}; vector DBConfig::GetOptions() { @@ -323,9 +327,9 @@ void DBConfig::ResetOption(optional_ptr db, const Configuratio option.reset_global(db.get(), *this); } -void DBConfig::SetOption(const string &name, Value value) { +void DBConfig::SetOption(const String &name, Value value) { lock_guard l(config_lock); - options.set_variables[name] = std::move(value); + options.set_variables[name.ToStdString()] = std::move(value); } void DBConfig::ResetOption(const String &name) { @@ -496,6 +500,8 @@ void DBConfig::SetDefaultTempDirectory() { options.temporary_directory = string(); } else if (DBConfig::IsInMemoryDatabase(options.database_path.c_str())) { options.temporary_directory = ".tmp"; + } else if (StringUtil::Contains(options.database_path, "?")) { + options.temporary_directory = StringUtil::Split(options.database_path, "?")[0] + ".tmp"; } else { options.temporary_directory = options.database_path + ".tmp"; } @@ -512,8 +518,7 @@ void DBConfig::CheckLock(const String &name) { return; } // not allowed! - throw InvalidInputException("Cannot change configuration option \"%s\" - the configuration has been locked", - name.ToStdString()); + throw InvalidInputException("Cannot change configuration option \"%s\" - the configuration has been locked", name); } idx_t DBConfig::GetSystemMaxThreads(FileSystem &fs) { @@ -536,6 +541,10 @@ idx_t DBConfig::GetSystemMaxThreads(FileSystem &fs) { } idx_t DBConfig::GetSystemAvailableMemory(FileSystem &fs) { + // System memory detection + auto memory = FileSystem::GetAvailableMemory(); + auto available_memory = memory.IsValid() ? memory.GetIndex() : DBConfigOptions().maximum_memory; + #ifdef __linux__ // Check SLURM environment variables first const char *slurm_mem_per_node = getenv("SLURM_MEM_PER_NODE"); @@ -557,16 +566,12 @@ idx_t DBConfig::GetSystemAvailableMemory(FileSystem &fs) { // Check cgroup memory limit auto cgroup_memory_limit = CGroups::GetMemoryLimit(fs); if (cgroup_memory_limit.IsValid()) { - return cgroup_memory_limit.GetIndex(); + auto cgroup_memory_limit_value = cgroup_memory_limit.GetIndex(); + return std::min(cgroup_memory_limit_value, available_memory); } #endif - // System memory detection - auto memory = FileSystem::GetAvailableMemory(); - if (!memory.IsValid()) { - return DBConfigOptions().maximum_memory; - } - return memory.GetIndex(); + return available_memory; } idx_t DBConfig::ParseMemoryLimit(const string &arg) { diff --git a/src/main/connection.cpp b/src/main/connection.cpp index e561a3cb9c7a..af76cfd1739e 100644 --- a/src/main/connection.cpp +++ b/src/main/connection.cpp @@ -19,7 +19,7 @@ namespace duckdb { Connection::Connection(DatabaseInstance &database) - : context(make_shared_ptr(database.shared_from_this())), warning_cb(nullptr) { + : context(make_shared_ptr(database.shared_from_this())) { auto &connection_manager = ConnectionManager::Get(database); connection_manager.AddConnection(*context); connection_manager.AssignConnectionId(*this); @@ -31,18 +31,15 @@ Connection::Connection(DatabaseInstance &database) } Connection::Connection(DuckDB &database) : Connection(*database.instance) { - // Initialization of warning_cb happens in the other constructor } -Connection::Connection(Connection &&other) noexcept : warning_cb(nullptr) { +Connection::Connection(Connection &&other) noexcept { std::swap(context, other.context); - std::swap(warning_cb, other.warning_cb); std::swap(connection_id, other.connection_id); } Connection &Connection::operator=(Connection &&other) noexcept { std::swap(context, other.context); - std::swap(warning_cb, other.warning_cb); std::swap(connection_id, other.connection_id); return *this; } diff --git a/src/main/database.cpp b/src/main/database.cpp index bfecdec1c092..3d644d4084a4 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -31,6 +31,7 @@ #include "duckdb/logging/logger.hpp" #include "duckdb/common/http_util.hpp" #include "mbedtls_wrapper.hpp" +#include "duckdb/main/database_file_path_manager.hpp" #ifndef DUCKDB_NO_THREADS #include "duckdb/common/thread.hpp" @@ -198,10 +199,8 @@ void DatabaseInstance::CreateMainDatabase() { Connection con(*this); con.BeginTransaction(); AttachOptions options(config.options); - auto initial_database = db_manager->AttachDatabase(*con.context, info, options); - initial_database->SetInitialDatabase(); - initial_database->Initialize(*con.context); - db_manager->FinalizeAttach(*con.context, info, std::move(initial_database)); + options.is_main_database = true; + db_manager->AttachDatabase(*con.context, info, options); con.Commit(); } @@ -297,17 +296,14 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf // initialize the secret manager config.secret_manager->Initialize(*this); - // resolve the type of teh database we are opening + // resolve the type of the database we are opening auto &fs = FileSystem::GetFileSystem(*this); DBPathAndType::ResolveDatabaseType(fs, config.options.database_path, config.options.database_type); // initialize the system catalog db_manager->InitializeSystemCatalog(); - if (config.options.database_type == "duckdb") { - config.options.database_type = string(); - } - if (!config.options.database_type.empty()) { + if (!config.options.database_type.empty() && !StringUtil::CIEquals(config.options.database_type, "duckdb")) { // if we are opening an extension database - load the extension if (!config.file_system) { throw InternalException("No file system!?"); @@ -476,6 +472,7 @@ void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path config.options.allocator_bulk_deallocation_flush_threshold); } config.db_cache_entry = std::move(new_config.db_cache_entry); + config.path_manager = std::move(new_config.path_manager); } DBConfig &DBConfig::GetConfig(ClientContext &context) { diff --git a/src/main/database_file_path_manager.cpp b/src/main/database_file_path_manager.cpp new file mode 100644 index 000000000000..2e107210a246 --- /dev/null +++ b/src/main/database_file_path_manager.cpp @@ -0,0 +1,88 @@ +#include "duckdb/main/database_file_path_manager.hpp" +#include "duckdb/common/exception/binder_exception.hpp" +#include "duckdb/parser/parsed_data/attach_info.hpp" +#include "duckdb/main/attached_database.hpp" + +namespace duckdb { + +DatabasePathInfo::DatabasePathInfo(DatabaseManager &manager, string name_p, AccessMode access_mode) + : name(std::move(name_p)), access_mode(access_mode) { + attached_databases.insert(manager); +} + +idx_t DatabaseFilePathManager::ApproxDatabaseCount() const { + lock_guard path_lock(db_paths_lock); + return db_paths.size(); +} + +InsertDatabasePathResult DatabaseFilePathManager::InsertDatabasePath(DatabaseManager &manager, const string &path, + const string &name, OnCreateConflict on_conflict, + AttachOptions &options) { + if (path.empty() || path == IN_MEMORY_PATH) { + return InsertDatabasePathResult::SUCCESS; + } + + lock_guard path_lock(db_paths_lock); + auto entry = db_paths.emplace(path, DatabasePathInfo(manager, name, options.access_mode)); + if (!entry.second) { + auto &existing = entry.first->second; + bool already_exists = false; + bool attached_in_this_system = false; + if (on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT && existing.name == name) { + already_exists = true; + attached_in_this_system = existing.attached_databases.find(manager) != existing.attached_databases.end(); + } + if (options.access_mode == AccessMode::READ_ONLY && existing.access_mode == AccessMode::READ_ONLY) { + if (attached_in_this_system) { + return InsertDatabasePathResult::ALREADY_EXISTS; + } + // all attaches are in read-only mode - there is no conflict, just increase the reference count + existing.attached_databases.insert(manager); + existing.reference_count++; + } else { + if (already_exists) { + if (attached_in_this_system) { + return InsertDatabasePathResult::ALREADY_EXISTS; + } + throw BinderException( + "Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is in " + "the process of being detached", + name, path); + } + throw BinderException( + "Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is already " + "attached by database \"%s\"", + name, path, existing.name); + } + } + options.stored_database_path = make_uniq(manager, *this, path, name); + return InsertDatabasePathResult::SUCCESS; +} + +void DatabaseFilePathManager::EraseDatabasePath(const string &path) { + if (path.empty() || path == IN_MEMORY_PATH) { + return; + } + lock_guard path_lock(db_paths_lock); + auto entry = db_paths.find(path); + if (entry != db_paths.end()) { + if (entry->second.reference_count <= 1) { + db_paths.erase(entry); + } else { + entry->second.reference_count--; + } + } +} + +void DatabaseFilePathManager::DetachDatabase(DatabaseManager &manager, const string &path) { + if (path.empty() || path == IN_MEMORY_PATH) { + return; + } + lock_guard path_lock(db_paths_lock); + auto entry = db_paths.find(path); + if (entry != db_paths.end()) { + entry->second.attached_databases.erase(manager); + } +} + +} // namespace duckdb diff --git a/src/main/database_manager.cpp b/src/main/database_manager.cpp index 008d5aaa4c21..1b6b070d43ca 100644 --- a/src/main/database_manager.cpp +++ b/src/main/database_manager.cpp @@ -10,12 +10,19 @@ #include "duckdb/storage/storage_manager.hpp" #include "duckdb/transaction/duck_transaction.hpp" #include "duckdb/transaction/duck_transaction_manager.hpp" +#include "duckdb/parser/parsed_data/alter_database_info.hpp" namespace duckdb { DatabaseManager::DatabaseManager(DatabaseInstance &db) : next_oid(0), current_query_number(1), current_transaction_id(0) { system = make_shared_ptr(db); + auto &config = DBConfig::GetConfig(db); + path_manager = config.path_manager; + if (!path_manager) { + // no shared path manager + path_manager = make_shared_ptr(); + } } DatabaseManager::~DatabaseManager() { @@ -39,23 +46,84 @@ void DatabaseManager::FinalizeStartup() { optional_ptr DatabaseManager::GetDatabase(ClientContext &context, const string &name) { auto &meta_transaction = MetaTransaction::Get(context); + // first check if we have a local reference to this database already + auto database = meta_transaction.GetReferencedDatabase(name); + if (database) { + // we do! return it + return database; + } lock_guard guard(databases_lock); + shared_ptr db; if (StringUtil::Lower(name) == TEMP_CATALOG) { - return meta_transaction.UseDatabase(context.client_data->temporary_objects); + db = context.client_data->temporary_objects; + } else { + db = GetDatabaseInternal(guard, name); + } + if (!db) { + return nullptr; } + return meta_transaction.UseDatabase(db); +} + +shared_ptr DatabaseManager::GetDatabase(const string &name) { + lock_guard guard(databases_lock); + return GetDatabaseInternal(guard, name); +} + +shared_ptr DatabaseManager::GetDatabaseInternal(const lock_guard &, const string &name) { if (StringUtil::Lower(name) == SYSTEM_CATALOG) { - return meta_transaction.UseDatabase(system); + return system; } auto entry = databases.find(name); if (entry == databases.end()) { // not found return nullptr; } - return meta_transaction.UseDatabase(entry->second); + return entry->second; } shared_ptr DatabaseManager::AttachDatabase(ClientContext &context, AttachInfo &info, AttachOptions &options) { + if (options.db_type.empty() || StringUtil::CIEquals(options.db_type, "duckdb")) { + // Start timing the ATTACH-delay step. + auto profiler = context.client_data->profiler; + profiler->StartTimer(MetricsType::WAITING_TO_ATTACH_LATENCY); + + while (InsertDatabasePath(info, options) == InsertDatabasePathResult::ALREADY_EXISTS) { + // database with this name and path already exists + // first check if it exists within this transaction + auto &meta_transaction = MetaTransaction::Get(context); + auto existing_db = meta_transaction.GetReferencedDatabaseOwning(info.name); + if (existing_db) { + profiler->EndTimer(MetricsType::WAITING_TO_ATTACH_LATENCY); + // it does! return it + return existing_db; + } + + // ... but it might not be done attaching yet! + // verify the database has actually finished attaching prior to returning + lock_guard guard(databases_lock); + auto entry = databases.find(info.name); + if (entry != databases.end()) { + // The database ACTUALLY exists, so we return it. + profiler->EndTimer(MetricsType::WAITING_TO_ATTACH_LATENCY); + return entry->second; + } + if (context.interrupted) { + profiler->EndTimer(MetricsType::WAITING_TO_ATTACH_LATENCY); + throw InterruptException(); + } + } + profiler->EndTimer(MetricsType::WAITING_TO_ATTACH_LATENCY); + } + + auto &config = DBConfig::GetConfig(context); + GetDatabaseType(context, info, config, options); + if (!options.db_type.empty()) { + // we only need to prevent duplicate opening of DuckDB files + // if this is not a DuckDB file but e.g. a CSV or Parquet file, we don't need to do this duplicate protection + options.stored_database_path.reset(); + } if (AttachedDatabase::NameIsReserved(info.name)) { throw BinderException("Attached database name \"%s\" cannot be used because it is a reserved name", info.name); } @@ -76,6 +144,20 @@ shared_ptr DatabaseManager::AttachDatabase(ClientContext &cont // now create the attached database auto &db = DatabaseInstance::GetDatabase(context); auto attached_db = db.CreateAttachedDatabase(context, info, options); + + //! Initialize the database. + if (options.is_main_database) { + attached_db->SetInitialDatabase(); + attached_db->Initialize(context); + } else { + attached_db->Initialize(context); + if (!options.default_table.name.empty()) { + attached_db->GetCatalog().SetDefaultTable(options.default_table.schema, options.default_table.name); + } + attached_db->FinalizeLoad(context); + } + + FinalizeAttach(context, info, attached_db); return attached_db; } @@ -86,17 +168,26 @@ optional_ptr DatabaseManager::FinalizeAttach(ClientContext &co if (default_database.empty()) { default_database = name; } - if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { - DetachDatabase(context, name, OnEntryNotFound::RETURN_NULL); - } + shared_ptr detached_db; { lock_guard guard(databases_lock); auto entry = databases.emplace(name, attached_db); if (!entry.second) { - throw BinderException("Failed to attach database: database with name \"%s\" already exists", name); + if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { + // override existing entry + detached_db = std::move(entry.first->second); + databases[name] = attached_db; + } else { + throw BinderException("Failed to attach database: database with name \"%s\" already exists", name); + } } } auto &meta_transaction = MetaTransaction::Get(context); + if (detached_db) { + meta_transaction.DetachDatabase(*detached_db); + detached_db->OnDetach(context); + detached_db.reset(); + } auto &db_ref = meta_transaction.UseDatabase(attached_db); auto &transaction = DuckTransaction::Get(context, *system); auto &transaction_manager = DuckTransactionManager::Get(*system); @@ -122,6 +213,54 @@ void DatabaseManager::DetachDatabase(ClientContext &context, const string &name, attached_db->OnDetach(context); } +void DatabaseManager::Alter(ClientContext &context, AlterInfo &info) { + auto &db_info = info.Cast(); + + switch (db_info.alter_database_type) { + case AlterDatabaseType::RENAME_DATABASE: { + auto &rename_info = db_info.Cast(); + RenameDatabase(context, db_info.catalog, rename_info.new_name, db_info.if_not_found); + break; + } + default: + throw InternalException("Unsupported ALTER DATABASE operation"); + } +} + +void DatabaseManager::RenameDatabase(ClientContext &context, const string &old_name, const string &new_name, + OnEntryNotFound if_not_found) { + if (AttachedDatabase::NameIsReserved(new_name)) { + throw BinderException("Database name \"%s\" cannot be used because it is a reserved name", new_name); + } + + shared_ptr attached_db; + { + lock_guard guard(databases_lock); + auto old_entry = databases.find(old_name); + if (old_entry == databases.end()) { + if (if_not_found == OnEntryNotFound::THROW_EXCEPTION) { + throw BinderException("Failed to rename database \"%s\": database not found", old_name); + } + return; + } + + auto new_entry = databases.find(new_name); + if (new_entry != databases.end()) { + throw BinderException("Failed to rename database \"%s\" to \"%s\": database with new name already exists", + old_name, new_name); + } + + attached_db = old_entry->second; + databases.erase(old_entry); + attached_db->SetName(new_name); + databases[new_name] = attached_db; + } + + if (default_database == old_name) { + default_database = new_name; + } +} + shared_ptr DatabaseManager::DetachInternal(const string &name) { shared_ptr attached_db; { @@ -136,56 +275,37 @@ shared_ptr DatabaseManager::DetachInternal(const string &name) return attached_db; } -void DatabaseManager::CheckPathConflict(const string &path, const string &name) { - if (path.empty() || path == IN_MEMORY_PATH) { - return; - } - - lock_guard path_lock(db_paths_lock); - auto entry = db_paths_to_name.find(path); - if (entry != db_paths_to_name.end()) { - throw BinderException("Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is already " - "attached by database \"%s\"", - name, path, entry->second); - } +idx_t DatabaseManager::ApproxDatabaseCount() { + return path_manager->ApproxDatabaseCount(); } -void DatabaseManager::InsertDatabasePath(const string &path, const string &name) { - if (path.empty() || path == IN_MEMORY_PATH) { - return; - } - - lock_guard path_lock(db_paths_lock); - auto entry = db_paths_to_name.emplace(path, name); - if (!entry.second) { - throw BinderException("Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is already " - "attached by database \"%s\"", - name, path, entry.first->second); - } -} - -void DatabaseManager::EraseDatabasePath(const string &path) { - if (path.empty() || path == IN_MEMORY_PATH) { - return; - } - lock_guard path_lock(db_paths_lock); - db_paths_to_name.erase(path); +InsertDatabasePathResult DatabaseManager::InsertDatabasePath(const AttachInfo &info, AttachOptions &options) { + return path_manager->InsertDatabasePath(*this, info.path, info.name, info.on_conflict, options); } vector DatabaseManager::GetAttachedDatabasePaths() { - lock_guard path_lock(db_paths_lock); - vector paths; - for (auto &entry : db_paths_to_name) { - paths.push_back(entry.first); + vector result; + lock_guard guard(databases_lock); + for (auto &entry : databases) { + auto &db_ref = *entry.second; + auto &catalog = db_ref.GetCatalog(); + if (catalog.InMemory() || catalog.IsSystemCatalog()) { + continue; + } + auto path = catalog.GetDBPath(); + if (path.empty()) { + continue; + } + result.push_back(std::move(path)); } - return paths; + return result; } void DatabaseManager::GetDatabaseType(ClientContext &context, AttachInfo &info, const DBConfig &config, AttachOptions &options) { // Test if the database is a DuckDB database file. - if (StringUtil::CIEquals(options.db_type, "DUCKDB")) { + if (StringUtil::CIEquals(options.db_type, "duckdb")) { options.db_type = ""; return; } @@ -193,8 +313,7 @@ void DatabaseManager::GetDatabaseType(ClientContext &context, AttachInfo &info, // Try to extract the database type from the path. if (options.db_type.empty()) { auto &fs = FileSystem::GetFileSystem(context); - CheckPathConflict(info.path, info.name); - DBPathAndType::CheckMagicBytes(QueryContext(context), fs, info.path, options.db_type); + DBPathAndType::CheckMagicBytes(context, fs, info.path, options.db_type); } if (options.db_type.empty()) { diff --git a/src/main/db_instance_cache.cpp b/src/main/db_instance_cache.cpp index 53571b41f689..57f4ee457fae 100644 --- a/src/main/db_instance_cache.cpp +++ b/src/main/db_instance_cache.cpp @@ -1,5 +1,6 @@ #include "duckdb/main/db_instance_cache.hpp" #include "duckdb/main/extension_helper.hpp" +#include "duckdb/main/database_file_path_manager.hpp" namespace duckdb { @@ -31,6 +32,13 @@ string GetDBAbsolutePath(const string &database_p, FileSystem &fs) { return fs.NormalizeAbsolutePath(fs.JoinPath(FileSystem::GetWorkingDirectory(), database)); } +DBInstanceCache::DBInstanceCache() { + path_manager = make_shared_ptr(); +} + +DBInstanceCache::~DBInstanceCache() { +} + shared_ptr DBInstanceCache::GetInstanceInternal(const string &database, const DBConfig &config, std::unique_lock &db_instances_lock) { D_ASSERT(db_instances_lock.owns_lock()); @@ -100,6 +108,7 @@ shared_ptr DBInstanceCache::CreateInstanceInternal(const string &databas instance_path = IN_MEMORY_PATH; } shared_ptr db_instance; + config.path_manager = path_manager; if (cache_instance) { D_ASSERT(db_instances.find(abs_database_path) == db_instances.end()); shared_ptr cache_entry = make_shared_ptr(); diff --git a/src/main/extension.cpp b/src/main/extension.cpp index cd786d863ec7..c982a4bc3071 100644 --- a/src/main/extension.cpp +++ b/src/main/extension.cpp @@ -7,6 +7,8 @@ namespace duckdb { +constexpr const idx_t ParsedExtensionMetaData::FOOTER_SIZE; + Extension::~Extension() { } diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index 88eeeda37780..96e559ec0436 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -522,6 +522,17 @@ void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs if (!info) { return; } + try { + LoadExternalExtensionInternal(db, fs, extension, *info); + } catch (std::exception &ex) { + ErrorData error(ex); + info->LoadFail(error); + throw; + } +} + +void ExtensionHelper::LoadExternalExtensionInternal(DatabaseInstance &db, FileSystem &fs, const string &extension, + ExtensionActiveLoad &info) { #ifdef DUCKDB_DISABLE_EXTENSION_LOAD throw PermissionException("Loading external extensions is disabled through a compile time flag"); #else @@ -538,7 +549,7 @@ void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs } try { - ExtensionLoader loader(*info); + ExtensionLoader loader(info); (*init_fun)(loader); loader.FinalizeLoad(); } catch (std::exception &e) { @@ -549,7 +560,7 @@ void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs D_ASSERT(extension_init_result.install_info); - info->FinishLoad(*extension_init_result.install_info); + info.FinishLoad(*extension_init_result.install_info); return; } @@ -589,7 +600,7 @@ void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs D_ASSERT(extension_init_result.install_info); - info->FinishLoad(*extension_init_result.install_info); + info.FinishLoad(*extension_init_result.install_info); return; } diff --git a/src/main/extension_manager.cpp b/src/main/extension_manager.cpp index 033f32ea1874..fbb2b9cea324 100644 --- a/src/main/extension_manager.cpp +++ b/src/main/extension_manager.cpp @@ -24,6 +24,14 @@ void ExtensionActiveLoad::FinishLoad(ExtensionInstallInfo &install_info) { DUCKDB_LOG_INFO(db, extension_name); } +void ExtensionActiveLoad::LoadFail(const ErrorData &error) { + auto &callbacks = DBConfig::GetConfig(db).extension_callbacks; + for (auto &callback : callbacks) { + callback->OnExtensionLoadFail(db, extension_name, error); + } + DUCKDB_LOG_INFO(db, "Failed to load extension '%s': %s", extension_name, error.Message()); +} + ExtensionManager::ExtensionManager(DatabaseInstance &db) : db(db) { } @@ -96,6 +104,10 @@ unique_ptr ExtensionManager::BeginLoad(const string &name) if (info->is_loaded) { return nullptr; } + auto &callbacks = DBConfig::GetConfig(db).extension_callbacks; + for (auto &callback : callbacks) { + callback->OnBeginExtensionLoad(db, extension_name); + } // extension is not loaded yet and we are in charge of loading it - return return result; } diff --git a/src/main/http/http_util.cpp b/src/main/http/http_util.cpp index 2a470c45a764..554346489d59 100644 --- a/src/main/http/http_util.cpp +++ b/src/main/http/http_util.cpp @@ -367,7 +367,9 @@ HTTPUtil::RunRequestWithRetry(const std::function(void) try { response = on_request(); - response->url = request.url; + if (response) { + response->url = request.url; + } } catch (IOException &e) { exception_error = e.what(); caught_e = std::current_exception(); @@ -379,8 +381,12 @@ HTTPUtil::RunRequestWithRetry(const std::function(void) // Note: request errors will always be retried bool should_retry = !response || response->ShouldRetry(); if (!should_retry) { + auto response_code = static_cast(response->status); + if (response_code >= 200 && response_code < 300) { + response->success = true; + return response; + } switch (response->status) { - case HTTPStatusCode::OK_200: case HTTPStatusCode::NotModified_304: response->success = true; break; diff --git a/src/main/profiling_info.cpp b/src/main/profiling_info.cpp index 8f744d51b457..6e60b9d25b86 100644 --- a/src/main/profiling_info.cpp +++ b/src/main/profiling_info.cpp @@ -23,12 +23,12 @@ ProfilingInfo::ProfilingInfo(const profiler_settings_t &n_settings, const idx_t // Reduce. if (depth == 0) { - auto op_metrics = DefaultOperatorSettings(); + auto op_metrics = OperatorScopeSettings(); for (const auto metric : op_metrics) { settings.erase(metric); } } else { - auto root_metrics = DefaultRootSettings(); + auto root_metrics = RootScopeSettings(); for (const auto metric : root_metrics) { settings.erase(metric); } @@ -37,32 +37,44 @@ ProfilingInfo::ProfilingInfo(const profiler_settings_t &n_settings, const idx_t } profiler_settings_t ProfilingInfo::DefaultSettings() { - return {MetricsType::QUERY_NAME, + return {MetricsType::ATTACH_LOAD_STORAGE_LATENCY, + MetricsType::ATTACH_REPLAY_WAL_LATENCY, MetricsType::BLOCKED_THREAD_TIME, - MetricsType::SYSTEM_PEAK_BUFFER_MEMORY, - MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE, + MetricsType::CHECKPOINT_LATENCY, MetricsType::CPU_TIME, - MetricsType::EXTRA_INFO, MetricsType::CUMULATIVE_CARDINALITY, - MetricsType::OPERATOR_NAME, - MetricsType::OPERATOR_TYPE, - MetricsType::OPERATOR_CARDINALITY, MetricsType::CUMULATIVE_ROWS_SCANNED, + MetricsType::EXTRA_INFO, + MetricsType::LATENCY, + MetricsType::OPERATOR_CARDINALITY, + MetricsType::OPERATOR_NAME, MetricsType::OPERATOR_ROWS_SCANNED, MetricsType::OPERATOR_TIMING, + MetricsType::OPERATOR_TYPE, MetricsType::RESULT_SET_SIZE, - MetricsType::LATENCY, MetricsType::ROWS_RETURNED, + MetricsType::SYSTEM_PEAK_BUFFER_MEMORY, + MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE, MetricsType::TOTAL_BYTES_READ, - MetricsType::TOTAL_BYTES_WRITTEN}; + MetricsType::TOTAL_BYTES_WRITTEN, + MetricsType::WAITING_TO_ATTACH_LATENCY, + MetricsType::QUERY_NAME}; } -profiler_settings_t ProfilingInfo::DefaultRootSettings() { - return {MetricsType::QUERY_NAME, MetricsType::BLOCKED_THREAD_TIME, MetricsType::LATENCY, - MetricsType::ROWS_RETURNED}; +profiler_settings_t ProfilingInfo::RootScopeSettings() { + return {MetricsType::ATTACH_LOAD_STORAGE_LATENCY, + MetricsType::ATTACH_REPLAY_WAL_LATENCY, + MetricsType::BLOCKED_THREAD_TIME, + MetricsType::CHECKPOINT_LATENCY, + MetricsType::LATENCY, + MetricsType::ROWS_RETURNED, + MetricsType::TOTAL_BYTES_READ, + MetricsType::TOTAL_BYTES_WRITTEN, + MetricsType::WAITING_TO_ATTACH_LATENCY, + MetricsType::QUERY_NAME}; } -profiler_settings_t ProfilingInfo::DefaultOperatorSettings() { +profiler_settings_t ProfilingInfo::OperatorScopeSettings() { return {MetricsType::OPERATOR_CARDINALITY, MetricsType::OPERATOR_ROWS_SCANNED, MetricsType::OPERATOR_TIMING, MetricsType::OPERATOR_NAME, MetricsType::OPERATOR_TYPE}; } @@ -83,6 +95,10 @@ void ProfilingInfo::ResetMetrics() { case MetricsType::BLOCKED_THREAD_TIME: case MetricsType::CPU_TIME: case MetricsType::OPERATOR_TIMING: + case MetricsType::WAITING_TO_ATTACH_LATENCY: + case MetricsType::ATTACH_LOAD_STORAGE_LATENCY: + case MetricsType::ATTACH_REPLAY_WAL_LATENCY: + case MetricsType::CHECKPOINT_LATENCY: metrics[metric] = Value::CreateValue(0.0); break; case MetricsType::OPERATOR_NAME: @@ -104,6 +120,7 @@ void ProfilingInfo::ResetMetrics() { metrics[metric] = Value::CreateValue(0); break; case MetricsType::EXTRA_INFO: + metrics[metric] = Value::MAP(InsertionOrderPreservingMap()); break; default: throw InternalException("MetricsType" + EnumUtil::ToString(metric) + "not implemented"); @@ -149,21 +166,10 @@ string ProfilingInfo::GetMetricAsString(const MetricsType metric) const { throw InternalException("Metric %s not enabled", EnumUtil::ToString(metric)); } - if (metric == MetricsType::EXTRA_INFO) { - string result; - for (auto &it : extra_info) { - if (!result.empty()) { - result += ", "; - } - result += StringUtil::Format("%s: %s", it.first, it.second); - } - return "\"" + result + "\""; - } - // The metric cannot be NULL and must be initialized. D_ASSERT(!metrics.at(metric).IsNull()); if (metric == MetricsType::OPERATOR_TYPE) { - auto type = PhysicalOperatorType(metrics.at(metric).GetValue()); + const auto type = PhysicalOperatorType(metrics.at(metric).GetValue()); return EnumUtil::ToString(type); } return metrics.at(metric).ToString(); @@ -178,18 +184,25 @@ void ProfilingInfo::WriteMetricsToJSON(yyjson_mut_doc *doc, yyjson_mut_val *dest if (metric == MetricsType::EXTRA_INFO) { auto extra_info_obj = yyjson_mut_obj(doc); - for (auto &it : extra_info) { - auto &key = it.first; - auto &value = it.second; - auto splits = StringUtil::Split(value, "\n"); + auto extra_info = metrics.at(metric); + auto children = MapValue::GetChildren(extra_info); + for (auto &child : children) { + auto struct_children = StructValue::GetChildren(child); + auto key = struct_children[0].GetValue(); + auto value = struct_children[1].GetValue(); + + auto key_mut = unsafe_yyjson_mut_strncpy(doc, key.c_str(), key.size()); + auto value_mut = unsafe_yyjson_mut_strncpy(doc, value.c_str(), value.size()); + + auto splits = StringUtil::Split(value_mut, "\n"); if (splits.size() > 1) { auto list_items = yyjson_mut_arr(doc); for (auto &split : splits) { yyjson_mut_arr_add_strcpy(doc, list_items, split.c_str()); } - yyjson_mut_obj_add_val(doc, extra_info_obj, key.c_str(), list_items); + yyjson_mut_obj_add_val(doc, extra_info_obj, key_mut, list_items); } else { - yyjson_mut_obj_add_strcpy(doc, extra_info_obj, key.c_str(), value.c_str()); + yyjson_mut_obj_add_strcpy(doc, extra_info_obj, key_mut, value_mut); } } yyjson_mut_obj_add_val(doc, dest, key_ptr, extra_info_obj); @@ -212,7 +225,11 @@ void ProfilingInfo::WriteMetricsToJSON(yyjson_mut_doc *doc, yyjson_mut_val *dest case MetricsType::LATENCY: case MetricsType::BLOCKED_THREAD_TIME: case MetricsType::CPU_TIME: - case MetricsType::OPERATOR_TIMING: { + case MetricsType::OPERATOR_TIMING: + case MetricsType::WAITING_TO_ATTACH_LATENCY: + case MetricsType::ATTACH_LOAD_STORAGE_LATENCY: + case MetricsType::ATTACH_REPLAY_WAL_LATENCY: + case MetricsType::CHECKPOINT_LATENCY: { yyjson_mut_obj_add_real(doc, dest, key_ptr, metrics[metric].GetValue()); break; } diff --git a/src/main/query_profiler.cpp b/src/main/query_profiler.cpp index 5259e73825eb..b4f15a8e68d3 100644 --- a/src/main/query_profiler.cpp +++ b/src/main/query_profiler.cpp @@ -16,6 +16,7 @@ #include "duckdb/planner/expression/bound_function_expression.hpp" #include "duckdb/storage/buffer/buffer_pool.hpp" #include "yyjson.hpp" +#include "yyjson_utils.hpp" #include #include @@ -38,10 +39,12 @@ bool QueryProfiler::IsDetailedEnabled() const { ProfilerPrintFormat QueryProfiler::GetPrintFormat(ExplainFormat format) const { auto print_format = ClientConfig::GetConfig(context).profiler_print_format; - if (format == ExplainFormat::DEFAULT) { - return print_format; - } switch (format) { + case ExplainFormat::DEFAULT: + if (print_format != ProfilerPrintFormat::NO_OUTPUT) { + return print_format; + } + DUCKDB_EXPLICIT_FALLTHROUGH; case ExplainFormat::TEXT: return ProfilerPrintFormat::QUERY_TREE; case ExplainFormat::JSON: @@ -100,9 +103,7 @@ void QueryProfiler::Reset() { phase_timings.clear(); phase_stack.clear(); running = false; - query_metrics.query = ""; - query_metrics.total_bytes_read = 0; - query_metrics.total_bytes_written = 0; + query_metrics.Reset(); } void QueryProfiler::StartQuery(const string &query, bool is_explain_analyze_p, bool start_at_optimizer) { @@ -279,6 +280,21 @@ void QueryProfiler::EndQuery() { if (info.Enabled(settings, MetricsType::RESULT_SET_SIZE)) { info.metrics[MetricsType::RESULT_SET_SIZE] = child_info.metrics[MetricsType::RESULT_SET_SIZE]; } + if (info.Enabled(settings, MetricsType::WAITING_TO_ATTACH_LATENCY)) { + info.metrics[MetricsType::WAITING_TO_ATTACH_LATENCY] = + query_metrics.waiting_to_attach_latency.Elapsed(); + } + if (info.Enabled(settings, MetricsType::ATTACH_LOAD_STORAGE_LATENCY)) { + info.metrics[MetricsType::ATTACH_LOAD_STORAGE_LATENCY] = + query_metrics.attach_load_storage_latency.Elapsed(); + } + if (info.Enabled(settings, MetricsType::ATTACH_REPLAY_WAL_LATENCY)) { + info.metrics[MetricsType::ATTACH_REPLAY_WAL_LATENCY] = + query_metrics.attach_replay_wal_latency.Elapsed(); + } + if (info.Enabled(settings, MetricsType::CHECKPOINT_LATENCY)) { + info.metrics[MetricsType::CHECKPOINT_LATENCY] = query_metrics.checkpoint_latency.Elapsed(); + } MoveOptimizerPhasesToRoot(); if (info.Enabled(settings, MetricsType::CUMULATIVE_OPTIMIZER_TIMING)) { @@ -320,6 +336,52 @@ void QueryProfiler::AddBytesWritten(const idx_t nr_bytes) { } } +void QueryProfiler::StartTimer(MetricsType type) { + if (!IsEnabled()) { + return; + } + + switch (type) { + case MetricsType::WAITING_TO_ATTACH_LATENCY: + query_metrics.waiting_to_attach_latency.Start(); + return; + case MetricsType::ATTACH_LOAD_STORAGE_LATENCY: + query_metrics.attach_load_storage_latency.Start(); + return; + case MetricsType::ATTACH_REPLAY_WAL_LATENCY: + query_metrics.attach_replay_wal_latency.Start(); + return; + case MetricsType::CHECKPOINT_LATENCY: + query_metrics.checkpoint_latency.Start(); + return; + default: + return; + } +} + +void QueryProfiler::EndTimer(MetricsType type) { + if (!IsEnabled()) { + return; + } + + switch (type) { + case MetricsType::WAITING_TO_ATTACH_LATENCY: + query_metrics.waiting_to_attach_latency.End(); + return; + case MetricsType::ATTACH_LOAD_STORAGE_LATENCY: + query_metrics.attach_load_storage_latency.End(); + return; + case MetricsType::ATTACH_REPLAY_WAL_LATENCY: + query_metrics.attach_replay_wal_latency.End(); + return; + case MetricsType::CHECKPOINT_LATENCY: + query_metrics.checkpoint_latency.End(); + return; + default: + return; + } +} + string QueryProfiler::ToString(ExplainFormat explain_format) const { return ToString(GetPrintFormat(explain_format)); } @@ -402,7 +464,7 @@ OperatorProfiler::OperatorProfiler(ClientContext &context) : context(context) { } // Reduce. - auto root_metrics = ProfilingInfo::DefaultRootSettings(); + auto root_metrics = ProfilingInfo::RootScopeSettings(); for (const auto metric : root_metrics) { settings.erase(metric); } @@ -555,7 +617,7 @@ void QueryProfiler::Flush(OperatorProfiler &profiler) { info.MetricSum(MetricsType::RESULT_SET_SIZE, node.second.result_set_size); } if (ProfilingInfo::Enabled(profiler.settings, MetricsType::EXTRA_INFO)) { - info.extra_info = node.second.extra_info; + info.metrics[MetricsType::EXTRA_INFO] = Value::MAP(node.second.extra_info); } if (ProfilingInfo::Enabled(profiler.settings, MetricsType::SYSTEM_PEAK_BUFFER_MEMORY)) { query_metrics.query_global_info.MetricMax(MetricsType::SYSTEM_PEAK_BUFFER_MEMORY, @@ -719,18 +781,24 @@ void QueryProfiler::QueryTreeToStream(std::ostream &ss) const { } } -InsertionOrderPreservingMap QueryProfiler::JSONSanitize(const InsertionOrderPreservingMap &input) { +Value QueryProfiler::JSONSanitize(const Value &input) { + D_ASSERT(input.type().id() == LogicalTypeId::MAP); + InsertionOrderPreservingMap result; - for (auto &it : input) { - auto key = it.first; + auto children = MapValue::GetChildren(input); + for (auto &child : children) { + auto struct_children = StructValue::GetChildren(child); + auto key = struct_children[0].GetValue(); + auto value = struct_children[1].GetValue(); + if (StringUtil::StartsWith(key, "__")) { key = StringUtil::Replace(key, "__", ""); key = StringUtil::Replace(key, "_", " "); key = StringUtil::Title(key); } - result[key] = it.second; + result[key] = value; } - return result; + return Value::MAP(result); } string QueryProfiler::JSONSanitize(const std::string &text) { @@ -770,7 +838,12 @@ string QueryProfiler::JSONSanitize(const std::string &text) { static yyjson_mut_val *ToJSONRecursive(yyjson_mut_doc *doc, ProfilingNode &node) { auto result_obj = yyjson_mut_obj(doc); auto &profiling_info = node.GetProfilingInfo(); - profiling_info.extra_info = QueryProfiler::JSONSanitize(profiling_info.extra_info); + + if (profiling_info.Enabled(profiling_info.settings, MetricsType::EXTRA_INFO)) { + profiling_info.metrics[MetricsType::EXTRA_INFO] = + QueryProfiler::JSONSanitize(profiling_info.metrics.at(MetricsType::EXTRA_INFO)); + } + profiling_info.WriteMetricsToJSON(doc, result_obj); auto children_list = yyjson_mut_arr(doc); @@ -782,44 +855,43 @@ static yyjson_mut_val *ToJSONRecursive(yyjson_mut_doc *doc, ProfilingNode &node) return result_obj; } -static string StringifyAndFree(yyjson_mut_doc *doc, yyjson_mut_val *object) { - auto data = yyjson_mut_val_write_opts(object, YYJSON_WRITE_ALLOW_INF_AND_NAN | YYJSON_WRITE_PRETTY, nullptr, - nullptr, nullptr); - if (!data) { - yyjson_mut_doc_free(doc); +static string StringifyAndFree(ConvertedJSONHolder &json_holder, yyjson_mut_val *object) { + json_holder.stringified_json = yyjson_mut_val_write_opts( + object, YYJSON_WRITE_ALLOW_INF_AND_NAN | YYJSON_WRITE_PRETTY, nullptr, nullptr, nullptr); + if (!json_holder.stringified_json) { throw InternalException("The plan could not be rendered as JSON, yyjson failed"); } - auto result = string(data); - free(data); - yyjson_mut_doc_free(doc); + auto result = string(json_holder.stringified_json); return result; } string QueryProfiler::ToJSON() const { lock_guard guard(lock); - auto doc = yyjson_mut_doc_new(nullptr); - auto result_obj = yyjson_mut_obj(doc); - yyjson_mut_doc_set_root(doc, result_obj); + ConvertedJSONHolder json_holder; + + json_holder.doc = yyjson_mut_doc_new(nullptr); + auto result_obj = yyjson_mut_obj(json_holder.doc); + yyjson_mut_doc_set_root(json_holder.doc, result_obj); if (query_metrics.query.empty() && !root) { - yyjson_mut_obj_add_str(doc, result_obj, "result", "empty"); - return StringifyAndFree(doc, result_obj); + yyjson_mut_obj_add_str(json_holder.doc, result_obj, "result", "empty"); + return StringifyAndFree(json_holder, result_obj); } if (!root) { - yyjson_mut_obj_add_str(doc, result_obj, "result", "error"); - return StringifyAndFree(doc, result_obj); + yyjson_mut_obj_add_str(json_holder.doc, result_obj, "result", "error"); + return StringifyAndFree(json_holder, result_obj); } auto &settings = root->GetProfilingInfo(); - settings.WriteMetricsToJSON(doc, result_obj); + settings.WriteMetricsToJSON(json_holder.doc, result_obj); // recursively print the physical operator tree - auto children_list = yyjson_mut_arr(doc); - yyjson_mut_obj_add_val(doc, result_obj, "children", children_list); - auto child = ToJSONRecursive(doc, *root->GetChild(0)); + auto children_list = yyjson_mut_arr(json_holder.doc); + yyjson_mut_obj_add_val(json_holder.doc, result_obj, "children", children_list); + auto child = ToJSONRecursive(json_holder.doc, *root->GetChild(0)); yyjson_mut_arr_add_val(children_list, child); - return StringifyAndFree(doc, result_obj); + return StringifyAndFree(json_holder, result_obj); } void QueryProfiler::WriteToFile(const char *path, string &info) const { @@ -869,7 +941,7 @@ unique_ptr QueryProfiler::CreateTree(const PhysicalOperator &root info.MetricSum(MetricsType::OPERATOR_TYPE, static_cast(root_p.type)); } if (info.Enabled(info.settings, MetricsType::EXTRA_INFO)) { - info.extra_info = root_p.ParamsToString(); + info.metrics[MetricsType::EXTRA_INFO] = Value::MAP(root_p.ParamsToString()); } tree_map.insert(make_pair(reference(root_p), reference(*node))); @@ -903,12 +975,13 @@ string QueryProfiler::RenderDisabledMessage(ProfilerPrintFormat format) const { } )"; case ProfilerPrintFormat::JSON: { - auto doc = yyjson_mut_doc_new(nullptr); - auto result_obj = yyjson_mut_obj(doc); - yyjson_mut_doc_set_root(doc, result_obj); + ConvertedJSONHolder json_holder; + json_holder.doc = yyjson_mut_doc_new(nullptr); + auto result_obj = yyjson_mut_obj(json_holder.doc); + yyjson_mut_doc_set_root(json_holder.doc, result_obj); - yyjson_mut_obj_add_str(doc, result_obj, "result", "disabled"); - return StringifyAndFree(doc, result_obj); + yyjson_mut_obj_add_str(json_holder.doc, result_obj, "result", "disabled"); + return StringifyAndFree(json_holder, result_obj); } default: throw InternalException("Unknown ProfilerPrintFormat \"%s\"", EnumUtil::ToString(format)); @@ -960,7 +1033,4 @@ void QueryProfiler::MoveOptimizerPhasesToRoot() { } } -void QueryProfiler::Propagate(QueryProfiler &) { -} - } // namespace duckdb diff --git a/src/main/relation.cpp b/src/main/relation.cpp index 9a28349e7059..b9e4d50ff49e 100644 --- a/src/main/relation.cpp +++ b/src/main/relation.cpp @@ -394,8 +394,8 @@ string Relation::ToString() { } // LCOV_EXCL_START -unique_ptr Relation::GetQueryNode() { - throw InternalException("Cannot create a query node from this node type"); +string Relation::GetQuery() { + return GetQueryNode()->ToString(); } void Relation::Head(idx_t limit) { diff --git a/src/main/relation/create_table_relation.cpp b/src/main/relation/create_table_relation.cpp index 2492f244b7bc..39aa65e3694a 100644 --- a/src/main/relation/create_table_relation.cpp +++ b/src/main/relation/create_table_relation.cpp @@ -29,6 +29,14 @@ BoundStatement CreateTableRelation::Bind(Binder &binder) { return binder.Bind(stmt.Cast()); } +unique_ptr CreateTableRelation::GetQueryNode() { + throw InternalException("Cannot create a query node from a create table relation"); +} + +string CreateTableRelation::GetQuery() { + return string(); +} + const vector &CreateTableRelation::Columns() { return columns; } diff --git a/src/main/relation/create_view_relation.cpp b/src/main/relation/create_view_relation.cpp index c00deef381c5..6f77f013f8d7 100644 --- a/src/main/relation/create_view_relation.cpp +++ b/src/main/relation/create_view_relation.cpp @@ -35,6 +35,14 @@ BoundStatement CreateViewRelation::Bind(Binder &binder) { return binder.Bind(stmt.Cast()); } +unique_ptr CreateViewRelation::GetQueryNode() { + throw InternalException("Cannot create a query node from an update relation"); +} + +string CreateViewRelation::GetQuery() { + return string(); +} + const vector &CreateViewRelation::Columns() { return columns; } diff --git a/src/main/relation/delete_relation.cpp b/src/main/relation/delete_relation.cpp index 64b3f231e820..2ec60f66458a 100644 --- a/src/main/relation/delete_relation.cpp +++ b/src/main/relation/delete_relation.cpp @@ -26,6 +26,14 @@ BoundStatement DeleteRelation::Bind(Binder &binder) { return binder.Bind(stmt.Cast()); } +unique_ptr DeleteRelation::GetQueryNode() { + throw InternalException("Cannot create a query node from a delete relation"); +} + +string DeleteRelation::GetQuery() { + return string(); +} + const vector &DeleteRelation::Columns() { return columns; } diff --git a/src/main/relation/explain_relation.cpp b/src/main/relation/explain_relation.cpp index f91e1d29f023..9f2976c9d081 100644 --- a/src/main/relation/explain_relation.cpp +++ b/src/main/relation/explain_relation.cpp @@ -20,6 +20,14 @@ BoundStatement ExplainRelation::Bind(Binder &binder) { return binder.Bind(explain.Cast()); } +unique_ptr ExplainRelation::GetQueryNode() { + throw InternalException("Cannot create a query node from an explain relation"); +} + +string ExplainRelation::GetQuery() { + return string(); +} + const vector &ExplainRelation::Columns() { return columns; } diff --git a/src/main/relation/insert_relation.cpp b/src/main/relation/insert_relation.cpp index 9728570a0598..84ef16ec6e47 100644 --- a/src/main/relation/insert_relation.cpp +++ b/src/main/relation/insert_relation.cpp @@ -24,6 +24,14 @@ BoundStatement InsertRelation::Bind(Binder &binder) { return binder.Bind(stmt.Cast()); } +unique_ptr InsertRelation::GetQueryNode() { + throw InternalException("Cannot create a query node from an insert relation"); +} + +string InsertRelation::GetQuery() { + return string(); +} + const vector &InsertRelation::Columns() { return columns; } diff --git a/src/main/relation/query_relation.cpp b/src/main/relation/query_relation.cpp index e0cf2e280e93..6ebebbc70f60 100644 --- a/src/main/relation/query_relation.cpp +++ b/src/main/relation/query_relation.cpp @@ -49,6 +49,10 @@ unique_ptr QueryRelation::GetQueryNode() { return std::move(select->node); } +string QueryRelation::GetQuery() { + return query; +} + unique_ptr QueryRelation::GetTableRef() { auto subquery_ref = make_uniq(GetSelectStatement(), GetAlias()); return std::move(subquery_ref); @@ -63,7 +67,6 @@ BoundStatement QueryRelation::Bind(Binder &binder) { if (first_bind) { auto &query_node = *select_stmt->node; auto &cte_map = query_node.cte_map; - vector> materialized_ctes; for (auto &kv : replacements) { auto &name = kv.first; auto &tableref = kv.second; @@ -84,28 +87,7 @@ BoundStatement QueryRelation::Bind(Binder &binder) { cte_info->query = std::move(select); cte_map.map[name] = std::move(cte_info); - - // We can not rely on CTE inlining anymore, so we need to add a materialized CTE node - // to the query node to ensure that the CTE exists - auto &cte_entry = cte_map.map[name]; - auto mat_cte = make_uniq(); - mat_cte->ctename = name; - mat_cte->query = cte_entry->query->node->Copy(); - mat_cte->aliases = cte_entry->aliases; - mat_cte->materialized = cte_entry->materialized; - materialized_ctes.push_back(std::move(mat_cte)); - } - - auto root = std::move(select_stmt->node); - while (!materialized_ctes.empty()) { - unique_ptr node_result; - node_result = std::move(materialized_ctes.back()); - node_result->cte_map = root->cte_map.Copy(); - node_result->child = std::move(root); - root = std::move(node_result); - materialized_ctes.pop_back(); } - select_stmt->node = std::move(root); } replacements.clear(); binder.SetBindingMode(saved_binding_mode); diff --git a/src/main/relation/setop_relation.cpp b/src/main/relation/setop_relation.cpp index a0253df37800..9a24fdd16b1e 100644 --- a/src/main/relation/setop_relation.cpp +++ b/src/main/relation/setop_relation.cpp @@ -20,8 +20,8 @@ unique_ptr SetOpRelation::GetQueryNode() { if (!setop_all) { result->modifiers.push_back(make_uniq()); } - result->left = left->GetQueryNode(); - result->right = right->GetQueryNode(); + result->children.push_back(left->GetQueryNode()); + result->children.push_back(right->GetQueryNode()); result->setop_type = setop_type; result->setop_all = setop_all; return std::move(result); diff --git a/src/main/relation/update_relation.cpp b/src/main/relation/update_relation.cpp index 9176cf2f20b2..81d85ca89f0b 100644 --- a/src/main/relation/update_relation.cpp +++ b/src/main/relation/update_relation.cpp @@ -35,6 +35,14 @@ BoundStatement UpdateRelation::Bind(Binder &binder) { return binder.Bind(stmt.Cast()); } +unique_ptr UpdateRelation::GetQueryNode() { + throw InternalException("Cannot create a query node from an update relation"); +} + +string UpdateRelation::GetQuery() { + return string(); +} + const vector &UpdateRelation::Columns() { return columns; } diff --git a/src/main/relation/write_csv_relation.cpp b/src/main/relation/write_csv_relation.cpp index 4795c7a513ea..f77d6f1eeab0 100644 --- a/src/main/relation/write_csv_relation.cpp +++ b/src/main/relation/write_csv_relation.cpp @@ -25,6 +25,14 @@ BoundStatement WriteCSVRelation::Bind(Binder &binder) { return binder.Bind(copy.Cast()); } +unique_ptr WriteCSVRelation::GetQueryNode() { + throw InternalException("Cannot create a query node from a write CSV relation"); +} + +string WriteCSVRelation::GetQuery() { + return string(); +} + const vector &WriteCSVRelation::Columns() { return columns; } diff --git a/src/main/relation/write_parquet_relation.cpp b/src/main/relation/write_parquet_relation.cpp index d6e403618dc9..b1dfdb29f911 100644 --- a/src/main/relation/write_parquet_relation.cpp +++ b/src/main/relation/write_parquet_relation.cpp @@ -25,6 +25,14 @@ BoundStatement WriteParquetRelation::Bind(Binder &binder) { return binder.Bind(copy.Cast()); } +unique_ptr WriteParquetRelation::GetQueryNode() { + throw InternalException("Cannot create a query node from a write parquet relation"); +} + +string WriteParquetRelation::GetQuery() { + return string(); +} + const vector &WriteParquetRelation::Columns() { return columns; } diff --git a/src/main/settings/autogenerated_settings.cpp b/src/main/settings/autogenerated_settings.cpp index f6ebaeed5887..989dd6e7ad1f 100644 --- a/src/main/settings/autogenerated_settings.cpp +++ b/src/main/settings/autogenerated_settings.cpp @@ -26,7 +26,7 @@ void AccessModeSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const } void AccessModeSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.access_mode = DBConfig().options.access_mode; + config.options.access_mode = DBConfigOptions().access_mode; } Value AccessModeSetting::GetSetting(const ClientContext &context) { @@ -48,7 +48,7 @@ void AllocatorBackgroundThreadsSetting::ResetGlobal(DatabaseInstance *db, DBConf if (!OnGlobalReset(db, config)) { return; } - config.options.allocator_background_threads = DBConfig().options.allocator_background_threads; + config.options.allocator_background_threads = DBConfigOptions().allocator_background_threads; } Value AllocatorBackgroundThreadsSetting::GetSetting(const ClientContext &context) { @@ -70,7 +70,7 @@ void AllowCommunityExtensionsSetting::ResetGlobal(DatabaseInstance *db, DBConfig if (!OnGlobalReset(db, config)) { return; } - config.options.allow_community_extensions = DBConfig().options.allow_community_extensions; + config.options.allow_community_extensions = DBConfigOptions().allow_community_extensions; } Value AllowCommunityExtensionsSetting::GetSetting(const ClientContext &context) { @@ -78,6 +78,28 @@ Value AllowCommunityExtensionsSetting::GetSetting(const ClientContext &context) return Value::BOOLEAN(config.options.allow_community_extensions); } +//===----------------------------------------------------------------------===// +// Allow Parser Override Extension +//===----------------------------------------------------------------------===// +void AllowParserOverrideExtensionSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { + if (!OnGlobalSet(db, config, input)) { + return; + } + config.options.allow_parser_override_extension = input.GetValue(); +} + +void AllowParserOverrideExtensionSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { + if (!OnGlobalReset(db, config)) { + return; + } + config.options.allow_parser_override_extension = DBConfigOptions().allow_parser_override_extension; +} + +Value AllowParserOverrideExtensionSetting::GetSetting(const ClientContext &context) { + auto &config = DBConfig::GetConfig(context); + return Value(config.options.allow_parser_override_extension); +} + //===----------------------------------------------------------------------===// // Allow Unredacted Secrets //===----------------------------------------------------------------------===// @@ -92,7 +114,7 @@ void AllowUnredactedSecretsSetting::ResetGlobal(DatabaseInstance *db, DBConfig & if (!OnGlobalReset(db, config)) { return; } - config.options.allow_unredacted_secrets = DBConfig().options.allow_unredacted_secrets; + config.options.allow_unredacted_secrets = DBConfigOptions().allow_unredacted_secrets; } Value AllowUnredactedSecretsSetting::GetSetting(const ClientContext &context) { @@ -114,7 +136,7 @@ void AllowUnsignedExtensionsSetting::ResetGlobal(DatabaseInstance *db, DBConfig if (!OnGlobalReset(db, config)) { return; } - config.options.allow_unsigned_extensions = DBConfig().options.allow_unsigned_extensions; + config.options.allow_unsigned_extensions = DBConfigOptions().allow_unsigned_extensions; } Value AllowUnsignedExtensionsSetting::GetSetting(const ClientContext &context) { @@ -137,7 +159,7 @@ void AutoinstallExtensionRepositorySetting::SetGlobal(DatabaseInstance *db, DBCo } void AutoinstallExtensionRepositorySetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.autoinstall_extension_repo = DBConfig().options.autoinstall_extension_repo; + config.options.autoinstall_extension_repo = DBConfigOptions().autoinstall_extension_repo; } Value AutoinstallExtensionRepositorySetting::GetSetting(const ClientContext &context) { @@ -153,7 +175,7 @@ void AutoinstallKnownExtensionsSetting::SetGlobal(DatabaseInstance *db, DBConfig } void AutoinstallKnownExtensionsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.autoinstall_known_extensions = DBConfig().options.autoinstall_known_extensions; + config.options.autoinstall_known_extensions = DBConfigOptions().autoinstall_known_extensions; } Value AutoinstallKnownExtensionsSetting::GetSetting(const ClientContext &context) { @@ -169,7 +191,7 @@ void AutoloadKnownExtensionsSetting::SetGlobal(DatabaseInstance *db, DBConfig &c } void AutoloadKnownExtensionsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.autoload_known_extensions = DBConfig().options.autoload_known_extensions; + config.options.autoload_known_extensions = DBConfigOptions().autoload_known_extensions; } Value AutoloadKnownExtensionsSetting::GetSetting(const ClientContext &context) { @@ -181,7 +203,7 @@ Value AutoloadKnownExtensionsSetting::GetSetting(const ClientContext &context) { // Checkpoint Threshold //===----------------------------------------------------------------------===// void CheckpointThresholdSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.checkpoint_wal_size = DBConfig().options.checkpoint_wal_size; + config.options.checkpoint_wal_size = DBConfigOptions().checkpoint_wal_size; } //===----------------------------------------------------------------------===// @@ -192,7 +214,7 @@ void CustomExtensionRepositorySetting::SetGlobal(DatabaseInstance *db, DBConfig } void CustomExtensionRepositorySetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.custom_extension_repo = DBConfig().options.custom_extension_repo; + config.options.custom_extension_repo = DBConfigOptions().custom_extension_repo; } Value CustomExtensionRepositorySetting::GetSetting(const ClientContext &context) { @@ -260,7 +282,7 @@ void DisableDatabaseInvalidationSetting::ResetGlobal(DatabaseInstance *db, DBCon if (!OnGlobalReset(db, config)) { return; } - config.options.disable_database_invalidation = DBConfig().options.disable_database_invalidation; + config.options.disable_database_invalidation = DBConfigOptions().disable_database_invalidation; } Value DisableDatabaseInvalidationSetting::GetSetting(const ClientContext &context) { @@ -282,7 +304,7 @@ void EnableExternalAccessSetting::ResetGlobal(DatabaseInstance *db, DBConfig &co if (!OnGlobalReset(db, config)) { return; } - config.options.enable_external_access = DBConfig().options.enable_external_access; + config.options.enable_external_access = DBConfigOptions().enable_external_access; } Value EnableExternalAccessSetting::GetSetting(const ClientContext &context) { @@ -298,7 +320,7 @@ void EnableHTTPMetadataCacheSetting::SetGlobal(DatabaseInstance *db, DBConfig &c } void EnableHTTPMetadataCacheSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.http_metadata_cache_enable = DBConfig().options.http_metadata_cache_enable; + config.options.http_metadata_cache_enable = DBConfigOptions().http_metadata_cache_enable; } Value EnableHTTPMetadataCacheSetting::GetSetting(const ClientContext &context) { @@ -372,7 +394,7 @@ void ExtensionDirectorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config } void ExtensionDirectorySetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.extension_directory = DBConfig().options.extension_directory; + config.options.extension_directory = DBConfigOptions().extension_directory; } Value ExtensionDirectorySetting::GetSetting(const ClientContext &context) { @@ -394,7 +416,7 @@ void ExternalThreadsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) if (!OnGlobalReset(db, config)) { return; } - config.options.external_threads = DBConfig().options.external_threads; + config.options.external_threads = DBConfigOptions().external_threads; } Value ExternalThreadsSetting::GetSetting(const ClientContext &context) { @@ -422,7 +444,7 @@ void HTTPProxySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const V } void HTTPProxySetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.http_proxy = DBConfig().options.http_proxy; + config.options.http_proxy = DBConfigOptions().http_proxy; } Value HTTPProxySetting::GetSetting(const ClientContext &context) { @@ -438,7 +460,7 @@ void HTTPProxyPasswordSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, } void HTTPProxyPasswordSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.http_proxy_password = DBConfig().options.http_proxy_password; + config.options.http_proxy_password = DBConfigOptions().http_proxy_password; } Value HTTPProxyPasswordSetting::GetSetting(const ClientContext &context) { @@ -454,7 +476,7 @@ void HTTPProxyUsernameSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, } void HTTPProxyUsernameSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.http_proxy_username = DBConfig().options.http_proxy_username; + config.options.http_proxy_username = DBConfigOptions().http_proxy_username; } Value HTTPProxyUsernameSetting::GetSetting(const ClientContext &context) { @@ -470,7 +492,7 @@ void LockConfigurationSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, } void LockConfigurationSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.lock_configuration = DBConfig().options.lock_configuration; + config.options.lock_configuration = DBConfigOptions().lock_configuration; } Value LockConfigurationSetting::GetSetting(const ClientContext &context) { @@ -504,7 +526,7 @@ void PinThreadsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const } void PinThreadsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.pin_threads = DBConfig().options.pin_threads; + config.options.pin_threads = DBConfigOptions().pin_threads; } Value PinThreadsSetting::GetSetting(const ClientContext &context) { @@ -520,7 +542,7 @@ void SchedulerProcessPartialSetting::SetGlobal(DatabaseInstance *db, DBConfig &c } void SchedulerProcessPartialSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.scheduler_process_partial = DBConfig().options.scheduler_process_partial; + config.options.scheduler_process_partial = DBConfigOptions().scheduler_process_partial; } Value SchedulerProcessPartialSetting::GetSetting(const ClientContext &context) { @@ -528,6 +550,13 @@ Value SchedulerProcessPartialSetting::GetSetting(const ClientContext &context) { return Value::BOOLEAN(config.options.scheduler_process_partial); } +//===----------------------------------------------------------------------===// +// Storage Block Prefetch +//===----------------------------------------------------------------------===// +void StorageBlockPrefetchSetting::OnSet(SettingCallbackInfo &info, Value ¶meter) { + EnumUtil::FromString(StringValue::Get(parameter)); +} + //===----------------------------------------------------------------------===// // Zstd Min String Length //===----------------------------------------------------------------------===// @@ -536,7 +565,7 @@ void ZstdMinStringLengthSetting::SetGlobal(DatabaseInstance *db, DBConfig &confi } void ZstdMinStringLengthSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.zstd_min_string_length = DBConfig().options.zstd_min_string_length; + config.options.zstd_min_string_length = DBConfigOptions().zstd_min_string_length; } Value ZstdMinStringLengthSetting::GetSetting(const ClientContext &context) { diff --git a/src/main/settings/custom_settings.cpp b/src/main/settings/custom_settings.cpp index a8bce753c5b0..0f83968b5f89 100644 --- a/src/main/settings/custom_settings.cpp +++ b/src/main/settings/custom_settings.cpp @@ -34,6 +34,14 @@ namespace duckdb { +constexpr const char *LoggingMode::Name; +constexpr const char *LoggingLevel::Name; +constexpr const char *EnableLogging::Name; +constexpr const char *LoggingStorage::Name; +constexpr const char *EnabledLogTypes::Name; +constexpr const char *DisabledLogTypes::Name; +constexpr const char *DisabledFilesystemsSetting::Name; + const string GetDefaultUserAgent() { return StringUtil::Format("duckdb/%s(%s)", DuckDB::LibraryVersion(), DuckDB::Platform()); } @@ -73,7 +81,7 @@ bool AllocatorBackgroundThreadsSetting::OnGlobalSet(DatabaseInstance *db, DBConf bool AllocatorBackgroundThreadsSetting::OnGlobalReset(DatabaseInstance *db, DBConfig &config) { if (db) { - TaskScheduler::GetScheduler(*db).SetAllocatorBackgroundThreads(DBConfig().options.allocator_background_threads); + TaskScheduler::GetScheduler(*db).SetAllocatorBackgroundThreads(DBConfigOptions().allocator_background_threads); } return true; } @@ -92,7 +100,7 @@ void AllocatorBulkDeallocationFlushThresholdSetting::SetGlobal(DatabaseInstance void AllocatorBulkDeallocationFlushThresholdSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { config.options.allocator_bulk_deallocation_flush_threshold = - DBConfig().options.allocator_bulk_deallocation_flush_threshold; + DBConfigOptions().allocator_bulk_deallocation_flush_threshold; if (db) { BufferManager::GetBufferManager(*db).GetBufferPool().SetAllocatorBulkDeallocationFlushThreshold( config.options.allocator_bulk_deallocation_flush_threshold); @@ -115,7 +123,7 @@ void AllocatorFlushThresholdSetting::SetGlobal(DatabaseInstance *db, DBConfig &c } void AllocatorFlushThresholdSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.allocator_flush_threshold = DBConfig().options.allocator_flush_threshold; + config.options.allocator_flush_threshold = DBConfigOptions().allocator_flush_threshold; if (db) { TaskScheduler::GetScheduler(*db).SetAllocatorFlushTreshold(config.options.allocator_flush_threshold); } @@ -142,7 +150,7 @@ bool AllowCommunityExtensionsSetting::OnGlobalSet(DatabaseInstance *db, DBConfig bool AllowCommunityExtensionsSetting::OnGlobalReset(DatabaseInstance *db, DBConfig &config) { if (db && !config.options.allow_community_extensions) { - if (DBConfig().options.allow_community_extensions) { + if (DBConfigOptions().allow_community_extensions) { throw InvalidInputException("Cannot upgrade allow_community_extensions setting while database is running"); } return false; @@ -150,6 +158,24 @@ bool AllowCommunityExtensionsSetting::OnGlobalReset(DatabaseInstance *db, DBConf return true; } +//===----------------------------------------------------------------------===// +// Allow Parser Override +//===----------------------------------------------------------------------===// +bool AllowParserOverrideExtensionSetting::OnGlobalSet(DatabaseInstance *db, DBConfig &config, const Value &input) { + auto new_value = input.GetValue(); + if (!StringUtil::CIEquals(new_value, "default") && !StringUtil::CIEquals(new_value, "fallback") && + !StringUtil::CIEquals(new_value, "strict")) { + throw InvalidInputException("Unrecognized value for parser override setting. Valid options are: \"default\", " + "\"fallback\", \"strict\"."); + } + return true; +} + +bool AllowParserOverrideExtensionSetting::OnGlobalReset(DatabaseInstance *db, DBConfig &config) { + config.options.allow_parser_override_extension = "default"; + return true; +} + //===----------------------------------------------------------------------===// // Allow Persistent Secrets //===----------------------------------------------------------------------===// @@ -239,7 +265,7 @@ void AllowedDirectoriesSetting::ResetGlobal(DatabaseInstance *db, DBConfig &conf if (!config.options.enable_external_access) { throw InvalidInputException("Cannot change allowed_directories when enable_external_access is disabled"); } - config.options.allowed_directories = DBConfig().options.allowed_directories; + config.options.allowed_directories = DBConfigOptions().allowed_directories; } Value AllowedDirectoriesSetting::GetSetting(const ClientContext &context) { @@ -273,7 +299,7 @@ void AllowedPathsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { if (!config.options.enable_external_access) { throw InvalidInputException("Cannot change allowed_paths when enable_external_access is disabled"); } - config.options.allowed_paths = DBConfig().options.allowed_paths; + config.options.allowed_paths = DBConfigOptions().allowed_paths; } Value AllowedPathsSetting::GetSetting(const ClientContext &context) { @@ -406,7 +432,7 @@ void CustomUserAgentSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) if (db) { throw InvalidInputException("Cannot change custom_user_agent setting while database is running"); } - config.options.custom_user_agent = DBConfig().options.custom_user_agent; + config.options.custom_user_agent = DBConfigOptions().custom_user_agent; } //===----------------------------------------------------------------------===// @@ -419,7 +445,7 @@ void DefaultBlockSizeSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, } void DefaultBlockSizeSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.default_block_alloc_size = DBConfig().options.default_block_alloc_size; + config.options.default_block_alloc_size = DBConfigOptions().default_block_alloc_size; } Value DefaultBlockSizeSetting::GetSetting(const ClientContext &context) { @@ -494,7 +520,7 @@ Value DefaultSecretStorageSetting::GetSetting(const ClientContext &context) { //===----------------------------------------------------------------------===// void DisabledCompressionMethodsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { auto list = StringUtil::Split(input.ToString(), ","); - set disabled_compression_methods; + vector disabled_compression_methods; for (auto &entry : list) { auto param = StringUtil::Lower(entry); StringUtil::Trim(param); @@ -512,19 +538,20 @@ void DisabledCompressionMethodsSetting::SetGlobal(DatabaseInstance *db, DBConfig if (compression_type == CompressionType::COMPRESSION_AUTO) { throw InvalidInputException("Unrecognized compression method \"%s\"", entry); } - disabled_compression_methods.insert(compression_type); + disabled_compression_methods.push_back(compression_type); } - config.options.disabled_compression_methods = std::move(disabled_compression_methods); + config.SetDisabledCompressionMethods(disabled_compression_methods); } void DisabledCompressionMethodsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.disabled_compression_methods = DBConfig().options.disabled_compression_methods; + vector disabled_compression_methods; + config.SetDisabledCompressionMethods(disabled_compression_methods); } Value DisabledCompressionMethodsSetting::GetSetting(const ClientContext &context) { auto &config = DBConfig::GetConfig(context); string result; - for (auto &optimizer : config.options.disabled_compression_methods) { + for (auto &optimizer : config.GetDisabledCompressionMethods()) { if (!result.empty()) { result += ","; } @@ -571,7 +598,7 @@ void DisabledOptimizersSetting::SetGlobal(DatabaseInstance *db, DBConfig &config } void DisabledOptimizersSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.disabled_optimizers = DBConfig().options.disabled_optimizers; + config.options.disabled_optimizers = DBConfigOptions().disabled_optimizers; } Value DisabledOptimizersSetting::GetSetting(const ClientContext &context) { @@ -653,7 +680,7 @@ void EnableExternalFileCacheSetting::SetGlobal(DatabaseInstance *db, DBConfig &c } void EnableExternalFileCacheSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.enable_external_file_cache = DBConfig().options.enable_external_file_cache; + config.options.enable_external_file_cache = DBConfigOptions().enable_external_file_cache; if (db) { ExternalFileCache::Get(*db).SetEnabled(config.options.enable_external_file_cache); } @@ -904,7 +931,7 @@ bool ExternalThreadsSetting::OnGlobalSet(DatabaseInstance *db, DBConfig &config, } bool ExternalThreadsSetting::OnGlobalReset(DatabaseInstance *db, DBConfig &config) { - idx_t new_external_threads = DBConfig().options.external_threads; + idx_t new_external_threads = DBConfigOptions().external_threads; if (db) { TaskScheduler::GetScheduler(*db).SetThreads(config.options.maximum_threads, new_external_threads); } @@ -944,7 +971,7 @@ void ForceBitpackingModeSetting::SetGlobal(DatabaseInstance *db, DBConfig &confi } void ForceBitpackingModeSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.force_bitpacking_mode = DBConfig().options.force_bitpacking_mode; + config.options.force_bitpacking_mode = DBConfigOptions().force_bitpacking_mode; } Value ForceBitpackingModeSetting::GetSetting(const ClientContext &context) { @@ -974,7 +1001,7 @@ void ForceCompressionSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, } void ForceCompressionSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.force_compression = DBConfig().options.force_compression; + config.options.force_compression = DBConfigOptions().force_compression; } Value ForceCompressionSetting::GetSetting(const ClientContext &context) { @@ -1088,8 +1115,7 @@ void LogQueryPathSetting::SetLocal(ClientContext &context, const Value &input) { void LogQueryPathSetting::ResetLocal(ClientContext &context) { auto &client_data = ClientData::Get(context); - // TODO: verify that this does the right thing - client_data.log_query_writer = std::move(ClientData(context).log_query_writer); + client_data.log_query_writer = nullptr; } Value LogQueryPathSetting::GetSetting(const ClientContext &context) { @@ -1369,7 +1395,7 @@ void StorageCompatibilityVersionSetting::SetGlobal(DatabaseInstance *db, DBConfi } void StorageCompatibilityVersionSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.serialization_compatibility = DBConfig().options.serialization_compatibility; + config.options.serialization_compatibility = DBConfigOptions().serialization_compatibility; } Value StorageCompatibilityVersionSetting::GetSetting(const ClientContext &context) { @@ -1417,7 +1443,7 @@ void TempDirectorySetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { throw PermissionException("Modifying the temp_directory has been disabled by configuration"); } config.SetDefaultTempDirectory(); - config.options.use_temporary_directory = DBConfig().options.use_temporary_directory; + config.options.use_temporary_directory = DBConfigOptions().use_temporary_directory; if (db) { auto &buffer_manager = BufferManager::GetBufferManager(*db); buffer_manager.SetTemporaryDirectory(config.options.temporary_directory); diff --git a/src/optimizer/CMakeLists.txt b/src/optimizer/CMakeLists.txt index 92e09839f22a..44c2009842d3 100644 --- a/src/optimizer/CMakeLists.txt +++ b/src/optimizer/CMakeLists.txt @@ -14,6 +14,7 @@ add_library_unity( column_lifetime_analyzer.cpp empty_result_pullup.cpp common_aggregate_optimizer.cpp + common_subplan_optimizer.cpp compressed_materialization.cpp cse_optimizer.cpp cte_filter_pusher.cpp @@ -34,6 +35,7 @@ add_library_unity( statistics_propagator.cpp limit_pushdown.cpp topn_optimizer.cpp + topn_window_elimination.cpp unnest_rewriter.cpp sampling_pushdown.cpp sum_rewriter.cpp) diff --git a/src/optimizer/common_subplan_optimizer.cpp b/src/optimizer/common_subplan_optimizer.cpp new file mode 100644 index 000000000000..0c3c9cb354fa --- /dev/null +++ b/src/optimizer/common_subplan_optimizer.cpp @@ -0,0 +1,575 @@ +#include "duckdb/optimizer/common_subplan_optimizer.hpp" + +#include "duckdb/optimizer/optimizer.hpp" +#include "duckdb/optimizer/cte_inlining.hpp" +#include "duckdb/optimizer/column_binding_replacer.hpp" +#include "duckdb/planner/operator/list.hpp" +#include "duckdb/common/serializer/memory_stream.hpp" +#include "duckdb/common/serializer/binary_serializer.hpp" + +namespace duckdb { + +//===--------------------------------------------------------------------===// +// Subplan Signature/Info +//===--------------------------------------------------------------------===// +struct PlanSignatureCreateState { + PlanSignatureCreateState() : stream(DEFAULT_BLOCK_ALLOC_SIZE), serializer(stream) { + } + + void Reset() { + to_canonical.clear(); + from_canonical.clear(); + table_indices.clear(); + expression_info.clear(); + } + + MemoryStream stream; + BinarySerializer serializer; + + unordered_map to_canonical; + unordered_map from_canonical; + + vector table_indices; + vector> expression_info; +}; + +class PlanSignature { +private: + PlanSignature(const MemoryStream &stream_p, idx_t offset_p, idx_t length_p, + vector> &&child_signatures_p, idx_t operator_count_p) + : stream(stream_p), offset(offset_p), length(length_p), + signature_hash(Hash(stream_p.GetData() + offset, length)), child_signatures(std::move(child_signatures_p)), + operator_count(operator_count_p) { + } + +public: + static unique_ptr Create(PlanSignatureCreateState &state, LogicalOperator &op, + vector> &&child_signatures, + const idx_t operator_count) { + state.Reset(); + if (!OperatorIsSupported(op)) { + return nullptr; + } + + if (op.type == LogicalOperatorType::LOGICAL_CHUNK_GET && + op.Cast().collection->Count() > 1000) { + // Avoid serializing massive amounts of data (this is here because of the "Test TPCH arrow roundtrip" test) + return nullptr; + } + + // Construct maps for converting column bindings to canonical representation and back + static constexpr idx_t CANONICAL_TABLE_INDEX_OFFSET = 10000000000000; + for (const auto &child_op : op.children) { + for (const auto &child_cb : child_op->GetColumnBindings()) { + const auto &original = child_cb.table_index; + auto it = state.to_canonical.find(original); + if (it != state.to_canonical.end()) { + continue; // We've seen this table index before + } + const auto canonical = CANONICAL_TABLE_INDEX_OFFSET + state.to_canonical.size(); + state.to_canonical[original] = canonical; + state.from_canonical[canonical] = original; + } + } + + // Convert operators to canonical table indices + ConvertTableIndices(op, state.table_indices); + + // Convert expressions to canonical (table indices, aliases, query locations) + bool can_materialize = ConvertExpressions(op, state.to_canonical, state.expression_info); + + // Temporarily move children here as we don't want to serialize them + auto children = std::move(op.children); + op.children.clear(); + + // TODO: to allow for better detection of equivalent plans, we could: + // 1. Sort the children of operators + // 2. Sort the expressions of operators + + // Serialize canonical representation of operator + const auto offset = state.stream.GetPosition(); + state.serializer.Begin(); + try { // Operators will throw if they cannot serialize, so we need to try/catch here + op.Serialize(state.serializer); + } catch (std::exception &) { + can_materialize = false; + } + state.serializer.End(); + const auto length = state.stream.GetPosition() - offset; + + // Convert back from canonical + ConvertTableIndices(op, state.table_indices); + ConvertExpressions(op, state.from_canonical, state.expression_info); + + // Restore children + op.children = std::move(children); + + if (can_materialize) { + return unique_ptr( + new PlanSignature(state.stream, offset, length, std::move(child_signatures), operator_count)); + } + return nullptr; + } + + idx_t OperatorCount() const { + return operator_count; + } + + hash_t HashSignature() const { + auto res = signature_hash; + for (auto &child : child_signatures) { + res = CombineHash(res, child.get().HashSignature()); + } + return res; + } + + bool Equals(const PlanSignature &other) const { + if (this->GetSignature() != other.GetSignature()) { + return false; + } + if (this->child_signatures.size() != other.child_signatures.size()) { + return false; + } + for (idx_t child_idx = 0; child_idx < this->child_signatures.size(); ++child_idx) { + if (!this->child_signatures[child_idx].get().Equals(other.child_signatures[child_idx].get())) { + return false; + } + } + return true; + } + +private: + String GetSignature() const { + return String(char_ptr_cast(stream.GetData() + offset), NumericCast(length)); + } + + static bool OperatorIsSupported(const LogicalOperator &op) { + switch (op.type) { + case LogicalOperatorType::LOGICAL_PROJECTION: + case LogicalOperatorType::LOGICAL_FILTER: + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: + case LogicalOperatorType::LOGICAL_WINDOW: + case LogicalOperatorType::LOGICAL_UNNEST: + case LogicalOperatorType::LOGICAL_LIMIT: + case LogicalOperatorType::LOGICAL_ORDER_BY: + case LogicalOperatorType::LOGICAL_TOP_N: + case LogicalOperatorType::LOGICAL_DISTINCT: + case LogicalOperatorType::LOGICAL_PIVOT: + case LogicalOperatorType::LOGICAL_GET: + case LogicalOperatorType::LOGICAL_CHUNK_GET: + case LogicalOperatorType::LOGICAL_EXPRESSION_GET: + case LogicalOperatorType::LOGICAL_DUMMY_SCAN: + case LogicalOperatorType::LOGICAL_EMPTY_RESULT: + case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: + case LogicalOperatorType::LOGICAL_ANY_JOIN: + case LogicalOperatorType::LOGICAL_CROSS_PRODUCT: + case LogicalOperatorType::LOGICAL_POSITIONAL_JOIN: + case LogicalOperatorType::LOGICAL_ASOF_JOIN: + case LogicalOperatorType::LOGICAL_UNION: + case LogicalOperatorType::LOGICAL_EXCEPT: + case LogicalOperatorType::LOGICAL_INTERSECT: + return true; + default: + // Unsupported: + // - case LogicalOperatorType::LOGICAL_COPY_TO_FILE: + // - case LogicalOperatorType::LOGICAL_SAMPLE: + // - case LogicalOperatorType::LOGICAL_COPY_DATABASE: + // - case LogicalOperatorType::LOGICAL_DELIM_GET: + // - case LogicalOperatorType::LOGICAL_CTE_REF: + // - case LogicalOperatorType::LOGICAL_JOIN: + // - case LogicalOperatorType::LOGICAL_DELIM_JOIN: + // - case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN: + // - case LogicalOperatorType::LOGICAL_RECURSIVE_CTE: + // - case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: + // - case LogicalOperatorType::LOGICAL_EXTENSION_OPERATOR + return false; + } + } + + template + static void ConvertTableIndices(LogicalOperator &op, vector &table_indices) { + switch (op.type) { + case LogicalOperatorType::LOGICAL_GET: { + ConvertTableIndicesGeneric(op, table_indices); + break; + } + case LogicalOperatorType::LOGICAL_CHUNK_GET: { + ConvertTableIndicesGeneric(op, table_indices); + break; + } + case LogicalOperatorType::LOGICAL_EXPRESSION_GET: { + ConvertTableIndicesGeneric(op, table_indices); + break; + } + case LogicalOperatorType::LOGICAL_DUMMY_SCAN: { + ConvertTableIndicesGeneric(op, table_indices); + break; + } + case LogicalOperatorType::LOGICAL_CTE_REF: { + ConvertTableIndicesGeneric(op, table_indices); + break; + } + case LogicalOperatorType::LOGICAL_PROJECTION: { + ConvertTableIndicesGeneric(op, table_indices); + break; + } + case LogicalOperatorType::LOGICAL_PIVOT: { + auto &pivot = op.Cast(); + if (TO_CANONICAL) { + table_indices.emplace_back(pivot.pivot_index); + } + pivot.pivot_index = TO_CANONICAL ? 0 : table_indices[0]; + break; + } + case LogicalOperatorType::LOGICAL_UNNEST: { + auto &unnest = op.Cast(); + if (TO_CANONICAL) { + table_indices.emplace_back(unnest.unnest_index); + } + unnest.unnest_index = TO_CANONICAL ? 0 : table_indices[0]; + break; + } + case LogicalOperatorType::LOGICAL_WINDOW: { + auto &window = op.Cast(); + if (TO_CANONICAL) { + table_indices.emplace_back(window.window_index); + } + window.window_index = TO_CANONICAL ? 0 : table_indices[0]; + break; + } + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: { + auto &aggregate = op.Cast(); + if (TO_CANONICAL) { + table_indices.emplace_back(aggregate.group_index); + table_indices.emplace_back(aggregate.aggregate_index); + table_indices.emplace_back(aggregate.groupings_index); + } + aggregate.group_index = TO_CANONICAL ? 0 : table_indices[0]; + aggregate.aggregate_index = TO_CANONICAL ? 1 : table_indices[1]; + aggregate.groupings_index = TO_CANONICAL ? 2 : table_indices[2]; + break; + } + case LogicalOperatorType::LOGICAL_UNION: + case LogicalOperatorType::LOGICAL_EXCEPT: + case LogicalOperatorType::LOGICAL_INTERSECT: { + auto &setop = op.Cast(); + if (TO_CANONICAL) { + table_indices.emplace_back(setop.table_index); + } + setop.table_index = TO_CANONICAL ? 0 : table_indices[0]; + break; + } + default: + break; + } + } + + template + static void ConvertTableIndicesGeneric(LogicalOperator &op, vector &table_idxs) { + auto &generic = op.Cast(); + if (TO_CANONICAL) { + table_idxs.emplace_back(generic.table_index); + } + generic.table_index = TO_CANONICAL ? 0 : table_idxs[0]; + } + + static bool ConvertExpressions(LogicalOperator &op, const unordered_map &table_index_mapping, + vector> &expression_info) { + bool can_materialize = true; + const auto to_canonical = expression_info.empty(); + idx_t info_idx = 0; + LogicalOperatorVisitor::EnumerateExpressions(op, [&](unique_ptr *expr) { + ExpressionIterator::EnumerateExpression(*expr, [&](unique_ptr &child) { + if (child->GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF) { + auto &col_ref = child->Cast(); + auto &table_index = col_ref.binding.table_index; + auto it = table_index_mapping.find(table_index); + D_ASSERT(it != table_index_mapping.end()); + table_index = it->second; + } + if (to_canonical) { + expression_info.emplace_back(std::move(child->alias), child->query_location); + child->alias.clear(); + child->query_location.SetInvalid(); + } else { + auto &info = expression_info[info_idx++]; + child->alias = std::move(info.first); + child->query_location = info.second; + } + if (child->IsVolatile()) { + can_materialize = false; + } + }); + }); + return can_materialize; + } + +private: + const MemoryStream &stream; + const idx_t offset; + const idx_t length; + + const hash_t signature_hash; + + const vector> child_signatures; + const idx_t operator_count; +}; + +struct PlanSignatureHash { + std::size_t operator()(const PlanSignature &k) const { + return k.HashSignature(); + } +}; + +struct PlanSignatureEquality { + bool operator()(const PlanSignature &a, const PlanSignature &b) const { + return a.Equals(b); + } +}; + +struct SubplanInfo { + explicit SubplanInfo(unique_ptr &op) : subplans({op}), lowest_common_ancestor(op) { + } + vector>> subplans; + reference> lowest_common_ancestor; +}; + +using subplan_map_t = unordered_map, SubplanInfo, PlanSignatureHash, PlanSignatureEquality>; + +//===--------------------------------------------------------------------===// +// CommonSubplanFinder +//===--------------------------------------------------------------------===// +class CommonSubplanFinder { +public: + CommonSubplanFinder() { + } + +private: + struct OperatorInfo { + OperatorInfo(unique_ptr &parent_p, const idx_t &depth_p) : parent(parent_p), depth(depth_p) { + } + + unique_ptr &parent; + const idx_t depth; + unique_ptr signature; + }; + + struct StackNode { + explicit StackNode(unique_ptr &op_p) : op(op_p), child_index(0) { + } + + bool HasMoreChildren() const { + return child_index < op->children.size(); + } + + unique_ptr &GetNextChild() { + D_ASSERT(child_index < op->children.size()); + return op->children[child_index++]; + }; + + unique_ptr &op; + idx_t child_index; + }; + +public: + subplan_map_t FindCommonSubplans(reference> root) { + // Find first operator with more than 1 child + while (root.get()->children.size() == 1) { + root = root.get()->children[0]; + } + + // Recurse through query plan using stack-based recursion + vector stack; + stack.emplace_back(root); + operator_infos.emplace(root, OperatorInfo(root, 0)); + + while (!stack.empty()) { + auto ¤t = stack.back(); + + // Depth-first + if (current.HasMoreChildren()) { + auto &child = current.GetNextChild(); + operator_infos.emplace(child, OperatorInfo(current.op, stack.size())); + stack.emplace_back(child); + continue; + } + + if (!RefersToSameObject(current.op, root.get())) { + // We have all child information for this operator now, compute signature + auto &signature = operator_infos.find(current.op)->second.signature; + signature = CreatePlanSignature(current.op); + + // Add to subplans (if we got actually got a signature) + if (signature) { + auto it = subplans.find(*signature); + if (it == subplans.end()) { + subplans.emplace(*signature, SubplanInfo(current.op)); + } else { + auto &info = it->second; + info.subplans.emplace_back(current.op); + info.lowest_common_ancestor = LowestCommonAncestor(info.lowest_common_ancestor, current.op); + } + } + } + + // Done with current + stack.pop_back(); + } + + // Filter out redundant or ineligible subplans before returning + for (auto it = subplans.begin(); it != subplans.end();) { + if (it->first.get().OperatorCount() == 1) { + it = subplans.erase(it); // Just one operator in this subplan + continue; + } + if (it->second.subplans.size() == 1) { + it = subplans.erase(it); // No other identical subplan + continue; + } + auto &subplan = it->second.subplans[0].get(); + auto &parent = operator_infos.find(subplan)->second.parent; + auto &parent_signature = operator_infos.find(parent)->second.signature; + if (parent_signature) { + auto parent_it = subplans.find(*parent_signature); + if (parent_it != subplans.end() && it->second.subplans.size() == parent_it->second.subplans.size()) { + it = subplans.erase(it); // Parent has exact same number of identical subplans + continue; + } + } + if (!CTEInlining::EndsInAggregateOrDistinct(*subplan)) { + it = subplans.erase(it); // Not eligible for materialization + continue; + } + it++; // This subplan might be useful + } + + return std::move(subplans); + } + +private: + unique_ptr CreatePlanSignature(const unique_ptr &op) { + vector> child_signatures; + idx_t operator_count = 1; + for (auto &child : op->children) { + auto it = operator_infos.find(child); + D_ASSERT(it != operator_infos.end()); + if (!it->second.signature) { + return nullptr; // Failed to create signature from one of the children + } + child_signatures.emplace_back(*it->second.signature); + operator_count += it->second.signature->OperatorCount(); + } + return PlanSignature::Create(state, *op, std::move(child_signatures), operator_count); + } + + unique_ptr &LowestCommonAncestor(reference> a, + reference> b) { + auto a_it = operator_infos.find(a); + auto b_it = operator_infos.find(b); + D_ASSERT(a_it != operator_infos.end() && b_it != operator_infos.end()); + + // Get parents of a and b until they're at the same depth + while (a_it->second.depth > b_it->second.depth) { + a = a_it->second.parent; + a_it = operator_infos.find(a); + D_ASSERT(a_it != operator_infos.end()); + } + while (b_it->second.depth > a_it->second.depth) { + b = b_it->second.parent; + b_it = operator_infos.find(b); + D_ASSERT(b_it != operator_infos.end()); + } + + // Move up one level at a time for both until ancestor is the same + while (!RefersToSameObject(a, b)) { + a_it = operator_infos.find(a); + b_it = operator_infos.find(b); + D_ASSERT(a_it != operator_infos.end() && b_it != operator_infos.end()); + a = a_it->second.parent; + b = b_it->second.parent; + } + + return a.get(); + } + +private: + //! Mapping from operator to info + reference_map_t, OperatorInfo> operator_infos; + //! Mapping from subplan signature to subplan information + subplan_map_t subplans; + //! State for creating PlanSignature with reusable data structures + PlanSignatureCreateState state; +}; + +//===--------------------------------------------------------------------===// +// CommonSubplanOptimizer +//===--------------------------------------------------------------------===// +CommonSubplanOptimizer::CommonSubplanOptimizer(Optimizer &optimizer_p) : optimizer(optimizer_p) { +} + +static void ConvertSubplansToCTE(Optimizer &optimizer, unique_ptr &op, SubplanInfo &subplan_info) { + const auto cte_index = optimizer.binder.GenerateTableIndex(); + const auto cte_name = StringUtil::Format("__common_subplan_1"); + + // Resolve types to be used for creating the materialized CTE and refs + op->ResolveOperatorTypes(); + + // Get types and names + const auto &types = subplan_info.subplans[0].get()->types; + vector col_names; + for (idx_t i = 0; i < types.size(); i++) { + col_names.emplace_back(StringUtil::Format("%s_col_%llu", cte_name, i)); + } + + // Create CTE refs and figure out column binding replacements + vector> cte_refs; + ColumnBindingReplacer replacer; + for (auto &subplan : subplan_info.subplans) { + cte_refs.emplace_back( + make_uniq(optimizer.binder.GenerateTableIndex(), cte_index, types, col_names)); + const auto old_bindings = subplan.get()->GetColumnBindings(); + const auto new_bindings = cte_refs.back()->GetColumnBindings(); + D_ASSERT(old_bindings.size() == new_bindings.size()); + for (idx_t i = 0; i < old_bindings.size(); i++) { + replacer.replacement_bindings.emplace_back(old_bindings[i], new_bindings[i]); + } + } + + // Create the materialized CTE and replace the common subplans with references to it + auto &lowest_common_ancestor = subplan_info.lowest_common_ancestor.get(); + auto cte = + make_uniq(cte_name, cte_index, types.size(), std::move(subplan_info.subplans[0].get()), + std::move(lowest_common_ancestor), CTEMaterialize::CTE_MATERIALIZE_DEFAULT); + for (idx_t i = 0; i < subplan_info.subplans.size(); i++) { + subplan_info.subplans[i].get() = std::move(cte_refs[i]); + } + lowest_common_ancestor = std::move(cte); + + // Replace bindings of subplans with those of the CTE refs + replacer.stop_operator = lowest_common_ancestor.get(); + replacer.VisitOperator(*op); // Replace from the root until CTE + replacer.VisitOperator(*lowest_common_ancestor->children[1]); // Replace in CTE child +} + +unique_ptr CommonSubplanOptimizer::Optimize(unique_ptr op) { + // Bottom-up identification of identical subplans + CommonSubplanFinder finder; + auto subplans = finder.FindCommonSubplans(op); + + // Identify the single best subplan (TODO: for now, in the future we should identify multiple) + if (subplans.empty()) { + return op; // No eligible subplans + } + auto best_it = subplans.begin(); + for (auto it = ++subplans.begin(); it != subplans.end(); it++) { + if (it->first.get().OperatorCount() > best_it->first.get().OperatorCount()) { + best_it = it; + } + } + + // Create a CTE! + ConvertSubplansToCTE(optimizer, op, best_it->second); + return op; +} + +} // namespace duckdb diff --git a/src/optimizer/cte_inlining.cpp b/src/optimizer/cte_inlining.cpp index 0b9e942eeef9..116d647682d2 100644 --- a/src/optimizer/cte_inlining.cpp +++ b/src/optimizer/cte_inlining.cpp @@ -55,10 +55,14 @@ static bool ContainsLimit(const LogicalOperator &op) { return false; } -static bool EndsInAggregateOrDistinct(const LogicalOperator &op) { - if (op.type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY || - op.type == LogicalOperatorType::LOGICAL_DISTINCT) { +bool CTEInlining::EndsInAggregateOrDistinct(const LogicalOperator &op) { + switch (op.type) { + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: + case LogicalOperatorType::LOGICAL_DISTINCT: + case LogicalOperatorType::LOGICAL_WINDOW: return true; + default: + break; } if (op.children.size() != 1) { return false; @@ -146,8 +150,7 @@ void CTEInlining::TryInlining(unique_ptr &op) { } } -bool CTEInlining::Inline(unique_ptr &op, LogicalOperator &materialized_cte, - bool requires_copy) { +bool CTEInlining::Inline(unique_ptr &op, LogicalOperator &materialized_cte, bool requires_copy) { if (op->type == LogicalOperatorType::LOGICAL_CTE_REF) { auto &cteref = op->Cast(); auto &cte = materialized_cte.Cast(); diff --git a/src/optimizer/filter_combiner.cpp b/src/optimizer/filter_combiner.cpp index 8e4a295b4889..ddbe82ab0dfb 100644 --- a/src/optimizer/filter_combiner.cpp +++ b/src/optimizer/filter_combiner.cpp @@ -1,5 +1,6 @@ #include "duckdb/optimizer/filter_combiner.hpp" +#include "duckdb/common/enums/expression_type.hpp" #include "duckdb/execution/expression_executor.hpp" #include "duckdb/optimizer/optimizer.hpp" #include "duckdb/planner/expression.hpp" @@ -907,6 +908,12 @@ FilterResult FilterCombiner::AddTransitiveFilters(BoundComparisonExpression &com idx_t left_equivalence_set = GetEquivalenceSet(left_node); idx_t right_equivalence_set = GetEquivalenceSet(right_node); if (left_equivalence_set == right_equivalence_set) { + if (comparison.GetExpressionType() == ExpressionType::COMPARE_GREATERTHAN || + comparison.GetExpressionType() == ExpressionType::COMPARE_LESSTHAN) { + // non equal comparison has equal equivalence set, then it is unsatisfiable + // e.g., j > i AND i < j is unsatisfiable + return FilterResult::UNSATISFIABLE; + } // this equality filter already exists, prune it return FilterResult::SUCCESS; } diff --git a/src/optimizer/filter_pushdown.cpp b/src/optimizer/filter_pushdown.cpp index dcb79fe60ade..7c13386d90b0 100644 --- a/src/optimizer/filter_pushdown.cpp +++ b/src/optimizer/filter_pushdown.cpp @@ -7,6 +7,7 @@ #include "duckdb/planner/operator/logical_filter.hpp" #include "duckdb/planner/operator/logical_join.hpp" #include "duckdb/planner/operator/logical_projection.hpp" +#include "duckdb/planner/operator/logical_empty_result.hpp" #include "duckdb/planner/operator/logical_window.hpp" namespace duckdb { @@ -207,17 +208,23 @@ unique_ptr FilterPushdown::PushdownJoin(unique_ptrfilter)); D_ASSERT(result != FilterResult::UNSUPPORTED); - (void)result; + if (result == FilterResult::UNSATISFIABLE) { + // one of the filters is unsatisfiable - abort filter pushdown + return FilterResult::UNSATISFIABLE; + } } filters.clear(); + return FilterResult::SUCCESS; } FilterResult FilterPushdown::AddFilter(unique_ptr expr) { - PushFilters(); + if (PushFilters() == FilterResult::UNSATISFIABLE) { + return FilterResult::UNSATISFIABLE; + } // split up the filters by AND predicate vector> expressions; expressions.push_back(std::move(expr)); @@ -275,6 +282,51 @@ unique_ptr FilterPushdown::PushFinalFilters(unique_ptr FilterPushdown::PushFiltersIntoDelimJoin(unique_ptr op) { + for (idx_t i = 0; i < filters.size(); i++) { + auto &f = *filters[i]; + for (auto &child : op->children) { + FilterPushdown pushdown(optimizer, convert_mark_joins); + + // check if filter bindings can be applied to the child bindings. + auto child_bindings = child->GetColumnBindings(); + unordered_set child_bindings_table; + for (auto &binding : child_bindings) { + child_bindings_table.insert(binding.table_index); + } + + // Check if ALL bindings of the filter are present in the child + bool should_push = true; + for (auto &binding : f.bindings) { + if (child_bindings_table.find(binding) == child_bindings_table.end()) { + should_push = false; + break; + } + } + + if (!should_push) { + continue; + } + + // copy the filter + auto filter_copy = f.filter->Copy(); + if (pushdown.AddFilter(std::move(filter_copy)) == FilterResult::UNSATISFIABLE) { + return make_uniq(std::move(op)); + } + + // push the filter into the child. + pushdown.GenerateFilters(); + child = pushdown.Rewrite(std::move(child)); + + // Don't push same filter again + filters.erase_at(i); + i--; + break; + } + } + return op; +} + unique_ptr FilterPushdown::FinishPushdown(unique_ptr op) { // unhandled type, first perform filter pushdown in its children for (auto &child : op->children) { diff --git a/src/optimizer/join_order/relation_statistics_helper.cpp b/src/optimizer/join_order/relation_statistics_helper.cpp index 0b9b6888bab7..6cf8dfea5059 100644 --- a/src/optimizer/join_order/relation_statistics_helper.cpp +++ b/src/optimizer/join_order/relation_statistics_helper.cpp @@ -225,25 +225,31 @@ RelationStats RelationStatisticsHelper::CombineStatsOfReorderableOperator(vector } RelationStats RelationStatisticsHelper::CombineStatsOfNonReorderableOperator(LogicalOperator &op, - vector child_stats) { - D_ASSERT(child_stats.size() == 2); + const vector &child_stats) { RelationStats ret; - idx_t child_1_card = child_stats[0].stats_initialized ? child_stats[0].cardinality : 0; - idx_t child_2_card = child_stats[1].stats_initialized ? child_stats[1].cardinality : 0; - ret.cardinality = MaxValue(child_1_card, child_2_card); + ret.cardinality = 0; + + // default predicted cardinality is the max of all child cardinalities + vector child_cardinalities; + for (auto &stats : child_stats) { + idx_t child_cardinality = stats.stats_initialized ? stats.cardinality : 0; + ret.cardinality = MaxValue(ret.cardinality, child_cardinality); + child_cardinalities.push_back(child_cardinality); + } switch (op.type) { case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: { + D_ASSERT(child_stats.size() == 2); auto &join = op.Cast(); switch (join.join_type) { case JoinType::RIGHT_ANTI: case JoinType::RIGHT_SEMI: - ret.cardinality = child_2_card; + ret.cardinality = child_cardinalities[1]; break; case JoinType::ANTI: case JoinType::SEMI: case JoinType::SINGLE: case JoinType::MARK: - ret.cardinality = child_1_card; + ret.cardinality = child_cardinalities[0]; break; default: break; @@ -254,18 +260,21 @@ RelationStats RelationStatisticsHelper::CombineStatsOfNonReorderableOperator(Log auto &setop = op.Cast(); if (setop.setop_all) { // setop returns all records - ret.cardinality = child_1_card + child_2_card; - } else { - ret.cardinality = MaxValue(child_1_card, child_2_card); + ret.cardinality = 0; + for (auto &child_cardinality : child_cardinalities) { + ret.cardinality += child_cardinality; + } } break; } case LogicalOperatorType::LOGICAL_INTERSECT: { - ret.cardinality = MinValue(child_1_card, child_2_card); + D_ASSERT(child_stats.size() == 2); + ret.cardinality = MinValue(child_cardinalities[0], child_cardinalities[1]); break; } case LogicalOperatorType::LOGICAL_EXCEPT: { - ret.cardinality = child_1_card; + D_ASSERT(child_stats.size() == 2); + ret.cardinality = child_cardinalities[0]; break; } default: @@ -274,8 +283,12 @@ RelationStats RelationStatisticsHelper::CombineStatsOfNonReorderableOperator(Log ret.stats_initialized = true; ret.filter_strength = 1; - ret.table_name = child_stats[0].table_name + " joined with " + child_stats[1].table_name; + ret.table_name = string(); for (auto &stats : child_stats) { + if (!ret.table_name.empty()) { + ret.table_name += " joined with "; + } + ret.table_name += stats.table_name; // MARK joins are nonreorderable. They won't return initialized stats // continue in this case. if (!stats.stats_initialized) { diff --git a/src/optimizer/late_materialization.cpp b/src/optimizer/late_materialization.cpp index a144df188b82..4e5b0f13e2e7 100644 --- a/src/optimizer/late_materialization.cpp +++ b/src/optimizer/late_materialization.cpp @@ -62,6 +62,10 @@ unique_ptr LateMaterialization::ConstructLHS(LogicalGet &get) { get.names, get.virtual_columns); new_get->GetMutableColumnIds() = get.GetColumnIds(); new_get->projection_ids = get.projection_ids; + new_get->parameters = get.parameters; + new_get->named_parameters = get.named_parameters; + new_get->input_table_types = get.input_table_types; + new_get->input_table_names = get.input_table_names; return new_get; } diff --git a/src/optimizer/optimizer.cpp b/src/optimizer/optimizer.cpp index ce6cb0045cf1..dfbbbd9012f7 100644 --- a/src/optimizer/optimizer.cpp +++ b/src/optimizer/optimizer.cpp @@ -32,8 +32,10 @@ #include "duckdb/optimizer/statistics_propagator.hpp" #include "duckdb/optimizer/sum_rewriter.hpp" #include "duckdb/optimizer/topn_optimizer.hpp" +#include "duckdb/optimizer/topn_window_elimination.hpp" #include "duckdb/optimizer/unnest_rewriter.hpp" #include "duckdb/optimizer/late_materialization.hpp" +#include "duckdb/optimizer/common_subplan_optimizer.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/planner.hpp" @@ -127,6 +129,12 @@ void Optimizer::RunBuiltInOptimizers() { plan = cte_inlining.Optimize(std::move(plan)); }); + // convert common subplans into materialized CTEs + RunOptimizer(OptimizerType::COMMON_SUBPLAN, [&]() { + CommonSubplanOptimizer common_subplan_optimizer(*this); + plan = common_subplan_optimizer.Optimize(std::move(plan)); + }); + // Rewrites SUM(x + C) into SUM(x) + C * COUNT(x) RunOptimizer(OptimizerType::SUM_REWRITER, [&]() { SumRewriterOptimizer optimizer(*this); @@ -257,6 +265,12 @@ void Optimizer::RunBuiltInOptimizers() { statistics_map = propagator.GetStatisticsMap(); }); + // rewrite row_number window function + filter on row_number to aggregate + RunOptimizer(OptimizerType::TOP_N_WINDOW_ELIMINATION, [&]() { + TopNWindowElimination topn_window_elimination(context, *this, &statistics_map); + plan = topn_window_elimination.Optimize(std::move(plan)); + }); + // remove duplicate aggregates RunOptimizer(OptimizerType::COMMON_AGGREGATE, [&]() { CommonAggregateOptimizer common_aggregate; diff --git a/src/optimizer/pushdown/pushdown_get.cpp b/src/optimizer/pushdown/pushdown_get.cpp index 90dbbb823951..ac4b6532ad94 100644 --- a/src/optimizer/pushdown/pushdown_get.cpp +++ b/src/optimizer/pushdown/pushdown_get.cpp @@ -4,6 +4,7 @@ #include "duckdb/planner/expression/bound_parameter_expression.hpp" #include "duckdb/planner/operator/logical_filter.hpp" #include "duckdb/planner/operator/logical_get.hpp" +#include "duckdb/planner/operator/logical_empty_result.hpp" namespace duckdb { unique_ptr FilterPushdown::PushdownGet(unique_ptr op) { @@ -48,7 +49,9 @@ unique_ptr FilterPushdown::PushdownGet(unique_ptr(std::move(op)); + } //! We generate the table filters that will be executed during the table scan vector pushdown_results; diff --git a/src/optimizer/pushdown/pushdown_inner_join.cpp b/src/optimizer/pushdown/pushdown_inner_join.cpp index 8370f4ca9ba6..e2e4730d11ce 100644 --- a/src/optimizer/pushdown/pushdown_inner_join.cpp +++ b/src/optimizer/pushdown/pushdown_inner_join.cpp @@ -14,6 +14,7 @@ unique_ptr FilterPushdown::PushdownInnerJoin(unique_ptrCast(); D_ASSERT(join.join_type == JoinType::INNER); if (op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN) { + op = PushFiltersIntoDelimJoin(std::move(op)); return FinishPushdown(std::move(op)); } // inner join: gather all the conditions of the inner join and add to the filter list diff --git a/src/optimizer/pushdown/pushdown_left_join.cpp b/src/optimizer/pushdown/pushdown_left_join.cpp index 9e56ed9d6efd..1ebf3cedd4e3 100644 --- a/src/optimizer/pushdown/pushdown_left_join.cpp +++ b/src/optimizer/pushdown/pushdown_left_join.cpp @@ -78,6 +78,7 @@ unique_ptr FilterPushdown::PushdownLeftJoin(unique_ptr &right_bindings) { auto &join = op->Cast(); if (op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN) { + op = PushFiltersIntoDelimJoin(std::move(op)); return FinishPushdown(std::move(op)); } FilterPushdown left_pushdown(optimizer, convert_mark_joins), right_pushdown(optimizer, convert_mark_joins); diff --git a/src/optimizer/pushdown/pushdown_semi_anti_join.cpp b/src/optimizer/pushdown/pushdown_semi_anti_join.cpp index 7d240e3f6f15..0b937fe25b5a 100644 --- a/src/optimizer/pushdown/pushdown_semi_anti_join.cpp +++ b/src/optimizer/pushdown/pushdown_semi_anti_join.cpp @@ -12,6 +12,7 @@ using Filter = FilterPushdown::Filter; unique_ptr FilterPushdown::PushdownSemiAntiJoin(unique_ptr op) { auto &join = op->Cast(); if (op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN) { + op = PushFiltersIntoDelimJoin(std::move(op)); return FinishPushdown(std::move(op)); } diff --git a/src/optimizer/pushdown/pushdown_set_operation.cpp b/src/optimizer/pushdown/pushdown_set_operation.cpp index c0f8aa1436b2..e56abdcd263f 100644 --- a/src/optimizer/pushdown/pushdown_set_operation.cpp +++ b/src/optimizer/pushdown/pushdown_set_operation.cpp @@ -29,54 +29,53 @@ unique_ptr FilterPushdown::PushdownSetOperation(unique_ptrtype == LogicalOperatorType::LOGICAL_INTERSECT); auto &setop = op->Cast(); - D_ASSERT(op->children.size() == 2); - auto left_bindings = op->children[0]->GetColumnBindings(); - auto right_bindings = op->children[1]->GetColumnBindings(); - if (left_bindings.size() != right_bindings.size()) { - throw InternalException("Filter pushdown - set operation LHS and RHS have incompatible counts"); - } - - // pushdown into set operation, we can duplicate the condition and pushdown the expressions into both sides - FilterPushdown left_pushdown(optimizer, convert_mark_joins), right_pushdown(optimizer, convert_mark_joins); - for (idx_t i = 0; i < filters.size(); i++) { - // first create a copy of the filter - auto right_filter = make_uniq(); - right_filter->filter = filters[i]->filter->Copy(); + for (auto &child : op->children) { + auto child_bindings = child->GetColumnBindings(); - // in the original filter, rewrite references to the result of the union into references to the left_index - ReplaceSetOpBindings(left_bindings, *filters[i], filters[i]->filter, setop); - // in the copied filter, rewrite references to the result of the union into references to the right_index - ReplaceSetOpBindings(right_bindings, *right_filter, right_filter->filter, setop); + FilterPushdown child_pushdown(optimizer, convert_mark_joins); + for (auto &original_filter : filters) { + // first create a copy of the filter + auto filter = make_uniq(); + filter->filter = original_filter->filter->Copy(); - // extract bindings again - filters[i]->ExtractBindings(); - right_filter->ExtractBindings(); + // rewrite references to the result of the union into references to the child index + ReplaceSetOpBindings(child_bindings, *filter, filter->filter, setop); - // move the filters into the child pushdown nodes - left_pushdown.filters.push_back(std::move(filters[i])); - right_pushdown.filters.push_back(std::move(right_filter)); - } + // extract bindings again + filter->ExtractBindings(); - op->children[0] = left_pushdown.Rewrite(std::move(op->children[0])); - op->children[1] = right_pushdown.Rewrite(std::move(op->children[1])); + // move the filters into the child pushdown nodes + child_pushdown.filters.push_back(std::move(filter)); + } - bool left_empty = op->children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT; - bool right_empty = op->children[1]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT; - if (left_empty && right_empty) { - // both empty: return empty result + // pushdown into the child + child = child_pushdown.Rewrite(std::move(child)); + } + bool all_empty = true; + for (auto &child : op->children) { + if (child->type != LogicalOperatorType::LOGICAL_EMPTY_RESULT) { + all_empty = false; + } + } + if (all_empty) { + // all sides are empty: the result must be empty return make_uniq(std::move(op)); } + if (op->type == LogicalOperatorType::LOGICAL_UNION) { + // for UNION (ALL) - delete all empty children and return + for (idx_t i = 0; i < op->children.size(); i++) { + if (op->children[i]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) { + op->children.erase(op->children.begin() + static_cast(i)); + i--; + } + } + return op; + } + bool left_empty = op->children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT; + bool right_empty = op->children[1]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT; if (left_empty && setop.setop_all) { // left child is empty result switch (op->type) { - case LogicalOperatorType::LOGICAL_UNION: - if (op->children[1]->type == LogicalOperatorType::LOGICAL_PROJECTION) { - // union with empty left side: return right child - auto &projection = op->children[1]->Cast(); - projection.table_index = setop.table_index; - return std::move(op->children[1]); - } - break; case LogicalOperatorType::LOGICAL_EXCEPT: // except: if left child is empty, return empty result case LogicalOperatorType::LOGICAL_INTERSECT: @@ -88,7 +87,6 @@ unique_ptr FilterPushdown::PushdownSetOperation(unique_ptrtype) { - case LogicalOperatorType::LOGICAL_UNION: case LogicalOperatorType::LOGICAL_EXCEPT: if (op->children[0]->type == LogicalOperatorType::LOGICAL_PROJECTION) { // union or except with empty right child: return left child diff --git a/src/optimizer/remove_unused_columns.cpp b/src/optimizer/remove_unused_columns.cpp index d7ff30ea2e20..20817633ab56 100644 --- a/src/optimizer/remove_unused_columns.cpp +++ b/src/optimizer/remove_unused_columns.cpp @@ -62,7 +62,6 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) { // This causes the duplicate eliminator to ignore functionality provided by grouping sets bool new_root = false; if (aggr.grouping_sets.size() > 1) { - ; new_root = true; } if (!everything_referenced && !new_root) { @@ -86,39 +85,49 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) { case LogicalOperatorType::LOGICAL_ASOF_JOIN: case LogicalOperatorType::LOGICAL_DELIM_JOIN: case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: { - if (!everything_referenced) { - auto &comp_join = op.Cast(); + if (everything_referenced) { + break; + } + auto &comp_join = op.Cast(); - if (comp_join.join_type != JoinType::INNER) { - break; + if (comp_join.join_type != JoinType::INNER) { + break; + } + // for inner joins with equality predicates in the form of (X=Y) + // we can replace any references to the RHS (Y) to references to the LHS (X) + // this reduces the amount of columns we need to extract from the join hash table + // (except in the case of floating point numbers which have +0 and -0, equal but different). + for (auto &cond : comp_join.conditions) { + if (cond.comparison != ExpressionType::COMPARE_EQUAL) { + continue; } - // for inner joins with equality predicates in the form of (X=Y) - // we can replace any references to the RHS (Y) to references to the LHS (X) - // this reduces the amount of columns we need to extract from the join hash table - // (except in the case of floating point numbers which have +0 and -0, equal but different). - for (auto &cond : comp_join.conditions) { - if (cond.comparison == ExpressionType::COMPARE_EQUAL) { - if (cond.left->GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF && - cond.right->GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF && - !(cond.left->Cast().return_type.IsFloating() && - cond.right->Cast().return_type.IsFloating())) { - // comparison join between two bound column refs - // we can replace any reference to the RHS (build-side) with a reference to the LHS (probe-side) - auto &lhs_col = cond.left->Cast(); - auto &rhs_col = cond.right->Cast(); - // if there are any columns that refer to the RHS, - auto colrefs = column_references.find(rhs_col.binding); - if (colrefs != column_references.end()) { - for (auto &entry : colrefs->second.bindings) { - auto &colref = entry.get(); - colref.binding = lhs_col.binding; - AddBinding(colref); - } - column_references.erase(rhs_col.binding); - } - } - } + if (cond.left->GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + continue; + } + if (cond.right->GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + continue; + } + if (cond.left->Cast().return_type.IsFloating()) { + continue; + } + if (cond.right->Cast().return_type.IsFloating()) { + continue; } + // comparison join between two bound column refs + // we can replace any reference to the RHS (build-side) with a reference to the LHS (probe-side) + auto &lhs_col = cond.left->Cast(); + auto &rhs_col = cond.right->Cast(); + // if there are any columns that refer to the RHS, + auto colrefs = column_references.find(rhs_col.binding); + if (colrefs == column_references.end()) { + continue; + } + for (auto &entry : colrefs->second.bindings) { + auto &colref = entry.get(); + colref.binding = lhs_col.binding; + AddBinding(colref); + } + column_references.erase(rhs_col.binding); } break; } @@ -135,40 +144,40 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) { entries.push_back(i); } ClearUnusedExpressions(entries, setop.table_index); - if (entries.size() < setop.column_count) { - if (entries.empty()) { - // no columns referenced: this happens in the case of a COUNT(*) - // extract the first column - entries.push_back(0); - } - // columns were cleared - setop.column_count = entries.size(); - - for (idx_t child_idx = 0; child_idx < op.children.size(); child_idx++) { - RemoveUnusedColumns remove(binder, context, true); - auto &child = op.children[child_idx]; + if (entries.size() >= setop.column_count) { + return; + } + if (entries.empty()) { + // no columns referenced: this happens in the case of a COUNT(*) + // extract the first column + entries.push_back(0); + } + // columns were cleared + setop.column_count = entries.size(); - // we push a projection under this child that references the required columns of the union - child->ResolveOperatorTypes(); - auto bindings = child->GetColumnBindings(); - vector> expressions; - expressions.reserve(entries.size()); - for (auto &column_idx : entries) { - expressions.push_back( - make_uniq(child->types[column_idx], bindings[column_idx])); - } - auto new_projection = - make_uniq(binder.GenerateTableIndex(), std::move(expressions)); - if (child->has_estimated_cardinality) { - new_projection->SetEstimatedCardinality(child->estimated_cardinality); - } - new_projection->children.push_back(std::move(child)); - op.children[child_idx] = std::move(new_projection); + for (idx_t child_idx = 0; child_idx < op.children.size(); child_idx++) { + RemoveUnusedColumns remove(binder, context, true); + auto &child = op.children[child_idx]; - remove.VisitOperator(*op.children[child_idx]); + // we push a projection under this child that references the required columns of the union + child->ResolveOperatorTypes(); + auto bindings = child->GetColumnBindings(); + vector> expressions; + expressions.reserve(entries.size()); + for (auto &column_idx : entries) { + expressions.push_back( + make_uniq(child->types[column_idx], bindings[column_idx])); } - return; + auto new_projection = make_uniq(binder.GenerateTableIndex(), std::move(expressions)); + if (child->has_estimated_cardinality) { + new_projection->SetEstimatedCardinality(child->estimated_cardinality); + } + new_projection->children.push_back(std::move(child)); + op.children[child_idx] = std::move(new_projection); + + remove.VisitOperator(*op.children[child_idx]); } + return; } for (auto &child : op.children) { RemoveUnusedColumns remove(binder, context, true); @@ -217,91 +226,94 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) { remove.VisitOperator(*op.children[0]); return; } - case LogicalOperatorType::LOGICAL_GET: + case LogicalOperatorType::LOGICAL_GET: { LogicalOperatorVisitor::VisitOperatorExpressions(op); - if (!everything_referenced) { - auto &get = op.Cast(); - if (!get.function.projection_pushdown) { - return; - } + if (everything_referenced) { + return; + } + auto &get = op.Cast(); + if (!get.function.projection_pushdown) { + return; + } - auto final_column_ids = get.GetColumnIds(); + auto final_column_ids = get.GetColumnIds(); - // Create "selection vector" of all column ids - vector proj_sel; - for (idx_t col_idx = 0; col_idx < final_column_ids.size(); col_idx++) { - proj_sel.push_back(col_idx); - } - // Create a copy that we can use to match ids later - auto col_sel = proj_sel; - // Clear unused ids, exclude filter columns that are projected out immediately - ClearUnusedExpressions(proj_sel, get.table_index, false); + // Create "selection vector" of all column ids + vector proj_sel; + for (idx_t col_idx = 0; col_idx < final_column_ids.size(); col_idx++) { + proj_sel.push_back(col_idx); + } + // Create a copy that we can use to match ids later + auto col_sel = proj_sel; + // Clear unused ids, exclude filter columns that are projected out immediately + ClearUnusedExpressions(proj_sel, get.table_index, false); - vector> filter_expressions; - // for every table filter, push a column binding into the column references map to prevent the column from - // being projected out - for (auto &filter : get.table_filters.filters) { - optional_idx index; - for (idx_t i = 0; i < final_column_ids.size(); i++) { - if (final_column_ids[i].GetPrimaryIndex() == filter.first) { - index = i; - break; - } - } - if (!index.IsValid()) { - throw InternalException("Could not find column index for table filter"); + vector> filter_expressions; + // for every table filter, push a column binding into the column references map to prevent the column from + // being projected out + for (auto &filter : get.table_filters.filters) { + optional_idx index; + for (idx_t i = 0; i < final_column_ids.size(); i++) { + if (final_column_ids[i].GetPrimaryIndex() == filter.first) { + index = i; + break; } + } + if (!index.IsValid()) { + throw InternalException("Could not find column index for table filter"); + } - auto column_type = get.GetColumnType(ColumnIndex(filter.first)); + auto column_type = get.GetColumnType(ColumnIndex(filter.first)); - ColumnBinding filter_binding(get.table_index, index.GetIndex()); - auto column_ref = make_uniq(std::move(column_type), filter_binding); - auto filter_expr = filter.second->ToExpression(*column_ref); - if (filter_expr->IsScalar()) { - filter_expr = std::move(column_ref); - } - VisitExpression(&filter_expr); - filter_expressions.push_back(std::move(filter_expr)); + ColumnBinding filter_binding(get.table_index, index.GetIndex()); + auto column_ref = make_uniq(std::move(column_type), filter_binding); + auto filter_expr = filter.second->ToExpression(*column_ref); + if (filter_expr->IsScalar()) { + filter_expr = std::move(column_ref); } + VisitExpression(&filter_expr); + filter_expressions.push_back(std::move(filter_expr)); + } - // Clear unused ids, include filter columns that are projected out immediately - ClearUnusedExpressions(col_sel, get.table_index); + // Clear unused ids, include filter columns that are projected out immediately + ClearUnusedExpressions(col_sel, get.table_index); - // Now set the column ids in the LogicalGet using the "selection vector" - vector column_ids; - column_ids.reserve(col_sel.size()); - for (auto col_sel_idx : col_sel) { - auto entry = column_references.find(ColumnBinding(get.table_index, col_sel_idx)); - if (entry == column_references.end()) { - throw InternalException("RemoveUnusedColumns - could not find referenced column"); - } - ColumnIndex new_index(final_column_ids[col_sel_idx].GetPrimaryIndex(), entry->second.child_columns); - column_ids.emplace_back(new_index); - } - if (column_ids.empty()) { - // this generally means we are only interested in whether or not anything exists in the table (e.g. - // EXISTS(SELECT * FROM tbl)) in this case, we just scan the row identifier column as it means we do not - // need to read any of the columns - column_ids.emplace_back(get.GetAnyColumn()); + // Now set the column ids in the LogicalGet using the "selection vector" + vector column_ids; + column_ids.reserve(col_sel.size()); + for (auto col_sel_idx : col_sel) { + auto entry = column_references.find(ColumnBinding(get.table_index, col_sel_idx)); + if (entry == column_references.end()) { + throw InternalException("RemoveUnusedColumns - could not find referenced column"); } - get.SetColumnIds(std::move(column_ids)); + ColumnIndex new_index(final_column_ids[col_sel_idx].GetPrimaryIndex(), entry->second.child_columns); + column_ids.emplace_back(new_index); + } + if (column_ids.empty()) { + // this generally means we are only interested in whether or not anything exists in the table (e.g. + // EXISTS(SELECT * FROM tbl)) in this case, we just scan the row identifier column as it means we do not + // need to read any of the columns + column_ids.emplace_back(get.GetAnyColumn()); + } + get.SetColumnIds(std::move(column_ids)); - if (get.function.filter_prune) { - // Now set the projection cols by matching the "selection vector" that excludes filter columns - // with the "selection vector" that includes filter columns - idx_t col_idx = 0; - get.projection_ids.clear(); - for (auto proj_sel_idx : proj_sel) { - for (; col_idx < col_sel.size(); col_idx++) { - if (proj_sel_idx == col_sel[col_idx]) { - get.projection_ids.push_back(col_idx); - break; - } - } + if (!get.function.filter_prune) { + return; + } + // Now set the projection cols by matching the "selection vector" that excludes filter columns + // with the "selection vector" that includes filter columns + idx_t col_idx = 0; + get.projection_ids.clear(); + for (auto proj_sel_idx : proj_sel) { + for (; col_idx < col_sel.size(); col_idx++) { + if (proj_sel_idx == col_sel[col_idx]) { + get.projection_ids.push_back(col_idx); + break; } } } return; + } case LogicalOperatorType::LOGICAL_DISTINCT: { auto &distinct = op.Cast(); if (distinct.distinct_type == DistinctType::DISTINCT_ON) { diff --git a/src/optimizer/rule/comparison_simplification.cpp b/src/optimizer/rule/comparison_simplification.cpp index 1e377528d0a3..dc778cfff393 100644 --- a/src/optimizer/rule/comparison_simplification.cpp +++ b/src/optimizer/rule/comparison_simplification.cpp @@ -56,13 +56,8 @@ unique_ptr ComparisonSimplificationRule::Apply(LogicalOperator &op, // Is the constant cast invertible? if (!cast_constant.IsNull() && !BoundCastExpression::CastIsInvertible(cast_expression.return_type, target_type)) { - // Is it actually invertible? - Value uncast_constant; - if (!cast_constant.TryCastAs(rewriter.context, constant_value.type(), uncast_constant, &error_message, - true) || - uncast_constant != constant_value) { - return nullptr; - } + // Cast is not invertible, so we do not rewrite this expression to ensure that the cast is executed + return nullptr; } //! We can cast, now we change our column_ref_expression from an operator cast to a column reference @@ -75,6 +70,7 @@ unique_ptr ComparisonSimplificationRule::Apply(LogicalOperator &op, expr.left = std::move(new_constant_expr); expr.right = std::move(child_expression); } + changes_made = true; } return nullptr; } diff --git a/src/optimizer/rule/regex_optimizations.cpp b/src/optimizer/rule/regex_optimizations.cpp index 24786867b55a..3a0697e99457 100644 --- a/src/optimizer/rule/regex_optimizations.cpp +++ b/src/optimizer/rule/regex_optimizations.cpp @@ -184,6 +184,13 @@ unique_ptr RegexOptimizationRule::Apply(LogicalOperator &op, vector< if (!escaped_like_string.exists) { return nullptr; } + + // if regexp had options, remove them so the new Contains Expression can be matched for other optimizers. + if (root.children.size() == 3) { + root.children.pop_back(); + D_ASSERT(root.children.size() == 2); + } + auto parameter = make_uniq(Value(std::move(escaped_like_string.like_string))); auto contains = make_uniq(root.return_type, GetStringContains(), std::move(root.children), nullptr); diff --git a/src/optimizer/statistics/operator/propagate_set_operation.cpp b/src/optimizer/statistics/operator/propagate_set_operation.cpp index 4c8c33ccd7d1..5a0fb090b323 100644 --- a/src/optimizer/statistics/operator/propagate_set_operation.cpp +++ b/src/optimizer/statistics/operator/propagate_set_operation.cpp @@ -4,6 +4,9 @@ namespace duckdb { void StatisticsPropagator::AddCardinalities(unique_ptr &stats, NodeStatistics &new_stats) { + if (!stats) { + return; + } if (!stats->has_estimated_cardinality || !new_stats.has_estimated_cardinality || !stats->has_max_cardinality || !new_stats.has_max_cardinality) { stats = nullptr; @@ -24,8 +27,59 @@ void StatisticsPropagator::AddCardinalities(unique_ptr &stats, N } } +unique_ptr StatisticsPropagator::PropagateUnion(LogicalSetOperation &setop, + unique_ptr &node_ptr) { + // first propagate statistics in the child nodes + vector> stats; + for (auto &child : setop.children) { + stats.push_back(PropagateStatistics(child)); + } + + // now fetch the column bindings of the children both sides + vector> child_bindings; + for (auto &child : setop.children) { + child_bindings.push_back(child->GetColumnBindings()); + } + for (idx_t i = 0; i < setop.column_count; i++) { + // for each column binding, we fetch the statistics from both the lhs and the rhs + unique_ptr new_stats; + for (idx_t child_idx = 0; child_idx < setop.children.size(); child_idx++) { + auto stats_entry = statistics_map.find(child_bindings[child_idx][i]); + if (stats_entry == statistics_map.end()) { + new_stats.reset(); + break; + } + auto &child_stats = stats_entry->second; + if (!new_stats) { + new_stats = child_stats->ToUnique(); + } else { + new_stats->Merge(*child_stats); + } + } + if (!new_stats) { + // no statistics on at least one of the sides: can't propagate stats + continue; + } + // propagate the stats for this column + ColumnBinding binding(setop.table_index, i); + statistics_map[binding] = std::move(new_stats); + } + // merge all cardinalities of the child stats together + for (idx_t i = 1; i < stats.size(); i++) { + if (!stats[i]) { + return nullptr; + } + AddCardinalities(stats[0], *stats[i]); + } + return std::move(stats[0]); +} + unique_ptr StatisticsPropagator::PropagateStatistics(LogicalSetOperation &setop, unique_ptr &node_ptr) { + if (setop.type == LogicalOperatorType::LOGICAL_UNION) { + return PropagateUnion(setop, node_ptr); + } + D_ASSERT(setop.children.size() == 2); // first propagate statistics in the child nodes auto left_stats = PropagateStatistics(setop.children[0]); auto right_stats = PropagateStatistics(setop.children[1]); @@ -46,11 +100,6 @@ unique_ptr StatisticsPropagator::PropagateStatistics(LogicalSetO } unique_ptr new_stats; switch (setop.type) { - case LogicalOperatorType::LOGICAL_UNION: - // union: merge the stats of the LHS and RHS together - new_stats = left_entry->second->ToUnique(); - new_stats->Merge(*right_entry->second); - break; case LogicalOperatorType::LOGICAL_EXCEPT: // except: use the stats of the LHS new_stats = left_entry->second->ToUnique(); @@ -70,9 +119,6 @@ unique_ptr StatisticsPropagator::PropagateStatistics(LogicalSetO if (!left_stats || !right_stats) { return nullptr; } - if (setop.type == LogicalOperatorType::LOGICAL_UNION) { - AddCardinalities(left_stats, *right_stats); - } return left_stats; } diff --git a/src/optimizer/topn_window_elimination.cpp b/src/optimizer/topn_window_elimination.cpp new file mode 100644 index 000000000000..957cef3675bd --- /dev/null +++ b/src/optimizer/topn_window_elimination.cpp @@ -0,0 +1,611 @@ +#include "duckdb/optimizer/topn_window_elimination.hpp" + +#include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/operator/logical_aggregate.hpp" +#include "duckdb/planner/operator/logical_get.hpp" +#include "duckdb/planner/operator/logical_filter.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" +#include "duckdb/planner/operator/logical_unnest.hpp" +#include "duckdb/planner/operator/logical_window.hpp" +#include "duckdb/function/scalar/nested_functions.hpp" +#include "duckdb/function/scalar/struct_functions.hpp" +#include "duckdb/optimizer/optimizer.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/planner/expression/bound_aggregate_expression.hpp" +#include "duckdb/planner/expression/bound_comparison_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_unnest_expression.hpp" +#include "duckdb/planner/expression/bound_window_expression.hpp" +#include "duckdb/function/function_binder.hpp" +#include "duckdb/main/database.hpp" + +namespace duckdb { + +namespace { + +idx_t GetGroupIdx(const unique_ptr &op) { + if (op->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + return op->Cast().group_index; + } + return op->GetTableIndex()[0]; +} + +idx_t GetAggregateIdx(const unique_ptr &op) { + if (op->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + return op->Cast().aggregate_index; + } + return op->GetTableIndex()[0]; +} + +LogicalType GetAggregateType(const unique_ptr &op) { + switch (op->type) { + case LogicalOperatorType::LOGICAL_UNNEST: { + const auto &logical_unnest = op->Cast(); + const idx_t unnest_offset = logical_unnest.children[0]->types.size(); + return logical_unnest.types[unnest_offset]; + } + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: { + const auto &logical_aggregate = op->Cast(); + const idx_t aggregate_column_idx = logical_aggregate.groups.size(); + return logical_aggregate.types[aggregate_column_idx]; + } + default: { + throw InternalException("Unnest or aggregate expected to extract aggregate type."); + } + } +} + +vector ExtractReturnTypes(const vector> &exprs) { + vector types; + types.reserve(exprs.size()); + for (const auto &expr : exprs) { + types.push_back(expr->return_type); + } + return types; +} + +bool BindingsReferenceRowNumber(const vector &bindings, const LogicalWindow &window) { + for (const auto &binding : bindings) { + if (binding.table_index == window.window_index) { + return true; + } + } + return false; +} + +} // namespace + +TopNWindowElimination::TopNWindowElimination(ClientContext &context_p, Optimizer &optimizer, + optional_ptr>> stats_p) + : context(context_p), optimizer(optimizer), stats(stats_p) { +} + +unique_ptr TopNWindowElimination::Optimize(unique_ptr op) { + auto &extension_manager = context.db->GetExtensionManager(); + if (!extension_manager.ExtensionIsLoaded("core_functions")) { + return op; + } + + ColumnBindingReplacer replacer; + op = OptimizeInternal(std::move(op), replacer); + if (!replacer.replacement_bindings.empty()) { + replacer.VisitOperator(*op); + } + return op; +} + +unique_ptr TopNWindowElimination::OptimizeInternal(unique_ptr op, + ColumnBindingReplacer &replacer) { + if (!CanOptimize(*op)) { + // Traverse through query plan to find grouped top-n pattern + if (op->children.size() > 1) { + // If an operator has multiple children, we do not want them to overwrite each other's stop operator. + // Thus, first update only the column binding in op, then set op as the new stop operator. + for (auto &child : op->children) { + ColumnBindingReplacer r2; + child = OptimizeInternal(std::move(child), r2); + + if (!r2.replacement_bindings.empty()) { + r2.VisitOperator(*op); + replacer.replacement_bindings.insert(replacer.replacement_bindings.end(), + r2.replacement_bindings.begin(), + r2.replacement_bindings.end()); + replacer.stop_operator = op; + } + } + } else if (!op->children.empty()) { + op->children[0] = OptimizeInternal(std::move(op->children[0]), replacer); + } + + return op; + } + // We have made sure that this is an operator sequence of filter -> N optional projections -> window + auto &filter = op->Cast(); + auto *child = filter.children[0].get(); + + // Get bindings and types from filter to use in top-most operator later + const auto topmost_bindings = filter.GetColumnBindings(); + auto new_bindings = TraverseProjectionBindings(topmost_bindings, child); + + D_ASSERT(child->type == LogicalOperatorType::LOGICAL_WINDOW); + auto &window = child->Cast(); + const idx_t window_idx = window.window_index; + + // Map the input column offsets of the group columns to the output offset if there are projections on the group + // We use an ordered map here because we need to iterate over them in order later + map group_projection_idxs; + auto aggregate_payload = GenerateAggregatePayload(new_bindings, window, group_projection_idxs); + const auto params = ExtractOptimizerParameters(window, filter, new_bindings, aggregate_payload); + + // Optimize window children + window.children[0] = Optimize(std::move(window.children[0])); + + op = CreateAggregateOperator(window, std::move(aggregate_payload), params); + op = TryCreateUnnestOperator(std::move(op), params); + op = CreateProjectionOperator(std::move(op), params, group_projection_idxs); + + D_ASSERT(op->type != LogicalOperatorType::LOGICAL_UNNEST); + + UpdateTopmostBindings(window_idx, op, group_projection_idxs, topmost_bindings, new_bindings, replacer); + replacer.stop_operator = op.get(); + + return unique_ptr(std::move(op)); +} + +unique_ptr +TopNWindowElimination::CreateAggregateExpression(vector> aggregate_params, + const bool requires_arg, + const TopNWindowEliminationParameters ¶ms) const { + auto &catalog = Catalog::GetSystemCatalog(context); + FunctionBinder function_binder(context); + + // If the value column can be null, we must use the nulls_last function to follow null ordering semantics + const bool change_to_arg = !requires_arg && params.can_be_null && params.limit > 1; + if (change_to_arg) { + // Copy value as argument + aggregate_params.insert(aggregate_params.begin() + 1, aggregate_params[0]->Copy()); + } + + D_ASSERT(params.order_type == OrderType::ASCENDING || params.order_type == OrderType::DESCENDING); + string fun_name = requires_arg || change_to_arg ? "arg_" : ""; + fun_name += params.order_type == OrderType::ASCENDING ? "min" : "max"; + fun_name += params.can_be_null && (requires_arg || change_to_arg) ? "_nulls_last" : ""; + + auto &fun_entry = catalog.GetEntry(context, DEFAULT_SCHEMA, fun_name); + const auto fun = fun_entry.functions.GetFunctionByArguments(context, ExtractReturnTypes(aggregate_params)); + return function_binder.BindAggregateFunction(fun, std::move(aggregate_params)); +} + +unique_ptr +TopNWindowElimination::CreateAggregateOperator(LogicalWindow &window, vector> args, + const TopNWindowEliminationParameters ¶ms) const { + auto &window_expr = window.expressions[0]->Cast(); + D_ASSERT(window_expr.orders.size() == 1); + + vector> aggregate_params; + aggregate_params.reserve(3); + + const bool use_arg = !args.empty(); + if (args.size() == 1) { + aggregate_params.push_back(std::move(args[0])); + } else if (args.size() > 1) { + // For more than one arg, we must use struct pack + auto &catalog = Catalog::GetSystemCatalog(context); + FunctionBinder function_binder(context); + auto &struct_pack_entry = catalog.GetEntry(context, DEFAULT_SCHEMA, "struct_pack"); + const auto struct_pack_fun = + struct_pack_entry.functions.GetFunctionByArguments(context, ExtractReturnTypes(args)); + auto struct_pack_expr = function_binder.BindScalarFunction(struct_pack_fun, std::move(args)); + aggregate_params.push_back(std::move(struct_pack_expr)); + } + + aggregate_params.push_back(std::move(window_expr.orders[0].expression)); + if (params.limit > 1) { + aggregate_params.push_back(std::move(make_uniq(Value::BIGINT(params.limit)))); + } + + auto aggregate_expr = CreateAggregateExpression(std::move(aggregate_params), use_arg, params); + + vector> select_list; + select_list.push_back(std::move(aggregate_expr)); + + auto aggregate = make_uniq(optimizer.binder.GenerateTableIndex(), + optimizer.binder.GenerateTableIndex(), std::move(select_list)); + aggregate->groupings_index = optimizer.binder.GenerateTableIndex(); + aggregate->groups = std::move(window_expr.partitions); + aggregate->children.push_back(std::move(window.children[0])); + aggregate->ResolveOperatorTypes(); + + return unique_ptr(std::move(aggregate)); +} + +unique_ptr +TopNWindowElimination::CreateRowNumberGenerator(unique_ptr aggregate_column_ref) const { + // Create unnest(generate_series(1, array_length(column_ref, 1))) function to generate row ids + FunctionBinder function_binder(context); + auto &catalog = Catalog::GetSystemCatalog(context); + + // array_length + auto &array_length_entry = catalog.GetEntry(context, DEFAULT_SCHEMA, "array_length"); + vector> array_length_exprs; + array_length_exprs.push_back(std::move(aggregate_column_ref)); + array_length_exprs.push_back(make_uniq(1)); + + const auto array_length_fun = array_length_entry.functions.GetFunctionByArguments( + context, {array_length_exprs[0]->return_type, array_length_exprs[1]->return_type}); + auto bound_array_length_fun = function_binder.BindScalarFunction(array_length_fun, std::move(array_length_exprs)); + + // generate_series + auto &generate_series_entry = + catalog.GetEntry(context, DEFAULT_SCHEMA, "generate_series"); + + vector> generate_series_exprs; + generate_series_exprs.push_back(make_uniq(1)); + generate_series_exprs.push_back(std::move(bound_array_length_fun)); + + const auto generate_series_fun = generate_series_entry.functions.GetFunctionByArguments( + context, {generate_series_exprs[0]->return_type, generate_series_exprs[1]->return_type}); + auto bound_generate_series_fun = + function_binder.BindScalarFunction(generate_series_fun, std::move(generate_series_exprs)); + + // unnest + auto unnest_row_number_expr = make_uniq(LogicalType::BIGINT); + unnest_row_number_expr->alias = "row_number"; + unnest_row_number_expr->child = std::move(bound_generate_series_fun); + + return unique_ptr(std::move(unnest_row_number_expr)); +} + +unique_ptr +TopNWindowElimination::TryCreateUnnestOperator(unique_ptr op, + const TopNWindowEliminationParameters ¶ms) const { + D_ASSERT(op->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY); + + auto &logical_aggregate = op->Cast(); + const idx_t aggregate_column_idx = logical_aggregate.groups.size(); + LogicalType aggregate_type = logical_aggregate.types[aggregate_column_idx]; + + if (params.limit <= 1) { + // LIMIT 1 -> we do not need to unnest + return std::move(op); + } + + // Create unnest expression for aggregate args + const auto aggregate_bindings = logical_aggregate.GetColumnBindings(); + auto aggregate_column_ref = + make_uniq(aggregate_type, aggregate_bindings[aggregate_column_idx]); + + vector> unnest_exprs; + + auto unnest_aggregate = make_uniq(ListType::GetChildType(aggregate_type)); + unnest_aggregate->child = aggregate_column_ref->Copy(); + unnest_exprs.push_back(std::move(unnest_aggregate)); + + if (params.include_row_number) { + // Create row number expression + unnest_exprs.push_back(CreateRowNumberGenerator(std::move(aggregate_column_ref))); + } + + auto unnest = make_uniq(optimizer.binder.GenerateTableIndex()); + unnest->expressions = std::move(unnest_exprs); + unnest->children.push_back(std::move(op)); + unnest->ResolveOperatorTypes(); + + return unique_ptr(std::move(unnest)); +} + +void TopNWindowElimination::AddStructExtractExprs( + vector> &exprs, const LogicalType &struct_type, + const unique_ptr &aggregate_column_ref) const { + FunctionBinder function_binder(context); + auto &catalog = Catalog::GetSystemCatalog(context); + auto &struct_extract_entry = + catalog.GetEntry(context, DEFAULT_SCHEMA, "struct_extract"); + const auto struct_extract_fun = + struct_extract_entry.functions.GetFunctionByArguments(context, {struct_type, LogicalType::VARCHAR}); + + const auto &child_types = StructType::GetChildTypes(struct_type); + for (idx_t i = 0; i < child_types.size(); i++) { + const auto &alias = child_types[i].first; + + vector> fun_args(2); + fun_args[0] = aggregate_column_ref->Copy(); + fun_args[1] = make_uniq(alias); + + auto bound_function = function_binder.BindScalarFunction(struct_extract_fun, std::move(fun_args)); + bound_function->alias = alias; + exprs.push_back(std::move(bound_function)); + } +} + +unique_ptr +TopNWindowElimination::CreateProjectionOperator(unique_ptr op, + const TopNWindowEliminationParameters ¶ms, + const map &group_idxs) const { + const auto aggregate_type = GetAggregateType(op); + const idx_t aggregate_table_idx = GetAggregateIdx(op); + const auto op_column_bindings = op->GetColumnBindings(); + + vector> proj_exprs; + // Only project necessary group columns + for (const auto &group_idx : group_idxs) { + proj_exprs.push_back( + make_uniq(op->types[group_idx.second], op_column_bindings[group_idx.second])); + } + + auto aggregate_column_ref = + make_uniq(aggregate_type, ColumnBinding(aggregate_table_idx, 0)); + + if (params.payload_type == TopNPayloadType::STRUCT_PACK) { + AddStructExtractExprs(proj_exprs, aggregate_type, aggregate_column_ref); + } else { + // No need for struct_unpack! Just reference the aggregate column + proj_exprs.push_back(std::move(aggregate_column_ref)); + } + + if (params.include_row_number) { + // If aggregate (i.e., limit 1): constant, if unnest: expect there to be a second column + if (op->type == LogicalOperatorType::LOGICAL_UNNEST) { + const idx_t row_number_offset = op->children[0]->types.size() + 1; + D_ASSERT(op->types.size() == row_number_offset + 1); // Row number should have been generated previously + proj_exprs.push_back(make_uniq(op->types[row_number_offset], + op_column_bindings[row_number_offset])); + } else { + proj_exprs.push_back(make_uniq(Value::BIGINT(1))); + } + } + + auto logical_projection = + make_uniq(optimizer.binder.GenerateTableIndex(), std::move(proj_exprs)); + logical_projection->children.push_back(std::move(op)); + logical_projection->ResolveOperatorTypes(); + + return unique_ptr(std::move(logical_projection)); +} + +bool TopNWindowElimination::CanOptimize(LogicalOperator &op) { + if (op.type != LogicalOperatorType::LOGICAL_FILTER) { + return false; + } + + const auto &filter = op.Cast(); + if (filter.expressions.size() != 1) { + return false; + } + + if (filter.expressions[0]->type != ExpressionType::COMPARE_LESSTHANOREQUALTO) { + return false; + } + + auto &filter_comparison = filter.expressions[0]->Cast(); + if (filter_comparison.right->type != ExpressionType::VALUE_CONSTANT) { + return false; + } + auto &filter_value = filter_comparison.right->Cast(); + if (filter_value.value.type() != LogicalType::BIGINT) { + return false; + } + if (filter_value.value.GetValue() < 1) { + return false; + } + + if (filter_comparison.left->type != ExpressionType::BOUND_COLUMN_REF) { + return false; + } + VisitExpression(&filter_comparison.left); + + auto *child = filter.children[0].get(); + while (child->type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &projection = child->Cast(); + if (column_references.size() != 1) { + column_references.clear(); + return false; + } + + const auto current_column_ref = column_references.begin()->first; + column_references.clear(); + D_ASSERT(current_column_ref.table_index == projection.table_index); + VisitExpression(&projection.expressions[current_column_ref.column_index]); + + child = child->children[0].get(); + } + + if (column_references.size() != 1) { + column_references.clear(); + return false; + } + const auto filter_col_idx = column_references.begin()->first.table_index; + column_references.clear(); + + if (child->type != LogicalOperatorType::LOGICAL_WINDOW) { + return false; + } + const auto &window = child->Cast(); + if (window.window_index != filter_col_idx) { + return false; + } + if (window.expressions.size() != 1) { + for (idx_t i = 1; i < window.expressions.size(); ++i) { + if (!window.expressions[i]->Equals(*window.expressions[0])) { + return false; + } + } + } + if (window.expressions[0]->type != ExpressionType::WINDOW_ROW_NUMBER) { + return false; + } + auto &window_expr = window.expressions[0]->Cast(); + + if (window_expr.orders.size() != 1) { + return false; + } + if (window_expr.orders[0].type != OrderType::DESCENDING && window_expr.orders[0].type != OrderType::ASCENDING) { + return false; + } + if (window_expr.orders[0].null_order != OrderByNullType::NULLS_LAST) { + return false; + } + + // We have found a grouped top-n window construct! + return true; +} + +vector> TopNWindowElimination::GenerateAggregatePayload(const vector &bindings, + const LogicalWindow &window, + map &group_idxs) { + vector> aggregate_args; + aggregate_args.reserve(bindings.size()); + + window.children[0]->ResolveOperatorTypes(); + const auto &window_child_types = window.children[0]->types; + const auto window_child_bindings = window.children[0]->GetColumnBindings(); + auto &window_expr = window.expressions[0]->Cast(); + + // Remember order of group columns to recreate that order in new bindings later + column_binding_map_t group_bindings; + for (idx_t i = 0; i < window_expr.partitions.size(); i++) { + auto &expr = window_expr.partitions[i]; + VisitExpression(&expr); + group_bindings[column_references.begin()->first] = i; + column_references.clear(); + } + + for (idx_t i = 0; i < bindings.size(); i++) { + const auto &binding = bindings[i]; + const auto group_binding = group_bindings.find(binding); + if (group_binding != group_bindings.end()) { + group_idxs[i] = group_binding->second; + continue; + } + if (binding.table_index == window.window_index) { + continue; + } + auto column_id = to_string(binding.column_index); // Use idx as struct pack/extract identifier + auto column_type = window_child_types[binding.column_index]; + const auto &column_binding = window_child_bindings[binding.column_index]; + + aggregate_args.push_back(make_uniq(column_id, column_type, column_binding)); + } + + if (aggregate_args.size() == 1) { + // If we only project the aggregate value itself, we do not need it as an arg + VisitExpression(&window_expr.orders[0].expression); + const auto aggregate_value_binding = column_references.begin()->first; + column_references.clear(); + + if (window_expr.orders[0].expression->type == ExpressionType::BOUND_COLUMN_REF && + aggregate_args[0]->Cast().binding == aggregate_value_binding) { + return {}; + } + } + + return aggregate_args; +} + +vector TopNWindowElimination::TraverseProjectionBindings(const std::vector &old_bindings, + LogicalOperator *&op) { + auto new_bindings = old_bindings; + + // Traverse child projections to retrieve projections on window output + while (op->type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &projection = op->Cast(); + + for (idx_t i = 0; i < new_bindings.size(); i++) { + auto &new_binding = new_bindings[i]; + D_ASSERT(new_binding.table_index == projection.table_index); + VisitExpression(&projection.expressions[new_binding.column_index]); + new_binding = column_references.begin()->first; + column_references.clear(); + } + op = op->children[0].get(); + } + + return new_bindings; +} + +void TopNWindowElimination::UpdateTopmostBindings(const idx_t window_idx, const unique_ptr &op, + const map &group_idxs, + const vector &topmost_bindings, + vector &new_bindings, + ColumnBindingReplacer &replacer) { + // The top-most operator's column order is [group][aggregate args][row number]. Now, set the new resulting bindings. + D_ASSERT(topmost_bindings.size() == new_bindings.size()); + replacer.replacement_bindings.reserve(new_bindings.size()); + set row_id_binding_idxs; + + const idx_t group_table_idx = GetGroupIdx(op); + const idx_t aggregate_table_idx = GetAggregateIdx(op); + + // Project the group columns + idx_t current_column_idx = 0; + for (auto group_idx : group_idxs) { + const idx_t group_referencing_idx = group_idx.first; + new_bindings[group_referencing_idx].table_index = group_table_idx; + new_bindings[group_referencing_idx].column_index = group_idx.second; + replacer.replacement_bindings.emplace_back(topmost_bindings[group_referencing_idx], + new_bindings[group_referencing_idx]); + current_column_idx++; + } + + if (group_table_idx != aggregate_table_idx) { + // If the topmost operator is not a projection, the table indexes are different, and we start back from 0 + current_column_idx = 0; + } + + // Project the args/value + for (idx_t i = 0; i < new_bindings.size(); i++) { + auto &binding = new_bindings[i]; + if (group_idxs.find(i) != group_idxs.end()) { + continue; + } + if (binding.table_index == window_idx) { + row_id_binding_idxs.insert(i); + continue; + } + binding.column_index = current_column_idx++; + binding.table_index = aggregate_table_idx; + replacer.replacement_bindings.emplace_back(topmost_bindings[i], binding); + } + + // Project the row number + for (const auto row_id_binding_idx : row_id_binding_idxs) { + // Let all projections on row id point to the last output column + auto &binding = new_bindings[row_id_binding_idx]; + binding.table_index = aggregate_table_idx; + binding.column_index = op->types.size() - 1; + replacer.replacement_bindings.emplace_back(topmost_bindings[row_id_binding_idx], binding); + } +} + +TopNWindowEliminationParameters +TopNWindowElimination::ExtractOptimizerParameters(const LogicalWindow &window, const LogicalFilter &filter, + const vector &bindings, + vector> &aggregate_payload) { + TopNWindowEliminationParameters params; + + auto &limit_expr = filter.expressions[0]->Cast().right; + params.limit = limit_expr->Cast().value.GetValue(); + params.include_row_number = BindingsReferenceRowNumber(bindings, window); + params.payload_type = aggregate_payload.size() > 1 ? TopNPayloadType::STRUCT_PACK : TopNPayloadType::SINGLE_COLUMN; + auto &window_expr = window.expressions[0]->Cast(); + params.order_type = window_expr.orders[0].type; + + VisitExpression(&window_expr.orders[0].expression); + if (params.payload_type == TopNPayloadType::SINGLE_COLUMN && !aggregate_payload.empty()) { + VisitExpression(&aggregate_payload[0]); + } + for (const auto &column_ref : column_references) { + const auto &column_stats = stats->find(column_ref.first); + if (column_stats == stats->end() || column_stats->second->CanHaveNull()) { + params.can_be_null = true; + } + } + column_references.clear(); + + return params; +} + +} // namespace duckdb diff --git a/src/parallel/task_executor.cpp b/src/parallel/task_executor.cpp index fa2c0087c3eb..9487a1427624 100644 --- a/src/parallel/task_executor.cpp +++ b/src/parallel/task_executor.cpp @@ -69,8 +69,10 @@ TaskExecutionResult BaseExecutorTask::Execute(TaskExecutionMode mode) { return TaskExecutionResult::TASK_FINISHED; } try { - TaskNotifier task_notifier {executor.context}; - ExecuteTask(); + { + TaskNotifier task_notifier {executor.context}; + ExecuteTask(); + } executor.FinishTask(); return TaskExecutionResult::TASK_FINISHED; } catch (std::exception &ex) { diff --git a/src/parser/expression/lambdaref_expression.cpp b/src/parser/expression/lambdaref_expression.cpp index fed844feae15..e71debfc0766 100644 --- a/src/parser/expression/lambdaref_expression.cpp +++ b/src/parser/expression/lambdaref_expression.cpp @@ -47,7 +47,7 @@ LambdaRefExpression::FindMatchingBinding(optional_ptr> &lam if (lambda_bindings) { for (idx_t i = lambda_bindings->size(); i > 0; i--) { if ((*lambda_bindings)[i - 1].HasMatchingBinding(column_name)) { - D_ASSERT((*lambda_bindings)[i - 1].alias.IsSet()); + D_ASSERT((*lambda_bindings)[i - 1].GetBindingAlias().IsSet()); return make_uniq(i - 1, column_name); } } diff --git a/src/parser/parsed_data/CMakeLists.txt b/src/parser/parsed_data/CMakeLists.txt index aaff3442b847..4d7bc11f5f19 100644 --- a/src/parser/parsed_data/CMakeLists.txt +++ b/src/parser/parsed_data/CMakeLists.txt @@ -5,6 +5,7 @@ add_library_unity( alter_scalar_function_info.cpp alter_table_function_info.cpp alter_table_info.cpp + alter_database_info.cpp attach_info.cpp comment_on_column_info.cpp copy_info.cpp diff --git a/src/parser/parsed_data/alter_database_info.cpp b/src/parser/parsed_data/alter_database_info.cpp new file mode 100644 index 000000000000..e737269862a1 --- /dev/null +++ b/src/parser/parsed_data/alter_database_info.cpp @@ -0,0 +1,46 @@ +#include "duckdb/parser/parsed_data/alter_database_info.hpp" +#include "duckdb/common/serializer/serializer.hpp" + +namespace duckdb { + +AlterDatabaseInfo::AlterDatabaseInfo(AlterDatabaseType alter_database_type) + : AlterInfo(AlterType::ALTER_DATABASE, string(), "", "", OnEntryNotFound::THROW_EXCEPTION), + alter_database_type(alter_database_type) { +} + +AlterDatabaseInfo::AlterDatabaseInfo(AlterDatabaseType alter_database_type, string catalog_p, + OnEntryNotFound if_not_found) + : AlterInfo(AlterType::ALTER_DATABASE, std::move(catalog_p), "", "", if_not_found), + alter_database_type(alter_database_type) { +} + +AlterDatabaseInfo::~AlterDatabaseInfo() { +} + +CatalogType AlterDatabaseInfo::GetCatalogType() const { + return CatalogType::DATABASE_ENTRY; +} + +RenameDatabaseInfo::RenameDatabaseInfo() : AlterDatabaseInfo(AlterDatabaseType::RENAME_DATABASE) { +} + +RenameDatabaseInfo::RenameDatabaseInfo(string catalog_p, string new_name_p, OnEntryNotFound if_not_found) + : AlterDatabaseInfo(AlterDatabaseType::RENAME_DATABASE, std::move(catalog_p), if_not_found), + new_name(std::move(new_name_p)) { +} + +unique_ptr RenameDatabaseInfo::Copy() const { + return make_uniq(catalog, new_name, if_not_found); +} + +string RenameDatabaseInfo::ToString() const { + string result; + result = "ALTER DATABASE "; + if (if_not_found == OnEntryNotFound::RETURN_NULL) { + result += "IF EXISTS "; + } + result += StringUtil::Format("%s RENAME TO %s", SQLIdentifier(catalog), SQLIdentifier(new_name)); + return result; +} + +} // namespace duckdb diff --git a/src/parser/parsed_expression_iterator.cpp b/src/parser/parsed_expression_iterator.cpp index 24bfd8e8a2bb..47e39dc76239 100644 --- a/src/parser/parsed_expression_iterator.cpp +++ b/src/parser/parsed_expression_iterator.cpp @@ -271,12 +271,6 @@ void ParsedExpressionIterator::EnumerateQueryNodeChildren( EnumerateQueryNodeChildren(*rcte_node.right, expr_callback, ref_callback); break; } - case QueryNodeType::CTE_NODE: { - auto &cte_node = node.Cast(); - EnumerateQueryNodeChildren(*cte_node.query, expr_callback, ref_callback); - EnumerateQueryNodeChildren(*cte_node.child, expr_callback, ref_callback); - break; - } case QueryNodeType::SELECT_NODE: { auto &sel_node = node.Cast(); for (idx_t i = 0; i < sel_node.select_list.size(); i++) { @@ -300,8 +294,9 @@ void ParsedExpressionIterator::EnumerateQueryNodeChildren( } case QueryNodeType::SET_OPERATION_NODE: { auto &setop_node = node.Cast(); - EnumerateQueryNodeChildren(*setop_node.left, expr_callback, ref_callback); - EnumerateQueryNodeChildren(*setop_node.right, expr_callback, ref_callback); + for (auto &child : setop_node.children) { + EnumerateQueryNodeChildren(*child, expr_callback, ref_callback); + } break; } default: diff --git a/src/parser/parser.cpp b/src/parser/parser.cpp index f931f0671375..8fff8df326e6 100644 --- a/src/parser/parser.cpp +++ b/src/parser/parser.cpp @@ -165,33 +165,36 @@ bool Parser::StripUnicodeSpaces(const string &query_str, string &new_query) { return ReplaceUnicodeSpaces(query_str, new_query, unicode_spaces); } -vector SplitQueryStringIntoStatements(const string &query) { - // Break sql string down into sql statements using the tokenizer - vector query_statements; - auto tokens = Parser::Tokenize(query); - idx_t next_statement_start = 0; - for (idx_t i = 1; i < tokens.size(); ++i) { - auto &t_prev = tokens[i - 1]; - auto &t = tokens[i]; - if (t_prev.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_OPERATOR) { - // LCOV_EXCL_START - for (idx_t c = t_prev.start; c <= t.start; ++c) { - if (query.c_str()[c] == ';') { - query_statements.emplace_back(query.substr(next_statement_start, t.start - next_statement_start)); - next_statement_start = tokens[i].start; - } +vector SplitQueries(const string &input_query) { + vector queries; + auto tokenized_input = Parser::Tokenize(input_query); + size_t last_split = 0; + + for (const auto &token : tokenized_input) { + if (token.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_OPERATOR && input_query[token.start] == ';') { + string segment = input_query.substr(last_split, token.start - last_split); + StringUtil::Trim(segment); + if (!segment.empty()) { + segment.append(";"); + queries.push_back(std::move(segment)); } - // LCOV_EXCL_STOP + last_split = token.start + 1; } } - query_statements.emplace_back(query.substr(next_statement_start, query.size() - next_statement_start)); - return query_statements; + string final_segment = input_query.substr(last_split); + StringUtil::Trim(final_segment); + if (!final_segment.empty()) { + final_segment.append(";"); + queries.push_back(std::move(final_segment)); + } + return queries; } void Parser::ParseQuery(const string &query) { Transformer transformer(options); string parser_error; optional_idx parser_error_location; + string parser_override_option = StringUtil::Lower(options.parser_override_setting); { // check if there are any unicode spaces in the string string new_query; @@ -202,6 +205,45 @@ void Parser::ParseQuery(const string &query) { } } { + if (options.extensions) { + for (auto &ext : *options.extensions) { + if (!ext.parser_override) { + continue; + } + if (StringUtil::CIEquals(parser_override_option, "default")) { + continue; + } + auto result = ext.parser_override(ext.parser_info.get(), query); + if (result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) { + statements = std::move(result.statements); + return; + } + if (StringUtil::CIEquals(parser_override_option, "strict")) { + if (result.type == ParserExtensionResultType::DISPLAY_ORIGINAL_ERROR) { + throw ParserException( + "Parser override failed to return a valid statement: %s\n\nConsider restarting the " + "database and " + "using the setting \"set allow_parser_override_extension=fallback\" to fallback to the " + "default parser.", + result.error.RawMessage()); + } + if (result.type == ParserExtensionResultType::DISPLAY_EXTENSION_ERROR) { + if (result.error.Type() == ExceptionType::NOT_IMPLEMENTED) { + throw NotImplementedException( + "Parser override has not yet implemented this transformer rule. (Original error: %s)", + result.error.RawMessage()); + } else if (result.error.Type() == ExceptionType::PARSER) { + throw ParserException("Parser override could not parse this query. (Original error: %s)", + result.error.RawMessage()); + } else { + result.error.Throw(); + } + } + } else if (StringUtil::CIEquals(parser_override_option, "fallback")) { + continue; + } + } + } PostgresParser::SetPreserveIdentifierCase(options.preserve_identifier_case); bool parsing_succeed = false; // Creating a new scope to prevent multiple PostgresParser destructors being called @@ -236,9 +278,9 @@ void Parser::ParseQuery(const string &query) { throw ParserException::SyntaxError(query, parser_error, parser_error_location); } else { // split sql string into statements and re-parse using extension - auto query_statements = SplitQueryStringIntoStatements(query); + auto queries = SplitQueries(query); idx_t stmt_loc = 0; - for (auto const &query_statement : query_statements) { + for (auto const &query_statement : queries) { ErrorData another_parser_error; // Creating a new scope to allow extensions to use PostgresParser, which is not reentrant { @@ -270,7 +312,9 @@ void Parser::ParseQuery(const string &query) { bool parsed_single_statement = false; for (auto &ext : *options.extensions) { D_ASSERT(!parsed_single_statement); - D_ASSERT(ext.parse_function); + if (!ext.parse_function) { + continue; + } auto result = ext.parse_function(ext.parser_info.get(), query_statement); if (result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) { auto statement = make_uniq(ext, std::move(result.parse_data)); diff --git a/src/parser/query_node/CMakeLists.txt b/src/parser/query_node/CMakeLists.txt index 9129b41486b1..45b6a23de62c 100644 --- a/src/parser/query_node/CMakeLists.txt +++ b/src/parser/query_node/CMakeLists.txt @@ -1,5 +1,11 @@ -add_library_unity(duckdb_query_node OBJECT recursive_cte_node.cpp cte_node.cpp - select_node.cpp set_operation_node.cpp) +add_library_unity( + duckdb_query_node + OBJECT + recursive_cte_node.cpp + cte_node.cpp + select_node.cpp + set_operation_node.cpp + statement_node.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/parser/query_node/cte_node.cpp b/src/parser/query_node/cte_node.cpp index 1e1f0e19909d..29c0599a52ed 100644 --- a/src/parser/query_node/cte_node.cpp +++ b/src/parser/query_node/cte_node.cpp @@ -1,42 +1,20 @@ #include "duckdb/parser/query_node/cte_node.hpp" #include "duckdb/common/serializer/serializer.hpp" #include "duckdb/common/serializer/deserializer.hpp" +#include "duckdb/parser/statement/select_statement.hpp" namespace duckdb { string CTENode::ToString() const { - string result; - result += child->ToString(); - return result; + throw InternalException("CTENode is a legacy type"); } bool CTENode::Equals(const QueryNode *other_p) const { - if (!QueryNode::Equals(other_p)) { - return false; - } - if (this == other_p) { - return true; - } - auto &other = other_p->Cast(); - - if (!query->Equals(other.query.get())) { - return false; - } - if (!child->Equals(other.child.get())) { - return false; - } - return true; + throw InternalException("CTENode is a legacy type"); } unique_ptr CTENode::Copy() const { - auto result = make_uniq(); - result->ctename = ctename; - result->query = query->Copy(); - result->child = child->Copy(); - result->aliases = aliases; - result->materialized = materialized; - this->CopyProperties(*result); - return std::move(result); + throw InternalException("CTENode is a legacy type"); } } // namespace duckdb diff --git a/src/parser/query_node/set_operation_node.cpp b/src/parser/query_node/set_operation_node.cpp index 9a8168a45aba..cdc188820dab 100644 --- a/src/parser/query_node/set_operation_node.cpp +++ b/src/parser/query_node/set_operation_node.cpp @@ -11,25 +11,27 @@ SetOperationNode::SetOperationNode() : QueryNode(QueryNodeType::SET_OPERATION_NO string SetOperationNode::ToString() const { string result; result = cte_map.ToString(); - result += "(" + left->ToString() + ") "; + result += "(" + children[0]->ToString() + ") "; - switch (setop_type) { - case SetOperationType::UNION: - result += setop_all ? "UNION ALL" : "UNION"; - break; - case SetOperationType::UNION_BY_NAME: - result += setop_all ? "UNION ALL BY NAME" : "UNION BY NAME"; - break; - case SetOperationType::EXCEPT: - result += setop_all ? "EXCEPT ALL" : "EXCEPT"; - break; - case SetOperationType::INTERSECT: - result += setop_all ? "INTERSECT ALL" : "INTERSECT"; - break; - default: - throw InternalException("Unsupported set operation type"); + for (idx_t i = 1; i < children.size(); i++) { + switch (setop_type) { + case SetOperationType::UNION: + result += setop_all ? "UNION ALL" : "UNION"; + break; + case SetOperationType::UNION_BY_NAME: + result += setop_all ? "UNION ALL BY NAME" : "UNION BY NAME"; + break; + case SetOperationType::EXCEPT: + result += setop_all ? "EXCEPT ALL" : "EXCEPT"; + break; + case SetOperationType::INTERSECT: + result += setop_all ? "INTERSECT ALL" : "INTERSECT"; + break; + default: + throw InternalException("Unsupported set operation type"); + } + result += " (" + children[i]->ToString() + ")"; } - result += " (" + right->ToString() + ")"; return result + ResultModifiersToString(); } @@ -47,11 +49,13 @@ bool SetOperationNode::Equals(const QueryNode *other_p) const { if (setop_all != other.setop_all) { return false; } - if (!left->Equals(other.left.get())) { + if (children.size() != other.children.size()) { return false; } - if (!right->Equals(other.right.get())) { - return false; + for (idx_t i = 0; i < children.size(); i++) { + if (!children[i]->Equals(other.children[i].get())) { + return false; + } } return true; } @@ -60,56 +64,72 @@ unique_ptr SetOperationNode::Copy() const { auto result = make_uniq(); result->setop_type = setop_type; result->setop_all = setop_all; - result->left = left->Copy(); - result->right = right->Copy(); + for (auto &child : children) { + result->children.push_back(child->Copy()); + } this->CopyProperties(*result); return std::move(result); } SetOperationNode::SetOperationNode(SetOperationType setop_type, unique_ptr left, unique_ptr right, - vector> children, bool setop_all) + vector> children_p, bool setop_all) : QueryNode(QueryNodeType::SET_OPERATION_NODE), setop_type(setop_type), setop_all(setop_all) { - if (left && right) { - // simple case - left/right are supplied - this->left = std::move(left); - this->right = std::move(right); - return; + if (children_p.empty()) { + if (!left || !right) { + throw SerializationException("Error deserializing SetOperationNode - left/right or children must be set"); + } + children.push_back(std::move(left)); + children.push_back(std::move(right)); + } else { + if (left || right) { + throw SerializationException("Error deserializing SetOperationNode - left/right or children must be set"); + } + children = std::move(children_p); + } + if (children.size() < 2) { + throw SerializationException("SetOperationNode must have at least two children"); + } +} + +unique_ptr SetOperationNode::SerializeChildNode(Serializer &serializer, idx_t index) const { + if (SerializeChildList(serializer)) { + // serialize new version - we are serializing all children in the new "children" field + return nullptr; } - if (children.size() == 2) { - this->left = std::move(children[0]); - this->right = std::move(children[1]); + // backwards compatibility - we are targeting an older version + // we need to serialize two children - "left" and "right" + if (index == 0) { + // for the left child, just directly emit the first child + return children[0]->Copy(); } - // we have multiple children - we need to construct a tree of set operation nodes - if (children.size() <= 1) { - throw SerializationException("Set Operation requires at least 2 children"); + if (index != 1) { + throw InternalException("SerializeChildNode should have index 0 or 1"); } - if (setop_type != SetOperationType::UNION) { - throw SerializationException("Multiple children in set-operations are only supported for UNION"); + vector> nodes; + for (idx_t i = 1; i < children.size(); i++) { + nodes.push_back(children[i]->Copy()); } - // construct a balanced tree from the union - while (children.size() > 2) { + // for the right child we construct a new tree by generating the set operation over all of the nodes + // we construct a balanced tree to avoid + while (nodes.size() > 1) { vector> new_children; - for (idx_t i = 0; i < children.size(); i += 2) { - if (i + 1 == children.size()) { - new_children.push_back(std::move(children[i])); + for (idx_t i = 0; i < nodes.size(); i += 2) { + if (i + 1 == nodes.size()) { + new_children.push_back(std::move(nodes[i])); } else { vector> empty_children; - auto setop_node = - make_uniq(setop_type, std::move(children[i]), std::move(children[i + 1]), - std::move(empty_children), setop_all); + auto setop_node = make_uniq(setop_type, std::move(nodes[i]), std::move(nodes[i + 1]), + std::move(empty_children), setop_all); new_children.push_back(std::move(setop_node)); } } - children = std::move(new_children); + nodes = std::move(new_children); } - // two children left - fill in the left/right of this node - this->left = std::move(children[0]); - this->right = std::move(children[1]); + return std::move(nodes[0]); } -vector> SetOperationNode::SerializeChildNodes() const { - // we always serialize children as left/right currently - return vector>(); +bool SetOperationNode::SerializeChildList(Serializer &serializer) const { + return serializer.ShouldSerialize(6); } } // namespace duckdb diff --git a/src/parser/query_node/statement_node.cpp b/src/parser/query_node/statement_node.cpp new file mode 100644 index 000000000000..66e7b8e5a602 --- /dev/null +++ b/src/parser/query_node/statement_node.cpp @@ -0,0 +1,40 @@ +#include "duckdb/parser/query_node/statement_node.hpp" + +namespace duckdb { + +StatementNode::StatementNode(SQLStatement &stmt_p) : QueryNode(QueryNodeType::STATEMENT_NODE), stmt(stmt_p) { +} + +//! Convert the query node to a string +string StatementNode::ToString() const { + return stmt.ToString(); +} + +bool StatementNode::Equals(const QueryNode *other_p) const { + if (!QueryNode::Equals(other_p)) { + return false; + } + if (this == other_p) { + return true; + } + auto &other = other_p->Cast(); + return RefersToSameObject(stmt, other.stmt); +} + +//! Create a copy of this SelectNode +unique_ptr StatementNode::Copy() const { + return make_uniq(stmt); +} + +//! Serializes a QueryNode to a stand-alone binary blob +//! Deserializes a blob back into a QueryNode + +void StatementNode::Serialize(Serializer &serializer) const { + throw InternalException("StatementNode cannot be serialized"); +} + +unique_ptr StatementNode::Deserialize(Deserializer &source) { + throw InternalException("StatementNode cannot be deserialized"); +} + +} // namespace duckdb diff --git a/src/parser/statement/relation_statement.cpp b/src/parser/statement/relation_statement.cpp index 9b3801495447..023d3cac920c 100644 --- a/src/parser/statement/relation_statement.cpp +++ b/src/parser/statement/relation_statement.cpp @@ -5,10 +5,7 @@ namespace duckdb { RelationStatement::RelationStatement(shared_ptr relation_p) : SQLStatement(StatementType::RELATION_STATEMENT), relation(std::move(relation_p)) { - if (relation->type == RelationType::QUERY_RELATION) { - auto &query_relation = relation->Cast(); - query = query_relation.query; - } + query = relation->GetQuery(); } unique_ptr RelationStatement::Copy() const { diff --git a/src/parser/tableref/bound_ref_wrapper.cpp b/src/parser/tableref/bound_ref_wrapper.cpp index a2ecb50864c9..6bf31269c360 100644 --- a/src/parser/tableref/bound_ref_wrapper.cpp +++ b/src/parser/tableref/bound_ref_wrapper.cpp @@ -2,7 +2,7 @@ namespace duckdb { -BoundRefWrapper::BoundRefWrapper(unique_ptr bound_ref_p, shared_ptr binder_p) +BoundRefWrapper::BoundRefWrapper(BoundStatement bound_ref_p, shared_ptr binder_p) : TableRef(TableReferenceType::BOUND_TABLE_REF), bound_ref(std::move(bound_ref_p)), binder(std::move(binder_p)) { } diff --git a/src/parser/transform/expression/transform_subquery.cpp b/src/parser/transform/expression/transform_subquery.cpp index bc8a9762dc51..986e46e25eaa 100644 --- a/src/parser/transform/expression/transform_subquery.cpp +++ b/src/parser/transform/expression/transform_subquery.cpp @@ -24,7 +24,6 @@ unique_ptr Transformer::TransformSubquery(duckdb_libpgquery::P subquery_expr->subquery = TransformSelectStmt(*root.subselect); SetQueryLocation(*subquery_expr, root.location); D_ASSERT(subquery_expr->subquery); - D_ASSERT(!subquery_expr->subquery->node->GetSelectList().empty()); switch (root.subLinkType) { case duckdb_libpgquery::PG_EXISTS_SUBLINK: { diff --git a/src/parser/transform/helpers/transform_cte.cpp b/src/parser/transform/helpers/transform_cte.cpp index d505804ab08c..f5f232fc3ff1 100644 --- a/src/parser/transform/helpers/transform_cte.cpp +++ b/src/parser/transform/helpers/transform_cte.cpp @@ -23,9 +23,16 @@ unique_ptr CommonTableExpressionInfo::Copy() { CommonTableExpressionInfo::~CommonTableExpressionInfo() { } +CTEMaterialize CommonTableExpressionInfo::GetMaterializedForSerialization(Serializer &serializer) const { + if (serializer.ShouldSerialize(7)) { + return materialized; + } + return CTEMaterialize::CTE_MATERIALIZE_DEFAULT; +} + void Transformer::ExtractCTEsRecursive(CommonTableExpressionMap &cte_map) { for (auto &cte_entry : stored_cte_map) { - for (auto &entry : cte_entry->map) { + for (auto &entry : cte_entry.get().map) { auto found_entry = cte_map.map.find(entry.first); if (found_entry != cte_map.map.end()) { // entry already present - use top-most entry @@ -40,7 +47,7 @@ void Transformer::ExtractCTEsRecursive(CommonTableExpressionMap &cte_map) { } void Transformer::TransformCTE(duckdb_libpgquery::PGWithClause &de_with_clause, CommonTableExpressionMap &cte_map) { - stored_cte_map.push_back(&cte_map); + stored_cte_map.push_back(cte_map); // TODO: might need to update in case of future lawsuit D_ASSERT(de_with_clause.ctes); @@ -76,7 +83,7 @@ void Transformer::TransformCTE(duckdb_libpgquery::PGWithClause &de_with_clause, } // we need a query if (!cte.ctequery || cte.ctequery->type != duckdb_libpgquery::T_PGSelectStmt) { - throw NotImplementedException("A CTE needs a SELECT"); + throw ParserException("A CTE needs a SELECT"); } // CTE transformation can either result in inlining for non recursive CTEs, or in recursive CTE bindings diff --git a/src/parser/transform/statement/CMakeLists.txt b/src/parser/transform/statement/CMakeLists.txt index e1bd47e4a502..c01ed4e1d3d8 100644 --- a/src/parser/transform/statement/CMakeLists.txt +++ b/src/parser/transform/statement/CMakeLists.txt @@ -2,6 +2,7 @@ add_library_unity( duckdb_transformer_statement OBJECT transform_alter_table.cpp + transform_alter_database.cpp transform_attach.cpp transform_detach.cpp transform_call.cpp diff --git a/src/parser/transform/statement/transform_alter_database.cpp b/src/parser/transform/statement/transform_alter_database.cpp new file mode 100644 index 000000000000..149b6f7a5eed --- /dev/null +++ b/src/parser/transform/statement/transform_alter_database.cpp @@ -0,0 +1,36 @@ +#include "duckdb/parser/statement/alter_statement.hpp" +#include "duckdb/parser/parsed_data/alter_database_info.hpp" +#include "duckdb/parser/transformer.hpp" + +namespace duckdb { + +unique_ptr Transformer::TransformAlterDatabase(duckdb_libpgquery::PGAlterDatabaseStmt &stmt) { + auto result = make_uniq(); + + auto database_name = stmt.dbname ? string(stmt.dbname) : string(); + if (database_name.empty()) { + throw ParserException("ALTER DATABASE requires a database name"); + } + + OnEntryNotFound if_not_found = OnEntryNotFound::THROW_EXCEPTION; + if (stmt.missing_ok) { + if_not_found = OnEntryNotFound::RETURN_NULL; + } + + switch (stmt.alter_type) { + case duckdb_libpgquery::PG_ALTER_DATABASE_RENAME: { + if (!stmt.new_name) { + throw ParserException("ALTER DATABASE RENAME requires a new name"); + } + auto info = make_uniq(database_name, string(stmt.new_name), if_not_found); + result->info = std::move(info); + break; + } + default: + throw ParserException("Unsupported ALTER DATABASE operation"); + } + + return result; +} + +} // namespace duckdb diff --git a/src/parser/transform/statement/transform_create_function.cpp b/src/parser/transform/statement/transform_create_function.cpp index 6490e29e5bfa..1525f634d2a6 100644 --- a/src/parser/transform/statement/transform_create_function.cpp +++ b/src/parser/transform/statement/transform_create_function.cpp @@ -47,6 +47,8 @@ unique_ptr Transformer::TransformMacroFunction(duckdb_libpgquery: default_expr = make_uniq(std::move(default_value)); default_expr->SetAlias(param.name); macro_func->default_parameters[param.name] = std::move(default_expr); + } else if (!macro_func->default_parameters.empty()) { + throw ParserException("Parameter without a default follows parameter with a default"); } } diff --git a/src/parser/transform/statement/transform_pivot_stmt.cpp b/src/parser/transform/statement/transform_pivot_stmt.cpp index 07dfb420d9dc..4572a3a36a16 100644 --- a/src/parser/transform/statement/transform_pivot_stmt.cpp +++ b/src/parser/transform/statement/transform_pivot_stmt.cpp @@ -95,7 +95,7 @@ unique_ptr Transformer::GenerateCreateEnumStmt(unique_ptr(); - select->node = TransformMaterializedCTE(std::move(subselect)); + select->node = std::move(subselect); info->query = std::move(select); info->type = LogicalType::INVALID; diff --git a/src/parser/transform/statement/transform_select.cpp b/src/parser/transform/statement/transform_select.cpp index 2e5135ef640d..16cd1a490144 100644 --- a/src/parser/transform/statement/transform_select.cpp +++ b/src/parser/transform/statement/transform_select.cpp @@ -26,13 +26,10 @@ unique_ptr Transformer::TransformSelectNodeInternal(duckdb_libpgquery throw ParserException("SELECT locking clause is not supported!"); } } - unique_ptr stmt = nullptr; if (select.pivot) { - stmt = TransformPivotStatement(select); - } else { - stmt = TransformSelectInternal(select); + return TransformPivotStatement(select); } - return TransformMaterializedCTE(std::move(stmt)); + return TransformSelectInternal(select); } unique_ptr Transformer::TransformSelectStmt(duckdb_libpgquery::PGSelectStmt &select, bool is_select) { diff --git a/src/parser/transform/statement/transform_select_node.cpp b/src/parser/transform/statement/transform_select_node.cpp index 56ef6c71c744..cb61ccd54450 100644 --- a/src/parser/transform/statement/transform_select_node.cpp +++ b/src/parser/transform/statement/transform_select_node.cpp @@ -51,6 +51,51 @@ void Transformer::TransformModifiers(duckdb_libpgquery::PGSelectStmt &stmt, Quer } } +bool Transformer::SetOperationsMatch(duckdb_libpgquery::PGSelectStmt &root, duckdb_libpgquery::PGNode &node) { + if (node.type != duckdb_libpgquery::T_PGSelectStmt) { + // not a select or set-op - set operations cannot match + return false; + } + auto &stmt = PGCast(node); + if (root.op != stmt.op || root.all != stmt.all) { + // set operation type does not match + return false; + } + if (root.op != duckdb_libpgquery::PG_SETOP_UNION && root.op != duckdb_libpgquery::PG_SETOP_UNION_BY_NAME) { + // only generate multi-child nodes for UNION/UNION ALL + return false; + } + // check if this is a "simple" set operation + if (stmt.withClause || stmt.sortClause || stmt.limitCount || stmt.limitOffset || stmt.sampleOptions) { + // it is not - we need to unfold it + return false; + } + return true; +} + +void Transformer::TransformSetOperationChildren(duckdb_libpgquery::PGSelectStmt &stmt, SetOperationNode &result) { + D_ASSERT(stmt.larg && stmt.rarg); + vector> set_operations; + set_operations.push_back(*stmt.larg); + set_operations.push_back(*stmt.rarg); + + for (idx_t i = 0; i < set_operations.size(); i++) { + auto &node = set_operations[i].get(); + // check if this set operation can be merged into the parents' set operation + if (!SetOperationsMatch(stmt, node)) { + // it cannot - transform the child + result.children.push_back(TransformSelectNode(node)); + } else { + // it can - recurse into children + // note that we must process the children in a specific order - so we need to expand the children in-place + auto &select = PGCast(node); + set_operations[i] = *select.larg; + set_operations.insert(set_operations.begin() + static_cast(i + 1), *select.rarg); + i--; + } + } +} + unique_ptr Transformer::TransformSelectInternal(duckdb_libpgquery::PGSelectStmt &stmt) { D_ASSERT(stmt.type == duckdb_libpgquery::T_PGSelectStmt); auto stack_checker = StackCheck(); @@ -129,13 +174,9 @@ unique_ptr Transformer::TransformSelectInternal(duckdb_libpgquery::PG node = make_uniq(); auto &result = node->Cast(); if (stmt.withClause) { - TransformCTE(*PGPointerCast(stmt.withClause), node->cte_map); - } - result.left = TransformSelectNode(*stmt.larg); - result.right = TransformSelectNode(*stmt.rarg); - if (!result.left || !result.right) { - throw InternalException("Failed to transform setop children."); + TransformCTE(*PGPointerCast(stmt.withClause), result.cte_map); } + TransformSetOperationChildren(stmt, result); result.setop_all = stmt.all; switch (stmt.op) { diff --git a/src/parser/transformer.cpp b/src/parser/transformer.cpp index d760adb27788..32ddaa87a421 100644 --- a/src/parser/transformer.cpp +++ b/src/parser/transformer.cpp @@ -173,6 +173,8 @@ unique_ptr Transformer::TransformStatementInternal(duckdb_libpgque return TransformCreateIndex(PGCast(stmt)); case duckdb_libpgquery::T_PGAlterTableStmt: return TransformAlter(PGCast(stmt)); + case duckdb_libpgquery::T_PGAlterDatabaseStmt: + return TransformAlterDatabase(PGCast(stmt)); case duckdb_libpgquery::T_PGRenameStmt: return TransformRename(PGCast(stmt)); case duckdb_libpgquery::T_PGPrepareStmt: @@ -230,32 +232,6 @@ unique_ptr Transformer::TransformStatementInternal(duckdb_libpgque } } -unique_ptr Transformer::TransformMaterializedCTE(unique_ptr root) { - // Extract materialized CTEs from cte_map - vector> materialized_ctes; - - for (auto &cte : root->cte_map.map) { - auto &cte_entry = cte.second; - auto mat_cte = make_uniq(); - mat_cte->ctename = cte.first; - mat_cte->query = TransformMaterializedCTE(cte_entry->query->node->Copy()); - mat_cte->aliases = cte_entry->aliases; - mat_cte->materialized = cte_entry->materialized; - materialized_ctes.push_back(std::move(mat_cte)); - } - - while (!materialized_ctes.empty()) { - unique_ptr node_result; - node_result = std::move(materialized_ctes.back()); - node_result->cte_map = root->cte_map.Copy(); - node_result->child = std::move(root); - root = std::move(node_result); - materialized_ctes.pop_back(); - } - - return root; -} - void Transformer::SetQueryLocation(ParsedExpression &expr, int query_location) { if (query_location < 0) { return; diff --git a/src/planner/bind_context.cpp b/src/planner/bind_context.cpp index b6e5df81f82e..cc1b3d25e459 100644 --- a/src/planner/bind_context.cpp +++ b/src/planner/bind_context.cpp @@ -38,16 +38,17 @@ optional_ptr BindContext::GetMatchingBinding(const string &column_name) optional_ptr result; for (auto &binding_ptr : bindings_list) { auto &binding = *binding_ptr; - auto is_using_binding = GetUsingBinding(column_name, binding.alias); + auto is_using_binding = GetUsingBinding(column_name, binding.GetBindingAlias()); if (is_using_binding) { continue; } if (binding.HasMatchingBinding(column_name)) { if (result || is_using_binding) { - throw BinderException("Ambiguous reference to column name \"%s\" (use: \"%s.%s\" " - "or \"%s.%s\")", - column_name, MinimumUniqueAlias(result->alias, binding.alias), column_name, - MinimumUniqueAlias(binding.alias, result->alias), column_name); + throw BinderException( + "Ambiguous reference to column name \"%s\" (use: \"%s.%s\" " + "or \"%s.%s\")", + column_name, MinimumUniqueAlias(result->GetBindingAlias(), binding.GetBindingAlias()), column_name, + MinimumUniqueAlias(binding.GetBindingAlias(), result->GetBindingAlias()), column_name); } result = &binding; } @@ -58,8 +59,8 @@ optional_ptr BindContext::GetMatchingBinding(const string &column_name) vector BindContext::GetSimilarBindings(const string &column_name) { vector> scores; for (auto &binding_ptr : bindings_list) { - auto binding = *binding_ptr; - for (auto &name : binding.names) { + auto &binding = *binding_ptr; + for (auto &name : binding.GetColumnNames()) { double distance = StringUtil::SimilarityRating(name, column_name); // check if we need to qualify the column auto matching_bindings = GetMatchingBindings(name); @@ -77,10 +78,6 @@ void BindContext::AddUsingBinding(const string &column_name, UsingColumnSet &set using_columns[column_name].insert(set); } -void BindContext::AddUsingBindingSet(unique_ptr set) { - using_column_sets.push_back(std::move(set)); -} - optional_ptr BindContext::GetUsingBinding(const string &column_name) { auto entry = using_columns.find(column_name); if (entry == using_columns.end()) { @@ -161,7 +158,7 @@ string BindContext::GetActualColumnName(Binding &binding, const string &column_n throw InternalException("Binding with name \"%s\" does not have a column named \"%s\"", binding.GetAlias(), column_name); } // LCOV_EXCL_STOP - return binding.names[binding_index]; + return binding.GetColumnNames()[binding_index]; } string BindContext::GetActualColumnName(const BindingAlias &binding_alias, const string &column_name) { @@ -204,7 +201,7 @@ unique_ptr BindContext::CreateColumnReference(const string &ta } static bool ColumnIsGenerated(Binding &binding, column_t index) { - if (binding.binding_type != BindingType::TABLE) { + if (binding.GetBindingType() != BindingType::TABLE) { return false; } auto &table_binding = binding.Cast(); @@ -243,10 +240,12 @@ unique_ptr BindContext::CreateColumnReference(const string &ca auto column_index = binding->GetBindingIndex(column_name); if (bind_type == ColumnBindType::EXPAND_GENERATED_COLUMNS && ColumnIsGenerated(*binding, column_index)) { return ExpandGeneratedColumn(binding->Cast(), column_name); - } else if (column_index < binding->names.size() && binding->names[column_index] != column_name) { + } + auto &column_names = binding->GetColumnNames(); + if (column_index < column_names.size() && column_names[column_index] != column_name) { // because of case insensitivity in the binder we rename the column to the original name // as it appears in the binding itself - result->SetAlias(binding->names[column_index]); + result->SetAlias(column_names[column_index]); } return std::move(result); } @@ -257,14 +256,6 @@ unique_ptr BindContext::CreateColumnReference(const string &sc return CreateColumnReference(catalog_name, schema_name, table_name, column_name, bind_type); } -optional_ptr BindContext::GetCTEBinding(const string &ctename) { - auto match = cte_bindings.find(ctename); - if (match == cte_bindings.end()) { - return nullptr; - } - return match->second.get(); -} - string GetCandidateAlias(const BindingAlias &main_alias, const BindingAlias &new_alias) { string candidate; if (!main_alias.GetCatalog().empty() && !new_alias.GetCatalog().empty()) { @@ -283,7 +274,7 @@ vector> BindContext::GetBindings(const BindingAlias &alias, E } vector> matching_bindings; for (auto &binding : bindings_list) { - if (binding->alias.Matches(alias)) { + if (binding->GetBindingAlias().Matches(alias)) { matching_bindings.push_back(*binding); } } @@ -291,7 +282,7 @@ vector> BindContext::GetBindings(const BindingAlias &alias, E // alias not found in this BindContext vector candidates; for (auto &binding : bindings_list) { - candidates.push_back(GetCandidateAlias(alias, binding->alias)); + candidates.push_back(GetCandidateAlias(alias, binding->GetBindingAlias())); } auto main_alias = GetCandidateAlias(alias, alias); string candidate_str = @@ -315,14 +306,14 @@ string BindContext::AmbiguityException(const BindingAlias &alias, const vector handled_using_columns; for (auto &entry : bindings_list) { auto &binding = *entry; - for (auto &column_name : binding.names) { - QualifiedColumnName qualified_column(binding.alias, column_name); + auto &column_names = binding.GetColumnNames(); + auto &binding_alias = binding.GetBindingAlias(); + for (auto &column_name : column_names) { + QualifiedColumnName qualified_column(binding_alias, column_name); if (CheckExclusionList(expr, qualified_column, exclusion_info)) { continue; } // check if this column is a USING column - auto using_binding_ptr = GetUsingBinding(column_name, binding.alias); + auto using_binding_ptr = GetUsingBinding(column_name, binding_alias); if (using_binding_ptr) { auto &using_binding = *using_binding_ptr; // it is! @@ -530,7 +524,7 @@ void BindContext::GenerateAllColumnExpressions(StarExpression &expr, continue; } auto new_expr = - CreateColumnReference(binding.alias, column_name, ColumnBindType::DO_NOT_EXPAND_GENERATED_COLUMNS); + CreateColumnReference(binding_alias, column_name, ColumnBindType::DO_NOT_EXPAND_GENERATED_COLUMNS); HandleRename(expr, qualified_column, *new_expr); new_select_list.push_back(std::move(new_expr)); } @@ -548,17 +542,20 @@ void BindContext::GenerateAllColumnExpressions(StarExpression &expr, } is_struct_ref = true; } + auto &binding_alias = binding->GetBindingAlias(); + auto &column_names = binding->GetColumnNames(); + auto &column_types = binding->GetColumnTypes(); if (is_struct_ref) { auto col_idx = binding->GetBindingIndex(expr.relation_name); - auto col_type = binding->types[col_idx]; + auto col_type = column_types[col_idx]; if (col_type.id() != LogicalTypeId::STRUCT) { throw BinderException(StringUtil::Format( "Cannot extract field from expression \"%s\" because it is not a struct", expr.ToString())); } auto &struct_children = StructType::GetChildTypes(col_type); vector column_names(3); - column_names[0] = binding->alias.GetAlias(); + column_names[0] = binding->GetAlias(); column_names[1] = expr.relation_name; for (auto &child : struct_children) { QualifiedColumnName qualified_name(child.first); @@ -571,13 +568,13 @@ void BindContext::GenerateAllColumnExpressions(StarExpression &expr, new_select_list.push_back(std::move(new_expr)); } } else { - for (auto &column_name : binding->names) { - QualifiedColumnName qualified_name(binding->alias, column_name); + for (auto &column_name : column_names) { + QualifiedColumnName qualified_name(binding_alias, column_name); if (CheckExclusionList(expr, qualified_name, exclusion_info)) { continue; } auto new_expr = - CreateColumnReference(binding->alias, column_name, ColumnBindType::DO_NOT_EXPAND_GENERATED_COLUMNS); + CreateColumnReference(binding_alias, column_name, ColumnBindType::DO_NOT_EXPAND_GENERATED_COLUMNS); HandleRename(expr, qualified_name, *new_expr); new_select_list.push_back(std::move(new_expr)); } @@ -613,10 +610,12 @@ void BindContext::GenerateAllColumnExpressions(StarExpression &expr, void BindContext::GetTypesAndNames(vector &result_names, vector &result_types) { for (auto &binding_entry : bindings_list) { auto &binding = *binding_entry; - D_ASSERT(binding.names.size() == binding.types.size()); - for (idx_t i = 0; i < binding.names.size(); i++) { - result_names.push_back(binding.names[i]); - result_types.push_back(binding.types[i]); + auto &column_names = binding.GetColumnNames(); + auto &column_types = binding.GetColumnTypes(); + D_ASSERT(column_names.size() == column_types.size()); + for (idx_t i = 0; i < column_names.size(); i++) { + result_names.push_back(column_names[i]); + result_types.push_back(column_types[i]); } } } @@ -686,7 +685,7 @@ vector BindContext::AliasColumnNames(const string &table_name, const vec return result; } -void BindContext::AddSubquery(idx_t index, const string &alias, SubqueryRef &ref, BoundQueryNode &subquery) { +void BindContext::AddSubquery(idx_t index, const string &alias, SubqueryRef &ref, BoundStatement &subquery) { auto names = AliasColumnNames(alias, subquery.names, ref.column_name_alias); AddGenericBinding(index, alias, names, subquery.types); } @@ -696,13 +695,13 @@ void BindContext::AddEntryBinding(idx_t index, const string &alias, const vector AddBinding(make_uniq(alias, types, names, index, entry)); } -void BindContext::AddView(idx_t index, const string &alias, SubqueryRef &ref, BoundQueryNode &subquery, +void BindContext::AddView(idx_t index, const string &alias, SubqueryRef &ref, BoundStatement &subquery, ViewCatalogEntry &view) { auto names = AliasColumnNames(alias, subquery.names, ref.column_name_alias); AddEntryBinding(index, alias, names, subquery.types, view.Cast()); } -void BindContext::AddSubquery(idx_t index, const string &alias, TableFunctionRef &ref, BoundQueryNode &subquery) { +void BindContext::AddSubquery(idx_t index, const string &alias, TableFunctionRef &ref, BoundStatement &subquery) { auto names = AliasColumnNames(alias, subquery.names, ref.column_name_alias); AddGenericBinding(index, alias, names, subquery.types); } @@ -712,33 +711,28 @@ void BindContext::AddGenericBinding(idx_t index, const string &alias, const vect AddBinding(make_uniq(BindingType::BASE, BindingAlias(alias), types, names, index)); } -void BindContext::AddCTEBinding(idx_t index, const string &alias, const vector &names, - const vector &types, bool using_key) { - auto binding = make_shared_ptr(BindingType::BASE, BindingAlias(alias), types, names, index); - - if (cte_bindings.find(alias) != cte_bindings.end()) { - throw BinderException("Duplicate CTE binding \"%s\" in query!", alias); +void BindContext::AddCTEBinding(unique_ptr binding) { + for (auto &cte_binding : cte_bindings) { + if (cte_binding->GetBindingAlias() == binding->GetBindingAlias()) { + throw BinderException("Duplicate CTE binding \"%s\" in query!", binding->GetBindingAlias().ToString()); + } } - cte_bindings[alias] = std::move(binding); - cte_references[alias] = make_shared_ptr(0); + cte_bindings.push_back(std::move(binding)); +} - if (using_key) { - auto recurring_alias = "recurring." + alias; - cte_bindings[recurring_alias] = - make_shared_ptr(BindingType::BASE, BindingAlias(recurring_alias), types, names, index); - cte_references[recurring_alias] = make_shared_ptr(0); - } +void BindContext::AddCTEBinding(idx_t index, BindingAlias alias_p, const vector &names, + const vector &types, CTEType cte_type) { + auto binding = make_uniq(std::move(alias_p), types, names, index, cte_type); + AddCTEBinding(std::move(binding)); } -void BindContext::RemoveCTEBinding(const std::string &alias) { - auto it = cte_bindings.find(alias); - if (it != cte_bindings.end()) { - cte_bindings.erase(it); - } - auto it2 = cte_references.find(alias); - if (it2 != cte_references.end()) { - cte_references.erase(it2); +optional_ptr BindContext::GetCTEBinding(const BindingAlias &ctename) { + for (auto &binding : cte_bindings) { + if (binding->GetBindingAlias().Matches(ctename)) { + return binding.get(); + } } + return nullptr; } void BindContext::AddContext(BindContext other) { @@ -755,7 +749,7 @@ void BindContext::AddContext(BindContext other) { vector BindContext::GetBindingAliases() { vector result; for (auto &binding : bindings_list) { - result.push_back(BindingAlias(binding->alias)); + result.push_back(binding->GetBindingAlias()); } return result; } @@ -782,7 +776,7 @@ void BindContext::RemoveContext(const vector &aliases) { // remove the binding from the list of bindings auto it = std::remove_if(bindings_list.begin(), bindings_list.end(), - [&](unique_ptr &x) { return x->alias == alias; }); + [&](unique_ptr &x) { return x->GetBindingAlias() == alias; }); bindings_list.erase(it, bindings_list.end()); } } diff --git a/src/planner/binder.cpp b/src/planner/binder.cpp index 1246f5a25354..9b3708be65ef 100644 --- a/src/planner/binder.cpp +++ b/src/planner/binder.cpp @@ -28,10 +28,6 @@ namespace duckdb { -Binder &Binder::GetRootBinder() { - return root_binder; -} - idx_t Binder::GetBinderDepth() const { return depth; } @@ -50,9 +46,11 @@ shared_ptr Binder::CreateBinder(ClientContext &context, optional_ptr parent_p, BinderType binder_type) - : context(context), bind_context(*this), parent(std::move(parent_p)), bound_tables(0), binder_type(binder_type), - entry_retriever(context), root_binder(parent ? parent->GetRootBinder() : *this), - depth(parent ? parent->GetBinderDepth() : 1) { + : context(context), bind_context(*this), parent(std::move(parent_p)), binder_type(binder_type), + global_binder_state(parent ? parent->global_binder_state : make_shared_ptr()), + query_binder_state(parent && binder_type == BinderType::REGULAR_BINDER ? parent->query_binder_state + : make_shared_ptr()), + entry_retriever(context), depth(parent ? parent->GetBinderDepth() : 1) { IncreaseDepth(); if (parent) { entry_retriever.Inherit(parent->entry_retriever); @@ -60,85 +58,22 @@ Binder::Binder(ClientContext &context, shared_ptr parent_p, BinderType b // We have to inherit macro and lambda parameter bindings and from the parent binder, if there is a parent. macro_binding = parent->macro_binding; lambda_bindings = parent->lambda_bindings; - - if (binder_type == BinderType::REGULAR_BINDER) { - // We have to inherit CTE bindings from the parent bind_context, if there is a parent. - bind_context.SetCTEBindings(parent->bind_context.GetCTEBindings()); - bind_context.cte_references = parent->bind_context.cte_references; - parameters = parent->parameters; - } - } -} - -unique_ptr Binder::BindMaterializedCTE(CommonTableExpressionMap &cte_map) { - // Extract materialized CTEs from cte_map - vector> materialized_ctes; - for (auto &cte : cte_map.map) { - auto &cte_entry = cte.second; - auto mat_cte = make_uniq(); - mat_cte->ctename = cte.first; - mat_cte->query = cte_entry->query->node->Copy(); - mat_cte->aliases = cte_entry->aliases; - mat_cte->materialized = cte_entry->materialized; - materialized_ctes.push_back(std::move(mat_cte)); - } - - if (materialized_ctes.empty()) { - return nullptr; - } - - unique_ptr cte_root = nullptr; - while (!materialized_ctes.empty()) { - unique_ptr node_result; - node_result = std::move(materialized_ctes.back()); - node_result->cte_map = cte_map.Copy(); - if (cte_root) { - node_result->child = std::move(cte_root); - } else { - node_result->child = nullptr; - } - cte_root = std::move(node_result); - materialized_ctes.pop_back(); } - - AddCTEMap(cte_map); - auto bound_cte = BindCTE(cte_root->Cast()); - - return bound_cte; } template BoundStatement Binder::BindWithCTE(T &statement) { - BoundStatement bound_statement; - auto bound_cte = BindMaterializedCTE(statement.template Cast().cte_map); - if (bound_cte) { - reference tail_ref = *bound_cte; - - while (tail_ref.get().child && tail_ref.get().child->type == QueryNodeType::CTE_NODE) { - tail_ref = tail_ref.get().child->Cast(); - } - - auto &tail = tail_ref.get(); - bound_statement = tail.child_binder->Bind(statement.template Cast()); - - tail.types = bound_statement.types; - tail.names = bound_statement.names; - - for (auto &c : tail.query_binder->correlated_columns) { - tail.child_binder->AddCorrelatedColumn(c); - } - MoveCorrelatedExpressions(*tail.child_binder); - - auto plan = std::move(bound_statement.plan); - bound_statement.plan = CreatePlan(*bound_cte, std::move(plan)); - } else { - bound_statement = Bind(statement.template Cast()); + auto &cte_map = statement.cte_map; + if (cte_map.map.empty()) { + return Bind(statement); } - return bound_statement; + + auto stmt_node = make_uniq(statement); + stmt_node->cte_map = cte_map.Copy(); + return Bind(*stmt_node); } BoundStatement Binder::Bind(SQLStatement &statement) { - root_statement = &statement; switch (statement.type) { case StatementType::SELECT_STATEMENT: return Bind(statement.Cast()); @@ -198,64 +133,12 @@ BoundStatement Binder::Bind(SQLStatement &statement) { } // LCOV_EXCL_STOP } -void Binder::AddCTEMap(CommonTableExpressionMap &cte_map) { - for (auto &cte_it : cte_map.map) { - AddCTE(cte_it.first, *cte_it.second); - } -} - -unique_ptr Binder::BindNode(QueryNode &node) { - // first we visit the set of CTEs and add them to the bind context - AddCTEMap(node.cte_map); - // now we bind the node - unique_ptr result; - switch (node.type) { - case QueryNodeType::SELECT_NODE: - result = BindNode(node.Cast()); - break; - case QueryNodeType::RECURSIVE_CTE_NODE: - result = BindNode(node.Cast()); - break; - case QueryNodeType::CTE_NODE: - result = BindNode(node.Cast()); - break; - default: - D_ASSERT(node.type == QueryNodeType::SET_OPERATION_NODE); - result = BindNode(node.Cast()); - break; - } - return result; -} - BoundStatement Binder::Bind(QueryNode &node) { - BoundStatement result; - auto bound_node = BindNode(node); - - result.names = bound_node->names; - result.types = bound_node->types; - - // and plan it - result.plan = CreatePlan(*bound_node); - return result; + return BindNode(node); } -unique_ptr Binder::CreatePlan(BoundQueryNode &node) { - switch (node.type) { - case QueryNodeType::SELECT_NODE: - return CreatePlan(node.Cast()); - case QueryNodeType::SET_OPERATION_NODE: - return CreatePlan(node.Cast()); - case QueryNodeType::RECURSIVE_CTE_NODE: - return CreatePlan(node.Cast()); - case QueryNodeType::CTE_NODE: - return CreatePlan(node.Cast()); - default: - throw InternalException("Unsupported bound query node type"); - } -} - -unique_ptr Binder::Bind(TableRef &ref) { - unique_ptr result; +BoundStatement Binder::Bind(TableRef &ref) { + BoundStatement result; switch (ref.type) { case TableReferenceType::BASE_TABLE: result = Bind(ref.Cast()); @@ -295,86 +178,33 @@ unique_ptr Binder::Bind(TableRef &ref) { default: throw InternalException("Unknown table ref type (%s)", EnumUtil::ToString(ref.type)); } - result->sample = std::move(ref.sample); - return result; -} - -unique_ptr Binder::CreatePlan(BoundTableRef &ref) { - unique_ptr root; - switch (ref.type) { - case TableReferenceType::BASE_TABLE: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::SUBQUERY: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::JOIN: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::TABLE_FUNCTION: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::EMPTY_FROM: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::EXPRESSION_LIST: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::COLUMN_DATA: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::CTE: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::PIVOT: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::DELIM_GET: - root = CreatePlan(ref.Cast()); - break; - case TableReferenceType::INVALID: - default: - throw InternalException("Unsupported bound table ref type (%s)", EnumUtil::ToString(ref.type)); - } - // plan the sample clause if (ref.sample) { - root = make_uniq(std::move(ref.sample), std::move(root)); + result.plan = make_uniq(std::move(ref.sample), std::move(result.plan)); } - return root; -} - -void Binder::AddCTE(const string &name, CommonTableExpressionInfo &info) { - D_ASSERT(!name.empty()); - auto entry = CTE_bindings.find(name); - if (entry != CTE_bindings.end()) { - throw InternalException("Duplicate CTE \"%s\" in query!", name); - } - CTE_bindings.insert(make_pair(name, reference(info))); + return result; } -vector> Binder::FindCTE(const string &name, bool skip) { - auto entry = CTE_bindings.find(name); - vector> ctes; - if (entry != CTE_bindings.end()) { - if (!skip || entry->second.get().query->node->type == QueryNodeType::RECURSIVE_CTE_NODE) { - ctes.push_back(entry->second); +optional_ptr Binder::GetCTEBinding(const BindingAlias &name) { + reference current_binder(*this); + optional_ptr result; + while (true) { + auto ¤t = current_binder.get(); + auto entry = current.bind_context.GetCTEBinding(name); + if (entry) { + // we only directly return the CTE if it can be referenced + // if it cannot be referenced (circular reference) we keep going up the stack + // to look for a CTE that can be referenced + if (entry->CanBeReferenced()) { + return entry; + } + result = entry; } + if (!current.parent || current.binder_type != BinderType::REGULAR_BINDER) { + break; + } + current_binder = *current.parent; } - if (parent && binder_type == BinderType::REGULAR_BINDER) { - auto parent_ctes = parent->FindCTE(name, name == alias); - ctes.insert(ctes.end(), parent_ctes.begin(), parent_ctes.end()); - } - return ctes; -} - -bool Binder::CTEIsAlreadyBound(CommonTableExpressionInfo &cte) { - if (bound_ctes.find(cte) != bound_ctes.end()) { - return true; - } - if (parent && binder_type == BinderType::REGULAR_BINDER) { - return parent->CTEIsAlreadyBound(cte); - } - return false; + return result; } void Binder::AddBoundView(ViewCatalogEntry &view) { @@ -390,13 +220,19 @@ void Binder::AddBoundView(ViewCatalogEntry &view) { } idx_t Binder::GenerateTableIndex() { - auto &root_binder = GetRootBinder(); - return root_binder.bound_tables++; + return global_binder_state->bound_tables++; } StatementProperties &Binder::GetStatementProperties() { - auto &root_binder = GetRootBinder(); - return root_binder.prop; + return global_binder_state->prop; +} + +optional_ptr Binder::GetParameters() { + return global_binder_state->parameters; +} + +void Binder::SetParameters(BoundParameterMap ¶meters) { + global_binder_state->parameters = parameters; } void Binder::PushExpressionBinder(ExpressionBinder &binder) { @@ -422,17 +258,11 @@ bool Binder::HasActiveBinder() { } vector> &Binder::GetActiveBinders() { - reference root = *this; - while (root.get().parent && root.get().binder_type == BinderType::REGULAR_BINDER) { - root = *root.get().parent; - } - auto &root_binder = root.get(); - return root_binder.active_binders; + return query_binder_state->active_binders; } void Binder::AddUsingBindingSet(unique_ptr set) { - auto &root_binder = GetRootBinder(); - root_binder.bind_context.AddUsingBindingSet(std::move(set)); + global_binder_state->using_column_sets.push_back(std::move(set)); } void Binder::MoveCorrelatedExpressions(Binder &other) { @@ -440,7 +270,7 @@ void Binder::MoveCorrelatedExpressions(Binder &other) { other.correlated_columns.clear(); } -void Binder::MergeCorrelatedColumns(vector &other) { +void Binder::MergeCorrelatedColumns(CorrelatedColumns &other) { for (idx_t i = 0; i < other.size(); i++) { AddCorrelatedColumn(other[i]); } @@ -449,7 +279,7 @@ void Binder::MergeCorrelatedColumns(vector &other) { void Binder::AddCorrelatedColumn(const CorrelatedColumnInfo &info) { // we only add correlated columns to the list if they are not already there if (std::find(correlated_columns.begin(), correlated_columns.end(), info) == correlated_columns.end()) { - correlated_columns.push_back(info); + correlated_columns.AddColumn(info); } } @@ -469,7 +299,6 @@ optional_ptr Binder::GetMatchingBinding(const string &catalog_name, con const string &table_name, const string &column_name, ErrorData &error) { optional_ptr binding; - D_ASSERT(!lambda_bindings); if (macro_binding && table_name == macro_binding->GetAlias()) { binding = optional_ptr(macro_binding.get()); } else { @@ -480,13 +309,11 @@ optional_ptr Binder::GetMatchingBinding(const string &catalog_name, con } void Binder::SetBindingMode(BindingMode mode) { - auto &root_binder = GetRootBinder(); - root_binder.mode = mode; + global_binder_state->mode = mode; } BindingMode Binder::GetBindingMode() { - auto &root_binder = GetRootBinder(); - return root_binder.mode; + return global_binder_state->mode; } void Binder::SetCanContainNulls(bool can_contain_nulls_p) { @@ -499,30 +326,26 @@ void Binder::SetAlwaysRequireRebind() { } void Binder::AddTableName(string table_name) { - auto &root_binder = GetRootBinder(); - root_binder.table_names.insert(std::move(table_name)); + global_binder_state->table_names.insert(std::move(table_name)); } void Binder::AddReplacementScan(const string &table_name, unique_ptr replacement) { - auto &root_binder = GetRootBinder(); - auto it = root_binder.replacement_scans.find(table_name); + auto it = global_binder_state->replacement_scans.find(table_name); replacement->column_name_alias.clear(); replacement->alias.clear(); - if (it == root_binder.replacement_scans.end()) { - root_binder.replacement_scans[table_name] = std::move(replacement); + if (it == global_binder_state->replacement_scans.end()) { + global_binder_state->replacement_scans[table_name] = std::move(replacement); } else { // A replacement scan by this name was previously registered, we can just use it } } const unordered_set &Binder::GetTableNames() { - auto &root_binder = GetRootBinder(); - return root_binder.table_names; + return global_binder_state->table_names; } case_insensitive_map_t> &Binder::GetReplacementScans() { - auto &root_binder = GetRootBinder(); - return root_binder.replacement_scans; + return global_binder_state->replacement_scans; } // FIXME: this is extremely naive diff --git a/src/planner/binder/expression/bind_columnref_expression.cpp b/src/planner/binder/expression/bind_columnref_expression.cpp index 026226221573..e8b83e95cec4 100644 --- a/src/planner/binder/expression/bind_columnref_expression.cpp +++ b/src/planner/binder/expression/bind_columnref_expression.cpp @@ -94,12 +94,12 @@ unique_ptr ExpressionBinder::QualifyColumnName(const string &c // bind as a macro column if (is_macro_column) { - return binder.bind_context.CreateColumnReference(binder.macro_binding->alias, column_name); + return binder.bind_context.CreateColumnReference(binder.macro_binding->GetBindingAlias(), column_name); } // bind as a regular column if (table_binding) { - return binder.bind_context.CreateColumnReference(table_binding->alias, column_name); + return binder.bind_context.CreateColumnReference(table_binding->GetBindingAlias(), column_name); } // it's not, find candidates and error @@ -155,6 +155,9 @@ void ExpressionBinder::QualifyColumnNames(unique_ptr &expr, case ExpressionType::FUNCTION: { // Special-handling for lambdas, which are inside function expressions. auto &function = expr->Cast(); + if (!IsUnnestFunction(function.function_name)) { + BindAndQualifyFunction(function, false); + } if (function.IsLambdaFunction()) { return QualifyColumnNamesInLambda(function, lambda_params); } @@ -273,11 +276,12 @@ unique_ptr ExpressionBinder::CreateStructPack(ColumnRefExpress } // We found the table, now create the struct_pack expression + auto &column_names = binding->GetColumnNames(); vector> child_expressions; - child_expressions.reserve(binding->names.size()); - for (const auto &column_name : binding->names) { + child_expressions.reserve(column_names.size()); + for (const auto &column_name : column_names) { child_expressions.push_back(binder.bind_context.CreateColumnReference( - binding->alias, column_name, ColumnBindType::DO_NOT_EXPAND_GENERATED_COLUMNS)); + binding->GetBindingAlias(), column_name, ColumnBindType::DO_NOT_EXPAND_GENERATED_COLUMNS)); } return make_uniq("struct_pack", std::move(child_expressions)); } @@ -309,7 +313,7 @@ unique_ptr ExpressionBinder::QualifyColumnNameWithManyDotsInte if (binding) { // part1 is a catalog - the column reference is "catalog.schema.table.column" struct_extract_start = 4; - return binder.bind_context.CreateColumnReference(binding->alias, col_ref.column_names[3]); + return binder.bind_context.CreateColumnReference(binding->GetBindingAlias(), col_ref.column_names[3]); } } ErrorData catalog_table_error; @@ -318,7 +322,7 @@ unique_ptr ExpressionBinder::QualifyColumnNameWithManyDotsInte if (binding) { // part1 is a catalog - the column reference is "catalog.table.column" struct_extract_start = 3; - return binder.bind_context.CreateColumnReference(binding->alias, col_ref.column_names[2]); + return binder.bind_context.CreateColumnReference(binding->GetBindingAlias(), col_ref.column_names[2]); } ErrorData schema_table_error; binding = binder.GetMatchingBinding(col_ref.column_names[0], col_ref.column_names[1], col_ref.column_names[2], @@ -327,7 +331,7 @@ unique_ptr ExpressionBinder::QualifyColumnNameWithManyDotsInte // part1 is a schema - the column reference is "schema.table.column" // any additional fields are turned into struct_extract calls struct_extract_start = 3; - return binder.bind_context.CreateColumnReference(binding->alias, col_ref.column_names[2]); + return binder.bind_context.CreateColumnReference(binding->GetBindingAlias(), col_ref.column_names[2]); } ErrorData table_column_error; binding = binder.GetMatchingBinding(col_ref.column_names[0], col_ref.column_names[1], table_column_error); @@ -336,7 +340,7 @@ unique_ptr ExpressionBinder::QualifyColumnNameWithManyDotsInte // the column reference is "table.column" // any additional fields are turned into struct_extract calls struct_extract_start = 2; - return binder.bind_context.CreateColumnReference(binding->alias, col_ref.column_names[1]); + return binder.bind_context.CreateColumnReference(binding->GetBindingAlias(), col_ref.column_names[1]); } // part1 could be a column ErrorData unused_error; @@ -357,7 +361,7 @@ unique_ptr ExpressionBinder::QualifyColumnNameWithManyDotsInte optional_idx schema_pos; optional_idx table_pos; for (const auto &binding_entry : binder.bind_context.GetBindingsList()) { - auto &alias = binding_entry->alias; + auto &alias = binding_entry->GetBindingAlias(); string catalog = alias.GetCatalog(); string schema = alias.GetSchema(); string table = alias.GetAlias(); @@ -480,7 +484,7 @@ unique_ptr ExpressionBinder::QualifyColumnName(ColumnRefExpres auto binding = binder.GetMatchingBinding(col_ref.column_names[0], col_ref.column_names[1], error); if (binding) { // it is! return the column reference directly - return binder.bind_context.CreateColumnReference(binding->alias, col_ref.GetColumnName()); + return binder.bind_context.CreateColumnReference(binding->GetBindingAlias(), col_ref.GetColumnName()); } // otherwise check if we can turn this into a struct extract diff --git a/src/planner/binder/expression/bind_function_expression.cpp b/src/planner/binder/expression/bind_function_expression.cpp index a747ef48dcdc..7274acf1662e 100644 --- a/src/planner/binder/expression/bind_function_expression.cpp +++ b/src/planner/binder/expression/bind_function_expression.cpp @@ -68,8 +68,8 @@ BindResult ExpressionBinder::TryBindLambdaOrJson(FunctionExpression &function, i json_bind_result.error.RawMessage()); } -BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t depth, - unique_ptr &expr_ptr) { +optional_ptr ExpressionBinder::BindAndQualifyFunction(FunctionExpression &function, bool allow_throw) { + D_ASSERT(!IsUnnestFunction(function.function_name)); // lookup the function in the catalog QueryErrorContext error_context(function.GetQueryLocation()); binder.BindSchemaOrCatalog(function.catalog, function.schema); @@ -82,6 +82,9 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t auto table_func = GetCatalogEntry(function.catalog, function.schema, table_function_lookup, OnEntryNotFound::RETURN_NULL); if (table_func) { + if (!allow_throw) { + return func; + } throw BinderException(function, "Function \"%s\" is a table function but it was used as a scalar function. This " "function has to be called in a FROM clause (similar to a table).", @@ -105,6 +108,9 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t if (error.HasError()) { // could not find the column - try to qualify the alias if (!QualifyColumnAlias(*colref)) { + if (!allow_throw) { + return func; + } // no alias found either - throw error.Throw(); } @@ -118,11 +124,19 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t } // rebind the function if (!func) { - func = - GetCatalogEntry(function.catalog, function.schema, function_lookup, OnEntryNotFound::THROW_EXCEPTION); + const auto on_entry_not_found = + allow_throw ? OnEntryNotFound::THROW_EXCEPTION : OnEntryNotFound::RETURN_NULL; + func = GetCatalogEntry(function.catalog, function.schema, function_lookup, on_entry_not_found); } } + return func; +} + +BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t depth, + unique_ptr &expr_ptr) { + auto func = BindAndQualifyFunction(function, true); + if (func->type != CatalogType::AGGREGATE_FUNCTION_ENTRY && (function.distinct || function.filter || !function.order_bys->orders.empty())) { throw InvalidInputException("Function \"%s\" is a %s. \"DISTINCT\", \"FILTER\", and \"ORDER BY\" are only " @@ -290,11 +304,13 @@ BindResult ExpressionBinder::BindLambdaFunction(FunctionExpression &function, Sc for (idx_t i = lambda_bindings->size(); i > 0; i--) { auto &binding = (*lambda_bindings)[i - 1]; - D_ASSERT(binding.names.size() == binding.types.size()); + auto &column_names = binding.GetColumnNames(); + auto &column_types = binding.GetColumnTypes(); + D_ASSERT(column_names.size() == column_types.size()); - for (idx_t column_idx = binding.names.size(); column_idx > 0; column_idx--) { - auto bound_lambda_param = make_uniq(binding.names[column_idx - 1], - binding.types[column_idx - 1], offset); + for (idx_t column_idx = column_names.size(); column_idx > 0; column_idx--) { + auto bound_lambda_param = make_uniq(column_names[column_idx - 1], + column_types[column_idx - 1], offset); offset++; bound_function_expr.children.push_back(std::move(bound_lambda_param)); } diff --git a/src/planner/binder/expression/bind_lambda.cpp b/src/planner/binder/expression/bind_lambda.cpp index 592daa245659..ad8402824794 100644 --- a/src/planner/binder/expression/bind_lambda.cpp +++ b/src/planner/binder/expression/bind_lambda.cpp @@ -12,25 +12,25 @@ namespace duckdb { -idx_t GetLambdaParamCount(const vector &lambda_bindings) { +idx_t GetLambdaParamCount(vector &lambda_bindings) { idx_t count = 0; for (auto &binding : lambda_bindings) { - count += binding.names.size(); + count += binding.GetColumnCount(); } return count; } -idx_t GetLambdaParamIndex(const vector &lambda_bindings, const BoundLambdaExpression &bound_lambda_expr, +idx_t GetLambdaParamIndex(vector &lambda_bindings, const BoundLambdaExpression &bound_lambda_expr, const BoundLambdaRefExpression &bound_lambda_ref_expr) { D_ASSERT(bound_lambda_ref_expr.lambda_idx < lambda_bindings.size()); idx_t offset = 0; // count the remaining lambda parameters BEFORE the current lambda parameter, // as these will be in front of the current lambda parameter in the input chunk for (idx_t i = bound_lambda_ref_expr.lambda_idx + 1; i < lambda_bindings.size(); i++) { - offset += lambda_bindings[i].names.size(); + offset += lambda_bindings[i].GetColumnCount(); } - offset += - lambda_bindings[bound_lambda_ref_expr.lambda_idx].names.size() - bound_lambda_ref_expr.binding.column_index - 1; + offset += lambda_bindings[bound_lambda_ref_expr.lambda_idx].GetColumnCount() - + bound_lambda_ref_expr.binding.column_index - 1; offset += bound_lambda_expr.parameter_count; return offset; } @@ -148,16 +148,18 @@ void ExpressionBinder::TransformCapturedLambdaColumn(unique_ptr &ori if (lambda_bindings && bound_lambda_ref.lambda_idx != lambda_bindings->size()) { auto &binding = (*lambda_bindings)[bound_lambda_ref.lambda_idx]; - D_ASSERT(binding.names.size() == binding.types.size()); + auto &column_names = binding.GetColumnNames(); + auto &column_types = binding.GetColumnTypes(); + D_ASSERT(column_names.size() == column_types.size()); // find the matching dummy column in the lambda binding - for (idx_t column_idx = 0; column_idx < binding.names.size(); column_idx++) { + for (idx_t column_idx = 0; column_idx < binding.GetColumnCount(); column_idx++) { if (column_idx == bound_lambda_ref.binding.column_index) { // now create the replacement auto index = GetLambdaParamIndex(*lambda_bindings, bound_lambda_expr, bound_lambda_ref); - replacement = make_uniq(binding.names[column_idx], - binding.types[column_idx], index); + replacement = + make_uniq(column_names[column_idx], column_types[column_idx], index); return; } } diff --git a/src/planner/binder/expression/bind_macro_expression.cpp b/src/planner/binder/expression/bind_macro_expression.cpp index cce06d712d05..5d2bce798653 100644 --- a/src/planner/binder/expression/bind_macro_expression.cpp +++ b/src/planner/binder/expression/bind_macro_expression.cpp @@ -98,6 +98,7 @@ void ExpressionBinder::UnfoldMacroExpression(FunctionExpression &function, Scala // validate the arguments and separate positional and default arguments vector> positional_arguments; InsertionOrderPreservingMap> named_arguments; + binder.lambda_bindings = lambda_bindings; auto bind_result = MacroFunction::BindMacroFunction(binder, macro_func.macros, macro_func.name, function, positional_arguments, named_arguments, depth); if (!bind_result.error.empty()) { diff --git a/src/planner/binder/expression/bind_parameter_expression.cpp b/src/planner/binder/expression/bind_parameter_expression.cpp index 109c0ecbd683..3fe02467e238 100644 --- a/src/planner/binder/expression/bind_parameter_expression.cpp +++ b/src/planner/binder/expression/bind_parameter_expression.cpp @@ -8,19 +8,19 @@ namespace duckdb { BindResult ExpressionBinder::BindExpression(ParameterExpression &expr, idx_t depth) { - if (!binder.parameters) { + auto parameters = binder.GetParameters(); + if (!parameters) { throw BinderException("Unexpected prepared parameter. This type of statement can't be prepared!"); } auto parameter_id = expr.identifier; - D_ASSERT(binder.parameters); // Check if a parameter value has already been supplied - auto ¶meter_data = binder.parameters->GetParameterData(); + auto ¶meter_data = parameters->GetParameterData(); auto param_data_it = parameter_data.find(parameter_id); if (param_data_it != parameter_data.end()) { // it has! emit a constant directly auto &data = param_data_it->second; - auto return_type = binder.parameters->GetReturnType(parameter_id); + auto return_type = parameters->GetReturnType(parameter_id); bool is_literal = return_type.id() == LogicalTypeId::INTEGER_LITERAL || return_type.id() == LogicalTypeId::STRING_LITERAL; auto constant = make_uniq(data.GetValue()); @@ -32,7 +32,7 @@ BindResult ExpressionBinder::BindExpression(ParameterExpression &expr, idx_t dep return BindResult(std::move(cast)); } - auto bound_parameter = binder.parameters->BindParameterExpression(expr); + auto bound_parameter = parameters->BindParameterExpression(expr); return BindResult(std::move(bound_parameter)); } diff --git a/src/planner/binder/expression/bind_star_expression.cpp b/src/planner/binder/expression/bind_star_expression.cpp index f48fc14e6710..4cc0e3a234e5 100644 --- a/src/planner/binder/expression/bind_star_expression.cpp +++ b/src/planner/binder/expression/bind_star_expression.cpp @@ -152,10 +152,15 @@ string Binder::ReplaceColumnsAlias(const string &alias, const string &column_nam void TryTransformStarLike(unique_ptr &root) { // detect "* LIKE [literal]" and similar expressions - if (root->GetExpressionClass() != ExpressionClass::FUNCTION) { + bool inverse = root->GetExpressionType() == ExpressionType::OPERATOR_NOT; + auto &expr = inverse ? root->Cast().children[0] : root; + if (!expr) { return; } - auto &function = root->Cast(); + if (expr->GetExpressionClass() != ExpressionClass::FUNCTION) { + return; + } + auto &function = expr->Cast(); if (function.children.size() < 2 || function.children.size() > 3) { return; } @@ -197,7 +202,7 @@ void TryTransformStarLike(unique_ptr &root) { auto original_alias = root->GetAlias(); auto star_expr = std::move(left); unique_ptr child_expr; - if (function.function_name == "regexp_full_match" && star.exclude_list.empty()) { + if (!inverse && function.function_name == "regexp_full_match" && star.exclude_list.empty()) { // * SIMILAR TO '[regex]' is equivalent to COLUMNS('[regex]') so we can just move the expression directly child_expr = std::move(right); } else { @@ -207,13 +212,20 @@ void TryTransformStarLike(unique_ptr &root) { vector named_parameters; named_parameters.push_back("__lambda_col"); function.children[0] = make_uniq("__lambda_col"); + function.children[1] = std::move(right); + + unique_ptr lambda_body = std::move(expr); + if (inverse) { + vector> root_children; + root_children.push_back(std::move(lambda_body)); + lambda_body = make_uniq(ExpressionType::OPERATOR_NOT, std::move(root_children)); + } + auto lambda = make_uniq(std::move(named_parameters), std::move(lambda_body)); - auto lambda = make_uniq(std::move(named_parameters), std::move(root)); vector> filter_children; filter_children.push_back(std::move(star_expr)); filter_children.push_back(std::move(lambda)); - auto list_filter = make_uniq("list_filter", std::move(filter_children)); - child_expr = std::move(list_filter); + child_expr = make_uniq("list_filter", std::move(filter_children)); } auto columns_expr = make_uniq(); diff --git a/src/planner/binder/expression/bind_subquery_expression.cpp b/src/planner/binder/expression/bind_subquery_expression.cpp index d413c88ed9e8..7f03f0e32d08 100644 --- a/src/planner/binder/expression/bind_subquery_expression.cpp +++ b/src/planner/binder/expression/bind_subquery_expression.cpp @@ -13,20 +13,16 @@ class BoundSubqueryNode : public QueryNode { static constexpr const QueryNodeType TYPE = QueryNodeType::BOUND_SUBQUERY_NODE; public: - BoundSubqueryNode(shared_ptr subquery_binder, unique_ptr bound_node, + BoundSubqueryNode(shared_ptr subquery_binder, BoundStatement bound_node, unique_ptr subquery) : QueryNode(QueryNodeType::BOUND_SUBQUERY_NODE), subquery_binder(std::move(subquery_binder)), bound_node(std::move(bound_node)), subquery(std::move(subquery)) { } shared_ptr subquery_binder; - unique_ptr bound_node; + BoundStatement bound_node; unique_ptr subquery; - const vector> &GetSelectList() const override { - throw InternalException("Cannot get select list of bound subquery node"); - } - string ToString() const override { throw InternalException("Cannot ToString bound subquery node"); } @@ -116,15 +112,15 @@ BindResult ExpressionBinder::BindExpression(SubqueryExpression &expr, idx_t dept idx_t expected_columns = 1; if (expr.child) { auto &child = BoundExpression::GetExpression(*expr.child); - ExtractSubqueryChildren(child, child_expressions, bound_subquery.bound_node->types); + ExtractSubqueryChildren(child, child_expressions, bound_subquery.bound_node.types); if (child_expressions.empty()) { child_expressions.push_back(std::move(child)); } expected_columns = child_expressions.size(); } - if (bound_subquery.bound_node->types.size() != expected_columns) { + if (bound_subquery.bound_node.types.size() != expected_columns) { throw BinderException(expr, "Subquery returns %zu columns - expected %d", - bound_subquery.bound_node->types.size(), expected_columns); + bound_subquery.bound_node.types.size(), expected_columns); } } // both binding the child and binding the subquery was successful @@ -132,7 +128,7 @@ BindResult ExpressionBinder::BindExpression(SubqueryExpression &expr, idx_t dept auto subquery_binder = std::move(bound_subquery.subquery_binder); auto bound_node = std::move(bound_subquery.bound_node); LogicalType return_type = - expr.subquery_type == SubqueryType::SCALAR ? bound_node->types[0] : LogicalType(LogicalTypeId::BOOLEAN); + expr.subquery_type == SubqueryType::SCALAR ? bound_node.types[0] : LogicalType(LogicalTypeId::BOOLEAN); if (return_type.id() == LogicalTypeId::UNKNOWN) { return_type = LogicalType::SQLNULL; } @@ -144,7 +140,7 @@ BindResult ExpressionBinder::BindExpression(SubqueryExpression &expr, idx_t dept for (idx_t child_idx = 0; child_idx < child_expressions.size(); child_idx++) { auto &child = child_expressions[child_idx]; auto child_type = ExpressionBinder::GetExpressionReturnType(*child); - auto &subquery_type = bound_node->types[child_idx]; + auto &subquery_type = bound_node.types[child_idx]; LogicalType compare_type; if (!LogicalType::TryGetMaxLogicalType(context, child_type, subquery_type, compare_type)) { throw BinderException( diff --git a/src/planner/binder/query_node/CMakeLists.txt b/src/planner/binder/query_node/CMakeLists.txt index fd28e0b483bf..709d23ce1749 100644 --- a/src/planner/binder/query_node/CMakeLists.txt +++ b/src/planner/binder/query_node/CMakeLists.txt @@ -5,10 +5,9 @@ add_library_unity( bind_setop_node.cpp bind_recursive_cte_node.cpp bind_cte_node.cpp + bind_statement_node.cpp bind_table_macro_node.cpp plan_query_node.cpp - plan_recursive_cte_node.cpp - plan_cte_node.cpp plan_select_node.cpp plan_setop.cpp plan_subquery.cpp) diff --git a/src/planner/binder/query_node/bind_cte_node.cpp b/src/planner/binder/query_node/bind_cte_node.cpp index 092b46ca6ee6..8cea37ebc277 100644 --- a/src/planner/binder/query_node/bind_cte_node.cpp +++ b/src/planner/binder/query_node/bind_cte_node.cpp @@ -1,91 +1,169 @@ -#include "duckdb/parser/expression/constant_expression.hpp" -#include "duckdb/parser/expression_map.hpp" -#include "duckdb/parser/query_node/select_node.hpp" #include "duckdb/parser/query_node/cte_node.hpp" #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/query_node/bound_cte_node.hpp" -#include "duckdb/planner/query_node/bound_select_node.hpp" +#include "duckdb/planner/operator/logical_materialized_cte.hpp" +#include "duckdb/parser/query_node/list.hpp" +#include "duckdb/parser/statement/select_statement.hpp" namespace duckdb { -unique_ptr Binder::BindNode(CTENode &statement) { - // first recursively visit the materialized CTE operations - // the left side is visited first and is added to the BindContext of the right side - D_ASSERT(statement.query); +struct BoundCTEData { + string ctename; + CTEMaterialize materialized; + idx_t setop_index; + shared_ptr child_binder; + shared_ptr cte_bind_state; +}; + +BoundStatement Binder::BindNode(QueryNode &node) { + reference current_binder(*this); + vector bound_ctes; + for (auto &cte : node.cte_map.map) { + bound_ctes.push_back(current_binder.get().PrepareCTE(cte.first, *cte.second)); + current_binder = *bound_ctes.back().child_binder; + } + BoundStatement result; + // now we bind the node + switch (node.type) { + case QueryNodeType::SELECT_NODE: + result = current_binder.get().BindNode(node.Cast()); + break; + case QueryNodeType::RECURSIVE_CTE_NODE: + result = current_binder.get().BindNode(node.Cast()); + break; + case QueryNodeType::SET_OPERATION_NODE: + result = current_binder.get().BindNode(node.Cast()); + break; + case QueryNodeType::STATEMENT_NODE: + result = current_binder.get().BindNode(node.Cast()); + break; + default: + throw InternalException("Unsupported query node type"); + } + for (idx_t i = bound_ctes.size(); i > 0; i--) { + auto &finish_binder = i == 1 ? *this : *bound_ctes[i - 2].child_binder; + result = finish_binder.FinishCTE(bound_ctes[i - 1], std::move(result)); + } + return result; +} - return BindCTE(statement); +CTEBindState::CTEBindState(Binder &parent_binder_p, QueryNode &cte_def_p, const vector &aliases_p) + : parent_binder(parent_binder_p), cte_def(cte_def_p), aliases(aliases_p), + active_binder_count(parent_binder.GetActiveBinders().size()) { } -unique_ptr Binder::BindCTE(CTENode &statement) { - auto result = make_uniq(); +CTEBindState::~CTEBindState() { +} - // first recursively visit the materialized CTE operations - // the left side is visited first and is added to the BindContext of the right side - D_ASSERT(statement.query); +bool CTEBindState::IsBound() const { + return query_binder.get() != nullptr; +} - result->ctename = statement.ctename; - result->materialized = statement.materialized; - result->setop_index = GenerateTableIndex(); +void CTEBindState::Bind(CTEBinding &binding) { + // we are lazily binding the CTE + // we need to bind it as if we were binding it during PrepareCTE + query_binder = Binder::CreateBinder(parent_binder.context, parent_binder); + + // we clear any expression binders that were added in the mean-time, to ensure we are not binding to any newly added + // correlated columns + auto &active_binders = parent_binder.GetActiveBinders(); + vector> stored_binders; + for (idx_t i = active_binder_count; i < active_binders.size(); i++) { + stored_binders.push_back(active_binders[i]); + } + active_binders.erase(active_binders.begin() + UnsafeNumericCast(active_binder_count), + active_binders.end()); + + // add this CTE to the query binder on the RHS with "CANNOT_BE_REFERENCED" to detect recursive references to + // ourselves + query_binder->bind_context.AddCTEBinding(binding.GetIndex(), binding.GetBindingAlias(), vector(), + vector(), CTEType::CANNOT_BE_REFERENCED); - result->query_binder = Binder::CreateBinder(context, this); - result->query = result->query_binder->BindNode(*statement.query); + // bind the actual CTE + query = query_binder->Bind(cte_def); + + // after binding - we add the active binders we removed back so we can leave the binder in its original state + for (auto &stored_binder : stored_binders) { + active_binders.push_back(stored_binder); + } // the result types of the CTE are the types of the LHS - result->types = result->query->types; + types = query.types; // names are picked from the LHS, unless aliases are explicitly specified - result->names = result->query->names; - for (idx_t i = 0; i < statement.aliases.size() && i < result->names.size(); i++) { - result->names[i] = statement.aliases[i]; + names = query.names; + for (idx_t i = 0; i < aliases.size() && i < names.size(); i++) { + names[i] = aliases[i]; } // Rename columns if duplicate names are detected idx_t index = 1; - vector names; + vector new_names; // Use a case-insensitive set to track names case_insensitive_set_t ci_names; - for (auto &n : result->names) { + for (auto &n : names) { string name = n; while (ci_names.find(name) != ci_names.end()) { name = n + "_" + std::to_string(index++); } - names.push_back(name); + new_names.push_back(name); ci_names.insert(name); } + names = std::move(new_names); +} + +BoundCTEData Binder::PrepareCTE(const string &ctename, CommonTableExpressionInfo &statement) { + BoundCTEData result; + + // first recursively visit the materialized CTE operations + // the left side is visited first and is added to the BindContext of the right side + D_ASSERT(statement.query); - // This allows the right side to reference the CTE - bind_context.AddGenericBinding(result->setop_index, statement.ctename, names, result->types); + result.ctename = ctename; + result.materialized = statement.materialized; + result.setop_index = GenerateTableIndex(); - result->child_binder = Binder::CreateBinder(context, this); + // instead of eagerly binding the CTE here we add the CTE bind state to the list of CTE bindings + // the CTE is bound lazily - when referenced for the first time we perform the binding + result.cte_bind_state = make_shared_ptr(*this, *statement.query->node, statement.aliases); + + result.child_binder = Binder::CreateBinder(context, this); // Add bindings of left side to temporary CTE bindings context - // If there is already a binding for the CTE, we need to remove it first // as we are binding a CTE currently, we take precendence over the existing binding. // This implements the CTE shadowing behavior. - result->child_binder->bind_context.RemoveCTEBinding(statement.ctename); - result->child_binder->bind_context.AddCTEBinding(result->setop_index, statement.ctename, names, result->types); - - if (statement.child) { - // Move all modifiers to the child node. - for (auto &modifier : statement.modifiers) { - statement.child->modifiers.push_back(std::move(modifier)); - } + auto cte_binding = make_uniq(BindingAlias(ctename), result.cte_bind_state, result.setop_index); + result.child_binder->bind_context.AddCTEBinding(std::move(cte_binding)); + return result; +} - statement.modifiers.clear(); +BoundStatement Binder::FinishCTE(BoundCTEData &bound_cte, BoundStatement child) { + if (!bound_cte.cte_bind_state->IsBound()) { + // CTE was not bound - just ignore it + return child; + } + auto &bind_state = *bound_cte.cte_bind_state; + for (auto &c : bind_state.query_binder->correlated_columns) { + bound_cte.child_binder->AddCorrelatedColumn(c); + } - result->child = result->child_binder->BindNode(*statement.child); - for (auto &c : result->query_binder->correlated_columns) { - result->child_binder->AddCorrelatedColumn(c); - } + BoundStatement result; + // the result types of the CTE are the types of the LHS + result.types = child.types; + result.names = child.names; - // the result types of the CTE are the types of the LHS - result->types = result->child->types; - result->names = result->child->names; + MoveCorrelatedExpressions(*bound_cte.child_binder); + MoveCorrelatedExpressions(*bind_state.query_binder); - MoveCorrelatedExpressions(*result->child_binder); - } + auto cte_query = std::move(bind_state.query.plan); + auto cte_child = std::move(child.plan); - MoveCorrelatedExpressions(*result->query_binder); + auto root = make_uniq(bound_cte.ctename, bound_cte.setop_index, result.types.size(), + std::move(cte_query), std::move(cte_child), bound_cte.materialized); + // check if there are any unplanned subqueries left in either child + has_unplanned_dependent_joins = has_unplanned_dependent_joins || + bound_cte.child_binder->has_unplanned_dependent_joins || + bind_state.query_binder->has_unplanned_dependent_joins; + result.plan = std::move(root); return result; } diff --git a/src/planner/binder/query_node/bind_recursive_cte_node.cpp b/src/planner/binder/query_node/bind_recursive_cte_node.cpp index 54e9e9fa5f66..7341d7edebd2 100644 --- a/src/planner/binder/query_node/bind_recursive_cte_node.cpp +++ b/src/planner/binder/query_node/bind_recursive_cte_node.cpp @@ -3,13 +3,12 @@ #include "duckdb/parser/query_node/select_node.hpp" #include "duckdb/parser/query_node/recursive_cte_node.hpp" #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/query_node/bound_recursive_cte_node.hpp" -#include "duckdb/planner/query_node/bound_select_node.hpp" +#include "duckdb/planner/operator/logical_set_operation.hpp" +#include "duckdb/planner/operator/logical_recursive_cte.hpp" namespace duckdb { -unique_ptr Binder::BindNode(RecursiveCTENode &statement) { - auto result = make_uniq(); +BoundStatement Binder::BindNode(RecursiveCTENode &statement) { // first recursively visit the recursive CTE operations // the left side is visited first and is added to the BindContext of the right side @@ -19,53 +18,55 @@ unique_ptr Binder::BindNode(RecursiveCTENode &statement) { throw BinderException("UNION ALL cannot be used with USING KEY in recursive CTE."); } - result->ctename = statement.ctename; - result->union_all = statement.union_all; - result->setop_index = GenerateTableIndex(); + auto ctename = statement.ctename; + auto union_all = statement.union_all; + auto setop_index = GenerateTableIndex(); - result->left_binder = Binder::CreateBinder(context, this); - result->left = result->left_binder->BindNode(*statement.left); + auto left_binder = Binder::CreateBinder(context, this); + auto left = left_binder->BindNode(*statement.left); + BoundStatement result; // the result types of the CTE are the types of the LHS - result->types = result->left->types; + result.types = left.types; // names are picked from the LHS, unless aliases are explicitly specified - result->names = result->left->names; - for (idx_t i = 0; i < statement.aliases.size() && i < result->names.size(); i++) { - result->names[i] = statement.aliases[i]; + result.names = left.names; + for (idx_t i = 0; i < statement.aliases.size() && i < result.names.size(); i++) { + result.names[i] = statement.aliases[i]; } // This allows the right side to reference the CTE recursively - bind_context.AddGenericBinding(result->setop_index, statement.ctename, result->names, result->types); + bind_context.AddGenericBinding(setop_index, statement.ctename, result.names, result.types); - result->right_binder = Binder::CreateBinder(context, this); + auto right_binder = Binder::CreateBinder(context, this); // Add bindings of left side to temporary CTE bindings context - // If there is already a binding for the CTE, we need to remove it first - // as we are binding a CTE currently, we take precendence over the existing binding. - // This implements the CTE shadowing behavior. - result->right_binder->bind_context.RemoveCTEBinding(statement.ctename); - result->right_binder->bind_context.AddCTEBinding(result->setop_index, statement.ctename, result->names, - result->types, !statement.key_targets.empty()); - - result->right = result->right_binder->BindNode(*statement.right); - for (auto &c : result->left_binder->correlated_columns) { - result->right_binder->AddCorrelatedColumn(c); + BindingAlias cte_alias(statement.ctename); + right_binder->bind_context.AddCTEBinding(setop_index, std::move(cte_alias), result.names, result.types); + if (!statement.key_targets.empty()) { + BindingAlias recurring_alias("recurring", statement.ctename); + right_binder->bind_context.AddCTEBinding(setop_index, std::move(recurring_alias), result.names, result.types); + } + + auto right = right_binder->BindNode(*statement.right); + for (auto &c : left_binder->correlated_columns) { + right_binder->AddCorrelatedColumn(c); } // move the correlated expressions from the child binders to this binder - MoveCorrelatedExpressions(*result->left_binder); - MoveCorrelatedExpressions(*result->right_binder); + MoveCorrelatedExpressions(*left_binder); + MoveCorrelatedExpressions(*right_binder); + vector> key_targets; // bind specified keys to the referenced column auto expression_binder = ExpressionBinder(*this, context); - for (unique_ptr &expr : statement.key_targets) { + for (auto &expr : statement.key_targets) { auto bound_expr = expression_binder.Bind(expr); D_ASSERT(bound_expr->type == ExpressionType::BOUND_COLUMN_REF); - result->key_targets.push_back(std::move(bound_expr)); + key_targets.push_back(std::move(bound_expr)); } // now both sides have been bound we can resolve types - if (result->left->types.size() != result->right->types.size()) { + if (left.types.size() != right.types.size()) { throw BinderException("Set operations can only apply to expressions with the " "same number of result columns"); } @@ -74,7 +75,42 @@ unique_ptr Binder::BindNode(RecursiveCTENode &statement) { throw NotImplementedException("FIXME: bind modifiers in recursive CTE"); } - return std::move(result); + // Generate the logical plan for the left and right sides of the set operation + left_binder->is_outside_flattened = is_outside_flattened; + right_binder->is_outside_flattened = is_outside_flattened; + + auto left_node = std::move(left.plan); + auto right_node = std::move(right.plan); + + // check if there are any unplanned subqueries left in either child + has_unplanned_dependent_joins = has_unplanned_dependent_joins || left_binder->has_unplanned_dependent_joins || + right_binder->has_unplanned_dependent_joins; + + // for both the left and right sides, cast them to the same types + left_node = CastLogicalOperatorToTypes(left.types, result.types, std::move(left_node)); + right_node = CastLogicalOperatorToTypes(right.types, result.types, std::move(right_node)); + + auto recurring_binding = right_binder->GetCTEBinding(BindingAlias("recurring", ctename)); + bool ref_recurring = recurring_binding && recurring_binding->IsReferenced(); + if (key_targets.empty() && ref_recurring) { + throw InvalidInputException("RECURRING can only be used with USING KEY in recursive CTE."); + } + + // Check if there is a reference to the recursive or recurring table, if not create a set operator. + auto cte_binding = right_binder->GetCTEBinding(BindingAlias(ctename)); + bool ref_cte = cte_binding && cte_binding->IsReferenced(); + if (!ref_cte && !ref_recurring) { + auto root = + make_uniq(setop_index, result.types.size(), std::move(left_node), + std::move(right_node), LogicalOperatorType::LOGICAL_UNION, union_all); + result.plan = std::move(root); + } else { + auto root = make_uniq(ctename, setop_index, result.types.size(), union_all, + std::move(key_targets), std::move(left_node), std::move(right_node)); + root->ref_recurring = ref_recurring; + result.plan = std::move(root); + } + return result; } } // namespace duckdb diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 4f52dfc4afcc..7bfc4b22ebf3 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -363,7 +363,7 @@ void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vect } } -unique_ptr Binder::BindNode(SelectNode &statement) { +BoundStatement Binder::BindNode(SelectNode &statement) { D_ASSERT(statement.from_table); // first bind the FROM table statement @@ -403,21 +403,22 @@ void Binder::BindWhereStarExpression(unique_ptr &expr) { } } -unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ptr from_table) { - D_ASSERT(from_table); +BoundStatement Binder::BindSelectNode(SelectNode &statement, BoundStatement from_table) { + D_ASSERT(from_table.plan); D_ASSERT(!statement.from_table); - auto result = make_uniq(); - result->projection_index = GenerateTableIndex(); - result->group_index = GenerateTableIndex(); - result->aggregate_index = GenerateTableIndex(); - result->groupings_index = GenerateTableIndex(); - result->window_index = GenerateTableIndex(); - result->prune_index = GenerateTableIndex(); - - result->from_table = std::move(from_table); + auto result_ptr = make_uniq(); + auto &result = *result_ptr; + result.projection_index = GenerateTableIndex(); + result.group_index = GenerateTableIndex(); + result.aggregate_index = GenerateTableIndex(); + result.groupings_index = GenerateTableIndex(); + result.window_index = GenerateTableIndex(); + result.prune_index = GenerateTableIndex(); + + result.from_table = std::move(from_table); // bind the sample clause if (statement.sample) { - result->sample_options = std::move(statement.sample); + result.sample_options = std::move(statement.sample); } // visit the select list and expand any "*" statements @@ -429,19 +430,19 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ } statement.select_list = std::move(new_select_list); - auto &bind_state = result->bind_state; + auto &bind_state = result.bind_state; for (idx_t i = 0; i < statement.select_list.size(); i++) { auto &expr = statement.select_list[i]; - result->names.push_back(expr->GetName()); + result.names.push_back(expr->GetName()); ExpressionBinder::QualifyColumnNames(*this, expr); if (!expr->GetAlias().empty()) { bind_state.alias_map[expr->GetAlias()] = i; - result->names[i] = expr->GetAlias(); + result.names[i] = expr->GetAlias(); } bind_state.projection_map[*expr] = i; bind_state.original_expressions.push_back(expr->Copy()); } - result->column_count = statement.select_list.size(); + result.column_count = statement.select_list.size(); // first visit the WHERE clause // the WHERE clause happens before the GROUP BY, PROJECTION or HAVING clauses @@ -452,12 +453,12 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ ColumnAliasBinder alias_binder(bind_state); WhereBinder where_binder(*this, context, &alias_binder); unique_ptr condition = std::move(statement.where_clause); - result->where_clause = where_binder.Bind(condition); + result.where_clause = where_binder.Bind(condition); } // now bind all the result modifiers; including DISTINCT and ORDER BY targets OrderBinder order_binder({*this}, statement, bind_state); - PrepareModifiers(order_binder, statement, *result); + PrepareModifiers(order_binder, statement, result); vector> unbound_groups; BoundGroupInformation info; @@ -465,7 +466,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ if (!group_expressions.empty()) { // the statement has a GROUP BY clause, bind it unbound_groups.resize(group_expressions.size()); - GroupBinder group_binder(*this, context, statement, result->group_index, bind_state, info.alias_map); + GroupBinder group_binder(*this, context, statement, result.group_index, bind_state, info.alias_map); for (idx_t i = 0; i < group_expressions.size(); i++) { // we keep a copy of the unbound expression; @@ -489,7 +490,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ if (!contains_subquery && requires_collation) { // if there is a collation on a group x, we should group by the collated expr, // but also push a first(x) aggregate in case x is selected (uncollated) - info.collated_groups[i] = result->aggregates.size(); + info.collated_groups[i] = result.aggregates.size(); auto first_fun = FirstFunctionGetter::GetFunction(bound_expr_ref.return_type); vector> first_children; @@ -499,9 +500,9 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ FunctionBinder function_binder(*this); auto function = function_binder.BindAggregateFunction(first_fun, std::move(first_children)); function->SetAlias("__collated_group"); - result->aggregates.push_back(std::move(function)); + result.aggregates.push_back(std::move(function)); } - result->groups.group_expressions.push_back(std::move(bound_expr)); + result.groups.group_expressions.push_back(std::move(bound_expr)); // in the unbound expression we DO bind the table names of any ColumnRefs // we do this to make sure that "table.a" and "a" are treated the same @@ -512,13 +513,13 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ info.map[*unbound_groups[i]] = i; } } - result->groups.grouping_sets = std::move(statement.groups.grouping_sets); + result.groups.grouping_sets = std::move(statement.groups.grouping_sets); // bind the HAVING clause, if any if (statement.having) { - HavingBinder having_binder(*this, context, *result, info, statement.aggregate_handling); + HavingBinder having_binder(*this, context, result, info, statement.aggregate_handling); ExpressionBinder::QualifyColumnNames(having_binder, statement.having); - result->having = having_binder.Bind(statement.having); + result.having = having_binder.Bind(statement.having); } // bind the QUALIFY clause, if any @@ -527,9 +528,9 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ if (statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) { throw BinderException("Combining QUALIFY with GROUP BY ALL is not supported yet"); } - QualifyBinder qualify_binder(*this, context, *result, info); + QualifyBinder qualify_binder(*this, context, result, info); ExpressionBinder::QualifyColumnNames(*this, statement.qualify); - result->qualify = qualify_binder.Bind(statement.qualify); + result.qualify = qualify_binder.Bind(statement.qualify); if (qualify_binder.HasBoundColumns()) { if (qualify_binder.BoundAggregates()) { throw BinderException("Cannot mix aggregates with non-aggregated columns!"); @@ -539,7 +540,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ } // after that, we bind to the SELECT list - SelectBinder select_binder(*this, context, *result, info); + SelectBinder select_binder(*this, context, result, info); // if we expand select-list expressions, e.g., via UNNEST, then we need to possibly // adjust the column index of the already bound ORDER BY modifiers, and not only set their types @@ -549,13 +550,13 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ for (idx_t i = 0; i < statement.select_list.size(); i++) { bool is_window = statement.select_list[i]->IsWindow(); - idx_t unnest_count = result->unnests.size(); + idx_t unnest_count = result.unnests.size(); LogicalType result_type; auto expr = select_binder.Bind(statement.select_list[i], &result_type, true); - bool is_original_column = i < result->column_count; + bool is_original_column = i < result.column_count; bool can_group_by_all = statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES && is_original_column; - result->bound_column_count++; + result.bound_column_count++; if (expr->GetExpressionType() == ExpressionType::BOUND_EXPANDED) { if (!is_original_column) { @@ -571,9 +572,9 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ for (auto &struct_expr : struct_expressions) { new_names.push_back(struct_expr->GetName()); - result->types.push_back(struct_expr->return_type); + result.types.push_back(struct_expr->return_type); internal_sql_types.push_back(struct_expr->return_type); - result->select_list.push_back(std::move(struct_expr)); + result.select_list.push_back(std::move(struct_expr)); } bind_state.AddExpandedColumn(struct_expressions.size()); continue; @@ -594,7 +595,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ if (is_window) { throw BinderException("Cannot group on a window clause"); } - if (result->unnests.size() > unnest_count) { + if (result.unnests.size() > unnest_count) { throw BinderException("Cannot group on an UNNEST or UNLIST clause"); } // we are forcing aggregates, and the node has columns bound @@ -602,10 +603,10 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ group_by_all_indexes.push_back(i); } - result->select_list.push_back(std::move(expr)); + result.select_list.push_back(std::move(expr)); if (is_original_column) { - new_names.push_back(std::move(result->names[i])); - result->types.push_back(result_type); + new_names.push_back(std::move(result.names[i])); + result.types.push_back(result_type); } internal_sql_types.push_back(result_type); @@ -617,31 +618,31 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ // push the GROUP BY ALL expressions into the group set for (auto &group_by_all_index : group_by_all_indexes) { - auto &expr = result->select_list[group_by_all_index]; + auto &expr = result.select_list[group_by_all_index]; auto group_ref = make_uniq( - expr->return_type, ColumnBinding(result->group_index, result->groups.group_expressions.size())); - result->groups.group_expressions.push_back(std::move(expr)); + expr->return_type, ColumnBinding(result.group_index, result.groups.group_expressions.size())); + result.groups.group_expressions.push_back(std::move(expr)); expr = std::move(group_ref); } set group_by_all_indexes_set; if (!group_by_all_indexes.empty()) { - idx_t num_set_indexes = result->groups.group_expressions.size(); + idx_t num_set_indexes = result.groups.group_expressions.size(); for (idx_t i = 0; i < num_set_indexes; i++) { group_by_all_indexes_set.insert(i); } - D_ASSERT(result->groups.grouping_sets.empty()); - result->groups.grouping_sets.push_back(group_by_all_indexes_set); + D_ASSERT(result.groups.grouping_sets.empty()); + result.groups.grouping_sets.push_back(group_by_all_indexes_set); } - result->column_count = new_names.size(); - result->names = std::move(new_names); - result->need_prune = result->select_list.size() > result->column_count; + result.column_count = new_names.size(); + result.names = std::move(new_names); + result.need_prune = result.select_list.size() > result.column_count; // in the normal select binder, we bind columns as if there is no aggregation // i.e. in the query [SELECT i, SUM(i) FROM integers;] the "i" will be bound as a normal column // since we have an aggregation, we need to either (1) throw an error, or (2) wrap the column in a FIRST() aggregate // we choose the former one [CONTROVERSIAL: this is the PostgreSQL behavior] - if (!result->groups.group_expressions.empty() || !result->aggregates.empty() || statement.having || - !result->groups.grouping_sets.empty()) { + if (!result.groups.group_expressions.empty() || !result.aggregates.empty() || statement.having || + !result.groups.grouping_sets.empty()) { if (statement.aggregate_handling == AggregateHandling::NO_AGGREGATES_ALLOWED) { throw BinderException("Aggregates cannot be present in a Project relation!"); } else { @@ -672,13 +673,19 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ // QUALIFY clause requires at least one window function to be specified in at least one of the SELECT column list or // the filter predicate of the QUALIFY clause - if (statement.qualify && result->windows.empty()) { + if (statement.qualify && result.windows.empty()) { throw BinderException("at least one window function must appear in the SELECT column or QUALIFY clause"); } // now that the SELECT list is bound, we set the types of DISTINCT/ORDER BY expressions - BindModifiers(*result, result->projection_index, result->names, internal_sql_types, bind_state); - return std::move(result); + BindModifiers(result, result.projection_index, result.names, internal_sql_types, bind_state); + + BoundStatement result_statement; + result_statement.types = result.types; + result_statement.names = result.names; + result_statement.plan = CreatePlan(result); + result_statement.extra_info.original_expressions = std::move(result.bind_state.original_expressions); + return result_statement; } } // namespace duckdb diff --git a/src/planner/binder/query_node/bind_setop_node.cpp b/src/planner/binder/query_node/bind_setop_node.cpp index 82d6c754bc0d..91a501b2fb40 100644 --- a/src/planner/binder/query_node/bind_setop_node.cpp +++ b/src/planner/binder/query_node/bind_setop_node.cpp @@ -10,267 +10,308 @@ #include "duckdb/planner/query_node/bound_select_node.hpp" #include "duckdb/planner/query_node/bound_set_operation_node.hpp" #include "duckdb/planner/expression_binder/select_bind_state.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" #include "duckdb/common/enum_util.hpp" namespace duckdb { -static void GatherAliases(BoundQueryNode &node, SelectBindState &bind_state, const vector &reorder_idx) { - if (node.type == QueryNodeType::SET_OPERATION_NODE) { - // setop, recurse - auto &setop = node.Cast(); +struct SetOpAliasGatherer { +public: + explicit SetOpAliasGatherer(SelectBindState &bind_state_p) : bind_state(bind_state_p) { + } - // create new reorder index - if (setop.setop_type == SetOperationType::UNION_BY_NAME) { - vector new_left_reorder_idx(setop.left_reorder_idx.size()); - vector new_right_reorder_idx(setop.right_reorder_idx.size()); - for (idx_t i = 0; i < setop.left_reorder_idx.size(); ++i) { - new_left_reorder_idx[i] = reorder_idx[setop.left_reorder_idx[i]]; - } + void GatherAliases(BoundStatement &stmt, const vector &reorder_idx); + void GatherSetOpAliases(SetOperationType setop_type, const vector &names, + vector &bound_children, const vector &reorder_idx); - for (idx_t i = 0; i < setop.right_reorder_idx.size(); ++i) { - new_right_reorder_idx[i] = reorder_idx[setop.right_reorder_idx[i]]; - } +private: + SelectBindState &bind_state; +}; - // use new reorder index - GatherAliases(*setop.left, bind_state, new_left_reorder_idx); - GatherAliases(*setop.right, bind_state, new_right_reorder_idx); - return; - } +void SetOpAliasGatherer::GatherAliases(BoundStatement &stmt, const vector &reorder_idx) { + if (stmt.extra_info.setop_type != SetOperationType::NONE) { + GatherSetOpAliases(stmt.extra_info.setop_type, stmt.names, stmt.extra_info.bound_children, reorder_idx); + return; + } - GatherAliases(*setop.left, bind_state, reorder_idx); - GatherAliases(*setop.right, bind_state, reorder_idx); - } else { - // query node - D_ASSERT(node.type == QueryNodeType::SELECT_NODE); - auto &select = node.Cast(); - // fill the alias lists with the names - for (idx_t i = 0; i < select.names.size(); i++) { - auto &name = select.names[i]; - // first check if the alias is already in there - auto entry = bind_state.alias_map.find(name); + // query node + auto &select_names = stmt.names; + // fill the alias lists with the names + D_ASSERT(reorder_idx.size() == select_names.size()); + for (idx_t i = 0; i < select_names.size(); i++) { + auto &name = select_names[i]; + // first check if the alias is already in there + auto entry = bind_state.alias_map.find(name); - idx_t index = reorder_idx[i]; + idx_t index = reorder_idx[i]; - if (entry == bind_state.alias_map.end()) { - // the alias is not in there yet, just assign it - bind_state.alias_map[name] = index; + if (entry == bind_state.alias_map.end()) { + // the alias is not in there yet, just assign it + bind_state.alias_map[name] = index; + } + } + // check if the expression matches one of the expressions in the original expression list + auto &select_list = stmt.extra_info.original_expressions; + for (idx_t i = 0; i < select_list.size(); i++) { + auto &expr = select_list[i]; + idx_t index = reorder_idx[i]; + // now check if the node is already in the set of expressions + auto expr_entry = bind_state.projection_map.find(*expr); + if (expr_entry != bind_state.projection_map.end()) { + // the node is in there + // repeat the same as with the alias: if there is an ambiguity we insert "-1" + if (expr_entry->second != index) { + bind_state.projection_map[*expr] = DConstants::INVALID_INDEX; } + } else { + // not in there yet, just place it in there + bind_state.projection_map[*expr] = index; } - // check if the expression matches one of the expressions in the original expression liset - for (idx_t i = 0; i < select.bind_state.original_expressions.size(); i++) { - auto &expr = select.bind_state.original_expressions[i]; - idx_t index = reorder_idx[i]; - // now check if the node is already in the set of expressions - auto expr_entry = bind_state.projection_map.find(*expr); - if (expr_entry != bind_state.projection_map.end()) { - // the node is in there - // repeat the same as with the alias: if there is an ambiguity we insert "-1" - if (expr_entry->second != index) { - bind_state.projection_map[*expr] = DConstants::INVALID_INDEX; + } +} + +void SetOpAliasGatherer::GatherSetOpAliases(SetOperationType setop_type, const vector &stmt_names, + vector &bound_children, const vector &reorder_idx) { + // create new reorder index + if (setop_type == SetOperationType::UNION_BY_NAME) { + auto &setop_names = stmt_names; + // for UNION BY NAME - create a new re-order index + case_insensitive_map_t reorder_map; + for (idx_t col_idx = 0; col_idx < setop_names.size(); ++col_idx) { + reorder_map[setop_names[col_idx]] = reorder_idx[col_idx]; + } + + // use new reorder index + for (auto &child : bound_children) { + vector new_reorder_idx; + auto &child_names = child.names; + for (idx_t col_idx = 0; col_idx < child_names.size(); col_idx++) { + auto &col_name = child_names[col_idx]; + auto entry = reorder_map.find(col_name); + if (entry == reorder_map.end()) { + throw InternalException("SetOp - Column name not found in reorder_map in UNION BY NAME"); } - } else { - // not in there yet, just place it in there - bind_state.projection_map[*expr] = index; + new_reorder_idx.push_back(entry->second); } + GatherAliases(child, new_reorder_idx); + } + } else { + for (auto &child : bound_children) { + GatherAliases(child, reorder_idx); } } } -static void BuildUnionByNameInfo(ClientContext &context, BoundSetOperationNode &result, bool can_contain_nulls) { - D_ASSERT(result.setop_type == SetOperationType::UNION_BY_NAME); - case_insensitive_map_t left_names_map; - case_insensitive_map_t right_names_map; +static void GatherAliases(BoundSetOperationNode &root, vector &child_statements, + SelectBindState &bind_state) { + SetOpAliasGatherer gatherer(bind_state); + vector reorder_idx; + for (idx_t i = 0; i < root.names.size(); i++) { + reorder_idx.push_back(i); + } + gatherer.GatherSetOpAliases(root.setop_type, root.names, child_statements, reorder_idx); +} - auto &left_node = *result.left; - auto &right_node = *result.right; +void Binder::BuildUnionByNameInfo(BoundSetOperationNode &result) { + D_ASSERT(result.setop_type == SetOperationType::UNION_BY_NAME); + vector> node_name_maps; + case_insensitive_set_t global_name_set; // Build a name_map to use to check if a name exists // We throw a binder exception if two same name in the SELECT list - for (idx_t i = 0; i < left_node.names.size(); ++i) { - if (left_names_map.find(left_node.names[i]) != left_names_map.end()) { - throw BinderException("UNION (ALL) BY NAME operation doesn't support duplicate names in the SELECT list - " - "the name \"%s\" occurs multiple times in the left-hand side", - left_node.names[i]); - } - left_names_map[left_node.names[i]] = i; - } - - for (idx_t i = 0; i < right_node.names.size(); ++i) { - if (right_names_map.find(right_node.names[i]) != right_names_map.end()) { - throw BinderException("UNION (ALL) BY NAME operation doesn't support duplicate names in the SELECT list - " - "the name \"%s\" occurs multiple times in the right-hand side", - right_node.names[i]); - } - if (left_names_map.find(right_node.names[i]) == left_names_map.end()) { - result.names.push_back(right_node.names[i]); + D_ASSERT(result.names.empty()); + for (auto &child : result.bound_children) { + auto &child_names = child.names; + case_insensitive_map_t node_name_map; + for (idx_t i = 0; i < child_names.size(); ++i) { + auto &col_name = child_names[i]; + if (node_name_map.find(col_name) != node_name_map.end()) { + throw BinderException( + "UNION (ALL) BY NAME operation doesn't support duplicate names in the SELECT list - " + "the name \"%s\" occurs multiple times", + col_name); + } + if (global_name_set.find(col_name) == global_name_set.end()) { + // column is not yet present in the result + result.names.push_back(col_name); + global_name_set.insert(col_name); + } + node_name_map[col_name] = i; } - right_names_map[right_node.names[i]] = i; + node_name_maps.push_back(std::move(node_name_map)); } idx_t new_size = result.names.size(); bool need_reorder = false; - vector left_reorder_idx(left_node.names.size()); - vector right_reorder_idx(right_node.names.size()); - // Construct return type and reorder_idxs - // reorder_idxs is used to gather correct alias_map - // and expression_map in GatherAlias(...) + // construct the return type of each of the columns for (idx_t i = 0; i < new_size; ++i) { - auto left_index = left_names_map.find(result.names[i]); - auto right_index = right_names_map.find(result.names[i]); - bool left_exist = left_index != left_names_map.end(); - bool right_exist = right_index != right_names_map.end(); - LogicalType result_type; - if (left_exist && right_exist) { - result_type = LogicalType::ForceMaxLogicalType(left_node.types[left_index->second], - right_node.types[right_index->second]); - if (left_index->second != i || right_index->second != i) { + auto &col_name = result.names[i]; + LogicalType result_type(LogicalTypeId::INVALID); + for (idx_t child_idx = 0; child_idx < result.bound_children.size(); ++child_idx) { + auto &child_types = result.bound_children[child_idx].types; + auto &child_name_map = node_name_maps[child_idx]; + // check if the column exists in this child node + auto entry = child_name_map.find(col_name); + if (entry == child_name_map.end()) { need_reorder = true; + } else { + auto col_idx_in_child = entry->second; + auto &child_col_type = child_types[col_idx_in_child]; + // the child exists in this node - compute the type + if (result_type.id() == LogicalTypeId::INVALID) { + result_type = child_col_type; + } else { + result_type = LogicalType::ForceMaxLogicalType(result_type, child_col_type); + } + if (i != col_idx_in_child) { + // the column exists - but the children are out-of-order, so we need to re-order anyway + need_reorder = true; + } } - left_reorder_idx[left_index->second] = i; - right_reorder_idx[right_index->second] = i; - } else if (left_exist) { - result_type = left_node.types[left_index->second]; - need_reorder = true; - left_reorder_idx[left_index->second] = i; - } else { - D_ASSERT(right_exist); - result_type = right_node.types[right_index->second]; - need_reorder = true; - right_reorder_idx[right_index->second] = i; } - + // compute the final type for each column if (!can_contain_nulls) { if (ExpressionBinder::ContainsNullType(result_type)) { result_type = ExpressionBinder::ExchangeNullType(result_type); } } - result.types.push_back(result_type); } - result.left_reorder_idx = std::move(left_reorder_idx); - result.right_reorder_idx = std::move(right_reorder_idx); - - // If reorder is required, collect reorder expressions for push projection - // into the two child nodes of union node - if (need_reorder) { - for (idx_t i = 0; i < new_size; ++i) { - auto left_index = left_names_map.find(result.names[i]); - auto right_index = right_names_map.find(result.names[i]); - bool left_exist = left_index != left_names_map.end(); - bool right_exist = right_index != right_names_map.end(); - unique_ptr left_reorder_expr; - unique_ptr right_reorder_expr; - if (left_exist && right_exist) { - left_reorder_expr = make_uniq( - left_node.types[left_index->second], ColumnBinding(left_node.GetRootIndex(), left_index->second)); - right_reorder_expr = - make_uniq(right_node.types[right_index->second], - ColumnBinding(right_node.GetRootIndex(), right_index->second)); - } else if (left_exist) { - left_reorder_expr = make_uniq( - left_node.types[left_index->second], ColumnBinding(left_node.GetRootIndex(), left_index->second)); - // create null value here - right_reorder_expr = make_uniq(Value(result.types[i])); + if (!need_reorder) { + // if all columns in the children of the set-operations are identical we don't need to re-order at all + // skip adding expressions entirely + return; + } + // If reorder is required, generate the expressions for each node + vector>> reorder_expressions; + reorder_expressions.resize(result.bound_children.size()); + for (idx_t i = 0; i < new_size; ++i) { + auto &col_name = result.names[i]; + for (idx_t child_idx = 0; child_idx < result.bound_children.size(); ++child_idx) { + auto &child = result.bound_children[child_idx]; + auto &child_name_map = node_name_maps[child_idx]; + // check if the column exists in this child node + auto entry = child_name_map.find(col_name); + unique_ptr expr; + if (entry == child_name_map.end()) { + // the column does not exist - push a `NULL` + expr = make_uniq(Value(result.types[i])); } else { - D_ASSERT(right_exist); - left_reorder_expr = make_uniq(Value(result.types[i])); - right_reorder_expr = - make_uniq(right_node.types[right_index->second], - ColumnBinding(right_node.GetRootIndex(), right_index->second)); + // the column exists - reference it + auto col_idx_in_child = entry->second; + auto &child_col_type = child.types[col_idx_in_child]; + auto root_idx = child.plan->GetRootIndex(); + expr = make_uniq(child_col_type, ColumnBinding(root_idx, col_idx_in_child)); } - result.left_reorder_exprs.push_back(std::move(left_reorder_expr)); - result.right_reorder_exprs.push_back(std::move(right_reorder_expr)); + reorder_expressions[child_idx].push_back(std::move(expr)); } } + // now push projections for each node + for (idx_t child_idx = 0; child_idx < result.bound_children.size(); ++child_idx) { + auto &child = result.bound_children[child_idx]; + auto &child_reorder_expressions = reorder_expressions[child_idx]; + // if we have re-order expressions push a projection + vector child_types; + for (auto &expr : child_reorder_expressions) { + child_types.push_back(expr->return_type); + } + auto child_projection = + make_uniq(GenerateTableIndex(), std::move(child_reorder_expressions)); + child_projection->children.push_back(std::move(child.plan)); + child.plan = std::move(child_projection); + child.types = std::move(child_types); + } } -static void GatherSetOpBinders(BoundQueryNode &node, Binder &binder, vector> &binders) { - if (node.type != QueryNodeType::SET_OPERATION_NODE) { - binders.push_back(binder); - return; +static void GatherSetOpBinders(vector &children, vector> &binders, + vector> &result) { + for (auto &child_binder : binders) { + result.push_back(*child_binder); + } + for (auto &child_node : children) { + GatherSetOpBinders(child_node.extra_info.bound_children, child_node.extra_info.child_binders, result); } - auto &setop_node = node.Cast(); - GatherSetOpBinders(*setop_node.left, *setop_node.left_binder, binders); - GatherSetOpBinders(*setop_node.right, *setop_node.right_binder, binders); } -unique_ptr Binder::BindNode(SetOperationNode &statement) { - auto result = make_uniq(); - result->setop_type = statement.setop_type; - result->setop_all = statement.setop_all; +BoundStatement Binder::BindNode(SetOperationNode &statement) { + BoundSetOperationNode result; + result.setop_type = statement.setop_type; + result.setop_all = statement.setop_all; // first recursively visit the set operations - // both the left and right sides have an independent BindContext and Binder - D_ASSERT(statement.left); - D_ASSERT(statement.right); - - result->setop_index = GenerateTableIndex(); - - result->left_binder = Binder::CreateBinder(context, this); - result->left_binder->can_contain_nulls = true; - result->left = result->left_binder->BindNode(*statement.left); - result->right_binder = Binder::CreateBinder(context, this); - result->right_binder->can_contain_nulls = true; - result->right = result->right_binder->BindNode(*statement.right); - - result->names = result->left->names; - - // move the correlated expressions from the child binders to this binder - MoveCorrelatedExpressions(*result->left_binder); - MoveCorrelatedExpressions(*result->right_binder); - - // now both sides have been bound we can resolve types - if (result->setop_type != SetOperationType::UNION_BY_NAME && - result->left->types.size() != result->right->types.size()) { - throw BinderException("Set operations can only apply to expressions with the " - "same number of result columns"); + // all children have an independent BindContext and Binder + result.setop_index = GenerateTableIndex(); + if (statement.children.size() < 2) { + throw InternalException("Set Operations must have at least 2 children"); + } + if (statement.children.size() != 2 && statement.setop_type != SetOperationType::UNION && + statement.setop_type != SetOperationType::UNION_BY_NAME) { + throw InternalException("Set Operation type must have exactly 2 children - except for UNION/UNION_BY_NAME"); + } + for (auto &child : statement.children) { + auto child_binder = Binder::CreateBinder(context, this); + child_binder->can_contain_nulls = true; + auto child_node = child_binder->BindNode(*child); + MoveCorrelatedExpressions(*child_binder); + result.bound_children.push_back(std::move(child_node)); + result.child_binders.push_back(std::move(child_binder)); } - if (result->setop_type == SetOperationType::UNION_BY_NAME) { - BuildUnionByNameInfo(context, *result, can_contain_nulls); + if (result.setop_type == SetOperationType::UNION_BY_NAME) { + // UNION BY NAME - merge the columns from all sides + BuildUnionByNameInfo(result); } else { + // UNION ALL BY POSITION - the columns of both sides must match exactly + result.names = result.bound_children[0].names; + auto result_columns = result.bound_children[0].types.size(); + for (idx_t i = 1; i < result.bound_children.size(); ++i) { + if (result.bound_children[i].types.size() != result_columns) { + throw BinderException("Set operations can only apply to expressions with the " + "same number of result columns"); + } + } + // figure out the types of the setop result by picking the max of both - for (idx_t i = 0; i < result->left->types.size(); i++) { - auto result_type = LogicalType::ForceMaxLogicalType(result->left->types[i], result->right->types[i]); + for (idx_t i = 0; i < result_columns; i++) { + auto result_type = result.bound_children[0].types[i]; + for (idx_t child_idx = 1; child_idx < result.bound_children.size(); ++child_idx) { + auto &child_types = result.bound_children[child_idx].types; + result_type = LogicalType::ForceMaxLogicalType(result_type, child_types[i]); + } if (!can_contain_nulls) { if (ExpressionBinder::ContainsNullType(result_type)) { result_type = ExpressionBinder::ExchangeNullType(result_type); } } - result->types.push_back(result_type); + result.types.push_back(result_type); } } SelectBindState bind_state; if (!statement.modifiers.empty()) { // handle the ORDER BY/DISTINCT clauses + vector> binders; + GatherSetOpBinders(result.bound_children, result.child_binders, binders); + GatherAliases(result, result.bound_children, bind_state); - // we recursively visit the children of this node to extract aliases and expressions that can be referenced - // in the ORDER BY - - if (result->setop_type == SetOperationType::UNION_BY_NAME) { - GatherAliases(*result->left, bind_state, result->left_reorder_idx); - GatherAliases(*result->right, bind_state, result->right_reorder_idx); - } else { - vector reorder_idx; - for (idx_t i = 0; i < result->names.size(); i++) { - reorder_idx.push_back(i); - } - GatherAliases(*result, bind_state, reorder_idx); - } // now we perform the actual resolution of the ORDER BY/DISTINCT expressions - vector> binders; - GatherSetOpBinders(*result->left, *result->left_binder, binders); - GatherSetOpBinders(*result->right, *result->right_binder, binders); OrderBinder order_binder(binders, bind_state); - PrepareModifiers(order_binder, statement, *result); + PrepareModifiers(order_binder, statement, result); } // finally bind the types of the ORDER/DISTINCT clause expressions - BindModifiers(*result, result->setop_index, result->names, result->types, bind_state); - return std::move(result); + BindModifiers(result, result.setop_index, result.names, result.types, bind_state); + + BoundStatement result_statement; + result_statement.types = result.types; + result_statement.names = result.names; + result_statement.plan = CreatePlan(result); + result_statement.extra_info.setop_type = statement.setop_type; + result_statement.extra_info.bound_children = std::move(result.bound_children); + result_statement.extra_info.child_binders = std::move(result.child_binders); + return result_statement; } } // namespace duckdb diff --git a/src/planner/binder/query_node/bind_statement_node.cpp b/src/planner/binder/query_node/bind_statement_node.cpp new file mode 100644 index 000000000000..6f6f9941a5cd --- /dev/null +++ b/src/planner/binder/query_node/bind_statement_node.cpp @@ -0,0 +1,26 @@ +#include "duckdb/parser/query_node/statement_node.hpp" +#include "duckdb/parser/statement/insert_statement.hpp" +#include "duckdb/parser/statement/update_statement.hpp" +#include "duckdb/parser/statement/delete_statement.hpp" +#include "duckdb/parser/statement/merge_into_statement.hpp" +#include "duckdb/planner/binder.hpp" + +namespace duckdb { + +BoundStatement Binder::BindNode(StatementNode &statement) { + // switch on type here to ensure we bind WITHOUT ctes to prevent infinite recursion + switch (statement.stmt.type) { + case StatementType::INSERT_STATEMENT: + return Bind(statement.stmt.Cast()); + case StatementType::DELETE_STATEMENT: + return Bind(statement.stmt.Cast()); + case StatementType::UPDATE_STATEMENT: + return Bind(statement.stmt.Cast()); + case StatementType::MERGE_INTO_STATEMENT: + return Bind(statement.stmt.Cast()); + default: + return Bind(statement.stmt); + } +} + +} // namespace duckdb diff --git a/src/planner/binder/query_node/plan_cte_node.cpp b/src/planner/binder/query_node/plan_cte_node.cpp deleted file mode 100644 index 5bd06c0e5c06..000000000000 --- a/src/planner/binder/query_node/plan_cte_node.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "duckdb/common/string_util.hpp" -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/expression/bound_cast_expression.hpp" -#include "duckdb/planner/operator/logical_materialized_cte.hpp" -#include "duckdb/planner/operator/logical_projection.hpp" -#include "duckdb/planner/operator/logical_set_operation.hpp" -#include "duckdb/planner/query_node/bound_cte_node.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundCTENode &node) { - // Generate the logical plan for the cte_query and child. - auto cte_query = CreatePlan(*node.query); - auto cte_child = CreatePlan(*node.child); - - auto root = make_uniq(node.ctename, node.setop_index, node.types.size(), - std::move(cte_query), std::move(cte_child), node.materialized); - - // check if there are any unplanned subqueries left in either child - has_unplanned_dependent_joins = has_unplanned_dependent_joins || node.child_binder->has_unplanned_dependent_joins || - node.query_binder->has_unplanned_dependent_joins; - - return VisitQueryNode(node, std::move(root)); -} - -unique_ptr Binder::CreatePlan(BoundCTENode &node, unique_ptr base) { - // Generate the logical plan for the cte_query and child. - auto cte_query = CreatePlan(*node.query); - unique_ptr root; - if (node.child && node.child->type == QueryNodeType::CTE_NODE) { - root = CreatePlan(node.child->Cast(), std::move(base)); - } else if (node.child) { - root = CreatePlan(*node.child); - } else { - root = std::move(base); - } - - // Only keep the materialized CTE, if it is used - if (node.child_binder->bind_context.cte_references[node.ctename] && - *node.child_binder->bind_context.cte_references[node.ctename] > 0) { - - // Push the CTE through single-child operators so query modifiers appear ABOVE the CTE (internal issue #2652) - // Otherwise, we may have a LIMIT on top of the CTE, and an ORDER BY in the query, and we can't make a TopN - reference> cte_child = root; - while (cte_child.get()->children.size() == 1 && cte_child.get()->type != LogicalOperatorType::LOGICAL_CTE_REF) { - cte_child = cte_child.get()->children[0]; - } - cte_child.get() = - make_uniq(node.ctename, node.setop_index, node.types.size(), std::move(cte_query), - std::move(cte_child.get()), node.materialized); - - // check if there are any unplanned subqueries left in either child - has_unplanned_dependent_joins = has_unplanned_dependent_joins || - node.child_binder->has_unplanned_dependent_joins || - node.query_binder->has_unplanned_dependent_joins; - } - return VisitQueryNode(node, std::move(root)); -} - -} // namespace duckdb diff --git a/src/planner/binder/query_node/plan_recursive_cte_node.cpp b/src/planner/binder/query_node/plan_recursive_cte_node.cpp deleted file mode 100644 index 5863db90d50c..000000000000 --- a/src/planner/binder/query_node/plan_recursive_cte_node.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "duckdb/planner/expression/bound_cast_expression.hpp" -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/operator/logical_projection.hpp" -#include "duckdb/planner/operator/logical_recursive_cte.hpp" -#include "duckdb/planner/operator/logical_set_operation.hpp" -#include "duckdb/planner/query_node/bound_recursive_cte_node.hpp" -#include "duckdb/common/string_util.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundRecursiveCTENode &node) { - // Generate the logical plan for the left and right sides of the set operation - node.left_binder->is_outside_flattened = is_outside_flattened; - node.right_binder->is_outside_flattened = is_outside_flattened; - - auto left_node = node.left_binder->CreatePlan(*node.left); - auto right_node = node.right_binder->CreatePlan(*node.right); - - // check if there are any unplanned subqueries left in either child - has_unplanned_dependent_joins = has_unplanned_dependent_joins || node.left_binder->has_unplanned_dependent_joins || - node.right_binder->has_unplanned_dependent_joins; - - // for both the left and right sides, cast them to the same types - left_node = CastLogicalOperatorToTypes(node.left->types, node.types, std::move(left_node)); - right_node = CastLogicalOperatorToTypes(node.right->types, node.types, std::move(right_node)); - - bool ref_recurring = node.right_binder->bind_context.cte_references["recurring." + node.ctename] && - *node.right_binder->bind_context.cte_references["recurring." + node.ctename] != 0; - - if (node.key_targets.empty() && ref_recurring) { - throw InvalidInputException("RECURRING can only be used with USING KEY in recursive CTE."); - } - - // Check if there is a reference to the recursive or recurring table, if not create a set operator. - if ((!node.right_binder->bind_context.cte_references[node.ctename] || - *node.right_binder->bind_context.cte_references[node.ctename] == 0) && - !ref_recurring) { - auto root = make_uniq(node.setop_index, node.types.size(), std::move(left_node), - std::move(right_node), LogicalOperatorType::LOGICAL_UNION, true); - return VisitQueryNode(node, std::move(root)); - } - - auto root = - make_uniq(node.ctename, node.setop_index, node.types.size(), node.union_all, - std::move(node.key_targets), std::move(left_node), std::move(right_node)); - root->ref_recurring = ref_recurring; - return VisitQueryNode(node, std::move(root)); -} - -} // namespace duckdb diff --git a/src/planner/binder/query_node/plan_select_node.cpp b/src/planner/binder/query_node/plan_select_node.cpp index 46e5d2e12751..10b206f245cc 100644 --- a/src/planner/binder/query_node/plan_select_node.cpp +++ b/src/planner/binder/query_node/plan_select_node.cpp @@ -16,10 +16,8 @@ unique_ptr Binder::PlanFilter(unique_ptr condition, } unique_ptr Binder::CreatePlan(BoundSelectNode &statement) { - unique_ptr root; - D_ASSERT(statement.from_table); - root = CreatePlan(*statement.from_table); - D_ASSERT(root); + D_ASSERT(statement.from_table.plan); + auto root = std::move(statement.from_table.plan); // plan the sample clause if (statement.sample_options) { diff --git a/src/planner/binder/query_node/plan_setop.cpp b/src/planner/binder/query_node/plan_setop.cpp index ba929bcd802f..a1a7f60b0653 100644 --- a/src/planner/binder/query_node/plan_setop.cpp +++ b/src/planner/binder/query_node/plan_setop.cpp @@ -10,8 +10,8 @@ namespace duckdb { // Optionally push a PROJECTION operator -unique_ptr Binder::CastLogicalOperatorToTypes(vector &source_types, - vector &target_types, +unique_ptr Binder::CastLogicalOperatorToTypes(const vector &source_types, + const vector &target_types, unique_ptr op) { D_ASSERT(op); // first check if we even need to cast @@ -93,45 +93,6 @@ unique_ptr Binder::CastLogicalOperatorToTypes(vector Binder::CreatePlan(BoundSetOperationNode &node) { - // Generate the logical plan for the left and right sides of the set operation - node.left_binder->is_outside_flattened = is_outside_flattened; - node.right_binder->is_outside_flattened = is_outside_flattened; - - auto left_node = node.left_binder->CreatePlan(*node.left); - auto right_node = node.right_binder->CreatePlan(*node.right); - - // Add a new projection to child node - D_ASSERT(node.left_reorder_exprs.size() == node.right_reorder_exprs.size()); - if (!node.left_reorder_exprs.empty()) { - D_ASSERT(node.setop_type == SetOperationType::UNION_BY_NAME); - vector left_types; - vector right_types; - // We are going to add a new projection operator, so collect the type - // of reorder exprs in order to call CastLogicalOperatorToTypes() - for (idx_t i = 0; i < node.left_reorder_exprs.size(); ++i) { - left_types.push_back(node.left_reorder_exprs[i]->return_type); - right_types.push_back(node.right_reorder_exprs[i]->return_type); - } - - auto left_projection = make_uniq(GenerateTableIndex(), std::move(node.left_reorder_exprs)); - left_projection->children.push_back(std::move(left_node)); - left_node = std::move(left_projection); - - auto right_projection = make_uniq(GenerateTableIndex(), std::move(node.right_reorder_exprs)); - right_projection->children.push_back(std::move(right_node)); - right_node = std::move(right_projection); - - left_node = CastLogicalOperatorToTypes(left_types, node.types, std::move(left_node)); - right_node = CastLogicalOperatorToTypes(right_types, node.types, std::move(right_node)); - } else { - left_node = CastLogicalOperatorToTypes(node.left->types, node.types, std::move(left_node)); - right_node = CastLogicalOperatorToTypes(node.right->types, node.types, std::move(right_node)); - } - - // check if there are any unplanned subqueries left in either child - has_unplanned_dependent_joins = has_unplanned_dependent_joins || node.left_binder->has_unplanned_dependent_joins || - node.right_binder->has_unplanned_dependent_joins; - // create actual logical ops for setops LogicalOperatorType logical_type = LogicalOperatorType::LOGICAL_INVALID; switch (node.setop_type) { @@ -146,13 +107,28 @@ unique_ptr Binder::CreatePlan(BoundSetOperationNode &node) { logical_type = LogicalOperatorType::LOGICAL_INTERSECT; break; default: - D_ASSERT(false); - break; + throw InternalException("Unsupported logical operator type for set-operation"); } + // Generate the logical plan for the children of the set operation - auto root = make_uniq(node.setop_index, node.types.size(), std::move(left_node), - std::move(right_node), logical_type, node.setop_all); + D_ASSERT(node.bound_children.size() >= 2); + vector> children; + for (idx_t child_idx = 0; child_idx < node.bound_children.size(); child_idx++) { + auto &child = node.bound_children[child_idx]; + auto &child_binder = *node.child_binders[child_idx]; + // construct the logical plan for the child node + auto child_node = std::move(child.plan); + // push casts for the target types + child_node = CastLogicalOperatorToTypes(child.types, node.types, std::move(child_node)); + // check if there are any unplanned subqueries left in any child + if (child_binder.has_unplanned_dependent_joins) { + has_unplanned_dependent_joins = true; + } + children.push_back(std::move(child_node)); + } + auto root = make_uniq(node.setop_index, node.types.size(), std::move(children), logical_type, + node.setop_all); return VisitQueryNode(node, std::move(root)); } diff --git a/src/planner/binder/query_node/plan_subquery.cpp b/src/planner/binder/query_node/plan_subquery.cpp index 2664903d3db3..29a419ab7eb1 100644 --- a/src/planner/binder/query_node/plan_subquery.cpp +++ b/src/planner/binder/query_node/plan_subquery.cpp @@ -186,9 +186,10 @@ static unique_ptr PlanUncorrelatedSubquery(Binder &binder, BoundSubq } } -static unique_ptr -CreateDuplicateEliminatedJoin(const vector &correlated_columns, JoinType join_type, - unique_ptr original_plan, bool perform_delim) { +static unique_ptr CreateDuplicateEliminatedJoin(const CorrelatedColumns &correlated_columns, + JoinType join_type, + unique_ptr original_plan, + bool perform_delim) { auto delim_join = make_uniq(join_type); delim_join->correlated_columns = correlated_columns; delim_join->perform_delim = perform_delim; @@ -216,7 +217,7 @@ static bool PerformDelimOnType(const LogicalType &type) { return true; } -static bool PerformDuplicateElimination(Binder &binder, vector &correlated_columns) { +static bool PerformDuplicateElimination(Binder &binder, CorrelatedColumns &correlated_columns) { if (!ClientConfig::GetConfig(binder.context).enable_optimizer) { // if optimizations are disabled we always do a delim join return true; @@ -235,7 +236,8 @@ static bool PerformDuplicateElimination(Binder &binder, vector Binder::PlanSubquery(BoundSubqueryExpression &expr, uniqu // first we translate the QueryNode of the subquery into a logical plan auto sub_binder = Binder::CreateBinder(context, this); sub_binder->is_outside_flattened = false; - auto subquery_root = sub_binder->CreatePlan(*expr.subquery); + auto subquery_root = std::move(expr.subquery.plan); D_ASSERT(subquery_root); // now we actually flatten the subquery @@ -403,7 +405,7 @@ void Binder::PlanSubqueries(unique_ptr &expr_ptr, unique_ptr Binder::PlanLateralJoin(unique_ptr left, unique_ptr right, - vector &correlated, JoinType join_type, + CorrelatedColumns &correlated, JoinType join_type, unique_ptr condition) { // scan the right operator for correlated columns // correlated LATERAL JOIN diff --git a/src/planner/binder/statement/bind_attach.cpp b/src/planner/binder/statement/bind_attach.cpp index 0e8655d2f12e..8ec2beca371e 100644 --- a/src/planner/binder/statement/bind_attach.cpp +++ b/src/planner/binder/statement/bind_attach.cpp @@ -1,7 +1,6 @@ #include "duckdb/planner/binder.hpp" #include "duckdb/parser/statement/attach_statement.hpp" #include "duckdb/parser/tableref/table_function_ref.hpp" -#include "duckdb/planner/tableref/bound_table_function.hpp" #include "duckdb/planner/operator/logical_simple.hpp" #include "duckdb/planner/expression_binder/table_function_binder.hpp" #include "duckdb/execution/expression_executor.hpp" diff --git a/src/planner/binder/statement/bind_call.cpp b/src/planner/binder/statement/bind_call.cpp index ba96927e8487..46be806cfeca 100644 --- a/src/planner/binder/statement/bind_call.cpp +++ b/src/planner/binder/statement/bind_call.cpp @@ -1,8 +1,6 @@ #include "duckdb/parser/statement/call_statement.hpp" #include "duckdb/parser/tableref/table_function_ref.hpp" #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/operator/logical_get.hpp" -#include "duckdb/planner/tableref/bound_table_function.hpp" #include "duckdb/parser/query_node/select_node.hpp" #include "duckdb/parser/expression/star_expression.hpp" diff --git a/src/planner/binder/statement/bind_copy.cpp b/src/planner/binder/statement/bind_copy.cpp index b7881a0a1d67..1757a2110a0b 100644 --- a/src/planner/binder/statement/bind_copy.cpp +++ b/src/planner/binder/statement/bind_copy.cpp @@ -36,7 +36,7 @@ void IsFormatExtensionKnown(const string &format) { // It's a match, we must throw throw CatalogException( "Copy Function with name \"%s\" is not in the catalog, but it exists in the %s extension.", format, - file_postfixes.extension); + std::string(file_postfixes.extension)); } } } @@ -551,8 +551,8 @@ BoundStatement Binder::Bind(CopyStatement &stmt, CopyToType copy_to_type) { // check if this matches the mode if (copy_option.mode != CopyOptionMode::READ_WRITE && copy_option.mode != copy_mode) { throw InvalidInputException("Option \"%s\" is not supported for %s - only for %s", provided_option, - stmt.info->is_from ? "reading" : "writing", - stmt.info->is_from ? "writing" : "reading"); + std::string(stmt.info->is_from ? "reading" : "writing"), + std::string(stmt.info->is_from ? "writing" : "reading")); } if (copy_option.type.id() != LogicalTypeId::ANY) { if (provided_entry.second.empty()) { diff --git a/src/planner/binder/statement/bind_create.cpp b/src/planner/binder/statement/bind_create.cpp index 76b43f60a517..91a03aadb49a 100644 --- a/src/planner/binder/statement/bind_create.cpp +++ b/src/planner/binder/statement/bind_create.cpp @@ -39,7 +39,6 @@ #include "duckdb/planner/operator/logical_projection.hpp" #include "duckdb/planner/parsed_data/bound_create_table_info.hpp" #include "duckdb/planner/query_node/bound_select_node.hpp" -#include "duckdb/planner/tableref/bound_basetableref.hpp" #include "duckdb/storage/storage_extension.hpp" #include "duckdb/common/extension_type_info.hpp" #include "duckdb/common/type_visitor.hpp" @@ -120,11 +119,11 @@ void Binder::SearchSchema(CreateInfo &info) { if (!info.temporary) { // non-temporary create: not read only if (info.catalog == TEMP_CATALOG) { - throw ParserException("Only TEMPORARY table names can use the \"%s\" catalog", TEMP_CATALOG); + throw ParserException("Only TEMPORARY table names can use the \"%s\" catalog", std::string(TEMP_CATALOG)); } } else { if (info.catalog != TEMP_CATALOG) { - throw ParserException("TEMPORARY table names can *only* use the \"%s\" catalog", TEMP_CATALOG); + throw ParserException("TEMPORARY table names can *only* use the \"%s\" catalog", std::string(TEMP_CATALOG)); } } } @@ -345,11 +344,7 @@ SchemaCatalogEntry &Binder::BindCreateFunctionInfo(CreateInfo &info) { try { dummy_binder->Bind(*query_node); } catch (const std::exception &ex) { - // TODO: we would like to do something like "error = ErrorData(ex);" here, - // but that breaks macro's like "create macro m(x) as table (from query_table(x));", - // because dummy-binding these always throws an error instead of a ParameterNotResolvedException. - // So, for now, we allow macro's with bind errors to be created. - // Binding is still useful because we can create the dependencies. + error = ErrorData(ex); } } @@ -548,23 +543,21 @@ BoundStatement Binder::Bind(CreateStatement &stmt) { create_index_info.table); auto table_ref = make_uniq(table_description); auto bound_table = Bind(*table_ref); - if (bound_table->type != TableReferenceType::BASE_TABLE) { + auto plan = std::move(bound_table.plan); + if (plan->type != LogicalOperatorType::LOGICAL_GET) { + throw BinderException("can only create an index on a base table"); + } + auto &get = plan->Cast(); + auto table_ptr = get.GetTable(); + if (!table_ptr) { throw BinderException("can only create an index on a base table"); } - auto &table_binding = bound_table->Cast(); - auto &table = table_binding.table; + auto &table = *table_ptr; if (table.temporary) { stmt.info->temporary = true; } properties.RegisterDBModify(table.catalog, context); - - // create a plan over the bound table - auto plan = CreatePlan(*bound_table); - if (plan->type != LogicalOperatorType::LOGICAL_GET) { - throw BinderException("Cannot create index on a view!"); - } - result.plan = table.catalog.BindCreateIndex(*this, stmt, table, std::move(plan)); break; } diff --git a/src/planner/binder/statement/bind_create_table.cpp b/src/planner/binder/statement/bind_create_table.cpp index ad70fe14abc0..f27ee73c6f77 100644 --- a/src/planner/binder/statement/bind_create_table.cpp +++ b/src/planner/binder/statement/bind_create_table.cpp @@ -289,7 +289,7 @@ void Binder::BindGeneratedColumns(BoundCreateTableInfo &info) { col.SetType(bound_expression->return_type); // Update the type in the binding, for future expansions - table_binding->types[i.index] = col.Type(); + table_binding->SetColumnType(i.index, col.Type()); } bound_indices.insert(i); } diff --git a/src/planner/binder/statement/bind_delete.cpp b/src/planner/binder/statement/bind_delete.cpp index e83a62ae361b..6b2b9d6fdc69 100644 --- a/src/planner/binder/statement/bind_delete.cpp +++ b/src/planner/binder/statement/bind_delete.cpp @@ -5,8 +5,6 @@ #include "duckdb/planner/operator/logical_delete.hpp" #include "duckdb/planner/operator/logical_filter.hpp" #include "duckdb/planner/operator/logical_get.hpp" -#include "duckdb/planner/bound_tableref.hpp" -#include "duckdb/planner/tableref/bound_basetableref.hpp" #include "duckdb/planner/operator/logical_cross_product.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" @@ -15,38 +13,34 @@ namespace duckdb { BoundStatement Binder::Bind(DeleteStatement &stmt) { // visit the table reference auto bound_table = Bind(*stmt.table); - if (bound_table->type != TableReferenceType::BASE_TABLE) { - throw BinderException("Can only delete from base table!"); + auto root = std::move(bound_table.plan); + if (root->type != LogicalOperatorType::LOGICAL_GET) { + throw BinderException("Can only delete from base table"); } - auto &table_binding = bound_table->Cast(); - auto &table = table_binding.table; - - auto root = CreatePlan(*bound_table); auto &get = root->Cast(); - D_ASSERT(root->type == LogicalOperatorType::LOGICAL_GET); - + auto table_ptr = get.GetTable(); + if (!table_ptr) { + throw BinderException("Can only delete from base table"); + } + auto &table = *table_ptr; if (!table.temporary) { // delete from persistent table: not read only! auto &properties = GetStatementProperties(); properties.RegisterDBModify(table.catalog, context); } - // Add CTEs as bindable - AddCTEMap(stmt.cte_map); - // plan any tables from the various using clauses if (!stmt.using_clauses.empty()) { unique_ptr child_operator; for (auto &using_clause : stmt.using_clauses) { // bind the using clause auto using_binder = Binder::CreateBinder(context, this); - auto bound_node = using_binder->Bind(*using_clause); - auto op = CreatePlan(*bound_node); + auto op = using_binder->Bind(*using_clause); if (child_operator) { // already bound a child: create a cross product to unify the two - child_operator = LogicalCrossProduct::Create(std::move(child_operator), std::move(op)); + child_operator = LogicalCrossProduct::Create(std::move(child_operator), std::move(op.plan)); } else { - child_operator = std::move(op); + child_operator = std::move(op.plan); } bind_context.AddContext(std::move(using_binder->bind_context)); } diff --git a/src/planner/binder/statement/bind_drop.cpp b/src/planner/binder/statement/bind_drop.cpp index 2b468defbcc7..9239e812e9b7 100644 --- a/src/planner/binder/statement/bind_drop.cpp +++ b/src/planner/binder/statement/bind_drop.cpp @@ -1,6 +1,5 @@ #include "duckdb/parser/statement/drop_statement.hpp" #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/bound_tableref.hpp" #include "duckdb/planner/operator/logical_simple.hpp" #include "duckdb/catalog/catalog.hpp" #include "duckdb/catalog/standard_entry.hpp" @@ -40,9 +39,34 @@ BoundStatement Binder::Bind(DropStatement &stmt) { // mark catalog as accessed properties.RegisterDBRead(*catalog, context); } - EntryLookupInfo entry_lookup(stmt.info->type, stmt.info->name); - auto entry = - Catalog::GetEntry(context, stmt.info->catalog, stmt.info->schema, entry_lookup, stmt.info->if_not_found); + optional_ptr entry; + if (stmt.info->type == CatalogType::MACRO_ENTRY) { + // We also support "DROP MACRO" (instead of "DROP MACRO TABLE") for table macros + // First try to drop a scalar macro + EntryLookupInfo macro_entry_lookup(stmt.info->type, stmt.info->name); + entry = Catalog::GetEntry(context, stmt.info->catalog, stmt.info->schema, macro_entry_lookup, + OnEntryNotFound::RETURN_NULL); + if (!entry) { + // Unable to find a scalar macro, try to drop a table macro + EntryLookupInfo table_macro_entry_lookup(CatalogType::TABLE_MACRO_ENTRY, stmt.info->name); + entry = Catalog::GetEntry(context, stmt.info->catalog, stmt.info->schema, table_macro_entry_lookup, + OnEntryNotFound::RETURN_NULL); + if (entry) { + // Change type to table macro so future lookups get the correct one + stmt.info->type = CatalogType::TABLE_MACRO_ENTRY; + } + } + + if (!entry) { + // Unable to find table macro, try again with original OnEntryNotFound to ensure we throw if necessary + entry = Catalog::GetEntry(context, stmt.info->catalog, stmt.info->schema, macro_entry_lookup, + stmt.info->if_not_found); + } + } else { + EntryLookupInfo entry_lookup(stmt.info->type, stmt.info->name); + entry = Catalog::GetEntry(context, stmt.info->catalog, stmt.info->schema, entry_lookup, + stmt.info->if_not_found); + } if (!entry) { break; } diff --git a/src/planner/binder/statement/bind_execute.cpp b/src/planner/binder/statement/bind_execute.cpp index cceb6796cd37..1202b01faefe 100644 --- a/src/planner/binder/statement/bind_execute.cpp +++ b/src/planner/binder/statement/bind_execute.cpp @@ -79,7 +79,7 @@ BoundStatement Binder::Bind(ExecuteStatement &stmt) { prepared = prepared_planner.PrepareSQLStatement(entry->second->unbound_statement->Copy()); rebound_plan = std::move(prepared_planner.plan); D_ASSERT(prepared->properties.bound_all_parameters); - this->bound_tables = prepared_planner.binder->bound_tables; + global_binder_state->bound_tables = prepared_planner.binder->global_binder_state->bound_tables; } // copy the properties of the prepared statement into the planner auto &properties = GetStatementProperties(); diff --git a/src/planner/binder/statement/bind_export.cpp b/src/planner/binder/statement/bind_export.cpp index ee162d2d8774..20d2606fe61a 100644 --- a/src/planner/binder/statement/bind_export.cpp +++ b/src/planner/binder/statement/bind_export.cpp @@ -140,21 +140,11 @@ unique_ptr Binder::UnionOperators(vector 1) { - vector> new_nodes; - for (idx_t i = 0; i < nodes.size(); i += 2) { - if (i + 1 == nodes.size()) { - new_nodes.push_back(std::move(nodes[i])); - } else { - auto copy_union = make_uniq(GenerateTableIndex(), 1U, std::move(nodes[i]), - std::move(nodes[i + 1]), - LogicalOperatorType::LOGICAL_UNION, true, false); - new_nodes.push_back(std::move(copy_union)); - } - } - nodes = std::move(new_nodes); + if (nodes.size() == 1) { + return std::move(nodes[0]); } - return std::move(nodes[0]); + return make_uniq(GenerateTableIndex(), 1U, std::move(nodes), + LogicalOperatorType::LOGICAL_UNION, true, false); } BoundStatement Binder::Bind(ExportStatement &stmt) { diff --git a/src/planner/binder/statement/bind_extension.cpp b/src/planner/binder/statement/bind_extension.cpp index b4fc0e86ba6a..6569315f7bab 100644 --- a/src/planner/binder/statement/bind_extension.cpp +++ b/src/planner/binder/statement/bind_extension.cpp @@ -5,8 +5,6 @@ namespace duckdb { BoundStatement Binder::Bind(ExtensionStatement &stmt) { - BoundStatement result; - // perform the planning of the function D_ASSERT(stmt.extension.plan_function); auto parse_result = @@ -18,11 +16,9 @@ BoundStatement Binder::Bind(ExtensionStatement &stmt) { properties.return_type = parse_result.return_type; // create the plan as a scan of the given table function - result.plan = BindTableFunction(parse_result.function, std::move(parse_result.parameters)); + auto result = BindTableFunction(parse_result.function, std::move(parse_result.parameters)); D_ASSERT(result.plan->type == LogicalOperatorType::LOGICAL_GET); auto &get = result.plan->Cast(); - result.names = get.names; - result.types = get.returned_types; get.ClearColumnIds(); for (idx_t i = 0; i < get.returned_types.size(); i++) { get.AddColumnId(i); diff --git a/src/planner/binder/statement/bind_insert.cpp b/src/planner/binder/statement/bind_insert.cpp index f2c8db644be8..fc7cf944976e 100644 --- a/src/planner/binder/statement/bind_insert.cpp +++ b/src/planner/binder/statement/bind_insert.cpp @@ -22,9 +22,6 @@ #include "duckdb/planner/expression/bound_default_expression.hpp" #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" -#include "duckdb/planner/bound_tableref.hpp" -#include "duckdb/planner/tableref/bound_basetableref.hpp" -#include "duckdb/planner/tableref/bound_dummytableref.hpp" #include "duckdb/parser/parsed_expression_iterator.hpp" #include "duckdb/storage/table_storage_info.hpp" #include "duckdb/parser/tableref/basetableref.hpp" @@ -277,7 +274,7 @@ unique_ptr Binder::GenerateMergeInto(InsertStatement &stmt, auto storage_info = table.GetStorageInfo(context); auto &columns = table.GetColumns(); // set up the columns on which to join - vector distinct_on_columns; + vector> all_distinct_on_columns; if (on_conflict_info.indexed_columns.empty()) { // When omitting the conflict target, we derive the join columns from the primary key/unique constraints // traverse the primary key/unique constraints @@ -292,6 +289,7 @@ unique_ptr Binder::GenerateMergeInto(InsertStatement &stmt, vector> and_children; auto &indexed_columns = index.column_set; + vector distinct_on_columns; for (auto &column : columns.Physical()) { if (!indexed_columns.count(column.Physical().index)) { continue; @@ -303,6 +301,7 @@ unique_ptr Binder::GenerateMergeInto(InsertStatement &stmt, and_children.push_back(std::move(new_condition)); distinct_on_columns.push_back(column.Name()); } + all_distinct_on_columns.push_back(std::move(distinct_on_columns)); if (and_children.empty()) { continue; } @@ -377,7 +376,7 @@ unique_ptr Binder::GenerateMergeInto(InsertStatement &stmt, throw BinderException("The specified columns as conflict target are not referenced by a UNIQUE/PRIMARY KEY " "CONSTRAINT or INDEX"); } - distinct_on_columns = on_conflict_info.indexed_columns; + all_distinct_on_columns.push_back(on_conflict_info.indexed_columns); merge_into->using_columns = std::move(on_conflict_info.indexed_columns); } @@ -445,17 +444,19 @@ unique_ptr Binder::GenerateMergeInto(InsertStatement &stmt, } } // push DISTINCT ON(unique_columns) - auto distinct_stmt = make_uniq(); - auto select_node = make_uniq(); - auto distinct = make_uniq(); - for (auto &col : distinct_on_columns) { - distinct->distinct_on_targets.push_back(make_uniq(col)); + for (auto &distinct_on_columns : all_distinct_on_columns) { + auto distinct_stmt = make_uniq(); + auto select_node = make_uniq(); + auto distinct = make_uniq(); + for (auto &col : distinct_on_columns) { + distinct->distinct_on_targets.push_back(make_uniq(col)); + } + select_node->modifiers.push_back(std::move(distinct)); + select_node->select_list.push_back(make_uniq()); + select_node->from_table = std::move(source); + distinct_stmt->node = std::move(select_node); + source = make_uniq(std::move(distinct_stmt), "excluded"); } - select_node->modifiers.push_back(std::move(distinct)); - select_node->select_list.push_back(make_uniq()); - select_node->from_table = std::move(source); - distinct_stmt->node = std::move(select_node); - source = make_uniq(std::move(distinct_stmt), "excluded"); merge_into->source = std::move(source); @@ -519,8 +520,6 @@ BoundStatement Binder::Bind(InsertStatement &stmt) { } auto insert = make_uniq(table, GenerateTableIndex()); - // Add CTEs as bindable - AddCTEMap(stmt.cte_map); auto values_list = stmt.GetValuesList(); diff --git a/src/planner/binder/statement/bind_logical_plan.cpp b/src/planner/binder/statement/bind_logical_plan.cpp index 5b187c8e3e21..3108b76aa2e6 100644 --- a/src/planner/binder/statement/bind_logical_plan.cpp +++ b/src/planner/binder/statement/bind_logical_plan.cpp @@ -32,7 +32,7 @@ BoundStatement Binder::Bind(LogicalPlanStatement &stmt) { if (parent) { throw InternalException("LogicalPlanStatement should be bound in root binder"); } - bound_tables = GetMaxTableIndex(*result.plan) + 1; + global_binder_state->bound_tables = GetMaxTableIndex(*result.plan) + 1; return result; } diff --git a/src/planner/binder/statement/bind_merge_into.cpp b/src/planner/binder/statement/bind_merge_into.cpp index 87a9726ecb97..62867f3525fb 100644 --- a/src/planner/binder/statement/bind_merge_into.cpp +++ b/src/planner/binder/statement/bind_merge_into.cpp @@ -1,6 +1,5 @@ #include "duckdb/planner/binder.hpp" #include "duckdb/parser/statement/merge_into_statement.hpp" -#include "duckdb/planner/tableref/bound_basetableref.hpp" #include "duckdb/planner/tableref/bound_joinref.hpp" #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/expression_binder/where_binder.hpp" @@ -178,11 +177,14 @@ BoundStatement Binder::Bind(MergeIntoStatement &stmt) { auto target_binder = Binder::CreateBinder(context, this); string table_alias = stmt.target->alias; auto bound_table = target_binder->Bind(*stmt.target); - if (bound_table->type != TableReferenceType::BASE_TABLE) { + if (bound_table.plan->type != LogicalOperatorType::LOGICAL_GET) { throw BinderException("Can only merge into base tables!"); } - auto &table_binding = bound_table->Cast(); - auto &table = table_binding.table; + auto table_ptr = bound_table.plan->Cast().GetTable(); + if (!table_ptr) { + throw BinderException("Can only merge into base tables!"); + } + auto &table = *table_ptr; if (!table.temporary) { // update of persistent table: not read only! auto &properties = GetStatementProperties(); @@ -198,9 +200,10 @@ BoundStatement Binder::Bind(MergeIntoStatement &stmt) { vector source_names; for (auto &binding_entry : source_binder->bind_context.GetBindingsList()) { auto &binding = *binding_entry; - for (idx_t c = 0; c < binding.names.size(); c++) { - source_aliases.push_back(binding.alias); - source_names.push_back(binding.names[c]); + auto &column_names = binding.GetColumnNames(); + for (idx_t c = 0; c < column_names.size(); c++) { + source_aliases.push_back(binding.GetBindingAlias()); + source_names.push_back(column_names[c]); } } @@ -231,11 +234,19 @@ BoundStatement Binder::Bind(MergeIntoStatement &stmt) { } auto bound_join_node = Bind(join); - auto root = CreatePlan(*bound_join_node); + auto root = std::move(bound_join_node.plan); + auto join_ref = reference(*root); + while (join_ref.get().children.size() == 1) { + join_ref = *join_ref.get().children[0]; + } + if (join_ref.get().children.size() != 2) { + throw NotImplementedException("Expected a join after binding a join operator - but got a %s", + join_ref.get().type); + } // kind of hacky, CreatePlan turns a RIGHT join into a LEFT join so the children get reversed from what we need bool inverted = join.type == JoinType::RIGHT; - auto &source = root->children[inverted ? 1 : 0]; - auto &get = root->children[inverted ? 0 : 1]->Cast(); + auto &source = join_ref.get().children[inverted ? 1 : 0]; + auto &get = join_ref.get().children[inverted ? 0 : 1]->Cast(); auto merge_into = make_uniq(table); merge_into->table_index = GenerateTableIndex(); diff --git a/src/planner/binder/statement/bind_pragma.cpp b/src/planner/binder/statement/bind_pragma.cpp index 3955cf89753e..b5fc04677b1c 100644 --- a/src/planner/binder/statement/bind_pragma.cpp +++ b/src/planner/binder/statement/bind_pragma.cpp @@ -2,6 +2,7 @@ #include "duckdb/parser/statement/pragma_statement.hpp" #include "duckdb/planner/operator/logical_pragma.hpp" #include "duckdb/catalog/catalog_entry/pragma_function_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" #include "duckdb/catalog/catalog.hpp" #include "duckdb/function/function_binder.hpp" #include "duckdb/planner/expression_binder/constant_binder.hpp" @@ -28,16 +29,32 @@ unique_ptr Binder::BindPragma(PragmaInfo &info, QueryErrorConte } // bind the pragma function - auto &entry = Catalog::GetEntry(context, INVALID_CATALOG, DEFAULT_SCHEMA, info.name); + auto entry = Catalog::GetEntry(context, INVALID_CATALOG, DEFAULT_SCHEMA, info.name, + OnEntryNotFound::RETURN_NULL); + if (!entry) { + // try to find whether a table extry might exist + auto table_entry = Catalog::GetEntry(context, INVALID_CATALOG, DEFAULT_SCHEMA, + info.name, OnEntryNotFound::RETURN_NULL); + if (table_entry) { + // there is a table entry with the same name, now throw more explicit error message + throw CatalogException("Pragma Function with name %s does not exist, but a table function with the same " + "name exists, try `CALL %s(...)`", + info.name, info.name); + } + // rebind to throw exception + entry = Catalog::GetEntry(context, INVALID_CATALOG, DEFAULT_SCHEMA, info.name, + OnEntryNotFound::THROW_EXCEPTION); + } + FunctionBinder function_binder(*this); ErrorData error; - auto bound_idx = function_binder.BindFunction(entry.name, entry.functions, params, error); + auto bound_idx = function_binder.BindFunction(entry->name, entry->functions, params, error); if (!bound_idx.IsValid()) { D_ASSERT(error.HasError()); error.AddQueryLocation(error_context); error.Throw(); } - auto bound_function = entry.functions.GetFunctionByOffset(bound_idx.GetIndex()); + auto bound_function = entry->functions.GetFunctionByOffset(bound_idx.GetIndex()); // bind and check named params BindNamedParameters(bound_function.named_parameters, named_parameters, error_context, bound_function.name); return make_uniq(std::move(bound_function), std::move(params), std::move(named_parameters)); diff --git a/src/planner/binder/statement/bind_prepare.cpp b/src/planner/binder/statement/bind_prepare.cpp index 74062e41333f..66fb40d6141e 100644 --- a/src/planner/binder/statement/bind_prepare.cpp +++ b/src/planner/binder/statement/bind_prepare.cpp @@ -8,7 +8,12 @@ namespace duckdb { BoundStatement Binder::Bind(PrepareStatement &stmt) { Planner prepared_planner(context); auto prepared_data = prepared_planner.PrepareSQLStatement(std::move(stmt.statement)); - this->bound_tables = prepared_planner.binder->bound_tables; + global_binder_state->bound_tables = prepared_planner.binder->global_binder_state->bound_tables; + + if (prepared_planner.properties.always_require_rebind) { + // we always need to rebind - don't keep the plan around + prepared_planner.plan.reset(); + } auto prepare = make_uniq(stmt.name, std::move(prepared_data), std::move(prepared_planner.plan)); // we can always prepare, even if the transaction has been invalidated diff --git a/src/planner/binder/statement/bind_simple.cpp b/src/planner/binder/statement/bind_simple.cpp index 583c9e992394..46758e416c41 100644 --- a/src/planner/binder/statement/bind_simple.cpp +++ b/src/planner/binder/statement/bind_simple.cpp @@ -60,16 +60,15 @@ BoundStatement Binder::BindAlterAddIndex(BoundStatement &result, CatalogEntry &e TableDescription table_description(table_info.catalog, table_info.schema, table_info.name); auto table_ref = make_uniq(table_description); auto bound_table = Bind(*table_ref); - if (bound_table->type != TableReferenceType::BASE_TABLE) { + if (bound_table.plan->type != LogicalOperatorType::LOGICAL_GET) { throw BinderException("can only add an index to a base table"); } - auto plan = CreatePlan(*bound_table); - auto &get = plan->Cast(); + auto &get = bound_table.plan->Cast(); get.names = column_list.GetColumnNames(); auto alter_table_info = unique_ptr_cast(std::move(alter_info)); - result.plan = table.catalog.BindAlterAddIndex(*this, table, std::move(plan), std::move(create_index_info), - std::move(alter_table_info)); + result.plan = table.catalog.BindAlterAddIndex(*this, table, std::move(bound_table.plan), + std::move(create_index_info), std::move(alter_table_info)); return std::move(result); } @@ -77,6 +76,16 @@ BoundStatement Binder::Bind(AlterStatement &stmt) { BoundStatement result; result.names = {"Success"}; result.types = {LogicalType::BOOLEAN}; + + // Special handling for ALTER DATABASE - doesn't use schema binding + if (stmt.info->type == AlterType::ALTER_DATABASE) { + auto &properties = GetStatementProperties(); + properties.return_type = StatementReturnType::NOTHING; + properties.RegisterDBModify(Catalog::GetSystemCatalog(context), context); + result.plan = make_uniq(LogicalOperatorType::LOGICAL_ALTER, std::move(stmt.info)); + return result; + } + BindSchemaOrCatalog(stmt.info->catalog, stmt.info->schema); optional_ptr entry; diff --git a/src/planner/binder/statement/bind_summarize.cpp b/src/planner/binder/statement/bind_summarize.cpp index 45b2b2f25c93..f8a68ae4c036 100644 --- a/src/planner/binder/statement/bind_summarize.cpp +++ b/src/planner/binder/statement/bind_summarize.cpp @@ -9,7 +9,6 @@ #include "duckdb/parser/tableref/showref.hpp" #include "duckdb/parser/tableref/basetableref.hpp" #include "duckdb/parser/expression/star_expression.hpp" -#include "duckdb/planner/bound_tableref.hpp" namespace duckdb { @@ -78,7 +77,7 @@ static unique_ptr SummarizeCreateNullPercentage(string column_ return make_uniq(LogicalType::DECIMAL(9, 2), std::move(case_expr)); } -unique_ptr Binder::BindSummarize(ShowRef &ref) { +BoundStatement Binder::BindSummarize(ShowRef &ref) { unique_ptr query; if (ref.query) { query = std::move(ref.query); diff --git a/src/planner/binder/statement/bind_update.cpp b/src/planner/binder/statement/bind_update.cpp index 650b23b89bf2..fafcbaf0221d 100644 --- a/src/planner/binder/statement/bind_update.cpp +++ b/src/planner/binder/statement/bind_update.cpp @@ -2,7 +2,6 @@ #include "duckdb/parser/statement/update_statement.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/tableref/bound_joinref.hpp" -#include "duckdb/planner/bound_tableref.hpp" #include "duckdb/planner/constraints/bound_check_constraint.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression/bound_default_expression.hpp" @@ -12,7 +11,6 @@ #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/operator/logical_projection.hpp" #include "duckdb/planner/operator/logical_update.hpp" -#include "duckdb/planner/tableref/bound_basetableref.hpp" #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp" #include "duckdb/storage/data_table.hpp" @@ -110,14 +108,15 @@ BoundStatement Binder::Bind(UpdateStatement &stmt) { // visit the table reference auto bound_table = Bind(*stmt.table); - if (bound_table->type != TableReferenceType::BASE_TABLE) { - throw BinderException("Can only update base table!"); + if (bound_table.plan->type != LogicalOperatorType::LOGICAL_GET) { + throw BinderException("Can only update base table"); } - auto &table_binding = bound_table->Cast(); - auto &table = table_binding.table; - - // Add CTEs as bindable - AddCTEMap(stmt.cte_map); + auto &bound_table_get = bound_table.plan->Cast(); + auto table_ptr = bound_table_get.GetTable(); + if (!table_ptr) { + throw BinderException("Can only update base table"); + } + auto &table = *table_ptr; optional_ptr get; if (stmt.from_table) { @@ -129,7 +128,7 @@ BoundStatement Binder::Bind(UpdateStatement &stmt) { get = &root->children[0]->Cast(); bind_context.AddContext(std::move(from_binder->bind_context)); } else { - root = CreatePlan(*bound_table); + root = std::move(bound_table.plan); get = &root->Cast(); } diff --git a/src/planner/binder/statement/bind_vacuum.cpp b/src/planner/binder/statement/bind_vacuum.cpp index 93e70fe5b8be..026f682b06c0 100644 --- a/src/planner/binder/statement/bind_vacuum.cpp +++ b/src/planner/binder/statement/bind_vacuum.cpp @@ -15,12 +15,18 @@ void Binder::BindVacuumTable(LogicalVacuum &vacuum, unique_ptr } D_ASSERT(vacuum.column_id_map.empty()); + auto bound_table = Bind(*info.ref); - if (bound_table->type != TableReferenceType::BASE_TABLE) { - throw InvalidInputException("can only vacuum or analyze base tables"); + if (bound_table.plan->type != LogicalOperatorType::LOGICAL_GET) { + throw BinderException("Can only vacuum or analyze base tables"); + } + auto table_scan = std::move(bound_table.plan); + auto &get = table_scan->Cast(); + auto table_ptr = get.GetTable(); + if (!table_ptr) { + throw BinderException("Can only vacuum or analyze base tables"); } - auto ref = unique_ptr_cast(std::move(bound_table)); - auto &table = ref->table; + auto &table = *table_ptr; vacuum.SetTable(table); vector> select_list; @@ -60,11 +66,6 @@ void Binder::BindVacuumTable(LogicalVacuum &vacuum, unique_ptr } info.columns = std::move(non_generated_column_names); - auto table_scan = CreatePlan(*ref); - D_ASSERT(table_scan->type == LogicalOperatorType::LOGICAL_GET); - - auto &get = table_scan->Cast(); - auto &column_ids = get.GetColumnIds(); D_ASSERT(select_list.size() == column_ids.size()); D_ASSERT(info.columns.size() == column_ids.size()); diff --git a/src/planner/binder/tableref/CMakeLists.txt b/src/planner/binder/tableref/CMakeLists.txt index 34e13a855b63..34689fe0de51 100644 --- a/src/planner/binder/tableref/CMakeLists.txt +++ b/src/planner/binder/tableref/CMakeLists.txt @@ -13,16 +13,7 @@ add_library_unity( bind_subqueryref.cpp bind_table_function.cpp bind_named_parameters.cpp - plan_basetableref.cpp - plan_delimgetref.cpp - plan_dummytableref.cpp - plan_expressionlistref.cpp - plan_column_data_ref.cpp - plan_joinref.cpp - plan_subqueryref.cpp - plan_table_function.cpp - plan_cteref.cpp - plan_pivotref.cpp) + plan_joinref.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/planner/binder/tableref/bind_basetableref.cpp b/src/planner/binder/tableref/bind_basetableref.cpp index e22a7b7845e4..25d0b7b2a082 100644 --- a/src/planner/binder/tableref/bind_basetableref.cpp +++ b/src/planner/binder/tableref/bind_basetableref.cpp @@ -11,15 +11,13 @@ #include "duckdb/parser/tableref/table_function_ref.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/operator/logical_get.hpp" -#include "duckdb/planner/tableref/bound_basetableref.hpp" -#include "duckdb/planner/tableref/bound_cteref.hpp" -#include "duckdb/planner/tableref/bound_dummytableref.hpp" -#include "duckdb/planner/tableref/bound_subqueryref.hpp" +#include "duckdb/planner/operator/logical_cteref.hpp" #include "duckdb/planner/expression_binder/constant_binder.hpp" #include "duckdb/catalog/catalog_search_path.hpp" #include "duckdb/planner/tableref/bound_at_clause.hpp" #include "duckdb/execution/expression_executor.hpp" #include "duckdb/parser/query_node/cte_node.hpp" +#include "duckdb/planner/operator/logical_dummy_scan.hpp" namespace duckdb { @@ -48,10 +46,10 @@ static bool TryLoadExtensionForReplacementScan(ClientContext &context, const str return false; } -unique_ptr Binder::BindWithReplacementScan(ClientContext &context, BaseTableRef &ref) { +BoundStatement Binder::BindWithReplacementScan(ClientContext &context, BaseTableRef &ref) { auto &config = DBConfig::GetConfig(context); if (!context.config.use_replacement_scans) { - return nullptr; + return BoundStatement(); } for (auto &scan : config.replacement_scans) { ReplacementScanInput input(ref.catalog_name, ref.schema_name, ref.table_name); @@ -80,7 +78,7 @@ unique_ptr Binder::BindWithReplacementScan(ClientContext &context } return Bind(*replacement_function); } - return nullptr; + return BoundStatement(); } unique_ptr Binder::BindAtClause(optional_ptr at_clause) { @@ -116,95 +114,38 @@ static vector ExchangeAllNullTypes(const vector &types return result; } -unique_ptr Binder::Bind(BaseTableRef &ref) { +BoundStatement Binder::Bind(BaseTableRef &ref) { QueryErrorContext error_context(ref.query_location); // CTEs and views are also referred to using BaseTableRefs, hence need to distinguish here // check if the table name refers to a CTE // CTE name should never be qualified (i.e. schema_name should be empty) // unless we want to refer to the recurring table of "using key". - vector> found_ctes; - if (ref.schema_name.empty() || ref.schema_name == "recurring") { - found_ctes = FindCTE(ref.table_name, ref.table_name == alias); - } + BindingAlias binding_alias(ref.schema_name, ref.table_name); + auto ctebinding = GetCTEBinding(binding_alias); + if (ctebinding && ctebinding->CanBeReferenced()) { + ctebinding->Reference(); - if (!found_ctes.empty()) { - // Check if there is a CTE binding in the BindContext - bool circular_cte = false; - for (auto found_cte : found_ctes) { - auto &cte = found_cte.get(); - auto ctebinding = bind_context.GetCTEBinding(ref.table_name); - if (ctebinding) { - // There is a CTE binding in the BindContext. - // This can only be the case if there is a recursive CTE, - // or a materialized CTE present. - auto index = GenerateTableIndex(); - auto materialized = cte.materialized; - - if (ref.schema_name == "recurring" && cte.key_targets.empty()) { - throw InvalidInputException("RECURRING can only be used with USING KEY in recursive CTE."); - } - - auto result = - make_uniq(index, ctebinding->index, materialized, ref.schema_name == "recurring"); - auto alias = ref.alias.empty() ? ref.table_name : ref.alias; - auto names = BindContext::AliasColumnNames(alias, ctebinding->names, ref.column_name_alias); - - bind_context.AddGenericBinding(index, alias, names, ctebinding->types); - - auto cte_reference = ref.schema_name.empty() ? ref.table_name : ref.schema_name + "." + ref.table_name; - - // Update references to CTE - auto cteref = bind_context.cte_references[cte_reference]; - - if (cteref == nullptr && ref.schema_name == "recurring") { - throw BinderException("There is a WITH item named \"%s\", but the recurring table cannot be " - "referenced from this part of the query.", - ref.table_name); - } - - (*cteref)++; - - result->types = ctebinding->types; - result->bound_columns = std::move(names); - return std::move(result); - } else { - if (CTEIsAlreadyBound(cte)) { - // remember error state - circular_cte = true; - // retry with next candidate CTE - continue; - } - - // If we have found a materialized CTE, but no corresponding CTE binding, - // something is wrong. - if (cte.materialized == CTEMaterialize::CTE_MATERIALIZE_ALWAYS) { - throw BinderException( - "There is a WITH item named \"%s\", but it cannot be referenced from this part of the query.", - ref.table_name); - } - - if (ref.schema_name == "recurring") { - throw BinderException("There is a WITH item named \"%s\", but the recurring table cannot be " - "referenced from this part of the query.", - ref.table_name); - } - } - } - if (circular_cte) { - auto replacement_scan_bind_result = BindWithReplacementScan(context, ref); - if (replacement_scan_bind_result) { - return replacement_scan_bind_result; - } + // There is a CTE binding in the BindContext. + // This can only be the case if there is a recursive CTE, + // or a materialized CTE present. + auto index = GenerateTableIndex(); - throw BinderException( - "Circular reference to CTE \"%s\", There are two possible solutions. \n1. use WITH RECURSIVE to " - "use recursive CTEs. \n2. If " - "you want to use the TABLE name \"%s\" the same as the CTE name, please explicitly add " - "\"SCHEMA\" before table name. You can try \"main.%s\" (main is the duckdb default schema)", - ref.table_name, ref.table_name, ref.table_name); - } + auto alias = ref.alias.empty() ? ref.table_name : ref.alias; + auto names = BindContext::AliasColumnNames(alias, ctebinding->GetColumnNames(), ref.column_name_alias); + + bind_context.AddGenericBinding(index, alias, names, ctebinding->GetColumnTypes()); + + bool is_recurring = ref.schema_name == "recurring"; + + BoundStatement result; + result.types = ctebinding->GetColumnTypes(); + result.names = names; + result.plan = + make_uniq(index, ctebinding->GetIndex(), result.types, std::move(names), is_recurring); + return result; } + // not a CTE // extract a table or view from the catalog auto at_clause = BindAtClause(ref.at_clause); @@ -229,14 +170,19 @@ unique_ptr Binder::Bind(BaseTableRef &ref) { vector types {LogicalType::INTEGER}; vector names {"__dummy_col" + to_string(table_index)}; bind_context.AddGenericBinding(table_index, ref_alias, names, types); - return make_uniq_base(table_index); + + BoundStatement result; + result.types = std::move(types); + result.names = std::move(names); + result.plan = make_uniq(table_index); + return result; } } if (!table_or_view) { // table could not be found: try to bind a replacement scan // Try replacement scan bind auto replacement_scan_bind_result = BindWithReplacementScan(context, ref); - if (replacement_scan_bind_result) { + if (replacement_scan_bind_result.plan) { return replacement_scan_bind_result; } @@ -245,7 +191,7 @@ unique_ptr Binder::Bind(BaseTableRef &ref) { auto extension_loaded = TryLoadExtensionForReplacementScan(context, full_path); if (extension_loaded) { replacement_scan_bind_result = BindWithReplacementScan(context, ref); - if (replacement_scan_bind_result) { + if (replacement_scan_bind_result.plan) { return replacement_scan_bind_result; } } @@ -261,6 +207,14 @@ unique_ptr Binder::Bind(BaseTableRef &ref) { } } + // if we found a CTE that cannot be referenced that means that there is a circular reference + if (ctebinding) { + D_ASSERT(!ctebinding->CanBeReferenced()); + throw BinderException(error_context, + "Circular reference to CTE \"%s\", use WITH RECURSIVE to " + "use recursive CTEs.", + ref.table_name); + } // could not find an alternative: bind again to get the error // note: this will always throw when using DuckDB as a catalog, but a second look-up might succeed // in catalogs that do not have transactional DDL @@ -270,7 +224,7 @@ unique_ptr Binder::Bind(BaseTableRef &ref) { switch (table_or_view->type) { case CatalogType::TABLE_ENTRY: { - // base table: create the BoundBaseTableRef node + // base table auto table_index = GenerateTableIndex(); auto &table = table_or_view->Cast(); @@ -313,7 +267,11 @@ unique_ptr Binder::Bind(BaseTableRef &ref) { } else { bind_context.AddBaseTable(table_index, ref.alias, table_names, table_types, col_ids, *table_entry); } - return make_uniq_base(table, std::move(logical_get)); + BoundStatement result; + result.types = table_types; + result.names = table_names; + result.plan = std::move(logical_get); + return result; } case CatalogType::VIEW_ENTRY: { // the node is a view: get the query that the view represents @@ -326,30 +284,6 @@ unique_ptr Binder::Bind(BaseTableRef &ref) { // The view may contain CTEs, but maybe only in the cte_map, so we need create CTE nodes for them auto query = view_catalog_entry.GetQuery().Copy(); - auto &select_stmt = query->Cast(); - - vector> materialized_ctes; - for (auto &cte : select_stmt.node->cte_map.map) { - auto &cte_entry = cte.second; - auto mat_cte = make_uniq(); - mat_cte->ctename = cte.first; - mat_cte->query = cte_entry->query->node->Copy(); - mat_cte->aliases = cte_entry->aliases; - mat_cte->materialized = cte_entry->materialized; - materialized_ctes.push_back(std::move(mat_cte)); - } - - auto root = std::move(select_stmt.node); - while (!materialized_ctes.empty()) { - unique_ptr node_result; - node_result = std::move(materialized_ctes.back()); - node_result->cte_map = root->cte_map.Copy(); - node_result->child = std::move(root); - root = std::move(node_result); - materialized_ctes.pop_back(); - } - select_stmt.node = std::move(root); - SubqueryRef subquery(unique_ptr_cast(std::move(query))); subquery.alias = ref.alias; @@ -375,15 +309,13 @@ unique_ptr Binder::Bind(BaseTableRef &ref) { throw BinderException("Contents of view were altered - view bound correlated columns"); } - D_ASSERT(bound_child->type == TableReferenceType::SUBQUERY); // verify that the types and names match up with the expected types and names if the view has type info defined - auto &bound_subquery = bound_child->Cast(); if (GetBindingMode() != BindingMode::EXTRACT_NAMES && GetBindingMode() != BindingMode::EXTRACT_QUALIFIED_NAMES && view_catalog_entry.HasTypes()) { // we bind the view subquery and the original view with different "can_contain_nulls", // but we don't want to throw an error when SQLNULL does not match up with INTEGER, // so we exchange all SQLNULL with INTEGER here before comparing - auto bound_types = ExchangeAllNullTypes(bound_subquery.subquery->types); + auto bound_types = ExchangeAllNullTypes(bound_child.types); auto view_types = ExchangeAllNullTypes(view_catalog_entry.types); if (bound_types != view_types) { auto actual_types = StringUtil::ToString(bound_types, ", "); @@ -392,17 +324,17 @@ unique_ptr Binder::Bind(BaseTableRef &ref) { "Contents of view were altered: types don't match! Expected [%s], but found [%s] instead", expected_types, actual_types); } - if (bound_subquery.subquery->names.size() == view_catalog_entry.names.size() && - bound_subquery.subquery->names != view_catalog_entry.names) { - auto actual_names = StringUtil::Join(bound_subquery.subquery->names, ", "); + if (bound_child.names.size() == view_catalog_entry.names.size() && + bound_child.names != view_catalog_entry.names) { + auto actual_names = StringUtil::Join(bound_child.names, ", "); auto expected_names = StringUtil::Join(view_catalog_entry.names, ", "); throw BinderException( "Contents of view were altered: names don't match! Expected [%s], but found [%s] instead", expected_names, actual_names); } } - bind_context.AddView(bound_subquery.subquery->GetRootIndex(), subquery.alias, subquery, - *bound_subquery.subquery, view_catalog_entry); + bind_context.AddView(bound_child.plan->GetRootIndex(), subquery.alias, subquery, bound_child, + view_catalog_entry); return bound_child; } default: diff --git a/src/planner/binder/tableref/bind_bound_table_ref.cpp b/src/planner/binder/tableref/bind_bound_table_ref.cpp index e31c2e83c60d..ace531ccf384 100644 --- a/src/planner/binder/tableref/bind_bound_table_ref.cpp +++ b/src/planner/binder/tableref/bind_bound_table_ref.cpp @@ -2,8 +2,8 @@ namespace duckdb { -unique_ptr Binder::Bind(BoundRefWrapper &ref) { - if (!ref.binder || !ref.bound_ref) { +BoundStatement Binder::Bind(BoundRefWrapper &ref) { + if (!ref.binder || !ref.bound_ref.plan) { throw InternalException("Rebinding bound ref that was already bound"); } bind_context.AddContext(std::move(ref.binder->bind_context)); diff --git a/src/planner/binder/tableref/bind_column_data_ref.cpp b/src/planner/binder/tableref/bind_column_data_ref.cpp index 635d23f7178b..d3c5ea4a22e5 100644 --- a/src/planner/binder/tableref/bind_column_data_ref.cpp +++ b/src/planner/binder/tableref/bind_column_data_ref.cpp @@ -1,20 +1,25 @@ #include "duckdb/planner/binder.hpp" #include "duckdb/parser/tableref/column_data_ref.hpp" -#include "duckdb/planner/tableref/bound_column_data_ref.hpp" #include "duckdb/planner/operator/logical_column_data_get.hpp" namespace duckdb { -unique_ptr Binder::Bind(ColumnDataRef &ref) { +BoundStatement Binder::Bind(ColumnDataRef &ref) { auto &collection = *ref.collection; auto types = collection.Types(); - auto result = make_uniq(std::move(ref.collection)); - result->bind_index = GenerateTableIndex(); - for (idx_t i = ref.expected_names.size(); i < types.size(); i++) { - ref.expected_names.push_back("col" + to_string(i + 1)); + + BoundStatement result; + result.names = std::move(ref.expected_names); + for (idx_t i = result.names.size(); i < types.size(); i++) { + result.names.push_back("col" + to_string(i + 1)); } - bind_context.AddGenericBinding(result->bind_index, ref.alias, ref.expected_names, types); - return unique_ptr_cast(std::move(result)); + result.types = types; + auto bind_index = GenerateTableIndex(); + bind_context.AddGenericBinding(bind_index, ref.alias, result.names, types); + + result.plan = + make_uniq_base(bind_index, std::move(types), std::move(ref.collection)); + return result; } } // namespace duckdb diff --git a/src/planner/binder/tableref/bind_delimgetref.cpp b/src/planner/binder/tableref/bind_delimgetref.cpp index f280404f9c0d..18c27cccfc8f 100644 --- a/src/planner/binder/tableref/bind_delimgetref.cpp +++ b/src/planner/binder/tableref/bind_delimgetref.cpp @@ -1,16 +1,21 @@ #include "duckdb/parser/tableref/delimgetref.hpp" #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/tableref/bound_delimgetref.hpp" +#include "duckdb/planner/operator/logical_delim_get.hpp" namespace duckdb { -unique_ptr Binder::Bind(DelimGetRef &ref) { +BoundStatement Binder::Bind(DelimGetRef &ref) { // Have to add bindings idx_t tbl_idx = GenerateTableIndex(); string internal_name = "__internal_delim_get_ref_" + std::to_string(tbl_idx); - bind_context.AddGenericBinding(tbl_idx, internal_name, ref.internal_aliases, ref.types); - return make_uniq(tbl_idx, ref.types); + BoundStatement result; + result.types = std::move(ref.types); + result.names = std::move(ref.internal_aliases); + result.plan = make_uniq(tbl_idx, result.types); + + bind_context.AddGenericBinding(tbl_idx, internal_name, result.names, result.types); + return result; } } // namespace duckdb diff --git a/src/planner/binder/tableref/bind_emptytableref.cpp b/src/planner/binder/tableref/bind_emptytableref.cpp index fe0e96f3d43c..b6ea93ab8dad 100644 --- a/src/planner/binder/tableref/bind_emptytableref.cpp +++ b/src/planner/binder/tableref/bind_emptytableref.cpp @@ -1,11 +1,13 @@ #include "duckdb/parser/tableref/emptytableref.hpp" #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/tableref/bound_dummytableref.hpp" +#include "duckdb/planner/operator/logical_dummy_scan.hpp" namespace duckdb { -unique_ptr Binder::Bind(EmptyTableRef &ref) { - return make_uniq(GenerateTableIndex()); +BoundStatement Binder::Bind(EmptyTableRef &ref) { + BoundStatement result; + result.plan = make_uniq(GenerateTableIndex()); + return result; } } // namespace duckdb diff --git a/src/planner/binder/tableref/bind_expressionlistref.cpp b/src/planner/binder/tableref/bind_expressionlistref.cpp index 7176fb682a28..139f94670eec 100644 --- a/src/planner/binder/tableref/bind_expressionlistref.cpp +++ b/src/planner/binder/tableref/bind_expressionlistref.cpp @@ -1,72 +1,87 @@ #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/tableref/bound_expressionlistref.hpp" #include "duckdb/parser/tableref/expressionlistref.hpp" #include "duckdb/planner/expression_binder/insert_binder.hpp" #include "duckdb/common/to_string.hpp" #include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/operator/logical_expression_get.hpp" +#include "duckdb/planner/operator/logical_dummy_scan.hpp" namespace duckdb { -unique_ptr Binder::Bind(ExpressionListRef &expr) { - auto result = make_uniq(); - result->types = expr.expected_types; - result->names = expr.expected_names; +BoundStatement Binder::Bind(ExpressionListRef &expr) { + BoundStatement result; + result.types = expr.expected_types; + result.names = expr.expected_names; + + vector>> values; auto prev_can_contain_nulls = this->can_contain_nulls; // bind value list InsertBinder binder(*this, context); binder.target_type = LogicalType(LogicalTypeId::INVALID); for (idx_t list_idx = 0; list_idx < expr.values.size(); list_idx++) { auto &expression_list = expr.values[list_idx]; - if (result->names.empty()) { + if (result.names.empty()) { // no names provided, generate them for (idx_t val_idx = 0; val_idx < expression_list.size(); val_idx++) { - result->names.push_back("col" + to_string(val_idx)); + result.names.push_back("col" + to_string(val_idx)); } } this->can_contain_nulls = true; vector> list; for (idx_t val_idx = 0; val_idx < expression_list.size(); val_idx++) { - if (!result->types.empty()) { - D_ASSERT(result->types.size() == expression_list.size()); - binder.target_type = result->types[val_idx]; + if (!result.types.empty()) { + D_ASSERT(result.types.size() == expression_list.size()); + binder.target_type = result.types[val_idx]; } auto bound_expr = binder.Bind(expression_list[val_idx]); list.push_back(std::move(bound_expr)); } - result->values.push_back(std::move(list)); + values.push_back(std::move(list)); this->can_contain_nulls = prev_can_contain_nulls; } - if (result->types.empty() && !expr.values.empty()) { + if (result.types.empty() && !expr.values.empty()) { // there are no types specified // we have to figure out the result types // for each column, we iterate over all of the expressions and select the max logical type // we initialize all types to SQLNULL - result->types.resize(expr.values[0].size(), LogicalType::SQLNULL); + result.types.resize(expr.values[0].size(), LogicalType::SQLNULL); // now loop over the lists and select the max logical type - for (idx_t list_idx = 0; list_idx < result->values.size(); list_idx++) { - auto &list = result->values[list_idx]; + for (idx_t list_idx = 0; list_idx < values.size(); list_idx++) { + auto &list = values[list_idx]; for (idx_t val_idx = 0; val_idx < list.size(); val_idx++) { - auto ¤t_type = result->types[val_idx]; + auto ¤t_type = result.types[val_idx]; auto next_type = ExpressionBinder::GetExpressionReturnType(*list[val_idx]); - result->types[val_idx] = LogicalType::MaxLogicalType(context, current_type, next_type); + result.types[val_idx] = LogicalType::MaxLogicalType(context, current_type, next_type); } } - for (auto &type : result->types) { + for (auto &type : result.types) { type = LogicalType::NormalizeType(type); } // finally do another loop over the expressions and add casts where required - for (idx_t list_idx = 0; list_idx < result->values.size(); list_idx++) { - auto &list = result->values[list_idx]; + for (idx_t list_idx = 0; list_idx < values.size(); list_idx++) { + auto &list = values[list_idx]; for (idx_t val_idx = 0; val_idx < list.size(); val_idx++) { list[val_idx] = - BoundCastExpression::AddCastToType(context, std::move(list[val_idx]), result->types[val_idx]); + BoundCastExpression::AddCastToType(context, std::move(list[val_idx]), result.types[val_idx]); } } } - result->bind_index = GenerateTableIndex(); - bind_context.AddGenericBinding(result->bind_index, expr.alias, result->names, result->types); - return std::move(result); + auto bind_index = GenerateTableIndex(); + bind_context.AddGenericBinding(bind_index, expr.alias, result.names, result.types); + + // values list, first plan any subqueries in the list + auto root = make_uniq_base(GenerateTableIndex()); + for (auto &expr_list : values) { + for (auto &expr : expr_list) { + PlanSubqueries(expr, root); + } + } + + auto expr_get = make_uniq(bind_index, result.types, std::move(values)); + expr_get->AddChild(std::move(root)); + result.plan = std::move(expr_get); + return result; } } // namespace duckdb diff --git a/src/planner/binder/tableref/bind_joinref.cpp b/src/planner/binder/tableref/bind_joinref.cpp index 257e275befd4..0a6420bfddef 100644 --- a/src/planner/binder/tableref/bind_joinref.cpp +++ b/src/planner/binder/tableref/bind_joinref.cpp @@ -55,7 +55,7 @@ bool Binder::TryFindBinding(const string &using_column, const string &join_side, } throw BinderException(error); } else { - result = binding.get().alias; + result = binding.get().GetBindingAlias(); } } return true; @@ -122,14 +122,14 @@ static vector RemoveDuplicateUsingColumns(const vector &using_co return result; } -unique_ptr Binder::BindJoin(Binder &parent_binder, TableRef &ref) { +BoundStatement Binder::BindJoin(Binder &parent_binder, TableRef &ref) { unnamed_subquery_index = parent_binder.unnamed_subquery_index; auto result = Bind(ref); parent_binder.unnamed_subquery_index = unnamed_subquery_index; return result; } -unique_ptr Binder::Bind(JoinRef &ref) { +BoundStatement Binder::Bind(JoinRef &ref) { auto result = make_uniq(ref.ref_type); result->left_binder = Binder::CreateBinder(context, this); result->right_binder = Binder::CreateBinder(context, this); @@ -188,7 +188,7 @@ unique_ptr Binder::Bind(JoinRef &ref) { case_insensitive_set_t lhs_columns; auto &lhs_binding_list = left_binder.bind_context.GetBindingsList(); for (auto &binding : lhs_binding_list) { - for (auto &column_name : binding->names) { + for (auto &column_name : binding->GetColumnNames()) { lhs_columns.insert(column_name); } } @@ -215,7 +215,7 @@ unique_ptr Binder::Bind(JoinRef &ref) { auto &rhs_binding_list = right_binder.bind_context.GetBindingsList(); for (auto &binding_ref : lhs_binding_list) { auto &binding = *binding_ref; - for (auto &column_name : binding.names) { + for (auto &column_name : binding.GetColumnNames()) { if (!left_candidates.empty()) { left_candidates += ", "; } @@ -224,7 +224,7 @@ unique_ptr Binder::Bind(JoinRef &ref) { } for (auto &binding_ref : rhs_binding_list) { auto &binding = *binding_ref; - for (auto &column_name : binding.names) { + for (auto &column_name : binding.GetColumnNames()) { if (!right_candidates.empty()) { right_candidates += ", "; } @@ -351,7 +351,13 @@ unique_ptr Binder::Bind(JoinRef &ref) { bind_context.RemoveContext(left_bindings); } - return std::move(result); + BoundStatement result_stmt; + result_stmt.types.insert(result_stmt.types.end(), result->left.types.begin(), result->left.types.end()); + result_stmt.types.insert(result_stmt.types.end(), result->right.types.begin(), result->right.types.end()); + result_stmt.names.insert(result_stmt.names.end(), result->left.names.begin(), result->left.names.end()); + result_stmt.names.insert(result_stmt.names.end(), result->right.names.begin(), result->right.names.end()); + result_stmt.plan = CreatePlan(*result); + return result_stmt; } } // namespace duckdb diff --git a/src/planner/binder/tableref/bind_pivot.cpp b/src/planner/binder/tableref/bind_pivot.cpp index 2eb2115301b7..51cfae932d74 100644 --- a/src/planner/binder/tableref/bind_pivot.cpp +++ b/src/planner/binder/tableref/bind_pivot.cpp @@ -9,18 +9,18 @@ #include "duckdb/parser/expression/conjunction_expression.hpp" #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/expression/function_expression.hpp" -#include "duckdb/planner/query_node/bound_select_node.hpp" #include "duckdb/parser/expression/star_expression.hpp" #include "duckdb/common/types/value_map.hpp" #include "duckdb/parser/parsed_expression_iterator.hpp" #include "duckdb/parser/expression/operator_expression.hpp" -#include "duckdb/planner/tableref/bound_subqueryref.hpp" #include "duckdb/planner/tableref/bound_pivotref.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" #include "duckdb/main/client_config.hpp" #include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp" #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp" #include "duckdb/main/query_result.hpp" +#include "duckdb/planner/operator/logical_aggregate.hpp" +#include "duckdb/planner/operator/logical_pivot.hpp" #include "duckdb/main/settings.hpp" namespace duckdb { @@ -58,10 +58,15 @@ static void ConstructPivots(PivotRef &ref, vector &pivot_valu } } -static void ExtractPivotExpressions(ParsedExpression &root_expr, case_insensitive_set_t &handled_columns) { +static void ExtractPivotExpressions(ParsedExpression &root_expr, case_insensitive_set_t &handled_columns, + optional_ptr macro_binding) { ParsedExpressionIterator::VisitExpression( root_expr, [&](const ColumnRefExpression &child_colref) { if (child_colref.IsQualified()) { + if (child_colref.column_names[0].find(DummyBinding::DUMMY_NAME) != string::npos && macro_binding && + macro_binding->HasMatchingBinding(child_colref.GetName())) { + throw ParameterNotResolvedException(); + } throw BinderException(child_colref, "PIVOT expression cannot contain qualified columns"); } handled_columns.insert(child_colref.GetColumnName()); @@ -378,24 +383,23 @@ static unique_ptr PivotFinalOperator(PivotBindState &bind_state, Piv return final_pivot_operator; } -void ExtractPivotAggregates(BoundTableRef &node, vector> &aggregates) { - if (node.type != TableReferenceType::SUBQUERY) { - throw InternalException("Pivot - Expected a subquery"); - } - auto &subq = node.Cast(); - if (subq.subquery->type != QueryNodeType::SELECT_NODE) { - throw InternalException("Pivot - Expected a select node"); - } - auto &select = subq.subquery->Cast(); - if (select.from_table->type != TableReferenceType::SUBQUERY) { - throw InternalException("Pivot - Expected another subquery"); - } - auto &subq2 = select.from_table->Cast(); - if (subq2.subquery->type != QueryNodeType::SELECT_NODE) { - throw InternalException("Pivot - Expected another select node"); +void ExtractPivotAggregates(BoundStatement &node, vector> &aggregates) { + reference op(*node.plan); + bool found_first_aggregate = false; + while (true) { + if (op.get().type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + if (found_first_aggregate) { + break; + } + found_first_aggregate = true; + } + if (op.get().children.size() != 1) { + throw InternalException("Pivot - expected an aggregate"); + } + op = *op.get().children[0]; } - auto &select2 = subq2.subquery->Cast(); - for (auto &aggr : select2.aggregates) { + auto &aggr_op = op.get().Cast(); + for (auto &aggr : aggr_op.expressions) { if (aggr->GetAlias() == "__collated_group") { continue; } @@ -412,15 +416,15 @@ string GetPivotAggregateName(const PivotValueElement &pivot_value, const string return name; } -unique_ptr Binder::BindBoundPivot(PivotRef &ref) { +BoundStatement Binder::BindBoundPivot(PivotRef &ref) { // bind the child table in a child binder - auto result = make_uniq(); - result->bind_index = GenerateTableIndex(); - result->child_binder = Binder::CreateBinder(context, this); - result->child = result->child_binder->Bind(*ref.source); + BoundPivotRef result; + result.bind_index = GenerateTableIndex(); + result.child_binder = Binder::CreateBinder(context, this); + result.child = result.child_binder->Bind(*ref.source); - auto &aggregates = result->bound_pivot.aggregates; - ExtractPivotAggregates(*result->child, aggregates); + auto &aggregates = result.bound_pivot.aggregates; + ExtractPivotAggregates(result.child, aggregates); if (aggregates.size() != ref.bound_aggregate_names.size()) { throw InternalException("Pivot aggregate count mismatch (expected %llu, found %llu)", ref.bound_aggregate_names.size(), aggregates.size()); @@ -428,7 +432,7 @@ unique_ptr Binder::BindBoundPivot(PivotRef &ref) { vector child_names; vector child_types; - result->child_binder->bind_context.GetTypesAndNames(child_names, child_types); + result.child_binder->bind_context.GetTypesAndNames(child_names, child_types); vector names; vector types; @@ -453,19 +457,23 @@ unique_ptr Binder::BindBoundPivot(PivotRef &ref) { pivot_str += "_" + str; } } - result->bound_pivot.pivot_values.push_back(std::move(pivot_str)); + result.bound_pivot.pivot_values.push_back(std::move(pivot_str)); names.push_back(std::move(name)); types.push_back(aggr->return_type); } } - result->bound_pivot.group_count = ref.bound_group_names.size(); - result->bound_pivot.types = types; + result.bound_pivot.group_count = ref.bound_group_names.size(); + result.bound_pivot.types = types; auto subquery_alias = ref.alias.empty() ? "__unnamed_pivot" : ref.alias; QueryResult::DeduplicateColumns(names); - bind_context.AddGenericBinding(result->bind_index, subquery_alias, names, types); + bind_context.AddGenericBinding(result.bind_index, subquery_alias, names, types); + + MoveCorrelatedExpressions(*result.child_binder); - MoveCorrelatedExpressions(*result->child_binder); - return std::move(result); + BoundStatement result_statement; + result_statement.plan = + make_uniq(result.bind_index, std::move(result.child.plan), std::move(result.bound_pivot)); + return result_statement; } unique_ptr Binder::BindPivot(PivotRef &ref, vector> all_columns) { @@ -492,7 +500,7 @@ unique_ptr Binder::BindPivot(PivotRef &ref, vector Binder::BindPivot(PivotRef &ref, vector Binder::BindUnpivot(Binder &child_binder, PivotRef &ref, return result_node; } -unique_ptr Binder::Bind(PivotRef &ref) { +BoundStatement Binder::Bind(PivotRef &ref) { if (!ref.source) { throw InternalException("Pivot without a source!?"); } @@ -858,13 +866,10 @@ unique_ptr Binder::Bind(PivotRef &ref) { } // bind the generated select node auto child_binder = Binder::CreateBinder(context, this); - auto bound_select_node = child_binder->BindNode(*select_node); - auto root_index = bound_select_node->GetRootIndex(); - BoundQueryNode *bound_select_ptr = bound_select_node.get(); + auto result = child_binder->BindNode(*select_node); + auto root_index = result.plan->GetRootIndex(); - unique_ptr result; MoveCorrelatedExpressions(*child_binder); - result = make_uniq(std::move(child_binder), std::move(bound_select_node)); auto subquery_alias = ref.alias.empty() ? "__unnamed_pivot" : ref.alias; SubqueryRef subquery_ref(nullptr, subquery_alias); subquery_ref.column_name_alias = std::move(ref.column_name_alias); @@ -872,16 +877,14 @@ unique_ptr Binder::Bind(PivotRef &ref) { // if a WHERE clause was provided - bind a subquery holding the WHERE clause // we need to bind a new subquery here because the WHERE clause has to be applied AFTER the unnest child_binder = Binder::CreateBinder(context, this); - child_binder->bind_context.AddSubquery(root_index, subquery_ref.alias, subquery_ref, *bound_select_ptr); + child_binder->bind_context.AddSubquery(root_index, subquery_ref.alias, subquery_ref, result); auto where_query = make_uniq(); where_query->select_list.push_back(make_uniq()); where_query->where_clause = std::move(where_clause); - bound_select_node = child_binder->BindSelectNode(*where_query, std::move(result)); - bound_select_ptr = bound_select_node.get(); - root_index = bound_select_node->GetRootIndex(); - result = make_uniq(std::move(child_binder), std::move(bound_select_node)); + result = child_binder->BindSelectNode(*where_query, std::move(result)); + root_index = result.plan->GetRootIndex(); } - bind_context.AddSubquery(root_index, subquery_ref.alias, subquery_ref, *bound_select_ptr); + bind_context.AddSubquery(root_index, subquery_ref.alias, subquery_ref, result); return result; } diff --git a/src/planner/binder/tableref/bind_showref.cpp b/src/planner/binder/tableref/bind_showref.cpp index b23456cab056..d2d91c3af276 100644 --- a/src/planner/binder/tableref/bind_showref.cpp +++ b/src/planner/binder/tableref/bind_showref.cpp @@ -5,12 +5,10 @@ #include "duckdb/parser/tableref/subqueryref.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/operator/logical_column_data_get.hpp" -#include "duckdb/planner/tableref/bound_table_function.hpp" #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/operator/logical_projection.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/catalog/catalog.hpp" -#include "duckdb/catalog/catalog_search_path.hpp" #include "duckdb/main/client_data.hpp" #include "duckdb/main/client_context.hpp" @@ -89,7 +87,7 @@ BaseTableColumnInfo FindBaseTableColumn(LogicalOperator &op, idx_t column_index) return FindBaseTableColumn(op, bindings[column_index]); } -unique_ptr Binder::BindShowQuery(ShowRef &ref) { +BoundStatement Binder::BindShowQuery(ShowRef &ref) { // bind the child plan of the DESCRIBE statement auto child_binder = Binder::CreateBinder(context, this); auto plan = child_binder->Bind(*ref.query); @@ -142,12 +140,17 @@ unique_ptr Binder::BindShowQuery(ShowRef &ref) { } collection->Append(append_state, output); - auto show = make_uniq(GenerateTableIndex(), return_types, std::move(collection)); - bind_context.AddGenericBinding(show->table_index, "__show_select", return_names, return_types); - return make_uniq(std::move(show)); + auto table_index = GenerateTableIndex(); + + BoundStatement result; + result.names = return_names; + result.types = return_types; + result.plan = make_uniq(table_index, return_types, std::move(collection)); + bind_context.AddGenericBinding(table_index, "__show_select", return_names, return_types); + return result; } -unique_ptr Binder::BindShowTable(ShowRef &ref) { +BoundStatement Binder::BindShowTable(ShowRef &ref) { auto lname = StringUtil::Lower(ref.table_name); string sql; @@ -193,7 +196,7 @@ unique_ptr Binder::BindShowTable(ShowRef &ref) { return Bind(*subquery); } -unique_ptr Binder::Bind(ShowRef &ref) { +BoundStatement Binder::Bind(ShowRef &ref) { if (ref.show_type == ShowType::SUMMARY) { return BindSummarize(ref); } diff --git a/src/planner/binder/tableref/bind_subqueryref.cpp b/src/planner/binder/tableref/bind_subqueryref.cpp index 5acbd06b5d03..cfa727927ad3 100644 --- a/src/planner/binder/tableref/bind_subqueryref.cpp +++ b/src/planner/binder/tableref/bind_subqueryref.cpp @@ -1,18 +1,14 @@ #include "duckdb/parser/tableref/subqueryref.hpp" #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/tableref/bound_subqueryref.hpp" namespace duckdb { -unique_ptr Binder::Bind(SubqueryRef &ref, optional_ptr cte) { +BoundStatement Binder::Bind(SubqueryRef &ref) { auto binder = Binder::CreateBinder(context, this); binder->can_contain_nulls = true; - if (cte) { - binder->bound_ctes.insert(*cte); - } auto subquery = binder->BindNode(*ref.subquery->node); binder->alias = ref.alias.empty() ? "unnamed_subquery" : ref.alias; - idx_t bind_index = subquery->GetRootIndex(); + idx_t bind_index = subquery.plan->GetRootIndex(); string subquery_alias; if (ref.alias.empty()) { auto index = unnamed_subquery_index++; @@ -24,10 +20,14 @@ unique_ptr Binder::Bind(SubqueryRef &ref, optional_ptr(std::move(binder), std::move(subquery)); - bind_context.AddSubquery(bind_index, subquery_alias, ref, *result->subquery); - MoveCorrelatedExpressions(*result->binder); - return std::move(result); + binder->is_outside_flattened = is_outside_flattened; + if (binder->has_unplanned_dependent_joins) { + has_unplanned_dependent_joins = true; + } + bind_context.AddSubquery(bind_index, subquery_alias, ref, subquery); + MoveCorrelatedExpressions(*binder); + + return subquery; } } // namespace duckdb diff --git a/src/planner/binder/tableref/bind_table_function.cpp b/src/planner/binder/tableref/bind_table_function.cpp index 0c6e1e0aaf96..abb0570115e2 100644 --- a/src/planner/binder/tableref/bind_table_function.cpp +++ b/src/planner/binder/tableref/bind_table_function.cpp @@ -13,9 +13,6 @@ #include "duckdb/planner/expression_binder/table_function_binder.hpp" #include "duckdb/planner/expression_binder/select_binder.hpp" #include "duckdb/planner/operator/logical_get.hpp" -#include "duckdb/planner/query_node/bound_select_node.hpp" -#include "duckdb/planner/tableref/bound_subqueryref.hpp" -#include "duckdb/planner/tableref/bound_table_function.hpp" #include "duckdb/function/function_binder.hpp" #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" @@ -79,32 +76,28 @@ static TableFunctionBindType GetTableFunctionBindType(TableFunctionCatalogEntry : TableFunctionBindType::STANDARD_TABLE_FUNCTION; } -void Binder::BindTableInTableOutFunction(vector> &expressions, - unique_ptr &subquery) { +void Binder::BindTableInTableOutFunction(vector> &expressions, BoundStatement &subquery) { auto binder = Binder::CreateBinder(this->context, this); - unique_ptr subquery_node; // generate a subquery and bind that (i.e. UNNEST([1,2,3]) becomes UNNEST((SELECT [1,2,3])) auto select_node = make_uniq(); select_node->select_list = std::move(expressions); select_node->from_table = make_uniq(); - subquery_node = std::move(select_node); binder->can_contain_nulls = true; - auto node = binder->BindNode(*subquery_node); - subquery = make_uniq(std::move(binder), std::move(node)); - MoveCorrelatedExpressions(*subquery->binder); + subquery = binder->BindNode(*select_node); + MoveCorrelatedExpressions(*binder); } bool Binder::BindTableFunctionParameters(TableFunctionCatalogEntry &table_function, vector> &expressions, vector &arguments, vector ¶meters, - named_parameter_map_t &named_parameters, - unique_ptr &subquery, ErrorData &error) { + named_parameter_map_t &named_parameters, BoundStatement &subquery, + ErrorData &error) { auto bind_type = GetTableFunctionBindType(table_function, expressions); if (bind_type == TableFunctionBindType::TABLE_IN_OUT_FUNCTION) { // bind table in-out function BindTableInTableOutFunction(expressions, subquery); // fetch the arguments from the subquery - arguments = subquery->subquery->types; + arguments = subquery.types; return true; } bool seen_subquery = false; @@ -142,9 +135,8 @@ bool Binder::BindTableFunctionParameters(TableFunctionCatalogEntry &table_functi auto binder = Binder::CreateBinder(this->context, this); binder->can_contain_nulls = true; auto &se = child->Cast(); - auto node = binder->BindNode(*se.subquery->node); - subquery = make_uniq(std::move(binder), std::move(node)); - MoveCorrelatedExpressions(*subquery->binder); + subquery = binder->BindNode(*se.subquery->node); + MoveCorrelatedExpressions(*binder); seen_subquery = true; arguments.emplace_back(LogicalTypeId::TABLE); parameters.emplace_back(Value()); @@ -188,11 +180,10 @@ static string GetAlias(const TableFunctionRef &ref) { return string(); } -unique_ptr Binder::BindTableFunctionInternal(TableFunction &table_function, - const TableFunctionRef &ref, vector parameters, - named_parameter_map_t named_parameters, - vector input_table_types, - vector input_table_names) { +BoundStatement Binder::BindTableFunctionInternal(TableFunction &table_function, const TableFunctionRef &ref, + vector parameters, named_parameter_map_t named_parameters, + vector input_table_types, + vector input_table_names) { auto function_name = GetAlias(ref); auto &column_name_alias = ref.column_name_alias; auto bind_index = GenerateTableIndex(); @@ -221,8 +212,12 @@ unique_ptr Binder::BindTableFunctionInternal(TableFunction &tab table_function.name); } } + BoundStatement result; bind_context.AddGenericBinding(bind_index, function_name, return_names, new_plan->types); - return new_plan; + result.names = return_names; + result.types = new_plan->types; + result.plan = std::move(new_plan); + return result; } } if (table_function.bind_replace) { @@ -234,7 +229,7 @@ unique_ptr Binder::BindTableFunctionInternal(TableFunction &tab if (!ref.column_name_alias.empty()) { new_plan->column_name_alias = ref.column_name_alias; } - return CreatePlan(*Bind(*new_plan)); + return Bind(*new_plan); } } if (!table_function.bind) { @@ -343,16 +338,24 @@ unique_ptr Binder::BindTableFunctionInternal(TableFunction &tab return_types.push_back(LogicalType::BIGINT); bind_context.AddGenericBinding(projection_index, function_name, return_names, return_types); - return std::move(projection); + BoundStatement result; + result.names = std::move(return_names); + result.types = std::move(return_types); + result.plan = std::move(projection); + return result; } // now add the table function to the bind context so its columns can be bound + BoundStatement result; bind_context.AddTableFunction(bind_index, function_name, return_names, return_types, get->GetMutableColumnIds(), get->GetTable().get(), std::move(virtual_columns)); - return std::move(get); + result.names = std::move(return_names); + result.types = std::move(return_types); + result.plan = std::move(get); + return result; } -unique_ptr Binder::BindTableFunction(TableFunction &function, vector parameters) { +BoundStatement Binder::BindTableFunction(TableFunction &function, vector parameters) { named_parameter_map_t named_parameters; vector input_table_types; vector input_table_names; @@ -364,7 +367,7 @@ unique_ptr Binder::BindTableFunction(TableFunction &function, v std::move(input_table_types), std::move(input_table_names)); } -unique_ptr Binder::Bind(TableFunctionRef &ref) { +BoundStatement Binder::Bind(TableFunctionRef &ref) { QueryErrorContext error_context(ref.query_location); D_ASSERT(ref.function->GetExpressionType() == ExpressionType::FUNCTION); @@ -388,7 +391,7 @@ unique_ptr Binder::Bind(TableFunctionRef &ref) { binder->can_contain_nulls = true; binder->alias = ref.alias.empty() ? "unnamed_query" : ref.alias; - unique_ptr query; + BoundStatement query; try { query = binder->BindNode(*query_node); } catch (std::exception &ex) { @@ -397,15 +400,14 @@ unique_ptr Binder::Bind(TableFunctionRef &ref) { error.Throw(); } - idx_t bind_index = query->GetRootIndex(); + idx_t bind_index = query.plan->GetRootIndex(); // string alias; string alias = (ref.alias.empty() ? "unnamed_query" + to_string(bind_index) : ref.alias); - auto result = make_uniq(std::move(binder), std::move(query)); // remember ref here is TableFunctionRef and NOT base class - bind_context.AddSubquery(bind_index, alias, ref, *result->subquery); - MoveCorrelatedExpressions(*result->binder); - return std::move(result); + bind_context.AddSubquery(bind_index, alias, ref, query); + MoveCorrelatedExpressions(*binder); + return query; } D_ASSERT(func_catalog.type == CatalogType::TABLE_FUNCTION_ENTRY); auto &function = func_catalog.Cast(); @@ -414,7 +416,7 @@ unique_ptr Binder::Bind(TableFunctionRef &ref) { vector arguments; vector parameters; named_parameter_map_t named_parameters; - unique_ptr subquery; + BoundStatement subquery; ErrorData error; if (!BindTableFunctionParameters(function, fexpr.children, arguments, parameters, named_parameters, subquery, error)) { @@ -437,9 +439,9 @@ unique_ptr Binder::Bind(TableFunctionRef &ref) { vector input_table_types; vector input_table_names; - if (subquery) { - input_table_types = subquery->subquery->types; - input_table_names = subquery->subquery->names; + if (subquery.plan) { + input_table_types = subquery.types; + input_table_names = subquery.names; } else if (table_function.in_out_function) { for (auto ¶m : parameters) { input_table_types.push_back(param.type()); @@ -457,7 +459,7 @@ unique_ptr Binder::Bind(TableFunctionRef &ref) { parameters[i] = parameters[i].CastAs(context, target_type); } } - } else if (subquery) { + } else if (subquery.plan) { for (idx_t i = 0; i < arguments.size(); i++) { auto target_type = i < table_function.arguments.size() ? table_function.arguments[i] : table_function.varargs; @@ -469,11 +471,39 @@ unique_ptr Binder::Bind(TableFunctionRef &ref) { } } - auto get = BindTableFunctionInternal(table_function, ref, std::move(parameters), std::move(named_parameters), - std::move(input_table_types), std::move(input_table_names)); - auto table_function_ref = make_uniq(std::move(get)); - table_function_ref->subquery = std::move(subquery); - return std::move(table_function_ref); + BoundStatement get; + try { + get = BindTableFunctionInternal(table_function, ref, std::move(parameters), std::move(named_parameters), + std::move(input_table_types), std::move(input_table_names)); + } catch (std::exception &ex) { + error = ErrorData(ex); + error.AddQueryLocation(ref); + error.Throw(); + } + + if (subquery.plan) { + auto child_node = std::move(subquery.plan); + + reference node = *get.plan; + + while (!node.get().children.empty()) { + D_ASSERT(node.get().children.size() == 1); + if (node.get().children.size() != 1) { + throw InternalException( + "Binder::CreatePlan: linear path expected, but found node with %d children", + node.get().children.size()); + } + node = *node.get().children[0]; + } + + D_ASSERT(node.get().type == LogicalOperatorType::LOGICAL_GET); + node.get().children.push_back(std::move(child_node)); + } + BoundStatement result_statement; + result_statement.names = get.names; + result_statement.types = get.types; + result_statement.plan = std::move(get.plan); + return result_statement; } } // namespace duckdb diff --git a/src/planner/binder/tableref/plan_basetableref.cpp b/src/planner/binder/tableref/plan_basetableref.cpp deleted file mode 100644 index 085498fbb64e..000000000000 --- a/src/planner/binder/tableref/plan_basetableref.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/operator/logical_get.hpp" -#include "duckdb/planner/tableref/bound_basetableref.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundBaseTableRef &ref) { - return std::move(ref.get); -} - -} // namespace duckdb diff --git a/src/planner/binder/tableref/plan_column_data_ref.cpp b/src/planner/binder/tableref/plan_column_data_ref.cpp deleted file mode 100644 index 83e965b5e5cd..000000000000 --- a/src/planner/binder/tableref/plan_column_data_ref.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/tableref/bound_column_data_ref.hpp" -#include "duckdb/planner/operator/logical_column_data_get.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundColumnDataRef &ref) { - auto types = ref.collection->Types(); - // Create a (potentially owning) LogicalColumnDataGet - auto root = make_uniq_base(ref.bind_index, std::move(types), - std::move(ref.collection)); - return root; -} - -} // namespace duckdb diff --git a/src/planner/binder/tableref/plan_cteref.cpp b/src/planner/binder/tableref/plan_cteref.cpp deleted file mode 100644 index 8bf1934fc3bd..000000000000 --- a/src/planner/binder/tableref/plan_cteref.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/operator/logical_cteref.hpp" -#include "duckdb/planner/tableref/bound_cteref.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundCTERef &ref) { - return make_uniq(ref.bind_index, ref.cte_index, ref.types, ref.bound_columns, ref.materialized_cte, - ref.is_recurring); -} - -} // namespace duckdb diff --git a/src/planner/binder/tableref/plan_delimgetref.cpp b/src/planner/binder/tableref/plan_delimgetref.cpp deleted file mode 100644 index b674b43df990..000000000000 --- a/src/planner/binder/tableref/plan_delimgetref.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/operator/logical_get.hpp" -#include "duckdb/planner/tableref/bound_basetableref.hpp" -#include "duckdb/planner/operator/logical_delim_get.hpp" -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundDelimGetRef &ref) { - return make_uniq(ref.bind_index, ref.column_types); -} - -} // namespace duckdb diff --git a/src/planner/binder/tableref/plan_dummytableref.cpp b/src/planner/binder/tableref/plan_dummytableref.cpp deleted file mode 100644 index f31fc929bcdc..000000000000 --- a/src/planner/binder/tableref/plan_dummytableref.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/operator/logical_dummy_scan.hpp" -#include "duckdb/planner/tableref/bound_dummytableref.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundEmptyTableRef &ref) { - return make_uniq(ref.bind_index); -} - -} // namespace duckdb diff --git a/src/planner/binder/tableref/plan_expressionlistref.cpp b/src/planner/binder/tableref/plan_expressionlistref.cpp deleted file mode 100644 index ba6253bce5f3..000000000000 --- a/src/planner/binder/tableref/plan_expressionlistref.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/tableref/bound_expressionlistref.hpp" -#include "duckdb/planner/operator/logical_expression_get.hpp" -#include "duckdb/planner/operator/logical_dummy_scan.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundExpressionListRef &ref) { - auto root = make_uniq_base(GenerateTableIndex()); - // values list, first plan any subqueries in the list - for (auto &expr_list : ref.values) { - for (auto &expr : expr_list) { - PlanSubqueries(expr, root); - } - } - // now create a LogicalExpressionGet from the set of expressions - // fetch the types - vector types; - for (auto &expr : ref.values[0]) { - types.push_back(expr->return_type); - } - auto expr_get = make_uniq(ref.bind_index, types, std::move(ref.values)); - expr_get->AddChild(std::move(root)); - return std::move(expr_get); -} - -} // namespace duckdb diff --git a/src/planner/binder/tableref/plan_joinref.cpp b/src/planner/binder/tableref/plan_joinref.cpp index 9de5829f238d..aa6193b13fe5 100644 --- a/src/planner/binder/tableref/plan_joinref.cpp +++ b/src/planner/binder/tableref/plan_joinref.cpp @@ -298,8 +298,8 @@ unique_ptr Binder::CreatePlan(BoundJoinRef &ref) { // Set the flag to ensure that children do not flatten before the root is_outside_flattened = false; } - auto left = CreatePlan(*ref.left); - auto right = CreatePlan(*ref.right); + auto left = std::move(ref.left.plan); + auto right = std::move(ref.right.plan); is_outside_flattened = old_is_outside_flattened; // For joins, depth of the bindings will be one higher on the right because of the lateral binder diff --git a/src/planner/binder/tableref/plan_pivotref.cpp b/src/planner/binder/tableref/plan_pivotref.cpp deleted file mode 100644 index 4d9482e5b2bb..000000000000 --- a/src/planner/binder/tableref/plan_pivotref.cpp +++ /dev/null @@ -1,13 +0,0 @@ -#include "duckdb/planner/tableref/bound_pivotref.hpp" -#include "duckdb/planner/operator/logical_pivot.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundPivotRef &ref) { - auto subquery = ref.child_binder->CreatePlan(*ref.child); - - auto result = make_uniq(ref.bind_index, std::move(subquery), std::move(ref.bound_pivot)); - return std::move(result); -} - -} // namespace duckdb diff --git a/src/planner/binder/tableref/plan_subqueryref.cpp b/src/planner/binder/tableref/plan_subqueryref.cpp deleted file mode 100644 index 821654460ab5..000000000000 --- a/src/planner/binder/tableref/plan_subqueryref.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/tableref/bound_subqueryref.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundSubqueryRef &ref) { - // generate the logical plan for the subquery - // this happens separately from the current LogicalPlan generation - ref.binder->is_outside_flattened = is_outside_flattened; - auto subquery = ref.binder->CreatePlan(*ref.subquery); - if (ref.binder->has_unplanned_dependent_joins) { - has_unplanned_dependent_joins = true; - } - return subquery; -} - -} // namespace duckdb diff --git a/src/planner/binder/tableref/plan_table_function.cpp b/src/planner/binder/tableref/plan_table_function.cpp deleted file mode 100644 index 6c2f9957a77d..000000000000 --- a/src/planner/binder/tableref/plan_table_function.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/tableref/bound_table_function.hpp" - -namespace duckdb { - -unique_ptr Binder::CreatePlan(BoundTableFunction &ref) { - if (ref.subquery) { - auto child_node = CreatePlan(*ref.subquery); - - reference node = *ref.get; - - while (!node.get().children.empty()) { - D_ASSERT(node.get().children.size() == 1); - if (node.get().children.size() != 1) { - throw InternalException( - "Binder::CreatePlan: linear path expected, but found node with %d children", - node.get().children.size()); - } - node = *node.get().children[0]; - } - - D_ASSERT(node.get().type == LogicalOperatorType::LOGICAL_GET); - node.get().children.push_back(std::move(child_node)); - } - return std::move(ref.get); -} - -} // namespace duckdb diff --git a/src/planner/collation_binding.cpp b/src/planner/collation_binding.cpp index 1ddefb9a894f..dd371bbc4284 100644 --- a/src/planner/collation_binding.cpp +++ b/src/planner/collation_binding.cpp @@ -8,6 +8,7 @@ #include "duckdb/function/function_binder.hpp" namespace duckdb { +constexpr const char *CollateCatalogEntry::Name; bool PushVarcharCollation(ClientContext &context, unique_ptr &source, const LogicalType &sql_type, CollationType type) { @@ -109,11 +110,34 @@ bool PushIntervalCollation(ClientContext &context, unique_ptr &sourc return true; } +bool PushVariantCollation(ClientContext &context, unique_ptr &source, const LogicalType &sql_type, + CollationType) { + if (sql_type.id() != LogicalTypeId::VARIANT) { + return false; + } + auto &catalog = Catalog::GetSystemCatalog(context); + auto &function_entry = catalog.GetEntry(context, DEFAULT_SCHEMA, "variant_normalize"); + if (function_entry.functions.Size() != 1) { + throw InternalException("variant_normalize should only have a single overload"); + } + auto source_alias = source->GetAlias(); + auto &scalar_function = function_entry.functions.GetFunctionReferenceByOffset(0); + vector> children; + children.push_back(std::move(source)); + + FunctionBinder function_binder(context); + auto function = function_binder.BindScalarFunction(scalar_function, std::move(children)); + function->SetAlias(source_alias); + source = std::move(function); + return true; +} + // timetz_byte_comparable CollationBinding::CollationBinding() { RegisterCollation(CollationCallback(PushVarcharCollation)); RegisterCollation(CollationCallback(PushTimeTZCollation)); RegisterCollation(CollationCallback(PushIntervalCollation)); + RegisterCollation(CollationCallback(PushVariantCollation)); } void CollationBinding::RegisterCollation(CollationCallback callback) { diff --git a/src/planner/expression_binder.cpp b/src/planner/expression_binder.cpp index 5141765bb28b..2207147337fa 100644 --- a/src/planner/expression_binder.cpp +++ b/src/planner/expression_binder.cpp @@ -103,7 +103,9 @@ BindResult ExpressionBinder::BindExpression(unique_ptr &expr, case ExpressionClass::STAR: return BindResult(BinderException::Unsupported(expr_ref, "STAR expression is not supported here")); default: - throw NotImplementedException("Unimplemented expression class"); + return BindResult( + NotImplementedException("Unimplemented expression class in ExpressionBinder::BindExpression: %s", + EnumUtil::ToString(expr_ref.GetExpressionClass()))); } } diff --git a/src/planner/expression_binder/lateral_binder.cpp b/src/planner/expression_binder/lateral_binder.cpp index 205b644e8025..0b693558acc1 100644 --- a/src/planner/expression_binder/lateral_binder.cpp +++ b/src/planner/expression_binder/lateral_binder.cpp @@ -3,7 +3,7 @@ #include "duckdb/planner/logical_operator_visitor.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression/bound_subquery_expression.hpp" -#include "duckdb/planner/tableref/bound_joinref.hpp" +#include "duckdb/planner/operator/logical_dependent_join.hpp" namespace duckdb { @@ -17,7 +17,7 @@ void LateralBinder::ExtractCorrelatedColumns(Expression &expr) { // add the correlated column info CorrelatedColumnInfo info(bound_colref); if (std::find(correlated_columns.begin(), correlated_columns.end(), info) == correlated_columns.end()) { - correlated_columns.push_back(std::move(info)); + correlated_columns.AddColumn(std::move(info)); // TODO is adding to the front OK here? } } } @@ -54,8 +54,7 @@ string LateralBinder::UnsupportedAggregateMessage() { return "LATERAL join cannot contain aggregates!"; } -static void ReduceColumnRefDepth(BoundColumnRefExpression &expr, - const vector &correlated_columns) { +static void ReduceColumnRefDepth(BoundColumnRefExpression &expr, const CorrelatedColumns &correlated_columns) { // don't need to reduce this if (expr.depth == 0) { return; @@ -69,8 +68,7 @@ static void ReduceColumnRefDepth(BoundColumnRefExpression &expr, } } -static void ReduceColumnDepth(vector &columns, - const vector &affected_columns) { +static void ReduceColumnDepth(CorrelatedColumns &columns, const CorrelatedColumns &affected_columns) { for (auto &s_correlated : columns) { for (auto &affected : affected_columns) { if (affected == s_correlated) { @@ -81,45 +79,44 @@ static void ReduceColumnDepth(vector &columns, } } -class ExpressionDepthReducerRecursive : public BoundNodeVisitor { +class ExpressionDepthReducerRecursive : public LogicalOperatorVisitor { public: - explicit ExpressionDepthReducerRecursive(const vector &correlated) - : correlated_columns(correlated) { + explicit ExpressionDepthReducerRecursive(const CorrelatedColumns &correlated) : correlated_columns(correlated) { } - void VisitExpression(unique_ptr &expression) override { - if (expression->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { - ReduceColumnRefDepth(expression->Cast(), correlated_columns); - } else if (expression->GetExpressionType() == ExpressionType::SUBQUERY) { - ReduceExpressionSubquery(expression->Cast(), correlated_columns); + void VisitExpression(unique_ptr *expression) override { + auto &expr = **expression; + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { + ReduceColumnRefDepth(expr.Cast(), correlated_columns); + } else if (expr.GetExpressionType() == ExpressionType::SUBQUERY) { + ReduceExpressionSubquery(expr.Cast(), correlated_columns); } - BoundNodeVisitor::VisitExpression(expression); + LogicalOperatorVisitor::VisitExpression(expression); } - void VisitBoundTableRef(BoundTableRef &ref) override { - if (ref.type == TableReferenceType::JOIN) { + void VisitOperator(LogicalOperator &op) override { + if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) { // rewrite correlated columns in child joins - auto &bound_join = ref.Cast(); + auto &bound_join = op.Cast(); ReduceColumnDepth(bound_join.correlated_columns, correlated_columns); } // visit the children of the table ref - BoundNodeVisitor::VisitBoundTableRef(ref); + LogicalOperatorVisitor::VisitOperator(op); } - static void ReduceExpressionSubquery(BoundSubqueryExpression &expr, - const vector &correlated_columns) { + static void ReduceExpressionSubquery(BoundSubqueryExpression &expr, const CorrelatedColumns &correlated_columns) { ReduceColumnDepth(expr.binder->correlated_columns, correlated_columns); ExpressionDepthReducerRecursive recursive(correlated_columns); - recursive.VisitBoundQueryNode(*expr.subquery); + recursive.VisitOperator(*expr.subquery.plan); } private: - const vector &correlated_columns; + const CorrelatedColumns &correlated_columns; }; class ExpressionDepthReducer : public LogicalOperatorVisitor { public: - explicit ExpressionDepthReducer(const vector &correlated) : correlated_columns(correlated) { + explicit ExpressionDepthReducer(const CorrelatedColumns &correlated) : correlated_columns(correlated) { } protected: @@ -133,10 +130,10 @@ class ExpressionDepthReducer : public LogicalOperatorVisitor { return nullptr; } - const vector &correlated_columns; + const CorrelatedColumns &correlated_columns; }; -void LateralBinder::ReduceExpressionDepth(LogicalOperator &op, const vector &correlated) { +void LateralBinder::ReduceExpressionDepth(LogicalOperator &op, const CorrelatedColumns &correlated) { ExpressionDepthReducer depth_reducer(correlated); depth_reducer.VisitOperator(op); } diff --git a/src/planner/expression_binder/table_function_binder.cpp b/src/planner/expression_binder/table_function_binder.cpp index 198bd072be17..720dbe37db6f 100644 --- a/src/planner/expression_binder/table_function_binder.cpp +++ b/src/planner/expression_binder/table_function_binder.cpp @@ -27,9 +27,13 @@ BindResult TableFunctionBinder::BindColumnReference(unique_ptr if (lambda_ref) { return BindLambdaReference(lambda_ref->Cast(), depth); } + if (binder.macro_binding && binder.macro_binding->HasMatchingBinding(col_ref.GetName())) { throw ParameterNotResolvedException(); } + } else if (col_ref.column_names[0].find(DummyBinding::DUMMY_NAME) != string::npos && binder.macro_binding && + binder.macro_binding->HasMatchingBinding(col_ref.GetName())) { + throw ParameterNotResolvedException(); } auto query_location = col_ref.GetQueryLocation(); diff --git a/src/planner/expression_iterator.cpp b/src/planner/expression_iterator.cpp index 8f3140e176eb..3d14079001de 100644 --- a/src/planner/expression_iterator.cpp +++ b/src/planner/expression_iterator.cpp @@ -4,8 +4,6 @@ #include "duckdb/planner/expression/list.hpp" #include "duckdb/planner/query_node/bound_select_node.hpp" #include "duckdb/planner/query_node/bound_set_operation_node.hpp" -#include "duckdb/planner/query_node/bound_recursive_cte_node.hpp" -#include "duckdb/planner/query_node/bound_cte_node.hpp" #include "duckdb/planner/tableref/list.hpp" #include "duckdb/common/enum_util.hpp" @@ -183,155 +181,4 @@ void ExpressionIterator::VisitExpressionClassMutable( *expr, [&](unique_ptr &child) { VisitExpressionClassMutable(child, expr_class, callback); }); } -void BoundNodeVisitor::VisitExpression(unique_ptr &expression) { - VisitExpressionChildren(*expression); -} - -void BoundNodeVisitor::VisitExpressionChildren(Expression &expr) { - ExpressionIterator::EnumerateChildren(expr, [&](unique_ptr &expr) { VisitExpression(expr); }); -} - -void BoundNodeVisitor::VisitBoundQueryNode(BoundQueryNode &node) { - switch (node.type) { - case QueryNodeType::SET_OPERATION_NODE: { - auto &bound_setop = node.Cast(); - VisitBoundQueryNode(*bound_setop.left); - VisitBoundQueryNode(*bound_setop.right); - break; - } - case QueryNodeType::RECURSIVE_CTE_NODE: { - auto &cte_node = node.Cast(); - VisitBoundQueryNode(*cte_node.left); - VisitBoundQueryNode(*cte_node.right); - break; - } - case QueryNodeType::CTE_NODE: { - auto &cte_node = node.Cast(); - VisitBoundQueryNode(*cte_node.child); - VisitBoundQueryNode(*cte_node.query); - break; - } - case QueryNodeType::SELECT_NODE: { - auto &bound_select = node.Cast(); - for (auto &expr : bound_select.select_list) { - VisitExpression(expr); - } - if (bound_select.where_clause) { - VisitExpression(bound_select.where_clause); - } - for (auto &expr : bound_select.groups.group_expressions) { - VisitExpression(expr); - } - if (bound_select.having) { - VisitExpression(bound_select.having); - } - for (auto &expr : bound_select.aggregates) { - VisitExpression(expr); - } - for (auto &entry : bound_select.unnests) { - for (auto &expr : entry.second.expressions) { - VisitExpression(expr); - } - } - for (auto &expr : bound_select.windows) { - VisitExpression(expr); - } - if (bound_select.from_table) { - VisitBoundTableRef(*bound_select.from_table); - } - break; - } - default: - throw NotImplementedException("Unimplemented query node in ExpressionIterator"); - } - for (idx_t i = 0; i < node.modifiers.size(); i++) { - switch (node.modifiers[i]->type) { - case ResultModifierType::DISTINCT_MODIFIER: - for (auto &expr : node.modifiers[i]->Cast().target_distincts) { - VisitExpression(expr); - } - break; - case ResultModifierType::ORDER_MODIFIER: - for (auto &order : node.modifiers[i]->Cast().orders) { - VisitExpression(order.expression); - } - break; - case ResultModifierType::LIMIT_MODIFIER: { - auto &limit_expr = node.modifiers[i]->Cast().limit_val.GetExpression(); - auto &offset_expr = node.modifiers[i]->Cast().offset_val.GetExpression(); - if (limit_expr) { - VisitExpression(limit_expr); - } - if (offset_expr) { - VisitExpression(offset_expr); - } - break; - } - default: - break; - } - } -} - -class LogicalBoundNodeVisitor : public LogicalOperatorVisitor { -public: - explicit LogicalBoundNodeVisitor(BoundNodeVisitor &parent) : parent(parent) { - } - - void VisitExpression(unique_ptr *expression) override { - auto &expr = **expression; - parent.VisitExpression(*expression); - VisitExpressionChildren(expr); - } - -protected: - BoundNodeVisitor &parent; -}; - -void BoundNodeVisitor::VisitBoundTableRef(BoundTableRef &ref) { - switch (ref.type) { - case TableReferenceType::EXPRESSION_LIST: { - auto &bound_expr_list = ref.Cast(); - for (auto &expr_list : bound_expr_list.values) { - for (auto &expr : expr_list) { - VisitExpression(expr); - } - } - break; - } - case TableReferenceType::JOIN: { - auto &bound_join = ref.Cast(); - if (bound_join.condition) { - VisitExpression(bound_join.condition); - } - VisitBoundTableRef(*bound_join.left); - VisitBoundTableRef(*bound_join.right); - break; - } - case TableReferenceType::SUBQUERY: { - auto &bound_subquery = ref.Cast(); - VisitBoundQueryNode(*bound_subquery.subquery); - break; - } - case TableReferenceType::TABLE_FUNCTION: { - auto &bound_table_function = ref.Cast(); - LogicalBoundNodeVisitor node_visitor(*this); - if (bound_table_function.get) { - node_visitor.VisitOperator(*bound_table_function.get); - } - if (bound_table_function.subquery) { - VisitBoundTableRef(*bound_table_function.subquery); - } - break; - } - case TableReferenceType::EMPTY_FROM: - case TableReferenceType::BASE_TABLE: - case TableReferenceType::CTE: - break; - default: - throw NotImplementedException("Unimplemented table reference type (%s) in ExpressionIterator", - EnumUtil::ToString(ref.type)); - } -} - } // namespace duckdb diff --git a/src/planner/logical_operator.cpp b/src/planner/logical_operator.cpp index e16062573fe3..016b7d6053c9 100644 --- a/src/planner/logical_operator.cpp +++ b/src/planner/logical_operator.cpp @@ -31,6 +31,19 @@ vector LogicalOperator::GetColumnBindings() { return {ColumnBinding(0, 0)}; } +idx_t LogicalOperator::GetRootIndex() { + auto bindings = GetColumnBindings(); + if (bindings.empty()) { + throw InternalException("Empty bindings in GetRootIndex"); + } + auto root_index = bindings[0].table_index; + for (idx_t i = 1; i < bindings.size(); i++) { + if (bindings[i].table_index != root_index) { + throw InternalException("GetRootIndex - multiple column bindings found"); + } + } + return root_index; +} void LogicalOperator::SetParamsEstimatedCardinality(InsertionOrderPreservingMap &result) const { if (has_estimated_cardinality) { result[RenderTreeNode::ESTIMATED_CARDINALITY] = StringUtil::Format("%llu", estimated_cardinality); diff --git a/src/planner/operator/logical_dependent_join.cpp b/src/planner/operator/logical_dependent_join.cpp index 2e46dbc78784..70af8444ac77 100644 --- a/src/planner/operator/logical_dependent_join.cpp +++ b/src/planner/operator/logical_dependent_join.cpp @@ -3,7 +3,7 @@ namespace duckdb { LogicalDependentJoin::LogicalDependentJoin(unique_ptr left, unique_ptr right, - vector correlated_columns, JoinType type, + CorrelatedColumns correlated_columns, JoinType type, unique_ptr condition) : LogicalComparisonJoin(type, LogicalOperatorType::LOGICAL_DEPENDENT_JOIN), join_condition(std::move(condition)), correlated_columns(std::move(correlated_columns)) { @@ -17,7 +17,7 @@ LogicalDependentJoin::LogicalDependentJoin(JoinType join_type) unique_ptr LogicalDependentJoin::Create(unique_ptr left, unique_ptr right, - vector correlated_columns, JoinType type, + CorrelatedColumns correlated_columns, JoinType type, unique_ptr condition) { return make_uniq(std::move(left), std::move(right), std::move(correlated_columns), type, std::move(condition)); diff --git a/src/planner/operator/logical_set_operation.cpp b/src/planner/operator/logical_set_operation.cpp index 72da2e0536f0..a6eeaed7f76d 100644 --- a/src/planner/operator/logical_set_operation.cpp +++ b/src/planner/operator/logical_set_operation.cpp @@ -4,6 +4,33 @@ namespace duckdb { +LogicalSetOperation::LogicalSetOperation(idx_t table_index, idx_t column_count, LogicalOperatorType type, + bool setop_all, bool allow_out_of_order) + : LogicalOperator(type), table_index(table_index), column_count(column_count), setop_all(setop_all), + allow_out_of_order(allow_out_of_order) { +} + +LogicalSetOperation::LogicalSetOperation(idx_t table_index, idx_t column_count, + vector> children_p, LogicalOperatorType type, + bool setop_all, bool allow_out_of_order) + : LogicalOperator(type), table_index(table_index), column_count(column_count), setop_all(setop_all), + allow_out_of_order(allow_out_of_order) { + D_ASSERT(type == LogicalOperatorType::LOGICAL_UNION || type == LogicalOperatorType::LOGICAL_EXCEPT || + type == LogicalOperatorType::LOGICAL_INTERSECT); + children = std::move(children_p); +} + +LogicalSetOperation::LogicalSetOperation(idx_t table_index, idx_t column_count, unique_ptr top, + unique_ptr bottom, LogicalOperatorType type, bool setop_all, + bool allow_out_of_order) + : LogicalOperator(type), table_index(table_index), column_count(column_count), setop_all(setop_all), + allow_out_of_order(allow_out_of_order) { + D_ASSERT(type == LogicalOperatorType::LOGICAL_UNION || type == LogicalOperatorType::LOGICAL_EXCEPT || + type == LogicalOperatorType::LOGICAL_INTERSECT); + children.push_back(std::move(top)); + children.push_back(std::move(bottom)); +} + vector LogicalSetOperation::GetTableIndex() const { return vector {table_index}; } diff --git a/src/planner/operator/logical_vacuum.cpp b/src/planner/operator/logical_vacuum.cpp index 36352a0ead56..ce4a769517cd 100644 --- a/src/planner/operator/logical_vacuum.cpp +++ b/src/planner/operator/logical_vacuum.cpp @@ -1,5 +1,5 @@ #include "duckdb/planner/operator/logical_vacuum.hpp" - +#include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/common/serializer/serializer.hpp" #include "duckdb/common/serializer/deserializer.hpp" @@ -46,11 +46,14 @@ unique_ptr LogicalVacuum::Deserialize(Deserializer &deserialize auto &context = deserializer.Get(); auto binder = Binder::CreateBinder(context); auto bound_table = binder->Bind(*info.ref); - if (bound_table->type != TableReferenceType::BASE_TABLE) { + if (bound_table.plan->type != LogicalOperatorType::LOGICAL_GET) { + throw InvalidInputException("can only vacuum or analyze base tables"); + } + auto table_ptr = bound_table.plan->Cast().GetTable(); + if (!table_ptr) { throw InvalidInputException("can only vacuum or analyze base tables"); } - auto ref = unique_ptr_cast(std::move(bound_table)); - auto &table = ref->table; + auto &table = *table_ptr; result->SetTable(table); // FIXME: we should probably verify that the 'column_id_map' and 'columns' are the same on the bound table after // deserialization? diff --git a/src/planner/planner.cpp b/src/planner/planner.cpp index 9f1cafa4dca2..a38bc2a6cc51 100644 --- a/src/planner/planner.cpp +++ b/src/planner/planner.cpp @@ -41,17 +41,13 @@ void Planner::CreatePlan(SQLStatement &statement) { bool parameters_resolved = true; try { profiler.StartPhase(MetricsType::PLANNER_BINDING); - binder->parameters = &bound_parameters; + binder->SetParameters(bound_parameters); auto bound_statement = binder->Bind(statement); profiler.EndPhase(); this->names = bound_statement.names; this->types = bound_statement.types; this->plan = std::move(bound_statement.plan); - auto max_tree_depth = ClientConfig::GetConfig(context).max_expression_depth; - CheckTreeDepth(*plan, max_tree_depth); - - this->plan = FlattenDependentJoins::DecorrelateIndependent(*binder, std::move(this->plan)); } catch (const std::exception &ex) { ErrorData error(ex); this->plan = nullptr; @@ -80,6 +76,12 @@ void Planner::CreatePlan(SQLStatement &statement) { throw; } } + if (this->plan) { + auto max_tree_depth = ClientConfig::GetConfig(context).max_expression_depth; + CheckTreeDepth(*plan, max_tree_depth); + + this->plan = FlattenDependentJoins::DecorrelateIndependent(*this->binder, std::move(this->plan)); + } this->properties = binder->GetStatementProperties(); this->properties.parameter_count = parameter_count; properties.bound_all_parameters = !bound_parameters.rebind && parameters_resolved; diff --git a/src/planner/subquery/flatten_dependent_join.cpp b/src/planner/subquery/flatten_dependent_join.cpp index 0b0ddc672db0..2fd7a50f4b98 100644 --- a/src/planner/subquery/flatten_dependent_join.cpp +++ b/src/planner/subquery/flatten_dependent_join.cpp @@ -18,9 +18,8 @@ namespace duckdb { -FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const vector &correlated, - bool perform_delim, bool any_join, - optional_ptr parent) +FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const CorrelatedColumns &correlated, bool perform_delim, + bool any_join, optional_ptr parent) : binder(binder), delim_offset(DConstants::INVALID_INDEX), correlated_columns(correlated), perform_delim(perform_delim), any_join(any_join), parent(parent) { for (idx_t i = 0; i < correlated_columns.size(); i++) { @@ -30,8 +29,7 @@ FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const vector &correlated_columns, +static void CreateDelimJoinConditions(LogicalComparisonJoin &delim_join, const CorrelatedColumns &correlated_columns, vector bindings, idx_t base_offset, bool perform_delim) { auto col_count = perform_delim ? correlated_columns.size() : 1; for (idx_t i = 0; i < col_count; i++) { @@ -50,7 +48,7 @@ static void CreateDelimJoinConditions(LogicalComparisonJoin &delim_join, unique_ptr FlattenDependentJoins::DecorrelateIndependent(Binder &binder, unique_ptr plan) { - vector correlated; + CorrelatedColumns correlated; FlattenDependentJoins flatten(binder, correlated); return flatten.Decorrelate(std::move(plan)); } @@ -80,12 +78,12 @@ unique_ptr FlattenDependentJoins::Decorrelate(unique_ptrsecond = false; // rewrite - idx_t lateral_depth = 0; + idx_t next_lateral_depth = 0; - RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth); + RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, next_lateral_depth); rewriter.VisitOperator(*plan); - RewriteCorrelatedExpressions recursive_rewriter(base_binding, correlated_map, lateral_depth, true); + RewriteCorrelatedExpressions recursive_rewriter(base_binding, correlated_map, next_lateral_depth, true); recursive_rewriter.VisitOperator(*plan); } else { op.children[0] = Decorrelate(std::move(op.children[0])); @@ -94,8 +92,8 @@ unique_ptr FlattenDependentJoins::Decorrelate(unique_ptr(op.correlated_columns[0].binding.table_index); + const auto &op_col = op.correlated_columns[op.correlated_columns.GetDelimIndex()]; + auto window = make_uniq(op_col.binding.table_index); auto row_number = make_uniq(ExpressionType::WINDOW_ROW_NUMBER, LogicalType::BIGINT, nullptr, nullptr); row_number->start = WindowBoundary::UNBOUNDED_PRECEDING; @@ -114,9 +112,9 @@ unique_ptr FlattenDependentJoins::Decorrelate(unique_ptrchildren[1], op.is_lateral_join, lateral_depth); if (delim_join->children[1]->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) { - auto &cte = delim_join->children[1]->Cast(); + auto &cte_ref = delim_join->children[1]->Cast(); // check if the left side of the CTE has correlated expressions - auto entry = flatten.has_correlated_expressions.find(*cte.children[0]); + auto entry = flatten.has_correlated_expressions.find(*cte_ref.children[0]); if (entry != flatten.has_correlated_expressions.end()) { if (!entry->second) { // the left side of the CTE has no correlated expressions, we can push the DEPENDENT_JOIN down @@ -132,7 +130,7 @@ unique_ptr FlattenDependentJoins::Decorrelate(unique_ptrchildren[1] = flatten.PushDownDependentJoin(std::move(delim_join->children[1]), propagate_null_values, lateral_depth); data_offset = flatten.data_offset; - auto left_offset = delim_join->children[0]->GetColumnBindings().size(); + const auto left_offset = delim_join->children[0]->GetColumnBindings().size(); if (!parent) { delim_offset = left_offset + flatten.delim_offset; } @@ -890,12 +888,16 @@ unique_ptr FlattenDependentJoins::PushDownDependentJoinInternal auto &setop = plan->Cast(); // set operator, push into both children #ifdef DEBUG - plan->children[0]->ResolveOperatorTypes(); - plan->children[1]->ResolveOperatorTypes(); - D_ASSERT(plan->children[0]->types == plan->children[1]->types); + for (auto &child : plan->children) { + child->ResolveOperatorTypes(); + } + for (idx_t i = 1; i < plan->children.size(); i++) { + D_ASSERT(plan->children[0]->types.size() == plan->children[i]->types.size()); + } #endif - plan->children[0] = PushDownDependentJoin(std::move(plan->children[0])); - plan->children[1] = PushDownDependentJoin(std::move(plan->children[1])); + for (auto &child : plan->children) { + child = PushDownDependentJoin(std::move(child)); + } for (idx_t i = 0; i < plan->children.size(); i++) { if (plan->children[i]->type == LogicalOperatorType::LOGICAL_CROSS_PRODUCT) { auto proj_index = binder.GenerateTableIndex(); @@ -920,10 +922,15 @@ unique_ptr FlattenDependentJoins::PushDownDependentJoinInternal // here we need to check the children. If they have reorderable bindings, you need to plan a projection // on top that will guarantee the order of the bindings. #ifdef DEBUG - D_ASSERT(plan->children[0]->GetColumnBindings().size() == plan->children[1]->GetColumnBindings().size()); - plan->children[0]->ResolveOperatorTypes(); - plan->children[1]->ResolveOperatorTypes(); - D_ASSERT(plan->children[0]->types == plan->children[1]->types); + for (idx_t i = 1; i < plan->children.size(); i++) { + D_ASSERT(plan->children[0]->GetColumnBindings().size() == plan->children[i]->GetColumnBindings().size()); + } + for (auto &child : plan->children) { + child->ResolveOperatorTypes(); + } + for (idx_t i = 1; i < plan->children.size(); i++) { + D_ASSERT(plan->children[0]->types.size() == plan->children[i]->types.size()); + } #endif // we have to refer to the setop index now base_binding.table_index = setop.table_index; diff --git a/src/planner/subquery/has_correlated_expressions.cpp b/src/planner/subquery/has_correlated_expressions.cpp index 9f1c679a1386..8554f3f5b95b 100644 --- a/src/planner/subquery/has_correlated_expressions.cpp +++ b/src/planner/subquery/has_correlated_expressions.cpp @@ -7,7 +7,7 @@ namespace duckdb { -HasCorrelatedExpressions::HasCorrelatedExpressions(const vector &correlated, bool lateral, +HasCorrelatedExpressions::HasCorrelatedExpressions(const CorrelatedColumns &correlated, bool lateral, idx_t lateral_depth) : has_correlated_expressions(false), lateral(lateral), correlated_columns(correlated), lateral_depth(lateral_depth) { diff --git a/src/planner/subquery/rewrite_correlated_expressions.cpp b/src/planner/subquery/rewrite_correlated_expressions.cpp index 903840dda3f9..10004d8f7f44 100644 --- a/src/planner/subquery/rewrite_correlated_expressions.cpp +++ b/src/planner/subquery/rewrite_correlated_expressions.cpp @@ -9,7 +9,6 @@ #include "duckdb/planner/expression_iterator.hpp" #include "duckdb/planner/tableref/bound_joinref.hpp" #include "duckdb/planner/operator/logical_dependent_join.hpp" -#include "duckdb/planner/tableref/bound_subqueryref.hpp" namespace duckdb { @@ -71,14 +70,14 @@ unique_ptr RewriteCorrelatedExpressions::VisitReplace(BoundColumnRef } //! Helper class used to recursively rewrite correlated expressions within nested subqueries. -class RewriteCorrelatedRecursive : public BoundNodeVisitor { +class RewriteCorrelatedRecursive : public LogicalOperatorVisitor { public: RewriteCorrelatedRecursive(ColumnBinding base_binding, column_binding_map_t &correlated_map); - void VisitBoundTableRef(BoundTableRef &ref) override; - void VisitExpression(unique_ptr &expression) override; + void VisitOperator(LogicalOperator &op) override; + void VisitExpression(unique_ptr *expression) override; - void RewriteCorrelatedSubquery(Binder &binder, BoundQueryNode &subquery); + void RewriteCorrelatedSubquery(Binder &binder, LogicalOperator &subquery); ColumnBinding base_binding; column_binding_map_t &correlated_map; @@ -92,7 +91,7 @@ unique_ptr RewriteCorrelatedExpressions::VisitReplace(BoundSubqueryE // subquery detected within this subquery // recursively rewrite it using the RewriteCorrelatedRecursive class RewriteCorrelatedRecursive rewrite(base_binding, correlated_map); - rewrite.RewriteCorrelatedSubquery(*expr.binder, *expr.subquery); + rewrite.RewriteCorrelatedSubquery(*expr.binder, *expr.subquery.plan); return nullptr; } @@ -101,40 +100,30 @@ RewriteCorrelatedRecursive::RewriteCorrelatedRecursive(ColumnBinding base_bindin : base_binding(base_binding), correlated_map(correlated_map) { } -void RewriteCorrelatedRecursive::VisitBoundTableRef(BoundTableRef &ref) { - if (ref.type == TableReferenceType::JOIN) { +void RewriteCorrelatedRecursive::VisitOperator(LogicalOperator &op) { + if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) { // rewrite correlated columns in child joins - auto &bound_join = ref.Cast(); - for (auto &corr : bound_join.correlated_columns) { + auto &dep_join = op.Cast(); + for (auto &corr : dep_join.correlated_columns) { auto entry = correlated_map.find(corr.binding); if (entry != correlated_map.end()) { corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second); } } - } else if (ref.type == TableReferenceType::SUBQUERY) { - auto &subquery = ref.Cast(); - RewriteCorrelatedSubquery(*subquery.binder, *subquery.subquery); - return; } // visit the children of the table ref - BoundNodeVisitor::VisitBoundTableRef(ref); + LogicalOperatorVisitor::VisitOperator(op); } -void RewriteCorrelatedRecursive::RewriteCorrelatedSubquery(Binder &binder, BoundQueryNode &subquery) { - // rewrite the binding in the correlated list of the subquery) - for (auto &corr : binder.correlated_columns) { - auto entry = correlated_map.find(corr.binding); - if (entry != correlated_map.end()) { - corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second); - } - } - VisitBoundQueryNode(subquery); +void RewriteCorrelatedRecursive::RewriteCorrelatedSubquery(Binder &binder, LogicalOperator &op) { + VisitOperator(op); } -void RewriteCorrelatedRecursive::VisitExpression(unique_ptr &expression) { - if (expression->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { +void RewriteCorrelatedRecursive::VisitExpression(unique_ptr *expression) { + auto &expr = **expression; + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { // bound column reference - auto &bound_colref = expression->Cast(); + auto &bound_colref = expr.Cast(); if (bound_colref.depth == 0) { // not a correlated column, ignore return; @@ -148,13 +137,13 @@ void RewriteCorrelatedRecursive::VisitExpression(unique_ptr &express bound_colref.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second); bound_colref.depth--; } - } else if (expression->GetExpressionType() == ExpressionType::SUBQUERY) { + } else if (expr.GetExpressionType() == ExpressionType::SUBQUERY) { // we encountered another subquery: rewrite recursively - auto &bound_subquery = expression->Cast(); - RewriteCorrelatedSubquery(*bound_subquery.binder, *bound_subquery.subquery); + auto &bound_subquery = expr.Cast(); + RewriteCorrelatedSubquery(*bound_subquery.binder, *bound_subquery.subquery.plan); } // recurse into the children of this subquery - BoundNodeVisitor::VisitExpression(expression); + LogicalOperatorVisitor::VisitExpression(expression); } RewriteCountAggregates::RewriteCountAggregates(column_binding_map_t &replacement_map) diff --git a/src/planner/subquery/rewrite_cte_scan.cpp b/src/planner/subquery/rewrite_cte_scan.cpp index 78b3b21eca10..f846d9b3638c 100644 --- a/src/planner/subquery/rewrite_cte_scan.cpp +++ b/src/planner/subquery/rewrite_cte_scan.cpp @@ -14,7 +14,7 @@ namespace duckdb { -RewriteCTEScan::RewriteCTEScan(idx_t table_index, const vector &correlated_columns) +RewriteCTEScan::RewriteCTEScan(idx_t table_index, const CorrelatedColumns &correlated_columns) : table_index(table_index), correlated_columns(correlated_columns) { } @@ -49,7 +49,7 @@ void RewriteCTEScan::VisitOperator(LogicalOperator &op) { // The correlated columns must be placed at the beginning of the // correlated_columns list. Otherwise, further column accesses // and rewrites will fail. - join.correlated_columns.emplace(join.correlated_columns.begin(), corr); + join.correlated_columns.AddColumn(std::move(corr)); } } } diff --git a/src/planner/table_binding.cpp b/src/planner/table_binding.cpp index d9bdd71c702a..c55d0be822b4 100644 --- a/src/planner/table_binding.cpp +++ b/src/planner/table_binding.cpp @@ -19,6 +19,10 @@ Binding::Binding(BindingType binding_type, BindingAlias alias_p, vector &Binding::GetColumnTypes() { + return types; +} + +const vector &Binding::GetColumnNames() { + return names; +} + +idx_t Binding::GetColumnCount() { + return GetColumnNames().size(); +} + +void Binding::SetColumnType(idx_t col_idx, LogicalType type_p) { + types[col_idx] = std::move(type_p); +} + string Binding::GetAlias() const { return alias.GetAlias(); } @@ -304,4 +336,42 @@ unique_ptr DummyBinding::ParamToArg(ColumnRefExpression &colre return arg; } +CTEBinding::CTEBinding(BindingAlias alias, vector types, vector names, idx_t index, + CTEType cte_type) + : Binding(BindingType::CTE, std::move(alias), std::move(types), std::move(names), index), cte_type(cte_type), + reference_count(0) { +} + +CTEBinding::CTEBinding(BindingAlias alias_p, shared_ptr bind_state_p, idx_t index) + : Binding(BindingType::CTE, std::move(alias_p), vector(), vector(), index), + cte_type(CTEType::CAN_BE_REFERENCED), reference_count(0), bind_state(std::move(bind_state_p)) { +} + +bool CTEBinding::CanBeReferenced() const { + return cte_type == CTEType::CAN_BE_REFERENCED; +} + +bool CTEBinding::IsReferenced() const { + return reference_count > 0; +} + +void CTEBinding::Reference() { + if (!CanBeReferenced()) { + throw InternalException("CTE cannot be referenced!"); + } + if (bind_state) { + // we have not bound the CTE yet - bind it + bind_state->Bind(*this); + + // copy over the names / types and initialize the binding + this->names = bind_state->names; + this->types = bind_state->types; + Initialize(); + + // finalize binding + bind_state.reset(); + } + reference_count++; +} + } // namespace duckdb diff --git a/src/storage/buffer/block_manager.cpp b/src/storage/buffer/block_manager.cpp index 47fef0ebf890..3f2064a5c787 100644 --- a/src/storage/buffer/block_manager.cpp +++ b/src/storage/buffer/block_manager.cpp @@ -34,19 +34,32 @@ shared_ptr BlockManager::RegisterBlock(block_id_t block_id) { } shared_ptr BlockManager::ConvertToPersistent(QueryContext context, block_id_t block_id, - shared_ptr old_block, BufferHandle old_handle) { + shared_ptr old_block, BufferHandle old_handle, + ConvertToPersistentMode mode) { // register a block with the new block id auto new_block = RegisterBlock(block_id); D_ASSERT(new_block->GetState() == BlockState::BLOCK_UNLOADED); D_ASSERT(new_block->Readers() == 0); + if (mode == ConvertToPersistentMode::THREAD_SAFE) { + // safe mode - create a copy of the old block and operate on that + // this ensures we don't modify the old block - which allows other concurrent operations on the old block to + // continue + auto old_block_copy = buffer_manager.AllocateMemory(old_block->GetMemoryTag(), this, false); + auto copy_pin = buffer_manager.Pin(old_block_copy); + memcpy(copy_pin.Ptr(), old_handle.Ptr(), GetBlockSize()); + old_block = std::move(old_block_copy); + old_handle = std::move(copy_pin); + } + auto lock = old_block->GetLock(); D_ASSERT(old_block->GetState() == BlockState::BLOCK_LOADED); D_ASSERT(old_block->GetBuffer(lock)); if (old_block->Readers() > 1) { - throw InternalException("BlockManager::ConvertToPersistent - cannot be called for block %d as old_block has " - "multiple readers active", - block_id); + throw InternalException( + "BlockManager::ConvertToPersistent in destructive mode - cannot be called for block %d as old_block has " + "multiple readers active", + block_id); } // Temp buffers can be larger than the storage block size. @@ -76,10 +89,11 @@ shared_ptr BlockManager::ConvertToPersistent(QueryContext context, } shared_ptr BlockManager::ConvertToPersistent(QueryContext context, block_id_t block_id, - shared_ptr old_block) { + shared_ptr old_block, + ConvertToPersistentMode mode) { // pin the old block to ensure we have it loaded in memory auto handle = buffer_manager.Pin(old_block); - return ConvertToPersistent(context, block_id, std::move(old_block), std::move(handle)); + return ConvertToPersistent(context, block_id, std::move(old_block), std::move(handle), mode); } void BlockManager::UnregisterBlock(block_id_t id) { diff --git a/src/storage/caching_file_system.cpp b/src/storage/caching_file_system.cpp index 3de905228d08..347107673928 100644 --- a/src/storage/caching_file_system.cpp +++ b/src/storage/caching_file_system.cpp @@ -79,6 +79,21 @@ FileHandle &CachingFileHandle::GetFileHandle() { return *file_handle; } +static bool ShouldExpandToFillGap(const idx_t current_length, const idx_t added_length) { + const idx_t MAX_BOUND_TO_BE_ADDED_LENGTH = 1048576; + + if (added_length > MAX_BOUND_TO_BE_ADDED_LENGTH) { + // Absolute value of what would be needed to added is too high + return false; + } + if (added_length > current_length) { + // Relative value of what would be needed to added is too high + return false; + } + + return true; +} + BufferHandle CachingFileHandle::Read(data_ptr_t &buffer, const idx_t nr_bytes, const idx_t location) { BufferHandle result; if (!external_file_cache.IsEnabled()) { @@ -90,30 +105,42 @@ BufferHandle CachingFileHandle::Read(data_ptr_t &buffer, const idx_t nr_bytes, c // Try to read from the cache, filling overlapping_ranges in the process vector> overlapping_ranges; - result = TryReadFromCache(buffer, nr_bytes, location, overlapping_ranges); + optional_idx start_location_of_next_range; + result = TryReadFromCache(buffer, nr_bytes, location, overlapping_ranges, start_location_of_next_range); if (result.IsValid()) { return result; // Success } + idx_t new_nr_bytes = nr_bytes; + if (start_location_of_next_range.IsValid()) { + const idx_t nr_bytes_to_be_added = start_location_of_next_range.GetIndex() - location - nr_bytes; + if (ShouldExpandToFillGap(nr_bytes, nr_bytes_to_be_added)) { + // Grow the range from location to start_location_of_next_range, so that to fill gaps in the cached ranges + new_nr_bytes = nr_bytes + nr_bytes_to_be_added; + } + } + // Finally, if we weren't able to find the file range in the cache, we have to create a new file range - result = external_file_cache.GetBufferManager().Allocate(MemoryTag::EXTERNAL_FILE_CACHE, nr_bytes); - auto new_file_range = make_shared_ptr(result.GetBlockHandle(), nr_bytes, location, version_tag); + result = external_file_cache.GetBufferManager().Allocate(MemoryTag::EXTERNAL_FILE_CACHE, new_nr_bytes); + auto new_file_range = + make_shared_ptr(result.GetBlockHandle(), new_nr_bytes, location, version_tag); buffer = result.Ptr(); // Interleave reading and copying from cached buffers if (OnDiskFile()) { // On-disk file: prefer interleaving reading and copying from cached buffers - ReadAndCopyInterleaved(overlapping_ranges, new_file_range, buffer, nr_bytes, location, true); + ReadAndCopyInterleaved(overlapping_ranges, new_file_range, buffer, new_nr_bytes, location, true); } else { - // Remote file: prefer interleaving reading and copying from cached buffers only if reduces number of real reads - if (ReadAndCopyInterleaved(overlapping_ranges, new_file_range, buffer, nr_bytes, location, false) <= 1) { - ReadAndCopyInterleaved(overlapping_ranges, new_file_range, buffer, nr_bytes, location, true); + // Remote file: prefer interleaving reading and copying from cached buffers only if reduces number of real + // reads + if (ReadAndCopyInterleaved(overlapping_ranges, new_file_range, buffer, new_nr_bytes, location, false) <= 1) { + ReadAndCopyInterleaved(overlapping_ranges, new_file_range, buffer, new_nr_bytes, location, true); } else { - GetFileHandle().Read(context, buffer, nr_bytes, location); + GetFileHandle().Read(context, buffer, new_nr_bytes, location); } } - return TryInsertFileRange(result, buffer, nr_bytes, location, new_file_range); + return TryInsertFileRange(result, buffer, new_nr_bytes, location, new_file_range); } BufferHandle CachingFileHandle::Read(data_ptr_t &buffer, idx_t &nr_bytes) { @@ -131,7 +158,12 @@ BufferHandle CachingFileHandle::Read(data_ptr_t &buffer, idx_t &nr_bytes) { // Try to read from the cache first vector> overlapping_ranges; - result = TryReadFromCache(buffer, nr_bytes, position, overlapping_ranges); + { + optional_idx start_location_of_next_range; + result = TryReadFromCache(buffer, nr_bytes, position, overlapping_ranges, start_location_of_next_range); + // start_location_of_next_range is in this case discarded + } + if (result.IsValid()) { position += nr_bytes; return result; // Success @@ -214,7 +246,8 @@ const string &CachingFileHandle::GetVersionTag(const unique_ptr } BufferHandle CachingFileHandle::TryReadFromCache(data_ptr_t &buffer, idx_t nr_bytes, idx_t location, - vector> &overlapping_ranges) { + vector> &overlapping_ranges, + optional_idx &start_location_of_next_range) { BufferHandle result; // Get read lock for cached ranges @@ -246,7 +279,8 @@ BufferHandle CachingFileHandle::TryReadFromCache(data_ptr_t &buffer, idx_t nr_by } while (it != ranges.end()) { if (it->second->location >= this_end) { - // We're past the requested location + // We're past the requested location, we are going to bail out, save start_location_of_next_range + start_location_of_next_range = it->second->location; break; } // Check if the cached range overlaps the requested one diff --git a/src/storage/checkpoint/row_group_writer.cpp b/src/storage/checkpoint/row_group_writer.cpp index 033446097d82..794dddca799e 100644 --- a/src/storage/checkpoint/row_group_writer.cpp +++ b/src/storage/checkpoint/row_group_writer.cpp @@ -12,6 +12,10 @@ RowGroupWriter::RowGroupWriter(TableCatalogEntry &table, PartialBlockManager &pa } } +DatabaseInstance &RowGroupWriter::GetDatabase() { + return table.ParentCatalog().GetDatabase(); +} + SingleFileRowGroupWriter::SingleFileRowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager, TableDataWriter &writer, MetadataWriter &table_data_writer) : RowGroupWriter(table, partial_block_manager), writer(writer), table_data_writer(table_data_writer) { diff --git a/src/storage/checkpoint/table_data_writer.cpp b/src/storage/checkpoint/table_data_writer.cpp index 0bde4acb367d..342ea1ff587d 100644 --- a/src/storage/checkpoint/table_data_writer.cpp +++ b/src/storage/checkpoint/table_data_writer.cpp @@ -63,8 +63,8 @@ void SingleFileTableDataWriter::WriteUnchangedTable(MetaBlockPointer pointer, id existing_rows = total_rows; } -void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, - Serializer &serializer) { +void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, + RowGroupCollection &collection, Serializer &serializer) { MetaBlockPointer pointer; idx_t total_rows; if (!existing_pointer.IsValid()) { @@ -94,6 +94,7 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat RowGroup::Serialize(row_group_pointer, row_group_serializer); row_group_serializer.End(); } + collection.FinalizeCheckpoint(pointer); } else { // we have existing metadata and the table is unchanged - write a pointer to the existing metadata pointer = existing_pointer; @@ -116,7 +117,7 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat if (!v1_0_0_storage) { options.emplace("v1_0_0_storage", v1_0_0_storage); } - auto index_storage_infos = info->GetIndexes().SerializeToDisk(context, options); + auto index_storage_infos = info.GetIndexes().SerializeToDisk(context, options); #ifdef DUCKDB_BLOCK_VERIFICATION for (auto &entry : index_storage_infos) { diff --git a/src/storage/checkpoint_manager.cpp b/src/storage/checkpoint_manager.cpp index 8618b38eba96..7e7b1c0c9dff 100644 --- a/src/storage/checkpoint_manager.cpp +++ b/src/storage/checkpoint_manager.cpp @@ -21,7 +21,6 @@ #include "duckdb/parser/parsed_data/create_schema_info.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/bound_tableref.hpp" #include "duckdb/planner/parsed_data/bound_create_table_info.hpp" #include "duckdb/storage/block_manager.hpp" #include "duckdb/storage/checkpoint/table_data_reader.hpp" @@ -279,7 +278,7 @@ void SingleFileCheckpointReader::LoadFromStorage() { return; } - if (block_manager.IsRemote()) { + if (block_manager.Prefetch()) { auto metadata_blocks = metadata_manager.GetBlocks(); auto &buffer_manager = BufferManager::GetBufferManager(storage.GetDatabase()); buffer_manager.Prefetch(metadata_blocks); diff --git a/src/storage/compression/bitpacking.cpp b/src/storage/compression/bitpacking.cpp index fa1ffaebab3f..f3188662ccfe 100644 --- a/src/storage/compression/bitpacking.cpp +++ b/src/storage/compression/bitpacking.cpp @@ -19,6 +19,7 @@ namespace duckdb { +constexpr const idx_t BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; static constexpr const idx_t BITPACKING_METADATA_GROUP_SIZE = STANDARD_VECTOR_SIZE > 512 ? STANDARD_VECTOR_SIZE : 2048; BitpackingMode BitpackingModeFromString(const string &str) { @@ -341,8 +342,6 @@ unique_ptr BitpackingInitAnalyze(ColumnData &col_data, PhysicalTyp template bool BitpackingAnalyze(AnalyzeState &state, Vector &input, idx_t count) { - auto &analyze_state = state.Cast>(); - // We use BITPACKING_METADATA_GROUP_SIZE tuples, which can exceed the block size. // In that case, we disable bitpacking. // we are conservative here by multiplying by 2 @@ -351,6 +350,7 @@ bool BitpackingAnalyze(AnalyzeState &state, Vector &input, idx_t count) { return false; } + auto &analyze_state = state.Cast>(); UnifiedVectorFormat vdata; input.ToUnifiedFormat(count, vdata); @@ -629,9 +629,9 @@ static T DeltaDecode(T *data, T previous_value, const size_t size) { template ::type> struct BitpackingScanState : public SegmentScanState { public: - explicit BitpackingScanState(ColumnSegment &segment) : current_segment(segment) { + explicit BitpackingScanState(const QueryContext &context, ColumnSegment &segment) : current_segment(segment) { auto &buffer_manager = BufferManager::GetBufferManager(segment.db); - handle = buffer_manager.Pin(segment.block); + handle = buffer_manager.Pin(context, segment.block); auto data_ptr = handle.Ptr(); // load offset to bitpacking widths pointer @@ -782,8 +782,8 @@ struct BitpackingScanState : public SegmentScanState { }; template -unique_ptr BitpackingInitScan(ColumnSegment &segment) { - auto result = make_uniq>(segment); +unique_ptr BitpackingInitScan(const QueryContext &context, ColumnSegment &segment) { + auto result = make_uniq>(context, segment); return std::move(result); } @@ -892,7 +892,7 @@ void BitpackingScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_c template void BitpackingFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) { - BitpackingScanState scan_state(segment); + BitpackingScanState scan_state(state.context, segment); scan_state.Skip(segment, NumericCast(row_id)); D_ASSERT(scan_state.current_group_offset < BITPACKING_METADATA_GROUP_SIZE); @@ -956,10 +956,10 @@ void BitpackingSkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_c // GetSegmentInfo //===--------------------------------------------------------------------===// template -InsertionOrderPreservingMap BitpackingGetSegmentInfo(ColumnSegment &segment) { +InsertionOrderPreservingMap BitpackingGetSegmentInfo(QueryContext context, ColumnSegment &segment) { map counts; auto tuple_count = segment.count.load(); - BitpackingScanState scan_state(segment); + BitpackingScanState scan_state(context, segment); for (idx_t i = 0; i < tuple_count; i += BITPACKING_METADATA_GROUP_SIZE) { if (i) { scan_state.LoadNextGroup(); diff --git a/src/storage/compression/dict_fsst.cpp b/src/storage/compression/dict_fsst.cpp index c43567c52be2..18c5dac2145d 100644 --- a/src/storage/compression/dict_fsst.cpp +++ b/src/storage/compression/dict_fsst.cpp @@ -56,7 +56,7 @@ struct DictFSSTCompressionStorage { static void Compress(CompressionState &state_p, Vector &scan_vector, idx_t count); static void FinalizeCompress(CompressionState &state_p); - static unique_ptr StringInitScan(ColumnSegment &segment); + static unique_ptr StringInitScan(const QueryContext &context, ColumnSegment &segment); template static void StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset); @@ -111,12 +111,15 @@ void DictFSSTCompressionStorage::FinalizeCompress(CompressionState &state_p) { //===--------------------------------------------------------------------===// // Scan //===--------------------------------------------------------------------===// -unique_ptr DictFSSTCompressionStorage::StringInitScan(ColumnSegment &segment) { +unique_ptr DictFSSTCompressionStorage::StringInitScan(const QueryContext &context, + ColumnSegment &segment) { auto &buffer_manager = BufferManager::GetBufferManager(segment.db); auto state = make_uniq(segment, buffer_manager.Pin(segment.block)); state->Initialize(true); - if (StringStats::HasMaxStringLength(segment.stats.statistics)) { - state->all_values_inlined = StringStats::MaxStringLength(segment.stats.statistics) <= string_t::INLINE_LENGTH; + + const auto &stats = segment.stats.statistics; + if (stats.GetStatsType() == StatisticsType::STRING_STATS && StringStats::HasMaxStringLength(stats)) { + state->all_values_inlined = StringStats::MaxStringLength(stats) <= string_t::INLINE_LENGTH; } return std::move(state); } @@ -187,12 +190,13 @@ static void DictFSSTFilter(ColumnSegment &segment, ColumnScanState &state, idx_t scan_state.filter_result = make_unsafe_uniq_array(scan_state.dict_count); // apply the filter + auto &dict_data = scan_state.dictionary->data; UnifiedVectorFormat vdata; - scan_state.dictionary->ToUnifiedFormat(scan_state.dict_count, vdata); + dict_data.ToUnifiedFormat(scan_state.dict_count, vdata); SelectionVector dict_sel; idx_t filter_count = scan_state.dict_count; - ColumnSegment::FilterSelection(dict_sel, *scan_state.dictionary, vdata, filter, filter_state, - scan_state.dict_count, filter_count); + ColumnSegment::FilterSelection(dict_sel, dict_data, vdata, filter, filter_state, scan_state.dict_count, + filter_count); // now set all matching tuples to true for (idx_t i = 0; i < filter_count; i++) { @@ -217,8 +221,7 @@ static void DictFSSTFilter(ColumnSegment &segment, ColumnScanState &state, idx_t } sel_count = approved_tuple_count; - result.Dictionary(*(scan_state.dictionary), scan_state.dict_count, dict_sel, vector_count); - DictionaryVector::SetDictionaryId(result, to_string(CastPointerToValue(&segment))); + result.Dictionary(scan_state.dictionary, dict_sel); return; } // fallback: scan + filter diff --git a/src/storage/compression/dict_fsst/decompression.cpp b/src/storage/compression/dict_fsst/decompression.cpp index 0546096bb575..f6befffe5a91 100644 --- a/src/storage/compression/dict_fsst/decompression.cpp +++ b/src/storage/compression/dict_fsst/decompression.cpp @@ -98,17 +98,18 @@ void CompressedStringScanState::Initialize(bool initialize_dictionary) { return; } - dictionary = make_buffer(segment.type, dict_count); - auto dict_child_data = FlatVector::GetData(*(dictionary)); - auto &validity = FlatVector::Validity(*dictionary); + dictionary = DictionaryVector::CreateReusableDictionary(segment.type, dict_count); + auto dict_child_data = FlatVector::GetData(dictionary->data); + auto &validity = FlatVector::Validity(dictionary->data); D_ASSERT(dict_count >= 1); validity.SetInvalid(0); + auto &dict_data = dictionary->data; uint32_t offset = 0; for (uint32_t i = 0; i < dict_count; i++) { //! We can uncompress during fetching, we need the length of the string inside the dictionary auto string_len = string_lengths[i]; - dict_child_data[i] = FetchStringFromDict(*dictionary, offset, i); + dict_child_data[i] = FetchStringFromDict(dict_data, offset, i); offset += string_len; } } @@ -158,7 +159,7 @@ void CompressedStringScanState::ScanToFlatVector(Vector &result, idx_t result_of if (dictionary) { // We have prepared the full dictionary, we can reference these strings directly - auto dictionary_values = FlatVector::GetData(*dictionary); + auto dictionary_values = FlatVector::GetData(dictionary->data); for (idx_t i = 0; i < scan_count; i++) { // Lookup dict offset in index buffer auto string_number = selvec.get_index(i + start_offset); @@ -223,8 +224,7 @@ void CompressedStringScanState::ScanToDictionaryVector(ColumnSegment &segment, V D_ASSERT(result_offset == 0); auto &selvec = GetSelVec(start, scan_count); - result.Dictionary(*(dictionary), dict_count, selvec, scan_count); - DictionaryVector::SetDictionaryId(result, to_string(CastPointerToValue(&segment))); + result.Dictionary(dictionary, selvec); result.Verify(result_offset + scan_count); } diff --git a/src/storage/compression/dictionary/decompression.cpp b/src/storage/compression/dictionary/decompression.cpp index 51f6945e2a8e..6e389d0269d6 100644 --- a/src/storage/compression/dictionary/decompression.cpp +++ b/src/storage/compression/dictionary/decompression.cpp @@ -48,10 +48,10 @@ void CompressedStringScanState::Initialize(ColumnSegment &segment, bool initiali return; } - dictionary = make_buffer(segment.type, index_buffer_count); + dictionary = DictionaryVector::CreateReusableDictionary(segment.type, index_buffer_count); dictionary_size = index_buffer_count; - auto dict_child_data = FlatVector::GetData(*(dictionary)); - FlatVector::SetNull(*dictionary, 0, true); + auto dict_child_data = FlatVector::GetData(dictionary->data); + FlatVector::SetNull(dictionary->data, 0, true); for (uint32_t i = 1; i < index_buffer_count; i++) { // NOTE: the passing of dict_child_vector, will not be used, its for big strings uint16_t str_len = GetStringLength(i); @@ -114,8 +114,7 @@ void CompressedStringScanState::ScanToDictionaryVector(ColumnSegment &segment, V } } - result.Dictionary(*(dictionary), dictionary_size, *sel_vec, scan_count); - DictionaryVector::SetDictionaryId(result, to_string(CastPointerToValue(&segment))); + result.Dictionary(dictionary, *sel_vec); } } // namespace duckdb diff --git a/src/storage/compression/dictionary_compression.cpp b/src/storage/compression/dictionary_compression.cpp index fa027edd951f..3529c119a0f6 100644 --- a/src/storage/compression/dictionary_compression.cpp +++ b/src/storage/compression/dictionary_compression.cpp @@ -57,7 +57,7 @@ struct DictionaryCompressionStorage { static void Compress(CompressionState &state_p, Vector &scan_vector, idx_t count); static void FinalizeCompress(CompressionState &state_p); - static unique_ptr StringInitScan(ColumnSegment &segment); + static unique_ptr StringInitScan(const QueryContext &context, ColumnSegment &segment); template static void StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset); @@ -118,7 +118,8 @@ void DictionaryCompressionStorage::FinalizeCompress(CompressionState &state_p) { //===--------------------------------------------------------------------===// // Scan //===--------------------------------------------------------------------===// -unique_ptr DictionaryCompressionStorage::StringInitScan(ColumnSegment &segment) { +unique_ptr DictionaryCompressionStorage::StringInitScan(const QueryContext &context, + ColumnSegment &segment) { auto &buffer_manager = BufferManager::GetBufferManager(segment.db); auto state = make_uniq(buffer_manager.Pin(segment.block)); state->Initialize(segment, true); diff --git a/src/storage/compression/empty_validity.cpp b/src/storage/compression/empty_validity.cpp index 0b6534153ab8..26abbcc64fbd 100644 --- a/src/storage/compression/empty_validity.cpp +++ b/src/storage/compression/empty_validity.cpp @@ -8,8 +8,7 @@ CompressionFunction EmptyValidityCompressionFun::GetFunction(PhysicalType type) } bool EmptyValidityCompressionFun::TypeIsSupported(const PhysicalType physical_type) { - D_ASSERT(physical_type == PhysicalType::BIT); - return true; + return physical_type == PhysicalType::BIT; } } // namespace duckdb diff --git a/src/storage/compression/fixed_size_uncompressed.cpp b/src/storage/compression/fixed_size_uncompressed.cpp index afd335daba4a..89c71852594b 100644 --- a/src/storage/compression/fixed_size_uncompressed.cpp +++ b/src/storage/compression/fixed_size_uncompressed.cpp @@ -143,10 +143,10 @@ struct FixedSizeScanState : public SegmentScanState { BufferHandle handle; }; -unique_ptr FixedSizeInitScan(ColumnSegment &segment) { +unique_ptr FixedSizeInitScan(const QueryContext &context, ColumnSegment &segment) { auto result = make_uniq(); auto &buffer_manager = BufferManager::GetBufferManager(segment.db); - result->handle = buffer_manager.Pin(segment.block); + result->handle = buffer_manager.Pin(context, segment.block); return std::move(result); } diff --git a/src/storage/compression/fsst.cpp b/src/storage/compression/fsst.cpp index cbb3b3ac71ad..eaf1c5e783bd 100644 --- a/src/storage/compression/fsst.cpp +++ b/src/storage/compression/fsst.cpp @@ -50,7 +50,7 @@ struct FSSTStorage { static void Compress(CompressionState &state_p, Vector &scan_vector, idx_t count); static void FinalizeCompress(CompressionState &state_p); - static unique_ptr StringInitScan(ColumnSegment &segment); + static unique_ptr StringInitScan(const QueryContext &context, ColumnSegment &segment); template static void StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset); @@ -569,7 +569,7 @@ struct FSSTScanState : public StringScanState { } }; -unique_ptr FSSTStorage::StringInitScan(ColumnSegment &segment) { +unique_ptr FSSTStorage::StringInitScan(const QueryContext &context, ColumnSegment &segment) { auto block_size = segment.GetBlockManager().GetBlockSize(); auto string_block_limit = StringUncompressed::GetStringBlockLimit(block_size); auto state = make_uniq(string_block_limit); @@ -585,8 +585,9 @@ unique_ptr FSSTStorage::StringInitScan(ColumnSegment &segment) } state->duckdb_fsst_decoder_ptr = state->duckdb_fsst_decoder.get(); - if (StringStats::HasMaxStringLength(segment.stats.statistics)) { - state->all_values_inlined = StringStats::MaxStringLength(segment.stats.statistics) <= string_t::INLINE_LENGTH; + const auto &stats = segment.stats.statistics; + if (stats.GetStatsType() == StatisticsType::STRING_STATS && StringStats::HasMaxStringLength(stats)) { + state->all_values_inlined = StringStats::MaxStringLength(stats) <= string_t::INLINE_LENGTH; } return std::move(state); diff --git a/src/storage/compression/numeric_constant.cpp b/src/storage/compression/numeric_constant.cpp index 2407eb4df5d4..411a85f6d524 100644 --- a/src/storage/compression/numeric_constant.cpp +++ b/src/storage/compression/numeric_constant.cpp @@ -11,7 +11,7 @@ namespace duckdb { //===--------------------------------------------------------------------===// // Scan //===--------------------------------------------------------------------===// -unique_ptr ConstantInitScan(ColumnSegment &segment) { +unique_ptr ConstantInitScan(const QueryContext &context, ColumnSegment &segment) { return nullptr; } @@ -263,7 +263,7 @@ bool ConstantFun::TypeIsSupported(const PhysicalType physical_type) { case PhysicalType::DOUBLE: return true; default: - throw InternalException("Unsupported type for constant function"); + return false; } } diff --git a/src/storage/compression/rle.cpp b/src/storage/compression/rle.cpp index 57ebaf1fa4b0..ed26d824db40 100644 --- a/src/storage/compression/rle.cpp +++ b/src/storage/compression/rle.cpp @@ -303,7 +303,7 @@ struct RLEScanState : public SegmentScanState { }; template -unique_ptr RLEInitScan(ColumnSegment &segment) { +unique_ptr RLEInitScan(const QueryContext &context, ColumnSegment &segment) { auto result = make_uniq>(segment); return std::move(result); } diff --git a/src/storage/compression/roaring/common.cpp b/src/storage/compression/roaring/common.cpp index 80f7004de889..3d9230787efe 100644 --- a/src/storage/compression/roaring/common.cpp +++ b/src/storage/compression/roaring/common.cpp @@ -208,7 +208,7 @@ void RoaringFinalizeCompress(CompressionState &state_p) { state.Finalize(); } -unique_ptr RoaringInitScan(ColumnSegment &segment) { +unique_ptr RoaringInitScan(const QueryContext &context, ColumnSegment &segment) { auto result = make_uniq(segment); return std::move(result); } diff --git a/src/storage/compression/string_uncompressed.cpp b/src/storage/compression/string_uncompressed.cpp index af3b826bf43f..201e9778757a 100644 --- a/src/storage/compression/string_uncompressed.cpp +++ b/src/storage/compression/string_uncompressed.cpp @@ -77,7 +77,8 @@ void UncompressedStringInitPrefetch(ColumnSegment &segment, PrefetchState &prefe } } -unique_ptr UncompressedStringStorage::StringInitScan(ColumnSegment &segment) { +unique_ptr UncompressedStringStorage::StringInitScan(const QueryContext &context, + ColumnSegment &segment) { auto result = make_uniq(); auto &buffer_manager = BufferManager::GetBufferManager(segment.db); result->handle = buffer_manager.Pin(segment.block); diff --git a/src/storage/compression/validity_uncompressed.cpp b/src/storage/compression/validity_uncompressed.cpp index 5a71b897469f..1e31061c29a4 100644 --- a/src/storage/compression/validity_uncompressed.cpp +++ b/src/storage/compression/validity_uncompressed.cpp @@ -207,7 +207,7 @@ struct ValidityScanState : public SegmentScanState { block_id_t block_id; }; -unique_ptr ValidityInitScan(ColumnSegment &segment) { +unique_ptr ValidityInitScan(const QueryContext &context, ColumnSegment &segment) { auto result = make_uniq(); auto &buffer_manager = BufferManager::GetBufferManager(segment.db); result->handle = buffer_manager.Pin(segment.block); diff --git a/src/storage/compression/zstd.cpp b/src/storage/compression/zstd.cpp index 40885528450c..58ddefa61458 100644 --- a/src/storage/compression/zstd.cpp +++ b/src/storage/compression/zstd.cpp @@ -81,7 +81,7 @@ struct ZSTDStorage { static void Compress(CompressionState &state_p, Vector &scan_vector, idx_t count); static void FinalizeCompress(CompressionState &state_p); - static unique_ptr StringInitScan(ColumnSegment &segment); + static unique_ptr StringInitScan(const QueryContext &context, ColumnSegment &segment); static void StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset); static void StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result); @@ -961,7 +961,7 @@ struct ZSTDScanState : public SegmentScanState { AllocatedData skip_buffer; }; -unique_ptr ZSTDStorage::StringInitScan(ColumnSegment &segment) { +unique_ptr ZSTDStorage::StringInitScan(const QueryContext &context, ColumnSegment &segment) { auto result = make_uniq(segment); return std::move(result); } diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index d18d72f6dd04..7cc6bf3e4dca 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -57,6 +57,12 @@ DataTable::DataTable(AttachedDatabase &db, shared_ptr table_io_m this->row_groups = make_shared_ptr(info, io_manager, types, 0); if (data && data->row_group_count > 0) { this->row_groups->Initialize(*data); + if (!HasIndexes()) { + // if we don't have indexes, always append a new row group upon appending + // we can clean up this row group again when vacuuming + // since we don't yet support vacuum when there are indexes, we only do this when there are no indexes + row_groups->SetAppendRequiresNewRowGroup(); + } } else { this->row_groups->InitializeEmpty(); D_ASSERT(row_groups->GetTotalRows() == 0); @@ -239,7 +245,7 @@ void DataTable::InitializeScan(ClientContext &context, DuckTransaction &transact state.checkpoint_lock = transaction.SharedLockTable(*info); auto &local_storage = LocalStorage::Get(transaction); state.Initialize(column_ids, context, table_filters); - row_groups->InitializeScan(state.table_state, column_ids, table_filters); + row_groups->InitializeScan(context, state.table_state, column_ids, table_filters); local_storage.InitializeScan(*this, state.local_state, table_filters); } @@ -247,7 +253,7 @@ void DataTable::InitializeScanWithOffset(DuckTransaction &transaction, TableScan const vector &column_ids, idx_t start_row, idx_t end_row) { state.checkpoint_lock = transaction.SharedLockTable(*info); state.Initialize(column_ids); - row_groups->InitializeScanWithOffset(state.table_state, column_ids, start_row, end_row); + row_groups->InitializeScanWithOffset(QueryContext(), state.table_state, column_ids, start_row, end_row); } idx_t DataTable::GetRowGroupSize() const { @@ -675,7 +681,7 @@ void DataTable::VerifyNewConstraint(LocalStorage &local_storage, DataTable &pare throw NotImplementedException("FIXME: ALTER COLUMN with such constraint is not supported yet"); } - parent.row_groups->VerifyNewConstraint(parent, constraint); + parent.row_groups->VerifyNewConstraint(local_storage.GetClientContext(), parent, constraint); local_storage.VerifyNewConstraint(parent, constraint); } @@ -899,12 +905,14 @@ void DataTable::FinalizeLocalAppend(LocalAppendState &state) { LocalStorage::FinalizeAppend(state); } -PhysicalIndex DataTable::CreateOptimisticCollection(ClientContext &context, unique_ptr collection) { +PhysicalIndex DataTable::CreateOptimisticCollection(ClientContext &context, + unique_ptr collection) { auto &local_storage = LocalStorage::Get(context, db); return local_storage.CreateOptimisticCollection(*this, std::move(collection)); } -RowGroupCollection &DataTable::GetOptimisticCollection(ClientContext &context, const PhysicalIndex collection_index) { +OptimisticWriteCollection &DataTable::GetOptimisticCollection(ClientContext &context, + const PhysicalIndex collection_index) { auto &local_storage = LocalStorage::Get(context, db); return local_storage.GetOptimisticCollection(*this, collection_index); } @@ -919,7 +927,7 @@ OptimisticDataWriter &DataTable::GetOptimisticWriter(ClientContext &context) { return local_storage.GetOptimisticWriter(*this); } -void DataTable::LocalMerge(ClientContext &context, RowGroupCollection &collection) { +void DataTable::LocalMerge(ClientContext &context, OptimisticWriteCollection &collection) { auto &local_storage = LocalStorage::Get(context, db); local_storage.LocalMerge(*this, collection); } @@ -1262,9 +1270,9 @@ void DataTable::RemoveFromIndexes(TableAppendState &state, DataChunk &chunk, Vec }); } -void DataTable::RemoveFromIndexes(Vector &row_identifiers, idx_t count) { +void DataTable::RemoveFromIndexes(const QueryContext &context, Vector &row_identifiers, idx_t count) { D_ASSERT(IsMainTable()); - row_groups->RemoveFromIndexes(info->indexes, row_identifiers, count); + row_groups->RemoveFromIndexes(context, info->indexes, row_identifiers, count); } //===--------------------------------------------------------------------===// @@ -1536,7 +1544,7 @@ void DataTable::Update(TableUpdateState &state, ClientContext &context, Vector & row_ids_slice.Slice(row_ids, sel_global_update, n_global_update); row_ids_slice.Flatten(n_global_update); - row_groups->Update(transaction, FlatVector::GetData(row_ids_slice), column_ids, updates_slice); + row_groups->Update(transaction, *this, FlatVector::GetData(row_ids_slice), column_ids, updates_slice); } } @@ -1560,7 +1568,7 @@ void DataTable::UpdateColumn(TableCatalogEntry &table, ClientContext &context, V updates.Flatten(); row_ids.Flatten(updates.size()); - row_groups->UpdateColumn(transaction, row_ids, column_path, updates); + row_groups->UpdateColumn(transaction, *this, row_ids, column_path, updates); } //===--------------------------------------------------------------------===// @@ -1598,19 +1606,26 @@ void DataTable::Checkpoint(TableDataWriter &writer, Serializer &serializer) { TableStatistics global_stats; row_groups->CopyStats(global_stats); row_groups->Checkpoint(writer, global_stats); + if (!HasIndexes()) { + row_groups->SetAppendRequiresNewRowGroup(); + } // The row group payload data has been written. Now write: // sample // column stats // row-group pointers // table pointer // index data - writer.FinalizeTable(global_stats, info.get(), serializer); + writer.FinalizeTable(global_stats, *info, *row_groups, serializer); } void DataTable::CommitDropColumn(const idx_t column_index) { row_groups->CommitDropColumn(column_index); } +void DataTable::Destroy() { + row_groups->Destroy(); +} + idx_t DataTable::ColumnCount() const { return column_definitions.size(); } @@ -1634,9 +1649,9 @@ void DataTable::CommitDropTable() { //===--------------------------------------------------------------------===// // Column Segment Info //===--------------------------------------------------------------------===// -vector DataTable::GetColumnSegmentInfo() { +vector DataTable::GetColumnSegmentInfo(const QueryContext &context) { auto lock = GetSharedCheckpointLock(); - return row_groups->GetColumnSegmentInfo(); + return row_groups->GetColumnSegmentInfo(context); } //===--------------------------------------------------------------------===// diff --git a/src/storage/local_storage.cpp b/src/storage/local_storage.cpp index b89183e26eef..39612b09b85c 100644 --- a/src/storage/local_storage.cpp +++ b/src/storage/local_storage.cpp @@ -16,14 +16,14 @@ namespace duckdb { LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &table) - : table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(context, table), - merged_storage(false) { + : context(context), table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), + optimistic_writer(context, table), merged_storage(false) { auto types = table.GetTypes(); auto data_table_info = table.GetDataTableInfo(); - auto &io_manager = TableIOManager::Get(table); - row_groups = make_shared_ptr(data_table_info, io_manager, types, MAX_ROW_ID, 0); - row_groups->InitializeEmpty(); + row_groups = optimistic_writer.CreateCollection(table, types, OptimisticWritePartialManagers::GLOBAL); + auto &collection = *row_groups->collection; + collection.InitializeEmpty(); data_table_info->GetIndexes().Scan([&](Index &index) { auto constraint = index.GetConstraintType(); @@ -63,14 +63,17 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &table) LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_data_table, LocalTableStorage &parent, const idx_t alter_column_index, const LogicalType &target_type, const vector &bound_columns, Expression &cast_expr) - : table_ref(new_data_table), allocator(Allocator::Get(new_data_table.db)), deleted_rows(parent.deleted_rows), - optimistic_collections(std::move(parent.optimistic_collections)), + : context(context), table_ref(new_data_table), allocator(Allocator::Get(new_data_table.db)), + deleted_rows(parent.deleted_rows), optimistic_collections(std::move(parent.optimistic_collections)), optimistic_writer(new_data_table, parent.optimistic_writer), merged_storage(parent.merged_storage) { // Alter the column type. - row_groups = parent.row_groups->AlterType(context, alter_column_index, target_type, bound_columns, cast_expr); - parent.row_groups->CommitDropColumn(alter_column_index); - parent.row_groups.reset(); + auto &parent_collection = *parent.row_groups->collection; + auto new_collection = + parent_collection.AlterType(context, alter_column_index, target_type, bound_columns, cast_expr); + parent_collection.CommitDropColumn(alter_column_index); + row_groups = std::move(parent.row_groups); + row_groups->collection = std::move(new_collection); append_indexes.Move(parent.append_indexes); } @@ -82,9 +85,11 @@ LocalTableStorage::LocalTableStorage(DataTable &new_data_table, LocalTableStorag optimistic_writer(new_data_table, parent.optimistic_writer), merged_storage(parent.merged_storage) { // Remove the column from the previous table storage. - row_groups = parent.row_groups->RemoveColumn(drop_column_index); - parent.row_groups->CommitDropColumn(drop_column_index); - parent.row_groups.reset(); + auto &parent_collection = *parent.row_groups->collection; + auto new_collection = parent_collection.RemoveColumn(drop_column_index); + parent_collection.CommitDropColumn(drop_column_index); + row_groups = std::move(parent.row_groups); + row_groups->collection = std::move(new_collection); append_indexes.Move(parent.append_indexes); } @@ -95,8 +100,10 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt, optimistic_collections(std::move(parent.optimistic_collections)), optimistic_writer(new_dt, parent.optimistic_writer), merged_storage(parent.merged_storage) { - row_groups = parent.row_groups->AddColumn(context, new_column, default_executor); - parent.row_groups.reset(); + auto &parent_collection = *parent.row_groups->collection; + auto new_collection = parent_collection.AddColumn(context, new_column, default_executor); + row_groups = std::move(parent.row_groups); + row_groups->collection = std::move(new_collection); append_indexes.Move(parent.append_indexes); } @@ -104,19 +111,21 @@ LocalTableStorage::~LocalTableStorage() { } void LocalTableStorage::InitializeScan(CollectionScanState &state, optional_ptr table_filters) { - if (row_groups->GetTotalRows() == 0) { + auto &collection = *row_groups->collection; + if (collection.GetTotalRows() == 0) { throw InternalException("No rows in LocalTableStorage row group for scan"); } - row_groups->InitializeScan(state, state.GetColumnIds(), table_filters.get()); + collection.InitializeScan(context, state, state.GetColumnIds(), table_filters.get()); } idx_t LocalTableStorage::EstimatedSize() { // count the appended rows - idx_t appended_rows = row_groups->GetTotalRows() - deleted_rows; + auto &collection = *row_groups->collection; + idx_t appended_rows = collection.GetTotalRows() - deleted_rows; // get the (estimated) size of a row (no compressions, etc.) idx_t row_size = 0; - auto &types = row_groups->GetTypes(); + auto &types = collection.GetTypes(); for (auto &type : types) { row_size += GetTypeIdSize(type.InternalType()); } @@ -144,8 +153,9 @@ void LocalTableStorage::WriteNewRowGroup() { } void LocalTableStorage::FlushBlocks() { - const idx_t row_group_size = row_groups->GetRowGroupSize(); - if (!merged_storage && row_groups->GetTotalRows() > row_group_size) { + auto &collection = *row_groups->collection; + const idx_t row_group_size = collection.GetRowGroupSize(); + if (!merged_storage && collection.GetTotalRows() > row_group_size) { optimistic_writer.WriteLastRowGroup(*row_groups); } optimistic_writer.FinalFlush(); @@ -205,16 +215,17 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen auto &index_list = data_table_info->GetIndexes(); ErrorData error; + auto &collection = *row_groups->collection; if (append_to_table) { // Appending to the table: we need to scan the entire chunk. DataChunk index_chunk; vector mapped_column_ids; if (table.HasIndexes() && index_list.HasUnbound()) { - TableIndexList::InitializeIndexChunk(index_chunk, row_groups->GetTypes(), mapped_column_ids, + TableIndexList::InitializeIndexChunk(index_chunk, collection.GetTypes(), mapped_column_ids, *data_table_info); } - row_groups->Scan(transaction, [&](DataChunk &table_chunk) -> bool { + collection.Scan(transaction, [&](DataChunk &table_chunk) -> bool { if (table.HasIndexes()) { if (index_list.HasUnbound()) { // The index chunk references all indexed columns. @@ -238,14 +249,14 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen } else { // We only append to the indexes. - error = AppendToIndexes(transaction, *row_groups, index_list, table.GetTypes(), append_state.current_row); + error = AppendToIndexes(transaction, collection, index_list, table.GetTypes(), append_state.current_row); } if (error.HasError()) { // Revert all appended row IDs. row_t current_row = append_state.row_start; // Remove the data from the indexes, if any. - row_groups->Scan(transaction, [&](DataChunk &chunk) -> bool { + collection.Scan(transaction, [&](DataChunk &chunk) -> bool { // Remove the chunk. try { table.RemoveFromIndexes(append_state, chunk, current_row); @@ -278,13 +289,13 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen } } -PhysicalIndex LocalTableStorage::CreateOptimisticCollection(unique_ptr collection) { +PhysicalIndex LocalTableStorage::CreateOptimisticCollection(unique_ptr collection) { lock_guard l(collections_lock); optimistic_collections.push_back(std::move(collection)); return PhysicalIndex(optimistic_collections.size() - 1); } -RowGroupCollection &LocalTableStorage::GetOptimisticCollection(const PhysicalIndex collection_index) { +OptimisticWriteCollection &LocalTableStorage::GetOptimisticCollection(const PhysicalIndex collection_index) { lock_guard l(collections_lock); auto &collection = optimistic_collections[collection_index.index]; return *collection; @@ -306,10 +317,10 @@ void LocalTableStorage::Rollback() { if (!collection) { continue; } - collection->CommitDropTable(); + collection->collection->CommitDropTable(); } optimistic_collections.clear(); - row_groups->CommitDropTable(); + row_groups->collection->CommitDropTable(); } //===--------------------------------------------------------------------===// @@ -398,7 +409,7 @@ LocalStorage &LocalStorage::Get(ClientContext &context, Catalog &catalog) { void LocalStorage::InitializeScan(DataTable &table, CollectionScanState &state, optional_ptr table_filters) { auto storage = table_manager.GetStorage(table); - if (storage == nullptr || storage->row_groups->GetTotalRows() == 0) { + if (storage == nullptr || storage->GetCollection().GetTotalRows() == 0) { return; } storage->InitializeScan(state, table_filters); @@ -415,22 +426,26 @@ void LocalStorage::InitializeParallelScan(DataTable &table, ParallelCollectionSc state.vector_index = 0; state.current_row_group = nullptr; } else { - storage->row_groups->InitializeParallelScan(state); + storage->GetCollection().InitializeParallelScan(state); } } +RowGroupCollection &LocalTableStorage::GetCollection() { + return *row_groups->collection; +} + bool LocalStorage::NextParallelScan(ClientContext &context, DataTable &table, ParallelCollectionScanState &state, CollectionScanState &scan_state) { auto storage = table_manager.GetStorage(table); if (!storage) { return false; } - return storage->row_groups->NextParallelScan(context, state, scan_state); + return storage->GetCollection().NextParallelScan(context, state, scan_state); } void LocalStorage::InitializeAppend(LocalAppendState &state, DataTable &table) { state.storage = &table_manager.GetOrCreateStorage(context, table); - state.storage->row_groups->InitializeAppend(TransactionData(transaction), state.append_state); + state.storage->GetCollection().InitializeAppend(TransactionData(transaction), state.append_state); } void LocalStorage::InitializeStorage(LocalAppendState &state, DataTable &table) { @@ -459,7 +474,7 @@ void LocalTableStorage::AppendToDeleteIndexes(Vector &row_ids, DataChunk &delete void LocalStorage::Append(LocalAppendState &state, DataChunk &table_chunk, DataTableInfo &data_table_info) { // Append to any unique indexes. auto storage = state.storage; - auto offset = NumericCast(MAX_ROW_ID) + storage->row_groups->GetTotalRows(); + auto offset = NumericCast(MAX_ROW_ID) + storage->GetCollection().GetTotalRows(); idx_t base_id = offset + state.append_state.total_append_count; if (!storage->append_indexes.Empty()) { @@ -482,7 +497,7 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &table_chunk, DataT } // Append the chunk to the local storage. - auto new_row_group = storage->row_groups->Append(table_chunk, state.append_state); + auto new_row_group = storage->GetCollection().Append(table_chunk, state.append_state); // Check if we should pre-emptively flush blocks to disk. if (new_row_group) { @@ -491,30 +506,32 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &table_chunk, DataT } void LocalStorage::FinalizeAppend(LocalAppendState &state) { - state.storage->row_groups->FinalizeAppend(state.append_state.transaction, state.append_state); + state.storage->GetCollection().FinalizeAppend(state.append_state.transaction, state.append_state); } -void LocalStorage::LocalMerge(DataTable &table, RowGroupCollection &collection) { +void LocalStorage::LocalMerge(DataTable &table, OptimisticWriteCollection &collection) { auto &storage = table_manager.GetOrCreateStorage(context, table); if (!storage.append_indexes.Empty()) { // append data to indexes if required - row_t base_id = MAX_ROW_ID + NumericCast(storage.row_groups->GetTotalRows()); - auto error = - storage.AppendToIndexes(transaction, collection, storage.append_indexes, table.GetTypes(), base_id); + row_t base_id = MAX_ROW_ID + NumericCast(storage.GetCollection().GetTotalRows()); + auto error = storage.AppendToIndexes(transaction, *collection.collection, storage.append_indexes, + table.GetTypes(), base_id); if (error.HasError()) { error.Throw(); } } - storage.row_groups->MergeStorage(collection, nullptr, nullptr); + storage.GetCollection().MergeStorage(*collection.collection, nullptr, nullptr); storage.merged_storage = true; } -PhysicalIndex LocalStorage::CreateOptimisticCollection(DataTable &table, unique_ptr collection) { +PhysicalIndex LocalStorage::CreateOptimisticCollection(DataTable &table, + unique_ptr collection) { auto &storage = table_manager.GetOrCreateStorage(context, table); return storage.CreateOptimisticCollection(std::move(collection)); } -RowGroupCollection &LocalStorage::GetOptimisticCollection(DataTable &table, const PhysicalIndex collection_index) { +OptimisticWriteCollection &LocalStorage::GetOptimisticCollection(DataTable &table, + const PhysicalIndex collection_index) { auto &storage = table_manager.GetOrCreateStorage(context, table); return storage.GetOptimisticCollection(collection_index); } @@ -547,11 +564,11 @@ idx_t LocalStorage::Delete(DataTable &table, Vector &row_ids, idx_t count) { // delete from unique indices (if any) if (!storage->append_indexes.Empty()) { - storage->row_groups->RemoveFromIndexes(storage->append_indexes, row_ids, count); + storage->GetCollection().RemoveFromIndexes(context, storage->append_indexes, row_ids, count); } auto ids = FlatVector::GetData(row_ids); - idx_t delete_count = storage->row_groups->Delete(TransactionData(0, 0), table, ids, count); + idx_t delete_count = storage->GetCollection().Delete(TransactionData(0, 0), table, ids, count); storage->deleted_rows += delete_count; return delete_count; } @@ -563,27 +580,27 @@ void LocalStorage::Update(DataTable &table, Vector &row_ids, const vector(row_ids); - storage->row_groups->Update(TransactionData(0, 0), ids, column_ids, updates); + storage->GetCollection().Update(TransactionData(0, 0), table, ids, column_ids, updates); } void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage, optional_ptr commit_state) { if (storage.is_dropped) { return; } - if (storage.row_groups->GetTotalRows() <= storage.deleted_rows) { + if (storage.GetCollection().GetTotalRows() <= storage.deleted_rows) { // all rows that we added were deleted // rollback any partial blocks that are still outstanding storage.Rollback(); return; } - auto append_count = storage.row_groups->GetTotalRows() - storage.deleted_rows; - const auto row_group_size = storage.row_groups->GetRowGroupSize(); + auto append_count = storage.GetCollection().GetTotalRows() - storage.deleted_rows; + const auto row_group_size = storage.GetCollection().GetRowGroupSize(); TableAppendState append_state; table.AppendLock(append_state); transaction.PushAppend(table, NumericCast(append_state.row_start), append_count); - if ((append_state.row_start == 0 || storage.row_groups->GetTotalRows() >= row_group_size) && + if ((append_state.row_start == 0 || storage.GetCollection().GetTotalRows() >= row_group_size) && storage.deleted_rows == 0) { // table is currently empty OR we are bulk appending: move over the storage directly // first flush any outstanding blocks @@ -593,7 +610,7 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage, optional_ storage.AppendToIndexes(transaction, append_state, false); } // finally move over the row groups - table.MergeStorage(*storage.row_groups, storage.append_indexes, commit_state); + table.MergeStorage(storage.GetCollection(), storage.append_indexes, commit_state); } else { // check if we have written data // if we have, we cannot merge to disk after all @@ -641,7 +658,7 @@ idx_t LocalStorage::AddedRows(DataTable &table) { if (!storage) { return 0; } - return storage->row_groups->GetTotalRows() - storage->deleted_rows; + return storage->GetCollection().GetTotalRows() - storage->deleted_rows; } vector LocalStorage::GetPartitionStats(DataTable &table) const { @@ -649,7 +666,7 @@ vector LocalStorage::GetPartitionStats(DataTable &table) co if (!storage) { return vector(); } - return storage->row_groups->GetPartitionStats(); + return storage->GetCollection().GetPartitionStats(); } void LocalStorage::DropTable(DataTable &table) { @@ -710,7 +727,7 @@ void LocalStorage::FetchChunk(DataTable &table, Vector &row_ids, idx_t count, co if (!storage) { throw InternalException("LocalStorage::FetchChunk - local storage not found"); } - storage->row_groups->Fetch(transaction, chunk, col_ids, row_ids, count, fetch_state); + storage->GetCollection().Fetch(transaction, chunk, col_ids, row_ids, count, fetch_state); } bool LocalStorage::CanFetch(DataTable &table, const row_t row_id) { @@ -718,7 +735,7 @@ bool LocalStorage::CanFetch(DataTable &table, const row_t row_id) { if (!storage) { throw InternalException("LocalStorage::CanFetch - local storage not found"); } - return storage->row_groups->CanFetch(transaction, row_id); + return storage->GetCollection().CanFetch(transaction, row_id); } TableIndexList &LocalStorage::GetIndexes(ClientContext &context, DataTable &table) { @@ -735,7 +752,7 @@ void LocalStorage::VerifyNewConstraint(DataTable &parent, const BoundConstraint if (!storage) { return; } - storage->row_groups->VerifyNewConstraint(parent, constraint); + storage->GetCollection().VerifyNewConstraint(context, parent, constraint); } } // namespace duckdb diff --git a/src/storage/metadata/metadata_manager.cpp b/src/storage/metadata/metadata_manager.cpp index d3efd4195cfe..8a32d6dc709d 100644 --- a/src/storage/metadata/metadata_manager.cpp +++ b/src/storage/metadata/metadata_manager.cpp @@ -31,6 +31,19 @@ MetadataBlock &MetadataBlock::operator=(MetadataBlock &&other) noexcept { return *this; } +string MetadataBlock::ToString() const { + string result; + for (idx_t i = 0; i < MetadataManager::METADATA_BLOCK_COUNT; i++) { + if (std::find(free_blocks.begin(), free_blocks.end(), i) != free_blocks.end()) { + if (!result.empty()) { + result += ", "; + } + result += to_string(i); + } + } + return "block_id: " + to_string(block_id) + " [" + result + "]"; +} + MetadataManager::MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager) : block_manager(block_manager), buffer_manager(buffer_manager) { } @@ -41,6 +54,8 @@ MetadataManager::~MetadataManager() { MetadataHandle MetadataManager::AllocateHandle() { // check if there is any free space left in an existing block // if not allocate a new block + MetadataPointer pointer; + unique_lock guard(block_lock); block_id_t free_block = INVALID_BLOCK; for (auto &kv : blocks) { auto &block = kv.second; @@ -50,13 +65,16 @@ MetadataHandle MetadataManager::AllocateHandle() { break; } } + guard.unlock(); if (free_block == INVALID_BLOCK || free_block > PeekNextBlockId()) { - free_block = AllocateNewBlock(); + free_block = AllocateNewBlock(guard); + } else { + guard.lock(); } + D_ASSERT(guard.owns_lock()); D_ASSERT(free_block != INVALID_BLOCK); // select the first free metadata block we can find - MetadataPointer pointer; pointer.block_index = UnsafeNumericCast(free_block); auto &block = blocks[free_block]; // the block is now dirty @@ -64,7 +82,7 @@ MetadataHandle MetadataManager::AllocateHandle() { if (block.block->BlockId() < MAXIMUM_BLOCK) { // this block is a disk-backed block, yet we are planning to write to it // we need to convert it into a transient block before we can write to it - ConvertToTransient(block); + ConvertToTransient(guard, block); D_ASSERT(block.block->BlockId() >= MAXIMUM_BLOCK); } D_ASSERT(!block.free_blocks.empty()); @@ -72,6 +90,7 @@ MetadataHandle MetadataManager::AllocateHandle() { // mark the block as used block.free_blocks.pop_back(); D_ASSERT(pointer.index < METADATA_BLOCK_COUNT); + guard.unlock(); // pin the block return Pin(pointer); } @@ -80,27 +99,36 @@ MetadataHandle MetadataManager::Pin(const MetadataPointer &pointer) { return Pin(QueryContext(), pointer); } -MetadataHandle MetadataManager::Pin(QueryContext context, const MetadataPointer &pointer) { +MetadataHandle MetadataManager::Pin(const QueryContext &context, const MetadataPointer &pointer) { D_ASSERT(pointer.index < METADATA_BLOCK_COUNT); - auto &block = blocks[UnsafeNumericCast(pointer.block_index)]; + shared_ptr block_handle; + { + lock_guard guard(block_lock); + auto &block = blocks[UnsafeNumericCast(pointer.block_index)]; #ifdef DEBUG - for (auto &free_block : block.free_blocks) { - if (free_block == pointer.index) { - throw InternalException("Pinning block %d.%d but it is marked as a free block", block.block_id, free_block); + for (auto &free_block : block.free_blocks) { + if (free_block == pointer.index) { + throw InternalException("Pinning block %d.%d but it is marked as a free block", block.block_id, + free_block); + } } - } #endif + block_handle = block.block; + } MetadataHandle handle; handle.pointer.block_index = pointer.block_index; handle.pointer.index = pointer.index; - handle.handle = buffer_manager.Pin(block.block); + handle.handle = buffer_manager.Pin(block_handle); return handle; } -void MetadataManager::ConvertToTransient(MetadataBlock &metadata_block) { +void MetadataManager::ConvertToTransient(unique_lock &block_lock, MetadataBlock &metadata_block) { + D_ASSERT(block_lock.owns_lock()); + auto old_block = metadata_block.block; + block_lock.unlock(); // pin the old block - auto old_buffer = buffer_manager.Pin(metadata_block.block); + auto old_buffer = buffer_manager.Pin(old_block); // allocate a new transient block to replace it auto new_buffer = buffer_manager.Allocate(MemoryTag::METADATA, &block_manager, false); @@ -108,14 +136,17 @@ void MetadataManager::ConvertToTransient(MetadataBlock &metadata_block) { // copy the data to the transient block memcpy(new_buffer.Ptr(), old_buffer.Ptr(), block_manager.GetBlockSize()); - metadata_block.block = std::move(new_block); - metadata_block.dirty = true; // unregister the old block block_manager.UnregisterBlock(metadata_block.block_id); + + block_lock.lock(); + metadata_block.block = std::move(new_block); + metadata_block.dirty = true; } -block_id_t MetadataManager::AllocateNewBlock() { +block_id_t MetadataManager::AllocateNewBlock(unique_lock &block_lock) { + D_ASSERT(!block_lock.owns_lock()); auto new_block_id = GetNextBlockId(); MetadataBlock new_block; @@ -128,11 +159,14 @@ block_id_t MetadataManager::AllocateNewBlock() { new_block.dirty = true; // zero-initialize the handle memset(handle.Ptr(), 0, block_manager.GetBlockSize()); - AddBlock(std::move(new_block)); + + block_lock.lock(); + AddBlock(block_lock, std::move(new_block)); return new_block_id; } -void MetadataManager::AddBlock(MetadataBlock new_block, bool if_exists) { +void MetadataManager::AddBlock(unique_lock &block_lock, MetadataBlock new_block, bool if_exists) { + D_ASSERT(block_lock.owns_lock()); if (blocks.find(new_block.block_id) != blocks.end()) { if (if_exists) { return; @@ -142,15 +176,17 @@ void MetadataManager::AddBlock(MetadataBlock new_block, bool if_exists) { blocks[new_block.block_id] = std::move(new_block); } -void MetadataManager::AddAndRegisterBlock(MetadataBlock block) { +void MetadataManager::AddAndRegisterBlock(unique_lock &block_lock, MetadataBlock block) { if (block.block) { throw InternalException("Calling AddAndRegisterBlock on block that already exists"); } if (block.block_id >= MAXIMUM_BLOCK) { throw InternalException("AddAndRegisterBlock called with a transient block id"); } + block_lock.unlock(); block.block = block_manager.RegisterBlock(block.block_id); - AddBlock(std::move(block), true); + block_lock.lock(); + AddBlock(block_lock, std::move(block), true); } MetaBlockPointer MetadataManager::GetDiskPointer(const MetadataPointer &pointer, uint32_t offset) { @@ -168,8 +204,14 @@ uint32_t MetaBlockPointer::GetBlockIndex() const { } MetadataPointer MetadataManager::FromDiskPointer(MetaBlockPointer pointer) { + unique_lock guard(block_lock); + return FromDiskPointerInternal(guard, pointer); +} + +MetadataPointer MetadataManager::FromDiskPointerInternal(unique_lock &block_lock, MetaBlockPointer pointer) { auto block_id = pointer.GetBlockId(); auto index = pointer.GetBlockIndex(); + auto entry = blocks.find(block_id); if (entry == blocks.end()) { // LCOV_EXCL_START throw InternalException("Failed to load metadata pointer (id %llu, idx %llu, ptr %llu)\n", block_id, index, @@ -182,11 +224,13 @@ MetadataPointer MetadataManager::FromDiskPointer(MetaBlockPointer pointer) { } MetadataPointer MetadataManager::RegisterDiskPointer(MetaBlockPointer pointer) { + unique_lock guard(block_lock); + auto block_id = pointer.GetBlockId(); MetadataBlock block; block.block_id = block_id; - AddAndRegisterBlock(std::move(block)); - return FromDiskPointer(pointer); + AddAndRegisterBlock(guard, std::move(block)); + return FromDiskPointerInternal(guard, pointer); } BlockPointer MetadataManager::ToBlockPointer(MetaBlockPointer meta_pointer, const idx_t metadata_block_size) { @@ -219,6 +263,7 @@ void MetadataManager::Flush() { // Write the blocks of the metadata manager to disk. const idx_t total_metadata_size = GetMetadataBlockSize() * METADATA_BLOCK_COUNT; + unique_lock guard(block_lock, std::defer_lock); for (auto &kv : blocks) { auto &block = kv.second; if (!block.dirty) { @@ -227,14 +272,21 @@ void MetadataManager::Flush() { } continue; } - auto handle = buffer_manager.Pin(block.block); + auto block_handle = block.block; + auto handle = buffer_manager.Pin(block_handle); // zero-initialize the few leftover bytes memset(handle.Ptr() + total_metadata_size, 0, block_manager.GetBlockSize() - total_metadata_size); D_ASSERT(kv.first == block.block_id); - if (block.block->BlockId() >= MAXIMUM_BLOCK) { + if (block_handle->BlockId() >= MAXIMUM_BLOCK) { // Convert the temporary block to a persistent block. - block.block = - block_manager.ConvertToPersistent(QueryContext(), kv.first, std::move(block.block), std::move(handle)); + // we cannot use ConvertToPersistent as another thread might still be reading the block + // so we use the safe version of ConvertToPersistent + auto new_block = block_manager.ConvertToPersistent(QueryContext(), kv.first, std::move(block_handle), + std::move(handle), ConvertToPersistentMode::THREAD_SAFE); + + guard.lock(); + block.block = std::move(new_block); + guard.unlock(); } else { // Already a persistent block, so we only need to write it. D_ASSERT(block.block->BlockId() == block.block_id); @@ -256,10 +308,12 @@ void MetadataManager::Read(ReadStream &source) { auto block_count = source.Read(); for (idx_t i = 0; i < block_count; i++) { auto block = MetadataBlock::Read(source); + + unique_lock guard(block_lock); auto entry = blocks.find(block.block_id); if (entry == blocks.end()) { // block does not exist yet - AddAndRegisterBlock(std::move(block)); + AddAndRegisterBlock(guard, std::move(block)); } else { // block was already created - only copy over the free list entry->second.free_blocks = std::move(block.free_blocks); @@ -294,18 +348,24 @@ idx_t MetadataBlock::FreeBlocksToInteger() { return result; } -void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) { - free_blocks.clear(); - if (free_list == 0) { - return; - } +vector MetadataBlock::BlocksFromInteger(idx_t free_list) { + vector blocks; for (idx_t i = 64; i > 0; i--) { auto index = i - 1; idx_t mask = idx_t(1) << index; if (free_list & mask) { - free_blocks.push_back(UnsafeNumericCast(index)); + blocks.push_back(UnsafeNumericCast(index)); } } + return blocks; +} + +void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) { + free_blocks.clear(); + if (free_list == 0) { + return; + } + free_blocks = BlocksFromInteger(free_list); } void MetadataManager::MarkBlocksAsModified() { @@ -330,6 +390,7 @@ void MetadataManager::MarkBlocksAsModified() { } modified_blocks.clear(); + for (auto &kv : blocks) { auto &block = kv.second; idx_t free_list = block.FreeBlocksToInteger(); @@ -339,6 +400,10 @@ void MetadataManager::MarkBlocksAsModified() { } void MetadataManager::ClearModifiedBlocks(const vector &pointers) { + if (pointers.empty()) { + return; + } + unique_lock guard(block_lock); for (auto &pointer : pointers) { auto block_id = pointer.GetBlockId(); auto block_index = pointer.GetBlockIndex(); @@ -354,6 +419,7 @@ void MetadataManager::ClearModifiedBlocks(const vector &pointe vector MetadataManager::GetMetadataInfo() const { vector result; + unique_lock guard(block_lock); for (auto &block : blocks) { MetadataBlockInfo block_info; block_info.block_id = block.second.block_id; @@ -371,17 +437,18 @@ vector MetadataManager::GetMetadataInfo() const { vector> MetadataManager::GetBlocks() const { vector> result; + unique_lock guard(block_lock); for (auto &entry : blocks) { result.push_back(entry.second.block); } return result; } -block_id_t MetadataManager::PeekNextBlockId() { +block_id_t MetadataManager::PeekNextBlockId() const { return block_manager.PeekFreeBlockId(); } -block_id_t MetadataManager::GetNextBlockId() { +block_id_t MetadataManager::GetNextBlockId() const { return block_manager.GetFreeBlockId(); } diff --git a/src/storage/optimistic_data_writer.cpp b/src/storage/optimistic_data_writer.cpp index 94dd5210e741..fbe481364b9f 100644 --- a/src/storage/optimistic_data_writer.cpp +++ b/src/storage/optimistic_data_writer.cpp @@ -2,9 +2,13 @@ #include "duckdb/storage/table/column_segment.hpp" #include "duckdb/storage/partial_block_manager.hpp" #include "duckdb/storage/table/column_checkpoint_state.hpp" +#include "duckdb/main/settings.hpp" namespace duckdb { +OptimisticWriteCollection::~OptimisticWriteCollection() { +} + OptimisticDataWriter::OptimisticDataWriter(ClientContext &context, DataTable &table) : context(context), table(table) { } @@ -27,56 +31,98 @@ bool OptimisticDataWriter::PrepareWrite() { // allocate the partial block-manager if none is allocated yet if (!partial_manager) { auto &block_manager = table.GetTableIOManager().GetBlockManagerForRowData(); - partial_manager = - make_uniq(QueryContext(context), block_manager, PartialBlockType::APPEND_TO_TABLE); + partial_manager = make_uniq(context, block_manager, PartialBlockType::APPEND_TO_TABLE); } return true; } -void OptimisticDataWriter::WriteNewRowGroup(RowGroupCollection &row_groups) { +unique_ptr OptimisticDataWriter::CreateCollection(DataTable &storage, + const vector &insert_types, + OptimisticWritePartialManagers type) { + auto table_info = storage.GetDataTableInfo(); + auto &io_manager = TableIOManager::Get(storage); + + // Create the local row group collection. + auto max_row_id = NumericCast(MAX_ROW_ID); + auto row_groups = make_shared_ptr(std::move(table_info), io_manager, insert_types, max_row_id); + + auto result = make_uniq(); + result->collection = std::move(row_groups); + if (type == OptimisticWritePartialManagers::PER_COLUMN) { + for (idx_t i = 0; i < insert_types.size(); i++) { + auto &block_manager = table.GetTableIOManager().GetBlockManagerForRowData(); + result->partial_block_managers.push_back(make_uniq( + QueryContext(context), block_manager, PartialBlockType::APPEND_TO_TABLE)); + } + } + return result; +} + +void OptimisticDataWriter::WriteNewRowGroup(OptimisticWriteCollection &row_groups) { // we finished writing a complete row group if (!PrepareWrite()) { return; } - // flush second-to-last row group - auto row_group = row_groups.GetRowGroup(-2); - FlushToDisk(*row_group); + + row_groups.complete_row_groups++; + auto unflushed_row_groups = row_groups.complete_row_groups - row_groups.last_flushed; + if (unflushed_row_groups >= DBConfig::GetSetting(context)) { + // we have crossed our flush threshold - flush any unwritten row groups to disk + vector> to_flush; + for (idx_t i = row_groups.last_flushed; i < row_groups.complete_row_groups; i++) { + to_flush.push_back(*row_groups.collection->GetRowGroup(NumericCast(i))); + } + FlushToDisk(row_groups, to_flush); + row_groups.last_flushed = row_groups.complete_row_groups; + } } -void OptimisticDataWriter::WriteLastRowGroup(RowGroupCollection &row_groups) { +void OptimisticDataWriter::WriteLastRowGroup(OptimisticWriteCollection &row_groups) { // we finished writing a complete row group if (!PrepareWrite()) { return; } - // flush second-to-last row group - auto row_group = row_groups.GetRowGroup(-1); - if (!row_group) { - return; + // flush the last batch of row groups + vector> to_flush; + for (idx_t i = row_groups.last_flushed; i < row_groups.complete_row_groups; i++) { + to_flush.push_back(*row_groups.collection->GetRowGroup(NumericCast(i))); + } + // add the last (incomplete) row group + to_flush.push_back(*row_groups.collection->GetRowGroup(-1)); + FlushToDisk(row_groups, to_flush); + + for (auto &partial_manager : row_groups.partial_block_managers) { + Merge(partial_manager); } - FlushToDisk(*row_group); + row_groups.partial_block_managers.clear(); } -void OptimisticDataWriter::FlushToDisk(RowGroup &row_group) { +void OptimisticDataWriter::FlushToDisk(OptimisticWriteCollection &collection, + const vector> &row_groups) { //! The set of column compression types (if any) vector compression_types; D_ASSERT(compression_types.empty()); for (auto &column : table.Columns()) { compression_types.push_back(column.CompressionType()); } - RowGroupWriteInfo info(*partial_manager, compression_types); - row_group.WriteToDisk(info); + RowGroupWriteInfo info(*partial_manager, compression_types, collection.partial_block_managers); + RowGroup::WriteToDisk(info, row_groups); } -void OptimisticDataWriter::Merge(OptimisticDataWriter &other) { - if (!other.partial_manager) { +void OptimisticDataWriter::Merge(unique_ptr &other_manager) { + if (!other_manager) { return; } if (!partial_manager) { - partial_manager = std::move(other.partial_manager); + partial_manager = std::move(other_manager); return; } - partial_manager->Merge(*other.partial_manager); - other.partial_manager.reset(); + partial_manager->Merge(*other_manager); + other_manager.reset(); +} + +void OptimisticDataWriter::Merge(OptimisticDataWriter &other) { + Merge(other.partial_manager); } void OptimisticDataWriter::FinalFlush() { diff --git a/src/storage/serialization/serialize_logical_operator.cpp b/src/storage/serialization/serialize_logical_operator.cpp index 83c37ec4fe98..4c70c25c1060 100644 --- a/src/storage/serialization/serialize_logical_operator.cpp +++ b/src/storage/serialization/serialize_logical_operator.cpp @@ -290,7 +290,6 @@ void LogicalCTERef::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(201, "cte_index", cte_index); serializer.WritePropertyWithDefault>(202, "chunk_types", chunk_types); serializer.WritePropertyWithDefault>(203, "bound_columns", bound_columns); - serializer.WriteProperty(204, "materialized_cte", materialized_cte); serializer.WritePropertyWithDefault(205, "is_recurring", is_recurring); } @@ -299,8 +298,7 @@ unique_ptr LogicalCTERef::Deserialize(Deserializer &deserialize auto cte_index = deserializer.ReadPropertyWithDefault(201, "cte_index"); auto chunk_types = deserializer.ReadPropertyWithDefault>(202, "chunk_types"); auto bound_columns = deserializer.ReadPropertyWithDefault>(203, "bound_columns"); - auto materialized_cte = deserializer.ReadProperty(204, "materialized_cte"); - auto result = duckdb::unique_ptr(new LogicalCTERef(table_index, cte_index, std::move(chunk_types), std::move(bound_columns), materialized_cte)); + auto result = duckdb::unique_ptr(new LogicalCTERef(table_index, cte_index, std::move(chunk_types), std::move(bound_columns))); deserializer.ReadPropertyWithDefault(205, "is_recurring", result->is_recurring); return std::move(result); } diff --git a/src/storage/serialization/serialize_nodes.cpp b/src/storage/serialization/serialize_nodes.cpp index b87ba38ecd88..ac39591773bf 100644 --- a/src/storage/serialization/serialize_nodes.cpp +++ b/src/storage/serialization/serialize_nodes.cpp @@ -252,7 +252,7 @@ ColumnList ColumnList::Deserialize(Deserializer &deserializer) { void CommonTableExpressionInfo::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault>(100, "aliases", aliases); serializer.WritePropertyWithDefault>(101, "query", query); - serializer.WriteProperty(102, "materialized", materialized); + serializer.WriteProperty(102, "materialized", GetMaterializedForSerialization(serializer)); serializer.WritePropertyWithDefault>>(103, "key_targets", key_targets); } diff --git a/src/storage/serialization/serialize_parse_info.cpp b/src/storage/serialization/serialize_parse_info.cpp index 01bf5bfd5069..a5616e0e60c1 100644 --- a/src/storage/serialization/serialize_parse_info.cpp +++ b/src/storage/serialization/serialize_parse_info.cpp @@ -9,6 +9,7 @@ #include "duckdb/parser/parsed_data/alter_info.hpp" #include "duckdb/parser/parsed_data/alter_table_info.hpp" #include "duckdb/parser/parsed_data/comment_on_column_info.hpp" +#include "duckdb/parser/parsed_data/alter_database_info.hpp" #include "duckdb/parser/parsed_data/attach_info.hpp" #include "duckdb/parser/parsed_data/copy_database_info.hpp" #include "duckdb/parser/parsed_data/copy_info.hpp" @@ -92,6 +93,9 @@ unique_ptr AlterInfo::Deserialize(Deserializer &deserializer) { auto allow_internal = deserializer.ReadPropertyWithDefault(205, "allow_internal"); unique_ptr result; switch (type) { + case AlterType::ALTER_DATABASE: + result = AlterDatabaseInfo::Deserialize(deserializer); + break; case AlterType::ALTER_TABLE: result = AlterTableInfo::Deserialize(deserializer); break; @@ -196,6 +200,24 @@ unique_ptr AlterViewInfo::Deserialize(Deserializer &deserializer) { return std::move(result); } +void AlterDatabaseInfo::Serialize(Serializer &serializer) const { + AlterInfo::Serialize(serializer); + serializer.WriteProperty(300, "alter_database_type", alter_database_type); +} + +unique_ptr AlterDatabaseInfo::Deserialize(Deserializer &deserializer) { + auto alter_database_type = deserializer.ReadProperty(300, "alter_database_type"); + unique_ptr result; + switch (alter_database_type) { + case AlterDatabaseType::RENAME_DATABASE: + result = RenameDatabaseInfo::Deserialize(deserializer); + break; + default: + throw SerializationException("Unsupported type for deserialization of AlterDatabaseInfo!"); + } + return std::move(result); +} + void AddColumnInfo::Serialize(Serializer &serializer) const { AlterTableInfo::Serialize(serializer); serializer.WriteProperty(400, "new_column", new_column); @@ -482,6 +504,17 @@ unique_ptr RenameColumnInfo::Deserialize(Deserializer &deseriali return std::move(result); } +void RenameDatabaseInfo::Serialize(Serializer &serializer) const { + AlterDatabaseInfo::Serialize(serializer); + serializer.WritePropertyWithDefault(400, "new_name", new_name); +} + +unique_ptr RenameDatabaseInfo::Deserialize(Deserializer &deserializer) { + auto result = duckdb::unique_ptr(new RenameDatabaseInfo()); + deserializer.ReadPropertyWithDefault(400, "new_name", result->new_name); + return std::move(result); +} + void RenameFieldInfo::Serialize(Serializer &serializer) const { AlterTableInfo::Serialize(serializer); serializer.WritePropertyWithDefault>(400, "column_path", column_path); diff --git a/src/storage/serialization/serialize_query_node.cpp b/src/storage/serialization/serialize_query_node.cpp index 807d8da715d4..25b167558c57 100644 --- a/src/storage/serialization/serialize_query_node.cpp +++ b/src/storage/serialization/serialize_query_node.cpp @@ -38,6 +38,9 @@ unique_ptr QueryNode::Deserialize(Deserializer &deserializer) { } result->modifiers = std::move(modifiers); result->cte_map = std::move(cte_map); + if (type == QueryNodeType::CTE_NODE) { + result = std::move(result->Cast().child); + } return result; } @@ -111,10 +114,12 @@ unique_ptr SelectNode::Deserialize(Deserializer &deserializer) { void SetOperationNode::Serialize(Serializer &serializer) const { QueryNode::Serialize(serializer); serializer.WriteProperty(200, "setop_type", setop_type); - serializer.WritePropertyWithDefault>(201, "left", left); - serializer.WritePropertyWithDefault>(202, "right", right); + serializer.WritePropertyWithDefault>(201, "left", SerializeChildNode(serializer, 0)); + serializer.WritePropertyWithDefault>(202, "right", SerializeChildNode(serializer, 1)); serializer.WritePropertyWithDefault(203, "setop_all", setop_all, true); - serializer.WritePropertyWithDefault>>(204, "children", SerializeChildNodes()); + if (serializer.ShouldSerialize(7)) { + serializer.WritePropertyWithDefault>>(204, "children", children); + } } unique_ptr SetOperationNode::Deserialize(Deserializer &deserializer) { diff --git a/src/storage/serialization/serialize_storage.cpp b/src/storage/serialization/serialize_storage.cpp index 29ac3038c51e..a6964ed2b0d3 100644 --- a/src/storage/serialization/serialize_storage.cpp +++ b/src/storage/serialization/serialize_storage.cpp @@ -25,7 +25,9 @@ BlockPointer BlockPointer::Deserialize(Deserializer &deserializer) { } void DataPointer::Serialize(Serializer &serializer) const { - serializer.WritePropertyWithDefault(100, "row_start", row_start); + if (!serializer.ShouldSerialize(7)) { + serializer.WritePropertyWithDefault(100, "row_start", row_start); + } serializer.WritePropertyWithDefault(101, "tuple_count", tuple_count); serializer.WriteProperty(102, "block_pointer", block_pointer); serializer.WriteProperty(103, "compression_type", compression_type); @@ -34,13 +36,12 @@ void DataPointer::Serialize(Serializer &serializer) const { } DataPointer DataPointer::Deserialize(Deserializer &deserializer) { - auto row_start = deserializer.ReadPropertyWithDefault(100, "row_start"); + deserializer.ReadDeletedProperty(100, "row_start"); auto tuple_count = deserializer.ReadPropertyWithDefault(101, "tuple_count"); auto block_pointer = deserializer.ReadProperty(102, "block_pointer"); auto compression_type = deserializer.ReadProperty(103, "compression_type"); auto statistics = deserializer.ReadProperty(104, "statistics"); DataPointer result(std::move(statistics)); - result.row_start = row_start; result.tuple_count = tuple_count; result.block_pointer = block_pointer; result.compression_type = compression_type; diff --git a/src/storage/serialization/serialize_types.cpp b/src/storage/serialization/serialize_types.cpp index 453961009b8d..963d5646e408 100644 --- a/src/storage/serialization/serialize_types.cpp +++ b/src/storage/serialization/serialize_types.cpp @@ -42,6 +42,9 @@ shared_ptr ExtraTypeInfo::Deserialize(Deserializer &deserializer) case ExtraTypeInfoType::GENERIC_TYPE_INFO: result = make_shared_ptr(type); break; + case ExtraTypeInfoType::GEO_TYPE_INFO: + result = GeoTypeInfo::Deserialize(deserializer); + break; case ExtraTypeInfoType::INTEGER_LITERAL_TYPE_INFO: result = IntegerLiteralTypeInfo::Deserialize(deserializer); break; @@ -136,6 +139,15 @@ unique_ptr ExtensionTypeInfo::Deserialize(Deserializer &deser return result; } +void GeoTypeInfo::Serialize(Serializer &serializer) const { + ExtraTypeInfo::Serialize(serializer); +} + +shared_ptr GeoTypeInfo::Deserialize(Deserializer &deserializer) { + auto result = duckdb::shared_ptr(new GeoTypeInfo()); + return std::move(result); +} + void IntegerLiteralTypeInfo::Serialize(Serializer &serializer) const { ExtraTypeInfo::Serialize(serializer); serializer.WriteProperty(200, "constant_value", constant_value); diff --git a/src/storage/single_file_block_manager.cpp b/src/storage/single_file_block_manager.cpp index 9185c67925b0..6d22ff4237c1 100644 --- a/src/storage/single_file_block_manager.cpp +++ b/src/storage/single_file_block_manager.cpp @@ -76,11 +76,11 @@ void EncryptCanary(MainHeader &main_header, const shared_ptr &e uint8_t canary_buffer[MainHeader::CANARY_BYTE_SIZE]; // we zero-out the iv and the (not yet) encrypted canary - uint8_t iv[16]; - memset(iv, 0, sizeof(iv)); + uint8_t iv[MainHeader::AES_IV_LEN]; + memset(iv, 0, MainHeader::AES_IV_LEN); memset(canary_buffer, 0, MainHeader::CANARY_BYTE_SIZE); - encryption_state->InitializeEncryption(iv, MainHeader::AES_NONCE_LEN, derived_key, + encryption_state->InitializeEncryption(iv, MainHeader::AES_IV_LEN, derived_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); encryption_state->Process(reinterpret_cast(MainHeader::CANARY), MainHeader::CANARY_BYTE_SIZE, canary_buffer, MainHeader::CANARY_BYTE_SIZE); @@ -91,15 +91,15 @@ void EncryptCanary(MainHeader &main_header, const shared_ptr &e bool DecryptCanary(MainHeader &main_header, const shared_ptr &encryption_state, data_ptr_t derived_key) { // just zero-out the iv - uint8_t iv[16]; - memset(iv, 0, sizeof(iv)); + uint8_t iv[MainHeader::AES_IV_LEN]; + memset(iv, 0, MainHeader::AES_IV_LEN); //! allocate a buffer for the decrypted canary data_t decrypted_canary[MainHeader::CANARY_BYTE_SIZE]; memset(decrypted_canary, 0, MainHeader::CANARY_BYTE_SIZE); //! Decrypt the canary - encryption_state->InitializeDecryption(iv, MainHeader::AES_NONCE_LEN, derived_key, + encryption_state->InitializeDecryption(iv, MainHeader::AES_IV_LEN, derived_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); encryption_state->Process(main_header.GetEncryptedCanary(), MainHeader::CANARY_BYTE_SIZE, decrypted_canary, MainHeader::CANARY_BYTE_SIZE); @@ -123,7 +123,7 @@ void MainHeader::Write(WriteStream &ser) { SerializeVersionNumber(ser, DuckDB::SourceID()); // We always serialize, and write zeros, if not set. - auto encryption_enabled = flags[0] & MainHeader::ENCRYPTED_DATABASE_FLAG; + auto encryption_enabled = IsEncrypted(); SerializeEncryptionMetadata(ser, encryption_metadata, encryption_enabled); SerializeDBIdentifier(ser, db_identifier); SerializeEncryptionMetadata(ser, encrypted_canary, encryption_enabled); @@ -168,7 +168,7 @@ MainHeader MainHeader::Read(ReadStream &source) { "%lld.\n" "The database file was created with %s.\n\n" "Newer DuckDB version might introduce backward incompatible changes (possibly guarded by compatibility " - "settings)" + "settings).\n" "See the storage page for migration strategy and more information: https://duckdb.org/internals/storage", header.version_number, VERSION_NUMBER_LOWER, VERSION_NUMBER_UPPER, version_text); } @@ -295,11 +295,11 @@ MainHeader ConstructMainHeader(idx_t version_number) { return header; } -void SingleFileBlockManager::StoreEncryptedCanary(DatabaseInstance &db, MainHeader &main_header, const string &key_id) { - const_data_ptr_t key = EncryptionEngine::GetKeyFromCache(db, key_id); +void SingleFileBlockManager::StoreEncryptedCanary(AttachedDatabase &db, MainHeader &main_header, const string &key_id) { + const_data_ptr_t key = EncryptionEngine::GetKeyFromCache(db.GetDatabase(), key_id); // Encrypt canary with the derived key - auto encryption_state = - db.GetEncryptionUtil()->CreateEncryptionState(key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); + auto encryption_state = db.GetDatabase().GetEncryptionUtil()->CreateEncryptionState( + main_header.GetEncryptionCipher(), MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); EncryptCanary(main_header, encryption_state, key); } @@ -322,7 +322,7 @@ void SingleFileBlockManager::StoreEncryptionMetadata(MainHeader &main_header) co offset++; Store(options.encryption_options.additional_authenticated_data, offset); offset++; - Store(options.encryption_options.cipher, offset); + Store(db.GetStorageManager().GetCipher(), offset); offset += 2; Store(options.encryption_options.key_length, offset); @@ -341,7 +341,7 @@ void SingleFileBlockManager::CheckAndAddEncryptionKey(MainHeader &main_header, s EncryptionKeyManager::DeriveKey(user_key, db_identifier, derived_key); auto encryption_state = db.GetDatabase().GetEncryptionUtil()->CreateEncryptionState( - derived_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); + main_header.GetEncryptionCipher(), MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); if (!DecryptCanary(main_header, encryption_state, derived_key)) { throw IOException("Wrong encryption key used to open the database file"); } @@ -390,8 +390,13 @@ void SingleFileBlockManager::CreateNewDatabase(QueryContext context) { EncryptionKeyManager::DeriveKey(*options.encryption_options.user_key, options.db_identifier, derived_key); options.encryption_options.user_key = nullptr; + // if no encryption cipher is specified, use GCM + if (db.GetStorageManager().GetCipher() == EncryptionTypes::INVALID) { + db.GetStorageManager().SetCipher(EncryptionTypes::GCM); + } + // Set the encrypted DB bit to 1. - main_header.flags[0] |= MainHeader::ENCRYPTED_DATABASE_FLAG; + main_header.SetEncrypted(); // The derived key is wiped in AddKeyToCache. options.encryption_options.derived_key_id = EncryptionEngine::AddKeyToCache(db.GetDatabase(), derived_key); @@ -407,7 +412,7 @@ void SingleFileBlockManager::CreateNewDatabase(QueryContext context) { // Always store the database identifier. StoreDBIdentifier(main_header, options.db_identifier); if (encryption_enabled) { - StoreEncryptedCanary(db.GetDatabase(), main_header, options.encryption_options.derived_key_id); + StoreEncryptedCanary(db, main_header, options.encryption_options.derived_key_id); } // Write the main database header. @@ -491,6 +496,18 @@ void SingleFileBlockManager::LoadExistingDatabase(QueryContext context) { // if encrypted, but no encryption key given throw CatalogException("Cannot open encrypted database \"%s\" without a key", path); } + + // if a cipher was provided, check if it is the same as in the config + auto stored_cipher = main_header.GetEncryptionCipher(); + auto config_cipher = db.GetStorageManager().GetCipher(); + if (config_cipher != EncryptionTypes::INVALID && config_cipher != stored_cipher) { + throw CatalogException("Cannot open encrypted database \"%s\" with a different cipher (%s) than the one " + "used to create it (%s)", + path, EncryptionTypes::CipherToString(config_cipher), + EncryptionTypes::CipherToString(stored_cipher)); + } + // this is ugly, but the storage manager does not know the cipher type before + db.GetStorageManager().SetCipher(stored_cipher); } options.version_number = main_header.version_number; @@ -573,7 +590,7 @@ void SingleFileBlockManager::ReadAndChecksum(QueryContext context, FileBuffer &b if (options.encryption_options.encryption_enabled && !skip_block_header) { auto key_id = options.encryption_options.derived_key_id; - EncryptionEngine::DecryptBlock(db.GetDatabase(), key_id, block.InternalBuffer(), block.Size(), delta); + EncryptionEngine::DecryptBlock(db, key_id, block.InternalBuffer(), block.Size(), delta); } CheckChecksum(block, location, delta, skip_block_header); @@ -604,7 +621,7 @@ void SingleFileBlockManager::ChecksumAndWrite(QueryContext context, FileBuffer & auto key_id = options.encryption_options.derived_key_id; temp_buffer_manager = make_uniq(Allocator::Get(db), block.GetBufferType(), block.Size(), GetBlockHeaderSize()); - EncryptionEngine::EncryptBlock(db.GetDatabase(), key_id, block, *temp_buffer_manager, delta); + EncryptionEngine::EncryptBlock(db, key_id, block, *temp_buffer_manager, delta); temp_buffer_manager->Write(context, *handle, location); } else { block.Write(context, *handle, location); @@ -849,6 +866,20 @@ bool SingleFileBlockManager::IsRemote() { return !handle->OnDiskFile(); } +bool SingleFileBlockManager::Prefetch() { + switch (DBConfig::GetSetting(db.GetDatabase())) { + case StorageBlockPrefetch::NEVER: + return false; + case StorageBlockPrefetch::DEBUG_FORCE_ALWAYS: + case StorageBlockPrefetch::ALWAYS_PREFETCH: + return !InMemory(); + case StorageBlockPrefetch::REMOTE_ONLY: + return IsRemote(); + default: + throw InternalException("Unknown StorageBlockPrefetch type"); + } +} + unique_ptr SingleFileBlockManager::ConvertBlock(block_id_t block_id, FileBuffer &source_buffer) { D_ASSERT(source_buffer.AllocSize() == GetBlockAllocSize()); // FIXME; maybe we should pass the block header size explicitly @@ -876,8 +907,8 @@ void SingleFileBlockManager::ReadBlock(data_ptr_t internal_buffer, uint64_t bloc uint64_t delta = GetBlockHeaderSize() - Storage::DEFAULT_BLOCK_HEADER_SIZE; if (options.encryption_options.encryption_enabled && !skip_block_header) { - EncryptionEngine::DecryptBlock(db.GetDatabase(), options.encryption_options.derived_key_id, internal_buffer, - block_size, delta); + EncryptionEngine::DecryptBlock(db, options.encryption_options.derived_key_id, internal_buffer, block_size, + delta); } CheckChecksum(internal_buffer, delta, skip_block_header); @@ -892,8 +923,8 @@ void SingleFileBlockManager::ReadBlock(Block &block, bool skip_block_header) con uint64_t delta = GetBlockHeaderSize() - Storage::DEFAULT_BLOCK_HEADER_SIZE; if (options.encryption_options.encryption_enabled && !skip_block_header) { - EncryptionEngine::DecryptBlock(db.GetDatabase(), options.encryption_options.derived_key_id, - block.InternalBuffer(), block.Size(), delta); + EncryptionEngine::DecryptBlock(db, options.encryption_options.derived_key_id, block.InternalBuffer(), + block.Size(), delta); } CheckChecksum(block, location, delta, skip_block_header); diff --git a/src/storage/standard_buffer_manager.cpp b/src/storage/standard_buffer_manager.cpp index b2f079ba316f..7939982e9f4e 100644 --- a/src/storage/standard_buffer_manager.cpp +++ b/src/storage/standard_buffer_manager.cpp @@ -12,6 +12,7 @@ #include "duckdb/storage/temporary_file_manager.hpp" #include "duckdb/storage/temporary_memory_manager.hpp" #include "duckdb/common/encryption_functions.hpp" +#include "duckdb/main/settings.hpp" namespace duckdb { @@ -246,14 +247,14 @@ void StandardBufferManager::BatchRead(vector> &handles, block_id_t first_block, block_id_t last_block) { auto &block_manager = handles[0]->block_manager; idx_t block_count = NumericCast(last_block - first_block + 1); -#ifndef DUCKDB_ALTERNATIVE_VERIFY if (block_count == 1) { - // prefetching with block_count == 1 has no performance impact since we can't batch reads - // skip the prefetch in this case - // we do it anyway if alternative_verify is on for extra testing - return; + if (DBConfig::GetSetting(db) != StorageBlockPrefetch::DEBUG_FORCE_ALWAYS) { + // prefetching with block_count == 1 has no performance impact since we can't batch reads + // skip the prefetch in this case + // we do it anyway if alternative_verify is on for extra testing + return; + } } -#endif // allocate a buffer to hold the data of all of the blocks auto total_block_size = block_count * block_manager.GetBlockAllocSize(); @@ -337,7 +338,7 @@ BufferHandle StandardBufferManager::Pin(shared_ptr &handle) { return Pin(QueryContext(), handle); } -BufferHandle StandardBufferManager::Pin(QueryContext context, shared_ptr &handle) { +BufferHandle StandardBufferManager::Pin(const QueryContext &context, shared_ptr &handle) { // we need to be careful not to return the BufferHandle to this block while holding the BlockHandle's lock // as exiting this function's scope may cause the destructor of the BufferHandle to be called while holding the lock // the destructor calls Unpin, which grabs the BlockHandle's lock again, causing a deadlock @@ -409,9 +410,9 @@ void StandardBufferManager::VerifyZeroReaders(BlockLock &lock, shared_ptr replacement_buffer; auto &allocator = Allocator::Get(db); - auto block_header_size = handle->block_manager.GetBlockHeaderSize(); - auto alloc_size = handle->GetMemoryUsage() - block_header_size; auto &buffer = handle->GetBuffer(lock); + auto block_header_size = buffer->GetHeaderSize(); + auto alloc_size = buffer->AllocSize() - block_header_size; if (handle->GetBufferType() == FileBufferType::BLOCK) { auto block = reinterpret_cast(buffer.get()); replacement_buffer = make_uniq(allocator, block->id, alloc_size, block_header_size); @@ -542,8 +543,10 @@ unique_ptr StandardBufferManager::ReadTemporaryBuffer(QueryContext c BlockHandle &block, unique_ptr reusable_buffer) { D_ASSERT(!temporary_directory.path.empty()); - D_ASSERT(temporary_directory.handle.get()); auto id = block.BlockId(); + if (!temporary_directory.handle) { + throw InternalException("ReadTemporaryBuffer called but temporary directory has not been instantiated yet"); + } if (temporary_directory.handle->GetTempFile().HasTemporaryBuffer(id)) { // This is a block that was offloaded to a regular .tmp file, the file contains blocks of a fixed size return temporary_directory.handle->GetTempFile().ReadTemporaryBuffer(context, id, std::move(reusable_buffer)); @@ -641,6 +644,10 @@ bool StandardBufferManager::HasFilesInTemporaryDirectory() const { return found; } +BlockManager &StandardBufferManager::GetTemporaryBlockManager() { + return *temp_block_manager; +} + vector StandardBufferManager::GetTemporaryFiles() { vector result; if (temporary_directory.path.empty()) { diff --git a/src/storage/statistics/CMakeLists.txt b/src/storage/statistics/CMakeLists.txt index 1a746647e8f2..e4df62beff32 100644 --- a/src/storage/statistics/CMakeLists.txt +++ b/src/storage/statistics/CMakeLists.txt @@ -9,7 +9,8 @@ add_library_unity( numeric_stats.cpp segment_statistics.cpp string_stats.cpp - struct_stats.cpp) + struct_stats.cpp + geometry_stats.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/storage/statistics/base_statistics.cpp b/src/storage/statistics/base_statistics.cpp index 106599002cd3..9eac3b9aa888 100644 --- a/src/storage/statistics/base_statistics.cpp +++ b/src/storage/statistics/base_statistics.cpp @@ -62,6 +62,9 @@ StatisticsType BaseStatistics::GetStatsType(const LogicalType &type) { if (type.id() == LogicalTypeId::SQLNULL) { return StatisticsType::BASE_STATS; } + if (type.id() == LogicalTypeId::GEOMETRY) { + return StatisticsType::GEOMETRY_STATS; + } switch (type.InternalType()) { case PhysicalType::BOOL: case PhysicalType::INT8: @@ -153,6 +156,9 @@ void BaseStatistics::Merge(const BaseStatistics &other) { case StatisticsType::ARRAY_STATS: ArrayStats::Merge(*this, other); break; + case StatisticsType::GEOMETRY_STATS: + GeometryStats::Merge(*this, other); + break; default: break; } @@ -174,6 +180,8 @@ BaseStatistics BaseStatistics::CreateUnknownType(LogicalType type) { return StructStats::CreateUnknown(std::move(type)); case StatisticsType::ARRAY_STATS: return ArrayStats::CreateUnknown(std::move(type)); + case StatisticsType::GEOMETRY_STATS: + return GeometryStats::CreateUnknown(std::move(type)); default: return BaseStatistics(std::move(type)); } @@ -191,6 +199,8 @@ BaseStatistics BaseStatistics::CreateEmptyType(LogicalType type) { return StructStats::CreateEmpty(std::move(type)); case StatisticsType::ARRAY_STATS: return ArrayStats::CreateEmpty(std::move(type)); + case StatisticsType::GEOMETRY_STATS: + return GeometryStats::CreateEmpty(std::move(type)); default: return BaseStatistics(std::move(type)); } @@ -329,6 +339,9 @@ void BaseStatistics::Serialize(Serializer &serializer) const { case StatisticsType::ARRAY_STATS: ArrayStats::Serialize(*this, serializer); break; + case StatisticsType::GEOMETRY_STATS: + GeometryStats::Serialize(*this, serializer); + break; default: break; } @@ -367,6 +380,9 @@ BaseStatistics BaseStatistics::Deserialize(Deserializer &deserializer) { case StatisticsType::ARRAY_STATS: ArrayStats::Deserialize(obj, stats); break; + case StatisticsType::GEOMETRY_STATS: + GeometryStats::Deserialize(obj, stats); + break; default: break; } @@ -397,13 +413,16 @@ string BaseStatistics::ToString() const { case StatisticsType::ARRAY_STATS: result = ArrayStats::ToString(*this) + result; break; + case StatisticsType::GEOMETRY_STATS: + result = GeometryStats::ToString(*this) + result; + break; default: break; } return result; } -void BaseStatistics::Verify(Vector &vector, const SelectionVector &sel, idx_t count) const { +void BaseStatistics::Verify(Vector &vector, const SelectionVector &sel, idx_t count, const bool ignore_has_null) const { D_ASSERT(vector.GetType() == this->type); switch (GetStatsType()) { case StatisticsType::NUMERIC_STATS: @@ -421,6 +440,9 @@ void BaseStatistics::Verify(Vector &vector, const SelectionVector &sel, idx_t co case StatisticsType::ARRAY_STATS: ArrayStats::Verify(*this, vector, sel, count); break; + case StatisticsType::GEOMETRY_STATS: + GeometryStats::Verify(*this, vector, sel, count); + break; default: break; } @@ -439,7 +461,7 @@ void BaseStatistics::Verify(Vector &vector, const SelectionVector &sel, idx_t co "Statistics mismatch: vector labeled as having only NULL values, but vector contains valid values: %s", vector.ToString(count)); } - if (!row_is_valid && !has_null) { + if (!row_is_valid && !has_null && !ignore_has_null) { throw InternalException( "Statistics mismatch: vector labeled as not having NULL values, but vector contains null values: %s", vector.ToString(count)); @@ -449,7 +471,7 @@ void BaseStatistics::Verify(Vector &vector, const SelectionVector &sel, idx_t co void BaseStatistics::Verify(Vector &vector, idx_t count) const { auto sel = FlatVector::IncrementalSelectionVector(); - Verify(vector, *sel, count); + Verify(vector, *sel, count, false); } BaseStatistics BaseStatistics::FromConstantType(const Value &input) { @@ -505,6 +527,14 @@ BaseStatistics BaseStatistics::FromConstantType(const Value &input) { } return result; } + case StatisticsType::GEOMETRY_STATS: { + auto result = GeometryStats::CreateEmpty(input.type()); + if (!input.IsNull()) { + auto &string_value = StringValue::Get(input); + GeometryStats::Update(result, string_t(string_value)); + } + return result; + } default: return BaseStatistics(input.type()); } diff --git a/src/storage/statistics/geometry_stats.cpp b/src/storage/statistics/geometry_stats.cpp new file mode 100644 index 000000000000..42f5efbd843b --- /dev/null +++ b/src/storage/statistics/geometry_stats.cpp @@ -0,0 +1,171 @@ +#include "duckdb/storage/statistics/geometry_stats.hpp" +#include "duckdb/storage/statistics/base_statistics.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/vector.hpp" +#include "duckdb/common/serializer/serializer.hpp" +#include "duckdb/common/serializer/deserializer.hpp" + +namespace duckdb { + +vector GeometryTypeSet::ToString(bool snake_case) const { + vector result; + for (auto d = 0; d < VERT_TYPES; d++) { + for (auto i = 0; i < PART_TYPES; i++) { + if (sets[d] & (1 << i)) { + string str; + switch (i) { + case 1: + str = snake_case ? "point" : "Point"; + break; + case 2: + str = snake_case ? "linestring" : "LineString"; + break; + case 3: + str = snake_case ? "polygon" : "Polygon"; + break; + case 4: + str = snake_case ? "multipoint" : "MultiPoint"; + break; + case 5: + str = snake_case ? "multilinestring" : "MultiLineString"; + break; + case 6: + str = snake_case ? "multipolygon" : "MultiPolygon"; + break; + case 7: + str = snake_case ? "geometrycollection" : "GeometryCollection"; + break; + default: + str = snake_case ? "unknown" : "Unknown"; + break; + } + switch (d) { + case 1: + str += snake_case ? "_z" : " Z"; + break; + case 2: + str += snake_case ? "_m" : " M"; + break; + case 3: + str += snake_case ? "_zm" : " ZM"; + break; + default: + break; + } + + result.push_back(str); + } + } + } + return result; +} + +BaseStatistics GeometryStats::CreateUnknown(LogicalType type) { + BaseStatistics result(std::move(type)); + result.InitializeUnknown(); + GetDataUnsafe(result).SetUnknown(); + return result; +} + +BaseStatistics GeometryStats::CreateEmpty(LogicalType type) { + BaseStatistics result(std::move(type)); + result.InitializeEmpty(); + GetDataUnsafe(result).SetEmpty(); + return result; +} + +void GeometryStats::Serialize(const BaseStatistics &stats, Serializer &serializer) { + const auto &data = GetDataUnsafe(stats); + + // Write extent + serializer.WriteObject(200, "extent", [&](Serializer &extent) { + extent.WriteProperty(101, "x_min", data.extent.x_min); + extent.WriteProperty(102, "x_max", data.extent.x_max); + extent.WriteProperty(103, "y_min", data.extent.y_min); + extent.WriteProperty(104, "y_max", data.extent.y_max); + extent.WriteProperty(105, "z_min", data.extent.z_min); + extent.WriteProperty(106, "z_max", data.extent.z_max); + extent.WriteProperty(107, "m_min", data.extent.m_min); + extent.WriteProperty(108, "m_max", data.extent.m_max); + }); + + // Write types + serializer.WriteObject(201, "types", [&](Serializer &types) { + types.WriteProperty(101, "types_xy", data.types.sets[0]); + types.WriteProperty(102, "types_xyz", data.types.sets[1]); + types.WriteProperty(103, "types_xym", data.types.sets[2]); + types.WriteProperty(104, "types_xyzm", data.types.sets[3]); + }); +} + +void GeometryStats::Deserialize(Deserializer &deserializer, BaseStatistics &base) { + auto &data = GetDataUnsafe(base); + + // Read extent + deserializer.ReadObject(200, "extent", [&](Deserializer &extent) { + extent.ReadProperty(101, "x_min", data.extent.x_min); + extent.ReadProperty(102, "x_max", data.extent.x_max); + extent.ReadProperty(103, "y_min", data.extent.y_min); + extent.ReadProperty(104, "y_max", data.extent.y_max); + extent.ReadProperty(105, "z_min", data.extent.z_min); + extent.ReadProperty(106, "z_max", data.extent.z_max); + extent.ReadProperty(107, "m_min", data.extent.m_min); + extent.ReadProperty(108, "m_max", data.extent.m_max); + }); + + // Read types + deserializer.ReadObject(201, "types", [&](Deserializer &types) { + types.ReadProperty(101, "types_xy", data.types.sets[0]); + types.ReadProperty(102, "types_xyz", data.types.sets[1]); + types.ReadProperty(103, "types_xym", data.types.sets[2]); + types.ReadProperty(104, "types_xyzm", data.types.sets[3]); + }); +} + +string GeometryStats::ToString(const BaseStatistics &stats) { + const auto &data = GetDataUnsafe(stats); + string result; + + result += "["; + result += StringUtil::Format("Extent: [X: [%f, %f], Y: [%f, %f], Z: [%f, %f], M: [%f, %f]", data.extent.x_min, + data.extent.x_max, data.extent.y_min, data.extent.y_max, data.extent.z_min, + data.extent.z_max, data.extent.m_min, data.extent.m_max); + result += StringUtil::Format("], Types: [%s]", StringUtil::Join(data.types.ToString(true), ", ")); + + result += "]"; + return result; +} + +void GeometryStats::Update(BaseStatistics &stats, const string_t &value) { + auto &data = GetDataUnsafe(stats); + data.Update(value); +} + +void GeometryStats::Merge(BaseStatistics &stats, const BaseStatistics &other) { + if (other.GetType().id() == LogicalTypeId::VALIDITY) { + return; + } + if (other.GetType().id() == LogicalTypeId::SQLNULL) { + return; + } + + auto &target = GetDataUnsafe(stats); + auto &source = GetDataUnsafe(other); + target.Merge(source); +} + +void GeometryStats::Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count) { + // TODO: Verify stats +} + +const GeometryStatsData &GeometryStats::GetDataUnsafe(const BaseStatistics &stats) { + D_ASSERT(stats.GetStatsType() == StatisticsType::GEOMETRY_STATS); + return stats.stats_union.geometry_data; +} + +GeometryStatsData &GeometryStats::GetDataUnsafe(BaseStatistics &stats) { + D_ASSERT(stats.GetStatsType() == StatisticsType::GEOMETRY_STATS); + return stats.stats_union.geometry_data; +} + +} // namespace duckdb diff --git a/src/storage/statistics/string_stats.cpp b/src/storage/statistics/string_stats.cpp index e7d23269231a..3fe22ecac1c9 100644 --- a/src/storage/statistics/string_stats.cpp +++ b/src/storage/statistics/string_stats.cpp @@ -170,6 +170,14 @@ void StringStats::Update(BaseStatistics &stats, const string_t &value) { } } +void StringStats::SetMin(BaseStatistics &stats, const string_t &value) { + ConstructValue(const_data_ptr_cast(value.GetData()), value.GetSize(), GetDataUnsafe(stats).min); +} + +void StringStats::SetMax(BaseStatistics &stats, const string_t &value) { + ConstructValue(const_data_ptr_cast(value.GetData()), value.GetSize(), GetDataUnsafe(stats).max); +} + void StringStats::Merge(BaseStatistics &stats, const BaseStatistics &other) { if (other.GetType().id() == LogicalTypeId::VALIDITY) { return; diff --git a/src/storage/statistics/struct_stats.cpp b/src/storage/statistics/struct_stats.cpp index 64993e04e3bd..c981c7157496 100644 --- a/src/storage/statistics/struct_stats.cpp +++ b/src/storage/statistics/struct_stats.cpp @@ -132,7 +132,7 @@ string StructStats::ToString(const BaseStatistics &stats) { void StructStats::Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count) { auto &child_entries = StructVector::GetEntries(vector); for (idx_t i = 0; i < child_entries.size(); i++) { - stats.child_stats[i].Verify(*child_entries[i], sel, count); + stats.child_stats[i].Verify(*child_entries[i], sel, count, true); } } diff --git a/src/storage/storage_info.cpp b/src/storage/storage_info.cpp index b4a7422e95e0..df847fc9d58e 100644 --- a/src/storage/storage_info.cpp +++ b/src/storage/storage_info.cpp @@ -4,6 +4,10 @@ #include "duckdb/common/optional_idx.hpp" namespace duckdb { +constexpr idx_t Storage::MAX_ROW_GROUP_SIZE; +constexpr idx_t Storage::MAX_BLOCK_ALLOC_SIZE; +constexpr idx_t Storage::MIN_BLOCK_ALLOC_SIZE; +constexpr idx_t Storage::DEFAULT_BLOCK_HEADER_SIZE; const uint64_t VERSION_NUMBER = 64; const uint64_t VERSION_NUMBER_LOWER = 64; @@ -83,13 +87,14 @@ static const StorageVersionInfo storage_version_info[] = { {"v1.3.1", 66}, {"v1.3.2", 66}, {"v1.4.0", 67}, + {"v1.5.0", 67}, {nullptr, 0} }; // END OF STORAGE VERSION INFO static_assert(DEFAULT_STORAGE_VERSION_INFO == VERSION_NUMBER, "Check on VERSION_INFO"); // START OF SERIALIZATION VERSION INFO -const uint64_t LATEST_SERIALIZATION_VERSION_INFO = 6; +const uint64_t LATEST_SERIALIZATION_VERSION_INFO = 7; const uint64_t DEFAULT_SERIALIZATION_VERSION_INFO = 1; static const SerializationVersionInfo serialization_version_info[] = { {"v0.10.0", 1}, @@ -108,7 +113,8 @@ static const SerializationVersionInfo serialization_version_info[] = { {"v1.3.1", 5}, {"v1.3.2", 5}, {"v1.4.0", 6}, - {"latest", 6}, + {"v1.5.0", 7}, + {"latest", 7}, {nullptr, 0} }; // END OF SERIALIZATION VERSION INFO diff --git a/src/storage/storage_manager.cpp b/src/storage/storage_manager.cpp index 89e8fbd1fb7b..ebf0472e05b5 100644 --- a/src/storage/storage_manager.cpp +++ b/src/storage/storage_manager.cpp @@ -5,6 +5,7 @@ #include "duckdb/common/serializer/memory_stream.hpp" #include "duckdb/main/attached_database.hpp" #include "duckdb/main/client_context.hpp" +#include "duckdb/main/client_data.hpp" #include "duckdb/main/database.hpp" #include "duckdb/storage/checkpoint_manager.hpp" #include "duckdb/storage/in_memory_block_manager.hpp" @@ -13,6 +14,9 @@ #include "duckdb/storage/storage_extension.hpp" #include "duckdb/storage/table/column_data.hpp" #include "duckdb/storage/table/in_memory_checkpoint.hpp" +#include "duckdb/catalog/duck_catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp" #include "mbedtls_wrapper.hpp" namespace duckdb { @@ -39,12 +43,18 @@ void StorageOptions::Initialize(const unordered_map &options) { block_header_size = DEFAULT_ENCRYPTION_BLOCK_HEADER_SIZE; encryption = true; } else if (entry.first == "encryption_cipher") { - throw BinderException("\"%s\" is not a valid cipher. Only AES GCM is supported.", entry.second.ToString()); + auto parsed_cipher = EncryptionTypes::StringToCipher(entry.second.ToString()); + if (parsed_cipher != EncryptionTypes::CipherType::GCM && + parsed_cipher != EncryptionTypes::CipherType::CTR) { + throw BinderException("\"%s\" is not a valid cipher. Try 'GCM' or 'CTR'.", entry.second.ToString()); + } + encryption_cipher = parsed_cipher; } else if (entry.first == "row_group_size") { row_group_size = entry.second.GetValue(); } else if (entry.first == "storage_version") { storage_version_user_provided = entry.second.ToString(); - storage_version = SerializationCompatibility::FromString(entry.second.ToString()).serialization_version; + storage_version = + SerializationCompatibility::FromString(storage_version_user_provided).serialization_version; } else if (entry.first == "compress") { if (entry.second.DefaultCastAs(LogicalType::BOOLEAN).GetValue()) { compress_in_memory = CompressInMemory::COMPRESS; @@ -136,6 +146,9 @@ bool StorageManager::InMemory() const { return path == IN_MEMORY_PATH; } +void StorageManager::Destroy() { +} + void StorageManager::Initialize(QueryContext context) { bool in_memory = InMemory(); if (in_memory && read_only) { @@ -205,7 +218,6 @@ void SingleFileStorageManager::LoadDatabase(QueryContext context) { // key is given upon ATTACH D_ASSERT(storage_options.block_header_size == DEFAULT_ENCRYPTION_BLOCK_HEADER_SIZE); options.encryption_options.encryption_enabled = true; - options.encryption_options.cipher = EncryptionTypes::StringToCipher(storage_options.encryption_cipher); options.encryption_options.user_key = std::move(storage_options.user_key); } @@ -311,13 +323,40 @@ void SingleFileStorageManager::LoadDatabase(QueryContext context) { } } - // load the db from storage + // Start timing the storage load step. + auto client_context = context.GetClientContext(); + if (client_context) { + auto profiler = client_context->client_data->profiler; + profiler->StartTimer(MetricsType::ATTACH_LOAD_STORAGE_LATENCY); + } + + // Load the checkpoint from storage. auto checkpoint_reader = SingleFileCheckpointReader(*this); checkpoint_reader.LoadFromStorage(); + // End timing the storage load step. + if (client_context) { + auto profiler = client_context->client_data->profiler; + profiler->EndTimer(MetricsType::ATTACH_LOAD_STORAGE_LATENCY); + } + + // Start timing the WAL replay step. + if (client_context) { + auto profiler = client_context->client_data->profiler; + profiler->StartTimer(MetricsType::ATTACH_REPLAY_WAL_LATENCY); + } + + // Replay the WAL. auto wal_path = GetWALPath(); wal = WriteAheadLog::Replay(fs, db, wal_path); + + // End timing the WAL replay step. + if (client_context) { + auto profiler = client_context->client_data->profiler; + profiler->EndTimer(MetricsType::ATTACH_REPLAY_WAL_LATENCY); + } } + if (row_group_size > 122880ULL && GetStorageVersion() < 4) { throw InvalidInputException("Unsupported row group size %llu - row group sizes >= 122_880 are only supported " "with STORAGE_VERSION '1.2.0' or above.\nExplicitly specify a newer storage " @@ -465,17 +504,35 @@ void SingleFileStorageManager::CreateCheckpoint(QueryContext context, Checkpoint if (db.GetStorageExtension()) { db.GetStorageExtension()->OnCheckpointStart(db, options); } + auto &config = DBConfig::Get(db); + // We only need to checkpoint if there is anything in the WAL. if (GetWALSize() > 0 || config.options.force_checkpoint || options.action == CheckpointAction::ALWAYS_CHECKPOINT) { - // we only need to checkpoint if there is anything in the WAL try { + + // Start timing the checkpoint. + auto client_context = context.GetClientContext(); + if (client_context) { + auto profiler = client_context->client_data->profiler; + profiler->StartTimer(MetricsType::CHECKPOINT_LATENCY); + } + + // Write the checkpoint. auto checkpointer = CreateCheckpointWriter(context, options); checkpointer->CreateCheckpoint(); + + // End timing the checkpoint. + if (client_context) { + auto profiler = client_context->client_data->profiler; + profiler->EndTimer(MetricsType::CHECKPOINT_LATENCY); + } + } catch (std::exception &ex) { ErrorData error(ex); throw FatalException("Failed to create checkpoint because of error: %s", error.RawMessage()); } } + if (!InMemory() && options.wal_action == CheckpointWALAction::DELETE_WAL) { ResetWAL(); } @@ -485,6 +542,33 @@ void SingleFileStorageManager::CreateCheckpoint(QueryContext context, Checkpoint } } +void SingleFileStorageManager::Destroy() { + if (!load_complete) { + return; + } + vector> schemas; + // we scan the set of committed schemas + auto &catalog = Catalog::GetCatalog(db).Cast(); + catalog.ScanSchemas([&](SchemaCatalogEntry &entry) { schemas.push_back(entry); }); + + vector> tables; + for (auto &schema : schemas) { + schema.get().Scan(CatalogType::TABLE_ENTRY, [&](CatalogEntry &entry) { + if (entry.internal) { + return; + } + if (entry.type == CatalogType::TABLE_ENTRY) { + tables.push_back(entry.Cast()); + } + }); + } + + for (auto &table : tables) { + auto &data_table = table.get().GetStorage(); + data_table.Destroy(); + } +} + DatabaseSize SingleFileStorageManager::GetDatabaseSize() { // All members default to zero DatabaseSize ds; diff --git a/src/storage/table/array_column_data.cpp b/src/storage/table/array_column_data.cpp index 05964339a0f4..849e1dec86d7 100644 --- a/src/storage/table/array_column_data.cpp +++ b/src/storage/table/array_column_data.cpp @@ -97,7 +97,7 @@ idx_t ArrayColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t c void ArrayColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, SelectionVector &sel, idx_t sel_count) { - bool is_supported = !child_column->type.IsNested(); + bool is_supported = !child_column->type.IsNested() && child_column->type.InternalType() != PhysicalType::VARCHAR; if (!is_supported) { ColumnData::Select(transaction, vector_index, state, result, sel, sel_count); return; @@ -224,13 +224,14 @@ idx_t ArrayColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &resul throw NotImplementedException("Array Fetch"); } -void ArrayColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) { +void ArrayColumnData::Update(TransactionData transaction, DataTable &data_table, idx_t column_index, + Vector &update_vector, row_t *row_ids, idx_t update_count) { throw NotImplementedException("Array Update is not supported."); } -void ArrayColumnData::UpdateColumn(TransactionData transaction, const vector &column_path, - Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) { +void ArrayColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table, + const vector &column_path, Vector &update_vector, row_t *row_ids, + idx_t update_count, idx_t depth) { throw NotImplementedException("Array Update Column is not supported"); } @@ -256,7 +257,7 @@ void ArrayColumnData::FetchRow(TransactionData transaction, ColumnFetchState &st // We need to fetch between [row_id * array_size, (row_id + 1) * array_size) auto child_state = make_uniq(); - child_state->Initialize(child_type, nullptr); + child_state->Initialize(state.context, child_type, nullptr); const auto child_offset = start + (UnsafeNumericCast(row_id) - start) * array_size; @@ -302,8 +303,8 @@ unique_ptr ArrayColumnData::CreateCheckpointState(RowGrou unique_ptr ArrayColumnData::Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &checkpoint_info) { - - auto checkpoint_state = make_uniq(row_group, *this, checkpoint_info.info.manager); + auto &partial_block_manager = checkpoint_info.GetPartialBlockManager(); + auto checkpoint_state = make_uniq(row_group, *this, partial_block_manager); checkpoint_state->validity_state = validity.Checkpoint(row_group, checkpoint_info); checkpoint_state->child_state = child_column->Checkpoint(row_group, checkpoint_info); return std::move(checkpoint_state); @@ -332,12 +333,12 @@ void ArrayColumnData::InitializeColumn(PersistentColumnData &column_data, BaseSt this->count = validity.count.load(); } -void ArrayColumnData::GetColumnSegmentInfo(idx_t row_group_index, vector col_path, +void ArrayColumnData::GetColumnSegmentInfo(const QueryContext &context, idx_t row_group_index, vector col_path, vector &result) { col_path.push_back(0); - validity.GetColumnSegmentInfo(row_group_index, col_path, result); + validity.GetColumnSegmentInfo(context, row_group_index, col_path, result); col_path.back() = 1; - child_column->GetColumnSegmentInfo(row_group_index, col_path, result); + child_column->GetColumnSegmentInfo(context, row_group_index, col_path, result); } void ArrayColumnData::Verify(RowGroup &parent) { diff --git a/src/storage/table/chunk_info.cpp b/src/storage/table/chunk_info.cpp index 3b7b11d7be84..702b4beb6847 100644 --- a/src/storage/table/chunk_info.cpp +++ b/src/storage/table/chunk_info.cpp @@ -1,10 +1,12 @@ #include "duckdb/storage/table/chunk_info.hpp" + #include "duckdb/transaction/transaction.hpp" #include "duckdb/common/exception/transaction_exception.hpp" #include "duckdb/common/serializer/serializer.hpp" #include "duckdb/common/serializer/deserializer.hpp" #include "duckdb/common/serializer/memory_stream.hpp" #include "duckdb/transaction/delete_info.hpp" +#include "duckdb/execution/index/fixed_size_allocator.hpp" namespace duckdb { @@ -40,7 +42,7 @@ void ChunkInfo::Write(WriteStream &writer) const { writer.Write(type); } -unique_ptr ChunkInfo::Read(ReadStream &reader) { +unique_ptr ChunkInfo::Read(FixedSizeAllocator &allocator, ReadStream &reader) { auto type = reader.Read(); switch (type) { case ChunkInfoType::EMPTY_INFO: @@ -48,7 +50,7 @@ unique_ptr ChunkInfo::Read(ReadStream &reader) { case ChunkInfoType::CONSTANT_INFO: return ChunkConstantInfo::Read(reader); case ChunkInfoType::VECTOR_INFO: - return ChunkVectorInfo::Read(reader); + return ChunkVectorInfo::Read(allocator, reader); default: throw SerializationException("Could not deserialize Chunk Info Type: unrecognized type"); } @@ -71,7 +73,7 @@ idx_t ChunkConstantInfo::TemplatedGetSelVector(transaction_t start_time, transac return 0; } -idx_t ChunkConstantInfo::GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) { +idx_t ChunkConstantInfo::GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) const { return TemplatedGetSelVector(transaction.start_time, transaction.transaction_id, sel_vector, max_count); } @@ -95,7 +97,7 @@ bool ChunkConstantInfo::HasDeletes() const { return is_deleted; } -idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) { +idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) const { return delete_id < TRANSACTION_ID_START ? max_count : 0; } @@ -128,49 +130,70 @@ unique_ptr ChunkConstantInfo::Read(ReadStream &reader) { //===--------------------------------------------------------------------===// // Vector info //===--------------------------------------------------------------------===// -ChunkVectorInfo::ChunkVectorInfo(idx_t start) - : ChunkInfo(start, ChunkInfoType::VECTOR_INFO), insert_id(0), same_inserted_id(true), any_deleted(false) { - for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { - inserted[i] = 0; - deleted[i] = NOT_DELETED_ID; +ChunkVectorInfo::ChunkVectorInfo(FixedSizeAllocator &allocator_p, idx_t start, transaction_t insert_id_p) + : ChunkInfo(start, ChunkInfoType::VECTOR_INFO), allocator(allocator_p), constant_insert_id(insert_id_p) { +} + +ChunkVectorInfo::~ChunkVectorInfo() { + if (AnyDeleted()) { + allocator.Free(deleted_data); + } + if (!HasConstantInsertionId()) { + allocator.Free(inserted_data); } } template idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector, idx_t max_count) const { - idx_t count = 0; - if (same_inserted_id && !any_deleted) { - // all tuples have the same inserted id: and no tuples were deleted - if (OP::UseInsertedVersion(start_time, transaction_id, insert_id)) { - return max_count; - } else { - return 0; + if (HasConstantInsertionId()) { + if (!AnyDeleted()) { + // all tuples have the same inserted id: and no tuples were deleted + if (OP::UseInsertedVersion(start_time, transaction_id, ConstantInsertId())) { + return max_count; + } else { + return 0; + } } - } else if (same_inserted_id) { - if (!OP::UseInsertedVersion(start_time, transaction_id, insert_id)) { + if (!OP::UseInsertedVersion(start_time, transaction_id, ConstantInsertId())) { return 0; } // have to check deleted flag + idx_t count = 0; + auto segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = segment.GetPtr(); for (idx_t i = 0; i < max_count; i++) { if (OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { sel_vector.set_index(count++, i); } } - } else if (!any_deleted) { + return count; + } + if (!AnyDeleted()) { // have to check inserted flag + auto insert_segment = allocator.GetHandle(GetInsertedPointer()); + auto inserted = insert_segment.GetPtr(); + + idx_t count = 0; for (idx_t i = 0; i < max_count; i++) { if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { sel_vector.set_index(count++, i); } } - } else { - // have to check both flags - for (idx_t i = 0; i < max_count; i++) { - if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i]) && - OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { - sel_vector.set_index(count++, i); - } + return count; + } + + idx_t count = 0; + // have to check both flags + auto insert_segment = allocator.GetHandle(GetInsertedPointer()); + auto inserted = insert_segment.GetPtr(); + + auto delete_segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = delete_segment.GetPtr(); + for (idx_t i = 0; i < max_count; i++) { + if (OP::UseInsertedVersion(start_time, transaction_id, inserted[i]) && + OP::UseDeletedVersion(start_time, transaction_id, deleted[i])) { + sel_vector.set_index(count++, i); } } return count; @@ -186,16 +209,76 @@ idx_t ChunkVectorInfo::GetCommittedSelVector(transaction_t min_start_id, transac return TemplatedGetSelVector(min_start_id, min_transaction_id, sel_vector, max_count); } -idx_t ChunkVectorInfo::GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) { +idx_t ChunkVectorInfo::GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) const { return GetSelVector(transaction.start_time, transaction.transaction_id, sel_vector, max_count); } bool ChunkVectorInfo::Fetch(TransactionData transaction, row_t row) { - return UseVersion(transaction, inserted[row]) && !UseVersion(transaction, deleted[row]); + transaction_t fetch_insert_id; + transaction_t fetch_deleted_id; + if (HasConstantInsertionId()) { + fetch_insert_id = ConstantInsertId(); + } else { + auto insert_segment = allocator.GetHandle(GetInsertedPointer()); + auto inserted = insert_segment.GetPtr(); + fetch_insert_id = inserted[row]; + } + if (!AnyDeleted()) { + fetch_deleted_id = NOT_DELETED_ID; + } else { + auto delete_segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = delete_segment.GetPtr(); + fetch_deleted_id = deleted[row]; + } + + return UseVersion(transaction, fetch_insert_id) && !UseVersion(transaction, fetch_deleted_id); +} + +IndexPointer ChunkVectorInfo::GetInsertedPointer() const { + if (HasConstantInsertionId()) { + throw InternalException("ChunkVectorInfo: insert id requested but insertions were not initialized"); + } + return inserted_data; +} + +IndexPointer ChunkVectorInfo::GetDeletedPointer() const { + if (!AnyDeleted()) { + throw InternalException("ChunkVectorInfo: deleted id requested but deletions were not initialized"); + } + return deleted_data; +} + +IndexPointer ChunkVectorInfo::GetInitializedInsertedPointer() { + if (HasConstantInsertionId()) { + transaction_t constant_id = ConstantInsertId(); + + inserted_data = allocator.New(); + inserted_data.SetMetadata(1); + auto segment = allocator.GetHandle(inserted_data); + auto inserted = segment.GetPtr(); + for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { + inserted[i] = constant_id; + } + } + return inserted_data; +} + +IndexPointer ChunkVectorInfo::GetInitializedDeletedPointer() { + if (!AnyDeleted()) { + deleted_data = allocator.New(); + deleted_data.SetMetadata(1); + auto segment = allocator.GetHandle(deleted_data); + auto deleted = segment.GetPtr(); + for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { + deleted[i] = NOT_DELETED_ID; + } + } + return deleted_data; } idx_t ChunkVectorInfo::Delete(transaction_t transaction_id, row_t rows[], idx_t count) { - any_deleted = true; + auto segment = allocator.GetHandle(GetInitializedDeletedPointer()); + auto deleted = segment.GetPtr(); idx_t deleted_tuples = 0; for (idx_t i = 0; i < count; i++) { @@ -220,6 +303,9 @@ idx_t ChunkVectorInfo::Delete(transaction_t transaction_id, row_t rows[], idx_t } void ChunkVectorInfo::CommitDelete(transaction_t commit_id, const DeleteInfo &info) { + auto segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = segment.GetPtr(); + if (info.is_consecutive) { for (idx_t i = 0; i < info.count; i++) { deleted[i] = commit_id; @@ -234,32 +320,45 @@ void ChunkVectorInfo::CommitDelete(transaction_t commit_id, const DeleteInfo &in void ChunkVectorInfo::Append(idx_t start, idx_t end, transaction_t commit_id) { if (start == 0) { - insert_id = commit_id; - } else if (insert_id != commit_id) { - same_inserted_id = false; - insert_id = NOT_DELETED_ID; + // first insert to this vector - just assign the commit id + constant_insert_id = commit_id; + return; + } + if (HasConstantInsertionId() && ConstantInsertId() == commit_id) { + // we are inserting again, but we have the same id as before - still the same insert id + return; } + + auto segment = allocator.GetHandle(GetInitializedInsertedPointer()); + auto inserted = segment.GetPtr(); for (idx_t i = start; i < end; i++) { inserted[i] = commit_id; } } void ChunkVectorInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t end) { - if (same_inserted_id) { - insert_id = commit_id; + if (HasConstantInsertionId()) { + constant_insert_id = commit_id; + return; } + auto segment = allocator.GetHandle(GetInsertedPointer()); + auto inserted = segment.GetPtr(); + for (idx_t i = start; i < end; i++) { inserted[i] = commit_id; } } bool ChunkVectorInfo::Cleanup(transaction_t lowest_transaction, unique_ptr &result) const { - if (any_deleted) { + if (AnyDeleted()) { // if any rows are deleted we can't clean-up return false; } // check if the insertion markers have to be used by all transactions going forward - if (!same_inserted_id) { + if (!HasConstantInsertionId()) { + auto segment = allocator.GetHandle(GetInsertedPointer()); + auto inserted = segment.GetPtr(); + for (idx_t idx = 1; idx < STANDARD_VECTOR_SIZE; idx++) { if (inserted[idx] > lowest_transaction) { // transaction was inserted after the lowest transaction start @@ -267,7 +366,7 @@ bool ChunkVectorInfo::Cleanup(transaction_t lowest_transaction, unique_ptr lowest_transaction) { + } else if (ConstantInsertId() > lowest_transaction) { // transaction was inserted after the lowest transaction start // we still need to use an older version - cannot compress return false; @@ -276,13 +375,31 @@ bool ChunkVectorInfo::Cleanup(transaction_t lowest_transaction, unique_ptr(); + idx_t delete_count = 0; for (idx_t i = 0; i < max_count; i++) { if (deleted[i] < TRANSACTION_ID_START) { @@ -319,15 +436,17 @@ void ChunkVectorInfo::Write(WriteStream &writer) const { mask.Write(writer, STANDARD_VECTOR_SIZE); } -unique_ptr ChunkVectorInfo::Read(ReadStream &reader) { +unique_ptr ChunkVectorInfo::Read(FixedSizeAllocator &allocator, ReadStream &reader) { auto start = reader.Read(); - auto result = make_uniq(start); - result->any_deleted = true; + auto result = make_uniq(allocator, start); ValidityMask mask; mask.Read(reader, STANDARD_VECTOR_SIZE); + + auto segment = allocator.GetHandle(result->GetInitializedDeletedPointer()); + auto deleted = segment.GetPtr(); for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { if (mask.RowIsValid(i)) { - result->deleted[i] = 0; + deleted[i] = 0; } } return std::move(result); diff --git a/src/storage/table/column_data.cpp b/src/storage/table/column_data.cpp index 082f42f615e5..2b48d90cff11 100644 --- a/src/storage/table/column_data.cpp +++ b/src/storage/table/column_data.cpp @@ -293,13 +293,13 @@ void ColumnData::FetchUpdateRow(TransactionData transaction, row_t row_id, Vecto updates->FetchRow(transaction, NumericCast(row_id), result, result_idx); } -void ColumnData::UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count, Vector &base_vector) { +void ColumnData::UpdateInternal(TransactionData transaction, DataTable &data_table, idx_t column_index, + Vector &update_vector, row_t *row_ids, idx_t update_count, Vector &base_vector) { lock_guard update_guard(update_lock); if (!updates) { updates = make_uniq(*this); } - updates->Update(transaction, column_index, update_vector, row_ids, update_count, base_vector); + updates->Update(transaction, data_table, column_index, update_vector, row_ids, update_count, base_vector); } idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, @@ -517,30 +517,37 @@ void ColumnData::AppendData(BaseStatistics &append_stats, ColumnAppendState &sta } } -void ColumnData::RevertAppend(row_t start_row) { +void ColumnData::RevertAppend(row_t start_row_p) { + idx_t start_row = NumericCast(start_row_p); auto l = data.Lock(); // check if this row is in the segment tree at all auto last_segment = data.GetLastSegment(l); if (!last_segment) { return; } - if (NumericCast(start_row) >= last_segment->start + last_segment->count) { + if (start_row >= last_segment->start + last_segment->count) { // the start row is equal to the final portion of the column data: nothing was ever appended here - D_ASSERT(NumericCast(start_row) == last_segment->start + last_segment->count); + D_ASSERT(start_row == last_segment->start + last_segment->count); return; } // find the segment index that the current row belongs to - idx_t segment_index = data.GetSegmentIndex(l, UnsafeNumericCast(start_row)); + idx_t segment_index = data.GetSegmentIndex(l, start_row); auto segment = data.GetSegmentByIndex(l, UnsafeNumericCast(segment_index)); - auto &transient = *segment; - D_ASSERT(transient.segment_type == ColumnSegmentType::TRANSIENT); + if (segment->start == start_row) { + // we are truncating exactly this segment - erase it entirely + data.EraseSegments(l, segment_index); + } else { + // we need to truncate within the segment + // remove any segments AFTER this segment: they should be deleted entirely + data.EraseSegments(l, segment_index + 1); - // remove any segments AFTER this segment: they should be deleted entirely - data.EraseSegments(l, segment_index); + auto &transient = *segment; + D_ASSERT(transient.segment_type == ColumnSegmentType::TRANSIENT); + segment->next = nullptr; + transient.RevertAppend(start_row); + } - this->count = UnsafeNumericCast(start_row) - this->start; - segment->next = nullptr; - transient.RevertAppend(UnsafeNumericCast(start_row)); + this->count = start_row - this->start; } idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) { @@ -565,26 +572,26 @@ void ColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, FetchUpdateRow(transaction, row_id, result, result_idx); } -idx_t ColumnData::FetchUpdateData(row_t *row_ids, Vector &base_vector) { - ColumnScanState state; +idx_t ColumnData::FetchUpdateData(ColumnScanState &state, row_t *row_ids, Vector &base_vector) { auto fetch_count = ColumnData::Fetch(state, row_ids[0], base_vector); base_vector.Flatten(fetch_count); return fetch_count; } -void ColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) { +void ColumnData::Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector, + row_t *row_ids, idx_t update_count) { Vector base_vector(type); - FetchUpdateData(row_ids, base_vector); + ColumnScanState state; + FetchUpdateData(state, row_ids, base_vector); - UpdateInternal(transaction, column_index, update_vector, row_ids, update_count, base_vector); + UpdateInternal(transaction, data_table, column_index, update_vector, row_ids, update_count, base_vector); } -void ColumnData::UpdateColumn(TransactionData transaction, const vector &column_path, Vector &update_vector, - row_t *row_ids, idx_t update_count, idx_t depth) { +void ColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table, const vector &column_path, + Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) { // this method should only be called at the end of the path in the base column case D_ASSERT(depth >= column_path.size()); - ColumnData::Update(transaction, column_path[0], update_vector, row_ids, update_count); + ColumnData::Update(transaction, data_table, column_path[0], update_vector, row_ids, update_count); } void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) { @@ -666,7 +673,8 @@ void ColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state, unique_ptr ColumnData::Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &checkpoint_info) { // scan the segments of the column data // set up the checkpoint state - auto checkpoint_state = CreateCheckpointState(row_group, checkpoint_info.info.manager); + auto &partial_block_manager = checkpoint_info.GetPartialBlockManager(); + auto checkpoint_state = CreateCheckpointState(row_group, partial_block_manager); checkpoint_state->global_stats = BaseStatistics::CreateEmpty(type).ToUnique(); auto &nodes = data.ReferenceSegments(); @@ -692,6 +700,7 @@ void ColumnData::InitializeColumn(PersistentColumnData &column_data, BaseStatist this->count = 0; for (auto &data_pointer : column_data.pointers) { // Update the count and statistics + data_pointer.row_start = start + count; this->count += data_pointer.tuple_count; // Merge the statistics. If this is a child column, the target_stats reference will point into the parents stats @@ -866,6 +875,17 @@ PersistentColumnData ColumnData::Serialize() { return result; } +void RealignColumnData(PersistentColumnData &column_data, idx_t new_start) { + idx_t current_start = new_start; + for (auto &pointer : column_data.pointers) { + pointer.row_start = current_start; + current_start += pointer.tuple_count; + } + for (auto &child : column_data.child_columns) { + RealignColumnData(child, new_start); + } +} + shared_ptr ColumnData::Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row, ReadStream &source, const LogicalType &type) { auto entry = ColumnData::CreateColumn(block_manager, info, column_index, start_row, type, nullptr); @@ -883,12 +903,15 @@ shared_ptr ColumnData::Deserialize(BlockManager &block_manager, Data deserializer.Unset(); deserializer.End(); + // re-align data segments, in case our start_row has changed + RealignColumnData(persistent_column_data, start_row); + // initialize the column entry->InitializeColumn(persistent_column_data, entry->stats->statistics); return entry; } -void ColumnData::GetColumnSegmentInfo(idx_t row_group_index, vector col_path, +void ColumnData::GetColumnSegmentInfo(const QueryContext &context, idx_t row_group_index, vector col_path, vector &result) { D_ASSERT(!col_path.empty()); @@ -937,7 +960,7 @@ void ColumnData::GetColumnSegmentInfo(idx_t row_group_index, vector col_p column_info.additional_blocks = segment_state->GetAdditionalBlocks(); } if (compression_function.get_segment_info) { - auto segment_info = compression_function.get_segment_info(*segment); + auto segment_info = compression_function.get_segment_info(context, *segment); vector sinfo; for (auto &item : segment_info) { auto &mode = item.first; diff --git a/src/storage/table/column_data_checkpointer.cpp b/src/storage/table/column_data_checkpointer.cpp index 08362da3f7e3..68c35f8427cb 100644 --- a/src/storage/table/column_data_checkpointer.cpp +++ b/src/storage/table/column_data_checkpointer.cpp @@ -372,8 +372,28 @@ void ColumnDataCheckpointer::WritePersistentSegments(ColumnCheckpointState &stat auto &col_data = state.column_data; auto nodes = col_data.data.MoveSegments(); + idx_t current_row = row_group.start; for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) { auto segment = nodes[segment_idx].node.get(); + if (segment->start != current_row) { + string extra_info; + for (auto &s : nodes) { + extra_info += "\n"; + extra_info += StringUtil::Format("Start %d, count %d", s.node->start, s.node->count.load()); + } + const_reference root = col_data; + while (root.get().HasParent()) { + root = root.get().Parent(); + } + throw InternalException( + "Failure in RowGroup::Checkpoint - column data pointer is unaligned with row group " + "start\nRow group start: %d\nRow group count %d\nCurrent row: %d\nSegment start: %d\nColumn index: " + "%d\nColumn type: %s\nRoot type: %s\nTable: %s.%s\nAll segments:%s", + row_group.start, row_group.count.load(), current_row, segment->start, root.get().column_index, + col_data.type, root.get().type, root.get().info.GetSchemaName(), root.get().info.GetTableName(), + extra_info); + } + current_row += segment->count; auto pointer = segment->GetDataPointer(); // merge the persistent stats into the global column stats diff --git a/src/storage/table/column_segment.cpp b/src/storage/table/column_segment.cpp index 347463fbe5e5..6ee49be195c3 100644 --- a/src/storage/table/column_segment.cpp +++ b/src/storage/table/column_segment.cpp @@ -80,7 +80,6 @@ ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr block } ColumnSegment::ColumnSegment(ColumnSegment &other, const idx_t start) - : SegmentBase(start, other.count.load()), db(other.db), type(std::move(other.type)), type_size(other.type_size), segment_type(other.segment_type), stats(std::move(other.stats)), block(std::move(other.block)), function(other.function), block_id(other.block_id), offset(other.offset), @@ -109,7 +108,7 @@ void ColumnSegment::InitializePrefetch(PrefetchState &prefetch_state, ColumnScan } void ColumnSegment::InitializeScan(ColumnScanState &state) { - state.scan_state = function.get().init_scan(*this); + state.scan_state = function.get().init_scan(state.context, *this); } void ColumnSegment::Scan(ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset, diff --git a/src/storage/table/in_memory_checkpoint.cpp b/src/storage/table/in_memory_checkpoint.cpp index 911c86747062..31a0170f348e 100644 --- a/src/storage/table/in_memory_checkpoint.cpp +++ b/src/storage/table/in_memory_checkpoint.cpp @@ -89,8 +89,8 @@ InMemoryTableDataWriter::InMemoryTableDataWriter(InMemoryCheckpointer &checkpoin void InMemoryTableDataWriter::WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) { } -void InMemoryTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, - Serializer &serializer) { +void InMemoryTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, + RowGroupCollection &collection, Serializer &serializer) { // nop: no need to write anything } diff --git a/src/storage/table/list_column_data.cpp b/src/storage/table/list_column_data.cpp index 7685d16ca884..5672482c73a0 100644 --- a/src/storage/table/list_column_data.cpp +++ b/src/storage/table/list_column_data.cpp @@ -263,13 +263,14 @@ idx_t ListColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result throw NotImplementedException("List Fetch"); } -void ListColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) { +void ListColumnData::Update(TransactionData transaction, DataTable &data_table, idx_t column_index, + Vector &update_vector, row_t *row_ids, idx_t update_count) { throw NotImplementedException("List Update is not supported."); } -void ListColumnData::UpdateColumn(TransactionData transaction, const vector &column_path, - Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) { +void ListColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table, + const vector &column_path, Vector &update_vector, row_t *row_ids, + idx_t update_count, idx_t depth) { throw NotImplementedException("List Update Column is not supported"); } @@ -312,7 +313,7 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta auto &child_type = ListType::GetChildType(result.GetType()); Vector child_scan(child_type, child_scan_count); // seek the scan towards the specified position and read [length] entries - child_state->Initialize(child_type, nullptr); + child_state->Initialize(state.context, child_type, nullptr); child_column->InitializeScanWithOffset(*child_state, start + start_offset); D_ASSERT(child_type.InternalType() == PhysicalType::STRUCT || child_state->row_index + child_scan_count - this->start <= child_column->GetMaxEntry()); @@ -391,13 +392,13 @@ void ListColumnData::InitializeColumn(PersistentColumnData &column_data, BaseSta child_column->InitializeColumn(column_data.child_columns[1], child_stats); } -void ListColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector col_path, - vector &result) { - ColumnData::GetColumnSegmentInfo(row_group_index, col_path, result); +void ListColumnData::GetColumnSegmentInfo(const QueryContext &context, idx_t row_group_index, vector col_path, + vector &result) { + ColumnData::GetColumnSegmentInfo(context, row_group_index, col_path, result); col_path.push_back(0); - validity.GetColumnSegmentInfo(row_group_index, col_path, result); + validity.GetColumnSegmentInfo(context, row_group_index, col_path, result); col_path.back() = 1; - child_column->GetColumnSegmentInfo(row_group_index, col_path, result); + child_column->GetColumnSegmentInfo(context, row_group_index, col_path, result); } } // namespace duckdb diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 6637551f242c..169d5c24ed51 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -21,18 +21,19 @@ #include "duckdb/transaction/duck_transaction.hpp" #include "duckdb/transaction/duck_transaction_manager.hpp" #include "duckdb/storage/table/row_id_column_data.hpp" +#include "duckdb/main/settings.hpp" namespace duckdb { RowGroup::RowGroup(RowGroupCollection &collection_p, idx_t start, idx_t count) : SegmentBase(start, count), collection(collection_p), version_info(nullptr), allocation_size(0), - row_id_is_loaded(false) { + row_id_is_loaded(false), has_changes(false) { Verify(); } RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer) : SegmentBase(pointer.row_start, pointer.tuple_count), collection(collection_p), version_info(nullptr), - allocation_size(0), row_id_is_loaded(false) { + allocation_size(0), row_id_is_loaded(false), has_changes(false) { // deserialize the columns if (pointer.data_pointers.size() != collection_p.GetTypes().size()) { throw IOException("Row group column count is unaligned with table column count. Corrupt file?"); @@ -53,7 +54,7 @@ RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer) RowGroup::RowGroup(RowGroupCollection &collection_p, PersistentRowGroupData &data) : SegmentBase(data.start, data.count), collection(collection_p), version_info(nullptr), - allocation_size(0), row_id_is_loaded(false) { + allocation_size(0), row_id_is_loaded(false), has_changes(false) { auto &block_manager = GetBlockManager(); auto &info = GetTableInfo(); auto &types = collection.get().GetTypes(); @@ -69,6 +70,9 @@ RowGroup::RowGroup(RowGroupCollection &collection_p, PersistentRowGroupData &dat void RowGroup::MoveToCollection(RowGroupCollection &collection_p, idx_t new_start) { lock_guard l(row_group_lock); + if (start != new_start) { + has_changes = true; + } this->collection = collection_p; this->start = new_start; for (idx_t c = 0; c < columns.size(); c++) { @@ -179,10 +183,11 @@ void RowGroup::InitializeEmpty(const vector &types) { } } -void ColumnScanState::Initialize(const LogicalType &type, const vector &children, - optional_ptr options) { +void ColumnScanState::Initialize(const QueryContext &context_p, const LogicalType &type, + const vector &children, optional_ptr options) { // Register the options in the state scan_options = options; + context = context_p; if (type.id() == LogicalTypeId::VALIDITY) { // validity - nothing to initialize @@ -197,7 +202,7 @@ void ColumnScanState::Initialize(const LogicalType &type, const vector options) { +void ColumnScanState::Initialize(const QueryContext &context_p, const LogicalType &type, + optional_ptr options) { vector children; - Initialize(type, children, options); + Initialize(context_p, type, children, options); } -void CollectionScanState::Initialize(const vector &types) { +void CollectionScanState::Initialize(const QueryContext &context, const vector &types) { auto &column_ids = GetColumnIds(); column_scans = make_unsafe_uniq_array(column_ids.size()); for (idx_t i = 0; i < column_ids.size(); i++) { @@ -241,7 +247,7 @@ void CollectionScanState::Initialize(const vector &types) { continue; } auto col_id = column_ids[i].GetPrimaryIndex(); - column_scans[i].Initialize(types[col_id], column_ids[i].GetChildIndexes(), &GetOptions()); + column_scans[i].Initialize(context, types[col_id], column_ids[i].GetChildIndexes(), &GetOptions()); } } @@ -306,7 +312,7 @@ unique_ptr RowGroup::AlterType(RowGroupCollection &new_collection, con column_data->InitializeAppend(append_state); // scan the original table, and fill the new column with the transformed value - scan_state.Initialize(GetCollection().GetTypes()); + scan_state.Initialize(executor.GetContext(), GetCollection().GetTypes()); InitializeScan(scan_state); DataChunk append_chunk; @@ -547,14 +553,7 @@ void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &s count = max_count; } auto &block_manager = GetBlockManager(); -#ifndef DUCKDB_ALTERNATIVE_VERIFY - // // in regular operation we only prefetch from remote file systems - // // when alternative verify is set, we always prefetch for testing purposes - if (block_manager.IsRemote()) -#else - if (!block_manager.InMemory()) -#endif - { + if (block_manager.Prefetch()) { PrefetchState prefetch_state; for (idx_t i = 0; i < column_ids.size(); i++) { const auto &column = column_ids[i]; @@ -724,7 +723,8 @@ shared_ptr RowGroup::GetOrCreateVersionInfoInternal() { // version info does not exist - need to create it lock_guard lock(row_group_lock); if (!owned_version_info) { - auto new_info = make_shared_ptr(start); + auto &buffer_manager = GetBlockManager().GetBufferManager(); + auto new_info = make_shared_ptr(buffer_manager, start); SetVersionInfo(std::move(new_info)); } return owned_version_info; @@ -820,7 +820,7 @@ void RowGroup::CommitAppend(transaction_t commit_id, idx_t row_group_start, idx_ void RowGroup::RevertAppend(idx_t row_group_start) { auto &vinfo = GetOrCreateVersionInfo(); vinfo.RevertAppend(row_group_start - this->start); - for (auto &column : columns) { + for (auto &column : GetColumns()) { column->RevertAppend(UnsafeNumericCast(row_group_start)); } SetCount(MinValue(row_group_start - this->start, this->count)); @@ -855,8 +855,8 @@ void RowGroup::CleanupAppend(transaction_t lowest_transaction, idx_t start, idx_ vinfo.CleanupAppend(lowest_transaction, start, count); } -void RowGroup::Update(TransactionData transaction, DataChunk &update_chunk, row_t *ids, idx_t offset, idx_t count, - const vector &column_ids) { +void RowGroup::Update(TransactionData transaction, DataTable &data_table, DataChunk &update_chunk, row_t *ids, + idx_t offset, idx_t count, const vector &column_ids) { #ifdef DEBUG for (size_t i = offset; i < offset + count; i++) { D_ASSERT(ids[i] >= row_t(this->start) && ids[i] < row_t(this->start + this->count)); @@ -869,23 +869,29 @@ void RowGroup::Update(TransactionData transaction, DataChunk &update_chunk, row_ if (offset > 0) { Vector sliced_vector(update_chunk.data[i], offset, offset + count); sliced_vector.Flatten(count); - col_data.Update(transaction, column.index, sliced_vector, ids + offset, count); + col_data.Update(transaction, data_table, column.index, sliced_vector, ids + offset, count); } else { - col_data.Update(transaction, column.index, update_chunk.data[i], ids, count); + col_data.Update(transaction, data_table, column.index, update_chunk.data[i], ids, count); } MergeStatistics(column.index, *col_data.GetUpdateStatistics()); } } -void RowGroup::UpdateColumn(TransactionData transaction, DataChunk &updates, Vector &row_ids, - const vector &column_path) { +void RowGroup::UpdateColumn(TransactionData transaction, DataTable &data_table, DataChunk &updates, Vector &row_ids, + idx_t offset, idx_t count, const vector &column_path) { D_ASSERT(updates.ColumnCount() == 1); auto ids = FlatVector::GetData(row_ids); auto primary_column_idx = column_path[0]; D_ASSERT(primary_column_idx < columns.size()); auto &col_data = GetColumn(primary_column_idx); - col_data.UpdateColumn(transaction, column_path, updates.data[0], ids, updates.size(), 1); + if (offset > 0) { + Vector sliced_vector(updates.data[0], offset, offset + count); + sliced_vector.Flatten(count); + col_data.UpdateColumn(transaction, data_table, column_path, sliced_vector, ids + offset, count, 1); + } else { + col_data.UpdateColumn(transaction, data_table, column_path, updates.data[0], ids, count, 1); + } MergeStatistics(primary_column_idx, *col_data.GetUpdateStatistics()); } @@ -911,39 +917,92 @@ void RowGroup::MergeIntoStatistics(TableStatistics &other) { } } +ColumnCheckpointInfo::ColumnCheckpointInfo(RowGroupWriteInfo &info, idx_t column_idx) + : column_idx(column_idx), info(info) { +} + +RowGroupWriteInfo::RowGroupWriteInfo(PartialBlockManager &manager, const vector &compression_types, + CheckpointType checkpoint_type) + : manager(manager), compression_types(compression_types), checkpoint_type(checkpoint_type) { +} + +RowGroupWriteInfo::RowGroupWriteInfo(PartialBlockManager &manager, const vector &compression_types, + vector> &column_partial_block_managers_p) + : manager(manager), compression_types(compression_types), checkpoint_type(CheckpointType::FULL_CHECKPOINT), + column_partial_block_managers(column_partial_block_managers_p) { +} + +PartialBlockManager &RowGroupWriteInfo::GetPartialBlockManager(idx_t column_idx) { + if (column_partial_block_managers && !column_partial_block_managers->empty()) { + return *column_partial_block_managers->at(column_idx); + } + return manager; +} + +PartialBlockManager &ColumnCheckpointInfo::GetPartialBlockManager() { + return info.GetPartialBlockManager(column_idx); +} + CompressionType ColumnCheckpointInfo::GetCompressionType() { return info.compression_types[column_idx]; } -RowGroupWriteData RowGroup::WriteToDisk(RowGroupWriteInfo &info) { - RowGroupWriteData result; - result.states.reserve(columns.size()); - result.statistics.reserve(columns.size()); +vector RowGroup::WriteToDisk(RowGroupWriteInfo &info, + const vector> &row_groups) { + vector result; + if (row_groups.empty()) { + return result; + } - // Checkpoint the individual columns of the row group - // Here we're iterating over columns. Each column can have multiple segments. + idx_t column_count = row_groups[0].get().GetColumnCount(); + for (auto &row_group : row_groups) { + D_ASSERT(column_count == row_group.get().GetColumnCount()); + RowGroupWriteData write_data; + write_data.states.reserve(column_count); + write_data.statistics.reserve(column_count); + result.push_back(std::move(write_data)); + } + + // Checkpoint the row groups + // In order to co-locate columns across different row groups, we write column-at-a-time + // i.e. we first write column #0 of all row groups, then column #1, ... + + // Each column can have multiple segments. // (Some columns will be wider than others, and require different numbers // of blocks to encode.) Segments cannot span blocks. // // Some of these columns are composite (list, struct). The data is written // first sequentially, and the pointers are written later, so that the // pointers all end up densely packed, and thus more cache-friendly. - for (idx_t column_idx = 0; column_idx < GetColumnCount(); column_idx++) { - auto &column = GetColumn(column_idx); - ColumnCheckpointInfo checkpoint_info(info, column_idx); - auto checkpoint_state = column.Checkpoint(*this, checkpoint_info); - D_ASSERT(checkpoint_state); + for (idx_t column_idx = 0; column_idx < column_count; column_idx++) { + for (idx_t row_group_idx = 0; row_group_idx < row_groups.size(); row_group_idx++) { + auto &row_group = row_groups[row_group_idx].get(); + auto &row_group_write_data = result[row_group_idx]; + auto &column = row_group.GetColumn(column_idx); + if (column.start != row_group.start) { + throw InternalException("RowGroup::WriteToDisk - child-column is unaligned with row group"); + } + ColumnCheckpointInfo checkpoint_info(info, column_idx); + auto checkpoint_state = column.Checkpoint(row_group, checkpoint_info); + D_ASSERT(checkpoint_state); - auto stats = checkpoint_state->GetStatistics(); - D_ASSERT(stats); + auto stats = checkpoint_state->GetStatistics(); + D_ASSERT(stats); - result.statistics.push_back(stats->Copy()); - result.states.push_back(std::move(checkpoint_state)); + row_group_write_data.statistics.push_back(stats->Copy()); + row_group_write_data.states.push_back(std::move(checkpoint_state)); + } } - D_ASSERT(result.states.size() == result.statistics.size()); return result; } +RowGroupWriteData RowGroup::WriteToDisk(RowGroupWriteInfo &info) { + vector> row_groups; + row_groups.push_back(*this); + auto result = WriteToDisk(info, row_groups); + return std::move(result[0]); +} + idx_t RowGroup::GetCommittedRowCount() { auto vinfo = GetVersionInfo(); if (!vinfo) { @@ -996,7 +1055,8 @@ vector RowGroup::GetColumnPointers() { } RowGroupWriteData RowGroup::WriteToDisk(RowGroupWriter &writer) { - if (!column_pointers.empty() && !HasChanges()) { + if (DBConfig::GetSetting(writer.GetDatabase()) && !column_pointers.empty() && + !HasChanges()) { // we have existing metadata and the row group has not been changed // re-use previous metadata RowGroupWriteData result; @@ -1034,8 +1094,10 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWrite row_group_pointer.has_metadata_blocks = has_metadata_blocks; row_group_pointer.extra_metadata_blocks = extra_metadata_blocks; row_group_pointer.deletes_pointers = deletes_pointers; - metadata_manager->ClearModifiedBlocks(write_data.existing_pointers); - metadata_manager->ClearModifiedBlocks(deletes_pointers); + if (metadata_manager) { + metadata_manager->ClearModifiedBlocks(write_data.existing_pointers); + metadata_manager->ClearModifiedBlocks(deletes_pointers); + } return row_group_pointer; } D_ASSERT(write_data.states.size() == columns.size()); @@ -1079,6 +1141,11 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWrite // this metadata block is not stored - add it to the extra metadata blocks row_group_pointer.extra_metadata_blocks.push_back(column_pointer.block_pointer); } + // set up the pointers correctly within this row group for future operations + column_pointers = row_group_pointer.data_pointers; + has_metadata_blocks = true; + extra_metadata_blocks = row_group_pointer.extra_metadata_blocks; + if (metadata_manager) { row_group_pointer.deletes_pointers = CheckpointDeletes(*metadata_manager); } @@ -1087,6 +1154,9 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWrite } bool RowGroup::HasChanges() const { + if (has_changes) { + return true; + } if (version_info.load()) { // we have deletes return true; @@ -1181,10 +1251,11 @@ PartitionStatistics RowGroup::GetPartitionStats() const { //===--------------------------------------------------------------------===// // GetColumnSegmentInfo //===--------------------------------------------------------------------===// -void RowGroup::GetColumnSegmentInfo(idx_t row_group_index, vector &result) { +void RowGroup::GetColumnSegmentInfo(const QueryContext &context, idx_t row_group_index, + vector &result) { for (idx_t col_idx = 0; col_idx < GetColumnCount(); col_idx++) { auto &col_data = GetColumn(col_idx); - col_data.GetColumnSegmentInfo(row_group_index, {col_idx}, result); + col_data.GetColumnSegmentInfo(context, row_group_index, {col_idx}, result); } } diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index ae3531efcc1a..c0dc479ad842 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -17,6 +17,7 @@ #include "duckdb/storage/table/scan_state.hpp" #include "duckdb/storage/table_storage_info.hpp" #include "duckdb/main/settings.hpp" +#include "duckdb/transaction/duck_transaction.hpp" namespace duckdb { @@ -65,7 +66,7 @@ RowGroupCollection::RowGroupCollection(shared_ptr info_p, BlockMa vector types_p, idx_t row_start_p, idx_t total_rows_p, idx_t row_group_size_p) : block_manager(block_manager), row_group_size(row_group_size_p), total_rows(total_rows_p), info(std::move(info_p)), - types(std::move(types_p)), row_start(row_start_p), allocation_size(0) { + types(std::move(types_p)), row_start(row_start_p), allocation_size(0), requires_new_row_group(false) { row_groups = make_shared_ptr(*this); } @@ -101,6 +102,10 @@ void RowGroupCollection::Initialize(PersistentTableData &data) { metadata_pointer = data.base_table_pointer; } +void RowGroupCollection::FinalizeCheckpoint(MetaBlockPointer pointer) { + metadata_pointer = pointer; +} + void RowGroupCollection::Initialize(PersistentCollectionData &data) { stats.InitializeEmpty(types); auto l = row_groups->Lock(); @@ -112,6 +117,10 @@ void RowGroupCollection::Initialize(PersistentCollectionData &data) { } } +void RowGroupCollection::SetAppendRequiresNewRowGroup() { + requires_new_row_group = true; +} + void RowGroupCollection::InitializeEmpty() { stats.InitializeEmpty(types); } @@ -121,10 +130,11 @@ void RowGroupCollection::AppendRowGroup(SegmentLock &l, idx_t start_row) { auto new_row_group = make_uniq(*this, start_row, 0U); new_row_group->InitializeEmpty(types); row_groups->AppendSegment(l, std::move(new_row_group)); + requires_new_row_group = false; } -RowGroup *RowGroupCollection::GetRowGroup(int64_t index) { - return (RowGroup *)row_groups->GetSegmentByIndex(index); +optional_ptr RowGroupCollection::GetRowGroup(int64_t index) { + return row_groups->GetSegmentByIndex(index); } void RowGroupCollection::Verify() { @@ -144,13 +154,14 @@ void RowGroupCollection::Verify() { //===--------------------------------------------------------------------===// // Scan //===--------------------------------------------------------------------===// -void RowGroupCollection::InitializeScan(CollectionScanState &state, const vector &column_ids, +void RowGroupCollection::InitializeScan(const QueryContext &context, CollectionScanState &state, + const vector &column_ids, optional_ptr table_filters) { auto row_group = row_groups->GetRootSegment(); D_ASSERT(row_group); state.row_groups = row_groups.get(); state.max_row = row_start + total_rows; - state.Initialize(GetTypes()); + state.Initialize(context, GetTypes()); while (row_group && !row_group->InitializeScan(state)) { row_group = row_groups->GetNextSegment(row_group); } @@ -160,26 +171,28 @@ void RowGroupCollection::InitializeCreateIndexScan(CreateIndexScanState &state) state.segment_lock = row_groups->Lock(); } -void RowGroupCollection::InitializeScanWithOffset(CollectionScanState &state, const vector &column_ids, - idx_t start_row, idx_t end_row) { +void RowGroupCollection::InitializeScanWithOffset(const QueryContext &context, CollectionScanState &state, + const vector &column_ids, idx_t start_row, + idx_t end_row) { auto row_group = row_groups->GetSegment(start_row); D_ASSERT(row_group); state.row_groups = row_groups.get(); state.max_row = end_row; - state.Initialize(GetTypes()); + state.Initialize(context, GetTypes()); idx_t start_vector = (start_row - row_group->start) / STANDARD_VECTOR_SIZE; if (!row_group->InitializeScanWithOffset(state, start_vector)) { throw InternalException("Failed to initialize row group scan with offset"); } } -bool RowGroupCollection::InitializeScanInRowGroup(CollectionScanState &state, RowGroupCollection &collection, - RowGroup &row_group, idx_t vector_index, idx_t max_row) { +bool RowGroupCollection::InitializeScanInRowGroup(const QueryContext &context, CollectionScanState &state, + RowGroupCollection &collection, RowGroup &row_group, + idx_t vector_index, idx_t max_row) { state.max_row = max_row; state.row_groups = collection.row_groups.get(); if (!state.column_scans) { // initialize the scan state - state.Initialize(collection.GetTypes()); + state.Initialize(context, collection.GetTypes()); } return row_group.InitializeScanWithOffset(state, vector_index); } @@ -233,7 +246,8 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec D_ASSERT(row_group); // initialize the scan for this row group - bool need_to_scan = InitializeScanInRowGroup(scan_state, *collection, *row_group, vector_index, max_row); + bool need_to_scan = + InitializeScanInRowGroup(context, scan_state, *collection, *row_group, vector_index, max_row); if (!need_to_scan) { // skip this row group continue; @@ -257,7 +271,7 @@ bool RowGroupCollection::Scan(DuckTransaction &transaction, const vectorLock(); - if (IsEmpty(l)) { + if (IsEmpty(l) || requires_new_row_group) { // empty row group collection: empty first row group - AppendRowGroup(l, row_start); + AppendRowGroup(l, row_start + total_rows); } state.start_row_group = row_groups->GetLastSegment(l); D_ASSERT(this->row_start + total_rows == state.start_row_group->start + state.start_row_group->count); @@ -493,12 +507,17 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row) { segment_index = segment_count - 1; } auto &segment = *row_groups->GetSegmentByIndex(l, UnsafeNumericCast(segment_index)); - - // remove any segments AFTER this segment: they should be deleted entirely - row_groups->EraseSegments(l, segment_index); - - segment.next = nullptr; - segment.RevertAppend(start_row); + if (segment.start == start_row) { + // we are truncating exactly this row group - erase it entirely + row_groups->EraseSegments(l, segment_index); + } else { + // we need to truncate within a row group + // remove any segments AFTER this segment: they should be deleted entirely + row_groups->EraseSegments(l, segment_index + 1); + + segment.next = nullptr; + segment.RevertAppend(start_row); + } } void RowGroupCollection::CleanupAppend(transaction_t lowest_transaction, idx_t start, idx_t count) { @@ -608,31 +627,37 @@ idx_t RowGroupCollection::Delete(TransactionData transaction, DataTable &table, //===--------------------------------------------------------------------===// // Update //===--------------------------------------------------------------------===// -void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const vector &column_ids, - DataChunk &updates) { +optional_ptr RowGroupCollection::NextUpdateRowGroup(row_t *ids, idx_t &pos, idx_t count) const { + auto row_group = row_groups->GetSegment(UnsafeNumericCast(ids[pos])); + + row_t base_id = + UnsafeNumericCast(row_group->start + ((UnsafeNumericCast(ids[pos]) - row_group->start) / + STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE)); + auto max_id = + MinValue(base_id + STANDARD_VECTOR_SIZE, UnsafeNumericCast(row_group->start + row_group->count)); + for (pos++; pos < count; pos++) { + D_ASSERT(ids[pos] >= 0); + // check if this id still belongs to this vector in this row group + if (ids[pos] < base_id) { + // id is before vector start -> it does not + break; + } + if (ids[pos] >= max_id) { + // id is after the maximum id in this vector -> it does not + break; + } + } + return row_group; +} + +void RowGroupCollection::Update(TransactionData transaction, DataTable &data_table, row_t *ids, + const vector &column_ids, DataChunk &updates) { D_ASSERT(updates.size() >= 1); idx_t pos = 0; do { idx_t start = pos; - auto row_group = row_groups->GetSegment(UnsafeNumericCast(ids[pos])); - row_t base_id = - UnsafeNumericCast(row_group->start + ((UnsafeNumericCast(ids[pos]) - row_group->start) / - STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE)); - auto max_id = MinValue(base_id + STANDARD_VECTOR_SIZE, - UnsafeNumericCast(row_group->start + row_group->count)); - for (pos++; pos < updates.size(); pos++) { - D_ASSERT(ids[pos] >= 0); - // check if this id still belongs to this vector in this row group - if (ids[pos] < base_id) { - // id is before vector start -> it does not - break; - } - if (ids[pos] >= max_id) { - // id is after the maximum id in this vector -> it does not - break; - } - } - row_group->Update(transaction, updates, ids, start, pos - start, column_ids); + auto row_group = NextUpdateRowGroup(ids, pos, updates.size()); + row_group->Update(transaction, data_table, updates, ids, start, pos - start, column_ids); auto l = stats.GetLock(); for (idx_t i = 0; i < column_ids.size(); i++) { @@ -642,7 +667,8 @@ void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const v } while (pos < updates.size()); } -void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_identifiers, idx_t count) { +void RowGroupCollection::RemoveFromIndexes(const QueryContext &context, TableIndexList &indexes, + Vector &row_identifiers, idx_t count) { auto row_ids = FlatVector::GetData(row_identifiers); // Collect all indexed columns. @@ -697,7 +723,7 @@ void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_ auto base_row_id = row_group_vector_idx * STANDARD_VECTOR_SIZE + row_group->start; // Fetch the current vector into fetch_chunk. - state.table_state.Initialize(GetTypes()); + state.table_state.Initialize(context, GetTypes()); row_group->InitializeScanWithOffset(state.table_state, row_group_vector_idx); row_group->ScanCommitted(state.table_state, fetch_chunk, TableScanType::TABLE_SCAN_COMMITTED_ROWS); fetch_chunk.Verify(); @@ -744,19 +770,20 @@ void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_ } } -void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_ids, const vector &column_path, - DataChunk &updates) { - auto first_id = FlatVector::GetValue(row_ids, 0); - if (first_id >= MAX_ROW_ID) { - throw NotImplementedException("Cannot update a column-path on transaction local data"); - } - // find the row_group this id belongs to - auto primary_column_idx = column_path[0]; - auto row_group = row_groups->GetSegment(UnsafeNumericCast(first_id)); - row_group->UpdateColumn(transaction, updates, row_ids, column_path); +void RowGroupCollection::UpdateColumn(TransactionData transaction, DataTable &data_table, Vector &row_ids, + const vector &column_path, DataChunk &updates) { + D_ASSERT(updates.size() >= 1); + auto ids = FlatVector::GetData(row_ids); + idx_t pos = 0; + do { + idx_t start = pos; + auto row_group = NextUpdateRowGroup(ids, pos, updates.size()); + row_group->UpdateColumn(transaction, data_table, updates, row_ids, start, pos - start, column_path); - auto lock = stats.GetLock(); - row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(*lock, primary_column_idx).Statistics()); + auto lock = stats.GetLock(); + auto primary_column_idx = column_path[0]; + row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(*lock, primary_column_idx).Statistics()); + } while (pos < updates.size()); } //===--------------------------------------------------------------------===// @@ -866,7 +893,7 @@ class VacuumTask : public BaseCheckpointTask { TableScanState scan_state; scan_state.Initialize(column_ids); - scan_state.table_state.Initialize(types); + scan_state.table_state.Initialize(QueryContext(), types); scan_state.table_state.max_row = idx_t(-1); idx_t merged_groups = 0; idx_t total_row_groups = vacuum_state.row_group_counts.size(); @@ -987,13 +1014,13 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi } idx_t merge_rows; idx_t next_idx = 0; - idx_t merge_count; - idx_t target_count; + idx_t merge_count = 0; + idx_t target_count = 0; bool perform_merge = false; // check if we can merge row groups adjacent to the current segment_idx // we try merging row groups into batches of 1-3 row groups // our goal is to reduce the amount of row groups - // hence we target_count should be less than merge_count for a marge to be worth it + // hence we target_count should be less than merge_count for a merge to be worth it // we greedily prefer to merge to the lowest target_count // i.e. we prefer to merge 2 row groups into 1, than 3 row groups into 2 const idx_t row_group_size = GetRowGroupSize(); @@ -1013,6 +1040,22 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi merge_rows += state.row_group_counts[next_idx]; merge_count++; } + if (next_idx == checkpoint_state.segments.size()) { + // in order to prevent poor performance when performing small appends, we only merge row groups at the end + // if we can reach a "target" size of twice the current size, or the max row group size + // this is to prevent repeated expensive checkpoints where: + // we have a row group with 100K rows + // merge it with a row group with 1 row, creating a row group with 100K+1 rows + // merge it with a row group with 1 row, creating a row group with 100K+2 rows + // etc. This leads to constant rewriting of the original 100K rows. + idx_t minimum_target = + MinValue(state.row_group_counts[segment_idx] * 2, row_group_size) * target_count; + if (merge_rows >= STANDARD_VECTOR_SIZE && merge_rows < minimum_target) { + // we haven't reached the minimum target - don't do this vacuum + next_idx = segment_idx + 1; + continue; + } + } if (target_count < merge_count) { // we can reduce "merge_count" row groups to "target_count" // perform the merge at this level @@ -1024,6 +1067,8 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi return false; } // schedule the vacuum task + DUCKDB_LOG(checkpoint_state.writer.GetDatabase(), CheckpointLogType, GetAttached(), *info, segment_idx, merge_count, + target_count, merge_rows, state.row_start); auto vacuum_task = make_uniq(checkpoint_state, state, segment_idx, merge_count, target_count, merge_rows, state.row_start); checkpoint_state.executor->ScheduleTask(std::move(vacuum_task)); @@ -1070,6 +1115,8 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl // schedule a checkpoint task for this row group entry.node->MoveToCollection(*this, vacuum_state.row_start); if (writer.GetCheckpointType() != CheckpointType::VACUUM_ONLY) { + DUCKDB_LOG(checkpoint_state.writer.GetDatabase(), CheckpointLogType, GetAttached(), *info, segment_idx, + *entry.node); auto checkpoint_task = GetCheckpointTask(checkpoint_state, segment_idx); checkpoint_state.executor->ScheduleTask(std::move(checkpoint_task)); } @@ -1086,7 +1133,7 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl // no errors - finalize the row groups // if the table already exists on disk - check if all row groups have stayed the same - if (metadata_pointer.IsValid()) { + if (DBConfig::GetSetting(writer.GetDatabase()) && metadata_pointer.IsValid()) { bool table_has_changes = false; for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) { auto &entry = segments[segment_idx]; @@ -1144,6 +1191,38 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl new_total_rows += row_group.count; } total_rows = new_total_rows; + l.Release(); + Verify(); +} + +//===--------------------------------------------------------------------===// +// Destroy +//===--------------------------------------------------------------------===// + +class DestroyTask : public BaseExecutorTask { +public: + DestroyTask(TaskExecutor &executor, unique_ptr row_group_p) + : BaseExecutorTask(executor), row_group(std::move(row_group_p)) { + } + + void ExecuteTask() override { + row_group.reset(); + } + +private: + unique_ptr row_group; +}; + +void RowGroupCollection::Destroy() { + auto l = row_groups->Lock(); + auto &segments = row_groups->ReferenceLoadedSegmentsMutable(l); + + TaskExecutor executor(TaskScheduler::GetScheduler(GetAttached().GetDatabase())); + for (auto &segment : segments) { + auto destroy_task = make_uniq(executor, std::move(segment.node)); + executor.ScheduleTask(std::move(destroy_task)); + } + executor.WorkOnTasks(); } //===--------------------------------------------------------------------===// @@ -1175,10 +1254,11 @@ vector RowGroupCollection::GetPartitionStats() const { //===--------------------------------------------------------------------===// // GetColumnSegmentInfo //===--------------------------------------------------------------------===// -vector RowGroupCollection::GetColumnSegmentInfo() { +vector RowGroupCollection::GetColumnSegmentInfo(const QueryContext &context) { vector result; - for (auto &row_group : row_groups->Segments()) { - row_group.GetColumnSegmentInfo(row_group.index, result); + auto lock = row_groups->Lock(); + for (auto &row_group : row_groups->Segments(lock)) { + row_group.GetColumnSegmentInfo(context, row_group.index, result); } return result; } @@ -1275,7 +1355,8 @@ shared_ptr RowGroupCollection::AlterType(ClientContext &cont return result; } -void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint) { +void RowGroupCollection::VerifyNewConstraint(const QueryContext &context, DataTable &parent, + const BoundConstraint &constraint) { if (total_rows == 0) { return; } @@ -1297,7 +1378,7 @@ void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConst CreateIndexScanState state; auto scan_type = TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED; state.Initialize(column_ids, nullptr); - InitializeScan(state.table_state, column_ids, nullptr); + InitializeScan(context, state.table_state, column_ids, nullptr); InitializeCreateIndexScan(state); diff --git a/src/storage/table/row_id_column_data.cpp b/src/storage/table/row_id_column_data.cpp index d869913bf8da..4bc3c4148ded 100644 --- a/src/storage/table/row_id_column_data.cpp +++ b/src/storage/table/row_id_column_data.cpp @@ -138,13 +138,14 @@ void RowIdColumnData::RevertAppend(row_t start_row) { throw InternalException("RowIdColumnData cannot be appended to"); } -void RowIdColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) { +void RowIdColumnData::Update(TransactionData transaction, DataTable &data_table, idx_t column_index, + Vector &update_vector, row_t *row_ids, idx_t update_count) { throw InternalException("RowIdColumnData cannot be updated"); } -void RowIdColumnData::UpdateColumn(TransactionData transaction, const vector &column_path, - Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) { +void RowIdColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table, + const vector &column_path, Vector &update_vector, row_t *row_ids, + idx_t update_count, idx_t depth) { throw InternalException("RowIdColumnData cannot be updated"); } diff --git a/src/storage/table/row_version_manager.cpp b/src/storage/table/row_version_manager.cpp index df4e463da2d1..67ebbfb8653b 100644 --- a/src/storage/table/row_version_manager.cpp +++ b/src/storage/table/row_version_manager.cpp @@ -7,7 +7,10 @@ namespace duckdb { -RowVersionManager::RowVersionManager(idx_t start) noexcept : start(start), has_changes(false) { +RowVersionManager::RowVersionManager(BufferManager &buffer_manager_p, idx_t start) noexcept + : allocator(STANDARD_VECTOR_SIZE * sizeof(transaction_t), buffer_manager_p.GetTemporaryBlockManager(), + MemoryTag::BASE_TABLE), + start(start), has_changes(false) { } void RowVersionManager::SetStart(idx_t new_start) { @@ -112,7 +115,7 @@ void RowVersionManager::AppendVersionInfo(TransactionData transaction, idx_t cou optional_ptr new_info; if (!vector_info[vector_idx]) { // first time appending to this vector: create new info - auto insert_info = make_uniq(start + vector_idx * STANDARD_VECTOR_SIZE); + auto insert_info = make_uniq(allocator, start + vector_idx * STANDARD_VECTOR_SIZE); new_info = insert_info.get(); vector_info[vector_idx] = std::move(insert_info); } else if (vector_info[vector_idx]->type == ChunkInfoType::VECTOR_INFO) { @@ -188,15 +191,12 @@ ChunkVectorInfo &RowVersionManager::GetVectorInfo(idx_t vector_idx) { if (!vector_info[vector_idx]) { // no info yet: create it - vector_info[vector_idx] = make_uniq(start + vector_idx * STANDARD_VECTOR_SIZE); + vector_info[vector_idx] = make_uniq(allocator, start + vector_idx * STANDARD_VECTOR_SIZE); } else if (vector_info[vector_idx]->type == ChunkInfoType::CONSTANT_INFO) { auto &constant = vector_info[vector_idx]->Cast(); // info exists but it's a constant info: convert to a vector info - auto new_info = make_uniq(start + vector_idx * STANDARD_VECTOR_SIZE); - new_info->insert_id = constant.insert_id; - for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { - new_info->inserted[i] = constant.insert_id; - } + auto new_info = + make_uniq(allocator, start + vector_idx * STANDARD_VECTOR_SIZE, constant.insert_id); vector_info[vector_idx] = std::move(new_info); } D_ASSERT(vector_info[vector_idx]->type == ChunkInfoType::VECTOR_INFO); @@ -262,7 +262,7 @@ shared_ptr RowVersionManager::Deserialize(MetaBlockPointer de if (!delete_pointer.IsValid()) { return nullptr; } - auto version_info = make_shared_ptr(start); + auto version_info = make_shared_ptr(manager.GetBufferManager(), start); MetadataReader source(manager, delete_pointer, &version_info->storage_pointers); auto chunk_count = source.Read(); D_ASSERT(chunk_count > 0); @@ -275,7 +275,7 @@ shared_ptr RowVersionManager::Deserialize(MetaBlockPointer de } version_info->FillVectorInfo(vector_index); - version_info->vector_info[vector_index] = ChunkInfo::Read(source); + version_info->vector_info[vector_index] = ChunkInfo::Read(version_info->GetAllocator(), source); } version_info->has_changes = false; return version_info; diff --git a/src/storage/table/standard_column_data.cpp b/src/storage/table/standard_column_data.cpp index 266161a777d1..ad8814ab4732 100644 --- a/src/storage/table/standard_column_data.cpp +++ b/src/storage/table/standard_column_data.cpp @@ -152,27 +152,29 @@ idx_t StandardColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &re return scan_count; } -void StandardColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) { +void StandardColumnData::Update(TransactionData transaction, DataTable &data_table, idx_t column_index, + Vector &update_vector, row_t *row_ids, idx_t update_count) { + ColumnScanState standard_state, validity_state; Vector base_vector(type); - auto standard_fetch = FetchUpdateData(row_ids, base_vector); - auto validity_fetch = validity.FetchUpdateData(row_ids, base_vector); + auto standard_fetch = FetchUpdateData(standard_state, row_ids, base_vector); + auto validity_fetch = validity.FetchUpdateData(validity_state, row_ids, base_vector); if (standard_fetch != validity_fetch) { throw InternalException("Unaligned fetch in validity and main column data for update"); } - UpdateInternal(transaction, column_index, update_vector, row_ids, update_count, base_vector); - validity.UpdateInternal(transaction, column_index, update_vector, row_ids, update_count, base_vector); + UpdateInternal(transaction, data_table, column_index, update_vector, row_ids, update_count, base_vector); + validity.UpdateInternal(transaction, data_table, column_index, update_vector, row_ids, update_count, base_vector); } -void StandardColumnData::UpdateColumn(TransactionData transaction, const vector &column_path, - Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) { +void StandardColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table, + const vector &column_path, Vector &update_vector, row_t *row_ids, + idx_t update_count, idx_t depth) { if (depth >= column_path.size()) { // update this column - ColumnData::Update(transaction, column_path[0], update_vector, row_ids, update_count); + ColumnData::Update(transaction, data_table, column_path[0], update_vector, row_ids, update_count); } else { // update the child column (i.e. the validity column) - validity.UpdateColumn(transaction, column_path, update_vector, row_ids, update_count, depth + 1); + validity.UpdateColumn(transaction, data_table, column_path, update_vector, row_ids, update_count, depth + 1); } } @@ -240,9 +242,10 @@ unique_ptr StandardColumnData::Checkpoint(RowGroup &row_g // to prevent reading the validity data immediately after it is checkpointed we first checkpoint the main column // this is necessary for concurrent checkpointing as due to the partial block manager checkpointed data might be // flushed to disk by a different thread than the one that wrote it, causing a data race - auto base_state = CreateCheckpointState(row_group, checkpoint_info.info.manager); + auto &partial_block_manager = checkpoint_info.GetPartialBlockManager(); + auto base_state = CreateCheckpointState(row_group, partial_block_manager); base_state->global_stats = BaseStatistics::CreateEmpty(type).ToUnique(); - auto validity_state_p = validity.CreateCheckpointState(row_group, checkpoint_info.info.manager); + auto validity_state_p = validity.CreateCheckpointState(row_group, partial_block_manager); validity_state_p->global_stats = BaseStatistics::CreateEmpty(validity.type).ToUnique(); auto &validity_state = *validity_state_p; @@ -293,11 +296,12 @@ void StandardColumnData::InitializeColumn(PersistentColumnData &column_data, Bas validity.InitializeColumn(column_data.child_columns[0], target_stats); } -void StandardColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector col_path, +void StandardColumnData::GetColumnSegmentInfo(const QueryContext &context, duckdb::idx_t row_group_index, + vector col_path, vector &result) { - ColumnData::GetColumnSegmentInfo(row_group_index, col_path, result); + ColumnData::GetColumnSegmentInfo(context, row_group_index, col_path, result); col_path.push_back(0); - validity.GetColumnSegmentInfo(row_group_index, std::move(col_path), result); + validity.GetColumnSegmentInfo(context, row_group_index, std::move(col_path), result); } void StandardColumnData::Verify(RowGroup &parent) { diff --git a/src/storage/table/struct_column_data.cpp b/src/storage/table/struct_column_data.cpp index 5137330efd0a..b1de02b2d984 100644 --- a/src/storage/table/struct_column_data.cpp +++ b/src/storage/table/struct_column_data.cpp @@ -207,17 +207,18 @@ idx_t StructColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &resu return scan_count; } -void StructColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, - idx_t update_count) { - validity.Update(transaction, column_index, update_vector, row_ids, update_count); +void StructColumnData::Update(TransactionData transaction, DataTable &data_table, idx_t column_index, + Vector &update_vector, row_t *row_ids, idx_t update_count) { + validity.Update(transaction, data_table, column_index, update_vector, row_ids, update_count); auto &child_entries = StructVector::GetEntries(update_vector); for (idx_t i = 0; i < child_entries.size(); i++) { - sub_columns[i]->Update(transaction, column_index, *child_entries[i], row_ids, update_count); + sub_columns[i]->Update(transaction, data_table, column_index, *child_entries[i], row_ids, update_count); } } -void StructColumnData::UpdateColumn(TransactionData transaction, const vector &column_path, - Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) { +void StructColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table, + const vector &column_path, Vector &update_vector, row_t *row_ids, + idx_t update_count, idx_t depth) { // we can never DIRECTLY update a struct column if (depth >= column_path.size()) { throw InternalException("Attempting to directly update a struct column - this should not be possible"); @@ -225,13 +226,13 @@ void StructColumnData::UpdateColumn(TransactionData transaction, const vector sub_columns.size()) { throw InternalException("Update column_path out of range"); } - sub_columns[update_column - 1]->UpdateColumn(transaction, column_path, update_vector, row_ids, update_count, - depth + 1); + sub_columns[update_column - 1]->UpdateColumn(transaction, data_table, column_path, update_vector, row_ids, + update_count, depth + 1); } } @@ -311,7 +312,8 @@ unique_ptr StructColumnData::CreateCheckpointState(RowGro unique_ptr StructColumnData::Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &checkpoint_info) { - auto checkpoint_state = make_uniq(row_group, *this, checkpoint_info.info.manager); + auto &partial_block_manager = checkpoint_info.GetPartialBlockManager(); + auto checkpoint_state = make_uniq(row_group, *this, partial_block_manager); checkpoint_state->validity_state = validity.Checkpoint(row_group, checkpoint_info); for (auto &sub_column : sub_columns) { checkpoint_state->child_states.push_back(sub_column->Checkpoint(row_group, checkpoint_info)); @@ -361,13 +363,13 @@ void StructColumnData::InitializeColumn(PersistentColumnData &column_data, BaseS this->count = validity.count.load(); } -void StructColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector col_path, - vector &result) { +void StructColumnData::GetColumnSegmentInfo(const QueryContext &context, idx_t row_group_index, vector col_path, + vector &result) { col_path.push_back(0); - validity.GetColumnSegmentInfo(row_group_index, col_path, result); + validity.GetColumnSegmentInfo(context, row_group_index, col_path, result); for (idx_t i = 0; i < sub_columns.size(); i++) { col_path.back() = i + 1; - sub_columns[i]->GetColumnSegmentInfo(row_group_index, col_path, result); + sub_columns[i]->GetColumnSegmentInfo(context, row_group_index, col_path, result); } } diff --git a/src/storage/table/update_segment.cpp b/src/storage/table/update_segment.cpp index 8056907bc489..c47851ead687 100644 --- a/src/storage/table/update_segment.cpp +++ b/src/storage/table/update_segment.cpp @@ -7,6 +7,7 @@ #include "duckdb/transaction/duck_transaction.hpp" #include "duckdb/transaction/update_info.hpp" #include "duckdb/transaction/undo_buffer.hpp" +#include "duckdb/storage/data_table.hpp" #include @@ -104,9 +105,10 @@ idx_t UpdateInfo::GetAllocSize(idx_t type_size) { return AlignValue(sizeof(UpdateInfo) + (sizeof(sel_t) + type_size) * STANDARD_VECTOR_SIZE); } -void UpdateInfo::Initialize(UpdateInfo &info, transaction_t transaction_id) { +void UpdateInfo::Initialize(UpdateInfo &info, DataTable &data_table, transaction_t transaction_id) { info.max = STANDARD_VECTOR_SIZE; info.version_number = transaction_id; + info.table = &data_table; info.segment = nullptr; info.prev.entry = nullptr; info.next.entry = nullptr; @@ -1236,11 +1238,11 @@ static idx_t SortSelectionVector(SelectionVector &sel, idx_t count, row_t *ids) return pos; } -UpdateInfo *CreateEmptyUpdateInfo(TransactionData transaction, idx_t type_size, idx_t count, +UpdateInfo *CreateEmptyUpdateInfo(TransactionData transaction, DataTable &data_table, idx_t type_size, idx_t count, unsafe_unique_array &data) { data = make_unsafe_uniq_array_uninitialized(UpdateInfo::GetAllocSize(type_size)); auto update_info = reinterpret_cast(data.get()); - UpdateInfo::Initialize(*update_info, transaction.transaction_id); + UpdateInfo::Initialize(*update_info, data_table, transaction.transaction_id); return update_info; } @@ -1258,8 +1260,8 @@ void UpdateSegment::InitializeUpdateInfo(idx_t vector_idx) { } } -void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vector &update_p, row_t *ids, idx_t count, - Vector &base_data) { +void UpdateSegment::Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_p, + row_t *ids, idx_t count, Vector &base_data) { // obtain an exclusive lock auto write_lock = lock.GetExclusiveLock(); @@ -1322,10 +1324,10 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect // no updates made yet by this transaction: initially the update info to empty if (transaction.transaction) { auto &dtransaction = transaction.transaction->Cast(); - node_ref = dtransaction.CreateUpdateInfo(type_size, count); + node_ref = dtransaction.CreateUpdateInfo(type_size, data_table, count); node = &UpdateInfo::Get(node_ref); } else { - node = CreateEmptyUpdateInfo(transaction, type_size, count, update_info_data); + node = CreateEmptyUpdateInfo(transaction, data_table, type_size, count, update_info_data); } node->segment = this; node->vector_index = vector_index; @@ -1360,7 +1362,7 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect idx_t alloc_size = UpdateInfo::GetAllocSize(type_size); auto handle = root->allocator.Allocate(alloc_size); auto &update_info = UpdateInfo::Get(handle); - UpdateInfo::Initialize(update_info, TRANSACTION_ID_START - 1); + UpdateInfo::Initialize(update_info, data_table, TRANSACTION_ID_START - 1); update_info.column_index = column_index; InitializeUpdateInfo(update_info, ids, sel, count, vector_index, vector_offset); @@ -1370,10 +1372,10 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect UndoBufferReference node_ref; optional_ptr transaction_node; if (transaction.transaction) { - node_ref = transaction.transaction->CreateUpdateInfo(type_size, count); + node_ref = transaction.transaction->CreateUpdateInfo(type_size, data_table, count); transaction_node = &UpdateInfo::Get(node_ref); } else { - transaction_node = CreateEmptyUpdateInfo(transaction, type_size, count, update_info_data); + transaction_node = CreateEmptyUpdateInfo(transaction, data_table, type_size, count, update_info_data); } InitializeUpdateInfo(*transaction_node, ids, sel, count, vector_index, vector_offset); diff --git a/src/storage/version_map.json b/src/storage/version_map.json index 74a266dbeee0..c1768ab702da 100644 --- a/src/storage/version_map.json +++ b/src/storage/version_map.json @@ -55,7 +55,8 @@ "v1.3.0": 66, "v1.3.1": 66, "v1.3.2": 66, - "v1.4.0": 67 + "v1.4.0": 67, + "v1.5.0": 67 }, "default": 64 }, @@ -77,7 +78,8 @@ "v1.3.1": 5, "v1.3.2": 5, "v1.4.0": 6, - "latest": 6 + "v1.5.0": 7, + "latest": 7 }, "default": 1 } diff --git a/src/storage/wal_replay.cpp b/src/storage/wal_replay.cpp index fb81641b36ff..695db78dfd3d 100644 --- a/src/storage/wal_replay.cpp +++ b/src/storage/wal_replay.cpp @@ -46,6 +46,7 @@ class ReplayState { optional_ptr current_table; MetaBlockPointer checkpoint_id; idx_t wal_version = 1; + optional_idx expected_checkpoint_id; struct ReplayIndexInfo { ReplayIndexInfo(TableIndexList &index_list, unique_ptr index, const string &table_schema, @@ -141,9 +142,10 @@ class WriteAheadLogDeserializer { auto &keys = EncryptionKeyManager::Get(state_p.db.GetDatabase()); auto &catalog = state_p.db.GetCatalog().Cast(); auto derived_key = keys.GetKey(catalog.GetEncryptionKeyId()); + //! initialize the decryption auto encryption_state = database.GetEncryptionUtil()->CreateEncryptionState( - derived_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); + state_p.db.GetStorageManager().GetCipher(), MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); encryption_state->InitializeDecryption(nonce.data(), nonce.size(), derived_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); @@ -192,10 +194,12 @@ class WriteAheadLogDeserializer { return false; } - bool DeserializeOnly() { + bool DeserializeOnly() const { return deserialize_only; } + static void ThrowVersionError(idx_t checkpoint_iteration, idx_t expected_checkpoint_iteration); + protected: void ReplayEntry(WALType wal_type); @@ -246,6 +250,7 @@ class WriteAheadLogDeserializer { MemoryStream stream; BinaryDeserializer deserializer; bool deserialize_only; + optional_idx expected_checkpoint_id; }; //===--------------------------------------------------------------------===// @@ -311,6 +316,11 @@ unique_ptr WriteAheadLog::ReplayInternal(AttachedDatabase &databa return nullptr; } } + if (checkpoint_state.expected_checkpoint_id.IsValid()) { + // we expected a checkpoint id - but no checkpoint has happened - abort! + auto expected_id = checkpoint_state.expected_checkpoint_id.GetIndex(); + WriteAheadLogDeserializer::ThrowVersionError(expected_id - 1, expected_id); + } // we need to recover from the WAL: actually set up the replay state ReplayState state(database, *con.context); @@ -336,7 +346,7 @@ unique_ptr WriteAheadLog::ReplayInternal(AttachedDatabase &databa } state.replay_index_infos.clear(); - successful_offset = reader.offset; + successful_offset = reader.CurrentOffset(); // check if the file is exhausted if (reader.Finished()) { // we finished reading the file: break @@ -454,31 +464,45 @@ void WriteAheadLogDeserializer::ReplayEntry(WALType entry_type) { //===--------------------------------------------------------------------===// // Replay Version //===--------------------------------------------------------------------===// +void WriteAheadLogDeserializer::ThrowVersionError(idx_t checkpoint_iteration, idx_t expected_checkpoint_iteration) { + string relation = checkpoint_iteration < expected_checkpoint_iteration ? "an older" : "a newer"; + throw IOException("This WAL was created for this database file, but the WAL checkpoint iteration does not " + "match the database file. " + "That means the WAL was created for %s version of this database. File checkpoint " + "iteration: %d, WAL checkpoint iteration: %d", + relation, expected_checkpoint_iteration, checkpoint_iteration); +} + void WriteAheadLogDeserializer::ReplayVersion() { state.wal_version = deserializer.ReadProperty(101, "version"); auto &single_file_block_manager = db.GetStorageManager().GetBlockManager().Cast(); - auto file_version_number = single_file_block_manager.GetVersionNumber(); - if (file_version_number > 66) { - data_t db_identifier[MainHeader::DB_IDENTIFIER_LEN]; - deserializer.ReadList(102, "db_identifier", [&](Deserializer::List &list, idx_t i) { - db_identifier[i] = list.ReadElement(); - }); - auto expected_db_identifier = single_file_block_manager.GetDBIdentifier(); - if (!MainHeader::CompareDBIdentifiers(db_identifier, expected_db_identifier)) { - throw IOException("WAL does not match database file."); - } + data_t db_identifier[MainHeader::DB_IDENTIFIER_LEN]; + bool is_set = false; + deserializer.ReadOptionalList(102, "db_identifier", [&](Deserializer::List &list, idx_t i) { + db_identifier[i] = list.ReadElement(); + is_set = true; + }); + auto checkpoint_iteration = deserializer.ReadPropertyWithDefault(103, "checkpoint_iteration"); + if (!is_set || !checkpoint_iteration.IsValid()) { + return; + } + auto expected_db_identifier = single_file_block_manager.GetDBIdentifier(); + if (!MainHeader::CompareDBIdentifiers(db_identifier, expected_db_identifier)) { + throw IOException("WAL does not match database file."); + } - auto expected_checkpoint_iteration = single_file_block_manager.GetCheckpointIteration(); - auto checkpoint_iteration = deserializer.ReadProperty(103, "checkpoint_iteration"); - if (expected_checkpoint_iteration != checkpoint_iteration) { - string relation = checkpoint_iteration < expected_checkpoint_iteration ? "older" : "newer"; - throw IOException("This WAL was created for this database file, but the WAL checkpoint iteration does not " - "match the database file. " - "That means the WAL was created for a %s version of this database. File checkpoint " - "iteration: %d, WAL checkpoint iteration: %d", - relation, expected_checkpoint_iteration, checkpoint_iteration); + auto wal_checkpoint_iteration = checkpoint_iteration.GetIndex(); + auto expected_checkpoint_iteration = single_file_block_manager.GetCheckpointIteration(); + if (expected_checkpoint_iteration != wal_checkpoint_iteration) { + if (wal_checkpoint_iteration + 1 == expected_checkpoint_iteration) { + // this iteration is exactly one lower than the expected iteration + // this can happen if we aborted AFTER checkpointing the file, but BEFORE truncating the WAL + // expect this situation to occur - we will throw an error if it does not later on + state.expected_checkpoint_id = expected_checkpoint_iteration; + return; } + ThrowVersionError(wal_checkpoint_iteration, expected_checkpoint_iteration); } } @@ -549,7 +573,7 @@ void WriteAheadLogDeserializer::ReplayIndexData(IndexStorageInfo &info) { // Convert the buffer handle to a persistent block and store the block id. if (!deserialize_only) { auto block_id = block_manager->GetFreeBlockId(); - block_manager->ConvertToPersistent(QueryContext(context), block_id, std::move(block_handle), + block_manager->ConvertToPersistent(context, block_id, std::move(block_handle), std::move(buffer_handle)); data_info.block_pointers[j].block_id = block_id; } diff --git a/src/storage/write_ahead_log.cpp b/src/storage/write_ahead_log.cpp index f5be901daa8a..57689386a4ae 100644 --- a/src/storage/write_ahead_log.cpp +++ b/src/storage/write_ahead_log.cpp @@ -154,8 +154,9 @@ class ChecksumWriter : public WriteStream { auto &db = wal.GetDatabase(); auto &keys = EncryptionKeyManager::Get(db.GetDatabase()); + auto encryption_state = db.GetDatabase().GetEncryptionUtil()->CreateEncryptionState( - keys.GetKey(encryption_key_id), MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); + db.GetStorageManager().GetCipher(), MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH); // temp buffer const idx_t ciphertext_size = size + sizeof(uint64_t); diff --git a/src/transaction/cleanup_state.cpp b/src/transaction/cleanup_state.cpp index f9a17f2651bf..96483dbc9e00 100644 --- a/src/transaction/cleanup_state.cpp +++ b/src/transaction/cleanup_state.cpp @@ -13,7 +13,7 @@ namespace duckdb { -CleanupState::CleanupState(transaction_t lowest_active_transaction) +CleanupState::CleanupState(const QueryContext &context, transaction_t lowest_active_transaction) : lowest_active_transaction(lowest_active_transaction), current_table(nullptr), count(0) { } @@ -97,7 +97,7 @@ void CleanupState::Flush() { // delete the tuples from all the indexes try { - current_table->RemoveFromIndexes(row_identifiers, count); + current_table->RemoveFromIndexes(context, row_identifiers, count); } catch (...) { // NOLINT: ignore errors here } diff --git a/src/transaction/commit_state.cpp b/src/transaction/commit_state.cpp index 0f5d75bd231c..6eba8ab10988 100644 --- a/src/transaction/commit_state.cpp +++ b/src/transaction/commit_state.cpp @@ -165,6 +165,12 @@ void CommitState::CommitEntry(UndoFlags type, data_ptr_t data) { case UndoFlags::INSERT_TUPLE: { // append: auto info = reinterpret_cast(data); + if (!info->table->IsMainTable()) { + auto table_name = info->table->GetTableName(); + auto table_modification = info->table->TableModification(); + throw TransactionException("Attempting to modify table %s but another transaction has %s this table", + table_name, table_modification); + } // mark the tuples as committed info->table->CommitAppend(commit_id, info->start_row, info->count); break; @@ -172,6 +178,12 @@ void CommitState::CommitEntry(UndoFlags type, data_ptr_t data) { case UndoFlags::DELETE_TUPLE: { // deletion: auto info = reinterpret_cast(data); + if (!info->table->IsMainTable()) { + auto table_name = info->table->GetTableName(); + auto table_modification = info->table->TableModification(); + throw TransactionException("Attempting to modify table %s but another transaction has %s this table", + table_name, table_modification); + } // mark the tuples as committed info->version_info->CommitDelete(info->vector_idx, commit_id, *info); break; @@ -179,6 +191,12 @@ void CommitState::CommitEntry(UndoFlags type, data_ptr_t data) { case UndoFlags::UPDATE_TUPLE: { // update: auto info = reinterpret_cast(data); + if (!info->table->IsMainTable()) { + auto table_name = info->table->GetTableName(); + auto table_modification = info->table->TableModification(); + throw TransactionException("Attempting to modify table %s but another transaction has %s this table", + table_name, table_modification); + } info->version_number = commit_id; break; } diff --git a/src/transaction/duck_transaction.cpp b/src/transaction/duck_transaction.cpp index dc6afccb7674..562faa614d09 100644 --- a/src/transaction/duck_transaction.cpp +++ b/src/transaction/duck_transaction.cpp @@ -32,8 +32,8 @@ TransactionData::TransactionData(transaction_t transaction_id_p, transaction_t s DuckTransaction::DuckTransaction(DuckTransactionManager &manager, ClientContext &context_p, transaction_t start_time, transaction_t transaction_id, idx_t catalog_version_p) : Transaction(manager, context_p), start_time(start_time), transaction_id(transaction_id), commit_id(0), - highest_active_query(0), catalog_version(catalog_version_p), awaiting_cleanup(false), - transaction_manager(manager), undo_buffer(*this, context_p), storage(make_uniq(context_p, *this)) { + catalog_version(catalog_version_p), awaiting_cleanup(false), transaction_manager(manager), + undo_buffer(*this, context_p), storage(make_uniq(context_p, *this)) { } DuckTransaction::~DuckTransaction() { @@ -126,11 +126,11 @@ void DuckTransaction::PushAppend(DataTable &table, idx_t start_row, idx_t row_co append_info->count = row_count; } -UndoBufferReference DuckTransaction::CreateUpdateInfo(idx_t type_size, idx_t entries) { +UndoBufferReference DuckTransaction::CreateUpdateInfo(idx_t type_size, DataTable &data_table, idx_t entries) { idx_t alloc_size = UpdateInfo::GetAllocSize(type_size); auto undo_entry = undo_buffer.CreateEntry(UndoFlags::UPDATE_TUPLE, alloc_size); auto &update_info = UpdateInfo::Get(undo_entry); - UpdateInfo::Initialize(update_info, transaction_id); + UpdateInfo::Initialize(update_info, data_table, transaction_id); return undo_entry; } @@ -246,14 +246,6 @@ ErrorData DuckTransaction::Commit(AttachedDatabase &db, transaction_t new_commit // no need to flush anything if we made no changes return ErrorData(); } - for (auto &entry : modified_tables) { - auto &tbl = entry.first.get(); - if (!tbl.IsMainTable()) { - return ErrorData( - TransactionException("Attempting to modify table %s but another transaction has %s this table", - tbl.GetTableName(), tbl.TableModification())); - } - } D_ASSERT(db.IsSystem() || db.IsTemporary() || !IsReadOnly()); UndoBuffer::IteratorState iterator_state; diff --git a/src/transaction/duck_transaction_manager.cpp b/src/transaction/duck_transaction_manager.cpp index cf8dc7c8c58e..2dfe7cd6c3f6 100644 --- a/src/transaction/duck_transaction_manager.cpp +++ b/src/transaction/duck_transaction_manager.cpp @@ -123,38 +123,45 @@ DuckTransactionManager::CanCheckpoint(DuckTransaction &transaction, unique_ptrtransaction_id) + "]"; } - other_transactions += "[" + to_string(active_transaction->transaction_id) + "]"; } - } - if (!other_transactions.empty()) { - // there are other transactions! - // these active transactions might need data from BEFORE this transaction - // we might need to change our strategy here based on what changes THIS transaction has made - if (undo_properties.has_dropped_entries) { - // this transaction has changed the catalog - we cannot checkpoint - return CheckpointDecision("Transaction has dropped catalog entries and there are other transactions " - "active\nActive transactions: " + - other_transactions); - } else if (undo_properties.has_updates) { + if (!other_transactions.empty()) { + // there are other transactions! + // these active transactions might need data from BEFORE this transaction + // we might need to change our strategy here based on what changes THIS transaction has made + if (undo_properties.has_dropped_entries) { + // this transaction has changed the catalog - we cannot checkpoint + return CheckpointDecision( + "Transaction has dropped catalog entries and there are other transactions " + "active\nActive transactions: " + + other_transactions); + } // this transaction has performed updates - we cannot checkpoint return CheckpointDecision( "Transaction has performed updates and there are other transactions active\nActive transactions: " + other_transactions); - } else { - // this transaction has performed deletes - we cannot vacuum - initiate a concurrent checkpoint instead - D_ASSERT(undo_properties.has_deletes); - checkpoint_type = CheckpointType::CONCURRENT_CHECKPOINT; } } + // otherwise - we need to do a concurrent checkpoint + checkpoint_type = CheckpointType::CONCURRENT_CHECKPOINT; } if (storage_manager.InMemory() && !storage_manager.CompressionIsEnabled()) { if (checkpoint_type == CheckpointType::CONCURRENT_CHECKPOINT) { @@ -209,7 +216,7 @@ void DuckTransactionManager::Checkpoint(ClientContext &context, bool force) { options.type = CheckpointType::CONCURRENT_CHECKPOINT; } - storage_manager.CreateCheckpoint(QueryContext(context), options); + storage_manager.CreateCheckpoint(context, options); } unique_ptr DuckTransactionManager::SharedCheckpointLock() { @@ -317,7 +324,7 @@ ErrorData DuckTransactionManager::CommitTransaction(ClientContext &context, Tran } // We do not need to hold the transaction lock during cleanup of transactions, - // as they (1) have been removed, or (2) exited old_transactions. + // as they (1) have been removed, or (2) enter cleanup_info. t_lock.unlock(); { @@ -346,7 +353,7 @@ ErrorData DuckTransactionManager::CommitTransaction(ClientContext &context, Tran options.type = checkpoint_decision.type; auto &storage_manager = db.GetStorageManager(); try { - storage_manager.CreateCheckpoint(QueryContext(context), options); + storage_manager.CreateCheckpoint(context, options); } catch (std::exception &ex) { error.Merge(ErrorData(ex)); } @@ -405,7 +412,6 @@ unique_ptr DuckTransactionManager::RemoveTransaction(DuckTransa idx_t t_index = active_transactions.size(); auto lowest_start_time = TRANSACTION_ID_START; auto lowest_transaction_id = MAX_TRANSACTION_ID; - auto lowest_active_query = MAXIMUM_QUERY_ID; for (idx_t i = 0; i < active_transactions.size(); i++) { if (active_transactions[i].get() == &transaction) { t_index = i; @@ -413,17 +419,13 @@ unique_ptr DuckTransactionManager::RemoveTransaction(DuckTransa } lowest_start_time = MinValue(lowest_start_time, active_transactions[i]->start_time); lowest_transaction_id = MinValue(lowest_transaction_id, active_transactions[i]->transaction_id); - transaction_t active_query = active_transactions[i]->active_query; - lowest_active_query = MinValue(lowest_active_query, active_query); } lowest_active_start = lowest_start_time; lowest_active_id = lowest_transaction_id; - auto lowest_stored_query = lowest_start_time; D_ASSERT(t_index != active_transactions.size()); // Decide if we need to store the transaction, or if we can schedule it for cleanup. auto current_transaction = std::move(active_transactions[t_index]); - auto current_query = DatabaseManager::Get(db).ActiveQueryNumber(); if (store_transaction) { // If the transaction made any changes, we need to keep it around. if (transaction.commit_id != 0) { @@ -432,9 +434,7 @@ unique_ptr DuckTransactionManager::RemoveTransaction(DuckTransa recently_committed_transactions.push_back(std::move(current_transaction)); } else { // The transaction was aborted. - // We might still need its information; add it to the set of transactions awaiting GC. - current_transaction->highest_active_query = current_query; - old_transactions.push_back(std::move(current_transaction)); + cleanup_info->transactions.push_back(std::move(current_transaction)); } } else if (transaction.ChangesMade()) { // We do not need to store the transaction, directly schedule it for cleanup. @@ -451,7 +451,6 @@ unique_ptr DuckTransactionManager::RemoveTransaction(DuckTransa idx_t i = 0; for (; i < recently_committed_transactions.size(); i++) { D_ASSERT(recently_committed_transactions[i]); - lowest_stored_query = MinValue(recently_committed_transactions[i]->start_time, lowest_stored_query); if (recently_committed_transactions[i]->commit_id >= lowest_start_time) { // recently_committed_transactions is ordered on commit_id. // Thus, if the current commit_id is greater than @@ -459,18 +458,8 @@ unique_ptr DuckTransactionManager::RemoveTransaction(DuckTransa break; } - // Changes made BEFORE this transaction are no longer relevant. - // We can schedule the transaction and its undo buffer for cleanup. recently_committed_transactions[i]->awaiting_cleanup = true; - - // HOWEVER: Any currently running QUERY can still be using - // the version information of the transaction. - // If we remove the UndoBuffer immediately, we have a race condition. - - // Store the current highest active query. - recently_committed_transactions[i]->highest_active_query = current_query; - // Move it to the list of transactions awaiting GC. - old_transactions.push_back(std::move(recently_committed_transactions[i])); + cleanup_info->transactions.push_back(std::move(recently_committed_transactions[i])); } if (i > 0) { @@ -480,34 +469,6 @@ unique_ptr DuckTransactionManager::RemoveTransaction(DuckTransa recently_committed_transactions.erase(start, end); } - // Check if we can clean up and free the memory of any old transactions. - i = active_transactions.empty() ? old_transactions.size() : 0; - for (; i < old_transactions.size(); i++) { - D_ASSERT(old_transactions[i]); - D_ASSERT(old_transactions[i]->highest_active_query > 0); - if (old_transactions[i]->highest_active_query >= lowest_active_query) { - // There is still a query running that could be using - // this transactions' data. - break; - } - } - - if (i > 0) { - // We garbage-collected old transactions: - // - Remove them from the list and schedule them for cleanup. - - // We can only safely do the actual memory cleanup when all the - // currently active queries have finished running! (actually, - // when all the currently active scans have finished running...). - - // Because we clean up asynchronously, we only clean up once we - // no longer need the transaction for anything (i.e., we can move it). - for (idx_t t_idx = 0; t_idx < i; t_idx++) { - cleanup_info->transactions.push_back(std::move(old_transactions[t_idx])); - } - old_transactions.erase(old_transactions.begin(), old_transactions.begin() + static_cast(i)); - } - return cleanup_info; } diff --git a/src/transaction/meta_transaction.cpp b/src/transaction/meta_transaction.cpp index 62a181429ad3..6fee5d96bcf8 100644 --- a/src/transaction/meta_transaction.cpp +++ b/src/transaction/meta_transaction.cpp @@ -64,7 +64,8 @@ Transaction &MetaTransaction::GetTransaction(AttachedDatabase &db) { #endif all_transactions.push_back(db); transactions.insert(make_pair(reference(db), TransactionReference(new_transaction))); - referenced_databases.insert(make_pair(reference(db), db.shared_from_this())); + auto shared_db = db.shared_from_this(); + UseDatabase(shared_db); return new_transaction; } else { @@ -184,10 +185,42 @@ void MetaTransaction::SetActiveQuery(transaction_t query_number) { } } +optional_ptr MetaTransaction::GetReferencedDatabase(const string &name) { + lock_guard guard(referenced_database_lock); + auto entry = used_databases.find(name); + if (entry != used_databases.end()) { + return entry->second.get(); + } + return nullptr; +} + +shared_ptr MetaTransaction::GetReferencedDatabaseOwning(const string &name) { + lock_guard guard(referenced_database_lock); + for (auto &entry : referenced_databases) { + if (StringUtil::CIEquals(entry.first.get().name, name)) { + return entry.second; + } + } + return nullptr; +} + +void MetaTransaction::DetachDatabase(AttachedDatabase &database) { + lock_guard guard(referenced_database_lock); + used_databases.erase(database.GetName()); +} + AttachedDatabase &MetaTransaction::UseDatabase(shared_ptr &database) { auto &db_ref = *database; + lock_guard guard(referenced_database_lock); auto entry = referenced_databases.find(db_ref); if (entry == referenced_databases.end()) { + auto used_entry = used_databases.emplace(db_ref.GetName(), db_ref); + if (!used_entry.second) { + // return used_entry.first->second.get(); + throw InternalException( + "Database name %s was already used by a different database for this meta transaction", + db_ref.GetName()); + } referenced_databases.emplace(reference(db_ref), database); } return db_ref; diff --git a/src/transaction/undo_buffer.cpp b/src/transaction/undo_buffer.cpp index 4408a972b7b8..beec05ec9367 100644 --- a/src/transaction/undo_buffer.cpp +++ b/src/transaction/undo_buffer.cpp @@ -15,6 +15,7 @@ #include "duckdb/transaction/delete_info.hpp" #include "duckdb/transaction/rollback_state.hpp" #include "duckdb/transaction/wal_write_state.hpp" +#include "duckdb/transaction/duck_transaction.hpp" namespace duckdb { constexpr uint32_t UNDO_ENTRY_HEADER_SIZE = sizeof(UndoFlags) + sizeof(uint32_t); @@ -176,7 +177,7 @@ void UndoBuffer::Cleanup(transaction_t lowest_active_transaction) { // the chunks) // (2) there is no active transaction with start_id < commit_id of this // transaction - CleanupState state(lowest_active_transaction); + CleanupState state(QueryContext(), lowest_active_transaction); UndoBuffer::IteratorState iterator_state; IterateEntries(iterator_state, [&](UndoFlags type, data_ptr_t data) { state.CleanupEntry(type, data); }); diff --git a/src/transaction/wal_write_state.cpp b/src/transaction/wal_write_state.cpp index 5fe17e0502f9..0036ad0c69b8 100644 --- a/src/transaction/wal_write_state.cpp +++ b/src/transaction/wal_write_state.cpp @@ -27,10 +27,10 @@ WALWriteState::WALWriteState(DuckTransaction &transaction_p, WriteAheadLog &log, : transaction(transaction_p), log(log), commit_state(commit_state), current_table_info(nullptr) { } -void WALWriteState::SwitchTable(DataTableInfo *table_info, UndoFlags new_op) { - if (current_table_info != table_info) { +void WALWriteState::SwitchTable(DataTableInfo &table_info, UndoFlags new_op) { + if (current_table_info != &table_info) { // write the current table to the log - log.WriteSetTable(table_info->GetSchemaName(), table_info->GetTableName()); + log.WriteSetTable(table_info.GetSchemaName(), table_info.GetTableName()); current_table_info = table_info; } } @@ -171,7 +171,7 @@ void WALWriteState::WriteCatalogEntry(CatalogEntry &entry, data_ptr_t dataptr) { void WALWriteState::WriteDelete(DeleteInfo &info) { // switch to the current table, if necessary - SwitchTable(info.table->GetDataTableInfo().get(), UndoFlags::DELETE_TUPLE); + SwitchTable(*info.table->GetDataTableInfo(), UndoFlags::DELETE_TUPLE); if (!delete_chunk) { delete_chunk = make_uniq(); @@ -198,7 +198,7 @@ void WALWriteState::WriteUpdate(UpdateInfo &info) { auto &column_data = info.segment->column_data; auto &table_info = column_data.GetTableInfo(); - SwitchTable(&table_info, UndoFlags::UPDATE_TUPLE); + SwitchTable(table_info, UndoFlags::UPDATE_TUPLE); // initialize the update chunk vector update_types; diff --git a/src/verification/deserialized_statement_verifier.cpp b/src/verification/deserialized_statement_verifier.cpp index 345828ea5ca6..1ade815d70ce 100644 --- a/src/verification/deserialized_statement_verifier.cpp +++ b/src/verification/deserialized_statement_verifier.cpp @@ -17,7 +17,9 @@ DeserializedStatementVerifier::Create(const SQLStatement &statement, auto &select_stmt = statement.Cast(); Allocator allocator; MemoryStream stream(allocator); - BinarySerializer::Serialize(select_stmt, stream); + SerializationOptions options; + options.serialization_compatibility = SerializationCompatibility::FromString("latest"); + BinarySerializer::Serialize(select_stmt, stream, options); stream.Rewind(); auto result = BinaryDeserializer::Deserialize(stream); diff --git a/src/verification/statement_verifier.cpp b/src/verification/statement_verifier.cpp index 81f4c4aba28f..14e4c0491ed8 100644 --- a/src/verification/statement_verifier.cpp +++ b/src/verification/statement_verifier.cpp @@ -1,5 +1,9 @@ #include "duckdb/verification/statement_verifier.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/set_operation_node.hpp" +#include "duckdb/parser/query_node/cte_node.hpp" + #include "duckdb/common/error_data.hpp" #include "duckdb/common/types/column/column_data_collection.hpp" #include "duckdb/parser/parser.hpp" @@ -15,13 +19,24 @@ namespace duckdb { +const vector> &StatementVerifier::GetSelectList(QueryNode &node) { + switch (node.type) { + case QueryNodeType::SELECT_NODE: + return node.Cast().select_list; + case QueryNodeType::SET_OPERATION_NODE: + return GetSelectList(*node.Cast().children[0]); + default: + return empty_select_list; + } +} + StatementVerifier::StatementVerifier(VerificationType type, string name, unique_ptr statement_p, optional_ptr> parameters_p) : type(type), name(std::move(name)), statement(std::move(statement_p)), select_statement(statement->type == StatementType::SELECT_STATEMENT ? &statement->Cast() : nullptr), parameters(parameters_p), - select_list(select_statement ? select_statement->node->GetSelectList() : empty_select_list) { + select_list(select_statement ? GetSelectList(*select_statement->node) : empty_select_list) { } StatementVerifier::StatementVerifier(unique_ptr statement_p, diff --git a/test/api/adbc/test_adbc.cpp b/test/api/adbc/test_adbc.cpp index 2513b6f904a4..ad7a56581f9f 100644 --- a/test/api/adbc/test_adbc.cpp +++ b/test/api/adbc/test_adbc.cpp @@ -3,6 +3,7 @@ #include "duckdb/common/adbc/adbc.hpp" #include "duckdb/common/adbc/wrappers.hpp" #include "duckdb/common/adbc/options.h" +#include "duckdb/common/arrow/nanoarrow/nanoarrow.hpp" #include namespace duckdb { @@ -213,6 +214,79 @@ TEST_CASE("ADBC - Test ingestion - Quoted Table and Schema", "[adbc]") { arrow_schema.release(&arrow_schema); } +TEST_CASE("ADBC - Test Ingestion - Funky identifiers", "[adbc]") { + if (!duckdb_lib) { + return; + } + ADBCTestDatabase db; + std::vector column_names = {"column.with.dots", "column\"with\"quotes", "column with whitespace"}; + std::string table_name = "my.table\"is 😵"; + std::string schema_name = "my schema\"is.😬"; + // Create a schema and allocate space for a single child + ArrowSchema arrow_schema; + duckdb_nanoarrow::ArrowSchemaInit(&arrow_schema, duckdb_nanoarrow::ArrowType::NANOARROW_TYPE_STRUCT); + duckdb_nanoarrow::ArrowSchemaAllocateChildren(&arrow_schema, static_cast(column_names.size())); + + // Create the child schemas and build the select query to get test data + std::string query = "SELECT "; + for (size_t i = 0; i < column_names.size(); i++) { + duckdb_nanoarrow::ArrowSchemaInit(arrow_schema.children[i], duckdb_nanoarrow::ArrowType::NANOARROW_TYPE_INT32); + duckdb_nanoarrow::ArrowSchemaSetName(arrow_schema.children[i], column_names.at(i).c_str()); + if (i > 0) { + query += ","; + } + query += std::to_string(i); + } + + // Create input data + auto &input_data = db.QueryArrow(query); + ArrowArray prepared_array; + input_data.get_next(&input_data, &prepared_array); + + // Create the schema + db.Query("CREATE SCHEMA " + KeywordHelper::WriteOptionallyQuoted(schema_name)); + + // Create ADBC statement that will create a table called "test" + AdbcStatement adbc_stmt; + AdbcError adbc_error; + InitializeADBCError(&adbc_error); + REQUIRE(SUCCESS(AdbcStatementNew(&db.adbc_connection, &adbc_stmt, &adbc_error))); + REQUIRE(SUCCESS( + AdbcStatementSetOption(&adbc_stmt, ADBC_INGEST_OPTION_MODE, ADBC_INGEST_OPTION_MODE_CREATE, &adbc_error))); + REQUIRE(SUCCESS( + AdbcStatementSetOption(&adbc_stmt, ADBC_INGEST_OPTION_TARGET_DB_SCHEMA, schema_name.c_str(), &adbc_error))); + REQUIRE( + SUCCESS(AdbcStatementSetOption(&adbc_stmt, ADBC_INGEST_OPTION_TARGET_TABLE, table_name.c_str(), &adbc_error))); + REQUIRE(SUCCESS(AdbcStatementBind(&adbc_stmt, &prepared_array, &arrow_schema, &adbc_error))); + + // Now execute: this should create the table and put the test data in + REQUIRE(SUCCESS(AdbcStatementExecuteQuery(&adbc_stmt, nullptr, nullptr, &adbc_error))); + + // Release everything + REQUIRE(SUCCESS(AdbcStatementRelease(&adbc_stmt, &adbc_error))); + if (arrow_schema.release) { + arrow_schema.release(&arrow_schema); + } + if (adbc_error.release) { + adbc_error.release(&adbc_error); + } + if (input_data.release) { + input_data.release(&input_data); + } + if (prepared_array.release) { + prepared_array.release(&prepared_array); + } + + // Check we can query + auto schema_table = + KeywordHelper::WriteOptionallyQuoted(schema_name) + "." + KeywordHelper::WriteOptionallyQuoted(table_name); + auto res = db.Query("select * from " + schema_table); + for (size_t i = 0; i < column_names.size(); i++) { + REQUIRE((res->ColumnName(i) == column_names.at(i))); + REQUIRE((res->GetValue(i, 0) == i)); + } +} + TEST_CASE("ADBC - Test ingestion - Lineitem", "[adbc]") { if (!duckdb_lib) { return; diff --git a/test/api/capi/CMakeLists.txt b/test/api/capi/CMakeLists.txt index 7f9eaea41a3e..7928e312ede7 100644 --- a/test/api/capi/CMakeLists.txt +++ b/test/api/capi/CMakeLists.txt @@ -3,6 +3,7 @@ add_library_unity( OBJECT capi_aggregate_functions.cpp capi_custom_type.cpp + capi_file_system.cpp capi_scalar_functions.cpp capi_table_functions.cpp test_capi.cpp diff --git a/test/api/capi/capi_file_system.cpp b/test/api/capi/capi_file_system.cpp new file mode 100644 index 000000000000..639b7ae8c71d --- /dev/null +++ b/test/api/capi/capi_file_system.cpp @@ -0,0 +1,159 @@ +#include "capi_tester.hpp" + +using namespace duckdb; +using namespace std; + +static void test_file_system(duckdb_file_system fs, string file_name) { + REQUIRE(fs != nullptr); + + duckdb_state state = DuckDBSuccess; + duckdb_file_handle file; + + auto file_path = TestDirectoryPath() + "/" + file_name; + + auto options = duckdb_create_file_open_options(); + state = duckdb_file_open_options_set_flag(options, DUCKDB_FILE_FLAG_WRITE, true); + REQUIRE(state == DuckDBSuccess); + state = duckdb_file_open_options_set_flag(options, DUCKDB_FILE_FLAG_READ, true); + REQUIRE(state == DuckDBSuccess); + + // Try to open non-existing file without create flag + state = duckdb_file_system_open(fs, file_path.c_str(), options, &file); + REQUIRE(state != DuckDBSuccess); + auto error_data = duckdb_file_system_error_data(fs); + auto error_type = duckdb_error_data_error_type(error_data); + REQUIRE(error_type == DUCKDB_ERROR_IO); + duckdb_destroy_error_data(&error_data); + + // Try to write to a null file handle + auto failed_bytes_written = duckdb_file_handle_write(file, "data", 4); + REQUIRE(failed_bytes_written == -1); + auto file_error_data = duckdb_file_handle_error_data(file); + auto has_error = duckdb_error_data_has_error(file_error_data); + REQUIRE(has_error == false); + duckdb_destroy_error_data(&file_error_data); + + // Set create flag + state = duckdb_file_open_options_set_flag(options, DUCKDB_FILE_FLAG_CREATE, true); + REQUIRE(state == DuckDBSuccess); + + // Create and open a file + state = duckdb_file_system_open(fs, file_path.c_str(), options, &file); + REQUIRE(state == DuckDBSuccess); + REQUIRE(file != nullptr); + + // Write to the file + const char *data = "Hello, DuckDB File System!"; + auto bytes_written = duckdb_file_handle_write(file, data, strlen(data)); + REQUIRE(bytes_written == (int64_t)strlen(data)); + auto position = duckdb_file_handle_tell(file); + REQUIRE(position == bytes_written); + auto size = duckdb_file_handle_size(file); + REQUIRE(size == bytes_written); + + // Sync + state = duckdb_file_handle_sync(file); + + // Seek to the beginning + state = duckdb_file_handle_seek(file, 0); + REQUIRE(state == DuckDBSuccess); + position = duckdb_file_handle_tell(file); + REQUIRE(position == 0); + + // Read from the file + char buffer[30]; + memset(buffer, 0, sizeof(buffer)); + auto bytes_read = duckdb_file_handle_read(file, buffer, sizeof(buffer) - 1); + REQUIRE(bytes_read == bytes_written); + REQUIRE(strcmp(buffer, data) == 0); + position = duckdb_file_handle_tell(file); + REQUIRE(position == bytes_read); + size = duckdb_file_handle_size(file); + REQUIRE(size == bytes_written); + + // Seek to the end + state = duckdb_file_handle_seek(file, bytes_written); + REQUIRE(state == DuckDBSuccess); + position = duckdb_file_handle_tell(file); + REQUIRE(position == bytes_written); + size = duckdb_file_handle_size(file); + REQUIRE(size == bytes_written); + + // Try to read from the end of the file + memset(buffer, 0, sizeof(buffer)); + bytes_read = duckdb_file_handle_read(file, buffer, sizeof(buffer) - 1); + REQUIRE(bytes_read == 0); // EOF + position = duckdb_file_handle_tell(file); + REQUIRE(position == bytes_written); + size = duckdb_file_handle_size(file); + REQUIRE(size == bytes_written); + + // Seek back to the beginning + state = duckdb_file_handle_seek(file, 0); + REQUIRE(state == DuckDBSuccess); + position = duckdb_file_handle_tell(file); + REQUIRE(position == 0); + size = duckdb_file_handle_size(file); + REQUIRE(size == bytes_written); + + // Close the file + duckdb_file_handle_close(file); + duckdb_destroy_file_handle(&file); + + // Open file again for reading + state = duckdb_file_system_open(fs, file_path.c_str(), options, &file); + REQUIRE(state == DuckDBSuccess); + REQUIRE(file != nullptr); + size = duckdb_file_handle_size(file); + REQUIRE(size == bytes_written); + // Check that the data is still there + memset(buffer, 0, sizeof(buffer)); + bytes_read = duckdb_file_handle_read(file, buffer, sizeof(buffer) - 1); + REQUIRE(bytes_read == bytes_written); + REQUIRE(strcmp(buffer, data) == 0); + position = duckdb_file_handle_tell(file); + REQUIRE(position == bytes_read); + size = duckdb_file_handle_size(file); + REQUIRE(size == bytes_written); + + // Close the file again + duckdb_file_handle_close(file); + duckdb_destroy_file_handle(&file); + + duckdb_destroy_file_open_options(&options); + + REQUIRE(file == nullptr); + REQUIRE(options == nullptr); + + // Try destroy again for good measure + duckdb_destroy_file_handle(&file); + duckdb_destroy_file_open_options(&options); + + REQUIRE(file == nullptr); + REQUIRE(options == nullptr); +} + +TEST_CASE("Test File System in C API", "[capi]") { + CAPITester tester; + duckdb_client_context context; + duckdb_file_system fs; + duckdb::unique_ptr result; + + REQUIRE(tester.OpenDatabase(nullptr)); + + // get a file system from the client context + duckdb_connection_get_client_context(tester.connection, &context); + fs = duckdb_client_context_get_file_system(context); + REQUIRE(fs != nullptr); + + test_file_system(fs, "test_file_capi_1.txt"); + + duckdb_destroy_file_system(&fs); + REQUIRE(fs == nullptr); + + duckdb_destroy_client_context(&context); + + // Try to destory fs again + duckdb_destroy_file_system(&fs); + REQUIRE(fs == nullptr); +} diff --git a/test/api/capi/capi_scalar_functions.cpp b/test/api/capi/capi_scalar_functions.cpp index 1d7f3c10164a..014df4dd8091 100644 --- a/test/api/capi/capi_scalar_functions.cpp +++ b/test/api/capi/capi_scalar_functions.cpp @@ -109,8 +109,8 @@ TEST_CASE("Test Scalar Functions C API", "[capi]") { REQUIRE(tester.OpenDatabase(nullptr)); CAPIRegisterAddition(tester.connection, "my_addition", DuckDBSuccess); - // try to register it again - this should be an error - CAPIRegisterAddition(tester.connection, "my_addition", DuckDBError); + // try to register it again - this should not be an error + CAPIRegisterAddition(tester.connection, "my_addition", DuckDBSuccess); // now call it result = tester.Query("SELECT my_addition(40, 2)"); @@ -387,8 +387,8 @@ TEST_CASE("Test Scalar Function Overloads C API", "[capi]") { REQUIRE(tester.OpenDatabase(nullptr)); CAPIRegisterAdditionOverloads(tester.connection, "my_addition", DuckDBSuccess); - // try to register it again - this should be an error - CAPIRegisterAdditionOverloads(tester.connection, "my_addition", DuckDBError); + // try to register it again - this should not be an error + CAPIRegisterAdditionOverloads(tester.connection, "my_addition", DuckDBSuccess); // now call it result = tester.Query("SELECT my_addition(40, 2)"); diff --git a/test/api/capi/capi_table_functions.cpp b/test/api/capi/capi_table_functions.cpp index c06ed192b70b..ed73ff2590f3 100644 --- a/test/api/capi/capi_table_functions.cpp +++ b/test/api/capi/capi_table_functions.cpp @@ -90,8 +90,9 @@ TEST_CASE("Test Table Functions C API", "[capi]") { REQUIRE(tester.OpenDatabase(nullptr)); capi_register_table_function(tester.connection, "my_function", my_bind, my_init, my_function); - // registering again causes an error - capi_register_table_function(tester.connection, "my_function", my_bind, my_init, my_function, DuckDBError); + + // registering again does not cause error, because we overload + capi_register_table_function(tester.connection, "my_function", my_bind, my_init, my_function); // now call it result = tester.Query("SELECT * FROM my_function(1)"); @@ -158,8 +159,8 @@ TEST_CASE("Test Table Function register errors in C API", "[capi]") { REQUIRE(tester.OpenDatabase(nullptr)); capi_register_table_function(tester.connection, "x", my_error_bind, my_init, my_function, DuckDBSuccess); - // Try to register it again with the same name, name collision - capi_register_table_function(tester.connection, "x", my_error_bind, my_init, my_function, DuckDBError); + // Try to register it again with the same name, is ok (because of overloading) + capi_register_table_function(tester.connection, "x", my_error_bind, my_init, my_function, DuckDBSuccess); } struct my_named_bind_data_struct { diff --git a/test/api/capi/test_capi_complex_types.cpp b/test/api/capi/test_capi_complex_types.cpp index f540fb63a898..1f2fa3cfa204 100644 --- a/test/api/capi/test_capi_complex_types.cpp +++ b/test/api/capi/test_capi_complex_types.cpp @@ -541,6 +541,15 @@ TEST_CASE("Binding values", "[capi]") { auto struct_value = duckdb_create_struct_value(struct_type, &value); duckdb_destroy_logical_type(&struct_type); + // Fail with out-of-bounds. + duckdb_prepared_statement prepared_fail; + REQUIRE(duckdb_prepare(tester.connection, "SELECT ?, ?", &prepared_fail) == DuckDBSuccess); + auto state = duckdb_bind_value(prepared_fail, 3, struct_value); + REQUIRE(state == DuckDBError); + auto error_msg = duckdb_prepare_error(prepared_fail); + REQUIRE(StringUtil::Contains(string(error_msg), "Can not bind to parameter number")); + duckdb_destroy_prepare(&prepared_fail); + duckdb::vector list_values {value}; auto list_value = duckdb_create_list_value(member_type, list_values.data(), member_count); diff --git a/test/api/capi/test_capi_prepared.cpp b/test/api/capi/test_capi_prepared.cpp index 414845108b5a..22e1b03e9628 100644 --- a/test/api/capi/test_capi_prepared.cpp +++ b/test/api/capi/test_capi_prepared.cpp @@ -17,6 +17,13 @@ TEST_CASE("Test prepared statements in C API", "[capi]") { REQUIRE(status == DuckDBSuccess); REQUIRE(stmt != nullptr); + REQUIRE(duckdb_prepared_statement_column_count(stmt) == 1); + REQUIRE(duckdb_prepared_statement_column_type(stmt, 0) == DUCKDB_TYPE_BIGINT); + auto logical_type = duckdb_prepared_statement_column_logical_type(stmt, 0); + REQUIRE(logical_type); + REQUIRE(duckdb_get_type_id(logical_type) == DUCKDB_TYPE_BIGINT); + duckdb_destroy_logical_type(&logical_type); + status = duckdb_bind_boolean(stmt, 1, true); REQUIRE(status == DuckDBSuccess); @@ -274,6 +281,9 @@ TEST_CASE("Test prepared statements in C API", "[capi]") { REQUIRE(status == DuckDBSuccess); REQUIRE(stmt != nullptr); + REQUIRE(duckdb_prepared_statement_column_count(stmt) == 1); + REQUIRE(duckdb_prepared_statement_column_type(stmt, 0) == DUCKDB_TYPE_INTEGER); + status = duckdb_execute_prepared(stmt, &res); REQUIRE(status == DuckDBError); duckdb_destroy_result(&res); @@ -292,9 +302,66 @@ TEST_CASE("Test prepared statements in C API", "[capi]") { REQUIRE(duckdb_param_type(nullptr, 0) == DUCKDB_TYPE_INVALID); REQUIRE(duckdb_param_type(stmt, 1) == DUCKDB_TYPE_INTEGER); + REQUIRE(duckdb_prepared_statement_column_count(stmt) == 1); + REQUIRE(duckdb_prepared_statement_column_type(stmt, 0) == DUCKDB_TYPE_HUGEINT); + duckdb_destroy_prepare(&stmt); } +TEST_CASE("Test duckdb_prepared_statement return value APIs", "[capi]") { + duckdb_database db; + duckdb_connection conn; + duckdb_prepared_statement stmt; + REQUIRE(duckdb_open("", &db) == DuckDBSuccess); + REQUIRE(duckdb_connect(db, &conn) == DuckDBSuccess); + + // Unambiguous return column types + REQUIRE(duckdb_prepare(conn, "select $1::TEXT, $2::integer, $3::BOOLEAN, $4::FLOAT, $5::DOUBLE", &stmt) == + DuckDBSuccess); + + REQUIRE(duckdb_prepared_statement_column_count(stmt) == 5); + auto expected_types = {DUCKDB_TYPE_VARCHAR, DUCKDB_TYPE_INTEGER, DUCKDB_TYPE_BOOLEAN, DUCKDB_TYPE_FLOAT, + DUCKDB_TYPE_DOUBLE}; + + for (idx_t i = 0; i < 5; i++) { + REQUIRE(duckdb_prepared_statement_column_type(stmt, i) == *next(expected_types.begin(), i)); + auto logical_type = duckdb_prepared_statement_column_logical_type(stmt, i); + REQUIRE(logical_type); + REQUIRE(duckdb_get_type_id(logical_type) == *next(expected_types.begin(), i)); + duckdb_destroy_logical_type(&logical_type); + } + + auto column_name = duckdb_prepared_statement_column_name(stmt, 0); + std::string col_name_str = column_name; + duckdb_free((void *)column_name); + REQUIRE(col_name_str == "CAST($1 AS VARCHAR)"); + + duckdb_destroy_prepare(&stmt); + + // Return columns contain ambiguous types + REQUIRE(duckdb_prepare(conn, "select $1::TEXT, $2::integer, $3, $4::BOOLEAN, $5::FLOAT, $6::DOUBLE", &stmt) == + DuckDBSuccess); + + REQUIRE(duckdb_prepared_statement_column_count(stmt) == 1); + REQUIRE(duckdb_prepared_statement_column_type(stmt, 0) == DUCKDB_TYPE_INVALID); + + auto logical_type = duckdb_prepared_statement_column_logical_type(stmt, 0); + REQUIRE(logical_type); + REQUIRE(duckdb_get_type_id(logical_type) == DUCKDB_TYPE_INVALID); + duckdb_destroy_logical_type(&logical_type); + + auto col_name_ptr = duckdb_prepared_statement_column_name(stmt, 0); + col_name_str = col_name_ptr; + duckdb_free((void *)col_name_ptr); + REQUIRE(col_name_str == "unknown"); + REQUIRE(duckdb_prepared_statement_column_name(stmt, 1) == nullptr); + REQUIRE(duckdb_prepared_statement_column_name(stmt, 5) == nullptr); + + duckdb_destroy_prepare(&stmt); + duckdb_disconnect(&conn); + duckdb_close(&db); +} + TEST_CASE("Test duckdb_param_type and duckdb_param_logical_type", "[capi]") { duckdb_database db; duckdb_connection conn; @@ -369,6 +436,9 @@ TEST_CASE("Test prepared statements with named parameters in C API", "[capi]") { REQUIRE(duckdb_parameter_name(stmt, 0) == (const char *)NULL); REQUIRE(duckdb_parameter_name(stmt, 2) == (const char *)NULL); + REQUIRE(duckdb_prepared_statement_column_count(stmt) == 1); + REQUIRE(duckdb_prepared_statement_column_type(stmt, 0) == DUCKDB_TYPE_BIGINT); + duckdb::vector expected_names = {"my_val"}; REQUIRE(names.size() == expected_names.size()); for (idx_t i = 0; i < expected_names.size(); i++) { diff --git a/test/api/capi/test_capi_profiling.cpp b/test/api/capi/test_capi_profiling.cpp index cb8e77c4be40..936feca75250 100644 --- a/test/api/capi/test_capi_profiling.cpp +++ b/test/api/capi/test_capi_profiling.cpp @@ -265,3 +265,152 @@ TEST_CASE("Test profiling after throwing an error", "[capi]") { tester.Cleanup(); } + +TEST_CASE("Test profiling with Extra Info enabled", "[capi]") { + CAPITester tester; + REQUIRE(tester.OpenDatabase(nullptr)); + + REQUIRE_NO_FAIL(tester.Query("PRAGMA enable_profiling = 'no_output'")); + duckdb::vector settings = {"EXTRA_INFO"}; + REQUIRE_NO_FAIL(tester.Query("PRAGMA custom_profiling_settings=" + BuildSettingsString(settings))); + REQUIRE_NO_FAIL(tester.Query("SELECT 1")); + + auto info = duckdb_get_profiling_info(tester.connection); + REQUIRE(info); + + // Retrieve the child node. + auto child_info = duckdb_profiling_info_get_child(info, 0); + REQUIRE(duckdb_profiling_info_get_child_count(child_info) != 0); + + auto map = duckdb_profiling_info_get_metrics(child_info); + REQUIRE(map); + auto count = duckdb_get_map_size(map); + REQUIRE(count != 0); + + bool found_extra_info = false; + for (idx_t i = 0; i < count; i++) { + auto key = duckdb_get_map_key(map, i); + REQUIRE(key); + auto key_c_str = duckdb_get_varchar(key); + auto key_str = duckdb::string(key_c_str); + + auto value = duckdb_get_map_value(map, i); + REQUIRE(value); + auto value_c_str = duckdb_get_varchar(value); + auto value_str = duckdb::string(value_c_str); + + if (key_str == EnumUtil::ToString(MetricsType::EXTRA_INFO)) { + REQUIRE(value_str.find("__order_by__")); + REQUIRE(value_str.find("ASC")); + found_extra_info = true; + } + + if (key) { + duckdb_destroy_value(&key); + duckdb_free(key_c_str); + } + if (value) { + duckdb_destroy_value(&value); + duckdb_free(value_c_str); + } + } + + REQUIRE(found_extra_info); + + duckdb_destroy_value(&map); + tester.Cleanup(); +} + +TEST_CASE("Test profiling with the appender", "[capi]") { + CAPITester tester; + duckdb::unique_ptr result; + REQUIRE(tester.OpenDatabase(nullptr)); + + tester.Query("CREATE TABLE tbl (i INT PRIMARY KEY, value VARCHAR)"); + REQUIRE_NO_FAIL(tester.Query("PRAGMA enable_profiling = 'no_output'")); + REQUIRE_NO_FAIL(tester.Query("SET profiling_coverage='ALL'")); + duckdb_appender appender; + + string query = "INSERT INTO tbl FROM my_appended_data"; + duckdb_logical_type types[2]; + types[0] = duckdb_create_logical_type(DUCKDB_TYPE_INTEGER); + types[1] = duckdb_create_logical_type(DUCKDB_TYPE_VARCHAR); + + auto status = duckdb_appender_create_query(tester.connection, query.c_str(), 2, types, "my_appended_data", nullptr, + &appender); + duckdb_destroy_logical_type(&types[0]); + duckdb_destroy_logical_type(&types[1]); + REQUIRE(status == DuckDBSuccess); + REQUIRE(duckdb_appender_error(appender) == nullptr); + + REQUIRE(duckdb_appender_begin_row(appender) == DuckDBSuccess); + REQUIRE(duckdb_append_int32(appender, 1) == DuckDBSuccess); + REQUIRE(duckdb_append_varchar(appender, "hello world") == DuckDBSuccess); + REQUIRE(duckdb_appender_end_row(appender) == DuckDBSuccess); + + REQUIRE(duckdb_appender_flush(appender) == DuckDBSuccess); + REQUIRE(duckdb_appender_close(appender) == DuckDBSuccess); + REQUIRE(duckdb_appender_destroy(&appender) == DuckDBSuccess); + + auto info = duckdb_get_profiling_info(tester.connection); + REQUIRE(info); + + // Check that the query name matches the appender query. + auto query_name = duckdb_profiling_info_get_value(info, "QUERY_NAME"); + REQUIRE(query_name); + auto query_name_c_str = duckdb_get_varchar(query_name); + auto query_name_str = duckdb::string(query_name_c_str); + REQUIRE(query_name_str == query); + duckdb_destroy_value(&query_name); + duckdb_free(query_name_c_str); + + duckdb::map cumulative_counter; + duckdb::map cumulative_result; + TraverseTree(info, cumulative_counter, cumulative_result, 0); + tester.Cleanup(); +} + +TEST_CASE("Test profiling with the non-query appender", "[capi]") { + CAPITester tester; + duckdb::unique_ptr result; + duckdb_state status; + + REQUIRE(tester.OpenDatabase(nullptr)); + tester.Query("CREATE TABLE test (i INTEGER)"); + REQUIRE_NO_FAIL(tester.Query("PRAGMA enable_profiling = 'no_output'")); + REQUIRE_NO_FAIL(tester.Query("SET profiling_coverage='ALL'")); + + duckdb_appender appender; + REQUIRE(duckdb_appender_create(tester.connection, nullptr, "test", &appender) == DuckDBSuccess); + REQUIRE(duckdb_appender_error(appender) == nullptr); + + // Appending a row. + REQUIRE(duckdb_appender_begin_row(appender) == DuckDBSuccess); + REQUIRE(duckdb_append_int32(appender, 42) == DuckDBSuccess); + // Finish and flush. + REQUIRE(duckdb_appender_end_row(appender) == DuckDBSuccess); + REQUIRE(duckdb_appender_flush(appender) == DuckDBSuccess); + REQUIRE(duckdb_appender_close(appender) == DuckDBSuccess); + REQUIRE(duckdb_appender_destroy(&appender) == DuckDBSuccess); + + auto info = duckdb_get_profiling_info(tester.connection); + REQUIRE(info); + + // Check that the query name matches the appender query. + auto query_name = duckdb_profiling_info_get_value(info, "QUERY_NAME"); + REQUIRE(query_name); + + auto query_name_c_str = duckdb_get_varchar(query_name); + auto query_name_str = duckdb::string(query_name_c_str); + + auto query = "INSERT INTO main.test FROM __duckdb_internal_appended_data"; + REQUIRE(query_name_str == query); + + duckdb_destroy_value(&query_name); + duckdb_free(query_name_c_str); + + duckdb::map cumulative_counter; + duckdb::map cumulative_result; + TraverseTree(info, cumulative_counter, cumulative_result, 0); + tester.Cleanup(); +} diff --git a/test/api/capi/test_capi_table_description.cpp b/test/api/capi/test_capi_table_description.cpp index 5064a422e24a..1273d1750b76 100644 --- a/test/api/capi/test_capi_table_description.cpp +++ b/test/api/capi/test_capi_table_description.cpp @@ -64,18 +64,37 @@ TEST_CASE("Test the table description in the C API", "[capi]") { REQUIRE(status == DuckDBSuccess); REQUIRE(duckdb_table_description_error(table_description) == nullptr); + SECTION("Passing nullptr to get_column_count") { + REQUIRE(duckdb_table_description_get_column_count(nullptr) == 0); + } SECTION("Passing nullptr to get_name") { REQUIRE(duckdb_table_description_get_column_name(nullptr, 0) == nullptr); } + SECTION("Passing nullptr to get_type") { + REQUIRE(duckdb_table_description_get_column_type(nullptr, 0) == nullptr); + } SECTION("Out of range column for get_name") { REQUIRE(duckdb_table_description_get_column_name(table_description, 1) == nullptr); } + SECTION("Out of range column for get_type") { + REQUIRE(duckdb_table_description_get_column_type(table_description, 1) == nullptr); + } + SECTION("get the column count") { + auto column_count = duckdb_table_description_get_column_count(table_description); + REQUIRE(column_count == 1); + } SECTION("In range column - get the name") { auto column_name = duckdb_table_description_get_column_name(table_description, 0); string expected = "my_column"; REQUIRE(!expected.compare(column_name)); duckdb_free(column_name); } + SECTION("In range column - get the type") { + auto column_type = duckdb_table_description_get_column_type(table_description, 0); + auto type_id = duckdb_get_type_id(column_type); + REQUIRE(type_id == DUCKDB_TYPE_INTEGER); + duckdb_destroy_logical_type(&column_type); + } duckdb_table_description_destroy(&table_description); } diff --git a/test/api/test_instance_cache.cpp b/test/api/test_instance_cache.cpp index 694eb02863d1..7abe20e55406 100644 --- a/test/api/test_instance_cache.cpp +++ b/test/api/test_instance_cache.cpp @@ -101,3 +101,107 @@ TEST_CASE("Test db creation does not block instance cache", "[api]") { REQUIRE(opening_slow_db_takes_remaining_time); REQUIRE(no_delay_for_db_creation); } + +TEST_CASE("Test attaching the same database path from different databases", "[api][.]") { + DBInstanceCache instance_cache; + auto test_path = TestCreatePath("instance_cache_reuse.db"); + + DBConfig config; + auto db1 = instance_cache.GetOrCreateInstance(":memory:", config, false); + auto db2 = instance_cache.GetOrCreateInstance(":memory:", config, false); + + Connection con1(*db1); + Connection con2(*db2); + SECTION("Regular ATTACH conflict") { + string attach_query = "ATTACH '" + test_path + "' AS db_ref"; + + REQUIRE_NO_FAIL(con1.Query(attach_query)); + + // fails - already attached in db1 + REQUIRE_FAIL(con2.Query(attach_query)); + + // if we detach from con1, we can now attach in con2 + REQUIRE_NO_FAIL(con1.Query("DETACH db_ref")); + + REQUIRE_NO_FAIL(con2.Query(attach_query)); + + // .. but not in con1 anymore! + REQUIRE_FAIL(con1.Query(attach_query)); + } + SECTION("ATTACH IF NOT EXISTS") { + string attach_query = "ATTACH IF NOT EXISTS '" + test_path + "' AS db_ref"; + + REQUIRE_NO_FAIL(con1.Query(attach_query)); + + // fails - already attached in db1 + REQUIRE_FAIL(con2.Query(attach_query)); + } +} + +TEST_CASE("Test attaching the same database path from different databases in read-only mode", "[api][.]") { + DBInstanceCache instance_cache; + auto test_path = TestCreatePath("instance_cache_reuse_readonly.db"); + + // create an empty database + { + DuckDB db(test_path); + Connection con(db); + REQUIRE_NO_FAIL(con.Query("CREATE TABLE IF NOT EXISTS integers AS FROM (VALUES (1), (2), (3)) t(i)")); + } + + DBConfig config; + auto db1 = instance_cache.GetOrCreateInstance(":memory:", config, false); + auto db2 = instance_cache.GetOrCreateInstance(":memory:", config, false); + auto db3 = instance_cache.GetOrCreateInstance(":memory:", config, false); + + Connection con1(*db1); + Connection con2(*db2); + Connection con3(*db3); + + SECTION("Regular ATTACH conflict") { + string attach_query = "ATTACH '" + test_path + "' AS db_ref"; + string read_only_attach = attach_query + " (READ_ONLY)"; + + REQUIRE_NO_FAIL(con1.Query(read_only_attach)); + + // succeeds - we can attach the same database multiple times in read-only mode + REQUIRE_NO_FAIL(con2.Query(read_only_attach)); + + // fails - we cannot attach in read-write + REQUIRE_FAIL(con3.Query(attach_query)); + + // if we detach from con1, we still cannot attach in read-write in con3 + REQUIRE_NO_FAIL(con1.Query("DETACH db_ref")); + REQUIRE_FAIL(con3.Query(attach_query)); + + // but if we detach in con2, we can attach in read-write mode now + REQUIRE_NO_FAIL(con2.Query("DETACH db_ref")); + REQUIRE_NO_FAIL(con3.Query(attach_query)); + + // and now we can no longer attach in read-only mode + REQUIRE_FAIL(con1.Query(read_only_attach)); + } + SECTION("ATTACH IF EXISTS") { + string attach_query = "ATTACH IF NOT EXISTS '" + test_path + "' AS db_ref"; + string read_only_attach = attach_query + " (READ_ONLY)"; + + REQUIRE_NO_FAIL(con1.Query(read_only_attach)); + + // succeeds - we can attach the same database multiple times in read-only mode + REQUIRE_NO_FAIL(con2.Query(read_only_attach)); + + // fails - we cannot attach in read-write + REQUIRE_FAIL(con3.Query(attach_query)); + + // if we detach from con1, we still cannot attach in read-write in con3 + REQUIRE_NO_FAIL(con1.Query("DETACH db_ref")); + REQUIRE_FAIL(con3.Query(attach_query)); + + // but if we detach in con2, we can attach in read-write mode now + REQUIRE_NO_FAIL(con2.Query("DETACH db_ref")); + REQUIRE_NO_FAIL(con3.Query(attach_query)); + + // and now we can no longer attach in read-only mode + REQUIRE_FAIL(con1.Query(read_only_attach)); + } +} diff --git a/test/api/test_lifecycle_hooks.cpp b/test/api/test_lifecycle_hooks.cpp index 47a4d36832f2..7a19465c93a0 100644 --- a/test/api/test_lifecycle_hooks.cpp +++ b/test/api/test_lifecycle_hooks.cpp @@ -2,6 +2,7 @@ #include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" #include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/main/extension_manager.hpp" #include "test_helpers.hpp" using namespace duckdb; @@ -53,7 +54,9 @@ TEST_CASE("Test ClientContextState", "[api]") { return nullptr; }); - ExtensionLoader loader(*db.instance, "test_extension"); + ExtensionInfo extension_info {}; + ExtensionActiveLoad load_info {*db.instance, extension_info, "test_extension"}; + ExtensionLoader loader {load_info}; loader.RegisterFunction(table_fun); SECTION("No error, No explicit transaction") { diff --git a/test/api/test_relation_api.cpp b/test/api/test_relation_api.cpp index 09a80318d62b..d04357fc40d1 100644 --- a/test/api/test_relation_api.cpp +++ b/test/api/test_relation_api.cpp @@ -1,6 +1,8 @@ #include "catch.hpp" #include "duckdb/common/file_system.hpp" #include "duckdb/common/enums/joinref_type.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/main/relation/value_relation.hpp" #include "iostream" #include "test_helpers.hpp" #include "duckdb/main/relation/materialized_relation.hpp" @@ -1061,6 +1063,60 @@ TEST_CASE("Test Relation Pending Query API", "[relation_api]") { } } +TEST_CASE("Test Relation Query setting query", "[relation_api]") { + DuckDB db; + Connection con(db); + + auto query = con.RelationFromQuery("SELECT current_query()"); + auto result = query->Limit(1)->Execute(); + REQUIRE(!result->Fetch()->GetValue(0, 0).ToString().empty()); +} + +TEST_CASE("Construct ValueRelation with RelationContextWrapper and operate on it", "[relation_api][txn][wrapper]") { + DuckDB db; + Connection con(db); + con.EnableQueryVerification(); + + // Build expressions to force the "expressions" overload (not the constants path) + duckdb::vector>> expressions; + { + duckdb::vector> row; + + { + duckdb::ConstantExpression ce1(duckdb::Value::INTEGER(1)); + row.push_back(ce1.Copy()); + } + { + duckdb::ConstantExpression ce2(duckdb::Value::INTEGER(2)); + row.push_back(ce2.Copy()); + } + expressions.push_back(std::move(row)); + } + + // Explicitly create a RelationContextWrapper from the client's context + auto rcw = duckdb::make_shared_ptr(con.context); + + // Manually construct a ValueRelation using the RelationContextWrapper + expressions ctor + duckdb::shared_ptr rel; + REQUIRE_NOTHROW(rel = duckdb::make_shared_ptr(rcw, std::move(expressions), + duckdb::vector {}, "vr")); + + // Base relation should execute (1 row, 2 columns) + duckdb::unique_ptr result; + REQUIRE_NOTHROW(result = rel->Execute()); + REQUIRE(CHECK_COLUMN(result, 0, {1})); + REQUIRE(CHECK_COLUMN(result, 1, {2})); + + // Project on top — should bind & execute under a valid transaction + REQUIRE_NOTHROW(result = rel->Project("1+1, 2+2")->Execute()); + REQUIRE(CHECK_COLUMN(result, 0, {2})); + REQUIRE(CHECK_COLUMN(result, 1, {4})); + + // Aggregate on top — also must bind & execute cleanly + REQUIRE_NOTHROW(result = rel->Aggregate("count(*)")->Execute()); + REQUIRE(CHECK_COLUMN(result, 0, {1})); +} + TEST_CASE("Test materialized relations", "[relation_api]") { auto db_path = TestCreatePath("relational_api_materialized_view.db"); { diff --git a/test/api/test_reset.cpp b/test/api/test_reset.cpp index fba810af2195..1285dabc227f 100644 --- a/test/api/test_reset.cpp +++ b/test/api/test_reset.cpp @@ -62,6 +62,7 @@ OptionValueSet GetValueForOption(const string &name, const LogicalType &type) { {"custom_extension_repository", {"duckdb.org/no-extensions-here", "duckdb.org/no-extensions-here"}}, {"autoinstall_extension_repository", {"duckdb.org/no-extensions-here", "duckdb.org/no-extensions-here"}}, {"lambda_syntax", {EnumUtil::ToString(LambdaSyntax::DISABLE_SINGLE_ARROW)}}, + {"allow_parser_override_extension", {"fallback"}}, {"profiling_coverage", {EnumUtil::ToString(ProfilingCoverage::ALL)}}, #ifdef DUCKDB_EXTENSION_AUTOLOAD_DEFAULT {"autoload_known_extensions", {!DUCKDB_EXTENSION_AUTOLOAD_DEFAULT}}, @@ -88,7 +89,7 @@ OptionValueSet GetValueForOption(const string &name, const LogicalType &type) { {"memory_limit", {"4.0 GiB"}}, {"storage_compatibility_version", {"v0.10.0"}}, {"ordered_aggregate_threshold", {Value::UBIGINT(idx_t(1) << 12)}}, - {"null_order", {"nulls_first"}}, + {"null_order", {"NULLS_FIRST"}}, {"debug_verify_vector", {"dictionary_expression"}}, {"perfect_ht_threshold", {0}}, {"pivot_filter_threshold", {999}}, @@ -120,6 +121,8 @@ OptionValueSet GetValueForOption(const string &name, const LogicalType &type) { {"allocator_bulk_deallocation_flush_threshold", {"4.0 GiB"}}, {"arrow_output_version", {"1.5"}}, {"enable_external_file_cache", {false}}, + {"experimental_metadata_reuse", {false}}, + {"storage_block_prefetch", {"always_prefetch"}}, {"pin_threads", {"off"}}}; // Every option that's not excluded has to be part of this map if (!value_map.count(name)) { diff --git a/test/catalog/CMakeLists.txt b/test/catalog/CMakeLists.txt index f5eb86226e41..02e076580f57 100644 --- a/test/catalog/CMakeLists.txt +++ b/test/catalog/CMakeLists.txt @@ -1,4 +1,5 @@ add_library_unity(test_catalog OBJECT test_catalog_version.cpp) + set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/test/catalog/test_catalog_version.cpp b/test/catalog/test_catalog_version.cpp index 19fb7f412ed9..5fe260a762cf 100644 --- a/test/catalog/test_catalog_version.cpp +++ b/test/catalog/test_catalog_version.cpp @@ -1,6 +1,7 @@ #include "catch.hpp" #include "test_helpers.hpp" #include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/main/extension_manager.hpp" #include "duckdb.hpp" @@ -86,7 +87,9 @@ TEST_CASE("Test catalog versioning", "[catalog]") { // system transactions do not register catalog version changes :/ duckdb::TableFunction tf("some_new_table_function", {}, SomeTableFunc); - ExtensionLoader loader(*db.instance, "test_catalog_extension"); + ExtensionInfo extension_info {}; + ExtensionActiveLoad load_info {*db.instance, extension_info, "test_catalog_extension"}; + ExtensionLoader loader {load_info}; loader.RegisterFunction(tf); con1.context->RunFunctionInTransaction([&]() { diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index c6089198742d..b59036815546 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -8,6 +8,7 @@ add_library_unity( test_numeric_cast.cpp test_parse_logical_type.cpp test_utf.cpp + test_storage_fuzz.cpp test_strftime.cpp test_string_util.cpp) diff --git a/test/common/test_file_system.cpp b/test/common/test_file_system.cpp index cd72a684ba40..9a9345a6d2fe 100644 --- a/test/common/test_file_system.cpp +++ b/test/common/test_file_system.cpp @@ -224,3 +224,15 @@ TEST_CASE("extract subsystem", "[file_system]") { vfs.SetDisabledFileSystems(disabled_subfilesystems); REQUIRE(vfs.ExtractSubSystem(target_fs) == nullptr); } + +TEST_CASE("re-register subsystem", "[file_system]") { + duckdb::VirtualFileSystem vfs; + + // First time registration should succeed. + auto local_filesystem = FileSystem::CreateLocal(); + vfs.RegisterSubSystem(std::move(local_filesystem)); + + // Re-register an already registered subfilesystem should throw. + auto second_local_filesystem = FileSystem::CreateLocal(); + REQUIRE_THROWS(vfs.RegisterSubSystem(std::move(second_local_filesystem))); +} diff --git a/test/common/test_storage_fuzz.cpp b/test/common/test_storage_fuzz.cpp new file mode 100644 index 000000000000..66faeee05943 --- /dev/null +++ b/test/common/test_storage_fuzz.cpp @@ -0,0 +1,517 @@ +#include "catch.hpp" +#include "duckdb.hpp" +#include "duckdb/common/common.hpp" +#include "duckdb/common/local_file_system.hpp" +#include "duckdb/common/virtual_file_system.hpp" +#include "duckdb/main/materialized_query_result.hpp" +#include "test_config.hpp" +#include "test_helpers.hpp" + +#include +#include + +using namespace duckdb; + +bool g_enable_verbose_output = false; +bool g_enable_info_output = true; + +#define PRINT_VERBOSE(x) \ + do { \ + if (g_enable_verbose_output) \ + std::cout << x << std::endl; \ + } while (0) + +#define PRINT_INFO(x) \ + do { \ + if (g_enable_info_output) \ + std::cout << x << std::endl; \ + } while (0) + +bool ends_with(const std::string &str, const std::string &suffix) { + // Ensure str is at least as long as suffix + if (str.length() < suffix.length()) { + return false; + } + + // Compare the ending characters + return str.compare(str.length() - suffix.length(), suffix.length(), suffix) == 0; +} + +class FaultInjectionFileSystem : public duckdb::LocalFileSystem { +public: + enum FaultInjectionSite { + WRITE = 0, + FSYNC = 1, + }; + + void Write(duckdb::FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override { + PRINT_VERBOSE("FS write offset=" << location << " bytes=" << nr_bytes); + if (is_db_file(handle)) { + ThrowInjectedFaultIfSet(FaultInjectionSite::WRITE); + } + return duckdb::LocalFileSystem::Write(handle, buffer, nr_bytes, location); + } + + void FileSync(duckdb::FileHandle &handle) override { + PRINT_VERBOSE("FS fsync " << handle.GetPath() << " file_size=" << handle.GetFileSize()); + if (is_db_file(handle)) { + ThrowInjectedFaultIfSet(FaultInjectionSite::FSYNC); + } + return duckdb::LocalFileSystem::FileSync(handle); + } + + void RemoveFile(const duckdb::string &filename, + duckdb::optional_ptr opener = nullptr) override { + PRINT_VERBOSE("FS remove " << filename); + return duckdb::LocalFileSystem::RemoveFile(filename, opener); + } + + void Truncate(duckdb::FileHandle &handle, int64_t new_size) override { + PRINT_VERBOSE("FS truncate " << handle.GetPath() << " from " << handle.GetFileSize() << " to " << new_size); + return duckdb::LocalFileSystem::Truncate(handle, new_size); + } + + // In linux - trim() is equivalent to zeroing out a range (albeit in a much more efficient manner). Let's + // reproduce this behavior regardless of whether the current environment supports it. + bool Trim(duckdb::FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) override { + PRINT_VERBOSE("FS trim " << handle.GetPath() << " offset=" << offset_bytes << " bytes=" << length_bytes); + + std::string nulls(length_bytes, '\0'); + duckdb::LocalFileSystem::Write(handle, (void *)nulls.data(), length_bytes, offset_bytes); + return true; + } + + // Will inject a single occurrence of a fault + void InjectFault(FaultInjectionSite site) { + std::lock_guard l(fault_m_); + // Make sure this is not called twice - as we will drop a fault + REQUIRE(faults.insert(site).second); + } + +protected: + void ThrowInjectedFaultIfSet(FaultInjectionSite site) { + std::lock_guard l(fault_m_); + auto it = faults.find(site); + if (it != faults.end()) { + faults.erase(it); + PRINT_VERBOSE("Injecting fault"); + throw duckdb::IOException("Injected fault"); + } + } + + bool is_wal_file(const duckdb::FileHandle &handle) { + return ends_with(handle.GetPath(), ".db.wal"); + } + + bool is_db_file(const duckdb::FileHandle &handle) { + return ends_with(handle.GetPath(), ".db"); + } + + bool is_wal_or_db_file(const duckdb::FileHandle &handle) { + return is_db_file(handle) || is_wal_file(handle); + } + +private: + std::mutex fault_m_; + std::unordered_set faults; +}; + +// This implementation of duckdb::FileSystem will cache writes to the database file in memory until fsync is called. +// It expects all read ranges to be perfectly aligned with previous writes. +class LazyFlushFileSystem : public FaultInjectionFileSystem { +public: + ~LazyFlushFileSystem() { + if (!unflushed_chunks.empty()) { + PRINT_INFO("Unflushed chunks on shutdown for db file"); + } + } + + void Write(duckdb::FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override { + PRINT_VERBOSE("FS write offset=" << location << " bytes=" << nr_bytes); + + // We only perform positional writes for the db file + REQUIRE(is_db_file(handle)); + + std::unique_lock l(m_); + + ThrowInjectedFaultIfSet(FaultInjectionSite::WRITE); + + // Store the data in memory until fsync occurs + PRINT_VERBOSE("Caching chunk " << location << " bytes " << nr_bytes); + + // TODO: be lazy - don't handle partial overwrites + REQUIRE(!partially_overlaps_existing_chunk(unflushed_chunks, location, nr_bytes)); + + auto it = unflushed_chunks.find(location); + if (it != unflushed_chunks.end()) { + // Check that if there is an existing chunk - it's size matches exactly + REQUIRE(it->second.size() == nr_bytes); + it->second = std::string((char *)buffer, nr_bytes); + } else { + unflushed_chunks.emplace(location, std::string((char *)buffer, nr_bytes)); + } + } + + int64_t Write(duckdb::FileHandle &handle, void *buffer, int64_t nr_bytes) override { + // Check appends only occur on the WAL + REQUIRE(is_wal_file(handle)); + + return duckdb::LocalFileSystem::Write(handle, buffer, nr_bytes); + } + + void Read(duckdb::FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override { + REQUIRE(is_db_file(handle)); + + { + // TODO: shared_lock + std::unique_lock l(m_); + + // We don't handle partial overlaps for now. + REQUIRE(!partially_overlaps_existing_chunk(unflushed_chunks, location, nr_bytes)); + + auto it = unflushed_chunks.find(location); + if (it != unflushed_chunks.end()) { + PRINT_VERBOSE("FS read cached chunk at offset=" << location << " bytes=" << nr_bytes); + const auto &data = it->second; + // Assume block-aligned reads + REQUIRE(data.size() == nr_bytes); + memcpy(buffer, data.data(), nr_bytes); + return; + } + } + + PRINT_VERBOSE("FS read disk chunk at offset=" << location << " bytes=" << nr_bytes); + return duckdb::LocalFileSystem::Read(handle, buffer, nr_bytes, location); + } + + int64_t Read(duckdb::FileHandle &handle, void *buffer, int64_t nr_bytes) override { + PRINT_VERBOSE("FS read at end of file, bytes=" << nr_bytes); + REQUIRE(is_wal_or_db_file(handle)); + + if (is_db_file(handle)) { + // Just make sure we don't miss the unflushed chunks + REQUIRE(unflushed_chunks.empty()); + } + return duckdb::LocalFileSystem::Read(handle, buffer, nr_bytes); + } + + void FileSync(duckdb::FileHandle &handle) override { + PRINT_VERBOSE("FS fsync " << handle.GetPath() << " file_size=" << handle.GetFileSize()); + + REQUIRE(is_wal_or_db_file(handle)); + + if (!is_db_file(handle)) { + return duckdb::LocalFileSystem::FileSync(handle); + } + + std::unique_lock l(m_); + + ThrowInjectedFaultIfSet(FaultInjectionSite::FSYNC); + + for (const auto &location_and_data : unflushed_chunks) { + auto location = location_and_data.first; + const auto &data = location_and_data.second; + PRINT_VERBOSE("Flushing chunk " << location << " size=" << data.size()); + duckdb::LocalFileSystem::Write(handle, (void *)data.data(), data.size(), location); + } + unflushed_chunks.clear(); + + duckdb::LocalFileSystem::FileSync(handle); + } + + bool Trim(duckdb::FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) override { + REQUIRE(is_db_file(handle)); + + std::unique_lock l(m_); + + // This is just simpler to implement + REQUIRE(unflushed_chunks.count(offset_bytes) == 0); + + return FaultInjectionFileSystem::Trim(handle, offset_bytes, length_bytes); + } + + void Truncate(duckdb::FileHandle &handle, int64_t new_size) override { + std::unique_lock l(m_); + + if (is_db_file(handle)) { + REQUIRE(unflushed_chunks.empty()); + } + + return duckdb::LocalFileSystem::Truncate(handle, new_size); + } + +private: + // Lock for modifying unflushed_chunks: + // 1. Adding to unflushed_chunks on write + // 2. Flushing unflushed_chunks on fsync + // 3. Reading from unflushed_chunks + std::mutex m_; + std::map unflushed_chunks; + + bool partially_overlaps_existing_chunk(const std::map &chunks, idx_t offset, size_t length) { + idx_t end = offset + length; + for (const auto &off_data : chunks) { + auto off = off_data.first; + const auto &data = off_data.second; + idx_t chunk_end = off + data.size(); + + // Check for any overlap + bool overlap = offset < chunk_end && off < end; + + // Exclude full containment and exact match + bool exact_match = (offset == off && length == data.size()); + + if (overlap && !exact_match) + return true; + } + return false; + } +}; + +template +void validate(ResultT &r, std::string expected_err_message = "") { + // For debugging + bool expected_err = !expected_err_message.empty(); + if (expected_err != r.HasError() && r.HasError()) { + PRINT_INFO("Unexpected: query failed with " << r.GetError()); + } + REQUIRE(expected_err == r.HasError()); + if (r.HasError()) { + REQUIRE(r.GetError().find(expected_err_message) != std::string::npos); + } +} + +void cleanup_db_file(const std::string &filename) { + bool removed_or_missing = std::remove(filename.c_str()) == 0 || errno == ENOENT; + REQUIRE(removed_or_missing); +} + +TEST_CASE("simple fault injection storage test", "[storage][.]") { + if (!TestConfiguration::TestRunStorageFuzzer()) { + SKIP_TEST("storage-fuzzer not enabled"); + return; + } + duckdb::DBConfig config; + + LazyFlushFileSystem *raw_fs = new LazyFlushFileSystem(); + config.file_system = duckdb::make_uniq(duckdb::unique_ptr(raw_fs)); + + std::string file_path = TestCreatePath("pig.db"); + + cleanup_db_file(file_path); + + { + duckdb::DuckDB db(file_path, &config); + + duckdb::Connection con(db); + + validate(*con.Query("CREATE TABLE IF NOT EXISTS t(i INTEGER)")); + validate(*con.Query("INSERT INTO t SELECT * FROM RANGE(0, 1000)")); + validate(*con.Query("INSERT INTO t SELECT * FROM RANGE(0, 1000000)")); + + auto res = con.Query("SELECT count(*) FROM t"); + validate(*res); + REQUIRE(res->GetValue(0, 0).ToString() == "1001000"); + + // Writes are ok - fsync are not ok + raw_fs->InjectFault(LazyFlushFileSystem::FaultInjectionSite::FSYNC); + + validate(*con.Query("INSERT INTO t SELECT * FROM RANGE(0, 1000000)"), + "TransactionContext Error: Failed to commit: Injected fault"); + + // Check that the tx was rolled back + auto res2 = con.Query("SELECT count(*) FROM t"); + validate(*res2); + REQUIRE(res2->GetValue(0, 0).ToString() == "1001000"); + } + { + duckdb::DuckDB db(file_path, &config); + duckdb::Connection con(db); + + auto res = con.Query("SELECT count(*) FROM t"); + validate(*res); + REQUIRE(res->GetValue(0, 0).ToString() == "1001000"); + } +} + +enum ActionType { + // This action will simply flip the setting true -> false or false -> true + TOGGLE_SKIP_CHECKPOINTS_ON_COMMIT = 0, + SMALL_WRITE = 2, + LARGE_WRITE = 3, + LARGE_WRITE_WITH_FAULT = 4, + UPDATE = 5, + DELETE = 6, + RESET_TABLE = 7, +}; + +TEST_CASE("fuzzed storage test", "[storage][.]") { + if (!TestConfiguration::TestRunStorageFuzzer()) { + SKIP_TEST("storage-fuzzer not enabled"); + return; + } + // DuckDB Configurations + duckdb::DBConfig config; + config.options.set_variables["debug_skip_checkpoint_on_commit"] = duckdb::Value(true); + config.options.trim_free_blocks = true; + config.options.checkpoint_on_shutdown = false; + + std::string file_path = TestCreatePath("pig.db"); + + cleanup_db_file(file_path); + + { + duckdb::DuckDB db(file_path, &config); + duckdb::Connection con(db); + validate(*con.Query("CREATE TABLE IF NOT EXISTS t(i INTEGER)")); + } + + std::map pct_to_action = {{0.1, ActionType::TOGGLE_SKIP_CHECKPOINTS_ON_COMMIT}, + {0.3, ActionType::LARGE_WRITE}, + {0.5, ActionType::SMALL_WRITE}, + {0.7, ActionType::UPDATE}, + {0.85, ActionType::DELETE}, + {1.0, ActionType::LARGE_WRITE_WITH_FAULT}}; + + // Randomly generated sequence of actions + std::vector actions = {}; + + int NUM_ACTIONS = 30; + for (int i = 0; i < NUM_ACTIONS; i++) { + double selection = (rand() % 100) / 100.0; + for (const auto &prob_type : pct_to_action) { + auto prob = prob_type.first; + auto type = prob_type.second; + if (selection > prob) { + continue; + } + actions.push_back(type); + break; + } + } + actions.push_back(RESET_TABLE); + for (int i = 0; i < NUM_ACTIONS; i++) { + double selection = (rand() % 100) / 100.0; + for (const auto &prob_type : pct_to_action) { + auto prob = prob_type.first; + auto type = prob_type.second; + if (selection > prob) { + continue; + } + actions.push_back(type); + break; + } + } + + uint64_t offset = 0; + bool skip_checkpoint_on_commit = true; + std::string expected_checksum = ""; + duckdb::unique_ptr previous_result; + for (const auto &action : actions) { + // Note: the injected file system has to be reset each time. DuckDB construction seems to be std::move'ing them + + /* + LazyFlushFileSystem *raw_fs = new LazyFlushFileSystem(); + config.file_system = + duckdb::make_uniq(duckdb::unique_ptr(raw_fs)); + */ + + FaultInjectionFileSystem *raw_fs = new FaultInjectionFileSystem(); + config.file_system = + duckdb::make_uniq(duckdb::unique_ptr(raw_fs)); + + duckdb::DuckDB db(file_path, &config); + duckdb::Connection con(db); + + // Compute a checksum + if (!expected_checksum.empty()) { + auto checksum = con.Query("SELECT bit_xor(hash(i)) FROM t"); + validate(*checksum); + auto computed_checksum = checksum->GetValue(0, 0).ToString(); + PRINT_INFO("Verifying checksum computed=" << computed_checksum << ", actual=" << expected_checksum); + if (computed_checksum != expected_checksum) { + auto result = con.Query("SELECT * FROM t ORDER BY ALL"); + string error; + ColumnDataCollection::ResultEquals(previous_result->Cast().Collection(), + result->Cast().Collection(), error); + Printer::PrintF("Checksum failure\nResult comparison:\n%s", error); + REQUIRE(computed_checksum == expected_checksum); + } + } + previous_result = con.Query("SELECT * FROM t ORDER BY ALL"); + + switch (action) { + case ActionType::TOGGLE_SKIP_CHECKPOINTS_ON_COMMIT: + skip_checkpoint_on_commit = !skip_checkpoint_on_commit; + PRINT_INFO("Setting skip commit=" << skip_checkpoint_on_commit); + config.options.set_variables["debug_skip_checkpoint_on_commit"] = duckdb::Value(skip_checkpoint_on_commit); + break; + case ActionType::SMALL_WRITE: { + std::string small_insert = "INSERT INTO t SELECT * FROM RANGE(" + std::to_string(offset) + ", " + + std::to_string(offset + 100) + ")"; + PRINT_INFO("RUN: " << small_insert); + validate(*con.Query(small_insert)); + offset += 100; + break; + } + case ActionType::LARGE_WRITE: { + std::string large_insert = "INSERT INTO t SELECT * FROM RANGE(" + std::to_string(offset) + ", " + + std::to_string(offset + 1000 * 1000) + ")"; + PRINT_INFO("RUN: " << large_insert); + validate(*con.Query(large_insert)); + offset += 1000 * 1000; + break; + } + case ActionType::UPDATE: { + if (offset != 0) { + uint64_t begin = rand() % offset; + uint64_t length = rand() % (offset - begin); + std::string update_query = "UPDATE t SET i = i * 2 WHERE i > " + std::to_string(begin) + " and i <" + + std::to_string(begin + length); + + PRINT_INFO("RUN: " << update_query); + validate(*con.Query(update_query)); + } + break; + } + case ActionType::DELETE: { + if (offset != 0) { + uint64_t begin = rand() % offset; + uint64_t length = rand() % (offset - begin); + std::string delete_query = + "DELETE FROM t WHERE i > " + std::to_string(begin) + " and i <" + std::to_string(begin + length); + + PRINT_INFO("RUN: " << delete_query); + validate(*con.Query(delete_query)); + break; + } + } + case ActionType::LARGE_WRITE_WITH_FAULT: { + raw_fs->InjectFault(LazyFlushFileSystem::FaultInjectionSite::FSYNC); + std::string large_insert = "INSERT INTO t SELECT * FROM RANGE(" + std::to_string(offset) + ", " + + std::to_string(offset + 1000 * 1000) + ")"; + + PRINT_INFO("RUN with fault: " << large_insert); + validate(*con.Query(large_insert), "Injected fault"); + break; + } + case ActionType::RESET_TABLE: { + std::string replace_query = "CREATE OR REPLACE TABLE t(i INTEGER)"; + PRINT_INFO("RUN with fault: " << replace_query); + validate(*con.Query(replace_query)); + break; + } + } + + // Compute a checksum (unless we injected a fault - which will invalidate the database) + if (action != ActionType::LARGE_WRITE_WITH_FAULT) { + auto checksum = con.Query("SELECT bit_xor(hash(i)) FROM t"); + validate(*checksum); + expected_checksum = checksum->GetValue(0, 0).ToString(); + + PRINT_INFO("Computed new checksum: " << expected_checksum); + } else { + PRINT_INFO("Keeping old checksum due to faults: " << expected_checksum); + } + } +} diff --git a/test/common/test_string_util.cpp b/test/common/test_string_util.cpp index 6713b9eec5f5..57fe78b937fa 100644 --- a/test/common/test_string_util.cpp +++ b/test/common/test_string_util.cpp @@ -369,3 +369,131 @@ TEST_CASE("Test path utilities", "[string_util]") { REQUIRE("\\tmp" == StringUtil::GetFilePath("\\tmp\\\\test.txt")); } } + +TEST_CASE("Test JSON Parsing", "[string_util]") { + auto complex_json = StringUtil::ParseJSONMap(R"JSON_LITERAL( + { + "crs": { + "$schema": "https://proj.org/schemas/v0.7/projjson.schema.json", + "type": "GeographicCRS", + "name": "WGS 84", + "datum_ensemble": { + "name": "World Geodetic System 1984 ensemble", + "members": [ + { + "name": "World Geodetic System 1984 (Transit)", + "id": { + "authority": "EPSG", + "code": 1166 + } + }, + { + "name": "World Geodetic System 1984 (G730)", + "id": { + "authority": "EPSG", + "code": 1152 + } + }, + { + "name": "World Geodetic System 1984 (G873)", + "id": { + "authority": "EPSG", + "code": 1153 + } + }, + { + "name": "World Geodetic System 1984 (G1150)", + "id": { + "authority": "EPSG", + "code": 1154 + } + }, + { + "name": "World Geodetic System 1984 (G1674)", + "id": { + "authority": "EPSG", + "code": 1155 + } + }, + { + "name": "World Geodetic System 1984 (G1762)", + "id": { + "authority": "EPSG", + "code": 1156 + } + }, + { + "name": "World Geodetic System 1984 (G2139)", + "id": { + "authority": "EPSG", + "code": 1309 + } + }, + { + "name": "World Geodetic System 1984 (G2296)", + "id": { + "authority": "EPSG", + "code": 1383 + } + } + ], + "ellipsoid": { + "name": "WGS 84", + "semi_major_axis": 6378137, + "inverse_flattening": 298.257223563 + }, + "accuracy": "2.0", + "id": { + "authority": "EPSG", + "code": 6326 + } + }, + "coordinate_system": { + "subtype": "ellipsoidal", + "axis": [ + { + "name": "Geodetic latitude", + "abbreviation": "Lat", + "direction": "north", + "unit": "degree" + }, + { + "name": "Geodetic longitude", + "abbreviation": "Lon", + "direction": "east", + "unit": "degree" + } + ] + }, + "scope": "Horizontal component of 3D system.", + "area": "World.", + "bbox": { + "south_latitude": -90, + "west_longitude": -180, + "north_latitude": 90, + "east_longitude": 180 + }, + "id": { + "authority": "EPSG", + "code": 4326 + } + }, + "crs_type": "projjson" +} )JSON_LITERAL"); + + complex_json = StringUtil::ParseJSONMap(R"JSON_LITERAL( + { + "int": 42, + "signed_int": -42, + "real": 1.5, + "null_val": null, + "arr": [1, 2, 3], + "obj": { + "str_val": "val" + }, + "empty_arr": [], + "bool_t": true, + "bool_f": false + } + )JSON_LITERAL"); +} diff --git a/test/configs/block_size_16kB.json b/test/configs/block_size_16kB.json index bb63e9d67733..8b350368ec3a 100644 --- a/test/configs/block_size_16kB.json +++ b/test/configs/block_size_16kB.json @@ -8,6 +8,13 @@ "paths": [ "test/sql/storage/constraints/foreignkey/foreign_key_persistent_memory_limit.test" ] + }, + { + "reason": "ALP and ALPRD are disabled for non-default block sizes.", + "paths": [ + "test/sql/storage/compression/alp/alp_min_max.test", + "test/sql/storage/compression/alprd/alprd_min_max.test" + ] } ] } diff --git a/test/configs/compressed_in_memory.json b/test/configs/compressed_in_memory.json index a1dd7e12f358..e23d08fec751 100644 --- a/test/configs/compressed_in_memory.json +++ b/test/configs/compressed_in_memory.json @@ -5,13 +5,18 @@ "on_load": "skip", "skip_compiled": "true", "skip_tests": [ - "test/fuzzer/sqlsmith/current_schemas_null.test", - "test/sql/settings/drop_set_schema.test", - "test/sql/catalog/test_set_search_path.test", - "test/sql/storage/temp_directory/max_swap_space_explicit.test", - "test/sql/storage/temp_directory/max_swap_space_inmemory.test", - "test/sql/storage/temp_directory/max_swap_space_error.test", - "test/sql/storage/buffer_manager_temp_dir.test", - "test/sql/pg_catalog/system_functions.test" + { + "reason": "Unknown, to be investigated", + "paths": [ + "test/fuzzer/sqlsmith/current_schemas_null.test", + "test/sql/settings/drop_set_schema.test", + "test/sql/catalog/test_set_search_path.test", + "test/sql/storage/temp_directory/max_swap_space_explicit.test", + "test/sql/storage/temp_directory/max_swap_space_inmemory.test", + "test/sql/storage/temp_directory/max_swap_space_error.test", + "test/sql/storage/buffer_manager_temp_dir.test", + "test/sql/pg_catalog/system_functions.test" + ] + } ] } diff --git a/test/configs/enable_verification.json b/test/configs/enable_verification.json old mode 100644 new mode 100755 index cbbae4a6c6c2..41f048a40551 --- a/test/configs/enable_verification.json +++ b/test/configs/enable_verification.json @@ -113,7 +113,8 @@ "test/sql/copy/parquet/parquet_hive.test", "test/sql/copy/parquet/union_by_name_hive_partitioning.test", "test/sql/optimizer/test_in_rewrite_rule.test", - "test/sql/storage/compression/rle/rle_constant.test" + "test/sql/storage/compression/rle/rle_constant.test", + "test/sql/types/geo/geometry_stats.test" ] }, { @@ -199,8 +200,6 @@ "paths": [ "test/fuzzer/duckfuzz/null_arguments.test", "test/issues/internal/test_5457.test", - "test/parquet/variant/variant_nanos_tz.test", - "test/parquet/variant/variant_nested_with_nulls.test", "test/sql/aggregate/aggregates/test_state_export.test", "test/sql/copy/parquet/bloom_filters.test", "test/sql/copy/parquet/corrupt_stats.test", @@ -210,7 +209,19 @@ "test/sql/optimizer/test_rowid_pushdown_plan.test", "test/sql/pg_catalog/system_functions.test", "test/sql/storage/compression/test_using_compression.test", - "test/sql/storage/types/struct/nested_struct_storage.test" + "test/sql/error/error_position.test" + ] + }, + { + "reason": "Serialization Error: Unsupported type for deserialization of AlterInfo!", + "paths": [ + "test/sql/alter/test_alter_database_rename.test" + ] + }, + { + "reason": "Profiling in combination with verification is disabled for SELECT statements", + "paths": [ + "test/sql/pragma/profiling/test_profiling_fs.test" ] } ] diff --git a/test/configs/enable_verification_for_debug.json b/test/configs/enable_verification_for_debug.json index 29d9fce3fe28..9306cfdbad98 100644 --- a/test/configs/enable_verification_for_debug.json +++ b/test/configs/enable_verification_for_debug.json @@ -113,7 +113,8 @@ "test/sql/copy/parquet/parquet_hive.test", "test/sql/copy/parquet/union_by_name_hive_partitioning.test", "test/sql/optimizer/test_in_rewrite_rule.test", - "test/sql/storage/compression/rle/rle_constant.test" + "test/sql/storage/compression/rle/rle_constant.test", + "test/sql/types/geo/geometry_stats.test" ] }, { @@ -724,7 +725,13 @@ "test/sql/optimizer/test_rowid_pushdown_plan.test", "test/sql/pg_catalog/system_functions.test", "test/sql/storage/compression/test_using_compression.test", - "test/sql/storage/types/struct/nested_struct_storage.test" + "test/sql/error/error_position.test" + ] + }, + { + "reason": "Profiling in combination with verification is disabled for SELECT statements", + "paths": [ + "test/sql/pragma/profiling/test_profiling_fs.test" ] } ] diff --git a/test/configs/encryption.json b/test/configs/encryption.json index 2fbc92ce0a9d..d1b06ec47b66 100644 --- a/test/configs/encryption.json +++ b/test/configs/encryption.json @@ -35,7 +35,8 @@ "test/sql/table_function/duckdb_databases.test", "test/sql/copy_database/copy_table_with_sequence.test", "test/sql/logging/file_system_logging_attach.test", - "test/sql/logging/file_system_logging_attach_deadlock.test" + "test/sql/logging/file_system_logging_attach_deadlock.test", + "test/sql/storage/read_duckdb/read_duckdb_basic.test" ] }, { diff --git a/test/configs/force_storage.json b/test/configs/force_storage.json index d73360b3b143..a29cf31ae9b1 100644 --- a/test/configs/force_storage.json +++ b/test/configs/force_storage.json @@ -21,7 +21,8 @@ "test/sql/attach/attach_issue_7660.test", "test/sql/attach/show_databases.test", "test/sql/attach/attach_views.test", - "test/sql/copy_database/copy_table_with_sequence.test" + "test/sql/copy_database/copy_table_with_sequence.test", + "test/sql/storage/read_duckdb/concurrent_duckdb_read.test_slow" ] } ] diff --git a/test/configs/force_storage_restart.json b/test/configs/force_storage_restart.json index 83e7aa972eba..c604727e96f6 100644 --- a/test/configs/force_storage_restart.json +++ b/test/configs/force_storage_restart.json @@ -23,7 +23,8 @@ "test/sql/attach/show_databases.test", "test/sql/attach/attach_views.test", "test/sql/copy_database/copy_table_with_sequence.test", - "test/sql/attach/attach_use_rollback.test" + "test/sql/attach/attach_use_rollback.test", + "test/sql/storage/read_duckdb/concurrent_duckdb_read.test_slow" ] } ] diff --git a/test/configs/latest_storage.json b/test/configs/latest_storage.json index 3dd6f3f69c2a..c6b36e6b010b 100644 --- a/test/configs/latest_storage.json +++ b/test/configs/latest_storage.json @@ -3,6 +3,9 @@ "on_init": "ATTACH '__TEST_DIR__/{BASE_TEST_NAME}__test__config__latest__storage.db' AS __test__config__latest__storage (STORAGE_VERSION 'latest'); SET storage_compatibility_version='latest';", "on_new_connection": "USE __test__config__latest__storage;", "on_load": "skip", + "settings": [ + {"name": "storage_compatibility_version", "value": "latest"} + ], "skip_compiled": "true", "skip_tests": [ { @@ -35,7 +38,8 @@ "test/sql/table_function/duckdb_databases.test", "test/sql/copy_database/copy_table_with_sequence.test", "test/sql/logging/file_system_logging_attach.test", - "test/sql/logging/file_system_logging_attach_deadlock.test" + "test/sql/logging/file_system_logging_attach_deadlock.test", + "test/sql/storage/read_duckdb/read_duckdb_basic.test" ] }, { diff --git a/test/configs/latest_storage_block_size_16kB.json b/test/configs/latest_storage_block_size_16kB.json index 1ba67e4a7779..5030b7faabcd 100644 --- a/test/configs/latest_storage_block_size_16kB.json +++ b/test/configs/latest_storage_block_size_16kB.json @@ -3,6 +3,9 @@ "on_init": "ATTACH '__TEST_DIR__/{BASE_TEST_NAME}__test__config__latest_storage_block_size_16kB.db' AS __test__config__latest_storage_block_size_16kB (STORAGE_VERSION 'latest'); SET storage_compatibility_version='latest';", "on_new_connection": "USE __test__config__latest_storage_block_size_16kB;", "on_load": "skip", + "settings": [ + {"name": "storage_compatibility_version", "value": "latest"} + ], "skip_compiled": "true", "block_size": "16384", "skip_tests": [ @@ -27,7 +30,14 @@ "test/sql/show_select/test_describe_all.test", "test/sql/storage/buffer_manager_temp_dir.test", "test/sql/table_function/duckdb_databases.test", - "test/sql/tpch/dbgen_error.test" + "test/sql/tpch/dbgen_error.test", + "test/sql/storage/read_duckdb/read_duckdb_basic.test" + ] + }, + { + "reason": "ALP is disabled for non-default block sizes.", + "paths": [ + "test/sql/storage/compression/alp/alp_min_max.test" ] } ] diff --git a/test/configs/one_schema_per_test.json b/test/configs/one_schema_per_test.json new file mode 100644 index 000000000000..84eebaaa82ad --- /dev/null +++ b/test/configs/one_schema_per_test.json @@ -0,0 +1,126 @@ +{ + "description": "Run with verification enabled (suitable for debug builds).", + "on_init": "ATTACH '{TEST_DIR}/unittester_mashup.db' as ____db (STORAGE_VERSION 'v1.4.0'); DROP SCHEMA IF EXISTS ____db.\"{BASE_TEST_NAME}\" CASCADE; CREATE SCHEMA ____db.\"{BASE_TEST_NAME}\";", + "on_new_connection": "USE ____db.\"{BASE_TEST_NAME}\";", + "on_load": "skip", + "skip_compiled": "true", + "skip_error_messages": [ + "Catalog Error", "Unable to connect" + ], + "skip_tests": [ + { + "reason": "Tests that relies on specific DB/schema name", + "paths": [ + "test/sql/attach/show_databases.test", + "test/sql/table_function/duckdb_sequences.test", + "test/sql/table_function/sqlite_master_quotes.test", + "test/sql/table_function/duckdb_constraints_fk.test", + "test/sql/table_function/duckdb_tables.test", + "test/sql/table_function/duckdb_schemas.test", + "test/sql/table_function/sqlite_master.test", + "test/sql/table_function/duckdb_databases.test", + "test/sql/table_function/duckdb_views.test", + "test/sql/keywords/keywords_in_expressions.test", + "test/sql/attach/show_databases.test", + "test/sql/attach/attach_show_all_tables.test", + "test/sql/attach/attach_table_info.test", + "test/sql/pg_catalog/sqlalchemy.test", + "test/sql/pg_catalog/system_functions.test", + "test/sql/storage/temp_directory/max_swap_space_error.test", + "test/sql/pragma/test_show_tables_temp_views.test", + "test/sql/catalog/view/test_view_sql.test", + "test/sql/catalog/view/test_view_sql_with_dependencies.test", + "test/sql/catalog/function/test_table_macro.test", + "test/sql/catalog/test_set_search_path.test", + "test/sql/generated_columns/virtual/group_by.test", + "test/sql/show_select/test_describe_all.test", + "test/sql/settings/drop_set_schema.test", + "test/fuzzer/sqlsmith/current_schemas_null.test", + "test/sql/attach/attach_issue_7660.test", + "test/sql/attach/attach_issue7711.test", + "test/sql/attach/attach_table_constraints.test" + ] + }, + { + "reason": "Tests that list types / tables / schemas DB-wide, so can't come with extra schemas", + "paths": [ + "test/sql/settings/setting_preserve_identifier_case.test", + "test/sql/types/enum/test_enum_duckdb_types.test", + "test/sql/generated_columns/virtual/gcol_duckdb_columns.test", + "test/sql/constraints/foreignkey/fk_4365.test", + "test/sql/catalog/sequence/test_duckdb_sequences.test", + "test/sql/catalog/comment_on_pg_description.test", + "test/sql/catalog/dependencies/test_alter_owning_table.test", + "test/sql/catalog/comment_on_extended.test", + "test/sql/catalog/comment_on_dependencies.test", + "test/sql/constraints/foreignkey/fk_4365.test", + "test/sql/pg_catalog/pg_views.test", + "test/sql/pg_catalog/pg_constraint.test", + "test/sql/pg_catalog/pg_sequence.test", + "test/sql/pg_catalog/pg_attribute.test", + "test/sql/catalog/sequence/test_duckdb_sequences.test", + "test/sql/pg_catalog/pg_sequences.test", + "test/sql/attach/attach_show_table.test", + "test/sql/attach/attach_catalog_error_early_out.test", + "test/sql/index/art/create_drop/test_art_create_if_exists.test", + "test/sql/export/export_indexes.test", + "test/sql/copy_database/copy_database_with_index.test", + "test/sql/table_function/information_schema.test", + "test/sql/table_function/information_schema_issue12867.test", + "test/sql/table_function/duckdb_indexes.test", + "test/sql/table_function/duckdb_constraints.test", + "test/sql/table_function/information_schema_fkey_constraint_names.test", + "test/sql/table_function/duckdb_columns.test", + "test/sql/table_function/duckdb_constraints_issue11284.test", + "test/sql/table_function/test_information_schema_columns.test", + "test/sql/table_function/duckdb_constraints_issue12863.test" + ] + }, + { + "reason": "Tests that rely on DB to be basically empty to start with", + "paths": [ + "test/sql/alter/rename_table/test_rename_table.test", + "test/sql/alter/rename_table/test_rename_table_transactions.test" + ] + }, + { + "reason": "Weird EXPORT to CSV/parquet corner case: Too many open files", + "paths": [ + "test/sql/index/art/storage/test_art_import.test", + "test/sql/copy/csv/test_export_force_quotes.test", + "test/sql/copy/csv/test_export_not_null.test", + "test/sql/export/export_quoted_union.test", + "test/sql/export/export_external_access.test", + "test/sql/export/empty_export.test", + "test/sql/export/export_quoted_enum.test", + "test/sql/export/parquet_export.test", + "test/sql/export/export_quoted_structs.test", + "test/sql/export/export_hive_path.test", + "test/sql/export/export_compression_level.test", + "test/sql/export/export_generated_columns.test", + "test/sql/export/parquet/export_parquet_struct.test", + "test/sql/export/parquet/export_parquet_list.test", + "test/sql/export/parquet/export_parquet_map.test", + "test/sql/export/parquet/export_parquet_enum.test", + "test/sql/export/parquet/export_parquet_bit.test", + "test/sql/export/parquet/export_parquet_hugeint.test", + "test/sql/export/parquet/export_parquet_union.test" + ] + }, + { + "reason": "Test that rely on empty directory", + "paths": [ + "test/sql/copy/partitioned/hive_partitioned_auto_detect.test", + "test/sql/copy/parquet/writer/partition_without_hive.test", + "test/sql/copy/parquet/writer/skip_empty_write.test" + ] + }, + { + "reason": "Tests that rely on exporting the whole database, that has in the meanime exploded in size", + "paths": [ + "test/sql/constraints/foreignkey/test_fk_export.test" + ] + } + + ] +} diff --git a/test/configs/peg_parser.json b/test/configs/peg_parser.json new file mode 100644 index 000000000000..e869a4e80a95 --- /dev/null +++ b/test/configs/peg_parser.json @@ -0,0 +1,51 @@ +{ + "description": "Test PEG Parser + transformer", + "skip_compiled": "true", + "on_init": "set allow_parser_override_extension=fallback;", + "statically_loaded_extensions": [ + "core_functions", + "autocomplete" + ], + "skip_tests": [ + { + "reason": "SIGBUS", + "paths": [ + "test/sql/catalog/sequence/sequence_offset_increment.test", + "test/sql/upsert/insert_or_replace/returning_nothing.test", + "test/sql/copy_database/copy_database_different_types.test", + "test/sql/copy_database/copy_table_with_sequence.test", + "test/sql/join/external/external_join_many_duplicates.test_slow" + ] + }, + { + "reason": "Arithmetic expression", + "paths": [ + "test/sql/function/operator/test_arithmetic_sqllogic.test", + "test/sql/projection/test_row_id_expression.test", + "test/sql/function/numeric/test_trigo.test", + "test/sql/function/generic/test_between.test" + ] + }, + { + "reason": "Expression Depth", + "paths": [ + "test/sql/overflow/expression_tree_depth.test" + ] + }, + { + "reason": "Timeout", + "paths": [ + "test/sql/copy/csv/afl/test_fuzz_3981.test_slow", + "test/sql/join/external/tpch_all_tables.test_slow", + "test/sql/storage/encryption/temp_files/encrypted_offloading_block_files.test_slow", + "test/sql/storage/temp_directory/offloading_block_files.test_slow" + ] + }, + { + "reason": "Setting option to 'fallback' changes behavior of first tests", + "paths": [ + "test/extension/loadable_parser_override.test" + ] + } + ] +} \ No newline at end of file diff --git a/test/configs/prefetch_all_storage.json b/test/configs/prefetch_all_storage.json new file mode 100644 index 000000000000..dc889cee9de6 --- /dev/null +++ b/test/configs/prefetch_all_storage.json @@ -0,0 +1,5 @@ +{ + "base_config": "test/configs/force_storage.json", + "description": "Run tests with debug prefetching enabled", + "on_init": "SET storage_block_prefetch='debug_force_always'" +} diff --git a/test/configs/storage_compatibility.json b/test/configs/storage_compatibility.json new file mode 100644 index 000000000000..5738d60599b6 --- /dev/null +++ b/test/configs/storage_compatibility.json @@ -0,0 +1,56 @@ +{ + "description": "Storage compatibility test.", + "initial_db": "bwc_storage_test.db", + "settings": [ + {"name": "storage_compatibility_version", "value": "v1.0.0"} + ], + "skip_compiled": "true", + "skip_tests": [ + { + "reason": "Contains explicit use of the memory catalog.", + "paths": [ + "test/sql/show_select/test_describe_all.test", + "test/sql/catalog/function/attached_macro.test", + "test/sql/catalog/test_temporary.test", + "test/sql/pragma/test_show_tables_temp_views.test", + "test/sql/pg_catalog/system_functions.test", + "test/sql/pg_catalog/sqlalchemy.test", + "test/sql/attach/attach_table_info.test", + "test/sql/attach/attach_defaults.test", + "test/sql/attach/attach_did_you_mean.test", + "test/sql/attach/attach_default_table.test", + "test/sql/attach/attach_show_all_tables.test", + "test/sql/attach/attach_issue7711.test", + "test/sql/attach/attach_issue_7660.test", + "test/sql/attach/show_databases.test", + "test/sql/attach/attach_views.test", + "test/sql/copy_database/copy_table_with_sequence.test" + ] + }, + { + "reason": "Stringification too slow", + "paths": [ + "test/sql/types/bignum/test_bignum_sum.test" + ] + }, + { + "reason": "Time (NS) not supported (new type).", + "paths": [ + "test/parquet/timens_parquet.test", + "test/sql/types/time/test_time_ns.test" + ] + }, + { + "reason": "Expected forwards compatibility failure.", + "paths": [ + "test/fuzzer/pedro/view_not_rebound_error_no_view_dependencies.test", + "test/issues/rigger/assertion_scale.test", + "test/issues/general/test_16662.test", + "test/sql/copy/csv/test_null_padding_projection.test", + "test/sql/types/geo/geometry.test", + "test/sql/types/geo/geometry_stats.test", + "test/sql/types/variant/variant_distinct.test" + ] + } + ] +} diff --git a/test/configs/wal_verification.json b/test/configs/wal_verification.json index 173bb6c06619..49142f20951b 100644 --- a/test/configs/wal_verification.json +++ b/test/configs/wal_verification.json @@ -25,7 +25,8 @@ "test/sql/pg_catalog/system_functions.test", "test/sql/pragma/test_show_tables_temp_views.test", "test/sql/show_select/test_describe_all.test", - "test/sql/attach/attach_use_rollback.test" + "test/sql/attach/attach_use_rollback.test", + "test/sql/storage/read_duckdb/read_duckdb_basic.test" ] }, {"reason": "Loading from disk is necessary to trigger expected OOM.", @@ -69,6 +70,7 @@ "test/sql/index/art/types/test_art_expression_key.test", "test/sql/index/art/types/test_art_union.test", "test/sql/types/alias/type_with_schema.test", + "test/sql/types/struct/create_type_struct.test", "test/sql/types/enum/test_alter_enum.test", "test/sql/types/enum/test_enum_schema.test", "test/sql/types/enum/test_enum_structs.test" @@ -93,6 +95,12 @@ "test/fuzzer/sqlsmith/nullif_map_with_config.test", "test/sql/storage/memory/in_memory_compress.test" ] + }, + { + "reason": "FIXME: Attempting to checkpoint read-only database.", + "paths": [ + "test/sql/storage/read_duckdb/read_duckdb_index.test" + ] } ] } \ No newline at end of file diff --git a/test/extension/autoloading_base.test b/test/extension/autoloading_base.test deleted file mode 100644 index 1aca13346aff..000000000000 --- a/test/extension/autoloading_base.test +++ /dev/null @@ -1,103 +0,0 @@ -# name: test/extension/autoloading_base.test -# description: Base tests for the autoloading mechanism for extensions -# group: [extension] - -require httpfs - -# This test assumes icu and json to be available in the LOCAL_EXTENSION_REPO and NOT linked into duckdb statically -# -> this should be the case for our autoloading tests where we have the local_extension_repo variable set -require-env LOCAL_EXTENSION_REPO - -# Ensure we have a clean extension directory without any preinstalled extensions -statement ok -set extension_directory='__TEST_DIR__/autoloading_base' - -query I -SELECT (count(*) > 0) FROM duckdb_extensions() WHERE install_path ILIKE '%duckdb_extension' ----- -false - -# All extensions reported by duckdb are either statically linked or not installed -query I -SELECT count(*) FROM duckdb_extensions() WHERE install_mode != 'NOT_INSTALLED' AND install_mode != 'STATICALLY_LINKED' ----- -0 - -### No autoloading nor installing: throw error with installation hint -statement ok -set autoload_known_extensions=false - -statement ok -set autoinstall_known_extensions=false - -statement error -SET s3_region='eu-west-1'; ----- -:.*Catalog Error.*Setting with name "s3_region" is not in the catalog.* - -statement error -select * from read_json_auto('data/json/example_n.ndjson'); ----- -:.*Catalog Error.*Table Function with name "read_json_auto" is not in the catalog.* - -statement error -select * from thistablefunctionwillnotexistfosho(); ----- -:.*Catalog Error.*Table Function with name thistablefunctionwillnotexistfosho does not exist.* - -### Autoloading and installing, but the autoloading repository is set to non-existent location -statement ok -set autoload_known_extensions=true - -statement ok -set autoinstall_known_extensions=true - -# Override the default repo with a non-existent local repo -statement ok -set autoinstall_extension_repository='/tmp/non-existent-repo'; - -# Error should inform the user on whats happening -statement error -SET s3_region='eu-west-1'; ----- -:Extension Autoloading Error.*An error occurred while trying to automatically install the required extension 'httpfs'.* - -statement error -select * from read_json_auto('data/json/example_n.ndjson'); ----- -:Extension Autoloading Error.*An error occurred while trying to automatically install the required extension 'json'.* - -# Now override with non-existent remote repo -statement ok -set autoinstall_extension_repository='http://duckdb.org/what/are/the/odds/we/actually/make/this/path/and/break/this/tests'; - -# Error should inform the user on whats happening -statement error -SET s3_region='eu-west-1'; ----- -:Extension Autoloading Error.*An error occurred while trying to automatically install the required extension 'httpfs'.* - -statement error -select * from read_json_auto('data/json/example_n.ndjson'); ----- -:Extension Autoloading Error.*An error occurred while trying to automatically install the required extension 'json'.* - -statement error -select * from thistablefunctionwillnotexistfosho(); ----- -:Catalog Error.*Table Function with name thistablefunctionwillnotexistfosho does not exist.* - -### Autoloading with correct tmp repo -statement ok -set autoinstall_extension_repository='${LOCAL_EXTENSION_REPO}'; - -statement ok -SET s3_region='eu-west-1'; - -statement ok -select * from read_json_auto('data/json/example_n.ndjson'); - -query I -SELECT (count(*) > 0) FROM duckdb_extensions() WHERE install_path ILIKE '%duckdb_extension'; ----- -true diff --git a/test/extension/autoloading_copy_function_target.test b/test/extension/autoloading_copy_function_target.test new file mode 100644 index 000000000000..84cab06e6f96 --- /dev/null +++ b/test/extension/autoloading_copy_function_target.test @@ -0,0 +1,19 @@ +# name: test/extension/autoloading_copy_function_target.test +# description: Tests for autoloading copy functions targets +# group: [extension] + +statement ok +set autoload_known_extensions=false + +statement ok +set autoinstall_known_extensions=false + +statement error +copy (select 1337 as edgy_hacker_number) TO 's3://non-existent-bucket/test1337.csv' +---- +:.*Missing Extension Error: File .*test1337.csv requires the extension httpfs to be loaded.* + +statement error +copy (select 1337 as edgy_hacker_number) TO 'azure://non-existent-bucket/test1337.csv' +---- +:.*Missing Extension Error: File .*test1337.csv requires the extension azure to be loaded.* diff --git a/test/extension/autoloading_current_setting.test b/test/extension/autoloading_current_setting.test deleted file mode 100644 index 119e63ff82c2..000000000000 --- a/test/extension/autoloading_current_setting.test +++ /dev/null @@ -1,47 +0,0 @@ -# name: test/extension/autoloading_current_setting.test -# description: More tests for extension autoloading. -# group: [extension] - -# This test assumes icu and json to be available in the LOCAL_EXTENSION_REPO and NOT linked into duckdb statically -# -> this should be the case for our autoloading tests where we have the local_extension_repo variable set -require-env LOCAL_EXTENSION_REPO - -require httpfs - -statement ok -set extension_directory='__TEST_DIR__/autoloading_current_setting' - -### No autoloading: throw error with installation hint -statement ok -set autoload_known_extensions=false - -statement ok -set autoinstall_known_extensions=false - -statement error -select current_setting('s3_region'); ----- -:.*Catalog Error.*Setting with name "s3_region" is not in the catalog.* - -### Autoloading, but but not autoinstall -statement ok -set autoload_known_extensions=true - -statement ok -set autoinstall_extension_repository='/tmp/non-existent-repo'; - -# Error should inform the user on whats happening -statement error -select current_setting('s3_region'); ----- -:.*Extension Autoloading Error.*An error occurred while trying to automatically install the required extension 'httpfs'.* - -### Autoloading with autoinstall and correct extension repo -statement ok -set autoinstall_extension_repository='${LOCAL_EXTENSION_REPO}'; - -statement ok -set autoinstall_known_extensions=true - -statement ok -select current_setting('s3_region'); diff --git a/test/extension/autoloading_filesystems.test b/test/extension/autoloading_filesystems.test deleted file mode 100644 index 8ad6b9027ca2..000000000000 --- a/test/extension/autoloading_filesystems.test +++ /dev/null @@ -1,47 +0,0 @@ -# name: test/extension/autoloading_filesystems.test -# description: Tests for autoloading with filesystems -# group: [extension] - -require httpfs - -# This test assumes icu and json to be available in the LOCAL_EXTENSION_REPO and NOT linked into duckdb statically -# -> this should be the case for our autoloading tests where we have the local_extension_repo variable set -require-env LOCAL_EXTENSION_REPO - -statement ok -set allow_persistent_secrets=false; - -# Ensure we have a clean extension directory without any preinstalled extensions -statement ok -set extension_directory='__TEST_DIR__/autoloading_filesystems' - -### No autoloading nor installing: throw error with installation hint -statement ok -set autoload_known_extensions=false - -statement ok -set autoinstall_known_extensions=false - -statement error -SELECT * FROM 's3://some-bucket/a-file.csv' ----- -Missing Extension Error: File s3://some-bucket/a-file.csv requires the extension httpfs to be loaded - -### With autoloading, install and correct repo -statement ok -set autoload_known_extensions=true - -statement ok -set autoinstall_known_extensions=true - -statement ok -set autoinstall_extension_repository='${LOCAL_EXTENSION_REPO}'; - -# Set an invalid endpoint to ensure we fail in the httpfs extension when trying to connect -statement ok -SET s3_endpoint='false_endpoint'; - -statement error -SELECT * FROM 's3://some-bucket/a-file.csv' ----- -Could not establish connection error for HTTP HEAD to 'https://some-bucket.false_endpoint/a-file.csv' diff --git a/test/extension/autoloading_load_only.test b/test/extension/autoloading_load_only.test deleted file mode 100644 index 5bbda345b59b..000000000000 --- a/test/extension/autoloading_load_only.test +++ /dev/null @@ -1,45 +0,0 @@ -# name: test/extension/autoloading_load_only.test -# description: Tests for autoloading with no autoinstall -# group: [extension] - -require httpfs - -# This test assumes icu and json to be available in the LOCAL_EXTENSION_REPO and NOT linked into duckdb statically -# -> this should be the case for our autoloading tests where we have the local_extension_repo variable set -require-env LOCAL_EXTENSION_REPO - -# Ensure we have a clean extension directory without any preinstalled extensions -statement ok -set extension_directory='__TEST_DIR__/autoloading_load_only' - -### No autoloading nor installing: throw error with installation hint -statement ok -set autoload_known_extensions=false - -statement ok -set autoinstall_known_extensions=false - -statement error -SET s3_region='eu-west-1'; ----- -:.*Catalog Error.*Setting with name "s3_region" is not in the catalog.* - -### Autoloading but not autoinstall, while the extension is not installed: still not working -statement ok -set autoload_known_extensions=true - -statement ok -set autoinstall_extension_repository='/tmp/non-existent-repo'; - -statement error -SET s3_region='eu-west-1'; ----- -:.*Extension Autoloading Error.*An error occurred while trying to automatically install the required extension 'httpfs'.* - -### Manually install the extension from the local repo -statement ok -INSTALL httpfs FROM '${LOCAL_EXTENSION_REPO}' - -# now autoloading works! -statement ok -SET s3_region='eu-west-1'; diff --git a/test/extension/autoloading_reset_setting.test b/test/extension/autoloading_reset_setting.test deleted file mode 100644 index 999102a0436a..000000000000 --- a/test/extension/autoloading_reset_setting.test +++ /dev/null @@ -1,49 +0,0 @@ -# name: test/extension/autoloading_reset_setting.test -# description: Testing reset setting that lives in an extension that can be autoloaded -# group: [extension] - -require httpfs - -# This test assumes httpfs and json to be available in the LOCAL_EXTENSION_REPO and NOT linked into duckdb statically -# -> this should be the case for our autoloading tests where we have the local_extension_repo variable set -require-env LOCAL_EXTENSION_REPO - -statement ok -set extension_directory='__TEST_DIR__/autoloading_reset_setting' - -### No autoloading: throw error with installation hint -statement ok -set autoload_known_extensions=false - -statement ok -set autoinstall_known_extensions=false - -# Testing reset setting -statement error -RESET s3_region; ----- -Catalog Error: Setting with name "s3_region" is not in the catalog, but it exists in the httpfs extension. - -### Autoloading, but no auto install -statement ok -set autoload_known_extensions=true - -statement ok -set autoinstall_extension_repository='/tmp/non-existent-repo'; - -# Error should inform the user on whats happening -statement error -RESET s3_region; ----- -Extension Autoloading Error: An error occurred while trying to automatically install the required extension 'httpfs': -Extension - -### Autoloading with correct tmp repo and autoinstall -statement ok -set autoinstall_extension_repository='${LOCAL_EXTENSION_REPO}'; - -statement ok -set autoinstall_known_extensions=true - -statement ok -RESET s3_region; diff --git a/test/extension/consistent_semicolon_extension_parse.test b/test/extension/consistent_semicolon_extension_parse.test new file mode 100644 index 000000000000..9d26ed2c85e6 --- /dev/null +++ b/test/extension/consistent_semicolon_extension_parse.test @@ -0,0 +1,26 @@ +# name: test/extension/consistent_semicolon_extension_parse.test +# description: Original issue: https://github.com/duckdb/duckdb/issues/18485 +# group: [extension] + +statement ok +LOAD '__BUILD_DIRECTORY__/test/extension/loadable_extension_demo.duckdb_extension'; + +statement error +quack (foo); +---- +Parser Error: This is not a quack: (foo); + +statement error +quack 's'; +---- +Parser Error: This is not a quack: 's'; + +statement error +unrecognizedkeyword ';';anotherunrecognizedkeyword';';;;;; +---- +Parser Error: syntax error at or near "unrecognizedkeyword" + +statement error +QUACK string_split('hello-world', ';');QUACK string_split('hello-world', ';') +---- +string_split('hello-world', ';'); \ No newline at end of file diff --git a/test/extension/duckdb_extension_settings.test b/test/extension/duckdb_extension_settings.test deleted file mode 100644 index 9fa62c6f5bc0..000000000000 --- a/test/extension/duckdb_extension_settings.test +++ /dev/null @@ -1,30 +0,0 @@ -# name: test/extension/duckdb_extension_settings.test -# description: settings for extensions -# group: [extension] - -require httpfs - -statement ok -SET autoinstall_known_extensions = true; - -statement ok -SET autoload_known_extensions = true; - -statement ok -SET extension_directory = '__TEST_DIR__/custom_extension_directory'; - -statement ok -SET custom_extension_repository = '__TEST_DIR__/not_existing_folder' - -statement error -FROM read_csv('https://some.org/file.csv'); ----- -not_existing_folder - -statement ok -SET autoinstall_extension_repository = '__TEST_DIR__/other_folder'; - -statement error -FROM read_csv('https://some.org/file.csv'); ----- -other_folder diff --git a/test/extension/loadable_extension_demo.cpp b/test/extension/loadable_extension_demo.cpp index 16a406703c94..20ca60987cf3 100644 --- a/test/extension/loadable_extension_demo.cpp +++ b/test/extension/loadable_extension_demo.cpp @@ -12,6 +12,10 @@ #include "duckdb/common/exception/conversion_exception.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/common/extension_type_info.hpp" +#include "duckdb/parser/sql_statement.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/tableref/emptytableref.hpp" using namespace duckdb; @@ -192,10 +196,11 @@ class QuackExtension : public ParserExtension { QuackExtension() { parse_function = QuackParseFunction; plan_function = QuackPlanFunction; + parser_override = QuackParser; } static ParserExtensionParseResult QuackParseFunction(ParserExtensionInfo *info, const string &query) { - auto lcase = StringUtil::Lower(StringUtil::Replace(query, ";", "")); + auto lcase = StringUtil::Lower(query); if (!StringUtil::Contains(lcase, "quack")) { // quack not found!? if (StringUtil::Contains(lcase, "quac")) { @@ -210,11 +215,14 @@ class QuackExtension : public ParserExtension { StringUtil::Trim(split); if (!split.empty()) { // we only accept quacks here + if (StringUtil::CIEquals(split, ";")) { + continue; + } return ParserExtensionParseResult("This is not a quack: " + split); } } // QUACK - return ParserExtensionParseResult(make_uniq(splits.size() + 1)); + return ParserExtensionParseResult(make_uniq(splits.size())); } static ParserExtensionPlanResult QuackPlanFunction(ParserExtensionInfo *info, ClientContext &context, @@ -228,6 +236,30 @@ class QuackExtension : public ParserExtension { result.return_type = StatementReturnType::QUERY_RESULT; return result; } + + static ParserOverrideResult QuackParser(ParserExtensionInfo *info, const string &query) { + vector queries = StringUtil::Split(query, ";"); + vector> statements; + for (const auto &query_input : queries) { + if (StringUtil::CIEquals(query_input, "override")) { + auto select_node = make_uniq(); + select_node->select_list.push_back( + make_uniq(Value("The DuckDB parser has been overridden"))); + select_node->from_table = make_uniq(); + auto select_statement = make_uniq(); + select_statement->node = std::move(select_node); + statements.push_back(std::move(select_statement)); + } + if (StringUtil::CIEquals(query_input, "over")) { + auto exception = ParserException("Parser overridden, query equaled \"over\" but not \"override\""); + return ParserOverrideResult(exception); + } + } + if (statements.empty()) { + return ParserOverrideResult(); + } + return ParserOverrideResult(std::move(statements)); + } }; static set test_loaded_extension_list; diff --git a/test/extension/loadable_parser_override.test b/test/extension/loadable_parser_override.test new file mode 100644 index 000000000000..b6589e7bc128 --- /dev/null +++ b/test/extension/loadable_parser_override.test @@ -0,0 +1,64 @@ +# name: test/extension/loadable_parser_override.test +# description: Try loading a parser override with an extension +# group: [extension] + +require skip_reload + +statement error +override +---- +Parser Error: syntax error at or near "override" + +statement ok +LOAD '__BUILD_DIRECTORY__/test/extension/loadable_extension_demo.duckdb_extension'; + +statement error +set allow_parser_override_extension=doesnotexist; +---- +Invalid Input Error: Unrecognized value for parser override setting. Valid options are: "default", "fallback", "strict". + +# Default behavior is not using the parser override +statement error +override +---- +Parser Error: syntax error at or near "override" + +# Fallback behavior is trying the parser override and if they error falling back to the default parser +statement ok +set allow_parser_override_extension=fallback; + +# The QuackParser can return a valid SQLStatement for this query +query I +override +---- +The DuckDB parser has been overridden + +# The parser cannot return a valid SQLStatement for this query, the default parser also gets an error +statement error +over +---- +Parser Error: syntax error at or near "over" + + +query I +SELECT 1; +---- +1 + +statement ok +set allow_parser_override_extension=strict; + +query I +override +---- +The DuckDB parser has been overridden + +statement error +over +---- +Parser Error: Parser override could not parse this query. (Original error: Parser overridden, query equaled "over" but not "override") + +statement error +SELECT 1; +---- +:.*Parser Error: Parser override failed.* \ No newline at end of file diff --git a/test/fuzzer/afl/alter_if_exists.test b/test/fuzzer/afl/alter_if_exists.test index d9a87eb4591d..bd77b968efef 100644 --- a/test/fuzzer/afl/alter_if_exists.test +++ b/test/fuzzer/afl/alter_if_exists.test @@ -8,3 +8,4 @@ ALTER TABLE IF EXISTS t0 ADD COLUMN c1 INT; statement error ALTER TABLE t0 ADD COLUMN c1 INT; ---- +Catalog Error: Table with name t0 does not exist \ No newline at end of file diff --git a/test/fuzzer/afl/invalid_foreign_key.test b/test/fuzzer/afl/invalid_foreign_key.test index 3be7cfc85e8e..7562c9a65851 100644 --- a/test/fuzzer/afl/invalid_foreign_key.test +++ b/test/fuzzer/afl/invalid_foreign_key.test @@ -8,3 +8,4 @@ PRAGMA enable_verification statement error create table y(y int, foreign key (y) references y(y)); ---- +:.*Binder Error: Failed to create foreign key.*referenced table "y".* \ No newline at end of file diff --git a/test/fuzzer/afl/window_function_binder_error.test b/test/fuzzer/afl/window_function_binder_error.test index 6ee138446eea..61ca5c919e4f 100644 --- a/test/fuzzer/afl/window_function_binder_error.test +++ b/test/fuzzer/afl/window_function_binder_error.test @@ -13,3 +13,4 @@ SELECT count(x) OVER () FROM (SELECT 1) x(x); statement error SELECT count(x) OVER () FROM (SELECT 1) x(x) GROUP BY ALL; ---- +Binder Error: Cannot group on a window clause \ No newline at end of file diff --git a/test/fuzzer/duckfuzz/arrow_scan_subquery_nullptr.test b/test/fuzzer/duckfuzz/arrow_scan_subquery_nullptr.test index 3c305adea560..46566b5f518d 100644 --- a/test/fuzzer/duckfuzz/arrow_scan_subquery_nullptr.test +++ b/test/fuzzer/duckfuzz/arrow_scan_subquery_nullptr.test @@ -6,3 +6,4 @@ statement error SELECT * FROM (SELECT * FROM arrow_scan(NULL, NULL, NULL)); Binder Error: arrow_scan: pointers cannot be null ---- +Parser Error: syntax error at or near "Binder" \ No newline at end of file diff --git a/test/fuzzer/duckfuzz/pivot_aggregate_mismatch.test b/test/fuzzer/duckfuzz/pivot_aggregate_mismatch.test index e5eab7a7703b..3ba4dc5c0e9b 100644 --- a/test/fuzzer/duckfuzz/pivot_aggregate_mismatch.test +++ b/test/fuzzer/duckfuzz/pivot_aggregate_mismatch.test @@ -2,9 +2,12 @@ # description: Pivot with non-aggregate function # group: [duckfuzz] +require icu; + statement ok PRAGMA enable_verification statement error FROM (VALUES (42)) t(i) PIVOT (current_date() FOR (i) IN (41)) ---- +Binder Error: Pivot expression must contain exactly one aggregate \ No newline at end of file diff --git a/test/fuzzer/pedro/complex_offset_clause_crash.test b/test/fuzzer/pedro/complex_offset_clause_crash.test index 71520689d13e..9bde286493a4 100644 --- a/test/fuzzer/pedro/complex_offset_clause_crash.test +++ b/test/fuzzer/pedro/complex_offset_clause_crash.test @@ -8,3 +8,4 @@ PRAGMA enable_verification statement error SELECT 6 OFFSET count(*) FILTER ((SELECT 2 UNION (SELECT 2) OFFSET (SELECT LAST))) OVER (); ---- +Not implemented Error: Unimplemented expression class \ No newline at end of file diff --git a/test/fuzzer/pedro/decimal_with_invalid_scale.test b/test/fuzzer/pedro/decimal_with_invalid_scale.test index 3c4ee191cd09..037995a3c8ca 100644 --- a/test/fuzzer/pedro/decimal_with_invalid_scale.test +++ b/test/fuzzer/pedro/decimal_with_invalid_scale.test @@ -5,3 +5,4 @@ statement error CREATE TABLE x(x DECIMAL(38763269, 77914819)); ---- +Parser Error: Width must be between 1 and 38 \ No newline at end of file diff --git a/test/fuzzer/pedro/force_no_cross_product.test b/test/fuzzer/pedro/force_no_cross_product.test index 4ab2b1c562b5..9ef28f218d25 100644 --- a/test/fuzzer/pedro/force_no_cross_product.test +++ b/test/fuzzer/pedro/force_no_cross_product.test @@ -8,3 +8,4 @@ PRAGMA debug_force_no_cross_product=1; statement error SELECT 1 FROM (SELECT 1), (SELECT 1); ---- +Invalid Input Error: Query requires a cross-product, but 'force_no_cross_product' \ No newline at end of file diff --git a/test/fuzzer/pedro/temp_sequence_durability.test b/test/fuzzer/pedro/temp_sequence_durability.test index 3f9612325fa2..a5c82a7aaf3b 100644 --- a/test/fuzzer/pedro/temp_sequence_durability.test +++ b/test/fuzzer/pedro/temp_sequence_durability.test @@ -18,3 +18,4 @@ restart statement error SELECT nextval('s0'); ---- +Catalog Error: Sequence with name s0 does not exist \ No newline at end of file diff --git a/test/fuzzer/pedro/update_default_segv.test b/test/fuzzer/pedro/update_default_segv.test index 3ec03ff8a36a..bccb594e62e0 100644 --- a/test/fuzzer/pedro/update_default_segv.test +++ b/test/fuzzer/pedro/update_default_segv.test @@ -18,3 +18,4 @@ CREATE TABLE t1(c4 UUID, c0 REAL AS(0)); statement error ALTER TABLE t1 ALTER c0 SET DEFAULT 0; ---- +Binder Error: Cannot SET DEFAULT for generated column "c0" \ No newline at end of file diff --git a/test/fuzzer/pedro/view_not_rebound_error_no_view_dependencies.test b/test/fuzzer/pedro/view_not_rebound_error_no_view_dependencies.test index b2719a97e973..29006da6a116 100644 --- a/test/fuzzer/pedro/view_not_rebound_error_no_view_dependencies.test +++ b/test/fuzzer/pedro/view_not_rebound_error_no_view_dependencies.test @@ -22,3 +22,4 @@ CREATE TABLE t1 (c2 INT); statement error SELECT 1 FROM t2 JOIN t1 ON (SELECT TRUE FROM t0); ---- +Binder Error: Referenced column "c1" not found in FROM clause \ No newline at end of file diff --git a/test/fuzzer/public/pivot_empty_enum.test b/test/fuzzer/public/pivot_empty_enum.test index 89caa26ba7b7..68d1dc7d3151 100644 --- a/test/fuzzer/public/pivot_empty_enum.test +++ b/test/fuzzer/public/pivot_empty_enum.test @@ -11,3 +11,4 @@ CREATE TABLE t1 (c01 INT); statement error PIVOT_WIDER t1 ON c01 IN ENUM; ---- +Binder Error: ENUM type is incomplete \ No newline at end of file diff --git a/test/fuzzer/sqlsmith/strptime_nullpointer.test b/test/fuzzer/sqlsmith/strptime_nullpointer.test index 8ec322a4b2f4..90dbb2b27e45 100644 --- a/test/fuzzer/sqlsmith/strptime_nullpointer.test +++ b/test/fuzzer/sqlsmith/strptime_nullpointer.test @@ -14,3 +14,4 @@ INSERT INTO t0(c0, c1) VALUES (534898561, 1156365055), (524523641, '0.4668052595 statement error SELECT NULL FROM t0 ORDER BY strptime(NOT(t0.c0 BETWEEN t0.rowid AND t0.rowid), 1407974306) ---- +Binder Error: No function matches the given name and argument types 'strptime \ No newline at end of file diff --git a/test/fuzzer/sqlsmith/timestamp_diff_overflow.test b/test/fuzzer/sqlsmith/timestamp_diff_overflow.test index 117edf6b7caf..e17d11c8210c 100644 --- a/test/fuzzer/sqlsmith/timestamp_diff_overflow.test +++ b/test/fuzzer/sqlsmith/timestamp_diff_overflow.test @@ -25,3 +25,4 @@ select from ts as ref_0 ---- +Conversion Error: Timestamp difference is out of bounds \ No newline at end of file diff --git a/test/fuzzer/sqlsmith/window_function_error.test b/test/fuzzer/sqlsmith/window_function_error.test index e557445c321d..2a13249248b6 100644 --- a/test/fuzzer/sqlsmith/window_function_error.test +++ b/test/fuzzer/sqlsmith/window_function_error.test @@ -12,3 +12,4 @@ statement error SELECT max(CAST(from_hex(CAST(ref_0.r_name AS VARCHAR)) AS BLOB)) OVER (ORDER BY ref_0.r_name) AS c5, FROM tbl AS ref_0 ---- +Invalid Input Error: Invalid input for hex digit: t \ No newline at end of file diff --git a/test/geoparquet/disabled.test b/test/geoparquet/disabled.test index 9a08bb5fb67f..c4ba9b51acd9 100644 --- a/test/geoparquet/disabled.test +++ b/test/geoparquet/disabled.test @@ -51,7 +51,7 @@ query I SELECT (decode(value)) as col FROM parquet_kv_metadata('__TEST_DIR__/data-point-out-enabled.parquet'); ---- -{"version":"1.1.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":["Point"],"bbox":[30.0,10.0,40.0,40.0]}}} +{"version":"1.0.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":["Point"],"bbox":[30.0,10.0,40.0,40.0]}}} # Now disable conversion diff --git a/test/geoparquet/mixed.test b/test/geoparquet/mixed.test index ae5470c1a186..d6679c6a5f50 100644 --- a/test/geoparquet/mixed.test +++ b/test/geoparquet/mixed.test @@ -49,4 +49,13 @@ query I SELECT (decode(value)) as col FROM parquet_kv_metadata('__TEST_DIR__/t1.parquet'); ---- -{"version":"1.1.0","primary_column":"geom","columns":{"geom":{"encoding":"WKB","geometry_types":["Point","LineString","Polygon","MultiPoint","MultiLineString","MultiPolygon","GeometryCollection","Point Z"],"bbox":[0.0,0.0,3.0,3.0]}}} +{"version":"1.0.0","primary_column":"geom","columns":{"geom":{"encoding":"WKB","geometry_types":["Point","LineString","Polygon","MultiPoint","MultiLineString","MultiPolygon","GeometryCollection","Point Z"],"bbox":[0.0,0.0,1.0,3.0,3.0,1.0]}}} + + +#------------------------------------------------------------------------------ +# Write with RETURN_STATS +#------------------------------------------------------------------------------ +query IIIIII +COPY (SELECT * FROM t1) TO '__TEST_DIR__/t1.parquet' (FORMAT 'parquet', RETURN_STATS); +---- +:.*t1.parquet 8 :\d+ :\d+ :{'"col"'={column_size_bytes=\d+, max=8, min=1, null_count=0}, '"geom"'={bbox_xmax=3.0, bbox_xmin=0.0, bbox_ymax=3.0, bbox_ymin=0.0, bbox_zmax=1.0, bbox_zmin=1.0, column_size_bytes=\d+, geo_types='\[point, linestring, polygon, multipoint, multilinestring, multipolygon, geometrycollection, point_z\]', null_count=0}} NULL diff --git a/test/geoparquet/roundtrip.test b/test/geoparquet/roundtrip.test index 8c313267b5a0..fa4c6e5de2c8 100644 --- a/test/geoparquet/roundtrip.test +++ b/test/geoparquet/roundtrip.test @@ -138,4 +138,4 @@ query I SELECT decode(value) as col FROM parquet_kv_metadata('__TEST_DIR__/data-multipolygon-out.parquet') WHERE key = 'geo'; ---- -{"version":"1.1.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":["MultiPolygon"],"bbox":[5.0,5.0,45.0,45.0]}}} \ No newline at end of file +{"version":"1.0.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":["MultiPolygon"],"bbox":[5.0,5.0,45.0,45.0]}}} \ No newline at end of file diff --git a/test/geoparquet/unsupported.test b/test/geoparquet/unsupported.test index 5ca0863eb3a8..9c01633e2f6d 100644 --- a/test/geoparquet/unsupported.test +++ b/test/geoparquet/unsupported.test @@ -8,14 +8,33 @@ require parquet #------------------------------------------------------------------------------ # Test unsupported geometry type #------------------------------------------------------------------------------ +# This is now ok, but we dont write the geoparquet metadata -statement error -COPY (SELECT 'POINT ZM (0 1 2 3)'::GEOMETRY) TO '__TEST_DIR__/t1.parquet' (FORMAT 'parquet'); +statement ok +COPY (SELECT 'POINT ZM (0 1 2 3)'::GEOMETRY as geometry) TO '__TEST_DIR__/t1.parquet' (FORMAT 'parquet'); + +# Not a geoparquet file +query I +SELECT (decode(value)) as col +FROM parquet_kv_metadata('__TEST_DIR__/t1.parquet'); +---- + +# But still a normal parquet file +query I +SELECT st_astext(st_geomfromwkb(geometry)) FROM '__TEST_DIR__/t1.parquet'; ---- -Invalid Input Error: Geoparquet does not support geometries with M coordinates +POINT ZM (0 1 2 3) +statement ok +COPY (SELECT 'POINT M (0 1 2)'::GEOMETRY as geometry) TO '__TEST_DIR__/t1.parquet' (FORMAT 'parquet'); + +query I +SELECT (decode(value)) as col +FROM parquet_kv_metadata('__TEST_DIR__/t1.parquet'); +---- -statement error -COPY (SELECT 'POINT M (0 1 2)'::GEOMETRY) TO '__TEST_DIR__/t1.parquet' (FORMAT 'parquet'); +# But still a normal parquet file +query I +SELECT st_astext(st_geomfromwkb(geometry)) FROM '__TEST_DIR__/t1.parquet'; ---- -Invalid Input Error: Geoparquet does not support geometries with M coordinates \ No newline at end of file +POINT M (0 1 2) \ No newline at end of file diff --git a/test/geoparquet/versions.test b/test/geoparquet/versions.test new file mode 100644 index 000000000000..b724b06026b1 --- /dev/null +++ b/test/geoparquet/versions.test @@ -0,0 +1,90 @@ +# name: test/geoparquet/versions.test +# group: [geoparquet] + +require spatial + +require parquet + +# DEFAULT (V1) + +statement ok +COPY (SELECT st_point(1,2) as geometry) +TO '__TEST_DIR__/test_default.parquet' (FORMAT PARQUET); + +query I +SELECT (decode(value)) as col +FROM parquet_kv_metadata('__TEST_DIR__/test_default.parquet'); +---- +{"version":"1.0.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":["Point"],"bbox":[1.0,2.0,1.0,2.0]}}} + +query I +SELECT geo_types from parquet_metadata('__TEST_DIR__/test_default.parquet'); +---- +NULL + +# V1 + +statement ok +COPY (SELECT st_point(1,2) as geometry) +TO '__TEST_DIR__/test_v1.parquet' (FORMAT PARQUET, GEOPARQUET_VERSION 'V1'); + +query I +SELECT (decode(value)) as col +FROM parquet_kv_metadata('__TEST_DIR__/test_v1.parquet'); +---- +{"version":"1.0.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":["Point"],"bbox":[1.0,2.0,1.0,2.0]}}} + +query I +SELECT geo_types from parquet_metadata('__TEST_DIR__/test_v1.parquet'); +---- +NULL + +# NONE + +statement ok +COPY (SELECT st_point(1,2) as geometry) +TO '__TEST_DIR__/test_none.parquet' (FORMAT PARQUET, GEOPARQUET_VERSION 'NONE'); + +query I +SELECT (decode(value)) as col +FROM parquet_kv_metadata('__TEST_DIR__/test_none.parquet'); +---- + +query I +SELECT geo_types from parquet_metadata('__TEST_DIR__/test_none.parquet'); +---- +[point] + +# BOTH + +statement ok +COPY (SELECT st_point(1,2) as geometry) +TO '__TEST_DIR__/test_both.parquet' (FORMAT PARQUET, GEOPARQUET_VERSION 'BOTH'); + +query I +SELECT (decode(value)) as col +FROM parquet_kv_metadata('__TEST_DIR__/test_both.parquet'); +---- +{"version":"1.0.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":["Point"],"bbox":[1.0,2.0,1.0,2.0]}}} + +query I +SELECT geo_types from parquet_metadata('__TEST_DIR__/test_both.parquet'); +---- +[point] + + +# V2 +statement ok +COPY (SELECT st_point(1,2) as geometry) +TO '__TEST_DIR__/test_v2.parquet' (FORMAT PARQUET, GEOPARQUET_VERSION 'V2'); + +query I +SELECT (decode(value)) as col +FROM parquet_kv_metadata('__TEST_DIR__/test_v2.parquet'); +---- +{"version":"2.0.0","primary_column":"geometry","columns":{"geometry":{"encoding":"WKB","geometry_types":["Point"],"bbox":[1.0,2.0,1.0,2.0]}}} + +query I +SELECT geo_types from parquet_metadata('__TEST_DIR__/test_v2.parquet'); +---- +[point] diff --git a/test/helpers/test_config.cpp b/test/helpers/test_config.cpp index fc06732dcae0..11f4bb1b412a 100644 --- a/test/helpers/test_config.cpp +++ b/test/helpers/test_config.cpp @@ -6,6 +6,7 @@ #include "duckdb/common/types/uuid.hpp" #include #include +#include namespace duckdb { @@ -23,6 +24,8 @@ static const TestConfigOption test_config_options[] = { {"comment", "Extra free form comment line", LogicalType::VARCHAR, nullptr}, {"initial_db", "Initial database path", LogicalType::VARCHAR, nullptr}, {"max_threads", "Max threads to use during tests", LogicalType::BIGINT, nullptr}, + {"base_config", "Config file to load and base initial settings on", LogicalType::VARCHAR, + TestConfiguration::LoadBaseConfig}, {"block_size", "Block Alloction Size; must be a power of 2", LogicalType::BIGINT, nullptr}, {"checkpoint_wal_size", "Size in bytes after which to trigger automatic checkpointing", LogicalType::BIGINT, nullptr}, @@ -30,6 +33,7 @@ static const TestConfigOption test_config_options[] = { {"force_restart", "Force restart the database between runs", LogicalType::BOOLEAN, nullptr}, {"summarize_failures", "Print a summary of all test failures after running", LogicalType::BOOLEAN, nullptr}, {"test_memory_leaks", "Run memory leak tests", LogicalType::BOOLEAN, nullptr}, + {"storage_fuzzer", "Run storage fuzzer tests", LogicalType::BOOLEAN, nullptr}, {"verify_vector", "Run vector verification for a specific vector type", LogicalType::VARCHAR, nullptr}, {"debug_initialize", "Initialize buffers with all 0 or all 1", LogicalType::VARCHAR, nullptr}, {"autoloading", "Loading strategy for extensions not bundled in", LogicalType::VARCHAR, nullptr}, @@ -38,6 +42,9 @@ static const TestConfigOption test_config_options[] = { {"on_init", "SQL statements to execute on init", LogicalType::VARCHAR, nullptr}, {"on_load", "SQL statements to execute on explicit load", LogicalType::VARCHAR, nullptr}, {"on_new_connection", "SQL statements to execute on connection", LogicalType::VARCHAR, nullptr}, + {"test_env", "The test variables", + LogicalType::LIST(LogicalType::STRUCT({{"env_name", LogicalType::VARCHAR}, {"env_value", LogicalType::VARCHAR}})), + nullptr}, {"skip_tests", "Tests to be skipped", LogicalType::LIST( LogicalType::STRUCT({{"reason", LogicalType::VARCHAR}, {"paths", LogicalType::LIST(LogicalType::VARCHAR)}})), @@ -49,6 +56,19 @@ static const TestConfigOption test_config_options[] = { {"statically_loaded_extensions", "Extensions to be loaded (from the statically available one)", LogicalType::LIST(LogicalType::VARCHAR), nullptr}, {"storage_version", "Database storage version to use by default", LogicalType::VARCHAR, nullptr}, + {"data_location", "Directory where static test files are read (defaults to `data/`)", LogicalType::VARCHAR, + nullptr}, + {"select_tag", "Select tests which match named tag (as singleton set; multiple sets are OR'd)", + LogicalType::VARCHAR, TestConfiguration::AppendSelectTagSet}, + {"select_tag_set", "Select tests which match _all_ named tags (multiple sets are OR'd)", + LogicalType::LIST(LogicalType::VARCHAR), TestConfiguration::AppendSelectTagSet}, + {"skip_tag", "Skip tests which match named tag (as singleton set; multiple sets are OR'd)", LogicalType::VARCHAR, + TestConfiguration::AppendSkipTagSet}, + {"skip_tag_set", "Skip tests which match _all_ named tags (multiple sets are OR'd)", + LogicalType::LIST(LogicalType::VARCHAR), TestConfiguration::AppendSkipTagSet}, + {"settings", "Configuration settings to apply", + LogicalType::LIST(LogicalType::STRUCT({{"name", LogicalType::VARCHAR}, {"value", LogicalType::VARCHAR}})), + nullptr}, {nullptr, nullptr, LogicalType::INVALID, nullptr}, }; @@ -192,6 +212,15 @@ bool TestConfiguration::ShouldSkipTest(const string &test_name) { return tests_to_be_skipped.count(test_name); } +string TestConfiguration::DataLocation() { + string res = GetOptionOrDefault("data_location", string("data/")); + // Force DataLocation to end with a '/' + if (res.back() != '/') { + res += "/"; + } + return res; +} + string TestConfiguration::OnInitCommand() { return GetOptionOrDefault("on_init", string()); } @@ -251,6 +280,11 @@ vector TestConfiguration::ErrorMessagesToBeSkipped() { return res; } +void TestConfiguration::LoadBaseConfig(const Value &input) { + auto &test_config = TestConfiguration::Get(); + test_config.LoadConfig(input.ToString()); +} + void TestConfiguration::ParseConnectScript(const Value &input) { auto init_cmd = ReadFileToString(input.ToString()); @@ -313,6 +347,7 @@ void TestConfiguration::LoadConfig(const string &config_path) { tests_to_be_skipped.insert(skipped_test.GetValue()); } } + options.erase("skip_tests"); } } @@ -323,6 +358,8 @@ void TestConfiguration::ProcessPath(string &path, const string &test_name) { auto base_test_name = StringUtil::Replace(test_name, "/", "_"); path = StringUtil::Replace(path, "{BASE_TEST_NAME}", base_test_name); + path = StringUtil::Replace(path, "__TEST_DIR__", TestDirectoryPath()); + path = StringUtil::Replace(path, "__WORKING_DIRECTORY__", FileSystem::GetWorkingDirectory()); } template @@ -366,6 +403,10 @@ bool TestConfiguration::GetTestMemoryLeaks() { return GetOptionOrDefault("test_memory_leaks", false); } +bool TestConfiguration::RunStorageFuzzer() { + return GetOptionOrDefault("storage_fuzzer", false); +} + bool TestConfiguration::GetSummarizeFailures() { return GetOptionOrDefault("summarize_failures", false); } @@ -378,6 +419,43 @@ string TestConfiguration::GetStorageVersion() { return GetOptionOrDefault("storage_version", string()); } +vector TestConfiguration::GetConfigSettings() { + vector result; + if (options.find("settings") != options.end()) { + auto entry = options["settings"]; + auto list_children = ListValue::GetChildren(entry); + for (const auto &value : list_children) { + auto &struct_children = StructValue::GetChildren(value); + ConfigSetting config_setting; + config_setting.name = StringValue::Get(struct_children[0]); + config_setting.value = StringValue::Get(struct_children[1]); + result.push_back(std::move(config_setting)); + } + } + return result; +} + +string TestConfiguration::GetTestEnv(const string &key, const string &default_value) { + if (test_env.empty() && options.find("test_env") != options.end()) { + auto entry = options["test_env"]; + auto list_children = ListValue::GetChildren(entry); + for (const auto &value : list_children) { + auto &struct_children = StructValue::GetChildren(value); + auto &env = StringValue::Get(struct_children[0]); + auto &env_value = StringValue::Get(struct_children[1]); + test_env[env] = env_value; + } + } + if (test_env.find(key) == test_env.end()) { + return default_value; + } + return test_env[key]; +} + +const unordered_map &TestConfiguration::GetTestEnvMap() { + return test_env; +} + DebugVectorVerification TestConfiguration::GetVectorVerification() { return EnumUtil::FromString(GetOptionOrDefault("verify_vector", "NONE")); } @@ -386,6 +464,68 @@ DebugInitialize TestConfiguration::GetDebugInitialize() { return EnumUtil::FromString(GetOptionOrDefault("debug_initialize", "NO_INITIALIZE")); } +vector> TestConfiguration::GetSelectTagSets() { + return select_tag_sets; +} + +vector> TestConfiguration::GetSkipTagSets() { + return skip_tag_sets; +} + +std::unordered_set make_tag_set(const Value &src_val) { + // handle both cases -- singleton VARCHAR/string, and set of strings + auto dst_set = std::unordered_set(); + if (src_val.type() == LogicalType::VARCHAR) { + dst_set.insert(src_val.GetValue()); + } else /* LIST(VARCHAR) */ { + for (auto &tag : ListValue::GetChildren(src_val)) { + dst_set.insert(tag.GetValue()); + } + } + return dst_set; +} + +void TestConfiguration::AppendSelectTagSet(const Value &tag_set) { + TestConfiguration::Get().select_tag_sets.push_back(make_tag_set(tag_set)); +} + +void TestConfiguration::AppendSkipTagSet(const Value &tag_set) { + TestConfiguration::Get().skip_tag_sets.push_back(make_tag_set(tag_set)); +} + +bool is_subset(const unordered_set &sub, const vector &super) { + for (const auto &elt : sub) { + if (std::find(super.begin(), super.end(), elt) == super.end()) { + return false; + } + } + return true; +} + +// NOTE: this model of policy assumes simply that all selects are applied to the All set, then +// all skips are applied to that result. (Typical alternative: CLI ordering where each +// select/skip operation is applied in sequence.) +TestConfiguration::SelectPolicy TestConfiguration::GetPolicyForTagSet(const vector &subject_tag_set) { + // Apply select_tag_set first then skip_tag_set; if both empty always NONE + auto policy = TestConfiguration::SelectPolicy::NONE; + // select: if >= 1 select_tag_set is subset of subject_tag_set + // if count(select_tag_sets) > 0 && no matches, SKIP + for (const auto &select_tag_set : select_tag_sets) { + policy = TestConfiguration::SelectPolicy::SKIP; // >=1 sets => SKIP || SELECT + if (is_subset(select_tag_set, subject_tag_set)) { + policy = TestConfiguration::SelectPolicy::SELECT; + break; + } + } + // skip: if >=1 skip_tag_set is subset of subject_tag_set, else passthrough + for (const auto &skip_tag_set : skip_tag_sets) { + if (is_subset(skip_tag_set, subject_tag_set)) { + return TestConfiguration::SelectPolicy::SKIP; + } + } + return policy; +} + bool TestConfiguration::TestForceStorage() { auto &test_config = TestConfiguration::Get(); return !test_config.GetInitialDBPath().empty(); @@ -401,6 +541,11 @@ bool TestConfiguration::TestMemoryLeaks() { return test_config.GetTestMemoryLeaks(); } +bool TestConfiguration::TestRunStorageFuzzer() { + auto &test_config = TestConfiguration::Get(); + return test_config.RunStorageFuzzer(); +} + FailureSummary::FailureSummary() : failures_summary_counter(0) { } diff --git a/test/include/test_config.hpp b/test/include/test_config.hpp index 1af03694e9cd..68ed3b67a084 100644 --- a/test/include/test_config.hpp +++ b/test/include/test_config.hpp @@ -16,15 +16,28 @@ #include "duckdb/common/atomic.hpp" #include "duckdb/common/mutex.hpp" #include "duckdb/common/enums/debug_initialize.hpp" +#include +#include namespace duckdb { enum class SortStyle : uint8_t { NO_SORT, ROW_SORT, VALUE_SORT }; +struct ConfigSetting { + string name; + Value value; +}; + class TestConfiguration { public: enum class ExtensionAutoLoadingMode { NONE = 0, AVAILABLE = 1, ALL = 2 }; + enum class SelectPolicy : uint8_t { + NONE, // does not match any explicit policy (default: policy=SELECT) + SELECT, // matches explicit select + SKIP // matches explicit skip + }; + static TestConfiguration &Get(); void Initialize(); @@ -43,12 +56,14 @@ class TestConfiguration { bool GetForceRestart(); bool GetCheckpointOnShutdown(); bool GetTestMemoryLeaks(); + bool RunStorageFuzzer(); bool GetSummarizeFailures(); bool GetSkipCompiledTests(); DebugVectorVerification GetVectorVerification(); DebugInitialize GetDebugInitialize(); ExtensionAutoLoadingMode GetExtensionAutoLoadingMode(); bool ShouldSkipTest(const string &test_name); + string DataLocation(); string OnInitCommand(); string OnLoadCommand(); string OnConnectionCommand(); @@ -57,18 +72,32 @@ class TestConfiguration { vector ExtensionToBeLoadedOnLoad(); vector ErrorMessagesToBeSkipped(); string GetStorageVersion(); + string GetTestEnv(const string &key, const string &default_value); + const unordered_map &GetTestEnvMap(); + vector> GetSelectTagSets(); + vector> GetSkipTagSets(); + SelectPolicy GetPolicyForTagSet(const vector &tag_set); + vector GetConfigSettings(); static bool TestForceStorage(); static bool TestForceReload(); static bool TestMemoryLeaks(); + static bool TestRunStorageFuzzer(); + static void LoadBaseConfig(const Value &input); static void ParseConnectScript(const Value &input); static void CheckSortStyle(const Value &input); static bool TryParseSortStyle(const string &sort_style, SortStyle &result); + static void AppendSelectTagSet(const Value &tag_set); + static void AppendSkipTagSet(const Value &tag_set); private: case_insensitive_map_t options; unordered_set tests_to_be_skipped; + unordered_map test_env; + + vector> select_tag_sets; + vector> skip_tag_sets; private: template diff --git a/test/issues/fuzz/function_pointer_crash_in_subquery.test b/test/issues/fuzz/function_pointer_crash_in_subquery.test index 1d06774250e3..70e82c513039 100644 --- a/test/issues/fuzz/function_pointer_crash_in_subquery.test +++ b/test/issues/fuzz/function_pointer_crash_in_subquery.test @@ -14,3 +14,4 @@ CREATE TABLE c0(test2 tinyint, s1 smallint, s2 integer, test1 bigint, i double, statement error SELECT * FROM c0 s1 INNER JOIN c0 s2 ON (SELECT s1.s2=s2 FROM c0 WHERE s2.s2=s2) ORDER BY s1.s2; ---- +Binder Error: Cannot extract field 's2' from expression "s1" \ No newline at end of file diff --git a/test/issues/general/test_19067.test b/test/issues/general/test_19067.test new file mode 100644 index 000000000000..ac50a431c748 --- /dev/null +++ b/test/issues/general/test_19067.test @@ -0,0 +1,171 @@ +# name: test/issues/general/test_19067.test +# description: Issue 19067 - Cascading Views using CTEs High Memory & CPU +# group: [general] + +statement ok +PRAGMA enable_verification + +statement ok +CREATE OR REPLACE VIEW view_1 AS +SELECT + i AS id, + i * 10 AS value, + 'item_' || i AS name, + (i % 3) + 1 AS category +FROM generate_series(1, 100) AS t(i); + +statement ok +CREATE OR REPLACE VIEW view_2 AS +WITH prev1 AS (SELECT * FROM view_1) +SELECT + *, + value * 2 AS double_value +FROM prev1; + +statement ok +CREATE OR REPLACE VIEW view_3 AS +WITH prev2 AS (SELECT * FROM view_2) +SELECT + *, + LENGTH(name) AS name_length +FROM prev2; + +statement ok +CREATE OR REPLACE VIEW view_4 AS +WITH prev3 AS (SELECT * FROM view_3) +SELECT + *, + CASE + WHEN category = 1 THEN 'Type A' + WHEN category = 2 THEN 'Type B' + ELSE 'Type C' + END AS category_desc +FROM prev3; + +statement ok +CREATE OR REPLACE VIEW view_5 AS +WITH prev AS (SELECT * FROM view_4) +SELECT + *, + value + id AS sum_value_id +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_6 AS +WITH prev AS (SELECT * FROM view_5) +SELECT + *, + UPPER(name) AS name_upper +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_7 AS +WITH prev AS (SELECT * FROM view_6) +SELECT + *, + ROW_NUMBER() OVER (ORDER BY id) AS row_num +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_8 AS +WITH prev AS (SELECT * FROM view_7) +SELECT + *, + CASE WHEN value > 500 THEN 'HIGH' ELSE 'LOW' END AS value_flag +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_9 AS +WITH prev AS (SELECT * FROM view_8) +SELECT + *, + id % 10 AS id_mod_10 +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_10 AS +WITH prev AS (SELECT * FROM view_9) +SELECT + *, + value / 10 AS value_divided +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_11 AS +WITH prev AS (SELECT * FROM view_10) +SELECT + *, + SUBSTRING(name, 1, 4) AS name_prefix +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_12 AS +WITH prev AS (SELECT * FROM view_11) +SELECT + *, + AVG(value) OVER () AS avg_value +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_13 AS +WITH prev AS (SELECT * FROM view_12) +SELECT + *, + value - avg_value AS diff_from_avg +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_14 AS +WITH prev AS (SELECT * FROM view_13) +SELECT + *, + RANK() OVER (ORDER BY value) AS value_rank +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_15 AS +WITH prev AS (SELECT * FROM view_14) +SELECT + *, + id % 2 = 0 AS is_even +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_16 AS +WITH prev AS (SELECT * FROM view_15) +SELECT + *, + name || '_' || category_desc AS full_name +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_17 AS +WITH prev AS (SELECT * FROM view_16) +SELECT + *, + id * id AS id_squared +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_18 AS +WITH prev AS (SELECT * FROM view_17) +SELECT + *, + COUNT(*) OVER (PARTITION BY category) AS category_count +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_19 AS +WITH prev AS (SELECT * FROM view_18) +SELECT + *, + ROUND((value * 100.0 / sum_value_id), 2) AS value_percentage +FROM prev; + +statement ok +CREATE OR REPLACE VIEW view_20 AS +WITH prev AS (SELECT * FROM view_19) +SELECT + *, + CASE WHEN value_rank <= 10 THEN 'TOP_10' ELSE 'OTHER' END AS final_category +FROM prev; \ No newline at end of file diff --git a/test/issues/general/test_5200.test b/test/issues/general/test_5200.test index e9de7d7ecc97..8bec2a057276 100644 --- a/test/issues/general/test_5200.test +++ b/test/issues/general/test_5200.test @@ -21,3 +21,4 @@ where t.n<9 and f.n<=t.n+1 ) select sum(f.f/t.f) from t,f where t.n=4 and t.s>0 and t.s=f.n; ---- +Out of Range Error: Overflow in multiplication of INT64 (2432902008176640000 * 21) \ No newline at end of file diff --git a/test/issues/internal/test_5994.test b/test/issues/internal/test_5994.test new file mode 100644 index 000000000000..6e0265f0b1ed --- /dev/null +++ b/test/issues/internal/test_5994.test @@ -0,0 +1,10 @@ +# name: test/issues/internal/test_5994.test +# description: Internal Issue 5994: (>=1.4) varchar != filter gives wrong result reading from specific parquet +# group: [internal] + +require parquet + +query I +SELECT COUNT(*) FROM 'data/parquet-testing/internal_5994.parquet' WHERE eventName != 'ListObjects'; +---- +118 diff --git a/test/issues/rigger/date_get_value.test b/test/issues/rigger/date_get_value.test index 5337798a3a5c..d05aae147e2c 100644 --- a/test/issues/rigger/date_get_value.test +++ b/test/issues/rigger/date_get_value.test @@ -11,3 +11,4 @@ INSERT INTO t0 VALUES (false, 1); statement error SELECT t0.c0, t0.c1 FROM t0 WHERE REVERSE((CASE 0.020672069925445347 WHEN (((DATE '1970-01-18')+(t0.c1)) NOT BETWEEN t0.c0 AND DATE '1970-01-10') THEN t0.c1 ELSE t0.c1 END )) GROUP BY t0.c0, t0.c1 UNION SELECT t0.c0, t0.c1 FROM t0 WHERE (NOT REVERSE((CASE 0.020672069925445347 WHEN (((DATE '1970-01-18')+(t0.c1)) NOT BETWEEN t0.c0 AND DATE '1970-01-10') THEN t0.c1 ELSE t0.c1 END ))) GROUP BY t0.c0, t0.c1 UNION SELECT t0.c0, t0.c1 FROM t0 WHERE ((REVERSE((CASE 0.020672069925445347 WHEN (((DATE '1970-01-18')+(t0.c1)) NOT BETWEEN t0.c0 AND DATE '1970-01-10') THEN t0.c1 ELSE t0.c1 END ))) IS NULL) GROUP BY t0.c0, t0.c1; ---- +Binder Error: Cannot mix values of type DATE and BOOLEAN in BETWEEN \ No newline at end of file diff --git a/test/issues/rigger/expression_equality_bug.test b/test/issues/rigger/expression_equality_bug.test index 08a8393890e9..9a717fd7fec1 100644 --- a/test/issues/rigger/expression_equality_bug.test +++ b/test/issues/rigger/expression_equality_bug.test @@ -11,3 +11,4 @@ CREATE TABLE t0(c0 BOOLEAN DEFAULT(1168904737), c1 BOOLEAN DEFAULT(DATE '1969-12 statement error SELECT t0.c0 FROM t0 WHERE (((t0.rowid)SIMILAR TO(TIMESTAMP '1970-01-11 11:17:02')) BETWEEN TIMESTAMP '1970-01-11 23:34:33' AND DATE '1970-01-20') UNION ALL SELECT t0.c0 FROM t0 WHERE (NOT (((t0.rowid)SIMILAR TO(TIMESTAMP '1970-01-11 11:17:02')) BETWEEN TIMESTAMP '1970-01-11 23:34:33' AND DATE '1970-01-20')) UNION ALL SELECT t0.c0 FROM t0 WHERE (((((t0.rowid)SIMILAR TO(TIMESTAMP '1970-01-11 11:17:02')) BETWEEN TIMESTAMP '1970-01-11 23:34:33' AND DATE '1970-01-20')) IS NULL); ---- +Binder Error: No function matches the given name and argument types 'regexp_full_match \ No newline at end of file diff --git a/test/issues/rigger/instr_crash.test b/test/issues/rigger/instr_crash.test index 0a3b419837e0..cb358b027717 100644 --- a/test/issues/rigger/instr_crash.test +++ b/test/issues/rigger/instr_crash.test @@ -26,3 +26,4 @@ COMMIT; statement error SELECT t96.c0, t0.rowid, t0.c1, t96.c2 FROM t0, t96 WHERE (TIMESTAMP '1969-12-28 13:59:01' LIKE INSTR(t0.c1, t96.c0) ESCAPE RTRIM(DATE '1969-12-23')) UNION ALL SELECT t96.c0, t0.rowid, t0.c1, t96.c2 FROM t0, t96 WHERE (NOT (TIMESTAMP '1969-12-28 13:59:01' LIKE INSTR(t0.c1, t96.c0) ESCAPE RTRIM(DATE '1969-12-23'))) UNION ALL SELECT t96.c0, t0.rowid, t0.c1, t96.c2 FROM t0, t96 WHERE (((TIMESTAMP '1969-12-28 13:59:01' LIKE INSTR(t0.c1, t96.c0) ESCAPE RTRIM(DATE '1969-12-23'))) IS NULL); ---- +Binder Error: No function matches the given name and argument types 'instr \ No newline at end of file diff --git a/test/issues/rigger/string_comparison_bug.test b/test/issues/rigger/string_comparison_bug.test index 04735879ef8a..fe77276cf8fd 100644 --- a/test/issues/rigger/string_comparison_bug.test +++ b/test/issues/rigger/string_comparison_bug.test @@ -221,3 +221,4 @@ PRAGMA disable_optimizer statement error SELECT * FROM t0 WHERE (('NN')<((t0.rowid NOT IN (t0.c1)))); ---- +Conversion Error: Could not convert string 'NN' to BOOL \ No newline at end of file diff --git a/test/ldbc/ldbc-empty.test b/test/ldbc/ldbc-empty.test index 52beeddca8b3..cacee98d423a 100644 --- a/test/ldbc/ldbc-empty.test +++ b/test/ldbc/ldbc-empty.test @@ -160,6 +160,7 @@ create table tag ( t_tagclassid bigint not null ); + # run the queries # Q1. Posting summary diff --git a/test/logging/test_logging.cpp b/test/logging/test_logging.cpp index e4ad7773be1f..76834bce87f9 100644 --- a/test/logging/test_logging.cpp +++ b/test/logging/test_logging.cpp @@ -4,6 +4,7 @@ #include "duckdb/main/database.hpp" #include "duckdb/logging/logger.hpp" #include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/main/extension_manager.hpp" #include "duckdb/function/table_function.hpp" #include "duckdb/logging/log_storage.hpp" #include "duckdb/logging/log_manager.hpp" @@ -168,7 +169,9 @@ TEST_CASE("Test thread context logger", "[logging][.]") { duckdb::TableFunction tf("test_thread_logger", {}, TestLoggingFunction, TestLoggingBind, nullptr, TestLoggingInitLocal); - ExtensionLoader loader(*db.instance, "log_test_extension"); + ExtensionInfo extension_info {}; + ExtensionActiveLoad load_info {*db.instance, extension_info, "log_test_extension"}; + ExtensionLoader loader {load_info}; loader.RegisterFunction(tf); REQUIRE_NO_FAIL(con.Query("set enable_logging=true;")); diff --git a/test/optimizer/common_subplan.test b/test/optimizer/common_subplan.test new file mode 100644 index 000000000000..f595bb2443ec --- /dev/null +++ b/test/optimizer/common_subplan.test @@ -0,0 +1,152 @@ +# name: test/optimizer/common_subplan.test +# description: Test the Common Subplan optimizer +# group: [optimizer] + +statement ok +pragma explain_output='optimized_only' + +# this should be automatically detected and materialized +query I +select t1.s + t2.s +from (select sum(range) s from range(10)) t1, + (select sum(range) s from range(10)) t2 +---- +90 + +query II +explain select t1.s + t2.s +from (select sum(range) s from range(10)) t1, + (select sum(range) s from range(10)) t2 +---- +logical_opt :.*CTE.* + +# this shouldn't because random() is volatile +query II +explain select t1.s + t2.s +from (select sum(random()) s from range(10)) t1, + (select sum(random()) s from range(10)) t2 +---- +logical_opt :.*CTE.* + +# common subplan inside of a materialized cte and outside of it +# should yield two ctes +query I +with cte as materialized ( + select sum(range) s from range(10) +) +select t1.s + t2.s +from cte t1, + (select sum(range) s from range(10)) t2 +---- +90 + +query II +explain with cte as materialized ( + select sum(range) s from range(10) +) +select t1.s + t2.s +from cte t1, + (select sum(range) s from range(10)) t2 +---- +logical_opt :.*CTE.*CTE.* + +require tpcds + +statement ok +call dsdgen(sf=0) + +# q44 +query II +explain +SELECT asceding.rnk, + i1.i_product_name best_performing, + i2.i_product_name worst_performing +FROM + (SELECT * + FROM + (SELECT item_sk, + rank() OVER ( + ORDER BY rank_col ASC) rnk + FROM + (SELECT ss_item_sk item_sk, + avg(ss_net_profit) rank_col + FROM store_sales ss1 + WHERE ss_store_sk = 4 + GROUP BY ss_item_sk + HAVING avg(ss_net_profit) > 0.9* + (SELECT avg(ss_net_profit) rank_col + FROM store_sales + WHERE ss_store_sk = 4 + AND ss_addr_sk IS NULL + GROUP BY ss_store_sk))V1)V11 + WHERE rnk < 11) asceding, + (SELECT * + FROM + (SELECT item_sk, + rank() OVER ( + ORDER BY rank_col DESC) rnk + FROM + (SELECT ss_item_sk item_sk, + avg(ss_net_profit) rank_col + FROM store_sales ss1 + WHERE ss_store_sk = 4 + GROUP BY ss_item_sk + HAVING avg(ss_net_profit) > 0.9* + (SELECT avg(ss_net_profit) rank_col + FROM store_sales + WHERE ss_store_sk = 4 + AND ss_addr_sk IS NULL + GROUP BY ss_store_sk))V2)V21 + WHERE rnk < 11) descending, + item i1, + item i2 +WHERE asceding.rnk = descending.rnk + AND i1.i_item_sk=asceding.item_sk + AND i2.i_item_sk=descending.item_sk +ORDER BY asceding.rnk +LIMIT 100; +---- +logical_opt :.*CTE.* + +# q65 +query II +explain +SELECT s_store_name, + i_item_desc, + sc.revenue, + i_current_price, + i_wholesale_cost, + i_brand +FROM store, + item, + (SELECT ss_store_sk, + avg(revenue) AS ave + FROM + (SELECT ss_store_sk, + ss_item_sk, + sum(ss_sales_price) AS revenue + FROM store_sales, + date_dim + WHERE ss_sold_date_sk = d_date_sk + AND d_month_seq BETWEEN 1176 AND 1176+11 + GROUP BY ss_store_sk, + ss_item_sk) sa + GROUP BY ss_store_sk) sb, + (SELECT ss_store_sk, + ss_item_sk, + sum(ss_sales_price) AS revenue + FROM store_sales, + date_dim + WHERE ss_sold_date_sk = d_date_sk + AND d_month_seq BETWEEN 1176 AND 1176+11 + GROUP BY ss_store_sk, + ss_item_sk) sc +WHERE sb.ss_store_sk = sc.ss_store_sk + AND sc.revenue <= 0.1 * sb.ave + AND s_store_sk = sc.ss_store_sk + AND i_item_sk = sc.ss_item_sk +ORDER BY s_store_name NULLS FIRST, + i_item_desc NULLS FIRST +LIMIT 100; +---- +logical_opt :.*CTE.* diff --git a/test/optimizer/pushdown/issue_18603.test b/test/optimizer/pushdown/issue_18603.test new file mode 100644 index 000000000000..823be9e10e27 --- /dev/null +++ b/test/optimizer/pushdown/issue_18603.test @@ -0,0 +1,46 @@ +# name: test/optimizer/pushdown/issue_18603.test +# description: Test filter pushdown with conflict comparison filters +# group: [pushdown] + +statement ok +pragma enable_verification + +statement ok +CREATE TABLE t0(c0 INT, c1 BOOLEAN); + +statement ok +CREATE TABLE t1(c0 INT); + +statement ok +INSERT INTO t0(c0, c1) VALUES (0, 0); + +statement ok +INSERT INTO t1(c0) VALUES (1); + +# test different order of filters +query III +SELECT * FROM t0 INNER JOIN t1 ON (t0.c1 > t0.c0) AND (t1.c0 > t0.c0) AND (t0.c0 < 7) AND (t0.c1 = t0.c0); +---- + +query II +EXPLAIN SELECT * FROM t0 INNER JOIN t1 ON (t0.c1 > t0.c0) AND (t1.c0 > t0.c0) AND (t0.c0 < 7) AND (t0.c1 = t0.c0); +---- +physical_plan :.*EMPTY_RESULT.* + +query II +EXPLAIN SELECT * FROM t0 INNER JOIN t1 ON (t0.c1 > t0.c0) AND (t1.c0 > t0.c0) AND (t0.c0 < 7) AND (t0.c1 = t0.c0); +---- +physical_plan :.*c0 > c0.* + +query II +EXPLAIN SELECT * FROM t0 INNER JOIN t1 ON (t0.c1 > t0.c0) AND (t1.c0 > t0.c0) AND (t0.c0 < 7) AND (t0.c1 = t0.c0); +---- +physical_plan :.*c0 < 7.* + +query III +SELECT * FROM t0 INNER JOIN t1 ON (t0.c1 = t0.c0) AND (t0.c1 > t0.c0) AND (t1.c0 > t0.c0) AND (t0.c0 < 7); +---- + +query III +SELECT * FROM t0 INNER JOIN t1 ON (t0.c1 = t0.c0) AND (t0.c0 < 7) AND (t0.c1 > t0.c0) AND (t1.c0 > t0.c0); +---- diff --git a/test/optimizer/pushdown/issue_18653.test b/test/optimizer/pushdown/issue_18653.test new file mode 100644 index 000000000000..2a65d728cf70 --- /dev/null +++ b/test/optimizer/pushdown/issue_18653.test @@ -0,0 +1,58 @@ +# name: test/optimizer/pushdown/issue_18653.test +# description: Performance issue with CROSS JOIN and LATERAL JOIN combined with unnest and json_each. Filter should be pushed down to reduce the joined rows +# group: [pushdown] + +statement ok +PRAGMA enable_verification + +statement ok +create table test_table as +select s.i as id, [1, 2, 3]::bigint[] as values from generate_series(1, 1000000) as s(i); + +statement ok +create index test_table_id_idx on test_table(id); + +query II +explain analyze +select id, value +from test_table +cross join unnest(values) as values(value) where id = 87100; +---- +analyzed_plan :.*LEFT_DELIM_JOIN.*FILTER.* + +query II +select id, value +from test_table +cross join unnest(values) as values(value) where id = 87100; +---- +87100 3 +87100 2 +87100 1 + +query II +explain analyze +select id, value +from test_table t +left join lateral unnest(t.values) as value on true +where id = 87100; +---- +analyzed_plan :.*LEFT_DELIM_JOIN.*FILTER.* + +require json + +statement ok +create table test_table2 as +select s.i as id, '{"key1": 1, "key2": 2, "key3": 3}'::JSON as values +from generate_series(1, 1000000) as s(i); + +statement ok +create index test_table2_id_idx on test_table2(id); + +query II +explain analyze +select t.id, key, value +from test_table2 t +cross join json_each(t.values) as kv(key, value) +where t.id = 87100; +---- +analyzed_plan :.*LEFT_DELIM_JOIN.*Filters:.*id=87100.* diff --git a/test/optimizer/regex_optimizer.test b/test/optimizer/regex_optimizer.test index 8d8b077df4a4..5491caa70050 100644 --- a/test/optimizer/regex_optimizer.test +++ b/test/optimizer/regex_optimizer.test @@ -228,3 +228,28 @@ statement error select count(s) from test where regexp_matches('aaa'); ---- Binder Error + +# Test regexp_matches with flags (like 'm') is properly optimized to contains +statement ok +DELETE FROM test; + +statement ok +INSERT INTO test VALUES ('hello world'), ('test'), ('hello again'); + +query II +explain analyze SELECT s FROM test WHERE regexp_matches(s, 'hello', 'm'); +---- +analyzed_plan :.*contains\(s, 'hello'\).* + +query I nosort +SELECT s FROM test WHERE regexp_matches(s, 'hello', 'm'); +---- +hello world +hello again + +# This used to trigger a debug assertion +query I +WITH fetch_schema AS ( + SELECT s FROM test WHERE regexp_matches(s, 'hello', 'm') +) SELECT * FROM fetch_schema LIMIT 0; +---- diff --git a/test/optimizer/test_try_cast_decimal.test b/test/optimizer/test_try_cast_decimal.test new file mode 100644 index 000000000000..16611c39322c --- /dev/null +++ b/test/optimizer/test_try_cast_decimal.test @@ -0,0 +1,37 @@ +# name: test/optimizer/test_try_cast_decimal.test +# description: Test try cast inconsistency +# group: [optimizer] + +statement ok +pragma enable_verification + +statement ok +CREATE TABLE t0(c0 INT , c1 BOOLEAN , PRIMARY KEY(c0)); + +statement ok +INSERT INTO t0(c0, c1) VALUES (890608529, false); + +query I +SELECT (true AND((TRY_CAST(t0.c0 AS DECIMAL(10,2)) > 0))) AS r FROM t0; +---- +NULL + +query I +SELECT (((TRY_CAST(t0.c0 AS DECIMAL(10,2)) > 0))) AS r FROM t0; +---- +NULL + +query I +SELECT (true OR((TRY_CAST(t0.c0 AS DECIMAL(10,2)) > 0))) AS r FROM t0; +---- +true + +statement error +SELECT (((CAST(t0.c0 AS DECIMAL(10,2)) > 0))) AS r FROM t0; +---- +Conversion Error + +query I +SELECT t0.c1 FROM t0 WHERE (true AND((TRY_CAST(t0.c0 AS DECIMAL(10,2)) > 0))); +---- + diff --git a/test/optimizer/topn_window_elimination.test b/test/optimizer/topn_window_elimination.test new file mode 100644 index 000000000000..e050c2f41c69 --- /dev/null +++ b/test/optimizer/topn_window_elimination.test @@ -0,0 +1,192 @@ +# name: test/optimizer/topn_window_elimination.test +# description: Test Top-N Window Elimination Rule +# group: [optimizer] + +statement ok +PRAGMA enable_verification + +statement ok +PRAGMA explain_output = OPTIMIZED_ONLY; + +statement ok +CREATE TABLE tbl AS SELECT * FROM VALUES (0, {'x': 0}, 0), (0, {'x': 1}, 1), (1, {'x': 2}, 2), (null, {'x': 3}, 3), (null, {'x': 4}, 4), t(grp, a, b) + +statement ok +CREATE TABLE tbl_with_null AS SELECT * FROM VALUES (0, [0], null), (0, [1], 1), (1, [2], null), (null, [3], 3), (null, [4], 4), t(grp, a, b) + +statement ok +CREATE MACRO window_fun(table_name, col_names, sort_order, topn) AS TABLE +FROM query( + 'SELECT * FROM (SELECT row_number() OVER (PARTITION BY grp ORDER BY b ' || sort_order || ') as rn, ' || array_to_string(col_names, ',') || ' FROM ' || table_name || ' QUALIFY rn <= ' || topn || ') ORDER BY ALL' +) + +statement ok +CREATE MACRO lateral_join(sort_order, topn) AS TABLE +FROM query ( + 'SELECT t1.* FROM tbl t1 INNER JOIN LATERAL (SELECT * FROM tbl t2 WHERE t1.grp = t2.grp ORDER BY b ' || sort_order || ' LIMIT ' || topn || ' ) ON true' +) + +# test min_max +foreach sort_order ASC DESC + +# test topn sizes +loop topn 1 3 + +# min/max +query II +EXPLAIN SELECT * FROM window_fun('tbl', ['grp'], '${sort_order}', ${topn}) +---- +logical_opt :.*FILTER.*WINDOW.* + +# arg_min/max, no struct_pack +query II +EXPLAIN SELECT * FROM window_fun('tbl', ['grp', 'a'], '${sort_order}', ${topn}) +---- +logical_opt :.*FILTER.*WINDOW.* + +# arg_min/max with struct_pack +query II +EXPLAIN SELECT * FROM window_fun('tbl', ['grp', 'a', 'b'], '${sort_order}', ${topn}) +---- +logical_opt :.*FILTER.*WINDOW.* + +# min/max and nulls +query II +EXPLAIN SELECT * FROM window_fun('tbl_with_null', ['grp'], '${sort_order}', ${topn}) +---- +logical_opt :.*FILTER.*WINDOW.* + +# arg_min/max, no struct_pack and nulls +query II +EXPLAIN SELECT * FROM window_fun('tbl_with_null', ['grp', 'a'], '${sort_order}', ${topn}) +---- +logical_opt :.*FILTER.*WINDOW.* + +# arg_min/max with struct_pack and nulls +query II +EXPLAIN SELECT * FROM window_fun('tbl_with_null', ['grp', 'a', 'b'], '${sort_order}', ${topn}) +---- +logical_opt :.*FILTER.*WINDOW.* + +# test lateral join +query II +EXPLAIN SELECT * FROM lateral_join('${sort_order}', ${topn}) +---- +logical_opt :.*FILTER.*WINDOW.* + +statement ok +CREATE TABLE ${sort_order}${topn}_1 AS SELECT * FROM window_fun('tbl', ['grp', 'a', 'b'], '${sort_order}', ${topn}) + +statement ok +CREATE TABLE ${sort_order}${topn}_2 AS SELECT * FROM window_fun('tbl', ['b', 'a', 'grp'], '${sort_order}', ${topn}) + +statement ok +CREATE TABLE ${sort_order}${topn}_3 AS SELECT * FROM window_fun('tbl', ['grp', 'a'], '${sort_order}', ${topn}) + +statement ok +CREATE TABLE ${sort_order}${topn}_4 AS SELECT * FROM window_fun('tbl', ['a', 'b'], '${sort_order}', ${topn}) + +statement ok +CREATE TABLE ${sort_order}${topn}_5 AS SELECT * FROM window_fun('tbl', ['a'], '${sort_order}', ${topn}) + +statement ok +CREATE TABLE ${sort_order}${topn}_6 AS SELECT * FROM lateral_join('${sort_order}', ${topn}) + +statement ok +CREATE TABLE ${sort_order}${topn}_7 AS SELECT * FROM window_fun('tbl_with_null', ['a', 'grp'], '${sort_order}', ${topn}) + +statement ok +CREATE TABLE ${sort_order}${topn}_8 AS SELECT * FROM window_fun('tbl_with_null', ['grp', 'a', 'b'], '${sort_order}', ${topn}) + +statement ok +SET disabled_optimizers = 'top_n_window_elimination' + +query IIII +SELECT * FROM ${sort_order}${topn}_1 EXCEPT SELECT * FROM window_fun('tbl', ['grp', 'a', 'b'], '${sort_order}', ${topn}) +---- + +query IIII +SELECT * FROM ${sort_order}${topn}_2 EXCEPT SELECT * FROM window_fun('tbl', ['b', 'a', 'grp'], '${sort_order}', ${topn}) +---- + +query III +SELECT * FROM ${sort_order}${topn}_3 EXCEPT SELECT * FROM window_fun('tbl', ['grp', 'a'], '${sort_order}', ${topn}) +---- + +query III +SELECT * FROM ${sort_order}${topn}_4 EXCEPT SELECT * FROM window_fun('tbl', ['a', 'b'], '${sort_order}', ${topn}) +---- + +query II +SELECT * FROM ${sort_order}${topn}_5 EXCEPT SELECT * FROM window_fun('tbl', ['a'], '${sort_order}', ${topn}) +---- + +query III +SELECT * FROM ${sort_order}${topn}_6 EXCEPT SELECT * FROM lateral_join('${sort_order}', ${topn}) +---- + +query III +SELECT * FROM ${sort_order}${topn}_7 EXCEPT SELECT * FROM window_fun('tbl_with_null', ['a', 'grp'], '${sort_order}', ${topn}) +---- + +query IIII +SELECT * FROM ${sort_order}${topn}_8 EXCEPT SELECT * FROM window_fun('tbl_with_null', ['grp', 'a', 'b'], '${sort_order}', ${topn}) +---- + +statement ok +SET disabled_optimizers = '' + +endloop + +endloop + +# Test vector functions +statement ok +CREATE TABLE vectors AS +SELECT [x,y,z] AS vec, row_number() OVER () AS id +FROM range(0,10) r(x), range(0,10) rr(y), range(0, 10) rrr(z) + +statement ok +CREATE OR REPLACE TABLE with_optimizer AS SELECT * EXCLUDE (rn) +FROM (SELECT *, list_distance(vec, [5,5,5]) AS dist, row_number() over (ORDER BY dist ASC) as rn FROM vectors) +WHERE rn <= 7 + +statement ok +set disabled_optimizers = 'top_n_window_elimination' + +statement ok +CREATE OR REPLACE TABLE without_optimizer AS SELECT * EXCLUDE (rn) +FROM (SELECT *, list_distance(vec, [5,5,5]) AS dist, row_number() over (ORDER BY dist ASC) as rn FROM vectors) +WHERE rn <= 7 + +statement ok +set disabled_optimizers = '' + +query III +SELECT * FROM (FROM with_optimizer) EXCEPT (FROM without_optimizer); +---- + +statement ok +CREATE OR REPLACE TABLE with_optimizer AS SELECT a.id, a.vec, neighbor.id as nbr_id, neighbor.vec as nbr_vec +FROM vectors as a, LATERAL ( + SELECT *, b.id + FROM vectors as b + ORDER BY list_distance(a.vec, b.vec) LIMIT 1 +) as neighbor +ORDER BY a.id, list_distance(a.vec, neighbor.vec); + +statement ok +set disabled_optimizers = 'top_n_window_elimination' + +statement ok +CREATE OR REPLACE TABLE without_optimizer AS SELECT a.id, a.vec, neighbor.id as nbr_id, neighbor.vec as nbr_vec +FROM vectors as a, LATERAL ( + SELECT *, b.id + FROM vectors as b + ORDER BY list_distance(a.vec, b.vec) LIMIT 1 +) as neighbor +ORDER BY a.id, list_distance(a.vec, neighbor.vec); + +query IIII +SELECT * FROM (FROM with_optimizer) EXCEPT (FROM without_optimizer); +--- diff --git a/test/optimizer/unnest_rewriter.test_slow b/test/optimizer/unnest_rewriter.test_slow index 01e8894e9a45..3163208939d9 100644 --- a/test/optimizer/unnest_rewriter.test_slow +++ b/test/optimizer/unnest_rewriter.test_slow @@ -124,7 +124,7 @@ EXPLAIN SELECT (SELECT UNNEST(i)) FROM (VALUES ([])) tbl(i); logical_opt :.*DELIM_JOIN.*SINGLE.* query II -EXPLAIN select * from (select [42, 43, 44]) t(a), (select unnest(t.a)) t2(b) where b=43; +EXPLAIN select * from (select [42, 43, 44]) t(a), (select unnest(t.a)) t2(b); ---- logical_opt :.*DELIM_JOIN.* diff --git a/test/parquet/parquet_fuzzer_issues.test b/test/parquet/parquet_fuzzer_issues.test new file mode 100644 index 000000000000..0296ef8e1998 --- /dev/null +++ b/test/parquet/parquet_fuzzer_issues.test @@ -0,0 +1,17 @@ +# name: test/parquet/parquet_fuzzer_issues.test +# description: Test Parquet fuzzer issues +# group: [parquet] + +require parquet + +# internal issue 6129 +statement error +from 'data/parquet-testing/broken/internal_6129.parquet' +---- +invalid number of miniblocks per block + +# internal issue 6165 +statement error +from 'data/parquet-testing/broken/internal_6165.parquet'; +---- +row group does not have enough columns diff --git a/test/parquet/variant/variant_all_types_shredded.test b/test/parquet/variant/variant_all_types_shredded.test new file mode 100644 index 000000000000..47efa02ae6ef --- /dev/null +++ b/test/parquet/variant/variant_all_types_shredded.test @@ -0,0 +1,55 @@ +# name: test/parquet/variant/variant_all_types_shredded.test +# group: [variant] + +require parquet + +statement ok +create macro data() as table ( + select COLUMNS([ + x for x in (*) if x NOT IN [ + 'utinyint', + 'usmallint', + 'uint', + 'ubigint', + 'hugeint', + 'uhugeint', + 'bignum', + 'timestamp_s', + 'timestamp_ms', + 'timestamp_tz', + 'time_tz', + 'interval', + 'bit', + 'dec_4_1', -- Parquet VARIANT doesn't have int16_t DECIMAL +-- Conversion isn't 1-to-1 + 'dec_9_4', -- can't roundtrip with json + 'dec_18_6', -- can't roundtrip with json + 'dec38_10', -- can't roundtrip with json + 'blob' -- data is base64-encoded in parquet read + ] + ])::VARIANT var from test_all_types() +) + +query I nosort expected_res +select * from data(); +---- + +foreach type bool tinyint smallint int bigint date time timestamp timestamp_ns timestamp_tz float double dec_9_4 dec_18_6 dec38_10 uuid varchar blob small_enum medium_enum large_enum int_array double_array date_array timestamp_array timestamptz_array varchar_array nested_int_array struct struct_of_arrays array_of_structs + +statement ok +SET VARIABLE type_str = (SELECT $$STRUCT("${type}" $$ || typeof("${type}") || ')' from test_all_types() limit 1); + +statement ok +COPY ( + FROM data() +) TO '__TEST_DIR__/all_types_shredded_${type}.parquet' ( + SHREDDING { + 'var': getvariable('type_str') + } +) + +query I nosort expected_res +select * from '__TEST_DIR__/all_types_shredded_${type}.parquet' +---- + +endloop diff --git a/test/parquet/variant/variant_basic.test b/test/parquet/variant/variant_basic.test index 668bccbeb4d4..8ef493778818 100644 --- a/test/parquet/variant/variant_basic.test +++ b/test/parquet/variant/variant_basic.test @@ -7,13 +7,13 @@ require parquet query II from 'data/parquet-testing/variant_array_array_string_and_integer.parquet'; ---- -1 [["string","iceberg",34],[34,null],[],["string","iceberg"],34] +1 [[string, iceberg, 34], [34, NULL], [], [string, iceberg], 34] # String query II from 'data/parquet-testing/variant_string.parquet'; ---- -1 "iceberg" +1 iceberg # BOOL TRUE query II @@ -25,13 +25,13 @@ from 'data/parquet-testing/variant_bool_true.parquet'; query II from 'data/parquet-testing/variant_decimal4_positive.parquet'; ---- -1 "123456.789" +1 123456.789 # UUID query II from 'data/parquet-testing/variant_uuid.parquet'; ---- -1 "f24f9b64-81fa-49d1-b74e-8c09a6e31c56" +1 f24f9b64-81fa-49d1-b74e-8c09a6e31c56 # Empty array query II @@ -53,13 +53,13 @@ from 'data/parquet-testing/variant_int32.parquet'; query II from 'data/parquet-testing/variant_binary.parquet'; ---- -1 "CgsMDQ==" +1 CgsMDQ== # Decimal16 query II from 'data/parquet-testing/variant_decimal16.parquet'; ---- -1 "9876543210.123456789" +1 9876543210.123456789 query II from 'data/parquet-testing/variant_int64.parquet'; @@ -70,30 +70,35 @@ from 'data/parquet-testing/variant_int64.parquet'; query II from 'data/parquet-testing/variant_timestamp_nanos_ntz.parquet'; ---- -1 "1957-11-07 12:33:54.123456789" +1 1957-11-07 12:33:54.123456789 # Array of strings (2-dimensional) query II from 'data/parquet-testing/variant_array_array_string.parquet'; ---- -1 [["string","iceberg"],["apple","banana"]] +1 [[string, iceberg], [apple, banana]] # TIMESTAMP_MICROS query II from 'data/parquet-testing/variant_timestamp_micros.parquet'; ---- -1 "1957-11-07 12:33:54.123456+00" +1 1957-11-07 12:33:54.123456+00 + +query II +from 'data/parquet-testing/variant_timestamp_micros_negative.parquet'; +---- +1 1957-11-07 12:33:54.123456 # Object {'a': .., 'c': ...} query II from 'data/parquet-testing/variant_object_primitives.parquet'; ---- -1 {"a":123456789,"c":"string"} +1 {'a': 123456789, 'c': string} query II from 'data/parquet-testing/variant_timestamp_micros_positive.parquet'; ---- -1 "2024-11-07 12:33:54.123456+00" +1 2024-11-07 12:33:54.123456+00 query II from 'data/parquet-testing/variant_int16_positive.parquet'; @@ -103,27 +108,22 @@ from 'data/parquet-testing/variant_int16_positive.parquet'; query II from 'data/parquet-testing/variant_time_ntz.parquet'; ---- -1 "12:33:54.123456" +1 12:33:54.123456 query II from 'data/parquet-testing/variant_decimal16_negative.parquet'; ---- -1 "-9876543210.123456789" +1 -9876543210.123456789 query II from 'data/parquet-testing/variant_timestamp_nanos1.parquet'; ---- -1 "1957-11-07 12:33:54.123457+00" +1 1957-11-07 12:33:54.123457+00 query II from 'data/parquet-testing/variant_decimal8_negative.parquet'; ---- -1 "-123456789.987654321" - -query II -from 'data/parquet-testing/variant_timestamp_micros_negative.parquet'; ----- -1 "1957-11-07 12:33:54.123456" +1 -123456789.987654321 query II from 'data/parquet-testing/variant_int8_positive.parquet'; @@ -133,7 +133,7 @@ from 'data/parquet-testing/variant_int8_positive.parquet'; query II from 'data/parquet-testing/variant_timestamp_nanos2.parquet'; ---- -1 "2024-11-07 12:33:54.123456+00" +1 2024-11-07 12:33:54.123456+00 query II from 'data/parquet-testing/variant_int8_negative.parquet'; @@ -143,12 +143,12 @@ from 'data/parquet-testing/variant_int8_negative.parquet'; query II from 'data/parquet-testing/variant_array_string.parquet'; ---- -1 ["iceberg","string"] +1 [iceberg, string] query II from 'data/parquet-testing/variant_date_negative.parquet'; ---- -1 "1957-11-07" +1 1957-11-07 query II from 'data/parquet-testing/variant_int64_positive.parquet'; @@ -158,7 +158,7 @@ from 'data/parquet-testing/variant_int64_positive.parquet'; query II from 'data/parquet-testing/variant_array_object_string_and_integer.parquet'; ---- -1 [{"a":123456789,"c":"string"},{"a":123456789,"c":"string"},"iceberg",34] +1 [{'a': 123456789, 'c': string}, {'a': 123456789, 'c': string}, iceberg, 34] query II from 'data/parquet-testing/variant_int32_positive.parquet'; @@ -184,22 +184,22 @@ from 'data/parquet-testing/variant_null.parquet'; query II from 'data/parquet-testing/variant_float_negative.parquet'; ---- -1 -10.109999656677246 +1 -10.11 query II from 'data/parquet-testing/variant_object_string_and_array.parquet'; ---- -1 {"a":123456789,"c":["string","iceberg"]} +1 {'a': 123456789, 'c': [string, iceberg]} query II from 'data/parquet-testing/variant_object_null_and_string.parquet'; ---- -1 {"a":null,"d":"iceberg"} +1 {'a': NULL, 'd': iceberg} query II from 'data/parquet-testing/variant_date_positive.parquet'; ---- -1 "2024-11-07" +1 2024-11-07 query II from 'data/parquet-testing/variant_bool_false.parquet'; @@ -209,12 +209,12 @@ from 'data/parquet-testing/variant_bool_false.parquet'; query II from 'data/parquet-testing/variant_array_object_string.parquet'; ---- -1 [{"a":123456789,"c":"string"},{"a":123456789,"c":"string"}] +1 [{'a': 123456789, 'c': string}, {'a': 123456789, 'c': string}] query II from 'data/parquet-testing/variant_decimal4_negative.parquet'; ---- -1 "-123456.789" +1 -123456.789 query II from 'data/parquet-testing/variant_double_positive.parquet'; @@ -224,4 +224,4 @@ from 'data/parquet-testing/variant_double_positive.parquet'; query II from 'data/parquet-testing/variant_timestamp_micros_ntz_positive.parquet'; ---- -1 "2024-11-07 12:33:54.123456" +1 2024-11-07 12:33:54.123456 diff --git a/test/parquet/variant/variant_basic_shredded_writing.test b/test/parquet/variant/variant_basic_shredded_writing.test new file mode 100644 index 000000000000..3bd8219f7120 --- /dev/null +++ b/test/parquet/variant/variant_basic_shredded_writing.test @@ -0,0 +1,47 @@ +# name: test/parquet/variant/variant_basic_shredded_writing.test +# group: [variant] + +require parquet + +statement ok +create macro data() AS TABLE ( + FROM (VALUES + ({'a': 21::INTEGER, 'b': NULL}::VARIANT), + ({'a': 42::INTEGER, 'd': 'test'}::VARIANT), + ([]::VARIANT), + (NULL::VARIANT), + ([{'b': True, 'c': 'test'}::VARIANT, 'test', 21, {'a': True}, [1::VARIANT, 2, True, 'false']]::VARIANT), + ('this is a long string'::VARIANT), + ('this is big enough to not be classified as a "short string" by parquet VARIANT'::VARIANT) + ) t(a) +) + +query I nosort expected_res +select * from data(); +---- + +statement ok +COPY ( + from data() t(a) +) TO '__TEST_DIR__/shredded_struct.parquet' ( + shredding { + a: 'STRUCT(a INTEGER, b VARIANT, c BOOLEAN)' + } +) + +query I nosort expected_res +select * from '__TEST_DIR__/shredded_struct.parquet'; +---- + +statement ok +COPY ( + select a from data() +) TO '__TEST_DIR__/shredded_list.parquet' ( + shredding { + a: 'VARCHAR[]' + } +) + +query I nosort expected_res +select * from '__TEST_DIR__/shredded_list.parquet'; +---- diff --git a/test/parquet/variant/variant_basic_writing.test b/test/parquet/variant/variant_basic_writing.test new file mode 100644 index 000000000000..7d9b6ae9f5ca --- /dev/null +++ b/test/parquet/variant/variant_basic_writing.test @@ -0,0 +1,116 @@ +# name: test/parquet/variant/variant_basic_writing.test +# group: [variant] + +require parquet + +require json + +# STRUCT(a INTEGER, b INTEGER[]) +statement ok +COPY (select + { + 'a': 42, + 'b': [null, 1, 2] + }::VARIANT + from range(10) +) TO '__TEST_DIR__/integer_variant.parquet'; + +query I +select * from '__TEST_DIR__/integer_variant.parquet'; +---- +{'a': 42, 'b': [NULL, 1, 2]} +{'a': 42, 'b': [NULL, 1, 2]} +{'a': 42, 'b': [NULL, 1, 2]} +{'a': 42, 'b': [NULL, 1, 2]} +{'a': 42, 'b': [NULL, 1, 2]} +{'a': 42, 'b': [NULL, 1, 2]} +{'a': 42, 'b': [NULL, 1, 2]} +{'a': 42, 'b': [NULL, 1, 2]} +{'a': 42, 'b': [NULL, 1, 2]} +{'a': 42, 'b': [NULL, 1, 2]} + +statement ok +COPY (select + '[["string","iceberg",-34],[-34,null],[],["string","iceberg"],-34]'::JSON::VARIANT + from range(5) +) TO '__TEST_DIR__/list_of_list_variant.parquet' + +query I +select * from '__TEST_DIR__/list_of_list_variant.parquet'; +---- +[[string, iceberg, -34], [-34, NULL], [], [string, iceberg], -34] +[[string, iceberg, -34], [-34, NULL], [], [string, iceberg], -34] +[[string, iceberg, -34], [-34, NULL], [], [string, iceberg], -34] +[[string, iceberg, -34], [-34, NULL], [], [string, iceberg], -34] +[[string, iceberg, -34], [-34, NULL], [], [string, iceberg], -34] + +statement ok +COPY ( + with cte as ( + FROM (VALUES + ({'a': 21, 'b': NULL}::VARIANT), + ([]::VARIANT), + (NULL::VARIANT), + ([{'b': True, 'c': 'test'}]::VARIANT), + ('this is a long string'::VARIANT), + ('this is big enough to not be classified as a "short string" by parquet VARIANT'::VARIANT) + ) t(a) + ) + select a from cte +) TO '__TEST_DIR__/varied_variant.parquet' + +query I +select * from '__TEST_DIR__/varied_variant.parquet'; +---- +{'a': 21, 'b': NULL} +[] +NULL +[{'b': true, 'c': test}] +this is a long string +this is big enough to not be classified as a "short string" by parquet VARIANT + +# VARIANT is only supported at the root for now +statement error +COPY (select [123::VARIANT]) TO '__TEST_DIR__/list_of_variant.parquet' +---- +Not implemented Error: Unimplemented type for Parquet "VARIANT" + +statement ok +create macro data() as table ( + select COLUMNS([ + x for x in (*) if x NOT IN [ + 'utinyint', + 'usmallint', + 'uint', + 'ubigint', + 'hugeint', + 'uhugeint', + 'bignum', + 'timestamp_s', + 'timestamp_ms', + 'timestamp_tz', + 'time_tz', + 'interval', + 'bit', + 'dec_4_1', -- Parquet VARIANT doesn't have int16_t DECIMAL +-- Conversion isn't 1-to-1 + 'dec_9_4', -- can't roundtrip with json + 'dec_18_6', -- can't roundtrip with json + 'dec38_10', -- can't roundtrip with json + 'blob' -- data is base64-encoded in parquet read + ] + ])::VARIANT as "\0" from test_all_types() +) + +statement ok +COPY ( + from data() +) TO '__TEST_DIR__/variant_test_all_types.parquet'; + +query I nosort expected_res +select * from data(); +---- + +query I nosort expected_res +select * from '__TEST_DIR__/variant_test_all_types.parquet'; +---- diff --git a/test/parquet/variant/variant_filtering.test b/test/parquet/variant/variant_filtering.test new file mode 100644 index 000000000000..95b1b6a61ff3 --- /dev/null +++ b/test/parquet/variant/variant_filtering.test @@ -0,0 +1,17 @@ +# name: test/parquet/variant/variant_filtering.test +# group: [variant] + +require parquet + +query II +from parquet_scan('data/parquet-testing/variant_array_object_string.parquet') WHERE var != [{a:123456789,c:'string'},{a:123456789,c:'string'}] +---- + +query I +select id from parquet_scan('data/parquet-testing/variant_array_object_string.parquet') WHERE var == [{a:123456789,c:'string'},{a:123456789,c:'string'}] +---- +1 + +query I +select id from parquet_scan('data/parquet-testing/variant_array_object_string.parquet') WHERE var == [{a:123456788,c:'string'},{a:123456789,c:'string'}] +---- diff --git a/test/parquet/variant/variant_list_of_struct_partial_shredding.test b/test/parquet/variant/variant_list_of_struct_partial_shredding.test new file mode 100644 index 000000000000..b2d036410fd4 --- /dev/null +++ b/test/parquet/variant/variant_list_of_struct_partial_shredding.test @@ -0,0 +1,62 @@ +# name: test/parquet/variant/variant_list_of_struct_partial_shredding.test +# group: [variant] + +require parquet + +statement ok +create macro data() AS TABLE ( + FROM (VALUES + ( + [ + {a:['foo'::VARIANT,42], b:true, c:{a:'nested1'}}::VARIANT, -- element of list in field 'a' is a different type + {a: 42, b: true, c:{a:'nested2'}}, -- field 'a' is a different type + {b: true, c:{a:'nested3'}}, -- field 'a' is missing + {a:[], b:false, c:{a:NULL}}, + {a: [], c:{a:'nested4'}} -- field 'b' is missing + ]::VARIANT + ), + ( + [] + ), + ( + [ + {a:NULL, b:NULL, c:{a:'inner'}}, + {a:['baz'], b:false, c:{a:NULL}} + ] + ), + ( + NULL + ), + ( + [ + {a:['alpha'], b:true, c:{a:'deep'}}::VARIANT, + {a: [[1,2]::VARIANT, 'hello', {a: 42}]}, -- fields 'b' and 'c' are missing, 'a' element is of a wrong type + {b: false}, -- fields 'a' and 'c' are missing + {a:[], b:NULL, c:{a:'leaf'}} + ] + ), + ( + [ + {a:NULL, b:false, c:{a:NULL}}, + {a:['x',NULL,'z'], b:true, c:{a:'final'}} + ] + ) + ) t(a) +); + +query I nosort expected_res +select * from data(); +---- + +statement ok +COPY ( + select a from data() +) TO '__TEST_DIR__/shredded_list_of_structs.parquet' ( + shredding { + a: 'STRUCT(a VARCHAR[], b BOOLEAN, c STRUCT(a VARCHAR))[]' + } +) + +query I nosort expected_res +select * from '__TEST_DIR__/shredded_list_of_structs.parquet'; +---- diff --git a/test/parquet/variant/variant_list_of_struct_shredding.test b/test/parquet/variant/variant_list_of_struct_shredding.test new file mode 100644 index 000000000000..c9f9d5d79714 --- /dev/null +++ b/test/parquet/variant/variant_list_of_struct_shredding.test @@ -0,0 +1,57 @@ +# name: test/parquet/variant/variant_list_of_struct_shredding.test +# group: [variant] + +require parquet + +statement ok +create macro data() AS TABLE ( + FROM (VALUES + ( + [ + {a:['foo','bar'], b:true, c:{a:'nested1'}}, + {a:[], b:false, c:{a:NULL}} + ]::VARIANT + ), + ( + [] + ), + ( + [ + {a:NULL, b:NULL, c:{a:'inner'}}, + {a:['baz'], b:false, c:{a:NULL}} + ] + ), + ( + NULL + ), + ( + [ + {a:['alpha'], b:true, c:{a:'deep'}}, + {a:[], b:NULL, c:{a:'leaf'}} + ] + ), + ( + [ + {a:NULL, b:false, c:{a:NULL}}, + {a:['x',NULL,'z'], b:true, c:{a:'final'}} + ] + ) + ) t(a) +); + +query I nosort expected_res +select * from data(); +---- + +statement ok +COPY ( + select a from data() +) TO '__TEST_DIR__/shredded_list_of_structs.parquet' ( + shredding { + a: 'STRUCT(a VARCHAR[], b BOOLEAN, c STRUCT(a VARCHAR))[]' + } +) + +query I nosort expected_res +select * from '__TEST_DIR__/shredded_list_of_structs.parquet'; +---- diff --git a/test/parquet/variant/variant_list_shredding.test b/test/parquet/variant/variant_list_shredding.test new file mode 100644 index 000000000000..a148414e14c5 --- /dev/null +++ b/test/parquet/variant/variant_list_shredding.test @@ -0,0 +1,32 @@ +# name: test/parquet/variant/variant_list_shredding.test +# group: [variant] + +require parquet + +statement ok +create macro data() AS TABLE ( + FROM (VALUES + ([['test', NULL, 'this is a long string'],[],['hello'],NULL,[],[1, 2, 3]::VARIANT]::VARIANT), + (NULL::VARIANT), + ([]::VARIANT), + ([[{'a': 'test'}::VARIANT, [1, 2, 3]]::VARIANT, {'a': 21}, {'b': 42}, [['hello']]]), + ([[], NULL, [1::VARIANT, 2, 'test'],['hello', 'world']]::VARIANT) + ) t(a) +) + +query I nosort expected_res +select * from data(); +---- + +statement ok +COPY ( + select a from data() +) TO '__TEST_DIR__/shredded_list_of_list_of_string.parquet' ( + shredding { + a: 'VARCHAR[][]' + } +) + +query I nosort expected_res +select * from '__TEST_DIR__/shredded_list_of_list_of_string.parquet'; +---- diff --git a/test/parquet/variant/variant_nanos_tz.test b/test/parquet/variant/variant_nanos_tz.test index f63581958a81..6cea0a39432d 100644 --- a/test/parquet/variant/variant_nanos_tz.test +++ b/test/parquet/variant/variant_nanos_tz.test @@ -3,29 +3,26 @@ require parquet -statement ok -set variant_legacy_encoding=true; - # Timestamp NS - negative (with timezone) (shredded) query II -from 'data/parquet-testing/variant_shredded_timestamp_nanos_tz_negative_no_logical_type.parquet'; +from 'data/parquet-testing/variant_shredded_timestamp_nanos_tz_negative.parquet'; ---- -1 "1957-11-07 12:33:54.123457+00" +1 1957-11-07 12:33:54.123457+00 # Timestamp NS - positive (with timezone) (shredded) query II -from 'data/parquet-testing/variant_shredded_timestamp_nanos_tz_positive_no_logical_type.parquet'; +from 'data/parquet-testing/variant_shredded_timestamp_nanos_tz_positive.parquet'; ---- -1 "2024-11-07 12:33:54.123456+00" +1 2024-11-07 12:33:54.123456+00 # Timestamp NS - positive (with timezone) (unshredded) query II -from 'data/parquet-testing/variant_timestamp_nanos_tz_positive_no_logical_type.parquet'; +from 'data/parquet-testing/variant_timestamp_nanos_tz_positive.parquet'; ---- -1 "2024-11-07 12:33:54.123456+00" +1 2024-11-07 12:33:54.123456+00 # Timestamp NS - negative (with timezone) (unshredded) query II -from 'data/parquet-testing/variant_timestamp_nanos_tz_negative_no_logical_type.parquet'; +from 'data/parquet-testing/variant_timestamp_nanos_tz_negative.parquet'; ---- -1 "1957-11-07 12:33:54.123457+00" +1 1957-11-07 12:33:54.123457+00 diff --git a/test/parquet/variant/variant_nested_with_nulls.test b/test/parquet/variant/variant_nested_with_nulls.test deleted file mode 100644 index 91aa2e3fcfd1..000000000000 --- a/test/parquet/variant/variant_nested_with_nulls.test +++ /dev/null @@ -1,44 +0,0 @@ -# name: test/parquet/variant/variant_nested_with_nulls.test -# group: [variant] - -require parquet - -query IIIIII -describe from parquet_scan('data/parquet-testing/variant_unshredded_nested_nulls.parquet') ----- -id BIGINT YES NULL NULL NULL -v STRUCT("value" BLOB, metadata BLOB) YES NULL NULL NULL -array_of_variants STRUCT("value" BLOB, metadata BLOB)[] YES NULL NULL NULL -struct_of_variants STRUCT(v STRUCT("value" BLOB, metadata BLOB)) YES NULL NULL NULL -map_of_variants MAP(VARCHAR, STRUCT("value" BLOB, metadata BLOB)) YES NULL NULL NULL -array_of_struct_of_variants STRUCT(v STRUCT("value" BLOB, metadata BLOB))[] YES NULL NULL NULL -struct_of_array_of_variants STRUCT(v STRUCT("value" BLOB, metadata BLOB)[]) YES NULL NULL NULL - -statement ok -set variant_legacy_encoding=true; - -# Now the variant column gets emitted as JSON -query IIIIII -describe from parquet_scan('data/parquet-testing/variant_unshredded_nested_nulls.parquet') ----- -id BIGINT YES NULL NULL NULL -v JSON YES NULL NULL NULL -array_of_variants JSON[] YES NULL NULL NULL -struct_of_variants STRUCT(v JSON) YES NULL NULL NULL -map_of_variants MAP(VARCHAR, JSON) YES NULL NULL NULL -array_of_struct_of_variants STRUCT(v JSON)[] YES NULL NULL NULL -struct_of_array_of_variants STRUCT(v JSON[]) YES NULL NULL NULL - -query IIIIIII -select * from parquet_scan('data/parquet-testing/variant_unshredded_nested_nulls.parquet') order by id limit 10; ----- -0 {"key":0} ['{"key":0}', NULL, '{"key":0}', NULL, '{"key":0}'] {'v': '{"key":0}'} {0='{"key":0}', nullKey=NULL} [{'v': '{"key":0}'}, {'v': NULL}, NULL, {'v': '{"key":0}'}, NULL, {'v': '{"key":0}'}] {'v': [NULL, '{"key":0}']} -0 {"key":0} ['{"key":0}', NULL, '{"key":0}', NULL, '{"key":0}'] {'v': '{"key":0}'} {0='{"key":0}', nullKey=NULL} [{'v': '{"key":0}'}, {'v': NULL}, NULL, {'v': '{"key":0}'}, NULL, {'v': '{"key":0}'}] {'v': [NULL, '{"key":0}']} -1 {"key":1} ['{"key":1}', NULL, '{"key":1}', NULL, '{"key":1}'] {'v': '{"key":1}'} {1='{"key":1}', nullKey=NULL} [{'v': '{"key":1}'}, {'v': NULL}, NULL, {'v': '{"key":1}'}, NULL, {'v': '{"key":1}'}] {'v': [NULL, '{"key":1}']} -1 {"key":1} ['{"key":1}', NULL, '{"key":1}', NULL, '{"key":1}'] {'v': '{"key":1}'} {1='{"key":1}', nullKey=NULL} [{'v': '{"key":1}'}, {'v': NULL}, NULL, {'v': '{"key":1}'}, NULL, {'v': '{"key":1}'}] {'v': [NULL, '{"key":1}']} -2 {"key":2} ['{"key":2}', NULL, '{"key":2}', NULL, '{"key":2}'] {'v': '{"key":2}'} {2='{"key":2}', nullKey=NULL} [{'v': '{"key":2}'}, {'v': NULL}, NULL, {'v': '{"key":2}'}, NULL, {'v': '{"key":2}'}] {'v': [NULL, '{"key":2}']} -3 {"key":3} ['{"key":3}', NULL, '{"key":3}', NULL, '{"key":3}'] {'v': '{"key":3}'} {3='{"key":3}', nullKey=NULL} [{'v': '{"key":3}'}, {'v': NULL}, NULL, {'v': '{"key":3}'}, NULL, {'v': '{"key":3}'}] {'v': [NULL, '{"key":3}']} -4 {"key":4} ['{"key":4}', NULL, '{"key":4}', NULL, '{"key":4}'] {'v': '{"key":4}'} {4='{"key":4}', nullKey=NULL} [{'v': '{"key":4}'}, {'v': NULL}, NULL, {'v': '{"key":4}'}, NULL, {'v': '{"key":4}'}] {'v': [NULL, '{"key":4}']} -5 {"key":5} ['{"key":5}', NULL, '{"key":5}', NULL, '{"key":5}'] {'v': '{"key":5}'} {5='{"key":5}', nullKey=NULL} [{'v': '{"key":5}'}, {'v': NULL}, NULL, {'v': '{"key":5}'}, NULL, {'v': '{"key":5}'}] {'v': [NULL, '{"key":5}']} -6 {"key":6} ['{"key":6}', NULL, '{"key":6}', NULL, '{"key":6}'] {'v': '{"key":6}'} {6='{"key":6}', nullKey=NULL} [{'v': '{"key":6}'}, {'v': NULL}, NULL, {'v': '{"key":6}'}, NULL, {'v': '{"key":6}'}] {'v': [NULL, '{"key":6}']} -7 {"key":7} ['{"key":7}', NULL, '{"key":7}', NULL, '{"key":7}'] {'v': '{"key":7}'} {7='{"key":7}', nullKey=NULL} [{'v': '{"key":7}'}, {'v': NULL}, NULL, {'v': '{"key":7}'}, NULL, {'v': '{"key":7}'}] {'v': [NULL, '{"key":7}']} diff --git a/test/parquet/variant/variant_partially_shredded.test b/test/parquet/variant/variant_partially_shredded.test index 1e7ee7e68c2e..400c351347c1 100644 --- a/test/parquet/variant/variant_partially_shredded.test +++ b/test/parquet/variant/variant_partially_shredded.test @@ -3,10 +3,6 @@ require parquet -query II nosort result -from 'data/parquet-testing/variant_partial_shredded0.parquet'; ----- - query II nosort result from 'data/parquet-testing/variant_partial_shredded1.parquet'; ---- @@ -187,3 +183,7 @@ from 'data/parquet-testing/variant_partial_shredded44.parquet'; query II nosort result from 'data/parquet-testing/variant_partial_shredded45.parquet'; ---- + +query II nosort result +from 'data/parquet-testing/variant_partial_shredded46.parquet'; +---- diff --git a/test/parquet/variant/variant_roundtrip.test_slow b/test/parquet/variant/variant_roundtrip.test_slow new file mode 100644 index 000000000000..7b18be593bde --- /dev/null +++ b/test/parquet/variant/variant_roundtrip.test_slow @@ -0,0 +1,37 @@ +# name: test/parquet/variant/variant_roundtrip.test_slow +# group: [variant] + +require parquet + +foreach parquet_file p2strings.parquet p2.parquet pandas-date.parquet parquet_with_json.parquet spark-store.parquet struct_skip_test.parquet timestamp.parquet candidate.parquet + +statement ok +COPY ( + SELECT + COLUMNS(*)::VARIANT + FROM read_parquet('data/parquet-testing/${parquet_file}') +) TO '__TEST_DIR__/variant_${parquet_file}' (FORMAT PARQUET); + +query I nosort expected_res +SELECT COLUMNS(*)::VARIANT FROM read_parquet('data/parquet-testing/${parquet_file}') + +query I nosort expected_res +SELECT COLUMNS(*)::VARIANT FROM read_parquet('__TEST_DIR__/variant_${parquet_file}') + +reset label expected_res + +endloop + +foreach parquet_file 7-set.snappy.arrow2.parquet adam_genotypes.parquet apkwan.parquet arrow_nan.parquet aws_kinesis.parquet aws1.snappy.parquet aws2.parquet bigdecimal.parquet binary_string.parquet blob.parquet boolean_stats.parquet bug13053-2.parquet bug13053.parquet bug14120-dict-nulls-only.parquet bug1554.parquet bug1588.parquet bug1589.parquet bug1618_struct_strings.parquet bug2267.parquet bug2557.parquet bug3734.parquet bug4442.parquet bug4859.parquet bug4903.parquet bug687_nulls.parquet byte_stream_split.parquet CASE_INSENSITIVE.PARQUET complex.parquet corrupt_stats.parquet data-types.parquet date.parquet delta_byte_array.parquet delta_length_byte_array.parquet empty.parquet enum.parquet file_row_number.parquet filter_bug1391.parquet fixed.parquet float16.parquet incorrect_index_page_offsets.parquet issue_6013.parquet issue10279_delta_encoding.parquet issue12621.parquet issue6630_1.parquet issue6630_2.parquet issue6990.parquet issue9417.parquet leftdate3_192_loop_1.parquet lineitem-top10000.gzip.parquet list_sort_segfault.parquet manyrowgroups.parquet manyrowgroups2.parquet map.parquet multi_bloom_a.parquet multi_bloom_b.parquet multi_bloom_c.parquet nan-float.parquet nullbyte_multiple.parquet nullbyte.parquet parquet_go.parquet rle_boolean_encoding.parquet seqs_table.parquet signed_stats.parquet silly-names.parquet simple.parquet sorted.zstd_18_131072_small.parquet spark-ontime.parquet struct.parquet test_unnest_rewriter.parquet timestamp-ms.parquet tz.parquet upsert_bug.parquet userdata1.parquet varchar_stats.parquet zstd.parquet + +statement ok +COPY ( + SELECT + COLUMNS(*)::VARIANT + FROM read_parquet('data/parquet-testing/${parquet_file}') +) TO '__TEST_DIR__/variant_${parquet_file}' (FORMAT PARQUET); + +statement ok +SELECT COLUMNS(*)::VARIANT FROM read_parquet('__TEST_DIR__/variant_${parquet_file}') + +endloop diff --git a/test/parquet/variant/variant_shredded.test b/test/parquet/variant/variant_shredded.test index 46c4225f2565..bcccc20282f9 100644 --- a/test/parquet/variant/variant_shredded.test +++ b/test/parquet/variant/variant_shredded.test @@ -7,13 +7,13 @@ require parquet query II from 'data/parquet-testing/variant_shredded_timestamp_nanos_ntz_positive.parquet'; ---- -1 "2024-11-07 12:33:54.123456789" +1 2024-11-07 12:33:54.123456789 # Float - negative query II from 'data/parquet-testing/variant_shredded_float_negative.parquet'; ---- -1 -10.109999656677246 +1 -10.11 # Int64 - negative query II @@ -25,37 +25,37 @@ from 'data/parquet-testing/variant_shredded_int64_negative.parquet'; query II from 'data/parquet-testing/variant_shredded_decimal16_negative.parquet'; ---- -1 "-9876543210.123456789" +1 -9876543210.123456789 # UUID query II from 'data/parquet-testing/variant_shredded_uuid.parquet'; ---- -1 "f24f9b64-81fa-49d1-b74e-8c09a6e31c56" +1 f24f9b64-81fa-49d1-b74e-8c09a6e31c56 # Decimal4 - negative query II from 'data/parquet-testing/variant_shredded_decimal4_negative.parquet'; ---- -1 "-123456.789" +1 -123456.789 # Decimal4 - positive query II from 'data/parquet-testing/variant_shredded_decimal4_positive.parquet'; ---- -1 "123456.789" +1 123456.789 # Timestamp Micros - negative (no timezone) query II from 'data/parquet-testing/variant_shredded_timestamp_micros_ntz_negative.parquet'; ---- -1 "1957-11-07 12:33:54.123456" +1 1957-11-07 12:33:54.123456 # Date - negative query II from 'data/parquet-testing/variant_shredded_date_negative.parquet'; ---- -1 "1957-11-07" +1 1957-11-07 # int8 - positive query II @@ -73,32 +73,32 @@ from 'data/parquet-testing/variant_shredded_int16_positive.parquet'; query II from 'data/parquet-testing/variant_shredded_decimal8_negative.parquet'; ---- -1 "-123456789.987654321" +1 -123456789.987654321 # string query II from 'data/parquet-testing/variant_shredded_string.parquet'; ---- -1 "iceberg" +1 iceberg -# FIXME: this is actually a Timestamp Nanos - positive (with timezone) -# Timestamp Micros - positive (with timezone) -query II -from 'data/parquet-testing/variant_shredded_timestamp_micros_tz_positive.parquet'; ----- -1 "2024-11-07 12:33:54.123456+00" +## FIXME: this is actually a Timestamp Nanos - positive (with timezone) +## Timestamp Micros - positive (with timezone) +#query II +#from 'data/parquet-testing/variant_shredded_timestamp_micros_tz_positive.parquet'; +#---- +#1 2024-11-07 12:33:54.123456+00 # binary query II from 'data/parquet-testing/variant_shredded_binary.parquet'; ---- -1 "CgsMDQ==" +1 CgsMDQ== # float - positive query II from 'data/parquet-testing/variant_shredded_float_positive.parquet'; ---- -1 10.109999656677246 +1 10.11 # double - positive query II @@ -110,13 +110,13 @@ from 'data/parquet-testing/variant_shredded_double_positive.parquet'; query II from 'data/parquet-testing/variant_shredded_decimal16_positive.parquet'; ---- -1 "9876543210.123456789" +1 9876543210.123456789 # Timestamp Micros - positive (no timezone) query II from 'data/parquet-testing/variant_shredded_timestamp_micros_ntz_positive.parquet'; ---- -1 "2024-11-07 12:33:54.123456" +1 2024-11-07 12:33:54.123456 # int16 - negative query II @@ -128,25 +128,25 @@ from 'data/parquet-testing/variant_shredded_int16_negative.parquet'; query II from 'data/parquet-testing/variant_shredded_timestamp_micros_tz_positive2.parquet'; ---- -1 "2024-11-07 12:33:54.123456+00" +1 2024-11-07 12:33:54.123456+00 # Timestamp Micros - negative (with timezone) query II from 'data/parquet-testing/variant_shredded_timestamp_micros_tz_negative.parquet'; ---- -1 "1957-11-07 12:33:54.123456+00" +1 1957-11-07 12:33:54.123456+00 # decimal8 - positive query II from 'data/parquet-testing/variant_shredded_decimal8_positive.parquet'; ---- -1 "123456789.987654321" +1 123456789.987654321 # Timestamp Nanos - negative (no timezone) query II from 'data/parquet-testing/variant_shredded_timestamp_nanos_ntz_negative.parquet'; ---- -1 "1957-11-07 12:33:54.123456789" +1 1957-11-07 12:33:54.123456789 # int32 - positive query II @@ -165,7 +165,7 @@ from 'data/parquet-testing/variant_shredded_int32_negative.parquet'; query II from 'data/parquet-testing/variant_shredded_timestamp_micros_tz_negative2.parquet'; ---- -1 "1957-11-07 12:33:54.123457+00" +1 1957-11-07 12:33:54.123457+00 # int8 - negative query II @@ -177,13 +177,13 @@ from 'data/parquet-testing/variant_shredded_int8_negative.parquet'; query II from 'data/parquet-testing/variant_shredded_time_micros_ntz.parquet'; ---- -1 "12:33:54.123456" +1 12:33:54.123456 # Date - positive query II from 'data/parquet-testing/variant_shredded_date_positive.parquet'; ---- -1 "2024-11-07" +1 2024-11-07 # bool - true query II diff --git a/test/parquet/variant/variant_shredded_nested.test b/test/parquet/variant/variant_shredded_nested.test index 22735cc3f057..016da62138cf 100644 --- a/test/parquet/variant/variant_shredded_nested.test +++ b/test/parquet/variant/variant_shredded_nested.test @@ -7,34 +7,34 @@ require parquet query II from 'data/parquet-testing/variant_shredded_array1.parquet'; ---- -1 [["string","iceberg"],["apple","banana"]] +1 [[string, iceberg], [apple, banana]] # Array query II from 'data/parquet-testing/variant_shredded_array2.parquet'; ---- -1 [{"a":123456789,"c":"string"},{"a":123456789,"c":"string"}] +1 [{'a': 123456789, 'c': string}, {'a': 123456789, 'c': string}] # Array query II from 'data/parquet-testing/variant_shredded_array3.parquet'; ---- -1 ["iceberg","string"] +1 [iceberg, string] # Object query II from 'data/parquet-testing/variant_shredded_object1.parquet'; ---- -1 {"a":123456789,"c":"string"} +1 {'a': 123456789, 'c': string} # Object query II from 'data/parquet-testing/variant_shredded_object2.parquet'; ---- -1 {"a":null,"d":"iceberg"} +1 {'a': NULL, 'd': iceberg} # Object query II from 'data/parquet-testing/variant_shredded_object3.parquet'; ---- -1 {"a":123456789,"c":["string","iceberg"]} +1 {'a': 123456789, 'c': [string, iceberg]} diff --git a/test/parquet/variant/variant_to_parquet_variant.test b/test/parquet/variant/variant_to_parquet_variant.test new file mode 100644 index 000000000000..be846388d7af --- /dev/null +++ b/test/parquet/variant/variant_to_parquet_variant.test @@ -0,0 +1,15 @@ +# name: test/parquet/variant/variant_to_parquet_variant.test +# group: [variant] + +require parquet + +query I +select variant_to_parquet_variant(NULL) +---- +{'metadata': \x11\x00\x00, 'value': \x00} + +# We don't expose the overload with a shredded type, only internally will we use that +statement error +select variant_to_parquet_variant(NULL, 'STRUCT(a VARCHAR)'::VARCHAR) +---- +Binder Error diff --git a/test/persistence/test_file_matches_wal.cpp b/test/persistence/test_file_matches_wal.cpp index 31e18328115b..e2253c5ec69b 100644 --- a/test/persistence/test_file_matches_wal.cpp +++ b/test/persistence/test_file_matches_wal.cpp @@ -55,14 +55,12 @@ TEST_CASE("Test replaying mismatching WAL files", "[persistence][.]") { result = con.Query("ATTACH '" + too_old_path_file + "';"); REQUIRE(result->HasError()); string error_msg = result->GetError(); - REQUIRE(StringUtil::Contains(error_msg, "That means the WAL was created for a older version of this database. File " - "checkpoint iteration: 1, WAL checkpoint iteration: 0")); + REQUIRE(StringUtil::Contains(error_msg, "older")); result = con.Query("ATTACH '" + too_new_path_file + "';"); REQUIRE(result->HasError()); error_msg = result->GetError(); - REQUIRE(StringUtil::Contains(error_msg, "That means the WAL was created for a newer version of this database. File " - "checkpoint iteration: 0, WAL checkpoint iteration: 1")); + REQUIRE(StringUtil::Contains(error_msg, "newer")); // Create and initialize a different file. string other_db_path = test_dir + "/my_other_db.db"; diff --git a/test/secrets/test_custom_secret_storage.cpp b/test/secrets/test_custom_secret_storage.cpp index a34cf14bd00c..ad77d2378f54 100644 --- a/test/secrets/test_custom_secret_storage.cpp +++ b/test/secrets/test_custom_secret_storage.cpp @@ -6,6 +6,7 @@ #include "duckdb/main/secret/secret_storage.hpp" #include "duckdb/main/secret/secret.hpp" #include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/main/extension_manager.hpp" using namespace duckdb; using namespace std; @@ -28,7 +29,9 @@ struct DemoSecretType { } static void RegisterDemoSecret(DatabaseInstance &instance, const string &type_name) { - ExtensionLoader loader(instance, "demo_secret_type_" + type_name); + ExtensionInfo extension_info {}; + ExtensionActiveLoad load_info {instance, extension_info, "demo_secret_type_" + type_name}; + ExtensionLoader loader {load_info}; SecretType secret_type; secret_type.name = type_name; secret_type.deserializer = KeyValueSecret::Deserialize; diff --git a/test/sql/aggregate/aggregates/arg_min_max_n_tpch.test b/test/sql/aggregate/aggregates/arg_min_max_n_tpch.test index 6f1ca382f960..4e6b73cfbb65 100644 --- a/test/sql/aggregate/aggregates/arg_min_max_n_tpch.test +++ b/test/sql/aggregate/aggregates/arg_min_max_n_tpch.test @@ -43,6 +43,10 @@ WHERE rid <= k GROUP BY ALL ORDER BY ALL; +# Disable top_n_window_elimination to prevent comparing max and max_by +statement ok +SET disabled_optimizers = 'top_n_window_elimination' + query II nosort top_resultset SELECT * FROM compute_top_k(lineitem, l_returnflag, l_orderkey, 3); diff --git a/test/sql/aggregate/aggregates/arg_min_max_nulls_last.test b/test/sql/aggregate/aggregates/arg_min_max_nulls_last.test new file mode 100644 index 000000000000..264ef5a4221b --- /dev/null +++ b/test/sql/aggregate/aggregates/arg_min_max_nulls_last.test @@ -0,0 +1,64 @@ +# name: test/sql/aggregate/aggregates/arg_min_max_nulls_last.test +# description: Test arg_min_nulls_last and arg_max_nulls_last +# group: [aggregates] + +statement ok +CREATE TABLE tbl AS SELECT * FROM VALUES (1, 5, 1), (1, NULL, 2), (1, 3, NULL), (2, NULL, NULL), (3, 1, NULL) t(grp, arg, val) + +query I +SELECT arg_max_nulls_last(arg, val) FROM tbl +---- +NULL + +query I +SELECT arg_max_nulls_last(arg, val, 1) FROM tbl +---- +[NULL] + +query I +SELECT arg_max_nulls_last(val, val, 4) FROM tbl +---- +[2, 1, NULL, NULL] + +query II +SELECT grp, arg_max_nulls_last(arg, val) FROM tbl GROUP BY grp ORDER BY grp +---- +1 NULL +2 NULL +3 1 + +query II +SELECT grp, arg_max_nulls_last(arg, val, 2) FROM tbl GROUP BY grp ORDER BY grp +---- +1 [NULL, 5] +2 [NULL] +3 [1] + +query I +SELECT arg_min_nulls_last(arg, val) FROM tbl +---- +5 + +query I +SELECT arg_min_nulls_last(arg, val, 1) FROM tbl +---- +[5] + +query I +SELECT arg_min_nulls_last(val, val, 4) FROM tbl +---- +[1, 2, NULL, NULL] + +query II +SELECT grp, arg_min_nulls_last(arg, val) FROM tbl GROUP BY grp ORDER BY grp +---- +1 5 +2 NULL +3 1 + +query II +SELECT grp, arg_min_nulls_last(arg, val, 2) FROM tbl GROUP BY grp ORDER BY grp +---- +1 [5, NULL] +2 [NULL] +3 [1] diff --git a/test/sql/aggregate/aggregates/arg_min_max_nulls_last_all_types.test_slow b/test/sql/aggregate/aggregates/arg_min_max_nulls_last_all_types.test_slow new file mode 100644 index 000000000000..b98d1c21b8e9 --- /dev/null +++ b/test/sql/aggregate/aggregates/arg_min_max_nulls_last_all_types.test_slow @@ -0,0 +1,43 @@ +# name: test/sql/aggregate/aggregates/arg_min_max_nulls_last_all_types.test_slow +# description: Test the ARG_MIN_NULLS_LAST and ARG_MAX_NULLS_LAST overloads with all types +# group: [aggregates] + +statement ok +PRAGMA enable_verification + +statement ok +create table all_types as from test_all_types() + +foreach col bool tinyint smallint int bigint hugeint uhugeint utinyint usmallint uint ubigint date time timestamp timestamp_s timestamp_ms timestamp_ns time_tz timestamp_tz float double dec_4_1 dec_9_4 dec_18_6 dec38_10 uuid interval varchar blob bit small_enum medium_enum large_enum int_array double_array date_array timestamp_array timestamptz_array varchar_array nested_int_array struct struct_of_arrays array_of_structs map union fixed_int_array fixed_varchar_array fixed_nested_int_array fixed_nested_varchar_array fixed_struct_array struct_of_fixed_array fixed_array_of_int_list list_of_fixed_int_array + +statement ok +CREATE OR REPLACE TABLE asc_ordered AS SELECT "${col}" FROM all_types ORDER BY "${col}" ASC NULLS LAST + +statement ok +CREATE OR REPLACE TABLE desc_ordered AS SELECT "${col}" FROM all_types ORDER BY "${col}" DESC NULLS LAST + +statement ok +CREATE OR REPLACE TABLE arg_min_result AS SELECT unnest(arg_min_nulls_last("${col}", "${col}", 3)) FROM all_types + +statement ok +CREATE OR REPLACE TABLE arg_max_result AS SELECT unnest(arg_max_nulls_last("${col}", "${col}", 3)) FROM all_types + +query II +SELECT * FROM (SELECT rowid, * FROM asc_ordered ORDER BY rowid) EXCEPT SELECT * FROM (SELECT rowid, * FROM arg_min_result ORDER BY rowid); + +query II +SELECT * FROM (SELECT rowid, * FROM desc_ordered ORDER BY rowid) EXCEPT SELECT * FROM (SELECT rowid, * FROM arg_max_result ORDER BY rowid); + +statement ok +CREATE OR REPLACE TABLE arg_min_result AS SELECT arg_min_nulls_last("${col}", "${col}") FROM all_types + +statement ok +CREATE OR REPLACE TABLE arg_max_result AS SELECT arg_max_nulls_last("${col}", "${col}") FROM all_types + +query II +SELECT * FROM (SELECT rowid, * FROM asc_ordered ORDER BY rowid LIMIT 1) EXCEPT SELECT * FROM (SELECT rowid, * FROM arg_min_result ORDER BY rowid); + +query II +SELECT * FROM (SELECT rowid, * FROM desc_ordered ORDER BY rowid LIMIT 1) EXCEPT SELECT * FROM (SELECT rowid, * FROM arg_max_result ORDER BY rowid); + +endloop diff --git a/test/sql/aggregate/aggregates/max_n_all_types_grouped.test b/test/sql/aggregate/aggregates/max_n_all_types_grouped.test index 58457990f566..f3e35889bfa1 100644 --- a/test/sql/aggregate/aggregates/max_n_all_types_grouped.test +++ b/test/sql/aggregate/aggregates/max_n_all_types_grouped.test @@ -27,9 +27,16 @@ CREATE OR REPLACE TABLE tbl AS SELECT * FROM (SELECT ${val_col} as val_col FROM all_types) CROSS JOIN (SELECT i % 2 as grp_col FROM range(5) as r(i)); +# Disable top_n_window_elimination to prevent comparing max and max_by +statement ok +SET disabled_optimizers = 'top_n_window_elimination' + statement ok CREATE OR REPLACE TABLE window_table AS SELECT * FROM compute_top_k(tbl, grp_col, val_col, 2) as rs(grp, res); +statement ok +SET disabled_optimizers = '' + statement ok CREATE OR REPLACE TABLE agg_table AS SELECT grp_col as grp, max(val_col, 2) as res FROM tbl GROUP BY ALL ORDER BY ALL; diff --git a/test/sql/aggregate/aggregates/test_count.test b/test/sql/aggregate/aggregates/test_count.test index 609cb0a19687..c9c8f796108a 100644 --- a/test/sql/aggregate/aggregates/test_count.test +++ b/test/sql/aggregate/aggregates/test_count.test @@ -42,3 +42,4 @@ SELECT COUNT(1 ORDER BY 1) statement error SELECT COUNT(DISTINCT *) FROM integers ---- +Binder Error: STAR expression is only allowed as the root element \ No newline at end of file diff --git a/test/sql/aggregate/aggregates/test_incorrect_aggregate.test b/test/sql/aggregate/aggregates/test_incorrect_aggregate.test index c20b97c61fd6..12982d4a1c4d 100644 --- a/test/sql/aggregate/aggregates/test_incorrect_aggregate.test +++ b/test/sql/aggregate/aggregates/test_incorrect_aggregate.test @@ -5,68 +5,84 @@ statement error SELECT COUNT(1, 2, 3) ---- +Binder Error: No function matches the given name and argument types 'count statement error SELECT COUNT(COUNT(1)) ---- +Binder Error: aggregate function calls cannot be nested statement error SELECT STDDEV_SAMP() ---- +Binder Error: No function matches the given name and argument types 'stddev_samp statement error SELECT STDDEV_SAMP(1, 2, 3) ---- +Binder Error: No function matches the given name and argument types 'stddev_samp statement error SELECT STDDEV_SAMP(STDDEV_SAMP(1)) ---- +Binder Error: aggregate function calls cannot be nested statement error SELECT SUM() ---- +Binder Error: No function matches the given name and argument types 'sum statement error SELECT SUM(1, 2, 3) ---- +Binder Error: No function matches the given name and argument types 'sum statement error SELECT SUM(SUM(1)) ---- +Binder Error: aggregate function calls cannot be nested statement error SELECT FIRST() ---- +Binder Error: No function matches the given name and argument types 'first statement error SELECT FIRST(1, 2, 3) ---- +Binder Error: No function matches the given name and argument types 'first statement error SELECT FIRST(FIRST(1)) ---- +Binder Error: aggregate function calls cannot be nested statement error SELECT MAX() ---- +Binder Error: No function matches the given name and argument types 'max statement error SELECT MAX(1, 2, 3) ---- +Binder Error: No function matches the given name and argument types 'max statement error SELECT MAX(MAX(1)) ---- +Binder Error: aggregate function calls cannot be nested statement error SELECT MIN() ---- +Binder Error: No function matches the given name and argument types 'min statement error SELECT MIN(1, 2, 3) ---- +Binder Error: No function matches the given name and argument types 'min statement error SELECT MIN(MIN(1)) ---- - +Binder Error: aggregate function calls cannot be nested \ No newline at end of file diff --git a/test/sql/aggregate/aggregates/test_quantile_cont.test b/test/sql/aggregate/aggregates/test_quantile_cont.test index d05df593ca1c..ae160cd9a3d5 100644 --- a/test/sql/aggregate/aggregates/test_quantile_cont.test +++ b/test/sql/aggregate/aggregates/test_quantile_cont.test @@ -18,6 +18,7 @@ create table quantile as select range r, random() from range(0,1000000,100) unio statement error SELECT quantile_cont(r, NULL) FROM quantile ---- +Binder Error: QUANTILE argument must not be NULL query R SELECT quantile_cont(r, 0.5) FROM quantile @@ -112,6 +113,7 @@ SELECT quantile_cont('00:00:00'::TIME + interval (r/100) second, 0.5) FROM quant statement error SELECT quantile_cont(interval (r/100) second, 0.5) FROM quantile ---- +:.*Binder Error: No function matches.*'quantile_cont.* # WITH TIME ZONE query I @@ -150,31 +152,37 @@ NULL statement error SELECT quantile_cont(r, -1.1) FROM quantile ---- +Binder Error: QUANTILE can only take parameters in the range [-1, 1] statement error SELECT quantile_cont(r, 1.1) FROM quantile ---- +Binder Error: QUANTILE can only take parameters in the range [-1, 1] statement error SELECT quantile_cont(r, "string") FROM quantile ---- +Binder Error: Referenced column "string" not found in FROM clause statement error SELECT quantile_cont(r, NULL) FROM quantile ---- +Binder Error: QUANTILE argument must not be NULL statement error SELECT quantile_cont(r::string, 0.5) FROM quantile ---- +:.*Binder Error: No function matches.*'quantile_cont.* statement error SELECT quantile_cont(r) FROM quantile ---- +:.*Binder Error: No function matches.*'quantile_cont.* statement error SELECT quantile_cont(r, 0.1, 50) FROM quantile ---- - +:.*Binder Error: No function matches.*'quantile_cont.* statement ok pragma threads=4 @@ -321,3 +329,4 @@ SELECT quantile_cont(t, 0.5) FROM bigints; statement error SELECT quantile_cont(r, random()) FROM quantile ---- +Binder Error: QUANTILE can only take constant parameters \ No newline at end of file diff --git a/test/sql/aggregate/aggregates/test_quantile_disc_list.test b/test/sql/aggregate/aggregates/test_quantile_disc_list.test index 16310bd14635..9dea5da71648 100644 --- a/test/sql/aggregate/aggregates/test_quantile_disc_list.test +++ b/test/sql/aggregate/aggregates/test_quantile_disc_list.test @@ -186,19 +186,24 @@ FROM VALUES (0), (1), (2), (10) AS tab(col); statement error SELECT quantile_disc(r, [-0.1, 0.5, 0.9]) FROM quantiles ---- +Binder Error: QUANTILE parameters must have consistent signs statement error SELECT quantile_disc(r, (0.1, 0.5, 1.1)) FROM quantiles ---- +:.*Binder Error: No function matches.*'quantile_disc.* statement error SELECT quantile_disc(r, [0.1, 0.5, NULL]) FROM quantiles ---- +Binder Error: QUANTILE parameter cannot be NULL statement error SELECT quantile_disc(r, ["0.1", "0.5", "0.9"]) FROM quantiles ---- +Binder Error: Referenced column "0.1" not found in FROM clause statement error SELECT quantile_disc(r, [0.1, 0.5, 0.9], 50) FROM quantiles ---- +:.*Binder Error: No function matches.*'quantile_disc.* \ No newline at end of file diff --git a/test/sql/aggregate/aggregates/test_stddev.test b/test/sql/aggregate/aggregates/test_stddev.test index 78503bcb775c..f5ea141152a1 100644 --- a/test/sql/aggregate/aggregates/test_stddev.test +++ b/test/sql/aggregate/aggregates/test_stddev.test @@ -139,11 +139,14 @@ select stddev(0) from range(10) statement error select stddev(a) from (values (1e301), (-1e301)) tbl(a) ---- +Out of Range Error: STDDEV_SAMP is out of range statement error select var_samp(a) from (values (1e301), (-1e301)) tbl(a) ---- +Out of Range Error: VARSAMP is out of range statement error select var_pop(a) from (values (1e301), (-1e301)) tbl(a) ---- +Out of Range Error: VARPOP is out of range \ No newline at end of file diff --git a/test/sql/aggregate/aggregates/test_sum.test b/test/sql/aggregate/aggregates/test_sum.test index 425cc8112cba..5acc91438475 100644 --- a/test/sql/aggregate/aggregates/test_sum.test +++ b/test/sql/aggregate/aggregates/test_sum.test @@ -74,6 +74,7 @@ SELECT SUM(b) FROM bigints statement error SELECT SUM(b)::BIGINT FROM bigints ---- +Conversion Error: Type INT128 with value 4611686018427388403500 # # Order by @@ -94,4 +95,4 @@ SELECT sum(n ORDER BY ABS(n))::BIGINT FROM doubles; statement error SELECT (sum(n) WITHIN GROUP(ORDER BY ABS(n)))::BIGINT FROM doubles; ---- - +Parser Error: Unknown ordered aggregate "sum" \ No newline at end of file diff --git a/test/sql/aggregate/grouping_sets/grouping.test b/test/sql/aggregate/grouping_sets/grouping.test index dc59cd902b8a..782c81ac22dd 100644 --- a/test/sql/aggregate/grouping_sets/grouping.test +++ b/test/sql/aggregate/grouping_sets/grouping.test @@ -186,32 +186,40 @@ NULL NULL 7 statement error SELECT GROUPING(); ---- +Parser Error: syntax error at or near ")" statement error SELECT GROUPING() FROM students; ---- +Parser Error: syntax error at or near ")" statement error SELECT GROUPING(NULL) FROM students; ---- +:.*Binder Error.*statement cannot be used.* statement error SELECT GROUPING(course) FROM students; ---- +:.*Binder Error.*statement cannot be used.* statement error SELECT GROUPING(course) FROM students GROUP BY (); ---- +:.*Binder Error.*statement cannot be used.* statement error SELECT GROUPING(type) FROM students GROUP BY course; ---- +:.*Binder Error.*must be a grouping column.* statement error SELECT GROUPING(course) FROM students WHERE GROUPING(course)=0 GROUP BY course; ---- +:.*Binder Error.*not supported.* # we have a limit on how many children the grouping clause can contain statement error SELECT GROUPING(course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course, course), course, type, COUNT(*) FROM students GROUP BY CUBE(course, type) ORDER BY 1, 2, 3, 4; ---- +:.*Binder Error.*statement cannot have more.* \ No newline at end of file diff --git a/test/sql/aggregate/grouping_sets/grouping_sets.test b/test/sql/aggregate/grouping_sets/grouping_sets.test index ee3fab8f6ab4..4b5d6a46c9eb 100644 --- a/test/sql/aggregate/grouping_sets/grouping_sets.test +++ b/test/sql/aggregate/grouping_sets/grouping_sets.test @@ -275,3 +275,4 @@ select count(*), course, type statement error select course from students group by (); ---- +Binder Error: column "course" must appear in the GROUP BY \ No newline at end of file diff --git a/test/sql/aggregate/grouping_sets/rollup.test b/test/sql/aggregate/grouping_sets/rollup.test index 2e970e1720bc..ccdf80b68a08 100644 --- a/test/sql/aggregate/grouping_sets/rollup.test +++ b/test/sql/aggregate/grouping_sets/rollup.test @@ -131,11 +131,14 @@ Math Masters 1 statement error select course, count(*) from students group by rollup () order by 1, 2; ---- +Parser Error: syntax error at or near ")" statement error select course, count(*) from students group by rollup (rollup (course)) order by 1, 2; ---- +Catalog Error: Scalar Function with name rollup does not exist statement error select course, count(*) from students group by rollup (grouping_sets (course)) order by 1, 2; ---- +Catalog Error: Scalar Function with name grouping_sets does not exist \ No newline at end of file diff --git a/test/sql/aggregate/qualify/test_qualify.test b/test/sql/aggregate/qualify/test_qualify.test index 3edc382521ff..33825ddda248 100644 --- a/test/sql/aggregate/qualify/test_qualify.test +++ b/test/sql/aggregate/qualify/test_qualify.test @@ -198,18 +198,22 @@ SELECT * FROM exam WINDOW w AS (ORDER BY mark) QUALIFY row_number() OVER w = 1 statement error SELECT * FROM exam QUALIFY row_number() OVER w = 1 WINDOW w AS (ORDER BY mark) ---- +Parser Error: syntax error at or near "WINDOW" # QUALIFY clause requires at least one window function to be specified in at least one of the SELECT column list or the filter predicate of the QUALIFY clause statement error SELECT b, avg(a) AS avga FROM test GROUP BY b QUALIFY avga > 10 ---- +Binder Error: at least one window function must appear in the SELECT # invalid window function statement error SELECT b FROM test QUALIFY avga() > 10 ---- +Catalog Error: Scalar Function with name avga does not exist # column not found in from clause and can't find in alias map statement error SELECT b, SUM(a) FROM test GROUP BY b QUALIFY row_number() OVER (PARTITION BY b) > sum; ---- +Binder Error: Referenced column sum not found in FROM clause \ No newline at end of file diff --git a/test/sql/alter/add_col/test_add_col_incorrect.test b/test/sql/alter/add_col/test_add_col_incorrect.test index fd08685d525c..dae83d62ebad 100644 --- a/test/sql/alter/add_col/test_add_col_incorrect.test +++ b/test/sql/alter/add_col/test_add_col_incorrect.test @@ -19,8 +19,11 @@ CREATE VIEW x(x) AS (SELECT 1) statement error ALTER VIEW x ADD COLUMN i INTEGER ---- +Parser Error: Adding columns is only supported for tables # Multiple alter per statement not supported yet statement error ALTER TABLE i ADD COLUMN j INT, ADD COLUMN k INT ---- +Parser Error: Only one ALTER command per statement is supported + diff --git a/test/sql/alter/alter_type/test_alter_type_dependencies.test b/test/sql/alter/alter_type/test_alter_type_dependencies.test index 09e3f6d916ee..7ce5ff6f4f0d 100644 --- a/test/sql/alter/alter_type/test_alter_type_dependencies.test +++ b/test/sql/alter/alter_type/test_alter_type_dependencies.test @@ -50,3 +50,4 @@ ALTER TABLE test ALTER i TYPE VARCHAR USING i::VARCHAR statement error EXECUTE v2 ---- +Invalid Input Error: Values were not provided \ No newline at end of file diff --git a/test/sql/alter/alter_type/test_alter_type_incorrect.test b/test/sql/alter/alter_type/test_alter_type_incorrect.test index e0c39e02b4cf..4755f6b789b7 100644 --- a/test/sql/alter/alter_type/test_alter_type_incorrect.test +++ b/test/sql/alter/alter_type/test_alter_type_incorrect.test @@ -12,22 +12,26 @@ INSERT INTO test VALUES (1, 1), (2, 2) statement error ALTER TABLE test ALTER blabla SET TYPE VARCHAR ---- +Binder Error: Table "test" does not have a column with name "blabla" # expression error statement error ALTER TABLE test ALTER i SET TYPE VARCHAR USING blabla ---- +Binder Error: Table does not contain column blabla referenced # cannot use aggregates/window functions statement error ALTER TABLE test ALTER i SET TYPE VARCHAR USING SUM(i) ---- +Binder Error: aggregate functions are not allowed in alter statement statement error ALTER TABLE test ALTER i SET TYPE VARCHAR USING row_id() OVER () ---- +Binder Error: window functions are not allowed in alter statement statement error ALTER TABLE test ALTER i SET TYPE VARCHAR USING othertable.j ---- - +Binder Error: Cannot extract field 'othertable' from expression "#0" \ No newline at end of file diff --git a/test/sql/alter/alter_type/test_alter_type_transactions.test b/test/sql/alter/alter_type/test_alter_type_transactions.test index 7973508382e4..f5f07f29afc5 100644 --- a/test/sql/alter/alter_type/test_alter_type_transactions.test +++ b/test/sql/alter/alter_type/test_alter_type_transactions.test @@ -20,6 +20,7 @@ ALTER TABLE test ALTER j TYPE VARCHAR statement error con2 ALTER TABLE test ALTER i TYPE VARCHAR ---- +TransactionContext Error: Catalog write-write conflict on alter with "test" statement ok con1 COMMIT @@ -49,12 +50,13 @@ ALTER TABLE test ALTER i TYPE VARCHAR statement error con2 INSERT INTO test (i, j) VALUES (3, 3) ---- +:.*TransactionContext Error: Transaction conflict.*altered by a different transaction.* # nor delete statement error con2 DELETE FROM test WHERE i=1 ---- -altered +:.*TransactionContext Error: Failed to commit.*altered this table.* query TI con1 SELECT * FROM test @@ -72,10 +74,12 @@ SELECT * FROM test statement error con2 UPDATE test SET i=1000 ---- +:.*TransactionContext Error: Transaction conflict.*altered by a different transaction.* statement error con2 UPDATE test SET j=100 ---- +:.*TransactionContext Error: Transaction conflict.*altered by a different transaction.* query TI con1 SELECT * FROM test @@ -116,6 +120,7 @@ ALTER TABLE test ALTER i TYPE VARCHAR statement error con2 COMMIT ---- +:.*TransactionContext Error: Failed to commit.*altered by a different transaction.* statement ok con1 DROP TABLE test @@ -136,6 +141,6 @@ ALTER TABLE test ALTER j TYPE VARCHAR # now con tries to add an index to that column: this should fail statement error con1 -CREATE INDEX i_index ON test(j +CREATE INDEX i_index ON test(j) ---- - +:.*TransactionContext Error: Transaction conflict.*altered or dropped.* \ No newline at end of file diff --git a/test/sql/alter/default/drop_default.test b/test/sql/alter/default/drop_default.test index 8bfab11646a0..2245a6ae263a 100644 --- a/test/sql/alter/default/drop_default.test +++ b/test/sql/alter/default/drop_default.test @@ -29,3 +29,4 @@ ALTER TABLE data ALTER COLUMN x DROP DEFAULT; statement error ALTER TABLE data ALTER COLUMN j DROP DEFAULT; ---- +Binder Error: Table "data" does not have a column with name "j" \ No newline at end of file diff --git a/test/sql/alter/drop_col/test_drop_col_check.test b/test/sql/alter/drop_col/test_drop_col_check.test index a16b9b6251f3..e566240dd7c1 100644 --- a/test/sql/alter/drop_col/test_drop_col_check.test +++ b/test/sql/alter/drop_col/test_drop_col_check.test @@ -46,4 +46,4 @@ SELECT * FROM test2 statement error ALTER TABLE test2 DROP COLUMN j ---- - +Catalog Error: Cannot drop column "j" \ No newline at end of file diff --git a/test/sql/alter/drop_col/test_drop_col_failure.test b/test/sql/alter/drop_col/test_drop_col_failure.test index 7cc248b461e0..108e70d908b2 100644 --- a/test/sql/alter/drop_col/test_drop_col_failure.test +++ b/test/sql/alter/drop_col/test_drop_col_failure.test @@ -12,6 +12,7 @@ INSERT INTO test VALUES (1, 1), (2, 2) statement error ALTER TABLE test DROP COLUMN blabla ---- +Binder Error: Table "test" does not have a column with name "blabla" # unless IF EXISTS is specified statement ok @@ -24,4 +25,4 @@ ALTER TABLE test DROP COLUMN i statement error ALTER TABLE test DROP COLUMN j ---- - +Catalog Error: Cannot drop column: table only has one column remaining \ No newline at end of file diff --git a/test/sql/alter/drop_col/test_drop_col_index.test b/test/sql/alter/drop_col/test_drop_col_index.test index 39d689769b9f..39e6e487f0b9 100644 --- a/test/sql/alter/drop_col/test_drop_col_index.test +++ b/test/sql/alter/drop_col/test_drop_col_index.test @@ -15,9 +15,10 @@ CREATE INDEX i_index ON test(j) statement error ALTER TABLE test DROP COLUMN j ---- +Catalog Error: Cannot drop this column: an index depends on it # we also cannot drop the column i (for now) because an index depends on a subsequent column statement error ALTER TABLE test DROP COLUMN i ---- - +Catalog Error: Cannot drop this column: an index depends on a column \ No newline at end of file diff --git a/test/sql/alter/drop_col/test_drop_col_transactions.test b/test/sql/alter/drop_col/test_drop_col_transactions.test index 321025251910..efbacb110ad5 100644 --- a/test/sql/alter/drop_col/test_drop_col_transactions.test +++ b/test/sql/alter/drop_col/test_drop_col_transactions.test @@ -20,6 +20,7 @@ ALTER TABLE test DROP COLUMN j statement error con2 ALTER TABLE test ADD COLUMN k INTEGER ---- +TransactionContext Error: Catalog write-write conflict statement ok con1 COMMIT @@ -49,13 +50,13 @@ ALTER TABLE test DROP COLUMN i statement error con2 INSERT INTO test (i, j) VALUES (3, 3) ---- -altered +:.*TransactionContext Error: Transaction conflict.*altered by a different transaction.* # nor delete statement error con2 DELETE FROM test WHERE i=1 ---- -altered +:.*TransactionContext Error: Failed to commit.*another transaction has altered this table.* query I con1 SELECT * FROM test @@ -73,6 +74,7 @@ SELECT * FROM test statement error con2 UPDATE test SET j=100 ---- +:.*TransactionContext Error: Transaction conflict.*altered by a different transaction.* query I con1 SELECT * FROM test @@ -116,6 +118,7 @@ ALTER TABLE test DROP COLUMN i statement error con2 COMMIT ---- +:.*TransactionContext Error: Failed to commit.*altered by a different transaction.* statement ok con1 DROP TABLE test @@ -136,6 +139,6 @@ ALTER TABLE test DROP COLUMN j # now con tries to add an index to that column: this should fail statement error con1 -CREATE INDEX i_index ON test(j +CREATE INDEX i_index ON test(j) ---- - +:.*TransactionContext Error: Transaction conflict: cannot add an index.* \ No newline at end of file diff --git a/test/sql/alter/rename_col/test_rename_col_check.test b/test/sql/alter/rename_col/test_rename_col_check.test index e4bfea04ae6a..990aa47fa7f4 100644 --- a/test/sql/alter/rename_col/test_rename_col_check.test +++ b/test/sql/alter/rename_col/test_rename_col_check.test @@ -12,6 +12,7 @@ INSERT INTO test (i, j) VALUES (1, 2), (2, 3) statement error INSERT INTO test (i, j) VALUES (100, 2) ---- +Constraint Error: CHECK constraint failed statement ok ALTER TABLE test RENAME COLUMN i TO k @@ -23,4 +24,4 @@ INSERT INTO test (k, j) VALUES (1, 2), (2, 3) statement error INSERT INTO test (k, j) VALUES (100, 2) ---- - +Constraint Error: CHECK constraint failed \ No newline at end of file diff --git a/test/sql/alter/rename_col/test_rename_col_not_null.test b/test/sql/alter/rename_col/test_rename_col_not_null.test index 3a27433e39e4..ba846b45c9e8 100644 --- a/test/sql/alter/rename_col/test_rename_col_not_null.test +++ b/test/sql/alter/rename_col/test_rename_col_not_null.test @@ -11,6 +11,7 @@ INSERT INTO test (i, j) VALUES (1, 2), (2, 3) statement error INSERT INTO test (i, j) VALUES (NULL, 2) ---- +Constraint Error: NOT NULL constraint failed: test.i statement ok ALTER TABLE test RENAME COLUMN i TO k @@ -22,4 +23,4 @@ INSERT INTO test (k, j) VALUES (1, 2), (2, 3) statement error INSERT INTO test (k, j) VALUES (NULL, 2) ---- - +Constraint Error: NOT NULL constraint failed: test.k \ No newline at end of file diff --git a/test/sql/alter/rename_col/test_rename_col_rollback.test b/test/sql/alter/rename_col/test_rename_col_rollback.test index 4a29bbfe741e..1450d4822c88 100644 --- a/test/sql/alter/rename_col/test_rename_col_rollback.test +++ b/test/sql/alter/rename_col/test_rename_col_rollback.test @@ -17,6 +17,7 @@ ALTER TABLE test RENAME COLUMN i TO k statement error SELECT i FROM test ---- +Binder Error: Referenced column "i" not found in FROM clause statement ok SELECT k FROM test @@ -32,4 +33,4 @@ SELECT i FROM test statement error SELECT k FROM test ---- - +Binder Error: Referenced column "k" not found in FROM clause \ No newline at end of file diff --git a/test/sql/alter/rename_col/test_rename_col_unique.test b/test/sql/alter/rename_col/test_rename_col_unique.test index e499ef34a2c1..9bfb37ff113e 100644 --- a/test/sql/alter/rename_col/test_rename_col_unique.test +++ b/test/sql/alter/rename_col/test_rename_col_unique.test @@ -11,15 +11,17 @@ INSERT INTO test (i, j) VALUES (1, 1), (2, 2) statement error INSERT INTO test (i, j) VALUES (1, 1) ---- +Constraint Error: Duplicate key "i: 1, j: 1" statement ok ALTER TABLE test RENAME COLUMN i TO k # the unique constraint should still work after altering the table +# error message refers to the old name of the column (was i, renamed to k) statement ok INSERT INTO test (k, j) VALUES (3, 3), (4, 4) statement error INSERT INTO test (k, j) VALUES (1, 1) ---- - +Constraint Error: Duplicate key \ No newline at end of file diff --git a/test/sql/alter/rename_schema/rename_schema.test b/test/sql/alter/rename_schema/rename_schema.test index 08261aae49f3..128a4d39032a 100644 --- a/test/sql/alter/rename_schema/rename_schema.test +++ b/test/sql/alter/rename_schema/rename_schema.test @@ -5,4 +5,4 @@ statement error ALTER SCHEMA a RENAME TO b; ---- -not yet supported +Not implemented Error: Altering schemas is not yet supported diff --git a/test/sql/alter/rename_table/test_rename_table_chain_rollback.test b/test/sql/alter/rename_table/test_rename_table_chain_rollback.test index a135a35933d4..01c1af5a7a3d 100644 --- a/test/sql/alter/rename_table/test_rename_table_chain_rollback.test +++ b/test/sql/alter/rename_table/test_rename_table_chain_rollback.test @@ -48,14 +48,17 @@ SELECT * FROM entry statement error con2 SELECT * FROM entry2 ---- +Catalog Error: Table with name entry2 does not exist statement error con2 SELECT * FROM entry3 ---- +Catalog Error: Table with name entry3 does not exist statement error con2 SELECT * FROM entry4 ---- +Catalog Error: Table with name entry4 does not exist statement ok con1 ROLLBACK @@ -68,5 +71,4 @@ SELECT * FROM entry statement error con2 SELECT * FROM entry4 ---- - - +Catalog Error: Table with name entry4 does not exist \ No newline at end of file diff --git a/test/sql/alter/rename_table/test_rename_table_incorrect.test b/test/sql/alter/rename_table/test_rename_table_incorrect.test index 81da8b9d67a5..df685bae8180 100644 --- a/test/sql/alter/rename_table/test_rename_table_incorrect.test +++ b/test/sql/alter/rename_table/test_rename_table_incorrect.test @@ -12,9 +12,10 @@ CREATE TABLE tbl2(i INTEGER) statement error ALTER TABLE non_table RENAME TO tbl ---- +Catalog Error: Table with name non_table does not exist # rename to an already existing table statement error ALTER TABLE tbl2 RENAME TO tbl ---- - +Catalog Error: Could not rename "tbl2" to "tbl" \ No newline at end of file diff --git a/test/sql/alter/rename_table/test_rename_table_transactions.test b/test/sql/alter/rename_table/test_rename_table_transactions.test index 85ac928a57e6..f2c56a181cd1 100644 --- a/test/sql/alter/rename_table/test_rename_table_transactions.test +++ b/test/sql/alter/rename_table/test_rename_table_transactions.test @@ -26,6 +26,7 @@ SELECT * FROM tbl2 statement error con1 SELECT * FROM tbl ---- +Catalog Error: Table with name tbl does not exist query I con2 SELECT * FROM tbl @@ -36,6 +37,7 @@ SELECT * FROM tbl statement error con2 SELECT * FROM tbl2 ---- +Catalog Error: Table with name tbl2 does not exist statement ok con1 COMMIT @@ -46,10 +48,12 @@ COMMIT statement error con1 SELECT * FROM tbl ---- +Catalog Error: Table with name tbl does not exist statement error con2 SELECT * FROM tbl ---- +Catalog Error: Table with name tbl does not exist query I con1 SELECT * FROM tbl2 @@ -71,6 +75,7 @@ CREATE TABLE tbl(i INTEGER); statement error con1 CREATE TABLE tbl2(i INTEGER); ---- +Catalog Error: Table with name "tbl2" already exists # we can drop and re-create the table statement ok con1 @@ -83,3 +88,4 @@ CREATE TABLE tbl2(i INTEGER); statement error con1 ALTER TABLE tbl RENAME TO tbl2 ---- +Catalog Error: Could not rename "tbl" to "tbl2" \ No newline at end of file diff --git a/test/sql/alter/rename_view/test_rename_view.test b/test/sql/alter/rename_view/test_rename_view.test index 53c155ec6540..07c7c33ebe4f 100644 --- a/test/sql/alter/rename_view/test_rename_view.test +++ b/test/sql/alter/rename_view/test_rename_view.test @@ -22,6 +22,7 @@ SELECT * FROM vw2 statement error SELECT * FROM vw ---- +Catalog Error: Table with name vw does not exist statement ok ROLLBACK @@ -35,6 +36,7 @@ SELECT * FROM vw; statement error SELECT * FROM vw2 ---- +Catalog Error: Table with name vw2 does not exist statement ok BEGIN TRANSACTION; @@ -54,6 +56,7 @@ SELECT * FROM vw2 statement error SELECT * FROM vw ---- +Catalog Error: Table with name vw does not exist statement ok CREATE VIEW vw AS SELECT i+1 AS i FROM tbl @@ -68,8 +71,10 @@ SELECT * FROM vw statement error ALTER VIEW sqlite_master RENAME TO my_sqlite_master ---- +Binder Error: Can not comment on System Catalog entries # cannot rename a view that does not exist statement error ALTER VIEW nonexistingview RENAME TO my_new_view ---- +Catalog Error: View with name nonexistingview does not exist \ No newline at end of file diff --git a/test/sql/alter/rename_view/test_rename_view_incorrect.test b/test/sql/alter/rename_view/test_rename_view_incorrect.test index 89e4c4a558ef..ae740012fe00 100644 --- a/test/sql/alter/rename_view/test_rename_view_incorrect.test +++ b/test/sql/alter/rename_view/test_rename_view_incorrect.test @@ -15,9 +15,10 @@ CREATE VIEW vw2 AS SELECT 1729 AS i statement error ALTER VIEW non_view RENAME TO vw ---- +:.*Catalog Error: View.*does not exist.* # rename to an already existing view statement error ALTER VIEW vw2 RENAME TO vw ---- - +Catalog Error: Could not rename \ No newline at end of file diff --git a/test/sql/alter/test_alter_database_rename.test b/test/sql/alter/test_alter_database_rename.test new file mode 100644 index 000000000000..c1a80484f21c --- /dev/null +++ b/test/sql/alter/test_alter_database_rename.test @@ -0,0 +1,46 @@ +# name: test/sql/alter/test_alter_database_rename.test +# description: Test ALTER DATABASE ... RENAME TO +# group: [alter] + +statement ok +ATTACH ':memory:' AS test_db; + +statement ok +CREATE TABLE test_db.sample AS SELECT i FROM range(100) t(i); + +query I +SELECT COUNT(*) FROM test_db.sample +---- +100 + +statement ok +ALTER DATABASE test_db RENAME TO renamed_db; + +query I +SELECT COUNT(*) FROM renamed_db.sample +---- +100 + +statement error +ALTER DATABASE non_existent RENAME TO something_else; +---- + +statement ok +ALTER DATABASE IF EXISTS non_existent RENAME TO something_else; + +statement ok +ATTACH ':memory:' AS another_db; + +statement error +ALTER DATABASE another_db RENAME TO renamed_db; +---- + +statement error +ALTER DATABSE renamed_db RENAME TO system; +---- +Parser Error: syntax error at or near "DATABSE" + +statement error +ALTER DATABSE renamed_db RENAME TO temp; +---- +Parser Error: syntax error at or near "DATABSE" \ No newline at end of file diff --git a/test/sql/alter/test_alter_if_exists.test b/test/sql/alter/test_alter_if_exists.test index 001909ab1d3f..6640391c03e1 100644 --- a/test/sql/alter/test_alter_if_exists.test +++ b/test/sql/alter/test_alter_if_exists.test @@ -89,6 +89,7 @@ ALTER TABLE IF EXISTS t1 ALTER COLUMN c0 TYPE varchar; statement error ALTER TABLE IF EXISTS t1 ALTER COLUMN IF EXISTS c0 TYPE varchar; ---- +Parser Error: syntax error at or near "EXISTS" # 'rowid' pseudo-column statement ok diff --git a/test/sql/attach/attach_dbname_quotes.test b/test/sql/attach/attach_dbname_quotes.test index cde53e1f8eb2..45d985d56f55 100644 --- a/test/sql/attach/attach_dbname_quotes.test +++ b/test/sql/attach/attach_dbname_quotes.test @@ -15,6 +15,9 @@ INSERT INTO "my""db".tbl VALUES (42) statement ok USE "my""db"; +statement ok +SET search_path=current_setting('search_path') + query I SELECT * FROM tbl ---- diff --git a/test/sql/attach/attach_fsspec.test b/test/sql/attach/attach_fsspec.test index f9d6ead27b92..e1beadec601b 100644 --- a/test/sql/attach/attach_fsspec.test +++ b/test/sql/attach/attach_fsspec.test @@ -10,3 +10,4 @@ not found statement error ATTACH 'file://dummy.csv' ---- +IO Error: Cannot open file "file://dummy.csv" \ No newline at end of file diff --git a/test/sql/attach/attach_httpfs.test b/test/sql/attach/attach_httpfs.test deleted file mode 100644 index 8742ecbb531e..000000000000 --- a/test/sql/attach/attach_httpfs.test +++ /dev/null @@ -1,45 +0,0 @@ -# name: test/sql/attach/attach_httpfs.test -# description: Test attach using httpfs -# group: [attach] - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -require-env S3_ATTACH_DB_PRESIGNED_URL - -# ATTACH a DuckDB database over HTTPFS -statement ok -ATTACH '${S3_ATTACH_DB_PRESIGNED_URL}' AS db (READONLY 1); - -query IIIII -SELECT * FROM db.integral_values ----- -1 2 3 4 5 -NULL NULL NULL NULL NULL - -statement error -CREATE TABLE db.integers(i INTEGER); ----- -read-only - -statement ok -SELECT * FROM db.all_types - -statement error -SELECT * FROM db.all_typez ----- -all_types - -statement ok -DETACH db diff --git a/test/sql/attach/attach_if_not_exists_detach.test b/test/sql/attach/attach_if_not_exists_detach.test new file mode 100644 index 000000000000..a6505cea234a --- /dev/null +++ b/test/sql/attach/attach_if_not_exists_detach.test @@ -0,0 +1,23 @@ +# name: test/sql/attach/attach_if_not_exists_detach.test +# description: Test ATTACH IF NOT EXISTS +# group: [attach] + +statement ok +ATTACH '__TEST_DIR__/attach_if_not_exists_detach.db' AS db1 + +# con1 opens a transaction for db1 that prevents it from being fully detached +statement ok con1 +BEGIN + +statement ok con1 +CREATE TABLE db1.tbl(i INTEGER); + +# con2 can detach db1, but it will not actually be detached +statement ok con2 +DETACH db1 + +# upon attaching we should fail +statement error con2 +ATTACH IF NOT EXISTS '__TEST_DIR__/attach_if_not_exists_detach.db' AS db1 +---- +the process of being detached diff --git a/test/sql/attach/attach_remote.test b/test/sql/attach/attach_remote.test deleted file mode 100644 index 285a941d48cd..000000000000 --- a/test/sql/attach/attach_remote.test +++ /dev/null @@ -1,17 +0,0 @@ -# name: test/sql/attach/attach_remote.test -# description: Test attaching of remote database -# group: [attach] - -require httpfs - -statement error -ATTACH 'https://duckdb.org/non_existing.db' AS db2 (READ_ONLY) ----- - -statement error -ATTACH 'https://duckdb.org/non_existing.db' AS db2 ----- - -statement error -ATTACH 'https://duckdb.org/non_existing.db' AS db2 (READ_WRITE) ----- diff --git a/test/sql/attach/attach_s3.test b/test/sql/attach/attach_s3.test deleted file mode 100644 index 555d074ccd5a..000000000000 --- a/test/sql/attach/attach_s3.test +++ /dev/null @@ -1,56 +0,0 @@ -# name: test/sql/attach/attach_s3.test -# description: Test attach using httpfs -# group: [attach] - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -require-env S3_ATTACH_DB - -statement ok -CREATE SECRET ( - TYPE S3, - PROVIDER config, - KEY_ID '${AWS_ACCESS_KEY_ID}', - SECRET '${AWS_SECRET_ACCESS_KEY}', - REGION '${AWS_DEFAULT_REGION}', - ENDPOINT '${DUCKDB_S3_ENDPOINT}', - USE_SSL '${DUCKDB_S3_USE_SSL}' -) - -# ATTACH a DuckDB database over HTTPFS -statement ok -ATTACH '${S3_ATTACH_DB}' AS db (READONLY 1); - -query IIIII -SELECT * FROM db.integral_values ----- -1 2 3 4 5 -NULL NULL NULL NULL NULL - -statement error -CREATE TABLE db.integers(i INTEGER); ----- -read-only - -statement ok -SELECT * FROM db.all_types - -statement error -SELECT * FROM db.all_typez ----- -all_types - -statement ok -DETACH db diff --git a/test/sql/attach/attach_s3_tpch.test_slow b/test/sql/attach/attach_s3_tpch.test_slow deleted file mode 100644 index 721be60ee238..000000000000 --- a/test/sql/attach/attach_s3_tpch.test_slow +++ /dev/null @@ -1,72 +0,0 @@ -# name: test/sql/attach/attach_s3_tpch.test_slow -# description: Test running TPC-H over a database attached over S3 -# group: [attach] - -require httpfs - -require tpch - -require-env S3_TEST_SERVER_AVAILABLE 1 - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -statement ok -CREATE SECRET ( - TYPE S3, - PROVIDER config, - KEY_ID '${AWS_ACCESS_KEY_ID}', - SECRET '${AWS_SECRET_ACCESS_KEY}', - REGION '${AWS_DEFAULT_REGION}', - ENDPOINT '${DUCKDB_S3_ENDPOINT}', - USE_SSL '${DUCKDB_S3_USE_SSL}' -) - -# ATTACH a DuckDB database over HTTPFS -statement ok -ATTACH 's3://test-bucket/presigned/lineitem_sf1.db' AS db (READONLY 1); - -statement ok -USE db - -loop i 1 9 - -query I -PRAGMA tpch(${i}) ----- -:extension/tpch/dbgen/answers/sf1/q0${i}.csv - -endloop - -loop i 10 23 - -query I -PRAGMA tpch(${i}) ----- -:extension/tpch/dbgen/answers/sf1/q${i}.csv - -endloop - -statement ok -USE memory - -statement ok -DETACH db - -statement ok -ATTACH 's3://test-bucket/presigned/lineitem_sf1.db' AS db (READONLY 1); - -statement ok -USE db - -query IIIIIIIIIIIIIIII -select count(distinct columns(*)) from lineitem; ----- -1500000 200000 10000 7 50 933900 11 9 3 2 2526 2466 2554 4 7 3610733 diff --git a/test/sql/attach/reattach_schema.test b/test/sql/attach/reattach_schema.test index dd346d73616c..b4e83da1d50d 100644 --- a/test/sql/attach/reattach_schema.test +++ b/test/sql/attach/reattach_schema.test @@ -77,7 +77,7 @@ USE new_name.my_schema statement error USE new_name.my_schema.my_table ---- -USE database +Parser Error: Expected "USE database" or "USE database.schema" query I SELECT * FROM my_table diff --git a/test/sql/binder/function_chaining_19035.test b/test/sql/binder/function_chaining_19035.test new file mode 100644 index 000000000000..e0d2242b4299 --- /dev/null +++ b/test/sql/binder/function_chaining_19035.test @@ -0,0 +1,43 @@ +# name: test/sql/binder/function_chaining_19035.test +# description: Lambda expression with macro function chaining +# group: [binder] + +statement ok +PRAGMA enable_verification + +statement ok +CREATE MACRO list_contains_macro(x, y) AS (SELECT list_contains(x, y)) + +statement error +SELECT list_filter([[1, 2, 1], [1, 2, 3], [1, 1, 1]], lambda x: list_contains_macro(x, 3)) +---- +:Binder Error.*subqueries in lambda expressions are not supported.* + +statement ok +CREATE TABLE tbl(a int[]); + +statement ok +INSERT INTO tbl VALUES ([5, 4, 3]), ([1, 2, 3]), (NULL), ([NULL, 101, 12]); + +query I +SELECT list_transform(a, lambda x, i: x + i + list_any_value(a)) FROM tbl; +---- +[11, 11, 11] +[3, 5, 7] +NULL +[NULL, 204, 116] + +query I +select ['a a ', ' b ', ' cc'].list_transform(lambda x: nullif(trim(x), '')) as trimmed_and_nulled +---- +[a a, b, cc] + +query I +select ['a a ', ' b ', ' cc'].list_transform(lambda x: x.trim().nullif('')) as trimmed_and_nulled; +---- +[a a, b, cc] + +query I +select ['a a ', ' b ', ' cd'].list_transform(lambda x: x.trim().nullif('').reverse()) as trimmed_and_nulled; +---- +[a a, b, dc] diff --git a/test/sql/binder/not_similar_to.test b/test/sql/binder/not_similar_to.test new file mode 100644 index 000000000000..0ddc9a417b31 --- /dev/null +++ b/test/sql/binder/not_similar_to.test @@ -0,0 +1,65 @@ +# name: test/sql/binder/not_similar_to.test +# description: Correctly return all columns NOT similar to pattern +# group: [binder] + +statement ok +PRAGMA enable_verification + +statement ok +create or replace table foo as select 'd' as a, 'e' as b, 'f' as c; + +# should select the column a and c +query II +select * similar to '(a|c)' from foo; +---- +d f + +# should select the column b +query I +select * not similar to '(a|c)' from foo; +---- +e + +# should select the column b +query I +SELECT * similar to 'b' FROM (select * not similar to '(a|c)' from foo); +---- +e + +statement error +SELECT * similar to 'a' FROM (select * not similar to '(a|c)' from foo); +---- +Binder Error: No matching columns found that match regex "a" + + +statement ok +CREATE TABLE t0(c0 VARCHAR); + +statement ok +INSERT INTO t0(c0) VALUES (0.1); + +query T +SELECT * FROM t0 WHERE REGEXP_MATCHES(t0.c0, '1'); +---- +0.1 + +query T +SELECT * FROM t0 WHERE NOT REGEXP_MATCHES(t0.c0, '1'); +---- + + +query I +SELECT 'aaa' NOT SIMILAR TO '[b-z]{3}'; +---- +1 + +statement ok +CREATE TABLE integers(col1 INTEGER, col2 INTEGER, k INTEGER) + +statement ok +INSERT INTO integers VALUES (1, 2, 3) + +query II +SELECT * LIKE 'col%' FROM integers +---- +1 2 \ No newline at end of file diff --git a/test/sql/binder/table_view_alias.test b/test/sql/binder/table_view_alias.test index 33bc29fcbfb2..051eedb2eec0 100644 --- a/test/sql/binder/table_view_alias.test +++ b/test/sql/binder/table_view_alias.test @@ -33,11 +33,14 @@ Referenced table "s1.t" not found statement error SELECT s1.x.c FROM s1.v AS x ---- +Referenced table "s1.x" not found statement error SELECT s1.v.c FROM s1.v AS x ---- +Referenced table "s1.v" not found statement error SELECT s1.v.c FROM s1.v AS v ---- +Referenced table "s1.v" not found \ No newline at end of file diff --git a/test/sql/binder/test_having_alias.test b/test/sql/binder/test_having_alias.test index 573ae895c1b8..82bd69cea256 100644 --- a/test/sql/binder/test_having_alias.test +++ b/test/sql/binder/test_having_alias.test @@ -22,6 +22,7 @@ SELECT i, COUNT(*) AS k FROM integers GROUP BY i HAVING k=1 ORDER BY i; statement error SELECT i, COUNT(*) AS k FROM integers GROUP BY i HAVING integers.k=1 ORDER BY i; ---- +Binder Error: column k must appear in the GROUP BY clause # column name wins over the alias query II @@ -77,8 +78,10 @@ SELECT COUNT(*) FROM (SELECT i, SUM(RANDOM()) AS k FROM integers GROUP BY i HAVI statement error SELECT COUNT(i) AS i FROM integers HAVING integers.i=5 ORDER BY i; ---- +Binder Error: column i must appear in the GROUP BY clause # recursive alias without aggregate statement error SELECT i + i AS i FROM integers HAVING i=5 ORDER BY i; ---- +Binder Error: column i must appear in the GROUP BY clause \ No newline at end of file diff --git a/test/sql/binder/test_string_alias.test b/test/sql/binder/test_string_alias.test index 89efacaa06ae..dee380ebe0b3 100644 --- a/test/sql/binder/test_string_alias.test +++ b/test/sql/binder/test_string_alias.test @@ -16,6 +16,7 @@ SELECT i AS 'hello world' FROM integers statement error SELECT i 'hello world' FROM integers ---- +:.*Catalog Error.*does not exist.* # double quotes work everywhere statement ok @@ -32,3 +33,4 @@ SELECT "hello world".i FROM integers AS 'hello world' statement error SELECT "hello world".i FROM integers 'hello world' ---- +Parser Error: syntax error diff --git a/test/sql/catalog/case_insensitive_using.test b/test/sql/catalog/case_insensitive_using.test index f0b70861f775..c1fe6e680daf 100644 --- a/test/sql/catalog/case_insensitive_using.test +++ b/test/sql/catalog/case_insensitive_using.test @@ -202,3 +202,4 @@ SELECT * FROM (SELECT 1 "hello", 2 "HeLlO") t1 JOIN (SELECT 1 "hello", 2 "HeLlO" statement error SELECT hello FROM (a JOIN b USING (hello)), (d JOIN e USING (hello)) ---- +:.*Binder Error: Ambiguous column reference.* \ No newline at end of file diff --git a/test/sql/catalog/dependencies/test_alter_owned_by.test b/test/sql/catalog/dependencies/test_alter_owned_by.test index 5c81c25315c5..936b9ce4ecda 100644 --- a/test/sql/catalog/dependencies/test_alter_owned_by.test +++ b/test/sql/catalog/dependencies/test_alter_owned_by.test @@ -30,13 +30,13 @@ ALTER SEQUENCE sequence1 OWNED BY tbl1; statement error ALTER SEQUENCE sequence2 OWNED BY sequence1; ---- -sequence1 can not become the owner, it is already owned by tbl1 +:.*Dependency Error.*is already owned.* # sequence1 is already owned by another entry statement error ALTER SEQUENCE sequence1 OWNED BY tbl2; ---- -Dependency Error: sequence1 is already owned by tbl1 +:.*Dependency Error.*is already owned.* statement ok create sequence sequence3; @@ -44,7 +44,7 @@ create sequence sequence3; statement error alter sequence sequence3 owned by sequence1 ---- -sequence1 can not become the owner, it is already owned by tbl1 +:.*Dependency Error.*is already owned.* statement ok DROP TABLE tbl1; @@ -52,3 +52,4 @@ DROP TABLE tbl1; statement error SELECT nextval('sequence1'); ---- +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/catalog/dependencies/test_prepare_dependencies_transactions.test b/test/sql/catalog/dependencies/test_prepare_dependencies_transactions.test index 7c866d21d681..6892d832c347 100644 --- a/test/sql/catalog/dependencies/test_prepare_dependencies_transactions.test +++ b/test/sql/catalog/dependencies/test_prepare_dependencies_transactions.test @@ -33,7 +33,7 @@ EXECUTE v statement error con2 DROP TABLE integers CASCADE ---- -Catalog write-write conflict on alter +:.*TransactionContext Error: Catalog write-write conflict on alter.* # now we rollback statement ok con2 @@ -43,6 +43,7 @@ ROLLBACK statement error con2 EXECUTE v ---- +:.*Catalog Error.*does not exist.* # case two: prepared statement is created inside transaction statement ok con1 @@ -77,3 +78,4 @@ COMMIT statement error con2 EXECUTE v ---- +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/catalog/dependencies/test_prepared_dependency.test b/test/sql/catalog/dependencies/test_prepared_dependency.test index f08170717a9e..6c6eee6279d2 100644 --- a/test/sql/catalog/dependencies/test_prepared_dependency.test +++ b/test/sql/catalog/dependencies/test_prepared_dependency.test @@ -23,6 +23,7 @@ DROP TABLE integers CASCADE statement error con2 EXECUTE v ---- +:.*Catalog Error.*does not exist.* # dependency on a sequence for prepare statement ok con1 @@ -42,4 +43,4 @@ DROP SEQUENCE seq CASCADE statement error con2 EXECUTE v ---- - +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/catalog/function/information_schema_macro.test b/test/sql/catalog/function/information_schema_macro.test index a10987b36087..7e379eb3a3d3 100644 --- a/test/sql/catalog/function/information_schema_macro.test +++ b/test/sql/catalog/function/information_schema_macro.test @@ -9,3 +9,4 @@ PRAGMA enable_verification statement error create macro information_schema.foo(a) as a; ---- +:.*Binder Error: Cannot create entry.* \ No newline at end of file diff --git a/test/sql/catalog/function/query_function.test b/test/sql/catalog/function/query_function.test index 1ef00218e192..ff75658b0455 100644 --- a/test/sql/catalog/function/query_function.test +++ b/test/sql/catalog/function/query_function.test @@ -114,6 +114,17 @@ SELECT * FROM query('CREATE TABLE tbl (a INT)'); ---- Parser Error: Expected a single SELECT statement +# test PIVOT statements in query() function +query I +SELECT * FROM query('SELECT * FROM (PIVOT (SELECT 1 AS col) ON col IN (1) using first(col))'); +---- +1 + +# PIVOT without explicit IN clause should give helpful error message +statement error +SELECT * FROM query('SELECT * FROM (PIVOT (SELECT 1 AS col) ON col using first(col))'); +---- +Parser Error: PIVOT statements without explicit IN clauses are not supported in query() function. Please specify the pivot values explicitly, e.g.: PIVOT ... ON col IN (val1, val2, ...) # test query_table() statement ok diff --git a/test/sql/catalog/function/test_drop_macro.test b/test/sql/catalog/function/test_drop_macro.test new file mode 100644 index 000000000000..451cda293ec1 --- /dev/null +++ b/test/sql/catalog/function/test_drop_macro.test @@ -0,0 +1,36 @@ +# name: test/sql/catalog/function/test_drop_macro.test +# description: Test dropping scalar/table macros +# group: [function] + +# create scalar and table macro with same name +statement ok +create macro m() as 42; + +statement ok +create macro m() as table (select 42 i); + +# table macros are explicitly dropped with "drop macro table" +# however, we also want to support dropping them with drop macro +# we disambiguate by first trying to drop the scalar macro, then the table macro +statement ok +drop macro m; + +# scalar macros are dropped first so this fails +statement error +select m(); +---- +is a table function but it was used as a scalar function + +# table macro is still available +query I +from m(); +---- +42 + +statement ok +drop macro m; + +statement error +from m(); +---- +with name m does not exist diff --git a/test/sql/catalog/function/test_macro_default_arg_with_dependencies.test b/test/sql/catalog/function/test_macro_default_arg_with_dependencies.test index 869e7edf2c02..b5d208b2b55d 100644 --- a/test/sql/catalog/function/test_macro_default_arg_with_dependencies.test +++ b/test/sql/catalog/function/test_macro_default_arg_with_dependencies.test @@ -68,10 +68,9 @@ Invalid default value # to test the dependencies we create this macro instead statement ok create macro my_macro2(i := 42) as ( - select min(a) + i from integers + select min(a) + i from integers ); - statement ok insert into integers values (5), (10), (13) diff --git a/test/sql/catalog/function/test_macro_issue_18927.test b/test/sql/catalog/function/test_macro_issue_18927.test new file mode 100644 index 000000000000..0a73871285c3 --- /dev/null +++ b/test/sql/catalog/function/test_macro_issue_18927.test @@ -0,0 +1,11 @@ +# name: test/sql/catalog/function/test_macro_issue_18927.test +# description: Test Issue 18927 - Dot operator (function chaining) fails with macros +# group: [function] + +statement ok +CREATE OR REPLACE MACRO my(s) AS s.lower().split('').aggregate('count'); + +query I +SELECT my('AA'); +---- +2 diff --git a/test/sql/catalog/function/test_macro_issue_19119.test b/test/sql/catalog/function/test_macro_issue_19119.test new file mode 100644 index 000000000000..3b793b2398c2 --- /dev/null +++ b/test/sql/catalog/function/test_macro_issue_19119.test @@ -0,0 +1,12 @@ +# name: test/sql/catalog/function/test_macro_issue_19119.test +# description: Test Issue 19119 - Regression: combination of window functions and RAISE_EVEN raises "Not implemented Error: Unimplemented expression class" +# group: [function] + +statement ok +CREATE TABLE t (x INT); + +statement ok +INSERT INTO t (x) VALUES (1), (2); + +statement ok +SELECT ROUND_EVEN(SUM(x) OVER (), 0) AS y FROM t; diff --git a/test/sql/catalog/function/test_macro_relpersistence_conflict.test b/test/sql/catalog/function/test_macro_relpersistence_conflict.test index 6fcffa3d3726..08029c315d53 100644 --- a/test/sql/catalog/function/test_macro_relpersistence_conflict.test +++ b/test/sql/catalog/function/test_macro_relpersistence_conflict.test @@ -70,3 +70,4 @@ restart statement error select test(4, 2) ---- +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/catalog/function/test_simple_macro.test b/test/sql/catalog/function/test_simple_macro.test index 4bd7d1894300..8f02feeb43b5 100644 --- a/test/sql/catalog/function/test_simple_macro.test +++ b/test/sql/catalog/function/test_simple_macro.test @@ -22,10 +22,12 @@ SELECT one() statement error SELECT one(1) ---- +does not support the supplied arguments statement error SELECT one(NULL) ---- +does not support the supplied arguments statement ok DROP MACRO one; @@ -98,6 +100,7 @@ DROP FUNCTION two; statement error CREATE MACRO add_macro(a) AS a + b ---- +column "b" not found statement ok CREATE MACRO add_macro(a, b) AS a + b @@ -159,27 +162,33 @@ false statement error SELECT IFELSE(); ---- +does not support the supplied arguments statement error SELECT IFELSE(1); ---- +does not support the supplied arguments statement error SELECT IFELSE(1, 2); ---- +does not support the supplied arguments statement error SELECT IFELSE(1, 2, 3, 4); ---- +does not support the supplied arguments # no duplicate macro function names statement error CREATE MACRO IFELSE(a,b) AS a+b ---- +already exists statement error CREATE MACRO ifelse(a,b) AS a+b ---- +already exists query T SELECT IFELSE('1', 'random', RANDOM()::VARCHAR) @@ -253,28 +262,44 @@ SELECT add_default5(3, b := 6) statement error SELECT add_default5(b := 6, 3) ---- +positional argument following named argument statement error -CREATE MACRO wrong_order(a, b := 3, c) +CREATE MACRO wrong_order(a, b := 3, c) AS a + b + c ---- +Parameter without a default follows parameter with a default statement error -CREATE MACRO wrong_order(a := 3, b) +CREATE MACRO wrong_order(a := 3, b) AS a + b ---- +Parameter without a default follows parameter with a default # only constant default values are allowed statement error CREATE MACRO select_plus_floats(a, f := b) AS (SELECT a + f FROM floats) ---- +Invalid default value -# +(FLOAT, VARCHAR) does not work - constant types are checked at create time +# +(FLOAT, VARCHAR) does not work - but types are only checked once the macro is called +statement ok +CREATE MACRO wrong_type(s := 'not a float') AS (SELECT b + s FROM floats) + +# this should fail statement error -CREATE MACRO wrong_type(s='not a float') AS (SELECT b + s FROM floats) +select wrong_type() +---- +Could not convert + +# but succeed if we pass the correct type +query I +select wrong_type(42) ---- +42.5 statement error CREATE MACRO two_default_params(a := 4, a := 2) AS a + a ---- +Duplicate parameter statement ok CREATE MACRO two_default_params(a := 4, b := 2) AS a + b @@ -297,22 +322,27 @@ SELECT two_default_params(b := 3) statement error SELECT two_default_params(a := 5, a := 3) ---- +has named argument repeated statement error SELECT two_default_params(b := 5, b := 3) ---- +has named argument repeated statement error CREATE MACRO macros.add_macro(a, b) AS a + b ---- +already exists statement error CREATE MACRO my_macro(a.b) AS 42; ---- +syntax error statement error CREATE MACRO my_macro(a.b.c) AS 42; ---- +syntax error statement ok CREATE MACRO my_macro(a) AS 42; @@ -320,10 +350,12 @@ CREATE MACRO my_macro(a) AS 42; statement error SELECT my_macro(x := 42); ---- +does not support the supplied arguments statement error SELECT my_macro(a := 42, a := 42); ---- +has named argument repeated # internal issue 1044 statement ok diff --git a/test/sql/catalog/function/test_table_macro_args.test b/test/sql/catalog/function/test_table_macro_args.test index 212856d8fa36..0c5beff02e69 100644 --- a/test/sql/catalog/function/test_table_macro_args.test +++ b/test/sql/catalog/function/test_table_macro_args.test @@ -138,13 +138,10 @@ spades # create macro with non existing table -statement ok -CREATE MACRO card_no_tbl() as TABLE SELECT * FROM suit_tbl; - statement error -SELECT * FROM card_no_tbl(); +CREATE MACRO card_no_tbl() as TABLE SELECT * FROM suit_tbl; ---- - +Catalog Error # wrong arg order statement error diff --git a/test/sql/catalog/sequence/sequence_overflow.test b/test/sql/catalog/sequence/sequence_overflow.test index d57032b6c222..b3c80f61953a 100644 --- a/test/sql/catalog/sequence/sequence_overflow.test +++ b/test/sql/catalog/sequence/sequence_overflow.test @@ -64,6 +64,7 @@ create sequence seq3 INCREMENT BY 1 MINVALUE 9223372036854775800 MAXVALUE 922337 statement error SELECT nextval('seq3') from generate_series(0,20); ---- +:.*Sequence Error.*reached maximum value of sequence.* statement ok create sequence seq4 INCREMENT BY -1 MINVALUE -9223372036854775808 MAXVALUE -9223372036854775800; @@ -71,6 +72,7 @@ create sequence seq4 INCREMENT BY -1 MINVALUE -9223372036854775808 MAXVALUE -922 statement error SELECT nextval('seq4') from generate_series(0,20); ---- +:.*Sequence Error.*reached minimum value of sequence.* statement ok create sequence seq5 INCREMENT BY 9223372036854775807 MINVALUE 9223372036854775800 MAXVALUE 9223372036854775807 CYCLE; @@ -106,6 +108,7 @@ create sequence seq6 INCREMENT BY 9223372036854775807 MINVALUE 92233720368547758 statement error SELECT nextval('seq6') from generate_series(0,20); ---- +:.*Sequence Error.*reached maximum value of sequence.* statement ok create sequence seq7 INCREMENT BY -9223372036854775808 MINVALUE -9223372036854775808 MAXVALUE -9223372036854775800; @@ -113,3 +116,4 @@ create sequence seq7 INCREMENT BY -9223372036854775808 MINVALUE -922337203685477 statement error SELECT nextval('seq7') from generate_series(0,20); ---- +:.*Sequence Error.*reached minimum value of sequence.* \ No newline at end of file diff --git a/test/sql/catalog/sequence/test_sequence.test b/test/sql/catalog/sequence/test_sequence.test index 1724d502789f..f75ddcde6875 100644 --- a/test/sql/catalog/sequence/test_sequence.test +++ b/test/sql/catalog/sequence/test_sequence.test @@ -16,6 +16,7 @@ CREATE SEQUENCE seq; statement error CREATE SEQUENCE seq; ---- +:.*Catalog Error.*already exists.* # ignore errors if sequence already exists statement ok @@ -80,17 +81,18 @@ NULL statement error SELECT nextval(a) FROM (VALUES ('seq'), (NULL), ('seq')) tbl1(a) ---- -non-constant sequences are no longer supported +:.*Not implemented Error.*non-constant sequences are no longer supported.* statement error SELECT currval(a) FROM (VALUES ('seq'), (NULL), ('seq')) tbl1(a) ---- -non-constant sequences are no longer supported +:.*Not implemented Error.*non-constant sequences are no longer supported.* # can't create a sequence that already exists statement error CREATE SEQUENCE seq; ---- +:.*Catalog Error.*already exists.* # drop the sequence statement ok @@ -100,6 +102,7 @@ DROP SEQUENCE seq; statement error DROP SEQUENCE seq; ---- +:.*Catalog Error: Sequence.*does not exist.* # but doesn't fail with IF EXISTS statement ok @@ -162,6 +165,7 @@ SELECT nextval('seq') statement error SELECT nextval('seq') ---- +:.*Sequence Error.*reached maximum value of sequence.* statement ok DROP SEQUENCE seq; @@ -263,6 +267,7 @@ SELECT currval('seq') statement error SELECT nextval('seq') ---- +:.*Sequence Error.*reached minimum value of sequence.* query I SELECT currval('seq') @@ -294,6 +299,7 @@ SELECT nextval('seq') statement error SELECT nextval('seq') ---- +:.*Sequence Error.*reached maximum value of sequence.* statement ok DROP SEQUENCE seq; @@ -367,11 +373,13 @@ DROP SEQUENCE seq; statement error CREATE SEQUENCE seq INCREMENT 1 START -1 CYCLE; ---- +:.*Parser Error.*cannot be less than MINVALUE.* # max_value defaults to -1, setting start to 1 gives start > max_value statement error CREATE SEQUENCE seq INCREMENT -1 START 1 CYCLE; ---- +:.*Parser Error.*cannot be greater than MAXVALUE.* # sequences in schemas statement ok @@ -411,31 +419,37 @@ SELECT currval('"a"."seq"'), currval('"b".seq'); statement error SELECT nextval('"a"."seq'); ---- +:.*Parser Error.*Unterminated quote.* # too many separators statement error SELECT nextval('a.b.c.d'); ---- +:.*Parser Error.*too many entries found.* # start exceeds max value statement error CREATE SEQUENCE seq MAXVALUE 5 START WITH 6; ---- +:.*Parser Error.*cannot be greater than MAXVALUE.* # start preceeds min value statement error CREATE SEQUENCE seq MINVALUE 5 START WITH 4; ---- +:.*Parser Error.*cannot be less than MINVALUE.* # min value bigger than max statement error CREATE SEQUENCE seq MINVALUE 7 MAXVALUE 5; ---- +:.*Parser Error.*must be less than MAXVALUE.* # increment must not be 0 statement error CREATE SEQUENCE seq INCREMENT 0; ---- +:.*Parser Error.*Increment must not be zero.* statement ok CREATE SEQUENCE seq; @@ -479,12 +493,12 @@ seq2 2 statement error SELECT s, nextval(s) FROM strings ---- -non-constant sequences are no longer supported +:.*Not implemented Error.*non-constant sequences are no longer supported.* statement error SELECT s, currval(s) FROM strings ---- -non-constant sequences are no longer supported +:.*Not implemented Error.*non-constant sequences are no longer supported.* # this will also cause an error if the sequence does not exist statement ok @@ -493,6 +507,7 @@ INSERT INTO strings VALUES ('nonexistant_seq') statement error SELECT s, nextval(s) FROM strings ---- +:.*Not implemented Error.*non-constant sequences are no longer supported.* # currval causes error for new sequence statement ok @@ -501,37 +516,45 @@ CREATE SEQUENCE fresh; statement error select currval('fresh'); ---- +:.*Sequence Error.*sequence is not yet defined in this session.* # convert inputs into varchar if that's not the case statement error select nextval(1 + 1); ---- +:.*Binder Error: No function matches.* statement error select currval(true); ---- +:.*Binder Error: No function matches.* # max value specified more than once statement error CREATE SEQUENCE wrongseq NO MAXVALUE MAXVALUE 2; ---- +:.*Parser Error: Maxvalue should be passed as most once.* # min value specified more than once statement error CREATE SEQUENCE wrongseq MINVALUE 10 MINVALUE 2; ---- +:.*Parser Error: Minvalue should be passed as most once.* # start value specified more than once statement error CREATE SEQUENCE wrongseq START 13 START WITH 3; ---- +:.*Parser Error: Start value should be passed as most once.* # cycle value specified more than once statement error CREATE SEQUENCE wrongseq CYCLE MAXVALUE 2 MINVALUE 1 NO CYCLE; ---- +:.*Parser Error: Cycle value should be passed as most once.* # increment value specified more than once statement error CREATE SEQUENCE wrongseq INCREMENT 2 INCREMENT BY -1; ---- +:.*Parser Error: Increment value should be passed as most once.* \ No newline at end of file diff --git a/test/sql/catalog/table/create_table_as_abort.test b/test/sql/catalog/table/create_table_as_abort.test index e88ec7f3d0ee..9140ac78d497 100644 --- a/test/sql/catalog/table/create_table_as_abort.test +++ b/test/sql/catalog/table/create_table_as_abort.test @@ -11,3 +11,4 @@ CREATE TABLE IF NOT EXISTS integers AS SELECT i1.i FROM range(10000000000000000) statement error CREATE TABLE integers AS SELECT i1.i FROM range(10000000000000000) i1(i); ---- +:.*Catalog Error.*already exists.* \ No newline at end of file diff --git a/test/sql/catalog/table/create_table_parameters.test b/test/sql/catalog/table/create_table_parameters.test index 01f15a9ad03f..76161c5cdc87 100644 --- a/test/sql/catalog/table/create_table_parameters.test +++ b/test/sql/catalog/table/create_table_parameters.test @@ -5,8 +5,10 @@ statement error CREATE TABLE t0 ( c1 INT DEFAULT ? ); ---- -DEFAULT values cannot contain parameters +:.*Binder Error.*cannot contain parameters.* +# error messages for `enable_verification`=(false|true) statement error CREATE TABLE t0 ( c1 INT CHECK (?) ); ---- +:(.*Invalid Input Error: Expected 2 parameters.*|.*Binder Error.*Unexpected prepared parameter.*) \ No newline at end of file diff --git a/test/sql/catalog/test_catalog_errors.test b/test/sql/catalog/test_catalog_errors.test index 7afb1790ad36..8c1561010dbe 100644 --- a/test/sql/catalog/test_catalog_errors.test +++ b/test/sql/catalog/test_catalog_errors.test @@ -12,26 +12,31 @@ CREATE VIEW vintegers AS SELECT 42; statement error CREATE OR REPLACE VIEW integers AS SELECT 42; ---- +:.*Catalog Error: Existing object integers.* # cannot use DROP VIEW to drop a table statement error DROP VIEW integers ---- +:.*Catalog Error: Existing object integers.* # cannot drop a table that does not exist statement error DROP TABLE blabla ---- +:.*Catalog Error.*does not exist.* # cannot alter a table that does not exist statement error ALTER TABLE blabla RENAME COLUMN i TO k ---- +:.*Catalog Error.*does not exist.* # cannot drop view with DROP TABLE statement error DROP TABLE IF EXISTS vintegers ---- +:.*Catalog Error: Existing object vintegers.* statement ok CREATE INDEX i_index ON integers(i); @@ -40,7 +45,7 @@ CREATE INDEX i_index ON integers(i); statement error CREATE INDEX i_index ON integers(i); ---- -already exists +:.*Catalog Error.*already exists.* # with IF NOT EXISTS it does not fail! statement ok @@ -54,6 +59,7 @@ DROP INDEX i_index statement error DROP INDEX i_index ---- +:.*Catalog Error.*does not exist.* # IF NOT EXISTS does not report the failure statement ok @@ -67,7 +73,7 @@ CREATE UNIQUE INDEX i_index ON integers(i); statement error CREATE UNIQUE INDEX i_index ON integers(i); ---- -already exists +:.*Catalog Error.*already exists.* # with IF NOT EXISTS it does not fail! statement ok @@ -80,3 +86,4 @@ DROP TABLE integers; statement error DROP INDEX i_index ---- +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/catalog/test_incorrect_table_creation.test b/test/sql/catalog/test_incorrect_table_creation.test index a399c5be592b..7a896d5489a5 100644 --- a/test/sql/catalog/test_incorrect_table_creation.test +++ b/test/sql/catalog/test_incorrect_table_creation.test @@ -14,10 +14,12 @@ CREATE TABLE integers(i INTEGER, PRIMARY KEY(i, i)) # multiple primary keys statement error -CREATE TABLE integers(i INTEGER, PRIMARY KEY(i), PRIMARY KEY(i) +CREATE TABLE integers(i INTEGER, PRIMARY KEY(i), PRIMARY KEY(i)) ---- +Parser Error: table "integers" has more than one primary key statement error -CREATE TABLE integers(i INTEGER PRIMARY KEY, PRIMARY KEY(i) +CREATE TABLE integers(i INTEGER PRIMARY KEY, PRIMARY KEY(i)) ---- +Parser Error: table "integers" has more than one primary key diff --git a/test/sql/catalog/test_standard_schema.test b/test/sql/catalog/test_standard_schema.test index 87c6f02d976a..c99398e60bef 100644 --- a/test/sql/catalog/test_standard_schema.test +++ b/test/sql/catalog/test_standard_schema.test @@ -6,6 +6,7 @@ statement error DROP SCHEMA main CASCADE; ---- +:.*Catalog Error: Cannot drop entry.* # create and drop an empty schema statement ok @@ -22,6 +23,7 @@ CREATE SCHEMA test; statement error CREATE SCHEMA test; ---- +:.*Catalog Error.*already exists.* # if not exists ignores error statement ok @@ -35,6 +37,7 @@ CREATE TABLE test.hello(i INTEGER); statement error CREATE TABLE test2.hello(i INTEGER); ---- +:.*Catalog Error.*does not exist.* # use the table in queries # insert into table @@ -45,6 +48,7 @@ INSERT INTO test.hello VALUES (2), (3), (4) statement error SELECT * FROM hello ---- +:.*Catalog Error.*does not exist.* # with schema specified should succeed query I @@ -58,6 +62,7 @@ SELECT * FROM test.hello statement error DROP SCHEMA test; ---- +:.*Dependency Error: Cannot drop entry.* # unless we use cascade to drop statement ok @@ -71,4 +76,4 @@ DROP SCHEMA IF EXISTS test; statement error DROP SCHEMA test; ---- - +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/catalog/test_temporary.test b/test/sql/catalog/test_temporary.test index 16081caf7972..f45a124ac653 100644 --- a/test/sql/catalog/test_temporary.test +++ b/test/sql/catalog/test_temporary.test @@ -34,7 +34,7 @@ SELECT * FROM s1 statement error CREATE TABLE temp.integersy(i INTEGER) ---- -Only TEMPORARY table names can use the "temp" catalog +:.*Parser Error.*Only TEMPORARY.* statement ok CREATE SCHEMA temp @@ -42,7 +42,7 @@ CREATE SCHEMA temp statement error CREATE TABLE temp.integersy(i INTEGER) ---- -Ambiguous +:.*Binder Error.*Ambiguous reference.* statement ok CREATE TABLE memory.temp.integersy(i INTEGER) @@ -53,6 +53,7 @@ DROP SCHEMA temp CASCADE statement error DROP TABLE memory.main.integersx ---- +:.*Catalog Error.*does not exist.* statement ok DROP TABLE integersx @@ -67,11 +68,13 @@ DROP TABLE temp.integersx statement error CREATE TEMPORARY TABLE integers2(i INTEGER) ON COMMIT DELETE ROWS ---- +:.*Not implemented Error: Only ON COMMIT.* # temp table already exists statement error CREATE TEMPORARY TABLE integers(i INTEGER) ---- +:.*Catalog Error.*already exists.* statement ok INSERT INTO integers VALUES (42) @@ -151,8 +154,10 @@ ROLLBACK statement error SELECT i from integers3 ---- +:.*Catalog Error.*does not exist.* # table is not visible to other cons statement error con2 INSERT INTO integers VALUES (42) ---- +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/catalog/view/recursive_view.test b/test/sql/catalog/view/recursive_view.test index 92237e7210a5..df6b4de26c08 100644 --- a/test/sql/catalog/view/recursive_view.test +++ b/test/sql/catalog/view/recursive_view.test @@ -21,6 +21,7 @@ CREATE OR REPLACE VIEW foo AS (SELECT * FROM foo); statement error SELECT * FROM foo; ---- +:.*Binder Error.*infinite recursion detected.* # more complex recursive view definition statement ok @@ -35,3 +36,4 @@ CREATE OR REPLACE VIEW foo AS (SELECT (SELECT * FROM foo2)); statement error SELECT * FROM foo; ---- +:.*Binder Error.*infinite recursion detected.* \ No newline at end of file diff --git a/test/sql/catalog/view/test_view.test b/test/sql/catalog/view/test_view.test index ff89e80769e1..82acf890594d 100644 --- a/test/sql/catalog/view/test_view.test +++ b/test/sql/catalog/view/test_view.test @@ -17,7 +17,7 @@ FROM t1 WHERE i < 43 statement error CREATE VIEW v1 AS SELECT 'whatever' ---- -Catalog Error: View with name "v1" already exists! +:.*Catalog Error.*already exists.* query I SELECT j FROM v1 WHERE j > 41 @@ -36,6 +36,7 @@ DROP VIEW v1 statement error SELECT j FROM v1 WHERE j > 41 ---- +:.*Catalog Error.*does not exist.* statement ok CREATE VIEW v1 AS SELECT 'whatever' @@ -56,6 +57,7 @@ SELECT * FROM v1 statement error INSERT INTO v1 VALUES (1) ---- +:.*Catalog Error.*not an table.* statement ok DROP VIEW v1 @@ -63,6 +65,7 @@ DROP VIEW v1 statement error DROP VIEW v1 ---- +:.*Catalog Error.*does not exist.* statement ok DROP VIEW IF EXISTS v1 @@ -70,3 +73,4 @@ DROP VIEW IF EXISTS v1 statement error CREATE VIEW v1 AS SELECT * FROM dontexist ---- +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/catalog/view/test_view_delete_update.test b/test/sql/catalog/view/test_view_delete_update.test index 22a733a0aab1..3606c84dfd47 100644 --- a/test/sql/catalog/view/test_view_delete_update.test +++ b/test/sql/catalog/view/test_view_delete_update.test @@ -17,9 +17,10 @@ CREATE VIEW v1 AS SELECT i AS j FROM t1 WHERE i < 43 statement error DELETE FROM v1; ---- +:.*Binder Error: Can only delete.* # try to update the view statement error UPDATE v1 SET j=1; ---- - +:.*Binder Error: Can only update.* \ No newline at end of file diff --git a/test/sql/collate/test_collate_list.test b/test/sql/collate/test_collate_list.test index 2496caed5b02..d6e48d3d91f1 100644 --- a/test/sql/collate/test_collate_list.test +++ b/test/sql/collate/test_collate_list.test @@ -8,4 +8,4 @@ PRAGMA collations statement error PRAGMA collations=3 ---- - +:.*Catalog Error: unrecognized configuration parameter.* \ No newline at end of file diff --git a/test/sql/collate/test_collate_orderby_column_number.test b/test/sql/collate/test_collate_orderby_column_number.test index 781b78deccbe..d476dd64be89 100644 --- a/test/sql/collate/test_collate_orderby_column_number.test +++ b/test/sql/collate/test_collate_orderby_column_number.test @@ -25,18 +25,22 @@ SELECT 999::VARCHAR ORDER BY 1 COLLATE NOCASE; statement error SELECT 999 FROM ORDER BY 1 COLLATE NOCASE; ---- +Parser Error: syntax error at or near "ORDER" statement error SELECT 'a' FROM ORDER BY -1 COLLATE NOCASE; ---- +Parser Error: syntax error at or near "ORDER" statement error SELECT 'a' FROM ORDER BY 0 COLLATE NOCASE; ---- +Parser Error: syntax error at or near "ORDER" statement error SELECT 'a' FROM ORDER BY -0 COLLATE NOCASE; ---- +Parser Error: syntax error at or near "ORDER" statement ok CREATE TABLE collate_test(s VARCHAR); diff --git a/test/sql/constraints/foreignkey/test_fk_cross_schema.test b/test/sql/constraints/foreignkey/test_fk_cross_schema.test index 597b1eac8c96..2402a59476d2 100644 --- a/test/sql/constraints/foreignkey/test_fk_cross_schema.test +++ b/test/sql/constraints/foreignkey/test_fk_cross_schema.test @@ -17,3 +17,4 @@ INSERT INTO s1.pk_integers VALUES (1), (2), (3) statement error CREATE TABLE s2.fk_integers(j INTEGER, FOREIGN KEY (j) REFERENCES s1.pk_intexgers(i)) ---- +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/constraints/foreignkey/test_fk_rollback.test b/test/sql/constraints/foreignkey/test_fk_rollback.test index fc60337f804d..cd71cc502832 100644 --- a/test/sql/constraints/foreignkey/test_fk_rollback.test +++ b/test/sql/constraints/foreignkey/test_fk_rollback.test @@ -18,7 +18,7 @@ CREATE TABLE fk_integers(j INTEGER, FOREIGN KEY (j) REFERENCES pk_integers(i)) statement error DROP TABLE pk_integers ---- -Could not drop the table because this table is main key table of the table "fk_integers" +:.*Catalog Error.*Could not drop the table.* statement ok ROLLBACK @@ -51,11 +51,14 @@ ROLLBACK statement error DELETE FROM pk_integers WHERE i=2; ---- +:.*Constraint Error: Violates foreign key constraint.* statement error INSERT INTO fk_integers VALUES (3); ---- +:.*Constraint Error: Violates foreign key constraint.* statement error DROP TABLE pk_integers ---- +:.*Catalog Error.*Could not drop the table.* \ No newline at end of file diff --git a/test/sql/constraints/primarykey/test_pk_many_columns.test b/test/sql/constraints/primarykey/test_pk_many_columns.test index dd1b75a919a8..08906d055ee4 100644 --- a/test/sql/constraints/primarykey/test_pk_many_columns.test +++ b/test/sql/constraints/primarykey/test_pk_many_columns.test @@ -9,6 +9,7 @@ CREATE TABLE numbers(a integer, b integer, c integer, d integer, e integer, PRIM statement error INSERT INTO numbers VALUES (1,1,1,1,1), (1,1,1,1,1) ---- +:.*Constraint Error.*constraint violation.* # insert unique values statement ok @@ -18,6 +19,7 @@ INSERT INTO numbers VALUES (1,1,1,1,1),(1,2,1,1,1),(1,1,2,1,1),(2,2,2,2,2) statement error INSERT INTO numbers VALUES (1,1,1,1,1),(1,1,1,1,4); ---- +:.*Constraint Error: Duplicate key.* # now insert just the second value statement ok @@ -27,4 +29,4 @@ INSERT INTO numbers VALUES (1,1,1,1,4); statement error UPDATE numbers SET c=1 WHERE c=2 ---- - +:.*Constraint Error: Duplicate key.* \ No newline at end of file diff --git a/test/sql/constraints/primarykey/test_pk_string.test b/test/sql/constraints/primarykey/test_pk_string.test index 54ea7630e4bf..286885d65b5f 100644 --- a/test/sql/constraints/primarykey/test_pk_string.test +++ b/test/sql/constraints/primarykey/test_pk_string.test @@ -9,6 +9,7 @@ CREATE TABLE numbers(i varchar PRIMARY KEY, j INTEGER) statement error INSERT INTO numbers VALUES ('1', 4), ('1', 5) ---- +:.*Constraint Error.*constraint violation.* # insert unique values statement ok @@ -24,6 +25,7 @@ SELECT * FROM numbers statement error INSERT INTO numbers VALUES ('6', 6), ('1', 4); ---- +:.*Constraint Error: Duplicate key.* # now insert just the first value statement ok @@ -40,9 +42,10 @@ SELECT * FROM numbers statement error INSERT INTO numbers VALUES (NULL, 4); ---- +:.*Constraint Error.*constraint failed.* # update NULL is also not allowed statement error UPDATE numbers SET i=NULL; ---- - +:.*Constraint Error.*constraint failed.* \ No newline at end of file diff --git a/test/sql/constraints/primarykey/test_primary_key.test b/test/sql/constraints/primarykey/test_primary_key.test index 1659df26afae..cb2cb6299afb 100644 --- a/test/sql/constraints/primarykey/test_primary_key.test +++ b/test/sql/constraints/primarykey/test_primary_key.test @@ -9,6 +9,7 @@ CREATE TABLE integers(i INTEGER PRIMARY KEY, j INTEGER) statement error INSERT INTO integers VALUES (3, 4), (3, 5) ---- +:.*Constraint Error.*constraint violation.* # insert unique values statement ok @@ -24,6 +25,7 @@ SELECT * FROM integers statement error INSERT INTO integers VALUES (6, 6), (3, 4); ---- +:.*Constraint Error.*violates primary key constraint.* # now insert just the first value statement ok @@ -40,11 +42,13 @@ SELECT * FROM integers statement error INSERT INTO integers VALUES (NULL, 4); ---- +:.*Constraint Error.*constraint failed.* # update NULL is also not allowed statement error UPDATE integers SET i=NULL; ---- +:.*Constraint Error.*constraint failed.* # insert the same value from multiple connections # NOTE: this tests current behavior @@ -71,4 +75,4 @@ COMMIT statement error con2 COMMIT ---- - +:.*TransactionContext Error.*constraint violation.* \ No newline at end of file diff --git a/test/sql/copy/copy_blob.test b/test/sql/copy/copy_blob.test new file mode 100644 index 000000000000..358cc79beda8 --- /dev/null +++ b/test/sql/copy/copy_blob.test @@ -0,0 +1,45 @@ +# name: test/sql/copy/copy_blob.test +# group: [copy] + +statement error +COPY (select 'foo') TO '__TEST_DIR__/test.blob' (FORMAT BLOB); +---- +Binder Error: "COPY (FORMAT BLOB)" only supports a single BLOB column + +statement error +COPY (select 'foo'::BLOB, 10) TO '__TEST_DIR__/test.blob' (FORMAT BLOB); +---- +Binder Error: "COPY (FORMAT BLOB)" only supports a single BLOB column + + +statement ok +COPY (select 'foo'::BLOB) TO '__TEST_DIR__/test.blob' (FORMAT BLOB); + +query III +select filename LIKE '%test.blob', content, size from read_blob('__TEST_DIR__/test.blob'); +---- +true foo 3 + + +statement error +COPY (select 'foo'::BLOB) TO '__TEST_DIR__/test.blob.gz' (FORMAT BLOB, ASDFGH); +---- +Binder Error: Unrecognized option for COPY (FORMAT BLOB): "ASDFGH" + +# With compression +statement ok +COPY (select 'foo'::BLOB) TO '__TEST_DIR__/test.blob.gz' (FORMAT BLOB); + +query II +select filename LIKE '%test.blob.gz', size from read_blob('__TEST_DIR__/test.blob.gz'); +---- +true 26 + +# With explicit compression +statement ok +COPY (select 'foo'::BLOB) TO '__TEST_DIR__/test2.blob' (FORMAT BLOB, COMPRESSION 'GZIP'); + +query II +select filename LIKE '%test2.blob', size from read_blob('__TEST_DIR__/test2.blob'); +---- +true 26 \ No newline at end of file diff --git a/test/sql/copy/csv/afl/test_fuzz_3981.test_slow b/test/sql/copy/csv/afl/test_fuzz_3981.test_slow deleted file mode 100644 index 19ad0bbcd0fe..000000000000 --- a/test/sql/copy/csv/afl/test_fuzz_3981.test_slow +++ /dev/null @@ -1,40 +0,0 @@ -# name: test/sql/copy/csv/afl/test_fuzz_3981.test_slow -# description: fuzzer generated csv files - should not raise internal exception (by failed assertion). -# group: [afl] - -statement ok -PRAGMA enable_verification - -query I -select count(file) from glob('data/csv/afl/3981/*'); ----- -7 - -statement maybe -FROM read_csv('data/csv/afl/3981/case_0.csv', compression='gzip'); ----- - -statement maybe -FROM read_csv('data/csv/afl/3981/case_1.csv', compression='gzip'); ----- - -statement maybe -FROM read_csv('data/csv/afl/3981/case_2.csv', compression='gzip'); ----- - -statement maybe -FROM read_csv('data/csv/afl/3981/case_3.csv', compression='gzip'); ----- - -statement maybe -FROM read_csv('data/csv/afl/3981/case_4.csv', compression='gzip'); ----- - -statement maybe -FROM read_csv('data/csv/afl/3981/case_5.csv', compression='gzip'); ----- - -statement maybe -FROM read_csv('data/csv/afl/3981/case_6.csv', compression='gzip'); ----- - diff --git a/test/sql/copy/csv/csv_glob_fallback.test b/test/sql/copy/csv/csv_glob_fallback.test index 144ace96c1da..481c3e54b894 100644 --- a/test/sql/copy/csv/csv_glob_fallback.test +++ b/test/sql/copy/csv/csv_glob_fallback.test @@ -15,3 +15,4 @@ SELECT * FROM 'data/csv/[avalon]_daily-avg.csv' statement error SELECT * FROM 'data/csv/[avxalon]_daily-avg.csv' ---- +:.*IO Error: No files found.* \ No newline at end of file diff --git a/test/sql/copy/csv/glob/copy_csv_glob_s3.test b/test/sql/copy/csv/glob/copy_csv_glob_s3.test deleted file mode 100644 index 7d0aae818c55..000000000000 --- a/test/sql/copy/csv/glob/copy_csv_glob_s3.test +++ /dev/null @@ -1,74 +0,0 @@ -# name: test/sql/copy/csv/glob/copy_csv_glob_s3.test -# description: Test globbing CSVs on s3 -# group: [glob] - -statement ok -PRAGMA enable_verification - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# copy files to S3 before beginning tests -statement ok -COPY (select * from 'data/csv/glob/a1/a1.csv') to 's3://test-bucket/copy_csv_glob_s3/copy/a1/a1.csv'; - -statement ok -COPY (select * from 'data/csv/glob/a2/a2.csv') to 's3://test-bucket/copy_csv_glob_s3/copy/a2/a2.csv'; - -statement ok -COPY (select * from 'data/csv/glob/a3/b1.csv') to 's3://test-bucket/copy_csv_glob_s3/copy/a3/b1.csv'; - -statement ok -COPY (select null) to 's3://test-bucket/glob/copy/empty/empty.csv'; - -statement ok -COPY (select * from 'data/csv/glob/i1/integer.csv') to 's3://test-bucket/copy_csv_glob_s3/copy/empty/integer.csv'; - -statement ok -CREATE TABLE dates(d DATE); - -statement ok -COPY dates FROM 's3://test-bucket/copy_csv_glob_s3/copy/a[123]/*.csv' (AUTO_DETECT 1); - -# simple globbing for both url styles -foreach urlstyle path vhost - -statement ok -SET s3_url_style='${urlstyle}' - -query I -SELECT * FROM dates ORDER BY 1 ----- -2019-06-05 -2019-06-15 -2019-06-25 -2019-07-05 -2019-07-15 -2019-07-25 -2019-08-05 -2019-08-15 -2019-08-25 - -# nothing matches the glob -statement error -INSERT INTO dates FROM read_csv('s3://test-bucket/copy_csv_glob_s3/copy/*/a*a.csv', auto_detect=1) ----- -No files found that match the pattern - -endloop diff --git a/test/sql/copy/csv/glob/read_csv_glob_s3.test b/test/sql/copy/csv/glob/read_csv_glob_s3.test deleted file mode 100644 index f84d13aca877..000000000000 --- a/test/sql/copy/csv/glob/read_csv_glob_s3.test +++ /dev/null @@ -1,174 +0,0 @@ -# name: test/sql/copy/csv/glob/read_csv_glob_s3.test -# description: Test globbing CSVs over s3 -# group: [glob] - -statement ok -PRAGMA enable_verification - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# Copy files to S3 before beginning tests -statement ok -COPY (select * from read_csv_auto('data/csv/glob/a1/a1.csv')) to 's3://test-bucket/read_csv_glob_s3/glob/a1/a1.csv' ( HEADER ); - -statement ok -COPY (select * from read_csv_auto('data/csv/glob/a2/a2.csv')) to 's3://test-bucket/read_csv_glob_s3/glob/a2/a2.csv' ( HEADER ); - -statement ok -COPY (select * from read_csv_auto('data/csv/glob/a3/b1.csv')) to 's3://test-bucket/read_csv_glob_s3/glob/a3/b1.csv' ( HEADER ); - -statement ok -COPY (select null) to 's3://test-bucket/read_csv_glob_s3/glob/empty/empty.csv' (HEADER 0); - -statement ok -COPY (select * from read_csv_auto('data/csv/glob/i1/integer.csv')) to 's3://test-bucket/read_csv_glob_s3/glob/i1/integer.csv' ( HEADER ); - -statement ok -COPY (select * from read_csv_auto('data/csv/glob/a1/a1.csv')) to 's3://test-bucket/read_csv_glob_s3/glob/f1/f*.csv' ( HEADER ); - -statement ok -COPY (select * from read_csv_auto('data/csv/glob/a2/a2.csv')) to 's3://test-bucket/read_csv_glob_s3/glob/f2/f[a].csv' ( HEADER ); - -foreach urlstyle path vhost - -statement ok -SET s3_url_style='${urlstyle}' - -# simple globbing -query I -SELECT * FROM read_csv('s3://test-bucket/read_csv_glob_s3/glob/a[123]/*.csv', auto_detect=1) ORDER BY 1 ----- -2019-06-05 -2019-06-15 -2019-06-25 -2019-07-05 -2019-07-15 -2019-07-25 -2019-08-05 -2019-08-15 -2019-08-25 - -query I -SELECT * FROM read_csv('s3://test-bucket/read_csv_glob_s3/*/a[123]/a*.csv', auto_detect=1) ORDER BY 1 ----- -2019-06-05 -2019-06-15 -2019-06-25 -2019-07-05 -2019-07-15 -2019-07-25 - -query II -SELECT a, b LIKE '%a1.csv%' FROM read_csv('s3://test-bucket/read_csv_glob_s3/gl*/a[123]/a*.csv', auto_detect=1, filename=1) t1(a,b) ORDER BY 1 ----- -2019-06-05 1 -2019-06-15 1 -2019-06-25 1 -2019-07-05 0 -2019-07-15 0 -2019-07-25 0 - -# read-csv auto fails here because of a type mismatch: most files contain dates, but one file contains integers -statement error -SELECT * FROM read_csv('s3://test-bucket/read_csv_glob_s3/glob/*/*.csv', auto_detect=1) ORDER BY 1 ----- - -# forcing string parsing works -query I -SELECT * FROM read_csv('s3://test-bucket/read_csv_glob_s3/glob/[aei]*/*.csv', columns=STRUCT_PACK(d := 'STRING')) ORDER BY 1 ----- -1 -2 -2019-06-05 -2019-06-15 -2019-06-25 -2019-07-05 -2019-07-15 -2019-07-25 -2019-08-05 -2019-08-15 -2019-08-25 -3 - -query II -SELECT a, b LIKE '%a_.csv' FROM read_csv('s3://test-bucket/read_csv_glob_s3/glob/[aei]*/*.csv', columns=STRUCT_PACK(d := 'STRING'), filename=1) t(a,b) ORDER BY 1 ----- -1 0 -2 0 -2019-06-05 1 -2019-06-15 1 -2019-06-25 1 -2019-07-05 1 -2019-07-15 1 -2019-07-25 1 -2019-08-05 0 -2019-08-15 0 -2019-08-25 0 -3 0 - -# test glob parsing -query I -SELECT COUNT(*) FROM glob('s3://test-bucket/read_csv_glob_s3/glob/[aei]*/*.csv') ----- -5 - -# nothing matches the glob -statement error -SELECT * FROM read_csv('s3://test-bucket/read_csv_glob_s3/glob/[aei]*/a*a.csv', auto_detect=1) ORDER BY 1 ----- - -query I -SELECT COUNT(*) FROM glob('s3://test-bucket/read_csv_glob_s3/glob/[aei]*/a*a.csv') ----- -0 - -query I -select count(*) from glob('s3://test-bucket/read_csv_glob_s3/glob/rewoiarwiouw3rajkawrasdf790273489*.csv') limit 10; ----- -0 - -# Escaping -query I -SELECT * FROM read_csv('s3://test-bucket/read_csv_glob_s3/glob/f*/f\*.csv', auto_detect=1) ORDER BY 1 ----- -2019-06-05 -2019-06-15 -2019-06-25 - -# TODO: for supporting this we need to combine s3 url encoding with duckdb pattern matching -#query I -#SELECT * FROM read_csv('s3://test-bucket/read_csv_glob_s3/glob/f2/f[a].csv', auto_detect=1) ORDER BY 1 -#---- -#2019-07-05 -#2019-07-15 -#2019-07-25 - -#query I -#SELECT * FROM read_csv('s3://test-bucket/read_csv_glob_s3/glob/f2/f\[a\].csv', auto_detect=1) ORDER BY 1 -#---- -#2019-07-05 -#2019-07-15 -#2019-07-25 - -statement error -SELECT * FROM read_csv('s3://test-bucket/read_csv_glob_s3/glob/e2/e[a].csv', auto_detect=1) ORDER BY 1 ----- - -endloop diff --git a/test/sql/copy/csv/parallel/csv_parallel_httpfs.test b/test/sql/copy/csv/parallel/csv_parallel_httpfs.test deleted file mode 100644 index 66847a4c7897..000000000000 --- a/test/sql/copy/csv/parallel/csv_parallel_httpfs.test +++ /dev/null @@ -1,36 +0,0 @@ -# name: test/sql/copy/csv/parallel/csv_parallel_httpfs.test -# description: This test issue #7336 and #7337 -# group: [parallel] - -statement ok -PRAGMA enable_verification - -require httpfs - -query IIII -select column00, column01, column02, column03 from read_csv_auto('https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv') ----- -1 AAAAAAAABAAAAAAA 980124 7135 -2 AAAAAAAACAAAAAAA 819667 1461 -3 AAAAAAAADAAAAAAA 1473522 6247 -4 AAAAAAAAEAAAAAAA 1703214 3986 -5 AAAAAAAAFAAAAAAA 953372 4470 -6 AAAAAAAAGAAAAAAA 213219 6374 -7 AAAAAAAAHAAAAAAA 68377 3219 -8 AAAAAAAAIAAAAAAA 1215897 2471 -9 AAAAAAAAJAAAAAAA 1168667 1404 -10 AAAAAAAAKAAAAAAA 1207553 5143 - -query IIIIIIIIIIIIIIIIII -from read_csv_auto('https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv'); ----- -1 AAAAAAAABAAAAAAA 980124 7135 32946 2452238 2452208 Mr. Javier Lewis Y 9 12 1936 CHILE NULL Javier.Lewis@VFAxlnZEvOx.org 2452508 -2 AAAAAAAACAAAAAAA 819667 1461 31655 2452318 2452288 Dr. Amy Moses Y 9 4 1966 TOGO NULL Amy.Moses@Ovk9KjHH.com 2452318 -3 AAAAAAAADAAAAAAA 1473522 6247 48572 2449130 2449100 Miss Latisha Hamilton Y 18 9 1979 NIUE NULL Latisha.Hamilton@V.com 2452313 -4 AAAAAAAAEAAAAAAA 1703214 3986 39558 2450030 2450000 Dr. Michael White Y 7 6 1983 MEXICO NULL Michael.White@i.org 2452361 -5 AAAAAAAAFAAAAAAA 953372 4470 36368 2449438 2449408 Sir Robert Moran N 8 5 1956 FIJI NULL Robert.Moran@Hh.edu 2452469 -6 AAAAAAAAGAAAAAAA 213219 6374 27082 2451883 2451853 Ms. Brunilda Sharp Y 4 12 1925 SURINAME NULL Brunilda.Sharp@T3pylZEUQjm.org 2452430 -7 AAAAAAAAHAAAAAAA 68377 3219 44814 2451438 2451408 Ms. Fonda Wiles N 24 4 1985 GAMBIA NULL Fonda.Wiles@S9KnyEtz9hv.org 2452360 -8 AAAAAAAAIAAAAAAA 1215897 2471 16598 2449406 2449376 Sir Ollie Shipman N 26 12 1938 KOREA, REPUBLIC OF NULL Ollie.Shipman@be.org 2452334 -9 AAAAAAAAJAAAAAAA 1168667 1404 49388 2452275 2452245 Sir Karl Gilbert N 26 10 1966 MONTSERRAT NULL Karl.Gilbert@Crg5KyP2IxX9C4d6.edu 2452454 -10 AAAAAAAAKAAAAAAA 1207553 5143 19580 2451353 2451323 Ms. Albert Brunson N 15 10 1973 JORDAN NULL Albert.Brunson@62.com 2452641 diff --git a/test/sql/copy/csv/parallel/test_parallel_csv.test b/test/sql/copy/csv/parallel/test_parallel_csv.test index b70d1df4f2f1..a7aedae225fe 100644 --- a/test/sql/copy/csv/parallel/test_parallel_csv.test +++ b/test/sql/copy/csv/parallel/test_parallel_csv.test @@ -139,19 +139,3 @@ HIO Hillsboro Oregon United States US 1 45.540394 -122.949825 North America Unit TPA Tampa Florida United States US 1 27.979722 -82.534722 North America United States, Mexico, & Canada PNQ Pune Maharashtra India IN 1 18.582222 73.919722 Asia India MCT Muscat Muscat Oman OM 1 23.6015386 58.2899376 Middle East South Africa, Kenya, & Middle East - - -require httpfs - -query II -select * from read_csv_auto("https://duckdb-public-gzip-test.s3.us-east-2.amazonaws.com/test.csv", header = 0); ----- -foo bar -foo bar - - -query II -from read_csv_auto("https://duckdb-public-gzip-test.s3.us-east-2.amazonaws.com/test.csv.gz", header = 0); ----- -foo bar -foo bar diff --git a/test/sql/copy/csv/recursive_query_csv.test_slow b/test/sql/copy/csv/recursive_query_csv.test_slow deleted file mode 100644 index e5e94eff465b..000000000000 --- a/test/sql/copy/csv/recursive_query_csv.test_slow +++ /dev/null @@ -1,215 +0,0 @@ -# name: test/sql/copy/csv/recursive_query_csv.test_slow -# description: Test read CSV function in a recursive CTE -# group: [csv] - -require httpfs - -statement ok -PRAGMA enable_verification - -# FIXME: bug in recursive CTE -mode skip - -query IIII -with recursive - base as - ( select * - from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/Real_Estate_Sales_2001-2021_GL.csv' - where '2003-01-01' < "date recorded" and "date recorded" < '2010-01-01' and "sale amount" > 1000000 - ) - , chains as - ( - select - town - , "date recorded" as begTS - , "date recorded" as endTS - , [struct_pack(date:= "date recorded", amt:="sale amount", type:="property type")] as chain - from base - where "property type" = 'Condo' - union all - select - chains.town - , chains.begTS - , base."date recorded" as endTS - , list_append(chains.chain, struct_pack(date:= "date recorded", amt:="sale amount", type:="property type")) as chain - from base, chains - where - base.town = chains.town - and - ( - (len(chains.chain) = 1 and list_contains(['Residential', 'Single Family'], base."property type")) - or (len(chains.chain) = 2 and base."property type" = 'Condo') - or (len(chains.chain) = 3 and list_contains(['Residential', 'Single Family'], base."property type")) - ) - and chains.endTS < base."date recorded" - and base."date recorded" < (chains.endTS + interval 6 days) - ) - select * from chains - order by all; ----- -Clinton 2007-08-22 2007-08-22 [{'date': 2007-08-22, 'amt': 1175000.0, 'type': Condo}] -Danbury 2007-05-02 2007-05-02 [{'date': 2007-05-02, 'amt': 3105000.0, 'type': Condo}] -Danbury 2007-05-09 2007-05-09 [{'date': 2007-05-09, 'amt': 1014205.0, 'type': Condo}] -Darien 2007-09-12 2007-09-12 [{'date': 2007-09-12, 'amt': 1150000.0, 'type': Condo}] -Fairfield 2007-06-15 2007-06-15 [{'date': 2007-06-15, 'amt': 1100000.0, 'type': Condo}] -Greenwich 2006-11-20 2006-11-20 [{'date': 2006-11-20, 'amt': 2050000.0, 'type': Condo}] -Greenwich 2006-11-20 2006-11-21 [{'date': 2006-11-20, 'amt': 2050000.0, 'type': Condo}, {'date': 2006-11-21, 'amt': 6500000.0, 'type': Single Family}] -Greenwich 2006-12-14 2006-12-14 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}] -Greenwich 2006-12-14 2006-12-15 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}, {'date': 2006-12-15, 'amt': 2195000.0, 'type': Single Family}] -Greenwich 2006-12-14 2006-12-15 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}, {'date': 2006-12-15, 'amt': 5500000.0, 'type': Single Family}] -Greenwich 2006-12-14 2006-12-18 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}, {'date': 2006-12-18, 'amt': 5010000.0, 'type': Single Family}] -Greenwich 2007-01-19 2007-01-19 [{'date': 2007-01-19, 'amt': 2227500.0, 'type': Condo}] -Greenwich 2007-01-19 2007-01-24 [{'date': 2007-01-19, 'amt': 2227500.0, 'type': Condo}, {'date': 2007-01-24, 'amt': 1750000.0, 'type': Single Family}] -Greenwich 2007-01-31 2007-01-31 [{'date': 2007-01-31, 'amt': 4600000.0, 'type': Condo}] -Greenwich 2007-02-27 2007-02-27 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}] -Greenwich 2007-02-27 2007-02-28 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}, {'date': 2007-02-28, 'amt': 2260000.0, 'type': Single Family}] -Greenwich 2007-02-27 2007-03-01 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}, {'date': 2007-03-01, 'amt': 1900000.0, 'type': Single Family}] -Greenwich 2007-02-27 2007-03-02 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}, {'date': 2007-03-02, 'amt': 6500000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-12 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}] -Greenwich 2007-03-12 2007-03-13 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-13, 'amt': 1600000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-16 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-16 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-20 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}] -Greenwich 2007-03-12 2007-03-20 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}] -Greenwich 2007-03-12 2007-03-22 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-22, 'amt': 1580000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-22 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-22, 'amt': 1580000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-23 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-23, 'amt': 2850000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-23 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-23, 'amt': 2850000.0, 'type': Single Family}] -Greenwich 2007-03-20 2007-03-20 [{'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}] -Greenwich 2007-03-20 2007-03-22 [{'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-22, 'amt': 1580000.0, 'type': Single Family}] -Greenwich 2007-03-20 2007-03-23 [{'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-23, 'amt': 2850000.0, 'type': Single Family}] -Greenwich 2007-03-30 2007-03-30 [{'date': 2007-03-30, 'amt': 1200000.0, 'type': Condo}] -Greenwich 2007-05-15 2007-05-15 [{'date': 2007-05-15, 'amt': 1215000.0, 'type': Condo}] -Greenwich 2007-05-15 2007-05-17 [{'date': 2007-05-15, 'amt': 1215000.0, 'type': Condo}, {'date': 2007-05-17, 'amt': 2250000.0, 'type': Single Family}] -Greenwich 2007-06-15 2007-06-15 [{'date': 2007-06-15, 'amt': 2264564.0, 'type': Condo}] -Greenwich 2007-06-15 2007-06-19 [{'date': 2007-06-15, 'amt': 2264564.0, 'type': Condo}, {'date': 2007-06-19, 'amt': 1470000.0, 'type': Single Family}] -Greenwich 2007-06-15 2007-06-19 [{'date': 2007-06-15, 'amt': 2264564.0, 'type': Condo}, {'date': 2007-06-19, 'amt': 1965000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-10 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}] -Greenwich 2007-07-10 2007-07-11 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-11, 'amt': 3150000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-11 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-11, 'amt': 3250000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-11 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-11, 'amt': 7050000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-12 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-12 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-17 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}] -Greenwich 2007-07-10 2007-07-17 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}] -Greenwich 2007-07-10 2007-07-19 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-19, 'amt': 3600000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-19 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-19, 'amt': 3600000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 7225000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 18000000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 7225000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 18000000.0, 'type': Single Family}] -Greenwich 2007-07-17 2007-07-17 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}] -Greenwich 2007-07-17 2007-07-19 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-19, 'amt': 3600000.0, 'type': Single Family}] -Greenwich 2007-07-17 2007-07-20 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 7225000.0, 'type': Single Family}] -Greenwich 2007-07-17 2007-07-20 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 18000000.0, 'type': Single Family}] -Greenwich 2007-08-16 2007-08-16 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}] -Greenwich 2007-08-16 2007-08-17 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-17, 'amt': 1925000.0, 'type': Single Family}] -Greenwich 2007-08-16 2007-08-17 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-17, 'amt': 3400000.0, 'type': Single Family}] -Greenwich 2007-08-16 2007-08-20 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-20, 'amt': 2590000.0, 'type': Single Family}] -Greenwich 2007-08-16 2007-08-21 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-21, 'amt': 4100000.0, 'type': Single Family}] -New Canaan 2007-01-22 2007-01-22 [{'date': 2007-01-22, 'amt': 1735000.0, 'type': Condo}] -New Canaan 2007-02-15 2007-02-15 [{'date': 2007-02-15, 'amt': 2230000.0, 'type': Condo}] - -# JoinDependentFilter triggers on this test, make sure that the result is the same with and without -statement ok -set disabled_optimizers to 'expression_rewriter' - -query IIII -with recursive - base as - ( select * - from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/Real_Estate_Sales_2001-2021_GL.csv' - where '2003-01-01' < "date recorded" and "date recorded" < '2010-01-01' and "sale amount" > 1000000 - ) - , chains as - ( - select - town - , "date recorded" as begTS - , "date recorded" as endTS - , [struct_pack(date:= "date recorded", amt:="sale amount", type:="property type")] as chain - from base - where "property type" = 'Condo' - union all - select - chains.town - , chains.begTS - , base."date recorded" as endTS - , list_append(chains.chain, struct_pack(date:= "date recorded", amt:="sale amount", type:="property type")) as chain - from base, chains - where - base.town = chains.town - and - ( - (len(chains.chain) = 1 and list_contains(['Residential', 'Single Family'], base."property type")) - or (len(chains.chain) = 2 and base."property type" = 'Condo') - or (len(chains.chain) = 3 and list_contains(['Residential', 'Single Family'], base."property type")) - ) - and chains.endTS < base."date recorded" - and base."date recorded" < (chains.endTS + interval 6 days) - ) - select * from chains - order by all; ----- -Clinton 2007-08-22 2007-08-22 [{'date': 2007-08-22, 'amt': 1175000.0, 'type': Condo}] -Danbury 2007-05-02 2007-05-02 [{'date': 2007-05-02, 'amt': 3105000.0, 'type': Condo}] -Danbury 2007-05-09 2007-05-09 [{'date': 2007-05-09, 'amt': 1014205.0, 'type': Condo}] -Darien 2007-09-12 2007-09-12 [{'date': 2007-09-12, 'amt': 1150000.0, 'type': Condo}] -Fairfield 2007-06-15 2007-06-15 [{'date': 2007-06-15, 'amt': 1100000.0, 'type': Condo}] -Greenwich 2006-11-20 2006-11-20 [{'date': 2006-11-20, 'amt': 2050000.0, 'type': Condo}] -Greenwich 2006-11-20 2006-11-21 [{'date': 2006-11-20, 'amt': 2050000.0, 'type': Condo}, {'date': 2006-11-21, 'amt': 6500000.0, 'type': Single Family}] -Greenwich 2006-12-14 2006-12-14 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}] -Greenwich 2006-12-14 2006-12-15 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}, {'date': 2006-12-15, 'amt': 2195000.0, 'type': Single Family}] -Greenwich 2006-12-14 2006-12-15 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}, {'date': 2006-12-15, 'amt': 5500000.0, 'type': Single Family}] -Greenwich 2006-12-14 2006-12-18 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}, {'date': 2006-12-18, 'amt': 5010000.0, 'type': Single Family}] -Greenwich 2007-01-19 2007-01-19 [{'date': 2007-01-19, 'amt': 2227500.0, 'type': Condo}] -Greenwich 2007-01-19 2007-01-24 [{'date': 2007-01-19, 'amt': 2227500.0, 'type': Condo}, {'date': 2007-01-24, 'amt': 1750000.0, 'type': Single Family}] -Greenwich 2007-01-31 2007-01-31 [{'date': 2007-01-31, 'amt': 4600000.0, 'type': Condo}] -Greenwich 2007-02-27 2007-02-27 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}] -Greenwich 2007-02-27 2007-02-28 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}, {'date': 2007-02-28, 'amt': 2260000.0, 'type': Single Family}] -Greenwich 2007-02-27 2007-03-01 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}, {'date': 2007-03-01, 'amt': 1900000.0, 'type': Single Family}] -Greenwich 2007-02-27 2007-03-02 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}, {'date': 2007-03-02, 'amt': 6500000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-12 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}] -Greenwich 2007-03-12 2007-03-13 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-13, 'amt': 1600000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-16 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-16 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-20 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}] -Greenwich 2007-03-12 2007-03-20 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}] -Greenwich 2007-03-12 2007-03-22 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-22, 'amt': 1580000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-22 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-22, 'amt': 1580000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-23 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-23, 'amt': 2850000.0, 'type': Single Family}] -Greenwich 2007-03-12 2007-03-23 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-23, 'amt': 2850000.0, 'type': Single Family}] -Greenwich 2007-03-20 2007-03-20 [{'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}] -Greenwich 2007-03-20 2007-03-22 [{'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-22, 'amt': 1580000.0, 'type': Single Family}] -Greenwich 2007-03-20 2007-03-23 [{'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-23, 'amt': 2850000.0, 'type': Single Family}] -Greenwich 2007-03-30 2007-03-30 [{'date': 2007-03-30, 'amt': 1200000.0, 'type': Condo}] -Greenwich 2007-05-15 2007-05-15 [{'date': 2007-05-15, 'amt': 1215000.0, 'type': Condo}] -Greenwich 2007-05-15 2007-05-17 [{'date': 2007-05-15, 'amt': 1215000.0, 'type': Condo}, {'date': 2007-05-17, 'amt': 2250000.0, 'type': Single Family}] -Greenwich 2007-06-15 2007-06-15 [{'date': 2007-06-15, 'amt': 2264564.0, 'type': Condo}] -Greenwich 2007-06-15 2007-06-19 [{'date': 2007-06-15, 'amt': 2264564.0, 'type': Condo}, {'date': 2007-06-19, 'amt': 1470000.0, 'type': Single Family}] -Greenwich 2007-06-15 2007-06-19 [{'date': 2007-06-15, 'amt': 2264564.0, 'type': Condo}, {'date': 2007-06-19, 'amt': 1965000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-10 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}] -Greenwich 2007-07-10 2007-07-11 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-11, 'amt': 3150000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-11 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-11, 'amt': 3250000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-11 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-11, 'amt': 7050000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-12 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-12 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-17 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}] -Greenwich 2007-07-10 2007-07-17 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}] -Greenwich 2007-07-10 2007-07-19 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-19, 'amt': 3600000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-19 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-19, 'amt': 3600000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 7225000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 18000000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 7225000.0, 'type': Single Family}] -Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 18000000.0, 'type': Single Family}] -Greenwich 2007-07-17 2007-07-17 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}] -Greenwich 2007-07-17 2007-07-19 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-19, 'amt': 3600000.0, 'type': Single Family}] -Greenwich 2007-07-17 2007-07-20 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 7225000.0, 'type': Single Family}] -Greenwich 2007-07-17 2007-07-20 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 18000000.0, 'type': Single Family}] -Greenwich 2007-08-16 2007-08-16 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}] -Greenwich 2007-08-16 2007-08-17 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-17, 'amt': 1925000.0, 'type': Single Family}] -Greenwich 2007-08-16 2007-08-17 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-17, 'amt': 3400000.0, 'type': Single Family}] -Greenwich 2007-08-16 2007-08-20 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-20, 'amt': 2590000.0, 'type': Single Family}] -Greenwich 2007-08-16 2007-08-21 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-21, 'amt': 4100000.0, 'type': Single Family}] -New Canaan 2007-01-22 2007-01-22 [{'date': 2007-01-22, 'amt': 1735000.0, 'type': Condo}] -New Canaan 2007-02-15 2007-02-15 [{'date': 2007-02-15, 'amt': 2230000.0, 'type': Condo}] diff --git a/test/sql/copy/csv/test_12314.test_slow b/test/sql/copy/csv/test_12314.test_slow deleted file mode 100644 index 0d765dc099f8..000000000000 --- a/test/sql/copy/csv/test_12314.test_slow +++ /dev/null @@ -1,18 +0,0 @@ -# name: test/sql/copy/csv/test_12314.test_slow -# description: Test CSV reading for issue 12314 -# group: [csv] - -require httpfs - -statement ok -PRAGMA enable_verification - -statement error -from read_csv('https://github.com/duckdb/duckdb-data/releases/download/v1.0/sample_data_12314.csv.gz',HEADER = 1, PARALLEL=false); ----- -Change the maximum length size, e.g., max_line_size=2097408 - -query I -select count(*) from read_csv('https://github.com/duckdb/duckdb-data/releases/download/v1.0/sample_data_12314.csv.gz',HEADER = 1, PARALLEL=false , max_line_size=2097408); ----- -26238 diff --git a/test/sql/copy/csv/test_csv_httpfs.test b/test/sql/copy/csv/test_csv_httpfs.test deleted file mode 100644 index 869a0b674d32..000000000000 --- a/test/sql/copy/csv/test_csv_httpfs.test +++ /dev/null @@ -1,355 +0,0 @@ -# name: test/sql/copy/csv/test_csv_httpfs.test -# description: This test triggers the http prefetch mechanism. -# group: [csv] - -require httpfs - -statement ok -PRAGMA enable_verification - - -#FIXME this test fails: file is nonexistent -mode skip - -query IIIIII rowsort -SELECT * from read_csv_auto('https://www.data.gouv.fr/fr/datasets/r/6d186965-f41b-41f3-9b23-88241cc6890c'); ----- -2020 Allemagne Germany 26.1 53196.069 200601.2 -2020 Autriche Austria 18.0 4723.5 26215.8 -2020 Belgique Belgium 28.999999999999996 9436.1 32553.0 -2020 Bulgarie Bulgaria 11.600000000000001 1124.1 9698.7 -2020 Chypre Cyprus 0.0 0.0 1627.6 -2020 Croatie Croatia 16.3 1094.8 6726.3 -2020 Danemark Denmark 11.600000000000001 1579.0 13601.4 -2020 Espagne Spain 17.4 14211.7 81512.9 -2020 Estonie Estonia 8.5 241.1 2827.3 -2020 Finlande Finland 2.8000000000000003 692.3 24674.4 -2020 France France 20.3 28278.9 139375.8 -2020 Grèce Greece 5.800000000000001 896.5 15401.9 -2020 Hongrie Hungary 30.5 5486.7 17872.4 -2020 Irlande Ireland 17.4 1968.477 11296.601 -2020 Italie Italy 29.2 33042.585 113119.475 -2020 Lettonie Latvia 8.200000000000001 323.605 3926.131 -2020 Lituanie Lithuania 10.7 584.104 5457.728 -2020 Luxembourg Luxembourg 16.5 623.165 3786.785 -2020 Malte Malta 0.0 0.0 547.5 -2020 Pays-Bas Netherlands 37.1 16588.314 44682.656 -2020 Pologne Poland 13.5 9323.205 69135.018 -2020 Portugal Portugal 11.1 1814.878 16354.725 -2020 Roumanie Romania 23.7 5626.161 23712.653 -2020 Royaume-Uni United Kingdom 32.4 39311.416 121414.483 -2020 République tchèque Czech Republic 21.4 5187.282 24263.896 -2020 Slovaquie Slovakia 25.0 2564.876 10248.401 -2020 Slovénie Slovenia 12.1 590.243 4861.315 -2020 Suède Sweden 1.5 475.195 31311.413 -2020 UE 28 Europe 28 22.5 238152.4 1056907.5 -2021 Allemagne Germany 26.760345686044435 51812.567 193616.957 -2021 Autriche Austria 18.720006775926056 4645.795 24817.272 -2021 Belgique Belgium 29.279402721103864 9088.083 31039.168 -2021 Bulgarie Bulgaria 12.368015142641884 1176.537 9512.739 -2021 Chypre Cyprus 0.0 0.0 1528.558 -2021 Croatie Croatia 17.10389029082304 1100.12 6431.987 -2021 Danemark Denmark 11.485631727184947 1508.152 13130.771 -2021 Espagne Spain 19.10173955663722 13815.0 72323.256 -2021 Estonie Estonia 8.988278645659518 245.094 2726.818 -2021 Finlande Finland 2.9937725178230212 694.288 23191.074 -2021 France France 20.649030024470434 26465.646 128168.955 -2021 Grèce Greece 7.580480506088059 1097.87 14482.855 -2021 Hongrie Hungary 32.344729318831554 5693.164 17601.52 -2021 Irlande Ireland 18.020604987495144 1953.468 10840.191 -2021 Italie Italy 30.86368769746751 31807.236 103057.147 -2021 Lettonie Latvia 8.502139837843602 322.927 3798.185 -2021 Lituanie Lithuania 11.029023816606903 582.797 5284.212 -2021 Luxembourg Luxembourg 17.282784281000467 564.365 3265.475 -2021 Malte Malta 0.0 0.0 499.875 -2021 Pays-Bas Netherlands 37.61392206122467 15896.316 42261.788 -2021 Pologne Poland 13.146720200313602 9235.656 70250.647 -2021 Portugal Portugal 11.437926753365227 1740.3 15215.17 -2021 Roumanie Romania 24.909638477223016 5846.885 23472.38 -2021 République tchèque Czech Republic 21.716683280446812 5158.445 23753.374 -2021 Slovaquie Slovakia 25.253930010417324 2427.134 9610.916 -2021 Slovénie Slovenia 13.141683407321874 582.024 4428.839 -2021 Suède Sweden 1.497679952802663 471.085 31454.317 -2021 UE 27 UE 27 21.894190365821018 193930.95399999994 885764.4460000001 - -query IIIIII rowsort res -SELECT * from read_csv('https://www.data.gouv.fr/fr/datasets/r/6d186965-f41b-41f3-9b23-88241cc6890c',DELIM=';',Columns={'annee_de_reference':'VARCHAR','pays':'VARCHAR','label_en':'VARCHAR','part_du_gaz_naturel_dans_la_consommation_finale_d_energie0':'VARCHAR','consommation_finale_de_gaz_naturel_mtep':'VARCHAR','consommation_finale_d_energie_totale_mtep':'VARCHAR'}); - - -query IIIIII rowsort res -SELECT * from read_csv('https://www.data.gouv.fr/fr/datasets/r/6d186965-f41b-41f3-9b23-88241cc6890c',DELIM=';',Columns={'annee_de_reference':'VARCHAR','pays':'VARCHAR','label_en':'VARCHAR','part_du_gaz_naturel_dans_la_consommation_finale_d_energie0':'VARCHAR','consommation_finale_de_gaz_naturel_mtep':'VARCHAR','consommation_finale_d_energie_totale_mtep':'VARCHAR'}); - - -# Give it a try to a request that returns length 0 -query I -SELECT count(*) from read_csv_auto('https://query1.finance.yahoo.com/v7/finance/download/^GSPC?period1=1512086400&period2=1670630400&interval=1d&events=history') ----- -1265 - -# Give it a try to a request that returns length 0 -query I -SELECT count(*) from read_csv_auto('https://query1.finance.yahoo.com/v7/finance/download/^GSPC?period1=1512086400&period2=1670630400&interval=1d&events=history') ----- -1265 - -# Give it a try to a request that returns length 0 -query I -SELECT count(*) from read_csv_auto('https://query1.finance.yahoo.com/v7/finance/download/^GSPC?period1=1512086400&period2=1670630400&interval=1d&events=history') ----- -1265 - -#Add test for 5924 -query IIIIII -select * from read_csv_auto('https://csvbase.com/meripaterson/stock-exchanges'); ----- -1 Africa Lesotho HYBSE NULL 2019-03-25 -2 Asia Kazakhstan Astana International Financial Centre AIXK 2018-11-18 -3 Africa South Africa ZAR X ZARX 2018-11-18 -4 South America Argentina Bolsas y Mercados Argentinos NULL 2018-04-02 -5 North America United States of America Delaware Board of Trade NULL 2018-04-02 -6 Australia & Oceania Australia Chi-X Asia Pacific NULL 2018-04-02 -7 Australia & Oceania Australia Chi-X Australia CHIA 2018-04-02 -8 South America Mexico BIVA BIVA 2018-01-06 -9 Africa South Africa Equity Express Securities Exchange NULL 2017-12-11 -10 Africa South Africa Cape Town Stock Exchange NULL 2021-10-22 -11 North America Curacao Dutch Caribbean Securities Exchange DCSX 2017-09-12 -12 North America Canada NEO NEOE 2017-09-06 -13 North America Canada Canadian Securities Exchange CNSX 2017-09-06 -14 Western Europe Germany XETRA XETR 2017-08-21 -15 Western Europe France Euronext Paris XPAR 2017-08-19 -16 Western Europe United Kingdom Euronext London XLDN 2017-08-19 -17 Eastern Europe Albania Tirana Stock Exchange XTIR 2017-08-16 -18 Africa Algeria Bourse d'Alger XALG 2017-08-16 -19 Africa Angola BODIVA XBDV 2017-08-16 -20 South America Argentina Buenos Aires Stock Exchange XBUE 2017-08-16 -21 South America Argentina Mercado Abierto Electrónico XMAB 2017-08-16 -22 Eastern Europe Armenia Armenia Securities Exchange XARM 2020-07-29 -23 Australia & Oceania Australia Australian Securities Exchange XASX 2017-08-16 -24 Australia & Oceania Australia Block Event BLEV 2017-08-16 -25 Australia & Oceania Australia IR Plus Securities Exchange SIMV 2017-08-16 -26 Australia & Oceania Australia National Stock Exchange of Australia XNEC 2017-08-16 -27 Australia & Oceania Australia Sydney Stock Exchange APXL 2017-08-16 -28 Western Europe Austria Wiener Börse XWBO 2017-08-16 -29 Asia Azerbaijan Baku Stock Exchange BSEX 2017-08-16 -30 North America Bahamas Bahamas International Securities Exchange XBAA 2017-08-16 -31 Middle East Bahrain Bahrain Bourse XBAH 2017-08-16 -32 Asia Bangladesh Chittagong Stock Exchange XCHG 2017-08-16 -33 Asia Bangladesh Dhaka Stock Exchange XDHA 2017-08-16 -34 North America Barbados Barbados Stock Exchange XBAB 2017-08-16 -35 Eastern Europe Belarus Belarusian Currency and Stock Exchange BCSE 2017-08-16 -36 Western Europe Belgium Euronext Brussels XBRU 2017-08-16 -37 North America Bermuda Bermuda Stock Exchange XBDA 2017-08-16 -38 Asia Bhutan Royal Securities Exchange of Bhutan NULL 2017-08-16 -39 South America Bolivia Bolsa de Valores de Bolivia XBOL 2017-08-16 -40 Eastern Europe Bosnia and Herzegovina Banja Luka Stock Exchange XBLB 2017-08-16 -41 Eastern Europe Bosnia and Herzegovina Sarajevo Stock Exchange XSSE 2017-08-16 -42 Africa Botswana Botswana Stock Exchange XBOT 2017-08-16 -43 South America Brazil B3 - Brasil Bolsa Balcão BVMF 2017-08-16 -44 South America Brazil Bolsa de Valores Minab - Espírito Santo BOVM 2017-08-16 -45 Eastern Europe Bulgaria Bulgarian Stock Exchange XBUL 2017-08-16 -46 Asia Cambodia Cambodia Securities Exchange XCSX 2017-08-16 -47 North America Canada Montreal Exchange XMOD 2017-08-16 -48 North America Canada Nasdaq Canada XCSD 2017-08-16 -49 North America Canada TMX TMXS 2017-08-16 -50 North America Canada Toronto Stock Exchange XTSE 2017-08-16 -51 Africa Cape Verde Bolsa de Valores de Cabo Verde XBVC 2017-08-16 -52 North America Cayman Islands Cayman Islands Stock Exchange XCAY 2017-08-16 -53 Western Europe Channel Islands Channel Islands Stock Exchange NULL 2017-08-16 -54 South America Chile Santiago Electronic Stock Exchange XBCL 2017-08-16 -55 South America Chile Santiago Stock Exchange XSGO 2017-08-16 -56 South America Chile Valparaiso Stock Exchange BOVA 2017-08-16 -57 Asia China Shanghai Stock Exchange XSHG 2017-08-16 -58 Asia China Shenzhen Stock Exchange XSHE 2017-08-16 -59 South America Colombia Bolsa de Valores de Colombia XBOG 2017-08-16 -60 North America Costa Rica Bolsa Nacional de Valores de Costa Rica XBNV 2017-08-16 -61 Eastern Europe Croatia Zagreb Stock Exchange XZAG 2017-08-16 -62 Eastern Europe Cyprus Cyprus Stock Exchange XCYS 2017-08-16 -63 Eastern Europe Czech Republic Prague Stock Exchange XPRAG 2017-08-16 -64 Eastern Europe Czech Republic RM-System Czech Stock Exchange XRMZ 2017-08-16 -65 Western Europe Denmark Nasdaq Copenhagen XCSE 2017-08-16 -66 North America Dominican Republic Bolsa de Valores de la República Dominicana XBVR 2017-08-16 -67 South America Ecuador Bolsa de Valores de Guayaquil XGUA 2017-08-16 -68 South America Ecuador Bolsa de Valores de Quito XQUI 2017-08-16 -69 Africa Egypt Egyptian Exchange XCAI 2017-08-16 -70 Africa Egypt Nilex NILX 2017-08-16 -71 North America El Salvador Bolsa de Valores de El Salvador XSVA 2017-08-16 -72 Eastern Europe Estonia Tallinn Stock Exchange XTAL 2017-08-16 -73 Australia & Oceania Fiji South Pacific Stock Exchange XSPS 2017-08-16 -74 Western Europe Finland Nasdaq Helsinki XHEL 2017-08-16 -75 Africa Gabon Bourse Régionale des Valeurs Mobilières d'Afrique Centrale NULL 2017-08-16 -76 Asia Georgia Georgian Stock Exchange XGSE 2017-08-16 -77 Western Europe Germany Börse Berlin XBER 2017-08-16 -78 Western Europe Germany Börse Düsseldorf XDUS 2017-08-16 -79 Western Europe Germany Börse Hamburg & Hannover HAMB 2017-08-16 -80 Western Europe Germany Börse München XMUN 2017-08-16 -81 Western Europe Germany Börse Stuttgart XSTU 2017-08-16 -82 Western Europe Germany Deutsche Börse Group XFRA 2017-08-16 -83 Western Europe Germany Eurex XEUR 2017-08-16 -84 Western Europe Germany Tradegate Exchange TGAT 2017-08-16 -85 Africa Ghana Ghana Stock Exchange XGHA 2017-08-16 -86 Western Europe Gibraltar Gibraltar Stock Exchange GSXL 2017-08-16 -87 Western Europe Greece Athens Stock Exchange ASEX 2017-08-16 -88 North America Guatemala Bolsa Nacional de Valores XGTG 2017-08-16 -89 Western Europe Guernsey International Stock Exchange XCIE 2017-08-16 -90 South America Guyana Guyana Stock Exchange GSCI 2017-08-16 -91 North America Haiti Haitian Stock Exchange NULL 2017-08-16 -92 North America Honduras Bolsa Centroamericana de Valores XBCV 2017-08-16 -93 Asia Hong Kong Hong Kong Growth Enterprise Market XGEM 2017-08-16 -94 Asia Hong Kong Hong Kong Stock Exchange XHKG 2017-08-16 -95 Eastern Europe Hungary Budapest Stock Exchange XBUD 2017-08-16 -96 Western Europe Iceland Nasdaq Iceland XICE 2017-08-16 -97 Asia India Ahmedabad Stock Exchange NULL 2017-08-16 -98 Asia India Bangalore Stock Exchange XBAN 2017-08-16 -99 Asia India Bombay Stock Exchange XBOM 2017-08-16 -100 Asia India BSE SME BSME 2017-08-16 -101 Asia India Calcutta Stock Exchange XCAL 2017-08-16 -102 Asia India Cochin Stock Exchange NULL 2017-08-16 -103 Asia India Coimbatore Stock Exchange NULL 2017-08-16 -104 Asia India Delhi Stock Exchange XDES 2017-08-16 -105 Asia India Inter-Connected Stock Exchange of India ISEX 2017-08-16 -106 Asia India Ludhiana Stock and Capital NULL 2017-08-16 -107 Asia India Metropolitan Stock Exchange NULL 2017-08-16 -108 Asia India National Stock Exchange of India XNSE 2017-08-16 -109 Asia India OTC Exchange of India OTCX 2017-08-16 -110 Asia India Pune Stock Exchange NULL 2017-08-16 -111 Asia India Saurashtra Kutch Stock Exchange NULL 2017-08-16 -112 Asia India United Stock Exchange of India XUSE 2017-08-16 -113 Asia India Vadodara Stock Exchange NULL 2017-08-16 -114 Asia Indonesia Indonesia Stock Exchange XIDX 2017-08-16 -115 Asia Iran Iran Fara Bourse NULL 2017-08-16 -116 Middle East Iran Tehran Stock Exchange XTEH 2017-08-16 -117 Middle East Iraq Iraq Stock Exchange XIQS 2017-08-16 -118 Western Europe Ireland Irish Stock Exchange XDUB 2017-08-16 -119 Middle East Israel Tel Aviv Stock Exchange XTAE 2017-08-16 -120 Western Europe Italy Borsa Italiana XMIL 2017-08-16 -121 Africa Ivory Coast Bourse Regionale des Valeurs Mobilieres XBRV 2017-08-16 -122 North America Jamaica Jamaica Stock Exchange XJAM 2017-08-16 -123 Asia Japan Chi-X Japan CHIJ 2017-08-16 -124 Asia Japan Daiwa Securities DRCT 2017-08-16 -125 Asia Japan Fukuoka Stock Exchange XFKA 2017-08-16 -126 Asia Japan Japan Exchange Group XJPX 2017-08-16 -127 Asia Japan Nagoya Stock Exchange XNGO 2017-08-16 -128 Asia Japan Sapporo Securities Exchange XSAP 2017-08-16 -129 Asia Japan SBI Japannext SBIJ 2017-08-16 -130 Middle East Jordan Amman Stock Exchange XAMM 2017-08-16 -131 Asia Kazakhstan Kazakhstan Stock Exchange XKAZ 2017-08-16 -132 Africa Kenya Nairobi Stock Exchange XNAI 2017-08-16 -133 Middle East Kuwait Kuwait Stock Exchange XKUW 2017-08-16 -134 Asia Kyrgyzstan Kyrgyz Stock Exchange XKSE 2017-08-16 -135 Asia Laos Lao Securities Exchange XLAO 2017-08-16 -136 Eastern Europe Latvia Riga Stock Exchange XRIS 2017-08-16 -137 Middle East Lebanon Beirut Stock Exchange XBEY 2017-08-16 -138 Africa Lesotho Maseru Securities Exchange NULL 2017-08-16 -139 Eastern Europe Lithuania Vilnius Stock Exchange XLIT 2017-08-16 -140 Western Europe Luxembourg Luxembourg Stock Exchange XLUX 2017-08-16 -141 Eastern Europe Macedonia Macedonian Stock Exchange XMAE 2017-08-16 -142 Africa Malawi Malawi Stock Exchange XMSW 2017-08-16 -143 Asia Malaysia Bursa Malaysia XKLS 2017-08-16 -144 Asia Maldives Maldives Stock Exchange MALX 2017-08-16 -145 Western Europe Malta Malta Stock Exchange XMAL 2017-08-16 -146 Western Europe Malta Malta Stock Exchange Prospects PROS 2017-08-16 -147 Africa Mauritius Stock Exchange of Mauritius XMAU 2017-08-16 -148 North America Mexico Bolsa Mexicana de Valores XMEX 2017-08-16 -149 Western Europe Moldova Moldova Stock Exchange XMOL 2017-08-16 -150 Asia Mongolia Mongolian Stock Exchange XULA 2017-08-16 -151 Eastern Europe Montenegro Montenegro Stock Exchange XMNX 2017-08-16 -152 Africa Morocco Casablanca Stock Exchange XCAS 2017-08-16 -153 Africa Mozambique Bolsa de Valores de Mozambique XBVM 2017-08-16 -154 Asia Myanmar Myanmar Securities Exchange Centre NULL 2017-08-16 -155 Asia Myanmar Yangon Stock Exchange NULL 2017-08-16 -156 Africa Namibia Namibian Stock Exchange XNAM 2017-08-16 -157 Asia Nepal Nepal Stock Exchange XNEP 2017-08-16 -158 Western Europe Netherlands Euronext Amsterdam XAMS 2017-08-16 -159 Western Europe Netherlands Nxchange XNXC 2017-08-16 -160 Australia & Oceania New Zealand New Zealand Exchange XNZE 2017-08-16 -161 North America Nicaragua Bolsa de Valores de Nicaragua XMAN 2017-08-16 -162 Africa Nigeria Nigerian Stock Exchange XNSA 2017-08-16 -163 Western Europe Norway Oslo Stock Exchange XOSL 2017-08-16 -164 Middle East Oman Muscat Securities Market XMUS 2017-08-16 -165 Asia Pakistan Lahore Stock Exchange NULL 2017-08-16 -166 Asia Pakistan Pakistan Stock Exchange XKAR 2017-08-16 -167 Middle East Palestine Palestine Securities Exchange XPAE 2017-08-16 -168 North America Panama Bolsa de Valores de Panama XPTY 2017-08-16 -169 Australia & Oceania Papua New Guinea Port Moresby Stock Exchange XPOM 2017-08-16 -170 South America Paraguay Bolsa de Valores & Productos de Asuncíon XVPA 2017-08-16 -171 South America Peru Bolsa de Valores de Lima XLIM 2017-08-16 -172 Asia Philippines Philippine Stock Exchange XPHS 2017-08-16 -173 Eastern Europe Poland NewConnect XNCO 2017-08-16 -174 Eastern Europe Poland Warsaw Stock Exchange XWAR 2017-08-16 -175 Western Europe Portugal Euronext Lisbon XLIS 2017-08-16 -176 Western Europe Portugal OPEX OPEX 2017-08-16 -177 Middle East Qatar Qatar Stock Exchange DSMD 2017-08-16 -178 Eastern Europe Romania Bucharest Stock Exchange XRAS 2017-08-16 -179 Eastern Europe Russia Moscow Exchange MISX 2017-08-16 -180 Eastern Europe Russia Saint Petersburg Stock Exchange XPET 2017-08-16 -181 Eastern Europe Russia Siberian Exchange XSIB 2017-08-16 -182 Africa Rwanda Rwanda Stock Exchange RSEX 2017-08-16 -183 North America Saint Kitts and Nevis Eastern Caribbean Securities Exchange XECS 2017-08-16 -184 Middle East Saudi Arabia Saudi Stock Exchange XSAU 2017-08-16 -185 Eastern Europe Serbia Belgrade Stock Exchange XBEL 2017-08-16 -186 Africa Seychelles Seychelles Securities Exchange (Trop-X) TRPX 2017-08-16 -187 Asia Singapore Singapore Exchange XSES 2017-08-16 -188 Eastern Europe Slovakia Bratislava Stock Exchange XBRA 2017-08-16 -189 Eastern Europe Slovenia Ljubljana Stock Exchange XLJU 2017-08-16 -190 Africa Somalia Somali Stock Exchange NULL 2017-08-16 -191 Africa South Africa A2X Markets A2XX 2017-08-16 -192 Africa South Africa Johannesburg Stock Exchange XJSE 2017-08-16 -193 Asia South Korea Korea New Exchange XKON 2017-08-16 -194 Asia South Korea Korea Stock Exchange XKRX 2017-08-16 -195 Asia South Korea KOSDAQ Securities Exchange XKOS 2017-08-16 -196 Western Europe Spain Bolsa de Bilbao XBIL 2017-08-16 -197 Western Europe Spain Bolsa de Madrid XMAD 2017-08-16 -198 Western Europe Spain Bolsa de Valencia XVAL 2017-08-16 -199 Western Europe Spain Borsa de Barcelona XBAR 2017-08-16 -200 Western Europe Spain Latibex XLAT 2017-08-16 -201 Asia Sri Lanka Colombo Stock Exchange XCOL 2017-08-16 -202 Africa Sudan Khartoum Stock Exchange XKHA 2017-08-16 -203 Africa Swaziland Swaziland Stock Exchange XSWA 2017-08-16 -204 Western Europe Sweden Aktietorget XSAT 2017-08-16 -205 Western Europe Sweden Nasdaq Stockholm XSTO 2017-08-16 -206 Western Europe Sweden Nordic Growth Market XNGM 2017-08-16 -207 Western Europe Switzerland Berne eXchange XBRN 2017-08-16 -208 Western Europe Switzerland SIX Swiss Exchange XSWX 2017-08-16 -209 Middle East Syria Damascus Securities Exchange XDSE 2017-08-16 -210 Asia Taiwan Taipei Exchange ROCO 2017-08-16 -211 Asia Taiwan Taiwan Stock Exchange XTAI 2017-08-16 -212 Africa Tanzania Dar-es-Salaam Stock Exchange XDAR 2017-08-16 -213 Asia Thailand Stock Exchange of Thailand XBKK 2017-08-16 -214 North America Trinidad and Tobago Trinidad and Tobago Stock Exchange XTRN 2017-08-16 -215 Africa Tunisia Bourse de Tunis XTUN 2017-08-16 -216 Eastern Europe Turkey Borsa İstanbul XIST 2017-08-16 -217 Africa Uganda Uganda Securities Exchange XUGA 2017-08-16 -218 Eastern Europe Ukraine East European Stock Exchange EESE 2017-08-16 -219 Eastern Europe Ukraine PFTS Ukraine Stock Exchange PFTS 2017-08-16 -220 Eastern Europe Ukraine Stock Exchange Perspectiva SEPE 2017-08-16 -221 Eastern Europe Ukraine Ukrainian Exchange UKEX 2017-08-16 -222 Middle East United Arab Emirates Abu Dhabi Securities Market XADS 2017-08-16 -223 Middle East United Arab Emirates Dubai Financial Market XDFM 2017-08-16 -224 Middle East United Arab Emirates Nasdaq Dubai DIFX 2017-08-16 -225 Western Europe United Kingdom Aquis Exchange AQXE 2017-08-16 -226 Western Europe United Kingdom Asset Match AMPX 2017-08-16 -227 Western Europe United Kingdom London Stock Exchange XLON 2017-08-16 -228 Western Europe United Kingdom NEX NEXS 2017-08-16 -229 Western Europe United Kingdom Turquoise TRQX 2017-08-16 -230 North America United States of America Bats BYX Exchange BYXD 2017-08-16 -231 North America United States of America Bats EDGA Exchange EDGA 2017-08-16 -232 North America United States of America Bats US BATS 2017-08-16 -233 North America United States of America BatsEDGX Exchange EDGX 2017-08-16 -234 North America United States of America Chicago Stock Exchange XCHI 2017-08-16 -235 North America United States of America Investors Exchange IEXG 2017-08-16 -236 North America United States of America NASDAQ XNAS 2017-08-16 -237 North America United States of America New York Stock Exchange XNYS 2017-08-16 -238 North America United States of America North American Derivatives Exchange NADEX HEGX 2017-08-16 -239 South America Uruguay Bolsa de Valores de Montevideo XMNT 2017-08-16 -240 South America Uruguay Bolsa Electronica de Valores de Uruguay BVUR 2017-08-16 -241 Asia Uzbekistan Tashkent Stock Exchange XSTE 2017-08-16 -242 Asia Vietnam Hanoi Stock Exchange HSTC 2017-08-16 -243 Asia Vietnam Ho Chi Minh Stock Exchange XSTC 2017-08-16 -244 Africa Zambia Lusaka Stock Exchange XLUS 2017-08-16 -245 Africa Zimbabwe Zimbabwe Stock Exchange XZIM 2017-08-16 -246 Eastern Europe Albania Albanian Securities Exchange XALS 2019-11-17 -247 North America United States of America Long-Term Stock Exchange LTSE 2020-09-14 -248 North America United States of America Miami International Securities Exchange MIHI 2020-09-24 -249 North America United States of America Members' Exchange NULL 2020-09-24 -250 Africa Zimbabwe Victoria Falls Stock Exchange NULL 2020-11-01 -251 Asia China Beijing Stock Exchange NULL 2021-12-27 diff --git a/test/sql/copy/csv/test_csv_httpfs.test_slow b/test/sql/copy/csv/test_csv_httpfs.test_slow deleted file mode 100644 index 44066653e285..000000000000 --- a/test/sql/copy/csv/test_csv_httpfs.test_slow +++ /dev/null @@ -1,33 +0,0 @@ -# name: test/sql/copy/csv/test_csv_httpfs.test_slow -# description: This test triggers the http prefetch mechanism. -# group: [csv] - -statement ok -pragma enable_verification; - -require httpfs - -require parquet - -#FIXME: remote changed? -mode skip - -# Add test for 3731 -query I -SELECT count(*) FROM read_csv_auto('https://datasets.imdbws.com/name.basics.tsv.gz', delim='\t', quote='') ----- -12783090 - -query I - copy ( - SELECT * - REPLACE ( - str_split(primaryProfession,',') as primaryProfession, - str_split(knownForTitles,',') as knownForTitles, - case WHEN regexp_matches(deathYear,'[0-9]+') THEN CAST(deathYear as integer) END as deathYear, - case WHEN regexp_matches(birthYear,'[0-9]+') THEN CAST(birthYear as integer) END as birthYear - ) - FROM read_csv_auto('https://datasets.imdbws.com/name.basics.tsv.gz', delim='\t', quote='') - ) to '__TEST_DIR__/name_basics.parquet' (FORMAT 'parquet', CODEC 'ZSTD') ----- -12783090 diff --git a/test/sql/copy/csv/test_csv_httpfs_prepared.test b/test/sql/copy/csv/test_csv_httpfs_prepared.test deleted file mode 100644 index 43362c033f61..000000000000 --- a/test/sql/copy/csv/test_csv_httpfs_prepared.test +++ /dev/null @@ -1,50 +0,0 @@ -# name: test/sql/copy/csv/test_csv_httpfs_prepared.test -# description: CSV Reading From HTTPFS in Prepared Statements -# group: [csv] - -require httpfs - -statement ok -PRAGMA enable_verification - -statement ok -PREPARE boaz_bug AS from read_csv_auto('https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv') order by 1 - -query ITIIIIITTTTIIITTTI -EXECUTE boaz_bug ----- -1 AAAAAAAABAAAAAAA 980124 7135 32946 2452238 2452208 Mr. Javier Lewis Y 9 12 1936 CHILE NULL Javier.Lewis@VFAxlnZEvOx.org 2452508 -2 AAAAAAAACAAAAAAA 819667 1461 31655 2452318 2452288 Dr. Amy Moses Y 9 4 1966 TOGO NULL Amy.Moses@Ovk9KjHH.com 2452318 -3 AAAAAAAADAAAAAAA 1473522 6247 48572 2449130 2449100 Miss Latisha Hamilton Y 18 9 1979 NIUE NULL Latisha.Hamilton@V.com 2452313 -4 AAAAAAAAEAAAAAAA 1703214 3986 39558 2450030 2450000 Dr. Michael White Y 7 6 1983 MEXICO NULL Michael.White@i.org 2452361 -5 AAAAAAAAFAAAAAAA 953372 4470 36368 2449438 2449408 Sir Robert Moran N 8 5 1956 FIJI NULL Robert.Moran@Hh.edu 2452469 -6 AAAAAAAAGAAAAAAA 213219 6374 27082 2451883 2451853 Ms. Brunilda Sharp Y 4 12 1925 SURINAME NULL Brunilda.Sharp@T3pylZEUQjm.org 2452430 -7 AAAAAAAAHAAAAAAA 68377 3219 44814 2451438 2451408 Ms. Fonda Wiles N 24 4 1985 GAMBIA NULL Fonda.Wiles@S9KnyEtz9hv.org 2452360 -8 AAAAAAAAIAAAAAAA 1215897 2471 16598 2449406 2449376 Sir Ollie Shipman N 26 12 1938 KOREA, REPUBLIC OF NULL Ollie.Shipman@be.org 2452334 -9 AAAAAAAAJAAAAAAA 1168667 1404 49388 2452275 2452245 Sir Karl Gilbert N 26 10 1966 MONTSERRAT NULL Karl.Gilbert@Crg5KyP2IxX9C4d6.edu 2452454 -10 AAAAAAAAKAAAAAAA 1207553 5143 19580 2451353 2451323 Ms. Albert Brunson N 15 10 1973 JORDAN NULL Albert.Brunson@62.com 2452641 - -statement ok -DEALLOCATE boaz_bug - -statement error -EXECUTE boaz_bug ----- -Prepared statement "boaz_bug" does not exist - -# Recreate prepared statement with different file - -#FIXME: FILE changed? -mode skip - -statement ok -PREPARE boaz_bug AS SELECT * from read_csv_auto('https://www.data.gouv.fr/fr/datasets/r/6d186965-f41b-41f3-9b23-88241cc6890c') order by all limit 5; - -query ITTRRR -EXECUTE boaz_bug ----- -2020 Allemagne Germany 26.1 53196.069 200601.2 -2020 Autriche Austria 18.0 4723.5 26215.8 -2020 Belgique Belgium 28.999999999999996 9436.1 32553.0 -2020 Bulgarie Bulgaria 11.600000000000001 1124.1 9698.7 -2020 Chypre Cyprus 0.0 0.0 1627.6 diff --git a/test/sql/copy/csv/test_csv_remote.test b/test/sql/copy/csv/test_csv_remote.test deleted file mode 100644 index 0555b6e51314..000000000000 --- a/test/sql/copy/csv/test_csv_remote.test +++ /dev/null @@ -1,34 +0,0 @@ -# name: test/sql/copy/csv/test_csv_remote.test -# description: Test reading csv files over http -# group: [csv] - -require httpfs - -statement ok -PRAGMA enable_verification - - -# regular csv file -query ITTTIITITTIIII nosort webpagecsv -SELECT * FROM read_csv_auto('data/csv/real/web_page.csv') ORDER BY 1; ----- - -# file with gzip -query IIIIIIIIIIIIIII nosort lineitemcsv -SELECT * FROM read_csv_auto('data/csv/lineitem1k.tbl.gz') ORDER BY ALL; ----- - -query ITTTIITITTIIII nosort webpagecsv -SELECT * FROM read_csv_auto('https://raw.githubusercontent.com/duckdb/duckdb/main/data/csv/real/web_page.csv') ORDER BY 1; ----- - -query IIIIIIIIIIIIIII nosort lineitemcsv -select * from read_csv_auto('https://raw.githubusercontent.com/duckdb/duckdb/main/data/csv/lineitem1k.tbl.gz') ORDER BY ALL; ----- - - -# Test load from url with query string -query IIIIIIIIIIII -FROM sniff_csv('https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv?v=1') ----- -, " (empty) \n (empty) 0 0 [{'name': column00, 'type': BIGINT}, {'name': column01, 'type': VARCHAR}, {'name': column02, 'type': BIGINT}, {'name': column03, 'type': BIGINT}, {'name': column04, 'type': BIGINT}, {'name': column05, 'type': BIGINT}, {'name': column06, 'type': BIGINT}, {'name': column07, 'type': VARCHAR}, {'name': column08, 'type': VARCHAR}, {'name': column09, 'type': VARCHAR}, {'name': column10, 'type': VARCHAR}, {'name': column11, 'type': BIGINT}, {'name': column12, 'type': BIGINT}, {'name': column13, 'type': BIGINT}, {'name': column14, 'type': VARCHAR}, {'name': column15, 'type': VARCHAR}, {'name': column16, 'type': VARCHAR}, {'name': column17, 'type': BIGINT}] NULL NULL NULL FROM read_csv('https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv?v=1', auto_detect=false, delim=',', quote='"', escape='', new_line='\n', skip=0, comment='', header=false, columns={'column00': 'BIGINT', 'column01': 'VARCHAR', 'column02': 'BIGINT', 'column03': 'BIGINT', 'column04': 'BIGINT', 'column05': 'BIGINT', 'column06': 'BIGINT', 'column07': 'VARCHAR', 'column08': 'VARCHAR', 'column09': 'VARCHAR', 'column10': 'VARCHAR', 'column11': 'BIGINT', 'column12': 'BIGINT', 'column13': 'BIGINT', 'column14': 'VARCHAR', 'column15': 'VARCHAR', 'column16': 'VARCHAR', 'column17': 'BIGINT'}); diff --git a/test/sql/copy/csv/test_csv_remote.test_slow b/test/sql/copy/csv/test_csv_remote.test_slow deleted file mode 100644 index 4df0fc6e9a26..000000000000 --- a/test/sql/copy/csv/test_csv_remote.test_slow +++ /dev/null @@ -1,15 +0,0 @@ -# name: test/sql/copy/csv/test_csv_remote.test_slow -# description: Test reading csv files over http, slow queries -# group: [csv] - -statement ok -pragma enable_verification; - -require httpfs - -# Read a compressed file (~44MB compressed, ~700MB uncompressed) over HTTP -query IIIIII -select count(*), min(strain), max(strain), min(strlen(sequence)), max(strlen(sequence)), avg(strlen(sequence)) -from read_csv_auto('https://raw.githubusercontent.com/duckdb/duckdb/main/data/csv/sequences.csv.gz', delim=','); ----- -100000 ARG/Cordoba-1006-155/2020 tiger/NY/040420/2020 17340 30643 29821.264410 diff --git a/test/sql/copy/csv/test_mixed_lines.test_slow b/test/sql/copy/csv/test_mixed_lines.test_slow deleted file mode 100644 index 9da5ceaba3dd..000000000000 --- a/test/sql/copy/csv/test_mixed_lines.test_slow +++ /dev/null @@ -1,32 +0,0 @@ -# name: test/sql/copy/csv/test_mixed_lines.test_slow -# description: Test mixed lines -# group: [csv] - -require httpfs - -statement ok -pragma enable_verification; - -statement error -from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/202204-clean-sane-header.csv' ----- -* Disable the parser's strict mode (strict_mode=false) to allow reading rows that do not comply with the CSV standard. - -query I -select count(*) from read_csv('https://github.com/duckdb/duckdb-data/releases/download/v1.0/202204-clean-sane-header.csv', strict_mode=false) ----- -50000 - -query IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII -from read_csv('https://github.com/duckdb/duckdb-data/releases/download/v1.0/202204-clean-sane-header.csv', strict_mode=false) order by all limit 10 ----- -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaa-aaaaaaaaaaaaaaaaa a1a-11111 a1111111-1aaa-11a1-a1a1-aa1a111111a1 aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaaaaaaa 1a1111 1a111111111111111 1a111111111111111 1a11111 aaa aaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaa-aa-aaa-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaa NULL NULL NULL aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa NULL aaaaaaa aaaaaaa 11a aa-aa-aaa-aaaaaa NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaa -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaa-aaaaaaaaaaaaaaaaa a1a-11111 a1111111-1aaa-11a1-a1a1-aa1a111111a1 aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaaaaaaa 1a1111 1a111111111111111 1a111111111111111 1a11111 aaa aaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaa-aaaa-aaaaaaaa-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaa NULL NULL NULL aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa NULL aaaaaaa aaaaaaa 11a aa-aaaa-aaaaaaaa-aaa NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaa -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaa-aaaaaaaaaaaaaaaaa a1a-11111 a11a11a1-111a-11a1-1a1a-1aa11a111111 aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaaaaaaa 1a1111 1a111111111111111 1a11111111111111 1a11111 aaa aaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa11 aaaaaaaaaaaaaaaaa11 NULL NULL NULL NULL NULL aaaaaaa aaaaaaa 11a aaaaaaaaaa NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaa -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaaaaaaaaaaaaa a1a-11111 111aaaa1-111a-1111-a1aa-111aa11aa111 aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaa 1a1111 1a111111111111111 1a111111111111111 1a11111 aaa aaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaa-aaaa-aaaaaaaa-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaa NULL NULL NULL aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa NULL aaaaaaa aaaaaaa 11a aa-aaaa-aaaaaaaa-aaa NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaa -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaaaaaaaaaaaaa a1a-11111 111aaaa1-111a-1111-a1aa-111aa11aa111 aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaa 1a1111 1a111111111111111 1a111111111111111 1a11111 aaa aaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaa-aaaa-aaaaaaaa-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaa NULL NULL NULL aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa NULL aaaaaaa aaaaaaa 11a aa-aaaa-aaaaaaaa-aaa NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaa -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaaaaaaaaaaaaa a1a-11111 111aaaa1-111a-1111-a1aa-111aa11aa111 aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaa 1a1111 1a111111111111111 1a111111111111111 1a11111 aaa aaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa11 aaaaaaaaaaaaaaaaa11 NULL NULL NULL NULL NULL aaaaaaa aaaaaaa 11a aaaaaaaaaa NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaa -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaaaaaaaaaaaaa a1a-11111 111aaaa1-111a-1111-a1aa-111aa11aa111 aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaa 1a1111 1a111111111111111 1a111111111111111 1a11111 aaa aaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa11 aaaaaaaaaaaaaaaaa11 NULL NULL NULL NULL NULL aaaaaaa aaaaaaa 11a aaaaaaaaaa NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaa -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaaaaaaaaaaaaa a1a-11111 111aaaa1-111a-1111-a1aa-111aa11aa111 aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaa 1a1111 1a111111111111111 1a111111111a-11 1a11111 aaa aaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaa-aaa-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaa NULL NULL NULL NULL NULL aaaaaaa aaaaaaa 11a aa-aaa-aaa NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaa -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaaaaaaaaaaaaa a1a-11111 aa1a111a-a1a1-1aa1-a1aa-1a1aa1aa11aa aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaa 1a111111 1a111111111111111 1a111111111111111 1a1111 aaa aaaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaa-a1a-aaa1aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1111111aaaa11a1aa1aaaa aa1111111aaaa11a1aa1aaaa NULL NULL NULL aaa-aaaaaaaa-aaaaaaaaaaaaaa-aaaaa-aaaaaa NULL aaaaaaa aaaaaaa 1aaaaaaaaa aa-a1a-aaa1 NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaaaa -11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 1111-11-11 11111111 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaa111111aaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaa 1111a1aa-1111-1111-aa11-11aa11a11111 aaaaaaaaaaaaaaaaaaaaaaaaaa 1111-11-11 aaaaaa-aaaaaaaaaaaaaaaa a1a-11111 aa1a111a-a1a1-1aa1-a1aa-1a1aa1aa11aa aaaaaaa aaaaaaa aaaaa NULL aaaaaaaaaaaaaaa 1a111111 1a111111111111111 1a111111111111111 1a1111 aaa aaaaaaaaaaaaa NULL aaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa1111a1aa-1111-1111-aa11-11aa11a11111aaaaaaaaaaaaaaaaaa-a1a-aaa1aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1111111aaaa11a1aa1aaaa aa1111111aaaa11a1aa1aaaa NULL NULL NULL aaa-aaaaaaaa-aaaaaaaaaaaaaa-aaaaa-aaaaaa NULL aaaaaaa aaaaaaa 1aaaaaaaaa aa-a1a-aaa1 NULL NULL NULL NULL aa-aaa-1111a aaaa NULL NULL NULL aaaaa aaaaaaaaaa aaaaa 1a1 aaaaaaaa NULL NULL NULL aaaaaaaaaaaa diff --git a/test/sql/copy/csv/test_null_padding_quoted_new_line.test_slow b/test/sql/copy/csv/test_null_padding_quoted_new_line.test_slow new file mode 100644 index 000000000000..ad6e26a0ab38 --- /dev/null +++ b/test/sql/copy/csv/test_null_padding_quoted_new_line.test_slow @@ -0,0 +1,23 @@ +# name: test/sql/copy/csv/test_null_padding_quoted_new_line.test_slow +# description: Test CSV with nullpading interoping with quoted new line +# group: [csv] + +statement ok +PRAGMA enable_verification + +statement ok +CREATE TABLE T as SELECT 'this line +has a new line' from range (1000000); + +statement ok +COPY T TO '__TEST_DIR__/test_null_padding_quoted_new_line.csv' (FORMAT CSV , FORCE_QUOTE *); + +statement error +select count(*) FROM read_csv('__TEST_DIR__/test_null_padding_quoted_new_line.csv', null_padding = true) +---- +The parallel scanner does not support null_padding in conjunction with quoted new lines. Please disable the parallel csv reader with parallel=false + +query I +select count(*) FROM read_csv('__TEST_DIR__/test_null_padding_quoted_new_line.csv') +---- +1000000 \ No newline at end of file diff --git a/test/sql/copy/csv/test_sniff_httpfs.test b/test/sql/copy/csv/test_sniff_httpfs.test deleted file mode 100644 index ea36b9be17ab..000000000000 --- a/test/sql/copy/csv/test_sniff_httpfs.test +++ /dev/null @@ -1,14 +0,0 @@ -# name: test/sql/copy/csv/test_sniff_httpfs.test -# description: Test sniff_csv functions over httpfs with auto-detection on compression -# group: [csv] - -require httpfs - -statement ok -PRAGMA enable_verification - -statement ok -from sniff_csv('https://github.com/duckdb/duckdb/raw/main/data/csv/who.csv.gz'); - -statement ok -from sniff_csv('https://github.com/duckdb/duckdb/raw/main/data/csv/who.csv.gz?v=1'); diff --git a/test/sql/copy/csv/test_url_with_plus.test b/test/sql/copy/csv/test_url_with_plus.test deleted file mode 100644 index cede25bb31dd..000000000000 --- a/test/sql/copy/csv/test_url_with_plus.test +++ /dev/null @@ -1,11 +0,0 @@ -# name: test/sql/copy/csv/test_url_with_plus.test -# description: Tests url with plus -# group: [csv] - -require httpfs - -statement ok -PRAGMA enable_verification - -statement ok -FROM read_csv('https://d37ci6vzurychx.cloudfront.net/misc/taxi+_zone_lookup.csv'); diff --git a/test/sql/copy/encryption/different_aes_ciphers.test b/test/sql/copy/encryption/different_aes_ciphers.test index a2f39754e2f0..ca147d7644f8 100644 --- a/test/sql/copy/encryption/different_aes_ciphers.test +++ b/test/sql/copy/encryption/different_aes_ciphers.test @@ -12,4 +12,74 @@ Binder Error: Not a valid key. A key cannot be empty statement error ATTACH '__TEST_DIR__/encrypted.duckdb' AS encrypted (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER 'random'); ---- -Binder Error: "random" is not a valid cipher. Only AES GCM is supported. \ No newline at end of file +Binder Error: "random" is not a valid cipher. Try 'GCM' or 'CTR'. + +statement error +ATTACH '__TEST_DIR__/encrypted.duckdb' AS encrypted (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER ''); +---- +Binder Error: "" is not a valid cipher. Try 'GCM' or 'CTR'. + +statement error +ATTACH '__TEST_DIR__/encrypted.duckdb' AS encrypted (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER 42); +---- +Binder Error: "42" is not a valid cipher. Try 'GCM' or 'CTR'. + +foreach cipher GCM CTR + +statement ok +ATTACH '__TEST_DIR__/encrypted_${cipher}.duckdb' AS encrypted_${cipher} (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); + +query I +select encrypted from duckdb_databases() where database_name = 'encrypted_${cipher}' and cipher='${cipher}'; +---- +true + +endloop + + +# we can create a database with a specific cipher (CTR) +statement ok +ATTACH '__TEST_DIR__/encrypted_default_cipher.duckdb' AS encrypted (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER 'CTR'); + +statement ok +create table encrypted.fuu as select 42; + +statement ok +DETACH encrypted + +# we can open it again by specifying that same cipher again +statement ok +ATTACH '__TEST_DIR__/encrypted_default_cipher.duckdb' AS encrypted (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER 'CTR'); + +query I +FROM encrypted.fuu +---- +42 + +statement ok +DETACH encrypted + +# or open it without specifying the cipher, it will be read from file +statement ok +ATTACH '__TEST_DIR__/encrypted_default_cipher.duckdb' AS encrypted (ENCRYPTION_KEY 'asdf'); + +query I +FROM encrypted.fuu +---- +42 + +statement ok +DETACH encrypted + +# but it will fail if we specify the wrong one +statement error +ATTACH '__TEST_DIR__/encrypted_default_cipher.duckdb' AS encrypted (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER 'GCM'); +---- +with a different cipher (GCM) than the one used to create it (CTR) + + +# CBC is disabled (for now) +statement error +ATTACH '__TEST_DIR__/CBC.duckdb' AS encrypted (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER 'CBC'); +---- +CBC encryption is disabled diff --git a/test/sql/copy/parquet/delta_byte_array_length_mismatch.test b/test/sql/copy/parquet/delta_byte_array_length_mismatch.test deleted file mode 100644 index 320c76f5ce31..000000000000 --- a/test/sql/copy/parquet/delta_byte_array_length_mismatch.test +++ /dev/null @@ -1,10 +0,0 @@ -# name: test/sql/copy/parquet/delta_byte_array_length_mismatch.test -# description: Test reading a delta -# group: [parquet] - -require parquet - -require httpfs - -statement ok -SELECT * FROM parquet_scan('https://github.com/duckdb/duckdb-data/releases/download/v1.0/delta_byte_array_length_mismatch.parquet') diff --git a/test/sql/copy/parquet/delta_byte_array_multiple_pages.test b/test/sql/copy/parquet/delta_byte_array_multiple_pages.test deleted file mode 100644 index 0e04ca58c09d..000000000000 --- a/test/sql/copy/parquet/delta_byte_array_multiple_pages.test +++ /dev/null @@ -1,23 +0,0 @@ -# name: test/sql/copy/parquet/delta_byte_array_multiple_pages.test -# description: Test delta byte array parquet file with multiple pages -# group: [parquet] - -require parquet - -require httpfs - -statement ok -CREATE TABLE delta_byte_array AS SELECT * FROM parquet_scan('https://github.com/duckdb/duckdb-data/releases/download/v1.0/delta_byte_array_multiple_pages.parquet') - -query I -SELECT COUNT(*) FROM delta_byte_array ----- -100000 - -query II -SELECT min(strlen(json_column)), max(strlen(json_column)) FROM delta_byte_array ----- -54 54 - - - diff --git a/test/sql/copy/parquet/incorrect_converted_type.test b/test/sql/copy/parquet/incorrect_converted_type.test index 8089977ff7f6..72c637183d93 100644 --- a/test/sql/copy/parquet/incorrect_converted_type.test +++ b/test/sql/copy/parquet/incorrect_converted_type.test @@ -7,44 +7,54 @@ require parquet statement error SELECT * FROM 'data/parquet-testing/broken/broken_bigint.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_date.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_int.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_smallint.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_timestamp.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_timestamp_ms.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_tinyint.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_ubigint.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_uinteger.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_usmallint.parquet'; ---- +:.*IO Error.*converted type.* statement error SELECT * FROM 'data/parquet-testing/broken/broken_utinyint.parquet'; ---- - +:.*IO Error.*converted type.* \ No newline at end of file diff --git a/test/sql/copy/parquet/parquet_2102.test_slow b/test/sql/copy/parquet/parquet_2102.test_slow deleted file mode 100644 index 5abce39bd56b..000000000000 --- a/test/sql/copy/parquet/parquet_2102.test_slow +++ /dev/null @@ -1,69 +0,0 @@ -# name: test/sql/copy/parquet/parquet_2102.test_slow -# description: Missing Column Data After Adding Left Join To Query in DuckDB Version 0.2.8 -# group: [parquet] - -require parquet - -require httpfs - -statement ok -CREATE TABLE view_one AS SELECT * FROM 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/issue2102_one.parquet'; - -statement ok -CREATE TABLE view_two AS SELECT * FROM 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/issue2102_two.parquet'; - -query I -SELECT COUNT(*) FROM view_one WHERE date IS NULL ----- -6219 - -statement ok -CREATE TABLE tbl1 AS SELECT one.id id, one.date date -FROM - view_one AS one -JOIN - view_two two ON two.id = one.id AND two.line = 1; - -query I -SELECT COUNT(*) FROM tbl1 ----- -691951 - -query I -SELECT COUNT(*) FROM tbl1 WHERE date IS NULL ----- -4742 - -statement ok -CREATE TABLE tbl2 AS SELECT one.id id, one.date date -FROM - view_one AS one -LEFT JOIN - view_two two ON two.id = one.id AND two.line = 1; - -query I -SELECT COUNT(*) FROM tbl2 ----- -695434 - -query I -SELECT COUNT(*) FROM tbl2 WHERE date IS NULL ----- -6219 - -statement ok -CREATE TABLE tbl3 AS SELECT one.id id, one.date date -FROM - view_one AS one -LEFT JOIN - view_two two ON two.id = one.id; - -query I -SELECT COUNT(*) FROM tbl3 ----- -768666 - -query I -SELECT COUNT(*) FROM tbl3 WHERE date IS NULL ----- -7124 diff --git a/test/sql/copy/parquet/parquet_4903.test b/test/sql/copy/parquet/parquet_4903.test index 3ed54c3fb854..59f7ec2f51cf 100644 --- a/test/sql/copy/parquet/parquet_4903.test +++ b/test/sql/copy/parquet/parquet_4903.test @@ -8,3 +8,4 @@ require parquet statement error SELECT type_param_constraints FROM 'data/parquet-testing/bug4903.parquet' limit 10 ---- +:.*Binder Error.*not found in FROM clause.* \ No newline at end of file diff --git a/test/sql/copy/parquet/parquet_5968.test b/test/sql/copy/parquet/parquet_5968.test deleted file mode 100644 index ed6f625d04cb..000000000000 --- a/test/sql/copy/parquet/parquet_5968.test +++ /dev/null @@ -1,24 +0,0 @@ -# name: test/sql/copy/parquet/parquet_5968.test -# description: Issue #5968: Segmentation fault on reading parquet file -# group: [parquet] - -require parquet - -require httpfs - -statement ok -CREATE TABLE issue_5968 AS FROM 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/issue_5968.parquet'; - -query I -SELECT COUNT(*) FROM issue_5968 ----- -2028587 - -query I -SELECT * FROM issue_5968 LIMIT 5 ----- -B00001 -B00001 -B00009 -B00009 -B00009 diff --git a/test/sql/copy/parquet/parquet_boolean_page.test_slow b/test/sql/copy/parquet/parquet_boolean_page.test_slow deleted file mode 100644 index ca305dff9f21..000000000000 --- a/test/sql/copy/parquet/parquet_boolean_page.test_slow +++ /dev/null @@ -1,21 +0,0 @@ -# name: test/sql/copy/parquet/parquet_boolean_page.test_slow -# description: Test that boolean values that cross column pages are correctly read -# group: [parquet] - -require parquet - -require httpfs - -statement ok -PRAGMA enable_verification - -query IIIII -SELECT - SUM(CASE WHEN is_successful THEN 1 ELSE 0 END), - SUM(CASE WHEN advanced_on_error_flag THEN 1 ELSE 0 END), - SUM(CASE WHEN safe_on_error_flag THEN 1 ELSE 0 END), - SUM(CASE WHEN rbi_flag THEN 1 ELSE 0 END), - SUM(CASE WHEN team_unearned_flag THEN 1 ELSE 0 END) -FROM read_parquet('https://github.com/duckdb/duckdb-data/releases/download/v1.0/event_baserunning_advance_attempt.parquet'); ----- -9252616 111041 7120 1609612 1860 diff --git a/test/sql/copy/parquet/parquet_encryption_httpfs.test b/test/sql/copy/parquet/parquet_encryption_httpfs.test deleted file mode 100644 index 370d2a96990b..000000000000 --- a/test/sql/copy/parquet/parquet_encryption_httpfs.test +++ /dev/null @@ -1,66 +0,0 @@ -# name: test/sql/copy/parquet/parquet_encryption_httpfs.test -# description: Test Parquet encryption with OpenSSL -# group: [parquet] - -require parquet - -require httpfs - -# parquet keys are not persisted across restarts -statement ok -PRAGMA enable_verification - -# add keys of 3 different lengths -statement ok -PRAGMA add_parquet_key('key128', '0123456789112345') - -statement ok -PRAGMA add_parquet_key('key192', '012345678911234501234567') - -statement ok -PRAGMA add_parquet_key('key256', '01234567891123450123456789112345') - -# test all valid AES key lengths -foreach key_len 128 192 256 - -statement ok -COPY (SELECT 42 i) to '__TEST_DIR__/encrypted${key_len}_openssl.parquet' (ENCRYPTION_CONFIG {footer_key: 'key${key_len}'}) - -query I -SELECT * FROM read_parquet('__TEST_DIR__/encrypted${key_len}_openssl.parquet', encryption_config={footer_key: 'key${key_len}'}) ----- -42 - -statement ok -CREATE OR REPLACE TABLE test (i INTEGER) - -statement ok -COPY test FROM '__TEST_DIR__/encrypted${key_len}_openssl.parquet' (ENCRYPTION_CONFIG {footer_key: 'key${key_len}'}) - -query I -SELECT * FROM test ----- -42 - -endloop - -# what happens if we don't try to decrypt even if the file is encrypted? -statement error -SELECT * FROM read_parquet('__TEST_DIR__/encrypted128_openssl.parquet') ----- -Invalid Input Error - -# what if we try to decrypt with the wrong key? -statement error -SELECT * FROM read_parquet('__TEST_DIR__/encrypted128_openssl.parquet', encryption_config={footer_key: 'key192'}) ----- -Invalid Input Error: Computed AES tag differs from read AES tag, are you using the right key? - -# what if we don't encrypt, but try to decrypt? -statement ok -COPY (SELECT 42 i) to '__TEST_DIR__/unencrypted.parquet' - -statement error -SELECT * FROM read_parquet('__TEST_DIR__/unencrypted.parquet', encryption_config={footer_key: 'key256'}) ----- -Invalid Input Error diff --git a/test/sql/copy/parquet/parquet_encryption_mbedtls_openssl.test b/test/sql/copy/parquet/parquet_encryption_mbedtls_openssl.test deleted file mode 100644 index cc493fb371f3..000000000000 --- a/test/sql/copy/parquet/parquet_encryption_mbedtls_openssl.test +++ /dev/null @@ -1,52 +0,0 @@ -# name: test/sql/copy/parquet/parquet_encryption_mbedtls_openssl.test -# description: Test Parquet encryption with OpenSSL -# group: [parquet] - -require parquet - -require httpfs - -# parquet keys are not persisted across restarts -statement ok -PRAGMA enable_verification - -# add keys of 3 different lengths -statement ok -PRAGMA add_parquet_key('key128', '0123456789112345') - -statement ok -PRAGMA add_parquet_key('key192', '012345678911234501234567') - -statement ok -PRAGMA add_parquet_key('key256', '01234567891123450123456789112345') - -# test all valid AES key lengths -foreach key_len 128 192 256 - -# write files with OpenSSL enabled -statement error -COPY (SELECT 42 i) to '__TEST_DIR__/encrypted${key_len}_openssl.parquet' (ENCRYPTION_CONFIG {footer_key: 'key${key_len}'}, DEBUG_USE_OPENSSL randomval) ----- -BOOL - -# write files with OpenSSL enabled -statement ok -COPY (SELECT 42 i) to '__TEST_DIR__/encrypted${key_len}_openssl.parquet' (ENCRYPTION_CONFIG {footer_key: 'key${key_len}'}, DEBUG_USE_OPENSSL true) - -# read OpenSSL encrypted files by using mbedtls -query I -SELECT * FROM read_parquet('__TEST_DIR__/encrypted${key_len}_openssl.parquet', encryption_config={footer_key: 'key${key_len}'}, debug_use_openssl=false) ----- -42 - -# write files with default mbedtls -statement ok -COPY (SELECT 42 i) to '__TEST_DIR__/encrypted${key_len}_mbedtls.parquet' (ENCRYPTION_CONFIG {footer_key: 'key${key_len}'}, DEBUG_USE_OPENSSL false) - -# read mbedtls encrypted files using OpenSSL -query I -SELECT * FROM read_parquet('__TEST_DIR__/encrypted${key_len}_mbedtls.parquet', encryption_config={footer_key: 'key${key_len}'}, debug_use_openssl=true) ----- -42 - -endloop diff --git a/test/sql/copy/parquet/parquet_external_access.test b/test/sql/copy/parquet/parquet_external_access.test index 48ba632b02ce..b0c4593b36ec 100644 --- a/test/sql/copy/parquet/parquet_external_access.test +++ b/test/sql/copy/parquet/parquet_external_access.test @@ -15,36 +15,44 @@ SET enable_external_access=false; statement error SELECT * FROM 'data/parquet-testing/arrow/lineitem-arrow.parquet' ---- +:.*Permission Error: Cannot access file.* # or their metadata statement error SELECT * FROM parquet_metadata('data/parquet-testing/arrow/lineitem-arrow.parquet') ---- +:.*Permission Error: Cannot access file.* statement error SELECT * FROM parquet_schema('data/parquet-testing/arrow/lineitem-arrow.parquet') ---- +:.*Permission Error: Cannot access file.* # also not in a list statement error SELECT * FROM parquet_scan(['data/parquet-testing/arrow/lineitem-arrow.parquet', 'data/parquet-testing/arrow/lineitem-arrow.parquet']) ---- +:.*Permission Error: Cannot access file.* # neither can we glob statement error SELECT * FROM glob('data/parquet-testing/arrow/lineitem-arrow.parquet') ---- +:.*Permission Error: Cannot access file.* # or copy to/from... statement error COPY lineitem FROM 'data/parquet-testing/arrow/lineitem-arrow.parquet' ---- +:.*Permission Error: Cannot access file.* statement error COPY lineitem TO '__TEST_DIR__/lineitem.parquet' ---- +:.*Permission Error: Cannot access file.* # we also can't just enable external access again statement error SET enable_external_access=true; ---- +:.*Invalid Input Error: Cannot change.*while database is running.* diff --git a/test/sql/copy/parquet/parquet_glob.test b/test/sql/copy/parquet/parquet_glob.test index 2137aba8358a..dd898cbff6df 100644 --- a/test/sql/copy/parquet/parquet_glob.test +++ b/test/sql/copy/parquet/parquet_glob.test @@ -69,11 +69,13 @@ FROM parquet_scan('data/parquet-testing/glob3/*/dir/*.parquet'); statement error select count(*) from parquet_scan('') ---- +:.*IO Error.*can only be set for.* # schema mismatch in parquet glob statement error select * from parquet_scan('data/parquet-testing/*.parquet') ---- +:.*Invalid Input Error: Failed to read file.* # parquet glob with COPY FROM statement ok @@ -95,3 +97,4 @@ CREATE TABLE vals2 (i INTEGER, j INTEGER) statement error COPY vals2 FROM '*/sql/*/parquet/*/glob/t?.parquet' (FORMAT PARQUET); ---- +:.*IO Error: No files found that match the pattern.* \ No newline at end of file diff --git a/test/sql/copy/parquet/parquet_glob_s3.test b/test/sql/copy/parquet/parquet_glob_s3.test deleted file mode 100644 index ea5df0c22e1e..000000000000 --- a/test/sql/copy/parquet/parquet_glob_s3.test +++ /dev/null @@ -1,186 +0,0 @@ -# name: test/sql/copy/parquet/parquet_glob_s3.test -# description: Test basic globbing of parquet files over s3 -# group: [parquet] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# this test was written before we implemented the external file cache -# when it is enabled, the request counts are different -# we disable it so this test still makes sense -statement ok -set enable_external_file_cache=false; - -# Copy files to S3 before beginning tests -statement ok -COPY (select * from 'data/parquet-testing/glob/t1.parquet') to 's3://test-bucket/parquet_glob_s3/glob/t1.parquet'; -COPY (select * from 'data/parquet-testing/glob/t2.parquet') to 's3://test-bucket/parquet_glob_s3/glob/t2.parquet'; -COPY (select * from 'data/parquet-testing/glob2/t1.parquet') to 's3://test-bucket/parquet_glob_s3/glob2/t1.parquet'; -COPY (select * from 'data/parquet-testing/glob/t1.parquet') to 's3://test-bucket/parquet_glob_s3/with+plus/t1.parquet'; -COPY (select * from 'data/parquet-testing/glob/t1.parquet') to 's3://test-bucket/parquet_glob_s3/with space/t1.parquet'; - -# parquet glob with COPY FROM -statement ok -CREATE TABLE vals (i INTEGER, j BLOB) - -statement ok -COPY vals FROM 's3://test-bucket/parquet_glob_s3/glob/t[0-9].parquet' (FORMAT PARQUET); - -query II -SELECT * FROM vals ORDER BY 1, 2 ----- -1 a -2 b - -# failed to copy: incorrect types found in parquet file -statement ok -CREATE TABLE vals2 (i INTEGER, j INTEGER) - -statement error -COPY vals2 FROM 's3://test-bucket/parquet_glob_s3/nonexistentfolderblablabla/t*.parquet' (FORMAT PARQUET); ----- - -# Test variety of urls with both url styles -foreach urlstyle path vhost - -statement ok -SET s3_url_style='${urlstyle}' - -# Begin tests -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/glob/t[0-9].parquet') ----- -2 - -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/glob/*') ----- -2 - -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/glob/*.parquet') ----- -2 - -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/g*/*.parquet') ----- -3 - -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/g*/t1.parquet') ----- -2 - -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/with*/*.parquet') ----- -2 - -# schema mismatch in parquet glob -statement error -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/notglob/*.parquet') ----- - -# parallel testing -statement ok -PRAGMA threads=4 - -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/glob/*') ----- -2 - -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/glob/*.parquet') ----- -2 - -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/g*/*.parquet') ----- -3 - -query I -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/g*/t1.parquet') ----- -2 - -# Question mark is not supported for S3 due to our use of query parameters -statement error -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/glob/t?.parquet') ----- -Invalid query parameters found. - -statement error -select count(*) from parquet_scan('s3://test-bucket/parquet_glob_s3/?lob/t?.parquet') ----- -Invalid query parameters found. - -# Finally, enabling url compatibility mode will disable globs allowing a user to query files with special chars -statement ok -SET s3_url_compatibility_mode=true; - -# Note that this is actually a file called '?.*[1-0]parquet??' which S3 should theoretically accept; -statement ok -COPY vals TO 's3://test-bucket/the_horror/?.*[1-0]parquetta??' (FORMAT parquet); - -query I -select count(*) from parquet_scan('s3://test-bucket/the_horror/?.*[1-0]parquetta??'); ----- -2 - -statement ok -SET s3_url_compatibility_mode=false; - -endloop - -# S3 glob gives us information necessary to skip HEAD requests -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM 's3://test-bucket/parquet_glob_s3/g*/*.parquet'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 0.*GET\: 3.*PUT\: 0.*\#POST\: 0.* - -statement ok -SET VARIABLE file_list = (SELECT LIST(file) FROM GLOB('s3://test-bucket/parquet_glob_s3/g*/*.parquet')) - -# sanity check for request count -# we expect 1 HEAD request per file for the open call, then 1 GET for the list call -# then for each file 1 for the metadata offset and 1 for the metadata -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM read_parquet(getvariable('file_list')); ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 3.*GET\: 3.*PUT\: 0.*\#POST\: 0.* - -statement ok -SET enable_http_metadata_cache=true; - -# metadata cache was just enabled, its still cold -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM read_parquet(getvariable('file_list')); ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 3.*GET\: 3.*PUT\: 0.*\#POST\: 0.* - -# now head request count should be 0 -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM read_parquet(getvariable('file_list')); ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 0.*GET\: 3.*PUT\: 0.*\#POST\: 0.* diff --git a/test/sql/copy/parquet/parquet_http_prefetch.test b/test/sql/copy/parquet/parquet_http_prefetch.test deleted file mode 100644 index 09303e8cbcb8..000000000000 --- a/test/sql/copy/parquet/parquet_http_prefetch.test +++ /dev/null @@ -1,41 +0,0 @@ -# name: test/sql/copy/parquet/parquet_http_prefetch.test -# description: This test triggers the http prefetch mechanism. -# group: [parquet] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -CREATE TABLE test_fetch_delay (a INT, b INT); - -statement ok -INSERT INTO test_fetch_delay (SELECT (i%2) * 2, (i%2) * 2 from range(0,2500000) as tbl(i)); - -statement ok -COPY test_fetch_delay to 's3://test-bucket/skip_delay.parquet'; - -statement ok -CREATE TABLE test as SELECT * from 's3://test-bucket/skip_delay.parquet' where a = 1; - -query I -SELECT COUNT(*) FROM test; ----- -0 diff --git a/test/sql/copy/parquet/parquet_metadata.test b/test/sql/copy/parquet/parquet_metadata.test index 2c21b0851296..78f9c5e05392 100644 --- a/test/sql/copy/parquet/parquet_metadata.test +++ b/test/sql/copy/parquet/parquet_metadata.test @@ -56,3 +56,39 @@ l_receiptdate BYTE_ARRAY VARCHAR l_shipinstruct BYTE_ARRAY VARCHAR l_shipmode BYTE_ARRAY VARCHAR l_comment BYTE_ARRAY VARCHAR + +# column_id +query II +SELECT column_id, name FROM parquet_schema('data/parquet-testing/lineitem-top10000.gzip.parquet') ORDER BY column_id; +---- +0 spark_schema +1 l_orderkey +2 l_partkey +3 l_suppkey +4 l_linenumber +5 l_quantity +6 l_extendedprice +7 l_discount +8 l_tax +9 l_returnflag +10 l_linestatus +11 l_shipdate +12 l_commitdate +13 l_receiptdate +14 l_shipinstruct +15 l_shipmode +16 l_comment + +query III +WITH per_file AS ( + SELECT file_name, COUNT(*) AS rows_per_file + FROM parquet_schema('data/parquet-testing/glob3/**/*.parquet') + GROUP BY file_name +) +SELECT + SUM(rows_per_file) AS total_rows, + MAX(rows_per_file) AS max_rows_per_filename, + (SELECT COUNT(DISTINCT column_id) FROM parquet_schema('data/parquet-testing/glob3/**/*.parquet')) AS distinct_column_ids +FROM per_file; +---- +9 3 3 diff --git a/test/sql/copy/parquet/parquet_metadata_glob.test_slow b/test/sql/copy/parquet/parquet_metadata_glob.test_slow new file mode 100644 index 000000000000..1571bc5ff591 --- /dev/null +++ b/test/sql/copy/parquet/parquet_metadata_glob.test_slow @@ -0,0 +1,22 @@ +# name: test/sql/copy/parquet/parquet_metadata_glob.test_slow +# description: Test parquet metadata function applied to many files +# group: [parquet] + +require parquet + +statement ok +set variable parquet_files = ( + select list(file) + from glob('data/parquet-testing/**.parquet') + where 'broken' not in file +) + +statement ok +SELECT * FROM parquet_metadata(getvariable('parquet_files')); + +query II +select row_group_bytes, row_group_compressed_bytes from parquet_metadata(getvariable('parquet_files')) +where file_name = 'data/parquet-testing/varchar_stats.parquet' +---- +200 208 +200 208 diff --git a/test/sql/copy/parquet/parquet_stats.test b/test/sql/copy/parquet/parquet_stats.test index a73a66482af3..409125a30945 100644 --- a/test/sql/copy/parquet/parquet_stats.test +++ b/test/sql/copy/parquet/parquet_stats.test @@ -210,10 +210,10 @@ false false query II select row_group_bytes, row_group_compressed_bytes from parquet_metadata('__TEST_DIR__/test.parquet'); ---- -27 1 +27 29 query II select row_group_bytes, row_group_compressed_bytes from parquet_metadata('data/parquet-testing/varchar_stats.parquet'); ---- -200 1 -200 1 +200 208 +200 208 diff --git a/test/sql/copy/parquet/parquet_write_codecs.test b/test/sql/copy/parquet/parquet_write_codecs.test index 6ee94c0e3901..4ab12e079bb5 100644 --- a/test/sql/copy/parquet/parquet_write_codecs.test +++ b/test/sql/copy/parquet/parquet_write_codecs.test @@ -29,13 +29,16 @@ endloop statement error COPY (SELECT 42, 'hello') TO '__TEST_DIR__/gzip.parquet' (FORMAT 'parquet', CODEC 'BLABLABLA'); ---- +:.*Binder Error: Expected codec argument.* # empty codec statement error COPY (SELECT 42, 'hello') TO '__TEST_DIR__/gzip.parquet' (FORMAT 'parquet', CODEC); ---- +:.*Invalid Input Error.*requires an argument.* # integer codec statement error COPY (SELECT 42, 'hello') TO '__TEST_DIR__/gzip.parquet' (FORMAT 'parquet', CODEC 3); ---- +:.*Invalid Input Error.*could not be cast as this type.* \ No newline at end of file diff --git a/test/sql/copy/parquet/snowflake_lineitem.test b/test/sql/copy/parquet/snowflake_lineitem.test deleted file mode 100644 index b22b9ad637f3..000000000000 --- a/test/sql/copy/parquet/snowflake_lineitem.test +++ /dev/null @@ -1,10 +0,0 @@ -# name: test/sql/copy/parquet/snowflake_lineitem.test -# description: Test parquet file exported from snowflake -# group: [parquet] - -require parquet - -require httpfs - -statement ok -CREATE TABLE snowflake_lineitem AS FROM 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/snowflake_lineitem_export.parquet' diff --git a/test/sql/copy/parquet/test_parquet_force_download.test b/test/sql/copy/parquet/test_parquet_force_download.test deleted file mode 100644 index 87bc98452296..000000000000 --- a/test/sql/copy/parquet/test_parquet_force_download.test +++ /dev/null @@ -1,139 +0,0 @@ -# name: test/sql/copy/parquet/test_parquet_force_download.test -# description: Test Force download -# group: [parquet] - -require parquet - -require httpfs - -require tpch - -statement ok -SET force_download=true; - -# we query the same file multiple times, so we have to disable the cache to verify the GET request count -statement ok -set enable_external_file_cache=false; - -query II -explain analyze SELECT id, first_name, last_name, email FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') ----- -analyzed_plan :.*GET: 1.* - -query I -SELECT count(*) FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') ----- -1000 - -statement ok -SET force_download=false; - -query II -explain analyze SELECT id, first_name, last_name, email FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') ----- -analyzed_plan :.*GET: 2.* - -query I -SELECT count(*) FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') ----- -1000 - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -statement ok -SET force_download=true; - -statement ok -SET threads=1 - -statement ok -create table user_info as SELECT * FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') -UNION ALL SELECT * FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') -UNION ALL SELECT * FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') -UNION ALL SELECT * FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') -UNION ALL SELECT * FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') -UNION ALL SELECT * FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') -UNION ALL SELECT * FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') -UNION ALL SELECT * FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') -UNION ALL SELECT * FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') - -# ROW_GROUP_SIZE for these tests used to be 2048, but because the parquet writer checked '>' instead of '>=' -# the row group sizes were actually two chunks, so 4096. This has since been fixed, and this needed to be updated -statement ok -COPY (from user_info) TO 's3://test-bucket/row-user-data.parquet' (FORMAT PARQUET, ROW_GROUP_SIZE 4096); - -statement ok -COPY (from user_info limit 100) TO 's3://test-bucket/row-user-data_1.parquet' (FORMAT PARQUET, ROW_GROUP_SIZE 4096); - -statement ok -PRAGMA threads=10 - -statement ok -PRAGMA verify_parallelism - -query II -explain analyze SELECT id, first_name, last_name, email FROM PARQUET_SCAN('s3://test-bucket/row-user-data.parquet') ----- -analyzed_plan :.*GET: 1.* - - -query I -SELECT count(*) FROM PARQUET_SCAN('s3://test-bucket/row-user-data.parquet') ----- -9000 - -query I -SELECT count(*) FROM PARQUET_SCAN('s3://test-bucket/row-user-data_1.parquet') ----- -100 - -statement ok -SET force_download=false; - -query I -SELECT count(*) FROM PARQUET_SCAN('s3://test-bucket/row-user-data.parquet') ----- -9000 - -query II -explain analyze SELECT id, first_name, last_name, email FROM PARQUET_SCAN('s3://test-bucket/row-user-data.parquet') ----- -analyzed_plan :.*GET: 4.* - -statement ok -SET force_download=true; - -query I -SELECT count(*) FROM (SELECT * FROM PARQUET_SCAN('s3://test-bucket/row-user-data.parquet') union all select * from PARQUET_SCAN('s3://test-bucket/row-user-data_1.parquet')) as t ----- -9100 - -query II -explain analyze SELECT id, first_name, last_name, email FROM PARQUET_SCAN('s3://test-bucket/row-user-data.parquet') union all select id, first_name, last_name, email from PARQUET_SCAN('s3://test-bucket/row-user-data_1.parquet') ----- -analyzed_plan :.*GET: 2.* - -statement ok -SET force_download=false; - -query I -SELECT count(*) FROM (SELECT * FROM PARQUET_SCAN('s3://test-bucket/row-user-data.parquet') union all select * from PARQUET_SCAN('s3://test-bucket/row-user-data_1.parquet')) as t ----- -9100 - -query II -explain analyze SELECT id, first_name, last_name, email FROM PARQUET_SCAN('s3://test-bucket/row-user-data.parquet') union all select id, first_name, last_name, email from PARQUET_SCAN('s3://test-bucket/row-user-data_1.parquet') ----- -analyzed_plan :.*GET: 6.* diff --git a/test/sql/copy/parquet/test_parquet_remote.test b/test/sql/copy/parquet/test_parquet_remote.test deleted file mode 100644 index d1c82e00f1e1..000000000000 --- a/test/sql/copy/parquet/test_parquet_remote.test +++ /dev/null @@ -1,83 +0,0 @@ -# name: test/sql/copy/parquet/test_parquet_remote.test -# description: Parquet read from S3/HTTPS -# group: [parquet] - -require httpfs - -require parquet - -# non existent host -statement error -SELECT * FROM PARQUET_SCAN('https://this-host-does-not-exist-for-sure/test.parquet'); ----- - -# non existent file -statement error -SELECT * FROM PARQUET_SCAN('https://duckdb.org/test.parquet'); ----- - -# missing path -statement error -SELECT * FROM PARQUET_SCAN('https://duckdb.org'); ----- - -# empty path -statement error -SELECT * FROM PARQUET_SCAN('https://duckdb.org/'); ----- - -# straightforward -query IIII -SELECT id, first_name, last_name, email FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') LIMIT 10; ----- -1 Amanda Jordan ajordan0@com.com -2 Albert Freeman afreeman1@is.gd -3 Evelyn Morgan emorgan2@altervista.org -4 Denise Riley driley3@gmpg.org -5 Carlos Burns cburns4@miitbeian.gov.cn -6 Kathryn White kwhite5@google.com -7 Samuel Holmes sholmes6@foxnews.com -8 Harry Howell hhowell7@eepurl.com -9 Jose Foster jfoster8@yelp.com -10 Emily Stewart estewart9@opensource.org - - -# with redirects -query IIII -SELECT id, first_name, last_name, email FROM PARQUET_SCAN('https://github.com/duckdb/duckdb/blob/main/data/parquet-testing/userdata1.parquet?raw=true') LIMIT 10; ----- -1 Amanda Jordan ajordan0@com.com -2 Albert Freeman afreeman1@is.gd -3 Evelyn Morgan emorgan2@altervista.org -4 Denise Riley driley3@gmpg.org -5 Carlos Burns cburns4@miitbeian.gov.cn -6 Kathryn White kwhite5@google.com -7 Samuel Holmes sholmes6@foxnews.com -8 Harry Howell hhowell7@eepurl.com -9 Jose Foster jfoster8@yelp.com -10 Emily Stewart estewart9@opensource.org - -# with explicit port nr -query IIII -SELECT id, first_name, last_name, email FROM PARQUET_SCAN('https://github.com:443/duckdb/duckdb/blob/main/data/parquet-testing/userdata1.parquet?raw=true') LIMIT 10; ----- -1 Amanda Jordan ajordan0@com.com -2 Albert Freeman afreeman1@is.gd -3 Evelyn Morgan emorgan2@altervista.org -4 Denise Riley driley3@gmpg.org -5 Carlos Burns cburns4@miitbeian.gov.cn -6 Kathryn White kwhite5@google.com -7 Samuel Holmes sholmes6@foxnews.com -8 Harry Howell hhowell7@eepurl.com -9 Jose Foster jfoster8@yelp.com -10 Emily Stewart estewart9@opensource.org - -query IIII -SELECT id, first_name, last_name, email FROM PARQUET_SCAN('https://github.com/duckdb/duckdb-data/releases/download/v1.0/us+er+da+ta.parquet') LIMIT 1; ----- -1 Amanda Jordan ajordan0@com.com - -query IIII -SELECT id, first_name, last_name, email FROM PARQUET_SCAN('https://github.com/duckdb/duckdb-data/releases/download/v1.0/us%2Ber%2Bda%2Bta.parquet') LIMIT 1; ----- -1 Amanda Jordan ajordan0@com.com diff --git a/test/sql/copy/parquet/test_parquet_remote_foreign_files.test b/test/sql/copy/parquet/test_parquet_remote_foreign_files.test deleted file mode 100644 index 0a210b4d18b1..000000000000 --- a/test/sql/copy/parquet/test_parquet_remote_foreign_files.test +++ /dev/null @@ -1,88 +0,0 @@ -# name: test/sql/copy/parquet/test_parquet_remote_foreign_files.test -# description: Test queries on tricky parquet files over http. Note: on GH connection issues, these tests fail silently -# group: [parquet] - -require parquet - -require httpfs - -# /data/parquet-testing/bug1554.parquet -query I -SELECT COUNT(backlink_count) FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug1554.parquet') WHERE http_status_code=200 ----- -0 - -query II -SELECT http_status_code, COUNT(backlink_count) FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug1554.parquet') GROUP BY http_status_code ORDER BY http_status_code ----- -200 0 -301 0 - -# /data/parquet-testing/bug1588.parquet - -query I -SELECT has_image_link FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug1588.parquet') where has_image_link = 1 ----- -1 -1 -1 - -# /data/parquet-testing/bug1589.parquet -query I -SELECT backlink_count FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug1589.parquet') LIMIT 1 ----- -NULL - -statement ok -SELECT * FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug1589.parquet') - - -query I -SELECT "inner"['str_field'] FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug1618_struct_strings.parquet') ----- -hello -NULL - -query I -SELECT "inner"['f64_field'] FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug1618_struct_strings.parquet') ----- -NULL -1.23 - -query I -SELECT "inner" FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug1618_struct_strings.parquet') ----- -{'str_field': hello, 'f64_field': NULL} -{'str_field': NULL, 'f64_field': 1.23} - -# /data/parquet-testing/struct.parquet -query I -select "inner"['f64_field'] from parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/struct.parquet'); ----- -NULL -1.23 - -# /data/parquet-testing/bug2267.parquet -query I -SELECT * FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug2267.parquet') ----- -[{'disabledPlans': [bea4c11e-220a-4e6d-8eb8-8ea15d019f90], 'skuId': c7df2760-2c81-4ef7-b578-5b5392b571df}, {'disabledPlans': [8a256a2b-b617-496d-b51b-e76466e88db0, 41781fb2-bc02-4b7c-bd55-b576c07bb09d, eec0eb4f-6444-4f95-aba0-50c24d67f998], 'skuId': 84a661c4-e949-4bd2-a560-ed7766fcaf2b}, {'disabledPlans': [], 'skuId': b05e124f-c7cc-45a0-a6aa-8cf78c946968}, {'disabledPlans': [], 'skuId': f30db892-07e9-47e9-837c-80727f46fd3d}] - -query I -SELECT assignedLicenses[1] FROM parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/bug2267.parquet') ----- -{'disabledPlans': [bea4c11e-220a-4e6d-8eb8-8ea15d019f90], 'skuId': c7df2760-2c81-4ef7-b578-5b5392b571df} - -# multiple files -query II -select * from parquet_scan(['https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/glob/t1.parquet', 'https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/glob/t2.parquet']) ----- -1 a -2 b - -# Malformed parquet to test fallback from prefetch -query IIII -select * from parquet_scan('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/arrow/nation.dict-malformed.parquet') limit 2; ----- -0 ALGERIA 0 haggle. carefully final deposits detect slyly agai -1 ARGENTINA 1 al foxes promise slyly according to the regular accounts. bold requests alon diff --git a/test/sql/copy/parquet/test_parquet_scan.test b/test/sql/copy/parquet/test_parquet_scan.test index af06f274dde9..6670cd3056d1 100644 --- a/test/sql/copy/parquet/test_parquet_scan.test +++ b/test/sql/copy/parquet/test_parquet_scan.test @@ -11,6 +11,7 @@ PRAGMA enable_verification statement error SELECT * FROM parquet_scan('does_not_exist') ---- +:.*IO Error: No files found.* # alltypes_plain.parquet query ITIIIIRRTTT @@ -247,3 +248,4 @@ SELECT FIRST(comments) OVER w, LAST(comments) OVER w FROM userdata1 WINDOW w AS statement error SELECT * FROM parquet_scan('data/parquet-testing/broken-arrow.parquet') ---- +:.*Invalid Error: Out of buffer.* \ No newline at end of file diff --git a/test/sql/copy/parquet/test_yellow_cab.test_slow b/test/sql/copy/parquet/test_yellow_cab.test_slow deleted file mode 100644 index 83ec356bbbad..000000000000 --- a/test/sql/copy/parquet/test_yellow_cab.test_slow +++ /dev/null @@ -1,34 +0,0 @@ -# name: test/sql/copy/parquet/test_yellow_cab.test_slow -# description: Test yellow cab parquet file -# group: [parquet] - -require parquet - -require httpfs - -statement ok -CREATE TABLE yellow_cab AS SELECT * FROM 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/yellowcab.parquet' - -statement ok -PRAGMA enable_verification - -query IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII -select min(VendorID::VARCHAR), max(VendorID::VARCHAR), min(tpep_pickup_datetime::VARCHAR), max(tpep_pickup_datetime::VARCHAR), min(tpep_dropoff_datetime::VARCHAR), max(tpep_dropoff_datetime::VARCHAR), min(passenger_count::VARCHAR), max(passenger_count::VARCHAR), min(trip_distance::VARCHAR), max(trip_distance::VARCHAR), min(pickup_longitude::VARCHAR), max(pickup_longitude::VARCHAR), min(pickup_latitude::VARCHAR), max(pickup_latitude::VARCHAR), min(RatecodeID::VARCHAR), max(RatecodeID::VARCHAR), min(store_and_fwd_flag::VARCHAR), max(store_and_fwd_flag::VARCHAR), min(dropoff_longitude::VARCHAR), max(dropoff_longitude::VARCHAR), min(dropoff_latitude::VARCHAR), max(dropoff_latitude::VARCHAR), min(payment_type::VARCHAR), max(payment_type::VARCHAR), min(fare_amount::VARCHAR), max(fare_amount::VARCHAR), min(extra::VARCHAR), max(extra::VARCHAR), min(mta_tax::VARCHAR), max(mta_tax::VARCHAR), min(tip_amount::VARCHAR), max(tip_amount::VARCHAR), min(tolls_amount::VARCHAR), max(tolls_amount::VARCHAR), min(improvement_surcharge::VARCHAR), max(improvement_surcharge::VARCHAR), min(total_amount::VARCHAR), max(total_amount::VARCHAR) from yellow_cab; ----- -1 2 2016-01-01 00:00:00 2016-01-29 12:08:57 2016-01-01 00:00:00 2016-01-30 12:05:11 0 8 .00 97.40 -0.13990700244903564 0 0 57.269275665283203 1 99 (empty) Y -73.210006713867188 0 0 41.317001342773437 1 4 -10 998 -0.5 2.0 -0.5 0.5 0 998.14 -10.5 9.75 -0.3 0.3 -10.8 998.3 - - -# writer round-trip -statement ok -COPY yellow_cab TO '__TEST_DIR__/yellowcab.parquet' (FORMAT PARQUET); - -query IIIIIIIIIIIIIIIIIII nosort yellowcab -SELECT * FROM yellow_cab ----- - -query IIIIIIIIIIIIIIIIIII nosort yellowcab -SELECT * FROM '__TEST_DIR__/yellowcab.parquet' ----- - - - diff --git a/test/sql/copy/parquet/writer/parquet_write_interval.test b/test/sql/copy/parquet/writer/parquet_write_interval.test index 38605f949dce..34dab93d6b41 100644 --- a/test/sql/copy/parquet/writer/parquet_write_interval.test +++ b/test/sql/copy/parquet/writer/parquet_write_interval.test @@ -36,3 +36,4 @@ NULL statement error COPY (SELECT -interval '1 day') TO '__TEST_DIR__/intervals.parquet' ---- +:.*IO Error.*do not support negative intervals.* \ No newline at end of file diff --git a/test/sql/copy/parquet/writer/write_map.test b/test/sql/copy/parquet/writer/write_map.test index 6bd626de8e85..e95dd6c9848c 100644 --- a/test/sql/copy/parquet/writer/write_map.test +++ b/test/sql/copy/parquet/writer/write_map.test @@ -30,11 +30,13 @@ INSERT INTO int_maps VALUES (MAP([NULL], [NULL])) ; ---- +:.*Invalid Input Error: Map keys can not be NULL.* # parquet does not support keys with null values statement error COPY string_map TO '__TEST_DIR__/int_maps.parquet' (FORMAT PARQUET) ---- +:.*Catalog Error.*does not exist!.* # string -> string map statement ok @@ -66,6 +68,7 @@ INSERT INTO string_map VALUES (MAP([NULL], [NULL])) ; ---- +:.*Invalid Input Error: Map keys can not be NULL.* # list -> list map statement ok @@ -96,3 +99,4 @@ INSERT INTO list_map VALUES (MAP([NULL], [NULL])) ; ---- +:.*Invalid Input Error: Map keys can not be NULL.* \ No newline at end of file diff --git a/test/sql/copy/s3/README.md b/test/sql/copy/s3/README.md deleted file mode 100644 index 24cfaa79f5e5..000000000000 --- a/test/sql/copy/s3/README.md +++ /dev/null @@ -1,69 +0,0 @@ - -In order to test these locally, `minio` is used. This requires Docker to be installed. - -### Installing Docker on MacOS - -Install `docker` using `homebrew`. - - -```bash -brew install docker --cask -``` - -Then open `/Applications/Docker`. Note that the first time you open the application you need to go to the `Applications` folder, right-click `Docker` and select `open`. - -### Setting Up Docker - -In order to finish setting up Docker, you need to open the Docker application, and login to your Docker account. Create a Docker account if you do not have one and finish setting up. - -### Running Minio - -Run the `install_s3_test_server` script. This requires root. This makes a few changes to your system, specifically to `/etc/hosts` to set up a few redirect interfaces to localhost. This only needs to be run once. - -```bash -sudo ./scripts/install_s3_test_server.sh -``` - -Then, if this has not been done yet, we need to generate some data: - -``` -./scripts/generate_presigned_url.sh -``` - -Then run the test server in the back-ground using Docker. Note that Docker must be opened for this to work. On MacOS you can open the docker gui (`/Applications/Docker`) and leave it open to accomplish this. - - -```bash -source ./scripts/run_s3_test_server.sh -``` - -Now set up the following environment variables to enable running of the tests. - -This can be done either manually: -```bash -export S3_TEST_SERVER_AVAILABLE=1 -export AWS_DEFAULT_REGION=eu-west-1 -export AWS_ACCESS_KEY_ID=minio_duckdb_user -export AWS_SECRET_ACCESS_KEY=minio_duckdb_user_password -export DUCKDB_S3_ENDPOINT=duckdb-minio.com:9000 -export DUCKDB_S3_USE_SSL=false -``` - -Or using the `set_s3_test_server_variables.sh` script - -```bash -# use source so it sets the environment variables in your current environment -source scripts/set_s3_test_server_variables.sh -``` - -Now you should be able to run the S3 tests using minio, e.g.: - -```bash -build/debug/test/unittest test/sql/copy/s3/s3_hive_partition.test -``` - -> minio uses port 9000. Clickhouse also uses port 9000. If the tests are not working and you have a running Clickhouse service - try killing it first, e.g. using `killall -9 clickhouse` - -#### Test Data - -The configuration for minio is stored in `scripts/minio_s3.yml`. Data is stored in `/tmp/minio_test_data`. \ No newline at end of file diff --git a/test/sql/copy/s3/download_config.test b/test/sql/copy/s3/download_config.test deleted file mode 100644 index 7ff2c79a1737..000000000000 --- a/test/sql/copy/s3/download_config.test +++ /dev/null @@ -1,128 +0,0 @@ -# name: test/sql/copy/s3/download_config.test -# description: Test S3 configuration -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -## Require that these environment variables are also set -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -CREATE TABLE test as SELECT * FROM range(0,10) tbl(i); - -foreach url_style path vhost -# Have to set these because they get altered during the loop -statement ok -SET s3_secret_access_key='${AWS_SECRET_ACCESS_KEY}'; - -statement ok -SET s3_access_key_id='${AWS_ACCESS_KEY_ID}'; - -statement ok -SET s3_endpoint='${DUCKDB_S3_ENDPOINT}'; - -statement ok -SET http_retries=2; - -statement ok -SET http_retry_wait_ms=10; - -statement ok -SET http_retry_backoff=1; - -statement ok -SET http_timeout=50000; - -statement ok -SET http_keep_alive=false; - -# Test the vhost style urls (this is the default) -statement ok -SET s3_url_style='${url_style}'; - -statement ok -COPY test TO 's3://test-bucket-public/root-dir/test_${url_style}_url_style.parquet'; - -# vhost style access -query I -SELECT i FROM "http://test-bucket-public.${DUCKDB_S3_ENDPOINT}/root-dir/test_${url_style}_url_style.parquet" LIMIT 3 ----- -0 -1 -2 - -# path style access -query I -SELECT i FROM "http://${DUCKDB_S3_ENDPOINT}/test-bucket-public/root-dir/test_${url_style}_url_style.parquet" LIMIT 3 ----- -0 -1 -2 - -# Test public access through s3 url -statement ok -SET s3_secret_access_key='';SET s3_access_key_id=''; - -query I -SELECT i FROM "s3://test-bucket-public/root-dir/test_${url_style}_url_style.parquet" LIMIT 3 ----- -0 -1 -2 - -endloop - -# empty url style is also allowed to select the default -statement ok -SET s3_secret_access_key='${AWS_SECRET_ACCESS_KEY}';SET s3_access_key_id='${AWS_ACCESS_KEY_ID}';SET s3_region='${AWS_DEFAULT_REGION}'; SET s3_endpoint='${DUCKDB_S3_ENDPOINT}'; SET s3_use_ssl=${DUCKDB_S3_USE_SSL}; - -statement ok -COPY test TO 's3://test-bucket-public/root-dir/test_default_url_style.parquet'; - -query I -SELECT i FROM "http://test-bucket-public.${DUCKDB_S3_ENDPOINT}/root-dir/test_default_url_style.parquet" LIMIT 3 ----- -0 -1 -2 - -# Incorrect path style throws error -statement ok -SET s3_url_style='handwritten'; - -statement error -COPY test TO 's3://test-bucket-public/root-dir/test2.parquet'; ----- - -# 404 -statement error -SELECT i FROM "http://test-bucket-public.${DUCKDB_S3_ENDPOINT}/root-dir/non-existent-file-ljaslkjdas.parquet" LIMIT 3 ----- -Unable to connect to URL "http://test-bucket-public. - -# Connection error -statement error -SELECT i FROM "http://test-bucket-public.duckdb-minio-non-existent-host.com:9000/root-dir/non-existent-file-ljaslkjdas.parquet" LIMIT 3 ----- -Could not establish connection error for HTTP HEAD to 'http://test-bucket-public. - -# S3 errors should throw on -statement error -SELECT * FROM parquet_scan('s3://this-aint-no-bucket/no-path/no-file'); ----- -Unable to connect to URL "http:// diff --git a/test/sql/copy/s3/fully_qualified_s3_url.test b/test/sql/copy/s3/fully_qualified_s3_url.test deleted file mode 100644 index 99b9113b1031..000000000000 --- a/test/sql/copy/s3/fully_qualified_s3_url.test +++ /dev/null @@ -1,204 +0,0 @@ -# name: test/sql/copy/s3/fully_qualified_s3_url.test -# description: Test S3, credentials override with query parameters -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -CREATE TABLE test as SELECT * FROM range(0,10) tbl(i); - -#set some false credentials to verify query param override -statement ok -SET s3_secret_access_key='false_pw';SET s3_access_key_id='false_key'; - -statement ok -SET s3_url_style='path'; - -statement error -COPY test TO 's3://test-bucket/s3_query_params/test.csv'; ----- -Unable to connect to URL - -#test with .csv file -statement ok -COPY test TO 's3://test-bucket/s3_query_params/test.csv?s3_access_key_id=${AWS_ACCESS_KEY_ID}&s3_secret_access_key=${AWS_SECRET_ACCESS_KEY}'; - -query I -SELECT i FROM "s3://test-bucket/s3_query_params/test.csv?s3_access_key_id=${AWS_ACCESS_KEY_ID}&s3_secret_access_key=${AWS_SECRET_ACCESS_KEY}" LIMIT 3 ----- -0 -1 -2 - -#test with .parquet file -statement ok -COPY test TO 's3://test-bucket/s3_query_params/test.parquet?s3_access_key_id=${AWS_ACCESS_KEY_ID}&s3_secret_access_key=${AWS_SECRET_ACCESS_KEY}' (FORMAT 'parquet'); - -query I -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_access_key_id=${AWS_ACCESS_KEY_ID}&s3_secret_access_key=${AWS_SECRET_ACCESS_KEY}" LIMIT 3 ----- -0 -1 -2 - -#test GLOB with .parquet file -query I -SELECT i FROM "s3://test-bucket/s3_query_params/*.parquet?s3_access_key_id=${AWS_ACCESS_KEY_ID}&s3_secret_access_key=${AWS_SECRET_ACCESS_KEY}" LIMIT 3 ----- -0 -1 -2 - -#global settings have not been modified by query parameters -query I -SELECT CURRENT_SETTING('s3_access_key_id'); ----- -false_key - -query I -SELECT CURRENT_SETTING('s3_secret_access_key'); ----- -false_pw - -#setting up a second s3 file with different credentials -statement ok -CREATE TABLE test_2 as SELECT * FROM range(100,110) tbl(j); - -statement ok -COPY test_2 TO 's3://test-bucket/s3_query_params/test_2.csv?s3_access_key_id=minio_duckdb_user_2&s3_secret_access_key=minio_duckdb_user_2_password' ; - -query I -SELECT j FROM "s3://test-bucket/s3_query_params/test_2.csv?s3_access_key_id=minio_duckdb_user_2&s3_secret_access_key=minio_duckdb_user_2_password" LIMIT 3 ----- -100 -101 -102 - -#test a joining of two tables with different credentials -query II -SELECT T1.i, T2.j FROM "s3://test-bucket/s3_query_params/test.parquet?s3_access_key_id=${AWS_ACCESS_KEY_ID}&s3_secret_access_key=${AWS_SECRET_ACCESS_KEY}" T1 - INNER JOIN "s3://test-bucket/s3_query_params/test_2.csv?s3_access_key_id=minio_duckdb_user_2&s3_secret_access_key=minio_duckdb_user_2_password" T2 - ON T1.i+100=T2.j LIMIT 3; ----- -0 100 -1 101 -2 102 - -statement ok -SET s3_secret_access_key='${AWS_SECRET_ACCESS_KEY}';SET s3_access_key_id='${AWS_ACCESS_KEY_ID}'; - -#test region param -statement ok -SET s3_region='false_region'; - -statement ok -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_region=${AWS_DEFAULT_REGION}" LIMIT 3 - -statement ok -SET s3_region='${AWS_DEFAULT_REGION}'; - -# test endpoint param -statement ok -SET s3_endpoint='false_endpoint'; - -statement ok -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_endpoint=${DUCKDB_S3_ENDPOINT}" LIMIT 3 - -statement ok -SET s3_endpoint='${DUCKDB_S3_ENDPOINT}'; - -#test secret_access_key -statement ok -SET s3_secret_access_key='false_acces_key'; - -statement ok -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_secret_access_key=${AWS_SECRET_ACCESS_KEY}" LIMIT 3 - -statement ok -SET s3_secret_access_key='${AWS_SECRET_ACCESS_KEY}'; - -#test access_key -statement ok -SET s3_access_key_id='false_acces_key_id'; - -statement ok -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_access_key_id=${AWS_ACCESS_KEY_ID}" LIMIT 3 - -statement ok -SET s3_access_key_id='${AWS_ACCESS_KEY_ID}'; - -#test use_ssl -statement ok -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_use_ssl=false" LIMIT 3 - -statement error -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_use_ssl=bla" LIMIT 3 ----- - -#test url_style -statement ok -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_url_style=vhost" LIMIT 3 - -statement ok -SET s3_url_style='path'; - -# test combinations -statement ok -SET s3_access_key_id='false_id'; SET s3_region='false_region'; - -statement ok -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_region=${AWS_DEFAULT_REGION}&s3_access_key_id=${AWS_ACCESS_KEY_ID}&s3_endpoint=${DUCKDB_S3_ENDPOINT}" LIMIT 3 - -statement ok -SET s3_access_key_id='${AWS_ACCESS_KEY_ID}'; SET s3_region='${AWS_DEFAULT_REGION}'; - -# test faulty input -statement error -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_region=${AWS_DEFAULT_REGION}&s3_access_key_id=incorrect_key_id" LIMIT 3 ----- - -statement error -SELECT i FROM "s3://test-bucket/s3_query_params/test.parquet?s3_region=${AWS_DEFAULT_REGION}&bla=bla" LIMIT 3 ----- - -# test endpoint valid path -statement ok -SET s3_endpoint='${DUCKDB_S3_ENDPOINT}/test-bucket'; SET s3_url_style='path'; - -statement ok -SELECT i FROM "s3://s3_query_params/test.parquet" LIMIT 3 - -# test endpoint invalid url style -statement ok -SET s3_url_style='vhost'; - -statement error -SELECT i FROM "s3://s3_query_params/test.parquet" LIMIT 3 ----- - -# test endpoint invalid path -statement ok -SET s3_endpoint='${DUCKDB_S3_ENDPOINT}/s3_query_params'; SET s3_url_style='path'; - -statement error -SELECT i FROM "s3://test-bucket/test.parquet" LIMIT 3 ----- diff --git a/test/sql/copy/s3/glob_s3_paging.test_slow b/test/sql/copy/s3/glob_s3_paging.test_slow deleted file mode 100644 index db1b5b7b0548..000000000000 --- a/test/sql/copy/s3/glob_s3_paging.test_slow +++ /dev/null @@ -1,94 +0,0 @@ -# name: test/sql/copy/s3/glob_s3_paging.test_slow -# description: Test globbing of a large number of parquet files to test the paging mechanism -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -set http_timeout=120000; - -# More retries (longest wait will be 25600ms) -statement ok -set http_retries=6; - -# Test should be a bit faster using the metadata cache -statement ok -SET enable_http_metadata_cache=true; - -foreach urlstyle path vhost - -statement ok -SET s3_url_style='${urlstyle}' - -## For both formats we generate 2000 files which we will glob to test the paging mechanism of aws ListObjectV2 call is handled properly -foreach format parquet csv - -foreach i 0 1 - -foreach j 0 1 2 3 4 5 6 7 8 9 - -foreach k 0 1 2 3 4 5 6 7 8 9 - -foreach l 0 1 2 3 4 5 6 7 8 9 - -statement ok -COPY (select (${i}${j}${k}${l})::INT as column0) to 's3://test-bucket/parquet_glob_s3_paging/paging/t${i}${j}${k}${l}-${urlstyle}-urls.${format}'; - -endloop - -endloop - -endloop - -endloop - -# Begin tests -query I -select sum(column0) from 's3://test-bucket/parquet_glob_s3_paging/paging/t*-${urlstyle}-urls.${format}' ----- -1999000 - -endloop - -endloop - -# test with parquet_metadata_cache = true -statement ok -SET parquet_metadata_cache=true; - -foreach urlstyle path vhost - -foreach format parquet - -loop i 0 2 - -# Begin tests -query I -select sum(column0) from 's3://test-bucket/parquet_glob_s3_paging/paging/t*-${urlstyle}-urls.${format}' ----- -1999000 - -endloop - -endloop - -endloop diff --git a/test/sql/copy/s3/hive_partitioned_write_s3.test b/test/sql/copy/s3/hive_partitioned_write_s3.test deleted file mode 100644 index 85ece6804e32..000000000000 --- a/test/sql/copy/s3/hive_partitioned_write_s3.test +++ /dev/null @@ -1,183 +0,0 @@ -# name: test/sql/copy/s3/hive_partitioned_write_s3.test -# description: basic tests for the hive partitioned write to s3 -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -CREATE SECRET ( - TYPE S3, - PROVIDER config, - KEY_ID '${AWS_ACCESS_KEY_ID}', - SECRET '${AWS_SECRET_ACCESS_KEY}', - REGION '${AWS_DEFAULT_REGION}', - ENDPOINT '${DUCKDB_S3_ENDPOINT}', - USE_SSL '${DUCKDB_S3_USE_SSL}' -) - -statement ok -set s3_endpoint='ensure.secret.is.used.instead.of.duckdb.setting.com' - -# Simple table that is easy to partition -statement ok -CREATE TABLE test as SELECT i%2 as part_col, (i+1)%5 as value_col, i as value2_col from range(0,10) tbl(i); - -statement ok -COPY test TO 's3://test-bucket/partitioned1' (FORMAT PARQUET, PARTITION_BY (part_col), OVERWRITE_OR_IGNORE TRUE); - -query III -SELECT part_col, value_col, value2_col FROM 's3://test-bucket/partitioned1/part_col=0/*.parquet' ORDER BY value2_col; ----- -0 1 0 -0 3 2 -0 0 4 -0 2 6 -0 4 8 - -query III -SELECT part_col, value_col, value2_col FROM 's3://test-bucket/partitioned1/part_col=1/*.parquet' ORDER BY value2_col; ----- -1 2 1 -1 4 3 -1 1 5 -1 3 7 -1 0 9 - -# Want a modified version of the partition_col? (for example to do custom string conversion?) No problem: -statement ok -COPY (SELECT * EXCLUDE (part_col), 'prefix-'::VARCHAR || part_col::VARCHAR as part_col FROM test) TO 's3://test-bucket/partitioned2' (FORMAT PARQUET, PARTITION_BY (part_col), OVERWRITE_OR_IGNORE TRUE); - -query III -SELECT part_col, value_col, value2_col FROM 's3://test-bucket/partitioned2/part_col=prefix-0/*.parquet' ORDER BY value2_col; ----- -prefix-0 1 0 -prefix-0 3 2 -prefix-0 0 4 -prefix-0 2 6 -prefix-0 4 8 - -query III -SELECT part_col, value_col, value2_col FROM 's3://test-bucket/partitioned2/part_col=prefix-1/*.parquet' ORDER BY value2_col; ----- -prefix-1 2 1 -prefix-1 4 3 -prefix-1 1 5 -prefix-1 3 7 -prefix-1 0 9 - -# Test partitioning by all -statement error -COPY test TO 's3://test-bucket/partitioned3' (FORMAT PARQUET, PARTITION_BY '*', OVERWRITE_OR_IGNORE TRUE); ----- -No column to write as all columns are specified as partition columns - -statement ok -COPY test TO 's3://test-bucket/partitioned3' (FORMAT PARQUET, PARTITION_BY '*', OVERWRITE_OR_IGNORE TRUE, WRITE_PARTITION_COLUMNS 1); - -query I -SELECT min(value2_col) as min_val -FROM parquet_scan('s3://test-bucket/partitioned3/part_col=*/value_col=*/value2_col=*/*.parquet', FILENAME=1) -GROUP BY filename -ORDER BY min_val ----- -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 - -# single col as param is also fine -statement ok -COPY test TO 's3://test-bucket/partitioned4' (FORMAT PARQUET, PARTITION_BY part_col, OVERWRITE_OR_IGNORE TRUE); - -query III -SELECT part_col, value_col, value2_col FROM parquet_scan('s3://test-bucket/partitioned4/*/*.parquet', HIVE_PARTITIONING=1) WHERE part_col=0 ORDER BY value2_col; ----- -0 1 0 -0 3 2 -0 0 4 -0 2 6 -0 4 8 - - -# a file already exist and, OVERWRITE_OR_IGNORE is not set, throw error -statement error -COPY test TO 's3://test-bucket/partitioned4' (FORMAT PARQUET, PARTITION_BY part_col); ----- -Directory - -# a file already exist and, OVERWRITE_OR_IGNORE is set to false, throw error -statement error -COPY test TO 's3://test-bucket/partitioned4' (FORMAT PARQUET, PARTITION_BY part_col, OVERWRITE_OR_IGNORE FALSE); ----- -Directory - -# Trailing slash ist auch gut! -statement ok -COPY test TO 's3://test-bucket/partitioned5/' (FORMAT PARQUET, PARTITION_BY part_col, OVERWRITE_OR_IGNORE TRUE); - -query III -SELECT part_col, value_col, value2_col FROM 's3://test-bucket/partitioned5/part_col=0/*.parquet' ORDER BY value2_col; ----- -0 1 0 -0 3 2 -0 0 4 -0 2 6 -0 4 8 - -# Cannot use the USE_TMP_FILE option simulatiously with partitioning -statement error -COPY test TO 's3://test-bucket/partitioned6' (FORMAT PARQUET, PARTITION_BY part_col, USE_TMP_FILE TRUE); ----- -Not implemented Error: Can't combine USE_TMP_FILE and PARTITION_BY for COPY - -# Technically it doesn't really matter, as currently out parition_by behaves similarly, but for clarity user should just -# EITHER use partition_by or per_thread_output. -statement error -COPY test TO 's3://test-bucket/partitioned6' (FORMAT PARQUET, PARTITION_BY part_col, PER_THREAD_OUTPUT TRUE); ----- -Not implemented Error: Can't combine PER_THREAD_OUTPUT and PARTITION_BY for COPY - -# partitioning csv files is also a thing -statement ok -COPY test TO 's3://test-bucket/partitioned7' (FORMAT CSV, PARTITION_BY part_col, OVERWRITE_OR_IGNORE TRUE); - -query III -SELECT part_col, value_col, value2_col FROM 's3://test-bucket/partitioned7/part_col=0/*.csv' ORDER BY value2_col; ----- -0 1 0 -0 3 2 -0 0 4 -0 2 6 -0 4 8 - -query III -SELECT part_col, value_col, value2_col FROM 's3://test-bucket/partitioned7/part_col=1/*.csv' ORDER BY value2_col; ----- -1 2 1 -1 4 3 -1 1 5 -1 3 7 -1 0 9 diff --git a/test/sql/copy/s3/hive_partitioned_write_s3.test_slow b/test/sql/copy/s3/hive_partitioned_write_s3.test_slow deleted file mode 100644 index 05c6d14064ab..000000000000 --- a/test/sql/copy/s3/hive_partitioned_write_s3.test_slow +++ /dev/null @@ -1,72 +0,0 @@ -# name: test/sql/copy/s3/hive_partitioned_write_s3.test_slow -# description: slow test for the hive partitioned write to s3 -# group: [s3] - -require parquet - -require httpfs - -require tpch - -require-env S3_TEST_SERVER_AVAILABLE 1 - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -pragma memory_limit='200MB' - -statement ok -set http_timeout=120000; - -# More retries (longest wait will be 25600ms) -statement ok -set http_retries=6; - -# around 200MB worth of data, will require the PartitionedColumnData to spill to disk -statement ok -COPY (SELECT i%2::INT32 as part_col, i::INT32 FROM range(0,25000000) tbl(i)) TO 's3://test-bucket/partitioned_memory_spill' (FORMAT parquet, PARTITION_BY part_col, overwrite_or_ignore TRUE); - -statement ok -pragma memory_limit='-1' - -statement ok -call dbgen(sf=1); - -# Partition by 2 columns -statement ok -COPY lineitem TO 's3://test-bucket/lineitem_sf1_partitioned' (FORMAT PARQUET, PARTITION_BY (l_returnflag, l_linestatus), overwrite_or_ignore TRUE); - -statement ok -DROP TABLE lineitem; - -statement ok -CREATE VIEW lineitem as SELECT * FROM parquet_scan('s3://test-bucket/lineitem_sf1_partitioned/*/*/*.parquet', HIVE_PARTITIONING=1); - -loop i 1 9 - -query I -PRAGMA tpch(${i}) ----- -:extension/tpch/dbgen/answers/sf1/q0${i}.csv - -endloop - -loop i 10 23 - -query I -PRAGMA tpch(${i}) ----- -:extension/tpch/dbgen/answers/sf1/q${i}.csv - -endloop diff --git a/test/sql/copy/s3/http_log.test b/test/sql/copy/s3/http_log.test deleted file mode 100644 index 854f77240f03..000000000000 --- a/test/sql/copy/s3/http_log.test +++ /dev/null @@ -1,43 +0,0 @@ -# name: test/sql/copy/s3/http_log.test -# description: Test http logger -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# Create some test data -statement ok -COPY (SELECT 'value-1' as value) TO 's3://test-bucket/http_log/test.parquet'; - -statement ok -CALL enable_logging('HTTP') - -statement ok -set logging_level='debug' - -query I -FROM 's3://test-bucket/http_log/test.parquet' ----- -value-1 - -query II rowsort -SELECT request.type, parse_filename(request.url) FROM duckdb_logs_parsed('HTTP'); ----- -GET test.parquet -HEAD test.parquet diff --git a/test/sql/copy/s3/http_proxy.test b/test/sql/copy/s3/http_proxy.test deleted file mode 100644 index a7d81b1ecbaf..000000000000 --- a/test/sql/copy/s3/http_proxy.test +++ /dev/null @@ -1,155 +0,0 @@ -# name: test/sql/copy/s3/http_proxy.test -# description: Test http proxy -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -require-env HTTP_PROXY_PUBLIC - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -PRAGMA enable_verification - -statement ok -COPY (SELECT 'value-1' as value) TO 's3://test-bucket/proxy-test/test.parquet'; - -query I -FROM 's3://test-bucket/proxy-test/test.parquet' ----- -value-1 - -# Lets try a faulty proxy first -statement ok -set http_proxy='blabla:1337' - -statement ok -set http_proxy_username='xxx' - -statement ok -set http_proxy_password='yyy' - -statement error -FROM 's3://test-bucket/proxy-test/test.parquet' ----- -Could not establish connection - -# Now a working one -statement ok -set http_proxy='${HTTP_PROXY_PUBLIC}' - -statement ok -RESET http_proxy_username - -statement ok -RESET http_proxy_password - -query I -FROM 's3://test-bucket/proxy-test/test.parquet' ----- -value-1 - -# And try the working one with an 'http://' prefix. -statement ok -set http_proxy='http://${HTTP_PROXY_PUBLIC}' - -query I -FROM 's3://test-bucket/proxy-test/test.parquet' ----- -value-1 - -# Now we revert to the failing one -statement ok -set http_proxy='blabla:1337' - -# But we create a HTTP secret with the proxy -statement ok -CREATE SECRET http1 ( - TYPE HTTP, - http_proxy '${HTTP_PROXY_PUBLIC}' -); - -# This works now, because it uses the secret -query I -FROM 's3://test-bucket/proxy-test/test.parquet' ----- -value-1 - -statement ok -DROP SECRET http1 - -require-env HTTP_PROXY - -statement error -FROM 's3://test-bucket/proxy-test/test.parquet' ----- -Could not establish connection - -statement ok -CREATE SECRET http1 ( - TYPE HTTP, - PROVIDER env -); - -# This works now, because it uses the secret -query I -FROM 's3://test-bucket/proxy-test/test.parquet' ----- -value-1 - -statement ok -DROP SECRET http1 - -require-env HTTP_PROXY_PRIVATE - -require-env HTTP_PROXY_PRIVATE_USERNAME - -require-env HTTP_PROXY_PRIVATE_PASSWORD - -# Let's try the private proxy -statement ok -CREATE SECRET http2 ( - TYPE HTTP, - http_proxy '${HTTP_PROXY_PRIVATE}', - http_proxy_username '${HTTP_PROXY_PRIVATE_USERNAME}', - http_proxy_password '${HTTP_PROXY_PRIVATE_PASSWORD}' -); - -# Correct auth means it works! -query I -FROM 's3://test-bucket/proxy-test/test.parquet' ----- -value-1 - -statement ok -DROP SECRET http2 - -# Now lets try incorrect auth -statement ok -CREATE SECRET http3 ( - TYPE HTTP, - http_proxy '${HTTP_PROXY_PRIVATE}', - http_proxy_username 'malicious', - http_proxy_password 'intruder' -); - -# We get a tasty HTTP 407 -statement error -FROM 's3://test-bucket/proxy-test/test.parquet' ----- -HTTP GET error on 'http://test-bucket.duckdb-minio.com:9000/proxy-test/test.parquet' (HTTP 407) diff --git a/test/sql/copy/s3/http_secret.test b/test/sql/copy/s3/http_secret.test deleted file mode 100644 index ea526c6bd8af..000000000000 --- a/test/sql/copy/s3/http_secret.test +++ /dev/null @@ -1,44 +0,0 @@ -# name: test/sql/copy/s3/http_secret.test -# description: Test http secret -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# Create some test data -statement ok -COPY (SELECT 'value-1' as value) TO 's3://test-bucket/http-secret-test/test.parquet'; - -statement ok -PRAGMA enable_verification - -# Create some wonky headers -statement ok -CREATE SECRET http3 ( - TYPE HTTP, - EXTRA_HTTP_HEADERS MAP{ - 'Authorization': 'Im very important', - 'CustomHeader': 'fliepflap' - } -); - -query I -FROM 's3://test-bucket/http-secret-test/test.parquet' ----- -value-1 diff --git a/test/sql/copy/s3/metadata_cache.test b/test/sql/copy/s3/metadata_cache.test deleted file mode 100644 index 06c80809b05e..000000000000 --- a/test/sql/copy/s3/metadata_cache.test +++ /dev/null @@ -1,90 +0,0 @@ -# name: test/sql/copy/s3/metadata_cache.test -# description: Test metadata cache that caches reponses from the initial HEAD requests to open a file. -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# this test was written before we implemented the external file cache -# when it is enabled, the request counts are different -# we disable it so this test still makes sense -statement ok -set enable_external_file_cache=false; - -statement ok -CREATE TABLE test as SELECT * FROM range(0,10) tbl(i); - -statement ok -CREATE TABLE test1 as SELECT * FROM range(10,20) tbl(i); - -query II -EXPLAIN ANALYZE COPY test TO 's3://test-bucket-public/root-dir/metadata_cache/test.parquet'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 1.*GET\: 0.*PUT\: 1.*\#POST\: 2.* - -query II -EXPLAIN ANALYZE COPY test TO 's3://test-bucket-public/root-dir/metadata_cache/test1.parquet'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 1.*GET\: 0.*PUT\: 1.*\#POST\: 2.* - -# Now we query the file metadata without the global metadata cache: There should be 1 HEAD request for the file size, -# then a GET for the pointer to the parquet metadata, then a GET for the metadata. -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM 's3://test-bucket-public/root-dir/metadata_cache/test.parquet'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 1.*GET\: 1.*PUT\: 0.*\#POST\: 0.* - -# Redoing query should still result in same request count -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM 's3://test-bucket-public/root-dir/metadata_cache/test.parquet'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 1.*GET\: 1.*PUT\: 0.*\#POST\: 0.* - -# Now enable the global metadata cache to store the results of the head requests, saving 1 HEAD per file -statement ok -SET enable_http_metadata_cache=true; - -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM 's3://test-bucket-public/root-dir/metadata_cache/test1.parquet'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 1.*GET\: 1.*PUT\: 0.*\#POST\: 0.* - -# Now with global metadata cache, we dont need to do the head request again. noice. -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM 's3://test-bucket-public/root-dir/metadata_cache/test1.parquet'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 0.*GET\: 1.*PUT\: 0.*\#POST\: 0.* - -# Now when we write a file to a cached url, this would break so the cache entry should be invalidated -statement ok -COPY (SELECT * from range(0,100) tbl(i)) TO 's3://test-bucket-public/root-dir/metadata_cache/test1.parquet'; - -# We need to do a new head request here -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM 's3://test-bucket-public/root-dir/metadata_cache/test1.parquet'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 1.*GET\: 1.*PUT\: 0.*\#POST\: 0.* - -# but now its cached again -query II -EXPLAIN ANALYZE SELECT COUNT(*) FROM 's3://test-bucket-public/root-dir/metadata_cache/test1.parquet'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 0.*GET\: 1.*PUT\: 0.*\#POST\: 0.* diff --git a/test/sql/copy/s3/parquet_s3_tpcds.test_slow b/test/sql/copy/s3/parquet_s3_tpcds.test_slow deleted file mode 100644 index 757a4b5064c1..000000000000 --- a/test/sql/copy/s3/parquet_s3_tpcds.test_slow +++ /dev/null @@ -1,96 +0,0 @@ -# name: test/sql/copy/s3/parquet_s3_tpcds.test_slow -# description: Test all tpcds queries on tpch sf0.01 over s3 -# group: [s3] - -require parquet - -require httpfs - -require tpcds - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# answers are generated from postgres -# hence check with NULLS LAST flag -statement ok -PRAGMA default_null_order='NULLS LAST' - -statement ok -SET enable_http_metadata_cache=true; - -statement ok -set http_timeout=120000; - -# More retries (longest wait will be 25600ms) -statement ok -set http_retries=6; - -statement ok -CREATE SCHEMA tpcds; - -statement ok -CALL dsdgen(sf=0.01, schema='tpcds'); - -foreach tbl call_center catalog_page catalog_returns catalog_sales customer customer_demographics customer_address date_dim household_demographics inventory income_band item promotion reason ship_mode store store_returns store_sales time_dim warehouse web_page web_returns web_sales web_site - -statement ok -COPY tpcds.${tbl} TO 's3://test-bucket/tpcds-sf0_01/${tbl}.parquet' (FORMAT 'PARQUET', COMPRESSION 'ZSTD'); - -statement ok -CREATE VIEW ${tbl} AS SELECT * FROM parquet_scan('s3://test-bucket/tpcds-sf0_01/${tbl}.parquet'); - -endloop - -# too slow queries: -# 64, 85 - -loop i 1 9 - -query I -PRAGMA tpcds(${i}) ----- -:extension/tpcds/dsdgen/answers/sf0.01/0${i}.csv - -endloop - -loop i 10 64 - -query I -PRAGMA tpcds(${i}) ----- -:extension/tpcds/dsdgen/answers/sf0.01/${i}.csv - -endloop - -loop i 65 85 - -query I -PRAGMA tpcds(${i}) ----- -:extension/tpcds/dsdgen/answers/sf0.01/${i}.csv - -endloop - -loop i 86 99 - -query I -PRAGMA tpcds(${i}) ----- -:extension/tpcds/dsdgen/answers/sf0.01/${i}.csv - -endloop diff --git a/test/sql/copy/s3/parquet_s3_tpch.test_slow b/test/sql/copy/s3/parquet_s3_tpch.test_slow deleted file mode 100644 index 425e24535e5c..000000000000 --- a/test/sql/copy/s3/parquet_s3_tpch.test_slow +++ /dev/null @@ -1,92 +0,0 @@ -# name: test/sql/copy/s3/parquet_s3_tpch.test_slow -# description: Test all tpch queries on tpch sf0.01 over s3 -# group: [s3] - -require parquet - -require httpfs - -require tpch - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -SET enable_http_metadata_cache=true; - -statement ok -set http_timeout=120000; - -# More retries (longest wait will be 25600ms) -statement ok -set http_retries=6; - -# Copy files to S3 before beginning tests -statement ok -CALL DBGEN(sf=0.01); - -# copy tpch files to S3 -statement ok -COPY lineitem to 's3://test-bucket/tpch-sf0_01/lineitem.parquet'; -COPY nation to 's3://test-bucket/tpch-sf0_01/nation.parquet'; -COPY region to 's3://test-bucket/tpch-sf0_01/region.parquet'; -COPY part to 's3://test-bucket/tpch-sf0_01/part.parquet'; -COPY supplier to 's3://test-bucket/tpch-sf0_01/supplier.parquet'; -COPY partsupp to 's3://test-bucket/tpch-sf0_01/partsupp.parquet'; -COPY customer to 's3://test-bucket/tpch-sf0_01/customer.parquet'; -COPY orders to 's3://test-bucket/tpch-sf0_01/orders.parquet'; - -# clears tables -statement ok -DROP TABLE lineitem; -DROP TABLE nation; -DROP TABLE region; -DROP TABLE part; -DROP TABLE supplier; -DROP TABLE partsupp; -DROP TABLE customer; -DROP TABLE orders; - -statement ok -CREATE VIEW lineitem as SELECT * FROM 's3://test-bucket/tpch-sf0_01/lineitem.parquet'; -CREATE VIEW nation as SELECT * FROM 's3://test-bucket/tpch-sf0_01/nation.parquet'; -CREATE VIEW region as SELECT * FROM 's3://test-bucket/tpch-sf0_01/region.parquet'; -CREATE VIEW part as SELECT * FROM 's3://test-bucket/tpch-sf0_01/part.parquet'; -CREATE VIEW supplier as SELECT * FROM 's3://test-bucket/tpch-sf0_01/supplier.parquet'; -CREATE VIEW partsupp as SELECT * FROM 's3://test-bucket/tpch-sf0_01/partsupp.parquet'; -CREATE VIEW customer as SELECT * FROM 's3://test-bucket/tpch-sf0_01/customer.parquet'; -CREATE VIEW orders as SELECT * FROM 's3://test-bucket/tpch-sf0_01/orders.parquet'; - - -# Run TPCH SF1 -loop i 1 9 - -query I -PRAGMA tpch(${i}) ----- -:extension/tpch/dbgen/answers/sf0.01/q0${i}.csv - -endloop - -loop i 10 23 - -query I -PRAGMA tpch(${i}) ----- -:extension/tpch/dbgen/answers/sf0.01/q${i}.csv - -endloop diff --git a/test/sql/copy/s3/s3_hive_partition.test b/test/sql/copy/s3/s3_hive_partition.test deleted file mode 100644 index 535a3ca5caf7..000000000000 --- a/test/sql/copy/s3/s3_hive_partition.test +++ /dev/null @@ -1,104 +0,0 @@ -# name: test/sql/copy/s3/s3_hive_partition.test -# description: Test the automatic parsing of the hive partitioning scheme -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -## Require that these environment variables are also set -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# Parquet filename name conflict -statement ok -CREATE TABLE test AS SELECT 1 as id, 'value1' as value; -CREATE TABLE test2 AS SELECT 2 as id, 'value2' as value; - -statement ok -COPY test TO 's3://test-bucket/hive-partitioning/simple/key_!-_.*()=zisiswurking1/test.parquet'; -COPY test2 TO 's3://test-bucket/hive-partitioning/simple/key_!-_.*()=zisiswurking2/test.parquet'; - -# test parsing hive partitioning scheme, with some common special characters -query III -select id, value, "key_!-_.*()" from parquet_scan('s3://test-bucket/hive-partitioning/simple/*/test.parquet', HIVE_PARTITIONING=1) ----- -1 value1 zisiswurking1 -2 value2 zisiswurking2 - -# Test some medium sized files -statement ok -CREATE TABLE test3 as SELECT id FROM range(0,10000) tbl(id); -CREATE TABLE test4 as SELECT id FROM range(10000,20000) tbl(id); - -statement ok -COPY test3 TO 's3://test-bucket/hive-partitioning/medium/part=1/part2=1/test.parquet'; -COPY test4 TO 's3://test-bucket/hive-partitioning/medium/part=1/part2=2/test.parquet'; -COPY test3 TO 's3://test-bucket/hive-partitioning/medium/part=1/part2=1/test.csv'; -COPY test4 TO 's3://test-bucket/hive-partitioning/medium/part=1/part2=2/test.csv'; - -query II -select min(id), max(id) from parquet_scan('s3://test-bucket/hive-partitioning/medium/*/*/test.parquet', HIVE_PARTITIONING=1) where part2=2 ----- -10000 19999 - -query II -select min(id), max(id) from parquet_scan('s3://test-bucket/hive-partitioning/medium/*/*/test.parquet', HIVE_PARTITIONING=1) where part2=1 ----- -0 9999 - -query II -select min(id), max(id) from read_csv_auto('s3://test-bucket/hive-partitioning/medium/*/*/test.csv', HIVE_PARTITIONING=1) where part2=2 ----- -10000 19999 - -query II -select min(id), max(id) from read_csv_auto('s3://test-bucket/hive-partitioning/medium/*/*/test.csv', HIVE_PARTITIONING=1) where part2=1 ----- -0 9999 - -# check cases where there are file filters AND table filters -statement ok -Create table t1 (a int, b int, c int); - -foreach i 0 1 2 3 4 5 6 7 8 9 - -statement ok -insert into t1 (select range, ${i}*10, ${i}*100 from range(0,10)); - -endloop - -statement ok -COPY (SELECT * FROM t1) TO 's3://test-bucket/hive-partitioning/filter-test-parquet' (FORMAT PARQUET, PARTITION_BY c, OVERWRITE_OR_IGNORE); - -statement ok -COPY (SELECT * FROM t1) TO 's3://test-bucket/hive-partitioning/filter-test-csv' (FORMAT CSV, PARTITION_BY c, OVERWRITE_OR_IGNORE); - -# There should be Table Filters (id < 50) and file filters (c = 500) -query II -EXPLAIN select a from parquet_scan('s3://test-bucket/hive-partitioning/filter-test-parquet/*/*.parquet', HIVE_PARTITIONING=1, HIVE_TYPES_AUTOCAST=0) where c=500 and a < 4; ----- -physical_plan :.*PARQUET_SCAN.*Filters:.*a<4.*File Filters:.* \(CAST\(c AS.*INTEGER\) = 500\).* - -# There should be Table Filters (id < 50) and file filters (c = 500) -query II -EXPLAIN select a from read_csv_auto('s3://test-bucket/hive-partitioning/filter-test-csv/*/*.csv', HIVE_PARTITIONING=1, HIVE_TYPES_AUTOCAST=0) where c=500 and a < 4; ----- -physical_plan :.*FILTER.*(a < 4).*READ_CSV_AUTO.*File Filters:.* \(CAST\(c AS.*INTEGER\) = 500\).* - -statement error -COPY (SELECT * FROM t1) TO 's3://test-bucket/hive-partitioning/filter-test-parquet' (FORMAT PARQUET, PARTITION_BY c, OVERWRITE); ----- -OVERWRITE is not supported for remote file systems diff --git a/test/sql/copy/s3/s3_presigned_read.test b/test/sql/copy/s3/s3_presigned_read.test deleted file mode 100644 index 21c344f4261b..000000000000 --- a/test/sql/copy/s3/s3_presigned_read.test +++ /dev/null @@ -1,40 +0,0 @@ -# name: test/sql/copy/s3/s3_presigned_read.test -# description: Read small csv/parquet files from S3 Presigned URL. -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -require-env S3_SMALL_CSV_PRESIGNED_URL - -require-env S3_SMALL_PARQUET_PRESIGNED_URL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -query I -SELECT phone FROM read_csv_auto('${S3_SMALL_CSV_PRESIGNED_URL}'); ----- -+318855443322 -+552244331122 -+12233445567 - -query I -SELECT i FROM '${S3_SMALL_PARQUET_PRESIGNED_URL}'; ----- -1 diff --git a/test/sql/copy/s3/s3_presigned_read.test_slow b/test/sql/copy/s3/s3_presigned_read.test_slow deleted file mode 100644 index 9bd3da94976e..000000000000 --- a/test/sql/copy/s3/s3_presigned_read.test_slow +++ /dev/null @@ -1,48 +0,0 @@ -# name: test/sql/copy/s3/s3_presigned_read.test_slow -# description: Read large csv/parquet files from S3 Presigned URL. -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - - -require-env S3_LARGE_PARQUET_PRESIGNED_URL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -set http_timeout=120000; - -# More retries (longest wait will be 25600ms) -statement ok -set http_retries=6; - -query I -SELECT - sum(l_extendedprice * l_discount) AS revenue -FROM - '${S3_LARGE_PARQUET_PRESIGNED_URL}' -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -123141078.2283 diff --git a/test/sql/copy/s3/starstar.test b/test/sql/copy/s3/starstar.test deleted file mode 100644 index a8ade2960c6a..000000000000 --- a/test/sql/copy/s3/starstar.test +++ /dev/null @@ -1,362 +0,0 @@ -# name: test/sql/copy/s3/starstar.test -# description: Test the glob "**" in s3 -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -## Require that these environment variables are also set -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# create a table -statement ok -CREATE TABLE mytable AS SELECT i as a, (i*2) as b, power(i,2) as c from range(0,10) tbl(i); - - - -# sanity check: the bucket is empty -query I -FROM GLOB('s3://test-bucket/glob_ss/*'); ----- - -query I -FROM GLOB('s3://test-bucket/glob_ss/**'); ----- - - - -# one file at the top -statement ok -COPY mytable TO 's3://test-bucket/glob_ss/t0.csv'; - -query I -FROM GLOB('s3://test-bucket/glob_ss/*'); ----- -s3://test-bucket/glob_ss/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**'); ----- -s3://test-bucket/glob_ss/t0.csv - - - -# and 1 file at depth 1 in dir 'a' -statement ok -COPY mytable TO 's3://test-bucket/glob_ss/a/t0.csv'; - -query I -FROM GLOB('s3://test-bucket/glob_ss/*/t0.csv'); ----- -s3://test-bucket/glob_ss/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/t0.csv'); ----- -s3://test-bucket/glob_ss/a/t0.csv -s3://test-bucket/glob_ss/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/*/t0.csv'); ----- -s3://test-bucket/glob_ss/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/*/*/t0.csv'); ----- - -query I -FROM GLOB('s3://test-bucket/glob_ss/**'); ----- -s3://test-bucket/glob_ss/a/t0.csv -s3://test-bucket/glob_ss/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/*'); ----- -s3://test-bucket/glob_ss/a/t0.csv -s3://test-bucket/glob_ss/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/*/**'); ----- -s3://test-bucket/glob_ss/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/a/**'); ----- -s3://test-bucket/glob_ss/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/*/a/t0.csv'); ----- - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/a/t0.csv'); ----- -s3://test-bucket/glob_ss/a/t0.csv - - - -# add 1 file at depth 2 in dir 'a/b/' -statement ok -COPY mytable TO 's3://test-bucket/glob_ss/a/b/t0.csv'; - -query I -FROM GLOB('s3://test-bucket/glob_ss/*/t0.csv'); ----- -s3://test-bucket/glob_ss/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/*/*/t0.csv'); ----- -s3://test-bucket/glob_ss/a/b/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/t0.csv'); ----- -s3://test-bucket/glob_ss/a/b/t0.csv -s3://test-bucket/glob_ss/a/t0.csv -s3://test-bucket/glob_ss/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/*/t0.csv'); ----- -s3://test-bucket/glob_ss/a/b/t0.csv -s3://test-bucket/glob_ss/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/*/*/*/t0.csv'); ----- - -query I -FROM GLOB('s3://test-bucket/glob_ss/**'); ----- -s3://test-bucket/glob_ss/a/b/t0.csv -s3://test-bucket/glob_ss/a/t0.csv -s3://test-bucket/glob_ss/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/*'); ----- -s3://test-bucket/glob_ss/a/b/t0.csv -s3://test-bucket/glob_ss/a/t0.csv -s3://test-bucket/glob_ss/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/*/**'); ----- -s3://test-bucket/glob_ss/a/b/t0.csv -s3://test-bucket/glob_ss/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/a/**'); ----- -s3://test-bucket/glob_ss/a/b/t0.csv -s3://test-bucket/glob_ss/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/*/a/t0.csv'); ----- - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/a/t0.csv'); ----- -s3://test-bucket/glob_ss/a/t0.csv - -### these next 3 require 'first_wildcard_pos == string::npos' to not trigger a return -# # sanity check -# query I -# FROM GLOB('s3://test-bucket/glob_ss/partitioned'); -# ---- - -# # sanity check -# query I -# SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/partitioned'); -# ---- -# 0 - -# # sanity check -# query I -# SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/partitioned/'); -# ---- -# 0 - -# sanity check -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/partitioned/*'); ----- -0 - -# partitioned mytable -statement ok -COPY mytable TO 's3://test-bucket/glob_ss/partitioned' (FORMAT PARQUET, PARTITION_BY (a, b)); - -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/partitioned/**'); ----- -10 - -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/partitioned/*/**'); ----- -10 - -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/partitioned/**/*'); ----- -10 - -query I -FROM GLOB('s3://test-bucket/glob_ss/partitioned/**/*.parquet'); ----- -s3://test-bucket/glob_ss/partitioned/a=0/b=0/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=1/b=2/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=2/b=4/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=3/b=6/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=4/b=8/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=5/b=10/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=6/b=12/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=7/b=14/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=8/b=16/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=9/b=18/data_0.parquet - -query I -FROM GLOB('s3://test-bucket/glob_ss/partitioned/**/*2/*.parquet'); ----- -s3://test-bucket/glob_ss/partitioned/a=1/b=2/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=6/b=12/data_0.parquet - - - -# use multiple ** -query I -FROM GLOB('s3://test-bucket/glob_ss/partitioned/**/*2/**/*.parquet'); ----- -s3://test-bucket/glob_ss/partitioned/a=1/b=2/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=2/b=4/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=6/b=12/data_0.parquet - - - -# fun stuff -statement ok -COPY mytable TO 's3://test-bucket/glob_ss/a/b/a/t0.csv'; -COPY mytable TO 's3://test-bucket/glob_ss/a/b/a/b/t0.csv'; -COPY mytable TO 's3://test-bucket/glob_ss/a/b/a/b/a/t0.csv'; -COPY mytable TO 's3://test-bucket/glob_ss/a/b/a/b/a/b/t0.csv'; - -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/*/**'); ----- -16 - -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/**/*'); ----- -17 - -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/**/a/*'); ----- -3 - -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/**/b/*'); ----- -3 - -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/**/[a-b]/*'); ----- -6 - -statement ok -COPY mytable TO 's3://test-bucket/glob_ss/a/b/a/b/a/b/c/d/e/t0.csv'; - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/b/**/*'); ----- -s3://test-bucket/glob_ss/a/b/a/b/a/b/c/d/e/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/t0.csv -s3://test-bucket/glob_ss/a/b/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/b*/**/*'); ----- -s3://test-bucket/glob_ss/a/b/a/b/a/b/c/d/e/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/t0.csv -s3://test-bucket/glob_ss/a/b/t0.csv -s3://test-bucket/glob_ss/partitioned/a=0/b=0/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=1/b=2/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=2/b=4/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=3/b=6/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=4/b=8/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=5/b=10/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=6/b=12/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=7/b=14/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=8/b=16/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=9/b=18/data_0.parquet - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/b*/*/**/*'); ----- -s3://test-bucket/glob_ss/a/b/a/b/a/b/c/d/e/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/a*/b*/*/**/*'); ----- -s3://test-bucket/glob_ss/a/b/a/b/a/b/c/d/e/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/t0.csv - -query I -FROM GLOB('s3://test-bucket/glob_ss/**/a*/b*/**/*'); ----- -s3://test-bucket/glob_ss/a/b/a/b/a/b/c/d/e/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/a/t0.csv -s3://test-bucket/glob_ss/a/b/a/b/t0.csv -s3://test-bucket/glob_ss/a/b/a/t0.csv -s3://test-bucket/glob_ss/a/b/t0.csv -s3://test-bucket/glob_ss/partitioned/a=0/b=0/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=1/b=2/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=2/b=4/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=3/b=6/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=4/b=8/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=5/b=10/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=6/b=12/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=7/b=14/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=8/b=16/data_0.parquet -s3://test-bucket/glob_ss/partitioned/a=9/b=18/data_0.parquet - -query I -SELECT COUNT(*) FROM GLOB('s3://test-bucket/glob_ss/**'); ----- -18 diff --git a/test/sql/copy/s3/upload_file_parallel.test_slow b/test/sql/copy/s3/upload_file_parallel.test_slow deleted file mode 100644 index dc5086a8eddb..000000000000 --- a/test/sql/copy/s3/upload_file_parallel.test_slow +++ /dev/null @@ -1,122 +0,0 @@ -# name: test/sql/copy/s3/upload_file_parallel.test_slow -# description: Copy large parquet files from and to S3 in parallel. -# group: [s3] - -require tpch - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -CALL DBGEN(sf=1) - -statement ok -set http_timeout=120000; - -# More retries (longest wait will be 25600ms) -statement ok -set http_retries=6; - -query I -SELECT - sum(l_extendedprice * l_discount) AS revenue -FROM - lineitem -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -123141078.2283 - -# We do this in parallel to also test synchronization of s3fs between 2 connections -concurrentloop threadid 0 2 - -statement ok -SET s3_endpoint='${DUCKDB_S3_ENDPOINT}';SET s3_use_ssl=${DUCKDB_S3_USE_SSL}; - -# Parquet file -statement ok -COPY lineitem TO 's3://test-bucket/multipart/export_large_${threadid}.parquet' (FORMAT 'parquet'); - -query I -SELECT - sum(l_extendedprice * l_discount) AS revenue -FROM - "s3://test-bucket/multipart/export_large_${threadid}.parquet" -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -123141078.2283 - -endloop - -statement ok -CALL dbgen(sf=0.01, suffix='_small'); - -query I -SELECT - sum(l_extendedprice * l_discount) AS revenue -FROM - lineitem_small -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -1193053.2253 - -# Upload and query 100 tiny files in parallel -concurrentloop threadid 0 100 - -statement ok -SET s3_secret_access_key='${AWS_SECRET_ACCESS_KEY}';SET s3_access_key_id='${AWS_ACCESS_KEY_ID}';SET s3_region='${AWS_DEFAULT_REGION}'; SET s3_endpoint='${DUCKDB_S3_ENDPOINT}';SET s3_use_ssl=${DUCKDB_S3_USE_SSL}; - -statement ok -SET s3_uploader_thread_limit=1 - -# Parquet file -statement ok -COPY lineitem_small TO 's3://test-bucket/multipart/export_small_${threadid}.parquet' (FORMAT 'parquet'); - -query I -SELECT - sum(l_extendedprice * l_discount) AS revenue -FROM - "s3://test-bucket/multipart/export_small_${threadid}.parquet" -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -1193053.2253 - -endloop diff --git a/test/sql/copy/s3/upload_large_file.test_slow b/test/sql/copy/s3/upload_large_file.test_slow deleted file mode 100644 index 12a07c390e39..000000000000 --- a/test/sql/copy/s3/upload_large_file.test_slow +++ /dev/null @@ -1,79 +0,0 @@ -# name: test/sql/copy/s3/upload_large_file.test_slow -# description: Copy large csv/parquet files from and to S3. -# group: [s3] - -require tpch - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -# confirm we use a reasonable amount of memory -statement ok -SET memory_limit='2.5GB'; - -statement ok -set http_timeout=120000; - -# More retries (longest wait will be 25600ms) -statement ok -set http_retries=6; - -# disable tmp dir to force OOM if we exceed our set limit -statement ok -PRAGMA temp_directory='' - -statement ok -SET s3_uploader_thread_limit = 5; - -statement ok -CALL DBGEN(sf=1) - -query I -SELECT - sum(l_extendedprice * l_discount) AS revenue -FROM - lineitem -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -123141078.2283 - -# Parquet file ~300MB -statement ok -COPY lineitem TO 's3://test-bucket/multipart/export_large.parquet' (FORMAT 'parquet'); - -query I -SELECT - sum(l_extendedprice * l_discount) AS revenue -FROM - "s3://test-bucket/multipart/export_large.parquet" -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -123141078.2283 diff --git a/test/sql/copy/s3/upload_large_json_file.test_slow b/test/sql/copy/s3/upload_large_json_file.test_slow deleted file mode 100644 index b7d77417001e..000000000000 --- a/test/sql/copy/s3/upload_large_json_file.test_slow +++ /dev/null @@ -1,87 +0,0 @@ -# name: test/sql/copy/s3/upload_large_json_file.test_slow -# description: Copy large json files from and to S3. -# group: [s3] - -require tpch - -require json - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -set http_timeout=120000; - -# More retries (longest wait will be 25600ms) -statement ok -set http_retries=6; - -statement ok -CALL DBGEN(sf=0.1) - -query I -SELECT - sum(l_extendedprice * l_discount) AS revenue -FROM - lineitem -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -11803420.2534 - -statement ok -COPY lineitem TO 's3://test-bucket/multipart/export_large.json' (FORMAT 'json'); - -query I -SELECT - sum(l_extendedprice * l_discount) AS revenue -FROM - "s3://test-bucket/multipart/export_large.json" -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -11803420.2534 - -# This query triggers an edge case where we apply an S3-specific optimization using multiple cached filehandles -query I -SELECT - sum(l_extendedprice * l_discount)/3 AS revenue -FROM - read_json_auto([ - 's3://test-bucket/multipart/export_large.json', - 's3://test-bucket/multipart/export_large.json', - 's3://test-bucket/multipart/export_large.json',]) -WHERE - l_shipdate >= CAST('1994-01-01' AS date) - AND l_shipdate < CAST('1995-01-01' AS date) - AND l_discount BETWEEN 0.05 - AND 0.07 - AND l_quantity < 24; ----- -11803420.2534 \ No newline at end of file diff --git a/test/sql/copy/s3/upload_small_file.test b/test/sql/copy/s3/upload_small_file.test deleted file mode 100644 index b2e8cb3d9ce3..000000000000 --- a/test/sql/copy/s3/upload_small_file.test +++ /dev/null @@ -1,77 +0,0 @@ -# name: test/sql/copy/s3/upload_small_file.test -# description: Copy small csv/parquet files from and to S3. -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -CREATE TABLE web_page as (SELECT * FROM "data/csv/real/web_page.csv"); - -query IIIIIIIIIIIIII -SELECT * FROM web_page LIMIT 10; ----- -1 AAAAAAAABAAAAAAA 1997-09-03 NULL 2450810 2452620 Y 98539 http://www.foo.com welcome 2531 8 3 4 -2 AAAAAAAACAAAAAAA 1997-09-03 2000-09-02 2450814 2452580 N NULL http://www.foo.com protected 1564 4 3 1 -3 AAAAAAAACAAAAAAA 2000-09-03 NULL 2450814 2452611 N NULL http://www.foo.com feedback 1564 4 3 4 -4 AAAAAAAAEAAAAAAA 1997-09-03 1999-09-03 2450812 2452579 N NULL http://www.foo.com general 3732 18 7 1 -5 AAAAAAAAEAAAAAAA 1999-09-04 2001-09-02 2450812 2452597 N NULL http://www.foo.com welcome 3732 18 3 1 -6 AAAAAAAAEAAAAAAA 2001-09-03 NULL 2450814 2452597 N NULL http://www.foo.com ad 3732 18 7 4 -7 AAAAAAAAHAAAAAAA 1997-09-03 NULL 2450815 2452574 N NULL http://www.foo.com feedback 3034 18 7 4 -8 AAAAAAAAIAAAAAAA 1997-09-03 2000-09-02 2450815 2452646 Y 1898 http://www.foo.com protected 3128 12 2 4 -9 AAAAAAAAIAAAAAAA 2000-09-03 NULL 2450807 2452579 Y 84146 http://www.foo.com welcome 3128 13 5 3 -10 AAAAAAAAKAAAAAAA 1997-09-03 1999-09-03 NULL 2452623 N NULL http://www.foo.com NULL NULL NULL NULL NULL - -# Parquet file -statement ok -COPY web_page TO 's3://test-bucket/multipart/web_page.parquet' (FORMAT 'parquet'); - -query IIIIIIIIIIIIII -SELECT * FROM "s3://test-bucket/multipart/web_page.parquet" LIMIT 10; ----- -1 AAAAAAAABAAAAAAA 1997-09-03 NULL 2450810 2452620 Y 98539 http://www.foo.com welcome 2531 8 3 4 -2 AAAAAAAACAAAAAAA 1997-09-03 2000-09-02 2450814 2452580 N NULL http://www.foo.com protected 1564 4 3 1 -3 AAAAAAAACAAAAAAA 2000-09-03 NULL 2450814 2452611 N NULL http://www.foo.com feedback 1564 4 3 4 -4 AAAAAAAAEAAAAAAA 1997-09-03 1999-09-03 2450812 2452579 N NULL http://www.foo.com general 3732 18 7 1 -5 AAAAAAAAEAAAAAAA 1999-09-04 2001-09-02 2450812 2452597 N NULL http://www.foo.com welcome 3732 18 3 1 -6 AAAAAAAAEAAAAAAA 2001-09-03 NULL 2450814 2452597 N NULL http://www.foo.com ad 3732 18 7 4 -7 AAAAAAAAHAAAAAAA 1997-09-03 NULL 2450815 2452574 N NULL http://www.foo.com feedback 3034 18 7 4 -8 AAAAAAAAIAAAAAAA 1997-09-03 2000-09-02 2450815 2452646 Y 1898 http://www.foo.com protected 3128 12 2 4 -9 AAAAAAAAIAAAAAAA 2000-09-03 NULL 2450807 2452579 Y 84146 http://www.foo.com welcome 3128 13 5 3 -10 AAAAAAAAKAAAAAAA 1997-09-03 1999-09-03 NULL 2452623 N NULL http://www.foo.com NULL NULL NULL NULL NULL - -# CSV file -statement ok -COPY web_page TO 's3://test-bucket/multipart/web_page.csv'; - -query IIIIIIIIIIIIII -SELECT * FROM "s3://test-bucket/multipart/web_page.csv" LIMIT 10; ----- -1 AAAAAAAABAAAAAAA 1997-09-03 NULL 2450810 2452620 Y 98539 http://www.foo.com welcome 2531 8 3 4 -2 AAAAAAAACAAAAAAA 1997-09-03 2000-09-02 2450814 2452580 N NULL http://www.foo.com protected 1564 4 3 1 -3 AAAAAAAACAAAAAAA 2000-09-03 NULL 2450814 2452611 N NULL http://www.foo.com feedback 1564 4 3 4 -4 AAAAAAAAEAAAAAAA 1997-09-03 1999-09-03 2450812 2452579 N NULL http://www.foo.com general 3732 18 7 1 -5 AAAAAAAAEAAAAAAA 1999-09-04 2001-09-02 2450812 2452597 N NULL http://www.foo.com welcome 3732 18 3 1 -6 AAAAAAAAEAAAAAAA 2001-09-03 NULL 2450814 2452597 N NULL http://www.foo.com ad 3732 18 7 4 -7 AAAAAAAAHAAAAAAA 1997-09-03 NULL 2450815 2452574 N NULL http://www.foo.com feedback 3034 18 7 4 -8 AAAAAAAAIAAAAAAA 1997-09-03 2000-09-02 2450815 2452646 Y 1898 http://www.foo.com protected 3128 12 2 4 -9 AAAAAAAAIAAAAAAA 2000-09-03 NULL 2450807 2452579 Y 84146 http://www.foo.com welcome 3128 13 5 3 -10 AAAAAAAAKAAAAAAA 1997-09-03 1999-09-03 NULL 2452623 N NULL http://www.foo.com NULL NULL NULL NULL NULL diff --git a/test/sql/copy/s3/url_encode.test b/test/sql/copy/s3/url_encode.test deleted file mode 100644 index 66cbd5c80874..000000000000 --- a/test/sql/copy/s3/url_encode.test +++ /dev/null @@ -1,145 +0,0 @@ -# name: test/sql/copy/s3/url_encode.test -# description: S3 Url encoding -# group: [s3] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -# override the default behaviour of skipping HTTP errors and connection failures: this test fails on connection issues -set ignore_error_messages - -statement ok -CREATE TABLE test_1 as (SELECT 1 FROM range(0,5)); -CREATE TABLE test_2 as (SELECT 2 FROM range(0,5)); -CREATE TABLE test_3 as (SELECT 3 FROM range(0,5)); -CREATE TABLE test_4 as (SELECT 4 FROM range(0,5)); - -foreach prefix s3:// r2:// s3a:// s3n:// - -statement ok -COPY test_1 TO '${prefix}test-bucket-public/url_encode/just because you can doesnt mean you should.parquet' (FORMAT 'parquet'); - -statement ok -COPY test_2 TO '${prefix}test-bucket-public/url_encode/just+dont+use+plus+or+spaces+please.parquet' (FORMAT 'parquet'); - -statement ok -COPY test_3 TO '${prefix}test-bucket-public/url_encode/should:avoid:using:colon:in:paths.parquet' (FORMAT 'parquet'); - -# For S3 urls spaces are fine -query I -SELECT * FROM "${prefix}test-bucket-public/url_encode/just because you can doesnt mean you should.parquet" LIMIT 1; ----- -1 - -# In S3 urls, + means a plus symbol -query I -SELECT * FROM "${prefix}test-bucket-public/url_encode/just+dont+use+plus+or+spaces+please.parquet" LIMIT 1; ----- -2 - -# Colons in S3 urls are encoded by duckdb internaly like boto3 (issue #5502) -query I -SELECT * FROM "${prefix}test-bucket-public/url_encode/should:avoid:using:colon:in:paths.parquet" LIMIT 1; ----- -3 - -# NOTE! For HTTP(s) urls, the + symbol is not encoded by duckdb, leaving it up to the server to decide if it should be interpreted -# as a space or a plus. In the case of AWS S3, they are interpreted as encoded spaces, however Minio does not -#query I -#SELECT * FROM "http://test-bucket-public.${DUCKDB_S3_ENDPOINT}/url_encode/just+because+you+can+doesnt+mean+you+should.parquet" LIMIT 1; -#---- -#1 - -# For HTTP urls, we also allow regular spaces, which will get encoded to %20 by duckdb -query I -SELECT * FROM "http://test-bucket-public.${DUCKDB_S3_ENDPOINT}/url_encode/just because you can doesnt mean you should.parquet" LIMIT 1; ----- -1 - -# For HTTP urls from AWS with + symbols, encoding them with %2B is required -query I -SELECT * FROM "http://test-bucket-public.${DUCKDB_S3_ENDPOINT}/url_encode/just%2Bdont%2Buse%2Bplus%2Bor%2Bspaces%2Bplease.parquet" LIMIT 1; ----- -2 - -# However Minio interprets them as spaces so this works too -query I -SELECT * FROM "http://test-bucket-public.${DUCKDB_S3_ENDPOINT}/url_encode/just+dont+use+plus+or+spaces+please.parquet" LIMIT 1; ----- -2 - -# Due to our support for query parameters, this will fail -statement error -COPY test_4 TO '${prefix}test-bucket-public/url_encode/question?marks?are?even?worse.parquet' (FORMAT 'parquet'); ----- -Invalid query parameters found. - -# Enabling url compatibility mode will disable both Globs and query params -# allowing a user to query those hard-to-reach files -statement ok -SET s3_url_compatibility_mode=true; - -statement ok -COPY test_4 TO '${prefix}test-bucket-public/url_encode/question?marks?and*stars[and]brackets.parquet' (FORMAT 'parquet'); - -query I -SELECT * FROM "${prefix}test-bucket-public/url_encode/question?marks?and*stars[and]brackets.parquet" LIMIT 1; ----- -4 - -# HTTP urls will be encoded here -query I -SELECT * FROM "http://test-bucket-public.${DUCKDB_S3_ENDPOINT}/url_encode/question%3Fmarks%3Fand%2Astars%5Band%5Dbrackets.parquet" LIMIT 1; ----- -4 - -statement ok -SET s3_url_compatibility_mode=false; - -# Check that the generated urls are correct -statement ok -set s3_endpoint='s3.some.random.endpoint.com'; - -statement error -SELECT * FROM '${prefix}test-bucket/whatever.parquet'; ----- -Could not establish connection error for HTTP HEAD to 'http://test-bucket.s3.some.random.endpoint.com/whatever.parquet' - -statement ok -set s3_endpoint='${DUCKDB_S3_ENDPOINT}' - -endloop - -# Check that the generated urls are correct for an empty endpoint -statement ok -set s3_endpoint=''; - -statement error -SELECT * FROM 's3://test-bucket/whatever.parquet'; ----- -:.*Unknown error for HTTP HEAD to 'http://test-bucket.s3.eu-west-1.amazonaws.com/whatever.parquet'.* - -statement error -SELECT * FROM 'r2://test-bucket/whatever.parquet'; ----- -:.*Unknown error for HTTP HEAD to 'http://test-bucket.s3.eu-west-1.amazonaws.com/whatever.parquet'.* - -statement error -SELECT * FROM 'gcs://test-bucket/whatever.parquet'; ----- -HTTP GET error on 'http://storage.googleapis.com/test-bucket/whatever.parquet' diff --git a/test/sql/copy/test_remote_head_forbidden.test b/test/sql/copy/test_remote_head_forbidden.test deleted file mode 100644 index 67d2e145b344..000000000000 --- a/test/sql/copy/test_remote_head_forbidden.test +++ /dev/null @@ -1,10 +0,0 @@ -# name: test/sql/copy/test_remote_head_forbidden.test -# description: Test Force download with server that doesn't want to give us the head -# group: [copy] - -require httpfs - -require json - -statement ok -FROM read_json('https://api.spring.io/projects/spring-boot/generations') diff --git a/test/sql/create/create_as.test b/test/sql/create/create_as.test index 9ca6ba0c6f00..50799dee1dd3 100644 --- a/test/sql/create/create_as.test +++ b/test/sql/create/create_as.test @@ -60,11 +60,12 @@ SELECT * FROM tbl1; statement error CREATE TABLE tbl4 IF NOT EXISTS AS SELECT 4; ---- +Parser Error: syntax error at or near "IF" statement error CREATE OR REPLACE TABLE tbl4 IF NOT EXISTS AS SELECT 4; ---- - +Parser Error: syntax error at or near "IF" ### CREATE TABLE t(col1, col2) AS SELECT ... statement ok diff --git a/test/sql/create/create_or_replace.test b/test/sql/create/create_or_replace.test index 84fbfa285e10..5bb294ca8e2f 100644 --- a/test/sql/create/create_or_replace.test +++ b/test/sql/create/create_or_replace.test @@ -17,7 +17,7 @@ CREATE VIEW integers2 AS SELECT 42 statement error CREATE OR REPLACE TABLE integers2(i INTEGER) ---- -is of type View +:.*Catalog Error.*is of type View.* statement ok CREATE TABLE IF NOT EXISTS integers(i INTEGER) @@ -28,3 +28,4 @@ INSERT INTO integers VALUES (1, 2); statement error CREATE OR REPLACE TABLE IF NOT EXISTS integers(i INTEGER) ---- +:.*Parser Error: syntax error.* \ No newline at end of file diff --git a/test/sql/create/create_table_as_error.test b/test/sql/create/create_table_as_error.test index f3da7a42b298..c9261a40bbb1 100644 --- a/test/sql/create/create_table_as_error.test +++ b/test/sql/create/create_table_as_error.test @@ -8,3 +8,4 @@ PRAGMA enable_verification statement error CREATE TABLE tbl AS EXECUTE tbl; ---- +:.*Parser Error.*requires a SELECT clause.* diff --git a/test/sql/create/create_table_compression.test b/test/sql/create/create_table_compression.test index 5c592cc1318c..9bae95efa23b 100644 --- a/test/sql/create/create_table_compression.test +++ b/test/sql/create/create_table_compression.test @@ -24,14 +24,17 @@ CREATE TABLE T (a INTEGER USING COMPRESSION 'bla') statement error CREATE TABLE T (a INTEGER USING COMPRESSION ) ---- +Parser Error: syntax error at or near ")" statement error CREATE TABLE T (a INTEGER NOT NULL USING COMPRESSION ) ---- +Parser Error: syntax error at or near ")" statement error CREATE TABLE T (a INTEGER USING COMPRESSION bla) ---- +Parser Error: Unrecognized option for column compression, expected none, uncompressed, rle, dictionary, pfor, bitpacking, fsst, chimp, patas, zstd, alp, alprd or roaring statement ok CREATE TABLE T (a INTEGER NOT NULL USING COMPRESSION RLE) diff --git a/test/sql/create/create_table_with_arraybounds.test b/test/sql/create/create_table_with_arraybounds.test index fed1682335b5..56cd1a032e9b 100644 --- a/test/sql/create/create_table_with_arraybounds.test +++ b/test/sql/create/create_table_with_arraybounds.test @@ -27,13 +27,18 @@ create type schema2.foo as VARCHAR; statement ok create type db2.schema3.bar as BOOL; -# Create a table with a USER[] type qualified with a schema -statement error +statement ok create table B ( vis schema2.foo[] ); + +statement ok +insert into b values (['foo', 'bar']); + +query I +from b; ---- -syntax error at or near +[foo, bar] # Create a table with a USER[] type qualified with a schema and a catalog statement error @@ -41,4 +46,4 @@ create table B ( vis db2.schema3.bar[] ); ---- -syntax error at or near +Catalog Error: Type with name bar does not exist! diff --git a/test/sql/create/create_using_index.test b/test/sql/create/create_using_index.test index 9b361558e366..643b2ae5a346 100644 --- a/test/sql/create/create_using_index.test +++ b/test/sql/create/create_using_index.test @@ -5,4 +5,5 @@ statement error CREATE TABLE t0 (i INT, CONSTRAINT any_constraint UNIQUE USING INDEX any_non_existed_index); ---- -not supported +Parser Error: UNIQUE USING INDEX is not supported + diff --git a/test/sql/cte/cte_bc.test b/test/sql/cte/cte_bc.test new file mode 100644 index 000000000000..2f97c238764b --- /dev/null +++ b/test/sql/cte/cte_bc.test @@ -0,0 +1,35 @@ +# name: test/sql/cte/cte_bc.test +# description: Test BC of reading CTEs +# group: [cte] + +# The database is written with a vector size of 2048. +require vector_size 2048 + +unzip data/storage/cte_v1.db.gz __TEST_DIR__/cte_v1.db + +unzip data/storage/cte_v1_4.db.gz __TEST_DIR__/cte_v1_4.db + +statement ok +ATTACH '__TEST_DIR__/cte_v1.db' (READ_ONLY) + +statement ok +ATTACH '__TEST_DIR__/cte_v1_4.db' (READ_ONLY) + +foreach cte_db cte_v1 cte_v1_4 + +query I +FROM ${cte_db}.v1 +---- +42 + +query I +FROM ${cte_db}.v2 +---- +42 + +query I +FROM ${cte_db}.v3 +---- +42 + +endloop diff --git a/test/sql/cte/cte_schema.test b/test/sql/cte/cte_schema.test new file mode 100644 index 000000000000..cc7a18093030 --- /dev/null +++ b/test/sql/cte/cte_schema.test @@ -0,0 +1,18 @@ +# name: test/sql/cte/cte_schema.test +# description: Test conflict between CTE and table in different schema +# group: [cte] + +statement ok +create schema s1; + +statement ok +create table s1.tbl(a varchar); + +statement ok +insert into s1.tbl values ('hello'); + +query II +with tbl as (select 'world' b) +select * from s1.tbl, tbl; +---- +hello world diff --git a/test/sql/cte/cte_with_replacement_scan.test b/test/sql/cte/cte_with_replacement_scan.test new file mode 100644 index 000000000000..b4e2b7d0b057 --- /dev/null +++ b/test/sql/cte/cte_with_replacement_scan.test @@ -0,0 +1,19 @@ +# name: test/sql/cte/cte_with_replacement_scan.test +# description: Test CTE with same name as file +# group: [cte] + +require parquet + +statement ok +PRAGMA enable_verification + +statement ok +copy (select 42 as answer) to '__TEST_DIR__/df.parquet'; + +query I +WITH '__TEST_DIR__/df.parquet' AS ( + SELECT answer FROM '__TEST_DIR__/df.parquet' +) +SELECT * FROM '__TEST_DIR__/df.parquet'; +---- +42 diff --git a/test/sql/cte/insert_cte_bug_3417.test b/test/sql/cte/insert_cte_bug_3417.test index a62cab32307e..1b74377821bf 100644 --- a/test/sql/cte/insert_cte_bug_3417.test +++ b/test/sql/cte/insert_cte_bug_3417.test @@ -14,3 +14,4 @@ CREATE TABLE table2 (table1_id INTEGER); statement error INSERT INTO table2 WITH cte AS (INSERT INTO table1 SELECT 1, 2 RETURNING id) SELECT id FROM cte; ---- +:.*Parser Error.*A CTE needs a SELECT.* \ No newline at end of file diff --git a/test/sql/cte/lazy_cte_bind.test b/test/sql/cte/lazy_cte_bind.test new file mode 100644 index 000000000000..abb87262aa00 --- /dev/null +++ b/test/sql/cte/lazy_cte_bind.test @@ -0,0 +1,12 @@ +# name: test/sql/cte/lazy_cte_bind.test +# description: Test that CTE binding is lazy +# group: [cte] + +statement ok +PRAGMA enable_verification + +query I +with cte as (select * from read_parquet('does/not/exist/file.parquet')) +select 42 +---- +42 diff --git a/test/sql/cte/materialized/automatic_cte_materialization.test_slow b/test/sql/cte/materialized/automatic_cte_materialization.test_slow index ce1062a5dc4d..ed4daef4431c 100644 --- a/test/sql/cte/materialized/automatic_cte_materialization.test_slow +++ b/test/sql/cte/materialized/automatic_cte_materialization.test_slow @@ -10,6 +10,9 @@ PRAGMA enable_verification statement ok PRAGMA explain_output='OPTIMIZED_ONLY' +statement ok +SET disabled_optimizers TO 'common_subplan' + statement ok CALL dbgen(sf=0.01); diff --git a/test/sql/cte/materialized/recursive_cte_error_materialized.test b/test/sql/cte/materialized/recursive_cte_error_materialized.test index 065e6929de7c..6ab3b1344ab1 100644 --- a/test/sql/cte/materialized/recursive_cte_error_materialized.test +++ b/test/sql/cte/materialized/recursive_cte_error_materialized.test @@ -29,3 +29,4 @@ SELECT source, path, target FROM tag_hierarchy ; ---- +:.*Conversion Error: Could not convert string.* \ No newline at end of file diff --git a/test/sql/cte/materialized/test_cte_in_cte_materialized.test b/test/sql/cte/materialized/test_cte_in_cte_materialized.test index 13ef3137d3c5..f79f4fc21fe0 100644 --- a/test/sql/cte/materialized/test_cte_in_cte_materialized.test +++ b/test/sql/cte/materialized/test_cte_in_cte_materialized.test @@ -35,6 +35,7 @@ with cte1 as MATERIALIZED (with b as MATERIALIZED (Select i as j from a) select statement error with cte1 as MATERIALIZED (select 42), cte1 as MATERIALIZED (select 42) select * FROM cte1; ---- +:.*Parser Error: Duplicate CTE name.* # refer to CTE in subquery tableref query I @@ -66,3 +67,4 @@ with cte as MATERIALIZED (Select i as j from a) select * from cte where j = (wit statement error with cte as MATERIALIZED (select * from cte) select * from cte ---- +:.*Binder Error.*Circular reference to CTE.* \ No newline at end of file diff --git a/test/sql/cte/materialized/test_recursive_cte_union_all_materialized.test b/test/sql/cte/materialized/test_recursive_cte_union_all_materialized.test index 7ef87572db09..6c01b010ff1b 100644 --- a/test/sql/cte/materialized/test_recursive_cte_union_all_materialized.test +++ b/test/sql/cte/materialized/test_recursive_cte_union_all_materialized.test @@ -97,19 +97,22 @@ SELECT * FROM t; statement error with recursive t as MATERIALIZED (select 1 as x union all select x+1 from t where x < 3 order by x) select * from t ---- +:.*Parser Error.*recursive query is not allowed.* # limit is not allowed in the recursive term of ctes statement error with recursive t as MATERIALIZED (select 1 as x union all select x+1 from t where x < 3 LIMIT 1) select * from t ---- +:.*Parser Error.*recursive query is not allowed.* # offset is not allowed in the recursive term of ctes statement error with recursive t as MATERIALIZED (select 1 as x union all select x+1 from t where x < 3 OFFSET 1) select * from t ---- +:.*Parser Error.*recursive query is not allowed.* # offset is not allowed in the recursive term of ctes statement error with recursive t as MATERIALIZED (select 1 as x union all select x+1 from t where x < 3 LIMIT 1 OFFSET 1) select * from t ---- - +:.*Parser Error.*recursive query is not allowed.* \ No newline at end of file diff --git a/test/sql/cte/recursive_cte_error.test b/test/sql/cte/recursive_cte_error.test index 561128375704..f07ec37080aa 100644 --- a/test/sql/cte/recursive_cte_error.test +++ b/test/sql/cte/recursive_cte_error.test @@ -29,3 +29,4 @@ SELECT source, path, target FROM tag_hierarchy ; ---- +:.*Conversion Error: Could not convert string.* \ No newline at end of file diff --git a/test/sql/cte/recursive_cte_key_variant.test b/test/sql/cte/recursive_cte_key_variant.test index eec5cda7d740..717ed505ccba 100644 --- a/test/sql/cte/recursive_cte_key_variant.test +++ b/test/sql/cte/recursive_cte_key_variant.test @@ -112,45 +112,45 @@ SELECT * FROM recurring.tbl; ---- Catalog Error: Table with name tbl does not exist! +# no using key statement error WITH RECURSIVE tbl2(a) AS (SELECT 1 UNION SELECT a.a + 1 FROM tbl2 AS a, recurring.tbl2 AS b WHERE a.a < 2) SELECT * FROM tbl2; ---- -Invalid Input Error: RECURRING can only be used with USING KEY in recursive CTE. +does not exist statement error WITH RECURSIVE tbl2(a,b) AS (SELECT 1, NULL UNION SELECT a.a+1, a.b FROM tbl2 AS a, (SELECT * FROM recurring.tbl2) AS b WHERE a.a < 2) SELECT * FROM tbl2; ---- -Invalid Input Error: RECURRING can only be used with USING KEY in recursive CTE. +does not exist # second cte references recurring table of first cte while first cte does not statement error WITH RECURSIVE tbl(a, b) USING KEY (a) AS (SELECT 5, 1 UNION SELECT a, b + 1 FROM tbl WHERE b < a), tbl1(a,b) AS (SELECT * FROM recurring.tbl) SELECT * FROM tbl1; ---- -cannot be referenced +does not exist # second cte references recurring table of first like the first cte statement error WITH RECURSIVE tbl(a, b) USING KEY (a) AS (SELECT * FROM ((VALUES (5, 1), (6,1)) UNION SELECT a, b + 1 FROM recurring.tbl WHERE b < a) WHERE a = 5), tbl1(a,b) AS (SELECT * FROM recurring.tbl) SELECT * FROM tbl1; ---- -cannot be referenced +does not exist statement error WITH RECURSIVE tbl2(a) AS (SELECT 1 UNION SELECT a.a+1 FROM tbl2 AS a, (SELECT * FROM recurring.tbl2) AS b WHERE a.a < 2) SELECT * FROM tbl2; ---- -Invalid Input Error: RECURRING can only be used with USING KEY in recursive CTE. +does not exist statement error WITH RECURSIVE tbl(a, b) USING KEY (a) AS (SELECT 5, 1 UNION SELECT a, b + 1 FROM tbl WHERE b < a),tbl1(a,b) AS (SELECT * FROM recurring.tbl) SELECT * FROM tbl1; ---- -cannot be referenced +does not exist statement error WITH RECURSIVE tbl(a, b) USING KEY (a) AS MATERIALIZED (SELECT 5, 1 UNION SELECT a, b + 1 FROM tbl WHERE b < a),tbl1(a,b) AS (SELECT * FROM recurring.tbl) SELECT * FROM tbl1; ---- -cannot be referenced - +does not exist ####################### # Connected components diff --git a/test/sql/cte/test_cte.test b/test/sql/cte/test_cte.test index 57bb31d0ad4e..044df3572b11 100644 --- a/test/sql/cte/test_cte.test +++ b/test/sql/cte/test_cte.test @@ -97,6 +97,14 @@ SELECT 1 UNION ALL (WITH cte AS (SELECT 42) SELECT * FROM cte); 1 42 +# cte in nested set operation node +query I +SELECT 1 UNION ALL (WITH cte AS (SELECT 42) SELECT * FROM cte UNION ALL SELECT * FROM cte); +---- +1 +42 +42 + # cte in recursive cte query I WITH RECURSIVE cte(d) AS ( @@ -220,3 +228,7 @@ SELECT b.x FROM (SELECT 1) _(x), LATERAL (SELECT * FROM cte) b(x) ---- +query III +WITH RECURSIVE rec(a, b, c) AS (select a,b,c from (values(1,2,3),(1,2,3)) s(a,b,c) union select 1,2,3) select * from rec; +---- +1 2 3 \ No newline at end of file diff --git a/test/sql/cte/test_cte_in_cte.test b/test/sql/cte/test_cte_in_cte.test index a10b3d2a24f8..8dff5822e80d 100644 --- a/test/sql/cte/test_cte_in_cte.test +++ b/test/sql/cte/test_cte_in_cte.test @@ -35,6 +35,7 @@ with cte1 as (with b as (Select i as j from a) select j from b), cte2 as (with c statement error with cte1 as (select 42), cte1 as (select 42) select * FROM cte1; ---- +:.*Parser Error: Duplicate CTE name.* # refer to CTE in subquery tableref query I @@ -60,3 +61,4 @@ with cte as (Select i as j from a) select * from cte where j = (with cte as (sel statement error with cte as (select * from cte) select * from cte ---- +:.*Binder Error.*Circular reference to CTE.* \ No newline at end of file diff --git a/test/sql/cte/test_giant_union_all.test_slow b/test/sql/cte/test_giant_union_all.test_slow new file mode 100644 index 000000000000..7e489f14f1e1 --- /dev/null +++ b/test/sql/cte/test_giant_union_all.test_slow @@ -0,0 +1,2010 @@ +# name: test/sql/cte/test_giant_union_all.test_slow +# description: Test giant chain of UNION ALL statements +# group: [cte] + +statement ok +CREATE VIEW union_view AS +SELECT 0 i +UNION ALL +SELECT 1 +UNION ALL +SELECT 2 +UNION ALL +SELECT 3 +UNION ALL +SELECT 4 +UNION ALL +SELECT 5 +UNION ALL +SELECT 6 +UNION ALL +SELECT 7 +UNION ALL +SELECT 8 +UNION ALL +SELECT 9 +UNION ALL +SELECT 10 +UNION ALL +SELECT 11 +UNION ALL +SELECT 12 +UNION ALL +SELECT 13 +UNION ALL +SELECT 14 +UNION ALL +SELECT 15 +UNION ALL +SELECT 16 +UNION ALL +SELECT 17 +UNION ALL +SELECT 18 +UNION ALL +SELECT 19 +UNION ALL +SELECT 20 +UNION ALL +SELECT 21 +UNION ALL +SELECT 22 +UNION ALL +SELECT 23 +UNION ALL +SELECT 24 +UNION ALL +SELECT 25 +UNION ALL +SELECT 26 +UNION ALL +SELECT 27 +UNION ALL +SELECT 28 +UNION ALL +SELECT 29 +UNION ALL +SELECT 30 +UNION ALL +SELECT 31 +UNION ALL +SELECT 32 +UNION ALL +SELECT 33 +UNION ALL +SELECT 34 +UNION ALL +SELECT 35 +UNION ALL +SELECT 36 +UNION ALL +SELECT 37 +UNION ALL +SELECT 38 +UNION ALL +SELECT 39 +UNION ALL +SELECT 40 +UNION ALL +SELECT 41 +UNION ALL +SELECT 42 +UNION ALL +SELECT 43 +UNION ALL +SELECT 44 +UNION ALL +SELECT 45 +UNION ALL +SELECT 46 +UNION ALL +SELECT 47 +UNION ALL +SELECT 48 +UNION ALL +SELECT 49 +UNION ALL +SELECT 50 +UNION ALL +SELECT 51 +UNION ALL +SELECT 52 +UNION ALL +SELECT 53 +UNION ALL +SELECT 54 +UNION ALL +SELECT 55 +UNION ALL +SELECT 56 +UNION ALL +SELECT 57 +UNION ALL +SELECT 58 +UNION ALL +SELECT 59 +UNION ALL +SELECT 60 +UNION ALL +SELECT 61 +UNION ALL +SELECT 62 +UNION ALL +SELECT 63 +UNION ALL +SELECT 64 +UNION ALL +SELECT 65 +UNION ALL +SELECT 66 +UNION ALL +SELECT 67 +UNION ALL +SELECT 68 +UNION ALL +SELECT 69 +UNION ALL +SELECT 70 +UNION ALL +SELECT 71 +UNION ALL +SELECT 72 +UNION ALL +SELECT 73 +UNION ALL +SELECT 74 +UNION ALL +SELECT 75 +UNION ALL +SELECT 76 +UNION ALL +SELECT 77 +UNION ALL +SELECT 78 +UNION ALL +SELECT 79 +UNION ALL +SELECT 80 +UNION ALL +SELECT 81 +UNION ALL +SELECT 82 +UNION ALL +SELECT 83 +UNION ALL +SELECT 84 +UNION ALL +SELECT 85 +UNION ALL +SELECT 86 +UNION ALL +SELECT 87 +UNION ALL +SELECT 88 +UNION ALL +SELECT 89 +UNION ALL +SELECT 90 +UNION ALL +SELECT 91 +UNION ALL +SELECT 92 +UNION ALL +SELECT 93 +UNION ALL +SELECT 94 +UNION ALL +SELECT 95 +UNION ALL +SELECT 96 +UNION ALL +SELECT 97 +UNION ALL +SELECT 98 +UNION ALL +SELECT 99 +UNION ALL +SELECT 100 +UNION ALL +SELECT 101 +UNION ALL +SELECT 102 +UNION ALL +SELECT 103 +UNION ALL +SELECT 104 +UNION ALL +SELECT 105 +UNION ALL +SELECT 106 +UNION ALL +SELECT 107 +UNION ALL +SELECT 108 +UNION ALL +SELECT 109 +UNION ALL +SELECT 110 +UNION ALL +SELECT 111 +UNION ALL +SELECT 112 +UNION ALL +SELECT 113 +UNION ALL +SELECT 114 +UNION ALL +SELECT 115 +UNION ALL +SELECT 116 +UNION ALL +SELECT 117 +UNION ALL +SELECT 118 +UNION ALL +SELECT 119 +UNION ALL +SELECT 120 +UNION ALL +SELECT 121 +UNION ALL +SELECT 122 +UNION ALL +SELECT 123 +UNION ALL +SELECT 124 +UNION ALL +SELECT 125 +UNION ALL +SELECT 126 +UNION ALL +SELECT 127 +UNION ALL +SELECT 128 +UNION ALL +SELECT 129 +UNION ALL +SELECT 130 +UNION ALL +SELECT 131 +UNION ALL +SELECT 132 +UNION ALL +SELECT 133 +UNION ALL +SELECT 134 +UNION ALL +SELECT 135 +UNION ALL +SELECT 136 +UNION ALL +SELECT 137 +UNION ALL +SELECT 138 +UNION ALL +SELECT 139 +UNION ALL +SELECT 140 +UNION ALL +SELECT 141 +UNION ALL +SELECT 142 +UNION ALL +SELECT 143 +UNION ALL +SELECT 144 +UNION ALL +SELECT 145 +UNION ALL +SELECT 146 +UNION ALL +SELECT 147 +UNION ALL +SELECT 148 +UNION ALL +SELECT 149 +UNION ALL +SELECT 150 +UNION ALL +SELECT 151 +UNION ALL +SELECT 152 +UNION ALL +SELECT 153 +UNION ALL +SELECT 154 +UNION ALL +SELECT 155 +UNION ALL +SELECT 156 +UNION ALL +SELECT 157 +UNION ALL +SELECT 158 +UNION ALL +SELECT 159 +UNION ALL +SELECT 160 +UNION ALL +SELECT 161 +UNION ALL +SELECT 162 +UNION ALL +SELECT 163 +UNION ALL +SELECT 164 +UNION ALL +SELECT 165 +UNION ALL +SELECT 166 +UNION ALL +SELECT 167 +UNION ALL +SELECT 168 +UNION ALL +SELECT 169 +UNION ALL +SELECT 170 +UNION ALL +SELECT 171 +UNION ALL +SELECT 172 +UNION ALL +SELECT 173 +UNION ALL +SELECT 174 +UNION ALL +SELECT 175 +UNION ALL +SELECT 176 +UNION ALL +SELECT 177 +UNION ALL +SELECT 178 +UNION ALL +SELECT 179 +UNION ALL +SELECT 180 +UNION ALL +SELECT 181 +UNION ALL +SELECT 182 +UNION ALL +SELECT 183 +UNION ALL +SELECT 184 +UNION ALL +SELECT 185 +UNION ALL +SELECT 186 +UNION ALL +SELECT 187 +UNION ALL +SELECT 188 +UNION ALL +SELECT 189 +UNION ALL +SELECT 190 +UNION ALL +SELECT 191 +UNION ALL +SELECT 192 +UNION ALL +SELECT 193 +UNION ALL +SELECT 194 +UNION ALL +SELECT 195 +UNION ALL +SELECT 196 +UNION ALL +SELECT 197 +UNION ALL +SELECT 198 +UNION ALL +SELECT 199 +UNION ALL +SELECT 200 +UNION ALL +SELECT 201 +UNION ALL +SELECT 202 +UNION ALL +SELECT 203 +UNION ALL +SELECT 204 +UNION ALL +SELECT 205 +UNION ALL +SELECT 206 +UNION ALL +SELECT 207 +UNION ALL +SELECT 208 +UNION ALL +SELECT 209 +UNION ALL +SELECT 210 +UNION ALL +SELECT 211 +UNION ALL +SELECT 212 +UNION ALL +SELECT 213 +UNION ALL +SELECT 214 +UNION ALL +SELECT 215 +UNION ALL +SELECT 216 +UNION ALL +SELECT 217 +UNION ALL +SELECT 218 +UNION ALL +SELECT 219 +UNION ALL +SELECT 220 +UNION ALL +SELECT 221 +UNION ALL +SELECT 222 +UNION ALL +SELECT 223 +UNION ALL +SELECT 224 +UNION ALL +SELECT 225 +UNION ALL +SELECT 226 +UNION ALL +SELECT 227 +UNION ALL +SELECT 228 +UNION ALL +SELECT 229 +UNION ALL +SELECT 230 +UNION ALL +SELECT 231 +UNION ALL +SELECT 232 +UNION ALL +SELECT 233 +UNION ALL +SELECT 234 +UNION ALL +SELECT 235 +UNION ALL +SELECT 236 +UNION ALL +SELECT 237 +UNION ALL +SELECT 238 +UNION ALL +SELECT 239 +UNION ALL +SELECT 240 +UNION ALL +SELECT 241 +UNION ALL +SELECT 242 +UNION ALL +SELECT 243 +UNION ALL +SELECT 244 +UNION ALL +SELECT 245 +UNION ALL +SELECT 246 +UNION ALL +SELECT 247 +UNION ALL +SELECT 248 +UNION ALL +SELECT 249 +UNION ALL +SELECT 250 +UNION ALL +SELECT 251 +UNION ALL +SELECT 252 +UNION ALL +SELECT 253 +UNION ALL +SELECT 254 +UNION ALL +SELECT 255 +UNION ALL +SELECT 256 +UNION ALL +SELECT 257 +UNION ALL +SELECT 258 +UNION ALL +SELECT 259 +UNION ALL +SELECT 260 +UNION ALL +SELECT 261 +UNION ALL +SELECT 262 +UNION ALL +SELECT 263 +UNION ALL +SELECT 264 +UNION ALL +SELECT 265 +UNION ALL +SELECT 266 +UNION ALL +SELECT 267 +UNION ALL +SELECT 268 +UNION ALL +SELECT 269 +UNION ALL +SELECT 270 +UNION ALL +SELECT 271 +UNION ALL +SELECT 272 +UNION ALL +SELECT 273 +UNION ALL +SELECT 274 +UNION ALL +SELECT 275 +UNION ALL +SELECT 276 +UNION ALL +SELECT 277 +UNION ALL +SELECT 278 +UNION ALL +SELECT 279 +UNION ALL +SELECT 280 +UNION ALL +SELECT 281 +UNION ALL +SELECT 282 +UNION ALL +SELECT 283 +UNION ALL +SELECT 284 +UNION ALL +SELECT 285 +UNION ALL +SELECT 286 +UNION ALL +SELECT 287 +UNION ALL +SELECT 288 +UNION ALL +SELECT 289 +UNION ALL +SELECT 290 +UNION ALL +SELECT 291 +UNION ALL +SELECT 292 +UNION ALL +SELECT 293 +UNION ALL +SELECT 294 +UNION ALL +SELECT 295 +UNION ALL +SELECT 296 +UNION ALL +SELECT 297 +UNION ALL +SELECT 298 +UNION ALL +SELECT 299 +UNION ALL +SELECT 300 +UNION ALL +SELECT 301 +UNION ALL +SELECT 302 +UNION ALL +SELECT 303 +UNION ALL +SELECT 304 +UNION ALL +SELECT 305 +UNION ALL +SELECT 306 +UNION ALL +SELECT 307 +UNION ALL +SELECT 308 +UNION ALL +SELECT 309 +UNION ALL +SELECT 310 +UNION ALL +SELECT 311 +UNION ALL +SELECT 312 +UNION ALL +SELECT 313 +UNION ALL +SELECT 314 +UNION ALL +SELECT 315 +UNION ALL +SELECT 316 +UNION ALL +SELECT 317 +UNION ALL +SELECT 318 +UNION ALL +SELECT 319 +UNION ALL +SELECT 320 +UNION ALL +SELECT 321 +UNION ALL +SELECT 322 +UNION ALL +SELECT 323 +UNION ALL +SELECT 324 +UNION ALL +SELECT 325 +UNION ALL +SELECT 326 +UNION ALL +SELECT 327 +UNION ALL +SELECT 328 +UNION ALL +SELECT 329 +UNION ALL +SELECT 330 +UNION ALL +SELECT 331 +UNION ALL +SELECT 332 +UNION ALL +SELECT 333 +UNION ALL +SELECT 334 +UNION ALL +SELECT 335 +UNION ALL +SELECT 336 +UNION ALL +SELECT 337 +UNION ALL +SELECT 338 +UNION ALL +SELECT 339 +UNION ALL +SELECT 340 +UNION ALL +SELECT 341 +UNION ALL +SELECT 342 +UNION ALL +SELECT 343 +UNION ALL +SELECT 344 +UNION ALL +SELECT 345 +UNION ALL +SELECT 346 +UNION ALL +SELECT 347 +UNION ALL +SELECT 348 +UNION ALL +SELECT 349 +UNION ALL +SELECT 350 +UNION ALL +SELECT 351 +UNION ALL +SELECT 352 +UNION ALL +SELECT 353 +UNION ALL +SELECT 354 +UNION ALL +SELECT 355 +UNION ALL +SELECT 356 +UNION ALL +SELECT 357 +UNION ALL +SELECT 358 +UNION ALL +SELECT 359 +UNION ALL +SELECT 360 +UNION ALL +SELECT 361 +UNION ALL +SELECT 362 +UNION ALL +SELECT 363 +UNION ALL +SELECT 364 +UNION ALL +SELECT 365 +UNION ALL +SELECT 366 +UNION ALL +SELECT 367 +UNION ALL +SELECT 368 +UNION ALL +SELECT 369 +UNION ALL +SELECT 370 +UNION ALL +SELECT 371 +UNION ALL +SELECT 372 +UNION ALL +SELECT 373 +UNION ALL +SELECT 374 +UNION ALL +SELECT 375 +UNION ALL +SELECT 376 +UNION ALL +SELECT 377 +UNION ALL +SELECT 378 +UNION ALL +SELECT 379 +UNION ALL +SELECT 380 +UNION ALL +SELECT 381 +UNION ALL +SELECT 382 +UNION ALL +SELECT 383 +UNION ALL +SELECT 384 +UNION ALL +SELECT 385 +UNION ALL +SELECT 386 +UNION ALL +SELECT 387 +UNION ALL +SELECT 388 +UNION ALL +SELECT 389 +UNION ALL +SELECT 390 +UNION ALL +SELECT 391 +UNION ALL +SELECT 392 +UNION ALL +SELECT 393 +UNION ALL +SELECT 394 +UNION ALL +SELECT 395 +UNION ALL +SELECT 396 +UNION ALL +SELECT 397 +UNION ALL +SELECT 398 +UNION ALL +SELECT 399 +UNION ALL +SELECT 400 +UNION ALL +SELECT 401 +UNION ALL +SELECT 402 +UNION ALL +SELECT 403 +UNION ALL +SELECT 404 +UNION ALL +SELECT 405 +UNION ALL +SELECT 406 +UNION ALL +SELECT 407 +UNION ALL +SELECT 408 +UNION ALL +SELECT 409 +UNION ALL +SELECT 410 +UNION ALL +SELECT 411 +UNION ALL +SELECT 412 +UNION ALL +SELECT 413 +UNION ALL +SELECT 414 +UNION ALL +SELECT 415 +UNION ALL +SELECT 416 +UNION ALL +SELECT 417 +UNION ALL +SELECT 418 +UNION ALL +SELECT 419 +UNION ALL +SELECT 420 +UNION ALL +SELECT 421 +UNION ALL +SELECT 422 +UNION ALL +SELECT 423 +UNION ALL +SELECT 424 +UNION ALL +SELECT 425 +UNION ALL +SELECT 426 +UNION ALL +SELECT 427 +UNION ALL +SELECT 428 +UNION ALL +SELECT 429 +UNION ALL +SELECT 430 +UNION ALL +SELECT 431 +UNION ALL +SELECT 432 +UNION ALL +SELECT 433 +UNION ALL +SELECT 434 +UNION ALL +SELECT 435 +UNION ALL +SELECT 436 +UNION ALL +SELECT 437 +UNION ALL +SELECT 438 +UNION ALL +SELECT 439 +UNION ALL +SELECT 440 +UNION ALL +SELECT 441 +UNION ALL +SELECT 442 +UNION ALL +SELECT 443 +UNION ALL +SELECT 444 +UNION ALL +SELECT 445 +UNION ALL +SELECT 446 +UNION ALL +SELECT 447 +UNION ALL +SELECT 448 +UNION ALL +SELECT 449 +UNION ALL +SELECT 450 +UNION ALL +SELECT 451 +UNION ALL +SELECT 452 +UNION ALL +SELECT 453 +UNION ALL +SELECT 454 +UNION ALL +SELECT 455 +UNION ALL +SELECT 456 +UNION ALL +SELECT 457 +UNION ALL +SELECT 458 +UNION ALL +SELECT 459 +UNION ALL +SELECT 460 +UNION ALL +SELECT 461 +UNION ALL +SELECT 462 +UNION ALL +SELECT 463 +UNION ALL +SELECT 464 +UNION ALL +SELECT 465 +UNION ALL +SELECT 466 +UNION ALL +SELECT 467 +UNION ALL +SELECT 468 +UNION ALL +SELECT 469 +UNION ALL +SELECT 470 +UNION ALL +SELECT 471 +UNION ALL +SELECT 472 +UNION ALL +SELECT 473 +UNION ALL +SELECT 474 +UNION ALL +SELECT 475 +UNION ALL +SELECT 476 +UNION ALL +SELECT 477 +UNION ALL +SELECT 478 +UNION ALL +SELECT 479 +UNION ALL +SELECT 480 +UNION ALL +SELECT 481 +UNION ALL +SELECT 482 +UNION ALL +SELECT 483 +UNION ALL +SELECT 484 +UNION ALL +SELECT 485 +UNION ALL +SELECT 486 +UNION ALL +SELECT 487 +UNION ALL +SELECT 488 +UNION ALL +SELECT 489 +UNION ALL +SELECT 490 +UNION ALL +SELECT 491 +UNION ALL +SELECT 492 +UNION ALL +SELECT 493 +UNION ALL +SELECT 494 +UNION ALL +SELECT 495 +UNION ALL +SELECT 496 +UNION ALL +SELECT 497 +UNION ALL +SELECT 498 +UNION ALL +SELECT 499 +UNION ALL +SELECT 500 +UNION ALL +SELECT 501 +UNION ALL +SELECT 502 +UNION ALL +SELECT 503 +UNION ALL +SELECT 504 +UNION ALL +SELECT 505 +UNION ALL +SELECT 506 +UNION ALL +SELECT 507 +UNION ALL +SELECT 508 +UNION ALL +SELECT 509 +UNION ALL +SELECT 510 +UNION ALL +SELECT 511 +UNION ALL +SELECT 512 +UNION ALL +SELECT 513 +UNION ALL +SELECT 514 +UNION ALL +SELECT 515 +UNION ALL +SELECT 516 +UNION ALL +SELECT 517 +UNION ALL +SELECT 518 +UNION ALL +SELECT 519 +UNION ALL +SELECT 520 +UNION ALL +SELECT 521 +UNION ALL +SELECT 522 +UNION ALL +SELECT 523 +UNION ALL +SELECT 524 +UNION ALL +SELECT 525 +UNION ALL +SELECT 526 +UNION ALL +SELECT 527 +UNION ALL +SELECT 528 +UNION ALL +SELECT 529 +UNION ALL +SELECT 530 +UNION ALL +SELECT 531 +UNION ALL +SELECT 532 +UNION ALL +SELECT 533 +UNION ALL +SELECT 534 +UNION ALL +SELECT 535 +UNION ALL +SELECT 536 +UNION ALL +SELECT 537 +UNION ALL +SELECT 538 +UNION ALL +SELECT 539 +UNION ALL +SELECT 540 +UNION ALL +SELECT 541 +UNION ALL +SELECT 542 +UNION ALL +SELECT 543 +UNION ALL +SELECT 544 +UNION ALL +SELECT 545 +UNION ALL +SELECT 546 +UNION ALL +SELECT 547 +UNION ALL +SELECT 548 +UNION ALL +SELECT 549 +UNION ALL +SELECT 550 +UNION ALL +SELECT 551 +UNION ALL +SELECT 552 +UNION ALL +SELECT 553 +UNION ALL +SELECT 554 +UNION ALL +SELECT 555 +UNION ALL +SELECT 556 +UNION ALL +SELECT 557 +UNION ALL +SELECT 558 +UNION ALL +SELECT 559 +UNION ALL +SELECT 560 +UNION ALL +SELECT 561 +UNION ALL +SELECT 562 +UNION ALL +SELECT 563 +UNION ALL +SELECT 564 +UNION ALL +SELECT 565 +UNION ALL +SELECT 566 +UNION ALL +SELECT 567 +UNION ALL +SELECT 568 +UNION ALL +SELECT 569 +UNION ALL +SELECT 570 +UNION ALL +SELECT 571 +UNION ALL +SELECT 572 +UNION ALL +SELECT 573 +UNION ALL +SELECT 574 +UNION ALL +SELECT 575 +UNION ALL +SELECT 576 +UNION ALL +SELECT 577 +UNION ALL +SELECT 578 +UNION ALL +SELECT 579 +UNION ALL +SELECT 580 +UNION ALL +SELECT 581 +UNION ALL +SELECT 582 +UNION ALL +SELECT 583 +UNION ALL +SELECT 584 +UNION ALL +SELECT 585 +UNION ALL +SELECT 586 +UNION ALL +SELECT 587 +UNION ALL +SELECT 588 +UNION ALL +SELECT 589 +UNION ALL +SELECT 590 +UNION ALL +SELECT 591 +UNION ALL +SELECT 592 +UNION ALL +SELECT 593 +UNION ALL +SELECT 594 +UNION ALL +SELECT 595 +UNION ALL +SELECT 596 +UNION ALL +SELECT 597 +UNION ALL +SELECT 598 +UNION ALL +SELECT 599 +UNION ALL +SELECT 600 +UNION ALL +SELECT 601 +UNION ALL +SELECT 602 +UNION ALL +SELECT 603 +UNION ALL +SELECT 604 +UNION ALL +SELECT 605 +UNION ALL +SELECT 606 +UNION ALL +SELECT 607 +UNION ALL +SELECT 608 +UNION ALL +SELECT 609 +UNION ALL +SELECT 610 +UNION ALL +SELECT 611 +UNION ALL +SELECT 612 +UNION ALL +SELECT 613 +UNION ALL +SELECT 614 +UNION ALL +SELECT 615 +UNION ALL +SELECT 616 +UNION ALL +SELECT 617 +UNION ALL +SELECT 618 +UNION ALL +SELECT 619 +UNION ALL +SELECT 620 +UNION ALL +SELECT 621 +UNION ALL +SELECT 622 +UNION ALL +SELECT 623 +UNION ALL +SELECT 624 +UNION ALL +SELECT 625 +UNION ALL +SELECT 626 +UNION ALL +SELECT 627 +UNION ALL +SELECT 628 +UNION ALL +SELECT 629 +UNION ALL +SELECT 630 +UNION ALL +SELECT 631 +UNION ALL +SELECT 632 +UNION ALL +SELECT 633 +UNION ALL +SELECT 634 +UNION ALL +SELECT 635 +UNION ALL +SELECT 636 +UNION ALL +SELECT 637 +UNION ALL +SELECT 638 +UNION ALL +SELECT 639 +UNION ALL +SELECT 640 +UNION ALL +SELECT 641 +UNION ALL +SELECT 642 +UNION ALL +SELECT 643 +UNION ALL +SELECT 644 +UNION ALL +SELECT 645 +UNION ALL +SELECT 646 +UNION ALL +SELECT 647 +UNION ALL +SELECT 648 +UNION ALL +SELECT 649 +UNION ALL +SELECT 650 +UNION ALL +SELECT 651 +UNION ALL +SELECT 652 +UNION ALL +SELECT 653 +UNION ALL +SELECT 654 +UNION ALL +SELECT 655 +UNION ALL +SELECT 656 +UNION ALL +SELECT 657 +UNION ALL +SELECT 658 +UNION ALL +SELECT 659 +UNION ALL +SELECT 660 +UNION ALL +SELECT 661 +UNION ALL +SELECT 662 +UNION ALL +SELECT 663 +UNION ALL +SELECT 664 +UNION ALL +SELECT 665 +UNION ALL +SELECT 666 +UNION ALL +SELECT 667 +UNION ALL +SELECT 668 +UNION ALL +SELECT 669 +UNION ALL +SELECT 670 +UNION ALL +SELECT 671 +UNION ALL +SELECT 672 +UNION ALL +SELECT 673 +UNION ALL +SELECT 674 +UNION ALL +SELECT 675 +UNION ALL +SELECT 676 +UNION ALL +SELECT 677 +UNION ALL +SELECT 678 +UNION ALL +SELECT 679 +UNION ALL +SELECT 680 +UNION ALL +SELECT 681 +UNION ALL +SELECT 682 +UNION ALL +SELECT 683 +UNION ALL +SELECT 684 +UNION ALL +SELECT 685 +UNION ALL +SELECT 686 +UNION ALL +SELECT 687 +UNION ALL +SELECT 688 +UNION ALL +SELECT 689 +UNION ALL +SELECT 690 +UNION ALL +SELECT 691 +UNION ALL +SELECT 692 +UNION ALL +SELECT 693 +UNION ALL +SELECT 694 +UNION ALL +SELECT 695 +UNION ALL +SELECT 696 +UNION ALL +SELECT 697 +UNION ALL +SELECT 698 +UNION ALL +SELECT 699 +UNION ALL +SELECT 700 +UNION ALL +SELECT 701 +UNION ALL +SELECT 702 +UNION ALL +SELECT 703 +UNION ALL +SELECT 704 +UNION ALL +SELECT 705 +UNION ALL +SELECT 706 +UNION ALL +SELECT 707 +UNION ALL +SELECT 708 +UNION ALL +SELECT 709 +UNION ALL +SELECT 710 +UNION ALL +SELECT 711 +UNION ALL +SELECT 712 +UNION ALL +SELECT 713 +UNION ALL +SELECT 714 +UNION ALL +SELECT 715 +UNION ALL +SELECT 716 +UNION ALL +SELECT 717 +UNION ALL +SELECT 718 +UNION ALL +SELECT 719 +UNION ALL +SELECT 720 +UNION ALL +SELECT 721 +UNION ALL +SELECT 722 +UNION ALL +SELECT 723 +UNION ALL +SELECT 724 +UNION ALL +SELECT 725 +UNION ALL +SELECT 726 +UNION ALL +SELECT 727 +UNION ALL +SELECT 728 +UNION ALL +SELECT 729 +UNION ALL +SELECT 730 +UNION ALL +SELECT 731 +UNION ALL +SELECT 732 +UNION ALL +SELECT 733 +UNION ALL +SELECT 734 +UNION ALL +SELECT 735 +UNION ALL +SELECT 736 +UNION ALL +SELECT 737 +UNION ALL +SELECT 738 +UNION ALL +SELECT 739 +UNION ALL +SELECT 740 +UNION ALL +SELECT 741 +UNION ALL +SELECT 742 +UNION ALL +SELECT 743 +UNION ALL +SELECT 744 +UNION ALL +SELECT 745 +UNION ALL +SELECT 746 +UNION ALL +SELECT 747 +UNION ALL +SELECT 748 +UNION ALL +SELECT 749 +UNION ALL +SELECT 750 +UNION ALL +SELECT 751 +UNION ALL +SELECT 752 +UNION ALL +SELECT 753 +UNION ALL +SELECT 754 +UNION ALL +SELECT 755 +UNION ALL +SELECT 756 +UNION ALL +SELECT 757 +UNION ALL +SELECT 758 +UNION ALL +SELECT 759 +UNION ALL +SELECT 760 +UNION ALL +SELECT 761 +UNION ALL +SELECT 762 +UNION ALL +SELECT 763 +UNION ALL +SELECT 764 +UNION ALL +SELECT 765 +UNION ALL +SELECT 766 +UNION ALL +SELECT 767 +UNION ALL +SELECT 768 +UNION ALL +SELECT 769 +UNION ALL +SELECT 770 +UNION ALL +SELECT 771 +UNION ALL +SELECT 772 +UNION ALL +SELECT 773 +UNION ALL +SELECT 774 +UNION ALL +SELECT 775 +UNION ALL +SELECT 776 +UNION ALL +SELECT 777 +UNION ALL +SELECT 778 +UNION ALL +SELECT 779 +UNION ALL +SELECT 780 +UNION ALL +SELECT 781 +UNION ALL +SELECT 782 +UNION ALL +SELECT 783 +UNION ALL +SELECT 784 +UNION ALL +SELECT 785 +UNION ALL +SELECT 786 +UNION ALL +SELECT 787 +UNION ALL +SELECT 788 +UNION ALL +SELECT 789 +UNION ALL +SELECT 790 +UNION ALL +SELECT 791 +UNION ALL +SELECT 792 +UNION ALL +SELECT 793 +UNION ALL +SELECT 794 +UNION ALL +SELECT 795 +UNION ALL +SELECT 796 +UNION ALL +SELECT 797 +UNION ALL +SELECT 798 +UNION ALL +SELECT 799 +UNION ALL +SELECT 800 +UNION ALL +SELECT 801 +UNION ALL +SELECT 802 +UNION ALL +SELECT 803 +UNION ALL +SELECT 804 +UNION ALL +SELECT 805 +UNION ALL +SELECT 806 +UNION ALL +SELECT 807 +UNION ALL +SELECT 808 +UNION ALL +SELECT 809 +UNION ALL +SELECT 810 +UNION ALL +SELECT 811 +UNION ALL +SELECT 812 +UNION ALL +SELECT 813 +UNION ALL +SELECT 814 +UNION ALL +SELECT 815 +UNION ALL +SELECT 816 +UNION ALL +SELECT 817 +UNION ALL +SELECT 818 +UNION ALL +SELECT 819 +UNION ALL +SELECT 820 +UNION ALL +SELECT 821 +UNION ALL +SELECT 822 +UNION ALL +SELECT 823 +UNION ALL +SELECT 824 +UNION ALL +SELECT 825 +UNION ALL +SELECT 826 +UNION ALL +SELECT 827 +UNION ALL +SELECT 828 +UNION ALL +SELECT 829 +UNION ALL +SELECT 830 +UNION ALL +SELECT 831 +UNION ALL +SELECT 832 +UNION ALL +SELECT 833 +UNION ALL +SELECT 834 +UNION ALL +SELECT 835 +UNION ALL +SELECT 836 +UNION ALL +SELECT 837 +UNION ALL +SELECT 838 +UNION ALL +SELECT 839 +UNION ALL +SELECT 840 +UNION ALL +SELECT 841 +UNION ALL +SELECT 842 +UNION ALL +SELECT 843 +UNION ALL +SELECT 844 +UNION ALL +SELECT 845 +UNION ALL +SELECT 846 +UNION ALL +SELECT 847 +UNION ALL +SELECT 848 +UNION ALL +SELECT 849 +UNION ALL +SELECT 850 +UNION ALL +SELECT 851 +UNION ALL +SELECT 852 +UNION ALL +SELECT 853 +UNION ALL +SELECT 854 +UNION ALL +SELECT 855 +UNION ALL +SELECT 856 +UNION ALL +SELECT 857 +UNION ALL +SELECT 858 +UNION ALL +SELECT 859 +UNION ALL +SELECT 860 +UNION ALL +SELECT 861 +UNION ALL +SELECT 862 +UNION ALL +SELECT 863 +UNION ALL +SELECT 864 +UNION ALL +SELECT 865 +UNION ALL +SELECT 866 +UNION ALL +SELECT 867 +UNION ALL +SELECT 868 +UNION ALL +SELECT 869 +UNION ALL +SELECT 870 +UNION ALL +SELECT 871 +UNION ALL +SELECT 872 +UNION ALL +SELECT 873 +UNION ALL +SELECT 874 +UNION ALL +SELECT 875 +UNION ALL +SELECT 876 +UNION ALL +SELECT 877 +UNION ALL +SELECT 878 +UNION ALL +SELECT 879 +UNION ALL +SELECT 880 +UNION ALL +SELECT 881 +UNION ALL +SELECT 882 +UNION ALL +SELECT 883 +UNION ALL +SELECT 884 +UNION ALL +SELECT 885 +UNION ALL +SELECT 886 +UNION ALL +SELECT 887 +UNION ALL +SELECT 888 +UNION ALL +SELECT 889 +UNION ALL +SELECT 890 +UNION ALL +SELECT 891 +UNION ALL +SELECT 892 +UNION ALL +SELECT 893 +UNION ALL +SELECT 894 +UNION ALL +SELECT 895 +UNION ALL +SELECT 896 +UNION ALL +SELECT 897 +UNION ALL +SELECT 898 +UNION ALL +SELECT 899 +UNION ALL +SELECT 900 +UNION ALL +SELECT 901 +UNION ALL +SELECT 902 +UNION ALL +SELECT 903 +UNION ALL +SELECT 904 +UNION ALL +SELECT 905 +UNION ALL +SELECT 906 +UNION ALL +SELECT 907 +UNION ALL +SELECT 908 +UNION ALL +SELECT 909 +UNION ALL +SELECT 910 +UNION ALL +SELECT 911 +UNION ALL +SELECT 912 +UNION ALL +SELECT 913 +UNION ALL +SELECT 914 +UNION ALL +SELECT 915 +UNION ALL +SELECT 916 +UNION ALL +SELECT 917 +UNION ALL +SELECT 918 +UNION ALL +SELECT 919 +UNION ALL +SELECT 920 +UNION ALL +SELECT 921 +UNION ALL +SELECT 922 +UNION ALL +SELECT 923 +UNION ALL +SELECT 924 +UNION ALL +SELECT 925 +UNION ALL +SELECT 926 +UNION ALL +SELECT 927 +UNION ALL +SELECT 928 +UNION ALL +SELECT 929 +UNION ALL +SELECT 930 +UNION ALL +SELECT 931 +UNION ALL +SELECT 932 +UNION ALL +SELECT 933 +UNION ALL +SELECT 934 +UNION ALL +SELECT 935 +UNION ALL +SELECT 936 +UNION ALL +SELECT 937 +UNION ALL +SELECT 938 +UNION ALL +SELECT 939 +UNION ALL +SELECT 940 +UNION ALL +SELECT 941 +UNION ALL +SELECT 942 +UNION ALL +SELECT 943 +UNION ALL +SELECT 944 +UNION ALL +SELECT 945 +UNION ALL +SELECT 946 +UNION ALL +SELECT 947 +UNION ALL +SELECT 948 +UNION ALL +SELECT 949 +UNION ALL +SELECT 950 +UNION ALL +SELECT 951 +UNION ALL +SELECT 952 +UNION ALL +SELECT 953 +UNION ALL +SELECT 954 +UNION ALL +SELECT 955 +UNION ALL +SELECT 956 +UNION ALL +SELECT 957 +UNION ALL +SELECT 958 +UNION ALL +SELECT 959 +UNION ALL +SELECT 960 +UNION ALL +SELECT 961 +UNION ALL +SELECT 962 +UNION ALL +SELECT 963 +UNION ALL +SELECT 964 +UNION ALL +SELECT 965 +UNION ALL +SELECT 966 +UNION ALL +SELECT 967 +UNION ALL +SELECT 968 +UNION ALL +SELECT 969 +UNION ALL +SELECT 970 +UNION ALL +SELECT 971 +UNION ALL +SELECT 972 +UNION ALL +SELECT 973 +UNION ALL +SELECT 974 +UNION ALL +SELECT 975 +UNION ALL +SELECT 976 +UNION ALL +SELECT 977 +UNION ALL +SELECT 978 +UNION ALL +SELECT 979 +UNION ALL +SELECT 980 +UNION ALL +SELECT 981 +UNION ALL +SELECT 982 +UNION ALL +SELECT 983 +UNION ALL +SELECT 984 +UNION ALL +SELECT 985 +UNION ALL +SELECT 986 +UNION ALL +SELECT 987 +UNION ALL +SELECT 988 +UNION ALL +SELECT 989 +UNION ALL +SELECT 990 +UNION ALL +SELECT 991 +UNION ALL +SELECT 992 +UNION ALL +SELECT 993 +UNION ALL +SELECT 994 +UNION ALL +SELECT 995 +UNION ALL +SELECT 996 +UNION ALL +SELECT 997 +UNION ALL +SELECT 998 +UNION ALL +SELECT 999 + +query IIII +SELECT COUNT(*), MIN(i), MAX(i), AVG(i) FROM union_view +---- +1000 0 999 499.5 diff --git a/test/sql/cte/test_moderate_union_all.test b/test/sql/cte/test_moderate_union_all.test new file mode 100644 index 000000000000..e2e60152a43b --- /dev/null +++ b/test/sql/cte/test_moderate_union_all.test @@ -0,0 +1,110 @@ +# name: test/sql/cte/test_moderate_union_all.test +# description: Test moderate chain of UNION ALL statements +# group: [cte] + +statement ok +CREATE VIEW union_view AS +SELECT 0 i +UNION ALL +SELECT 1 +UNION ALL +SELECT 2 +UNION ALL +SELECT 3 +UNION ALL +SELECT 4 +UNION ALL +SELECT 5 +UNION ALL +SELECT 6 +UNION ALL +SELECT 7 +UNION ALL +SELECT 8 +UNION ALL +SELECT 9 +UNION ALL +SELECT 10 +UNION ALL +SELECT 11 +UNION ALL +SELECT 12 +UNION ALL +SELECT 13 +UNION ALL +SELECT 14 +UNION ALL +SELECT 15 +UNION ALL +SELECT 16 +UNION ALL +SELECT 17 +UNION ALL +SELECT 18 +UNION ALL +SELECT 19 +UNION ALL +SELECT 20 +UNION ALL +SELECT 21 +UNION ALL +SELECT 22 +UNION ALL +SELECT 23 +UNION ALL +SELECT 24 +UNION ALL +SELECT 25 +UNION ALL +SELECT 26 +UNION ALL +SELECT 27 +UNION ALL +SELECT 28 +UNION ALL +SELECT 29 +UNION ALL +SELECT 30 +UNION ALL +SELECT 31 +UNION ALL +SELECT 32 +UNION ALL +SELECT 33 +UNION ALL +SELECT 34 +UNION ALL +SELECT 35 +UNION ALL +SELECT 36 +UNION ALL +SELECT 37 +UNION ALL +SELECT 38 +UNION ALL +SELECT 39 +UNION ALL +SELECT 40 +UNION ALL +SELECT 41 +UNION ALL +SELECT 42 +UNION ALL +SELECT 43 +UNION ALL +SELECT 44 +UNION ALL +SELECT 45 +UNION ALL +SELECT 46 +UNION ALL +SELECT 47 +UNION ALL +SELECT 48 +UNION ALL +SELECT 49 + +query IIII +SELECT COUNT(*), MIN(i), MAX(i), AVG(i) FROM union_view +---- +50 0 49 24.5 diff --git a/test/sql/cte/test_recursive_cte_union_all.test b/test/sql/cte/test_recursive_cte_union_all.test index f6dccbf413c8..4210f3192604 100644 --- a/test/sql/cte/test_recursive_cte_union_all.test +++ b/test/sql/cte/test_recursive_cte_union_all.test @@ -123,19 +123,22 @@ SELECT * FROM t; statement error with recursive t as (select 1 as x union all select x+1 from t where x < 3 order by x) select * from t ---- +:.*Parser Error.*recursive query is not allowed.* # limit is not allowed in the recursive term of ctes statement error with recursive t as (select 1 as x union all select x+1 from t where x < 3 LIMIT 1) select * from t ---- +:.*Parser Error.*recursive query is not allowed.* # offset is not allowed in the recursive term of ctes statement error with recursive t as (select 1 as x union all select x+1 from t where x < 3 OFFSET 1) select * from t ---- +:.*Parser Error.*recursive query is not allowed.* # offset is not allowed in the recursive term of ctes statement error with recursive t as (select 1 as x union all select x+1 from t where x < 3 LIMIT 1 OFFSET 1) select * from t ---- - +:.*Parser Error.*recursive query is not allowed.* \ No newline at end of file diff --git a/test/sql/delete/test_issue_1834.test_slow b/test/sql/delete/test_issue_1834.test_slow deleted file mode 100644 index 029f6a737776..000000000000 --- a/test/sql/delete/test_issue_1834.test_slow +++ /dev/null @@ -1,26 +0,0 @@ -# name: test/sql/delete/test_issue_1834.test_slow -# description: Deleting with DELETE USING causes a segmentation fault -# group: [delete] - -require httpfs - -statement ok -CREATE TABLE Person_likes_Comment (creationDate timestamp without time zone not null, id bigint not null, likes_Comment bigint not null); - -statement ok -CREATE TABLE Person_Delete_candidates (deletionDate timestamp without time zone not null, id bigint); - -statement ok -COPY Person_likes_Comment FROM 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/Person_likes_Comment.csv' (DELIMITER '|', TIMESTAMPFORMAT '%Y-%m-%dT%H:%M:%S.%g+00:00'); - -statement ok -COPY Person_Delete_candidates FROM 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/Person_Delete_candidates.csv' (DELIMITER '|', HEADER, TIMESTAMPFORMAT '%Y-%m-%dT%H:%M:%S.%g+00:00'); - -statement ok -DELETE FROM Person_likes_Comment USING Person_Delete_candidates WHERE Person_Delete_candidates.id = Person_likes_Comment.id; - -# all tuples fulfilling this predicate should have been deleted -query I -SELECT COUNT(*) FROM Person_likes_Comment, Person_Delete_candidates WHERE Person_Delete_candidates.id = Person_likes_Comment.id; ----- -0 diff --git a/test/sql/delete/test_using_delete.test b/test/sql/delete/test_using_delete.test index ea8f98ef9481..7071b50ef250 100644 --- a/test/sql/delete/test_using_delete.test +++ b/test/sql/delete/test_using_delete.test @@ -82,8 +82,10 @@ SELECT * FROM a; statement error DELETE FROM a USING b WHERE a.i=b.i; ---- +:.*Catalog Error.*does not exist.* # column does not exist statement error DELETE FROM a USING a b WHERE a.i=b.j; ---- +:.*Binder Error.*does not have a column.* \ No newline at end of file diff --git a/test/sql/error/error_position.test b/test/sql/error/error_position.test index a8e5d959cfac..33f5a19fcf91 100644 --- a/test/sql/error/error_position.test +++ b/test/sql/error/error_position.test @@ -2,14 +2,12 @@ # description: Test error position # group: [error] -# error within table macro -statement ok -create macro checksum(x) as table SELECT bit_xor(md5_number(CAST(COLUMNS(*) AS VARCHAR))) FROM query_table(table_name); - statement ok set errors_as_json=true; +# error within table macro statement error -select * from checksum('tbl'); +create macro checksum(x) as table SELECT bit_xor(md5_number(CAST(COLUMNS(*) AS VARCHAR))) FROM query_table(table_name); ---- -:.*"position".*:.*"14".* +:.*"position".*:.*"95".* + diff --git a/test/sql/error/escape_percent_sign.test b/test/sql/error/escape_percent_sign.test index 15e808d1e37d..59f1d784582a 100644 --- a/test/sql/error/escape_percent_sign.test +++ b/test/sql/error/escape_percent_sign.test @@ -9,3 +9,4 @@ SELECT case when i%2 <> 0 then [1] else NULL end FROM range(10000) tbl(i); statement error select count(*) from list_int where l is distinct from NULL; ---- +:.*Binder Error.*not found.* \ No newline at end of file diff --git a/test/sql/error/incorrect_sql.test b/test/sql/error/incorrect_sql.test index 2e02404da4a9..6c0dfd2bb881 100644 --- a/test/sql/error/incorrect_sql.test +++ b/test/sql/error/incorrect_sql.test @@ -6,14 +6,17 @@ statement error SELEC 42; ---- +Parser Error: syntax error at or near "SELEC" statement error SELEC 42, 'thisisareallylongstringloremipsumblablathisisareallylongstringloremipsumblablalthisisareallylongstringloremipsumblablalthisisareallylongstringloremipsumblablal'; ---- +Parser Error: syntax error at or near "SELEC" statement error SELEC 42, '🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆🦆'; ---- +Parser Error: syntax error at or near "SELEC" # unrecognized column statement error @@ -58,17 +61,20 @@ SELECT * FROM READ_CSV('x', hello=3); statement error SELECT 42 WHERE 1=1 WHERE 1=0; ---- +Parser Error: syntax error at or near "WHERE" # multiple statements without semi colon statement error SELECT 42 SELECT 42; ---- +Parser Error: syntax error at or near "SELECT" # multiple statements, but error is only in second statement statement error SELECT 42; SELEC 42; ---- +Parser Error: syntax error at or near "SELEC" # non-existent table statement error diff --git a/test/sql/error/subquery_single_column.test b/test/sql/error/subquery_single_column.test index a8c38f32a824..6e0abb04e3ce 100644 --- a/test/sql/error/subquery_single_column.test +++ b/test/sql/error/subquery_single_column.test @@ -6,3 +6,4 @@ statement error SELECT (SELECT 42, 84) ---- +:.*Binder Error: Subquery returns.* \ No newline at end of file diff --git a/test/sql/error/unknown_window_function.test b/test/sql/error/unknown_window_function.test index b9b5bf769575..4b409d30ac90 100644 --- a/test/sql/error/unknown_window_function.test +++ b/test/sql/error/unknown_window_function.test @@ -5,3 +5,4 @@ statement error SELECT substr('hello', 3, 2) OVER (); ---- +:.*Catalog Error: substr is not an aggregate function.* \ No newline at end of file diff --git a/test/sql/explain/test_explain_analyze.test b/test/sql/explain/test_explain_analyze.test index 70a3351d9edf..8d55d8cb9730 100644 --- a/test/sql/explain/test_explain_analyze.test +++ b/test/sql/explain/test_explain_analyze.test @@ -86,4 +86,13 @@ analyzed_plan :.+EXPLAIN_ANALYZE.* query II EXPLAIN (ANALYZE, FORMAT html) SELECT SUM(i) FROM integers ---- -analyzed_plan :.+
EXPLAIN_ANALYZE
.* \ No newline at end of file +analyzed_plan :.+
EXPLAIN_ANALYZE
.* + +# Make sure that EXPLAIN ANALYZE still works when no output is specified +statement ok +PRAGMA enable_profiling = 'no_output'; + +query II +EXPLAIN ANALYZE SELECT 42; +---- +analyzed_plan :.+EXPLAIN_ANALYZE.* diff --git a/test/sql/export/export_compression_level.test b/test/sql/export/export_compression_level.test index 78a94b05347e..b4b380616aa8 100644 --- a/test/sql/export/export_compression_level.test +++ b/test/sql/export/export_compression_level.test @@ -19,7 +19,7 @@ INSERT INTO test_table VALUES (3, 'Charlie', 234.56, '2024-01-03 14:15:00'); statement ok -EXPORT DATABASE 'my_duckdb' ( +EXPORT DATABASE '__TEST_DIR__/my_duckdb' ( FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 12, @@ -30,7 +30,7 @@ statement ok DROP TABLE test_table; statement ok -IMPORT DATABASE 'my_duckdb'; +IMPORT DATABASE '__TEST_DIR__/my_duckdb'; query IIII FROM test_table; diff --git a/test/sql/export/export_generated_columns.test b/test/sql/export/export_generated_columns.test index 24704e80d783..174c8650d322 100644 --- a/test/sql/export/export_generated_columns.test +++ b/test/sql/export/export_generated_columns.test @@ -30,11 +30,13 @@ INSERT INTO tbl VALUES(5); statement error INSERT INTO tbl VALUES(2,3) ---- +:.*Binder Error: table tbl has 1 columns.* # 'x' can not be removed, as 'gen_x' depends on it statement error ALTER TABLE tbl DROP COLUMN x; ---- +:.*Catalog Error.*column is a dependency.* statement ok EXPORT DATABASE '__TEST_DIR__/export_generated_columns' (FORMAT CSV); @@ -55,13 +57,15 @@ SELECT * FROM tbl statement error INSERT INTO tbl VALUES(2,3) ---- +:.*Binder Error: table tbl has 1 columns.* statement error drop macro my_macro; ---- -Dependency Error: Cannot drop entry "my_macro" because there are entries that depend on it +:.*Dependency Error.*Cannot drop entry.* # 'x' can not be removed, as 'gen_x' depends on it statement error ALTER TABLE tbl DROP COLUMN x; ---- +:.*Catalog Error.*column is a dependency.* \ No newline at end of file diff --git a/test/sql/export/export_hive_path.test b/test/sql/export/export_hive_path.test new file mode 100644 index 000000000000..1c644206130f --- /dev/null +++ b/test/sql/export/export_hive_path.test @@ -0,0 +1,26 @@ +# name: test/sql/export/export_hive_path.test +# description: Test export database to hive path +# group: [export] + +statement ok +BEGIN + +statement ok +CREATE TABLE integers(i INTEGER); + +statement ok +INSERT INTO integers VALUES (42); + +statement ok +EXPORT DATABASE '__TEST_DIR__/d=1992-01-01' + +statement ok +ROLLBACK + +statement ok +IMPORT DATABASE '__TEST_DIR__/d=1992-01-01' + +query I +FROM integers +---- +42 diff --git a/test/sql/export/parquet_export.test b/test/sql/export/parquet_export.test index 0db0fc6b42dd..28d44e459342 100644 --- a/test/sql/export/parquet_export.test +++ b/test/sql/export/parquet_export.test @@ -38,6 +38,7 @@ SELECT SUM(i), SUM(j) FROM integers statement error INSERT INTO integers VALUES (NULL, NULL) ---- +:.*Constraint Error.*constraint failed.* statement ok DROP TABLE integers @@ -72,6 +73,7 @@ SELECT SUM(i), SUM(j) FROM integers statement error INSERT INTO integers VALUES (NULL, NULL) ---- +:.*Constraint Error.*constraint failed.* statement ok DROP TABLE integers @@ -106,3 +108,4 @@ SELECT SUM(i), SUM(j) FROM integers statement error INSERT INTO integers VALUES (NULL, NULL) ---- +:.*Constraint Error.*constraint failed.* \ No newline at end of file diff --git a/test/sql/extensions/version_is_valid_httpfs.test b/test/sql/extensions/version_is_valid_httpfs.test deleted file mode 100644 index 4fc2319c4ed7..000000000000 --- a/test/sql/extensions/version_is_valid_httpfs.test +++ /dev/null @@ -1,21 +0,0 @@ -# name: test/sql/extensions/version_is_valid_httpfs.test -# description: Test version metadata on load -# group: [extensions] - -require-env LOCAL_EXTENSION_REPO - -require httpfs - -statement ok -SET autoinstall_known_extensions=true; - -statement ok -SET autoload_known_extensions=true; - -statement ok -SET enable_server_cert_verification = true; - -query I -SELECT count(*) FROM duckdb_extensions() WHERE extension_version != '' AND extension_name == 'httpfs'; ----- -1 diff --git a/test/sql/filter/test_alias_filter.test b/test/sql/filter/test_alias_filter.test index 81e1e8fb5c39..3e9c3e33987a 100644 --- a/test/sql/filter/test_alias_filter.test +++ b/test/sql/filter/test_alias_filter.test @@ -25,6 +25,7 @@ SELECT i % 2 AS k FROM integers WHERE k<>0; statement error SELECT i % 2 AS k FROM integers WHERE integers.k<>0; ---- +:.*Binder Error.*does not have a column.* # columns take priority query I @@ -54,3 +55,4 @@ SELECT i % 2 AS k FROM integers WHERE k=k; statement error SELECT i % 2 AS o, COUNT(i) AS c FROM integers WHERE c = 0 GROUP BY o; ---- +:.*Binder Error.*cannot contain aggregates.* \ No newline at end of file diff --git a/test/sql/filter/test_illegal_filters.test b/test/sql/filter/test_illegal_filters.test index 2e835f5d9ced..72510f104bff 100644 --- a/test/sql/filter/test_illegal_filters.test +++ b/test/sql/filter/test_illegal_filters.test @@ -15,4 +15,4 @@ INSERT INTO integers VALUES (2, 12) statement error SELECT * FROM integers WHERE SUM(a)>10 ---- - +:.*Binder Error.*cannot contain aggregates.* \ No newline at end of file diff --git a/test/sql/filter/test_variant_filter.test b/test/sql/filter/test_variant_filter.test new file mode 100644 index 000000000000..999a6311dd95 --- /dev/null +++ b/test/sql/filter/test_variant_filter.test @@ -0,0 +1,89 @@ +# name: test/sql/filter/test_variant_filter.test +# description: test comparison logic for VARIANT columns +# group: [filter] + +query I +WITH cte as ( + select '12'::VARIANT a +) +select IF(a == 12, 1, 0) from cte +---- +0 + +query I +WITH cte as ( + select '12'::VARIANT a +) +select IF(a == [1,2,3], 1, 0) from cte +---- +0 + +query I +WITH cte as ( + select '[1,2,3]'::VARIANT a +) +select IF(a == [1,2,3], 1, 0) from cte +---- +0 + +# Different lists, not equal +query I +WITH cte as ( + select [1,2,3]::VARIANT a +) +select IF(a == [3, 2, 1], 1, 0) from cte +---- +0 + +# Exact equality of list +query I +WITH cte as ( + select [1,2,3]::VARIANT a +) +select IF(a == [1,2,3], 1, 0) from cte +---- +1 + +query I +WITH cte as ( + select [1::VARIANT,'2',3]::VARIANT a +) +select IF(a == [1,2,3], 1, 0) from cte +---- +0 + +# Not even remotely equal +query I +WITH cte as ( + select {a: 21, b: [1,2,3]}::VARIANT a +) +select IF(a == [1,2,3], 1, 0) from cte +---- +0 + +# Compare equal because objects are compared by key, not position +query I +WITH cte as ( + select {a: 21, b: [1,2,3]}::VARIANT a +) +select IF(a == {b: [1,2,3], a: 21}, 1, 0) from cte +---- +1 + +# GreaterThan / LessThan + +query I +WITH cte as ( + select {a: 21, b: [1,2,3]}::VARIANT a +) +select IF(a < [1,2,3], 1, 0) from cte +---- +0 + +query I +WITH cte as ( + select {a: 21, b: [1,2,3]}::VARIANT a +) +select IF(a > [1,2,3], 1, 0) from cte +---- +1 diff --git a/test/sql/function/blob/base64.test b/test/sql/function/blob/base64.test index 501fe3899c1a..b2dcc21bd9e0 100644 --- a/test/sql/function/blob/base64.test +++ b/test/sql/function/blob/base64.test @@ -87,8 +87,10 @@ select from_base64('AAAA'); statement error SELECT from_base64('ab'); ---- +:.*Conversion Error: Could not decode string.* # unknown bytes statement error SELECT from_base64('üab'); ---- +:.*Conversion Error: Could not decode string.* \ No newline at end of file diff --git a/test/sql/function/date/date_trunc_stats.test b/test/sql/function/date/date_trunc_stats.test index 930227a0cd91..a0ef6763f370 100644 --- a/test/sql/function/date/date_trunc_stats.test +++ b/test/sql/function/date/date_trunc_stats.test @@ -15,3 +15,4 @@ SELECT date_trunc('DAY', A0) FROM T1 statement error SELECT datetrunc('milliseconds', DATE '-2005205-7-28'); ---- +:.*Conversion Error.*not in timestamp range.* \ No newline at end of file diff --git a/test/sql/function/date/test_strftime.test b/test/sql/function/date/test_strftime.test index 2c0a42331161..aebfda539eb1 100644 --- a/test/sql/function/date/test_strftime.test +++ b/test/sql/function/date/test_strftime.test @@ -132,16 +132,19 @@ endloop statement error SELECT strftime(d, d::VARCHAR) FROM dates ORDER BY d; ---- +:.*Invalid Input Error.*must be a constant.* # unterminated escape statement error SELECT strftime(DATE '1992-01-01', '%'); ---- +:.*Invalid Input Error.*Failed to parse format.* # unrecognized code statement error SELECT strftime(DATE '1992-01-01', '%R'); ---- +:.*Invalid Input Error.*Failed to parse format.* # millisecond specifier %g query IIII @@ -155,13 +158,15 @@ select strftime(strptime('023', '%g'), '%g'), strftime(strptime('0', '%g'), '%g' statement error SELECT strptime('-1', '%g'); ---- +:.*Invalid Input Error.*Could not parse string.* statement error SELECT strptime('1000', '%g'); ---- +:.*Invalid Input Error.*Could not parse string.* # this won't work without explicit casts statement error SELECT strftime('%Y', '1992-01-01'); ---- - +:.*Binder Error.*Could not choose a best candidate.* \ No newline at end of file diff --git a/test/sql/function/date/test_strftime_exhaustive.test b/test/sql/function/date/test_strftime_exhaustive.test index 4a365834852b..a05a1167975d 100644 --- a/test/sql/function/date/test_strftime_exhaustive.test +++ b/test/sql/function/date/test_strftime_exhaustive.test @@ -414,4 +414,4 @@ SELECT strftime(DATE '-4869706-10-11','%-yi'); statement error SELECT strftime(date '-99999-01-01', random()::varchar) ---- - +:.*Invalid Input Error.*must be a constant.* \ No newline at end of file diff --git a/test/sql/function/enum/test_enum_first.test b/test/sql/function/enum/test_enum_first.test index d2f64e90b968..e3a14c59bf97 100644 --- a/test/sql/function/enum/test_enum_first.test +++ b/test/sql/function/enum/test_enum_first.test @@ -16,3 +16,4 @@ red statement error SELECT enum_first('bla') ---- +:.*Binder Error.*This function needs an ENUM.* \ No newline at end of file diff --git a/test/sql/function/enum/test_enum_last.test b/test/sql/function/enum/test_enum_last.test index 5c01107573f8..7bac6d274956 100644 --- a/test/sql/function/enum/test_enum_last.test +++ b/test/sql/function/enum/test_enum_last.test @@ -16,3 +16,4 @@ purple statement error SELECT enum_last('bla') ---- +:.*Binder Error.*This function needs an ENUM.* \ No newline at end of file diff --git a/test/sql/function/enum/test_enum_range.test b/test/sql/function/enum/test_enum_range.test index f34c9fc41b23..ad3f04e09e6b 100644 --- a/test/sql/function/enum/test_enum_range.test +++ b/test/sql/function/enum/test_enum_range.test @@ -40,19 +40,24 @@ SELECT enum_range_boundary('orange'::rainbow, NULL) statement error SELECT enum_range_boundary('orange'::rainbow, 'brl'::currency) ---- +:.*Binder Error.*parameters need to link.* statement error SELECT enum_range_boundary(NULL, NULL) ---- +:.*Binder Error.*This function needs an ENUM.* statement error SELECT enum_last('bla') ---- +:.*Binder Error.*This function needs an ENUM.* statement error SELECT enum_range_boundary('orange'::rainbow, 1) ---- +:.*Binder Error.*This function needs an ENUM.* statement error SELECT enum_range_boundary(1, 'orange'::rainbow) ---- +:.*Binder Error.*This function needs an ENUM.* \ No newline at end of file diff --git a/test/sql/function/generic/test_approx_database_count.test b/test/sql/function/generic/test_approx_database_count.test index ef67ddb9a2fc..28388db3ebce 100644 --- a/test/sql/function/generic/test_approx_database_count.test +++ b/test/sql/function/generic/test_approx_database_count.test @@ -1,5 +1,5 @@ # name: test/sql/function/generic/test_approx_database_count.test -# description: Test the approx_database_count scalar function. +# description: Test the approx_database_count table function. # group: [generic] statement ok diff --git a/test/sql/function/generic/test_connection_count.test b/test/sql/function/generic/test_connection_count.test new file mode 100644 index 000000000000..3fc0623ead4a --- /dev/null +++ b/test/sql/function/generic/test_connection_count.test @@ -0,0 +1,17 @@ +# name: test/sql/function/generic/test_connection_count.test +# description: Test the connection_count table function. +# group: [generic] + +statement ok +PRAGMA enable_verification + +statement ok con1 +BEGIN + +statement ok con2 +BEGIN + +query I +SELECT count FROM duckdb_connection_count(); +---- +3 diff --git a/test/sql/function/list/aggregates/bit_and.test b/test/sql/function/list/aggregates/bit_and.test index 1101f1fed016..503c31d1c114 100644 --- a/test/sql/function/list/aggregates/bit_and.test +++ b/test/sql/function/list/aggregates/bit_and.test @@ -45,3 +45,4 @@ NULL 1 NULL statement error SELECT list_bit_and() ---- +:.*Binder Error.*does not support the supplied arguments.* \ No newline at end of file diff --git a/test/sql/function/list/aggregates/bit_or.test b/test/sql/function/list/aggregates/bit_or.test index 7e5d980df0ac..f1b9e1f98339 100644 --- a/test/sql/function/list/aggregates/bit_or.test +++ b/test/sql/function/list/aggregates/bit_or.test @@ -45,3 +45,4 @@ NULL 1 NULL statement error SELECT list_bit_or() ---- +:.*Binder Error.*does not support the supplied arguments.* \ No newline at end of file diff --git a/test/sql/function/list/aggregates/bit_xor.test b/test/sql/function/list/aggregates/bit_xor.test index 668e9401a927..22f6823a1276 100644 --- a/test/sql/function/list/aggregates/bit_xor.test +++ b/test/sql/function/list/aggregates/bit_xor.test @@ -45,3 +45,4 @@ NULL 0 NULL statement error SELECT list_bit_xor() ---- +:.*Binder Error.*does not support the supplied arguments.* \ No newline at end of file diff --git a/test/sql/function/list/aggregates/count.test b/test/sql/function/list/aggregates/count.test index 4caa004b26e2..a88f399a0f29 100644 --- a/test/sql/function/list/aggregates/count.test +++ b/test/sql/function/list/aggregates/count.test @@ -34,3 +34,4 @@ NULL statement error select list_count() ---- +:.*Binder Error.*does not support the supplied arguments.* \ No newline at end of file diff --git a/test/sql/function/list/aggregates/kurtosis.test b/test/sql/function/list/aggregates/kurtosis.test index b145ab3af714..738d1efcfdd1 100644 --- a/test/sql/function/list/aggregates/kurtosis.test +++ b/test/sql/function/list/aggregates/kurtosis.test @@ -16,6 +16,7 @@ NULL statement error select list_kurtosis([2e304, 2e305, 2e306, 2e307]) ---- +:.*Out of Range Error.*out of range.* statement ok create table aggr(k int[]); @@ -50,3 +51,4 @@ NULL statement error select list_kurtosis() ---- +:.*Binder Error.*does not support the supplied arguments.* \ No newline at end of file diff --git a/test/sql/function/list/aggregates/var_stddev.test b/test/sql/function/list/aggregates/var_stddev.test index 6505d47eb307..46ce8aa9d1ea 100644 --- a/test/sql/function/list/aggregates/var_stddev.test +++ b/test/sql/function/list/aggregates/var_stddev.test @@ -126,19 +126,23 @@ select list_aggr([0, 0], 'stddev') statement error select list_aggr([1e301, -1e301], 'stddev') ---- +:.*Out of Range Error.*out of range.* statement error select list_var_samp([1e301, -1e301]) ---- +:.*Out of Range Error.*out of range.* statement error select list_var_pop([1e301, -1e301]) ---- +:.*Out of Range Error.*out of range.* # incorrect usage statement error SELECT list_stddev_samp() ---- +:.*Binder Error.*does not support the supplied arguments.* # stddev_pop unexpectedly does not fetch any rows, test for list_stddev_pop statement ok @@ -150,3 +154,4 @@ INSERT INTO t0 VALUES([1E200, 0]); statement error SELECT list_stddev_pop(c0) FROM t0; ---- +:.*Out of Range Error.*out of range.* \ No newline at end of file diff --git a/test/sql/function/list/generate_subscripts.test b/test/sql/function/list/generate_subscripts.test index c89f30d97c35..1a95a3a7493d 100644 --- a/test/sql/function/list/generate_subscripts.test +++ b/test/sql/function/list/generate_subscripts.test @@ -20,3 +20,4 @@ SELECT generate_subscripts(NULL, 1) statement error SELECT generate_subscripts([[1,2],[3,4],[5,6]], 2) ---- +:.*Not implemented Error.*not implemented.* \ No newline at end of file diff --git a/test/sql/function/list/lambdas/arrow/lambda_scope_deprecated.test b/test/sql/function/list/lambdas/arrow/lambda_scope_deprecated.test index 0e4405c8f76c..33b387583eca 100644 --- a/test/sql/function/list/lambdas/arrow/lambda_scope_deprecated.test +++ b/test/sql/function/list/lambdas/arrow/lambda_scope_deprecated.test @@ -168,5 +168,4 @@ SELECT list_transform([1, 2, 3], "x.y" -> "x.y" + x.y) AS l FROM (VALUES (42), ( statement error SELECT list_reduce([1, 2, 3, 4], x *++++++++* y -> x - y) AS l; ---- -Invalid lambda parameters - +Binder Error: Invalid lambda parameters! diff --git a/test/sql/function/list/list_contains.test b/test/sql/function/list/list_contains.test index 8b3a408a76fb..dfecbb598ac3 100644 --- a/test/sql/function/list/list_contains.test +++ b/test/sql/function/list/list_contains.test @@ -224,6 +224,7 @@ SELECT list_contains([NULL,7], 7) statement error SELECT list_contains([[1,2,3],[1],[1,2,3]) ---- +Parser Error: syntax error at or near ")" statement error SELECT list_contains([[1,2,3],[1],[1,2,3]]) diff --git a/test/sql/function/list/list_position.test b/test/sql/function/list/list_position.test index adc918fe4e47..16b361460341 100644 --- a/test/sql/function/list/list_position.test +++ b/test/sql/function/list/list_position.test @@ -276,6 +276,8 @@ SELECT list_position([NULL,7], 7) statement error SELECT list_position([[1,2,3],[1],[1,2,3]) ---- +Parser Error: syntax error at or near ")" + statement error SELECT list_position([[1,2,3],[1],[1,2,3]]) diff --git a/test/sql/function/list/list_reverse.test b/test/sql/function/list/list_reverse.test index f3731a0c9706..d12e1e7c6873 100644 --- a/test/sql/function/list/list_reverse.test +++ b/test/sql/function/list/list_reverse.test @@ -134,6 +134,7 @@ SELECT list_reverse(42) statement error SELECT list_reverse ([1, 3, 2, 42, 117,, NULL]) ---- +Parser Error: syntax error at or near "," statement ok CREATE TABLE palindromes (s VARCHAR); diff --git a/test/sql/function/numeric/test_factorial.test b/test/sql/function/numeric/test_factorial.test index 11c9e642beee..ecb13b0ad78f 100644 --- a/test/sql/function/numeric/test_factorial.test +++ b/test/sql/function/numeric/test_factorial.test @@ -47,3 +47,4 @@ SELECT factorial(30); statement error SELECT factorial(40); ---- +:.*Out of Range Error.*out of range.* \ No newline at end of file diff --git a/test/sql/function/numeric/test_gamma.test b/test/sql/function/numeric/test_gamma.test index 7f7003d9d23a..a342dbfb6038 100644 --- a/test/sql/function/numeric/test_gamma.test +++ b/test/sql/function/numeric/test_gamma.test @@ -10,6 +10,7 @@ NULL statement error SELECT gamma(0) ---- +:.*Out of Range Error: cannot take gamma.* query I SELECT gamma(-1) @@ -50,6 +51,7 @@ SELECT gamma(2::hugeint) statement error SELECT gamma('asdf') ---- +:.*Conversion Error: Could not convert string.* query I SELECT lgamma(NULL) @@ -59,6 +61,7 @@ NULL statement error SELECT lgamma(0) ---- +:.*Out of Range Error: cannot take.* query I SELECT lgamma(-1) @@ -104,3 +107,4 @@ SELECT lgamma(2::hugeint) statement error SELECT lgamma('asdf') ---- +:.*Conversion Error: Could not convert string.* \ No newline at end of file diff --git a/test/sql/function/numeric/test_invalid_math.test b/test/sql/function/numeric/test_invalid_math.test index 6e693054916a..7ae164d03506 100644 --- a/test/sql/function/numeric/test_invalid_math.test +++ b/test/sql/function/numeric/test_invalid_math.test @@ -4,32 +4,32 @@ # cannot take log of negative number or zero statement error -SELECTl log(0); +SELECT log(0); ---- statement error -SELECTl log(-1); +SELECT log(-1); ---- statement error -SELECTl ln(0); +SELECT ln(0); ---- statement error -SELECTl ln(-1); +SELECT ln(-1); ---- statement error -SELECTl log10(0); +SELECT log10(0); ---- statement error -SELECTl log10(-1); +SELECT log10(-1); ---- # cannot take square root of a negative number statement error -SELECTl sqrt(-1); +SELECT sqrt(-1); ---- query I diff --git a/test/sql/function/numeric/test_pg_math.test b/test/sql/function/numeric/test_pg_math.test index 91e88ca3eda6..e394ec618bd1 100644 --- a/test/sql/function/numeric/test_pg_math.test +++ b/test/sql/function/numeric/test_pg_math.test @@ -102,22 +102,27 @@ select log(2, 64) statement error select log(0, 64) ---- +:.*Out of Range Error.*cannot take logarithm.* statement error select log(2, 0) ---- +:.*Out of Range Error.*cannot take logarithm.* statement error select log(-1, 64) ---- +:.*Out of Range Error.*cannot take logarithm.* statement error select log(2, -1) ---- +:.*Out of Range Error.*cannot take logarithm.* statement error select log(1, 64) ---- +:.*Out of Range Error.*divison by zero.* query I select log(2, 1) @@ -158,11 +163,14 @@ NaN statement error select log('-Inf'::DOUBLE, 64) ---- +:.*Out of Range Error.*cannot take logarithm.* statement error select log(64, '-Inf'::DOUBLE) ---- +:.*Out of Range Error.*cannot take logarithm.* statement error select log(2, -1) ---- +:.*Out of Range Error.*cannot take logarithm.* \ No newline at end of file diff --git a/test/sql/function/numeric/test_type_resolution.test b/test/sql/function/numeric/test_type_resolution.test index 5bd78e883f8f..9b77e35d4289 100644 --- a/test/sql/function/numeric/test_type_resolution.test +++ b/test/sql/function/numeric/test_type_resolution.test @@ -35,6 +35,7 @@ SELECT 1::TINYINT + 1::DOUBLE statement error SELECT 1::TINYINT + 1::VARCHAR ---- +:.*Binder Error: No function matches.* query I SELECT 1::SMALLINT + 1::TINYINT @@ -69,6 +70,7 @@ SELECT 1::SMALLINT + 1::DOUBLE statement error SELECT 1::SMALLINT + 1::VARCHAR ---- +:.*Binder Error: No function matches.* query I SELECT 1::INTEGER + 1::TINYINT @@ -103,6 +105,7 @@ SELECT 1::INTEGER + 1::DOUBLE statement error SELECT 1::INTEGER + 1::VARCHAR ---- +:.*Binder Error: No function matches.* query I SELECT 1::BIGINT + 1::TINYINT @@ -137,6 +140,7 @@ SELECT 1::BIGINT + 1::DOUBLE statement error SELECT 1::BIGINT + 1::VARCHAR ---- +:.*Binder Error: No function matches.* query R SELECT 1::REAL + 1::TINYINT @@ -171,6 +175,7 @@ SELECT 1::REAL + 1::DOUBLE statement error SELECT 1::REAL + 1::VARCHAR ---- +:.*Binder Error: No function matches.* query R SELECT 1::DOUBLE + 1::TINYINT @@ -205,4 +210,4 @@ SELECT 1::DOUBLE + 1::DOUBLE statement error SELECT 1::DOUBLE + 1::VARCHAR ---- - +:.*Binder Error: No function matches.* \ No newline at end of file diff --git a/test/sql/function/operator/test_date_arithmetic.test b/test/sql/function/operator/test_date_arithmetic.test index 624199e14214..f82c311c6e0a 100644 --- a/test/sql/function/operator/test_date_arithmetic.test +++ b/test/sql/function/operator/test_date_arithmetic.test @@ -187,3 +187,4 @@ NULL 20:08:10.001-15:59 NULL statement error SELECT '294247-01-10'::DATE + '04:00:54.775808'::TIME; ---- +:.*Out of Range Error.*out of range.* \ No newline at end of file diff --git a/test/sql/function/string/regex_extract.test b/test/sql/function/string/regex_extract.test index 04ba0776b95c..1a3a079e9bb0 100644 --- a/test/sql/function/string/regex_extract.test +++ b/test/sql/function/string/regex_extract.test @@ -44,10 +44,12 @@ baz statement error SELECT regexp_extract('foobarbaz', '(b..)(b..)', -1) ---- +:.*Invalid Input Error: Group index must be.* statement error SELECT regexp_extract('foobarbaz', '(b..)(b..)', 42) ---- +:.*Invalid Input Error: Group index must be.* statement ok CREATE TABLE test (s VARCHAR, p VARCHAR, i INT) @@ -63,6 +65,7 @@ INSERT INTO test VALUES statement error SELECT regexp_extract(s, p, i) FROM test ---- +:.*Invalid Input Error.*must be a constant.* query T SELECT regexp_extract(s, p, 0) FROM test @@ -85,7 +88,7 @@ bar statement error SELECT regexp_extract(s, '(b..)(b..)', i) FROM test ---- -Invalid Input Error: Group specification field must be a constant! +:.*Invalid Input Error.*must be a constant.* # null values query T @@ -106,3 +109,4 @@ NULL statement error SELECT regexp_extract('foobarbaz', 'b..', '1') ---- +:.*Binder Error.*Could not choose a best candidate.* \ No newline at end of file diff --git a/test/sql/function/string/regex_search.test b/test/sql/function/string/regex_search.test index 7aebb0227bab..ee9b40b5edaf 100644 --- a/test/sql/function/string/regex_search.test +++ b/test/sql/function/string/regex_search.test @@ -87,6 +87,7 @@ SELECT regexp_matches('foobarbequebaz', '(bar)(beque)') statement error SELECT regexp_matches('', '\X') ---- +:.*Invalid Input Error: invalid escape sequence.* statement ok CREATE TABLE regex(s STRING, p STRING) @@ -156,7 +157,7 @@ SELECT regexp_matches(c0, '.*SD.*', ' i ') from t0; statement error SELECT regexp_matches(c0, '.*SD.*', NULL) from t0; ---- -must not be NULL +:.*Invalid Input Error.*must not be NULL.* # this also works with tables statement ok @@ -180,16 +181,19 @@ SELECT regexp_matches(v, 'h.*', 'c') FROM test ORDER BY v statement error SELECT regexp_matches(v, 'h.*', v) FROM test ORDER BY v ---- +:.*Invalid Input Error.*must be a constant.* # throw on invalid options statement error SELECT regexp_matches(c0, '.*SD.*', 'q') from t0; ---- +:.*Invalid Input Error.*Unrecognized.* # can only use "g" with regexp replace statement error SELECT regexp_matches(c0, '.*SD.*', 'g') from t0; ---- +:.*Invalid Input Error.*only valid for regexp_replace.* # error in non-constant regex statement ok @@ -198,3 +202,4 @@ INSERT INTO regex VALUES ('asdf', '(') statement error SELECT regexp_matches(s, p) FROM regex ---- +:.*Invalid Input Error.*missing.* \ No newline at end of file diff --git a/test/sql/function/string/sha1.test b/test/sql/function/string/sha1.test index 9edfdb59af48..11afb3af06a5 100644 --- a/test/sql/function/string/sha1.test +++ b/test/sql/function/string/sha1.test @@ -49,6 +49,7 @@ da4b9237bacccdf19c0760cab7aec4a8359010b0 356a192b7913b04c54574d18c28d46e6395428a statement error SELECT sha1() ---- +:.*Binder Error: No function matches.* query I SELECT sha1(''::blob) @@ -58,3 +59,4 @@ da39a3ee5e6b4b0d3255bfef95601890afd80709 statement error SELECT sha1(42) ---- +:.*Binder Error: No function matches.* \ No newline at end of file diff --git a/test/sql/function/string/sha256.test b/test/sql/function/string/sha256.test index 98519a789f65..c70fc5fd9462 100644 --- a/test/sql/function/string/sha256.test +++ b/test/sql/function/string/sha256.test @@ -49,3 +49,4 @@ d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35 6b86b273ff34fce statement error SELECT sha256() ---- +:.*Binder Error: No function matches.* \ No newline at end of file diff --git a/test/sql/function/string/test_ascii.test b/test/sql/function/string/test_ascii.test index beccf3552972..30249c3a16bd 100644 --- a/test/sql/function/string/test_ascii.test +++ b/test/sql/function/string/test_ascii.test @@ -50,6 +50,7 @@ NULL statement error SELECT ASCII() ---- +:.*Binder Error: No function matches.* query T SELECT CHR(97) @@ -74,11 +75,14 @@ NULL statement error SELECT CHR(-10) ---- +:.*Invalid Input Error: Invalid UTF8.* statement error SELECT CHR(1073741824) ---- +:.*Invalid Input Error: Invalid UTF8.* statement error SELECT CHR() ---- +:.*Binder Error: No function matches.* \ No newline at end of file diff --git a/test/sql/function/string/test_ilike_escape.test b/test/sql/function/string/test_ilike_escape.test index 716fa264a7b1..b3f5ca5c7a71 100644 --- a/test/sql/function/string/test_ilike_escape.test +++ b/test/sql/function/string/test_ilike_escape.test @@ -73,8 +73,10 @@ NULL statement error select 'a%c' ilike 'a$%C' escape '///'; ---- +:.*Syntax Error: Invalid escape string.* # escape must be a constant statement error SELECT str ILIKE pat ESCAPE str FROM tbl ---- +:.*Syntax Error: Invalid escape string.* \ No newline at end of file diff --git a/test/sql/function/string/test_like.test b/test/sql/function/string/test_like.test index 0291d6556f43..cde174b47dd2 100644 --- a/test/sql/function/string/test_like.test +++ b/test/sql/function/string/test_like.test @@ -290,4 +290,4 @@ aaa statement error SELECT 'hello' LIKE 'hê?llo' COLLATE idontexist; ---- - +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/function/string/test_replace.test b/test/sql/function/string/test_replace.test index 39740a5e1937..89ab50e47cd2 100644 --- a/test/sql/function/string/test_replace.test +++ b/test/sql/function/string/test_replace.test @@ -70,12 +70,14 @@ Motöread statement error select REPLACE(1) ---- +:.*Binder Error: No function matches.* statement error select REPLACE(1, 2) ---- +:.*Binder Error: No function matches.* statement error select REPLACE(1, 2, 3, 4) ---- - +:.*Binder Error: No function matches.* \ No newline at end of file diff --git a/test/sql/function/string/test_reverse.test b/test/sql/function/string/test_reverse.test index e00558570ca9..0294084b5031 100644 --- a/test/sql/function/string/test_reverse.test +++ b/test/sql/function/string/test_reverse.test @@ -44,12 +44,14 @@ daeHrötoM statement error select REVERSE() ---- +:^Binder Error: No function matches.* statement error select REVERSE(1, 2) ---- +:^Binder Error: No function matches.* statement error select REVERSE('hello', 'world') ---- - +:^Binder Error: No function matches.* \ No newline at end of file diff --git a/test/sql/function/string/test_right.test b/test/sql/function/string/test_right.test index fdd86cd314aa..1af32a40c55f 100644 --- a/test/sql/function/string/test_right.test +++ b/test/sql/function/string/test_right.test @@ -116,3 +116,4 @@ SELECT "right"('a', -9223372036854775808); statement error SELECT right_grapheme('a', 9223372036854775808); ---- +:^Binder Error: No function matches.* \ No newline at end of file diff --git a/test/sql/function/string/test_similar_to.test b/test/sql/function/string/test_similar_to.test index 9397ea3e57d8..4c9e7d9a51e2 100644 --- a/test/sql/function/string/test_similar_to.test +++ b/test/sql/function/string/test_similar_to.test @@ -134,4 +134,4 @@ aaa statement error SELECT s FROM strings WHERE s SIMILAR TO 'ab.*%' {escape ''} ---- - +:.*Parser Error: syntax error.* \ No newline at end of file diff --git a/test/sql/function/string/test_split_part.test b/test/sql/function/string/test_split_part.test index e2acb1a6edec..bb2c2a0f5eab 100644 --- a/test/sql/function/string/test_split_part.test +++ b/test/sql/function/string/test_split_part.test @@ -94,11 +94,14 @@ NULL statement error select split_part() ---- +:.*Binder Error.*does not support the supplied arguments.* statement error select split_part('a') ---- +:.*Binder Error.*does not support the supplied arguments.* statement error select split_part('a','a') ---- +:.*Binder Error.*does not support the supplied arguments.* \ No newline at end of file diff --git a/test/sql/function/string/test_string_split.test b/test/sql/function/string/test_string_split.test index 41c009b284bb..cbd6c2b87c94 100644 --- a/test/sql/function/string/test_string_split.test +++ b/test/sql/function/string/test_string_split.test @@ -346,12 +346,15 @@ select UNNEST(string_split_regex('1,2,3,4,*,6', '(\*|,)')) statement error select string_split() ---- +:.*Binder Error.*No function matches.* statement error select string_split('a') ---- +:.*Binder Error.*No function matches.* # incorrect regex statement error SELECT string_split_regex(a, '[') FROM test ORDER BY a; ---- +:.*Catalog Error.*does not exist.* \ No newline at end of file diff --git a/test/sql/function/string/test_to_base.test b/test/sql/function/string/test_to_base.test index 4103cfeb4e2d..ecb5cf18c77f 100644 --- a/test/sql/function/string/test_to_base.test +++ b/test/sql/function/string/test_to_base.test @@ -8,12 +8,12 @@ PRAGMA enable_verification statement error SELECT to_base(-10, 2) ---- -number must be greater than or equal to 0 +Invalid Input Error: 'to_base' number must be greater than or equal to 0 statement error SELECT to_base(-10, 2, 64) -number must be greater than or equal to 0 ---- +Invalid Input Error: 'to_base' number must be greater than or equal to 0 statement error SELECT to_base(10, 1) diff --git a/test/sql/function/string/test_translate.test b/test/sql/function/string/test_translate.test index f885426d2f24..18445af82645 100644 --- a/test/sql/function/string/test_translate.test +++ b/test/sql/function/string/test_translate.test @@ -127,12 +127,14 @@ Hi statement error select TRANSLATE(1) ---- +:.*Binder Error.*No function matches.* statement error select TRANSLATE(1, 2) ---- +:.*Binder Error.*No function matches.* statement error select TRANSLATE(1, 2, 3, 4) ---- - +:.*Binder Error.*No function matches.* \ No newline at end of file diff --git a/test/sql/function/string/test_trim.test b/test/sql/function/string/test_trim.test index d44c1e76e8f1..591c905418ec 100644 --- a/test/sql/function/string/test_trim.test +++ b/test/sql/function/string/test_trim.test @@ -113,24 +113,29 @@ NULL NULL NULL statement error select LTRIM() ---- +:.*Binder Error: No function matches.*ltrim().* statement error select LTRIM('hello', 'world', 'aaa') ---- +:.*Binder Error: No function matches.*ltrim().* statement error select RTRIM() ---- +:.*Binder Error: No function matches.*rtrim().* statement error select RTRIM('hello', 'world', 'aaa') ---- +:.*Binder Error: No function matches.*rtrim().* statement error select TRIM() ---- +Parser Error: syntax error at or near ")" statement error select TRIM('hello', 'world', 'aaa') ---- - +:.*Binder Error: No function matches.*trim().* \ No newline at end of file diff --git a/test/sql/function/timestamp/test_icu_strptime.test b/test/sql/function/timestamp/test_icu_strptime.test index 4b0902763032..a9f2e6f63cb2 100644 --- a/test/sql/function/timestamp/test_icu_strptime.test +++ b/test/sql/function/timestamp/test_icu_strptime.test @@ -513,6 +513,12 @@ ORDER BY ALL endloop +# Offset patterns should also trigger casting to th current time zone +query I +SELECT STRPTIME('2025-09-16', ['%Y-%m-%d', '%Y-%m-%d%z']); +---- +2025-09-16 00:00:00-07 + # # Try # diff --git a/test/sql/function/variant/variant_extract.test b/test/sql/function/variant/variant_extract.test index 96c69d1c69c6..98fc35bbe66e 100644 --- a/test/sql/function/variant/variant_extract.test +++ b/test/sql/function/variant/variant_extract.test @@ -1,6 +1,9 @@ # name: test/sql/function/variant/variant_extract.test # group: [variant] +statement ok +pragma enable_verification; + require json query I diff --git a/test/sql/function/variant/variant_typeof.test b/test/sql/function/variant/variant_typeof.test index 3abb90654c17..f2e9ba67947c 100644 --- a/test/sql/function/variant/variant_typeof.test +++ b/test/sql/function/variant/variant_typeof.test @@ -1,6 +1,9 @@ # name: test/sql/function/variant/variant_typeof.test # group: [variant] +statement ok +pragma enable_verification; + query I select variant_typeof({'a': 42}::VARIANT); ---- @@ -18,6 +21,11 @@ OBJECT(bool, tinyint, smallint, int, bigint, hugeint, uhugeint, utinyint, usmall OBJECT(bool, tinyint, smallint, int, bigint, hugeint, uhugeint, utinyint, usmallint, uint, ubigint, bignum, date, time, timestamp, timestamp_s, timestamp_ms, timestamp_ns, time_tz, timestamp_tz, float, double, dec_4_1, dec_9_4, dec_18_6, dec38_10, uuid, interval, varchar, blob, bit, small_enum, medium_enum, large_enum, int_array, double_array, date_array, timestamp_array, timestamptz_array, varchar_array, nested_int_array, struct, struct_of_arrays, array_of_structs, map, union, fixed_int_array, fixed_varchar_array, fixed_nested_int_array, fixed_nested_varchar_array, fixed_struct_array, struct_of_fixed_array, fixed_array_of_int_list, list_of_fixed_int_array) OBJECT(bool, tinyint, smallint, int, bigint, hugeint, uhugeint, utinyint, usmallint, uint, ubigint, bignum, date, time, timestamp, timestamp_s, timestamp_ms, timestamp_ns, time_tz, timestamp_tz, float, double, dec_4_1, dec_9_4, dec_18_6, dec38_10, uuid, interval, varchar, blob, bit, small_enum, medium_enum, large_enum, int_array, double_array, date_array, timestamp_array, timestamptz_array, varchar_array, nested_int_array, struct, struct_of_arrays, array_of_structs, map, union, fixed_int_array, fixed_varchar_array, fixed_nested_int_array, fixed_nested_varchar_array, fixed_struct_array, struct_of_fixed_array, fixed_array_of_int_list, list_of_fixed_int_array) +statement error +CREATE TABLE T (v VARIANT); +---- +A table cannot be created from a VARIANT column yet + statement error create table all_types as select struct_pack(*COLUMNS(*))::VARIANT test from test_all_types(); ---- diff --git a/test/sql/generated_columns/stored/basic.test b/test/sql/generated_columns/stored/basic.test index 42560e8c8cab..92eb495e052f 100644 --- a/test/sql/generated_columns/stored/basic.test +++ b/test/sql/generated_columns/stored/basic.test @@ -11,3 +11,4 @@ CREATE TABLE tbl ( gcol AS (price) STORED, ); ---- +:.*Invalid Input Error.*Can not create.* \ No newline at end of file diff --git a/test/sql/generated_columns/virtual/check.test b/test/sql/generated_columns/virtual/check.test index 3df462c3310b..aa4b5b1a6235 100644 --- a/test/sql/generated_columns/virtual/check.test +++ b/test/sql/generated_columns/virtual/check.test @@ -21,6 +21,7 @@ CREATE TABLE tbl ( amount_sold AS (price) CHECK (amount_sold > 5) ); ---- +:.*Binder Error.*Constraints on generated columns.* # Also when used in this way statement error @@ -30,6 +31,7 @@ CREATE TABLE tbl ( CHECK (amount_sold > price) ); ---- +:.*Binder Error.*Constraints on generated columns.* statement ok CREATE TABLE chk ( @@ -45,3 +47,4 @@ INSERT INTO chk VALUES (6); statement error INSERT INTO chk VALUES (3); ---- +:.*Constraint Error.*constraint failed.* \ No newline at end of file diff --git a/test/sql/generated_columns/virtual/circular_dependency_stresstest.test b/test/sql/generated_columns/virtual/circular_dependency_stresstest.test index ac5b210ec8c5..b4ae581eae53 100644 --- a/test/sql/generated_columns/virtual/circular_dependency_stresstest.test +++ b/test/sql/generated_columns/virtual/circular_dependency_stresstest.test @@ -20,6 +20,7 @@ CREATE TABLE tbl ( x INTEGER ); ---- +:.*Invalid Input Error.*Circular dependency.* statement ok CREATE TABLE tbl ( @@ -52,3 +53,4 @@ CREATE TABLE circular ( x INTEGER, ); ---- +:.*Invalid Input Error.*Circular dependency.* \ No newline at end of file diff --git a/test/sql/generated_columns/virtual/create_table.test b/test/sql/generated_columns/virtual/create_table.test index 2b754319980c..8cf3484ba28e 100644 --- a/test/sql/generated_columns/virtual/create_table.test +++ b/test/sql/generated_columns/virtual/create_table.test @@ -9,6 +9,7 @@ PRAGMA enable_verification statement error CREATE TEMP TABLE t0(c0 AS (1)); ---- +:.*Binder Error.*not supported.* # Function doesn't exist statement error @@ -17,7 +18,7 @@ CREATE TABLE unit ( total_profit BOOLEAN GENERATED ALWAYS AS(non_existant_function() * price) VIRTUAL, ); ---- -non_existant_function +:.*Catalog Error.*does not exist.* # Column doesn't exist statement error @@ -25,7 +26,7 @@ CREATE TABLE unit ( total_profit BOOLEAN GENERATED ALWAYS AS(price * 2) VIRTUAL, ); ---- -price +:.*Binder Error.*price.* #Expression (word) is cast to BOOLEAN statement ok @@ -44,6 +45,7 @@ SELECT * FROM tbl statement error INSERT INTO tbl VALUES ('string', 5, 12); ---- +:.*Constraint Error.*Incorrect value for generated column.* # Expression contains a subquery statement error @@ -54,7 +56,7 @@ CREATE TABLE unit ( amount_sold INTEGER, ); ---- -Expression of generated column "total_profit" contains a subquery, which isn't allowed +:.*Parser Error.*isn't allowed.* statement ok CREATE MACRO my_macro() AS ( @@ -70,7 +72,7 @@ CREATE TABLE unit ( amount_sold INTEGER, ); ---- -Failed to bind generated column 'total_profit' because the expression contains a subquery +:.*Binder Error.*Failed to bind generated column.* # Duplicate column definition statement error @@ -81,6 +83,7 @@ CREATE TABLE unit ( amount_sold INTEGER, ); ---- +:.*Catalog Error.*already exists.* statement ok CREATE TABLE unit ( @@ -105,3 +108,4 @@ ALTER TABLE unit DROP COLUMN total_profit; statement error SELECT total_profit FROM unit; ---- +:.*Binder Error.*not found.* \ No newline at end of file diff --git a/test/sql/generated_columns/virtual/default.test b/test/sql/generated_columns/virtual/default.test index 02472cdd8879..6ef1e5edcc85 100644 --- a/test/sql/generated_columns/virtual/default.test +++ b/test/sql/generated_columns/virtual/default.test @@ -18,6 +18,8 @@ INSERT INTO unit VALUES (4,5) statement error ALTER TABLE unit ADD COLUMN total_profit INTEGER DEFAULT 1 GENERATED ALWAYS AS (price * amount_sold) VIRTUAL; ---- +Parser Error: syntax error at or near "GENERATED" + # Generated column failed to create statement error diff --git a/test/sql/generated_columns/virtual/drop.test b/test/sql/generated_columns/virtual/drop.test index e8dfa72ecc6c..3fec06940a3a 100644 --- a/test/sql/generated_columns/virtual/drop.test +++ b/test/sql/generated_columns/virtual/drop.test @@ -23,3 +23,4 @@ ALTER TABLE unit DROP COLUMN total_profit; statement error SELECT total_profit FROM unit; ---- +:.*Binder Error.*not found .* \ No newline at end of file diff --git a/test/sql/generated_columns/virtual/drop_dependency.test b/test/sql/generated_columns/virtual/drop_dependency.test index 2c22040742c0..41addbf209f1 100644 --- a/test/sql/generated_columns/virtual/drop_dependency.test +++ b/test/sql/generated_columns/virtual/drop_dependency.test @@ -14,6 +14,7 @@ INSERT INTO unit VALUES (5,4) statement error ALTER TABLE unit ADD COLUMN total_profit INTEGER GENERATED ALWAYS AS (price * amount_sold) VIRTUAL; ---- +:.*Parser Error.*not supported.* statement ok ALTER TABLE unit DROP COLUMN price; @@ -30,3 +31,4 @@ CREATE TABLE tbl ( statement error ALTER TABLE tbl DROP COLUMN price; ---- +:.*Catalog Error.*Cannot drop column.* \ No newline at end of file diff --git a/test/sql/generated_columns/virtual/foreign_key.test b/test/sql/generated_columns/virtual/foreign_key.test index 1377d3a5f7c4..45b978e0f406 100644 --- a/test/sql/generated_columns/virtual/foreign_key.test +++ b/test/sql/generated_columns/virtual/foreign_key.test @@ -18,7 +18,7 @@ CREATE TABLE tbl2 ( b AS (a) REFERENCES tbl1(a), ); ---- -generated +:.*Binder Error.*Failed to create foreign key.* #Creating a foreign key constraint on a generated column statement error @@ -28,7 +28,7 @@ CREATE TABLE tbl2 ( FOREIGN KEY (b) REFERENCES tbl1 (a) ); ---- -generated +:.*Binder Error.*Failed to create foreign key.* # Referencing a generated column is not supported because in order to do that # you have to create a UNIQUE/PRIMARY KEY constraint on a generated column first - which isn't supported (yet) @@ -40,7 +40,7 @@ CREATE TABLE tbl2 ( also_price AS (price) UNIQUE ); ---- -generated +:.*Binder Error.*not supported.* statement ok CREATE TABLE tbl2 ( @@ -55,7 +55,7 @@ CREATE TABLE tbl3 ( FOREIGN KEY (a) REFERENCES tbl2 (also_price) ); ---- -no primary key +:.*Binder Error.*Failed to create foreign key.* #Show that normal uses of FOREIGN_KEY constraint work mixed with generated columns statement ok @@ -81,3 +81,4 @@ INSERT INTO a VALUES (5); statement error INSERT INTO a VALUES (5); ---- +:.*Constraint Error.*Duplicate key.* \ No newline at end of file diff --git a/test/sql/generated_columns/virtual/insert.test b/test/sql/generated_columns/virtual/insert.test index 786020a818af..b5873dfa69cd 100644 --- a/test/sql/generated_columns/virtual/insert.test +++ b/test/sql/generated_columns/virtual/insert.test @@ -16,6 +16,7 @@ CREATE TABLE test ( statement error INSERT INTO test (bar) VALUES (5) ---- +:.*Binder Error.*Cannot insert.* #Using column names statement ok @@ -35,6 +36,7 @@ SELECT * FROM test statement error INSERT INTO test VALUES (22) ---- +:.*Binder Error.*table test has 2 columns.* #Using VALUES correctly statement ok @@ -44,6 +46,7 @@ INSERT INTO test VALUES (22, 10); statement error INSERT INTO test VALUES (22, 10, 10) ---- +:.*Binder Error.*table test has 2 columns.* statement ok CREATE TABLE tbl ( @@ -77,3 +80,4 @@ CREATE TABLE tbl2 ( statement error INSERT INTO tbl2 VALUES ('test'); ---- +:.*Constraint Error.*Incorrect value for generated column.* \ No newline at end of file diff --git a/test/sql/generated_columns/virtual/null.test b/test/sql/generated_columns/virtual/null.test index d28da2cf2ad4..08f76446c1c9 100644 --- a/test/sql/generated_columns/virtual/null.test +++ b/test/sql/generated_columns/virtual/null.test @@ -12,6 +12,7 @@ CREATE TABLE unit ( amount_sold AS (price) NOT NULL, ); ---- +:.*Binder Error.*not supported yet.* # Show that proper NOT NULL constraints still work when mixed in when generated columns statement ok @@ -28,3 +29,4 @@ INSERT INTO tbl VALUES (5); statement error INSERT INTO tbl VALUES (null); ---- +:.*Constraint Error.*constraint failed.* \ No newline at end of file diff --git a/test/sql/httpfs/hffs.test b/test/sql/httpfs/hffs.test deleted file mode 100644 index 82231fb9a679..000000000000 --- a/test/sql/httpfs/hffs.test +++ /dev/null @@ -1,42 +0,0 @@ -# name: test/sql/httpfs/hffs.test -# description: Ensure the HuggingFace filesystem works as expected -# group: [httpfs] - -require parquet - -require httpfs - -statement error -FROM parquet_scan('hf://') ----- -IO Error: Failed to parse 'hf://'. Please format url like: 'hf://datasets/my-username/my-dataset/path/to/file.parquet' - -statement error -FROM 'hf://file.parquet' ----- -IO Error: Failed to parse 'hf://file.parquet'. Please format url like: 'hf://datasets/my-username/my-dataset/path/to/file.parquet' - -statement error -FROM 'hf://yepthisdoesntwork/file.parquet' ----- -IO Error: Failed to parse: 'hf://yepthisdoesntwork/file.parquet'. Currently DuckDB only supports querying datasets or spaces, so the url should start with 'hf://datasets' or 'hf://spaces' - -statement error -FROM 'hf://stil/not/file.parquet' ----- -IO Error: Failed to parse: 'hf://stil/not/file.parquet'. Currently DuckDB only supports querying datasets or spaces, so the url should start with 'hf://datasets' or 'hf://spaces' - -statement error -FROM 'hf://datasets/file.parquet' ----- -IO Error: Failed to parse 'hf://datasets/file.parquet'. Please format url like: 'hf://datasets/my-username/my-dataset/path/to/file.parquet' - -statement error -FROM 'hf://datasets/myname/file.parquet' ----- -IO Error: Failed to parse 'hf://datasets/myname/file.parquet'. Please format url like: 'hf://datasets/my-username/my-dataset/path/to/file.parquet' - -statement error -FROM 'hf://datasets/**/file.parquet' ----- -IO Error: Failed to parse 'hf://datasets/**/file.parquet'. Please format url like: 'hf://datasets/my-username/my-dataset/path/to/file.parquet' diff --git a/test/sql/httpfs/hffs.test_slow b/test/sql/httpfs/hffs.test_slow deleted file mode 100644 index 5870366b404e..000000000000 --- a/test/sql/httpfs/hffs.test_slow +++ /dev/null @@ -1,180 +0,0 @@ -# name: test/sql/httpfs/hffs.test_slow -# description: Ensure the HuggingFace filesystem works as expected -# group: [httpfs] - -require parquet - -require httpfs - -# FIXME: currently this will not fail the Linux HTTPFS ci job if it fails, because it might do so due to networking issues -# however having a CI job dedicated to remote tests that may spuriously fail would solve this - -# Non existent repos get a 401 -statement error -FROM parquet_scan('hf://datasets/samansmink/non-existent/*.parquet'); ----- - -# Globbing non-existent repo is also 401 -statement error -FROM parquet_scan('hf://datasets/samansmink/non-existent/**/*.parquet'); ----- - -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_tests/hive_data/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -1 value1 hf://datasets/samansmink/duckdb_ci_tests/hive_data/part=a/date=2012-01-01/test.parquet -2 value2 hf://datasets/samansmink/duckdb_ci_tests/hive_data/part=b/date=2013-01-01/test.parquet - - -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_tests/hive_data/*/*/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -1 value1 hf://datasets/samansmink/duckdb_ci_tests/hive_data/part=a/date=2012-01-01/test.parquet -2 value2 hf://datasets/samansmink/duckdb_ci_tests/hive_data/part=b/date=2013-01-01/test.parquet - -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_tests/hive_data/part=[ab]/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -1 value1 hf://datasets/samansmink/duckdb_ci_tests/hive_data/part=a/date=2012-01-01/test.parquet -2 value2 hf://datasets/samansmink/duckdb_ci_tests/hive_data/part=b/date=2013-01-01/test.parquet - -# This ensures the next query is forced to use pagination, testing our support for it -statement ok -set hf_max_per_page=1; - -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_tests/hive_data/part=[b]/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -2 value2 hf://datasets/samansmink/duckdb_ci_tests/hive_data/part=b/date=2013-01-01/test.parquet - -statement ok -reset hf_max_per_page; - -# Ensure we only open 1 of the files here to confirm filter pushdown has eliminated the other paths -query II rowsort -explain analyze SELECT id, part FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_tests/hive_data/**/*.parquet') where part='a'; ----- -analyzed_plan :.*HTTP Stats.*\#HEAD\: 1 .* - -statement ok -set hf_max_per_page=1; - -# Branches can be specified, including the special branch types with '~' -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_tests@~parquet/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -1 value1 hf://datasets/samansmink/duckdb_ci_tests@~parquet/default/test/0000.parquet -2 value2 hf://datasets/samansmink/duckdb_ci_tests@~parquet/default/test/0001.parquet - -# Secret provider 'config' (default) allows setting the token directly -statement ok -CREATE SECRET hf_token (TYPE HUGGINGFACE, token 'some_hf_token'); - -# Secret provider 'credential chain' scans several places for a token -statement ok -CREATE SECRET hf_token_from_credential_chain (TYPE HUGGINGFACE, PROVIDER credential_chain); - -statement ok -DROP SECRET hf_token - -statement ok -DROP SECRET hf_token_from_credential_chain - -# Private bucket is not allowed without credentials -statement error -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_private/hive_data/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -401 - -# Ensure spaces work too -query I -select size from read_text('hf://spaces/samansmink/duckdb_ci_tests/README.md'); ----- -199 - -# FIXME: push auth key into CI for this to ensure it is tested in CI properly -require-env HUGGING_FACE_TOKEN - -statement ok -CREATE SECRET hf1 (TYPE HUGGINGFACE, TOKEN '${HUGGING_FACE_TOKEN}'); - -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_private/hive_data/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -1 value1 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=a/date=2012-01-01/test.parquet -2 value2 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=b/date=2013-01-01/test.parquet - -statement ok -DROP SECRET hf1 - -# Same can be achieved with an http secret setting the bearer token manually - -statement ok -CREATE SECRET http1 (TYPE HTTP, BEARER_TOKEN '${HUGGING_FACE_TOKEN}'); - -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_private/hive_data/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -1 value1 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=a/date=2012-01-01/test.parquet -2 value2 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=b/date=2013-01-01/test.parquet - -statement ok -DROP SECRET http1 - -# Note that the huggingface secret takes precedence over the http secret - -statement ok -CREATE SECRET hf2 (TYPE HUGGINGFACE, TOKEN '${HUGGING_FACE_TOKEN}'); - -statement ok -CREATE SECRET http2 (TYPE HTTP, BEARER_TOKEN 'hocus pocus this token is bogus'); - -# Works because hf secret is selected -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_private/hive_data/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -1 value1 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=a/date=2012-01-01/test.parquet -2 value2 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=b/date=2013-01-01/test.parquet - -statement ok -DROP SECRET hf2; - -# the http secret does not work -statement error -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_private/hive_data/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -401 - -statement ok -DROP SECRET http2 - -# Finally we can also manually set the bearer token header -statement ok -CREATE SECRET http3 ( - TYPE HTTP, - EXTRA_HTTP_HEADERS MAP{ - 'Authorization': 'Bearer ${HUGGING_FACE_TOKEN}', - } -); - -# Works because hf secret is selected -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_private/hive_data/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -1 value1 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=a/date=2012-01-01/test.parquet -2 value2 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=b/date=2013-01-01/test.parquet - -statement ok -DROP SECRET http3 - -# FIXME: test this from CI as well -require-env HUGGING_FACE_TOKEN_IN_CACHE - -statement ok -CREATE SECRET hf1 (TYPE HUGGINGFACE, PROVIDER credential_chain); - -query III rowsort -FROM parquet_scan('hf://datasets/samansmink/duckdb_ci_private/hive_data/**/*.parquet', FILENAME=1, hive_partitioning=0); ----- -1 value1 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=a/date=2012-01-01/test.parquet -2 value2 hf://datasets/samansmink/duckdb_ci_private/hive_data/part=b/date=2013-01-01/test.parquet diff --git a/test/sql/httpfs/internal_issue_2490.test b/test/sql/httpfs/internal_issue_2490.test deleted file mode 100644 index 4aa11f76a92c..000000000000 --- a/test/sql/httpfs/internal_issue_2490.test +++ /dev/null @@ -1,10 +0,0 @@ -# name: test/sql/httpfs/internal_issue_2490.test -# description: Internal issue 2490 - Wrong URL encoding leads to 404 for redirects with httplib v0.14.3 -# group: [httpfs] - -require httpfs - -require parquet - -statement ok -FROM 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/us+er+da+ta.parquet' LIMIT 1; diff --git a/test/sql/index/art/issues/test_art_fuzzer_persisted.test b/test/sql/index/art/issues/test_art_fuzzer_persisted.test index 3d4085eba592..1ec3380c0e08 100644 --- a/test/sql/index/art/issues/test_art_fuzzer_persisted.test +++ b/test/sql/index/art/issues/test_art_fuzzer_persisted.test @@ -14,7 +14,7 @@ statement ok CREATE INDEX i1 ON t1 (c1); statement ok -PRAGMA MEMORY_LIMIT='2MB'; +PRAGMA MEMORY_LIMIT='4MB'; statement ok CHECKPOINT; diff --git a/test/sql/index/art/storage/test_upsert_reclaim_space.test_slow b/test/sql/index/art/storage/test_upsert_reclaim_space.test_slow index fc59d3fb4879..d5f78c5f0a6a 100644 --- a/test/sql/index/art/storage/test_upsert_reclaim_space.test_slow +++ b/test/sql/index/art/storage/test_upsert_reclaim_space.test_slow @@ -61,7 +61,7 @@ DELETE FROM tbl; # We should not have significantly move blocks than in previous iterations. query I -SELECT current.total_blocks < blocks_del_tbl.total_blocks + 4 +SELECT current.total_blocks < blocks_del_tbl.total_blocks + 6 FROM pragma_database_size() AS current, blocks_del_tbl; ---- 1 @@ -78,7 +78,7 @@ CHECKPOINT; # We should not have significantly move blocks than in previous iterations. query I -SELECT current.total_blocks < blocks_idx.total_blocks + 4 +SELECT current.total_blocks < blocks_idx.total_blocks + 6 FROM pragma_database_size() AS current, blocks_idx; ---- 1 diff --git a/test/sql/join/asof/test_asof_join_doubles.test b/test/sql/join/asof/test_asof_join_doubles.test index f5a71924e941..4ed5ac10895f 100644 --- a/test/sql/join/asof/test_asof_join_doubles.test +++ b/test/sql/join/asof/test_asof_join_doubles.test @@ -105,7 +105,7 @@ statement ok INSERT INTO events0 VALUES (10, 4); # RIGHT Window version -query II nosort right_inequality +query II nosort SELECT p.ts, e.value FROM range(0,10) p(ts) @@ -129,20 +129,40 @@ ORDER BY p.ts ASC NULLS LAST NULL 4 # RIGHT ON inequality only -query II nosort right_inequality +query II nosort SELECT p.ts, e.value FROM range(0,10) p(ts) ASOF RIGHT JOIN events0 e ON p.ts >= e.begin ORDER BY p.ts ASC NULLS LAST ---- +1 0 +2 0 +3 1 +4 1 +5 1 +6 2 +7 2 +8 3 +9 3 +NULL 4 # RIGHT USING inequality only -query II nosort right_inequality +query II nosort SELECT p.begin, e.value FROM range(0,10) p(begin) ASOF RIGHT JOIN events0 e USING (begin) ORDER BY p.begin ASC NULLS LAST ---- +1 0 +2 0 +3 1 +4 1 +5 1 +6 2 +7 2 +8 3 +9 3 +NULL 4 # # With equality @@ -325,3 +345,10 @@ USING (key, begin) ORDER BY 1 ASC NULLS FIRST, 2 ---- +# INNER with empty RHS +query II +SELECT p.ts, e.value +FROM range(0,10) p(ts) ASOF JOIN (from events0 where value < 0) e +ON p.ts >= e.begin +ORDER BY p.ts ASC +---- diff --git a/test/sql/join/asof/test_asof_join_predicates.test b/test/sql/join/asof/test_asof_join_predicates.test new file mode 100644 index 000000000000..741312ab623e --- /dev/null +++ b/test/sql/join/asof/test_asof_join_predicates.test @@ -0,0 +1,114 @@ +# name: test/sql/join/asof/test_asof_join_predicates.test +# description: Test As-Of join NLJ rewrite for non-comparison predicates +# group: [asof] + +statement ok +PRAGMA enable_verification; + +# Issue 18309 +statement ok +CREATE TABLE tt1 (i INTEGER, j VARCHAR); + +statement ok + INSERT INTO tt1 VALUES + (2, 'A'), + (4, 'B'), + (5, 'A'); + +statement ok +CREATE TABLE tt2 (i INTEGER, j VARCHAR, k VARCHAR); + +statement ok +INSERT INTO tt2 VALUES + (1, 'A', 'I'), + (3, 'B', 'II'); + +query II +explain +SELECT tt1.i, tt2.k + FROM tt1 + ASOF JOIN tt2 ON + tt1.j = tt2.j AND tt1.i >= tt2.i + ORDER BY tt1.i; +---- +physical_plan :.*NESTED_LOOP_JOIN.* + +query II +SELECT tt1.i, tt2.k + FROM tt1 + ASOF JOIN tt2 ON + (tt1.j = tt2.j OR tt1.j = tt2.j) AND tt1.i >= tt2.i + ORDER BY tt1.i; +---- +2 I +4 II +5 I + +# Issue 19027 +statement ok +create table l (id integer, date timestamp, item varchar); + +statement ok +insert into l values + (0, '2025-01-01', 'A'); + +statement ok +create table r (id integer, date timestamp, item varchar, valuei double); + +statement ok +insert into r values + (0, '2025-01-01', 'A', 8.0), + (0, '2025-01-01', 'B', 12.0); + +query II +explain +select + l.id, + l.date, + l.item as litem, + r.item as ritem, + valuei +from l +asof left join r + on l.id = r.id and l.date >= r.date + and (l.item = r.item or l.item = '*'); +---- +physical_plan :.*NESTED_LOOP_JOIN.* + +query IIIII +select + l.id, + l.date, + l.item as litem, + r.item as ritem, + valuei +from l +asof left join r + on l.id = r.id and l.date >= r.date + and (l.item = r.item or l.item = '*'); +---- +0 2025-01-01 00:00:00 A A 8.0 + +# Issue 19251 +statement ok +create temp table tbl1 as + select unnest(range(1000)) % 10 as x, '2022-01-01'::timestamp + to_days(unnest(range(1000))) as ts; + +statement ok +create temp table tbl2 as + select unnest(range(1000)) % 10 as x, '2022-01-01'::timestamp + to_hours(unnest(range(1000))) as ts; + +query II +explain +from tbl1 asof join +tbl2 on tbl1.x = tbl2.x + and tbl1.ts >= tbl2.ts + and (tbl1.ts - tbl2.ts) < interval '1' hours; +---- +physical_plan :.*NESTED_LOOP_JOIN.* + +statement ok +from tbl1 asof join +tbl2 on tbl1.x = tbl2.x + and tbl1.ts >= tbl2.ts + and (tbl1.ts - tbl2.ts) < interval '1' hours; diff --git a/test/sql/join/asof/test_asof_join_timestamps.test b/test/sql/join/asof/test_asof_join_timestamps.test index 316387189c3e..374de532e5b0 100644 --- a/test/sql/join/asof/test_asof_join_timestamps.test +++ b/test/sql/join/asof/test_asof_join_timestamps.test @@ -29,6 +29,18 @@ INSERT INTO probe0 VALUES ('infinity') ; +statement ok +CREATE TABLE asof_nulls ( + time TIMESTAMP, + value FLOAT +); + +statement ok +INSERT INTO asof_nulls (time, value) VALUES ('2025-07-15 00:00:00', 42); + +statement ok +INSERT INTO asof_nulls (time, value) VALUES ('2025-07-15 01:00:00', null); + # Compare NLJ optimisation to operator foreach threshold 0 32 @@ -208,4 +220,12 @@ ORDER BY p.begin ASC ---- 2023-03-21 12:00:00 +# Return NULLs +query II +SELECT time_series.time, asof_nulls.value +FROM (VALUES ('2025-07-15 02:00:00'::TIMESTAMP)) as time_series(time) +ASOF LEFT JOIN asof_nulls ON asof_nulls.time <= time_series.time; +---- +2025-07-15 02:00:00 NULL + endloop diff --git a/test/sql/join/inner/test_using_join.test b/test/sql/join/inner/test_using_join.test index 37f731af4bd7..c24ddcf05af2 100644 --- a/test/sql/join/inner/test_using_join.test +++ b/test/sql/join/inner/test_using_join.test @@ -59,10 +59,12 @@ SELECT a+1 FROM t1 JOIN t2 USING(a) ORDER BY a statement error SELECT t2.a, t2.b, t2.c FROM t1 JOIN t2 USING(a+b) ---- +Parser Error: syntax error at or near "+" statement error SELECT t2.a, t2.b, t2.c FROM t1 JOIN t2 USING("") ---- +Parser Error: zero-length delimited identifier at or near """" statement error SELECT t2.a, t2.b, t2.c FROM t1 JOIN t2 USING(d) @@ -71,6 +73,7 @@ SELECT t2.a, t2.b, t2.c FROM t1 JOIN t2 USING(d) statement error SELECT t2.a, t2.b, t2.c FROM t1 JOIN t2 USING(t1.a) ---- +Parser Error: syntax error at or near "." query IIII SELECT * FROM t1 JOIN t2 USING(a,b) diff --git a/test/sql/json/issues/issue19357.test b/test/sql/json/issues/issue19357.test new file mode 100644 index 000000000000..295498ddc0ed --- /dev/null +++ b/test/sql/json/issues/issue19357.test @@ -0,0 +1,20 @@ +# name: test/sql/json/issues/issue19357.test +# description: Test issue 19357 - Expected unified vector format of type VARCHAR, but found type INT32 +# group: [issues] + +require json + +query I +SELECT TO_JSON({'key_1': 'one'}) AS WITHOUT_KEEP_NULL +---- +{"key_1":"one"} + +query I +SELECT JSON_OBJECT('key_1', 'one', 'key_2', NULL) AS KEEP_NULL_1 +---- +{"key_1":"one","key_2":null} + +statement error +SELECT JSON_OBJECT('key_1', 'one', NULL, 'two') AS KEEP_NULL_2 +---- +json_object() keys must be VARCHAR diff --git a/test/sql/json/scalar/test_json_create.test b/test/sql/json/scalar/test_json_create.test index 0816bdb00017..ab0c969528c4 100644 --- a/test/sql/json/scalar/test_json_create.test +++ b/test/sql/json/scalar/test_json_create.test @@ -27,7 +27,7 @@ select to_json({n: 42}) statement error select to_json({n: 42}, {extra: 'argument'}) ---- -Invalid Input Error: to_json() takes exactly one argument +to_json() takes exactly one argument query T select to_json(union_value(n := 42)) @@ -141,7 +141,7 @@ select json_array(a, b, c, d, e) from test [-777,4.2,"goose",[4,2],null] query T -select json_object(a, a, b, b, c, c, d, d, e, e) from test +select json_object(a::varchar, a, b::varchar, b, c, c, d::varchar, d, e::varchar, e) from test ---- {"0":0,"0.5":0.5,"short":"short","[0, 1, 2, 3, 4, 5, 6, 7, 9]":[0,1,2,3,4,5,6,7,9],"33":33} {"42":42,"1.0":1.0,"looooooooooooooong":"looooooooooooooong","[]":[],"42":42} diff --git a/test/sql/json/scalar/test_json_transform.test b/test/sql/json/scalar/test_json_transform.test index d26b1dbb33b6..4c33abfa2fa2 100644 --- a/test/sql/json/scalar/test_json_transform.test +++ b/test/sql/json/scalar/test_json_transform.test @@ -524,4 +524,26 @@ NULL statement error select json_transform_strict('42', '"TIMESTAMP"') ---- -:.*Invalid Input Error.*Unable to cast.* \ No newline at end of file +:.*Invalid Input Error.*Unable to cast.* + +# enum tests +statement ok +CREATE OR REPLACE TYPE test_enum AS ENUM ('a', 'b', 'c'); + +query T +select json_transform('{"test": "a"}', '{"test": "test_enum"}') +---- +{'test': a} + +query T +select json_transform('{"test": ["a","b"]}', '{"test": "test_enum[]"}') +---- +{'test': [a, b]} + +query T +select json_transform('{"test": ["a","b"]}', '{"test": "test_enum[2]"}') +---- +{'test': [a, b]} + +statement ok +DROP TYPE test_enum; \ No newline at end of file diff --git a/test/sql/json/scalar/test_json_type.test b/test/sql/json/scalar/test_json_type.test index a9b75ee48533..f3846d20569f 100644 --- a/test/sql/json/scalar/test_json_type.test +++ b/test/sql/json/scalar/test_json_type.test @@ -218,3 +218,19 @@ query T SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[6]'); ---- NULL + +# test LIST(JSON) compatibility +query T +select json_type(json_extract('{"duck":[1,2,3]}', '$..duck')) +---- +ARRAY + +query T +select json_type(json_extract('{"a":{"duck":[1]},"b":{"duck":[2,3]}}', '$..duck')) +---- +ARRAY + +query T +select json_type(json_extract('{"duck":[1,2,3]}', '$.duck[*]')) +---- +ARRAY diff --git a/test/sql/json/table/read_json_auto.test_slow b/test/sql/json/table/read_json_auto.test_slow index efed7eee5fb9..eac953ddc6ed 100644 --- a/test/sql/json/table/read_json_auto.test_slow +++ b/test/sql/json/table/read_json_auto.test_slow @@ -352,29 +352,3 @@ statement error select * from read_json_auto(['data/json/empty_array.json', 'data/json/example_n.ndjson'], maximum_sample_files=NULL); ---- Binder Error - -statement ok -pragma disable_verification - -require httpfs - -# this is one big object - yyjson uses it as a benchmark -query II -select typeof("type"), typeof(features) from read_json_auto('https://github.com/duckdb/duckdb-data/releases/download/v1.0/canada.json', maximum_depth=3); ----- -VARCHAR STRUCT("type" JSON, properties JSON, geometry JSON)[] - -# let's crank up maximum_depth and see if we can fully unnest this big object -query II -select typeof("type"), typeof(features) from read_json_auto('https://github.com/duckdb/duckdb-data/releases/download/v1.0/canada.json', maximum_depth=8); ----- -VARCHAR STRUCT("type" VARCHAR, properties STRUCT("name" VARCHAR), geometry STRUCT("type" VARCHAR, coordinates DOUBLE[][][]))[] - -# ^ fully unnested, no more JSON type in there - -# the "coordinates" array in "features.geometry" is huge, let's just check the length - not all the values -query IIIII -select type, features[1].type, features[1].properties.name, features[1].geometry.type, length(features[1].geometry.coordinates) -from read_json_auto('https://github.com/duckdb/duckdb-data/releases/download/v1.0/canada.json', maximum_depth=8); ----- -FeatureCollection Feature Canada Polygon 480 diff --git a/test/sql/json/table/read_json_objects.test b/test/sql/json/table/read_json_objects.test index ecf73b6214e5..8064d4a4cef7 100644 --- a/test/sql/json/table/read_json_objects.test +++ b/test/sql/json/table/read_json_objects.test @@ -142,27 +142,6 @@ SELECT count(*) from read_ndjson_objects('data/json/example_*n.ndjson') ---- 10 -require httpfs - -# same file but hosted on github -query I -select * from read_json_objects('https://github.com/duckdb/duckdb-data/releases/download/v1.0/example_rn.ndjson', format='nd') ----- -{"id":1,"name":"O Brother, Where Art Thou?"} -{"id":2,"name":"Home for the Holidays"} -{"id":3,"name":"The Firm"} -{"id":4,"name":"Broadcast News"} -{"id":5,"name":"Raising Arizona"} - -query I -select * from read_ndjson_objects('https://github.com/duckdb/duckdb-data/releases/download/v1.0/example_rn.ndjson') ----- -{"id":1,"name":"O Brother, Where Art Thou?"} -{"id":2,"name":"Home for the Holidays"} -{"id":3,"name":"The Firm"} -{"id":4,"name":"Broadcast News"} -{"id":5,"name":"Raising Arizona"} - # empty file query I select * from read_json_objects('data/json/empty.ndjson') diff --git a/test/sql/json/table/test_json_table_in_out.test_slow b/test/sql/json/table/test_json_table_in_out.test_slow index 76d93d2d7dc4..74e0ae42bbac 100644 --- a/test/sql/json/table/test_json_table_in_out.test_slow +++ b/test/sql/json/table/test_json_table_in_out.test_slow @@ -739,3 +739,10 @@ $.my_array[0] $.my_array $.my_array[0].my_key $.my_array[0] $.my_array[1] $.my_array $.my_array[1].my_key $.my_array[1] + +# internal issues 5772 and 5776 +statement ok +create table all_types as select * exclude(small_enum, medium_enum, large_enum) from test_all_types() limit 0; + +statement ok +SELECT NULL FROM json_each(6051, NULL) diff --git a/test/sql/json/test_json_copy.test_slow b/test/sql/json/test_json_copy.test_slow index 51ba2e05fd49..dfc0951bbe93 100644 --- a/test/sql/json/test_json_copy.test_slow +++ b/test/sql/json/test_json_copy.test_slow @@ -77,6 +77,17 @@ select * from roundtrip ---- +# test issue 18816 +statement ok +copy (select 42 i) +to '__TEST_DIR__/json_batch' +(format json, per_thread_output true, overwrite true); + +statement ok +copy (select 42 i) +to '__TEST_DIR__/json_batch' +(format json, per_thread_output true, append true); + # test issue #6305 statement ok copy ( diff --git a/test/sql/json/test_json_empty_object.test b/test/sql/json/test_json_empty_object.test index 7d8a3abc43a2..72f7443040f3 100644 --- a/test/sql/json/test_json_empty_object.test +++ b/test/sql/json/test_json_empty_object.test @@ -30,8 +30,9 @@ MAP(VARCHAR, JSON) # can't force it to have an empty struct statement error -select * from read_json('__TEST_DIR__/my.json', columns={a: 'STRUCT()') +select * from read_json('__TEST_DIR__/my.json', columns={a: 'STRUCT()'}) ---- +Invalid Input Error: Value "STRUCT()" can not be converted to a DuckDB Type. # test issue 6443 statement ok diff --git a/test/sql/logging/file_system_logging.test b/test/sql/logging/file_system_logging.test deleted file mode 100644 index 6aa2ed0ca189..000000000000 --- a/test/sql/logging/file_system_logging.test +++ /dev/null @@ -1,56 +0,0 @@ -# name: test/sql/logging/file_system_logging.test -# group: [logging] - -require parquet - -require noforcestorage - -statement ok -set enable_logging = true; - -statement ok -set logging_level='trace'; - -statement ok -COPY (SELECT 1 as a) TO '__TEST_DIR__/test.csv' - -statement ok -FROM '__TEST_DIR__/test.csv' - -statement ok -pragma threads=1 - -# Note: regex for test stability -query IIII -SELECT scope, type, log_level, regexp_replace(message, '\"path\":.*test.csv"', '"test.csv"') -FROM duckdb_logs -WHERE type = 'FileSystem' -ORDER BY timestamp ----- -CONNECTION FileSystem TRACE {"fs":"LocalFileSystem","test.csv","op":"OPEN"} -CONNECTION FileSystem TRACE {"fs":"LocalFileSystem","test.csv","op":"WRITE","bytes":"4","pos":"0"} -CONNECTION FileSystem TRACE {"fs":"LocalFileSystem","test.csv","op":"CLOSE"} -CONNECTION FileSystem TRACE {"fs":"LocalFileSystem","test.csv","op":"OPEN"} -CONNECTION FileSystem TRACE {"fs":"LocalFileSystem","test.csv","op":"READ","bytes":"4","pos":"0"} -CONNECTION FileSystem TRACE {"fs":"LocalFileSystem","test.csv","op":"READ","bytes":"0","pos":"4"} -CONNECTION FileSystem TRACE {"fs":"LocalFileSystem","test.csv","op":"CLOSE"} - -statement ok -CALL truncate_duckdb_logs(); - -require httpfs - -statement ok -FROM 'https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv' - -# FIXME: investigate why we call READ twice? -query IIII -SELECT scope, type, log_level, regexp_replace(message, '\"path\":.*test.csv"', '"test.csv"') -FROM duckdb_logs -WHERE type = 'FileSystem' AND message NOT LIKE '%duckdb_extension%' -ORDER BY timestamp ----- -CONNECTION FileSystem TRACE {"fs":"HTTPFileSystem","path":"https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv","op":"OPEN"} -CONNECTION FileSystem TRACE {"fs":"HTTPFileSystem","path":"https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv","op":"READ","bytes":"1276","pos":"0"} -CONNECTION FileSystem TRACE {"fs":"HTTPFileSystem","path":"https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv","op":"READ","bytes":"0","pos":"1276"} -CONNECTION FileSystem TRACE {"fs":"HTTPFileSystem","path":"https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv","op":"CLOSE"} diff --git a/test/sql/logging/http_logging.test b/test/sql/logging/http_logging.test deleted file mode 100644 index 031e43e0e1fc..000000000000 --- a/test/sql/logging/http_logging.test +++ /dev/null @@ -1,45 +0,0 @@ -# name: test/sql/logging/http_logging.test -# group: [logging] - -require parquet - -require httpfs - -statement ok -CALL enable_logging('HTTP'); - -statement ok -FROM 'https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv' - -query IIII -SELECT - request.type, - request.url, - response.status, - response.reason, -FROM duckdb_logs_parsed('HTTP') WHERE response.status != 'ServiceUnavailable_503' ----- -HEAD https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv OK_200 OK -GET https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv PartialContent_206 Partial Content - -query II -SELECT request.headers['Range'], response.headers['Content-Range'] -FROM duckdb_logs_parsed('HTTP') -WHERE request.type='GET' ----- -bytes=0-1275 bytes 0-1275/1276 - -statement ok -CALL truncate_duckdb_logs() - -# This old option still exists, however it now logs to the duckdb log instead of printing straight to stdout -statement ok -set enable_http_logging=false; - -statement ok -FROM 'https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv' - -query I -select count(*) FROM duckdb_logs_parsed('HTTP'); ----- -0 diff --git a/test/sql/logging/test_logging_function.test b/test/sql/logging/test_logging_function.test index 6fb6a6f34295..c8059d76f668 100644 --- a/test/sql/logging/test_logging_function.test +++ b/test/sql/logging/test_logging_function.test @@ -8,6 +8,11 @@ query IIIIIIIIII from duckdb_logs ---- +statement error +PRAGMA enable_logging; +---- +Pragma Function with name enable_logging does not exist, but a table function with the same name exists, try + statement ok CALL enable_logging(); diff --git a/test/sql/merge/merge_into_parenthesis_bug.test b/test/sql/merge/merge_into_parenthesis_bug.test new file mode 100644 index 000000000000..fc03a329c9fd --- /dev/null +++ b/test/sql/merge/merge_into_parenthesis_bug.test @@ -0,0 +1,37 @@ +# name: test/sql/merge/merge_into_parenthesis_bug.test +# description: Test MERGE issue when condition is in parenthesis +# group: [merge] + +statement ok +PRAGMA enable_verification + +statement ok +CREATE TABLE my_timeseries (ts TIMESTAMP, x DOUBLE PRECISION, y DOUBLE PRECISION); + +statement ok +insert into my_timeseries VALUES ('2025-09-15', 43, 39) + +statement ok +CREATE TABLE my_timeseries_new (ts TIMESTAMP, x DOUBLE PRECISION, y DOUBLE PRECISION); + +statement ok +insert into my_timeseries_new VALUES ('2025-09-15', 43, 39) + +statement ok +MERGE INTO my_timeseries old + USING my_timeseries_new new + ON ( + old.x = new.x AND + ( old.ts != new.ts + OR old.x = 1 + ) + ) +WHEN MATCHED THEN UPDATE; + +statement ok +MERGE INTO my_timeseries old + USING my_timeseries_new new + USING(ts) +WHEN MATCHED AND ( + old.x IS DISTINCT FROM new.y +) THEN UPDATE; diff --git a/test/sql/parallelism/interquery/concurrent_attach_detach.cpp b/test/sql/parallelism/interquery/concurrent_attach_detach.cpp index 24b9d5564954..6410e46e3548 100644 --- a/test/sql/parallelism/interquery/concurrent_attach_detach.cpp +++ b/test/sql/parallelism/interquery/concurrent_attach_detach.cpp @@ -1,14 +1,19 @@ #include "catch.hpp" -#include "test_helpers.hpp" -#include "duckdb/common/mutex.hpp" #include "duckdb/common/atomic.hpp" #include "duckdb/common/map.hpp" +#include "duckdb/common/mutex.hpp" +#include "duckdb/common/vector.hpp" +#include "duckdb/common/optional_idx.hpp" +#include "test_helpers.hpp" -#include +#include #include using namespace duckdb; -using namespace std; + +enum class AttachTaskType { CREATE_TABLE, LOOKUP, APPEND, APPLY_CHANGES, DESCRIBE_TABLE, CHECKPOINT }; + +namespace { string test_dir_path; const string prefix = "db_"; @@ -22,128 +27,472 @@ string getDBName(idx_t i) { return prefix + to_string(i); } -const idx_t dbCount = 10; -const idx_t workerCount = 40; -const idx_t iterationCount = 100; -atomic success; +const idx_t db_count = 10; +const idx_t worker_count = 40; +const idx_t iteration_count = 100; +const idx_t nr_initial_rows = 2050; + +vector> logging; +atomic success {true}; -void execQuery(Connection &conn, const string &query) { +duckdb::unique_ptr execQuery(Connection &conn, const string &query) { auto result = conn.Query(query); if (result->HasError()) { - Printer::Print(result->GetError()); + Printer::PrintF("Failed to execute query %s:\n------\n%s\n-------", query, result->GetError()); success = false; } + return result; } +struct TableInfo { + idx_t size; +}; + struct DBInfo { mutex mu; - idx_t count = 0; + idx_t table_count = 0; + vector tables; +}; + +struct AttachTask { + AttachTaskType type; + duckdb::optional_idx db_id; + duckdb::optional_idx tbl_id; + duckdb::optional_idx tbl_size; + std::vector ids; + bool actual_describe = false; }; -DBInfo dbInfos[dbCount]; +struct AttachWorker; class DBPoolMgr { public: mutex mu; map m; - void addWorker(Connection &conn, idx_t i) { - lock_guard lock(mu); + void addWorker(AttachWorker &worker, const idx_t i); + void removeWorker(AttachWorker &worker, const idx_t i); - if (m.find(i) != m.end()) { - m[i]++; - return; - } + DBInfo db_infos[db_count]; +}; - m[i] = 1; - string query = "ATTACH '" + getDBPath(i) + "'"; - execQuery(conn, query); +struct AttachWorker { +public: + AttachWorker(DuckDB &db, idx_t worker_id, vector &logs, DBPoolMgr &db_pool) + : conn(db), worker_id(worker_id), logs(logs), db_pool(db_pool) { } - void removeWorker(Connection &conn, idx_t i) { - lock_guard lock(mu); +public: + duckdb::unique_ptr execQuery(const string &query) { + return ::execQuery(conn, query); + } + void Work(); - m[i]--; - if (m[i] != 0) { - return; +private: + AttachTask RandomTask(); + void createTbl(AttachTask &task); + void lookup(AttachTask &task); + void append_internal(AttachTask &task); + void append(AttachTask &task); + void delete_internal(AttachTask &task); + void apply_changes(AttachTask &task); + void describe_tbl(AttachTask &task); + void checkpoint_db(AttachTask &task); + void GetRandomTable(AttachTask &task); + void addLog(const string &msg) { + logs.push_back(msg); + } + +public: + Connection conn; + idx_t worker_id; + vector &logs; + DBPoolMgr &db_pool; +}; + +void DBPoolMgr::addWorker(AttachWorker &worker, const idx_t i) { + lock_guard lock(mu); + + if (m.find(i) != m.end()) { + m[i]++; + return; + } + m[i] = 1; + + string query = "ATTACH '" + getDBPath(i) + "'"; + worker.execQuery(query); +} + +void DBPoolMgr::removeWorker(AttachWorker &worker, const idx_t i) { + lock_guard lock(mu); + + m[i]--; + if (m[i] != 0) { + return; + } + + m.erase(i); + string query = "DETACH " + getDBName(i); + worker.execQuery(query); +} + +void AttachWorker::createTbl(AttachTask &task) { + auto db_id = task.db_id.GetIndex(); + auto &db_infos = db_pool.db_infos; + lock_guard lock(db_infos[db_id].mu); + auto tbl_id = db_infos[db_id].table_count; + db_infos[db_id].tables.emplace_back(TableInfo {nr_initial_rows}); + db_infos[db_id].table_count++; + + string tbl_path = StringUtil::Format("%s.tbl_%d", getDBName(db_id), tbl_id); + string create_sql = StringUtil::Format( + "CREATE TABLE %s(i BIGINT PRIMARY KEY, s VARCHAR, ts TIMESTAMP, obj STRUCT(key1 UBIGINT, key2 VARCHAR))", + tbl_path); + addLog("; q: " + create_sql); + execQuery(create_sql); + string insert_sql = "INSERT INTO " + tbl_path + + " SELECT " + "range::UBIGINT AS i, " + "range::VARCHAR AS s, " + // Note: We increment timestamps by 1 millisecond (i.e., 1000 microseconds). + "epoch_ms(range) AS ts, " + "{'key1': range::UBIGINT, 'key2': range::VARCHAR} AS obj " + "FROM range(" + + to_string(nr_initial_rows) + ")"; + addLog("; q: " + insert_sql); + execQuery(insert_sql); +} + +void AttachWorker::lookup(AttachTask &task) { + if (!task.tbl_id.IsValid()) { + return; + } + auto db_id = task.db_id.GetIndex(); + auto tbl_id = task.tbl_id.GetIndex(); + auto expected_max_val = task.tbl_size.GetIndex() - 1; + + // Run the query. + auto table_name = getDBName(db_id) + ".tbl_" + to_string(tbl_id); + string query = "SELECT i, s, ts, obj FROM " + table_name + " WHERE i = " + to_string(expected_max_val); + addLog("q: " + query); + auto result = execQuery(query); + if (result->RowCount() == 0) { + addLog("FAILURE - No rows returned from query"); + success = false; + } + if (!CHECK_COLUMN(result, 0, {Value::UBIGINT(expected_max_val)})) { + success = false; + return; + } + if (!CHECK_COLUMN(result, 1, {to_string(expected_max_val)})) { + success = false; + return; + } + if (!CHECK_COLUMN(result, 2, {Value::TIMESTAMP(timestamp_t {static_cast(expected_max_val * 1000)})})) { + success = false; + return; + } + if (!CHECK_COLUMN( + result, 3, + {Value::STRUCT({{"key1", Value::UBIGINT(expected_max_val)}, {"key2", to_string(expected_max_val)}})})) { + success = false; + return; + } +} + +void AttachWorker::append_internal(AttachTask &task) { + auto db_id = task.db_id.GetIndex(); + auto tbl_id = task.tbl_id.GetIndex(); + auto tbl_str = "tbl_" + to_string(tbl_id); + // set appender + addLog("db: " + getDBName(db_id) + "; table: " + tbl_str + "; append rows"); + + try { + Appender appender(conn, getDBName(db_id), DEFAULT_SCHEMA, tbl_str); + DataChunk chunk; + + child_list_t struct_children; + struct_children.emplace_back(make_pair("key1", LogicalTypeId::UBIGINT)); + struct_children.emplace_back(make_pair("key2", LogicalTypeId::VARCHAR)); + + const vector types = {LogicalType::UBIGINT, LogicalType::VARCHAR, LogicalType::TIMESTAMP, + LogicalType::STRUCT(struct_children)}; + + // fill up datachunk + chunk.Initialize(*conn.context, types); + // int + auto &col_ubigint = chunk.data[0]; + auto data_ubigint = FlatVector::GetData(col_ubigint); + // varchar + auto &col_varchar = chunk.data[1]; + auto data_varchar = FlatVector::GetData(col_varchar); + // timestamp + auto &col_ts = chunk.data[2]; + auto data_ts = FlatVector::GetData(col_ts); + // struct + auto &col_struct = chunk.data[3]; + auto &data_struct_entries = StructVector::GetEntries(col_struct); + auto &entry_ubigint = data_struct_entries[0]; + auto data_struct_ubigint = FlatVector::GetData(*entry_ubigint); + auto &entry_varchar = data_struct_entries[1]; + auto data_struct_varchar = FlatVector::GetData(*entry_varchar); + + for (idx_t i = 0; i < task.ids.size(); i++) { + auto row_idx = task.ids[i]; + data_ubigint[i] = row_idx; + data_varchar[i] = StringVector::AddString(col_varchar, to_string(row_idx)); + data_ts[i] = timestamp_t {static_cast(1000 * (row_idx))}; + data_struct_ubigint[i] = row_idx; + data_struct_varchar[i] = StringVector::AddString(*entry_varchar, to_string(row_idx)); } - m.erase(i); - string query = "DETACH " + getDBName(i); - execQuery(conn, query); + chunk.SetCardinality(task.ids.size()); + appender.AppendDataChunk(chunk); + appender.Close(); + + } catch (const std::exception &e) { + addLog("Caught exception when using Appender: " + string(e.what())); + success = false; + return; + } catch (...) { + addLog("Caught error when using Appender!"); + success = false; + return; + } +} + +void AttachWorker::append(AttachTask &task) { + if (!task.tbl_id.IsValid()) { + return; } -}; + auto db_id = task.db_id.GetIndex(); + auto tbl_id = task.tbl_id.GetIndex(); + auto &db_infos = db_pool.db_infos; + lock_guard lock(db_infos[db_id].mu); + auto current_num_rows = db_infos[db_id].tables[tbl_id].size; + idx_t append_count = STANDARD_VECTOR_SIZE; -DBPoolMgr dbPool; + for (idx_t i = 0; i < append_count; i++) { + task.ids.push_back(current_num_rows + i); + } + + append_internal(task); + db_infos[db_id].tables[tbl_id].size += append_count; +} + +void AttachWorker::delete_internal(AttachTask &task) { + auto db_id = task.db_id.GetIndex(); + auto tbl_id = task.tbl_id.GetIndex(); + auto &ids = task.ids; + auto tbl_str = "tbl_" + to_string(tbl_id); + + string delete_list; + for (auto delete_idx : ids) { + if (!delete_list.empty()) { + delete_list += ", "; + } + delete_list += "(" + to_string(delete_idx) + ")"; + } + string delete_sql = + StringUtil::Format("WITH ids (id) AS (VALUES %s) DELETE FROM %s.%s.%s AS t USING ids WHERE t.i = ids.id", + delete_list, getDBName(db_id), DEFAULT_SCHEMA, tbl_str); + addLog("q: " + delete_sql); + execQuery(delete_sql); +} -void lookup(Connection &conn, idx_t i) { - unique_lock lock(dbInfos[i].mu); - auto maxTblId = dbInfos[i].count; - lock.unlock(); +void AttachWorker::apply_changes(AttachTask &task) { + if (!task.tbl_id.IsValid()) { + return; + } + auto db_id = task.db_id.GetIndex(); + auto &db_infos = db_pool.db_infos; + lock_guard lock(db_infos[db_id].mu); + execQuery("BEGIN"); + delete_internal(task); + append_internal(task); + execQuery("COMMIT"); +} - if (maxTblId == 0) { +void AttachWorker::describe_tbl(AttachTask &task) { + if (!task.tbl_id.IsValid()) { return; } + auto db_id = task.db_id.GetIndex(); + auto tbl_id = task.tbl_id.GetIndex(); + auto tbl_str = "tbl_" + to_string(tbl_id); + auto actual_describe = task.actual_describe; + string describe_sql; + if (actual_describe) { + describe_sql = StringUtil::Format("DESCRIBE %s.%s.%s", getDBName(db_id), DEFAULT_SCHEMA, tbl_str); + } else { + describe_sql = StringUtil::Format("SELECT 1 FROM %s.%s.%s LIMIT 1", getDBName(db_id), DEFAULT_SCHEMA, tbl_str); + } - auto tblId = std::rand() % maxTblId; - string query = "SELECT i, s FROM " + getDBName(i) + ".tbl_" + to_string(tblId) + " WHERE i = 2049"; - execQuery(conn, query); + addLog("q: " + describe_sql); + execQuery(describe_sql); } -void createLookupTbl(Connection &conn, idx_t i) { - lock_guard lock(dbInfos[i].mu); - auto tblId = dbInfos[i].count; - dbInfos[i].count++; +void AttachWorker::checkpoint_db(AttachTask &task) { + auto db_id = task.db_id.GetIndex(); + auto &db_infos = db_pool.db_infos; + unique_lock lock(db_infos[db_id].mu); + string checkpoint_sql = "CHECKPOINT " + getDBName(db_id); + addLog("q: " + checkpoint_sql); + // checkpoint can fail, we don't care + conn.Query(checkpoint_sql); +} - string query = "CREATE TABLE " + getDBName(i) + ".tbl_" + to_string(tblId) + - " AS SELECT range AS i, range::VARCHAR AS s FROM range(10000)"; - execQuery(conn, query); +void AttachWorker::GetRandomTable(AttachTask &task) { + auto &db_infos = db_pool.db_infos; + auto db_id = task.db_id.GetIndex(); + lock_guard lock(db_infos[db_id].mu); + auto max_tbl_id = db_infos[db_id].table_count; + + if (max_tbl_id == 0) { + return; + } + + task.tbl_id = std::rand() % max_tbl_id; + task.tbl_size = db_infos[db_id].tables[task.tbl_id.GetIndex()].size; } -void workUnit(std::unique_ptr conn) { - for (int i = 0; i < iterationCount; i++) { - idx_t scenarioId = std::rand() % 2; - idx_t dbId = std::rand() % dbCount; +AttachTask AttachWorker::RandomTask() { + AttachTask result; + idx_t scenario_id = std::rand() % 10; + result.db_id = std::rand() % db_count; + auto db_id = result.db_id.GetIndex(); + switch (scenario_id) { + case 0: + result.type = AttachTaskType::CREATE_TABLE; + GetRandomTable(result); + break; + case 1: + result.type = AttachTaskType::LOOKUP; + GetRandomTable(result); + break; + case 2: + result.type = AttachTaskType::APPEND; + GetRandomTable(result); + break; + case 3: + result.type = AttachTaskType::APPLY_CHANGES; + GetRandomTable(result); + if (result.tbl_id.IsValid()) { + auto current_num_rows = result.tbl_size.GetIndex(); + idx_t delete_count = std::rand() % (STANDARD_VECTOR_SIZE / 3); + if (delete_count == 0) { + delete_count = 1; + } - dbPool.addWorker(*conn, dbId); + unordered_set unique_ids; + for (idx_t i = 0; i < delete_count; i++) { + unique_ids.insert(std::rand() % current_num_rows); + } + for (auto &id : unique_ids) { + result.ids.push_back(id); + } + } + break; + case 4: + case 5: + case 6: + case 7: + case 8: + result.type = AttachTaskType::DESCRIBE_TABLE; + GetRandomTable(result); + result.actual_describe = std::rand() % 2 == 0; + break; + default: + result.type = AttachTaskType::CHECKPOINT; + break; + } + return result; +} - switch (scenarioId) { - case 0: - lookup(*conn, dbId); - break; - case 1: - createLookupTbl(*conn, dbId); - break; - default: - throw runtime_error("invalid scenario"); +void AttachWorker::Work() { + for (idx_t i = 0; i < iteration_count; i++) { + if (!success) { + return; } - dbPool.removeWorker(*conn, dbId); + try { + auto task = RandomTask(); + + db_pool.addWorker(*this, task.db_id.GetIndex()); + + switch (task.type) { + case AttachTaskType::CREATE_TABLE: + createTbl(task); + break; + case AttachTaskType::LOOKUP: + lookup(task); + break; + case AttachTaskType::APPEND: + append(task); + break; + case AttachTaskType::APPLY_CHANGES: + apply_changes(task); + break; + case AttachTaskType::DESCRIBE_TABLE: + describe_tbl(task); + break; + case AttachTaskType::CHECKPOINT: + checkpoint_db(task); + break; + default: + addLog("invalid task type"); + success = false; + return; + } + db_pool.removeWorker(*this, task.db_id.GetIndex()); + + } catch (const std::exception &e) { + addLog("Caught exception when running iterations: " + string(e.what())); + success = false; + return; + } catch (...) { + addLog("Caught unknown when using running iterations"); + success = false; + return; + } } } -TEST_CASE("Run a concurrent ATTACH/DETACH scenario", "[attach][.]") { - test_dir_path = TestDirectoryPath(); +void workUnit(std::unique_ptr worker) { + worker->Work(); +} +TEST_CASE("Run a concurrent ATTACH/DETACH scenario", "[interquery][.]") { + test_dir_path = TestDirectoryPath(); + DBPoolMgr db_pool; DuckDB db(nullptr); - Connection initConn(db); + Connection init_conn(db); - execQuery(initConn, "SET catalog_error_max_schemas = '0'"); - execQuery(initConn, "SET threads = '1'"); - // execQuery(initConn, "SET default_block_size = '16384'"); - // execQuery(initConn, "SET storage_compatibility_version = 'v1.3.2'"); + execQuery(init_conn, "SET catalog_error_max_schemas = '0'"); + execQuery(init_conn, "SET threads = '1'"); + execQuery(init_conn, "SET storage_compatibility_version = 'latest'"); + execQuery(init_conn, "CALL enable_logging()"); + execQuery(init_conn, "PRAGMA enable_profiling='no_output'"); - success = true; - std::vector workers; - for (int i = 0; i < workerCount; i++) { - auto conn = make_uniq(db); - workers.emplace_back(workUnit, std::move(conn)); + logging.resize(worker_count); + vector workers; + for (idx_t i = 0; i < worker_count; i++) { + auto worker = make_uniq(db, i, logging[i], db_pool); + workers.emplace_back(workUnit, std::move(worker)); } for (auto &worker : workers) { worker.join(); } if (!success) { + for (idx_t worker_id = 0; worker_id < logging.size(); worker_id++) { + for (auto &log : logging[worker_id]) { + Printer::PrintF("thread %d; %s", worker_id, log); + } + } FAIL(); } + ClearTestDirectory(); } + +} // anonymous namespace diff --git a/test/sql/parser/test_long_error.test b/test/sql/parser/test_long_error.test index 34cda11a9cc9..e6315f1bd545 100644 --- a/test/sql/parser/test_long_error.test +++ b/test/sql/parser/test_long_error.test @@ -5,7 +5,9 @@ statement error xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ---- +:.*Parser Error.*syntax error at or near.* statement error SELECT xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ---- +:.*Binder Error.*Referenced column.* \ No newline at end of file diff --git a/test/sql/peg_parser/alter.test b/test/sql/peg_parser/alter.test new file mode 100644 index 000000000000..be8c7e7c43a2 --- /dev/null +++ b/test/sql/peg_parser/alter.test @@ -0,0 +1,8 @@ +# name: test/sql/peg_parser/alter.test +# description: Test analyze and vacuum statements in peg parser +# group: [peg_parser] + +require autocomplete + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$ALTER TABLE tbl SET SORTED BY (i DESC NULLS FIRST)$TEST_PEG_PARSER$); diff --git a/test/sql/peg_parser/copy_expression.test b/test/sql/peg_parser/copy_expression.test new file mode 100644 index 000000000000..40c90a384a84 --- /dev/null +++ b/test/sql/peg_parser/copy_expression.test @@ -0,0 +1,12 @@ +# name: test/sql/peg_parser/copy_expression.test +# description: Test copy expression syntax in peg parser +# group: [peg_parser] + +require autocomplete + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$COPY (SELECT * FROM foo) TO ('foo') WITH (HEADER)$TEST_PEG_PARSER$); + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$COPY (SELECT * FROM range(5) t(i)) TO (getvariable('copy_target')) WITH (HEADER)$TEST_PEG_PARSER$); + diff --git a/test/sql/peg_parser/create_macro.test b/test/sql/peg_parser/create_macro.test index 347dbdaa8275..38be62a9286d 100644 --- a/test/sql/peg_parser/create_macro.test +++ b/test/sql/peg_parser/create_macro.test @@ -6,3 +6,10 @@ require autocomplete statement ok CALL check_peg_parser($TEST_PEG_PARSER$create macro my_range(x) as table from range(x);$TEST_PEG_PARSER$); + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$create macro m(s varchar) as s || 'c'$TEST_PEG_PARSER$); + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$create or replace macro m(s varchar := 'cc') as s || 'c'$TEST_PEG_PARSER$); + diff --git a/test/sql/peg_parser/create_type.test b/test/sql/peg_parser/create_type.test new file mode 100644 index 000000000000..e417bc47ebbb --- /dev/null +++ b/test/sql/peg_parser/create_type.test @@ -0,0 +1,9 @@ +# name: test/sql/peg_parser/create_type.test +# description: Test create table syntax in peg parser +# group: [peg_parser] + +require autocomplete + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$CREATE TYPE t4 AS UNION ( v0 SETOF t4 );$TEST_PEG_PARSER$); + diff --git a/test/sql/peg_parser/index.test b/test/sql/peg_parser/index.test index 0bf8d3ff2322..5fea353a9409 100644 --- a/test/sql/peg_parser/index.test +++ b/test/sql/peg_parser/index.test @@ -4,6 +4,9 @@ require autocomplete +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$create unique index part_comp_key_index on insertconflicttest(key, fruit) where 1;$TEST_PEG_PARSER$); + statement ok CALL check_peg_parser($TEST_PEG_PARSER$CREATE INDEX my_ip_idx ON t1 USING HNSW (vec) WITH (metric = 'ip');$TEST_PEG_PARSER$); diff --git a/test/sql/peg_parser/insert.test b/test/sql/peg_parser/insert.test new file mode 100644 index 000000000000..b1c45ce3fe4a --- /dev/null +++ b/test/sql/peg_parser/insert.test @@ -0,0 +1,16 @@ +# name: test/sql/peg_parser/insert.test +# description: Test analyze and vacuum statements in peg parser +# group: [peg_parser] + +require autocomplete + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$SELECT TRUE OFFSET 1 ROWS;$TEST_PEG_PARSER$); + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$INSERT INTO v0 BY POSITION ( SELECT TRUE ) OFFSET 1 ROWS RETURNING v0;$TEST_PEG_PARSER$); + +statement error +CALL check_peg_parser($TEST_PEG_PARSER$INSERT INTO table2 WITH cte AS (INSERT INTO table1 SELECT 1) SELECT id FROM cte;$TEST_PEG_PARSER$); +---- +:.*Binder Error.* diff --git a/test/sql/peg_parser/show_table.test b/test/sql/peg_parser/show_table.test new file mode 100644 index 000000000000..5c65c73416c9 --- /dev/null +++ b/test/sql/peg_parser/show_table.test @@ -0,0 +1,9 @@ +# name: test/sql/peg_parser/show_table.test +# description: Test create macro syntax in peg parser +# group: [peg_parser] + +require autocomplete + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$SHOW TABLES FROM db.main$TEST_PEG_PARSER$); + diff --git a/test/sql/peg_parser/transformer/peg_transformer.test b/test/sql/peg_parser/transformer/peg_transformer.test new file mode 100644 index 000000000000..14c068fd5b98 --- /dev/null +++ b/test/sql/peg_parser/transformer/peg_transformer.test @@ -0,0 +1,17 @@ +# name: test/sql/peg_parser/transformer/peg_transformer.test +# description: Test analyze and vacuum statements in peg parser +# group: [transformer] + +require autocomplete + +require skip_reload + +require no_extension_autoloading "FIXME: to be reviewed whether this can be lifted" + +statement ok +set allow_parser_override_extension=strict; + +statement error +select 1; +---- +Not implemented Error: Parser override has not yet implemented this transformer rule. (Original error: No transformer function found for rule 'SelectStatement') diff --git a/test/sql/peg_parser/transformer/set_statement.test b/test/sql/peg_parser/transformer/set_statement.test new file mode 100644 index 000000000000..4edca3f61374 --- /dev/null +++ b/test/sql/peg_parser/transformer/set_statement.test @@ -0,0 +1,27 @@ +# name: test/sql/peg_parser/transformer/set_statement.test +# description: Test lambda syntax in peg parser +# group: [transformer] + +require autocomplete + +require icu + +statement ok +set allow_parser_override_extension=strict; + +statement ok +SET threads=1; + +statement ok +set VARIABLE dataset_size = 122880; + +statement error +SET GLOBAL enable_progress_bar=true; +---- +Catalog Error: option "enable_progress_bar" cannot be set globally + +statement ok +RESET threads; + +statement ok +SET TimeZone='UTC'; \ No newline at end of file diff --git a/test/sql/peg_parser/transformer/use_statement.test b/test/sql/peg_parser/transformer/use_statement.test new file mode 100644 index 000000000000..9dac8a8d4f41 --- /dev/null +++ b/test/sql/peg_parser/transformer/use_statement.test @@ -0,0 +1,24 @@ +# name: test/sql/peg_parser/transformer/use_statement.test +# description: Test use statement with new transformer +# group: [transformer] + +require autocomplete + +require skip_reload + +require no_extension_autoloading "FIXME: to be reviewed whether this can be lifted" + +statement ok +ATTACH ':memory:' as "my""db"; + +statement ok +CREATE TABLE "my""db".tbl(i int); + +statement ok +INSERT INTO "my""db".tbl VALUES (42) + +statement ok +set allow_parser_override_extension=strict; + +statement ok +USE "my""db"; diff --git a/test/sql/peg_parser/with_ordinality.test b/test/sql/peg_parser/with_ordinality.test new file mode 100644 index 000000000000..78f5778037cd --- /dev/null +++ b/test/sql/peg_parser/with_ordinality.test @@ -0,0 +1,11 @@ +# name: test/sql/peg_parser/with_ordinality.test +# description: Test with ordinality syntax in peg parser +# group: [peg_parser] + +require autocomplete + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$FROM foo() WITH ORDINALITY AS _(ordinality);$TEST_PEG_PARSER$); + +statement ok +CALL check_peg_parser($TEST_PEG_PARSER$SELECT a,my_range,my_ordinality FROM test AS t(a), LATERAL range(t.a) WITH ORDINALITY AS _(my_range,my_ordinality) ORDER BY a,my_range;$TEST_PEG_PARSER$); diff --git a/test/sql/pivot/test_pivot.test b/test/sql/pivot/test_pivot.test index 3af9a9f5e606..01d452a878d0 100644 --- a/test/sql/pivot/test_pivot.test +++ b/test/sql/pivot/test_pivot.test @@ -266,7 +266,7 @@ SELECT * AS p ORDER BY EMPID; ---- - +Parser Error: syntax error at or near ")" # star statement error @@ -276,7 +276,7 @@ SELECT * AS p ORDER BY EMPID; ---- -must contain columns or lists of columns +Parser Error: PIVOT IN list must contain columns or lists of columns query I FROM diff --git a/test/sql/pragma/profiling/test_attach_and_checkpoint_latency.test b/test/sql/pragma/profiling/test_attach_and_checkpoint_latency.test new file mode 100644 index 000000000000..97257c196669 --- /dev/null +++ b/test/sql/pragma/profiling/test_attach_and_checkpoint_latency.test @@ -0,0 +1,102 @@ +# name: test/sql/pragma/profiling/test_attach_and_checkpoint_latency.test +# group: [profiling] + +require json + +require noforcestorage + +require skip_reload + +# Setup. + +statement ok +SET threads = 1; + +statement ok +SET wal_autocheckpoint = '1TB'; + +statement ok +PRAGMA disable_checkpoint_on_shutdown; + +statement ok +PRAGMA profiling_output = '__TEST_DIR__/profile_fs.json'; + +statement ok +PRAGMA custom_profiling_settings='{"WAITING_TO_ATTACH_LATENCY": "true", "ATTACH_LOAD_STORAGE_LATENCY": "true", "ATTACH_REPLAY_WAL_LATENCY": "true", "CHECKPOINT_LATENCY": "true"}'; + +statement ok +SET profiling_coverage='ALL'; + +# Finished setup. + +# CHECKPOINT_LATENCY. + +statement ok +ATTACH '__TEST_DIR__/profile_fs.db'; + +statement ok +CREATE TABLE profile_fs.tbl AS SELECT range AS id FROM range(100_000); + +statement ok +PRAGMA enable_profiling = 'json'; + +statement ok +CHECKPOINT profile_fs; + +statement ok +PRAGMA disable_profiling; + +statement ok +CREATE OR REPLACE TABLE metrics_output AS SELECT * FROM '__TEST_DIR__/profile_fs.json'; + +query I +SELECT + CASE WHEN checkpoint_latency > 0 THEN 'true' + ELSE 'false' END +FROM metrics_output; +---- +true + +# WAITING_TO_ATTACH_LATENCY, ATTACH_LOAD_STORAGE_LATENCY and ATTACH_REPLAY_WAL_LATENCY. + +statement ok +CREATE TABLE profile_fs.other_tbl AS SELECT range AS id FROM range(100_000); + +statement ok +DETACH profile_fs; + +statement ok +PRAGMA enable_profiling = 'json'; + +statement ok +ATTACH '__TEST_DIR__/profile_fs.db'; + +statement ok +PRAGMA disable_profiling; + +statement ok +CREATE OR REPLACE TABLE metrics_output AS SELECT * FROM '__TEST_DIR__/profile_fs.json'; + +query I +SELECT + CASE WHEN waiting_to_attach_latency > 0 THEN 'true' + ELSE 'false' END +FROM metrics_output; +---- +true + +query I +SELECT + CASE WHEN attach_load_storage_latency > 0 THEN 'true' + ELSE 'false' END +FROM metrics_output; +---- +true + +query I +SELECT + CASE WHEN attach_replay_wal_latency > 0 THEN 'true' + ELSE 'false' END +FROM metrics_output; +---- +true diff --git a/test/sql/pragma/profiling/test_custom_profiling_optimizer.test b/test/sql/pragma/profiling/test_custom_profiling_optimizer.test index 791dddb42e75..bbf753c222ec 100644 --- a/test/sql/pragma/profiling/test_custom_profiling_optimizer.test +++ b/test/sql/pragma/profiling/test_custom_profiling_optimizer.test @@ -169,7 +169,10 @@ SELECT unnest(res) FROM ( string_split(setting, ', ') AS res ) ORDER BY ALL; ---- +"ATTACH_LOAD_STORAGE_LATENCY": "true" +"ATTACH_REPLAY_WAL_LATENCY": "true" "BLOCKED_THREAD_TIME": "true" +"CHECKPOINT_LATENCY": "true" "CPU_TIME": "true" "CUMULATIVE_CARDINALITY": "true" "CUMULATIVE_ROWS_SCANNED": "true" @@ -187,6 +190,7 @@ SELECT unnest(res) FROM ( "SYSTEM_PEAK_TEMP_DIR_SIZE": "true" "TOTAL_BYTES_READ": "true" "TOTAL_BYTES_WRITTEN": "true" +"WAITING_TO_ATTACH_LATENCY": "true" statement ok CREATE OR REPLACE TABLE metrics_output AS SELECT * FROM '__TEST_DIR__/profiling_output.json'; diff --git a/test/sql/pragma/profiling/test_default_profiling_settings.test b/test/sql/pragma/profiling/test_default_profiling_settings.test index cf6ed0400f1d..70105c4c78e1 100644 --- a/test/sql/pragma/profiling/test_default_profiling_settings.test +++ b/test/sql/pragma/profiling/test_default_profiling_settings.test @@ -29,7 +29,10 @@ SELECT unnest(res) FROM ( string_split(setting, ', ') AS res ) ORDER BY ALL; ---- +"ATTACH_LOAD_STORAGE_LATENCY": "true" +"ATTACH_REPLAY_WAL_LATENCY": "true" "BLOCKED_THREAD_TIME": "true" +"CHECKPOINT_LATENCY": "true" "CPU_TIME": "true" "CUMULATIVE_CARDINALITY": "true" "CUMULATIVE_ROWS_SCANNED": "true" @@ -47,6 +50,7 @@ SELECT unnest(res) FROM ( "SYSTEM_PEAK_TEMP_DIR_SIZE": "true" "TOTAL_BYTES_READ": "true" "TOTAL_BYTES_WRITTEN": "true" +"WAITING_TO_ATTACH_LATENCY": "true" statement ok CREATE OR REPLACE TABLE metrics_output AS SELECT * FROM '__TEST_DIR__/profiling_output.json'; diff --git a/test/sql/pragma/profiling/test_profiling_fs.test b/test/sql/pragma/profiling/test_profiling_fs.test index bf02ebcbbc3e..be55d3c1f48c 100644 --- a/test/sql/pragma/profiling/test_profiling_fs.test +++ b/test/sql/pragma/profiling/test_profiling_fs.test @@ -8,6 +8,9 @@ require noforcestorage require skip_reload +# FIXME: Enable when more read paths have been added to profiling. +require no_alternative_verify + statement ok SET threads = 1; @@ -68,10 +71,10 @@ statement ok PRAGMA custom_profiling_settings='{"TOTAL_BYTES_READ": "true", "TOTAL_BYTES_WRITTEN": "true"}'; statement ok -SET profiling_coverage='ALL'; +ATTACH '__TEST_DIR__/profile_fs.db'; statement ok -ATTACH '__TEST_DIR__/profile_fs.db'; +SELECT * FROM profile_fs.tbl; statement ok PRAGMA disable_profiling; @@ -79,9 +82,6 @@ PRAGMA disable_profiling; statement ok CREATE OR REPLACE TABLE metrics_output AS SELECT * FROM '__TEST_DIR__/profile_fs.json'; -statement ok -SELECT total_bytes_read FROM metrics_output; - query I SELECT CASE WHEN total_bytes_read > 0 THEN 'true' diff --git a/test/sql/pragma/test_enable_profile.test b/test/sql/pragma/test_enable_profile.test index 0d57a5f456dd..22851b5034b8 100644 --- a/test/sql/pragma/test_enable_profile.test +++ b/test/sql/pragma/test_enable_profile.test @@ -6,10 +6,13 @@ statement error PRAGMA enable_profiling() ---- +Parser Error: syntax error at or near ")" + statement error PRAGMA enable_profiling='unsupported' ---- +Parser Error: Unrecognized print format unsupported, supported formats: [json, query_tree, query_tree_optimizer, no_output] statement error PRAGMA profiling_output diff --git a/test/sql/pragma/test_memory_limit.test b/test/sql/pragma/test_memory_limit.test index 3ae8f5cc3376..ccd279b9cfed 100644 --- a/test/sql/pragma/test_memory_limit.test +++ b/test/sql/pragma/test_memory_limit.test @@ -125,6 +125,7 @@ PRAGMA memory_limit statement error PRAGMA memory_limit() ---- +Parser Error: syntax error at or near ")" statement error PRAGMA memory_limit(1, 2) diff --git a/test/sql/pragma/test_pragma_parsing.test b/test/sql/pragma/test_pragma_parsing.test index 6553ecf20c95..7183787af945 100644 --- a/test/sql/pragma/test_pragma_parsing.test +++ b/test/sql/pragma/test_pragma_parsing.test @@ -6,11 +6,13 @@ statement error PRAG ---- +Parser Error: syntax error at or near "PRAG" # Pragma without a keyword statement error PRAGMA ---- +:.*Parser Error.*syntax error at end of input.* # Unknown pragma error statement error diff --git a/test/sql/prepared/test_basic_prepare.test b/test/sql/prepared/test_basic_prepare.test index 316ca76e4cd5..e777cb04a57b 100644 --- a/test/sql/prepared/test_basic_prepare.test +++ b/test/sql/prepared/test_basic_prepare.test @@ -42,10 +42,12 @@ EXECUTE s1(42, 'dpfkg') statement error PREPARE EXPLAIN SELECT 42 ---- +Parser Error: syntax error at or near "SELECT" statement error PREPARE CREATE TABLE a(i INTEGER) ---- +Parser Error: syntax error at or near "CREATE" statement error SELECT * FROM a; diff --git a/test/sql/projection/test_scalar_projection.test b/test/sql/projection/test_scalar_projection.test index ed78a924115c..e245a6df8f4a 100644 --- a/test/sql/projection/test_scalar_projection.test +++ b/test/sql/projection/test_scalar_projection.test @@ -49,10 +49,12 @@ SELECT 1 AS a, a * 2 statement error SELECT ---- +Parser Error: SELECT clause without selection list statement error SELECT FROM (SELECT 42) v1 ---- +Parser Error: SELECT clause without selection list # Scalar query from SQLLogicTests query I diff --git a/test/sql/projection/test_value_list.test b/test/sql/projection/test_value_list.test index 4a3800b1948a..be462fff1a2d 100644 --- a/test/sql/projection/test_value_list.test +++ b/test/sql/projection/test_value_list.test @@ -62,7 +62,7 @@ SELECT * FROM (VALUES ((SELECT MIN(a) FROM test), 2, 3), ((SELECT MAX(b) FROM te statement error SELECT * FROM (VALUES ('hello', 2), (1 + 3, '5'), (DATE '1992-09-20', 3)) v1; ---- -an explicit cast is required +:.*Not implemented Error.*an explicit cast is required.* # value list with NULLs query TI @@ -93,7 +93,7 @@ SELECT * FROM (VALUES (3), ('42')) v1; statement error SELECT * FROM (VALUES (3), ('hello')) v1; ---- -hello +:.*Conversion Error.*hello.* query I SELECT typeof(x) FROM (VALUES (DATE '1992-01-01'), ('1992-01-01')) v1(x) LIMIT 1; @@ -110,11 +110,13 @@ hello statement error SELECT * FROM (VALUES (1, 2, 3), (1,2)) v1; ---- +:.*Parser Error.*lists must all be the same length.* # default in value list is not allowed statement error SELECT * FROM (VALUES (DEFAULT, 2, 3), (1,2)) v1; ---- +:.*Parser Error.*lists must all be the same length.* # VALUES list for INSERT statement ok @@ -134,6 +136,7 @@ hello statement error INSERT INTO varchars VALUES (1, 2), ('hello', 3), (DEFAULT, DEFAULT); ---- +:.*Binder Error.*table varchars has 1 columns.* statement error INSERT INTO varchars (v) VALUES (1, 2), ('hello', 3), (DEFAULT, DEFAULT); @@ -142,9 +145,10 @@ INSERT INTO varchars (v) VALUES (1, 2), ('hello', 3), (DEFAULT, DEFAULT); statement error INSERT INTO varchars (v) VALUES (1, 2), ('hello'), (DEFAULT, DEFAULT); ---- +:.*Parser Error.*lists must all be the same length.* # operation on default not allowed statement error INSERT INTO varchars (v) VALUES (DEFAULT IS NULL); ---- - +:.*Binder Error.*not allowed.* \ No newline at end of file diff --git a/test/sql/secrets/create_secret.test_slow b/test/sql/secrets/create_secret.test_slow deleted file mode 100644 index 602ee129ae38..000000000000 --- a/test/sql/secrets/create_secret.test_slow +++ /dev/null @@ -1,76 +0,0 @@ -# name: test/sql/secrets/create_secret.test_slow -# description: Test secret creation using the default s3 secret provider -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -require httpfs - -# Ensure any currently stored secrets don't interfere with the test -statement ok -set allow_persistent_secrets=false; - -statement ok -reset s3_use_ssl - -# Create an S3 secret using the default provider (for s3, this will be the "config" provider, requiring the user to set all) -statement ok -CREATE SECRET default_provider_secret ( - TYPE S3, - KEY_ID 'my_key', - SECRET 'my_secret', - REGION 'my_region', - ENDPOINT 'invalid-on-purpose' -) - -# The secret will be created for the default scope for this type -query III -SELECT name, type, scope FROM duckdb_secrets() WHERE name='default_provider_secret'; ----- -default_provider_secret s3 ['s3://', 's3n://', 's3a://'] - -# Note the endpoint is now using the one in the default_provider_secret -statement error -FROM 's3://test-bucket/test.csv' ----- -HTTP HEAD to 'https://test-bucket.invalid-on-purpose/test.csv' - -# Now create an S3 secret using the default (config) provider by explicitly passing it -statement ok -CREATE SECRET secret_scope_1 ( - TYPE S3, - PROVIDER config, - SCOPE 's3://b1', - ENDPOINT 'invalid-on-purpose-2' -) - -query III -SELECT name, type, scope FROM duckdb_secrets() WHERE name='secret_scope_1'; ----- -secret_scope_1 s3 ['s3://b1'] - -# Longest match of credential scope takes the win so, this is will grab the secret_scope_1 secret -statement error -FROM 's3://b1/test.csv' ----- -Could not establish connection error for HTTP HEAD to 'https://b1.invalid-on-purpose-2/test.csv' - -# Now confirm we can also set multiple scopes -statement ok -CREATE SECRET secret_scope_2 ( - TYPE S3, - PROVIDER config, - SCOPE ['s3://b2', 's3://b3'], - ENDPOINT 'invalid-on-purpose-3' -) - -query III -SELECT name, type, scope FROM duckdb_secrets() WHERE name='secret_scope_2'; ----- -secret_scope_2 s3 ['s3://b2', 's3://b3'] - -statement error -FROM 's3://b2/test.csv' ----- -Could not establish connection error for HTTP HEAD to 'https://b2.invalid-on-purpose-3/test.csv' diff --git a/test/sql/secrets/create_secret_binding.test b/test/sql/secrets/create_secret_binding.test deleted file mode 100644 index bf1aa428f2f2..000000000000 --- a/test/sql/secrets/create_secret_binding.test +++ /dev/null @@ -1,92 +0,0 @@ -# name: test/sql/secrets/create_secret_binding.test -# description: Test secret binding & types -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -require httpfs - -# Ensure any currently stored secrets don't interfere with the test -statement ok -set allow_persistent_secrets=false; - -# Binder autocasts options, also both with single quotes and without is allowed -statement ok -CREATE SECRET s1 ( - TYPE R2, - PROVIDER config, - SCOPE ['s3://my_r2_scope', 's3://my_r2_scope2'], - ACCOUNT_ID 'some_bogus_account', - KEY_ID '123', - USE_SSL 1, - URL_COMPATIBILITY_MODE false -) - -query I nosort s1 -FROM duckdb_secrets(); ----- - -statement ok -DROP SECRET s1 - -# Create the secret again but in a different way to demonstrate casting and case insensitivity of param names -statement ok -CREATE SECRET s1 ( - TYPE R2, - PROVIDER config, - SCOPE ['s3://my_r2_scope', 's3://my_r2_scope2'], - account_id 'some_bogus_account', - key_id 123, - USE_SSL 'true', - URL_COMPATIBILITY_MODE '0' -) - -query I nosort s1 -FROM duckdb_secrets(); ----- - -### Now let's try some incorrect inputs - -# Incorrect type -statement error -CREATE SECRET incorrect_type ( - TYPE R2, - PROVIDER config, - USE_SSL 'fliepflap' -) ----- -Binder Error: Failed to cast option 'use_ssl' to type 'BOOLEAN': 'Could not convert string 'fliepflap' to BOOL' - -# Incorrect param altogether -statement error -CREATE SECRET incorrect_type ( - TYPE R2, - PROVIDER config, - FLIEPFLAP true -) ----- -Binder Error: Unknown parameter 'fliepflap' for secret type 'r2' with provider 'config' - -# Incorrect param for this type, but correct for other -statement error -CREATE SECRET incorrect_type ( - TYPE S3, - PROVIDER config, - ACCOUNT_ID 'my_acount' -) ----- -Binder Error: Unknown parameter 'account_id' for secret type 's3' with provider 'config' - -# Params can only occur once -statement error -CREATE SECRET duplicate_param ( - TYPE R2, - PROVIDER config, - account_id 'some_bogus_account', - key_id 123, - KEY_ID 12098, - account_id blablabla -) ----- -Binder Error: Duplicate query param found while parsing create secret: 'key_id' diff --git a/test/sql/secrets/create_secret_cascading.test_slow b/test/sql/secrets/create_secret_cascading.test_slow deleted file mode 100644 index 8dd8cb8ddb79..000000000000 --- a/test/sql/secrets/create_secret_cascading.test_slow +++ /dev/null @@ -1,58 +0,0 @@ -# name: test/sql/secrets/create_secret_cascading.test_slow -# description: Test the cascading mechanism of secret settings -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -require httpfs - -# Ensure any currently stored secrets don't interfere with the test -statement ok -set allow_persistent_secrets=false; - -statement ok -set s3_endpoint = 'invalid-on-purpose-setting' - -statement ok -set s3_url_style = 'path' - -statement ok -set s3_use_ssl = false - -# This secret overrides only the url style, not the endpoint -statement ok -CREATE SECRET s1 ( - TYPE S3, - REGION 'my_region', - URL_STYLE 'vhost', - SCOPE 's3://url-style-only' -) - -# This secret overrides both the url style and the endpoint -statement ok -CREATE SECRET s2 ( - TYPE S3, - REGION 'my_region', - URL_STYLE 'vhost', - ENDPOINT 'invalid-on-purpose-secret', - SCOPE 's3://url-style-and-endpoint' -) - -# Only the url style from the secret is used -statement error -FROM 's3://url-style-only/test.csv' ----- -Could not establish connection error for HTTP HEAD to 'http://url-style-only.invalid-on-purpose-setting/test.csv' - -# Both Url style and endpoint are used now -statement error -FROM 's3://url-style-and-endpoint/test.csv' ----- -Could not establish connection error for HTTP HEAD to 'http://url-style-and-endpoint.invalid-on-purpose-secret/test.csv' - -# This request matches none of the secrets, we use the settings -statement error -FROM 's3://test-bucket/test.csv' ----- -Could not establish connection error for HTTP HEAD to 'http://invalid-on-purpose-setting/test-bucket/test.csv' diff --git a/test/sql/secrets/create_secret_defaults.test b/test/sql/secrets/create_secret_defaults.test deleted file mode 100644 index fd15aed24a83..000000000000 --- a/test/sql/secrets/create_secret_defaults.test +++ /dev/null @@ -1,60 +0,0 @@ -# name: test/sql/secrets/create_secret_defaults.test -# description: Test default values during secret creation -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -require httpfs - -# Ensure any currently stored secrets don't interfere with the test -statement ok -set allow_persistent_secrets=false; - -statement ok -DROP SECRET IF EXISTS s1; - -# Without name we use the __default_ name. The default config for for the S3 type is config -statement ok -CREATE SECRET ( - TYPE S3, - KEY_ID 'my_key', - SECRET 'my_secret' -) - -query IIII -SELECT name, provider, type, scope FROM duckdb_secrets(); ----- -__default_s3 config s3 ['s3://', 's3n://', 's3a://'] - -# Without name we use the __default_ name. The default config for for the R2 type is config -statement ok -CREATE SECRET ( - TYPE R2, - KEY_ID 'my_key', - SECRET 'my_secret', - ACCOUNT_ID 'my_account_id' -) - -query IIII -SELECT name, provider, type, scope FROM duckdb_secrets() ORDER BY name; ----- -__default_r2 config r2 ['r2://'] -__default_s3 config s3 ['s3://', 's3n://', 's3a://'] - - -# Without name we use the __default_ name. The default config for for the R2 type is config -statement ok -CREATE SECRET ( - TYPE GCS, - KEY_ID 'my_key', - SECRET 'my_secret' -) - -# duckdb_secrets with all defaults looks like this now -query IIIIII -SELECT name, persistent, storage, provider, type, scope FROM duckdb_secrets() ORDER BY name; ----- -__default_gcs 0 memory config gcs ['gcs://', 'gs://'] -__default_r2 0 memory config r2 ['r2://'] -__default_s3 0 memory config s3 ['s3://', 's3n://', 's3a://'] \ No newline at end of file diff --git a/test/sql/secrets/create_secret_gcs.test_slow b/test/sql/secrets/create_secret_gcs.test_slow deleted file mode 100644 index 3d2163980a18..000000000000 --- a/test/sql/secrets/create_secret_gcs.test_slow +++ /dev/null @@ -1,34 +0,0 @@ -# name: test/sql/secrets/create_secret_gcs.test_slow -# description: Test secret creation using the default gcs secret provider -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -require httpfs - -# Ensure any currently stored secrets don't interfere with the test -statement ok -set allow_persistent_secrets=false; - -statement ok -reset s3_use_ssl; - -# GCS Secrets automatically default to the correct endpoint for Google Cloud Storage -statement ok -CREATE SECRET ( - TYPE GCS, - KEY_ID 'my_key', - SECRET 'my_secret' -) - -# The secret will be created for the default scope -query IIII -SELECT name, type, provider, scope FROM duckdb_secrets(); ----- -__default_gcs gcs config ['gcs://', 'gs://'] - -statement error -FROM 'gcs://test-bucket/test.csv' ----- -https://storage.googleapis.com/test-bucket/test.csv diff --git a/test/sql/secrets/create_secret_hffs.test b/test/sql/secrets/create_secret_hffs.test deleted file mode 100644 index 0224ead6d00c..000000000000 --- a/test/sql/secrets/create_secret_hffs.test +++ /dev/null @@ -1,31 +0,0 @@ -# name: test/sql/secrets/create_secret_hffs.test -# description: Test huggingface secrets -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -require httpfs - -statement ok -set allow_persistent_secrets=false; - -# Manually setting token is simplest -statement ok -CREATE SECRET hf1 ( - TYPE HUGGINGFACE, - TOKEN 'bla' -) - -# Cache provider will automatically try to fetch the token from the cache -statement ok -CREATE SECRET hf2 ( - TYPE HUGGINGFACE, - PROVIDER 'credential_chain' -) - -query IIII -SELECT name, type, provider, scope FROM duckdb_secrets() order by name; ----- -hf1 huggingface config ['hf://'] -hf2 huggingface credential_chain ['hf://'] diff --git a/test/sql/secrets/create_secret_invalid_map.test b/test/sql/secrets/create_secret_invalid_map.test deleted file mode 100644 index fb512700a028..000000000000 --- a/test/sql/secrets/create_secret_invalid_map.test +++ /dev/null @@ -1,24 +0,0 @@ -# name: test/sql/secrets/create_secret_invalid_map.test -# description: Test throwing input errors on multi map input. -# group: [secrets] - -require httpfs - -statement ok -PRAGMA enable_verification; - -statement error -CREATE PERSISTENT SECRET http_multimap ( - TYPE HTTP, - EXTRA_HTTP_HEADERS MAP{123: 'quack1', 123 : 'quack2'} -); ----- -:Invalid Input Error.*Map keys must be unique.* - -statement error -CREATE PERSISTENT SECRET http_multimap ( - TYPE HTTP, - EXTRA_HTTP_HEADERS MAP{NULL: 'quack1', 123 : 'quack2'} -); ----- -:Invalid Input Error.*Map keys can not be NULL.* \ No newline at end of file diff --git a/test/sql/secrets/create_secret_minio.test b/test/sql/secrets/create_secret_minio.test deleted file mode 100644 index 11dcb0ebe1ee..000000000000 --- a/test/sql/secrets/create_secret_minio.test +++ /dev/null @@ -1,78 +0,0 @@ -# name: test/sql/secrets/create_secret_minio.test -# description: Test s3 secrets actually work using minio -# group: [secrets] - -require parquet - -require httpfs - -require-env S3_TEST_SERVER_AVAILABLE 1 - -# Require that these environment variables are also set - -require-env AWS_DEFAULT_REGION - -require-env AWS_ACCESS_KEY_ID - -require-env AWS_SECRET_ACCESS_KEY - -require-env DUCKDB_S3_ENDPOINT - -require-env DUCKDB_S3_USE_SSL - -set ignore_error_messages - -load __TEST_DIR__/persistent_secrets.db - -statement ok -PRAGMA enable_verification; - -statement ok -set secret_directory='__TEST_DIR__/create_secret_minio' - -# first need to unset the duckdb settings: currently the env variables are loaded automatically making all queries auth -statement ok -set s3_access_key_id=''; - -statement ok -set s3_secret_access_key=''; - -statement error -copy (select 1 as a) to 's3://test- /test-file.parquet' ----- - -# Now we create a scoped secret with correct credentials -statement ok -CREATE PERSISTENT SECRET ( - TYPE S3, - PROVIDER config, - SCOPE 's3://test-bucket/only-this-file-gets-auth.parquet', - KEY_ID '${AWS_ACCESS_KEY_ID}', - SECRET '${AWS_SECRET_ACCESS_KEY}', - REGION '${AWS_DEFAULT_REGION}', - ENDPOINT '${DUCKDB_S3_ENDPOINT}', - USE_SSL '${DUCKDB_S3_USE_SSL}' -) - -# scope doesn't match! query still fails -statement error -copy (select 1 as a) to 's3://test-bucket/test-file.parquet' ----- - -# scope matches, the secret is chosen and the query will succeed -statement ok -copy (select 1 as a) to 's3://test-bucket/only-this-file-gets-auth.parquet' - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_minio' - -# persistent secrets survive restart -statement ok -copy (select 1 as a) to 's3://test-bucket/only-this-file-gets-auth.parquet' - -# Its still scoped -statement error -copy (select 1 as a) to 's3://test-bucket/no-auth-here.parquet' ----- \ No newline at end of file diff --git a/test/sql/secrets/create_secret_name_conflicts.test b/test/sql/secrets/create_secret_name_conflicts.test deleted file mode 100644 index 8c480807b96a..000000000000 --- a/test/sql/secrets/create_secret_name_conflicts.test +++ /dev/null @@ -1,89 +0,0 @@ -# name: test/sql/secrets/create_secret_name_conflicts.test -# description: Test name conflict behaviour for secrets -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -load __TEST_DIR__/persistent_secrets.db - -require httpfs - -statement ok -set secret_directory='__TEST_DIR__/create_secret_name_conflicts' - -statement ok -CREATE TEMPORARY SECRET s1 ( TYPE S3 ) - -statement error -CREATE TEMPORARY SECRET s1 ( TYPE S3 ) ----- -Invalid Input Error: Temporary secret with name 's1' already exists! - -statement ok -CREATE PERSISTENT SECRET s1 ( TYPE S3 ) - -statement error -CREATE PERSISTENT SECRET s1 ( TYPE S3 ) ----- -Persistent secret with name 's1' already exists in secret storage 'local_file'! - -statement error -DROP SECRET s1; ----- -Invalid Input Error: Ambiguity found for secret name 's1', secret occurs in multiple storages - -statement error -DROP SECRET s1 FROM bogus; ----- -Invalid Input Error: Unknown storage type found for drop secret: 'bogus' - -statement ok -DROP TEMPORARY SECRET s1; - -# Re-dropping the temp s1 is now erroneous -statement error -DROP TEMPORARY SECRET s1; ----- -Invalid Input Error: Failed to remove non-existent secret with name 's1' - -query II -SELECT name, storage FROM duckdb_secrets() ----- -s1 local_file - -# Now we will do it again but while the permanent secret is still lazily loaded -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_name_conflicts' - -statement ok -CREATE TEMPORARY SECRET s1 ( TYPE S3 ) - -# Now the drop should be ambiguous again: but the persistent secret will be lazily loaded now -statement error -DROP SECRET s1; ----- -Invalid Input Error: Ambiguity found for secret name 's1', secret occurs in multiple storages - -# Fully specified drop statement this time -statement ok -DROP PERSISTENT SECRET s1 FROM LOCAL_FILE; - -# Now a semi-weird case: this will create if not exists only within its own storage: therefore this does actually create -# the secret -statement ok -CREATE PERSISTENT SECRET IF NOT EXISTS s1 ( TYPE S3 ) - -query II -SELECT name, storage FROM duckdb_secrets() ORDER BY storage ----- -s1 local_file -s1 memory - -statement ok -DROP PERSISTENT SECRET s1; - -statement ok -DROP SECRET s1; \ No newline at end of file diff --git a/test/sql/secrets/create_secret_non_writable_persistent_dir.test b/test/sql/secrets/create_secret_non_writable_persistent_dir.test deleted file mode 100644 index 1091eb6337bc..000000000000 --- a/test/sql/secrets/create_secret_non_writable_persistent_dir.test +++ /dev/null @@ -1,46 +0,0 @@ -# name: test/sql/secrets/create_secret_non_writable_persistent_dir.test -# description: Test persistent secrets when the secret dir is non-writable -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -load __TEST_DIR__/create_secret_non_writable_persistent_dir.db - -require httpfs - -# First we create any file -statement ok -COPY (SELECT 1 as a) to '__TEST_DIR__/file_to_prevent_the_secret_dir_from_being_created.csv' - -# Then we set the secret dir to this. -statement ok -set secret_directory='__TEST_DIR__/file_to_prevent_the_secret_dir_from_being_created.csv' - -# Now on creation of a tmp secret, the secret manager is initialized, but the persistent secret directory creation is impossible -statement ok -CREATE SECRET my_tmp_secret ( - TYPE S3, - SCOPE 's3://bucket1' -) - -# This now fails with the message that we could not create the persistent secret directory -statement error -CREATE PERSISTENT SECRET my_tmp_secret ( - TYPE S3, - SCOPE 's3://bucket2' -) ----- - -restart - -# Try with a correct, deeply nested path: AOK? -statement ok -set secret_directory='__TEST_DIR__/create_secret_non_writable_persistent_dir/a/deeply/nested/folder/will/be/created' - -statement maybe -CREATE PERSISTENT SECRET my_tmp_secret ( - TYPE S3, - SCOPE 's3://bucket2' -) ----- diff --git a/test/sql/secrets/create_secret_overwriting.test b/test/sql/secrets/create_secret_overwriting.test deleted file mode 100644 index 39e73147d291..000000000000 --- a/test/sql/secrets/create_secret_overwriting.test +++ /dev/null @@ -1,73 +0,0 @@ -# name: test/sql/secrets/create_secret_overwriting.test -# description: Test secret overwriting and deleting -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -require httpfs - -# Ensure any currently stored secrets don't interfere with the test -statement ok -set allow_persistent_secrets=false; - -# Create some s3 secret -statement ok -CREATE SECRET my_secret ( - TYPE S3, - SCOPE 's3://bucket1' -) - -query II -SELECT name, scope FROM duckdb_secrets(); ----- -my_secret ['s3://bucket1'] - -statement error -CREATE SECRET my_secret ( - TYPE S3, - KEY_ID 'my_key', - SECRET 'my_secret', - SCOPE 's3://bucket1' -) ----- -Invalid Input Error: Temporary secret with name 'my_secret' already exists! - -# We should be able to replace the secret though -statement ok -CREATE OR REPLACE SECRET my_secret ( - TYPE S3, - SCOPE 's3://bucket2' -) - -query II -SELECT name, scope FROM duckdb_secrets(); ----- -my_secret ['s3://bucket2'] - -# We can also ignore if we want to -statement ok -CREATE SECRET IF NOT EXISTS my_secret ( - TYPE S3, - SCOPE 's3://bucket5' -) - -query II -SELECT name, scope FROM duckdb_secrets(); ----- -my_secret ['s3://bucket2'] - -# Now try dropping a secret that does not exist -statement error -DROP SECRET my_secret_does_not_exist; ----- -Failed to remove non-existent secret with name 'my_secret_does_not_exist' - -# Drop one that does exist -statement ok -DROP SECRET my_secret; - -# Secret be gone! -query II -SELECT name, scope FROM duckdb_secrets(); ----- diff --git a/test/sql/secrets/create_secret_persistence.test b/test/sql/secrets/create_secret_persistence.test deleted file mode 100644 index bc44ab7d8910..000000000000 --- a/test/sql/secrets/create_secret_persistence.test +++ /dev/null @@ -1,195 +0,0 @@ -# name: test/sql/secrets/create_secret_persistence.test -# description: Test secret persistence -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -load __TEST_DIR__/persistent_secrets.db - -require httpfs - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence' - -# Create some s3 secret, the normally the default is TEMPORARY -statement ok -CREATE SECRET my_tmp_secret ( - TYPE S3, - SCOPE 's3://bucket1' -) - -# Explicitly stating -statement ok -CREATE TEMPORARY SECRET my_tmp_secret_2 ( - TYPE S3, - SCOPE 's3://bucket2' -) - -statement ok -CREATE OR REPLACE PERSISTENT SECRET my_tmp_secret_3 ( - TYPE S3, - SCOPE 's3://bucket3' -) - -query III -SELECT name, storage, scope FROM duckdb_secrets() where storage='memory' order by name; ----- -my_tmp_secret memory ['s3://bucket1'] -my_tmp_secret_2 memory ['s3://bucket2'] - -query II -SELECT name, scope FROM duckdb_secrets() where storage != 'memory'; ----- -my_tmp_secret_3 ['s3://bucket3'] - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence' - -# Persistent secrets are restored automatically -query II -SELECT name, scope FROM duckdb_secrets(); ----- -my_tmp_secret_3 ['s3://bucket3'] - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence' - -# Trying to create same name TMP secret fails -statement error -CREATE PERSISTENT SECRET my_tmp_secret_3 ( - TYPE S3, - SCOPE 's3://bucket3_not_used' -) ----- -Invalid Input Error: Persistent secret with name 'my_tmp_secret_3' already exists in secret storage 'local_file'! - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence' - -# Trying to create same name PERSISTENT secret fails -statement error -CREATE PERSISTENT SECRET my_tmp_secret_3 ( - TYPE S3, - SCOPE 's3://bucket3_not_used' -) ----- -Invalid Input Error: Persistent secret with name 'my_tmp_secret_3' already exists in secret storage 'local_file'! - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence' - -# Note: this will be a temporary secret: there are now 2 secrets with the same name -statement ok -CREATE SECRET IF NOT EXISTS my_tmp_secret_3 ( - TYPE S3, - SCOPE 's3://bucket3_not_used' -) - -# Secret is unmodified -query III -SELECT name, storage, scope FROM duckdb_secrets() where name='my_tmp_secret_3' order by storage; ----- -my_tmp_secret_3 local_file ['s3://bucket3'] -my_tmp_secret_3 memory ['s3://bucket3_not_used'] - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence' - -# Ignoring already existing persistent secret is fine -statement ok -CREATE PERSISTENT SECRET IF NOT EXISTS my_tmp_secret_3 ( - TYPE S3, - SCOPE 's3://bucket3_not_used' -) - -# Running second time, code path slightly different as the secret is lazy loaded in previous step -statement ok -CREATE PERSISTENT SECRET IF NOT EXISTS my_tmp_secret_3 ( - TYPE S3, - SCOPE 's3://bucket3_not_used' -) - -# Secret is unmodified -query II -SELECT name, scope FROM duckdb_secrets(); ----- -my_tmp_secret_3 ['s3://bucket3'] - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence' - -# Secret is still unmodified after restart -query II -SELECT name, scope FROM duckdb_secrets() ; ----- -my_tmp_secret_3 ['s3://bucket3'] - -# Now we do actually update the persistent secret -statement ok -CREATE OR REPLACE PERSISTENT SECRET my_tmp_secret_3 ( - TYPE S3, - SCOPE 's3://bucket3_updated' -) - -# Its updated! -query II -SELECT name, scope FROM duckdb_secrets(); ----- -my_tmp_secret_3 ['s3://bucket3_updated'] - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence' - -# Survives restart! -query II -SELECT name, scope FROM duckdb_secrets(); ----- -my_tmp_secret_3 ['s3://bucket3_updated'] - -# Now we add another secret, to reliably test deletion of the other one -statement ok -CREATE PERSISTENT SECRET IF NOT EXISTS my_tmp_secret_4 ( - TYPE S3, - SCOPE 's3://another_secret' -) - -query II -SELECT name, scope FROM duckdb_secrets() order by name; ----- -my_tmp_secret_3 ['s3://bucket3_updated'] -my_tmp_secret_4 ['s3://another_secret'] - -statement ok -DROP SECRET my_tmp_secret_3; - -# my_tmp_secret_3 is deleted -query II -SELECT name, scope FROM duckdb_secrets(); ----- -my_tmp_secret_4 ['s3://another_secret'] - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence' - -# Secret is actually deleted -query III -SELECT name, storage, scope FROM duckdb_secrets() order by name; ----- -my_tmp_secret_4 local_file ['s3://another_secret'] diff --git a/test/sql/secrets/create_secret_persistence_error_handling.test b/test/sql/secrets/create_secret_persistence_error_handling.test deleted file mode 100644 index 93c102e2728d..000000000000 --- a/test/sql/secrets/create_secret_persistence_error_handling.test +++ /dev/null @@ -1,46 +0,0 @@ -# name: test/sql/secrets/create_secret_persistence_error_handling.test -# description: Test secret persistence with buggy secrets -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -load __TEST_DIR__/create_secret_persistence_error_handling.db - -require httpfs - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence_error_handling' - -# Hacky way to make duckdb create the create_secret_persistence_error_handling dir -statement ok -COPY (select 1 as a, 2 as b ) to '__TEST_DIR__/create_secret_persistence_error_handling/' (FORMAT csv, PARTITION_BY a) - -# Now write a corrupt secret file -statement ok -COPY (select 1 as a ) to '__TEST_DIR__/create_secret_persistence_error_handling/s1.duckdb_secret' (FORMAT csv) - -statement error -FROM duckdb_secrets(); ----- - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence_error_handling2' - -statement ok -CREATE PERSISTENT SECRET s1 (TYPE S3); - -restart no_extension_load - -statement ok -set secret_directory='__TEST_DIR__/create_secret_persistence_error_handling2' - -# Disable autoloading -statement ok -SET autoload_known_extensions=false; - -# Force persistent deserialization; we can deserialize generic key/value secrets -statement ok -from duckdb_secrets(); diff --git a/test/sql/secrets/create_secret_r2.test b/test/sql/secrets/create_secret_r2.test deleted file mode 100644 index ff9e7099a060..000000000000 --- a/test/sql/secrets/create_secret_r2.test +++ /dev/null @@ -1,65 +0,0 @@ -# name: test/sql/secrets/create_secret_r2.test -# description: Test secret creation using the default r2 secret provider -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -require httpfs - -statement ok -set secret_directory='__TEST_DIR__/create_secret_r2' - -# R2 is secrets will instead of requiring manually constructing the endpoint of .r2.cloudflarestorage.com, -# use the account_id to configure it. Also the region is not required at all. Also the scope defaults to r2:// -statement ok -CREATE SECRET ( - TYPE R2, - ACCOUNT_ID 'some_bogus_account', - KEY_ID 'my_key', - SECRET 'my_secret' -) - -# The secret will be created for the default scope -query IIII -SELECT name, type, provider, scope FROM duckdb_secrets(); ----- -__default_r2 r2 config ['r2://'] - -# -statement error -FROM 's3://test-bucket/test.csv' -error for HTTP HEAD to 'https://some_bogus_account.r2.cloudflarestorage.com/test-bucket/test.csv' ----- - -# Account ID is only for R2, trying to set this for S3 will fail -statement error -CREATE SECRET ( - TYPE S3, - ACCOUNT_ID 'some_bogus_account', - KEY_ID 'my_key', - SECRET 'my_secret' -) ----- -Binder Error: Unknown parameter 'account_id' for secret type 's3' with default provider 'config' - -# Account ID is only for R2, trying to set this for GCS will fail -statement error -CREATE SECRET ( - TYPE GCS, - PROVIDER config, - ACCOUNT_ID 'some_bogus_account', - KEY_ID 'my_key', - SECRET 'my_secret' -) ----- -Binder Error: Unknown parameter 'account_id' for secret type 'gcs' with provider 'config' - -# Ensure secret lookup works correctly; -statement ok -CREATE SECRET test( - TYPE R2, - ACCOUNT_ID 'some_bogus_account', - KEY_ID 'my_key', - SECRET 'my_secret' -) diff --git a/test/sql/secrets/create_secret_r2_serialization.test b/test/sql/secrets/create_secret_r2_serialization.test deleted file mode 100644 index 6ba8d54ad8e9..000000000000 --- a/test/sql/secrets/create_secret_r2_serialization.test +++ /dev/null @@ -1,70 +0,0 @@ -# name: test/sql/secrets/create_secret_r2_serialization.test -# description: Demo of secret serialization -# group: [secrets] - -# NOTE: this is a testing feature that will be removed / replaced with actual persistent secrets. - -require httpfs - -require parquet - -load __TEST_DIR__/test_serialize_secrets.db - -statement ok -PRAGMA enable_verification; - -statement ok -set secret_directory='__TEST_DIR__/create_secret_r2_serialization' - -statement ok -CREATE OR REPLACE PERSISTENT SECRET s1 ( - TYPE S3, - PROVIDER config, - SCOPE 's3://my_scope', - KEY_ID 'mekey', - SECRET 'mesecret', - REGION 'meregion', - SESSION_TOKEN 'mesesh', - ENDPOINT 'meendpoint', - URL_STYLE 'mahstyle', - USE_SSL true, - URL_COMPATIBILITY_MODE true -) - -query IIII -select name, type, provider, scope FROM duckdb_secrets(); ----- -s1 s3 config ['s3://my_scope'] - -query I nosort secret_to_string -select * from duckdb_secrets(); ----- - -restart - -# Now setting the secret dir somehwere nonexistent will yield no persistent secrets -statement ok -set secret_directory='__TEST_DIR__/does_not_exist2' - -query I -select count(*) FROM duckdb_secrets(); ----- -0 - -restart - -# However setting it to the dir that does, we can suddenly see our persisted secrets -statement ok -set secret_directory='__TEST_DIR__/create_secret_r2_serialization' - -# After restart secret is still there -query IIII -select name, type, provider, scope FROM duckdb_secrets(); ----- -s1 s3 config ['s3://my_scope'] - -# Even more: it matches the exact string note that we don't disable redaction here to ensure we cover -# redaction set serialization with this test -query I nosort secret_to_string -select * from duckdb_secrets(); ----- \ No newline at end of file diff --git a/test/sql/secrets/create_secret_s3_serialization.test b/test/sql/secrets/create_secret_s3_serialization.test deleted file mode 100644 index 4d127e672f30..000000000000 --- a/test/sql/secrets/create_secret_s3_serialization.test +++ /dev/null @@ -1,99 +0,0 @@ -# name: test/sql/secrets/create_secret_s3_serialization.test -# description: Test serialization of the S3/GCS/r2 secrets -# group: [secrets] - -require httpfs - -require parquet - -load __TEST_DIR__/test_serialize_secrets.db - -statement ok -PRAGMA enable_verification; - -statement ok -set secret_directory='__TEST_DIR__/create_secret_s3_serialization' - -statement ok -CREATE OR REPLACE PERSISTENT SECRET s1 ( - TYPE S3, - PROVIDER config, - SCOPE 's3://my_s3_scope', - KEY_ID 'mekey', - SECRET 'mesecret', - REGION 'meregion', - SESSION_TOKEN 'mesesh', - ENDPOINT 'meendpoint', - URL_STYLE 'mahstyle', - USE_SSL true, - URL_COMPATIBILITY_MODE true -) - -statement ok -CREATE OR REPLACE PERSISTENT SECRET s2 ( - TYPE R2, - PROVIDER config, - SCOPE 's3://my_r2_scope', - ACCOUNT_ID 'some_bogus_account', - KEY_ID 'mekey', - SECRET 'mesecret', - SESSION_TOKEN 'mesesh', - URL_STYLE 'mahstyle', - USE_SSL 1, - URL_COMPATIBILITY_MODE 1 -) - -statement ok -CREATE OR REPLACE PERSISTENT SECRET s3 ( - TYPE GCS, - PROVIDER config, - SCOPE 's3://my_gcs_scope', - KEY_ID 'mekey', - SECRET 'mesecret', - SESSION_TOKEN 'mesesh', - URL_STYLE 'mahstyle', - USE_SSL true, - URL_COMPATIBILITY_MODE true -) - -query IIII -select name, type, provider, scope FROM duckdb_secrets() order by name; ----- -s1 s3 config ['s3://my_s3_scope'] -s2 r2 config ['s3://my_r2_scope'] -s3 gcs config ['s3://my_gcs_scope'] - -# Note: this query prints the tokens as an unredacted string -query I nosort secret_to_string -select secret_string from duckdb_secrets(redact=false) order by type; ----- - -restart - -# Now setting the secret dir somehwere nonexistent will yield no persistent secrets -statement ok -set secret_directory='__TEST_DIR__/does_not_exist1' - -query I -select count(*) FROM duckdb_secrets(redact=false); ----- -0 - -restart - -# However setting it to the dir that does, we can suddenly see our persisted secrets -statement ok -set secret_directory='__TEST_DIR__/create_secret_s3_serialization' - -# After restart secrets are still there -query IIII -select name, type, provider, scope FROM duckdb_secrets() order by name; ----- -s1 s3 config ['s3://my_s3_scope'] -s2 r2 config ['s3://my_r2_scope'] -s3 gcs config ['s3://my_gcs_scope'] - -# Note: this query prints the tokens as an unredacted string -query I nosort secret_to_string -select secret_string from duckdb_secrets(redact=false) order by type; ----- \ No newline at end of file diff --git a/test/sql/secrets/create_secret_scope_matching.test b/test/sql/secrets/create_secret_scope_matching.test deleted file mode 100644 index 3d5dd2aac35a..000000000000 --- a/test/sql/secrets/create_secret_scope_matching.test +++ /dev/null @@ -1,61 +0,0 @@ -# name: test/sql/secrets/create_secret_scope_matching.test -# description: Test scope matching behaviour is correct -# group: [secrets] - -load __TEST_DIR__/create_secret_scope_matching.db - -statement ok -PRAGMA enable_verification; - -require httpfs - -statement ok -set secret_directory='__TEST_DIR__/create_secret_scope_matching' - -# No match -query I -SELECT name FROM which_secret('s3://', 's3') ----- - -statement ok -CREATE TEMPORARY SECRET t1 ( TYPE S3 ) - -statement ok -CREATE TEMPORARY SECRET t2 ( TYPE S3 ) - -statement ok -CREATE SECRET p1 IN LOCAL_FILE ( TYPE S3 ) - -# This ties within the same storage: the two temporary secrets s1 and s2 both score identically. We solve this by -# tie-breaking on secret name alphabetical ordering -query I -SELECT name FROM which_secret('s3://', 's3') ----- -t1 - -query III -FROM which_secret('s3://', 's3') ----- -t1 TEMPORARY memory - -statement ok -DROP SECRET t1 - -# Temporary secrets take preference over temporary ones -query I -SELECT name FROM which_secret('s3://', 's3') ----- -t2 - -statement ok -DROP SECRET t2 - -query I -SELECT name FROM which_secret('s3://', 's3') ----- -p1 - -statement maybe -DROP SECRET p1 ----- -Invalid Input Error: Failed to remove non-existent secret diff --git a/test/sql/secrets/create_secret_settings.test b/test/sql/secrets/create_secret_settings.test deleted file mode 100644 index 6200424aa8b2..000000000000 --- a/test/sql/secrets/create_secret_settings.test +++ /dev/null @@ -1,71 +0,0 @@ -# name: test/sql/secrets/create_secret_settings.test -# description: Test setting secret settings -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -load __TEST_DIR__/secrets_settings.db - -require httpfs - -statement ok -set secret_directory='__TEST_DIR__/create_secret_settings1' - -statement ok -set allow_persistent_secrets=true; - -# Create some s3 secret, the normally the default is TEMPORARY -statement ok -CREATE PERSISTENT SECRET my_perm_secret ( - TYPE S3, - SCOPE 's3://bucket1' -) - -query II -SELECT name, scope from duckdb_secrets(); ----- -my_perm_secret ['s3://bucket1'] - -statement error -set secret_directory='__TEST_DIR__/create_secret_settings2' ----- -Invalid Input Error: Changing Secret Manager settings after the secret manager is used is not allowed! - -statement error -set allow_persistent_secrets=false; ----- -Invalid Input Error: Changing Secret Manager settings after the secret manager is used is not allowed! - -# This setting CAN be modified after init -statement ok -set default_secret_storage = 'local_file' - -statement ok -reset default_secret_storage; - -restart - -# When disabling secrets, we won't read the one that we wrote earlier -statement ok -set allow_persistent_secrets=false - -query I -select count(*) from duckdb_secrets(); ----- -0 - -restart - -# Switch settings back and it works again -statement ok -set allow_persistent_secrets=true - -# setting the path right it will work -statement ok -set secret_directory='__TEST_DIR__/create_secret_settings1' - -query II -SELECT name, scope from duckdb_secrets(); ----- -my_perm_secret ['s3://bucket1'] diff --git a/test/sql/secrets/create_secret_storage_backends.test b/test/sql/secrets/create_secret_storage_backends.test deleted file mode 100644 index 5daa06d68813..000000000000 --- a/test/sql/secrets/create_secret_storage_backends.test +++ /dev/null @@ -1,111 +0,0 @@ -# name: test/sql/secrets/create_secret_storage_backends.test -# description: Test different storage backends -# group: [secrets] - -load __TEST_DIR__/create_secret_storage_backends.db - -statement ok -PRAGMA enable_verification; - -require httpfs - -# Ensure any currently stored secrets don't interfere with the test -statement ok -set allow_persistent_secrets=false; - -statement error -CREATE TEMPORARY SECRET s1 IN LOCAL_FILE ( TYPE S3 ) ----- -Invalid Input Error: Persistent secrets are disabled. Restart DuckDB and enable persistent secrets through 'SET allow_persistent_secrets=true' - -statement error -CREATE PERSISTENT SECRET s1 IN NON_EXISTENT_SECRET_STORAGE ( TYPE S3 ) ----- -Invalid Input Error: Persistent secrets are disabled. Restart DuckDB and enable persistent secrets through 'SET allow_persistent_secrets=true' - -# We have disabled the permanent secrets, so this should fail -statement error -CREATE PERSISTENT SECRET perm_s1 ( TYPE S3 ) ----- -Invalid Input Error: Persistent secrets are disabled. Restart DuckDB and enable persistent secrets through 'SET allow_persistent_secrets=true' - -restart - -# Enable persistent secrets so we can set a 'secret_directory' -statement ok -set allow_persistent_secrets=true; - -statement ok -set secret_directory='__TEST_DIR__/create_secret_storages' - -# Default for persistent secret is currently LOCAL_FILE (only native persistent storage method currently) -statement ok -CREATE PERSISTENT SECRET perm_s1 ( TYPE S3 ) - -# Specifying IN ... implies persistent, hence this is okay -statement ok -CREATE SECRET perm_s2 IN LOCAL_FILE ( TYPE S3 ) - -# Explicitly stating temporary is cool -statement ok -CREATE TEMPORARY SECRET temp_s1 ( TYPE s3 ); - -# Not specifying it will use the system default (which is temp) -statement ok -CREATE SECRET temp_s2 ( TYPE s3 ); - -query IIIIII -SELECT * EXCLUDE (secret_string) FROM duckdb_secrets() ORDER BY name ----- -perm_s1 s3 config true local_file ['s3://', 's3n://', 's3a://'] -perm_s2 s3 config true local_file ['s3://', 's3n://', 's3a://'] -temp_s1 s3 config false memory ['s3://', 's3n://', 's3a://'] -temp_s2 s3 config false memory ['s3://', 's3n://', 's3a://'] - -restart - -# Since extensions can add secret storage backends, we allow switching the default backend -statement ok -set default_secret_storage='currently-non-existent' - -statement ok -set secret_directory='__TEST_DIR__/create_secret_storages' - -statement error -CREATE PERSISTENT SECRET s1 ( TYPE S3 ) ----- -Secret storage 'currently-non-existent' not found! - -# We can still work around this broken default by specifying the storage explicitly -statement ok -CREATE PERSISTENT SECRET s1 IN LOCAL_FILE ( TYPE S3 ) - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_storages' - -# Let's restore and now things work again -statement ok -reset default_secret_storage - -statement ok -CREATE PERSISTENT SECRET s2 ( TYPE S3 ) - -query IIIIII -SELECT * EXCLUDE (secret_string) FROM duckdb_secrets() ORDER BY name ----- -perm_s1 s3 config true local_file ['s3://', 's3n://', 's3a://'] -perm_s2 s3 config true local_file ['s3://', 's3n://', 's3a://'] -s1 s3 config true local_file ['s3://', 's3n://', 's3a://'] -s2 s3 config true local_file ['s3://', 's3n://', 's3a://'] - -statement maybe -DROP SECRET perm_s1; ----- -Invalid Input Error: Failed to remove non-existent secret - -statement maybe -DROP SECRET perm_s2; ----- -Invalid Input Error: Failed to remove non-existent secret diff --git a/test/sql/secrets/create_secret_transactional.test b/test/sql/secrets/create_secret_transactional.test deleted file mode 100644 index 79a4157b4530..000000000000 --- a/test/sql/secrets/create_secret_transactional.test +++ /dev/null @@ -1,146 +0,0 @@ -# name: test/sql/secrets/create_secret_transactional.test -# description: Test secret transactional safety -# group: [secrets] - -statement ok -PRAGMA enable_verification; - -require httpfs - -load __TEST_DIR__/create_secret_transactional.db - -statement ok -set secret_directory='__TEST_DIR__/create_secret_transactional' - -statement ok -PRAGMA threads=1 - -foreach secret_type TEMPORARY PERSISTENT - -statement ok con1 -BEGIN TRANSACTION - -statement ok con1 -CREATE ${secret_type} SECRET s1 (TYPE S3) - -statement ok con2 -BEGIN TRANSACTION - -statement ok con2 -CREATE ${secret_type} SECRET s2 (TYPE S3) - -query I con1 -SELECT name FROM duckdb_secrets(); ----- -s1 - -query I con2 -SELECT name FROM duckdb_secrets(); ----- -s2 - -statement ok con1 -COMMIT - -# Transaction 2 still only sees own secret: it has not commited yet -query I con2 -SELECT name FROM duckdb_secrets(); ----- -s2 - -# New transaction will see only committed secret -query I con3 -SELECT name FROM duckdb_secrets(); ----- -s1 - -statement ok con2 -COMMIT - -# Now both are visible -query I con3 -SELECT name FROM duckdb_secrets() ORDER BY name; ----- -s1 -s2 - -statement ok con1 -BEGIN TRANSACTION - -statement ok con1 -DROP SECRET s1; - -# Drop not yet commited: con3 will not see it yet -query I con3 -SELECT name FROM duckdb_secrets() ORDER BY name; ----- -s1 -s2 - -# Commit the drop -statement ok con1 -COMMIT - -# Drop now visible to con3 -query I con3 -SELECT name FROM duckdb_secrets(); ----- -s2 - -# Clean up for loop end -statement ok -DROP SECRET s2 - -endloop - -# Now lets test transactional safety of lazily loaded persistent secrets - -statement ok -CREATE PERSISTENT SECRET perm_s1 (TYPE S3) - -restart - -statement ok -set secret_directory='__TEST_DIR__/create_secret_transactional' - -# After restart, we create 2 connections that each add their own tmp secret; the perm secret is now lazily loaded! -statement ok con1 -BEGIN TRANSACTION - -statement ok con1 -CREATE SECRET tmp_s1 (TYPE S3) - -statement ok con2 -BEGIN TRANSACTION - -statement ok con2 -CREATE SECRET tmp_s2 (TYPE S3) - -# Now con1 drops the lazily loaded perm secret -statement ok con1 -DROP SECRET perm_s1; - -query I con1 -SELECT name FROM duckdb_secrets(); ----- -tmp_s1 - -# con2 still has both secrets -query I con2 -SELECT name FROM duckdb_secrets() ORDER BY name; ----- -perm_s1 -tmp_s2 - -statement ok con1 -COMMIT - -statement ok con2 -COMMIT - -# Now the deletion is visible to con2 -query I con2 -SELECT name FROM duckdb_secrets() ORDER BY name; ----- -tmp_s1 -tmp_s2 \ No newline at end of file diff --git a/test/sql/secrets/persistent_key_value_secret.test b/test/sql/secrets/persistent_key_value_secret.test deleted file mode 100644 index 89448fa9a811..000000000000 --- a/test/sql/secrets/persistent_key_value_secret.test +++ /dev/null @@ -1,28 +0,0 @@ -# name: test/sql/secrets/persistent_key_value_secret.test -# group: [secrets] - -load __TEST_DIR__/persistent_extra_headers - -require httpfs - -require json - -statement ok -CREATE PERSISTENT SECRET http ( - TYPE HTTP, - EXTRA_HTTP_HEADERS MAP { - 'Authorization': 'Bearer sk_test_not_valid_key' - } -); - -restart - -# Because this is an https host, the 'EXTRA_HTTP_HEADERS' will be used, as long as this doesn't crash anything -# we are happy with this test throwing an IO error. -statement error -select - unnest(data) as customers -from - read_json('https://non.existant/endpoint'); ----- -IO Error: Could not establish connection error for HTTP HEAD to 'https://non.existant/endpoint' diff --git a/test/sql/secrets/secret_compatibility_httpfs.test b/test/sql/secrets/secret_compatibility_httpfs.test deleted file mode 100644 index 3cbbb1102406..000000000000 --- a/test/sql/secrets/secret_compatibility_httpfs.test +++ /dev/null @@ -1,23 +0,0 @@ -# name: test/sql/secrets/secret_compatibility_httpfs.test -# description: Test secret compatibility across versions -# group: [secrets] - -require httpfs - -require-env TEST_PERSISTENT_SECRETS_AVAILABLE - -# Ensure any currently stored secrets don't interfere with the test -statement ok -set secret_directory='./data/secrets/httpfs' - -mode output_result - -query IIIIIII -from duckdb_secrets() order by name; ----- -s3_config_secret_v1_1_2 s3 config true local_file ['s3://', 's3n://', 's3a://'] name=s3_config_secret_v1_1_2;type=s3;provider=config;serializable=true;scope=s3://,s3n://,s3a://;region=us-east-2;use_ssl=false -s3_config_secret_v1_1_3 s3 config true local_file ['s3://', 's3n://', 's3a://'] name=s3_config_secret_v1_1_3;type=s3;provider=config;serializable=true;scope=s3://,s3n://,s3a://;region=us-east-2;use_ssl=false -s3_config_secret_v_1_0_0 s3 config true local_file ['s3://', 's3n://', 's3a://'] name=s3_config_secret_v_1_0_0;type=s3;provider=config;serializable=true;scope=s3://,s3n://,s3a://;endpoint=s3.amazonaws.com;key_id=;region=us-east-2;s3_url_compatibility_mode=0;secret=redacted;session_token=redacted;url_style=;use_ssl=0 -s3_secret_chain_v_1_0_0 s3 credential_chain true local_file ['s3://', 's3n://', 's3a://'] name=s3_secret_chain_v_1_0_0;type=s3;provider=credential_chain;serializable=true;scope=s3://,s3n://,s3a://;endpoint=s3.amazonaws.com;region=us-east-2;use_ssl=false -s3_secret_chain_v_1_1_2 s3 credential_chain true local_file ['s3://', 's3n://', 's3a://'] name=s3_secret_chain_v_1_1_2;type=s3;provider=credential_chain;serializable=true;scope=s3://,s3n://,s3a://;endpoint=s3.amazonaws.com;region=us-east-2;use_ssl=false -s3_secret_chain_v_1_1_3 s3 credential_chain true local_file ['s3://', 's3n://', 's3a://'] name=s3_secret_chain_v_1_1_3;type=s3;provider=credential_chain;serializable=true;scope=s3://,s3n://,s3a://;endpoint=s3.amazonaws.com;region=us-east-2;use_ssl=false \ No newline at end of file diff --git a/test/sql/secrets/secret_types_function.test b/test/sql/secrets/secret_types_function.test deleted file mode 100644 index c1fd676571bb..000000000000 --- a/test/sql/secrets/secret_types_function.test +++ /dev/null @@ -1,20 +0,0 @@ -# name: test/sql/secrets/secret_types_function.test -# description: Test duckdb_secret_types function -# group: [secrets] - -query III -FROM duckdb_secret_types() WHERE type IN ['s3', 'r2', 'gcs', 'http'] ORDER BY type ----- -http config (empty) - -require httpfs - -require no_extension_autoloading "EXPECTED: The duckdb_secret_types() function does not trigger autoloading httpfs" - -query III -FROM duckdb_secret_types() WHERE type IN ['s3', 'r2', 'gcs', 'http'] ORDER BY type ----- -gcs config httpfs -http config (empty) -r2 config httpfs -s3 config httpfs diff --git a/test/sql/settings/errors_as_json.test b/test/sql/settings/errors_as_json.test index fd73a0dfa8f3..3ca13ba68b84 100644 --- a/test/sql/settings/errors_as_json.test +++ b/test/sql/settings/errors_as_json.test @@ -21,7 +21,7 @@ COLUMN_NOT_FOUND statement error SECT cbl FROM (VALUES (42)) t(col) ---- -SYNTAX_ERROR +:.*Parser Error.*syntax error at or near.* statement error select corr('hello', 'world') diff --git a/test/sql/settings/setting_alias.test b/test/sql/settings/setting_alias.test new file mode 100644 index 000000000000..f205e500c694 --- /dev/null +++ b/test/sql/settings/setting_alias.test @@ -0,0 +1,26 @@ +# name: test/sql/settings/setting_alias.test +# description: Test behavior of aliased settings +# group: [settings] + +require skip_reload + +query II +SELECT current_setting('null_order'), (SELECT value FROM duckdb_settings() WHERE name='null_order') +---- +NULLS_LAST NULLS_LAST + +statement ok +SET null_order='NULLS_FIRST' + +query II +SELECT current_setting('null_order'), (SELECT value FROM duckdb_settings() WHERE name='null_order') +---- +NULLS_FIRST NULLS_FIRST + +statement ok +RESET null_order + +query II +SELECT current_setting('null_order'), (SELECT value FROM duckdb_settings() WHERE name='null_order') +---- +NULLS_LAST NULLS_LAST diff --git a/test/sql/settings/test_disabled_file_system_httpfs.test b/test/sql/settings/test_disabled_file_system_httpfs.test deleted file mode 100644 index 9cc1ee839bb6..000000000000 --- a/test/sql/settings/test_disabled_file_system_httpfs.test +++ /dev/null @@ -1,28 +0,0 @@ -# name: test/sql/settings/test_disabled_file_system_httpfs.test -# description: Test disabled file systems with HTTPFS -# group: [settings] - -require skip_reload - -require no_extension_autoloading "EXPECTED: Test disable loading from local file system" - -statement ok -PRAGMA enable_verification - -require httpfs - -statement ok -SET disabled_filesystems='LocalFileSystem'; - -# httpfs works -statement ok -from read_csv_auto('https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv'); - -statement ok -SET disabled_filesystems='LocalFileSystem,HTTPFileSystem'; - -# not if we disable it -statement error -from read_csv_auto('https://github.com/duckdb/duckdb/raw/main/data/csv/customer.csv'); ----- -File system HTTPFileSystem has been disabled by configuration diff --git a/test/sql/storage/checkpoint/concurrent_load_delete.test_slow b/test/sql/storage/checkpoint/concurrent_load_delete.test_slow new file mode 100644 index 000000000000..5ca46e260010 --- /dev/null +++ b/test/sql/storage/checkpoint/concurrent_load_delete.test_slow @@ -0,0 +1,51 @@ +# name: test/sql/storage/checkpoint/concurrent_load_delete.test_slow +# description: Test concurrent delete load workflow +# group: [checkpoint] + +load __TEST_DIR__/concurrent_delete_load.db + +statement ok +create or replace table z(id integer); + +statement ok +insert into z from range(10_000_000); + +loop i 0 100 + +statement ok +PRAGMA disable_checkpoint_on_shutdown + +statement ok +SET checkpoint_threshold='1TB' + +concurrentloop c 0 7 + +onlyif c=0 +statement ok +FORCE CHECKPOINT + +onlyif c>0&&c<=5 +statement ok +SELECT SUM(id) FROM z + +onlyif c=5 +statement ok +DELETE FROM z WHERE id%((random() * 3)::UBIGINT)=0 + +onlyif c=6 +statement ok +INSERT INTO z FROM range(1000, 100000 + ${c} * 100000) + +endloop + +restart + +endloop + +statement ok +CHECKPOINT + +restart + +statement ok +SELECT SUM(id) FROM z diff --git a/test/sql/storage/checkpoint/keep_small_row_groups.test_slow b/test/sql/storage/checkpoint/keep_small_row_groups.test_slow new file mode 100644 index 000000000000..384cfaf2ca99 --- /dev/null +++ b/test/sql/storage/checkpoint/keep_small_row_groups.test_slow @@ -0,0 +1,39 @@ +# name: test/sql/storage/checkpoint/keep_small_row_groups.test_slow +# description: Test keeping small row groups +# group: [checkpoint] + +load __TEST_DIR__/keep_small_row_groups.db + +foreach type noindex index + +onlyif type=noindex +statement ok +create or replace table z(id integer, a varchar, b varchar, c varchar); + +onlyif type=index +statement ok +create or replace table z(id integer primary key, a varchar, b varchar, c varchar); + +statement ok +insert into z select i, sha256(i::varchar) as a, sha256((i**2)::varchar) as b, sha256((i**3)::varchar) as c from range(100000) r(i); + +statement ok +CHECKPOINT; + +# we insert 100K rows, as part of small 100 row inserts, and checkpoint for each iteration +loop i 0 100 + +statement ok +insert into z select 100000 + 100 * ${i} + i, 'a','b','c' from range(100) t(i); + +statement ok +CHECKPOINT; + +endloop + +query I +SELECT COUNT(DISTINCT row_group_id) < 5 FROM pragma_storage_info('z') +---- +true + +endloop diff --git a/test/sql/storage/compact_block_size/create_table_compression.test b/test/sql/storage/compact_block_size/create_table_compression.test index 7f93928912af..5d5c4278d25c 100644 --- a/test/sql/storage/compact_block_size/create_table_compression.test +++ b/test/sql/storage/compact_block_size/create_table_compression.test @@ -22,10 +22,12 @@ CREATE TABLE T (a INTEGER USING COMPRESSION 'bla') statement error CREATE TABLE T (a INTEGER USING COMPRESSION ) ---- +Parser Error: syntax error at or near ")" statement error CREATE TABLE T (a INTEGER NOT NULL USING COMPRESSION ) ---- +Parser Error: syntax error at or near ")" statement error CREATE TABLE T (a INTEGER USING COMPRESSION bla) diff --git a/test/sql/storage/compression/alp/alp_min_max.test b/test/sql/storage/compression/alp/alp_min_max.test index 2b30c17f7a73..9c276fe0c3d2 100644 --- a/test/sql/storage/compression/alp/alp_min_max.test +++ b/test/sql/storage/compression/alp/alp_min_max.test @@ -1,14 +1,13 @@ # name: test/sql/storage/compression/alp/alp_min_max.test # group: [alp] -# load the DB from disk load __TEST_DIR__/alp_min_max.db statement ok PRAGMA enable_verification statement ok -pragma force_compression='alp'; +PRAGMA force_compression='alp'; foreach type DOUBLE FLOAT @@ -17,7 +16,6 @@ CREATE TABLE all_types AS SELECT ${type} FROM test_all_types(); loop i 0 15 - statement ok INSERT INTO all_types SELECT ${type} FROM all_types; diff --git a/test/sql/storage/compression/alp/alp_stress_test.test_slow b/test/sql/storage/compression/alp/alp_stress_test.test_slow index adaea15af26a..9d76a32bc12f 100644 --- a/test/sql/storage/compression/alp/alp_stress_test.test_slow +++ b/test/sql/storage/compression/alp/alp_stress_test.test_slow @@ -1,9 +1,6 @@ # name: test/sql/storage/compression/alp/alp_stress_test.test_slow # group: [alp] -# FIXME: for small block sizes (16KB), this test returns a Constant segment in Line 69 -require block_size 262144 - load __TEST_DIR__/alp_min_max.db foreach type DOUBLE FLOAT @@ -64,11 +61,6 @@ insert into ${compression}_tbl select * from temp_table; statement ok checkpoint -# And verify that no other compression is used -query I -SELECT compression FROM pragma_storage_info('${compression}_tbl') WHERE segment_type == '${type}' AND compression != '${compression}'; ----- - # compression endloop diff --git a/test/sql/storage/compression/alprd/alprd_min_max.test b/test/sql/storage/compression/alprd/alprd_min_max.test index 4d9c94ae0403..471c5e5a3258 100644 --- a/test/sql/storage/compression/alprd/alprd_min_max.test +++ b/test/sql/storage/compression/alprd/alprd_min_max.test @@ -1,14 +1,13 @@ # name: test/sql/storage/compression/alprd/alprd_min_max.test # group: [alprd] -# load the DB from disk load __TEST_DIR__/alprd_min_max.db statement ok PRAGMA enable_verification statement ok -pragma force_compression='alprd'; +PRAGMA force_compression='alprd'; foreach type DOUBLE FLOAT diff --git a/test/sql/storage/compression/alprd/alprd_stress_test.test_slow b/test/sql/storage/compression/alprd/alprd_stress_test.test_slow index bead967e040d..e405d68dba30 100644 --- a/test/sql/storage/compression/alprd/alprd_stress_test.test_slow +++ b/test/sql/storage/compression/alprd/alprd_stress_test.test_slow @@ -1,9 +1,6 @@ # name: test/sql/storage/compression/alprd/alprd_stress_test.test_slow # group: [alprd] -# FIXME: for small block sizes (16KB), this test returns a Constant segment in Line 69 -require block_size 262144 - load __TEST_DIR__/alprd_min_max.db foreach type DOUBLE FLOAT @@ -64,11 +61,6 @@ insert into ${compression}_tbl select * from temp_table; statement ok checkpoint -# And verify that no other compression is used -query I -SELECT compression FROM pragma_storage_info('${compression}_tbl') WHERE segment_type == '${type}' AND compression != '${compression}'; ----- - # compression endloop diff --git a/test/sql/storage/compression/dict_fsst/dictionary_covers_validity.test b/test/sql/storage/compression/dict_fsst/dictionary_covers_validity.test index 9b6d5e104936..f956ef2c2814 100644 --- a/test/sql/storage/compression/dict_fsst/dictionary_covers_validity.test +++ b/test/sql/storage/compression/dict_fsst/dictionary_covers_validity.test @@ -12,18 +12,16 @@ CREATE TABLE tbl AS SELECT 'a': i, 'b': NULL::VARCHAR } col -FROM range(5000) t(i); - -statement ok -set force_compression='dict_fsst'; - -statement ok -INSERT INTO tbl VALUES ( +FROM range(5000) t(i) +union all +select { 'a': 10000, 'b': 'hello' } -); + +statement ok +set force_compression='dict_fsst'; statement ok force checkpoint; @@ -43,12 +41,18 @@ statement ok set force_compression='zstd'; statement ok -INSERT INTO tbl VALUES ( +CREATE OR REPLACE TABLE tbl AS SELECT + { + 'a': i, + 'b': NULL::VARCHAR + } col +FROM range(5000) t(i) +union all +select { 'a': 10000, 'b': 'hello' - } -); + } FROM range(2) statement ok force checkpoint; diff --git a/test/sql/storage/compression/fsst/fsst_disable_compression.test b/test/sql/storage/compression/fsst/fsst_disable_compression.test index ce9ee40f2062..79d54c3f35cd 100644 --- a/test/sql/storage/compression/fsst/fsst_disable_compression.test +++ b/test/sql/storage/compression/fsst/fsst_disable_compression.test @@ -4,6 +4,8 @@ require no_latest_storage +require skip_reload + # load the DB from disk load __TEST_DIR__/test_disabled_compression_methods.db readwrite v1.0.0 diff --git a/test/sql/storage/compression/string/empty.test b/test/sql/storage/compression/string/empty.test index 9bf17421167c..2ddabc62504e 100644 --- a/test/sql/storage/compression/string/empty.test +++ b/test/sql/storage/compression/string/empty.test @@ -54,11 +54,7 @@ SELECT lower(compression)='${compression}' FROM pragma_storage_info('test_empty' 1 statement ok -CREATE TABLE test_empty_large AS SELECT '' as a from range(0,10000); - -statement ok -INSERT INTO test_empty_large VALUES('A'); -INSERT INTO test_empty_large VALUES(''); +CREATE TABLE test_empty_large AS SELECT '' as a from range(0,10000) union all select 'A' union all select ''; statement ok CHECKPOINT diff --git a/test/sql/storage/compression/string/null.test_slow b/test/sql/storage/compression/string/null.test_slow index 6e936d1e3905..731f693a02b2 100644 --- a/test/sql/storage/compression/string/null.test_slow +++ b/test/sql/storage/compression/string/null.test_slow @@ -67,7 +67,7 @@ true # mix with non-null values statement ok -INSERT INTO nulls VALUES (1), (1), (1), (2), (2), (2) +CREATE OR REPLACE TABLE nulls AS FROM nulls UNION ALL VALUES (1), (1), (1), (2), (2), (2) query III SELECT COUNT(*), COUNT(i::INT), SUM(i::INT) FROM nulls diff --git a/test/sql/storage/concurrent_attach_if_not_exists.test_slow b/test/sql/storage/concurrent_attach_if_not_exists.test_slow new file mode 100644 index 000000000000..724367a8183d --- /dev/null +++ b/test/sql/storage/concurrent_attach_if_not_exists.test_slow @@ -0,0 +1,26 @@ +# name: test/sql/storage/concurrent_attach_if_not_exists.test_slow +# description: Test concurrent attaching +# group: [storage] + +concurrentloop x 0 10 + +foreach name db1 db2 db3 db4 db5 + +statement maybe +attach if not exists '__TEST_DIR__/concurrent_attach_${name}.duckdb' AS ${name} +---- +detach + +statement maybe +detach ${name} +---- +database not found + +statement maybe +attach if not exists '__TEST_DIR__/concurrent_attach_${name}.duckdb' AS ${name} +---- +detach + +endloop + +endloop diff --git a/test/sql/storage/concurrent_attach_if_not_exists_create.test b/test/sql/storage/concurrent_attach_if_not_exists_create.test new file mode 100644 index 000000000000..1d751db610ee --- /dev/null +++ b/test/sql/storage/concurrent_attach_if_not_exists_create.test @@ -0,0 +1,17 @@ +# name: test/sql/storage/concurrent_attach_if_not_exists_create.test +# description: Test concurrent ATTACH IF NOT EXISTS followed by a create +# group: [storage] + +concurrentloop x 0 10 + +foreach name db1 db2 db3 db4 db5 + +statement ok +attach if not exists '__TEST_DIR__/concurrent_attach_${name}.duckdb' AS ${name} + +statement ok +create table ${name}.tbl${x}(i INTEGER) + +endloop + +endloop diff --git a/test/sql/storage/concurrent_attach_or_replace.test_slow b/test/sql/storage/concurrent_attach_or_replace.test_slow new file mode 100644 index 000000000000..20dbbec88e5a --- /dev/null +++ b/test/sql/storage/concurrent_attach_or_replace.test_slow @@ -0,0 +1,52 @@ +# name: test/sql/storage/concurrent_attach_or_replace.test_slow +# description: Test concurrent running attach or replace +# group: [storage] + +# create the databases +loop db_index 0 20 + +statement ok +attach '__TEST_DIR__/attached_db${db_index}.duckdb' AS db_name + +statement ok +create table db_name.integers(i integer); + +statement ok +detach db_name + +endloop + +statement ok +attach '__TEST_DIR__/attached_db0.duckdb' AS db_name + +concurrentloop i 0 10 + +loop db_index 1 20 + +onlyif i<1 +statement ok +begin + +onlyif i<1 +statement ok +attach or replace '__TEST_DIR__/attached_db${db_index}.duckdb' AS db_name + +onlyif i<1 +statement ok +select count(*) from db_name.integers + +onlyif i<1 +statement ok +commit + +endloop + +loop insert_idx 0 100 + +onlyif i>=1 +statement ok +insert into db_name.integers values (42); + +endloop + +endloop diff --git a/test/sql/storage/encryption/temp_files/encrypt_asof_join_merge.test_slow b/test/sql/storage/encryption/temp_files/encrypt_asof_join_merge.test_slow index a0c66b8cb2ca..1c705ae1c66c 100644 --- a/test/sql/storage/encryption/temp_files/encrypt_asof_join_merge.test_slow +++ b/test/sql/storage/encryption/temp_files/encrypt_asof_join_merge.test_slow @@ -2,8 +2,10 @@ # description: Test merge queue and repartitioning with encrypted temporary files # group: [temp_files] +foreach cipher GCM CTR + statement ok -ATTACH '__TEST_DIR__/encrypted_temp_files.db' AS enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_temp_files_${cipher}.db' AS enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok SET temp_file_encryption=true; @@ -35,4 +37,8 @@ WITH build AS ( ) SELECT SUM(v) AS v, COUNT(*) AS n FROM probe ASOF JOIN build USING(k, t) ---- -26790 1488 \ No newline at end of file +26790 1488 + +restart + +endloop \ No newline at end of file diff --git a/test/sql/storage/encryption/temp_files/encrypted_offloading_block_files.test_slow b/test/sql/storage/encryption/temp_files/encrypted_offloading_block_files.test_slow index 95a4f65b72dc..280803a839bc 100644 --- a/test/sql/storage/encryption/temp_files/encrypted_offloading_block_files.test_slow +++ b/test/sql/storage/encryption/temp_files/encrypted_offloading_block_files.test_slow @@ -1,13 +1,16 @@ # name: test/sql/storage/encryption/temp_files/encrypted_offloading_block_files.test_slow # group: [temp_files] +foreach cipher GCM CTR + + require block_size 262144 load __TEST_DIR__/offloading_block_files.db # ATTACH DB statement ok -ATTACH '__TEST_DIR__/encrypted.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted.db' as enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok SET temp_file_encryption=true; @@ -34,3 +37,7 @@ SET memory_limit = '1GB'; statement error SELECT * FROM tbl ORDER BY random_value; ---- + +restart + +endloop \ No newline at end of file diff --git a/test/sql/storage/encryption/temp_files/encrypted_out_of_core.test_slow b/test/sql/storage/encryption/temp_files/encrypted_out_of_core.test_slow deleted file mode 100644 index db4d38c7b2d9..000000000000 --- a/test/sql/storage/encryption/temp_files/encrypted_out_of_core.test_slow +++ /dev/null @@ -1,59 +0,0 @@ -# name: test/sql/storage/encryption/temp_files/encrypted_out_of_core.test_slow -# description: Encrypted large joins in persistent databases have a leftover temporary directory. -# group: [temp_files] - -require tpch - -load __TEST_DIR__/leftover_temp_files.db - -statement ok -ATTACH '__TEST_DIR__/encrypted_temp_files.db' AS enc (ENCRYPTION_KEY 'asdf'); - -statement ok -SET temp_file_encryption=true; - -statement ok -USE enc; - -statement ok -SET threads=8 - -statement ok -SET memory_limit='1GB'; - -statement ok -CALL dbgen(sf=1); - -statement ok -ALTER TABLE lineitem RENAME TO lineitem1 - -statement ok -CREATE TABLE lineitem2 AS FROM lineitem1 - -# creating and dropping a table with an ORDER BY -statement ok -CREATE OR REPLACE TEMPORARY TABLE ans as select l1.*, l1.* from lineitem1 l1 ORDER BY l_orderkey, l_returnflag - -statement ok -DROP TABLE ans; - -# performing a small hash join -statement ok -CREATE OR REPLACE TEMPORARY TABLE ans as select l1.*, l2.* from lineitem1 l1 JOIN (FROM lineitem2 l2 WHERE l_orderkey<10000) AS l2 USING (l_orderkey, l_linenumber) - -statement ok -DROP TABLE ans; - -# performing a large window function -statement ok -CREATE OR REPLACE TEMPORARY TABLE ans as select l1.*, row_number() OVER (PARTITION BY l_orderkey, l_linenumber ORDER BY l_orderkey) from lineitem1 l1 - -statement ok -DROP TABLE ans; - -# performing a large hash join -statement ok -CREATE OR REPLACE TEMPORARY TABLE ans as select l1.*, l2.* from lineitem1 l1 JOIN lineitem2 l2 USING (l_orderkey, l_linenumber) - -statement ok -DROP TABLE ans; \ No newline at end of file diff --git a/test/sql/storage/encryption/temp_files/encrypted_tmp_file_setting.test b/test/sql/storage/encryption/temp_files/encrypted_tmp_file_setting.test index c4ed642df07c..4c402d78465b 100644 --- a/test/sql/storage/encryption/temp_files/encrypted_tmp_file_setting.test +++ b/test/sql/storage/encryption/temp_files/encrypted_tmp_file_setting.test @@ -1,6 +1,8 @@ # name: test/sql/storage/encryption/temp_files/encrypted_tmp_file_setting.test # group: [temp_files] +foreach cipher GCM CTR + require block_size 262144 statement ok @@ -45,3 +47,6 @@ SET temp_file_encryption = true; ---- Permission Error: Existing temporary files found: Modifying the temp_file_encryption setting while there are existing temporary files is disabled. +restart + +endloop \ No newline at end of file diff --git a/test/sql/storage/encryption/temp_files/encrypted_tpch_join.test_slow b/test/sql/storage/encryption/temp_files/encrypted_tpch_join.test_slow index 4cbc621c6e90..2491d644956f 100644 --- a/test/sql/storage/encryption/temp_files/encrypted_tpch_join.test_slow +++ b/test/sql/storage/encryption/temp_files/encrypted_tpch_join.test_slow @@ -1,6 +1,8 @@ # name: test/sql/storage/encryption/temp_files/encrypted_tpch_join.test_slow # group: [temp_files] +foreach cipher GCM CTR + require tpch statement ok @@ -10,7 +12,7 @@ statement ok SET memory_limit = '1GB'; statement ok -ATTACH '__TEST_DIR__/tpch_enc.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/tpch_enc_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok USE enc; @@ -27,3 +29,7 @@ CREATE TABLE lineitem2 AS FROM lineitem1; statement ok CREATE OR REPLACE TABLE ans as select l1.* , l2.* from lineitem1 l1 JOIN lineitem2 l2 USING (l_orderkey , l_linenumber); + +restart + +endloop \ No newline at end of file diff --git a/test/sql/storage/encryption/temp_files/temp_directory_enable_external_access.test b/test/sql/storage/encryption/temp_files/temp_directory_enable_external_access.test index fc4d4665c724..1a9a97017460 100644 --- a/test/sql/storage/encryption/temp_files/temp_directory_enable_external_access.test +++ b/test/sql/storage/encryption/temp_files/temp_directory_enable_external_access.test @@ -1,13 +1,16 @@ # name: test/sql/storage/encryption/temp_files/temp_directory_enable_external_access.test # group: [temp_files] +foreach cipher GCM CTR + + require block_size 262144 statement ok SET temp_directory='__TEST_DIR__/test_temp_dir' statement ok -ATTACH '__TEST_DIR__/encrypted_temp_files.db' AS enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_temp_files_${cipher}.db' AS enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok USE enc; @@ -32,3 +35,7 @@ disabled by configuration # 10M row table with 8B rows -> 80MB uncompressed - this requires the temp directory statement ok CREATE TEMPORARY TABLE tbl AS FROM range(10_000_000) + +restart + +endloop \ No newline at end of file diff --git a/test/sql/storage/encryption/wal/encrypted_wal_blob_storage.test b/test/sql/storage/encryption/wal/encrypted_wal_blob_storage.test index 042c0d64ce25..d92c055832cc 100644 --- a/test/sql/storage/encryption/wal/encrypted_wal_blob_storage.test +++ b/test/sql/storage/encryption/wal/encrypted_wal_blob_storage.test @@ -2,6 +2,9 @@ # description: Test BLOB with persistent storage with an encrypted WAL # group: [wal] +foreach cipher GCM CTR + + load __TEST_DIR__/any_file.db statement ok @@ -12,7 +15,7 @@ PRAGMA wal_autocheckpoint='1TB'; # # load the DB from disk statement ok -ATTACH '__TEST_DIR__/encrypted_blob_storage.db' AS enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_blob_storage_${cipher}.db' AS enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); # create a table with hugeints statement ok @@ -25,7 +28,7 @@ statement ok DETACH enc statement ok -ATTACH '__TEST_DIR__/encrypted_blob_storage.db' AS enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_blob_storage_${cipher}.db' AS enc (ENCRYPTION_KEY 'asdf'); query I SELECT * FROM enc.blobs @@ -38,5 +41,7 @@ NULL U\xAA\xFFU\xAA\xFFU\xAA\xFF\x01 U\xAA\xFFU\xAA\xFFU\xAA\xFF\x01 -# endloop +restart + +endloop diff --git a/test/sql/storage/encryption/wal/encrypted_wal_lazy_creation.test b/test/sql/storage/encryption/wal/encrypted_wal_lazy_creation.test index 9a8353c18f35..a704efb73c28 100644 --- a/test/sql/storage/encryption/wal/encrypted_wal_lazy_creation.test +++ b/test/sql/storage/encryption/wal/encrypted_wal_lazy_creation.test @@ -4,6 +4,9 @@ # If we enable alternative verification, then we disable checkpointing on shutdown. # Thus, we'll keep the WAL alive when detaching the database. + +foreach cipher GCM CTR + require no_alternative_verify require noforcestorage @@ -11,7 +14,7 @@ require noforcestorage require skip_reload statement ok -ATTACH '__TEST_DIR__/attach_no_wal.db' (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/attach_no_wal_${cipher}.db' AS attach_no_wal (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok CREATE TABLE attach_no_wal.integers(i INTEGER); @@ -23,11 +26,11 @@ statement ok DETACH attach_no_wal; statement ok -ATTACH '__TEST_DIR__/attach_no_wal.db' (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/attach_no_wal_${cipher}.db' AS attach_no_wal (ENCRYPTION_KEY 'asdf'); # Verify that we don't have a WAL after attaching. query I -SELECT COUNT(*) FROM glob('__TEST_DIR__/attach_no_wal.db.wal'); +SELECT COUNT(*) FROM glob('__TEST_DIR__/attach_no_wal_${cipher}.db.wal'); ---- 0 @@ -35,4 +38,8 @@ SELECT COUNT(*) FROM glob('__TEST_DIR__/attach_no_wal.db.wal'); query I SELECT COUNT(*) FROM attach_no_wal.integers; ---- -10000 \ No newline at end of file +10000 + +restart + +endloop \ No newline at end of file diff --git a/test/sql/storage/encryption/wal/encrypted_wal_pragmas.test b/test/sql/storage/encryption/wal/encrypted_wal_pragmas.test index 28ac656b627f..d42b1b73d99c 100644 --- a/test/sql/storage/encryption/wal/encrypted_wal_pragmas.test +++ b/test/sql/storage/encryption/wal/encrypted_wal_pragmas.test @@ -2,10 +2,13 @@ # description: test encrypted wal debug PRAGMAS # group: [wal] +foreach cipher GCM CTR + + load __TEST_DIR__/any_wal_db.db statement ok -ATTACH '__TEST_DIR__/encrypted_wal_restart.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_wal_restart_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok PRAGMA disable_checkpoint_on_shutdown @@ -38,14 +41,14 @@ DETACH enc # WAL replay succeeds statement ok -ATTACH '__TEST_DIR__/encrypted_wal_restart.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_wal_restart_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf'); statement ok DETACH enc # now set debug pragma at the start statement ok -ATTACH '__TEST_DIR__/encrypted_wal_restart_new.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_wal_restart_new_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok PRAGMA disable_checkpoint_on_shutdown @@ -72,7 +75,7 @@ SELECT * FROM enc.test ORDER BY 1 restart statement ok -ATTACH '__TEST_DIR__/encrypted_wal_restart_new.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_wal_restart_new_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf'); query IT SELECT * FROM enc.test ORDER BY 1 @@ -87,7 +90,7 @@ restart # this should work statement ok -ATTACH '__TEST_DIR__/encrypted_wal_disable_enable.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_wal_disable_enable_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok PRAGMA wal_autocheckpoint='1TB'; @@ -114,7 +117,7 @@ INSERT INTO enc.test VALUES (10, 'hello') restart statement ok -ATTACH '__TEST_DIR__/encrypted_wal_disable_enable.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_wal_disable_enable_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); query IT SELECT * FROM enc.test ORDER BY 1 @@ -125,3 +128,6 @@ SELECT * FROM enc.test ORDER BY 1 13 22 +restart + +endloop \ No newline at end of file diff --git a/test/sql/storage/encryption/wal/encrypted_wal_restart.test b/test/sql/storage/encryption/wal/encrypted_wal_restart.test index 65b57eebe24c..197990dfa63a 100644 --- a/test/sql/storage/encryption/wal/encrypted_wal_restart.test +++ b/test/sql/storage/encryption/wal/encrypted_wal_restart.test @@ -2,10 +2,12 @@ # description: test wal restart # group: [wal] +foreach cipher GCM CTR + load __TEST_DIR__/any_wal_db.db statement ok -ATTACH '__TEST_DIR__/encrypted_wal_restart.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_wal_restart_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok PRAGMA disable_checkpoint_on_shutdown @@ -23,7 +25,7 @@ INSERT INTO enc.test VALUES (11, 22), (13, 22), (12, 21) restart statement ok -ATTACH '__TEST_DIR__/encrypted_wal_restart.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_wal_restart_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); statement ok PRAGMA disable_checkpoint_on_shutdown @@ -54,12 +56,12 @@ SELECT * FROM enc.test ORDER BY 1 restart -statement error -ATTACH '__TEST_DIR__/encrypted_wal_restart.db' as enc (ENCRYPTION_KEY 'xxxx'); ----- +# statement error +# ATTACH '__TEST_DIR__/encrypted_wal_restart_${cipher}.db' as enc (ENCRYPTION_KEY 'xxxx', ENCRYPTION_CIPHER '${cipher}'); +# ---- statement ok -ATTACH '__TEST_DIR__/encrypted_wal_restart.db' as enc (ENCRYPTION_KEY 'asdf'); +ATTACH '__TEST_DIR__/encrypted_wal_restart_${cipher}.db' as enc (ENCRYPTION_KEY 'asdf', ENCRYPTION_CIPHER '${cipher}'); query IT SELECT * FROM enc.test ORDER BY 1 @@ -67,4 +69,8 @@ SELECT * FROM enc.test ORDER BY 1 10 hello 11 22 12 21 -13 22 \ No newline at end of file +13 22 + +endloop + +restart \ No newline at end of file diff --git a/test/sql/storage/external_file_cache/external_file_cache_httpfs.test b/test/sql/storage/external_file_cache/external_file_cache_httpfs.test deleted file mode 100644 index 2efa3613731b..000000000000 --- a/test/sql/storage/external_file_cache/external_file_cache_httpfs.test +++ /dev/null @@ -1,18 +0,0 @@ -# name: test/sql/storage/external_file_cache/external_file_cache_httpfs.test -# description: Test the external file cache for HTTPFS reads -# group: [external_file_cache] - -require parquet - -require httpfs - -# first query caches the data -statement ok -from 's3://duckdb-blobs/data/shakespeare.parquet'; - - -# second query should only have a head request, no gets -query II -explain analyze from 's3://duckdb-blobs/data/shakespeare.parquet'; ----- -analyzed_plan :.*GET: 0.* diff --git a/test/sql/storage/external_file_cache/external_file_cache_read_blob.test_slow b/test/sql/storage/external_file_cache/external_file_cache_read_blob.test_slow deleted file mode 100644 index 9edf1622b5ee..000000000000 --- a/test/sql/storage/external_file_cache/external_file_cache_read_blob.test_slow +++ /dev/null @@ -1,25 +0,0 @@ -# name: test/sql/storage/external_file_cache/external_file_cache_read_blob.test_slow -# description: Test the external file cache for read_blob HTTPFS reads -# group: [external_file_cache] - -require parquet - -require httpfs - -# first read_blob should do 1 GET -query II -explain analyze from read_blob('s3://duckdb-blobs/data/shakespeare.parquet'); ----- -analyzed_plan :.*GET: 1.* - -# second one should do 0 -query II -explain analyze from read_blob('s3://duckdb-blobs/data/shakespeare.parquet'); ----- -analyzed_plan :.*GET: 0.* - -# although the read was cached using read_blob, the parquet reader can read from cache -query II -explain analyze from 's3://duckdb-blobs/data/shakespeare.parquet'; ----- -analyzed_plan :.*GET: 0.* diff --git a/test/sql/storage/invalid_unicode_scrambled.test_slow b/test/sql/storage/invalid_unicode_scrambled.test_slow deleted file mode 100644 index 2af3750177dd..000000000000 --- a/test/sql/storage/invalid_unicode_scrambled.test_slow +++ /dev/null @@ -1,14 +0,0 @@ -# name: test/sql/storage/invalid_unicode_scrambled.test_slow -# description: Issue #1650 - "invalid unicode detected in segment statistics" when inserting structs with strings and NULL values -# group: [storage] - -require httpfs - -require parquet - -statement ok -create or replace table blah as (with -us as (select distinct * from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/invalid_unicode_scrambled.parquet') select Address from -us); - - diff --git a/test/sql/storage/metadata/multi_table_metadata_reuse.test b/test/sql/storage/metadata/multi_table_metadata_reuse.test new file mode 100644 index 000000000000..ac01e83eaddb --- /dev/null +++ b/test/sql/storage/metadata/multi_table_metadata_reuse.test @@ -0,0 +1,58 @@ +# name: test/sql/storage/metadata/multi_table_metadata_reuse.test +# description: Test metadata reuse with multiple tables and restarts +# group: [metadata] + +load __TEST_DIR__/multi_table_metadata_reuse.db + +statement ok +SET experimental_metadata_reuse=true; + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE TABLE ducklake_table(end_snapshot BIGINT); + +statement ok +CREATE TABLE ducklake_column(end_snapshot BIGINT); + +statement ok +COMMIT; + +restart + +statement ok +SET experimental_metadata_reuse=true; + +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO ducklake_table VALUES (1); + +statement ok +INSERT INTO ducklake_column VALUES (1); + +statement ok +COMMIT; + +restart + +statement ok +SET experimental_metadata_reuse=true; + +statement ok +BEGIN TRANSACTION; + +statement ok +UPDATE ducklake_table SET end_snapshot = 3 + +statement ok +UPDATE ducklake_column SET end_snapshot = 3 + +statement ok +COMMIT; + +statement ok +CREATE TABLE my_table (a INTEGER, b INTEGER); + diff --git a/test/sql/storage/parallel/batch_insert_filtered_row_groups.test_slow b/test/sql/storage/parallel/batch_insert_filtered_row_groups.test_slow index 2a94bf9d7b16..d1476c73509c 100644 --- a/test/sql/storage/parallel/batch_insert_filtered_row_groups.test_slow +++ b/test/sql/storage/parallel/batch_insert_filtered_row_groups.test_slow @@ -29,11 +29,11 @@ query I SELECT * FROM test QUALIFY i <= lag(i) over () ---- -# ensure that we still write 122880 as our row group size count, even for different block sizes +# ensure that we still write close to our row group size as our row group size count, even for different block sizes query I -SELECT MAX(count) FROM pragma_storage_info('test') +SELECT MAX(count) > 100000 FROM pragma_storage_info('test') ---- -122880 +true # The median differs between block sizes because the upper bound of the segment size is the block size. require block_size 262144 diff --git a/test/sql/storage/read_duckdb/concurrent_duckdb_read.test_slow b/test/sql/storage/read_duckdb/concurrent_duckdb_read.test_slow new file mode 100644 index 000000000000..1e891a5238e2 --- /dev/null +++ b/test/sql/storage/read_duckdb/concurrent_duckdb_read.test_slow @@ -0,0 +1,37 @@ +# name: test/sql/storage/read_duckdb/concurrent_duckdb_read.test_slow +# description: Test concurrent read of a DuckDB file +# group: [read_duckdb] + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_concurrent.db' + +statement ok +CREATE TABLE read_duckdb_concurrent.my_tbl AS SELECT * FROM range(1000000) t(i) + +statement ok +DETACH read_duckdb_concurrent + +concurrentloop i 0 10 + +loop x 0 100 + +onlyif i=0 +query I +SHOW DATABASES +---- +memory + +onlyif i=0 +query I +SHOW TABLES +---- + +endloop + +onlyif i>0 +query II +SELECT COUNT(*), SUM(i) FROM read_duckdb('__TEST_DIR__/read_duckdb_concurrent.db') +---- +1000000 499999500000 + +endloop diff --git a/test/sql/storage/read_duckdb/read_duckdb_basic.test b/test/sql/storage/read_duckdb/read_duckdb_basic.test new file mode 100644 index 000000000000..9956857502f8 --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_basic.test @@ -0,0 +1,91 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_basic.test +# description: Test basic usage of read_duckdb +# group: [read_duckdb] + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_test.db' + +statement ok +CREATE TABLE read_duckdb_test.my_tbl AS SELECT 42 i + +# we cannot read a database that is already attached +statement error +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_test.db') +---- +already attached by database + +statement ok +DETACH read_duckdb_test + +query I +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_test.db') +---- +42 + +query I +SELECT COUNT(*) FROM duckdb_databases +---- +1 + +# globbing +statement ok +ATTACH '__TEST_DIR__/read_duckdb_test2.db' + +statement ok +CREATE TABLE read_duckdb_test2.other_tbl AS SELECT 100 i + +statement ok +DETACH read_duckdb_test2 + +query I +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_test*.db') +---- +42 +100 + +# replacement scan +query I +SELECT * FROM '__TEST_DIR__/read_duckdb_test*.db' +---- +42 +100 + +# prepared statement +statement ok +PREPARE v1 AS FROM read_duckdb('__TEST_DIR__/read_duckdb_test*.db') + +query I +EXECUTE v1 +---- +42 +100 + +# multi-table +statement ok +ATTACH '__TEST_DIR__/read_duckdb_test.db' + +statement ok +CREATE TABLE read_duckdb_test.my_tbl2 AS SELECT 84 j + +statement ok +DETACH read_duckdb_test + +statement error +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_test.db') +---- +multiple tables + +query I +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_test.db', table_name='my_tbl') +---- +42 + +query I +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_test.db', table_name='my_tbl2') +---- +84 + +statement error +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_test.db', table_name='my_tblx') +---- +my_tbl2 diff --git a/test/sql/storage/read_duckdb/read_duckdb_generated.test b/test/sql/storage/read_duckdb/read_duckdb_generated.test new file mode 100644 index 000000000000..d4b82a6af1cd --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_generated.test @@ -0,0 +1,30 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_generated.test +# description: Test read_duckdb with generated columns +# group: [read_duckdb] + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_generated.db' AS rd + +statement ok +CREATE TABLE rd.tbl ( + price INTEGER, + amount_sold INTEGER, + total_profit AS (price * amount_sold), + non_generated INTEGER +); + +statement ok +INSERT INTO rd.tbl VALUES (5,4, 100); + +statement ok +DETACH rd + +statement error +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_generated.db') +---- +total_profit + +query III +SELECT non_generated + 42, price, amount_sold FROM read_duckdb('__TEST_DIR__/read_duckdb_generated.db') +---- +142 5 4 diff --git a/test/sql/storage/read_duckdb/read_duckdb_index.test b/test/sql/storage/read_duckdb/read_duckdb_index.test new file mode 100644 index 000000000000..a9fc0daa5e1a --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_index.test @@ -0,0 +1,28 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_index.test +# description: Test read_duckdb with an index +# group: [read_duckdb] + +# FIXME: bug in WAL replay with read-only mode attach +require no_alternative_verify + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_index.db' + +statement ok +CREATE TABLE read_duckdb_index.my_tbl(i INTEGER PRIMARY KEY); + +statement ok +INSERT INTO read_duckdb_index.my_tbl SELECT i + 1 FROM range(1000000) t(i); + +statement ok +DETACH read_duckdb_index + +query I +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_index.db') WHERE i=42873 +---- +42873 + +query II +SELECT *, rowid FROM read_duckdb('__TEST_DIR__/read_duckdb_index.db') WHERE i=42873 +---- +42873 42872 diff --git a/test/sql/storage/read_duckdb/read_duckdb_schema.test b/test/sql/storage/read_duckdb/read_duckdb_schema.test new file mode 100644 index 000000000000..eb95a5aa221b --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_schema.test @@ -0,0 +1,42 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_schema.test +# description: Test basic usage of read_duckdb +# group: [read_duckdb] + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_schema.db' AS rd + +statement ok +CREATE SCHEMA rd.s1; + + +statement ok +CREATE SCHEMA rd.s2; + +statement ok +CREATE TABLE rd.s1.my_tbl AS SELECT 42 i + +statement ok +CREATE TABLE rd.s2.my_tbl AS SELECT 84 i + +statement ok +DETACH rd + +statement error +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_schema.db') +---- +s1.my_tbl + +query I +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_schema.db', schema_name='s1', table_name='my_tbl') +---- +42 + +query I +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_schema.db', schema_name='s2', table_name='my_tbl') +---- +84 + +statement error +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_schema.db', schema_name='s3', table_name='my_tbl') +---- +schema_name="s3", table_name="my_tbl" diff --git a/test/sql/storage/read_duckdb/read_duckdb_suggested.test b/test/sql/storage/read_duckdb/read_duckdb_suggested.test new file mode 100644 index 000000000000..9c228054232c --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_suggested.test @@ -0,0 +1,26 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_suggested.test +# description: Test suggestions +# group: [read_duckdb] + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_suggested.db' AS suggested + +foreach table_name aaa bbb ccc ddd eee fff ggg hhh iii jjj kkk lll mmm nnn ooo ppp qqq rrr sss ttt uuu vvv + +statement ok +CREATE TABLE suggested.${table_name} AS SELECT 42 i + +endloop + +statement ok +DETACH suggested + +statement error +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_suggested.db', table_name='ffg') +---- +fff + +statement error +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_suggested.db', table_name='vvx') +---- +vvv diff --git a/test/sql/storage/read_duckdb/read_duckdb_top_n.test b/test/sql/storage/read_duckdb/read_duckdb_top_n.test new file mode 100644 index 000000000000..bf5e7dde8db0 --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_top_n.test @@ -0,0 +1,54 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_top_n.test +# group: [read_duckdb] + +# create a bunch of tables +loop i 0 5 + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_top_n${i}.db' AS rd + +statement ok +CREATE TABLE rd.tbl AS SELECT ${i} grp, i val FROM range(${i} * 1000, ${i} * 1000 + 1000) t(i) + +statement ok +DETACH rd + +endloop + +query II +FROM '__TEST_DIR__/read_duckdb_top_n*.db' ORDER BY val DESC LIMIT 5 +---- +4 4999 +4 4998 +4 4997 +4 4996 +4 4995 + + +query II +FROM '__TEST_DIR__/read_duckdb_top_n*.db' WHERE grp<=2 ORDER BY val DESC LIMIT 5 +---- +2 2999 +2 2998 +2 2997 +2 2996 +2 2995 + +query I +SELECT val FROM '__TEST_DIR__/read_duckdb_top_n*.db' WHERE grp<=2 ORDER BY val DESC LIMIT 5 +---- +2999 +2998 +2997 +2996 +2995 + +query I +SELECT SUM(val) FROM '__TEST_DIR__/read_duckdb_top_n*.db' WHERE grp%2=0 +---- +7498500 + +query I +SELECT SUM(val) FROM '__TEST_DIR__/read_duckdb_top_n*.db' WHERE grp::VARCHAR = '2' +---- +2499500 diff --git a/test/sql/storage/read_duckdb/read_duckdb_tpch.test_slow b/test/sql/storage/read_duckdb/read_duckdb_tpch.test_slow new file mode 100644 index 000000000000..e0f1c5c634ea --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_tpch.test_slow @@ -0,0 +1,78 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_tpch.test_slow +# group: [read_duckdb] + +require tpch + +statement ok +CALL dbgen(sf=1, suffix='_original'); + +foreach tbl lineitem nation orders supplier part partsupp region customer + +statement ok +ATTACH '__TEST_DIR__/${tbl}_sf1.db' AS ${tbl} + +statement ok +CREATE TABLE ${tbl}.${tbl} AS FROM ${tbl}_original + +statement ok +DETACH ${tbl} + +statement ok +CREATE VIEW ${tbl} AS FROM read_duckdb('__TEST_DIR__/${tbl}_sf1.db') + +endloop + +loop i 1 9 + +query I +PRAGMA tpch(${i}) +---- +:extension/tpch/dbgen/answers/sf1/q0${i}.csv + +endloop + +loop i 10 23 + +query I +PRAGMA tpch(${i}) +---- +:extension/tpch/dbgen/answers/sf1/q${i}.csv + +endloop + +# top-n +query IIIIIIII +SELECT o_orderkey, o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority FROM orders ORDER BY o_orderkey LIMIT 5; +---- +1 36901 O 173665.47 1996-01-02 5-LOW Clerk#000000951 0 +2 78002 O 46929.18 1996-12-01 1-URGENT Clerk#000000880 0 +3 123314 F 193846.25 1993-10-14 5-LOW Clerk#000000955 0 +4 136777 O 32151.78 1995-10-11 5-LOW Clerk#000000124 0 +5 44485 F 144659.20 1994-07-30 5-LOW Clerk#000000925 0 + +query IIIIIIII +SELECT o_orderkey, o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority FROM orders ORDER BY o_orderkey DESC LIMIT 5; +---- +6000000 110063 O 37625.29 1996-08-31 2-HIGH Clerk#000000411 0 +5999975 113398 F 63216.65 1993-07-25 1-URGENT Clerk#000000813 0 +5999974 55448 F 92750.90 1993-07-28 3-MEDIUM Clerk#000000776 0 +5999973 32071 O 68906.56 1997-07-13 4-NOT SPECIFIED Clerk#000000130 0 +5999972 143594 O 114856.68 1996-05-02 3-MEDIUM Clerk#000000536 0 + +query IIIIIIIIIIIIIII +SELECT l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode FROM lineitem ORDER BY l_shipdate, l_orderkey LIMIT 5; +---- +721220 177803 5355 2 19 35735.20 0.08 0.03 R F 1992-01-02 1992-02-04 1992-01-09 TAKE BACK RETURN SHIP +842980 188156 5711 4 5 6220.75 0.01 0.03 A F 1992-01-02 1992-03-20 1992-01-20 COLLECT COD REG AIR +904677 56678 1689 1 43 70290.81 0.08 0.01 R F 1992-01-02 1992-03-22 1992-01-14 COLLECT COD AIR +990147 154290 4291 1 6 8065.74 0.10 0.01 R F 1992-01-02 1992-03-01 1992-01-15 NONE REG AIR +1054181 16217 6218 1 45 50994.45 0.03 0.08 R F 1992-01-02 1992-02-05 1992-01-15 NONE MAIL + +query IIIIIIIIIIIIIII +SELECT l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode FROM lineitem ORDER BY l_orderkey DESC, l_shipdate DESC LIMIT 5; +---- +6000000 32255 2256 1 5 5936.25 0.04 0.03 N O 1996-11-02 1996-11-19 1996-12-01 TAKE BACK RETURN MAIL +6000000 96127 6128 2 28 31447.36 0.01 0.02 N O 1996-09-22 1996-10-01 1996-10-21 NONE AIR +5999975 37131 2138 3 18 19226.34 0.04 0.01 A F 1993-11-17 1993-08-28 1993-12-08 DELIVER IN PERSON FOB +5999975 6452 1453 2 7 9509.15 0.04 0.00 A F 1993-11-02 1993-09-23 1993-11-19 DELIVER IN PERSON SHIP +5999975 7272 2273 1 32 37736.64 0.07 0.01 R F 1993-10-07 1993-09-30 1993-10-21 COLLECT COD REG AIR diff --git a/test/sql/storage/read_duckdb/read_duckdb_transaction.test b/test/sql/storage/read_duckdb/read_duckdb_transaction.test new file mode 100644 index 000000000000..1eb67bc2fca3 --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_transaction.test @@ -0,0 +1,28 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_transaction.test +# description: Test using read_duckdb in a transaction +# group: [read_duckdb] + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_transaction.db' AS rd + +statement ok +CREATE TABLE rd.my_tbl AS SELECT 42 i + +statement ok +DETACH rd + +statement ok +BEGIN + +query I +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_transaction.db') +---- +42 + +query I +SELECT * FROM read_duckdb('__TEST_DIR__/read_duckdb_transaction.db') +---- +42 + +statement ok +COMMIT diff --git a/test/sql/storage/read_duckdb/read_duckdb_union_by_name.test b/test/sql/storage/read_duckdb/read_duckdb_union_by_name.test new file mode 100644 index 000000000000..aea0a749c10a --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_union_by_name.test @@ -0,0 +1,32 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_union_by_name.test +# description: Test read_duckdb with union by name +# group: [read_duckdb] + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_unionbyname1.db' AS rd + +statement ok +CREATE TABLE rd.my_tbl AS SELECT 100 i, 84 col1 + +statement ok +DETACH rd + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_unionbyname2.db' AS rd + +statement ok +CREATE TABLE rd.my_tbl AS SELECT 200 i, 84 col2 + +statement ok +DETACH rd + +query III +SELECT * FROM read_duckdb(['__TEST_DIR__/read_duckdb_unionbyname1.db', '__TEST_DIR__/read_duckdb_unionbyname2.db'], union_by_name=true); +---- +100 84 NULL +200 NULL 84 + +statement error +SELECT * FROM read_duckdb(['__TEST_DIR__/read_duckdb_unionbyname1.db', '__TEST_DIR__/read_duckdb_unionbyname2.db']); +---- +could not be found in file diff --git a/test/sql/storage/read_duckdb/read_duckdb_virtual_columns.test b/test/sql/storage/read_duckdb/read_duckdb_virtual_columns.test new file mode 100644 index 000000000000..2c84aa385dae --- /dev/null +++ b/test/sql/storage/read_duckdb/read_duckdb_virtual_columns.test @@ -0,0 +1,23 @@ +# name: test/sql/storage/read_duckdb/read_duckdb_virtual_columns.test +# description: Test basic usage of read_duckdb +# group: [read_duckdb] + +foreach dbindex 1 2 + +statement ok +ATTACH '__TEST_DIR__/read_duckdb_virt${dbindex}.db' AS rd + +statement ok +CREATE TABLE rd.my_tbl AS SELECT 42 + i AS v FROM range(0, ${dbindex}) t(i) + +statement ok +DETACH rd + +endloop + +query III +SELECT rowid, (filename.replace('\', '/').split('/'))[3], * FROM read_duckdb('__TEST_DIR__/read_duckdb_virt[1-2].db') ORDER BY filename, rowid +---- +0 read_duckdb_virt1.db 42 +0 read_duckdb_virt2.db 42 +1 read_duckdb_virt2.db 43 diff --git a/test/sql/storage/update/wal_restart_update_insert.test b/test/sql/storage/update/wal_restart_update_insert.test new file mode 100644 index 000000000000..1c9e204c1976 --- /dev/null +++ b/test/sql/storage/update/wal_restart_update_insert.test @@ -0,0 +1,45 @@ +# name: test/sql/storage/update/wal_restart_update_insert.test +# description: Test WAL restart with updates and insertions +# group: [update] + +# load the DB from disk +load __TEST_DIR__/wal_restart_update_insert.test.db + +statement ok +CREATE TABLE test(i INTEGER PRIMARY KEY, j INTEGER); + +statement ok +INSERT INTO test SELECT r, r FROM range(2000) t(r); + +statement ok +CHECKPOINT + +statement ok +PRAGMA disable_checkpoint_on_shutdown + +statement ok +SET checkpoint_threshold='1TB' + +statement ok +INSERT INTO test SELECT r, r FROM range(2000,200000) t(r) + +statement ok +UPDATE test SET j=j+1 + +restart + +statement ok +PRAGMA disable_checkpoint_on_shutdown + +statement ok +SET checkpoint_threshold='1TB' + +statement ok +INSERT INTO test SELECT r, r FROM range(200000,400000) t(r) + +restart + +query I +select count(*) FROM test +---- +400000 diff --git a/test/sql/storage/wal/wal_promote_version.test b/test/sql/storage/wal/wal_promote_version.test new file mode 100644 index 000000000000..13626db43b2f --- /dev/null +++ b/test/sql/storage/wal/wal_promote_version.test @@ -0,0 +1,36 @@ +# name: test/sql/storage/wal/wal_promote_version.test +# description: WAL promote version +# group: [wal] + +statement ok +PRAGMA disable_checkpoint_on_shutdown; + +statement ok +SET checkpoint_threshold='1TB' + +statement ok +ATTACH '__TEST_DIR__/wal_promote.db'; + +statement ok +CREATE TABLE wal_promote.T AS (FROM range(10)); + +statement ok +DETACH wal_promote; + +statement ok +ATTACH '__TEST_DIR__/wal_promote.db' (STORAGE_VERSION 'latest'); + +statement ok +INSERT INTO wal_promote.T VALUES (42); + +statement ok +DETACH wal_promote; + +statement ok +ATTACH '__TEST_DIR__/wal_promote.db' (STORAGE_VERSION 'latest'); + +statement ok +INSERT INTO wal_promote.T VALUES (42); + +statement ok +DETACH wal_promote; diff --git a/test/sql/storage_version/storage_version.dbtest.duckdb b/test/sql/storage_version/storage_version.dbtest.duckdb new file mode 100644 index 000000000000..dd796961dbd9 Binary files /dev/null and b/test/sql/storage_version/storage_version.dbtest.duckdb differ diff --git a/test/sql/subquery/any_all/test_any_all.test b/test/sql/subquery/any_all/test_any_all.test index 3f40cbf31c83..0e5699d3d87c 100644 --- a/test/sql/subquery/any_all/test_any_all.test +++ b/test/sql/subquery/any_all/test_any_all.test @@ -84,9 +84,9 @@ NULL statement error SELECT 2 ^ ANY(SELECT * FROM integers) ---- -comparisons +Parser Error: ANY and ALL operators require one of =,<>,>,<,>=,<= comparisons! statement error SELECT 2 ^ ANY([1, 2, 3]) ---- -Unsupported comparison +Parser Error: Unsupported comparison "^" for ANY/ALL subquery \ No newline at end of file diff --git a/test/sql/subquery/complex/correlated_internal_issue_5975.test b/test/sql/subquery/complex/correlated_internal_issue_5975.test new file mode 100644 index 000000000000..6b77afbee9b5 --- /dev/null +++ b/test/sql/subquery/complex/correlated_internal_issue_5975.test @@ -0,0 +1,179 @@ +# name: test/sql/subquery/complex/correlated_internal_issue_5975.test +# description: Test internal issue 5975 - INTERNAL Error: Failed to bind column reference "" +# group: [complex] + +statement ok +CREATE TABLE my_logs ( + featherEventId UUID, + "duckInfo.gooseEmail" VARCHAR, + "duckInfo.gooseSubject" VARCHAR +); + +statement ok +CREATE TYPE MY_ENUM AS ENUM ( + 'EnumField1', + 'EnumField2', + 'EnumField3', + 'EnumField4', + 'EnumField5', + 'EnumField6', + 'EnumField7', + 'EnumField8' +); + +statement ok +CREATE OR REPLACE MACRO swan_MY_ENUM (sa) AS ( + WITH sa_parts AS ( + SELECT STRING_SPLIT(sa, '@') AS emailParts + ) + SELECT 'EnumField2'::MY_ENUM + FROM sa_parts +); + +statement ok +CREATE OR REPLACE MACRO swan_email_info (duckEmail) AS ( + SELECT + CASE + WHEN ENDS_WITH(duckEmail, 'duckdblabs.com') THEN STRUCT_PACK( + subject := 'serviceAccount:' || duckEmail, + type := swan_MY_ENUM (duckEmail) + ) + WHEN duckEmail = 'my@duckdblabs.com' + OR duckEmail = 'EnumField8' THEN STRUCT_PACK( + subject := 'EnumField8', + type := 'EnumField8'::MY_ENUM + ) + WHEN REGEXP_MATCHES(duckEmail, '[\w-.+]+@(([\w-]+).)+[\w-]{2,4}') THEN STRUCT_PACK( + subject := 'user:' || duckEmail, + type := 'EnumField1'::MY_ENUM + ) + END AS duckInfo +); + +statement ok +CREATE OR REPLACE MACRO swan_subject_info (duckSubject) AS ( + WITH + subjectComponents AS ( + SELECT + duckSubject AS subject, + STRING_SPLIT(duckSubject, ':') AS parts + ) + SELECT + CASE + WHEN parts[1] = 'EnumField2' THEN STRUCT_PACK( + subject := subject, + type := swan_MY_ENUM (parts[2]) + ) + WHEN parts[1] = 'EnumField1' THEN STRUCT_PACK(subject := subject, type := 'EnumField1'::MY_ENUM) + WHEN REGEXP_MATCHES( + subject, + 'duckdb.org' + ) THEN STRUCT_PACK( + subject := subject, + type := 'EnumField6'::MY_ENUM + ) + WHEN NOT REGEXP_FULL_MATCH(subject, '.+@.+\..+') THEN STRUCT_PACK( + subject := subject, + type := 'EnumField7'::MY_ENUM + ) + END AS duckInfo + FROM + subjectComponents +); + +statement ok +WITH + duck_info AS ( + SELECT + featherEventId, + "duckInfo.gooseEmail" AS email, + "duckInfo.gooseSubject" AS subject, + HASH('email' || "duckInfo.gooseEmail") AS emailHash, + HASH( + 'subject' || "duckInfo.gooseSubject" + ) AS subjectHash, + FROM + my_logs + ), + unique_emails AS ( + SELECT + email, + emailHash + FROM + duck_info + WHERE + email IS NOT NULL + GROUP BY + 1, + 2 + ), + email_info AS ( + SELECT + emailHash AS swanHash, + swan_email_info (email) AS swanInfo + FROM + unique_emails + ), + unique_subjects AS ( + SELECT + subject, + subjectHash + FROM + duck_info + WHERE + subject IS NOT NULL + GROUP BY + 1, + 2 + ), + subject_info AS ( + SELECT + subjectHash AS swanHash, + swan_subject_info (subject) AS swanInfo + FROM + unique_subjects + ), + swan_info AS ( + SELECT + * + FROM + email_info + UNION ALL + SELECT + * + FROM + subject_info + ), + goose_id AS ( + SELECT + featherEventId, + CASE + WHEN subject IS NOT NULL THEN subjectHash + WHEN email IS NOT NULL THEN emailHash + END AS swanHash, + FROM + duck_info + ), + normalized_ids AS ( + SELECT + featherEventId, + swanInfo.subject AS featherIdentityId, + swanInfo.type AS featherIdentityType, + FROM + goose_id + LEFT JOIN swan_info USING (swanHash) + ), + chicken_info AS ( + SELECT + featherEventId, + featherIdentityId, + featherIdentityType, + FROM + normalized_ids + ) +SELECT + featherEventId, + featherIdentityId, + featherIdentityType, +FROM + chicken_info; \ No newline at end of file diff --git a/test/sql/subquery/scalar/test_correlated_set_op.test b/test/sql/subquery/scalar/test_correlated_set_op.test new file mode 100644 index 000000000000..0b90f1becfa6 --- /dev/null +++ b/test/sql/subquery/scalar/test_correlated_set_op.test @@ -0,0 +1,37 @@ +# name: test/sql/subquery/scalar/test_correlated_set_op.test +# description: Test correlated subqueries with set operations +# group: [scalar] + +statement ok +SET default_null_order='nulls_first'; + +statement ok +CREATE TABLE integers(i INTEGER); + +statement ok +INSERT INTO integers VALUES (1), (2), (3), (NULL); + +# scalar select with correlation in different UNION ALL branches +query II +SELECT i, (SELECT SUM(x) FROM (SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT i1.i) t(x)) FROM integers i1 ORDER BY i; +---- +NULL 3 +1 4 +2 5 +3 6 + +query II +SELECT i, (SELECT SUM(x) FROM (SELECT i1.i UNION ALL SELECT 2 UNION ALL SELECT 1) t(x)) FROM integers i1 ORDER BY i; +---- +NULL 3 +1 4 +2 5 +3 6 + +query II +SELECT i, (SELECT SUM(x) FROM (SELECT 2 UNION ALL SELECT i1.i UNION ALL SELECT 1) t(x)) FROM integers i1 ORDER BY i; +---- +NULL 3 +1 4 +2 5 +3 6 diff --git a/test/sql/transactions/delete_and_drop_in_same_transaction.test b/test/sql/transactions/delete_and_drop_in_same_transaction.test new file mode 100644 index 000000000000..2090192edcf7 --- /dev/null +++ b/test/sql/transactions/delete_and_drop_in_same_transaction.test @@ -0,0 +1,23 @@ +# name: test/sql/transactions/delete_and_drop_in_same_transaction.test +# group: [transactions] + +statement ok +CREATE OR REPLACE TABLE SampleTable AS +SELECT DISTINCT id +FROM (VALUES + ('one'), + ('two'), + ('three') +) AS t(id); + +statement ok +BEGIN TRANSACTION; + +statement ok +DELETE FROM sampletable; + +statement ok +DROP TABLE SampleTable; + +statement ok +COMMIT TRANSACTION; diff --git a/test/sql/transactions/test_transaction_abort.test b/test/sql/transactions/test_transaction_abort.test index 5639f9a8c382..517a5fa8134f 100644 --- a/test/sql/transactions/test_transaction_abort.test +++ b/test/sql/transactions/test_transaction_abort.test @@ -20,6 +20,7 @@ BEGIN TRANSACTION statement error SELEC 42 ---- +Parser Error: syntax error at or near "SELEC" query I SELECT 42 diff --git a/test/sql/types/bignum/test_bignum_sum.test b/test/sql/types/bignum/test_bignum_sum.test index ba124324c20d..e640fef564c1 100644 --- a/test/sql/types/bignum/test_bignum_sum.test +++ b/test/sql/types/bignum/test_bignum_sum.test @@ -253,6 +253,11 @@ SELECT 170141183460469231731687303715884105728::BIGNUM + 1::BIGNUM; ---- 170141183460469231731687303715884105729 +query I +select (170141183460469231731687::BIGNUM - 170141183460469231731686::BIGNUM) + 1::BIGNUM +---- +2 + # Even bigger bignum addition query I SELECT 340282366920938463463374607431768211456::BIGNUM + 340282366920938463463374607431768211456::BIGNUM; diff --git a/test/sql/types/enum/test_enum_schema.test b/test/sql/types/enum/test_enum_schema.test index 69db4ff98fac..b57de0331699 100644 --- a/test/sql/types/enum/test_enum_schema.test +++ b/test/sql/types/enum/test_enum_schema.test @@ -120,4 +120,31 @@ CREATE TYPE "foo".bar AS ENUM ('a', 'b'); statement ok CREATE TABLE foo.baz ( bar_col "foo".bar NOT NULL -); \ No newline at end of file +); + +statement ok +drop schema "foo" cascade; + +# Test schema-qualified enum types with arrays +# https://github.com/duckdb/duckdb/issues/17041 +statement ok +CREATE SCHEMA foo; + +statement ok +CREATE TYPE foo.bar AS ENUM ('a', 'b', 'c'); + +statement ok +CREATE TABLE foo.test ( + qualified_array foo.bar[] +); + +statement ok +INSERT INTO foo.test VALUES (['a', 'b']); + +query I +SELECT * FROM foo.test; +---- +[a, b] + +statement ok +DROP SCHEMA foo CASCADE; \ No newline at end of file diff --git a/test/sql/types/geo/geometry.test b/test/sql/types/geo/geometry.test new file mode 100644 index 000000000000..c991309eb0f9 --- /dev/null +++ b/test/sql/types/geo/geometry.test @@ -0,0 +1,123 @@ +# name: test/sql/types/geo/geometry.test +# group: [geo] + +statement ok +create table t1(id INT, g GEOMETRY); + +# Note: We print multipoints as POINTs, without extra parentheses, like PostGIS does +# But both syntaxes are accepted input + +statement ok +insert into t1 values + (1, 'POINT(0 1)'), + (2, 'LINESTRING(0 0, 1 1, 2 2)'), + (3, 'POLYGON((0 0, 4 0, 4 4, 0 4, 0 0))'), + (4, 'MULTIPOINT((1 1), (2 2), (3 3))'), + (5, 'MULTIPOINT(1 1, 2 2, 3 3)'), --alternative syntax + (6, 'MULTILINESTRING((0 0, 1 1), (2 2, 3 3))'), + (7, 'MULTIPOLYGON(((0 0, 4 0, 4 4, 0 4, 0 0)), ((5 5, 7 5, 7 7, 5 7, 5 5)))'), + (8, 'GEOMETRYCOLLECTION(POINT(1 1), LINESTRING(0 0, 1 1))'), + (9, NULL); + +query II +select id, g::VARCHAR from t1 order by id; +---- +1 POINT (0 1) +2 LINESTRING (0 0, 1 1, 2 2) +3 POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0)) +4 MULTIPOINT (1 1, 2 2, 3 3) +5 MULTIPOINT (1 1, 2 2, 3 3) +6 MULTILINESTRING ((0 0, 1 1), (2 2, 3 3)) +7 MULTIPOLYGON (((0 0, 4 0, 4 4, 0 4, 0 0)), ((5 5, 7 5, 7 7, 5 7, 5 5))) +8 GEOMETRYCOLLECTION (POINT (1 1), LINESTRING (0 0, 1 1)) +9 NULL + +# Handle empty geometries +statement ok +create table t2(id INT, g GEOMETRY); + +statement ok +insert into t2 values + (1, 'POINT EMPTY'), + (2, 'LINESTRING EMPTY'), + (3, 'POLYGON EMPTY'), + (4, 'MULTIPOINT EMPTY'), + (5, 'MULTILINESTRING EMPTY'), + (6, 'MULTIPOLYGON EMPTY'), + (7, 'GEOMETRYCOLLECTION EMPTY'), + +query II +select id, g::VARCHAR from t2 order by id; +---- +1 POINT EMPTY +2 LINESTRING EMPTY +3 POLYGON EMPTY +4 MULTIPOINT EMPTY +5 MULTILINESTRING EMPTY +6 MULTIPOLYGON EMPTY +7 GEOMETRYCOLLECTION EMPTY + +# Special case +query I +SELECT 'MULTIPOINT(EMPTY, 2 2, EMPTY)'::GEOMETRY::VARCHAR; +---- +MULTIPOINT (EMPTY, 2 2, EMPTY) + + +# Z/M/ZM coordinates +statement ok +create table t3(id INT, g GEOMETRY); + +statement ok +insert into t3 values + (1, 'POINT Z (1 2 3)'), + (2, 'POINT M (1 2 3)'), + (3, 'POINT ZM (1 2 3 4)'), + (4, 'LINESTRING Z (0 0 0, 1 1 1, 2 2 2)'), + (5, 'LINESTRING M (0 0 0, 1 1 1, 2 2 2)'), + (6, 'LINESTRING ZM (0 0 0 0, 1 1 1 1, 2 2 2 2)'), + (7, 'POLYGON Z ((0 0 0, 4 0 0, 4 4 0, 0 4 0, 0 0 0))'), + (8, 'POLYGON M ((0 0 0, 4 0 0, 4 4 0, 0 4 0, 0 0 0))'), + (9, 'POLYGON ZM ((0 0 0 0, 4 0 0 0, 4 4 0 0, 0 4 0 0, 0 0 0 0))'), + (10, 'MULTIPOINT Z ((1 1 1), (2 2 2), (3 3 3))'), + (11, 'MULTIPOINT M ((1 1 1), (2 2 2), (3 3 3))'), + (12, 'MULTIPOINT ZM ((1 1 1 1), (2 2 2 2), (3 3 3 3))'), + (13, 'MULTILINESTRING Z ((0 0 0, 1 1 1), (2 2 2, 3 3 3))'), + (14, 'MULTILINESTRING M ((0 0 0, 1 1 1), (2 2 2, 3 3 3))'), + (15, 'MULTILINESTRING ZM ((0 0 0 0, 1 1 1 1), (2 2 2 2, 3 3 3 3))'), + (16, 'MULTIPOLYGON Z (((0 0 0,4 0 0,4 4 0,0 4 0,0 0 0)),((5 5 5,7 5 5,7 7 5,5 7 5,5 5 5)))'), + (17, 'MULTIPOLYGON M (((0 0 0,4 0 0,4 4 0,0 4 0,0 0 0)),((5 5 5,7 5 5,7 7 5,5 7 5,5 5 5)))'), + (18, 'MULTIPOLYGON ZM (((0 0 0 0,4 0 0 0,4 4 0 0,0 4 0 0,0 0 0 0)),((5 5 5 5,7 5 5 5,7 7 5 5,5 7 5 5,5 5 5 5)))'), + (19, 'GEOMETRYCOLLECTION Z (POINT Z (1 1 1), LINESTRING Z (0 0 0, 1 1 1))'), + (20, 'GEOMETRYCOLLECTION M (POINT M (1 1 1), LINESTRING M (0 0 0, 1 1 1))'), + (21, 'GEOMETRYCOLLECTION ZM (POINT ZM (1 1 1 1), LINESTRING ZM (0 0 0 0, 1 1 1 1))'); + +query II +select id, g::VARCHAR from t3 order by id; +---- +1 POINT Z (1 2 3) +2 POINT M (1 2 3) +3 POINT ZM (1 2 3 4) +4 LINESTRING Z (0 0 0, 1 1 1, 2 2 2) +5 LINESTRING M (0 0 0, 1 1 1, 2 2 2) +6 LINESTRING ZM (0 0 0 0, 1 1 1 1, 2 2 2 2) +7 POLYGON Z ((0 0 0, 4 0 0, 4 4 0, 0 4 0, 0 0 0)) +8 POLYGON M ((0 0 0, 4 0 0, 4 4 0, 0 4 0, 0 0 0)) +9 POLYGON ZM ((0 0 0 0, 4 0 0 0, 4 4 0 0, 0 4 0 0, 0 0 0 0)) +10 MULTIPOINT Z (1 1 1, 2 2 2, 3 3 3) +11 MULTIPOINT M (1 1 1, 2 2 2, 3 3 3) +12 MULTIPOINT ZM (1 1 1 1, 2 2 2 2, 3 3 3 3) +13 MULTILINESTRING Z ((0 0 0, 1 1 1), (2 2 2, 3 3 3)) +14 MULTILINESTRING M ((0 0 0, 1 1 1), (2 2 2, 3 3 3)) +15 MULTILINESTRING ZM ((0 0 0 0, 1 1 1 1), (2 2 2 2, 3 3 3 3)) +16 MULTIPOLYGON Z (((0 0 0, 4 0 0, 4 4 0, 0 4 0, 0 0 0)), ((5 5 5, 7 5 5, 7 7 5, 5 7 5, 5 5 5))) +17 MULTIPOLYGON M (((0 0 0, 4 0 0, 4 4 0, 0 4 0, 0 0 0)), ((5 5 5, 7 5 5, 7 7 5, 5 7 5, 5 5 5))) +18 MULTIPOLYGON ZM (((0 0 0 0, 4 0 0 0, 4 4 0 0, 0 4 0 0, 0 0 0 0)), ((5 5 5 5, 7 5 5 5, 7 7 5 5, 5 7 5 5, 5 5 5 5))) +19 GEOMETRYCOLLECTION Z (POINT Z (1 1 1), LINESTRING Z (0 0 0, 1 1 1)) +20 GEOMETRYCOLLECTION M (POINT M (1 1 1), LINESTRING M (0 0 0, 1 1 1)) +21 GEOMETRYCOLLECTION ZM (POINT ZM (1 1 1 1), LINESTRING ZM (0 0 0 0, 1 1 1 1)) + +statement error +select 'GEOMETRYCOLLECTION Z (POINT Z (1 1 2), LINESTRING (0 0, 1 1))'::GEOMETRY; +---- +Invalid Input Error: Geometry has inconsistent Z/M dimensions \ No newline at end of file diff --git a/test/sql/types/geo/geometry_stats.test b/test/sql/types/geo/geometry_stats.test new file mode 100644 index 000000000000..036e9c8da3d5 --- /dev/null +++ b/test/sql/types/geo/geometry_stats.test @@ -0,0 +1,61 @@ +# name: test/sql/types/geo/geometry_stats.test +# group: [geo] + +# Stats are not preserved when serialized in the stats() function, so this doesnt work +require no_alternative_verify + +statement ok +create table t1(id INT, g GEOMETRY); + +statement ok +insert into t1 values (0, 'POINT(0 0)'::GEOMETRY); + +query I +select stats(g) from t1 limit 1; +---- +[Extent: [X: [0.000000, 0.000000], Y: [0.000000, 0.000000], Z: [inf, -inf], M: [inf, -inf]], Types: [point]][Has Null: false, Has No Null: true][Approx Unique: 1] + +statement ok +insert into t1 values (1, 'POINT(-2 2)'::GEOMETRY); + +query I +select stats(g) from t1 limit 1; +---- +[Extent: [X: [-2.000000, 0.000000], Y: [0.000000, 2.000000], Z: [inf, -inf], M: [inf, -inf]], Types: [point]][Has Null: false, Has No Null: true][Approx Unique: 2] + +statement ok +insert into t1 values (3, 'POINT(2 -2)'::GEOMETRY); + +query I +select stats(g) from t1 limit 1; +---- +[Extent: [X: [-2.000000, 2.000000], Y: [-2.000000, 2.000000], Z: [inf, -inf], M: [inf, -inf]], Types: [point]][Has Null: false, Has No Null: true][Approx Unique: 3] + +# Insert Z-Value Geometry +statement ok +insert into t1 values (4, 'LINESTRING Z (0 0 0, 1 1 1, 2 2 2)'::GEOMETRY); + +query I +select stats(g) from t1 limit 1; +---- +[Extent: [X: [-2.000000, 2.000000], Y: [-2.000000, 2.000000], Z: [0.000000, 2.000000], M: [inf, -inf]], Types: [point, linestring_z]][Has Null: false, Has No Null: true][Approx Unique: 4] + +# Insert M-Value Geometry +statement ok +insert into t1 values (5, 'POLYGON M ((0 0 2, 4 0 2, 4 4 2, 0 4 2, 0 0 2))'::GEOMETRY); + +query I +select stats(g) from t1 limit 1; +---- +[Extent: [X: [-2.000000, 4.000000], Y: [-2.000000, 4.000000], Z: [0.000000, 2.000000], M: [2.000000, 2.000000]], Types: [point, linestring_z, polygon_m]][Has Null: false, Has No Null: true][Approx Unique: 5] + +# Insert ZM-Value Geometry +statement ok +insert into t1 values (6, 'MULTILINESTRING ZM ((0 0 -10 10, 1 1 -10 10), (2 2 2 1, 3 3 3 1))'::GEOMETRY); + +query I +select stats(g) from t1 limit 1; +---- +[Extent: [X: [-2.000000, 4.000000], Y: [-2.000000, 4.000000], Z: [-10.000000, 3.000000], M: [1.000000, 10.000000]], Types: [point, linestring_z, polygon_m, multilinestring_zm]][Has Null: false, Has No Null: true][Approx Unique: 6] + + diff --git a/test/sql/types/struct/create_type_struct.test b/test/sql/types/struct/create_type_struct.test new file mode 100644 index 000000000000..f9fb36faa248 --- /dev/null +++ b/test/sql/types/struct/create_type_struct.test @@ -0,0 +1,38 @@ +# name: test/sql/types/struct/create_type_struct.test +# description: Test create struct list with schema +# group: [struct] + +# Original issue: https://github.com/duckdb/duckdb/issues/18040 + +statement ok +CREATE SCHEMA app; + +statement ok +CREATE TYPE app.item AS STRUCT ( + id uuid, + code UINTEGER +); + +statement ok +CREATE TYPE app.product as STRUCT ( + id uuid, + items app.item[] +); + +statement ok +CREATE SCHEMA app2; + +statement ok +SET SEARCH_PATH TO app2; + +statement ok +CREATE TYPE app2.item AS STRUCT ( + id uuid, + code UINTEGER +); + +statement ok +CREATE TYPE app2.product as STRUCT ( + id uuid, + items item[] +); \ No newline at end of file diff --git a/test/sql/types/struct/update_empty_row.test b/test/sql/types/struct/update_empty_row.test index deb5f869e2c7..67b0ba9c89c9 100644 --- a/test/sql/types/struct/update_empty_row.test +++ b/test/sql/types/struct/update_empty_row.test @@ -5,4 +5,4 @@ statement error UPDATE t0 SET (c0) = ROW(); ---- -expected 1 values, got 0 +:.*Parser Error.*expected 1 values, got 0.* \ No newline at end of file diff --git a/test/sql/types/variant/variant_distinct.test b/test/sql/types/variant/variant_distinct.test new file mode 100644 index 000000000000..19964ce3dfcb --- /dev/null +++ b/test/sql/types/variant/variant_distinct.test @@ -0,0 +1,800 @@ +# name: test/sql/types/variant/variant_distinct.test +# description: Test VARIANT distinctions +# group: [variant] + +statement ok +PRAGMA enable_verification + +# Constant single integer column distinctions +query T +SELECT [1]::VARIANT IS NOT DISTINCT FROM [2] +---- +false + +query T +SELECT [1]::VARIANT IS NOT DISTINCT FROM [1] +---- +true + +query T +SELECT NULL IS NOT DISTINCT FROM [1]::VARIANT +---- +false + +query T +SELECT [1] IS NOT DISTINCT FROM NULL::VARIANT +---- +false + +query T +SELECT [1]::VARIANT IS DISTINCT FROM [2] +---- +true + +query T +SELECT [1]::VARIANT IS DISTINCT FROM [1] +---- +false + +query T +SELECT NULL::VARIANT IS DISTINCT FROM [1] +---- +true + +query T +SELECT [1]::VARIANT IS DISTINCT FROM NULL +---- +true + +statement ok +CREATE VIEW list_int1 AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + ([1], [1]), + ([1], [2]), + ([2], [1]), + (NULL, [1]), + ([2], NULL), + (NULL, NULL) + ) tbl(l, r); + +query T +SELECT l IS NOT DISTINCT FROM r FROM list_int1 +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM list_int1 +---- +false +true +true +true +true +false + +# Constant multiple integer column distinctions + +query T +SELECT [1]::VARIANT IS NOT DISTINCT FROM [1, 2] +---- +false + +query T +SELECT [1]::VARIANT IS NOT DISTINCT FROM [1] +---- +true + +query T +SELECT NULL::VARIANT IS NOT DISTINCT FROM [1] +---- +false + +query T +SELECT [1] IS NOT DISTINCT FROM NULL::VARIANT +---- +false + +query T +SELECT [1]::VARIANT IS DISTINCT FROM [1, 2] +---- +true + +query T +SELECT [1]::VARIANT IS DISTINCT FROM [1] +---- +false + +query T +SELECT NULL::VARIANT IS DISTINCT FROM [1] +---- +true + +query T +SELECT [1] IS DISTINCT FROM NULL::VARIANT +---- +true + +statement ok +CREATE VIEW list_int AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + ([1], [1]), + ([1], [1, 2]), + ([1, 2], [1]), + (NULL, [1]), + ([1, 2], NULL), + (NULL, NULL) + ) tbl(l, r); + +query T +SELECT l IS NOT DISTINCT FROM r FROM list_int +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM list_int +---- +false +true +true +true +true +false + +# Constant empty integer column distinctions + +query T +SELECT []::VARIANT IS NOT DISTINCT FROM [1, 2] +---- +false + +query T +SELECT []::VARIANT IS NOT DISTINCT FROM [] +---- +true + +query T +SELECT NULL::VARIANT IS NOT DISTINCT FROM [] +---- +false + +query T +SELECT []::VARIANT IS NOT DISTINCT FROM NULL +---- +false + +query T +SELECT []::VARIANT IS DISTINCT FROM [1, 2] +---- +true + +query T +SELECT []::VARIANT IS DISTINCT FROM [] +---- +false + +query T +SELECT NULL::VARIANT IS DISTINCT FROM [] +---- +true + +query T +SELECT []::VARIANT IS DISTINCT FROM NULL +---- +true + +statement ok +CREATE VIEW list_int_empty AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + ([], []), + ([], [1, 2]), + ([1, 2], []), + (NULL, []), + ([1, 2], NULL), + (NULL, NULL) + ) tbl(l, r); + +query T +SELECT l IS NOT DISTINCT FROM r FROM list_int_empty +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM list_int_empty +---- +false +true +true +true +true +false + +# List of strings +query T +SELECT ['duck']::VARIANT IS NOT DISTINCT FROM ['duck', 'goose'] +---- +false + +query T +SELECT ['duck']::VARIANT IS NOT DISTINCT FROM ['duck'] +---- +true + +query T +SELECT NULL::VARIANT IS NOT DISTINCT FROM ['duck'] +---- +false + +query T +SELECT ['duck']::VARIANT IS NOT DISTINCT FROM NULL +---- +false + +query T +SELECT ['duck']::VARIANT IS DISTINCT FROM ['duck', 'goose'] +---- +true + +query T +SELECT ['duck']::VARIANT IS DISTINCT FROM ['duck'] +---- +false + +query T +SELECT NULL::VARIANT IS DISTINCT FROM ['duck'] +---- +true + +query T +SELECT ['duck']::VARIANT IS DISTINCT FROM NULL +---- +true + +statement ok +CREATE VIEW list_str AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + (['duck'], ['duck']), + (['duck'], ['duck', 'goose']), + (['duck', 'goose'], ['duck']), + (NULL, ['duck']), + (['duck', 'goose'], NULL), + (NULL, NULL) + ) tbl(l, r); + +query T +SELECT l IS NOT DISTINCT FROM r FROM list_str +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM list_str +---- +false +true +true +true +true +false + +# List of structs + +query T +SELECT [{'x': 'duck', 'y': 1}]::VARIANT IS NOT DISTINCT FROM [{'x': 'duck', 'y': 1}, {'x': 'goose', 'y': 2}] +---- +false + +query T +SELECT [{'x': 'duck', 'y': 1}]::VARIANT IS NOT DISTINCT FROM [{'x': 'duck', 'y': 1}] +---- +true + +query T +SELECT NULL::VARIANT IS NOT DISTINCT FROM [{'x': 'duck', 'y': 1}] +---- +false + +query T +SELECT [{'x': 'duck', 'y': 1}]::VARIANT IS NOT DISTINCT FROM NULL +---- +false + +query T +SELECT [{'x': 'duck', 'y': 1}]::VARIANT IS DISTINCT FROM [{'x': 'duck', 'y': 1}, {'x': 'goose', 'y': 2}] +---- +true + +query T +SELECT [{'x': 'duck', 'y': 1}]::VARIANT IS DISTINCT FROM [{'x': 'duck', 'y': 1}] +---- +false + +query T +SELECT NULL::VARIANT IS DISTINCT FROM [{'x': 'duck', 'y': 1}] +---- +true + +query T +SELECT [{'x': 'duck', 'y': 1}]::VARIANT IS DISTINCT FROM NULL +---- +true + +statement ok +CREATE VIEW list_of_struct AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + ([{'x': 'duck', 'y': 1}], [{'x': 'duck', 'y': 1}]), + ([{'x': 'duck', 'y': 1}], [{'x': 'duck', 'y': 1}, {'x': 'goose', 'y': 2}]), + ([{'x': 'duck', 'y': 1}, {'x': 'goose', 'y': 2}], [{'x': 'duck', 'y': 1}]), + (NULL, [{'x': 'duck', 'y': 1}]), + ([{'x': 'duck', 'y': 1}, {'x': 'goose', 'y': 2}], NULL), + (NULL, NULL) + ) tbl(l, r); + + +query T +SELECT l IS NOT DISTINCT FROM r FROM list_of_struct +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM list_of_struct +---- +false +true +true +true +true +false + +# Filter by constant +query T +select CASE WHEN a::INT32::VARIANT < 4 THEN [a,a+1,a+2] ELSE NULL END IS NOT DISTINCT FROM [1::INT32,2,3] from range(5) tbl(a); +---- +false +true +false +false +false + +query T +select CASE WHEN a::INT32::VARIANT < 4 THEN [a,a+1,a+2] ELSE NULL END IS DISTINCT FROM [1::INT32,2,3] from range(5) tbl(a); +---- +true +false +true +true +true + +foreach type + +# Constant single integer column distinct +query T +SELECT {'x': 1::${type}}::VARIANT IS NOT DISTINCT FROM {'x': 2::${type}} +---- +false + +query T +SELECT {'x': 1::${type}}::VARIANT IS NOT DISTINCT FROM {'x': 1::${type}} +---- +true + +query T +SELECT NULL::VARIANT IS NOT DISTINCT FROM {'x': 1::${type}} +---- +false + +query T +SELECT {'x': 1::${type}}::VARIANT IS DISTINCT FROM {'x': 2::${type}} +---- +true + +query T +SELECT {'x': 1::${type}}::VARIANT IS DISTINCT FROM {'x': 1::${type}} +---- +false + +query T +SELECT {'x': 1::${type}}::VARIANT IS DISTINCT FROM NULL +---- +true + +statement ok +CREATE OR REPLACE VIEW struct_int AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + ({'x': 1::${type}}, {'x': 1::${type}}), + ({'x': 1::${type}}, {'x': 2::${type}}), + ({'x': 2::${type}}, {'x': 1::${type}}), + (NULL, {'x': 1::${type}}), + ({'x': 2::${type}}, NULL), + (NULL, NULL) + ) tbl(l, r); + +query T +SELECT l IS NOT DISTINCT FROM r FROM struct_int +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM struct_int +---- +false +true +true +true +true +false + +endloop + +# Constant single string column distinct +query T +SELECT {'x': 'duck'}::VARIANT IS NOT DISTINCT FROM {'x': 'goose'} +---- +false + +query T +SELECT {'x': 'duck'}::VARIANT IS NOT DISTINCT FROM {'x': 'duck'} +---- +true + + +query T +SELECT {'x': 'duck'}::VARIANT IS NOT DISTINCT FROM NULL +---- +false + +query T +SELECT NULL::VARIANT IS NOT DISTINCT FROM {'x': 'duck'} +---- +false + +query T +SELECT {'x': 'duck'}::VARIANT IS DISTINCT FROM {'x': 'goose'} +---- +true + +query T +SELECT {'x': 'duck'}::VARIANT IS DISTINCT FROM {'x': 'duck'} +---- +false + +query T +SELECT {'x': 'duck'}::VARIANT IS DISTINCT FROM NULL +---- +true + +query T +SELECT NULL::VARIANT IS DISTINCT FROM {'x': 'duck'} +---- +true + +statement ok +CREATE VIEW struct_str AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + ({'x': 'duck'}, {'x': 'duck'}), + ({'x': 'duck'}, {'x': 'goose'}), + ({'x': 'goose'}, {'x': 'duck'}), + (NULL, {'x': 'duck'}), + ({'x': 'goose'}, NULL), + (NULL, NULL) + ) tbl(l, r); + +query T +SELECT l IS NOT DISTINCT FROM r FROM struct_str +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM struct_str +---- +false +true +true +true +true +false + +# Constant string, integer column distinct +query T +SELECT {'x': 'duck', 'y': 1}::VARIANT IS NOT DISTINCT FROM {'x': 'goose', 'y': 2} +---- +false + +query T +SELECT {'x': 'duck', 'y': 1}::VARIANT IS NOT DISTINCT FROM {'x': 'duck', 'y': 1} +---- +true + +query T +SELECT NULL::VARIANT IS NOT DISTINCT FROM {'x': 'duck', 'y': 1} +---- +false + +query T +SELECT {'x': 'duck', 'y': 1}::VARIANT IS NOT DISTINCT FROM NULL +---- +false + +query T +SELECT {'x': 'duck', 'y': 1}::VARIANT IS DISTINCT FROM {'x': 'goose', 'y': 2} +---- +true + +query T +SELECT {'x': 'duck', 'y': 1}::VARIANT IS DISTINCT FROM {'x': 'duck', 'y': 1} +---- +false + +query T +SELECT NULL::VARIANT IS DISTINCT FROM {'x': 'duck', 'y': 1} +---- +true + +query T +SELECT {'x': 'duck', 'y': 1}::VARIANT IS DISTINCT FROM NULL +---- +true + +statement ok +CREATE VIEW struct_str_int AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + ({'x': 'duck', 'y': 1}, {'x': 'duck', 'y': 1}), + ({'x': 'duck', 'y': 1}, {'x': 'goose', 'y': 2}), + ({'x': 'goose', 'y': 2}, {'x': 'duck', 'y': 1}), + (NULL, {'x': 'duck', 'y': 1}), + ({'x': 'goose', 'y': 2}, NULL), + (NULL, NULL) + ) tbl(l, r); + +query T +SELECT l IS NOT DISTINCT FROM r FROM struct_str_int +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM struct_str_int +---- +false +true +true +true +true +false + +# Nested structs + +query T +SELECT {'x': 1, 'y': {'a': 'duck', 'b': 1.5}}::VARIANT IS NOT DISTINCT FROM {'x': 2, 'y': {'a': 'goose', 'b': 2.5}} +---- +false + +query T +SELECT {'x': 1, 'y': {'a': 'duck', 'b': 1.5}}::VARIANT IS NOT DISTINCT FROM {'x': 1, 'y': {'a': 'duck', 'b': 1.5}} +---- +true + +query T +SELECT NULL::VARIANT IS NOT DISTINCT FROM {'x': 1, 'y': {'a': 'duck', 'b': 1.5}} +---- +false + +query T +SELECT {'x': 1, 'y': {'a': 'duck', 'b': 1.5}}::VARIANT IS NOT DISTINCT FROM NULL +---- +false + +query T +SELECT {'x': 1, 'y': {'a': 'duck', 'b': 1.5}}::VARIANT IS DISTINCT FROM {'x': 2, 'y': {'a': 'goose', 'b': 2.5}} +---- +true + +query T +SELECT {'x': 1, 'y': {'a': 'duck', 'b': 1.5}}::VARIANT IS DISTINCT FROM {'x': 1, 'y': {'a': 'duck', 'b': 1.5}} +---- +false + +query T +SELECT NULL::VARIANT IS DISTINCT FROM {'x': 1, 'y': {'a': 'duck', 'b': 1.5}} +---- +true + +query T +SELECT {'x': 1, 'y': {'a': 'duck', 'b': 1.5}} IS DISTINCT FROM NULL::VARIANT +---- +true + +statement ok +CREATE VIEW struct_nested AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + ({'x': 1, 'y': {'a': 'duck', 'b': 1.5}}, {'x': 1, 'y': {'a': 'duck', 'b': 1.5}}), + ({'x': 1, 'y': {'a': 'duck', 'b': 1.5}}, {'x': 2, 'y': {'a': 'goose', 'b': 2.5}}), + ({'x': 2, 'y': {'a': 'goose', 'b': 2.5}}, {'x': 1, 'y': {'a': 'duck', 'b': 1.5}}), + (NULL, {'x': 1, 'y': {'a': 'duck', 'b': 1.5}}), + ({'x': 2, 'y': {'a': 'goose', 'b': 2.5}}, NULL), + (NULL, NULL) + ) tbl(l, r); + +query T +SELECT l IS NOT DISTINCT FROM r FROM struct_nested +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM struct_nested +---- +false +true +true +true +true +false + +# List nested inside struct +query T +SELECT {'x': 1, 'y': ['duck', 'somateria']}::VARIANT IS NOT DISTINCT FROM {'x': 2, 'y': ['goose']} +---- +false + +query T +SELECT {'x': 1, 'y': ['duck', 'somateria']}::VARIANT IS NOT DISTINCT FROM {'x': 1, 'y': ['duck', 'somateria']} +---- +true + +query T +SELECT NULL::VARIANT IS NOT DISTINCT FROM {'x': 1, 'y': ['duck', 'somateria']} +---- +false + +query T +SELECT {'x': 1, 'y': ['duck', 'somateria']}::VARIANT IS NOT DISTINCT FROM NULL +---- +false + +query T +SELECT {'x': 1, 'y': ['duck', 'somateria']}::VARIANT IS DISTINCT FROM {'x': 2, 'y': ['goose']} +---- +true + +query T +SELECT {'x': 1, 'y': ['duck', 'somateria']}::VARIANT IS DISTINCT FROM {'x': 1, 'y': ['duck', 'somateria']} +---- +false + +query T +SELECT NULL::VARIANT IS DISTINCT FROM {'x': 1, 'y': ['duck', 'somateria']} +---- +true + +query T +SELECT {'x': 1, 'y': ['duck', 'somateria']}::VARIANT IS DISTINCT FROM NULL +---- +true + +statement ok +CREATE VIEW list_in_struct AS SELECT COLUMNS(*)::VARIANT FROM (VALUES + ({'x': 1, 'y': ['duck', 'somateria']}, {'x': 1, 'y': ['duck', 'somateria']}), + ({'x': 1, 'y': ['duck', 'somateria']}, {'x': 2, 'y': ['goose']}), + ({'x': 2, 'y': ['goose']}, {'x': 1, 'y': ['duck', 'somateria']}), + (NULL, {'x': 1, 'y': ['duck', 'somateria']}), + ({'x': 2, 'y': ['goose']}, NULL), + (NULL, NULL) + ) tbl(l, r); + +query T +SELECT l IS NOT DISTINCT FROM r FROM list_in_struct +---- +true +false +false +false +false +true + +query T +SELECT l IS DISTINCT FROM r FROM list_in_struct +---- +false +true +true +true +true +false + +# Filter by constant +query T +WITH cte as ( + select a::INT32 a from range(5) tbl(a) +) +select + CASE + WHEN a::VARIANT < 4 + THEN { + 'x': a, + 'y': a+1, + 'z': a+2 + }::VARIANT + ELSE NULL + END IS NOT DISTINCT FROM { + 'x': 1::INT32, + 'y': 2::INT32, + 'z': 3::INT32 + } +from cte; +---- +false +true +false +false +false + +query T +WITH cte as ( + select a::INT32 a from range(5) tbl(a) +) +select CASE WHEN a < 4 THEN {'x': a, 'z': a+2, 'y': a+1}::VARIANT ELSE NULL END IS DISTINCT FROM {'x': 1, 'y': 2, 'z': 3} +from cte; +---- +true +false +true +true +true + +# Without the collation that pushes 'variant_normalize', +# the 'variant_extract' (performed by the ['nested'] call) will break the comparison +query T +WITH cte as ( + select a::INT32 a from range(5) tbl(a) +) +select CASE WHEN a < 4 THEN ({ + 'nested': { + 'x': a, + 'z': a+2, + 'y': a+1 + } +}::VARIANT)['nested'] ELSE NULL END IS DISTINCT FROM { + 'x': 1, + 'y': 2, + 'z': 3 +} +from cte; +---- +true +false +true +true +true diff --git a/test/sql/update/update_error_suggestions.test b/test/sql/update/update_error_suggestions.test index a257ed21b575..c78eb5940f48 100644 --- a/test/sql/update/update_error_suggestions.test +++ b/test/sql/update/update_error_suggestions.test @@ -16,4 +16,4 @@ UPDATE tbl SET myco=42 statement error UPDATE tbl SET tbl.mycol=42 ---- -not supported +Parser Error: Qualified column names in UPDATE .. SET not supported diff --git a/test/sql/upsert/upsert_default.test b/test/sql/upsert/upsert_default.test index afb099bc949e..d52b1078f041 100644 --- a/test/sql/upsert/upsert_default.test +++ b/test/sql/upsert/upsert_default.test @@ -17,14 +17,14 @@ insert into tbl(b) VALUES (3), (5), (6); statement ok insert into tbl(b) VALUES (7), (3), (4) ON CONFLICT do update set c = 5, a = 10; -query III +query III rowsort select * from tbl; ---- 10 3 5 +5 4 10 5 5 10 5 6 10 5 7 10 -5 4 10 statement ok create table t (i int primary key, j int); diff --git a/test/sql/upsert/upsert_multiple_constraints.test b/test/sql/upsert/upsert_multiple_constraints.test new file mode 100644 index 000000000000..4b9a09ee9b54 --- /dev/null +++ b/test/sql/upsert/upsert_multiple_constraints.test @@ -0,0 +1,16 @@ +# name: test/sql/upsert/upsert_multiple_constraints.test +# description: Test upsert with a mix of primary / unique constraints +# group: [upsert] + +statement ok +CREATE TABLE IF NOT EXISTS tbl (id INTEGER PRIMARY KEY, uniq INTEGER, UNIQUE(uniq)); + +query I +INSERT OR IGNORE INTO tbl VALUES (1, 1), (2, 1); +---- +1 + +query I +SELECT uniq FROM tbl +---- +1 diff --git a/test/sql/vacuum/test_analyze.test b/test/sql/vacuum/test_analyze.test index 045db6c803a8..76e6de79db24 100644 --- a/test/sql/vacuum/test_analyze.test +++ b/test/sql/vacuum/test_analyze.test @@ -38,7 +38,7 @@ CREATE VIEW testview AS SELECT * FROM test; statement error ANALYZE testview; ---- -can only vacuum or analyze base tables +Can only vacuum or analyze base tables statement ok INSERT INTO test SELECT range % 5000, range % 5000 FROM range(10000); diff --git a/test/sql/window/test_evil_window.test b/test/sql/window/test_evil_window.test index 1baca0eef8de..39f43d4f4e32 100644 --- a/test/sql/window/test_evil_window.test +++ b/test/sql/window/test_evil_window.test @@ -77,3 +77,11 @@ personnel 100.000000 sales 34.246575 sales 100.000000 +statement ok +CREATE TABLE empty_unsorted(c0 VARCHAR); + +statement ok +SELECT * +FROM empty_unsorted +WHERE(NOT(false = ANY([]))) +ORDER BY(~((SUM(true) OVER() - SUM(true) OVER())::INT)) ASC; diff --git a/test/sql/window/test_window_exclude.test_slow b/test/sql/window/test_window_exclude.test_slow index 1af1f9900c67..36e81b59a333 100644 --- a/test/sql/window/test_window_exclude.test_slow +++ b/test/sql/window/test_window_exclude.test_slow @@ -877,7 +877,9 @@ NULL 5200 6000 statement error SELECT sum(i) OVER (EXCLUDE CURRENT ROW) FROM generate_series(1,10) AS _(i); ---- +Parser Error: syntax error at or near "CURRENT" statement error SELECT sum(i) OVER (ROWS UNBOUNDED PRECEDING EXCLUDE GROUPS) FROM generate_series(1,10) AS _(i); ---- +Parser Error: syntax error at or near "GROUPS" \ No newline at end of file diff --git a/test/sqlite/README.md b/test/sqlite/README.md new file mode 100644 index 000000000000..d1a5cd814a85 --- /dev/null +++ b/test/sqlite/README.md @@ -0,0 +1,42 @@ +# SQLLogic Test Runner + +## Origin + +Here you'll find source code originating from +[SQLite's SQLLogicTest](https://sqlite.org/sqllogictest/doc/trunk/about.wiki). +DuckDB has extended functionality in several ways, including several new expressions +(test_env, set/reset, tags). + +## Usage Notes + +### Environment: test_env and require-env + +Environment variables can be managed in 2 ways: `test_env` which allows variables to have defaults set, and `require-env` which is a select/skip predicate for a test file. + +For examples of `test_env` usage see the `duckdb/ducklake` extension tests. + +When a file `require-env FOO`, or `require-env FOO=bar` a test will only execute if FOO is set, or in the latter case, set to `bar`. + +### Tags: explicit and implicit + +SQL test files also support a `tags` attribute of the form: + +```text +tags optimization memory>=64GB +``` + +The tags are free-form, and can be used when executing tests for both selection and skipping, a la: + +```bash +build/release/test/unittest --skip-tag 'slow' --select-tag-set "['memory>=64GB', 'env[TEST_DATA]']" +``` + +Tags can be specified individually, or as a set (which is treated as an `AND` predicate). +Each specification is an `OR`, and selects are processed before skips. + +Additionally some implicit tags are computed when an SQL test file is parsed. +All `require-env` and `test_env` expressions will be added as tags of the form `env[VAR]`, and +`env[VAR]=VALUE` (when specified). + +For an extensive example of tag matching expectations, see the file +`test/sqlite/validate_tags_usage.sh` which unit tests these behaviors. diff --git a/test/sqlite/result_helper.cpp b/test/sqlite/result_helper.cpp index 608ff5aefb40..5f4675ab0f3e 100644 --- a/test/sqlite/result_helper.cpp +++ b/test/sqlite/result_helper.cpp @@ -309,7 +309,11 @@ bool TestResultHelper::CheckStatementResult(const Statement &statement, ExecuteC error = !error; } if (result.HasError() && !statement.expected_error.empty()) { - if (!StringUtil::Contains(result.GetError(), statement.expected_error)) { + // We run both comparions on purpose, we might move to only the second but might require some changes in + // tests + // This is due to some errors containing absolute paths, some relatives + if (!StringUtil::Contains(result.GetError(), statement.expected_error) && + !StringUtil::Contains(result.GetError(), runner.ReplaceKeywords(statement.expected_error))) { bool success = false; if (StringUtil::StartsWith(statement.expected_error, ":") || StringUtil::StartsWith(statement.expected_error, ":")) { @@ -487,7 +491,9 @@ bool TestResultHelper::CompareValues(SQLLogicTestLogger &logger, MaterializedQue Value lvalue, rvalue; bool error = false; // simple first test: compare string value directly - if (lvalue_str == rvalue_str) { + // We run both comparions on purpose, we might move to only the second but might require some changes in tests + // This is due to some results containing absolute paths, some relatives + if (lvalue_str == rvalue_str || lvalue_str == runner.ReplaceKeywords(rvalue_str)) { return true; } if (StringUtil::StartsWith(rvalue_str, ":") || StringUtil::StartsWith(rvalue_str, ":")) { diff --git a/test/sqlite/sqllogic_command.cpp b/test/sqlite/sqllogic_command.cpp index 774bed99af0a..4a5332442a5f 100644 --- a/test/sqlite/sqllogic_command.cpp +++ b/test/sqlite/sqllogic_command.cpp @@ -23,7 +23,7 @@ static void query_break(int line) { (void)line; } -static Connection *GetConnection(DuckDB &db, +static Connection *GetConnection(SQLLogicTestRunner &runner, DuckDB &db, unordered_map> &named_connection_map, string con_name) { auto entry = named_connection_map.find(con_name); @@ -34,9 +34,9 @@ static Connection *GetConnection(DuckDB &db, auto &test_config = TestConfiguration::Get(); auto init_cmd = test_config.OnConnectionCommand(); if (!init_cmd.empty()) { - auto res = con->Query(init_cmd); + auto res = con->Query(runner.ReplaceKeywords(init_cmd)); if (res->HasError()) { - FAIL("Startup queries provided via on_init failed: " + res->GetError()); + FAIL("Startup queries provided via on_new_connection failed: " + res->GetError()); } } auto res = con.get(); @@ -61,9 +61,9 @@ Connection *Command::CommandConnection(ExecuteContext &context) const { auto &test_config = TestConfiguration::Get(); auto init_cmd = test_config.OnConnectionCommand(); if (!init_cmd.empty()) { - auto res = context.con->Query(init_cmd); + auto res = context.con->Query(runner.ReplaceKeywords(init_cmd)); if (res->HasError()) { - string error_msg = "Startup queries provided via on_init failed: " + res->GetError(); + string error_msg = "Startup queries provided via on_new_connection failed: " + res->GetError(); if (context.is_parallel) { throw std::runtime_error(error_msg); } else { @@ -81,7 +81,7 @@ Connection *Command::CommandConnection(ExecuteContext &context) const { if (context.is_parallel) { throw std::runtime_error("Named connections not supported in parallel loop"); } - return GetConnection(*runner.db, runner.named_connection_map, connection_name); + return GetConnection(runner, *runner.db, runner.named_connection_map, connection_name); } } @@ -166,6 +166,7 @@ void Command::RestartDatabase(ExecuteContext &context, Connection *&connection, unique_ptr Command::ExecuteQuery(ExecuteContext &context, Connection *connection, string file_name, idx_t query_line) const { query_break(query_line); + if (TestConfiguration::TestForceReload() && TestConfiguration::TestForceStorage()) { RestartDatabase(context, connection, context.sql_query); } diff --git a/test/sqlite/sqllogic_parser.cpp b/test/sqlite/sqllogic_parser.cpp index 7ac4bb90e3b1..df59aedb3770 100644 --- a/test/sqlite/sqllogic_parser.cpp +++ b/test/sqlite/sqllogic_parser.cpp @@ -163,11 +163,13 @@ bool SQLLogicParser::IsSingleLineStatement(SQLLogicToken &token) { case SQLLogicTokenType::SQLLOGIC_ENDLOOP: case SQLLogicTokenType::SQLLOGIC_REQUIRE: case SQLLogicTokenType::SQLLOGIC_REQUIRE_ENV: + case SQLLogicTokenType::SQLLOGIC_TEST_ENV: case SQLLogicTokenType::SQLLOGIC_LOAD: case SQLLogicTokenType::SQLLOGIC_RESTART: case SQLLogicTokenType::SQLLOGIC_RECONNECT: case SQLLogicTokenType::SQLLOGIC_SLEEP: case SQLLogicTokenType::SQLLOGIC_UNZIP: + case SQLLogicTokenType::SQLLOGIC_TAGS: return true; case SQLLogicTokenType::SQLLOGIC_SKIP_IF: @@ -182,6 +184,42 @@ bool SQLLogicParser::IsSingleLineStatement(SQLLogicToken &token) { } } +// (All) Context statements must precede all non-header statements +bool SQLLogicParser::IsTestCommand(SQLLogicTokenType &type) { + switch (type) { + case SQLLogicTokenType::SQLLOGIC_QUERY: + case SQLLogicTokenType::SQLLOGIC_STATEMENT: + return true; + + case SQLLogicTokenType::SQLLOGIC_CONCURRENT_FOREACH: + case SQLLogicTokenType::SQLLOGIC_CONCURRENT_LOOP: + case SQLLogicTokenType::SQLLOGIC_ENDLOOP: + case SQLLogicTokenType::SQLLOGIC_FOREACH: + case SQLLogicTokenType::SQLLOGIC_HALT: + case SQLLogicTokenType::SQLLOGIC_HASH_THRESHOLD: + case SQLLogicTokenType::SQLLOGIC_INVALID: + case SQLLogicTokenType::SQLLOGIC_LOAD: + case SQLLogicTokenType::SQLLOGIC_LOOP: + case SQLLogicTokenType::SQLLOGIC_MODE: + case SQLLogicTokenType::SQLLOGIC_ONLY_IF: + case SQLLogicTokenType::SQLLOGIC_RECONNECT: + case SQLLogicTokenType::SQLLOGIC_REQUIRE: + case SQLLogicTokenType::SQLLOGIC_REQUIRE_ENV: + case SQLLogicTokenType::SQLLOGIC_RESET: + case SQLLogicTokenType::SQLLOGIC_RESTART: + case SQLLogicTokenType::SQLLOGIC_SET: + case SQLLogicTokenType::SQLLOGIC_SKIP_IF: + case SQLLogicTokenType::SQLLOGIC_SLEEP: + case SQLLogicTokenType::SQLLOGIC_TAGS: + case SQLLogicTokenType::SQLLOGIC_TEST_ENV: + case SQLLogicTokenType::SQLLOGIC_UNZIP: + return false; + + default: + throw std::runtime_error("Unknown SQLLogic token found!"); + } +} + SQLLogicTokenType SQLLogicParser::CommandToToken(const string &token) { if (token == "skipif") { return SQLLogicTokenType::SQLLOGIC_SKIP_IF; @@ -215,6 +253,8 @@ SQLLogicTokenType SQLLogicParser::CommandToToken(const string &token) { return SQLLogicTokenType::SQLLOGIC_REQUIRE; } else if (token == "require-env") { return SQLLogicTokenType::SQLLOGIC_REQUIRE_ENV; + } else if (token == "test-env") { + return SQLLogicTokenType::SQLLOGIC_TEST_ENV; } else if (token == "load") { return SQLLogicTokenType::SQLLOGIC_LOAD; } else if (token == "restart") { @@ -225,6 +265,8 @@ SQLLogicTokenType SQLLogicParser::CommandToToken(const string &token) { return SQLLogicTokenType::SQLLOGIC_SLEEP; } else if (token == "unzip") { return SQLLogicTokenType::SQLLOGIC_UNZIP; + } else if (token == "tags") { + return SQLLogicTokenType::SQLLOGIC_TAGS; } Fail("Unrecognized parameter %s", token); return SQLLogicTokenType::SQLLOGIC_INVALID; diff --git a/test/sqlite/sqllogic_parser.hpp b/test/sqlite/sqllogic_parser.hpp index 76d0b5efb21e..eff512ad229c 100644 --- a/test/sqlite/sqllogic_parser.hpp +++ b/test/sqlite/sqllogic_parser.hpp @@ -32,11 +32,13 @@ enum class SQLLogicTokenType { SQLLOGIC_ENDLOOP, SQLLOGIC_REQUIRE, SQLLOGIC_REQUIRE_ENV, + SQLLOGIC_TEST_ENV, SQLLOGIC_LOAD, SQLLOGIC_RESTART, SQLLOGIC_RECONNECT, SQLLOGIC_SLEEP, - SQLLOGIC_UNZIP + SQLLOGIC_UNZIP, + SQLLOGIC_TAGS }; class SQLLogicToken { @@ -60,6 +62,7 @@ class SQLLogicParser { public: static bool EmptyOrComment(const string &line); static bool IsSingleLineStatement(SQLLogicToken &token); + static bool IsTestCommand(SQLLogicTokenType &type); //! Does the next line contain a comment, empty line, or is the end of the file bool NextLineEmptyOrComment(); diff --git a/test/sqlite/sqllogic_test_runner.cpp b/test/sqlite/sqllogic_test_runner.cpp index 19cbca369b8a..72559950dfa3 100644 --- a/test/sqlite/sqllogic_test_runner.cpp +++ b/test/sqlite/sqllogic_test_runner.cpp @@ -58,6 +58,9 @@ SQLLogicTestRunner::SQLLogicTestRunner(string dbpath) : dbpath(std::move(dbpath) } else if (config->options.autoload_known_extensions) { local_extension_repo = string(DUCKDB_BUILD_DIRECTORY) + "/repository"; } + for (auto &entry : test_config.GetConfigSettings()) { + config->SetOptionByName(entry.name, entry.value); + } } SQLLogicTestRunner::~SQLLogicTestRunner() { @@ -114,10 +117,14 @@ void SQLLogicTestRunner::EndLoop() { } ExtensionLoadResult SQLLogicTestRunner::LoadExtension(DuckDB &db, const std::string &extension) { - Connection con(db); - auto result = con.Query("LOAD " + extension); - if (!result->HasError()) { - return ExtensionLoadResult::LOADED_EXTENSION; + auto &test_config = TestConfiguration::Get(); + if (test_config.GetExtensionAutoLoadingMode() != TestConfiguration::ExtensionAutoLoadingMode::NONE) { + // try LOAD extension + Connection con(db); + auto result = con.Query("LOAD " + extension); + if (!result->HasError()) { + return ExtensionLoadResult::LOADED_EXTENSION; + } } return ExtensionHelper::LoadExtension(db, extension); } @@ -175,7 +182,7 @@ void SQLLogicTestRunner::Reconnect() { } auto &test_config = TestConfiguration::Get(); - auto init_cmd = test_config.OnInitCommand() + test_config.OnConnectionCommand(); + auto init_cmd = test_config.OnInitCommand() + ";" + test_config.OnConnectionCommand(); if (!init_cmd.empty()) { test_config.ProcessPath(init_cmd, file_name); auto res = con->Query(ReplaceKeywords(init_cmd)); @@ -223,10 +230,18 @@ string SQLLogicTestRunner::ReplaceKeywords(string input) { auto &value = it.second; input = StringUtil::Replace(input, StringUtil::Format("${%s}", name), value); } - input = StringUtil::Replace(input, "{UUID}", UUID::ToString(UUID::GenerateRandomUUID())); - input = StringUtil::Replace(input, "__TEST_DIR__", TestDirectoryPath()); - input = StringUtil::Replace(input, "__WORKING_DIRECTORY__", FileSystem::GetWorkingDirectory()); + auto &test_config = TestConfiguration::Get(); + test_config.ProcessPath(input, file_name); input = StringUtil::Replace(input, "__BUILD_DIRECTORY__", DUCKDB_BUILD_DIRECTORY); + + string data_location = test_config.DataLocation(); + + input = StringUtil::Replace(input, "'data/", string("'") + data_location); + input = StringUtil::Replace(input, "\"data/", string("\"") + data_location); + if (StringUtil::StartsWith(input, "data/")) { + input = data_location + input.substr(5); + } + return input; } @@ -598,7 +613,14 @@ RequireResult SQLLogicTestRunner::CheckRequire(SQLLogicParser &parser, const vec bool perform_install = false; bool perform_load = false; if (!config->options.autoload_known_extensions) { - auto result = SQLLogicTestRunner::LoadExtension(*db, param); + auto result = ExtensionLoadResult::NOT_LOADED; + try { + result = SQLLogicTestRunner::LoadExtension(*db, param); + } catch (std::exception &ex) { + ErrorData error_data(ex); + parser.Fail("extension '%s' load threw an exception: %s", param, error_data.Message()); + } + if (result == ExtensionLoadResult::LOADED_EXTENSION) { // add the extension to the list of loaded extensions extensions.insert(param); @@ -689,6 +711,14 @@ bool TryParseConditions(SQLLogicParser &parser, const string &condition_text, ve return true; } +// add implicit tags from environment variables, with value if available +void add_env_tag(vector &tags, const string &name, const string *value = nullptr) { + tags.emplace_back(StringUtil::Format("env[%s]", name)); + if (value != nullptr) { + tags.emplace_back(StringUtil::Format("env[%s]=%s", name, *value)); + } +} + void SQLLogicTestRunner::ExecuteFile(string script) { auto &test_config = TestConfiguration::Get(); if (test_config.ShouldSkipTest(script)) { @@ -699,6 +729,9 @@ void SQLLogicTestRunner::ExecuteFile(string script) { file_name = script; SQLLogicParser parser; idx_t skip_level = 0; + bool test_expr_executed = false; + bool file_tags_expr_seen = false; + vector file_tags; // gets both implicit and file-spec'd // for the original SQLite tests we convert floating point numbers to integers // for our own tests this is undesirable since it hides certain errors @@ -728,6 +761,13 @@ void SQLLogicTestRunner::ExecuteFile(string script) { FAIL("Could not find test script '" + script + "'. Perhaps run `make sqlite`. "); } + if (StringUtil::EndsWith(script, ".test_slow")) { + file_tags.emplace_back("slow"); + } + if (StringUtil::EndsWith(script, ".test_coverage")) { + file_tags.emplace_back("coverage"); + } + /* Loop over all records in the file */ while (parser.NextStatement()) { // tokenize the current line @@ -738,6 +778,15 @@ void SQLLogicTestRunner::ExecuteFile(string script) { parser.Fail("all test statements need to be separated by an empty line"); } + // Check tags first time we hit test statements, since all explicit & implicit tags now present + if (parser.IsTestCommand(token.type) && !test_expr_executed) { + if (test_config.GetPolicyForTagSet(file_tags) == TestConfiguration::SelectPolicy::SKIP) { + SKIP_TEST("select tag-set"); + return; + } + test_expr_executed = true; + } + vector conditions; bool skip_statement = false; while (token.type == SQLLogicTokenType::SQLLOGIC_SKIP_IF || token.type == SQLLogicTokenType::SQLLOGIC_ONLY_IF) { @@ -1018,6 +1067,25 @@ void SQLLogicTestRunner::ExecuteFile(string script) { SKIP_TEST("require " + token.parameters[0]); return; } + } else if (token.type == SQLLogicTokenType::SQLLOGIC_TEST_ENV) { + if (InLoop()) { + parser.Fail("test-env cannot be called in a loop"); + } + + if (token.parameters.size() != 2) { + parser.Fail("test-env requires 2 arguments: "); + } + auto env_var = token.parameters[0]; + auto env_actual = test_config.GetTestEnv(env_var, token.parameters[1]); + + // Check if we have something defining from our test + if (environment_variables.count(env_var)) { + parser.Fail(StringUtil::Format("Environment/Test variable '%s' has already been defined", env_var)); + } + + environment_variables[env_var] = env_actual; + add_env_tag(file_tags, env_var, &env_actual); + } else if (token.type == SQLLogicTokenType::SQLLOGIC_REQUIRE_ENV) { if (InLoop()) { parser.Fail("require-env cannot be called in a loop"); @@ -1051,12 +1119,15 @@ void SQLLogicTestRunner::ExecuteFile(string script) { SKIP_TEST("require-env " + token.parameters[0] + " " + token.parameters[1]); return; } + + file_tags.emplace_back(StringUtil::Format("env[%s]=%s", token.parameters[0], token.parameters[1])); } if (environment_variables.count(env_var)) { parser.Fail(StringUtil::Format("Environment variable '%s' has already been defined", env_var)); } environment_variables[env_var] = env_actual; + add_env_tag(file_tags, token.parameters[0], token.parameters.size() == 2 ? &token.parameters[1] : nullptr); } else if (token.type == SQLLogicTokenType::SQLLOGIC_LOAD) { auto &test_config = TestConfiguration::Get(); @@ -1136,6 +1207,26 @@ void SQLLogicTestRunner::ExecuteFile(string script) { auto command = make_uniq(*this, input_path, extraction_path); ExecuteCommand(std::move(command)); + } else if (token.type == SQLLogicTokenType::SQLLOGIC_TAGS) { + // NOTE: tags-before-test-commands is the low bar right now + // 1 better: all non-command lines precede command lines + // Mo better: parse first, build entire context before execution; allows e.g. + // - implicit tag scans of e.g. strings, vars, etc., like '${ENVVAR}', '__TEST_DIR__', 'ATTACH' + // - faster subset runs + // - tag match runs to generate lists + if (test_expr_executed) { + parser.Fail("tags expression must precede test commands"); + } + if (file_tags_expr_seen) { + parser.Fail("tags may be only specified once"); + } + file_tags_expr_seen = true; + if (token.parameters.empty()) { + parser.Fail("tags requires >= 1 argument, e.g.: [tag2 .. tagN]"); + } + + // extend file_tags for jit eval + file_tags.insert(file_tags.begin(), token.parameters.begin(), token.parameters.end()); } } if (InLoop()) { diff --git a/test/sqlite/tags/tags-1-2-3.test_slow b/test/sqlite/tags/tags-1-2-3.test_slow new file mode 100644 index 000000000000..14acdf7e1e65 --- /dev/null +++ b/test/sqlite/tags/tags-1-2-3.test_slow @@ -0,0 +1,9 @@ +# name: test/sqlite/tags/tags-1-2-3.test_slow +# group: [tags] + +tags 1 2 3 + +require-env VALIDATE_TAGS + +statement ok +SELECT 'tagged 1 2 3'; diff --git a/test/sqlite/tags/tags-1-2.test b/test/sqlite/tags/tags-1-2.test new file mode 100644 index 000000000000..768f5eff319d --- /dev/null +++ b/test/sqlite/tags/tags-1-2.test @@ -0,0 +1,9 @@ +# name: test/sqlite/tags/tags-1-2.test +# group: [tags] + +tags 1 2 + +require-env VALIDATE_TAGS + +statement ok +SELECT 'tagged 1 2'; diff --git a/test/sqlite/tags/tags-1.test b/test/sqlite/tags/tags-1.test new file mode 100644 index 000000000000..1775052735f0 --- /dev/null +++ b/test/sqlite/tags/tags-1.test @@ -0,0 +1,9 @@ +# name: test/sqlite/tags/tags-1.test +# group: [tags] + +tags 1 + +require-env VALIDATE_TAGS + +statement ok +SELECT 'tagged 1'; diff --git a/test/sqlite/tags/tags-a.test b/test/sqlite/tags/tags-a.test new file mode 100644 index 000000000000..07098abd9370 --- /dev/null +++ b/test/sqlite/tags/tags-a.test @@ -0,0 +1,9 @@ +# name: test/sqlite/tags/tags-a.test +# group: [tags] + +tags a + +require-env VALIDATE_TAGS + +statement ok +SELECT 'tagged a'; diff --git a/test/sqlite/test_sqllogictest.cpp b/test/sqlite/test_sqllogictest.cpp index 3c292311b9d2..8466fa11df7e 100644 --- a/test/sqlite/test_sqllogictest.cpp +++ b/test/sqlite/test_sqllogictest.cpp @@ -61,6 +61,11 @@ static void testRunner() { runner.output_sql = Catch::getCurrentContext().getConfig()->outputSQL(); runner.enable_verification = VERIFICATION; + // Copy configured env vars + for (auto &kv : test_config.GetTestEnvMap()) { + runner.environment_variables[kv.first] = kv.second; + } + string prev_directory; // We assume the test working dir for extensions to be one dir above the test/sql. Note that this is very hacky. diff --git a/test/sqlite/validate_tags_usage.sh b/test/sqlite/validate_tags_usage.sh new file mode 100755 index 000000000000..d44039baeea9 --- /dev/null +++ b/test/sqlite/validate_tags_usage.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +## +# assumes $SCRIPT_DIR/../../build/debug/test/unittest to be ready to run +# + +ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." &>/dev/null && pwd) +cd "$ROOT" + +: ${UNITTEST:="build/debug/test/unittest --output-sql=true"} +: ${TESTS_SPEC:='test/sqlite/tags/*'} +export VALIDATE_TAGS=1 + +run() { + $UNITTEST "$@" "$TESTS_SPEC" 2>&1 | grep "SELECT 'tagged" +} + +expect() { + output=$(cat) + local errs=0 + if [[ "$#" -eq 0 ]]; then + [[ $(echo -n "$output" | wc -l) -eq 0 ]] && { + echo -n "✅ - ok" + } || { + printf "\n ❌ - error - matches found but none expected:\n%s" "$output" + } + else + for elt in "$@"; do + echo -n "$output" | grep -q "'tagged $elt'" || { + printf "\n ❌ - error - missing %s" "$elt" + errs=$(($errs + 1)) + } + done + [[ $errs -eq 0 ]] && { + echo -n " ✅ - ok" + } + fi +} + +test() { + local args="$1" + shift + echo -n "test $args -- " + run $args | expect "$@" + echo +} + +test_select() { + # select tags + test "--select-tag 1" "1" "1 2" "1 2 3" + test "--select-tag-set ['1']" "1" "1 2" "1 2 3" + + test "--select-tag 2" "1 2" "1 2 3" + test "--select-tag-set ['2']" "1 2" "1 2 3" + + test "--select-tag 3" "1 2 3" + test "--select-tag-set ['3']" "1 2 3" + + test "--select-tag-set ['1','3']" "1 2 3" + test "--select-tag-set ['2','3']" "1 2 3" + test "--select-tag-set ['1','2']" "1 2" "1 2 3" + test "--select-tag-set ['1','2','3']" "1 2 3" + test "--select-tag-set ['1','2','3','4']" +} + +test_skip() { + # skip tags + test "--skip-tag 1" + test "--skip-tag 1" + + test "--skip-tag 1" + test "--skip-tag-set ['1']" + + test "--skip-tag 2" "1" + test "--skip-tag-set ['2']" "1" + + test "--skip-tag 3" "1" "1 2" + test "--skip-tag-set ['3']" "1" "1 2" + + test "--skip-tag-set ['1','3']" "1" "1 2" + test "--skip-tag-set ['2','3']" "1" "1 2" + test "--skip-tag-set ['1','2']" "1" + test "--skip-tag-set ['1','2','3']" "1" "1 2" + test "--skip-tag-set ['1','2','3','a']" "1" "1 2" "1 2 3" "a" +} + +test_combo() { + # crossover + test "--select-tag 1 --skip-tag 2" "1" + test "--skip-tag-set ['1','2','3'] --select-tag 2" "1 2" + test "--select-tag 1 --skip-tag 1" + test "--select-tag noexist --skip-tag 1" + test "--select-tag 3 --skip-tag noexist" "1 2 3" + + # confirm BNF behavior + test "--select-tag 3 --select-tag a" "1 2 3" "a" + test "--skip-tag 3 --skip-tag a" "1" "1 2" +} + +test_implicit_env() { + test "--select-tag env[VALIDATE_TAGS]" "1" "1" "1 2" "1 2 3" "a" + test "--select-tag env[VALIDATE_TAGS]=0" + # NOTE: =1 not set because it's a require, not a test-env + test "--select-tag env[VALIDATE_TAGS]=1" +} + +test "" "1" "1 2" "1 2 3" +test_select +test_skip +test_combo +test_implicit_env diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index a7e32c8e9b89..5294f426512f 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -17,10 +17,10 @@ if(NOT AMALGAMATION_BUILD) endif() if(NOT WIN32 - AND NOT SUN - AND ${BUILD_UNITTESTS}) - add_subdirectory(imdb) - if(${BUILD_TPCE}) + AND ${BUILD_UNITTESTS} AND ${BUILD_TPCE}) add_subdirectory(tpce-tool) - endif() endif() + +if (${BUILD_BENCHMARKS}) + add_subdirectory(imdb) +endif() \ No newline at end of file diff --git a/third_party/httplib/httplib.hpp b/third_party/httplib/httplib.hpp index 8778497477a0..409c47d0bb3c 100644 --- a/third_party/httplib/httplib.hpp +++ b/third_party/httplib/httplib.hpp @@ -7077,7 +7077,12 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { } auto location = res.get_header_value("location"); - if (location.empty()) { return false; } + if (location.empty()) { + // s3 requests will not return a location header, and instead a + // X-Amx-Region-Bucket header. Return true so all response headers + // are returned to the httpfs/calling extension + return true; + } const Regex re( R"((?:(https?):)?(?://(?:\[([\d:]+)\]|([^:/?#]+))(?::(\d+))?)?([^?#]*)(\?[^#]*)?(?:#.*)?)"); @@ -8555,7 +8560,11 @@ inline long SSLClient::get_openssl_verify_result() const { inline SSL_CTX *SSLClient::ssl_context() const { return ctx_; } inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) { - return is_valid() && ClientImpl::create_and_connect_socket(socket, error); + if (!is_valid()) { + error = Error::SSLConnection; + return false; + } + return ClientImpl::create_and_connect_socket(socket, error); } // Assumes that socket_mutex_ is locked and that there are no requests in flight diff --git a/third_party/libpg_query/grammar/statements.list b/third_party/libpg_query/grammar/statements.list index 59f9184e2452..d0ed58a57e86 100644 --- a/third_party/libpg_query/grammar/statements.list +++ b/third_party/libpg_query/grammar/statements.list @@ -1,3 +1,4 @@ +AlterDatabaseStmt AlterObjectSchemaStmt AlterSeqStmt AlterTableStmt diff --git a/third_party/libpg_query/grammar/statements/alter_database.y b/third_party/libpg_query/grammar/statements/alter_database.y new file mode 100644 index 000000000000..bdb264690236 --- /dev/null +++ b/third_party/libpg_query/grammar/statements/alter_database.y @@ -0,0 +1,25 @@ +/***************************************************************************** + * + * Alter Database Statement + * + *****************************************************************************/ +AlterDatabaseStmt: + ALTER DATABASE ColId RENAME TO ColId + { + PGAlterDatabaseStmt *n = makeNode(PGAlterDatabaseStmt); + n->dbname = $3; + n->new_name = $6; + n->alter_type = PG_ALTER_DATABASE_RENAME; + n->missing_ok = false; + $$ = (PGNode *)n; + } + | ALTER DATABASE IF_P EXISTS ColId RENAME TO ColId + { + PGAlterDatabaseStmt *n = makeNode(PGAlterDatabaseStmt); + n->dbname = $5; + n->new_name = $8; + n->alter_type = PG_ALTER_DATABASE_RENAME; + n->missing_ok = true; + $$ = (PGNode *)n; + } + ; diff --git a/third_party/libpg_query/grammar/statements/select.y b/third_party/libpg_query/grammar/statements/select.y index 85de7b687a41..a9c1e188d1d6 100644 --- a/third_party/libpg_query/grammar/statements/select.y +++ b/third_party/libpg_query/grammar/statements/select.y @@ -1725,9 +1725,10 @@ Typename: SimpleTypename opt_array_bounds $$->arrayBounds = list_make1(makeInteger(-1)); $$->setof = true; } - | qualified_typename + | qualified_typename opt_array_bounds { $$ = makeTypeNameFromNameList($1); + $$->arrayBounds = $2; } | RowOrStruct '(' colid_type_list ')' opt_array_bounds { diff --git a/third_party/libpg_query/include/nodes/parsenodes.hpp b/third_party/libpg_query/include/nodes/parsenodes.hpp index 2806ae4e287d..bfdc6ca202ce 100755 --- a/third_party/libpg_query/include/nodes/parsenodes.hpp +++ b/third_party/libpg_query/include/nodes/parsenodes.hpp @@ -2086,6 +2086,18 @@ typedef struct PGCopyDatabaseStmt { const char *copy_database_flag; } PGCopyDatabaseStmt; +typedef enum PGAlterDatabaseType { + PG_ALTER_DATABASE_RENAME +} PGAlterDatabaseType; + +typedef struct PGAlterDatabaseStmt { + PGNodeTag type; + const char *dbname; + const char *new_name; + PGAlterDatabaseType alter_type; + bool missing_ok; +} PGAlterDatabaseStmt; + /* ---------------------- * Interval Constant * ---------------------- diff --git a/third_party/libpg_query/src_backend_parser_gram.cpp b/third_party/libpg_query/src_backend_parser_gram.cpp index 11bf35b9bb35..f222526b5c7b 100644 --- a/third_party/libpg_query/src_backend_parser_gram.cpp +++ b/third_party/libpg_query/src_backend_parser_gram.cpp @@ -1645,18 +1645,18 @@ union yyalloc #endif /* YYFINAL -- State number of the termination state. */ -#define YYFINAL 886 +#define YYFINAL 889 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 76887 +#define YYLAST 77384 /* YYNTOKENS -- Number of terminals. */ #define YYNTOKENS 538 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 509 +#define YYNNTS 510 /* YYNRULES -- Number of rules. */ -#define YYNRULES 2254 +#define YYNRULES 2257 /* YYNRULES -- Number of states. */ -#define YYNSTATES 3801 +#define YYNSTATES 3815 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ #define YYUNDEFTOK 2 @@ -1757,1079 +1757,1081 @@ static const yytype_uint16 yyprhs[] = 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, - 81, 83, 85, 87, 89, 91, 93, 95, 96, 101, - 108, 113, 120, 125, 132, 137, 144, 146, 149, 153, - 156, 158, 162, 165, 169, 171, 175, 177, 180, 183, - 185, 188, 191, 194, 197, 203, 207, 214, 221, 225, - 232, 236, 241, 248, 255, 262, 268, 274, 281, 291, - 296, 302, 310, 317, 322, 331, 336, 339, 344, 348, - 355, 360, 363, 366, 369, 372, 374, 377, 378, 380, - 383, 386, 389, 391, 395, 400, 403, 405, 406, 409, - 413, 416, 420, 422, 425, 427, 429, 431, 433, 435, - 437, 439, 442, 445, 447, 449, 451, 453, 455, 462, - 469, 478, 485, 494, 501, 510, 517, 526, 535, 546, - 555, 566, 568, 569, 579, 581, 586, 591, 599, 602, - 604, 608, 611, 614, 615, 620, 624, 625, 627, 628, - 631, 635, 641, 644, 647, 648, 657, 663, 664, 670, - 676, 684, 687, 688, 690, 692, 694, 698, 701, 702, - 704, 705, 707, 711, 713, 717, 719, 722, 724, 728, - 731, 738, 748, 757, 760, 765, 767, 769, 770, 772, - 776, 779, 784, 790, 796, 805, 813, 815, 816, 825, - 837, 848, 849, 851, 852, 854, 856, 857, 860, 862, - 865, 867, 871, 876, 880, 890, 903, 905, 909, 911, - 915, 919, 920, 925, 932, 934, 937, 939, 941, 942, - 944, 947, 950, 952, 955, 958, 960, 963, 967, 970, - 973, 976, 979, 983, 987, 991, 993, 997, 999, 1000, - 1002, 1005, 1008, 1014, 1022, 1023, 1026, 1029, 1033, 1037, - 1040, 1043, 1046, 1048, 1050, 1051, 1054, 1057, 1058, 1061, - 1071, 1084, 1096, 1097, 1100, 1102, 1104, 1106, 1108, 1110, - 1112, 1116, 1117, 1119, 1122, 1124, 1126, 1129, 1132, 1136, - 1138, 1140, 1143, 1146, 1148, 1151, 1155, 1161, 1165, 1168, - 1174, 1176, 1178, 1180, 1181, 1187, 1195, 1201, 1204, 1208, - 1210, 1212, 1215, 1218, 1219, 1223, 1228, 1233, 1234, 1238, - 1241, 1242, 1246, 1248, 1250, 1252, 1254, 1256, 1258, 1260, - 1262, 1264, 1266, 1270, 1274, 1276, 1279, 1282, 1285, 1288, - 1291, 1294, 1295, 1299, 1303, 1307, 1308, 1310, 1313, 1315, - 1318, 1321, 1324, 1327, 1330, 1334, 1337, 1340, 1342, 1346, - 1348, 1350, 1352, 1354, 1358, 1360, 1363, 1364, 1366, 1369, - 1370, 1372, 1376, 1377, 1380, 1381, 1385, 1389, 1391, 1397, - 1401, 1403, 1407, 1409, 1412, 1414, 1419, 1425, 1431, 1438, - 1442, 1450, 1455, 1467, 1469, 1473, 1476, 1479, 1482, 1483, - 1487, 1489, 1491, 1494, 1497, 1500, 1503, 1505, 1506, 1508, - 1511, 1518, 1523, 1530, 1535, 1542, 1551, 1553, 1555, 1557, - 1559, 1562, 1564, 1567, 1569, 1572, 1574, 1576, 1578, 1580, - 1584, 1588, 1592, 1596, 1598, 1601, 1604, 1606, 1610, 1612, - 1614, 1616, 1620, 1622, 1624, 1625, 1627, 1629, 1631, 1641, - 1644, 1645, 1649, 1650, 1652, 1653, 1657, 1661, 1664, 1666, - 1673, 1677, 1681, 1684, 1687, 1689, 1690, 1696, 1699, 1702, - 1703, 1711, 1713, 1715, 1717, 1720, 1726, 1735, 1743, 1749, - 1758, 1766, 1771, 1776, 1778, 1782, 1784, 1786, 1790, 1792, - 1796, 1798, 1800, 1803, 1808, 1812, 1814, 1818, 1821, 1826, - 1831, 1840, 1852, 1862, 1870, 1871, 1875, 1879, 1881, 1883, - 1887, 1888, 1890, 1891, 1893, 1894, 1896, 1897, 1899, 1903, - 1906, 1907, 1910, 1911, 1913, 1914, 1916, 1918, 1920, 1924, - 1928, 1930, 1932, 1936, 1940, 1944, 1948, 1952, 1956, 1961, - 1965, 1968, 1970, 1972, 1974, 1978, 1980, 1984, 1986, 1988, - 1990, 1994, 1998, 2002, 2004, 2007, 2012, 2017, 2020, 2024, - 2030, 2036, 2038, 2040, 2044, 2045, 2057, 2069, 2080, 2093, - 2095, 2098, 2104, 2109, 2114, 2119, 2124, 2132, 2138, 2143, - 2151, 2158, 2168, 2178, 2183, 2185, 2187, 2189, 2191, 2193, - 2195, 2197, 2203, 2205, 2207, 2211, 2213, 2216, 2219, 2222, - 2226, 2228, 2232, 2241, 2247, 2248, 2250, 2253, 2255, 2259, - 2261, 2264, 2265, 2268, 2269, 2273, 2277, 2282, 2287, 2292, - 2297, 2301, 2304, 2306, 2308, 2309, 2311, 2313, 2314, 2317, - 2319, 2325, 2327, 2328, 2331, 2334, 2335, 2337, 2338, 2342, - 2348, 2350, 2354, 2359, 2363, 2365, 2367, 2368, 2371, 2374, - 2375, 2378, 2381, 2383, 2385, 2387, 2388, 2391, 2396, 2402, - 2407, 2410, 2414, 2416, 2418, 2420, 2423, 2426, 2428, 2431, - 2435, 2436, 2438, 2439, 2445, 2447, 2452, 2459, 2462, 2464, - 2465, 2470, 2471, 2473, 2475, 2479, 2484, 2485, 2487, 2489, - 2492, 2495, 2498, 2500, 2502, 2505, 2508, 2510, 2512, 2514, - 2516, 2518, 2520, 2524, 2528, 2529, 2531, 2535, 2537, 2540, - 2542, 2544, 2546, 2548, 2550, 2553, 2558, 2563, 2569, 2571, - 2573, 2576, 2577, 2580, 2581, 2583, 2587, 2589, 2590, 2592, - 2595, 2599, 2602, 2607, 2610, 2614, 2617, 2618, 2620, 2623, - 2624, 2629, 2635, 2637, 2640, 2643, 2644, 2646, 2650, 2652, - 2655, 2658, 2663, 2668, 2672, 2676, 2680, 2684, 2688, 2692, - 2696, 2698, 2703, 2708, 2718, 2728, 2732, 2733, 2736, 2739, - 2740, 2746, 2750, 2752, 2754, 2758, 2764, 2768, 2770, 2773, - 2775, 2779, 2785, 2787, 2790, 2794, 2799, 2805, 2810, 2816, - 2821, 2828, 2834, 2839, 2845, 2851, 2857, 2860, 2865, 2867, - 2869, 2870, 2872, 2877, 2883, 2888, 2889, 2892, 2895, 2898, - 2900, 2902, 2904, 2906, 2907, 2912, 2915, 2917, 2920, 2923, - 2928, 2931, 2938, 2941, 2943, 2947, 2952, 2953, 2956, 2957, - 2960, 2961, 2963, 2967, 2971, 2974, 2975, 2978, 2983, 2985, - 2987, 2989, 2990, 2993, 2997, 3003, 3010, 3013, 3017, 3019, - 3025, 3031, 3037, 3041, 3045, 3049, 3054, 3055, 3057, 3059, - 3061, 3063, 3065, 3068, 3073, 3075, 3077, 3079, 3081, 3084, - 3088, 3089, 3091, 3093, 3095, 3097, 3099, 3102, 3105, 3108, - 3111, 3114, 3116, 3120, 3121, 3123, 3125, 3127, 3129, 3135, - 3138, 3140, 3142, 3144, 3146, 3151, 3153, 3156, 3159, 3161, - 3165, 3169, 3172, 3174, 3175, 3181, 3184, 3190, 3193, 3195, - 3199, 3203, 3204, 3206, 3208, 3210, 3212, 3214, 3216, 3218, - 3220, 3222, 3224, 3226, 3228, 3230, 3232, 3234, 3236, 3238, - 3240, 3242, 3244, 3246, 3248, 3250, 3252, 3254, 3256, 3258, - 3260, 3262, 3264, 3266, 3268, 3270, 3272, 3274, 3276, 3278, - 3280, 3282, 3286, 3290, 3294, 3298, 3302, 3306, 3310, 3311, - 3313, 3317, 3321, 3327, 3330, 3333, 3337, 3341, 3345, 3349, - 3353, 3357, 3361, 3365, 3369, 3373, 3377, 3381, 3385, 3389, - 3393, 3396, 3399, 3403, 3407, 3410, 3413, 3417, 3421, 3427, - 3432, 3439, 3443, 3449, 3454, 3461, 3466, 3473, 3479, 3487, - 3491, 3494, 3499, 3503, 3506, 3511, 3515, 3519, 3523, 3527, - 3532, 3536, 3541, 3545, 3550, 3556, 3563, 3570, 3578, 3585, - 3593, 3600, 3608, 3612, 3617, 3622, 3629, 3631, 3636, 3641, - 3647, 3652, 3659, 3661, 3665, 3668, 3671, 3675, 3679, 3683, - 3687, 3691, 3695, 3699, 3703, 3707, 3711, 3715, 3719, 3723, - 3727, 3731, 3734, 3737, 3743, 3750, 3757, 3765, 3767, 3770, - 3772, 3774, 3776, 3779, 3782, 3787, 3791, 3793, 3795, 3797, - 3799, 3802, 3804, 3806, 3808, 3810, 3812, 3814, 3816, 3819, - 3824, 3827, 3831, 3835, 3840, 3844, 3850, 3857, 3865, 3875, - 3883, 3891, 3897, 3899, 3901, 3903, 3909, 3916, 3923, 3928, - 3933, 3938, 3943, 3950, 3956, 3962, 3968, 3973, 3980, 3985, - 3993, 4003, 4009, 4010, 4016, 4021, 4022, 4024, 4025, 4028, - 4029, 4031, 4035, 4039, 4042, 4045, 4046, 4053, 4055, 4056, - 4060, 4061, 4065, 4069, 4073, 4074, 4076, 4081, 4084, 4087, - 4090, 4093, 4096, 4100, 4103, 4106, 4110, 4111, 4116, 4120, - 4122, 4128, 4132, 4134, 4138, 4140, 4143, 4147, 4149, 4153, - 4155, 4158, 4160, 4161, 4163, 4165, 4167, 4169, 4171, 4173, - 4175, 4177, 4179, 4181, 4183, 4185, 4187, 4189, 4191, 4193, - 4195, 4197, 4199, 4201, 4206, 4208, 4213, 4215, 4220, 4222, - 4225, 4227, 4230, 4232, 4235, 4237, 4241, 4243, 4247, 4249, - 4252, 4254, 4258, 4260, 4263, 4265, 4266, 4268, 4272, 4274, - 4278, 4282, 4284, 4288, 4292, 4293, 4295, 4297, 4299, 4301, - 4303, 4305, 4307, 4309, 4311, 4313, 4315, 4317, 4319, 4321, - 4323, 4328, 4332, 4335, 4339, 4340, 4344, 4348, 4351, 4354, - 4356, 4357, 4360, 4363, 4367, 4370, 4372, 4374, 4378, 4380, - 4382, 4388, 4390, 4393, 4398, 4401, 4402, 4404, 4405, 4407, - 4409, 4412, 4416, 4422, 4430, 4438, 4440, 4441, 4442, 4445, - 4446, 4449, 4453, 4457, 4461, 4467, 4475, 4483, 4484, 4487, - 4489, 4490, 4492, 4493, 4495, 4499, 4501, 4504, 4508, 4511, - 4513, 4517, 4522, 4525, 4527, 4531, 4533, 4537, 4539, 4542, - 4544, 4545, 4549, 4551, 4555, 4557, 4560, 4565, 4568, 4569, - 4573, 4575, 4579, 4581, 4584, 4589, 4592, 4593, 4595, 4599, - 4601, 4605, 4607, 4610, 4612, 4616, 4618, 4620, 4623, 4625, - 4627, 4630, 4632, 4634, 4637, 4645, 4648, 4654, 4658, 4662, - 4664, 4666, 4668, 4670, 4672, 4674, 4676, 4678, 4680, 4682, - 4684, 4686, 4688, 4690, 4693, 4696, 4700, 4704, 4705, 4707, - 4709, 4711, 4717, 4721, 4722, 4724, 4726, 4728, 4730, 4732, - 4734, 4739, 4747, 4754, 4757, 4758, 4760, 4762, 4764, 4766, - 4780, 4797, 4799, 4802, 4803, 4805, 4806, 4808, 4809, 4812, - 4813, 4815, 4816, 4823, 4832, 4839, 4848, 4855, 4864, 4868, - 4871, 4873, 4874, 4881, 4888, 4890, 4892, 4894, 4896, 4898, - 4900, 4903, 4905, 4907, 4909, 4911, 4913, 4918, 4925, 4929, - 4932, 4937, 4941, 4947, 4949, 4950, 4952, 4954, 4955, 4957, - 4959, 4961, 4963, 4965, 4967, 4969, 4971, 4973, 4975, 4977, - 4979, 4981, 4983, 4985, 4987, 4989, 4991, 4993, 4995, 4997, - 4999, 5001, 5003, 5005, 5007, 5009, 5011, 5013, 5015, 5017, - 5019, 5021, 5023, 5025, 5027, 5029, 5031, 5035, 5037, 5039, - 5041, 5043, 5045, 5047, 5050, 5052, 5054, 5057, 5061, 5065, - 5069, 5073, 5075, 5079, 5083, 5086, 5090, 5094, 5096, 5098, - 5100, 5104, 5110, 5112, 5114, 5116, 5118, 5122, 5125, 5130, - 5137, 5144, 5145, 5147, 5149, 5151, 5152, 5155, 5158, 5163, - 5170, 5176, 5181, 5188, 5190, 5192, 5194, 5196, 5198, 5200, - 5201, 5203, 5207, 5209, 5210, 5218, 5222, 5224, 5227, 5231, - 5234, 5235, 5238, 5239, 5242, 5247, 5253, 5262, 5270, 5273, - 5277, 5283, 5285, 5286, 5289, 5290, 5292, 5293, 5296, 5298, - 5302, 5306, 5307, 5310, 5314, 5318, 5322, 5326, 5328, 5330, - 5332, 5335, 5339, 5342, 5345, 5348, 5353, 5356, 5360, 5365, - 5369, 5371, 5373, 5375, 5377, 5379, 5381, 5382, 5384, 5388, - 5391, 5401, 5414, 5426, 5439, 5454, 5458, 5463, 5468, 5469, - 5477, 5488, 5498, 5501, 5505, 5506, 5511, 5513, 5515, 5517, - 5519, 5521, 5523, 5525, 5527, 5529, 5531, 5533, 5535, 5537, - 5539, 5541, 5543, 5545, 5547, 5549, 5551, 5553, 5555, 5557, - 5559, 5561, 5563, 5565, 5567, 5569, 5571, 5573, 5575, 5577, - 5579, 5581, 5583, 5585, 5587, 5589, 5591, 5593, 5595, 5597, - 5599, 5601, 5603, 5605, 5607, 5609, 5611, 5613, 5615, 5617, - 5619, 5621, 5623, 5625, 5627, 5629, 5631, 5633, 5635, 5637, - 5639, 5641, 5643, 5645, 5647, 5649, 5651, 5653, 5655, 5657, - 5659, 5661, 5663, 5665, 5667, 5669, 5671, 5673, 5675, 5677, - 5679, 5681, 5683, 5685, 5687, 5689, 5691, 5693, 5695, 5697, - 5699, 5701, 5703, 5705, 5707, 5709, 5711, 5713, 5715, 5717, - 5719, 5721, 5723, 5725, 5727, 5729, 5731, 5733, 5735, 5737, - 5739, 5741, 5743, 5745, 5747, 5749, 5751, 5753, 5755, 5757, - 5759, 5761, 5763, 5765, 5767, 5769, 5771, 5773, 5775, 5777, - 5779, 5781, 5783, 5785, 5787, 5789, 5791, 5793, 5795, 5797, - 5799, 5801, 5803, 5805, 5807, 5809, 5811, 5813, 5815, 5817, - 5819, 5821, 5823, 5825, 5827, 5829, 5831, 5833, 5835, 5837, - 5839, 5841, 5843, 5845, 5847, 5849, 5851, 5853, 5855, 5857, - 5859, 5861, 5863, 5865, 5867, 5869, 5871, 5873, 5875, 5877, - 5879, 5881, 5883, 5885, 5887, 5889, 5891, 5893, 5895, 5897, - 5899, 5901, 5903, 5905, 5907, 5909, 5911, 5913, 5915, 5917, - 5919, 5921, 5923, 5925, 5927, 5929, 5931, 5933, 5935, 5937, - 5939, 5941, 5943, 5945, 5947, 5949, 5951, 5953, 5955, 5957, - 5959, 5961, 5963, 5965, 5967, 5969, 5971, 5973, 5975, 5977, - 5979, 5981, 5983, 5985, 5987, 5989, 5991, 5993, 5995, 5997, - 5999, 6001, 6003, 6005, 6007, 6009, 6011, 6013, 6015, 6017, - 6019, 6021, 6023, 6025, 6027, 6029, 6031, 6033, 6035, 6037, - 6039, 6041, 6043, 6045, 6047, 6049, 6051, 6053, 6055, 6057, - 6059, 6061, 6063, 6065, 6067, 6069, 6071, 6073, 6075, 6077, - 6079, 6081, 6083, 6085, 6087, 6089, 6091, 6093, 6095, 6097, - 6099, 6101, 6103, 6105, 6107, 6109, 6111, 6113, 6115, 6117, - 6119, 6121, 6123, 6125, 6127, 6129, 6131, 6133, 6135, 6137, - 6139, 6141, 6143, 6145, 6147, 6149, 6151, 6153, 6155, 6157, - 6159, 6161, 6163, 6165, 6167, 6169, 6171, 6173, 6175, 6177, - 6179, 6181, 6183, 6185, 6187, 6189, 6191, 6193, 6195, 6197, - 6199, 6201, 6203, 6205, 6207, 6209, 6211, 6213, 6215, 6217, - 6219, 6221, 6223, 6225, 6227, 6229, 6231, 6233, 6235, 6237, - 6239, 6241, 6243, 6245, 6247, 6249, 6251, 6253, 6255, 6257, - 6259, 6261, 6263, 6265, 6267, 6269, 6271, 6273, 6275, 6277, - 6279, 6281, 6283, 6285, 6287, 6289, 6291, 6293, 6295, 6297, - 6299, 6301, 6303, 6305, 6307, 6309, 6311, 6313, 6315, 6317, - 6319, 6321, 6323, 6325, 6327, 6329, 6331, 6333, 6335, 6337, - 6339, 6341, 6343, 6345, 6347, 6349, 6351, 6353, 6355, 6357, - 6359, 6361, 6363, 6365, 6367, 6369, 6371, 6373, 6375, 6377, - 6379, 6381, 6383, 6385, 6387, 6389, 6391, 6393, 6395, 6397, - 6399, 6401, 6403, 6405, 6407, 6409, 6411, 6413, 6415, 6417, - 6419, 6421, 6423, 6425, 6427, 6429, 6431, 6433, 6435, 6437, - 6439, 6441, 6443, 6445, 6447, 6449, 6451, 6453, 6455, 6457, - 6459, 6461, 6463, 6465, 6467, 6469, 6471, 6473, 6475, 6477, - 6479, 6481, 6483, 6485, 6487, 6489, 6491, 6493, 6495, 6497, - 6499, 6501, 6503, 6505, 6507, 6509, 6511, 6513, 6515, 6517, - 6519, 6521, 6523, 6525, 6527, 6529, 6531, 6533, 6535, 6537, - 6539, 6541, 6543, 6545, 6547, 6549, 6551, 6553, 6555, 6557, - 6559, 6561, 6563, 6565, 6567, 6569, 6571, 6573, 6575, 6577, - 6579, 6581, 6583, 6585, 6587, 6589, 6591, 6593, 6595, 6597, - 6599, 6601, 6603, 6605, 6607, 6609, 6611, 6613, 6615, 6617, - 6619, 6621, 6623, 6625, 6627, 6629, 6631, 6633, 6635, 6637, - 6639, 6641, 6643, 6645, 6647, 6649, 6651, 6653, 6655, 6657, - 6659, 6661, 6663, 6665, 6667, 6669, 6671, 6673, 6675, 6677, - 6679, 6681, 6683, 6685, 6687, 6689, 6691, 6693, 6695, 6697, - 6699, 6701, 6703, 6705, 6707, 6709, 6711, 6713, 6715, 6717, - 6719, 6721, 6723, 6725, 6727, 6729, 6731, 6733, 6735, 6737, - 6739, 6741, 6743, 6745, 6747, 6749, 6751, 6753, 6755, 6757, - 6759, 6761, 6763, 6765, 6767, 6769, 6771, 6773, 6775, 6777, - 6779, 6781, 6783, 6785, 6787 + 81, 83, 85, 87, 89, 91, 93, 95, 97, 98, + 103, 110, 115, 122, 127, 134, 139, 146, 148, 151, + 155, 158, 160, 164, 167, 171, 173, 177, 179, 182, + 185, 187, 190, 193, 196, 199, 205, 209, 216, 223, + 227, 234, 238, 243, 250, 257, 264, 270, 276, 283, + 293, 298, 304, 312, 319, 324, 333, 338, 341, 346, + 350, 357, 362, 365, 368, 371, 374, 376, 379, 380, + 382, 385, 388, 391, 393, 397, 402, 405, 407, 408, + 411, 415, 418, 422, 424, 427, 429, 431, 433, 435, + 437, 439, 441, 444, 447, 449, 451, 453, 455, 457, + 464, 471, 480, 487, 496, 503, 512, 519, 528, 537, + 548, 557, 568, 570, 571, 581, 583, 588, 593, 601, + 604, 606, 610, 613, 616, 617, 622, 626, 627, 629, + 630, 633, 637, 643, 646, 649, 650, 659, 665, 666, + 672, 678, 686, 689, 690, 692, 694, 696, 700, 703, + 704, 706, 707, 709, 713, 715, 719, 721, 724, 726, + 730, 733, 740, 750, 759, 762, 767, 769, 771, 772, + 774, 778, 781, 786, 792, 798, 807, 815, 817, 818, + 827, 839, 850, 851, 853, 854, 856, 858, 859, 862, + 864, 867, 869, 873, 878, 882, 892, 905, 907, 911, + 913, 917, 921, 922, 927, 934, 936, 939, 941, 943, + 944, 946, 949, 952, 954, 957, 960, 962, 965, 969, + 972, 975, 978, 981, 985, 989, 993, 995, 999, 1001, + 1002, 1004, 1007, 1010, 1017, 1026, 1032, 1040, 1041, 1044, + 1047, 1051, 1055, 1058, 1061, 1064, 1066, 1068, 1069, 1072, + 1075, 1076, 1079, 1089, 1102, 1114, 1115, 1118, 1120, 1122, + 1124, 1126, 1128, 1130, 1134, 1135, 1137, 1140, 1142, 1144, + 1147, 1150, 1154, 1156, 1158, 1161, 1164, 1166, 1169, 1173, + 1179, 1183, 1186, 1192, 1194, 1196, 1198, 1199, 1205, 1213, + 1219, 1222, 1226, 1228, 1230, 1233, 1236, 1237, 1241, 1246, + 1251, 1252, 1256, 1259, 1260, 1264, 1266, 1268, 1270, 1272, + 1274, 1276, 1278, 1280, 1282, 1284, 1288, 1292, 1294, 1297, + 1300, 1303, 1306, 1309, 1312, 1313, 1317, 1321, 1325, 1326, + 1328, 1331, 1333, 1336, 1339, 1342, 1345, 1348, 1352, 1355, + 1358, 1360, 1364, 1366, 1368, 1370, 1372, 1376, 1378, 1381, + 1382, 1384, 1387, 1388, 1390, 1394, 1395, 1398, 1399, 1403, + 1407, 1409, 1415, 1419, 1421, 1425, 1427, 1430, 1432, 1437, + 1443, 1449, 1456, 1460, 1468, 1473, 1485, 1487, 1491, 1494, + 1497, 1500, 1501, 1505, 1507, 1509, 1512, 1515, 1518, 1521, + 1523, 1524, 1526, 1529, 1536, 1541, 1548, 1553, 1560, 1569, + 1571, 1573, 1575, 1577, 1580, 1582, 1585, 1587, 1590, 1592, + 1594, 1596, 1598, 1602, 1606, 1610, 1614, 1616, 1619, 1622, + 1624, 1628, 1630, 1632, 1634, 1638, 1640, 1642, 1643, 1645, + 1647, 1649, 1659, 1662, 1663, 1667, 1668, 1670, 1671, 1675, + 1679, 1682, 1684, 1691, 1695, 1699, 1702, 1705, 1707, 1708, + 1714, 1717, 1720, 1721, 1729, 1731, 1733, 1735, 1738, 1744, + 1753, 1761, 1767, 1776, 1784, 1789, 1794, 1796, 1800, 1802, + 1804, 1808, 1810, 1814, 1816, 1818, 1821, 1826, 1830, 1832, + 1836, 1839, 1844, 1849, 1858, 1870, 1880, 1888, 1889, 1893, + 1897, 1899, 1901, 1905, 1906, 1908, 1909, 1911, 1912, 1914, + 1915, 1917, 1921, 1924, 1925, 1928, 1929, 1931, 1932, 1934, + 1936, 1938, 1942, 1946, 1948, 1950, 1954, 1958, 1962, 1966, + 1970, 1974, 1979, 1983, 1986, 1988, 1990, 1992, 1996, 1998, + 2002, 2004, 2006, 2008, 2012, 2016, 2020, 2022, 2025, 2030, + 2035, 2038, 2042, 2048, 2054, 2056, 2058, 2062, 2063, 2075, + 2087, 2098, 2111, 2113, 2116, 2122, 2127, 2132, 2137, 2142, + 2150, 2156, 2161, 2169, 2176, 2186, 2196, 2201, 2203, 2205, + 2207, 2209, 2211, 2213, 2215, 2221, 2223, 2225, 2229, 2231, + 2234, 2237, 2240, 2244, 2246, 2250, 2259, 2265, 2266, 2268, + 2271, 2273, 2277, 2279, 2282, 2283, 2286, 2287, 2291, 2295, + 2300, 2305, 2310, 2315, 2319, 2322, 2324, 2326, 2327, 2329, + 2331, 2332, 2335, 2337, 2343, 2345, 2346, 2349, 2352, 2353, + 2355, 2356, 2360, 2366, 2368, 2372, 2377, 2381, 2383, 2385, + 2386, 2389, 2392, 2393, 2396, 2399, 2401, 2403, 2405, 2406, + 2409, 2414, 2420, 2425, 2428, 2432, 2434, 2436, 2438, 2441, + 2444, 2446, 2449, 2453, 2454, 2456, 2457, 2463, 2465, 2470, + 2477, 2480, 2482, 2483, 2488, 2489, 2491, 2493, 2497, 2502, + 2503, 2505, 2507, 2510, 2513, 2516, 2518, 2520, 2523, 2526, + 2528, 2530, 2532, 2534, 2536, 2538, 2542, 2546, 2547, 2549, + 2553, 2555, 2558, 2560, 2562, 2564, 2566, 2568, 2571, 2576, + 2581, 2587, 2589, 2591, 2594, 2595, 2598, 2599, 2601, 2605, + 2607, 2608, 2610, 2613, 2617, 2620, 2625, 2628, 2632, 2635, + 2636, 2638, 2641, 2642, 2647, 2653, 2655, 2658, 2661, 2662, + 2664, 2668, 2670, 2673, 2676, 2681, 2686, 2690, 2694, 2698, + 2702, 2706, 2710, 2714, 2716, 2721, 2726, 2736, 2746, 2750, + 2751, 2754, 2757, 2758, 2764, 2768, 2770, 2772, 2776, 2782, + 2786, 2788, 2791, 2793, 2797, 2803, 2805, 2808, 2812, 2817, + 2823, 2828, 2834, 2839, 2846, 2852, 2857, 2863, 2869, 2875, + 2878, 2883, 2885, 2887, 2888, 2890, 2895, 2901, 2906, 2907, + 2910, 2913, 2916, 2918, 2920, 2922, 2924, 2925, 2930, 2933, + 2935, 2938, 2941, 2946, 2949, 2956, 2959, 2961, 2965, 2970, + 2971, 2974, 2975, 2978, 2979, 2981, 2985, 2989, 2992, 2993, + 2996, 3001, 3003, 3005, 3007, 3008, 3011, 3015, 3021, 3028, + 3031, 3035, 3038, 3044, 3050, 3056, 3060, 3064, 3068, 3073, + 3074, 3076, 3078, 3080, 3082, 3084, 3087, 3092, 3094, 3096, + 3098, 3100, 3103, 3107, 3108, 3110, 3112, 3114, 3116, 3118, + 3121, 3124, 3127, 3130, 3133, 3135, 3139, 3140, 3142, 3144, + 3146, 3148, 3154, 3157, 3159, 3161, 3163, 3165, 3170, 3172, + 3175, 3178, 3180, 3184, 3188, 3191, 3193, 3194, 3200, 3203, + 3209, 3212, 3214, 3218, 3222, 3223, 3225, 3227, 3229, 3231, + 3233, 3235, 3237, 3239, 3241, 3243, 3245, 3247, 3249, 3251, + 3253, 3255, 3257, 3259, 3261, 3263, 3265, 3267, 3269, 3271, + 3273, 3275, 3277, 3279, 3281, 3283, 3285, 3287, 3289, 3291, + 3293, 3295, 3297, 3299, 3301, 3305, 3309, 3313, 3317, 3321, + 3325, 3329, 3330, 3332, 3336, 3340, 3346, 3349, 3352, 3356, + 3360, 3364, 3368, 3372, 3376, 3380, 3384, 3388, 3392, 3396, + 3400, 3404, 3408, 3412, 3415, 3418, 3422, 3426, 3429, 3432, + 3436, 3440, 3446, 3451, 3458, 3462, 3468, 3473, 3480, 3485, + 3492, 3498, 3506, 3510, 3513, 3518, 3522, 3525, 3530, 3534, + 3538, 3542, 3546, 3551, 3555, 3560, 3564, 3569, 3575, 3582, + 3589, 3597, 3604, 3612, 3619, 3627, 3631, 3636, 3641, 3648, + 3650, 3655, 3660, 3666, 3671, 3678, 3680, 3684, 3687, 3690, + 3694, 3698, 3702, 3706, 3710, 3714, 3718, 3722, 3726, 3730, + 3734, 3738, 3742, 3746, 3750, 3753, 3756, 3762, 3769, 3776, + 3784, 3786, 3789, 3791, 3793, 3795, 3798, 3801, 3806, 3810, + 3812, 3814, 3816, 3818, 3821, 3823, 3825, 3827, 3829, 3831, + 3833, 3835, 3838, 3843, 3846, 3850, 3854, 3859, 3863, 3869, + 3876, 3884, 3894, 3902, 3910, 3916, 3918, 3920, 3922, 3928, + 3935, 3942, 3947, 3952, 3957, 3962, 3969, 3975, 3981, 3987, + 3992, 3999, 4004, 4012, 4022, 4028, 4029, 4035, 4040, 4041, + 4043, 4044, 4047, 4048, 4050, 4054, 4058, 4061, 4064, 4065, + 4072, 4074, 4075, 4079, 4080, 4084, 4088, 4092, 4093, 4095, + 4100, 4103, 4106, 4109, 4112, 4115, 4119, 4122, 4125, 4129, + 4130, 4135, 4139, 4141, 4147, 4151, 4153, 4157, 4159, 4162, + 4166, 4168, 4172, 4174, 4177, 4179, 4180, 4182, 4184, 4186, + 4188, 4190, 4192, 4194, 4196, 4198, 4200, 4202, 4204, 4206, + 4208, 4210, 4212, 4214, 4216, 4218, 4220, 4225, 4227, 4232, + 4234, 4239, 4241, 4244, 4246, 4249, 4251, 4254, 4256, 4260, + 4262, 4266, 4268, 4271, 4273, 4277, 4279, 4282, 4284, 4285, + 4287, 4291, 4293, 4297, 4301, 4303, 4307, 4311, 4312, 4314, + 4316, 4318, 4320, 4322, 4324, 4326, 4328, 4330, 4332, 4334, + 4336, 4338, 4340, 4342, 4347, 4351, 4354, 4358, 4359, 4363, + 4367, 4370, 4373, 4375, 4376, 4379, 4382, 4386, 4389, 4391, + 4393, 4397, 4399, 4401, 4407, 4409, 4412, 4417, 4420, 4421, + 4423, 4424, 4426, 4428, 4431, 4435, 4441, 4449, 4457, 4459, + 4460, 4461, 4464, 4465, 4468, 4472, 4476, 4480, 4486, 4494, + 4502, 4503, 4506, 4508, 4509, 4511, 4512, 4514, 4518, 4520, + 4523, 4527, 4530, 4532, 4536, 4541, 4544, 4546, 4550, 4552, + 4556, 4558, 4561, 4563, 4564, 4568, 4570, 4574, 4576, 4579, + 4584, 4587, 4588, 4592, 4594, 4598, 4600, 4603, 4608, 4611, + 4612, 4614, 4618, 4620, 4624, 4626, 4629, 4631, 4635, 4637, + 4639, 4642, 4644, 4646, 4649, 4651, 4653, 4656, 4664, 4667, + 4673, 4677, 4681, 4683, 4685, 4687, 4689, 4691, 4693, 4695, + 4697, 4699, 4701, 4703, 4705, 4707, 4709, 4712, 4715, 4719, + 4723, 4724, 4726, 4728, 4730, 4736, 4740, 4741, 4743, 4745, + 4747, 4749, 4751, 4753, 4758, 4766, 4773, 4776, 4777, 4779, + 4781, 4783, 4785, 4799, 4816, 4818, 4821, 4822, 4824, 4825, + 4827, 4828, 4831, 4832, 4834, 4835, 4842, 4851, 4858, 4867, + 4874, 4883, 4887, 4890, 4892, 4893, 4900, 4907, 4909, 4911, + 4913, 4915, 4917, 4919, 4922, 4924, 4926, 4928, 4930, 4932, + 4937, 4944, 4948, 4951, 4956, 4960, 4966, 4968, 4969, 4971, + 4973, 4974, 4976, 4978, 4980, 4982, 4984, 4986, 4988, 4990, + 4992, 4994, 4996, 4998, 5000, 5002, 5004, 5006, 5008, 5010, + 5012, 5014, 5016, 5018, 5020, 5022, 5024, 5026, 5028, 5030, + 5032, 5034, 5036, 5038, 5040, 5042, 5044, 5046, 5048, 5050, + 5054, 5056, 5058, 5060, 5062, 5064, 5066, 5069, 5071, 5073, + 5076, 5080, 5084, 5088, 5092, 5094, 5098, 5102, 5105, 5109, + 5113, 5115, 5117, 5119, 5123, 5129, 5131, 5133, 5135, 5137, + 5141, 5144, 5149, 5156, 5163, 5164, 5166, 5168, 5170, 5171, + 5174, 5177, 5182, 5189, 5195, 5200, 5207, 5209, 5211, 5213, + 5215, 5217, 5219, 5220, 5222, 5226, 5228, 5229, 5237, 5241, + 5243, 5246, 5250, 5253, 5254, 5257, 5258, 5261, 5266, 5272, + 5281, 5289, 5292, 5296, 5302, 5304, 5305, 5308, 5309, 5311, + 5312, 5315, 5317, 5321, 5325, 5326, 5329, 5333, 5337, 5341, + 5345, 5347, 5349, 5351, 5354, 5358, 5361, 5364, 5367, 5372, + 5375, 5379, 5384, 5388, 5390, 5392, 5394, 5396, 5398, 5400, + 5401, 5403, 5407, 5410, 5420, 5433, 5445, 5458, 5473, 5477, + 5482, 5487, 5488, 5496, 5507, 5517, 5520, 5524, 5525, 5530, + 5532, 5534, 5536, 5538, 5540, 5542, 5544, 5546, 5548, 5550, + 5552, 5554, 5556, 5558, 5560, 5562, 5564, 5566, 5568, 5570, + 5572, 5574, 5576, 5578, 5580, 5582, 5584, 5586, 5588, 5590, + 5592, 5594, 5596, 5598, 5600, 5602, 5604, 5606, 5608, 5610, + 5612, 5614, 5616, 5618, 5620, 5622, 5624, 5626, 5628, 5630, + 5632, 5634, 5636, 5638, 5640, 5642, 5644, 5646, 5648, 5650, + 5652, 5654, 5656, 5658, 5660, 5662, 5664, 5666, 5668, 5670, + 5672, 5674, 5676, 5678, 5680, 5682, 5684, 5686, 5688, 5690, + 5692, 5694, 5696, 5698, 5700, 5702, 5704, 5706, 5708, 5710, + 5712, 5714, 5716, 5718, 5720, 5722, 5724, 5726, 5728, 5730, + 5732, 5734, 5736, 5738, 5740, 5742, 5744, 5746, 5748, 5750, + 5752, 5754, 5756, 5758, 5760, 5762, 5764, 5766, 5768, 5770, + 5772, 5774, 5776, 5778, 5780, 5782, 5784, 5786, 5788, 5790, + 5792, 5794, 5796, 5798, 5800, 5802, 5804, 5806, 5808, 5810, + 5812, 5814, 5816, 5818, 5820, 5822, 5824, 5826, 5828, 5830, + 5832, 5834, 5836, 5838, 5840, 5842, 5844, 5846, 5848, 5850, + 5852, 5854, 5856, 5858, 5860, 5862, 5864, 5866, 5868, 5870, + 5872, 5874, 5876, 5878, 5880, 5882, 5884, 5886, 5888, 5890, + 5892, 5894, 5896, 5898, 5900, 5902, 5904, 5906, 5908, 5910, + 5912, 5914, 5916, 5918, 5920, 5922, 5924, 5926, 5928, 5930, + 5932, 5934, 5936, 5938, 5940, 5942, 5944, 5946, 5948, 5950, + 5952, 5954, 5956, 5958, 5960, 5962, 5964, 5966, 5968, 5970, + 5972, 5974, 5976, 5978, 5980, 5982, 5984, 5986, 5988, 5990, + 5992, 5994, 5996, 5998, 6000, 6002, 6004, 6006, 6008, 6010, + 6012, 6014, 6016, 6018, 6020, 6022, 6024, 6026, 6028, 6030, + 6032, 6034, 6036, 6038, 6040, 6042, 6044, 6046, 6048, 6050, + 6052, 6054, 6056, 6058, 6060, 6062, 6064, 6066, 6068, 6070, + 6072, 6074, 6076, 6078, 6080, 6082, 6084, 6086, 6088, 6090, + 6092, 6094, 6096, 6098, 6100, 6102, 6104, 6106, 6108, 6110, + 6112, 6114, 6116, 6118, 6120, 6122, 6124, 6126, 6128, 6130, + 6132, 6134, 6136, 6138, 6140, 6142, 6144, 6146, 6148, 6150, + 6152, 6154, 6156, 6158, 6160, 6162, 6164, 6166, 6168, 6170, + 6172, 6174, 6176, 6178, 6180, 6182, 6184, 6186, 6188, 6190, + 6192, 6194, 6196, 6198, 6200, 6202, 6204, 6206, 6208, 6210, + 6212, 6214, 6216, 6218, 6220, 6222, 6224, 6226, 6228, 6230, + 6232, 6234, 6236, 6238, 6240, 6242, 6244, 6246, 6248, 6250, + 6252, 6254, 6256, 6258, 6260, 6262, 6264, 6266, 6268, 6270, + 6272, 6274, 6276, 6278, 6280, 6282, 6284, 6286, 6288, 6290, + 6292, 6294, 6296, 6298, 6300, 6302, 6304, 6306, 6308, 6310, + 6312, 6314, 6316, 6318, 6320, 6322, 6324, 6326, 6328, 6330, + 6332, 6334, 6336, 6338, 6340, 6342, 6344, 6346, 6348, 6350, + 6352, 6354, 6356, 6358, 6360, 6362, 6364, 6366, 6368, 6370, + 6372, 6374, 6376, 6378, 6380, 6382, 6384, 6386, 6388, 6390, + 6392, 6394, 6396, 6398, 6400, 6402, 6404, 6406, 6408, 6410, + 6412, 6414, 6416, 6418, 6420, 6422, 6424, 6426, 6428, 6430, + 6432, 6434, 6436, 6438, 6440, 6442, 6444, 6446, 6448, 6450, + 6452, 6454, 6456, 6458, 6460, 6462, 6464, 6466, 6468, 6470, + 6472, 6474, 6476, 6478, 6480, 6482, 6484, 6486, 6488, 6490, + 6492, 6494, 6496, 6498, 6500, 6502, 6504, 6506, 6508, 6510, + 6512, 6514, 6516, 6518, 6520, 6522, 6524, 6526, 6528, 6530, + 6532, 6534, 6536, 6538, 6540, 6542, 6544, 6546, 6548, 6550, + 6552, 6554, 6556, 6558, 6560, 6562, 6564, 6566, 6568, 6570, + 6572, 6574, 6576, 6578, 6580, 6582, 6584, 6586, 6588, 6590, + 6592, 6594, 6596, 6598, 6600, 6602, 6604, 6606, 6608, 6610, + 6612, 6614, 6616, 6618, 6620, 6622, 6624, 6626, 6628, 6630, + 6632, 6634, 6636, 6638, 6640, 6642, 6644, 6646, 6648, 6650, + 6652, 6654, 6656, 6658, 6660, 6662, 6664, 6666, 6668, 6670, + 6672, 6674, 6676, 6678, 6680, 6682, 6684, 6686, 6688, 6690, + 6692, 6694, 6696, 6698, 6700, 6702, 6704, 6706, 6708, 6710, + 6712, 6714, 6716, 6718, 6720, 6722, 6724, 6726, 6728, 6730, + 6732, 6734, 6736, 6738, 6740, 6742, 6744, 6746, 6748, 6750, + 6752, 6754, 6756, 6758, 6760, 6762, 6764, 6766, 6768, 6770, + 6772, 6774, 6776, 6778, 6780, 6782, 6784, 6786, 6788, 6790, + 6792, 6794, 6796, 6798, 6800, 6802, 6804, 6806 }; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ static const yytype_int16 yyrhs[] = { 539, 0, -1, 540, -1, 540, 531, 541, -1, 541, - -1, 979, -1, 608, -1, 542, -1, 1017, -1, 1018, - -1, 1034, -1, 980, -1, 982, -1, 700, -1, 1037, - -1, 688, -1, 969, -1, 596, -1, 594, -1, 621, - -1, 589, -1, 557, -1, 1013, -1, 1019, -1, 615, - -1, 671, -1, 604, -1, 987, -1, 985, -1, 986, - -1, 972, -1, 568, -1, 1004, -1, 677, -1, 593, - -1, 966, -1, 566, -1, 713, -1, 617, -1, 603, - -1, 699, -1, 620, -1, 1008, -1, 1026, -1, 998, - -1, 1029, -1, 1035, -1, -1, 33, 427, 808, 554, - -1, 33, 427, 194, 154, 808, 554, -1, 33, 205, - 558, 554, -1, 33, 205, 194, 154, 558, 554, -1, - 33, 390, 558, 554, -1, 33, 390, 194, 154, 558, - 554, -1, 33, 480, 558, 554, -1, 33, 480, 194, - 154, 558, 554, -1, 545, -1, 543, 545, -1, 395, - 118, 857, -1, 138, 118, -1, 365, -1, 365, 610, - 611, -1, 395, 612, -1, 395, 178, 670, -1, 553, - -1, 546, 532, 553, -1, 548, -1, 547, 548, -1, - 530, 564, -1, 559, -1, 559, 547, -1, 549, 648, - -1, 549, 649, -1, 27, 550, -1, 27, 194, 280, - 154, 550, -1, 27, 83, 550, -1, 27, 83, 194, - 280, 154, 550, -1, 395, 316, 60, 528, 905, 529, - -1, 363, 316, 60, -1, 395, 406, 60, 528, 742, - 529, -1, 363, 406, 60, -1, 33, 567, 559, 544, - -1, 33, 567, 559, 138, 280, 285, -1, 33, 567, - 559, 395, 280, 285, -1, 33, 567, 559, 395, 413, - 614, -1, 33, 567, 559, 395, 636, -1, 33, 567, - 559, 363, 636, -1, 33, 567, 559, 395, 416, 559, - -1, 33, 567, 559, 27, 178, 670, 41, 193, 624, - -1, 33, 567, 559, 543, -1, 33, 567, 559, 138, - 193, -1, 33, 567, 559, 138, 193, 194, 154, -1, - 138, 567, 194, 154, 549, 675, -1, 138, 567, 549, - 675, -1, 33, 567, 559, 556, 451, 820, 817, 552, - -1, 33, 567, 559, 555, -1, 27, 638, -1, 33, - 94, 954, 622, -1, 470, 94, 954, -1, 138, 94, - 194, 154, 954, 675, -1, 138, 94, 954, 675, -1, - 395, 248, -1, 395, 460, -1, 395, 636, -1, 363, - 636, -1, 555, -1, 467, 857, -1, -1, 632, -1, - 395, 632, -1, 27, 632, -1, 138, 646, -1, 551, - -1, 554, 532, 551, -1, 299, 528, 546, 529, -1, - 395, 108, -1, 395, -1, -1, 112, 954, -1, 112, - 332, 954, -1, 112, 31, -1, 112, 332, 31, -1, - 560, -1, 559, 562, -1, 3, -1, 1040, -1, 1041, - -1, 559, -1, 5, -1, 5, -1, 563, -1, 562, - 563, -1, 530, 564, -1, 565, -1, 3, -1, 1044, - -1, 1040, -1, 1046, -1, 33, 379, 954, 359, 440, - 954, -1, 33, 427, 808, 359, 440, 954, -1, 33, - 427, 194, 154, 808, 359, 440, 954, -1, 33, 390, - 558, 359, 440, 954, -1, 33, 390, 194, 154, 558, - 359, 440, 954, -1, 33, 480, 558, 359, 440, 954, - -1, 33, 480, 194, 154, 558, 359, 440, 954, -1, - 33, 205, 558, 359, 440, 954, -1, 33, 205, 194, - 154, 558, 359, 440, 954, -1, 33, 427, 808, 359, - 567, 549, 440, 954, -1, 33, 427, 194, 154, 808, - 359, 567, 549, 440, 954, -1, 33, 427, 808, 359, - 94, 954, 440, 954, -1, 33, 427, 194, 154, 808, - 359, 94, 954, 440, 954, -1, 83, -1, -1, 573, - 215, 576, 222, 570, 571, 569, 577, 579, -1, 713, - -1, 309, 580, 472, 713, -1, 528, 584, 529, 713, - -1, 528, 584, 529, 309, 580, 472, 713, -1, 118, - 473, -1, 558, -1, 558, 41, 559, -1, 60, 271, - -1, 60, 327, -1, -1, 528, 587, 529, 814, -1, - 295, 94, 954, -1, -1, 725, -1, -1, 559, 928, - -1, 588, 517, 857, -1, 528, 581, 529, 517, 857, - -1, 300, 361, -1, 300, 195, -1, -1, 295, 92, - 572, 134, 464, 395, 586, 814, -1, 295, 92, 572, - 134, 281, -1, -1, 559, 582, 583, 744, 745, -1, - 869, 582, 583, 744, 745, -1, 528, 857, 529, 582, - 583, 744, 745, -1, 367, 934, -1, -1, 466, -1, - 426, -1, 588, -1, 581, 532, 588, -1, 81, 961, - -1, -1, 961, -1, -1, 574, -1, 584, 532, 574, - -1, 575, -1, 585, 532, 575, -1, 585, -1, 585, - 532, -1, 578, -1, 587, 532, 578, -1, 559, 928, - -1, 101, 669, 451, 558, 41, 590, -1, 101, 669, - 451, 194, 280, 154, 558, 41, 590, -1, 101, 300, - 361, 669, 451, 558, 41, 590, -1, 145, 714, -1, - 145, 528, 591, 529, -1, 821, -1, 592, -1, -1, - 561, -1, 592, 532, 561, -1, 329, 559, -1, 329, - 559, 517, 1003, -1, 329, 559, 528, 907, 529, -1, - 101, 669, 390, 558, 595, -1, 101, 669, 390, 194, - 280, 154, 558, 595, -1, 101, 300, 361, 669, 390, - 558, 595, -1, 609, -1, -1, 101, 598, 386, 597, - 599, 528, 602, 529, -1, 101, 598, 386, 194, 280, - 154, 597, 599, 528, 602, 529, -1, 101, 300, 361, - 598, 386, 597, 599, 528, 602, 529, -1, -1, 559, - -1, -1, 434, -1, 320, -1, -1, 201, 3, -1, - 857, -1, 565, 600, -1, 601, -1, 602, 532, 601, - -1, 573, 464, 159, 658, -1, 153, 954, 607, -1, - 101, 669, 427, 1039, 41, 153, 954, 607, 1038, -1, - 101, 669, 427, 194, 280, 154, 1039, 41, 153, 954, - 607, 1038, -1, 857, -1, 964, 13, 857, -1, 605, - -1, 606, 532, 605, -1, 528, 606, 529, -1, -1, - 33, 390, 558, 609, -1, 33, 390, 194, 154, 558, - 609, -1, 612, -1, 609, 612, -1, 490, -1, 514, - -1, -1, 4, -1, 519, 4, -1, 520, 4, -1, - 614, -1, 41, 824, -1, 61, 611, -1, 107, -1, - 278, 107, -1, 204, 613, 611, -1, 255, 611, -1, - 266, 611, -1, 278, 255, -1, 278, 266, -1, 310, - 60, 961, -1, 390, 271, 961, -1, 411, 610, 611, - -1, 365, -1, 365, 610, 611, -1, 60, -1, -1, - 957, -1, 519, 957, -1, 520, 957, -1, 138, 598, - 386, 559, 616, -1, 138, 598, 386, 194, 154, 559, - 616, -1, -1, 174, 3, -1, 23, 618, -1, 53, - 618, 619, -1, 411, 618, 619, -1, 87, 618, -1, - 144, 618, -1, 372, 618, -1, 493, -1, 442, -1, - -1, 347, 296, -1, 347, 495, -1, -1, 465, 558, - -1, 101, 669, 427, 558, 528, 656, 529, 643, 635, - -1, 101, 669, 427, 194, 280, 154, 558, 528, 656, - 529, 643, 635, -1, 101, 300, 361, 669, 427, 558, - 528, 656, 529, 643, 635, -1, -1, 622, 647, -1, - 664, -1, 1046, -1, 899, -1, 611, -1, 561, -1, - 279, -1, 528, 609, 529, -1, -1, 561, -1, 278, - 26, -1, 366, -1, 64, -1, 395, 285, -1, 395, - 118, -1, 94, 954, 628, -1, 628, -1, 642, -1, - 81, 961, -1, 280, 285, -1, 285, -1, 457, 655, - -1, 335, 229, 655, -1, 75, 528, 857, 529, 637, - -1, 467, 89, 954, -1, 118, 858, -1, 353, 558, - 658, 667, 634, -1, 482, -1, 417, -1, 629, -1, - -1, 178, 670, 41, 193, 624, -1, 178, 670, 41, - 528, 857, 529, 630, -1, 41, 528, 857, 529, 630, - -1, 646, 625, -1, 295, 464, 626, -1, 633, -1, - 660, -1, 633, 660, -1, 660, 633, -1, -1, 295, - 87, 138, -1, 295, 87, 123, 375, -1, 295, 87, - 334, 375, -1, -1, 528, 640, 529, -1, 278, 207, - -1, -1, 94, 954, 665, -1, 665, -1, 86, -1, - 95, -1, 119, -1, 193, -1, 206, -1, 413, -1, - 416, -1, 31, -1, 661, -1, 640, 532, 661, -1, - 467, 205, 652, -1, 120, -1, 280, 120, -1, 209, - 121, -1, 209, 197, -1, 490, 636, -1, 490, 293, - -1, 492, 293, -1, -1, 528, 651, 529, -1, 645, - 203, 639, -1, 645, 151, 639, -1, -1, 565, -1, - 280, 120, -1, 120, -1, 209, 197, -1, 209, 121, - -1, 280, 469, -1, 278, 207, -1, 821, 659, -1, - 820, 631, 659, -1, 559, 648, -1, 559, 649, -1, - 654, -1, 651, 532, 654, -1, 559, -1, 650, -1, - 668, -1, 638, -1, 565, 517, 623, -1, 565, -1, - 490, 644, -1, -1, 666, -1, 666, 532, -1, -1, - 559, -1, 528, 662, 529, -1, -1, 659, 627, -1, - -1, 295, 123, 626, -1, 565, 517, 623, -1, 565, - -1, 565, 530, 565, 517, 623, -1, 565, 530, 565, - -1, 657, -1, 662, 532, 657, -1, 662, -1, 662, - 532, -1, 821, -1, 958, 962, 523, 451, -1, 396, - 958, 962, 523, 451, -1, 75, 528, 857, 529, 622, - -1, 457, 528, 663, 529, 655, 622, -1, 457, 641, - 622, -1, 335, 229, 528, 663, 529, 655, 622, -1, - 335, 229, 641, 622, -1, 171, 229, 528, 663, 529, - 353, 558, 658, 667, 634, 622, -1, 653, -1, 666, - 532, 653, -1, 252, 175, -1, 252, 314, -1, 252, - 401, -1, -1, 240, 558, 645, -1, 434, -1, 432, - -1, 244, 434, -1, 244, 432, -1, 180, 434, -1, - 180, 432, -1, 460, -1, -1, 34, -1, 60, 118, - -1, 138, 672, 194, 154, 674, 675, -1, 138, 672, - 674, 675, -1, 138, 673, 194, 154, 951, 675, -1, - 138, 673, 951, 675, -1, 138, 676, 954, 295, 961, - 675, -1, 138, 676, 194, 154, 954, 295, 961, 675, + -1, 615, -1, 980, -1, 608, -1, 542, -1, 1018, + -1, 1019, -1, 1035, -1, 981, -1, 983, -1, 701, + -1, 1038, -1, 689, -1, 970, -1, 596, -1, 594, + -1, 622, -1, 589, -1, 557, -1, 1014, -1, 1020, + -1, 616, -1, 672, -1, 604, -1, 988, -1, 986, + -1, 987, -1, 973, -1, 568, -1, 1005, -1, 678, + -1, 593, -1, 967, -1, 566, -1, 714, -1, 618, + -1, 603, -1, 700, -1, 621, -1, 1009, -1, 1027, + -1, 999, -1, 1030, -1, 1036, -1, -1, 33, 427, + 809, 554, -1, 33, 427, 194, 154, 809, 554, -1, + 33, 205, 558, 554, -1, 33, 205, 194, 154, 558, + 554, -1, 33, 390, 558, 554, -1, 33, 390, 194, + 154, 558, 554, -1, 33, 480, 558, 554, -1, 33, + 480, 194, 154, 558, 554, -1, 545, -1, 543, 545, + -1, 395, 118, 858, -1, 138, 118, -1, 365, -1, + 365, 610, 611, -1, 395, 612, -1, 395, 178, 671, + -1, 553, -1, 546, 532, 553, -1, 548, -1, 547, + 548, -1, 530, 564, -1, 559, -1, 559, 547, -1, + 549, 649, -1, 549, 650, -1, 27, 550, -1, 27, + 194, 280, 154, 550, -1, 27, 83, 550, -1, 27, + 83, 194, 280, 154, 550, -1, 395, 316, 60, 528, + 906, 529, -1, 363, 316, 60, -1, 395, 406, 60, + 528, 743, 529, -1, 363, 406, 60, -1, 33, 567, + 559, 544, -1, 33, 567, 559, 138, 280, 285, -1, + 33, 567, 559, 395, 280, 285, -1, 33, 567, 559, + 395, 413, 614, -1, 33, 567, 559, 395, 637, -1, + 33, 567, 559, 363, 637, -1, 33, 567, 559, 395, + 416, 559, -1, 33, 567, 559, 27, 178, 671, 41, + 193, 625, -1, 33, 567, 559, 543, -1, 33, 567, + 559, 138, 193, -1, 33, 567, 559, 138, 193, 194, + 154, -1, 138, 567, 194, 154, 549, 676, -1, 138, + 567, 549, 676, -1, 33, 567, 559, 556, 451, 821, + 818, 552, -1, 33, 567, 559, 555, -1, 27, 639, + -1, 33, 94, 955, 623, -1, 470, 94, 955, -1, + 138, 94, 194, 154, 955, 676, -1, 138, 94, 955, + 676, -1, 395, 248, -1, 395, 460, -1, 395, 637, + -1, 363, 637, -1, 555, -1, 467, 858, -1, -1, + 633, -1, 395, 633, -1, 27, 633, -1, 138, 647, + -1, 551, -1, 554, 532, 551, -1, 299, 528, 546, + 529, -1, 395, 108, -1, 395, -1, -1, 112, 955, + -1, 112, 332, 955, -1, 112, 31, -1, 112, 332, + 31, -1, 560, -1, 559, 562, -1, 3, -1, 1041, + -1, 1042, -1, 559, -1, 5, -1, 5, -1, 563, + -1, 562, 563, -1, 530, 564, -1, 565, -1, 3, + -1, 1045, -1, 1041, -1, 1047, -1, 33, 379, 955, + 359, 440, 955, -1, 33, 427, 809, 359, 440, 955, + -1, 33, 427, 194, 154, 809, 359, 440, 955, -1, + 33, 390, 558, 359, 440, 955, -1, 33, 390, 194, + 154, 558, 359, 440, 955, -1, 33, 480, 558, 359, + 440, 955, -1, 33, 480, 194, 154, 558, 359, 440, + 955, -1, 33, 205, 558, 359, 440, 955, -1, 33, + 205, 194, 154, 558, 359, 440, 955, -1, 33, 427, + 809, 359, 567, 549, 440, 955, -1, 33, 427, 194, + 154, 809, 359, 567, 549, 440, 955, -1, 33, 427, + 809, 359, 94, 955, 440, 955, -1, 33, 427, 194, + 154, 809, 359, 94, 955, 440, 955, -1, 83, -1, + -1, 573, 215, 576, 222, 570, 571, 569, 577, 579, + -1, 714, -1, 309, 580, 472, 714, -1, 528, 584, + 529, 714, -1, 528, 584, 529, 309, 580, 472, 714, + -1, 118, 473, -1, 558, -1, 558, 41, 559, -1, + 60, 271, -1, 60, 327, -1, -1, 528, 587, 529, + 815, -1, 295, 94, 955, -1, -1, 726, -1, -1, + 559, 929, -1, 588, 517, 858, -1, 528, 581, 529, + 517, 858, -1, 300, 361, -1, 300, 195, -1, -1, + 295, 92, 572, 134, 464, 395, 586, 815, -1, 295, + 92, 572, 134, 281, -1, -1, 559, 582, 583, 745, + 746, -1, 870, 582, 583, 745, 746, -1, 528, 858, + 529, 582, 583, 745, 746, -1, 367, 935, -1, -1, + 466, -1, 426, -1, 588, -1, 581, 532, 588, -1, + 81, 962, -1, -1, 962, -1, -1, 574, -1, 584, + 532, 574, -1, 575, -1, 585, 532, 575, -1, 585, + -1, 585, 532, -1, 578, -1, 587, 532, 578, -1, + 559, 929, -1, 101, 670, 451, 558, 41, 590, -1, + 101, 670, 451, 194, 280, 154, 558, 41, 590, -1, + 101, 300, 361, 670, 451, 558, 41, 590, -1, 145, + 715, -1, 145, 528, 591, 529, -1, 822, -1, 592, + -1, -1, 561, -1, 592, 532, 561, -1, 329, 559, + -1, 329, 559, 517, 1004, -1, 329, 559, 528, 908, + 529, -1, 101, 670, 390, 558, 595, -1, 101, 670, + 390, 194, 280, 154, 558, 595, -1, 101, 300, 361, + 670, 390, 558, 595, -1, 609, -1, -1, 101, 598, + 386, 597, 599, 528, 602, 529, -1, 101, 598, 386, + 194, 280, 154, 597, 599, 528, 602, 529, -1, 101, + 300, 361, 598, 386, 597, 599, 528, 602, 529, -1, + -1, 559, -1, -1, 434, -1, 320, -1, -1, 201, + 3, -1, 858, -1, 565, 600, -1, 601, -1, 602, + 532, 601, -1, 573, 464, 159, 659, -1, 153, 955, + 607, -1, 101, 670, 427, 1040, 41, 153, 955, 607, + 1039, -1, 101, 670, 427, 194, 280, 154, 1040, 41, + 153, 955, 607, 1039, -1, 858, -1, 965, 13, 858, + -1, 605, -1, 606, 532, 605, -1, 528, 606, 529, + -1, -1, 33, 390, 558, 609, -1, 33, 390, 194, + 154, 558, 609, -1, 612, -1, 609, 612, -1, 490, + -1, 514, -1, -1, 4, -1, 519, 4, -1, 520, + 4, -1, 614, -1, 41, 825, -1, 61, 611, -1, + 107, -1, 278, 107, -1, 204, 613, 611, -1, 255, + 611, -1, 266, 611, -1, 278, 255, -1, 278, 266, + -1, 310, 60, 962, -1, 390, 271, 962, -1, 411, + 610, 611, -1, 365, -1, 365, 610, 611, -1, 60, + -1, -1, 958, -1, 519, 958, -1, 520, 958, -1, + 33, 109, 559, 359, 440, 559, -1, 33, 109, 194, + 154, 559, 359, 440, 559, -1, 138, 598, 386, 559, + 617, -1, 138, 598, 386, 194, 154, 559, 617, -1, + -1, 174, 3, -1, 23, 619, -1, 53, 619, 620, + -1, 411, 619, 620, -1, 87, 619, -1, 144, 619, + -1, 372, 619, -1, 493, -1, 442, -1, -1, 347, + 296, -1, 347, 495, -1, -1, 465, 558, -1, 101, + 670, 427, 558, 528, 657, 529, 644, 636, -1, 101, + 670, 427, 194, 280, 154, 558, 528, 657, 529, 644, + 636, -1, 101, 300, 361, 670, 427, 558, 528, 657, + 529, 644, 636, -1, -1, 623, 648, -1, 665, -1, + 1047, -1, 900, -1, 611, -1, 561, -1, 279, -1, + 528, 609, 529, -1, -1, 561, -1, 278, 26, -1, + 366, -1, 64, -1, 395, 285, -1, 395, 118, -1, + 94, 955, 629, -1, 629, -1, 643, -1, 81, 962, + -1, 280, 285, -1, 285, -1, 457, 656, -1, 335, + 229, 656, -1, 75, 528, 858, 529, 638, -1, 467, + 89, 955, -1, 118, 859, -1, 353, 558, 659, 668, + 635, -1, 482, -1, 417, -1, 630, -1, -1, 178, + 671, 41, 193, 625, -1, 178, 671, 41, 528, 858, + 529, 631, -1, 41, 528, 858, 529, 631, -1, 647, + 626, -1, 295, 464, 627, -1, 634, -1, 661, -1, + 634, 661, -1, 661, 634, -1, -1, 295, 87, 138, + -1, 295, 87, 123, 375, -1, 295, 87, 334, 375, + -1, -1, 528, 641, 529, -1, 278, 207, -1, -1, + 94, 955, 666, -1, 666, -1, 86, -1, 95, -1, + 119, -1, 193, -1, 206, -1, 413, -1, 416, -1, + 31, -1, 662, -1, 641, 532, 662, -1, 467, 205, + 653, -1, 120, -1, 280, 120, -1, 209, 121, -1, + 209, 197, -1, 490, 637, -1, 490, 293, -1, 492, + 293, -1, -1, 528, 652, 529, -1, 646, 203, 640, + -1, 646, 151, 640, -1, -1, 565, -1, 280, 120, + -1, 120, -1, 209, 197, -1, 209, 121, -1, 280, + 469, -1, 278, 207, -1, 822, 660, -1, 821, 632, + 660, -1, 559, 649, -1, 559, 650, -1, 655, -1, + 652, 532, 655, -1, 559, -1, 651, -1, 669, -1, + 639, -1, 565, 517, 624, -1, 565, -1, 490, 645, + -1, -1, 667, -1, 667, 532, -1, -1, 559, -1, + 528, 663, 529, -1, -1, 660, 628, -1, -1, 295, + 123, 627, -1, 565, 517, 624, -1, 565, -1, 565, + 530, 565, 517, 624, -1, 565, 530, 565, -1, 658, + -1, 663, 532, 658, -1, 663, -1, 663, 532, -1, + 822, -1, 959, 963, 523, 451, -1, 396, 959, 963, + 523, 451, -1, 75, 528, 858, 529, 623, -1, 457, + 528, 664, 529, 656, 623, -1, 457, 642, 623, -1, + 335, 229, 528, 664, 529, 656, 623, -1, 335, 229, + 642, 623, -1, 171, 229, 528, 664, 529, 353, 558, + 659, 668, 635, 623, -1, 654, -1, 667, 532, 654, + -1, 252, 175, -1, 252, 314, -1, 252, 401, -1, + -1, 240, 558, 646, -1, 434, -1, 432, -1, 244, + 434, -1, 244, 432, -1, 180, 434, -1, 180, 432, + -1, 460, -1, -1, 34, -1, 60, 118, -1, 138, + 673, 194, 154, 675, 676, -1, 138, 673, 675, 676, + -1, 138, 674, 194, 154, 952, 676, -1, 138, 674, + 952, 676, -1, 138, 677, 955, 295, 962, 676, -1, + 138, 677, 194, 154, 955, 295, 962, 676, -1, 427, + -1, 390, -1, 176, -1, 249, -1, 249, 427, -1, + 480, -1, 254, 480, -1, 205, -1, 171, 427, -1, + 82, -1, 98, -1, 379, -1, 413, -1, 435, 383, + 313, -1, 435, 383, 130, -1, 435, 383, 433, -1, + 435, 383, 91, -1, 451, -1, 25, 257, -1, 148, + 445, -1, 158, -1, 171, 108, 494, -1, 341, -1, + 393, -1, 962, -1, 675, 532, 962, -1, 64, -1, + 366, -1, -1, 326, -1, 376, -1, 445, -1, 573, + 256, 222, 1015, 467, 792, 808, 688, 579, -1, 37, + 858, -1, -1, 528, 584, 529, -1, -1, 521, -1, + -1, 464, 395, 586, -1, 464, 395, 521, -1, 464, + 571, -1, 123, -1, 215, 680, 473, 528, 906, 529, + -1, 215, 571, 681, -1, 215, 118, 473, -1, 134, + 281, -1, 146, 683, -1, 858, -1, -1, 486, 253, + 679, 436, 682, -1, 60, 407, -1, 60, 431, -1, + -1, 486, 280, 253, 685, 679, 436, 682, -1, 684, + -1, 686, -1, 687, -1, 687, 688, -1, 101, 670, + 696, 558, 693, -1, 101, 670, 696, 194, 280, 154, + 558, 693, -1, 101, 300, 361, 670, 696, 558, 693, + -1, 101, 670, 696, 558, 695, -1, 101, 670, 696, + 194, 280, 154, 558, 695, -1, 101, 300, 361, 670, + 696, 558, 695, -1, 697, 41, 427, 716, -1, 697, + 41, 427, 715, -1, 691, -1, 692, 532, 691, -1, + 690, -1, 692, -1, 697, 41, 858, -1, 694, -1, + 695, 532, 694, -1, 176, -1, 249, -1, 528, 529, + -1, 528, 698, 532, 529, -1, 528, 698, 529, -1, + 699, -1, 698, 532, 699, -1, 965, 821, -1, 965, + 821, 13, 858, -1, 965, 821, 14, 858, -1, 573, + 464, 1015, 395, 586, 788, 1016, 579, -1, 99, 711, + 558, 659, 709, 703, 707, 713, 704, 610, 708, -1, + 99, 528, 714, 529, 440, 707, 713, 610, 708, -1, + 99, 174, 109, 559, 440, 559, 702, -1, -1, 528, + 379, 529, -1, 528, 108, 529, -1, 174, -1, 440, + -1, 705, 125, 561, -1, -1, 467, -1, -1, 41, + -1, -1, 340, -1, -1, 710, -1, 528, 1025, 529, + -1, 490, 293, -1, -1, 710, 712, -1, -1, 56, + -1, -1, 56, -1, 293, -1, 173, -1, 124, 706, + 561, -1, 285, 706, 561, -1, 103, -1, 189, -1, + 345, 706, 561, -1, 147, 706, 561, -1, 170, 345, + 663, -1, 170, 345, 521, -1, 315, 60, 663, -1, + 315, 60, 521, -1, 170, 280, 285, 663, -1, 170, + 285, 663, -1, 142, 561, -1, 561, -1, 414, -1, + 415, -1, 3, 530, 559, -1, 3, -1, 528, 858, + 529, -1, 863, -1, 716, -1, 715, -1, 528, 716, + 529, -1, 528, 715, 529, -1, 528, 1030, 529, -1, + 719, -1, 717, 742, -1, 717, 741, 779, 748, -1, + 717, 741, 747, 780, -1, 726, 717, -1, 726, 717, + 742, -1, 726, 717, 741, 779, 748, -1, 726, 717, + 741, 747, 780, -1, 719, -1, 715, -1, 388, 739, + 934, -1, -1, 388, 739, 934, 733, 788, 815, 768, + 777, 876, 778, 753, -1, 388, 738, 936, 733, 788, + 815, 768, 777, 876, 778, 753, -1, 174, 789, 718, + 733, 815, 768, 777, 876, 778, 753, -1, 174, 789, + 388, 738, 936, 733, 815, 768, 777, 876, 778, 753, + -1, 787, -1, 427, 809, -1, 717, 456, 736, 737, + 717, -1, 717, 456, 736, 717, -1, 717, 220, 736, + 717, -1, 717, 149, 736, 717, -1, 721, 792, 467, + 936, -1, 721, 792, 467, 936, 183, 60, 954, -1, + 721, 792, 183, 60, 954, -1, 721, 792, 295, 725, + -1, 721, 792, 295, 725, 183, 60, 954, -1, 721, + 792, 295, 725, 467, 936, -1, 721, 792, 295, 725, + 467, 936, 183, 60, 954, -1, 722, 792, 295, 936, + 222, 271, 955, 720, 954, -1, 722, 792, 295, 936, + -1, 472, -1, 473, -1, 321, -1, 323, -1, 462, + -1, 322, -1, 859, -1, 859, 201, 528, 716, 529, + -1, 795, -1, 723, -1, 724, 532, 723, -1, 724, + -1, 724, 532, -1, 490, 727, -1, 514, 727, -1, + 490, 351, 727, -1, 728, -1, 727, 532, 728, -1, + 955, 964, 729, 41, 732, 528, 969, 529, -1, 467, + 229, 528, 730, 529, -1, -1, 731, -1, 731, 532, + -1, 926, -1, 731, 532, 926, -1, 254, -1, 280, + 254, -1, -1, 222, 734, -1, -1, 434, 735, 558, + -1, 432, 735, 558, -1, 244, 434, 735, 558, -1, + 244, 432, 735, 558, -1, 180, 434, 735, 558, -1, + 180, 432, 735, 558, -1, 460, 735, 558, -1, 427, + 558, -1, 558, -1, 427, -1, -1, 31, -1, 133, + -1, -1, 60, 271, -1, 133, -1, 133, 295, 528, + 906, 529, -1, 31, -1, -1, 195, 287, -1, 364, + 287, -1, -1, 742, -1, -1, 301, 60, 743, -1, + 301, 60, 31, 745, 746, -1, 744, -1, 743, 532, + 744, -1, 858, 467, 900, 746, -1, 858, 745, 746, + -1, 42, -1, 127, -1, -1, 513, 166, -1, 513, + 234, -1, -1, 749, 750, -1, 750, 749, -1, 749, + -1, 750, -1, 747, -1, -1, 241, 762, -1, 241, + 762, 532, 763, -1, 164, 767, 764, 766, 296, -1, + 164, 767, 766, 296, -1, 292, 763, -1, 292, 764, + 766, -1, 4, -1, 9, -1, 863, -1, 751, 523, + -1, 751, 319, -1, 751, -1, 751, 375, -1, 467, + 377, 755, -1, -1, 559, -1, -1, 754, 528, 752, + 529, 758, -1, 752, -1, 752, 528, 559, 529, -1, + 752, 528, 559, 532, 9, 529, -1, 429, 755, -1, + 756, -1, -1, 360, 528, 9, 529, -1, -1, 439, + -1, 479, -1, 759, 14, 858, -1, 47, 528, 760, + 529, -1, -1, 858, -1, 31, -1, 858, 523, -1, + 4, 319, -1, 9, 319, -1, 858, -1, 860, -1, + 519, 765, -1, 520, 765, -1, 958, -1, 4, -1, + 374, -1, 375, -1, 166, -1, 277, -1, 183, 60, + 770, -1, 183, 60, 31, -1, -1, 771, -1, 769, + 532, 771, -1, 769, -1, 769, 532, -1, 858, -1, + 772, -1, 774, -1, 773, -1, 775, -1, 528, 529, + -1, 373, 528, 906, 529, -1, 104, 528, 906, 529, + -1, 184, 397, 528, 770, 529, -1, 184, -1, 185, + -1, 188, 858, -1, -1, 342, 858, -1, -1, 781, + -1, 169, 347, 296, -1, 779, -1, -1, 782, -1, + 781, 782, -1, 783, 784, 785, -1, 169, 464, -1, + 169, 278, 229, 464, -1, 169, 398, -1, 169, 229, + 398, -1, 290, 951, -1, -1, 284, -1, 402, 247, + -1, -1, 473, 528, 906, 529, -1, 786, 532, 528, + 906, 529, -1, 786, -1, 786, 532, -1, 174, 790, + -1, -1, 792, -1, 789, 532, 792, -1, 789, -1, + 789, 532, -1, 560, 19, -1, 809, 804, 761, 757, + -1, 791, 809, 761, 757, -1, 810, 805, 757, -1, + 791, 810, 757, -1, 787, 803, 757, -1, 235, 810, + 805, -1, 715, 804, 757, -1, 791, 715, 757, -1, + 235, 715, 804, -1, 802, -1, 528, 802, 529, 803, + -1, 791, 528, 802, 529, -1, 792, 321, 528, 936, + 169, 798, 793, 529, 804, -1, 792, 462, 794, 528, + 799, 169, 801, 529, 804, -1, 183, 60, 953, -1, + -1, 202, 287, -1, 150, 287, -1, -1, 859, 201, + 528, 936, 529, -1, 859, 201, 560, -1, 861, -1, + 864, -1, 528, 904, 529, -1, 796, 201, 528, 936, + 529, -1, 796, 201, 560, -1, 797, -1, 798, 797, + -1, 560, -1, 528, 953, 529, -1, 799, 201, 528, + 936, 529, -1, 800, -1, 801, 800, -1, 528, 802, + 529, -1, 792, 102, 227, 792, -1, 792, 806, 227, + 792, 808, -1, 792, 227, 792, 808, -1, 792, 274, + 806, 227, 792, -1, 792, 274, 227, 792, -1, 792, + 43, 806, 227, 792, 808, -1, 792, 43, 227, 792, + 808, -1, 792, 328, 227, 792, -1, 792, 38, 227, + 792, 808, -1, 792, 389, 227, 792, 808, -1, 41, + 560, 528, 953, 529, -1, 41, 560, -1, 559, 528, + 953, 529, -1, 559, -1, 803, -1, -1, 803, -1, + 41, 528, 816, 529, -1, 41, 560, 528, 816, 529, + -1, 559, 528, 816, 529, -1, -1, 175, 807, -1, + 238, 807, -1, 370, 807, -1, 389, -1, 38, -1, + 211, -1, 305, -1, -1, 467, 528, 953, 529, -1, + 295, 858, -1, 558, -1, 558, 521, -1, 296, 558, + -1, 296, 528, 558, 529, -1, 870, 814, -1, 375, + 174, 528, 812, 529, 814, -1, 870, 813, -1, 811, + -1, 812, 532, 811, -1, 41, 528, 816, 529, -1, + -1, 514, 302, -1, -1, 487, 858, -1, -1, 817, + -1, 816, 532, 817, -1, 560, 822, 818, -1, 81, + 962, -1, -1, 559, 822, -1, 819, 532, 559, 822, + -1, 374, -1, 420, -1, 822, -1, -1, 825, 824, + -1, 396, 825, 824, -1, 825, 40, 526, 958, 527, + -1, 396, 825, 40, 526, 958, 527, -1, 825, 40, + -1, 396, 825, 40, -1, 823, 824, -1, 820, 528, + 819, 529, 824, -1, 250, 528, 910, 529, 824, -1, + 456, 528, 819, 529, 824, -1, 3, 530, 3, -1, + 823, 530, 3, -1, 824, 526, 527, -1, 824, 526, + 958, 527, -1, -1, 827, -1, 829, -1, 831, -1, + 835, -1, 841, -1, 842, 857, -1, 842, 528, 958, + 529, -1, 829, -1, 832, -1, 836, -1, 841, -1, + 961, 828, -1, 528, 907, 529, -1, -1, 218, -1, + 219, -1, 403, -1, 55, -1, 348, -1, 167, 830, + -1, 137, 331, -1, 116, 828, -1, 113, 828, -1, + 288, 828, -1, 58, -1, 528, 958, 529, -1, -1, + 833, -1, 834, -1, 833, -1, 834, -1, 57, 840, + 528, 906, 529, -1, 57, 840, -1, 837, -1, 838, + -1, 837, -1, 838, -1, 839, 528, 958, 529, -1, + 839, -1, 73, 840, -1, 72, 840, -1, 474, -1, + 273, 73, 840, -1, 273, 72, 840, -1, 275, 840, + -1, 477, -1, -1, 439, 528, 958, 529, 843, -1, + 439, 843, -1, 438, 528, 958, 529, 843, -1, 438, + 843, -1, 221, -1, 514, 438, 511, -1, 492, 438, + 511, -1, -1, 508, -1, 509, -1, 268, -1, 269, + -1, 110, -1, 111, -1, 191, -1, 192, -1, 264, + -1, 265, -1, 384, -1, 385, -1, 262, -1, 263, + -1, 258, -1, 259, -1, 484, -1, 485, -1, 343, + -1, 344, -1, 114, -1, 115, -1, 70, -1, 69, + -1, 261, -1, 260, -1, 844, -1, 845, -1, 846, + -1, 847, -1, 848, -1, 849, -1, 850, -1, 851, + -1, 852, -1, 853, -1, 854, -1, 855, -1, 856, + -1, 844, 440, 845, -1, 846, 440, 847, -1, 846, + 440, 848, -1, 846, 440, 849, -1, 847, 440, 848, + -1, 847, 440, 849, -1, 848, 440, 849, -1, -1, + 860, -1, 858, 11, 822, -1, 858, 81, 962, -1, + 858, 47, 438, 511, 858, -1, 519, 858, -1, 520, + 858, -1, 858, 519, 858, -1, 858, 520, 858, -1, + 858, 521, 858, -1, 858, 522, 858, -1, 858, 15, + 858, -1, 858, 523, 858, -1, 858, 524, 858, -1, + 858, 16, 858, -1, 858, 515, 858, -1, 858, 516, + 858, -1, 858, 517, 858, -1, 858, 20, 858, -1, + 858, 21, 858, -1, 858, 22, 858, -1, 858, 899, + 858, -1, 899, 858, -1, 858, 899, -1, 858, 37, + 858, -1, 858, 300, 858, -1, 280, 858, -1, 512, + 858, -1, 858, 179, 858, -1, 858, 240, 858, -1, + 858, 240, 858, 147, 858, -1, 858, 512, 240, 858, + -1, 858, 512, 240, 858, 147, 858, -1, 858, 196, + 858, -1, 858, 196, 858, 147, 858, -1, 858, 512, + 196, 858, -1, 858, 512, 196, 858, 147, 858, -1, + 858, 400, 440, 858, -1, 858, 400, 440, 858, 147, + 858, -1, 858, 512, 400, 440, 858, -1, 858, 512, + 400, 440, 858, 147, 858, -1, 858, 224, 285, -1, + 858, 225, -1, 858, 224, 280, 285, -1, 858, 280, + 285, -1, 858, 283, -1, 231, 952, 19, 858, -1, + 858, 17, 858, -1, 858, 18, 858, -1, 888, 307, + 888, -1, 858, 224, 447, -1, 858, 224, 280, 447, + -1, 858, 224, 162, -1, 858, 224, 280, 162, -1, + 858, 224, 458, -1, 858, 224, 280, 458, -1, 858, + 224, 133, 174, 858, -1, 858, 224, 280, 133, 174, + 858, -1, 858, 224, 290, 528, 910, 529, -1, 858, + 224, 280, 290, 528, 910, 529, -1, 858, 54, 933, + 859, 37, 858, -1, 858, 512, 54, 933, 859, 37, + 858, -1, 858, 54, 424, 859, 37, 858, -1, 858, + 512, 54, 424, 859, 37, 858, -1, 858, 201, 920, + -1, 858, 512, 201, 920, -1, 858, 901, 896, 715, + -1, 858, 901, 896, 528, 858, 529, -1, 118, -1, + 84, 528, 858, 529, -1, 461, 528, 858, 529, -1, + 521, 84, 528, 858, 529, -1, 521, 942, 946, 950, + -1, 559, 530, 521, 942, 946, 950, -1, 860, -1, + 859, 11, 822, -1, 519, 859, -1, 520, 859, -1, + 859, 519, 859, -1, 859, 520, 859, -1, 859, 521, + 859, -1, 859, 522, 859, -1, 859, 15, 859, -1, + 859, 523, 859, -1, 859, 524, 859, -1, 859, 16, + 859, -1, 859, 515, 859, -1, 859, 516, 859, -1, + 859, 517, 859, -1, 859, 20, 859, -1, 859, 21, + 859, -1, 859, 22, 859, -1, 859, 899, 859, -1, + 899, 859, -1, 859, 899, -1, 859, 224, 133, 174, + 859, -1, 859, 224, 280, 133, 174, 859, -1, 859, + 224, 290, 528, 910, 529, -1, 859, 224, 280, 290, + 528, 910, 529, -1, 861, -1, 862, 932, -1, 927, + -1, 957, -1, 715, -1, 715, 562, -1, 154, 715, + -1, 776, 528, 906, 529, -1, 528, 858, 529, -1, + 864, -1, 888, -1, 533, -1, 10, -1, 534, 565, + -1, 863, -1, 866, -1, 867, -1, 869, -1, 921, + -1, 865, -1, 872, -1, 40, 715, -1, 40, 526, + 907, 527, -1, 535, 9, -1, 526, 907, 527, -1, + 536, 891, 537, -1, 250, 536, 895, 537, -1, 956, + 528, 529, -1, 956, 528, 742, 740, 529, -1, 956, + 528, 908, 741, 740, 529, -1, 956, 528, 476, 909, + 741, 740, 529, -1, 956, 528, 908, 532, 476, 909, + 741, 740, 529, -1, 956, 528, 31, 908, 741, 740, + 529, -1, 956, 528, 133, 908, 741, 740, 529, -1, + 868, 873, 874, 875, 879, -1, 871, -1, 868, -1, + 871, -1, 82, 169, 528, 858, 529, -1, 67, 528, + 858, 41, 822, 529, -1, 450, 528, 858, 41, 822, + 529, -1, 161, 528, 911, 529, -1, 308, 528, 913, + 529, -1, 327, 528, 915, 529, -1, 422, 528, 916, + 529, -1, 444, 528, 858, 41, 822, 529, -1, 446, + 528, 59, 919, 529, -1, 446, 528, 236, 919, 529, + -1, 446, 528, 441, 919, 529, -1, 446, 528, 919, + 529, -1, 286, 528, 858, 532, 858, 529, -1, 80, + 528, 906, 529, -1, 526, 858, 169, 952, 201, 858, + 527, -1, 526, 858, 169, 952, 201, 860, 194, 858, + 527, -1, 491, 183, 528, 742, 529, -1, -1, 165, + 528, 487, 858, 529, -1, 165, 528, 858, 529, -1, + -1, 157, -1, -1, 489, 877, -1, -1, 878, -1, + 877, 532, 878, -1, 559, 41, 880, -1, 306, 880, + -1, 306, 559, -1, -1, 528, 881, 882, 741, 883, + 529, -1, 559, -1, -1, 315, 60, 905, -1, -1, + 346, 884, 886, -1, 375, 884, 886, -1, 186, 884, + 886, -1, -1, 885, -1, 54, 885, 37, 885, -1, + 453, 330, -1, 453, 168, -1, 105, 374, -1, 858, + 330, -1, 858, 168, -1, 150, 105, 374, -1, 150, + 183, -1, 150, 437, -1, 150, 278, 303, -1, -1, + 374, 528, 906, 529, -1, 374, 528, 529, -1, 887, + -1, 528, 905, 532, 858, 529, -1, 560, 19, 858, + -1, 889, -1, 890, 532, 889, -1, 890, -1, 890, + 532, -1, 858, 19, 858, -1, 892, -1, 893, 532, + 892, -1, 893, -1, 893, 532, -1, 894, -1, -1, + 39, -1, 405, -1, 31, -1, 8, -1, 898, -1, + 519, -1, 520, -1, 521, -1, 522, -1, 15, -1, + 523, -1, 524, -1, 16, -1, 515, -1, 516, -1, + 517, -1, 20, -1, 21, -1, 22, -1, 8, -1, + 297, 528, 902, 529, -1, 897, -1, 297, 528, 902, + 529, -1, 897, -1, 297, 528, 902, 529, -1, 240, + -1, 512, 240, -1, 179, -1, 512, 179, -1, 196, + -1, 512, 196, -1, 897, -1, 559, 530, 902, -1, + 860, -1, 903, 532, 860, -1, 903, -1, 903, 532, + -1, 858, -1, 905, 532, 858, -1, 905, -1, 905, + 532, -1, 906, -1, -1, 909, -1, 908, 532, 909, + -1, 858, -1, 965, 13, 858, -1, 965, 14, 858, + -1, 822, -1, 910, 532, 822, -1, 912, 174, 858, + -1, -1, 3, -1, 844, -1, 845, -1, 846, -1, + 847, -1, 848, -1, 849, -1, 850, -1, 851, -1, + 852, -1, 853, -1, 854, -1, 855, -1, 856, -1, + 561, -1, 858, 914, 917, 918, -1, 858, 914, 917, + -1, 324, 858, -1, 859, 201, 859, -1, -1, 858, + 917, 918, -1, 858, 918, 917, -1, 858, 917, -1, + 858, 918, -1, 905, -1, -1, 174, 858, -1, 169, + 858, -1, 858, 174, 906, -1, 174, 906, -1, 906, + -1, 715, -1, 528, 906, 529, -1, 927, -1, 864, + -1, 66, 925, 922, 924, 144, -1, 923, -1, 922, + 923, -1, 486, 858, 436, 858, -1, 140, 858, -1, + -1, 858, -1, -1, 559, -1, 559, -1, 559, 562, + -1, 526, 858, 527, -1, 526, 928, 19, 928, 527, + -1, 526, 928, 19, 928, 19, 928, 527, -1, 526, + 928, 19, 520, 19, 928, 527, -1, 858, -1, -1, + -1, 929, 563, -1, -1, 528, 529, -1, 528, 908, + 529, -1, 530, 564, 930, -1, 526, 858, 527, -1, + 526, 928, 19, 928, 527, -1, 526, 928, 19, 928, + 19, 928, 527, -1, 526, 928, 19, 520, 19, 928, + 527, -1, -1, 932, 931, -1, 46, -1, -1, 936, + -1, -1, 937, -1, 935, 532, 937, -1, 935, -1, + 935, 532, -1, 858, 41, 966, -1, 858, 3, -1, + 858, -1, 559, 19, 858, -1, 150, 528, 941, 529, + -1, 150, 939, -1, 560, -1, 939, 530, 560, -1, + 939, -1, 940, 532, 939, -1, 940, -1, 940, 532, + -1, 938, -1, -1, 858, 41, 559, -1, 943, -1, + 944, 532, 943, -1, 944, -1, 944, 532, -1, 361, + 528, 945, 529, -1, 361, 943, -1, -1, 939, 41, + 559, -1, 947, -1, 948, 532, 947, -1, 948, -1, + 948, 532, -1, 359, 528, 949, 529, -1, 359, 947, + -1, -1, 558, -1, 951, 532, 558, -1, 955, -1, + 952, 532, 955, -1, 952, -1, 952, 532, -1, 953, + -1, 528, 953, 529, -1, 560, -1, 960, -1, 559, + 562, -1, 958, -1, 4, -1, 561, 929, -1, 6, + -1, 7, -1, 956, 561, -1, 956, 528, 908, 741, + 740, 529, 561, -1, 826, 561, -1, 842, 528, 858, + 529, 857, -1, 842, 958, 857, -1, 842, 561, 857, + -1, 447, -1, 162, -1, 285, -1, 9, -1, 3, + -1, 1041, -1, 1046, -1, 3, -1, 1041, -1, 1043, + -1, 3, -1, 1041, -1, 1044, -1, 559, -1, 559, + 963, -1, 530, 564, -1, 963, 530, 564, -1, 528, + 953, 529, -1, -1, 959, -1, 565, -1, 5, -1, + 332, 955, 968, 41, 969, -1, 528, 910, 529, -1, + -1, 714, -1, 568, -1, 700, -1, 701, -1, 1014, + -1, 1030, -1, 101, 379, 558, 971, -1, 101, 379, + 194, 280, 154, 558, 971, -1, 101, 300, 361, 379, + 558, 971, -1, 971, 972, -1, -1, 622, -1, 973, + -1, 594, -1, 1036, -1, 101, 979, 205, 976, 977, + 295, 558, 975, 528, 587, 529, 978, 815, -1, 101, + 979, 205, 976, 194, 280, 154, 653, 295, 558, 975, + 528, 587, 529, 978, 815, -1, 559, -1, 467, 974, + -1, -1, 90, -1, -1, 653, -1, -1, 490, 637, + -1, -1, 457, -1, -1, 33, 427, 809, 395, 379, + 955, -1, 33, 427, 194, 154, 809, 395, 379, 955, + -1, 33, 390, 558, 395, 379, 955, -1, 33, 390, + 194, 154, 558, 395, 379, 955, -1, 33, 480, 558, + 395, 379, 955, -1, 33, 480, 194, 154, 558, 395, + 379, 955, -1, 170, 76, 982, -1, 76, 982, -1, + 559, -1, -1, 85, 295, 985, 558, 224, 984, -1, + 85, 295, 83, 858, 224, 984, -1, 561, -1, 285, -1, 427, -1, 390, -1, 176, -1, 249, -1, 249, - 427, -1, 480, -1, 254, 480, -1, 205, -1, 171, - 427, -1, 82, -1, 98, -1, 379, -1, 413, -1, - 435, 383, 313, -1, 435, 383, 130, -1, 435, 383, - 433, -1, 435, 383, 91, -1, 451, -1, 25, 257, - -1, 148, 445, -1, 158, -1, 171, 108, 494, -1, - 341, -1, 393, -1, 961, -1, 674, 532, 961, -1, - 64, -1, 366, -1, -1, 326, -1, 376, -1, 445, - -1, 573, 256, 222, 1014, 467, 791, 807, 687, 579, - -1, 37, 857, -1, -1, 528, 584, 529, -1, -1, - 521, -1, -1, 464, 395, 586, -1, 464, 395, 521, - -1, 464, 571, -1, 123, -1, 215, 679, 473, 528, - 905, 529, -1, 215, 571, 680, -1, 215, 118, 473, - -1, 134, 281, -1, 146, 682, -1, 857, -1, -1, - 486, 253, 678, 436, 681, -1, 60, 407, -1, 60, - 431, -1, -1, 486, 280, 253, 684, 678, 436, 681, - -1, 683, -1, 685, -1, 686, -1, 686, 687, -1, - 101, 669, 695, 558, 692, -1, 101, 669, 695, 194, - 280, 154, 558, 692, -1, 101, 300, 361, 669, 695, - 558, 692, -1, 101, 669, 695, 558, 694, -1, 101, - 669, 695, 194, 280, 154, 558, 694, -1, 101, 300, - 361, 669, 695, 558, 694, -1, 696, 41, 427, 715, - -1, 696, 41, 427, 714, -1, 690, -1, 691, 532, - 690, -1, 689, -1, 691, -1, 696, 41, 857, -1, - 693, -1, 694, 532, 693, -1, 176, -1, 249, -1, - 528, 529, -1, 528, 697, 532, 529, -1, 528, 697, - 529, -1, 698, -1, 697, 532, 698, -1, 964, 820, - -1, 964, 820, 13, 857, -1, 964, 820, 14, 857, - -1, 573, 464, 1014, 395, 586, 787, 1015, 579, -1, - 99, 710, 558, 658, 708, 702, 706, 712, 703, 610, - 707, -1, 99, 528, 713, 529, 440, 706, 712, 610, - 707, -1, 99, 174, 109, 559, 440, 559, 701, -1, - -1, 528, 379, 529, -1, 528, 108, 529, -1, 174, - -1, 440, -1, 704, 125, 561, -1, -1, 467, -1, - -1, 41, -1, -1, 340, -1, -1, 709, -1, 528, - 1024, 529, -1, 490, 293, -1, -1, 709, 711, -1, - -1, 56, -1, -1, 56, -1, 293, -1, 173, -1, - 124, 705, 561, -1, 285, 705, 561, -1, 103, -1, - 189, -1, 345, 705, 561, -1, 147, 705, 561, -1, - 170, 345, 662, -1, 170, 345, 521, -1, 315, 60, - 662, -1, 315, 60, 521, -1, 170, 280, 285, 662, - -1, 170, 285, 662, -1, 142, 561, -1, 561, -1, - 414, -1, 415, -1, 3, 530, 559, -1, 3, -1, - 528, 857, 529, -1, 862, -1, 715, -1, 714, -1, - 528, 715, 529, -1, 528, 714, 529, -1, 528, 1029, - 529, -1, 718, -1, 716, 741, -1, 716, 740, 778, - 747, -1, 716, 740, 746, 779, -1, 725, 716, -1, - 725, 716, 741, -1, 725, 716, 740, 778, 747, -1, - 725, 716, 740, 746, 779, -1, 718, -1, 714, -1, - 388, 738, 933, -1, -1, 388, 738, 933, 732, 787, - 814, 767, 776, 875, 777, 752, -1, 388, 737, 935, - 732, 787, 814, 767, 776, 875, 777, 752, -1, 174, - 788, 717, 732, 814, 767, 776, 875, 777, 752, -1, - 174, 788, 388, 737, 935, 732, 814, 767, 776, 875, - 777, 752, -1, 786, -1, 427, 808, -1, 716, 456, - 735, 736, 716, -1, 716, 456, 735, 716, -1, 716, - 220, 735, 716, -1, 716, 149, 735, 716, -1, 720, - 791, 467, 935, -1, 720, 791, 467, 935, 183, 60, - 953, -1, 720, 791, 183, 60, 953, -1, 720, 791, - 295, 724, -1, 720, 791, 295, 724, 183, 60, 953, - -1, 720, 791, 295, 724, 467, 935, -1, 720, 791, - 295, 724, 467, 935, 183, 60, 953, -1, 721, 791, - 295, 935, 222, 271, 954, 719, 953, -1, 721, 791, - 295, 935, -1, 472, -1, 473, -1, 321, -1, 323, - -1, 462, -1, 322, -1, 858, -1, 858, 201, 528, - 715, 529, -1, 794, -1, 722, -1, 723, 532, 722, - -1, 723, -1, 723, 532, -1, 490, 726, -1, 514, - 726, -1, 490, 351, 726, -1, 727, -1, 726, 532, - 727, -1, 954, 963, 728, 41, 731, 528, 968, 529, - -1, 467, 229, 528, 729, 529, -1, -1, 730, -1, - 730, 532, -1, 925, -1, 730, 532, 925, -1, 254, - -1, 280, 254, -1, -1, 222, 733, -1, -1, 434, - 734, 558, -1, 432, 734, 558, -1, 244, 434, 734, - 558, -1, 244, 432, 734, 558, -1, 180, 434, 734, - 558, -1, 180, 432, 734, 558, -1, 460, 734, 558, - -1, 427, 558, -1, 558, -1, 427, -1, -1, 31, - -1, 133, -1, -1, 60, 271, -1, 133, -1, 133, - 295, 528, 905, 529, -1, 31, -1, -1, 195, 287, - -1, 364, 287, -1, -1, 741, -1, -1, 301, 60, - 742, -1, 301, 60, 31, 744, 745, -1, 743, -1, - 742, 532, 743, -1, 857, 467, 899, 745, -1, 857, - 744, 745, -1, 42, -1, 127, -1, -1, 513, 166, - -1, 513, 234, -1, -1, 748, 749, -1, 749, 748, - -1, 748, -1, 749, -1, 746, -1, -1, 241, 761, - -1, 241, 761, 532, 762, -1, 164, 766, 763, 765, - 296, -1, 164, 766, 765, 296, -1, 292, 762, -1, - 292, 763, 765, -1, 4, -1, 9, -1, 862, -1, - 750, 523, -1, 750, 319, -1, 750, -1, 750, 375, - -1, 467, 377, 754, -1, -1, 559, -1, -1, 753, - 528, 751, 529, 757, -1, 751, -1, 751, 528, 559, - 529, -1, 751, 528, 559, 532, 9, 529, -1, 429, - 754, -1, 755, -1, -1, 360, 528, 9, 529, -1, - -1, 439, -1, 479, -1, 758, 14, 857, -1, 47, - 528, 759, 529, -1, -1, 857, -1, 31, -1, 857, - 523, -1, 4, 319, -1, 9, 319, -1, 857, -1, - 859, -1, 519, 764, -1, 520, 764, -1, 957, -1, - 4, -1, 374, -1, 375, -1, 166, -1, 277, -1, - 183, 60, 769, -1, 183, 60, 31, -1, -1, 770, - -1, 768, 532, 770, -1, 768, -1, 768, 532, -1, - 857, -1, 771, -1, 773, -1, 772, -1, 774, -1, - 528, 529, -1, 373, 528, 905, 529, -1, 104, 528, - 905, 529, -1, 184, 397, 528, 769, 529, -1, 184, - -1, 185, -1, 188, 857, -1, -1, 342, 857, -1, - -1, 780, -1, 169, 347, 296, -1, 778, -1, -1, - 781, -1, 780, 781, -1, 782, 783, 784, -1, 169, - 464, -1, 169, 278, 229, 464, -1, 169, 398, -1, - 169, 229, 398, -1, 290, 950, -1, -1, 284, -1, - 402, 247, -1, -1, 473, 528, 905, 529, -1, 785, - 532, 528, 905, 529, -1, 785, -1, 785, 532, -1, - 174, 789, -1, -1, 791, -1, 788, 532, 791, -1, - 788, -1, 788, 532, -1, 560, 19, -1, 808, 803, - 760, 756, -1, 790, 808, 760, 756, -1, 809, 804, - 756, -1, 790, 809, 756, -1, 786, 802, 756, -1, - 235, 809, 804, -1, 714, 803, 756, -1, 790, 714, - 756, -1, 235, 714, 803, -1, 801, -1, 528, 801, - 529, 802, -1, 790, 528, 801, 529, -1, 791, 321, - 528, 935, 169, 797, 792, 529, 803, -1, 791, 462, - 793, 528, 798, 169, 800, 529, 803, -1, 183, 60, - 952, -1, -1, 202, 287, -1, 150, 287, -1, -1, - 858, 201, 528, 935, 529, -1, 858, 201, 560, -1, - 860, -1, 863, -1, 528, 903, 529, -1, 795, 201, - 528, 935, 529, -1, 795, 201, 560, -1, 796, -1, - 797, 796, -1, 560, -1, 528, 952, 529, -1, 798, - 201, 528, 935, 529, -1, 799, -1, 800, 799, -1, - 528, 801, 529, -1, 791, 102, 227, 791, -1, 791, - 805, 227, 791, 807, -1, 791, 227, 791, 807, -1, - 791, 274, 805, 227, 791, -1, 791, 274, 227, 791, - -1, 791, 43, 805, 227, 791, 807, -1, 791, 43, - 227, 791, 807, -1, 791, 328, 227, 791, -1, 791, - 38, 227, 791, 807, -1, 791, 389, 227, 791, 807, - -1, 41, 560, 528, 952, 529, -1, 41, 560, -1, - 559, 528, 952, 529, -1, 559, -1, 802, -1, -1, - 802, -1, 41, 528, 815, 529, -1, 41, 560, 528, - 815, 529, -1, 559, 528, 815, 529, -1, -1, 175, - 806, -1, 238, 806, -1, 370, 806, -1, 389, -1, - 38, -1, 211, -1, 305, -1, -1, 467, 528, 952, - 529, -1, 295, 857, -1, 558, -1, 558, 521, -1, - 296, 558, -1, 296, 528, 558, 529, -1, 869, 813, - -1, 375, 174, 528, 811, 529, 813, -1, 869, 812, - -1, 810, -1, 811, 532, 810, -1, 41, 528, 815, - 529, -1, -1, 514, 302, -1, -1, 487, 857, -1, - -1, 816, -1, 815, 532, 816, -1, 560, 821, 817, - -1, 81, 961, -1, -1, 559, 821, -1, 818, 532, - 559, 821, -1, 374, -1, 420, -1, 821, -1, -1, - 824, 823, -1, 396, 824, 823, -1, 824, 40, 526, - 957, 527, -1, 396, 824, 40, 526, 957, 527, -1, - 824, 40, -1, 396, 824, 40, -1, 822, -1, 819, - 528, 818, 529, 823, -1, 250, 528, 909, 529, 823, - -1, 456, 528, 818, 529, 823, -1, 3, 530, 3, - -1, 822, 530, 3, -1, 823, 526, 527, -1, 823, - 526, 957, 527, -1, -1, 826, -1, 828, -1, 830, - -1, 834, -1, 840, -1, 841, 856, -1, 841, 528, - 957, 529, -1, 828, -1, 831, -1, 835, -1, 840, - -1, 960, 827, -1, 528, 906, 529, -1, -1, 218, - -1, 219, -1, 403, -1, 55, -1, 348, -1, 167, - 829, -1, 137, 331, -1, 116, 827, -1, 113, 827, - -1, 288, 827, -1, 58, -1, 528, 957, 529, -1, - -1, 832, -1, 833, -1, 832, -1, 833, -1, 57, - 839, 528, 905, 529, -1, 57, 839, -1, 836, -1, - 837, -1, 836, -1, 837, -1, 838, 528, 957, 529, - -1, 838, -1, 73, 839, -1, 72, 839, -1, 474, - -1, 273, 73, 839, -1, 273, 72, 839, -1, 275, - 839, -1, 477, -1, -1, 439, 528, 957, 529, 842, - -1, 439, 842, -1, 438, 528, 957, 529, 842, -1, - 438, 842, -1, 221, -1, 514, 438, 511, -1, 492, - 438, 511, -1, -1, 508, -1, 509, -1, 268, -1, - 269, -1, 110, -1, 111, -1, 191, -1, 192, -1, - 264, -1, 265, -1, 384, -1, 385, -1, 262, -1, - 263, -1, 258, -1, 259, -1, 484, -1, 485, -1, - 343, -1, 344, -1, 114, -1, 115, -1, 70, -1, - 69, -1, 261, -1, 260, -1, 843, -1, 844, -1, - 845, -1, 846, -1, 847, -1, 848, -1, 849, -1, - 850, -1, 851, -1, 852, -1, 853, -1, 854, -1, - 855, -1, 843, 440, 844, -1, 845, 440, 846, -1, - 845, 440, 847, -1, 845, 440, 848, -1, 846, 440, - 847, -1, 846, 440, 848, -1, 847, 440, 848, -1, - -1, 859, -1, 857, 11, 821, -1, 857, 81, 961, - -1, 857, 47, 438, 511, 857, -1, 519, 857, -1, - 520, 857, -1, 857, 519, 857, -1, 857, 520, 857, - -1, 857, 521, 857, -1, 857, 522, 857, -1, 857, - 15, 857, -1, 857, 523, 857, -1, 857, 524, 857, - -1, 857, 16, 857, -1, 857, 515, 857, -1, 857, - 516, 857, -1, 857, 517, 857, -1, 857, 20, 857, - -1, 857, 21, 857, -1, 857, 22, 857, -1, 857, - 898, 857, -1, 898, 857, -1, 857, 898, -1, 857, - 37, 857, -1, 857, 300, 857, -1, 280, 857, -1, - 512, 857, -1, 857, 179, 857, -1, 857, 240, 857, - -1, 857, 240, 857, 147, 857, -1, 857, 512, 240, - 857, -1, 857, 512, 240, 857, 147, 857, -1, 857, - 196, 857, -1, 857, 196, 857, 147, 857, -1, 857, - 512, 196, 857, -1, 857, 512, 196, 857, 147, 857, - -1, 857, 400, 440, 857, -1, 857, 400, 440, 857, - 147, 857, -1, 857, 512, 400, 440, 857, -1, 857, - 512, 400, 440, 857, 147, 857, -1, 857, 224, 285, - -1, 857, 225, -1, 857, 224, 280, 285, -1, 857, - 280, 285, -1, 857, 283, -1, 231, 951, 19, 857, - -1, 857, 17, 857, -1, 857, 18, 857, -1, 887, - 307, 887, -1, 857, 224, 447, -1, 857, 224, 280, - 447, -1, 857, 224, 162, -1, 857, 224, 280, 162, - -1, 857, 224, 458, -1, 857, 224, 280, 458, -1, - 857, 224, 133, 174, 857, -1, 857, 224, 280, 133, - 174, 857, -1, 857, 224, 290, 528, 909, 529, -1, - 857, 224, 280, 290, 528, 909, 529, -1, 857, 54, - 932, 858, 37, 857, -1, 857, 512, 54, 932, 858, - 37, 857, -1, 857, 54, 424, 858, 37, 857, -1, - 857, 512, 54, 424, 858, 37, 857, -1, 857, 201, - 919, -1, 857, 512, 201, 919, -1, 857, 900, 895, - 714, -1, 857, 900, 895, 528, 857, 529, -1, 118, - -1, 84, 528, 857, 529, -1, 461, 528, 857, 529, - -1, 521, 84, 528, 857, 529, -1, 521, 941, 945, - 949, -1, 559, 530, 521, 941, 945, 949, -1, 859, - -1, 858, 11, 821, -1, 519, 858, -1, 520, 858, - -1, 858, 519, 858, -1, 858, 520, 858, -1, 858, - 521, 858, -1, 858, 522, 858, -1, 858, 15, 858, - -1, 858, 523, 858, -1, 858, 524, 858, -1, 858, - 16, 858, -1, 858, 515, 858, -1, 858, 516, 858, - -1, 858, 517, 858, -1, 858, 20, 858, -1, 858, - 21, 858, -1, 858, 22, 858, -1, 858, 898, 858, - -1, 898, 858, -1, 858, 898, -1, 858, 224, 133, - 174, 858, -1, 858, 224, 280, 133, 174, 858, -1, - 858, 224, 290, 528, 909, 529, -1, 858, 224, 280, - 290, 528, 909, 529, -1, 860, -1, 861, 931, -1, - 926, -1, 956, -1, 714, -1, 714, 562, -1, 154, - 714, -1, 775, 528, 905, 529, -1, 528, 857, 529, - -1, 863, -1, 887, -1, 533, -1, 10, -1, 534, - 565, -1, 862, -1, 865, -1, 866, -1, 868, -1, - 920, -1, 864, -1, 871, -1, 40, 714, -1, 40, - 526, 906, 527, -1, 535, 9, -1, 526, 906, 527, - -1, 536, 890, 537, -1, 250, 536, 894, 537, -1, - 955, 528, 529, -1, 955, 528, 741, 739, 529, -1, - 955, 528, 907, 740, 739, 529, -1, 955, 528, 476, - 908, 740, 739, 529, -1, 955, 528, 907, 532, 476, - 908, 740, 739, 529, -1, 955, 528, 31, 907, 740, - 739, 529, -1, 955, 528, 133, 907, 740, 739, 529, - -1, 867, 872, 873, 874, 878, -1, 870, -1, 867, - -1, 870, -1, 82, 169, 528, 857, 529, -1, 67, - 528, 857, 41, 821, 529, -1, 450, 528, 857, 41, - 821, 529, -1, 161, 528, 910, 529, -1, 308, 528, - 912, 529, -1, 327, 528, 914, 529, -1, 422, 528, - 915, 529, -1, 444, 528, 857, 41, 821, 529, -1, - 446, 528, 59, 918, 529, -1, 446, 528, 236, 918, - 529, -1, 446, 528, 441, 918, 529, -1, 446, 528, - 918, 529, -1, 286, 528, 857, 532, 857, 529, -1, - 80, 528, 905, 529, -1, 526, 857, 169, 951, 201, - 857, 527, -1, 526, 857, 169, 951, 201, 859, 194, - 857, 527, -1, 491, 183, 528, 741, 529, -1, -1, - 165, 528, 487, 857, 529, -1, 165, 528, 857, 529, - -1, -1, 157, -1, -1, 489, 876, -1, -1, 877, - -1, 876, 532, 877, -1, 559, 41, 879, -1, 306, - 879, -1, 306, 559, -1, -1, 528, 880, 881, 740, - 882, 529, -1, 559, -1, -1, 315, 60, 904, -1, - -1, 346, 883, 885, -1, 375, 883, 885, -1, 186, - 883, 885, -1, -1, 884, -1, 54, 884, 37, 884, - -1, 453, 330, -1, 453, 168, -1, 105, 374, -1, - 857, 330, -1, 857, 168, -1, 150, 105, 374, -1, - 150, 183, -1, 150, 437, -1, 150, 278, 303, -1, - -1, 374, 528, 905, 529, -1, 374, 528, 529, -1, - 886, -1, 528, 904, 532, 857, 529, -1, 560, 19, - 857, -1, 888, -1, 889, 532, 888, -1, 889, -1, - 889, 532, -1, 857, 19, 857, -1, 891, -1, 892, - 532, 891, -1, 892, -1, 892, 532, -1, 893, -1, - -1, 39, -1, 405, -1, 31, -1, 8, -1, 897, - -1, 519, -1, 520, -1, 521, -1, 522, -1, 15, - -1, 523, -1, 524, -1, 16, -1, 515, -1, 516, - -1, 517, -1, 20, -1, 21, -1, 22, -1, 8, - -1, 297, 528, 901, 529, -1, 896, -1, 297, 528, - 901, 529, -1, 896, -1, 297, 528, 901, 529, -1, - 240, -1, 512, 240, -1, 179, -1, 512, 179, -1, - 196, -1, 512, 196, -1, 896, -1, 559, 530, 901, - -1, 859, -1, 902, 532, 859, -1, 902, -1, 902, - 532, -1, 857, -1, 904, 532, 857, -1, 904, -1, - 904, 532, -1, 905, -1, -1, 908, -1, 907, 532, - 908, -1, 857, -1, 964, 13, 857, -1, 964, 14, - 857, -1, 821, -1, 909, 532, 821, -1, 911, 174, - 857, -1, -1, 3, -1, 843, -1, 844, -1, 845, - -1, 846, -1, 847, -1, 848, -1, 849, -1, 850, - -1, 851, -1, 852, -1, 853, -1, 854, -1, 855, - -1, 561, -1, 857, 913, 916, 917, -1, 857, 913, - 916, -1, 324, 857, -1, 858, 201, 858, -1, -1, - 857, 916, 917, -1, 857, 917, 916, -1, 857, 916, - -1, 857, 917, -1, 904, -1, -1, 174, 857, -1, - 169, 857, -1, 857, 174, 905, -1, 174, 905, -1, - 905, -1, 714, -1, 528, 905, 529, -1, 926, -1, - 863, -1, 66, 924, 921, 923, 144, -1, 922, -1, - 921, 922, -1, 486, 857, 436, 857, -1, 140, 857, - -1, -1, 857, -1, -1, 559, -1, 559, -1, 559, - 562, -1, 526, 857, 527, -1, 526, 927, 19, 927, - 527, -1, 526, 927, 19, 927, 19, 927, 527, -1, - 526, 927, 19, 520, 19, 927, 527, -1, 857, -1, - -1, -1, 928, 563, -1, -1, 528, 529, -1, 528, - 907, 529, -1, 530, 564, 929, -1, 526, 857, 527, - -1, 526, 927, 19, 927, 527, -1, 526, 927, 19, - 927, 19, 927, 527, -1, 526, 927, 19, 520, 19, - 927, 527, -1, -1, 931, 930, -1, 46, -1, -1, - 935, -1, -1, 936, -1, 934, 532, 936, -1, 934, - -1, 934, 532, -1, 857, 41, 965, -1, 857, 3, - -1, 857, -1, 559, 19, 857, -1, 150, 528, 940, - 529, -1, 150, 938, -1, 560, -1, 938, 530, 560, - -1, 938, -1, 939, 532, 938, -1, 939, -1, 939, - 532, -1, 937, -1, -1, 857, 41, 559, -1, 942, - -1, 943, 532, 942, -1, 943, -1, 943, 532, -1, - 361, 528, 944, 529, -1, 361, 942, -1, -1, 938, - 41, 559, -1, 946, -1, 947, 532, 946, -1, 947, - -1, 947, 532, -1, 359, 528, 948, 529, -1, 359, - 946, -1, -1, 558, -1, 950, 532, 558, -1, 954, - -1, 951, 532, 954, -1, 951, -1, 951, 532, -1, - 952, -1, 528, 952, 529, -1, 560, -1, 959, -1, - 559, 562, -1, 957, -1, 4, -1, 561, 928, -1, - 6, -1, 7, -1, 955, 561, -1, 955, 528, 907, - 740, 739, 529, 561, -1, 825, 561, -1, 841, 528, - 857, 529, 856, -1, 841, 957, 856, -1, 841, 561, - 856, -1, 447, -1, 162, -1, 285, -1, 9, -1, - 3, -1, 1040, -1, 1045, -1, 3, -1, 1040, -1, - 1042, -1, 3, -1, 1040, -1, 1043, -1, 559, -1, - 559, 962, -1, 530, 564, -1, 962, 530, 564, -1, - 528, 952, 529, -1, -1, 958, -1, 565, -1, 5, - -1, 332, 954, 967, 41, 968, -1, 528, 909, 529, - -1, -1, 713, -1, 568, -1, 699, -1, 700, -1, - 1013, -1, 1029, -1, 101, 379, 558, 970, -1, 101, - 379, 194, 280, 154, 558, 970, -1, 101, 300, 361, - 379, 558, 970, -1, 970, 971, -1, -1, 621, -1, - 972, -1, 594, -1, 1035, -1, 101, 978, 205, 975, - 976, 295, 558, 974, 528, 587, 529, 977, 814, -1, - 101, 978, 205, 975, 194, 280, 154, 652, 295, 558, - 974, 528, 587, 529, 977, 814, -1, 559, -1, 467, - 973, -1, -1, 90, -1, -1, 652, -1, -1, 490, - 636, -1, -1, 457, -1, -1, 33, 427, 808, 395, - 379, 954, -1, 33, 427, 194, 154, 808, 395, 379, - 954, -1, 33, 390, 558, 395, 379, 954, -1, 33, - 390, 194, 154, 558, 395, 379, 954, -1, 33, 480, - 558, 395, 379, 954, -1, 33, 480, 194, 154, 558, - 395, 379, 954, -1, 170, 76, 981, -1, 76, 981, - -1, 559, -1, -1, 85, 295, 984, 558, 224, 983, - -1, 85, 295, 83, 857, 224, 983, -1, 561, -1, - 285, -1, 427, -1, 390, -1, 176, -1, 249, -1, - 249, 427, -1, 480, -1, 109, -1, 205, -1, 379, - -1, 451, -1, 156, 109, 561, 707, -1, 156, 109, - 559, 440, 561, 707, -1, 200, 109, 561, -1, 155, - 990, -1, 155, 994, 988, 990, -1, 155, 478, 990, - -1, 155, 528, 993, 529, 990, -1, 478, -1, -1, - 995, -1, 611, -1, -1, 979, -1, 608, -1, 542, - -1, 1034, -1, 980, -1, 700, -1, 1037, -1, 688, - -1, 969, -1, 594, -1, 621, -1, 589, -1, 557, - -1, 1013, -1, 671, -1, 604, -1, 972, -1, 568, - -1, 1004, -1, 677, -1, 593, -1, 966, -1, 566, - -1, 713, -1, 617, -1, 699, -1, 603, -1, 1008, - -1, 1026, -1, 998, -1, 1029, -1, 1035, -1, 3, - -1, 1040, -1, 1044, -1, 991, -1, 561, -1, 996, - -1, 993, 532, 996, -1, 36, -1, 35, -1, 447, - -1, 162, -1, 295, -1, 992, -1, 997, 989, -1, - 991, -1, 994, -1, 395, 999, -1, 395, 244, 999, - -1, 395, 394, 999, -1, 395, 180, 999, -1, 395, - 475, 999, -1, 1000, -1, 1033, 174, 105, -1, 438, - 511, 1002, -1, 379, 561, -1, 1033, 440, 1003, -1, - 1033, 517, 1003, -1, 857, -1, 561, -1, 3, -1, - 841, 561, 856, -1, 841, 528, 957, 529, 561, -1, - 611, -1, 118, -1, 244, -1, 1001, -1, 1003, 532, - 1001, -1, 243, 1006, -1, 1005, 216, 1006, 1007, -1, - 1005, 216, 1006, 174, 559, 1007, -1, 1005, 216, 1006, - 174, 561, 1007, -1, -1, 170, -1, 561, -1, 559, - -1, -1, 479, 561, -1, 479, 559, -1, 468, 1010, - 1012, 988, -1, 468, 1010, 1012, 988, 558, 963, -1, - 468, 1010, 1012, 988, 1017, -1, 468, 528, 1011, 529, - -1, 468, 528, 1011, 529, 558, 963, -1, 994, -1, - 478, -1, 173, -1, 175, -1, 3, -1, 175, -1, - -1, 1009, -1, 1011, 532, 1009, -1, 173, -1, -1, - 573, 123, 174, 1014, 1016, 1015, 579, -1, 448, 734, - 1014, -1, 808, -1, 808, 559, -1, 808, 41, 559, - -1, 487, 857, -1, -1, 467, 789, -1, -1, 994, - 988, -1, 994, 988, 558, 963, -1, 48, 1020, 561, - 1021, 1025, -1, 48, 194, 280, 154, 1020, 561, 1021, - 1025, -1, 48, 300, 361, 1020, 561, 1021, 1025, -1, - 129, 565, -1, 129, 109, 565, -1, 129, 109, 194, - 154, 565, -1, 109, -1, -1, 41, 559, -1, -1, - 857, -1, -1, 565, 1022, -1, 1023, -1, 1024, 532, - 1023, -1, 528, 1024, 529, -1, -1, 363, 1028, -1, - 363, 244, 1028, -1, 363, 394, 1028, -1, 363, 180, - 1028, -1, 363, 475, 1028, -1, 1033, -1, 31, -1, - 1027, -1, 438, 511, -1, 442, 226, 239, -1, 1031, - 713, -1, 423, 713, -1, 423, 558, -1, 1031, 428, - 174, 558, -1, 1031, 558, -1, 1031, 438, 511, -1, - 1031, 442, 226, 239, -1, 1031, 31, 1032, -1, 1031, - -1, 128, -1, 127, -1, 399, -1, 1030, -1, 428, - -1, -1, 559, -1, 1033, 530, 559, -1, 62, 867, - -1, 101, 669, 480, 558, 658, 977, 41, 713, 1036, - -1, 101, 669, 480, 194, 280, 154, 558, 658, 977, - 41, 713, 1036, -1, 101, 300, 361, 669, 480, 558, - 658, 977, 41, 713, 1036, -1, 101, 669, 351, 480, - 558, 528, 662, 529, 977, 41, 713, 1036, -1, 101, - 300, 361, 669, 351, 480, 558, 528, 662, 529, 977, - 41, 713, 1036, -1, 490, 75, 298, -1, 490, 65, - 75, 298, -1, 490, 244, 75, 298, -1, -1, 101, - 669, 427, 1039, 41, 713, 1038, -1, 101, 669, 427, - 194, 280, 154, 1039, 41, 713, 1038, -1, 101, 300, - 361, 669, 427, 1039, 41, 713, 1038, -1, 490, 108, - -1, 490, 278, 108, -1, -1, 558, 658, 643, 635, - -1, 23, -1, 24, -1, 25, -1, 26, -1, 27, - -1, 28, -1, 29, -1, 30, -1, 32, -1, 33, - -1, 34, -1, 44, -1, 45, -1, 48, -1, 49, - -1, 51, -1, 52, -1, 53, -1, 61, -1, 62, - -1, 63, -1, 64, -1, 65, -1, 68, -1, 69, - -1, 70, -1, 71, -1, 74, -1, 76, -1, 77, - -1, 78, -1, 79, -1, 85, -1, 86, -1, 87, - -1, 88, -1, 89, -1, 91, -1, 92, -1, 93, - -1, 95, -1, 96, -1, 97, -1, 98, -1, 99, - -1, 100, -1, 103, -1, 104, -1, 105, -1, 106, - -1, 107, -1, 108, -1, 109, -1, 110, -1, 111, - -1, 112, -1, 114, -1, 115, -1, 117, -1, 119, - -1, 121, -1, 122, -1, 123, -1, 124, -1, 125, - -1, 126, -1, 129, -1, 130, -1, 131, -1, 132, - -1, 135, -1, 136, -1, 137, -1, 138, -1, 139, - -1, 141, -1, 142, -1, 143, -1, 145, -1, 146, - -1, 147, -1, 148, -1, 150, -1, 151, -1, 152, - -1, 153, -1, 155, -1, 156, -1, 157, -1, 158, - -1, 159, -1, 160, -1, 163, -1, 165, -1, 166, - -1, 168, -1, 170, -1, 172, -1, 176, -1, 177, - -1, 180, -1, 181, -1, 182, -1, 186, -1, 187, - -1, 189, -1, 190, -1, 191, -1, 192, -1, 193, - -1, 194, -1, 195, -1, 197, -1, 198, -1, 199, - -1, 200, -1, 202, -1, 203, -1, 204, -1, 205, - -1, 206, -1, 207, -1, 208, -1, 210, -1, 213, - -1, 214, -1, 215, -1, 216, -1, 217, -1, 223, - -1, 226, -1, 228, -1, 229, -1, 230, -1, 232, - -1, 233, -1, 234, -1, 237, -1, 239, -1, 242, - -1, 243, -1, 244, -1, 245, -1, 246, -1, 247, - -1, 248, -1, 249, -1, 251, -1, 252, -1, 253, - -1, 254, -1, 255, -1, 256, -1, 257, -1, 258, - -1, 259, -1, 260, -1, 261, -1, 262, -1, 263, - -1, 264, -1, 265, -1, 266, -1, 267, -1, 268, - -1, 269, -1, 270, -1, 271, -1, 272, -1, 276, - -1, 277, -1, 278, -1, 281, -1, 282, -1, 284, - -1, 287, -1, 289, -1, 290, -1, 291, -1, 293, - -1, 294, -1, 297, -1, 298, -1, 299, -1, 302, - -1, 303, -1, 306, -1, 309, -1, 310, -1, 311, - -1, 312, -1, 313, -1, 314, -1, 315, -1, 316, - -1, 317, -1, 318, -1, 319, -1, 320, -1, 325, - -1, 326, -1, 329, -1, 330, -1, 332, -1, 333, - -1, 334, -1, 336, -1, 337, -1, 338, -1, 339, - -1, 340, -1, 341, -1, 343, -1, 344, -1, 345, - -1, 346, -1, 347, -1, 349, -1, 350, -1, 351, - -1, 352, -1, 354, -1, 355, -1, 356, -1, 357, - -1, 358, -1, 359, -1, 360, -1, 361, -1, 362, - -1, 363, -1, 364, -1, 365, -1, 366, -1, 368, - -1, 369, -1, 371, -1, 372, -1, 373, -1, 375, - -1, 376, -1, 377, -1, 378, -1, 379, -1, 380, - -1, 381, -1, 382, -1, 383, -1, 384, -1, 385, - -1, 386, -1, 387, -1, 390, -1, 391, -1, 392, - -1, 393, -1, 394, -1, 395, -1, 397, -1, 398, - -1, 401, -1, 402, -1, 404, -1, 406, -1, 407, - -1, 408, -1, 409, -1, 410, -1, 411, -1, 412, - -1, 413, -1, 414, -1, 415, -1, 416, -1, 417, - -1, 418, -1, 419, -1, 421, -1, 425, -1, 426, - -1, 428, -1, 430, -1, 431, -1, 432, -1, 433, - -1, 434, -1, 435, -1, 437, -1, 442, -1, 443, - -1, 445, -1, 448, -1, 449, -1, 451, -1, 452, - -1, 453, -1, 454, -1, 455, -1, 458, -1, 459, - -1, 460, -1, 463, -1, 464, -1, 465, -1, 466, - -1, 468, -1, 469, -1, 470, -1, 471, -1, 472, - -1, 475, -1, 477, -1, 479, -1, 480, -1, 481, - -1, 482, -1, 483, -1, 484, -1, 485, -1, 488, - -1, 491, -1, 492, -1, 493, -1, 494, -1, 495, - -1, 496, -1, 508, -1, 509, -1, 510, -1, 511, - -1, 54, -1, 55, -1, 57, -1, 58, -1, 72, - -1, 73, -1, 80, -1, 84, -1, 113, -1, 116, - -1, 154, -1, 161, -1, 167, -1, 178, -1, 184, - -1, 185, -1, 212, -1, 218, -1, 219, -1, 221, - -1, 250, -1, 273, -1, 275, -1, 279, -1, 286, - -1, 288, -1, 304, -1, 308, -1, 327, -1, 331, - -1, 348, -1, 374, -1, 396, -1, 403, -1, 420, - -1, 422, -1, 438, -1, 439, -1, 444, -1, 446, - -1, 450, -1, 473, -1, 474, -1, 497, -1, 498, - -1, 499, -1, 500, -1, 501, -1, 502, -1, 503, - -1, 504, -1, 505, -1, 506, -1, 507, -1, 43, - -1, 47, -1, 50, -1, 56, -1, 82, -1, 90, - -1, 102, -1, 173, -1, 175, -1, 178, -1, 179, - -1, 196, -1, 211, -1, 224, -1, 225, -1, 227, - -1, 238, -1, 240, -1, 250, -1, 274, -1, 283, - -1, 305, -1, 307, -1, 328, -1, 370, -1, 400, - -1, 420, -1, 429, -1, 478, -1, 38, -1, 43, - -1, 47, -1, 50, -1, 56, -1, 60, -1, 82, - -1, 84, -1, 90, -1, 102, -1, 173, -1, 175, - -1, 179, -1, 196, -1, 211, -1, 224, -1, 225, - -1, 227, -1, 238, -1, 240, -1, 274, -1, 283, - -1, 305, -1, 307, -1, 328, -1, 370, -1, 389, - -1, 400, -1, 429, -1, 450, -1, 461, -1, 478, - -1, 38, -1, 43, -1, 47, -1, 50, -1, 54, - -1, 55, -1, 56, -1, 57, -1, 58, -1, 60, - -1, 73, -1, 72, -1, 80, -1, 82, -1, 84, - -1, 90, -1, 102, -1, 113, -1, 116, -1, 154, - -1, 161, -1, 167, -1, 173, -1, 175, -1, 178, - -1, 179, -1, 184, -1, 185, -1, 196, -1, 211, - -1, 212, -1, 219, -1, 221, -1, 218, -1, 224, - -1, 225, -1, 227, -1, 238, -1, 240, -1, 250, - -1, 273, -1, 274, -1, 275, -1, 279, -1, 283, - -1, 286, -1, 288, -1, 305, -1, 304, -1, 307, - -1, 308, -1, 327, -1, 328, -1, 331, -1, 348, - -1, 370, -1, 374, -1, 389, -1, 396, -1, 400, - -1, 403, -1, 420, -1, 422, -1, 429, -1, 438, - -1, 439, -1, 444, -1, 446, -1, 450, -1, 461, - -1, 473, -1, 474, -1, 478, -1, 497, -1, 498, - -1, 499, -1, 500, -1, 501, -1, 502, -1, 503, - -1, 504, -1, 505, -1, 506, -1, 507, -1, 38, - -1, 43, -1, 47, -1, 50, -1, 56, -1, 60, - -1, 82, -1, 84, -1, 90, -1, 102, -1, 173, - -1, 175, -1, 178, -1, 179, -1, 196, -1, 211, - -1, 224, -1, 225, -1, 227, -1, 238, -1, 240, - -1, 250, -1, 274, -1, 283, -1, 305, -1, 307, - -1, 328, -1, 370, -1, 389, -1, 400, -1, 420, - -1, 429, -1, 450, -1, 461, -1, 478, -1, 31, - -1, 35, -1, 36, -1, 37, -1, 39, -1, 40, - -1, 41, -1, 42, -1, 46, -1, 59, -1, 66, - -1, 67, -1, 75, -1, 81, -1, 83, -1, 94, - -1, 101, -1, 118, -1, 120, -1, 127, -1, 128, - -1, 133, -1, 134, -1, 140, -1, 144, -1, 149, - -1, 162, -1, 164, -1, 169, -1, 171, -1, 174, - -1, 183, -1, 188, -1, 201, -1, 209, -1, 220, - -1, 222, -1, 231, -1, 235, -1, 236, -1, 241, - -1, 280, -1, 285, -1, 292, -1, 295, -1, 296, - -1, 300, -1, 301, -1, 321, -1, 322, -1, 323, - -1, 324, -1, 335, -1, 342, -1, 353, -1, 367, - -1, 388, -1, 399, -1, 405, -1, 423, -1, 424, - -1, 427, -1, 436, -1, 440, -1, 441, -1, 447, - -1, 456, -1, 457, -1, 462, -1, 467, -1, 476, - -1, 486, -1, 487, -1, 489, -1, 490, -1 + 427, -1, 480, -1, 109, -1, 205, -1, 379, -1, + 451, -1, 156, 109, 561, 708, -1, 156, 109, 559, + 440, 561, 708, -1, 200, 109, 561, -1, 155, 991, + -1, 155, 995, 989, 991, -1, 155, 478, 991, -1, + 155, 528, 994, 529, 991, -1, 478, -1, -1, 996, + -1, 611, -1, -1, 980, -1, 608, -1, 542, -1, + 1035, -1, 981, -1, 701, -1, 1038, -1, 689, -1, + 970, -1, 594, -1, 622, -1, 589, -1, 557, -1, + 1014, -1, 672, -1, 604, -1, 973, -1, 568, -1, + 1005, -1, 678, -1, 593, -1, 967, -1, 566, -1, + 714, -1, 618, -1, 700, -1, 603, -1, 1009, -1, + 1027, -1, 999, -1, 1030, -1, 1036, -1, 3, -1, + 1041, -1, 1045, -1, 992, -1, 561, -1, 997, -1, + 994, 532, 997, -1, 36, -1, 35, -1, 447, -1, + 162, -1, 295, -1, 993, -1, 998, 990, -1, 992, + -1, 995, -1, 395, 1000, -1, 395, 244, 1000, -1, + 395, 394, 1000, -1, 395, 180, 1000, -1, 395, 475, + 1000, -1, 1001, -1, 1034, 174, 105, -1, 438, 511, + 1003, -1, 379, 561, -1, 1034, 440, 1004, -1, 1034, + 517, 1004, -1, 858, -1, 561, -1, 3, -1, 842, + 561, 857, -1, 842, 528, 958, 529, 561, -1, 611, + -1, 118, -1, 244, -1, 1002, -1, 1004, 532, 1002, + -1, 243, 1007, -1, 1006, 216, 1007, 1008, -1, 1006, + 216, 1007, 174, 559, 1008, -1, 1006, 216, 1007, 174, + 561, 1008, -1, -1, 170, -1, 561, -1, 559, -1, + -1, 479, 561, -1, 479, 559, -1, 468, 1011, 1013, + 989, -1, 468, 1011, 1013, 989, 558, 964, -1, 468, + 1011, 1013, 989, 1018, -1, 468, 528, 1012, 529, -1, + 468, 528, 1012, 529, 558, 964, -1, 995, -1, 478, + -1, 173, -1, 175, -1, 3, -1, 175, -1, -1, + 1010, -1, 1012, 532, 1010, -1, 173, -1, -1, 573, + 123, 174, 1015, 1017, 1016, 579, -1, 448, 735, 1015, + -1, 809, -1, 809, 559, -1, 809, 41, 559, -1, + 487, 858, -1, -1, 467, 790, -1, -1, 995, 989, + -1, 995, 989, 558, 964, -1, 48, 1021, 561, 1022, + 1026, -1, 48, 194, 280, 154, 1021, 561, 1022, 1026, + -1, 48, 300, 361, 1021, 561, 1022, 1026, -1, 129, + 565, -1, 129, 109, 565, -1, 129, 109, 194, 154, + 565, -1, 109, -1, -1, 41, 559, -1, -1, 858, + -1, -1, 565, 1023, -1, 1024, -1, 1025, 532, 1024, + -1, 528, 1025, 529, -1, -1, 363, 1029, -1, 363, + 244, 1029, -1, 363, 394, 1029, -1, 363, 180, 1029, + -1, 363, 475, 1029, -1, 1034, -1, 31, -1, 1028, + -1, 438, 511, -1, 442, 226, 239, -1, 1032, 714, + -1, 423, 714, -1, 423, 558, -1, 1032, 428, 174, + 558, -1, 1032, 558, -1, 1032, 438, 511, -1, 1032, + 442, 226, 239, -1, 1032, 31, 1033, -1, 1032, -1, + 128, -1, 127, -1, 399, -1, 1031, -1, 428, -1, + -1, 559, -1, 1034, 530, 559, -1, 62, 868, -1, + 101, 670, 480, 558, 659, 978, 41, 714, 1037, -1, + 101, 670, 480, 194, 280, 154, 558, 659, 978, 41, + 714, 1037, -1, 101, 300, 361, 670, 480, 558, 659, + 978, 41, 714, 1037, -1, 101, 670, 351, 480, 558, + 528, 663, 529, 978, 41, 714, 1037, -1, 101, 300, + 361, 670, 351, 480, 558, 528, 663, 529, 978, 41, + 714, 1037, -1, 490, 75, 298, -1, 490, 65, 75, + 298, -1, 490, 244, 75, 298, -1, -1, 101, 670, + 427, 1040, 41, 714, 1039, -1, 101, 670, 427, 194, + 280, 154, 1040, 41, 714, 1039, -1, 101, 300, 361, + 670, 427, 1040, 41, 714, 1039, -1, 490, 108, -1, + 490, 278, 108, -1, -1, 558, 659, 644, 636, -1, + 23, -1, 24, -1, 25, -1, 26, -1, 27, -1, + 28, -1, 29, -1, 30, -1, 32, -1, 33, -1, + 34, -1, 44, -1, 45, -1, 48, -1, 49, -1, + 51, -1, 52, -1, 53, -1, 61, -1, 62, -1, + 63, -1, 64, -1, 65, -1, 68, -1, 69, -1, + 70, -1, 71, -1, 74, -1, 76, -1, 77, -1, + 78, -1, 79, -1, 85, -1, 86, -1, 87, -1, + 88, -1, 89, -1, 91, -1, 92, -1, 93, -1, + 95, -1, 96, -1, 97, -1, 98, -1, 99, -1, + 100, -1, 103, -1, 104, -1, 105, -1, 106, -1, + 107, -1, 108, -1, 109, -1, 110, -1, 111, -1, + 112, -1, 114, -1, 115, -1, 117, -1, 119, -1, + 121, -1, 122, -1, 123, -1, 124, -1, 125, -1, + 126, -1, 129, -1, 130, -1, 131, -1, 132, -1, + 135, -1, 136, -1, 137, -1, 138, -1, 139, -1, + 141, -1, 142, -1, 143, -1, 145, -1, 146, -1, + 147, -1, 148, -1, 150, -1, 151, -1, 152, -1, + 153, -1, 155, -1, 156, -1, 157, -1, 158, -1, + 159, -1, 160, -1, 163, -1, 165, -1, 166, -1, + 168, -1, 170, -1, 172, -1, 176, -1, 177, -1, + 180, -1, 181, -1, 182, -1, 186, -1, 187, -1, + 189, -1, 190, -1, 191, -1, 192, -1, 193, -1, + 194, -1, 195, -1, 197, -1, 198, -1, 199, -1, + 200, -1, 202, -1, 203, -1, 204, -1, 205, -1, + 206, -1, 207, -1, 208, -1, 210, -1, 213, -1, + 214, -1, 215, -1, 216, -1, 217, -1, 223, -1, + 226, -1, 228, -1, 229, -1, 230, -1, 232, -1, + 233, -1, 234, -1, 237, -1, 239, -1, 242, -1, + 243, -1, 244, -1, 245, -1, 246, -1, 247, -1, + 248, -1, 249, -1, 251, -1, 252, -1, 253, -1, + 254, -1, 255, -1, 256, -1, 257, -1, 258, -1, + 259, -1, 260, -1, 261, -1, 262, -1, 263, -1, + 264, -1, 265, -1, 266, -1, 267, -1, 268, -1, + 269, -1, 270, -1, 271, -1, 272, -1, 276, -1, + 277, -1, 278, -1, 281, -1, 282, -1, 284, -1, + 287, -1, 289, -1, 290, -1, 291, -1, 293, -1, + 294, -1, 297, -1, 298, -1, 299, -1, 302, -1, + 303, -1, 306, -1, 309, -1, 310, -1, 311, -1, + 312, -1, 313, -1, 314, -1, 315, -1, 316, -1, + 317, -1, 318, -1, 319, -1, 320, -1, 325, -1, + 326, -1, 329, -1, 330, -1, 332, -1, 333, -1, + 334, -1, 336, -1, 337, -1, 338, -1, 339, -1, + 340, -1, 341, -1, 343, -1, 344, -1, 345, -1, + 346, -1, 347, -1, 349, -1, 350, -1, 351, -1, + 352, -1, 354, -1, 355, -1, 356, -1, 357, -1, + 358, -1, 359, -1, 360, -1, 361, -1, 362, -1, + 363, -1, 364, -1, 365, -1, 366, -1, 368, -1, + 369, -1, 371, -1, 372, -1, 373, -1, 375, -1, + 376, -1, 377, -1, 378, -1, 379, -1, 380, -1, + 381, -1, 382, -1, 383, -1, 384, -1, 385, -1, + 386, -1, 387, -1, 390, -1, 391, -1, 392, -1, + 393, -1, 394, -1, 395, -1, 397, -1, 398, -1, + 401, -1, 402, -1, 404, -1, 406, -1, 407, -1, + 408, -1, 409, -1, 410, -1, 411, -1, 412, -1, + 413, -1, 414, -1, 415, -1, 416, -1, 417, -1, + 418, -1, 419, -1, 421, -1, 425, -1, 426, -1, + 428, -1, 430, -1, 431, -1, 432, -1, 433, -1, + 434, -1, 435, -1, 437, -1, 442, -1, 443, -1, + 445, -1, 448, -1, 449, -1, 451, -1, 452, -1, + 453, -1, 454, -1, 455, -1, 458, -1, 459, -1, + 460, -1, 463, -1, 464, -1, 465, -1, 466, -1, + 468, -1, 469, -1, 470, -1, 471, -1, 472, -1, + 475, -1, 477, -1, 479, -1, 480, -1, 481, -1, + 482, -1, 483, -1, 484, -1, 485, -1, 488, -1, + 491, -1, 492, -1, 493, -1, 494, -1, 495, -1, + 496, -1, 508, -1, 509, -1, 510, -1, 511, -1, + 54, -1, 55, -1, 57, -1, 58, -1, 72, -1, + 73, -1, 80, -1, 84, -1, 113, -1, 116, -1, + 154, -1, 161, -1, 167, -1, 178, -1, 184, -1, + 185, -1, 212, -1, 218, -1, 219, -1, 221, -1, + 250, -1, 273, -1, 275, -1, 279, -1, 286, -1, + 288, -1, 304, -1, 308, -1, 327, -1, 331, -1, + 348, -1, 374, -1, 396, -1, 403, -1, 420, -1, + 422, -1, 438, -1, 439, -1, 444, -1, 446, -1, + 450, -1, 473, -1, 474, -1, 497, -1, 498, -1, + 499, -1, 500, -1, 501, -1, 502, -1, 503, -1, + 504, -1, 505, -1, 506, -1, 507, -1, 43, -1, + 47, -1, 50, -1, 56, -1, 82, -1, 90, -1, + 102, -1, 173, -1, 175, -1, 178, -1, 179, -1, + 196, -1, 211, -1, 224, -1, 225, -1, 227, -1, + 238, -1, 240, -1, 250, -1, 274, -1, 283, -1, + 305, -1, 307, -1, 328, -1, 370, -1, 400, -1, + 420, -1, 429, -1, 478, -1, 38, -1, 43, -1, + 47, -1, 50, -1, 56, -1, 60, -1, 82, -1, + 84, -1, 90, -1, 102, -1, 173, -1, 175, -1, + 179, -1, 196, -1, 211, -1, 224, -1, 225, -1, + 227, -1, 238, -1, 240, -1, 274, -1, 283, -1, + 305, -1, 307, -1, 328, -1, 370, -1, 389, -1, + 400, -1, 429, -1, 450, -1, 461, -1, 478, -1, + 38, -1, 43, -1, 47, -1, 50, -1, 54, -1, + 55, -1, 56, -1, 57, -1, 58, -1, 60, -1, + 73, -1, 72, -1, 80, -1, 82, -1, 84, -1, + 90, -1, 102, -1, 113, -1, 116, -1, 154, -1, + 161, -1, 167, -1, 173, -1, 175, -1, 178, -1, + 179, -1, 184, -1, 185, -1, 196, -1, 211, -1, + 212, -1, 219, -1, 221, -1, 218, -1, 224, -1, + 225, -1, 227, -1, 238, -1, 240, -1, 250, -1, + 273, -1, 274, -1, 275, -1, 279, -1, 283, -1, + 286, -1, 288, -1, 305, -1, 304, -1, 307, -1, + 308, -1, 327, -1, 328, -1, 331, -1, 348, -1, + 370, -1, 374, -1, 389, -1, 396, -1, 400, -1, + 403, -1, 420, -1, 422, -1, 429, -1, 438, -1, + 439, -1, 444, -1, 446, -1, 450, -1, 461, -1, + 473, -1, 474, -1, 478, -1, 497, -1, 498, -1, + 499, -1, 500, -1, 501, -1, 502, -1, 503, -1, + 504, -1, 505, -1, 506, -1, 507, -1, 38, -1, + 43, -1, 47, -1, 50, -1, 56, -1, 60, -1, + 82, -1, 84, -1, 90, -1, 102, -1, 173, -1, + 175, -1, 178, -1, 179, -1, 196, -1, 211, -1, + 224, -1, 225, -1, 227, -1, 238, -1, 240, -1, + 250, -1, 274, -1, 283, -1, 305, -1, 307, -1, + 328, -1, 370, -1, 389, -1, 400, -1, 420, -1, + 429, -1, 450, -1, 461, -1, 478, -1, 31, -1, + 35, -1, 36, -1, 37, -1, 39, -1, 40, -1, + 41, -1, 42, -1, 46, -1, 59, -1, 66, -1, + 67, -1, 75, -1, 81, -1, 83, -1, 94, -1, + 101, -1, 118, -1, 120, -1, 127, -1, 128, -1, + 133, -1, 134, -1, 140, -1, 144, -1, 149, -1, + 162, -1, 164, -1, 169, -1, 171, -1, 174, -1, + 183, -1, 188, -1, 201, -1, 209, -1, 220, -1, + 222, -1, 231, -1, 235, -1, 236, -1, 241, -1, + 280, -1, 285, -1, 292, -1, 295, -1, 296, -1, + 300, -1, 301, -1, 321, -1, 322, -1, 323, -1, + 324, -1, 335, -1, 342, -1, 353, -1, 367, -1, + 388, -1, 399, -1, 405, -1, 423, -1, 424, -1, + 427, -1, 436, -1, 440, -1, 441, -1, 447, -1, + 456, -1, 457, -1, 462, -1, 467, -1, 476, -1, + 486, -1, 487, -1, 489, -1, 490, -1 }; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ static const yytype_uint16 yyrline[] = { - 0, 521, 521, 537, 549, 558, 559, 560, 561, 562, - 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, - 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, - 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, - 593, 594, 595, 596, 597, 598, 599, 601, 9, 18, - 27, 36, 45, 54, 63, 72, 85, 87, 93, 94, - 99, 103, 107, 118, 126, 130, 138, 139, 143, 150, - 151, 156, 163, 173, 182, 191, 200, 209, 217, 225, - 233, 241, 250, 258, 266, 275, 284, 293, 302, 319, - 328, 337, 346, 356, 369, 384, 393, 401, 416, 424, - 434, 444, 451, 458, 466, 473, 484, 485, 490, 494, - 499, 504, 512, 513, 518, 522, 523, 524, 7, 13, - 19, 25, 9, 13, 44, 45, 46, 50, 51, 55, - 59, 60, 64, 70, 75, 76, 77, 78, 6, 15, - 25, 35, 45, 55, 65, 75, 85, 95, 106, 117, - 127, 140, 141, 9, 23, 29, 36, 42, 49, 59, - 63, 71, 72, 73, 77, 86, 95, 102, 103, 108, - 120, 125, 150, 155, 160, 166, 176, 186, 192, 203, - 214, 229, 230, 236, 237, 242, 243, 249, 250, 254, - 255, 260, 262, 268, 269, 273, 274, 277, 278, 283, - 7, 15, 23, 34, 42, 50, 68, 69, 72, 76, - 7, 14, 22, 9, 19, 29, 42, 43, 7, 17, - 27, 40, 41, 45, 46, 47, 51, 52, 57, 61, - 68, 72, 7, 7, 14, 31, 51, 55, 65, 69, - 75, 76, 9, 17, 29, 30, 34, 35, 36, 41, - 42, 43, 48, 52, 56, 60, 64, 68, 72, 76, - 80, 84, 88, 92, 97, 101, 105, 112, 113, 117, - 118, 119, 7, 16, 28, 29, 2, 10, 17, 24, - 32, 40, 51, 52, 53, 57, 58, 59, 2, 7, - 21, 36, 56, 57, 84, 85, 86, 87, 88, 89, - 93, 94, 99, 104, 105, 106, 107, 108, 113, 120, - 121, 122, 139, 146, 153, 163, 173, 185, 193, 202, - 220, 221, 225, 226, 230, 239, 262, 276, 283, 288, - 290, 292, 294, 297, 300, 301, 302, 303, 308, 312, - 313, 318, 325, 330, 331, 332, 333, 334, 335, 336, - 337, 343, 344, 348, 353, 360, 367, 374, 386, 387, - 388, 389, 393, 398, 399, 400, 405, 410, 411, 412, - 413, 414, 415, 420, 440, 466, 474, 484, 485, 489, - 493, 494, 495, 499, 503, 511, 512, 517, 518, 519, - 523, 531, 532, 537, 538, 542, 547, 551, 555, 560, - 568, 569, 573, 574, 578, 579, 585, 596, 609, 623, - 637, 651, 665, 688, 692, 699, 703, 711, 716, 723, - 733, 734, 735, 736, 737, 744, 751, 752, 757, 758, - 9, 19, 29, 39, 49, 59, 73, 74, 75, 76, - 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, - 87, 88, 89, 90, 95, 96, 97, 98, 99, 100, - 105, 106, 111, 112, 113, 118, 119, 120, 7, 29, - 30, 34, 35, 39, 39, 43, 51, 58, 65, 71, - 80, 87, 95, 101, 111, 112, 116, 126, 127, 128, - 132, 142, 142, 146, 147, 7, 16, 25, 34, 43, - 52, 64, 74, 84, 88, 95, 99, 103, 113, 117, - 124, 125, 130, 134, 138, 145, 149, 156, 164, 172, - 7, 1, 30, 49, 61, 62, 63, 67, 68, 73, - 77, 94, 95, 99, 100, 105, 106, 110, 111, 115, - 119, 124, 125, 130, 134, 139, 143, 147, 151, 155, - 159, 163, 167, 171, 175, 179, 183, 187, 191, 195, - 199, 211, 212, 213, 214, 215, 216, 217, 47, 48, - 52, 53, 54, 72, 73, 80, 88, 96, 104, 112, - 120, 131, 132, 159, 164, 172, 188, 205, 223, 241, - 242, 261, 265, 269, 273, 277, 287, 298, 308, 317, - 328, 339, 351, 366, 384, 384, 388, 388, 392, 392, - 396, 402, 409, 413, 414, 418, 419, 433, 440, 447, - 457, 458, 461, 475, 476, 480, 481, 485, 486, 490, - 491, 492, 496, 507, 515, 520, 525, 530, 535, 543, - 551, 556, 561, 568, 569, 573, 574, 575, 579, 586, - 587, 591, 592, 596, 597, 598, 602, 603, 607, 608, - 624, 625, 628, 637, 648, 649, 650, 653, 654, 655, - 659, 660, 661, 662, 666, 667, 671, 673, 689, 691, - 696, 699, 704, 708, 712, 719, 723, 727, 731, 738, - 743, 750, 751, 755, 760, 764, 768, 776, 783, 784, - 789, 790, 795, 796, 800, 810, 811, 816, 817, 822, - 824, 826, 831, 851, 852, 854, 859, 860, 864, 865, - 868, 869, 894, 895, 900, 904, 905, 909, 910, 914, - 915, 916, 917, 918, 922, 935, 942, 949, 956, 957, - 961, 962, 966, 967, 971, 972, 976, 977, 981, 982, - 986, 997, 998, 999, 1000, 1004, 1005, 1010, 1011, 1012, - 1021, 1027, 1036, 1037, 1050, 1051, 1055, 1056, 1060, 1061, - 1065, 1076, 1083, 1090, 1098, 1106, 1116, 1124, 1133, 1142, - 1151, 1155, 1160, 1165, 1176, 1190, 1191, 1194, 1195, 1196, - 1199, 1207, 1217, 1218, 1219, 1222, 1230, 1239, 1243, 1250, - 1251, 1255, 1264, 1268, 1293, 1297, 1310, 1324, 1339, 1351, - 1364, 1378, 1392, 1405, 1420, 1439, 1445, 1450, 1456, 1463, - 1464, 1472, 1476, 1480, 1486, 1493, 1498, 1499, 1500, 1501, - 1502, 1503, 1507, 1508, 1520, 1521, 1526, 1533, 1540, 1547, - 1579, 1590, 1603, 1608, 1609, 1612, 1613, 1616, 1617, 1622, - 1623, 1628, 1632, 1638, 1659, 1667, 1681, 1684, 1688, 1688, - 1691, 1692, 1694, 1699, 1706, 1711, 1717, 1722, 1728, 1732, - 1739, 1746, 1756, 1757, 1761, 1763, 1766, 1770, 1771, 1772, - 1773, 1774, 1775, 1780, 1800, 1801, 1802, 1803, 1814, 1828, - 1829, 1835, 1840, 1845, 1850, 1855, 1860, 1865, 1870, 1876, - 1882, 1888, 1895, 1917, 1926, 1930, 1938, 1942, 1950, 1962, - 1983, 1987, 1993, 1997, 2010, 2018, 2028, 2030, 2032, 2034, - 2036, 2038, 2043, 2044, 2051, 2060, 2068, 2077, 2088, 2096, - 2097, 2098, 2102, 2102, 2105, 2105, 2108, 2108, 2111, 2111, - 2114, 2114, 2117, 2117, 2120, 2120, 2123, 2123, 2126, 2126, - 2129, 2129, 2132, 2132, 2135, 2135, 2138, 2138, 2141, 2143, - 2145, 2147, 2149, 2151, 2153, 2155, 2157, 2159, 2161, 2163, - 2165, 2167, 2172, 2177, 2183, 2190, 2195, 2201, 2207, 2238, - 2240, 2242, 2250, 2265, 2267, 2269, 2271, 2273, 2275, 2277, - 2279, 2281, 2283, 2285, 2287, 2289, 2291, 2293, 2295, 2298, - 2300, 2302, 2305, 2307, 2309, 2311, 2313, 2318, 2323, 2330, - 2335, 2342, 2347, 2354, 2359, 2367, 2375, 2383, 2391, 2409, - 2417, 2425, 2433, 2441, 2449, 2457, 2465, 2469, 2485, 2493, - 2501, 2509, 2517, 2525, 2533, 2537, 2541, 2545, 2549, 2557, - 2565, 2573, 2581, 2601, 2623, 2634, 2641, 2655, 2663, 2668, - 2678, 2687, 2708, 2710, 2712, 2714, 2716, 2718, 2720, 2722, - 2724, 2726, 2728, 2730, 2732, 2734, 2736, 2738, 2740, 2742, - 2744, 2746, 2748, 2750, 2754, 2758, 2762, 2776, 2777, 2791, - 2792, 2793, 2804, 2828, 2839, 2849, 2853, 2857, 2864, 2868, - 2875, 2882, 2883, 2884, 2885, 2886, 2887, 2888, 2889, 2900, - 2905, 2914, 2920, 2927, 2947, 2951, 2958, 2965, 2973, 2981, - 2992, 3012, 3048, 3059, 3060, 3067, 3073, 3075, 3077, 3081, - 3090, 3095, 3102, 3117, 3124, 3128, 3132, 3136, 3140, 3150, - 3159, 3181, 3182, 3186, 3187, 3188, 3192, 3193, 3200, 3201, - 3205, 3206, 3211, 3219, 3221, 3235, 3238, 3265, 3266, 3269, - 3270, 3278, 3286, 3294, 3303, 3313, 3331, 3377, 3386, 3395, - 3404, 3413, 3425, 3426, 3427, 3428, 3429, 3443, 3444, 3447, - 3448, 3452, 3462, 3463, 3467, 3468, 3472, 3479, 3480, 3485, - 3486, 3491, 3492, 3495, 3496, 3497, 3500, 3501, 3504, 3505, - 3506, 3507, 3508, 3509, 3510, 3511, 3512, 3513, 3514, 3515, - 3516, 3517, 3520, 3522, 3527, 3529, 3534, 3536, 3538, 3540, - 3542, 3544, 3546, 3548, 3562, 3564, 3569, 3573, 3580, 3585, - 3591, 3595, 3602, 3607, 3614, 3619, 3627, 3631, 3637, 3641, - 3650, 3661, 3662, 3666, 3670, 3677, 3678, 3679, 3680, 3681, - 3682, 3683, 3684, 3685, 3686, 3687, 3688, 3689, 3690, 3691, - 3701, 3705, 3712, 3719, 3720, 3736, 3740, 3745, 3749, 3764, - 3769, 3773, 3776, 3779, 3780, 3781, 3784, 3791, 3792, 3793, - 3803, 3817, 3818, 3822, 3833, 3834, 3837, 3838, 3846, 3852, - 3856, 3863, 3871, 3879, 3887, 3897, 3898, 3903, 3904, 3908, - 3909, 3910, 3914, 3923, 3931, 3939, 3948, 3963, 3964, 3969, - 3970, 3980, 3981, 3985, 3986, 3990, 3991, 3994, 4010, 4018, - 4026, 4036, 4037, 4041, 4045, 4051, 4053, 4058, 4059, 4063, - 4064, 4067, 4071, 4072, 4076, 4077, 4080, 4081, 4082, 4085, - 4089, 4090, 4094, 4095, 4097, 4098, 4099, 4109, 4110, 4114, - 4116, 4122, 4123, 4127, 4128, 4131, 4142, 4145, 4156, 4160, - 4164, 4176, 4180, 4189, 4196, 4234, 4238, 4242, 4246, 4250, - 4254, 4258, 4264, 4281, 4282, 4283, 4286, 4287, 4288, 4291, - 4292, 4293, 4296, 4297, 4300, 4302, 4307, 4308, 4311, 4315, - 4316, 7, 18, 19, 23, 24, 25, 26, 27, 28, - 7, 26, 50, 73, 80, 85, 86, 87, 88, 8, - 33, 62, 66, 67, 72, 73, 78, 79, 83, 84, - 89, 90, 7, 16, 25, 34, 43, 52, 5, 12, - 22, 23, 7, 15, 26, 27, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39, 7, 19, 33, 9, - 16, 26, 33, 44, 45, 50, 51, 52, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, - 92, 93, 94, 99, 100, 105, 109, 117, 118, 123, - 124, 125, 131, 136, 144, 145, 10, 16, 22, 28, - 34, 44, 45, 53, 64, 76, 84, 95, 101, 105, - 109, 124, 131, 132, 133, 137, 138, 7, 17, 26, - 35, 46, 47, 49, 50, 53, 54, 55, 8, 22, - 36, 48, 56, 70, 71, 72, 73, 74, 87, 88, - 93, 94, 98, 99, 7, 18, 31, 35, 42, 53, - 54, 60, 61, 9, 19, 7, 16, 25, 37, 44, - 51, 60, 61, 65, 66, 77, 78, 82, 89, 93, - 100, 105, 2, 7, 12, 17, 22, 31, 38, 48, - 49, 56, 3, 10, 17, 24, 32, 39, 46, 53, - 60, 69, 69, 71, 71, 73, 73, 75, 76, 6, - 8, 21, 34, 47, 65, 87, 88, 89, 90, 11, - 24, 37, 54, 55, 56, 61, 74, 74, 74, 74, + 0, 522, 522, 538, 550, 559, 560, 561, 562, 563, + 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, + 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, + 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, + 594, 595, 596, 597, 598, 599, 600, 601, 603, 9, + 18, 27, 36, 45, 54, 63, 72, 85, 87, 93, + 94, 99, 103, 107, 118, 126, 130, 138, 139, 143, + 150, 151, 156, 163, 173, 182, 191, 200, 209, 217, + 225, 233, 241, 250, 258, 266, 275, 284, 293, 302, + 319, 328, 337, 346, 356, 369, 384, 393, 401, 416, + 424, 434, 444, 451, 458, 466, 473, 484, 485, 490, + 494, 499, 504, 512, 513, 518, 522, 523, 524, 7, + 13, 19, 25, 9, 13, 44, 45, 46, 50, 51, + 55, 59, 60, 64, 70, 75, 76, 77, 78, 6, + 15, 25, 35, 45, 55, 65, 75, 85, 95, 106, + 117, 127, 140, 141, 9, 23, 29, 36, 42, 49, + 59, 63, 71, 72, 73, 77, 86, 95, 102, 103, + 108, 120, 125, 150, 155, 160, 166, 176, 186, 192, + 203, 214, 229, 230, 236, 237, 242, 243, 249, 250, + 254, 255, 260, 262, 268, 269, 273, 274, 277, 278, + 283, 7, 15, 23, 34, 42, 50, 68, 69, 72, + 76, 7, 14, 22, 9, 19, 29, 42, 43, 7, + 17, 27, 40, 41, 45, 46, 47, 51, 52, 57, + 61, 68, 72, 7, 7, 14, 31, 51, 55, 65, + 69, 75, 76, 9, 17, 29, 30, 34, 35, 36, + 41, 42, 43, 48, 52, 56, 60, 64, 68, 72, + 76, 80, 84, 88, 92, 97, 101, 105, 112, 113, + 117, 118, 119, 7, 16, 7, 16, 28, 29, 2, + 10, 17, 24, 32, 40, 51, 52, 53, 57, 58, + 59, 2, 7, 21, 36, 56, 57, 84, 85, 86, + 87, 88, 89, 93, 94, 99, 104, 105, 106, 107, + 108, 113, 120, 121, 122, 139, 146, 153, 163, 173, + 185, 193, 202, 220, 221, 225, 226, 230, 239, 262, + 276, 283, 288, 290, 292, 294, 297, 300, 301, 302, + 303, 308, 312, 313, 318, 325, 330, 331, 332, 333, + 334, 335, 336, 337, 343, 344, 348, 353, 360, 367, + 374, 386, 387, 388, 389, 393, 398, 399, 400, 405, + 410, 411, 412, 413, 414, 415, 420, 440, 466, 474, + 484, 485, 489, 493, 494, 495, 499, 503, 511, 512, + 517, 518, 519, 523, 531, 532, 537, 538, 542, 547, + 551, 555, 560, 568, 569, 573, 574, 578, 579, 585, + 596, 609, 623, 637, 651, 665, 688, 692, 699, 703, + 711, 716, 723, 733, 734, 735, 736, 737, 744, 751, + 752, 757, 758, 9, 19, 29, 39, 49, 59, 73, + 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 95, 96, 97, + 98, 99, 100, 105, 106, 111, 112, 113, 118, 119, + 120, 7, 29, 30, 34, 35, 39, 39, 43, 51, + 58, 65, 71, 80, 87, 95, 101, 111, 112, 116, + 126, 127, 128, 132, 142, 142, 146, 147, 7, 16, + 25, 34, 43, 52, 64, 74, 84, 88, 95, 99, + 103, 113, 117, 124, 125, 130, 134, 138, 145, 149, + 156, 164, 172, 7, 1, 30, 49, 61, 62, 63, + 67, 68, 73, 77, 94, 95, 99, 100, 105, 106, + 110, 111, 115, 119, 124, 125, 130, 134, 139, 143, + 147, 151, 155, 159, 163, 167, 171, 175, 179, 183, + 187, 191, 195, 199, 211, 212, 213, 214, 215, 216, + 217, 47, 48, 52, 53, 54, 72, 73, 80, 88, + 96, 104, 112, 120, 131, 132, 159, 164, 172, 188, + 205, 223, 241, 242, 261, 265, 269, 273, 277, 287, + 298, 308, 317, 328, 339, 351, 366, 384, 384, 388, + 388, 392, 392, 396, 402, 409, 413, 414, 418, 419, + 433, 440, 447, 457, 458, 461, 475, 476, 480, 481, + 485, 486, 490, 491, 492, 496, 507, 515, 520, 525, + 530, 535, 543, 551, 556, 561, 568, 569, 573, 574, + 575, 579, 586, 587, 591, 592, 596, 597, 598, 602, + 603, 607, 608, 624, 625, 628, 637, 648, 649, 650, + 653, 654, 655, 659, 660, 661, 662, 666, 667, 671, + 673, 689, 691, 696, 699, 704, 708, 712, 719, 723, + 727, 731, 738, 743, 750, 751, 755, 760, 764, 768, + 776, 783, 784, 789, 790, 795, 796, 800, 810, 811, + 816, 817, 822, 824, 826, 831, 851, 852, 854, 859, + 860, 864, 865, 868, 869, 894, 895, 900, 904, 905, + 909, 910, 914, 915, 916, 917, 918, 922, 935, 942, + 949, 956, 957, 961, 962, 966, 967, 971, 972, 976, + 977, 981, 982, 986, 997, 998, 999, 1000, 1004, 1005, + 1010, 1011, 1012, 1021, 1027, 1036, 1037, 1050, 1051, 1055, + 1056, 1060, 1061, 1065, 1076, 1083, 1090, 1098, 1106, 1116, + 1124, 1133, 1142, 1151, 1155, 1160, 1165, 1176, 1190, 1191, + 1194, 1195, 1196, 1199, 1207, 1217, 1218, 1219, 1222, 1230, + 1239, 1243, 1250, 1251, 1255, 1264, 1268, 1293, 1297, 1310, + 1324, 1339, 1351, 1364, 1378, 1392, 1405, 1420, 1439, 1445, + 1450, 1456, 1463, 1464, 1472, 1476, 1480, 1486, 1493, 1498, + 1499, 1500, 1501, 1502, 1503, 1507, 1508, 1520, 1521, 1526, + 1533, 1540, 1547, 1579, 1590, 1603, 1608, 1609, 1612, 1613, + 1616, 1617, 1622, 1623, 1628, 1632, 1638, 1659, 1667, 1681, + 1684, 1688, 1688, 1691, 1692, 1694, 1699, 1706, 1711, 1717, + 1722, 1728, 1733, 1740, 1747, 1757, 1758, 1762, 1764, 1767, + 1771, 1772, 1773, 1774, 1775, 1776, 1781, 1801, 1802, 1803, + 1804, 1815, 1829, 1830, 1836, 1841, 1846, 1851, 1856, 1861, + 1866, 1871, 1877, 1883, 1889, 1896, 1918, 1927, 1931, 1939, + 1943, 1951, 1963, 1984, 1988, 1994, 1998, 2011, 2019, 2029, + 2031, 2033, 2035, 2037, 2039, 2044, 2045, 2052, 2061, 2069, + 2078, 2089, 2097, 2098, 2099, 2103, 2103, 2106, 2106, 2109, + 2109, 2112, 2112, 2115, 2115, 2118, 2118, 2121, 2121, 2124, + 2124, 2127, 2127, 2130, 2130, 2133, 2133, 2136, 2136, 2139, + 2139, 2142, 2144, 2146, 2148, 2150, 2152, 2154, 2156, 2158, + 2160, 2162, 2164, 2166, 2168, 2173, 2178, 2184, 2191, 2196, + 2202, 2208, 2239, 2241, 2243, 2251, 2266, 2268, 2270, 2272, + 2274, 2276, 2278, 2280, 2282, 2284, 2286, 2288, 2290, 2292, + 2294, 2296, 2299, 2301, 2303, 2306, 2308, 2310, 2312, 2314, + 2319, 2324, 2331, 2336, 2343, 2348, 2355, 2360, 2368, 2376, + 2384, 2392, 2410, 2418, 2426, 2434, 2442, 2450, 2458, 2466, + 2470, 2486, 2494, 2502, 2510, 2518, 2526, 2534, 2538, 2542, + 2546, 2550, 2558, 2566, 2574, 2582, 2602, 2624, 2635, 2642, + 2656, 2664, 2669, 2679, 2688, 2709, 2711, 2713, 2715, 2717, + 2719, 2721, 2723, 2725, 2727, 2729, 2731, 2733, 2735, 2737, + 2739, 2741, 2743, 2745, 2747, 2749, 2751, 2755, 2759, 2763, + 2777, 2778, 2792, 2793, 2794, 2805, 2829, 2840, 2850, 2854, + 2858, 2865, 2869, 2876, 2883, 2884, 2885, 2886, 2887, 2888, + 2889, 2890, 2901, 2906, 2915, 2921, 2928, 2948, 2952, 2959, + 2966, 2974, 2982, 2993, 3013, 3049, 3060, 3061, 3068, 3074, + 3076, 3078, 3082, 3091, 3096, 3103, 3118, 3125, 3129, 3133, + 3137, 3141, 3151, 3160, 3182, 3183, 3187, 3188, 3189, 3193, + 3194, 3201, 3202, 3206, 3207, 3212, 3220, 3222, 3236, 3239, + 3266, 3267, 3270, 3271, 3279, 3287, 3295, 3304, 3314, 3332, + 3378, 3387, 3396, 3405, 3414, 3426, 3427, 3428, 3429, 3430, + 3444, 3445, 3448, 3449, 3453, 3463, 3464, 3468, 3469, 3473, + 3480, 3481, 3486, 3487, 3492, 3493, 3496, 3497, 3498, 3501, + 3502, 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, 3513, + 3514, 3515, 3516, 3517, 3518, 3521, 3523, 3528, 3530, 3535, + 3537, 3539, 3541, 3543, 3545, 3547, 3549, 3563, 3565, 3570, + 3574, 3581, 3586, 3592, 3596, 3603, 3608, 3615, 3620, 3628, + 3632, 3638, 3642, 3651, 3662, 3663, 3667, 3671, 3678, 3679, + 3680, 3681, 3682, 3683, 3684, 3685, 3686, 3687, 3688, 3689, + 3690, 3691, 3692, 3702, 3706, 3713, 3720, 3721, 3737, 3741, + 3746, 3750, 3765, 3770, 3774, 3777, 3780, 3781, 3782, 3785, + 3792, 3793, 3794, 3804, 3818, 3819, 3823, 3834, 3835, 3838, + 3839, 3847, 3853, 3857, 3864, 3872, 3880, 3888, 3898, 3899, + 3904, 3905, 3909, 3910, 3911, 3915, 3924, 3932, 3940, 3949, + 3964, 3965, 3970, 3971, 3981, 3982, 3986, 3987, 3991, 3992, + 3995, 4011, 4019, 4027, 4037, 4038, 4042, 4046, 4052, 4054, + 4059, 4060, 4064, 4065, 4068, 4072, 4073, 4077, 4078, 4081, + 4082, 4083, 4086, 4090, 4091, 4095, 4096, 4098, 4099, 4100, + 4110, 4111, 4115, 4117, 4123, 4124, 4128, 4129, 4132, 4143, + 4146, 4157, 4161, 4165, 4177, 4181, 4190, 4197, 4235, 4239, + 4243, 4247, 4251, 4255, 4259, 4265, 4282, 4283, 4284, 4287, + 4288, 4289, 4292, 4293, 4294, 4297, 4298, 4301, 4303, 4308, + 4309, 4312, 4316, 4317, 7, 18, 19, 23, 24, 25, + 26, 27, 28, 7, 26, 50, 73, 80, 85, 86, + 87, 88, 8, 33, 62, 66, 67, 72, 73, 78, + 79, 83, 84, 89, 90, 7, 16, 25, 34, 43, + 52, 5, 12, 22, 23, 7, 15, 26, 27, 30, + 31, 32, 33, 34, 35, 36, 37, 38, 39, 7, + 19, 33, 9, 16, 26, 33, 44, 45, 50, 51, + 52, 57, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, + 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, + 86, 87, 88, 92, 93, 94, 99, 100, 105, 109, + 117, 118, 123, 124, 125, 131, 136, 144, 145, 10, + 16, 22, 28, 34, 44, 45, 53, 64, 76, 84, + 95, 101, 105, 109, 124, 131, 132, 133, 137, 138, + 7, 17, 26, 35, 46, 47, 49, 50, 53, 54, + 55, 8, 22, 36, 48, 56, 70, 71, 72, 73, + 74, 87, 88, 93, 94, 98, 99, 7, 18, 31, + 35, 42, 53, 54, 60, 61, 9, 19, 7, 16, + 25, 37, 44, 51, 60, 61, 65, 66, 77, 78, + 82, 89, 93, 100, 105, 2, 7, 12, 17, 22, + 31, 38, 48, 49, 56, 3, 10, 17, 24, 32, + 39, 46, 53, 60, 69, 69, 71, 71, 73, 73, + 75, 76, 6, 8, 21, 34, 47, 65, 87, 88, + 89, 90, 11, 24, 37, 54, 55, 56, 61, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, @@ -2862,19 +2864,19 @@ static const yytype_uint16 yyrline[] = 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, - 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, + 74, 74, 74, 74, 74, 74, 74, 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, + 75, 75, 75, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, - 76, 76, 76, 76, 76, 76, 76, 76, 76, 77, - 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 76, 76, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, - 77, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 77, 77, 77, 77, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, @@ -2882,18 +2884,18 @@ static const yytype_uint16 yyrline[] = 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 79, 79, 79, 79, 79, + 78, 78, 78, 78, 78, 78, 78, 78, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80 + 80, 80, 80, 80, 80, 80, 80, 80 }; #endif @@ -3009,29 +3011,29 @@ static const char *const yytname[] = "UpdateExtensionsStmt", "ExecuteStmt", "execute_param_expr", "execute_param_list", "execute_param_clause", "AlterSeqStmt", "SeqOptList", "opt_with", "NumericOnly", "SeqOptElem", "opt_by", - "SignedIconst", "DropSecretStmt", "opt_storage_drop_specifier", - "TransactionStmt", "opt_transaction", "opt_transaction_type", "UseStmt", - "CreateStmt", "ConstraintAttributeSpec", "def_arg", - "OptParenthesizedSeqOptList", "generic_option_arg", "key_action", - "ColConstraint", "ColConstraintElem", "GeneratedColumnType", - "opt_GeneratedColumnType", "GeneratedConstraintElem", - "generic_option_elem", "key_update", "key_actions", "OnCommitOption", - "reloptions", "opt_no_inherit", "TableConstraint", "TableLikeOption", - "reloption_list", "ExistingIndex", "ConstraintAttr", "OptWith", - "definition", "TableLikeOptionList", "generic_option_name", - "ConstraintAttributeElem", "regularColumnDef", "generatedColumnDef", - "columnDef", "def_list", "index_name", "TableElement", "def_elem", - "opt_definition", "OptTableElementList", "columnElem", "opt_column_list", - "ColQualList", "key_delete", "reloption_elem", "columnList", - "columnList_opt_comma", "func_type", "ConstraintElem", - "TableElementList", "key_match", "TableLikeClause", "OptTemp", - "generated_when", "DropStmt", "drop_type_any_name", "drop_type_name", - "any_name_list", "opt_drop_behavior", "drop_type_name_on_any_name", - "MergeIntoStmt", "opt_and_clause", "opt_insert_column_list", - "opt_star_expr", "matched_clause_action", "opt_error_message", - "matched_clause", "opt_source_or_target", "not_matched_clause", - "matched_or_not_matched_clause", "merge_match_list", - "CreateFunctionStmt", "table_macro_definition", + "SignedIconst", "AlterDatabaseStmt", "DropSecretStmt", + "opt_storage_drop_specifier", "TransactionStmt", "opt_transaction", + "opt_transaction_type", "UseStmt", "CreateStmt", + "ConstraintAttributeSpec", "def_arg", "OptParenthesizedSeqOptList", + "generic_option_arg", "key_action", "ColConstraint", "ColConstraintElem", + "GeneratedColumnType", "opt_GeneratedColumnType", + "GeneratedConstraintElem", "generic_option_elem", "key_update", + "key_actions", "OnCommitOption", "reloptions", "opt_no_inherit", + "TableConstraint", "TableLikeOption", "reloption_list", "ExistingIndex", + "ConstraintAttr", "OptWith", "definition", "TableLikeOptionList", + "generic_option_name", "ConstraintAttributeElem", "regularColumnDef", + "generatedColumnDef", "columnDef", "def_list", "index_name", + "TableElement", "def_elem", "opt_definition", "OptTableElementList", + "columnElem", "opt_column_list", "ColQualList", "key_delete", + "reloption_elem", "columnList", "columnList_opt_comma", "func_type", + "ConstraintElem", "TableElementList", "key_match", "TableLikeClause", + "OptTemp", "generated_when", "DropStmt", "drop_type_any_name", + "drop_type_name", "any_name_list", "opt_drop_behavior", + "drop_type_name_on_any_name", "MergeIntoStmt", "opt_and_clause", + "opt_insert_column_list", "opt_star_expr", "matched_clause_action", + "opt_error_message", "matched_clause", "opt_source_or_target", + "not_matched_clause", "matched_or_not_matched_clause", + "merge_match_list", "CreateFunctionStmt", "table_macro_definition", "table_macro_definition_parens", "table_macro_list_internal", "table_macro_list", "macro_definition", "macro_definition_list", "macro_alias", "param_list", "MacroParameterList", "MacroParameter", @@ -3205,228 +3207,228 @@ static const yytype_uint16 yyr1[] = 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, 541, - 541, 541, 541, 541, 541, 541, 541, 541, 542, 542, - 542, 542, 542, 542, 542, 542, 543, 543, 544, 544, - 545, 545, 545, 545, 546, 546, 547, 547, 548, 549, - 549, 550, 550, 551, 551, 551, 551, 551, 551, 551, + 541, 541, 541, 541, 541, 541, 541, 541, 541, 542, + 542, 542, 542, 542, 542, 542, 542, 543, 543, 544, + 544, 545, 545, 545, 545, 546, 546, 547, 547, 548, + 549, 549, 550, 550, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, 551, - 551, 551, 551, 551, 551, 551, 552, 552, 553, 553, - 553, 553, 554, 554, 555, 556, 556, 556, 557, 557, - 557, 557, 558, 558, 559, 559, 559, 560, 560, 561, - 562, 562, 563, 564, 565, 565, 565, 565, 566, 566, + 551, 551, 551, 551, 551, 551, 551, 552, 552, 553, + 553, 553, 553, 554, 554, 555, 556, 556, 556, 557, + 557, 557, 557, 558, 558, 559, 559, 559, 560, 560, + 561, 562, 562, 563, 564, 565, 565, 565, 565, 566, 566, 566, 566, 566, 566, 566, 566, 566, 566, 566, - 566, 567, 567, 568, 569, 569, 569, 569, 569, 570, - 570, 571, 571, 571, 572, 572, 572, 573, 573, 574, - 575, 575, 576, 576, 576, 577, 577, 577, 578, 578, - 578, 579, 579, 580, 580, 581, 581, 582, 582, 583, - 583, 584, 584, 585, 585, 586, 586, 587, 587, 588, - 589, 589, 589, 590, 590, 590, 591, 591, 592, 592, - 593, 593, 593, 594, 594, 594, 595, 595, 596, 596, - 596, 597, 597, 598, 598, 598, 599, 599, 600, 601, - 602, 602, 603, 604, 604, 604, 605, 605, 606, 606, - 607, 607, 608, 608, 609, 609, 610, 610, 610, 611, - 611, 611, 611, 612, 612, 612, 612, 612, 612, 612, - 612, 612, 612, 612, 612, 612, 612, 613, 613, 614, - 614, 614, 615, 615, 616, 616, 617, 617, 617, 617, - 617, 617, 618, 618, 618, 619, 619, 619, 620, 621, - 621, 621, 622, 622, 623, 623, 623, 623, 623, 623, - 624, 624, 625, 626, 626, 626, 626, 626, 627, 627, - 627, 627, 628, 628, 628, 628, 628, 628, 628, 628, - 629, 629, 630, 630, 631, 631, 631, 632, 633, 634, - 634, 634, 634, 634, 635, 635, 635, 635, 636, 637, - 637, 638, 638, 639, 639, 639, 639, 639, 639, 639, - 639, 640, 640, 641, 642, 642, 642, 642, 643, 643, - 643, 643, 644, 645, 645, 645, 646, 647, 647, 647, - 647, 647, 647, 648, 649, 650, 650, 651, 651, 652, - 653, 653, 653, 654, 654, 655, 655, 656, 656, 656, - 657, 658, 658, 659, 659, 660, 661, 661, 661, 661, - 662, 662, 663, 663, 664, 664, 664, 665, 665, 665, - 665, 665, 665, 666, 666, 667, 667, 667, 667, 668, - 669, 669, 669, 669, 669, 669, 669, 669, 670, 670, - 671, 671, 671, 671, 671, 671, 672, 672, 672, 672, - 672, 672, 672, 672, 672, 672, 672, 672, 672, 672, - 672, 672, 672, 672, 673, 673, 673, 673, 673, 673, - 674, 674, 675, 675, 675, 676, 676, 676, 677, 678, - 678, 679, 679, 680, 680, 681, 681, 681, 681, 681, - 681, 681, 681, 681, 682, 682, 683, 684, 684, 684, - 685, 686, 686, 687, 687, 688, 688, 688, 688, 688, - 688, 689, 690, 691, 691, 692, 692, 693, 694, 694, - 695, 695, 696, 696, 696, 697, 697, 698, 698, 698, - 699, 700, 700, 700, 701, 701, 701, 702, 702, 703, - 703, 704, 704, 705, 705, 706, 706, 707, 707, 708, - 708, 709, 709, 710, 710, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 712, 712, 712, 712, 712, 712, 712, 713, 713, - 714, 714, 714, 715, 715, 715, 715, 715, 715, 715, - 715, 716, 716, 717, 717, 718, 718, 718, 718, 718, - 718, 718, 718, 718, 718, 718, 718, 718, 718, 718, - 718, 718, 718, 718, 719, 719, 720, 720, 721, 721, - 722, 722, 722, 723, 723, 724, 724, 725, 725, 725, - 726, 726, 727, 728, 728, 729, 729, 730, 730, 731, - 731, 731, 732, 732, 733, 733, 733, 733, 733, 733, - 733, 733, 733, 734, 734, 735, 735, 735, 736, 737, - 737, 738, 738, 739, 739, 739, 740, 740, 741, 741, - 742, 742, 743, 743, 744, 744, 744, 745, 745, 745, - 746, 746, 746, 746, 747, 747, 748, 748, 748, 748, - 749, 749, 750, 750, 750, 751, 751, 751, 751, 752, - 752, 753, 753, 754, 754, 754, 754, 755, 756, 756, - 757, 757, 758, 758, 759, 760, 760, 761, 761, 761, - 761, 761, 762, 763, 763, 763, 764, 764, 765, 765, - 766, 766, 767, 767, 767, 768, 768, 769, 769, 770, - 770, 770, 770, 770, 771, 772, 773, 774, 775, 775, - 776, 776, 777, 777, 778, 778, 779, 779, 780, 780, - 781, 782, 782, 782, 782, 783, 783, 784, 784, 784, - 785, 785, 786, 786, 787, 787, 788, 788, 789, 789, - 790, 791, 791, 791, 791, 791, 791, 791, 791, 791, - 791, 791, 791, 791, 791, 792, 792, 793, 793, 793, - 794, 794, 795, 795, 795, 796, 796, 797, 797, 798, - 798, 799, 800, 800, 801, 801, 801, 801, 801, 801, - 801, 801, 801, 801, 801, 802, 802, 802, 802, 803, - 803, 804, 804, 804, 804, 804, 805, 805, 805, 805, - 805, 805, 806, 806, 807, 807, 808, 808, 808, 808, - 809, 809, 810, 811, 811, 812, 812, 813, 813, 814, - 814, 815, 815, 816, 817, 817, 818, 818, 819, 819, - 820, 820, 821, 821, 821, 821, 821, 821, 821, 821, - 821, 821, 822, 822, 823, 823, 823, 824, 824, 824, - 824, 824, 824, 824, 825, 825, 825, 825, 826, 827, - 827, 828, 828, 828, 828, 828, 828, 828, 828, 828, - 828, 828, 829, 829, 830, 830, 831, 831, 832, 833, - 834, 834, 835, 835, 836, 837, 838, 838, 838, 838, - 838, 838, 839, 839, 840, 840, 840, 840, 841, 842, - 842, 842, 843, 843, 844, 844, 845, 845, 846, 846, - 847, 847, 848, 848, 849, 849, 850, 850, 851, 851, - 852, 852, 853, 853, 854, 854, 855, 855, 856, 856, - 856, 856, 856, 856, 856, 856, 856, 856, 856, 856, - 856, 856, 856, 856, 856, 856, 856, 856, 856, 857, - 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, - 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, - 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, - 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, - 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, - 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, + 566, 566, 567, 567, 568, 569, 569, 569, 569, 569, + 570, 570, 571, 571, 571, 572, 572, 572, 573, 573, + 574, 575, 575, 576, 576, 576, 577, 577, 577, 578, + 578, 578, 579, 579, 580, 580, 581, 581, 582, 582, + 583, 583, 584, 584, 585, 585, 586, 586, 587, 587, + 588, 589, 589, 589, 590, 590, 590, 591, 591, 592, + 592, 593, 593, 593, 594, 594, 594, 595, 595, 596, + 596, 596, 597, 597, 598, 598, 598, 599, 599, 600, + 601, 602, 602, 603, 604, 604, 604, 605, 605, 606, + 606, 607, 607, 608, 608, 609, 609, 610, 610, 610, + 611, 611, 611, 611, 612, 612, 612, 612, 612, 612, + 612, 612, 612, 612, 612, 612, 612, 612, 613, 613, + 614, 614, 614, 615, 615, 616, 616, 617, 617, 618, + 618, 618, 618, 618, 618, 619, 619, 619, 620, 620, + 620, 621, 622, 622, 622, 623, 623, 624, 624, 624, + 624, 624, 624, 625, 625, 626, 627, 627, 627, 627, + 627, 628, 628, 628, 628, 629, 629, 629, 629, 629, + 629, 629, 629, 630, 630, 631, 631, 632, 632, 632, + 633, 634, 635, 635, 635, 635, 635, 636, 636, 636, + 636, 637, 638, 638, 639, 639, 640, 640, 640, 640, + 640, 640, 640, 640, 641, 641, 642, 643, 643, 643, + 643, 644, 644, 644, 644, 645, 646, 646, 646, 647, + 648, 648, 648, 648, 648, 648, 649, 650, 651, 651, + 652, 652, 653, 654, 654, 654, 655, 655, 656, 656, + 657, 657, 657, 658, 659, 659, 660, 660, 661, 662, + 662, 662, 662, 663, 663, 664, 664, 665, 665, 665, + 666, 666, 666, 666, 666, 666, 667, 667, 668, 668, + 668, 668, 669, 670, 670, 670, 670, 670, 670, 670, + 670, 671, 671, 672, 672, 672, 672, 672, 672, 673, + 673, 673, 673, 673, 673, 673, 673, 673, 673, 673, + 673, 673, 673, 673, 673, 673, 673, 674, 674, 674, + 674, 674, 674, 675, 675, 676, 676, 676, 677, 677, + 677, 678, 679, 679, 680, 680, 681, 681, 682, 682, + 682, 682, 682, 682, 682, 682, 682, 683, 683, 684, + 685, 685, 685, 686, 687, 687, 688, 688, 689, 689, + 689, 689, 689, 689, 690, 691, 692, 692, 693, 693, + 694, 695, 695, 696, 696, 697, 697, 697, 698, 698, + 699, 699, 699, 700, 701, 701, 701, 702, 702, 702, + 703, 703, 704, 704, 705, 705, 706, 706, 707, 707, + 708, 708, 709, 709, 710, 710, 711, 711, 712, 712, + 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, + 712, 712, 712, 712, 713, 713, 713, 713, 713, 713, + 713, 714, 714, 715, 715, 715, 716, 716, 716, 716, + 716, 716, 716, 716, 717, 717, 718, 718, 719, 719, + 719, 719, 719, 719, 719, 719, 719, 719, 719, 719, + 719, 719, 719, 719, 719, 719, 719, 720, 720, 721, + 721, 722, 722, 723, 723, 723, 724, 724, 725, 725, + 726, 726, 726, 727, 727, 728, 729, 729, 730, 730, + 731, 731, 732, 732, 732, 733, 733, 734, 734, 734, + 734, 734, 734, 734, 734, 734, 735, 735, 736, 736, + 736, 737, 738, 738, 739, 739, 740, 740, 740, 741, + 741, 742, 742, 743, 743, 744, 744, 745, 745, 745, + 746, 746, 746, 747, 747, 747, 747, 748, 748, 749, + 749, 749, 749, 750, 750, 751, 751, 751, 752, 752, + 752, 752, 753, 753, 754, 754, 755, 755, 755, 755, + 756, 757, 757, 758, 758, 759, 759, 760, 761, 761, + 762, 762, 762, 762, 762, 763, 764, 764, 764, 765, + 765, 766, 766, 767, 767, 768, 768, 768, 769, 769, + 770, 770, 771, 771, 771, 771, 771, 772, 773, 774, + 775, 776, 776, 777, 777, 778, 778, 779, 779, 780, + 780, 781, 781, 782, 783, 783, 783, 783, 784, 784, + 785, 785, 785, 786, 786, 787, 787, 788, 788, 789, + 789, 790, 790, 791, 792, 792, 792, 792, 792, 792, + 792, 792, 792, 792, 792, 792, 792, 792, 793, 793, + 794, 794, 794, 795, 795, 796, 796, 796, 797, 797, + 798, 798, 799, 799, 800, 801, 801, 802, 802, 802, + 802, 802, 802, 802, 802, 802, 802, 802, 803, 803, + 803, 803, 804, 804, 805, 805, 805, 805, 805, 806, + 806, 806, 806, 806, 806, 807, 807, 808, 808, 809, + 809, 809, 809, 810, 810, 811, 812, 812, 813, 813, + 814, 814, 815, 815, 816, 816, 817, 818, 818, 819, + 819, 820, 820, 821, 821, 822, 822, 822, 822, 822, + 822, 822, 822, 822, 822, 823, 823, 824, 824, 824, + 825, 825, 825, 825, 825, 825, 825, 826, 826, 826, + 826, 827, 828, 828, 829, 829, 829, 829, 829, 829, + 829, 829, 829, 829, 829, 830, 830, 831, 831, 832, + 832, 833, 834, 835, 835, 836, 836, 837, 838, 839, + 839, 839, 839, 839, 839, 840, 840, 841, 841, 841, + 841, 842, 843, 843, 843, 844, 844, 845, 845, 846, + 846, 847, 847, 848, 848, 849, 849, 850, 850, 851, + 851, 852, 852, 853, 853, 854, 854, 855, 855, 856, + 856, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 857, 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, - 858, 858, 858, 858, 858, 858, 858, 859, 859, 860, - 860, 860, 860, 860, 860, 861, 861, 861, 862, 862, - 862, 863, 863, 863, 863, 863, 863, 863, 863, 863, - 863, 864, 865, 866, 867, 867, 867, 867, 867, 867, - 867, 868, 868, 869, 869, 870, 870, 870, 870, 870, - 870, 870, 870, 870, 870, 870, 870, 870, 870, 871, - 871, 872, 872, 873, 873, 873, 874, 874, 875, 875, - 876, 876, 877, 878, 878, 878, 879, 880, 880, 881, - 881, 882, 882, 882, 882, 883, 883, 884, 884, 884, - 884, 884, 885, 885, 885, 885, 885, 886, 886, 887, - 887, 888, 889, 889, 890, 890, 891, 892, 892, 893, - 893, 894, 894, 895, 895, 895, 896, 896, 897, 897, - 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - 897, 897, 898, 898, 899, 899, 900, 900, 900, 900, - 900, 900, 900, 900, 901, 901, 902, 902, 903, 903, - 904, 904, 905, 905, 906, 906, 907, 907, 908, 908, - 908, 909, 909, 910, 910, 911, 911, 911, 911, 911, - 911, 911, 911, 911, 911, 911, 911, 911, 911, 911, - 912, 912, 913, 914, 914, 915, 915, 915, 915, 915, - 915, 916, 917, 918, 918, 918, 919, 919, 919, 919, - 920, 921, 921, 922, 923, 923, 924, 924, 925, 926, - 926, 563, 563, 563, 563, 927, 927, 928, 928, 929, - 929, 929, 930, 930, 930, 930, 930, 931, 931, 932, - 932, 933, 933, 934, 934, 935, 935, 936, 936, 936, - 936, 937, 937, 938, 938, 939, 939, 940, 940, 941, - 941, 942, 943, 943, 944, 944, 945, 945, 945, 946, - 947, 947, 948, 948, 949, 949, 949, 950, 950, 951, - 951, 952, 952, 953, 953, 954, 955, 955, 956, 956, - 956, 956, 956, 956, 956, 956, 956, 956, 956, 956, - 956, 956, 957, 958, 958, 958, 959, 959, 959, 960, - 960, 960, 961, 961, 962, 962, 963, 963, 964, 965, - 965, 966, 967, 967, 968, 968, 968, 968, 968, 968, - 969, 969, 969, 970, 970, 971, 971, 971, 971, 972, - 972, 973, 974, 974, 975, 975, 976, 976, 977, 977, - 978, 978, 979, 979, 979, 979, 979, 979, 980, 980, - 981, 981, 982, 982, 983, 983, 984, 984, 984, 984, - 984, 984, 984, 984, 984, 984, 985, 985, 986, 987, - 987, 987, 987, 988, 988, 989, 989, 989, 990, 990, - 990, 990, 990, 990, 990, 990, 990, 990, 990, 990, - 990, 990, 990, 990, 990, 990, 990, 990, 990, 990, - 990, 990, 990, 990, 990, 990, 990, 990, 990, 990, - 991, 991, 991, 992, 992, 993, 993, 994, 994, 995, - 995, 995, 995, 996, 997, 997, 998, 998, 998, 998, - 998, 999, 999, 999, 999, 1000, 1000, 1001, 1002, 1002, - 1002, 1002, 1002, 1002, 1002, 1003, 1003, 1004, 1004, 1004, - 1004, 1005, 1005, 1006, 1006, 1007, 1007, 1007, 1008, 1008, - 1008, 1008, 1008, 1009, 1009, 1009, 1009, 1009, 1010, 1010, - 1011, 1011, 1012, 1012, 1013, 1013, 1014, 1014, 1014, 1015, - 1015, 1016, 1016, 1017, 1017, 1018, 1018, 1018, 1019, 1019, - 1019, 1020, 1020, 1021, 1021, 1022, 1022, 1023, 1024, 1024, - 1025, 1025, 1026, 1026, 1026, 1026, 1026, 1027, 1027, 1028, - 1028, 1028, 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1029, - 1029, 1030, 1030, 1031, 1031, 1032, 1032, 1033, 1033, 1034, - 1035, 1035, 1035, 1035, 1035, 1036, 1036, 1036, 1036, 1037, - 1037, 1037, 1038, 1038, 1038, 1039, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, 1040, - 1040, 1040, 1040, 1040, 1040, 1040, 1041, 1041, 1041, 1041, + 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, + 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, + 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, + 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, + 858, 858, 858, 858, 858, 858, 858, 858, 858, 858, + 858, 858, 858, 858, 858, 859, 859, 859, 859, 859, + 859, 859, 859, 859, 859, 859, 859, 859, 859, 859, + 859, 859, 859, 859, 859, 859, 859, 859, 859, 859, + 860, 860, 861, 861, 861, 861, 861, 861, 862, 862, + 862, 863, 863, 863, 864, 864, 864, 864, 864, 864, + 864, 864, 864, 864, 865, 866, 867, 868, 868, 868, + 868, 868, 868, 868, 869, 869, 870, 870, 871, 871, + 871, 871, 871, 871, 871, 871, 871, 871, 871, 871, + 871, 871, 872, 872, 873, 873, 874, 874, 874, 875, + 875, 876, 876, 877, 877, 878, 879, 879, 879, 880, + 881, 881, 882, 882, 883, 883, 883, 883, 884, 884, + 885, 885, 885, 885, 885, 886, 886, 886, 886, 886, + 887, 887, 888, 888, 889, 890, 890, 891, 891, 892, + 893, 893, 894, 894, 895, 895, 896, 896, 896, 897, + 897, 898, 898, 898, 898, 898, 898, 898, 898, 898, + 898, 898, 898, 898, 898, 899, 899, 900, 900, 901, + 901, 901, 901, 901, 901, 901, 901, 902, 902, 903, + 903, 904, 904, 905, 905, 906, 906, 907, 907, 908, + 908, 909, 909, 909, 910, 910, 911, 911, 912, 912, + 912, 912, 912, 912, 912, 912, 912, 912, 912, 912, + 912, 912, 912, 913, 913, 914, 915, 915, 916, 916, + 916, 916, 916, 916, 917, 918, 919, 919, 919, 920, + 920, 920, 920, 921, 922, 922, 923, 924, 924, 925, + 925, 926, 927, 927, 563, 563, 563, 563, 928, 928, + 929, 929, 930, 930, 930, 931, 931, 931, 931, 931, + 932, 932, 933, 933, 934, 934, 935, 935, 936, 936, + 937, 937, 937, 937, 938, 938, 939, 939, 940, 940, + 941, 941, 942, 942, 943, 944, 944, 945, 945, 946, + 946, 946, 947, 948, 948, 949, 949, 950, 950, 950, + 951, 951, 952, 952, 953, 953, 954, 954, 955, 956, + 956, 957, 957, 957, 957, 957, 957, 957, 957, 957, + 957, 957, 957, 957, 957, 958, 959, 959, 959, 960, + 960, 960, 961, 961, 961, 962, 962, 963, 963, 964, + 964, 965, 966, 966, 967, 968, 968, 969, 969, 969, + 969, 969, 969, 970, 970, 970, 971, 971, 972, 972, + 972, 972, 973, 973, 974, 975, 975, 976, 976, 977, + 977, 978, 978, 979, 979, 980, 980, 980, 980, 980, + 980, 981, 981, 982, 982, 983, 983, 984, 984, 985, + 985, 985, 985, 985, 985, 985, 985, 985, 985, 986, + 986, 987, 988, 988, 988, 988, 989, 989, 990, 990, + 990, 991, 991, 991, 991, 991, 991, 991, 991, 991, + 991, 991, 991, 991, 991, 991, 991, 991, 991, 991, + 991, 991, 991, 991, 991, 991, 991, 991, 991, 991, + 991, 991, 991, 992, 992, 992, 993, 993, 994, 994, + 995, 995, 996, 996, 996, 996, 997, 998, 998, 999, + 999, 999, 999, 999, 1000, 1000, 1000, 1000, 1001, 1001, + 1002, 1003, 1003, 1003, 1003, 1003, 1003, 1003, 1004, 1004, + 1005, 1005, 1005, 1005, 1006, 1006, 1007, 1007, 1008, 1008, + 1008, 1009, 1009, 1009, 1009, 1009, 1010, 1010, 1010, 1010, + 1010, 1011, 1011, 1012, 1012, 1013, 1013, 1014, 1014, 1015, + 1015, 1015, 1016, 1016, 1017, 1017, 1018, 1018, 1019, 1019, + 1019, 1020, 1020, 1020, 1021, 1021, 1022, 1022, 1023, 1023, + 1024, 1025, 1025, 1026, 1026, 1027, 1027, 1027, 1027, 1027, + 1028, 1028, 1029, 1029, 1029, 1030, 1030, 1030, 1030, 1030, + 1030, 1030, 1030, 1030, 1031, 1031, 1032, 1032, 1033, 1033, + 1034, 1034, 1035, 1036, 1036, 1036, 1036, 1036, 1037, 1037, + 1037, 1037, 1038, 1038, 1038, 1039, 1039, 1039, 1040, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, + 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, - 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1043, - 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1042, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, 1043, - 1043, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, - 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, + 1043, 1043, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, - 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, - 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, - 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, - 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, 1044, - 1044, 1044, 1044, 1044, 1044, 1045, 1045, 1045, 1045, 1045, + 1044, 1044, 1044, 1044, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, + 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, + 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, + 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, + 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, + 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1045, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - 1046, 1046, 1046, 1046, 1046 + 1046, 1046, 1046, 1047, 1047, 1047, 1047, 1047, 1047, 1047, + 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, + 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, + 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, + 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, + 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, + 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, + 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047 }; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ @@ -3436,164 +3438,166 @@ static const yytype_uint8 yyr2[] = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 0, 4, 6, - 4, 6, 4, 6, 4, 6, 1, 2, 3, 2, - 1, 3, 2, 3, 1, 3, 1, 2, 2, 1, - 2, 2, 2, 2, 5, 3, 6, 6, 3, 6, - 3, 4, 6, 6, 6, 5, 5, 6, 9, 4, - 5, 7, 6, 4, 8, 4, 2, 4, 3, 6, - 4, 2, 2, 2, 2, 1, 2, 0, 1, 2, - 2, 2, 1, 3, 4, 2, 1, 0, 2, 3, - 2, 3, 1, 2, 1, 1, 1, 1, 1, 1, - 1, 2, 2, 1, 1, 1, 1, 1, 6, 6, - 8, 6, 8, 6, 8, 6, 8, 8, 10, 8, - 10, 1, 0, 9, 1, 4, 4, 7, 2, 1, - 3, 2, 2, 0, 4, 3, 0, 1, 0, 2, - 3, 5, 2, 2, 0, 8, 5, 0, 5, 5, - 7, 2, 0, 1, 1, 1, 3, 2, 0, 1, - 0, 1, 3, 1, 3, 1, 2, 1, 3, 2, - 6, 9, 8, 2, 4, 1, 1, 0, 1, 3, - 2, 4, 5, 5, 8, 7, 1, 0, 8, 11, - 10, 0, 1, 0, 1, 1, 0, 2, 1, 2, - 1, 3, 4, 3, 9, 12, 1, 3, 1, 3, - 3, 0, 4, 6, 1, 2, 1, 1, 0, 1, - 2, 2, 1, 2, 2, 1, 2, 3, 2, 2, - 2, 2, 3, 3, 3, 1, 3, 1, 0, 1, - 2, 2, 5, 7, 0, 2, 2, 3, 3, 2, - 2, 2, 1, 1, 0, 2, 2, 0, 2, 9, - 12, 11, 0, 2, 1, 1, 1, 1, 1, 1, - 3, 0, 1, 2, 1, 1, 2, 2, 3, 1, - 1, 2, 2, 1, 2, 3, 5, 3, 2, 5, - 1, 1, 1, 0, 5, 7, 5, 2, 3, 1, - 1, 2, 2, 0, 3, 4, 4, 0, 3, 2, - 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 3, 3, 1, 2, 2, 2, 2, 2, - 2, 0, 3, 3, 3, 0, 1, 2, 1, 2, - 2, 2, 2, 2, 3, 2, 2, 1, 3, 1, - 1, 1, 1, 3, 1, 2, 0, 1, 2, 0, - 1, 3, 0, 2, 0, 3, 3, 1, 5, 3, - 1, 3, 1, 2, 1, 4, 5, 5, 6, 3, - 7, 4, 11, 1, 3, 2, 2, 2, 0, 3, - 1, 1, 2, 2, 2, 2, 1, 0, 1, 2, - 6, 4, 6, 4, 6, 8, 1, 1, 1, 1, - 2, 1, 2, 1, 2, 1, 1, 1, 1, 3, - 3, 3, 3, 1, 2, 2, 1, 3, 1, 1, - 1, 3, 1, 1, 0, 1, 1, 1, 9, 2, - 0, 3, 0, 1, 0, 3, 3, 2, 1, 6, - 3, 3, 2, 2, 1, 0, 5, 2, 2, 0, - 7, 1, 1, 1, 2, 5, 8, 7, 5, 8, - 7, 4, 4, 1, 3, 1, 1, 3, 1, 3, - 1, 1, 2, 4, 3, 1, 3, 2, 4, 4, - 8, 11, 9, 7, 0, 3, 3, 1, 1, 3, - 0, 1, 0, 1, 0, 1, 0, 1, 3, 2, - 0, 2, 0, 1, 0, 1, 1, 1, 3, 3, - 1, 1, 3, 3, 3, 3, 3, 3, 4, 3, - 2, 1, 1, 1, 3, 1, 3, 1, 1, 1, - 3, 3, 3, 1, 2, 4, 4, 2, 3, 5, - 5, 1, 1, 3, 0, 11, 11, 10, 12, 1, - 2, 5, 4, 4, 4, 4, 7, 5, 4, 7, - 6, 9, 9, 4, 1, 1, 1, 1, 1, 1, - 1, 5, 1, 1, 3, 1, 2, 2, 2, 3, - 1, 3, 8, 5, 0, 1, 2, 1, 3, 1, - 2, 0, 2, 0, 3, 3, 4, 4, 4, 4, - 3, 2, 1, 1, 0, 1, 1, 0, 2, 1, - 5, 1, 0, 2, 2, 0, 1, 0, 3, 5, - 1, 3, 4, 3, 1, 1, 0, 2, 2, 0, - 2, 2, 1, 1, 1, 0, 2, 4, 5, 4, - 2, 3, 1, 1, 1, 2, 2, 1, 2, 3, - 0, 1, 0, 5, 1, 4, 6, 2, 1, 0, - 4, 0, 1, 1, 3, 4, 0, 1, 1, 2, - 2, 2, 1, 1, 2, 2, 1, 1, 1, 1, - 1, 1, 3, 3, 0, 1, 3, 1, 2, 1, - 1, 1, 1, 1, 2, 4, 4, 5, 1, 1, - 2, 0, 2, 0, 1, 3, 1, 0, 1, 2, - 3, 2, 4, 2, 3, 2, 0, 1, 2, 0, - 4, 5, 1, 2, 2, 0, 1, 3, 1, 2, - 2, 4, 4, 3, 3, 3, 3, 3, 3, 3, - 1, 4, 4, 9, 9, 3, 0, 2, 2, 0, - 5, 3, 1, 1, 3, 5, 3, 1, 2, 1, - 3, 5, 1, 2, 3, 4, 5, 4, 5, 4, - 6, 5, 4, 5, 5, 5, 2, 4, 1, 1, - 0, 1, 4, 5, 4, 0, 2, 2, 2, 1, - 1, 1, 1, 0, 4, 2, 1, 2, 2, 4, - 2, 6, 2, 1, 3, 4, 0, 2, 0, 2, - 0, 1, 3, 3, 2, 0, 2, 4, 1, 1, - 1, 0, 2, 3, 5, 6, 2, 3, 1, 5, - 5, 5, 3, 3, 3, 4, 0, 1, 1, 1, - 1, 1, 2, 4, 1, 1, 1, 1, 2, 3, - 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, - 2, 1, 3, 0, 1, 1, 1, 1, 5, 2, - 1, 1, 1, 1, 4, 1, 2, 2, 1, 3, - 3, 2, 1, 0, 5, 2, 5, 2, 1, 3, - 3, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 4, + 6, 4, 6, 4, 6, 4, 6, 1, 2, 3, + 2, 1, 3, 2, 3, 1, 3, 1, 2, 2, + 1, 2, 2, 2, 2, 5, 3, 6, 6, 3, + 6, 3, 4, 6, 6, 6, 5, 5, 6, 9, + 4, 5, 7, 6, 4, 8, 4, 2, 4, 3, + 6, 4, 2, 2, 2, 2, 1, 2, 0, 1, + 2, 2, 2, 1, 3, 4, 2, 1, 0, 2, + 3, 2, 3, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 2, 2, 1, 1, 1, 1, 1, 6, + 6, 8, 6, 8, 6, 8, 6, 8, 8, 10, + 8, 10, 1, 0, 9, 1, 4, 4, 7, 2, + 1, 3, 2, 2, 0, 4, 3, 0, 1, 0, + 2, 3, 5, 2, 2, 0, 8, 5, 0, 5, + 5, 7, 2, 0, 1, 1, 1, 3, 2, 0, + 1, 0, 1, 3, 1, 3, 1, 2, 1, 3, + 2, 6, 9, 8, 2, 4, 1, 1, 0, 1, + 3, 2, 4, 5, 5, 8, 7, 1, 0, 8, + 11, 10, 0, 1, 0, 1, 1, 0, 2, 1, + 2, 1, 3, 4, 3, 9, 12, 1, 3, 1, + 3, 3, 0, 4, 6, 1, 2, 1, 1, 0, + 1, 2, 2, 1, 2, 2, 1, 2, 3, 2, + 2, 2, 2, 3, 3, 3, 1, 3, 1, 0, + 1, 2, 2, 6, 8, 5, 7, 0, 2, 2, + 3, 3, 2, 2, 2, 1, 1, 0, 2, 2, + 0, 2, 9, 12, 11, 0, 2, 1, 1, 1, + 1, 1, 1, 3, 0, 1, 2, 1, 1, 2, + 2, 3, 1, 1, 2, 2, 1, 2, 3, 5, + 3, 2, 5, 1, 1, 1, 0, 5, 7, 5, + 2, 3, 1, 1, 2, 2, 0, 3, 4, 4, + 0, 3, 2, 0, 3, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 3, 3, 1, 2, 2, + 2, 2, 2, 2, 0, 3, 3, 3, 0, 1, + 2, 1, 2, 2, 2, 2, 2, 3, 2, 2, + 1, 3, 1, 1, 1, 1, 3, 1, 2, 0, + 1, 2, 0, 1, 3, 0, 2, 0, 3, 3, + 1, 5, 3, 1, 3, 1, 2, 1, 4, 5, + 5, 6, 3, 7, 4, 11, 1, 3, 2, 2, + 2, 0, 3, 1, 1, 2, 2, 2, 2, 1, + 0, 1, 2, 6, 4, 6, 4, 6, 8, 1, + 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, + 1, 1, 3, 3, 3, 3, 1, 2, 2, 1, + 3, 1, 1, 1, 3, 1, 1, 0, 1, 1, + 1, 9, 2, 0, 3, 0, 1, 0, 3, 3, + 2, 1, 6, 3, 3, 2, 2, 1, 0, 5, + 2, 2, 0, 7, 1, 1, 1, 2, 5, 8, + 7, 5, 8, 7, 4, 4, 1, 3, 1, 1, + 3, 1, 3, 1, 1, 2, 4, 3, 1, 3, + 2, 4, 4, 8, 11, 9, 7, 0, 3, 3, + 1, 1, 3, 0, 1, 0, 1, 0, 1, 0, + 1, 3, 2, 0, 2, 0, 1, 0, 1, 1, + 1, 3, 3, 1, 1, 3, 3, 3, 3, 3, + 3, 4, 3, 2, 1, 1, 1, 3, 1, 3, + 1, 1, 1, 3, 3, 3, 1, 2, 4, 4, + 2, 3, 5, 5, 1, 1, 3, 0, 11, 11, + 10, 12, 1, 2, 5, 4, 4, 4, 4, 7, + 5, 4, 7, 6, 9, 9, 4, 1, 1, 1, + 1, 1, 1, 1, 5, 1, 1, 3, 1, 2, + 2, 2, 3, 1, 3, 8, 5, 0, 1, 2, + 1, 3, 1, 2, 0, 2, 0, 3, 3, 4, + 4, 4, 4, 3, 2, 1, 1, 0, 1, 1, + 0, 2, 1, 5, 1, 0, 2, 2, 0, 1, + 0, 3, 5, 1, 3, 4, 3, 1, 1, 0, + 2, 2, 0, 2, 2, 1, 1, 1, 0, 2, + 4, 5, 4, 2, 3, 1, 1, 1, 2, 2, + 1, 2, 3, 0, 1, 0, 5, 1, 4, 6, + 2, 1, 0, 4, 0, 1, 1, 3, 4, 0, + 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, + 1, 1, 1, 1, 1, 3, 3, 0, 1, 3, + 1, 2, 1, 1, 1, 1, 1, 2, 4, 4, + 5, 1, 1, 2, 0, 2, 0, 1, 3, 1, + 0, 1, 2, 3, 2, 4, 2, 3, 2, 0, + 1, 2, 0, 4, 5, 1, 2, 2, 0, 1, + 3, 1, 2, 2, 4, 4, 3, 3, 3, 3, + 3, 3, 3, 1, 4, 4, 9, 9, 3, 0, + 2, 2, 0, 5, 3, 1, 1, 3, 5, 3, + 1, 2, 1, 3, 5, 1, 2, 3, 4, 5, + 4, 5, 4, 6, 5, 4, 5, 5, 5, 2, + 4, 1, 1, 0, 1, 4, 5, 4, 0, 2, + 2, 2, 1, 1, 1, 1, 0, 4, 2, 1, + 2, 2, 4, 2, 6, 2, 1, 3, 4, 0, + 2, 0, 2, 0, 1, 3, 3, 2, 0, 2, + 4, 1, 1, 1, 0, 2, 3, 5, 6, 2, + 3, 2, 5, 5, 5, 3, 3, 3, 4, 0, + 1, 1, 1, 1, 1, 2, 4, 1, 1, 1, + 1, 2, 3, 0, 1, 1, 1, 1, 1, 2, + 2, 2, 2, 2, 1, 3, 0, 1, 1, 1, + 1, 5, 2, 1, 1, 1, 1, 4, 1, 2, + 2, 1, 3, 3, 2, 1, 0, 5, 2, 5, + 2, 1, 3, 3, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 3, 3, 3, 3, 3, 3, 3, 0, 1, - 3, 3, 5, 2, 2, 3, 3, 3, 3, 3, + 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, + 3, 0, 1, 3, 3, 5, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 2, 2, 3, 3, 2, 2, 3, 3, 5, 4, - 6, 3, 5, 4, 6, 4, 6, 5, 7, 3, - 2, 4, 3, 2, 4, 3, 3, 3, 3, 4, - 3, 4, 3, 4, 5, 6, 6, 7, 6, 7, - 6, 7, 3, 4, 4, 6, 1, 4, 4, 5, - 4, 6, 1, 3, 2, 2, 3, 3, 3, 3, + 3, 3, 3, 2, 2, 3, 3, 2, 2, 3, + 3, 5, 4, 6, 3, 5, 4, 6, 4, 6, + 5, 7, 3, 2, 4, 3, 2, 4, 3, 3, + 3, 3, 4, 3, 4, 3, 4, 5, 6, 6, + 7, 6, 7, 6, 7, 3, 4, 4, 6, 1, + 4, 4, 5, 4, 6, 1, 3, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 2, 2, 5, 6, 6, 7, 1, 2, 1, - 1, 1, 2, 2, 4, 3, 1, 1, 1, 1, - 2, 1, 1, 1, 1, 1, 1, 1, 2, 4, - 2, 3, 3, 4, 3, 5, 6, 7, 9, 7, - 7, 5, 1, 1, 1, 5, 6, 6, 4, 4, - 4, 4, 6, 5, 5, 5, 4, 6, 4, 7, - 9, 5, 0, 5, 4, 0, 1, 0, 2, 0, - 1, 3, 3, 2, 2, 0, 6, 1, 0, 3, - 0, 3, 3, 3, 0, 1, 4, 2, 2, 2, - 2, 2, 3, 2, 2, 3, 0, 4, 3, 1, - 5, 3, 1, 3, 1, 2, 3, 1, 3, 1, - 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, + 3, 3, 3, 3, 2, 2, 5, 6, 6, 7, + 1, 2, 1, 1, 1, 2, 2, 4, 3, 1, + 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, + 1, 2, 4, 2, 3, 3, 4, 3, 5, 6, + 7, 9, 7, 7, 5, 1, 1, 1, 5, 6, + 6, 4, 4, 4, 4, 6, 5, 5, 5, 4, + 6, 4, 7, 9, 5, 0, 5, 4, 0, 1, + 0, 2, 0, 1, 3, 3, 2, 2, 0, 6, + 1, 0, 3, 0, 3, 3, 3, 0, 1, 4, + 2, 2, 2, 2, 2, 3, 2, 2, 3, 0, + 4, 3, 1, 5, 3, 1, 3, 1, 2, 3, + 1, 3, 1, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 4, 1, 4, 1, 4, 1, 2, - 1, 2, 1, 2, 1, 3, 1, 3, 1, 2, - 1, 3, 1, 2, 1, 0, 1, 3, 1, 3, - 3, 1, 3, 3, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 4, 1, 4, 1, + 4, 1, 2, 1, 2, 1, 2, 1, 3, 1, + 3, 1, 2, 1, 3, 1, 2, 1, 0, 1, + 3, 1, 3, 3, 1, 3, 3, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 4, 3, 2, 3, 0, 3, 3, 2, 2, 1, - 0, 2, 2, 3, 2, 1, 1, 3, 1, 1, - 5, 1, 2, 4, 2, 0, 1, 0, 1, 1, - 2, 3, 5, 7, 7, 1, 0, 0, 2, 0, - 2, 3, 3, 3, 5, 7, 7, 0, 2, 1, - 0, 1, 0, 1, 3, 1, 2, 3, 2, 1, - 3, 4, 2, 1, 3, 1, 3, 1, 2, 1, - 0, 3, 1, 3, 1, 2, 4, 2, 0, 3, - 1, 3, 1, 2, 4, 2, 0, 1, 3, 1, - 3, 1, 2, 1, 3, 1, 1, 2, 1, 1, - 2, 1, 1, 2, 7, 2, 5, 3, 3, 1, + 1, 1, 1, 4, 3, 2, 3, 0, 3, 3, + 2, 2, 1, 0, 2, 2, 3, 2, 1, 1, + 3, 1, 1, 5, 1, 2, 4, 2, 0, 1, + 0, 1, 1, 2, 3, 5, 7, 7, 1, 0, + 0, 2, 0, 2, 3, 3, 3, 5, 7, 7, + 0, 2, 1, 0, 1, 0, 1, 3, 1, 2, + 3, 2, 1, 3, 4, 2, 1, 3, 1, 3, + 1, 2, 1, 0, 3, 1, 3, 1, 2, 4, + 2, 0, 3, 1, 3, 1, 2, 4, 2, 0, + 1, 3, 1, 3, 1, 2, 1, 3, 1, 1, + 2, 1, 1, 2, 1, 1, 2, 7, 2, 5, + 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 2, 3, 3, + 0, 1, 1, 1, 5, 3, 0, 1, 1, 1, + 1, 1, 1, 4, 7, 6, 2, 0, 1, 1, + 1, 1, 13, 16, 1, 2, 0, 1, 0, 1, + 0, 2, 0, 1, 0, 6, 8, 6, 8, 6, + 8, 3, 2, 1, 0, 6, 6, 1, 1, 1, + 1, 1, 1, 2, 1, 1, 1, 1, 1, 4, + 6, 3, 2, 4, 3, 5, 1, 0, 1, 1, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 2, 2, 3, 3, 0, 1, 1, - 1, 5, 3, 0, 1, 1, 1, 1, 1, 1, - 4, 7, 6, 2, 0, 1, 1, 1, 1, 13, - 16, 1, 2, 0, 1, 0, 1, 0, 2, 0, - 1, 0, 6, 8, 6, 8, 6, 8, 3, 2, - 1, 0, 6, 6, 1, 1, 1, 1, 1, 1, - 2, 1, 1, 1, 1, 1, 4, 6, 3, 2, - 4, 3, 5, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, + 1, 1, 1, 1, 1, 1, 2, 1, 1, 2, + 3, 3, 3, 3, 1, 3, 3, 2, 3, 3, + 1, 1, 1, 3, 5, 1, 1, 1, 1, 3, + 2, 4, 6, 6, 0, 1, 1, 1, 0, 2, + 2, 4, 6, 5, 4, 6, 1, 1, 1, 1, + 1, 1, 0, 1, 3, 1, 0, 7, 3, 1, + 2, 3, 2, 0, 2, 0, 2, 4, 5, 8, + 7, 2, 3, 5, 1, 0, 2, 0, 1, 0, + 2, 1, 3, 3, 0, 2, 3, 3, 3, 3, + 1, 1, 1, 2, 3, 2, 2, 2, 4, 2, + 3, 4, 3, 1, 1, 1, 1, 1, 1, 0, + 1, 3, 2, 9, 12, 11, 12, 14, 3, 4, + 4, 0, 7, 10, 9, 2, 3, 0, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, - 1, 1, 1, 2, 1, 1, 2, 3, 3, 3, - 3, 1, 3, 3, 2, 3, 3, 1, 1, 1, - 3, 5, 1, 1, 1, 1, 3, 2, 4, 6, - 6, 0, 1, 1, 1, 0, 2, 2, 4, 6, - 5, 4, 6, 1, 1, 1, 1, 1, 1, 0, - 1, 3, 1, 0, 7, 3, 1, 2, 3, 2, - 0, 2, 0, 2, 4, 5, 8, 7, 2, 3, - 5, 1, 0, 2, 0, 1, 0, 2, 1, 3, - 3, 0, 2, 3, 3, 3, 3, 1, 1, 1, - 2, 3, 2, 2, 2, 4, 2, 3, 4, 3, - 1, 1, 1, 1, 1, 1, 0, 1, 3, 2, - 9, 12, 11, 12, 14, 3, 4, 4, 0, 7, - 10, 9, 2, 3, 0, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -3655,9 +3659,7 @@ static const yytype_uint8 yyr2[] = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1, 1 }; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state @@ -3665,5426 +3667,5427 @@ static const yytype_uint8 yyr2[] = means the default is an error. */ static const yytype_uint16 yydefact[] = { - 168, 284, 0, 1488, 1487, 1562, 284, 0, 1421, 0, - 284, 544, 427, 0, 1592, 1591, 0, 223, 284, 0, - 168, 0, 1522, 0, 0, 0, 606, 609, 607, 0, - 0, 0, 284, 652, 0, 1593, 284, 0, 0, 644, - 608, 0, 1539, 0, 0, 0, 0, 0, 2, 4, - 7, 21, 36, 31, 0, 20, 34, 18, 17, 39, - 26, 6, 24, 38, 41, 19, 25, 33, 15, 40, - 13, 37, 582, 568, 657, 581, 0, 0, 167, 762, - 589, 35, 16, 30, 5, 11, 12, 28, 29, 27, - 1444, 44, 32, 0, 42, 22, 8, 9, 23, 43, - 45, 1594, 1590, 10, 46, 14, 283, 282, 276, 0, - 0, 0, 0, 0, 1561, 0, 0, 0, 287, 124, - 1616, 1617, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, - 1626, 2000, 1627, 1628, 2001, 1629, 1630, 2002, 1631, 1632, - 1633, 1946, 1947, 2003, 1948, 1949, 1634, 1635, 1636, 1637, - 1638, 1639, 1640, 1641, 1642, 1950, 1951, 1643, 1644, 1645, - 1646, 1647, 1952, 2004, 1953, 1648, 1649, 1650, 1651, 1652, - 2005, 1653, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1661, - 2006, 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, - 1671, 1954, 1672, 1673, 1955, 1674, 1675, 1676, 1677, 1678, - 1679, 1680, 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, - 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, - 1699, 1700, 1701, 1956, 1702, 1703, 1704, 1705, 1706, 1707, - 1957, 1708, 1709, 1710, 1958, 1711, 1712, 1713, 2007, 2008, - 1714, 1715, 1959, 2010, 1716, 1717, 1718, 1960, 1961, 1719, - 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1727, 2011, 1728, - 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, - 1739, 2012, 1962, 1740, 1741, 1742, 1743, 1744, 1963, 1964, - 1965, 1745, 2013, 2014, 1746, 2015, 1747, 1748, 1749, 1750, - 1751, 1752, 1753, 2016, 1754, 2017, 1755, 1756, 1757, 1758, - 1759, 1760, 1761, 1762, 1966, 1763, 1764, 1765, 1766, 1767, - 1768, 1769, 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, - 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1967, 2019, 1968, - 1785, 1786, 1787, 1969, 1788, 1789, 2020, 1790, 1970, 1791, - 1971, 1792, 1793, 1794, 1795, 1796, 1797, 1798, 1799, 1800, - 1801, 1972, 2021, 1802, 2022, 1973, 1803, 1804, 1805, 1806, - 1807, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1816, - 1974, 2023, 1817, 1818, 1975, 1819, 1820, 1821, 1822, 1823, - 1824, 1825, 1826, 1827, 1828, 1829, 1830, 1831, 1832, 1976, - 1833, 1834, 1835, 1836, 1837, 1838, 1839, 1840, 1841, 1842, - 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851, 2024, - 1852, 1853, 1854, 1977, 1855, 1856, 1857, 1858, 1859, 1860, - 1861, 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, - 1871, 1872, 1873, 1978, 1874, 1875, 2025, 1876, 1877, 1979, - 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, - 1888, 1889, 1890, 1891, 1892, 1980, 1893, 1981, 1894, 1895, - 1896, 2027, 1897, 1898, 1899, 1900, 1901, 1902, 1903, 1982, - 1983, 1904, 1905, 1984, 1906, 1985, 1907, 1908, 1986, 1909, - 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, - 1920, 1921, 1922, 1923, 1924, 1925, 1987, 1988, 1926, 1927, - 2028, 1928, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1936, - 1937, 1938, 1939, 1940, 1941, 1989, 1990, 1991, 1992, 1993, - 1994, 1995, 1996, 1997, 1998, 1999, 1942, 1943, 1944, 1945, - 0, 1599, 0, 1346, 125, 126, 1368, 124, 1959, 1966, - 1980, 1420, 1419, 125, 0, 279, 543, 0, 0, 0, - 0, 0, 0, 225, 0, 421, 420, 1410, 426, 0, - 0, 0, 128, 120, 1819, 127, 1345, 118, 134, 2180, - 2181, 2182, 2183, 2061, 2184, 2185, 2186, 2187, 2062, 2188, - 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2189, 2070, 2190, - 2191, 2072, 2071, 2192, 2073, 2193, 2074, 2194, 2075, 2076, - 2195, 2196, 2077, 1668, 2078, 2079, 2197, 2198, 2199, 2200, - 2201, 2202, 2203, 2204, 2205, 2080, 2081, 2206, 2207, 2082, - 2208, 2209, 2083, 2210, 2084, 2085, 2086, 2211, 2087, 2088, - 2212, 2089, 2213, 2214, 2090, 2091, 2094, 2092, 2215, 2093, - 2216, 2095, 2096, 2097, 2217, 2218, 2219, 2098, 2099, 2220, - 2100, 2101, 2102, 2103, 2104, 2221, 2105, 2222, 2106, 2107, - 2223, 2224, 2225, 2226, 2227, 2109, 2108, 2110, 2111, 2228, - 2229, 2230, 2231, 2112, 2113, 2114, 2232, 2233, 2115, 2234, - 2235, 2116, 2117, 2236, 2118, 2119, 2237, 2120, 2121, 2238, - 2122, 2123, 2239, 2240, 2241, 2124, 2242, 2125, 2126, 2243, - 2244, 2127, 2128, 2245, 2129, 2246, 2247, 2130, 2248, 2249, - 2131, 2132, 2250, 2133, 2251, 2252, 2253, 2254, 2134, 2135, - 2136, 2137, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 1558, - 136, 135, 137, 0, 445, 446, 0, 456, 0, 438, - 443, 439, 0, 465, 458, 466, 447, 437, 459, 448, - 436, 224, 0, 467, 453, 441, 0, 0, 0, 0, - 280, 241, 427, 0, 168, 0, 1450, 1460, 1470, 1465, - 1459, 1468, 1457, 1474, 1463, 1449, 1472, 1458, 1462, 1467, - 1455, 1473, 1453, 1471, 1469, 1456, 1464, 1448, 1452, 1439, - 1444, 1477, 1466, 1475, 1461, 1476, 1478, 1451, 1479, 1454, - 0, 1421, 0, 1952, 2004, 1957, 0, 1970, 0, 1973, - 1974, 1855, 1981, 1984, 1985, 1986, 1987, 0, 836, 127, - 122, 820, 0, 584, 0, 766, 780, 820, 825, 1113, - 848, 1114, 0, 129, 1524, 1523, 1517, 210, 1383, 1578, - 1716, 1757, 1872, 1982, 1904, 1926, 1597, 1579, 1572, 1577, - 281, 651, 649, 0, 1302, 1716, 1757, 1859, 1872, 1982, - 1926, 1496, 1501, 0, 287, 1584, 127, 122, 1583, 0, - 590, 643, 0, 288, 1538, 0, 1543, 0, 1835, 617, - 620, 1377, 618, 582, 0, 0, 1, 168, 0, 174, - 0, 0, 647, 647, 0, 647, 0, 574, 0, 0, - 582, 577, 581, 763, 1443, 1553, 0, 1596, 1896, 1982, - 1904, 1586, 1582, 1726, 0, 0, 1726, 0, 1726, 0, - 1726, 0, 0, 1562, 1564, 0, 277, 1286, 0, 1347, - 130, 0, 0, 1432, 1428, 1433, 1429, 1434, 1427, 1426, - 1435, 1431, 0, 0, 0, 392, 425, 424, 423, 422, - 427, 1726, 1394, 221, 510, 511, 0, 0, 0, 0, - 0, 0, 1405, 121, 119, 1726, 1559, 454, 455, 0, - 444, 440, 442, 0, 0, 1726, 1372, 464, 460, 1726, - 464, 1339, 1726, 0, 0, 233, 0, 420, 1441, 1480, - 2131, 1494, 0, 1495, 1485, 1447, 1481, 1482, 168, 0, - 542, 1418, 0, 0, 0, 1234, 820, 825, 0, 0, - 838, 0, 1254, 0, 1260, 0, 0, 0, 820, 589, - 0, 780, 837, 123, 770, 0, 818, 819, 699, 699, - 652, 0, 633, 0, 699, 706, 699, 830, 0, 0, - 833, 831, 0, 833, 0, 0, 0, 833, 829, 789, - 0, 706, 0, 818, 821, 699, 0, 840, 1438, 0, - 0, 0, 0, 1575, 1573, 1574, 1580, 0, 1576, 0, - 0, 1349, 1351, 1352, 1202, 1362, 1089, 0, 1947, 1948, - 1949, 1277, 1950, 1951, 1953, 1954, 1955, 1046, 1688, 1956, - 1360, 1958, 1960, 1961, 1963, 1964, 1965, 0, 1966, 1967, - 1968, 0, 1361, 1971, 1797, 1976, 1977, 1979, 1982, 1983, - 1359, 0, 1988, 0, 0, 0, 1320, 1225, 0, 1088, - 0, 0, 0, 1279, 1287, 1081, 0, 0, 884, 885, - 906, 907, 886, 912, 913, 915, 887, 0, 1309, 979, - 1077, 1297, 1091, 1086, 1096, 1092, 1093, 1132, 1094, 1112, - 1097, 1169, 1087, 0, 1095, 1079, 1305, 633, 1303, 0, - 1080, 1348, 633, 1301, 1499, 1497, 1504, 1498, 0, 1500, - 0, 0, 0, 278, 123, 1546, 1545, 1537, 1535, 1536, - 1534, 1533, 1540, 0, 1542, 1444, 1279, 1220, 1222, 0, - 619, 0, 0, 624, 571, 570, 572, 3, 0, 0, - 0, 0, 1706, 0, 645, 646, 0, 0, 0, 0, - 0, 0, 0, 0, 747, 672, 673, 675, 744, 748, - 756, 0, 0, 0, 0, 0, 578, 0, 1377, 1525, - 1595, 1589, 0, 1587, 0, 0, 0, 152, 152, 0, - 0, 0, 0, 0, 112, 50, 105, 0, 0, 0, - 0, 255, 268, 0, 0, 0, 0, 0, 265, 0, - 0, 248, 52, 242, 244, 0, 152, 0, 48, 0, - 0, 0, 54, 1562, 0, 0, 1571, 285, 286, 1285, - 0, 132, 133, 131, 124, 0, 2145, 2000, 2001, 2002, - 2003, 2150, 2004, 1953, 2005, 2006, 0, 2007, 2008, 1959, - 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 1966, 2019, - 2020, 2021, 2022, 2023, 2024, 2173, 2025, 1980, 2027, 1986, - 2178, 0, 2028, 1104, 655, 1228, 657, 1226, 1378, 0, - 125, 1365, 0, 1430, 0, 0, 0, 0, 540, 0, - 0, 0, 0, 1390, 1726, 222, 226, 0, 1726, 217, - 1726, 392, 0, 1726, 0, 1726, 392, 1726, 0, 1404, - 1407, 0, 457, 452, 450, 449, 451, 1726, 274, 0, - 0, 1373, 462, 463, 0, 431, 0, 0, 433, 0, - 0, 238, 0, 236, 0, 427, 168, 0, 249, 1490, - 1491, 1489, 0, 0, 1484, 1446, 252, 269, 1493, 1483, - 1492, 1445, 1440, 0, 0, 1436, 537, 0, 0, 0, - 1235, 955, 954, 936, 937, 952, 953, 938, 939, 946, - 947, 957, 956, 944, 945, 940, 941, 934, 935, 950, - 951, 942, 943, 948, 949, 932, 933, 1249, 1236, 1237, - 1238, 1239, 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, - 1248, 0, 0, 779, 776, 0, 0, 0, 0, 0, - 0, 1279, 0, 1052, 1087, 0, 0, 0, 1220, 1259, - 0, 0, 0, 0, 0, 0, 1220, 1265, 0, 0, - 804, 816, 0, 692, 698, 777, 775, 0, 1302, 767, - 0, 850, 780, 778, 0, 699, 774, 0, 830, 0, - 829, 0, 0, 832, 826, 0, 827, 0, 0, 0, - 0, 828, 0, 0, 0, 0, 0, 699, 0, 816, - 0, 773, 847, 1507, 1515, 211, 0, 1369, 2029, 2030, - 2031, 2032, 894, 2033, 923, 901, 2034, 923, 923, 2035, - 2036, 2037, 2038, 890, 890, 903, 2039, 2040, 2041, 2042, - 2043, 891, 892, 928, 2044, 2045, 2046, 2047, 2048, 0, - 0, 2049, 923, 2050, 890, 2051, 2052, 2053, 895, 2054, - 858, 2055, 0, 2056, 893, 859, 2057, 931, 931, 2058, - 0, 2059, 918, 2060, 0, 1231, 868, 876, 877, 878, - 879, 904, 905, 880, 910, 911, 881, 978, 0, 890, - 1370, 1371, 168, 1581, 1598, 0, 1225, 1098, 922, 909, - 1276, 0, 917, 916, 0, 1225, 899, 898, 897, 1083, - 0, 896, 0, 1182, 923, 923, 921, 1004, 900, 0, - 0, 0, 0, 0, 927, 0, 925, 0, 1005, 983, - 984, 0, 0, 1319, 1328, 1220, 1224, 0, 1081, 1220, - 0, 1090, 1100, 0, 1172, 1174, 0, 0, 0, 1280, - 1350, 1082, 0, 1355, 0, 0, 978, 978, 1308, 1202, - 0, 1192, 1195, 0, 0, 1199, 1200, 1201, 0, 0, - 0, 1300, 0, 1210, 1212, 0, 0, 1020, 1208, 0, - 1023, 0, 0, 0, 0, 1196, 1197, 1198, 1188, 1189, - 1190, 1191, 1193, 1194, 1206, 1187, 1001, 0, 1078, 0, - 1135, 0, 1000, 1306, 765, 0, 1353, 765, 1509, 1513, - 1514, 1508, 1512, 0, 1503, 1502, 1505, 1506, 0, 1547, - 1531, 0, 1528, 1223, 760, 621, 1341, 0, 0, 0, - 1552, 173, 172, 0, 0, 232, 0, 594, 593, 666, - 658, 660, 666, 0, 592, 0, 720, 721, 0, 0, - 0, 0, 753, 751, 1349, 1362, 708, 676, 707, 0, - 0, 680, 0, 712, 979, 746, 576, 670, 671, 674, - 575, 0, 749, 0, 759, 0, 613, 615, 598, 612, - 610, 595, 603, 747, 675, 0, 1554, 0, 0, 1518, - 1585, 1588, 0, 0, 0, 0, 0, 1726, 0, 0, - 861, 73, 69, 96, 342, 151, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 104, 101, 0, 0, 102, - 103, 0, 0, 0, 0, 1369, 253, 254, 267, 0, - 258, 259, 256, 260, 261, 0, 0, 246, 247, 0, - 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1564, 1563, 0, 1555, 1281, 1286, 657, - 657, 657, 0, 0, 0, 0, 655, 656, 0, 0, - 0, 0, 0, 536, 390, 400, 0, 0, 0, 1394, - 221, 0, 0, 0, 0, 0, 0, 0, 427, 1397, - 1395, 1393, 1396, 1398, 0, 0, 0, 0, 0, 213, - 216, 0, 389, 361, 0, 0, 0, 0, 1409, 0, - 0, 505, 503, 506, 495, 508, 498, 0, 1726, 379, - 1406, 0, 1560, 0, 0, 272, 464, 1374, 0, 461, - 464, 1340, 0, 464, 240, 0, 0, 1442, 1486, 250, - 270, 251, 271, 542, 1566, 1568, 0, 545, 550, 534, - 0, 534, 0, 547, 551, 534, 546, 0, 534, 541, - 0, 1128, 0, 1118, 0, 0, 839, 0, 0, 1119, - 1054, 1055, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1072, 1071, 1120, 843, 0, 846, 0, 0, 1257, 1258, - 0, 1121, 0, 0, 1264, 0, 0, 0, 1126, 0, - 781, 0, 0, 682, 683, 691, 687, 694, 0, 697, - 684, 633, 583, 1716, 1757, 0, 644, 644, 644, 642, - 632, 0, 724, 782, 0, 772, 0, 0, 0, 805, - 0, 0, 807, 809, 0, 0, 812, 0, 788, 787, - 0, 0, 771, 0, 0, 851, 0, 1345, 0, 0, - 212, 0, 0, 0, 876, 0, 0, 0, 866, 862, - 0, 958, 959, 960, 961, 962, 963, 964, 965, 966, - 967, 968, 969, 970, 882, 1382, 0, 888, 1385, 0, - 1386, 1387, 1384, 1381, 1388, 1389, 0, 0, 0, 0, - 1275, 1271, 0, 0, 0, 0, 0, 1177, 1179, 1181, - 0, 920, 919, 1186, 1192, 1195, 1199, 1200, 1201, 1196, - 1197, 1198, 1188, 1189, 1190, 1191, 1193, 1194, 0, 1214, - 0, 1168, 0, 0, 0, 0, 0, 0, 0, 0, - 1313, 1312, 0, 1336, 0, 1101, 1085, 0, 0, 1175, - 1102, 1310, 1320, 1288, 0, 0, 0, 1358, 1357, 980, - 989, 992, 1025, 1026, 996, 997, 998, 1002, 1380, 1379, - 1307, 0, 1299, 0, 0, 981, 1006, 1011, 0, 1266, - 1269, 1042, 1268, 0, 1030, 0, 1019, 0, 1028, 1032, - 1007, 1022, 0, 1003, 0, 1300, 1211, 1213, 0, 1209, - 0, 993, 994, 995, 985, 986, 987, 988, 990, 991, - 999, 1185, 1183, 1184, 0, 1286, 0, 1298, 0, 0, - 1137, 0, 0, 1027, 1304, 0, 850, 657, 850, 0, - 978, 1548, 1377, 1541, 1377, 1530, 1221, 1342, 1376, 0, - 631, 0, 1550, 159, 163, 0, 0, 1287, 193, 195, - 765, 0, 664, 665, 669, 0, 0, 669, 648, 591, - 1977, 1855, 0, 0, 0, 0, 713, 754, 0, 745, - 710, 711, 0, 709, 1349, 714, 1348, 715, 718, 719, - 681, 1337, 755, 757, 0, 750, 0, 1343, 597, 616, - 0, 0, 0, 0, 0, 580, 579, 761, 1525, 1525, - 1527, 1526, 0, 51, 0, 1726, 75, 0, 0, 0, - 0, 0, 0, 292, 71, 72, 0, 394, 0, 70, - 66, 292, 117, 1726, 464, 1726, 464, 1620, 1689, 1873, - 0, 64, 366, 108, 0, 145, 78, 80, 397, 0, - 351, 0, 0, 98, 113, 138, 0, 0, 53, 243, - 257, 262, 141, 266, 263, 1414, 264, 152, 0, 49, - 0, 139, 0, 1412, 0, 0, 55, 143, 1416, 1564, - 1571, 0, 0, 1285, 0, 655, 655, 655, 653, 654, - 1105, 0, 1227, 0, 1229, 1230, 1019, 1424, 1423, 1425, - 1422, 524, 535, 0, 391, 0, 539, 527, 528, 536, - 1392, 226, 0, 217, 392, 0, 0, 392, 0, 1394, - 0, 0, 221, 227, 0, 0, 0, 0, 0, 390, - 382, 380, 413, 0, 387, 381, 0, 0, 337, 0, - 1614, 0, 1694, 200, 205, 0, 0, 0, 0, 1363, - 2146, 2147, 2148, 2149, 2151, 2152, 2153, 2154, 2155, 2156, - 2157, 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, - 2167, 2168, 2169, 2170, 2171, 2172, 2174, 2175, 2176, 2177, - 2178, 2179, 512, 0, 515, 861, 1364, 0, 0, 0, - 0, 0, 274, 275, 430, 1375, 432, 0, 434, 239, - 237, 1437, 1565, 1567, 538, 0, 533, 0, 560, 0, - 0, 0, 0, 0, 0, 0, 0, 1115, 1233, 0, - 1252, 1251, 1053, 1060, 1063, 1067, 1068, 1069, 1253, 0, - 0, 0, 1064, 1065, 1066, 1056, 1057, 1058, 1059, 1061, - 1062, 1070, 848, 0, 0, 842, 1262, 1261, 1255, 1256, - 0, 1123, 1124, 1125, 1263, 0, 0, 817, 686, 688, - 685, 0, 0, 850, 644, 644, 644, 644, 641, 0, - 0, 0, 849, 0, 741, 702, 703, 0, 0, 813, - 811, 0, 835, 0, 808, 0, 814, 0, 799, 0, - 806, 855, 822, 0, 0, 824, 1516, 872, 0, 867, - 863, 0, 0, 0, 873, 0, 0, 0, 0, 0, - 0, 0, 1232, 0, 650, 1099, 0, 0, 0, 1272, - 0, 1047, 889, 902, 1024, 0, 1180, 1103, 0, 1203, - 1167, 930, 929, 931, 931, 1048, 0, 1315, 1317, 0, - 0, 0, 0, 1327, 0, 1050, 0, 1221, 1171, 1173, - 1328, 1084, 914, 978, 0, 0, 0, 0, 0, 0, - 0, 1031, 1021, 0, 1029, 1033, 0, 0, 0, 1015, - 0, 0, 1013, 1043, 1009, 0, 0, 1044, 1285, 0, - 1289, 0, 0, 1136, 1145, 768, 764, 724, 655, 724, - 0, 1510, 1532, 1529, 0, 629, 0, 0, 1551, 0, - 182, 0, 0, 0, 0, 0, 185, 199, 196, 1550, - 0, 0, 659, 661, 0, 1204, 669, 663, 717, 716, - 0, 679, 752, 677, 0, 758, 0, 614, 0, 600, - 0, 791, 0, 0, 1519, 1520, 0, 0, 0, 341, - 0, 0, 0, 292, 0, 402, 0, 409, 0, 0, - 394, 373, 68, 67, 97, 0, 0, 0, 60, 116, - 89, 81, 56, 95, 0, 0, 100, 0, 93, 110, - 111, 109, 114, 0, 302, 327, 0, 0, 338, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1571, 1557, 1570, 1286, 1286, 1282, 0, 0, - 0, 657, 1106, 0, 523, 565, 562, 563, 0, 561, - 248, 567, 401, 0, 0, 0, 215, 389, 0, 0, - 1409, 497, 500, 1391, 427, 0, 226, 0, 230, 0, - 0, 217, 392, 0, 365, 375, 376, 361, 388, 359, - 358, 360, 0, 1615, 241, 0, 1609, 0, 207, 203, - 392, 1408, 0, 0, 514, 0, 517, 860, 504, 0, - 509, 0, 0, 507, 0, 1403, 273, 464, 1569, 548, - 553, 0, 559, 555, 554, 549, 557, 556, 552, 1116, - 1127, 1250, 0, 0, 0, 0, 841, 844, 0, 1122, - 1117, 815, 0, 0, 724, 0, 0, 0, 0, 635, - 634, 640, 0, 0, 1139, 0, 705, 810, 0, 0, - 0, 797, 786, 792, 793, 0, 0, 0, 853, 852, - 823, 876, 0, 856, 876, 0, 876, 0, 874, 0, - 883, 971, 972, 973, 974, 975, 976, 977, 908, 0, - 1274, 1270, 1176, 1178, 1215, 926, 924, 1049, 1318, 1311, - 1314, 1220, 1322, 1324, 0, 0, 0, 0, 1335, 0, - 1170, 1336, 1356, 982, 0, 0, 1012, 1267, 1034, 0, - 0, 0, 1008, 1203, 0, 0, 0, 0, 0, 1017, - 0, 1293, 1286, 0, 1292, 0, 0, 0, 0, 1111, - 769, 741, 0, 741, 0, 1278, 0, 625, 627, 630, - 168, 1549, 0, 1544, 160, 161, 162, 0, 0, 0, - 177, 154, 0, 0, 0, 194, 182, 170, 667, 668, - 0, 662, 678, 1338, 1344, 599, 0, 1081, 0, 0, - 596, 0, 146, 292, 0, 0, 74, 0, 411, 353, - 403, 386, 368, 0, 0, 0, 293, 0, 428, 0, - 0, 374, 0, 0, 0, 0, 354, 0, 0, 313, - 0, 0, 386, 0, 393, 309, 310, 0, 59, 90, - 0, 86, 0, 115, 0, 0, 0, 0, 0, 62, - 85, 0, 57, 861, 464, 464, 65, 1369, 2029, 2030, - 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, - 2041, 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2166, 2049, - 299, 2050, 1797, 2051, 2052, 2053, 2054, 2055, 0, 2056, - 859, 2057, 2058, 2246, 2059, 2060, 1188, 1189, 298, 297, - 396, 294, 404, 296, 0, 1370, 295, 399, 352, 0, - 0, 142, 1415, 0, 140, 0, 1413, 149, 147, 144, - 1417, 1556, 0, 0, 1109, 1110, 1107, 655, 0, 0, - 0, 0, 542, 530, 0, 0, 0, 1614, 202, 0, - 0, 1726, 0, 0, 229, 228, 218, 0, 1409, 214, - 389, 0, 419, 337, 861, 414, 0, 1614, 1612, 0, - 0, 208, 0, 206, 1409, 1608, 496, 499, 513, 516, - 0, 0, 0, 0, 582, 501, 0, 0, 0, 435, - 558, 1073, 0, 0, 0, 0, 695, 0, 701, 741, - 639, 638, 637, 636, 723, 1663, 1960, 1854, 0, 727, - 722, 725, 730, 732, 731, 733, 729, 740, 0, 743, - 704, 834, 1216, 1218, 0, 0, 0, 0, 798, 800, - 0, 802, 0, 854, 870, 0, 871, 0, 869, 864, - 875, 1273, 1316, 1325, 1326, 1321, 1330, 1332, 0, 0, - 0, 979, 1051, 1040, 1038, 1035, 0, 1036, 1016, 0, - 0, 1014, 1010, 0, 1045, 0, 0, 1290, 0, 1131, - 0, 1134, 1148, 1144, 1143, 1139, 1106, 1139, 1511, 623, - 626, 0, 181, 158, 184, 183, 0, 1287, 191, 0, - 0, 182, 0, 491, 492, 493, 182, 0, 186, 520, - 0, 0, 611, 790, 604, 605, 0, 407, 76, 0, - 386, 0, 292, 370, 369, 372, 367, 371, 0, 429, - 0, 0, 311, 0, 318, 356, 357, 355, 312, 386, - 392, 314, 0, 0, 0, 82, 61, 58, 63, 83, - 0, 0, 84, 87, 855, 99, 92, 1369, 0, 0, - 0, 77, 79, 0, 0, 1284, 1283, 0, 526, 525, - 564, 566, 522, 531, 248, 0, 0, 0, 361, 1611, - 0, 0, 0, 389, 0, 231, 0, 0, 0, 1614, - 0, 0, 289, 0, 334, 0, 234, 1613, 201, 204, - 0, 0, 0, 1600, 518, 519, 0, 0, 1401, 1402, - 0, 1074, 0, 1075, 845, 0, 0, 693, 1139, 0, - 0, 0, 734, 728, 0, 1138, 1140, 0, 690, 1219, - 794, 0, 796, 0, 820, 0, 820, 803, 865, 857, - 1323, 1333, 1334, 1329, 1129, 0, 1037, 1041, 1039, 1018, - 1286, 1286, 1294, 1291, 1133, 1147, 1150, 743, 1354, 743, - 628, 622, 0, 0, 169, 0, 0, 166, 153, 470, - 0, 494, 468, 171, 1205, 601, 602, 0, 292, 0, - 385, 408, 323, 301, 0, 0, 0, 308, 315, 418, - 317, 0, 91, 107, 0, 0, 398, 150, 148, 1108, - 542, 0, 220, 1409, 337, 1608, 0, 0, 0, 0, - 361, 241, 1610, 350, 343, 344, 345, 346, 347, 348, - 349, 364, 363, 335, 336, 209, 0, 0, 0, 0, - 502, 1403, 0, 188, 197, 0, 188, 1076, 696, 0, - 743, 0, 0, 0, 726, 0, 0, 742, 0, 587, - 1217, 0, 785, 783, 0, 784, 1331, 0, 0, 0, - 0, 657, 690, 690, 155, 0, 156, 192, 0, 0, - 0, 0, 0, 489, 392, 410, 384, 0, 377, 321, - 320, 322, 326, 0, 324, 0, 340, 0, 333, 301, - 0, 94, 0, 405, 521, 529, 0, 291, 1602, 389, - 0, 219, 1608, 337, 1614, 1608, 0, 1605, 0, 0, - 0, 0, 190, 1409, 0, 190, 0, 690, 736, 0, - 735, 1142, 1141, 692, 795, 0, 1130, 1296, 1295, 0, - 1154, 586, 585, 0, 0, 0, 0, 469, 0, 0, - 470, 418, 0, 362, 0, 0, 323, 0, 316, 415, - 416, 417, 0, 329, 319, 330, 88, 106, 406, 0, - 389, 1603, 290, 235, 1601, 1606, 1607, 0, 188, 187, - 666, 189, 850, 198, 666, 700, 588, 737, 689, 801, - 1149, 0, 0, 0, 0, 0, 165, 850, 176, 0, - 478, 0, 485, 163, 163, 486, 487, 488, 0, 333, - 383, 378, 300, 325, 339, 0, 0, 0, 331, 0, - 332, 1608, 0, 190, 669, 1399, 669, 1946, 1664, 1911, - 0, 1166, 1155, 1166, 1166, 1146, 157, 164, 0, 482, - 483, 484, 0, 0, 474, 0, 0, 477, 0, 292, - 305, 0, 304, 0, 395, 328, 1604, 1409, 666, 178, - 179, 0, 1159, 1158, 1157, 1161, 1160, 0, 1153, 1151, - 1152, 850, 481, 0, 473, 480, 0, 476, 475, 490, - 412, 303, 307, 306, 850, 669, 0, 0, 1163, 0, - 1164, 175, 471, 0, 1400, 180, 1156, 1162, 1165, 0, - 479 + 169, 287, 0, 1491, 1490, 1565, 287, 0, 1424, 0, + 287, 547, 430, 0, 1595, 1594, 0, 224, 287, 0, + 169, 0, 1525, 0, 0, 0, 609, 612, 610, 0, + 0, 0, 287, 655, 0, 1596, 287, 0, 0, 647, + 611, 0, 1542, 0, 0, 0, 0, 0, 2, 4, + 8, 22, 37, 32, 0, 21, 35, 19, 18, 40, + 27, 7, 5, 25, 39, 42, 20, 26, 34, 16, + 41, 14, 38, 585, 571, 660, 584, 0, 0, 168, + 765, 592, 36, 17, 31, 6, 12, 13, 29, 30, + 28, 1447, 45, 33, 0, 43, 23, 9, 10, 24, + 44, 46, 1597, 1593, 11, 47, 15, 286, 285, 279, + 0, 0, 0, 0, 0, 0, 1564, 0, 0, 0, + 290, 125, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, + 1627, 1628, 1629, 2003, 1630, 1631, 2004, 1632, 1633, 2005, + 1634, 1635, 1636, 1949, 1950, 2006, 1951, 1952, 1637, 1638, + 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1953, 1954, 1646, + 1647, 1648, 1649, 1650, 1955, 2007, 1956, 1651, 1652, 1653, + 1654, 1655, 2008, 1656, 1657, 1658, 1659, 1660, 1661, 1662, + 1663, 1664, 2009, 1665, 1666, 1667, 1668, 1669, 1670, 1671, + 1672, 1673, 1674, 1957, 1675, 1676, 1958, 1677, 1678, 1679, + 1680, 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, 1689, + 1690, 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, 1699, + 1700, 1701, 1702, 1703, 1704, 1959, 1705, 1706, 1707, 1708, + 1709, 1710, 1960, 1711, 1712, 1713, 1961, 1714, 1715, 1716, + 2010, 2011, 1717, 1718, 1962, 2013, 1719, 1720, 1721, 1963, + 1964, 1722, 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1730, + 2014, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, 1739, + 1740, 1741, 1742, 2015, 1965, 1743, 1744, 1745, 1746, 1747, + 1966, 1967, 1968, 1748, 2016, 2017, 1749, 2018, 1750, 1751, + 1752, 1753, 1754, 1755, 1756, 2019, 1757, 2020, 1758, 1759, + 1760, 1761, 1762, 1763, 1764, 1765, 1969, 1766, 1767, 1768, + 1769, 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, + 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1970, + 2022, 1971, 1788, 1789, 1790, 1972, 1791, 1792, 2023, 1793, + 1973, 1794, 1974, 1795, 1796, 1797, 1798, 1799, 1800, 1801, + 1802, 1803, 1804, 1975, 2024, 1805, 2025, 1976, 1806, 1807, + 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1816, 1817, + 1818, 1819, 1977, 2026, 1820, 1821, 1978, 1822, 1823, 1824, + 1825, 1826, 1827, 1828, 1829, 1830, 1831, 1832, 1833, 1834, + 1835, 1979, 1836, 1837, 1838, 1839, 1840, 1841, 1842, 1843, + 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, + 1854, 2027, 1855, 1856, 1857, 1980, 1858, 1859, 1860, 1861, + 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1871, + 1872, 1873, 1874, 1875, 1876, 1981, 1877, 1878, 2028, 1879, + 1880, 1982, 1881, 1882, 1883, 1884, 1885, 1886, 1887, 1888, + 1889, 1890, 1891, 1892, 1893, 1894, 1895, 1983, 1896, 1984, + 1897, 1898, 1899, 2030, 1900, 1901, 1902, 1903, 1904, 1905, + 1906, 1985, 1986, 1907, 1908, 1987, 1909, 1988, 1910, 1911, + 1989, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, 1920, + 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1990, 1991, + 1929, 1930, 2031, 1931, 1932, 1933, 1934, 1935, 1936, 1937, + 1938, 1939, 1940, 1941, 1942, 1943, 1944, 1992, 1993, 1994, + 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 1945, 1946, + 1947, 1948, 0, 1602, 0, 1349, 126, 127, 1371, 125, + 1962, 1969, 1983, 1423, 1422, 126, 0, 282, 546, 0, + 0, 0, 0, 0, 0, 226, 0, 424, 423, 1413, + 429, 0, 0, 0, 129, 121, 1822, 128, 1348, 119, + 135, 2183, 2184, 2185, 2186, 2064, 2187, 2188, 2189, 2190, + 2065, 2191, 2066, 2067, 2068, 2069, 2070, 2071, 2072, 2192, + 2073, 2193, 2194, 2075, 2074, 2195, 2076, 2196, 2077, 2197, + 2078, 2079, 2198, 2199, 2080, 1671, 2081, 2082, 2200, 2201, + 2202, 2203, 2204, 2205, 2206, 2207, 2208, 2083, 2084, 2209, + 2210, 2085, 2211, 2212, 2086, 2213, 2087, 2088, 2089, 2214, + 2090, 2091, 2215, 2092, 2216, 2217, 2093, 2094, 2097, 2095, + 2218, 2096, 2219, 2098, 2099, 2100, 2220, 2221, 2222, 2101, + 2102, 2223, 2103, 2104, 2105, 2106, 2107, 2224, 2108, 2225, + 2109, 2110, 2226, 2227, 2228, 2229, 2230, 2112, 2111, 2113, + 2114, 2231, 2232, 2233, 2234, 2115, 2116, 2117, 2235, 2236, + 2118, 2237, 2238, 2119, 2120, 2239, 2121, 2122, 2240, 2123, + 2124, 2241, 2125, 2126, 2242, 2243, 2244, 2127, 2245, 2128, + 2129, 2246, 2247, 2130, 2131, 2248, 2132, 2249, 2250, 2133, + 2251, 2252, 2134, 2135, 2253, 2136, 2254, 2255, 2256, 2257, + 2137, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 2145, 2146, + 2147, 1561, 137, 136, 138, 0, 448, 449, 0, 459, + 0, 441, 446, 442, 0, 468, 461, 469, 450, 440, + 462, 451, 439, 225, 0, 470, 456, 444, 0, 0, + 0, 0, 283, 242, 0, 430, 0, 169, 0, 1453, + 1463, 1473, 1468, 1462, 1471, 1460, 1477, 1466, 1452, 1475, + 1461, 1465, 1470, 1458, 1476, 1456, 1474, 1472, 1459, 1467, + 1451, 1455, 1442, 1447, 1480, 1469, 1478, 1464, 1479, 1481, + 1454, 1482, 1457, 0, 1424, 0, 1955, 2007, 1960, 0, + 1973, 0, 1976, 1977, 1858, 1984, 1987, 1988, 1989, 1990, + 0, 839, 128, 123, 823, 0, 587, 0, 769, 783, + 823, 828, 1116, 851, 1117, 0, 130, 1527, 1526, 1520, + 211, 1386, 1581, 1719, 1760, 1875, 1985, 1907, 1929, 1600, + 1582, 1575, 1580, 284, 654, 652, 0, 1305, 1719, 1760, + 1862, 1875, 1985, 1929, 1499, 1504, 0, 290, 1587, 128, + 123, 1586, 0, 593, 646, 0, 291, 1541, 0, 1546, + 0, 1838, 620, 623, 1380, 621, 585, 0, 0, 1, + 169, 0, 175, 0, 0, 650, 650, 0, 650, 0, + 577, 0, 0, 585, 580, 584, 766, 1446, 1556, 0, + 1599, 1899, 1985, 1907, 1589, 1585, 1729, 0, 1729, 0, + 0, 1729, 0, 1729, 0, 1729, 0, 0, 1565, 1567, + 0, 280, 1289, 0, 1350, 131, 0, 0, 1435, 1431, + 1436, 1432, 1437, 1430, 1429, 1438, 1434, 0, 0, 0, + 395, 428, 427, 426, 425, 430, 1729, 1397, 222, 513, + 514, 0, 0, 0, 0, 0, 0, 1408, 122, 120, + 1729, 1562, 457, 458, 0, 447, 443, 445, 0, 0, + 1729, 1375, 467, 463, 1729, 467, 1342, 1729, 0, 0, + 234, 0, 423, 1444, 1483, 2134, 1497, 0, 1498, 1488, + 1450, 1484, 1485, 169, 0, 545, 1421, 0, 0, 0, + 1237, 823, 828, 0, 0, 841, 0, 1257, 0, 1263, + 0, 0, 0, 823, 592, 0, 783, 840, 124, 773, + 0, 821, 822, 702, 702, 655, 0, 636, 0, 702, + 709, 702, 833, 0, 0, 836, 834, 0, 836, 0, + 0, 0, 836, 832, 792, 0, 709, 0, 821, 824, + 702, 0, 843, 1441, 0, 0, 0, 0, 1578, 1576, + 1577, 1583, 0, 1579, 0, 0, 1352, 1354, 1355, 1205, + 1365, 1092, 0, 1950, 1951, 1952, 1280, 1953, 1954, 1956, + 1957, 1958, 1049, 1691, 1959, 1363, 1961, 1963, 1964, 1966, + 1967, 1968, 0, 1969, 1970, 1971, 0, 1364, 1974, 1800, + 1979, 1980, 1982, 1985, 1986, 1362, 0, 1991, 0, 0, + 0, 1323, 1228, 0, 1091, 0, 0, 0, 1282, 1290, + 1084, 0, 0, 887, 888, 909, 910, 889, 915, 916, + 918, 890, 0, 1312, 982, 1080, 1300, 1094, 1089, 1099, + 1095, 1096, 1135, 1097, 1115, 1100, 1172, 1090, 0, 1098, + 1082, 1308, 636, 1306, 0, 1083, 1351, 636, 1304, 1502, + 1500, 1507, 1501, 0, 1503, 0, 0, 0, 281, 124, + 1549, 1548, 1540, 1538, 1539, 1537, 1536, 1543, 0, 1545, + 1447, 1282, 1223, 1225, 0, 622, 0, 0, 627, 574, + 573, 575, 3, 0, 0, 0, 0, 1709, 0, 648, + 649, 0, 0, 0, 0, 0, 0, 0, 0, 750, + 675, 676, 678, 747, 751, 759, 0, 0, 0, 0, + 0, 581, 0, 1380, 1528, 1598, 1592, 0, 1590, 0, + 0, 0, 0, 0, 153, 153, 0, 0, 0, 0, + 0, 113, 51, 106, 0, 0, 0, 0, 256, 269, + 0, 0, 0, 0, 0, 266, 0, 0, 249, 53, + 243, 245, 0, 153, 0, 49, 0, 0, 0, 55, + 1565, 0, 0, 1574, 288, 289, 1288, 0, 133, 134, + 132, 125, 0, 2148, 2003, 2004, 2005, 2006, 2153, 2007, + 1956, 2008, 2009, 0, 2010, 2011, 1962, 2013, 2014, 2015, + 2016, 2017, 2018, 2019, 2020, 1969, 2022, 2023, 2024, 2025, + 2026, 2027, 2176, 2028, 1983, 2030, 1989, 2181, 0, 2031, + 1107, 658, 1231, 660, 1229, 1381, 0, 126, 1368, 0, + 1433, 0, 0, 0, 0, 543, 0, 0, 0, 0, + 1393, 1729, 223, 227, 0, 1729, 218, 1729, 395, 0, + 1729, 0, 1729, 395, 1729, 0, 1407, 1410, 0, 460, + 455, 453, 452, 454, 1729, 277, 0, 0, 1376, 465, + 466, 0, 434, 0, 0, 436, 0, 0, 239, 0, + 237, 0, 430, 169, 0, 250, 1493, 1494, 1492, 0, + 0, 1487, 1449, 253, 270, 1496, 1486, 1495, 1448, 1443, + 0, 0, 1439, 540, 0, 0, 0, 1238, 958, 957, + 939, 940, 955, 956, 941, 942, 949, 950, 960, 959, + 947, 948, 943, 944, 937, 938, 953, 954, 945, 946, + 951, 952, 935, 936, 1252, 1239, 1240, 1241, 1242, 1243, + 1244, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 0, 0, + 782, 779, 0, 0, 0, 0, 0, 0, 1282, 0, + 1055, 1090, 0, 0, 0, 1223, 1262, 0, 0, 0, + 0, 0, 0, 1223, 1268, 0, 0, 807, 819, 0, + 695, 701, 780, 778, 0, 1305, 770, 0, 853, 783, + 781, 0, 702, 777, 0, 833, 0, 832, 0, 0, + 835, 829, 0, 830, 0, 0, 0, 0, 831, 0, + 0, 0, 0, 0, 702, 0, 819, 0, 776, 850, + 1510, 1518, 212, 0, 1372, 2032, 2033, 2034, 2035, 897, + 2036, 926, 904, 2037, 926, 926, 2038, 2039, 2040, 2041, + 893, 893, 906, 2042, 2043, 2044, 2045, 2046, 894, 895, + 931, 2047, 2048, 2049, 2050, 2051, 0, 0, 2052, 926, + 2053, 893, 2054, 2055, 2056, 898, 2057, 861, 2058, 0, + 2059, 896, 862, 2060, 934, 934, 2061, 0, 2062, 921, + 2063, 0, 1234, 879, 879, 880, 881, 882, 907, 908, + 883, 913, 914, 884, 981, 0, 893, 1373, 1374, 169, + 1584, 1601, 0, 1228, 1101, 925, 912, 1279, 0, 920, + 919, 0, 1228, 902, 901, 900, 1086, 0, 899, 0, + 1185, 926, 926, 924, 1007, 903, 0, 0, 0, 0, + 0, 930, 0, 928, 0, 1008, 986, 987, 0, 0, + 1322, 1331, 1223, 1227, 0, 1084, 1223, 0, 1093, 1103, + 0, 1175, 1177, 0, 0, 0, 1283, 1353, 1085, 0, + 1358, 0, 0, 981, 981, 1311, 1205, 0, 1195, 1198, + 0, 0, 1202, 1203, 1204, 0, 0, 0, 1303, 0, + 1213, 1215, 0, 0, 1023, 1211, 0, 1026, 0, 0, + 0, 0, 1199, 1200, 1201, 1191, 1192, 1193, 1194, 1196, + 1197, 1209, 1190, 1004, 0, 1081, 0, 1138, 0, 1003, + 1309, 768, 0, 1356, 768, 1512, 1516, 1517, 1511, 1515, + 0, 1506, 1505, 1508, 1509, 0, 1550, 1534, 0, 1531, + 1226, 763, 624, 1344, 0, 0, 0, 1555, 174, 173, + 0, 0, 233, 0, 597, 596, 669, 661, 663, 669, + 0, 595, 0, 723, 724, 0, 0, 0, 0, 756, + 754, 1352, 1365, 711, 679, 710, 0, 0, 683, 0, + 715, 982, 749, 579, 673, 674, 677, 578, 0, 752, + 0, 762, 0, 616, 618, 601, 615, 613, 598, 606, + 750, 678, 0, 1557, 0, 0, 1521, 1588, 1591, 0, + 0, 0, 0, 0, 0, 0, 1729, 0, 0, 864, + 74, 70, 97, 345, 152, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 105, 102, 0, 0, 103, 104, + 0, 0, 0, 0, 1372, 254, 255, 268, 0, 259, + 260, 257, 261, 262, 0, 0, 247, 248, 0, 0, + 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1567, 1566, 0, 1558, 1284, 1289, 660, 660, + 660, 0, 0, 0, 0, 658, 659, 0, 0, 0, + 0, 0, 539, 393, 403, 0, 0, 0, 1397, 222, + 0, 0, 0, 0, 0, 0, 0, 430, 1400, 1398, + 1396, 1399, 1401, 0, 0, 0, 0, 0, 214, 217, + 0, 392, 364, 0, 0, 0, 0, 1412, 0, 0, + 508, 506, 509, 498, 511, 501, 0, 1729, 382, 1409, + 0, 1563, 0, 0, 275, 467, 1377, 0, 464, 467, + 1343, 0, 467, 241, 0, 0, 1445, 1489, 251, 271, + 252, 272, 545, 1569, 1571, 0, 548, 553, 537, 0, + 537, 0, 550, 554, 537, 549, 0, 537, 544, 0, + 1131, 0, 1121, 0, 0, 842, 0, 0, 1122, 1057, + 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, + 1074, 1123, 846, 0, 849, 0, 0, 1260, 1261, 0, + 1124, 0, 0, 1267, 0, 0, 0, 1129, 0, 784, + 0, 0, 685, 686, 694, 690, 697, 0, 700, 687, + 636, 586, 1719, 1760, 0, 647, 647, 647, 645, 635, + 0, 727, 785, 0, 775, 0, 0, 0, 808, 0, + 0, 810, 812, 0, 0, 815, 0, 791, 790, 0, + 0, 774, 0, 0, 854, 0, 1348, 0, 0, 213, + 0, 0, 0, 879, 0, 0, 0, 871, 869, 865, + 0, 961, 962, 963, 964, 965, 966, 967, 968, 969, + 970, 971, 972, 973, 885, 1385, 0, 891, 1388, 0, + 1389, 1390, 1387, 1384, 1391, 1392, 0, 0, 0, 0, + 1278, 1274, 0, 0, 0, 0, 0, 1180, 1182, 1184, + 0, 923, 922, 1189, 1195, 1198, 1202, 1203, 1204, 1199, + 1200, 1201, 1191, 1192, 1193, 1194, 1196, 1197, 0, 1217, + 0, 1171, 0, 0, 0, 0, 0, 0, 0, 0, + 1316, 1315, 0, 1339, 0, 1104, 1088, 0, 0, 1178, + 1105, 1313, 1323, 1291, 0, 0, 0, 1361, 1360, 983, + 992, 995, 1028, 1029, 999, 1000, 1001, 1005, 1383, 1382, + 1310, 0, 1302, 0, 0, 984, 1009, 1014, 0, 1269, + 1272, 1045, 1271, 0, 1033, 0, 1022, 0, 1031, 1035, + 1010, 1025, 0, 1006, 0, 1303, 1214, 1216, 0, 1212, + 0, 996, 997, 998, 988, 989, 990, 991, 993, 994, + 1002, 1188, 1186, 1187, 0, 1289, 0, 1301, 0, 0, + 1140, 0, 0, 1030, 1307, 0, 853, 660, 853, 0, + 981, 1551, 1380, 1544, 1380, 1533, 1224, 1345, 1379, 0, + 634, 0, 1553, 160, 164, 0, 0, 1290, 194, 196, + 768, 0, 667, 668, 672, 0, 0, 672, 651, 594, + 1980, 1858, 0, 0, 0, 0, 716, 757, 0, 748, + 713, 714, 0, 712, 1352, 717, 1351, 718, 721, 722, + 684, 1340, 758, 760, 0, 753, 0, 1346, 600, 619, + 0, 0, 0, 0, 0, 583, 582, 764, 1528, 1528, + 1530, 1529, 0, 273, 0, 52, 0, 1729, 76, 0, + 0, 0, 0, 0, 0, 295, 72, 73, 0, 397, + 0, 71, 67, 295, 118, 1729, 467, 1729, 467, 1623, + 1692, 1876, 0, 65, 369, 109, 0, 146, 79, 81, + 400, 0, 354, 0, 0, 99, 114, 139, 0, 0, + 54, 244, 258, 263, 142, 267, 264, 1417, 265, 153, + 0, 50, 0, 140, 0, 1415, 0, 0, 56, 144, + 1419, 1567, 1574, 0, 0, 1288, 0, 658, 658, 658, + 656, 657, 1108, 0, 1230, 0, 1232, 1233, 1022, 1427, + 1426, 1428, 1425, 527, 538, 0, 394, 0, 542, 530, + 531, 539, 1395, 227, 0, 218, 395, 0, 0, 395, + 0, 1397, 0, 0, 222, 228, 0, 0, 0, 0, + 0, 393, 385, 383, 416, 0, 390, 384, 0, 0, + 340, 0, 1617, 0, 1697, 201, 206, 0, 0, 0, + 0, 1366, 2149, 2150, 2151, 2152, 2154, 2155, 2156, 2157, + 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2167, + 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2175, 2177, 2178, + 2179, 2180, 2181, 2182, 515, 0, 518, 864, 1367, 0, + 0, 0, 0, 0, 277, 278, 433, 1378, 435, 0, + 437, 240, 238, 1440, 1568, 1570, 541, 0, 536, 0, + 563, 0, 0, 0, 0, 0, 0, 0, 0, 1118, + 1236, 0, 1255, 1254, 1056, 1063, 1066, 1070, 1071, 1072, + 1256, 0, 0, 0, 1067, 1068, 1069, 1059, 1060, 1061, + 1062, 1064, 1065, 1073, 851, 0, 0, 845, 1265, 1264, + 1258, 1259, 0, 1126, 1127, 1128, 1266, 0, 0, 820, + 689, 691, 688, 0, 0, 853, 647, 647, 647, 647, + 644, 0, 0, 0, 852, 0, 744, 705, 706, 0, + 0, 816, 814, 0, 838, 0, 811, 0, 817, 0, + 802, 0, 809, 858, 825, 0, 0, 827, 1519, 875, + 0, 870, 866, 0, 0, 0, 876, 0, 0, 0, + 0, 0, 0, 0, 1235, 0, 653, 1102, 0, 0, + 0, 1275, 0, 1050, 892, 905, 1027, 0, 1183, 1106, + 0, 1206, 1170, 933, 932, 934, 934, 1051, 0, 1318, + 1320, 0, 0, 0, 0, 1330, 0, 1053, 0, 1224, + 1174, 1176, 1331, 1087, 917, 981, 0, 0, 0, 0, + 0, 0, 0, 1034, 1024, 0, 1032, 1036, 0, 0, + 0, 1018, 0, 0, 1016, 1046, 1012, 0, 0, 1047, + 1288, 0, 1292, 0, 0, 1139, 1148, 771, 767, 727, + 658, 727, 0, 1513, 1535, 1532, 0, 632, 0, 0, + 1554, 0, 183, 0, 0, 0, 0, 0, 186, 200, + 197, 1553, 0, 0, 662, 664, 0, 1207, 672, 666, + 720, 719, 0, 682, 755, 680, 0, 761, 0, 617, + 0, 603, 0, 794, 0, 0, 1522, 1523, 0, 0, + 0, 0, 344, 0, 0, 0, 295, 0, 405, 0, + 412, 0, 0, 397, 376, 69, 68, 98, 0, 0, + 0, 61, 117, 90, 82, 57, 96, 0, 0, 101, + 0, 94, 111, 112, 110, 115, 0, 305, 330, 0, + 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1574, 1560, 1573, 1289, 1289, + 1285, 0, 0, 0, 660, 1109, 0, 526, 568, 565, + 566, 0, 564, 249, 570, 404, 0, 0, 0, 216, + 392, 0, 0, 1412, 500, 503, 1394, 430, 0, 227, + 0, 231, 0, 0, 218, 395, 0, 368, 378, 379, + 364, 391, 362, 361, 363, 0, 1618, 242, 0, 1612, + 0, 208, 204, 395, 1411, 0, 0, 517, 0, 520, + 863, 507, 0, 512, 0, 0, 510, 0, 1406, 276, + 467, 1572, 551, 556, 0, 562, 558, 557, 552, 560, + 559, 555, 1119, 1130, 1253, 0, 0, 0, 0, 844, + 847, 0, 1125, 1120, 818, 0, 0, 727, 0, 0, + 0, 0, 638, 637, 643, 0, 0, 1142, 0, 708, + 813, 0, 0, 0, 800, 789, 795, 796, 0, 0, + 0, 856, 855, 826, 879, 0, 859, 879, 0, 879, + 877, 0, 0, 886, 974, 975, 976, 977, 978, 979, + 980, 911, 0, 1277, 1273, 1179, 1181, 1218, 929, 927, + 1052, 1321, 1314, 1317, 1223, 1325, 1327, 0, 0, 0, + 0, 1338, 0, 1173, 1339, 1359, 985, 0, 0, 1015, + 1270, 1037, 0, 0, 0, 1011, 1206, 0, 0, 0, + 0, 0, 1020, 0, 1296, 1289, 0, 1295, 0, 0, + 0, 0, 1114, 772, 744, 0, 744, 0, 1281, 0, + 628, 630, 633, 169, 1552, 0, 1547, 161, 162, 163, + 0, 0, 0, 178, 155, 0, 0, 0, 195, 183, + 171, 670, 671, 0, 665, 681, 1341, 1347, 602, 0, + 1084, 0, 0, 599, 0, 274, 147, 295, 0, 0, + 75, 0, 414, 356, 406, 389, 371, 0, 0, 0, + 296, 0, 431, 0, 0, 377, 0, 0, 0, 0, + 357, 0, 0, 316, 0, 0, 389, 0, 396, 312, + 313, 0, 60, 91, 0, 87, 0, 116, 0, 0, + 0, 0, 0, 63, 86, 0, 58, 864, 467, 467, + 66, 1372, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, + 2040, 2041, 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2049, + 2050, 2051, 2169, 2052, 302, 2053, 1800, 2054, 2055, 2056, + 2057, 2058, 0, 2059, 862, 2060, 2061, 2249, 2062, 2063, + 1191, 1192, 301, 300, 399, 297, 407, 299, 0, 1373, + 298, 402, 355, 0, 0, 143, 1418, 0, 141, 0, + 1416, 150, 148, 145, 1420, 1559, 0, 0, 1112, 1113, + 1110, 658, 0, 0, 0, 0, 545, 533, 0, 0, + 0, 1617, 203, 0, 0, 1729, 0, 0, 230, 229, + 219, 0, 1412, 215, 392, 0, 422, 340, 864, 417, + 0, 1617, 1615, 0, 0, 209, 0, 207, 1412, 1611, + 499, 502, 516, 519, 0, 0, 0, 0, 585, 504, + 0, 0, 0, 438, 561, 1076, 0, 0, 0, 0, + 698, 0, 704, 744, 642, 641, 640, 639, 726, 1666, + 1963, 1857, 0, 730, 725, 728, 733, 735, 734, 736, + 732, 743, 0, 746, 707, 837, 1219, 1221, 0, 0, + 0, 0, 801, 803, 0, 805, 0, 857, 873, 0, + 874, 0, 872, 878, 867, 1276, 1319, 1328, 1329, 1324, + 1333, 1335, 0, 0, 0, 982, 1054, 1043, 1041, 1038, + 0, 1039, 1019, 0, 0, 1017, 1013, 0, 1048, 0, + 0, 1293, 0, 1134, 0, 1137, 1151, 1147, 1146, 1142, + 1109, 1142, 1514, 626, 629, 0, 182, 159, 185, 184, + 0, 1290, 192, 0, 0, 183, 0, 494, 495, 496, + 183, 0, 187, 523, 0, 0, 614, 793, 607, 608, + 0, 410, 77, 0, 389, 0, 295, 373, 372, 375, + 370, 374, 0, 432, 0, 0, 314, 0, 321, 359, + 360, 358, 315, 389, 395, 317, 0, 0, 0, 83, + 62, 59, 64, 84, 0, 0, 85, 88, 858, 100, + 93, 1372, 0, 0, 0, 78, 80, 0, 0, 1287, + 1286, 0, 529, 528, 567, 569, 525, 534, 249, 0, + 0, 0, 364, 1614, 0, 0, 0, 392, 0, 232, + 0, 0, 0, 1617, 0, 0, 292, 0, 337, 0, + 235, 1616, 202, 205, 0, 0, 0, 1603, 521, 522, + 0, 0, 1404, 1405, 0, 1077, 0, 1078, 848, 0, + 0, 696, 1142, 0, 0, 0, 737, 731, 0, 1141, + 1143, 0, 693, 1222, 797, 0, 799, 0, 823, 0, + 823, 806, 868, 860, 1326, 1336, 1337, 1332, 1132, 0, + 1040, 1044, 1042, 1021, 1289, 1289, 1297, 1294, 1136, 1150, + 1153, 746, 1357, 746, 631, 625, 0, 0, 170, 0, + 0, 167, 154, 473, 0, 497, 471, 172, 1208, 604, + 605, 0, 295, 0, 388, 411, 326, 304, 0, 0, + 0, 311, 318, 421, 320, 0, 92, 108, 0, 0, + 401, 151, 149, 1111, 545, 0, 221, 1412, 340, 1611, + 0, 0, 0, 0, 364, 242, 1613, 353, 346, 347, + 348, 349, 350, 351, 352, 367, 366, 338, 339, 210, + 0, 0, 0, 0, 505, 1406, 0, 189, 198, 0, + 189, 1079, 699, 0, 746, 0, 0, 0, 729, 0, + 0, 745, 0, 590, 1220, 0, 788, 786, 0, 787, + 1334, 0, 0, 0, 0, 660, 693, 693, 156, 0, + 157, 193, 0, 0, 0, 0, 0, 492, 395, 413, + 387, 0, 380, 324, 323, 325, 329, 0, 327, 0, + 343, 0, 336, 304, 0, 95, 0, 408, 524, 532, + 0, 294, 1605, 392, 0, 220, 1611, 340, 1617, 1611, + 0, 1608, 0, 0, 0, 0, 191, 1412, 0, 191, + 0, 693, 739, 0, 738, 1145, 1144, 695, 798, 0, + 1133, 1299, 1298, 0, 1157, 589, 588, 0, 0, 0, + 0, 472, 0, 0, 473, 421, 0, 365, 0, 0, + 326, 0, 319, 418, 419, 420, 0, 332, 322, 333, + 89, 107, 409, 0, 392, 1606, 293, 236, 1604, 1609, + 1610, 0, 189, 188, 669, 190, 853, 199, 669, 703, + 591, 740, 692, 804, 1152, 0, 0, 0, 0, 0, + 166, 853, 177, 0, 481, 0, 488, 164, 164, 489, + 490, 491, 0, 336, 386, 381, 303, 328, 342, 0, + 0, 0, 334, 0, 335, 1611, 0, 191, 672, 1402, + 672, 1949, 1667, 1914, 0, 1169, 1158, 1169, 1169, 1149, + 158, 165, 0, 485, 486, 487, 0, 0, 477, 0, + 0, 480, 0, 295, 308, 0, 307, 0, 398, 331, + 1607, 1412, 669, 179, 180, 0, 1162, 1161, 1160, 1164, + 1163, 0, 1156, 1154, 1155, 853, 484, 0, 476, 483, + 0, 479, 478, 493, 415, 306, 310, 309, 853, 672, + 0, 0, 1166, 0, 1167, 176, 474, 0, 1403, 181, + 1159, 1165, 1168, 0, 482 }; /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int16 yydefgoto[] = { - -1, 47, 48, 49, 766, 2760, 2761, 2762, 2340, 2329, - 2330, 1810, 1811, 1244, 3601, 2341, 1245, 1246, 2764, 767, - 818, 1186, 867, 1124, 1659, 930, 1281, 1282, 768, 1817, - 769, 3010, 2254, 2703, 3580, 54, 3308, 2258, 1200, 3311, - 3544, 3003, 3306, 2705, 3622, 3680, 3309, 2259, 2260, 3545, - 2261, 770, 2443, 3192, 3193, 771, 772, 1909, 58, 1346, - 559, 1906, 3174, 2828, 2829, 773, 774, 1381, 1382, 985, - 775, 1910, 1849, 3129, 1264, 1839, 1396, 62, 1935, 776, - 108, 926, 64, 777, 2747, 3130, 3594, 2775, 3754, 3064, - 3065, 3591, 3592, 2750, 2343, 3663, 3664, 2843, 1830, 3658, - 2430, 3531, 2349, 2323, 3066, 2438, 3490, 3182, 2344, 3046, - 2835, 2836, 2431, 3587, 1930, 2432, 3588, 3332, 2433, 1885, - 1913, 2751, 3665, 2350, 1886, 2746, 3131, 1814, 2434, 3598, - 2435, 560, 3050, 778, 757, 758, 977, 1375, 759, 779, - 3582, 3745, 3775, 3705, 3740, 3313, 3650, 3314, 3315, 3316, - 780, 1921, 1922, 1923, 1924, 1925, 1926, 961, 1927, 2483, - 2484, 781, 782, 2804, 2409, 3384, 3385, 2507, 2403, 1405, - 1888, 1406, 549, 1969, 2810, 783, 1125, 73, 74, 1032, - 75, 3326, 76, 77, 1786, 1787, 1788, 869, 879, 880, - 1739, 2996, 2997, 2697, 1491, 2040, 872, 1206, 1755, 853, - 854, 1874, 896, 1877, 1750, 1751, 2264, 2712, 1779, 1780, - 1215, 1216, 2026, 2027, 3559, 2028, 2029, 1484, 1485, 3427, - 2577, 2578, 1495, 1767, 1771, 1772, 2285, 2275, 1758, 2574, - 3229, 3230, 3231, 3232, 3233, 3234, 3235, 1126, 2904, 3438, - 1775, 1776, 1218, 1219, 1220, 1784, 2295, 79, 80, 2236, - 2685, 2686, 824, 825, 3247, 1515, 1789, 2910, 2911, 2912, - 3250, 3251, 3252, 826, 1027, 1028, 1055, 1050, 1504, 2052, - 827, 828, 2003, 2004, 2545, 1057, 2042, 2064, 2065, 2918, - 2602, 1584, 2326, 1585, 1586, 2079, 1587, 1127, 1588, 1616, - 1128, 1621, 1590, 1129, 1130, 1131, 1593, 1132, 1133, 1134, - 1135, 1609, 1136, 1137, 1634, 2081, 2082, 2083, 2084, 2085, - 2086, 2087, 2088, 2089, 2090, 2091, 2092, 2093, 2094, 1187, - 1790, 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, - 1148, 830, 1149, 1150, 1710, 2230, 2684, 3239, 3435, 3436, - 2989, 3294, 3466, 3571, 3694, 3731, 3732, 3768, 1151, 1152, - 1654, 1655, 1656, 2117, 2118, 2119, 2120, 2224, 1704, 1705, - 1153, 3133, 1707, 2140, 3243, 3244, 1188, 1477, 1647, 1326, - 1327, 1598, 1451, 1452, 1458, 1978, 1466, 1470, 2008, 2009, - 1478, 2191, 1154, 2110, 2111, 2620, 1611, 2998, 1155, 1280, - 1660, 2984, 2227, 1708, 2184, 1162, 1156, 1163, 1158, 1643, - 2957, 2638, 2639, 1644, 2643, 2953, 2954, 2153, 2958, 3267, - 3268, 2645, 2292, 1736, 2297, 2298, 981, 1159, 1160, 1161, - 1328, 533, 1599, 3681, 1371, 1193, 1329, 2180, 784, 1062, - 2103, 785, 1343, 1901, 786, 3419, 3208, 1360, 1931, 2447, - 561, 787, 788, 542, 86, 2398, 942, 87, 88, 89, - 905, 1398, 789, 991, 1400, 992, 90, 1401, 994, 995, - 791, 861, 862, 1524, 1724, 1525, 792, 93, 836, 1799, - 793, 1182, 876, 1183, 1185, 794, 1203, 2700, 2252, 96, - 97, 98, 117, 1276, 2503, 1955, 1956, 1866, 795, 847, - 848, 885, 101, 102, 1231, 849, 797, 798, 3413, 799, - 2846, 1352, 543, 535, 536, 1601, 731, 1331, 732 + -1, 47, 48, 49, 769, 2773, 2774, 2775, 2352, 2341, + 2342, 1819, 1820, 1251, 3615, 2353, 1252, 1253, 2777, 770, + 821, 1191, 870, 1129, 1666, 935, 1288, 1289, 771, 1826, + 772, 3023, 2264, 2715, 3594, 54, 3322, 2268, 1205, 3325, + 3558, 3016, 3320, 2717, 3636, 3694, 3323, 2269, 2270, 3559, + 2271, 773, 2455, 3206, 3207, 774, 775, 1918, 58, 1353, + 561, 1915, 3188, 2841, 2842, 776, 777, 1388, 1389, 990, + 778, 1919, 1858, 3143, 1271, 1848, 1403, 62, 63, 1944, + 779, 109, 931, 65, 780, 2760, 3144, 3608, 2788, 3768, + 3078, 3079, 3605, 3606, 2763, 2355, 3677, 3678, 2856, 1839, + 3672, 2442, 3545, 2361, 2335, 3080, 2450, 3504, 3196, 2356, + 3060, 2848, 2849, 2443, 3601, 1939, 2444, 3602, 3346, 2445, + 1894, 1922, 2764, 3679, 2362, 1895, 2759, 3145, 1823, 2446, + 3612, 2447, 562, 3064, 781, 759, 760, 982, 1382, 761, + 782, 3596, 3759, 3789, 3719, 3754, 3327, 3664, 3328, 3329, + 3330, 783, 1930, 1931, 1932, 1933, 1934, 1935, 966, 1936, + 2495, 2496, 784, 785, 2817, 2421, 3398, 3399, 2519, 2415, + 1412, 1897, 1413, 551, 1978, 2823, 786, 1130, 74, 75, + 1037, 76, 3340, 77, 78, 1793, 1794, 1795, 872, 882, + 883, 1746, 3009, 3010, 2709, 1498, 2049, 875, 1211, 1762, + 856, 857, 1883, 899, 1886, 1757, 1758, 2274, 2724, 1786, + 1787, 1220, 1221, 2035, 2036, 3573, 2037, 2038, 1491, 1492, + 3441, 2589, 2590, 1502, 1774, 1778, 1779, 2295, 2285, 1765, + 2586, 3243, 3244, 3245, 3246, 3247, 3248, 3249, 1131, 2917, + 3452, 1782, 1783, 1223, 1224, 1225, 1791, 2305, 80, 81, + 2246, 2697, 2698, 827, 828, 3261, 1522, 1796, 2923, 2924, + 2925, 3264, 3265, 3266, 829, 1032, 1033, 1060, 1055, 1511, + 2061, 830, 831, 2012, 2013, 2557, 1062, 2051, 2073, 2074, + 2931, 2614, 1591, 2338, 1592, 1593, 2087, 1594, 1132, 1595, + 1623, 1133, 1628, 1597, 1134, 1135, 1136, 1600, 1137, 1138, + 1139, 1140, 1616, 1141, 1142, 1641, 2091, 2092, 2093, 2094, + 2095, 2096, 2097, 2098, 2099, 2100, 2101, 2102, 2103, 2104, + 1192, 1797, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, + 1152, 1153, 833, 1154, 1155, 1717, 2240, 2696, 3253, 3449, + 3450, 3002, 3308, 3480, 3585, 3708, 3745, 3746, 3782, 1156, + 1157, 1661, 1662, 1663, 2127, 2128, 2129, 2130, 2234, 1711, + 1712, 1158, 3147, 1714, 2150, 3257, 3258, 1193, 1484, 1654, + 1333, 1334, 1605, 1458, 1459, 1465, 1987, 1473, 1477, 2017, + 2018, 1485, 2201, 1159, 2120, 2121, 2632, 1618, 3011, 1160, + 1287, 1667, 2997, 2237, 1715, 2194, 1167, 1161, 1168, 1163, + 1650, 2970, 2650, 2651, 1651, 2655, 2966, 2967, 2163, 2971, + 3281, 3282, 2657, 2302, 1743, 2307, 2308, 986, 1164, 1165, + 1166, 1335, 535, 1606, 3695, 1378, 1198, 1336, 2190, 787, + 1067, 2113, 788, 1350, 1910, 789, 3433, 3222, 1367, 1940, + 2459, 563, 790, 791, 544, 87, 2410, 947, 88, 89, + 90, 908, 1405, 792, 996, 1407, 997, 91, 1408, 999, + 1000, 794, 864, 865, 1531, 1731, 1532, 795, 94, 839, + 1806, 796, 1187, 879, 1188, 1190, 797, 1208, 2712, 2262, + 97, 98, 99, 119, 1283, 2515, 1964, 1965, 1875, 798, + 850, 851, 888, 102, 103, 1236, 852, 800, 801, 3427, + 802, 2859, 1359, 545, 537, 538, 1608, 733, 1338, 734 }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -#define YYPACT_NINF -3281 +#define YYPACT_NINF -3295 static const int yypact[] = { - 6743, -13, 747, -3281, -3281, 245, -13, 53022, 69774, 225, - -13, 213, 3740, 55552, -3281, -3281, 49968, 7871, -13, 59594, - 76341, 339, 382, 34049, 687, 60103, -3281, -3281, -3281, 69774, - 59594, 60612, -13, 351, 70283, -3281, -13, 37105, 56061, 64, - -3281, 59594, 77, 281, 61121, 59594, 2317, 830, 345, -3281, - -3281, -3281, -3281, -3281, 263, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, 176, -3281, 868, 181, 34049, 34049, 133, 332, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - 427, -3281, -3281, 712, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, 36595, -3281, -3281, -3281, -3281, -3281, -3281, 61630, - 59594, 62139, 56570, 62648, -3281, 678, 574, 1150, 733, 190, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, 212, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, 539, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, 218, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - 655, -3281, 633, -3281, 221, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, 1252, -3281, -3281, 1087, 3035, 59594, - 409, 697, 843, -3281, 63157, -3281, 823, -3281, -3281, 833, - 1645, 1023, -3281, -3281, 57079, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, 50477, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, 981, -3281, -3281, 806, -3281, 324, -3281, - -3281, 834, 793, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, 902, -3281, -3281, -3281, 907, 70792, 63666, 64175, - -3281, 788, 2504, 8637, 76359, 33029, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - 427, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - 60103, 69774, 803, 814, 1177, 824, 35067, 831, 37615, 836, - 846, 1216, 887, 891, 912, 928, 281, 33539, 879, 655, - 1416, 64684, 64684, -18, 34558, 1165, -3281, 64684, 65193, -3281, - 969, -3281, 1150, -3281, -3281, -3281, -3281, 450, 935, -3281, - 65702, 65702, 65702, 986, 1287, 65702, -3281, -3281, -3281, 1016, - -3281, -3281, 1256, 20995, 20995, 71301, 71301, 1150, 71301, 1044, - 71301, -3281, -3281, 84, 733, -3281, 655, -3281, -3281, 133, - -3281, -3281, 56061, -3281, -3281, 300, 1387, 20995, 59594, 1030, - -3281, 1042, 1030, 1064, 1073, 1077, -3281, 6743, 1414, 1304, - 1402, 57588, 361, 361, 1560, 361, 708, 970, 2265, 3452, - -3281, 2119, -3281, 1108, -3281, 59594, 60103, 1206, 1466, 1132, - 1447, -3281, -3281, 1528, 911, 1327, 1548, 7918, 1555, 1046, - 1567, 1111, 1569, 1605, 1688, 49, -3281, 20995, 50986, 655, - -3281, 12471, 20995, -3281, -3281, -3281, 1298, -3281, -3281, -3281, - -3281, -3281, 59594, 69774, 1202, 1217, -3281, -3281, -3281, -3281, - 2135, 1470, -3281, 71810, -3281, -3281, 1284, 66211, 66720, 67229, - 67738, 68247, 1695, -3281, -3281, 1624, -3281, -3281, -3281, 1294, - -3281, -3281, -3281, 241, 72319, 1639, 1272, 132, -3281, 1651, - 137, -3281, 1662, 1517, 15121, -3281, 1459, -3281, -3281, -3281, - 281, -3281, 447, -3281, -3281, 46902, -3281, -3281, 76359, 1383, - 1321, -3281, 20995, 20995, 1325, 7362, 64684, 65193, 20995, 59594, - -3281, 20995, 26335, 1329, 20995, 20995, 13005, 20995, 31019, 64684, - 1165, 1330, -3281, 642, -3281, 59594, 1337, -3281, 1441, 1441, - 351, 34049, 1657, 33539, 1441, 1826, 1441, -3281, 274, 1659, - 1579, -3281, 34049, 1579, 1012, 1365, 1674, 1579, -3281, 699, - 1676, 1826, 38124, 1370, -3281, 1441, 1612, -3281, -3281, 20995, - 15121, 58097, 1877, -3281, -3281, -3281, -3281, 1680, -3281, 69774, - 1395, -3281, -3281, -3281, -3281, -3281, -3281, 789, 1916, 163, - 1923, 20995, 163, 163, 1407, 222, 222, -3281, 1606, 1412, - -3281, 226, 1413, 1418, 1939, 1943, 195, 59594, 165, 1075, - 163, 20995, -3281, 222, 1421, 1950, 1429, 1956, 162, 192, - -3281, 1434, 227, 20995, 20995, 20995, 731, 20995, 11403, -3281, - 50986, 1955, 59594, 257, -3281, 655, 1437, 1150, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, 1438, -3281, 200, 7393, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, 1478, -3281, -3281, - -3281, -3281, 1664, 20995, -3281, -3281, 1442, 1657, -3281, 228, - -3281, -3281, 1657, -3281, -3281, -3281, -3281, -3281, 244, -3281, - 1868, 20995, 20995, -3281, 655, 72828, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, 689, -3281, 427, 669, 48608, 1465, 1469, - 1030, 59594, 59594, 1534, -3281, -3281, -3281, -3281, 56061, 321, - 1780, 56061, 191, 1618, -3281, -3281, 133, 133, 16189, 931, - 236, 112, 16723, 21529, 1845, 1725, 249, 644, 1849, -3281, - 1730, 1959, 26335, 20995, 20995, 708, 970, 20995, 1042, 119, - -3281, -3281, 59594, -3281, 1783, 59594, 53531, 839, 989, 1498, - 1589, 57, 625, 1936, -3281, 1504, -3281, 1597, 59594, 75358, - 250, -3281, 1979, 250, 250, 314, 1980, 1608, 283, 1774, - 915, 317, 1504, 3660, -3281, 56061, 168, 1072, 1504, 59594, - 1611, 1644, 1504, 1605, 1150, 69774, 1526, -3281, -3281, 44347, - 2036, -3281, -3281, -3281, 201, 15121, -3281, 1290, 1297, 1312, - 1323, -3281, 792, 224, 1355, 1364, 15121, 1371, 1382, 205, - 1393, 1436, 1615, 1634, 1648, 1684, 1724, 1729, 185, 1735, - 1740, 1743, 1746, 1753, 1755, -3281, 1787, 211, 1795, 232, - 1434, 15121, 1797, -3281, 194, 48608, 33, -3281, -3281, 1805, - 215, -3281, 48859, -3281, 1835, 1625, 1628, 69774, 1581, 59594, - 1687, 1776, 1924, 1978, 1800, -3281, 1882, 59594, 1804, 3660, - 1806, 1559, 2047, 1810, 2050, 1813, 1217, 1814, 1572, -3281, - 73337, 50986, -3281, -3281, -3281, -3281, -3281, 1949, 1932, 69774, - 50986, 1577, -3281, -3281, 69774, -3281, 59594, 59594, -3281, 59594, - 69774, -3281, 700, 48608, 2084, 845, 76359, 52004, -3281, -3281, - -3281, -3281, 1059, 1126, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, 1150, 50986, -3281, 3856, 47479, 1590, 20995, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, 1591, 1952, -3281, -3281, 6178, 1594, 47556, 1595, 26335, - 26335, 655, 1260, -3281, -3281, 26335, 1600, 52513, 47426, 1598, - 1602, 47813, 17257, 20995, 17257, 17257, 47950, -3281, 1603, 48002, - 64684, 1585, 59594, 30507, -3281, -3281, -3281, 20995, 20995, 1165, - 58591, 1647, 1607, -3281, 1609, 1441, -3281, 34049, -3281, 34049, - -3281, 1908, 34049, -3281, -3281, 2953, -3281, 34049, 1917, 20995, - 34049, -3281, 34049, 1859, 1860, 1627, 34049, 1441, 59594, 1629, - 59594, -3281, -3281, 48608, -3281, 1620, 705, 1626, -3281, -3281, - -3281, -3281, -3281, -3281, 1682, -3281, -3281, 1682, 1682, -3281, - -3281, -3281, -3281, 1632, 1632, 1637, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, 1640, - 1075, -3281, 1682, -3281, 1632, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, 75358, -3281, -3281, -3281, -3281, 531, 631, -3281, - 1641, -3281, -3281, -3281, 1642, -3281, 1649, 2131, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, 9067, 711, 1632, - -3281, -3281, 6089, -3281, -3281, 20995, 20995, -3281, -3281, 1646, - 48608, 1692, -3281, -3281, 20995, 20995, -3281, -3281, -3281, -3281, - 2173, -3281, 216, 20995, 1682, 1682, -3281, 6838, -3281, 43724, - 17791, 1754, 1756, 2173, -3281, 2173, -3281, 20995, 6838, 2180, - 2180, 1665, 38633, -3281, 1834, 48083, -3281, 1672, 2233, 6433, - 1673, -3281, -3281, 2185, -3281, 1678, 1669, 20995, 45375, 208, - 655, 655, 20995, -3281, 2173, 20995, 9886, 9886, -3281, 247, - 58097, 20995, 20995, 20995, 20995, 20995, 20995, 20995, 20995, 49459, - 1770, 204, 69774, 20995, 20995, 29993, 1038, -3281, 20995, 1927, - -3281, 1685, 20995, 1775, 391, 20995, 20995, 20995, 20995, 20995, - 20995, 20995, 20995, 20995, -3281, -3281, 29479, 337, 672, 2031, - 2051, 14, 333, 20995, 2043, 12471, -3281, 2043, -3281, -3281, - -3281, -3281, -3281, 231, -3281, -3281, 1620, 1620, 69774, -3281, - 59594, 300, 55043, 20995, -3281, -3281, 1690, 1701, 1995, 2190, - 1766, -3281, -3281, 59594, 1778, -3281, 42196, 2017, -3281, 354, - 1715, -3281, 47408, 1982, 2017, 133, -3281, -3281, 26869, 1851, - 2025, 1961, -3281, -3281, 1940, 1941, -3281, 1723, 48937, 22063, - 22063, -3281, 1398, 48608, 1451, -3281, -3281, -3281, -3281, -3281, - -3281, 91, -3281, 59594, 113, 39142, -3281, 1731, 259, -3281, - 2530, 2082, 2045, 1845, 644, 1742, -3281, 60103, 60103, -3281, - -3281, -3281, 1411, 1741, 73846, 59594, 2044, 1992, 2048, -45, - 58097, -3281, 1749, -3281, -3281, -3281, 59594, 69774, 68756, 74355, - 51495, 59594, 2224, 2225, 50986, -3281, -3281, 2227, 2235, -3281, - -3281, 59594, 1028, 59594, 8689, -3281, -3281, -3281, -3281, 250, - -3281, -3281, -3281, -3281, -3281, 69774, 59594, -3281, -3281, 250, - 69774, 59594, 250, -3281, 1563, 59594, 59594, 69774, 59594, 1759, - 59594, 59594, 1150, 1688, -3281, 50986, -3281, -3281, 22597, 45, - 45, 1996, 2006, 2009, 1769, 13539, 194, -3281, 20995, 20995, - 189, 308, 69774, 1964, -3281, -3281, 716, 2012, 141, -3281, - 69774, 1819, 59594, 59594, 59594, 59594, 59594, 59594, 1610, -3281, - -3281, -3281, -3281, -3281, 2155, 2307, 1784, 1785, 2157, -3281, - 3660, 2160, 54040, 840, 2978, 2163, 59100, 2164, 1833, 2170, - 32037, -3281, -3281, 1793, -3281, -3281, 1796, 2286, 2053, -3281, - -3281, 2039, -3281, 69774, 2332, -3281, 132, -3281, 50986, -3281, - 137, -3281, 2052, 269, -3281, 15121, 20995, -3281, -3281, -3281, - -3281, -3281, -3281, 1321, 20995, -3281, 727, -3281, -3281, 2304, - 1150, 2304, 544, -3281, -3281, 2304, -3281, 2288, 2304, -3281, - 58097, -3281, 7991, -3281, 20995, 20995, -3281, 20995, 2172, -3281, - 2338, 2338, 58097, 26335, 26335, 26335, 26335, 26335, 26335, 754, - 1421, 26335, 26335, 26335, 26335, 26335, 26335, 26335, 26335, 26335, - 27403, 392, -3281, -3281, 740, 2309, 20995, 20995, 2182, 2172, - 20995, -3281, 58097, 1823, -3281, 1827, 1828, 20995, -3281, 58097, - -3281, 59594, 1832, -3281, -3281, -3281, 542, 1836, 1838, -3281, - -3281, 1657, -3281, 919, 1045, 59594, 2189, 4323, 5373, -3281, - -3281, 20995, 2179, -3281, 387, -3281, 2953, 2953, 34049, -3281, - 20995, 1840, -3281, -3281, 34049, 2200, -3281, 2953, -3281, -3281, - 39651, 2953, -3281, 58097, 755, -3281, 59594, 58097, 818, 20995, - -3281, 15121, 2367, 58097, 2331, 69774, 69774, 2371, 1857, 1862, - 2173, 1951, -3281, 1953, 1958, 1960, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, 58097, -3281, -3281, 219, - -3281, -3281, -3281, -3281, -3281, -3281, 1861, 1872, 20995, 20995, - 125, -3281, 8079, 1866, 1875, 20995, 48337, -3281, 1869, -3281, - 1852, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, 1878, -3281, - 1876, -3281, 1881, 1902, 1903, 1887, 1889, 8132, 20995, 59594, - -3281, 1891, 23131, 2066, 59594, -3281, -3281, 20995, 20995, 59594, - -3281, 48608, 2257, -3281, 1897, 1905, 8386, -3281, -3281, -3281, - 223, 405, 48956, 333, 2365, 2365, 2365, 6838, -3281, -3281, - -3281, 1920, -3281, 26335, 26335, -3281, 3058, 1275, 11403, -3281, - -3281, -3281, -3281, 2261, -3281, 1179, -3281, 1909, -3281, -3281, - 3776, -3281, 43724, 3840, 20995, 217, -3281, 20995, 29993, 20995, - 1998, 2365, 2365, 2365, 313, 313, 223, 223, 223, 405, - 333, -3281, -3281, -3281, 1911, 20995, 50986, -3281, 1913, 1914, - 2279, 1429, 20995, -3281, -3281, 34049, 1647, 33, 1647, 2173, - 9886, -3281, 1042, -3281, 1042, -3281, 48608, 59594, -3281, 1915, - 683, 34049, 1962, 2406, 2390, 34049, 69774, -3281, -3281, 1919, - 2043, 1935, -3281, -3281, 1944, 20995, 3099, 1944, -3281, 2017, - 6, 2162, 1185, 1185, 1398, 2165, -3281, -3281, 1999, -3281, - -3281, -3281, 20995, 15655, 1455, -3281, 1463, -3281, -3281, -3281, - -3281, -3281, 1922, -3281, 2209, -3281, 59594, -3281, -3281, 26335, - 2399, 20995, 40160, 2402, 2193, -3281, -3281, -3281, 1986, 1986, - -3281, -3281, 2027, 1504, 20995, 2188, -3281, 182, 1942, 2318, - -1, 2266, 69774, -3281, -3281, -3281, 363, 384, 50986, 1749, - -3281, -3281, 237, 2319, 269, 2321, 269, 50986, 50986, 50986, - 854, -3281, -3281, -3281, 1150, -3281, -3281, -3281, -84, 884, - -3281, 1957, 1965, -3281, -3281, -3281, 2037, 1761, 1504, 3660, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, 202, 1801, 1504, - 2040, -3281, 2046, -3281, 2054, 1895, 1504, -3281, -3281, 1688, - 1526, 900, 18325, 48608, 253, 194, 194, 194, -3281, -3281, - -3281, 15121, -3281, 1966, 48608, 48608, 171, -3281, -3281, -3281, - -3281, 1968, -3281, 175, -3281, 69774, -3281, -3281, -3281, 1964, - 1978, 1882, 59594, 3660, 1969, 2438, 2446, 1217, 1572, -3281, - 2127, 761, 69774, -3281, 50986, 69774, 59594, 59594, 59594, 54549, - -3281, -3281, -3281, 1971, 1970, -3281, 28, 2208, 2210, 59594, - 2014, 59594, 1983, -3281, -3281, 59594, 1984, 2465, 59594, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, 904, -3281, 58097, -3281, 1572, 1572, 18859, - 2353, 59594, 1932, -3281, -3281, -3281, -3281, 69774, -3281, -3281, - 48608, -3281, 48608, -3281, -3281, 50986, -3281, 1150, -3281, 1150, - 2228, 69774, 45884, 1150, 46393, 1150, 1981, -3281, 48608, 8721, - 48608, 2182, -3281, 255, 2338, 1719, 1719, 1719, 5868, 2341, - 285, 1988, 1719, 1719, 1719, 290, 290, 255, 255, 255, - 2338, 392, 969, 52513, 1989, -3281, 48608, 48608, -3281, -3281, - 1991, -3281, -3281, -3281, -3281, 1993, 1994, -3281, -3281, -3281, - -3281, 69774, 198, 1647, 64, 64, 64, 64, -3281, 59594, - 59594, 59594, 48608, 2464, 2339, -3281, -3281, 2512, 2000, -3281, - -3281, 2953, 48608, 59594, -3281, 28451, -3281, 59594, -3281, 2359, - -3281, 2449, -3281, 59594, 905, -3281, -3281, -3281, 913, 2011, - 1862, 58097, 943, 959, -3281, 2173, 184, 2003, 1583, 949, - 1033, 1471, -3281, 56061, -3281, -3281, 2013, 48414, 20995, -3281, - 2391, -3281, -3281, -3281, 48608, 20995, 20995, -3281, 43724, -3281, - -3281, -3281, -3281, 458, 458, -3281, 8835, 1891, 2008, 2018, - 59594, 11403, 48528, -3281, 40669, -3281, 124, 9286, 48608, -3281, - 1834, -3281, -3281, 9886, 20995, 3088, 3316, 20995, 2019, 20995, - 2369, -3281, -3281, 2021, -3281, -3281, 58097, 20995, 2024, 4036, - 26335, 26335, 4867, -3281, 5063, 20995, 11403, -3281, 44389, 2535, - 2029, 1996, 19393, -3281, 2249, 2030, -3281, 2179, 194, 2179, - 2034, -3281, -3281, -3281, 69774, -3281, 2305, 2049, -3281, 20995, - 2197, 69774, 160, 1864, 2953, 978, -3281, 655, 42196, 1962, - 20995, 629, -3281, -3281, 2055, -3281, 1944, -3281, -3281, -3281, - 2269, -3281, -3281, -3281, 59594, -3281, 2042, -3281, 39142, 2385, - 11937, -3281, 39142, 59594, -3281, -3281, 59594, 9487, 2418, -3281, - 69774, 69774, 69774, -3281, 69774, 2041, 2059, 872, 2056, 436, - -3281, 1671, -3281, -3281, 872, 2398, 299, 1984, 283, 2975, - 485, -3281, -3281, -3281, 2128, 59594, -3281, 69774, -3281, -3281, - -3281, -3281, -3281, 51495, -3281, -3281, 43214, 50986, -3281, 50986, - 20995, 20995, 59594, 59594, 59594, 59594, 69774, 59594, 59594, 59594, - 59594, 59594, 1526, -3281, -3281, 20995, 20995, -3281, 2060, 2062, - 2063, 1996, -3281, 425, -3281, 2068, -3281, -3281, 20995, -3281, - 317, -3281, -3281, 175, 2071, 2072, -3281, 54040, 3035, 59100, - 1833, -3281, 1796, 1978, 939, 69265, 1882, 20995, -3281, 979, - 1004, 3660, 2075, 2539, -3281, -3281, -3281, 840, 54040, -3281, - -3281, -3281, 2495, -3281, 788, 251, -3281, 2555, 1584, -3281, - 1217, -3281, 3035, 1572, -3281, 32531, 1850, -3281, -3281, 2563, - -3281, 2566, 3035, 48608, 69774, 2141, -3281, 269, -3281, -3281, - -3281, 69774, 2077, -3281, 2077, -3281, -3281, 2077, -3281, -3281, - -3281, -3281, 26335, 2436, 2085, 58097, -3281, -3281, 59594, -3281, - -3281, -3281, 1009, 2083, 2179, 59594, 59594, 59594, 59594, -3281, - -3281, -3281, 19927, 20995, 2125, 20995, -3281, -3281, 2086, 14073, - 2416, -3281, 27937, -3281, -3281, 2091, 39651, 69774, -3281, -3281, - -3281, -3281, 2173, -3281, -3281, 69774, -3281, 2094, -3281, 2095, - -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, 20995, - 48608, -3281, 48608, -3281, -3281, -3281, -3281, -3281, 59594, -3281, - -3281, 7562, -3281, 2092, 2097, 69774, 59594, 143, -3281, 20995, - -3281, 2066, -3281, 419, 20995, 20995, 3058, -3281, 6858, 20995, - 58097, 1015, 3058, 370, 20995, 5494, 5756, 20995, 20995, 5583, - 9834, -3281, 23665, 14587, -3281, 2098, 20995, 9887, 42705, -3281, - 34049, 2339, 2100, 2339, 1150, -3281, 2101, 2099, -3281, -3281, - 6089, 48608, 20995, -3281, -3281, -3281, -3281, 2150, 401, 36085, - 2337, -3281, 2156, 2132, 69774, -3281, 2197, 48608, -3281, -3281, - 43724, -3281, -3281, -3281, -3281, -3281, 2584, 1658, 2122, 2124, - -3281, 1394, -3281, -3281, 69774, 2126, -3281, 2129, 872, -3281, - 69774, 2167, -3281, 298, 2454, 196, -3281, 20995, -3281, 2547, - 2626, 1671, 2142, 69774, 59594, 26335, -3281, 307, 243, -3281, - 2442, 59594, 2167, 2583, -3281, -3281, -3281, 436, -3281, 2479, - 2389, -3281, 250, -3281, 20995, 436, 2392, 265, 69774, -3281, - -3281, 3783, -3281, 58097, 269, 269, -3281, 1626, 2151, 2152, - 2153, 2166, 2168, 2169, 2171, 2174, 2177, 2178, 2181, 2184, - 2187, 2191, 2192, 2194, 2196, 2198, 2199, 2203, 1640, 2204, - -3281, 2205, 2055, 2206, 2211, 2212, 2216, 2217, 74864, 2219, - 2220, 2221, 2222, 1641, 2223, 2226, 1059, 1126, -3281, -3281, - -3281, -3281, -3281, -3281, 1272, 2230, -3281, 2176, -3281, 2159, - 1027, -3281, -3281, 2240, -3281, 2245, -3281, -3281, -3281, -3281, - -3281, -3281, 2175, 2183, -3281, -3281, -3281, 194, 2229, 2232, - 69774, 10025, 1321, 158, 50986, 69774, 2238, 2014, -3281, 2645, - 764, 2407, 2236, 2243, -3281, 48608, -3281, 50986, 1833, -3281, - 54040, 3023, 720, 2210, 58097, -3281, 289, 2014, -3281, 2586, - 59100, -3281, 2239, 2244, 1833, 2207, -3281, 1796, -3281, -3281, - 20995, 20995, 2268, 20995, 164, -3281, 2405, 69774, 2247, -3281, - 2077, 5900, 26335, 58097, 1039, 1062, -3281, 2703, 2378, 2339, - -3281, -3281, -3281, -3281, -3281, 2252, -20, 2254, 10869, 2253, - -3281, -3281, -3281, -3281, -3281, -3281, 48608, 48608, 69774, 2413, - 48608, -3281, -3281, 2255, 2262, 41178, 2697, 2264, -3281, -3281, - 2585, -3281, 31528, -3281, 1862, 2267, 1862, 58097, 1862, -3281, - -3281, 48608, 1891, 20995, -3281, -3281, -3281, 2263, 2270, 69774, - 44659, 2590, -3281, 3058, 3058, 6858, 1085, -3281, 3058, 20995, - 20995, 3058, 3058, 20995, -3281, 20461, 261, -3281, 1089, -3281, - 10079, -3281, 75852, -3281, -3281, 2125, 1150, 2125, -3281, -3281, - 69774, 2271, 2274, -3281, -3281, -3281, 2324, -3281, -3281, 1093, - 2705, 2197, 170, -3281, -3281, 2156, 2197, 20995, -3281, -3281, - 2280, 39142, -3281, -3281, -3281, -3281, 39142, 872, -3281, 2445, - 2167, 2275, -3281, -3281, -3281, -3281, -3281, -3281, 44241, -3281, - 96, 20995, -3281, 790, 5868, -3281, -3281, -3281, -3281, 2167, - 1217, -3281, 59594, 2769, 2657, -3281, -3281, 48608, -3281, -3281, - 2173, 2173, -3281, -3281, 2449, -3281, -3281, 2283, 1272, -87, - 43214, -3281, -3281, 59594, 59594, -3281, -3281, 2285, -3281, -3281, - -3281, -3281, -3281, -3281, 317, 2690, 1106, 1112, 840, -3281, - 3035, 59594, 2662, 54040, 50986, -3281, 2776, 2289, 59594, 2014, - 1990, 1990, -3281, 2444, -3281, 2448, -3281, -3281, -3281, -3281, - 1150, 2783, 340, -3281, 48608, 48608, 1412, 59594, -3281, -3281, - 35576, 5900, 1155, -3281, -3281, 2296, 2298, -3281, 2125, 20995, - 2301, 20995, -3281, 24199, 2789, 2300, -3281, 20995, 2366, 28965, - -3281, 20995, -3281, 59594, 64684, 2308, 64684, -3281, -3281, -3281, - -3281, 59594, -3281, -3281, -3281, 20995, -3281, 3058, 3058, 3058, - 20995, 20995, -3281, -3281, -3281, -3281, 2520, 2413, -3281, 2413, - -3281, -3281, 20995, 3035, 655, 3267, 69774, 36, -3281, 2800, - 2588, -3281, -3281, 48608, -3281, -3281, -3281, 59594, -3281, 50986, - -3281, 872, 501, 2310, 20995, 44259, 2554, -3281, -3281, 2595, - -3281, 2668, -3281, 2383, 626, 2411, -3281, -3281, -3281, -3281, - 1321, 1150, -3281, 1833, 2210, 2207, 2335, 59594, 1175, 3035, - 840, 788, -3281, -3281, -3281, -3281, -3281, -3281, -3281, -3281, - -3281, -3281, -3281, -3281, -3281, -3281, 3035, 2790, 2568, 2792, - -3281, 2141, 20995, 92, -3281, 1184, 2791, -3281, -3281, 2855, - 2413, 2342, 24199, 2344, -3281, 2346, 69774, 48608, 2493, -3281, - -3281, 2347, -3281, -3281, 20995, -3281, -3281, 44870, 2352, 2370, - 2832, 1996, 2366, 2366, -3281, 401, -3281, -3281, 2799, 35576, - 2760, 20995, 2463, 2840, 1217, 872, 2384, 1186, -3281, -3281, - -3281, -3281, -3281, 3660, -3281, 44294, 2628, 148, 2609, 2310, - 20995, -3281, 2456, -3281, -3281, -3281, 2867, -3281, -3281, 54040, - 2381, -3281, 2207, 2210, 2014, 2207, 2612, -3281, 2613, 2386, - 44312, 69774, 69774, 1833, 35576, 69774, 2387, 2366, -3281, 2393, - -3281, -3281, -3281, 30507, -3281, 2395, -3281, -3281, -3281, 20995, - 186, -3281, -3281, 2440, 59594, 1188, 74, 48608, 359, 659, - 2800, 2595, 43214, -3281, 50986, 1954, 501, 2706, -3281, -3281, - -3281, -3281, 328, 2620, -3281, 2624, -3281, 48608, -3281, 3035, - 54040, -3281, -3281, -3281, -3281, -3281, -3281, 35576, 2791, -3281, - 354, -3281, 1647, -3281, 354, -3281, -3281, -3281, -3281, -3281, - 1598, 24733, 24733, 24733, 2396, 3035, -3281, 1647, -3281, 2525, - -3281, 2640, 20995, 161, 254, -3281, -3281, -3281, 2490, 2609, - -3281, -3281, -3281, -3281, -3281, 737, 737, 2804, -3281, 2466, - -3281, 2207, 1190, 69774, 1944, -3281, 1944, 25801, 2557, 210, - 47461, 2784, -3281, 2784, 2784, -3281, -3281, -3281, 42196, -3281, - -3281, 48608, 2469, 69774, 2422, 2472, 41687, -3281, 359, -3281, - -3281, 2911, -3281, 246, -3281, -3281, -3281, 1833, 354, -3281, - -3281, 2909, -3281, -3281, -3281, -3281, -3281, 183, -3281, -3281, - -3281, 1647, -3281, 1204, -3281, -3281, 2425, -3281, -3281, -3281, - 872, -3281, -3281, -3281, 1647, 1944, 25267, 2576, -3281, 2653, - -3281, -3281, -3281, 20995, -3281, -3281, -3281, -3281, -3281, 2430, - -3281 + 7837, 387, 787, -3295, -3295, 708, 387, 53010, 69762, 148, + 387, 124, 3023, 55540, -3295, -3295, 49956, 3804, 387, 59582, + 76838, 461, 467, 34057, 552, 60091, -3295, -3295, -3295, 69762, + 59582, 60600, 387, 353, 70271, -3295, 387, 37113, 56049, 525, + -3295, 59582, 92, 297, 61109, 59582, 4028, 888, 445, -3295, + -3295, -3295, -3295, -3295, 323, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, 185, -3295, 134, 214, 34057, 34057, 955, + 449, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, 546, -3295, -3295, 838, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, 36603, -3295, -3295, -3295, -3295, -3295, -3295, + 70780, 61618, 59582, 62127, 56558, 62636, -3295, 827, 707, 1115, + 841, 181, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, 201, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, 634, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, 225, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, 565, -3295, 665, -3295, 226, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, 145, -3295, -3295, 1091, + 2223, 59582, 411, 519, 885, -3295, 63145, -3295, 880, -3295, + -3295, 898, 963, 1049, -3295, -3295, 57067, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, 50465, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, 1013, -3295, -3295, 829, -3295, + 270, -3295, -3295, 866, 819, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, 922, -3295, -3295, -3295, 925, 71289, + 63654, 64163, -3295, 812, 711, 987, 6452, 76856, 33037, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, 546, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, 60091, 69762, 824, 836, 1189, 840, 35075, + 844, 37623, 857, 863, 1241, 899, 914, 927, 941, 297, + 33547, 950, 565, 1472, 64672, 64672, -36, 34566, 2495, -3295, + 64672, 65181, -3295, 980, -3295, 1115, -3295, -3295, -3295, -3295, + 395, 968, -3295, 65690, 65690, 65690, 1000, 1287, 65690, -3295, + -3295, -3295, 1007, -3295, -3295, 1236, 21003, 21003, 71798, 71798, + 1115, 71798, 1031, 71798, -3295, -3295, 374, 841, -3295, 565, + -3295, -3295, 955, -3295, -3295, 56049, -3295, -3295, 292, 1375, + 21003, 59582, 1014, -3295, 1035, 1014, 1041, 1061, 1066, -3295, + 7837, 1404, 1299, 1381, 57576, 370, 370, 1537, 370, 786, + 923, 4316, 2204, -3295, 1297, -3295, 1098, -3295, 59582, 60091, + 1200, 1460, 1129, 1417, -3295, -3295, 1491, 1331, 1502, 773, + 1346, 1560, 6505, 1565, 1090, 1567, 1123, 1598, 1552, 1713, + 41, -3295, 21003, 50974, 565, -3295, 12479, 21003, -3295, -3295, + -3295, 1333, -3295, -3295, -3295, -3295, -3295, 59582, 69762, 1246, + 1237, -3295, -3295, -3295, -3295, 1709, 1497, -3295, 72307, -3295, + -3295, 1309, 66199, 66708, 67217, 67726, 68235, 1702, -3295, -3295, + 1650, -3295, -3295, -3295, 1301, -3295, -3295, -3295, 199, 72816, + 1653, 1306, 123, -3295, 1684, 245, -3295, 1686, 1536, 15129, + -3295, 1489, -3295, -3295, -3295, 297, -3295, 626, -3295, -3295, + 46910, -3295, -3295, 76856, 1419, 1350, -3295, 21003, 21003, 1356, + 5835, 64672, 65181, 21003, 59582, -3295, 21003, 26343, 1357, 21003, + 21003, 13013, 21003, 31027, 64672, 2495, 1358, -3295, 710, -3295, + 59582, 1363, -3295, 1464, 1464, 353, 34057, 1675, 33547, 1464, + 1848, 1464, -3295, 919, 1672, 1595, -3295, 34057, 1595, 1138, + 1376, 1678, 1595, -3295, 281, 1681, 1848, 38132, 1379, -3295, + 1464, 1610, -3295, -3295, 21003, 15129, 58085, 1883, -3295, -3295, + -3295, -3295, 1691, -3295, 69762, 1403, -3295, -3295, -3295, -3295, + -3295, -3295, 772, 1920, 174, 1927, 21003, 174, 174, 1407, + 228, 228, -3295, 1607, 1414, -3295, 232, 1418, 1423, 1947, + 1949, 189, 59582, 167, 1008, 174, 21003, -3295, 228, 1427, + 1953, 1431, 1960, 204, 216, -3295, 1438, 234, 21003, 21003, + 21003, 312, 21003, 11411, -3295, 50974, 1961, 59582, 263, -3295, + 565, 1441, 1115, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + 1447, -3295, 227, 7043, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, 1483, -3295, -3295, -3295, -3295, 1671, 21003, -3295, + -3295, 1448, 1675, -3295, 237, -3295, -3295, 1675, -3295, -3295, + -3295, -3295, -3295, 255, -3295, 1876, 21003, 21003, -3295, 565, + 73325, -3295, -3295, -3295, -3295, -3295, -3295, -3295, 675, -3295, + 546, 655, 48616, 1450, 1454, 1014, 59582, 59582, 1525, -3295, + -3295, -3295, -3295, 56049, 200, 1781, 56049, 182, 1611, -3295, + -3295, 955, 955, 16197, 1843, 309, 83, 16731, 21537, 1838, + 1719, 236, 913, 1841, -3295, 1723, 1955, 26343, 21003, 21003, + 786, 923, 21003, 1035, 104, -3295, -3295, 59582, -3295, 1779, + 69762, 1580, 59582, 53519, 855, 886, 1494, 1583, 54, 897, + 1930, -3295, 1493, -3295, 1586, 59582, 75855, 252, -3295, 1968, + 252, 252, 264, 1970, 1592, 290, 1765, 1302, -85, 1493, + 2981, -3295, 56049, 161, 1477, 1493, 59582, 1597, 1503, 1493, + 1552, 1115, 69762, 1514, -3295, -3295, 9937, 2016, -3295, -3295, + -3295, 194, 15129, -3295, 1128, 1203, 1492, 1507, -3295, 426, + 209, 1569, 1596, 15129, 1608, 1641, 198, 1646, 1673, 1680, + 1697, 1718, 1724, 1726, 1728, 179, 1730, 1732, 1743, 1750, + 1770, 1772, -3295, 1774, 205, 1786, 239, 1438, 15129, 1788, + -3295, 150, 48616, 32, -3295, -3295, 1801, 221, -3295, 48867, + -3295, 1815, 1605, 1621, 69762, 1557, 59582, 1662, 1202, 1897, + 1966, 1782, -3295, 1867, 59582, 1790, 2981, 1797, 1551, 2031, + 1800, 2041, 1804, 1237, 1805, 1559, -3295, 73834, 50974, -3295, + -3295, -3295, -3295, -3295, 1936, 1918, 69762, 50974, 1561, -3295, + -3295, 69762, -3295, 59582, 59582, -3295, 59582, 69762, -3295, 694, + 48616, 2081, 131, 76856, 51992, -3295, -3295, -3295, -3295, 444, + 562, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + 1115, 50974, -3295, 3327, 47487, 1571, 21003, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, 1572, 1922, + -3295, -3295, 6933, 1577, 47564, 1588, 26343, 26343, 565, 1833, + -3295, -3295, 26343, 1591, 52501, 47434, 1566, 1593, 47821, 17265, + 21003, 17265, 17265, 47958, -3295, 1599, 48010, 64672, 1581, 59582, + 30515, -3295, -3295, -3295, 21003, 21003, 2495, 58579, 1628, 1603, + -3295, 1606, 1464, -3295, 34057, -3295, 34057, -3295, 1899, 34057, + -3295, -3295, 4506, -3295, 34057, 1913, 21003, 34057, -3295, 34057, + 1846, 1859, 1619, 34057, 1464, 59582, 1620, 59582, -3295, -3295, + 48616, -3295, 1622, 727, 1623, -3295, -3295, -3295, -3295, -3295, + -3295, 1679, -3295, -3295, 1679, 1679, -3295, -3295, -3295, -3295, + 1624, 1624, 1634, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, 1639, 1008, -3295, 1679, + -3295, 1624, -3295, -3295, -3295, -3295, -3295, -3295, -3295, 75855, + -3295, -3295, -3295, -3295, 350, 645, -3295, 1640, -3295, -3295, + -3295, 1642, -3295, 1643, 2132, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, 8165, 739, 1624, -3295, -3295, 1975, + -3295, -3295, 21003, 21003, -3295, -3295, 1648, 48616, 1688, -3295, + -3295, 21003, 21003, -3295, -3295, -3295, -3295, 2168, -3295, 196, + 21003, 1679, 1679, -3295, 2991, -3295, 43732, 17799, 1744, 1751, + 2168, -3295, 2168, -3295, 21003, 2991, 2170, 2170, 1660, 38641, + -3295, 1831, 48091, -3295, 1666, 2242, 7772, 1663, -3295, -3295, + 2175, -3295, 1664, 1665, 21003, 45383, 195, 565, 565, 21003, + -3295, 2168, 21003, 6176, 6176, -3295, 210, 58085, 21003, 21003, + 21003, 21003, 21003, 21003, 21003, 21003, 49447, 1761, 223, 69762, + 21003, 21003, 30001, 1677, -3295, 21003, 1916, -3295, 1685, 21003, + 1763, 1079, 21003, 21003, 21003, 21003, 21003, 21003, 21003, 21003, + 21003, -3295, -3295, 29487, 265, 669, 2023, 2049, -31, 421, + 21003, 2042, 12479, -3295, 2042, -3295, -3295, -3295, -3295, -3295, + 238, -3295, -3295, 1622, 1622, 69762, -3295, 59582, 292, 55031, + 21003, -3295, -3295, 1683, 1690, 1988, 2179, 1754, -3295, -3295, + 59582, 1755, -3295, 42204, 2003, -3295, 351, 1692, -3295, 47416, + 1954, 2003, 955, -3295, -3295, 26877, 1828, 1998, 1934, -3295, + -3295, 1914, 1921, -3295, 1700, 48945, 22071, 22071, -3295, 1373, + 48616, 1397, -3295, -3295, -3295, -3295, -3295, -3295, 632, -3295, + 59582, 119, 39150, -3295, 1711, 133, -3295, 2038, 2058, 2024, + 1838, 913, 1716, -3295, 60091, 60091, -3295, -3295, -3295, 1889, + 69762, 1734, 1722, 74343, 59582, 2026, 1976, 2028, -43, 58085, + -3295, 1729, -3295, -3295, -3295, 59582, 69762, 68744, 74852, 51483, + 59582, 2212, 2216, 50974, -3295, -3295, 2224, 2228, -3295, -3295, + 59582, 495, 59582, 6962, -3295, -3295, -3295, -3295, 252, -3295, + -3295, -3295, -3295, -3295, 69762, 59582, -3295, -3295, 252, 69762, + 59582, 252, -3295, 1764, 59582, 59582, 69762, 59582, 1906, 59582, + 59582, 1115, 1713, -3295, 50974, -3295, -3295, 22605, 52, 52, + 1963, 1996, 2002, 1771, 13547, 150, -3295, 21003, 21003, 354, + 336, 69762, 1950, -3295, -3295, 750, 1999, 118, -3295, 69762, + 1813, 59582, 59582, 59582, 59582, 59582, 59582, 1511, -3295, -3295, + -3295, -3295, -3295, 2154, 2306, 1784, 1785, 2156, -3295, 2981, + 2163, 54028, 826, 3313, 2164, 59088, 2167, 1832, 2171, 32045, + -3295, -3295, 1794, -3295, -3295, 1796, 2283, 2050, -3295, -3295, + 2034, -3295, 69762, 2328, -3295, 123, -3295, 50974, -3295, 245, + -3295, 2043, 424, -3295, 15129, 21003, -3295, -3295, -3295, -3295, + -3295, -3295, 1350, 21003, -3295, 778, -3295, -3295, 2295, 1115, + 2295, 536, -3295, -3295, 2295, -3295, 2279, 2295, -3295, 58085, + -3295, 8363, -3295, 21003, 21003, -3295, 21003, 2166, -3295, 2334, + 2334, 58085, 26343, 26343, 26343, 26343, 26343, 26343, 813, 1427, + 26343, 26343, 26343, 26343, 26343, 26343, 26343, 26343, 26343, 27411, + 440, -3295, -3295, 780, 2305, 21003, 21003, 2182, 2166, 21003, + -3295, 58085, 1829, -3295, 1830, 1835, 21003, -3295, 58085, -3295, + 59582, 1836, -3295, -3295, -3295, 27, 1834, 1842, -3295, -3295, + 1675, -3295, 949, 1006, 59582, 1649, 2236, 3005, -3295, -3295, + 21003, 2177, -3295, -22, -3295, 4506, 4506, 34057, -3295, 21003, + 1847, -3295, -3295, 34057, 2192, -3295, 4506, -3295, -3295, 39659, + 4506, -3295, 58085, 788, -3295, 59582, 58085, 810, 21003, -3295, + 15129, 2363, 58085, 2337, 69762, 69762, 2377, 1857, 1860, 1857, + 2168, 1959, -3295, 1964, 1967, 1969, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, 58085, -3295, -3295, 327, + -3295, -3295, -3295, -3295, -3295, -3295, 1861, 1862, 21003, 21003, + 122, -3295, 8444, 1864, 1865, 21003, 48345, -3295, 1869, -3295, + 1866, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, 1878, -3295, + 1884, -3295, 1885, 1901, 1905, 1888, 1891, 8557, 21003, 59582, + -3295, 1900, 23139, 2059, 59582, -3295, -3295, 21003, 21003, 59582, + -3295, 48616, 2277, -3295, 1907, 1909, 8961, -3295, -3295, -3295, + 254, 412, 3909, 421, 4000, 4000, 4000, 2991, -3295, -3295, + -3295, 1924, -3295, 26343, 26343, -3295, 2417, 3326, 11411, -3295, + -3295, -3295, -3295, 2270, -3295, 1056, -3295, 1923, -3295, -3295, + 3371, -3295, 43732, 8191, 21003, 229, -3295, 21003, 30001, 21003, + 2009, 4000, 4000, 4000, 398, 398, 254, 254, 254, 412, + 421, -3295, -3295, -3295, 1925, 21003, 50974, -3295, 1929, 1931, + 2298, 1431, 21003, -3295, -3295, 34057, 1628, 32, 1628, 2168, + 6176, -3295, 1035, -3295, 1035, -3295, 48616, 59582, -3295, 1932, + 583, 34057, 1965, 2420, 2398, 34057, 69762, -3295, -3295, 1937, + 2042, 1956, -3295, -3295, 1957, 21003, 1758, 1957, -3295, 2003, + 34, 2176, 962, 962, 1373, 2178, -3295, -3295, 2011, -3295, + -3295, -3295, 21003, 15663, 1442, -3295, 1446, -3295, -3295, -3295, + -3295, -3295, 1944, -3295, 2219, -3295, 59582, -3295, -3295, 26343, + 2407, 21003, 40168, 2419, 2209, -3295, -3295, -3295, 2005, 2005, + -3295, -3295, 2045, -3295, 2046, 1493, 21003, 2202, -3295, 329, + 1973, 2338, 341, 2286, 69762, -3295, -3295, -3295, 316, 342, + 50974, 1729, -3295, -3295, 224, 2339, 424, 2341, 424, 50974, + 50974, 50974, 815, -3295, -3295, -3295, 1115, -3295, -3295, -3295, + 317, 816, -3295, 1974, 1977, -3295, -3295, -3295, 2056, 1725, + 1493, 2981, -3295, -3295, -3295, -3295, -3295, -3295, -3295, 190, + 1792, 1493, 2064, -3295, 2068, -3295, 2071, 1935, 1493, -3295, + -3295, 1713, 1514, 830, 18333, 48616, 208, 150, 150, 150, + -3295, -3295, -3295, 15129, -3295, 1983, 48616, 48616, 146, -3295, + -3295, -3295, -3295, 1985, -3295, 178, -3295, 69762, -3295, -3295, + -3295, 1950, 1966, 1867, 59582, 2981, 1986, 2474, 2475, 1237, + 1559, -3295, 2157, 37, 69762, -3295, 50974, 69762, 59582, 59582, + 59582, 54537, -3295, -3295, -3295, 1990, 1991, -3295, 47, 2229, + 2222, 59582, 2037, 59582, 2001, -3295, -3295, 59582, 2008, 2489, + 59582, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, 831, -3295, 58085, -3295, 1559, + 1559, 18867, 2385, 59582, 1918, -3295, -3295, -3295, -3295, 69762, + -3295, -3295, 48616, -3295, 48616, -3295, -3295, 50974, -3295, 1115, + -3295, 1115, 2255, 69762, 45892, 1115, 46401, 1115, 2012, -3295, + 48616, 9074, 48616, 2182, -3295, 260, 2334, 811, 811, 811, + 3261, 2368, 249, 2019, 811, 811, 811, 333, 333, 260, + 260, 260, 2334, 440, 980, 52501, 2021, -3295, 48616, 48616, + -3295, -3295, 2022, -3295, -3295, -3295, -3295, 2027, 2036, -3295, + -3295, -3295, -3295, 69762, 186, 1628, 525, 525, 525, 525, + -3295, 59582, 59582, 59582, 48616, 2490, 2364, -3295, -3295, 2549, + 2039, -3295, -3295, 4506, 48616, 59582, -3295, 28459, -3295, 59582, + -3295, 2397, -3295, 2486, -3295, 59582, 848, -3295, -3295, -3295, + 867, 2044, 1857, 58085, 868, 869, -3295, 152, 2168, 2047, + 1556, 1544, 938, 1443, -3295, 56049, -3295, -3295, 2048, 48422, + 21003, -3295, 2429, -3295, -3295, -3295, 48616, 21003, 21003, -3295, + 43732, -3295, -3295, -3295, -3295, -81, -81, -3295, 9493, 1900, + 2051, 2053, 59582, 11411, 48536, -3295, 40677, -3295, 73, 9629, + 48616, -3295, 1831, -3295, -3295, 6176, 21003, 3078, 5169, 21003, + 2055, 21003, 2405, -3295, -3295, 2052, -3295, -3295, 58085, 21003, + 2062, 3889, 26343, 26343, 4426, -3295, 4459, 21003, 11411, -3295, + 44397, 2566, 2066, 1963, 19401, -3295, 2289, 2060, -3295, 2177, + 150, 2177, 2069, -3295, -3295, -3295, 69762, -3295, 2345, 2072, + -3295, 21003, 2234, 69762, 557, 2266, 4506, 873, -3295, 565, + 42204, 1965, 21003, 250, -3295, -3295, 2074, -3295, 1957, -3295, + -3295, -3295, 2308, -3295, -3295, -3295, 59582, -3295, 2079, -3295, + 39150, 2426, 11945, -3295, 39150, 59582, -3295, -3295, 69762, 59582, + 9796, 2456, -3295, 69762, 69762, 69762, -3295, 69762, 2082, 2088, + 1024, 2091, 814, -3295, 2406, -3295, -3295, 1024, 2442, 279, + 2008, 290, 3000, 473, -3295, -3295, -3295, 2173, 59582, -3295, + 69762, -3295, -3295, -3295, -3295, -3295, 51483, -3295, -3295, 43222, + 50974, -3295, 50974, 21003, 21003, 59582, 59582, 59582, 59582, 69762, + 59582, 59582, 59582, 59582, 59582, 1514, -3295, -3295, 21003, 21003, + -3295, 2092, 2093, 2096, 1963, -3295, 206, -3295, 2097, -3295, + -3295, 21003, -3295, -85, -3295, -3295, 178, 2098, 2100, -3295, + 54028, 2223, 59088, 1832, -3295, 1796, 1966, 1044, 69253, 1867, + 21003, -3295, 874, 884, 2981, 2101, 2591, -3295, -3295, -3295, + 826, 54028, -3295, -3295, -3295, 2546, -3295, 812, 259, -3295, + 2593, 1406, -3295, 1237, -3295, 2223, 1559, -3295, 32539, 1821, + -3295, -3295, 2594, -3295, 2595, 2223, 48616, 69762, 2172, -3295, + 424, -3295, -3295, -3295, 69762, 2105, -3295, 2105, -3295, -3295, + 2105, -3295, -3295, -3295, -3295, 26343, 2464, 2112, 58085, -3295, + -3295, 59582, -3295, -3295, -3295, 894, 2113, 2177, 59582, 59582, + 59582, 59582, -3295, -3295, -3295, 19935, 21003, 2152, 21003, -3295, + -3295, 2114, 14081, 2443, -3295, 27945, -3295, -3295, 2116, 39659, + 69762, -3295, -3295, -3295, -3295, 2168, -3295, -3295, 69762, -3295, + -3295, 2125, 2126, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + -3295, -3295, 21003, 48616, -3295, 48616, -3295, -3295, -3295, -3295, + -3295, 59582, -3295, -3295, 7617, -3295, 2124, 2129, 69762, 59582, + 128, -3295, 21003, -3295, 2059, -3295, 452, 21003, 21003, 2417, + -3295, 10249, 21003, 58085, 902, 2417, 293, 21003, 5189, 5219, + 21003, 21003, 5372, 9902, -3295, 23673, 14595, -3295, 2136, 21003, + 10043, 42713, -3295, 34057, 2364, 2138, 2364, 1115, -3295, 2139, + 2137, -3295, -3295, 1975, 48616, 21003, -3295, -3295, -3295, -3295, + 2188, -5, 36093, 2378, -3295, 2191, 2161, 69762, -3295, 2234, + 48616, -3295, -3295, 43732, -3295, -3295, -3295, -3295, -3295, 2614, + 1917, 2150, 2159, -3295, 1370, -3295, -3295, -3295, 69762, 2160, + -3295, 2180, 1024, -3295, 69762, 2193, -3295, 269, 2483, 157, + -3295, 21003, -3295, 2577, 2658, 2406, 2174, 69762, 59582, 26343, + -3295, 285, 272, -3295, 2471, 59582, 2193, 2616, -3295, -3295, + -3295, 814, -3295, 2513, 2423, -3295, 252, -3295, 21003, 814, + 2425, 153, 69762, -3295, -3295, 2770, -3295, 58085, 424, 424, + -3295, 1623, 2181, 2186, 2187, 2190, 2194, 2197, 2199, 2201, + 2206, 2208, 2210, 2213, 2214, 2215, 2217, 2220, 2225, 2227, + 2230, 2240, 1639, 2243, -3295, 2244, 2074, 2245, 2246, 2247, + 2248, 2251, 75361, 2254, 2258, 2259, 2260, 1640, 2263, 2267, + 444, 562, -3295, -3295, -3295, -3295, -3295, -3295, 1306, 2268, + -3295, 2195, -3295, 2189, 904, -3295, -3295, 2281, -3295, 2292, + -3295, -3295, -3295, -3295, -3295, -3295, 2207, 2231, -3295, -3295, + -3295, 150, 2235, 2271, 69762, 10346, 1350, 121, 50974, 69762, + 2274, 2037, -3295, 2701, 771, 2466, 2221, 2278, -3295, 48616, + -3295, 50974, 1832, -3295, 54028, 3636, 664, 2222, 58085, -3295, + 737, 2037, -3295, 2644, 59088, -3295, 2276, 2264, 1832, 2309, + -3295, 1796, -3295, -3295, 21003, 21003, 2380, 21003, 160, -3295, + 2514, 69762, 2284, -3295, 2105, 4604, 26343, 58085, 952, 960, + -3295, 2804, 2455, 2364, -3295, -3295, -3295, -3295, -3295, 2290, + 439, 2291, 10877, 2294, -3295, -3295, -3295, -3295, -3295, -3295, + 48616, 48616, 69762, 2479, 48616, -3295, -3295, 2296, 2293, 41186, + 2767, 2300, -3295, -3295, 2629, -3295, 31536, -3295, 1857, 2310, + 1857, 58085, 1857, -3295, -3295, 48616, 1900, 21003, -3295, -3295, + -3295, 2304, 2303, 69762, 44667, 2645, -3295, 2417, 2417, 10249, + 961, -3295, 2417, 21003, 21003, 2417, 2417, 21003, -3295, 20469, + 231, -3295, 969, -3295, 44249, -3295, 76349, -3295, -3295, 2152, + 1115, 2152, -3295, -3295, 69762, 2311, 2313, -3295, -3295, -3295, + 2366, -3295, -3295, 990, 2759, 2234, 191, -3295, -3295, 2191, + 2234, 21003, -3295, -3295, 2323, 39150, -3295, -3295, -3295, -3295, + 39150, 1024, -3295, 2501, 2193, 2330, -3295, -3295, -3295, -3295, + -3295, -3295, 44267, -3295, 79, 21003, -3295, 1001, 3261, -3295, + -3295, -3295, -3295, 2193, 1237, -3295, 59582, 2823, 2712, -3295, + -3295, 48616, -3295, -3295, 2168, 2168, -3295, -3295, 2486, -3295, + -3295, 2340, 1306, 401, 43222, -3295, -3295, 59582, 59582, -3295, + -3295, 2342, -3295, -3295, -3295, -3295, -3295, -3295, -85, 2742, + 996, 1015, 826, -3295, 2223, 59582, 2721, 54028, 50974, -3295, + 2835, 2353, 59582, 2037, 1178, 1178, -3295, 2510, -3295, 2517, + -3295, -3295, -3295, -3295, 1115, 2848, 301, -3295, 48616, 48616, + 1414, 59582, -3295, -3295, 35584, 4604, 1023, -3295, -3295, 2367, + 2371, -3295, 2152, 21003, 2372, 21003, -3295, 24207, 2860, 2373, + -3295, 21003, 2436, 28973, -3295, 21003, -3295, 59582, 64672, 2376, + 64672, -3295, -3295, -3295, -3295, 59582, -3295, -3295, -3295, 21003, + -3295, 2417, 2417, 2417, 21003, 21003, -3295, -3295, -3295, -3295, + 2592, 2479, -3295, 2479, -3295, -3295, 21003, 2223, 565, 3646, + 69762, 12, -3295, 2869, 2657, -3295, -3295, 48616, -3295, -3295, + -3295, 59582, -3295, 50974, -3295, 1024, 8, 2384, 21003, 44302, + 2628, -3295, -3295, 2662, -3295, 2723, -3295, 2450, 630, 2468, + -3295, -3295, -3295, -3295, 1350, 1115, -3295, 1832, 2222, 2309, + 2392, 59582, 1029, 2223, 826, 812, -3295, -3295, -3295, -3295, + -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, -3295, + 2223, 2846, 2624, 2850, -3295, 2172, 21003, 110, -3295, 1033, + 2842, -3295, -3295, 2915, 2479, 2399, 24207, 2401, -3295, 2403, + 69762, 48616, 2555, -3295, -3295, 2413, -3295, -3295, 21003, -3295, + -3295, 44878, 2408, 2416, 2874, 1963, 2436, 2436, -3295, -5, + -3295, -3295, 2852, 35584, 2810, 21003, 2511, 2889, 1237, 1024, + 2433, 1045, -3295, -3295, -3295, -3295, -3295, 2981, -3295, 44320, + 2673, 601, 2659, 2384, 21003, -3295, 2502, -3295, -3295, -3295, + 2914, -3295, -3295, 54028, 2428, -3295, 2309, 2222, 2037, 2309, + 2660, -3295, 2668, 2439, 44355, 69762, 69762, 1832, 35584, 69762, + 2448, 2436, -3295, 2449, -3295, -3295, -3295, 30515, -3295, 2457, + -3295, -3295, -3295, 21003, 169, -3295, -3295, 2496, 59582, 1075, + 82, 48616, 276, 434, 2869, 2662, 43222, -3295, 50974, 1028, + 8, 2765, -3295, -3295, -3295, -3295, 241, 2690, -3295, 2692, + -3295, 48616, -3295, 2223, 54028, -3295, -3295, -3295, -3295, -3295, + -3295, 35584, 2842, -3295, 351, -3295, 1628, -3295, 351, -3295, + -3295, -3295, -3295, -3295, 1566, 24741, 24741, 24741, 2461, 2223, + -3295, 1628, -3295, 2596, -3295, 2713, 21003, 129, 246, -3295, + -3295, -3295, 2557, 2659, -3295, -3295, -3295, -3295, -3295, 533, + 533, 2872, -3295, 2533, -3295, 2309, 1103, 69762, 1957, -3295, + 1957, 25809, 2626, 211, 47469, 2853, -3295, 2853, 2853, -3295, + -3295, -3295, 42204, -3295, -3295, 48616, 2525, 69762, 2488, 2531, + 41695, -3295, 276, -3295, -3295, 2984, -3295, 268, -3295, -3295, + -3295, 1832, 351, -3295, -3295, 2977, -3295, -3295, -3295, -3295, + -3295, 951, -3295, -3295, -3295, 1628, -3295, 1110, -3295, -3295, + 2491, -3295, -3295, -3295, 1024, -3295, -3295, -3295, 1628, 1957, + 25275, 2647, -3295, 2715, -3295, -3295, -3295, 21003, -3295, -3295, + -3295, -3295, -3295, 2498, -3295 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int16 yypgoto[] = { - -3281, -3281, -3281, 2073, 79, -3281, -3281, 206, -3281, -3281, - 636, -1739, -1738, 1136, -3281, 199, -783, 641, -3281, 81, - 4828, 2910, 4436, 230, -528, -893, -1295, 4, 87, -1170, - 7, -3281, -3281, -1828, -3281, -1553, -502, 270, -3281, -3281, - -647, -2717, -595, -3281, -3155, -3226, -762, -3281, -2624, -3159, - -2136, 95, -2575, -3281, -3281, 97, 5, -2173, -3281, -1724, - 72, -2181, -3281, -195, -2847, 98, 100, 1040, -3281, -2723, - 102, -902, -1235, -956, -1231, -3281, -93, -3281, 496, 107, - 924, 2130, -3281, 17, -2276, -3102, -610, -3281, -726, -3281, - -351, -3281, -659, -3281, -850, -672, -711, -2941, -1181, -3281, - 1763, -401, -3281, 682, -3281, -2726, -3281, -3281, 666, -3281, - 1199, 1208, -3281, -3281, -2245, 187, -640, -2767, -2727, -2259, - -918, 276, -639, 248, -2203, -1222, -3281, 714, -3281, -623, - -3281, -908, -2484, 110, -3281, -3281, 1660, -922, -3281, 115, - -618, -3281, -3281, -714, -3281, -3281, -3281, -3281, -3281, -280, - 129, -3281, 550, -3281, -2236, 551, -2207, 1697, -605, -3281, - 203, 10, 11, -3281, -3281, -3281, -3281, -775, 635, -1899, - -3281, -3281, -3281, -3281, 229, 3, 3797, -40, -31, -3281, - -27, -3281, -3281, -3281, 757, -3281, -3281, 1, 61, 1870, - -3281, -3281, -3281, -3281, -1045, -3281, -1850, 647, -3281, 2032, - 2033, -1819, -858, -56, 278, 800, -1676, -2205, -621, 1274, - 1854, 1856, -3281, 510, -2753, -3281, -556, -3281, -655, -3281, - -3281, -3281, 2028, -3281, 798, 1326, -1593, -1580, -3281, -2322, - -3281, -471, -348, -3281, -3281, -3281, -3281, -3281, -2641, -3043, - -570, 1293, -3281, 1871, -3281, -3281, -3281, -3281, 51, -1585, - 3064, 837, -3281, 67, -3281, -3281, -3281, -3281, 178, -3281, - 1031, -160, -3281, -474, -680, -803, 2087, 58, 179, -1922, - 35, 271, 552, -3281, -3281, 555, -2152, -1457, 505, -264, - 1026, -3281, -2359, -1510, -3281, -1973, -1214, -3281, -3281, -914, - 1220, -3281, -3281, -3281, 1529, 2143, -3281, -3281, 2158, 2526, - -3281, -925, 3150, -724, -1048, 2108, -959, 2111, -949, -953, - -957, 2112, 2113, 2117, 2121, 2133, 2136, 2138, -1589, 5823, - -152, 1384, -2255, -3281, -1458, -1620, -3281, -3281, -3281, 27, - -3281, -1426, 85, -3281, -3281, -3281, -3281, -2902, -3281, -450, - -3281, -431, -3281, -3281, -3281, -1804, -3280, -1838, -3281, 2534, - 968, -3281, -3281, 503, -3281, -3281, -3281, -3281, -1599, -3281, - 7222, 864, -3281, -2093, -3281, -3281, -1002, -844, -790, -1029, - -1249, -2006, -3281, -3281, -3281, -3281, -3281, -3281, -1038, -1747, - -173, 925, -3281, -3281, 1022, -3281, -3281, -166, -1516, -1815, - -2186, -3281, -3281, -3281, 930, 1652, 135, -839, -1668, -3281, - -1578, -3281, -3281, 982, -2518, -3281, -3281, 495, -2773, -3281, - -3281, 193, -3281, -699, -1163, -2591, 69, 15, -3281, 1896, - -2683, -3281, -3281, -753, -2816, -1143, -947, -3281, 140, -3281, - 147, 145, -1715, -3281, 19, -3281, -393, -3281, -3281, -2737, - -3281, 149, 150, 2350, -3281, 1277, -3281, -3281, -3281, -3281, - -547, -3281, -642, 2161, -3281, -3281, 24, -3281, 1768, -3281, - 151, 302, -3281, 1090, -3281, 762, 152, -3281, 2256, -397, - 153, 1430, -3281, -3281, -3281, 13, -408, 451, -3281, 1431, - -3281, -3281, -663, -1698, -3281, 660, 1301, -2139, 154, -3281, - 585, 8, -3281, -3281, -3281, 99, 155, 23, -3175, 156, - -2877, -1730, -7, -3281, -3281, -3281, -634, -3281, -2682 + -3295, -3295, -3295, 2140, 84, -3295, -3295, 258, -3295, -3295, + 692, -1729, -1733, 1193, -3295, 251, -758, 691, -3295, 87, + 4693, 2849, 4942, 617, -531, -916, -1266, 18, 89, -1171, + 3, -3295, -3295, -1872, -3295, -1556, -451, 320, -3295, -3295, + -595, -2703, -545, -3295, -3200, -3232, -711, -3295, -2677, -3165, + -2165, 100, -2587, -3295, -3295, 106, 17, -2228, -3295, -1768, + 64, -2183, -3295, -144, -2859, 109, 114, 1095, -3295, -2738, + 117, -910, -1248, -972, -1243, -3295, -40, -3295, -3295, 548, + 126, 1319, 2183, -3295, 19, -2283, -3037, -557, -3295, -673, + -3295, -299, -3295, -611, -3295, -901, -619, -660, -2977, -1186, + -3295, 1823, -351, -3295, 736, -3295, -2742, -3295, -3295, 719, + -3295, 1251, 1252, -3295, -3295, -2366, 222, -594, -2789, -2736, + -2297, -914, 313, -602, 286, -2221, -1299, -3295, 753, -3295, + -586, -3295, -923, -1975, 130, -3295, -3295, 1707, -954, -3295, + 132, -580, -3295, -3295, -677, -3295, -3295, -3295, -3295, -3295, + -242, 135, -3295, 592, -3295, -2263, 595, -2217, 1748, -642, + -3295, 233, 7, 10, -3295, -3295, -3295, -3295, -799, 671, + -1916, -3295, -3295, -3295, -3295, 277, 4, 3275, -41, -32, + -3295, -28, -3295, -3295, -3295, 793, -3295, -3295, 2, 51, + 1915, -3295, -3295, -3295, -3295, -1046, -3295, -1893, 566, -3295, + 2070, 2075, -1821, -891, -33, 315, 837, -1734, -2199, -620, + 1312, 1895, 1902, -3295, 543, -2735, -3295, -528, -3295, -721, + -3295, -3295, -3295, 2065, -3295, 828, 1359, -1552, -1550, -3295, + -2327, -3295, -443, -321, -3295, -3295, -3295, -3295, -3295, -2437, + -3063, -599, 1327, -3295, 1908, -3295, -3295, -3295, -3295, 62, + -1599, 3105, 871, -3295, 203, -3295, -3295, -3295, -3295, 212, + -3295, 1060, -133, -3295, -478, -702, -809, 2122, 147, 217, + -1911, -9, -375, 584, -3295, -3295, 586, -2160, -1466, 538, + -237, 1062, -3295, -2373, -1602, -3295, -1527, -1211, -3295, -3295, + -783, 1498, -3295, -3295, -3295, 2882, 2887, -3295, -3295, 3069, + 3471, -3295, -931, 3592, 1076, -1058, 2135, -973, 2141, -941, + -958, -955, 2143, 2145, 2146, 2147, 2149, 2151, 2153, -1581, + 5816, -959, 3798, -2280, -3295, -1447, -1624, -3295, -3295, -3295, + 50, -3295, -1430, 127, -3295, -3295, -3295, -3295, -2924, -3295, + -424, -3295, -420, -3295, -3295, -3295, -1845, -3294, -1884, -3295, + 4395, 981, -3295, -3295, 524, -3295, -3295, -3295, -3295, -1588, + -3295, 7216, 889, -3295, -2115, -3295, -3295, -1005, -874, -473, + -1035, -1256, -2000, -3295, -3295, -3295, -3295, -3295, -3295, -1188, + -1839, -234, 946, -3295, -3295, 1046, -3295, -3295, -147, -1519, + -1823, -2190, -3295, -3295, -3295, 953, 1682, 158, -848, -1680, + -3295, -1586, -3295, -3295, 1004, -2531, -3295, -3295, 510, -2806, + -3295, -3295, 215, -3295, -694, -1158, -2593, 46, 26, -3295, + 1121, -2691, -3295, -3295, -733, -2851, -1150, -951, -3295, 136, + -3295, 173, 138, -1757, -3295, 23, -3295, -372, -3295, -3295, + -2754, -3295, 142, 149, 2386, -3295, 1290, -3295, -3295, -3295, + -3295, -555, -3295, -652, 2198, -3295, -3295, 30, -3295, 1806, + -3295, 165, 394, -3295, 1116, -3295, 689, 170, -3295, 2312, + -444, 171, 1457, -3295, -3295, -3295, 11, -541, 481, -3295, + 1466, -3295, -3295, -662, -1724, -3295, 695, 1339, -2168, 175, + -3295, 447, 15, -3295, -3295, -3295, 76, 176, 24, -3131, + 177, -2845, -1759, -7, -3295, -3295, -3295, -635, -3295, -2694 }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ -#define YYTABLE_NINF -2180 +#define YYTABLE_NINF -2183 static const yytype_int16 yytable[] = { - 534, 78, 929, 71, 978, 57, 884, 53, 100, 730, - 69, 70, 1469, 95, 1157, 1263, 534, 65, 897, 83, - 729, 78, 532, 104, 1051, 2030, 1852, 1338, 796, 1737, - 2139, 1526, 1853, 1189, 531, 1836, 1283, 1384, 532, 1395, - 868, 2005, 1341, 1225, 790, 2234, 1439, 901, 1443, 2099, - 829, 902, 1442, 2384, 2501, 2754, 1441, 2393, 1378, 980, - 1825, 1636, 2717, 2068, 2151, 2190, 2316, 2598, 1819, 534, - 534, 2707, 1871, 870, 822, 1937, 2267, 2167, 2168, 50, - 2336, 51, 567, 3169, 2687, 1796, 2689, 52, 761, 756, - 3166, 532, 532, 3134, 3136, 55, 1857, 56, 59, 838, - 60, 2600, 61, 829, 829, 912, 882, 63, 831, 2668, - 66, 3183, 1714, 881, 881, 67, 1650, 1717, 2372, 2745, - 2706, 3187, 988, 2952, 2579, 2580, 2856, 822, 822, 68, - 1283, 997, 2238, 863, 1262, 2586, 1268, 3025, 1272, 2590, - 81, 3030, 1029, 898, 899, 82, 2812, 919, 1054, 84, - 85, 91, 92, 94, 99, 103, 105, 1612, 1613, 1408, - 2169, 831, 831, 2415, -502, 2380, 2411, -931, -923, 2192, - -2018, -1425, 1617, 3621, 2410, 1626, -569, 2287, 2805, 915, - 833, -573, 2821, 3266, 3269, 1076, 2569, 2570, 2571, 1628, - -2018, -392, 2290, 1075, 833, -1366, 1372, -931, -2166, -2166, - -928, 1372, 2023, 1453, -928, 833, -1366, 2024, 1076, 1075, - -2009, 2822, 1722, -1347, -1363, -1363, -2026, -2009, -2157, -2157, - -1367, 2702, 2830, -2026, -2175, -2175, -1367, -890, -1364, -1364, - 2814, -903, -918, 833, 1670, 2115, 833, -2152, -2152, 1672, - 2816, 2793, 3402, 998, 3168, -2177, -2177, 1718, 1388, 833, - 2182, 1815, 874, 1075, 1388, 835, 1869, 1803, 1170, 1075, - 1274, 2548, 1855, 2182, 2755, 2618, 1982, 1870, 3506, 546, - 1680, 1984, 2796, 1646, 1075, 1214, 1657, -569, -1186, 3742, - 3461, 1283, -573, -532, 1745, 1815, -1186, -248, 3787, 3493, - 3389, 1023, -248, 1797, 1837, 3351, 2784, 1840, 1841, 3319, - 2327, 1982, -718, 1177, 1682, 1983, 1984, 23, 2872, 2874, - 3406, 2877, 1498, 833, 2702, 2407, 3336, 3386, 3369, 2022, - 1759, 2839, 2193, 3659, 1670, 2959, 1217, 1054, 1671, 1672, - 2913, 3578, 1363, 1372, 894, 3, 4, 1597, 1174, 1029, - 3608, 1759, 888, 1021, 1670, 1277, 894, 924, 1671, 1672, - 3295, 2194, 3297, 1806, 114, 3698, 1402, 2022, 2074, 3188, - 1680, 997, 1719, 3347, 3782, 2991, 3788, 2993, 2221, 1760, - 1030, 1364, 3691, 1822, 1486, 2756, 2222, 3430, 3763, 1493, - 1680, 1496, 851, 1795, 1791, 1792, 888, 547, 2231, 1872, - 1760, 3625, 1204, 3467, 1682, 3469, 2262, 2293, 1622, 3684, - 1521, -1207, 1756, 1982, 2748, 3537, 2444, 1983, 1984, -1207, - 2679, 2894, 3403, 1210, 1682, 3538, 1670, 3068, 2883, 3333, - 3645, 1842, 2321, 3479, 3572, -860, 3573, 3404, 3345, 106, - 1670, 3005, 969, 2776, 889, 3368, 3505, 3671, 1918, 115, - 3674, 3396, 2300, 1938, 1723, 2205, 2777, 3761, 800, 1040, - 3480, 3715, 1680, 3397, 26, 27, 28, 3411, 801, 1761, - 2516, 3789, 3660, 1823, 1176, 1553, 2321, 3038, 1876, 2195, - 3048, -569, 2522, 1178, 2396, 1179, -573, 1341, 889, 2197, - 107, 2263, 3700, 2322, 852, 1041, 1682, 3006, 1720, 1762, - 1212, 871, 3069, 3701, 1205, 3334, 3049, 3758, 1373, 3039, - 1682, 1499, 2550, 1373, 3346, 3702, 3796, 3627, -738, 2555, - 1762, 3021, 1043, 1757, 1031, 2294, 1741, 1808, 3722, 890, - 544, 33, 3522, 3723, 1171, 1597, 3550, 2742, 3348, 3189, - 1636, 3783, 3692, 3158, 1630, 2944, 1239, 2745, 3699, 2745, - 3764, 2749, 2232, -569, 1278, 116, 3756, 3518, -573, 3661, - 3710, 944, 3504, 2591, 1365, 1763, 1824, 2591, 1873, 1492, - 38, 3693, -860, 3488, 3579, 1875, 2798, 2799, 2800, 1843, - 2206, 2637, 3219, 3607, 3703, 2884, 1763, 2071, 3428, 3070, - 1844, 2408, 3498, 3353, 3539, 1824, 2612, 2207, 2190, 2806, - 2807, 3358, 2208, 2399, 3478, 40, 730, 1661, 1798, 3482, - 2757, 1172, 2758, 2139, 1793, 875, 43, 966, 1856, 2594, - 1862, 2109, 1612, 1613, 1069, 3408, 978, 3196, 927, 3206, - 3790, 1939, 928, 3405, 3494, 3383, 2392, 1943, 2183, 2014, - 1617, 2209, 2759, 964, -472, 1373, 2198, 1626, 1732, 1809, - 1608, 2670, 2785, 2495, 1047, 3173, 3197, 2199, 2031, 3746, - 1628, 2691, -1186, 3151, 1631, 1794, 1377, 2913, 3179, 2907, - 2971, 46, 3514, 1500, 1374, 3337, -569, 2715, 3210, 1377, - 2055, -573, 3672, 2640, 1366, 2709, 1632, 1940, 3566, 1853, - 2287, 2792, 1742, 2613, 1631, 2097, 2237, 3134, 3136, 3743, - 1633, -923, 2192, -2018, 2720, -502, -502, 2833, 2826, 2121, - 2122, 1623, -1425, 2808, 2823, -569, 1632, -569, 1119, 1120, - -573, 2928, -573, -2018, 2895, 2896, 2897, 2898, -1366, 1337, - 1635, 1623, -392, -928, 3364, 884, 2301, 891, 1665, -1366, - 3485, 1119, 1120, -2009, 927, 3486, -1347, 3673, 928, -2026, - -2009, 548, 2223, -1367, 1947, 3450, -2026, 1703, 1377, -1367, - 1615, 970, 1614, 997, 1620, -918, 1715, 3327, 996, 2239, - 1017, 2106, 1646, 1392, 1393, 78, 1283, 2163, 1283, 1392, - 1393, 1646, 796, 1847, 2881, -1207, 3606, 884, 1180, 1999, - 2797, 2812, 3012, 927, 3360, 3361, 2142, 1658, 3462, 993, - 1740, 2210, 3716, 1744, 3613, 3018, 832, 1848, 3614, 534, - 2020, 3750, -248, -248, 3159, -2151, -2151, 1847, 1210, 877, - 534, 1996, 1997, 1998, 1999, 1641, 2107, 534, 2164, 3641, - 3642, 532, 2392, 3704, 2510, 2113, 2575, 3304, 983, 2511, - 886, 1848, 532, 829, 1700, 1701, 1702, 1703, 901, 532, - 2045, 946, 902, 947, 829, 1226, 534, 534, 1597, 1513, - 2758, 829, 1698, 1699, 1700, 1701, 1702, 1703, 2556, 1035, - 1462, 2558, 2062, 3019, 903, 3052, 2576, 3305, 1019, 2992, - 534, 3400, 1210, 1826, 3686, 1324, 887, 1211, 3318, 3214, - 3081, 1642, 3166, 2360, 1020, 1212, 3682, 2529, 78, 2512, - 71, 831, 57, 2363, 53, 100, 2366, 69, 70, 1181, - 95, 1514, 831, 2556, 65, 904, 83, 1175, 3055, 831, - 104, 1994, 1995, 1996, 1997, 1998, 1999, 2559, 3589, 2327, - 534, 730, 1815, 3401, 1330, 534, 1175, 3320, 906, 2185, - 118, 2680, 2359, 1816, 545, 923, 1213, 2695, 1236, 1190, - 2521, 1827, 760, 3397, 1237, 2811, 1597, 881, 3254, 1212, - 1631, 3256, 109, 3258, 863, 863, 850, 863, 922, 863, - 864, 1004, 3387, 2696, 3276, 2914, 50, 1059, 51, 3134, - 3136, 2549, 1632, 2485, 52, 2857, 1386, 1330, 1060, 1387, - 3152, 3153, 55, 3590, 56, 59, 2563, 60, 996, 61, - 2421, 1753, 3042, 884, 63, 534, 534, 66, 1384, 78, - 1213, 534, 67, 3036, 534, 534, 796, 534, 534, 534, - 534, 2385, 2386, 2387, 2494, 3751, 68, 892, 2496, 2313, - 3784, 2498, 1340, 1631, 534, 550, 534, 81, 3085, 2139, - 1000, 1828, 82, 2752, 2530, 534, 84, 85, 91, 92, - 94, 99, 103, 105, 2531, 1632, 532, 3145, 532, 1238, - 1498, 2358, 534, 1330, 1600, 1236, 3491, 532, 829, 1633, - 829, 1237, 1058, 1949, 2962, 2560, 3706, -2018, 1075, 829, - 3496, 2369, 1815, 1236, 534, 3059, 2376, 1007, 884, 1237, - 925, 3043, 822, 1818, 1019, 1829, 1597, 1166, 893, 551, - 3707, 2923, 2361, 822, 534, 1036, 1501, 2364, 1489, 2692, - 1020, 2693, 1508, 3752, 2030, 23, 534, 534, 534, 1505, - 534, 534, 956, 730, 3771, 1891, 831, 2005, 831, 550, - 1661, 3474, 3778, 1631, 1651, 3060, 110, 831, 1853, 948, - 1951, 949, 3753, 2726, -656, 1075, 835, 111, 1236, -656, - 1417, 1418, 2801, 3061, 1237, 1632, 534, 1624, 1625, 3602, - 3044, 957, 3045, 1824, 1892, 833, 1938, 1164, 1165, 1635, - 1167, 931, 1169, 1826, 534, 534, 1238, 3286, 927, 894, - -1347, 2193, 928, 2554, 112, 1747, 1748, 2715, 1754, 902, - 902, 927, 902, 551, 1238, 928, 2509, 1040, 2825, 2718, - 2513, 3391, 1597, 2515, 1075, 927, 943, 2786, 2225, 1658, - 2194, 534, 2226, 1037, 950, 534, 534, 3422, 1038, -224, - 1239, -656, 3585, 1425, 1426, 534, 534, 534, 1730, 953, - 534, 1731, 1506, 1041, 1339, 1394, 1511, 113, 962, 1944, - 1650, 1827, 1945, 1175, 2070, 1437, 1175, 2071, 967, 1507, - 2095, 960, 1600, 2096, 1895, 2404, 1597, 3062, 2405, 1238, - 1043, 968, 26, 27, 28, 2840, 2504, 3063, 1597, 2505, - 881, 971, -656, 3382, 2616, 2851, 3132, 1039, 1074, 2542, - 1240, 1982, 2543, 972, 1241, 1983, 1984, 555, 1330, 987, - 1985, 1986, 1987, 1669, 2592, 973, 1670, 2593, 1597, 1330, - 1671, 1672, 2914, 974, 1851, 1597, 3328, 1425, 1426, 2013, - 1854, 2015, 2016, -2146, -2146, 558, 1242, 1980, 1981, 2444, - -2147, -2147, 2660, 2001, 1330, 1606, 984, 46, 2195, 33, - 1826, 1828, 1680, 2196, 895, -2148, -2148, 1239, 2197, -2180, - 2436, 1002, 2437, 1431, 1432, 932, -2149, -2149, 3377, 1597, - 1040, 2661, 1003, 1597, 2658, 1239, 1004, 2595, 1899, 1597, - 2593, 2564, 1005, 2565, 730, 2811, 1682, 1663, 38, 1008, - 1900, 933, 1902, 730, 1011, 1932, 1903, 1666, -2153, -2153, - 3262, 555, 1597, 987, 1012, 1829, 1041, -2154, -2154, 2688, - 996, 1243, 1047, 2772, -2155, -2155, 2773, 78, 1827, 1716, - 1013, 1241, 1042, 40, 796, -2156, -2156, 730, 1721, 558, - 1022, 1500, 534, 1043, 43, 1266, -2158, -2158, 1954, 1241, - 1239, 993, 2766, 2778, 2768, 1014, 2779, 1431, 1432, 1015, - 2908, 2139, 2657, 1242, 2915, 1063, 1064, 1065, 934, 2794, - 1068, 3215, 2505, 2854, 2920, 1024, 2855, 2593, 1236, 1044, - 1016, 1267, 2921, 1824, 1237, 2096, 1941, 2099, 1942, -2159, - -2159, 1858, 534, 534, -2180, 2646, 1017, 935, 534, 46, - 534, 1988, 2729, 1061, 2662, 534, 534, 534, 534, 2663, - 1270, -2180, 2924, 3780, 1241, 2925, -2180, 2566, 1828, 2567, - 534, 534, 532, 1056, 1989, 2198, 1045, 2769, 2926, 2771, - 534, 2925, 534, 1046, 829, 534, 2199, 1066, 1243, 2820, - 534, 936, 534, 534, 1863, 534, 1271, 3013, 3176, 534, - 3014, 3177, 532, 1067, 532, -2180, 1243, 532, 3035, 3759, - 3037, 3760, 532, 3072, 829, 532, 829, 532, 3079, 829, - 3725, 532, 1829, 3178, 829, 1047, 2405, 829, 3216, 829, - 1207, 3217, 1209, 829, 3277, 3737, 1069, 2096, 822, 1238, - 822, 1070, 831, 822, 1048, 1168, 3372, 1990, 822, 2265, - 1184, 822, 1191, 822, 2046, 1600, 2047, 822, 3423, 2049, - 1192, 2096, 1691, 2857, 2053, 3162, 3071, 2056, 3080, 2057, - 3795, 1243, 831, 2061, 831, 2945, 2946, 831, 1198, 833, - 1236, 3424, 831, 1194, 2593, 831, 1237, 831, 534, 534, - 1824, 831, 1195, 78, 1199, 2102, 1196, 534, 534, 2098, - 2105, 3604, 2100, 2101, 3456, 2104, 534, 2096, 3463, 3791, - 1208, 2071, 3475, 534, 1201, 3476, 2664, 1049, -2160, -2160, - 534, 937, 3794, 1953, 1230, 3512, 1227, 2665, 3177, 1650, - 1232, 3513, 938, 1233, 2405, 3568, 3569, -2161, -2161, 2931, - 534, 730, 2934, 2936, 2937, 534, 2933, 2935, 534, 1324, - 2932, -2162, -2162, 1600, 534, 534, 534, 534, 534, 534, - 534, 534, 730, 1234, 2327, -2180, 534, 534, 534, 939, - 2444, 534, 1235, 2179, 3547, 534, 1247, 2096, 534, 534, - 534, 534, 534, 534, 534, 534, 534, -2163, -2163, 534, - 532, 1238, 1248, 940, 3611, 1597, 534, 3177, 1330, 1265, - 1239, 14, 15, 3623, 114, 3653, 3624, 3697, 3654, 3757, - 3624, 1269, 3624, 1273, 2269, 1333, 534, 1074, 902, 1275, - 1982, 1336, 941, 3792, 1983, 1984, 3476, -2164, -2164, -2180, - -2180, -2180, -2165, -2165, 2867, 1337, 3052, 3449, -2167, -2167, - 1342, 534, 3053, -2168, -2168, 1181, -2169, -2169, 23, -2170, - -2170, 1597, 534, 534, 1347, 3054, -2171, -2171, -2172, -2172, - 2312, 2715, 2288, 2289, 1241, 1991, 1992, 1993, 1361, 1994, - 1995, 1996, 1997, 1998, 1999, 1359, 1236, -2180, 1362, 3055, - 550, 3056, 1237, 1369, 1698, 1699, 1700, 1701, 1702, 1703, - -2174, -2174, 1370, 1600, 2234, 1376, 1242, -582, -2176, -2176, - -2179, -2179, 1380, 730, 2163, -1411, 1379, 730, 1878, 1879, - 1385, 954, -582, 1403, 2342, -713, -713, -582, 2348, -717, - -717, 2523, 2524, 2525, 2526, 2527, 2528, -716, -716, 2532, - 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 1404, - 3079, 1427, 1428, 1409, 551, 1431, 1432, 1467, 730, 1480, - 3132, 534, 1239, 3200, 3201, 1482, 3324, 3325, 1330, 1954, - 1483, 534, 534, 1494, 2317, 3744, 3747, 1597, -582, 1490, - 3057, 1243, 2859, 2861, 1503, 2331, 1502, 2334, 3733, 3734, - 2345, 3029, 1826, 1509, 955, 3769, 3770, 1238, 1520, -582, - 2353, 1510, 2355, 1516, 2074, 26, 27, 28, 2485, 1600, - 2420, 2734, 2735, 2486, 1522, 2362, 3170, 2440, 1602, 1603, - 2365, -894, 2367, 1605, 2370, 2371, 1241, 2373, -901, 2377, - 2378, 730, 3194, 1726, 1727, 1614, 3139, 1618, 1330, 534, - 46, -738, 1597, 3157, -891, 3209, -739, 534, -892, 1629, - -582, 3058, 954, 2240, 3288, -895, 3059, 1630, 2368, -582, - 1827, -893, 1637, 1600, 1652, 1662, 1664, 534, 534, 1709, - 534, 1711, 33, 1725, 1713, 1600, 534, 534, 534, 534, - 534, 534, 3007, 35, 534, 534, 534, 534, 534, 534, - 534, 534, 534, 534, 3546, 1249, 956, 1733, 1734, 534, - 534, 1738, 1743, 534, 3724, 1600, 3060, 37, 3726, 1826, - 534, 38, 1600, 1746, 1211, 1250, 1990, 1213, 1781, 1785, - 1783, 3523, 1801, 1861, 3061, 955, 1820, 2309, 2311, 1821, - 1831, 2655, 2656, 1243, 534, 957, 1832, 1833, 23, 1838, - 1845, 534, 555, 534, 987, 1850, 40, 534, 1846, 1826, - 1828, 1860, 1597, 2715, 1865, 1868, 1600, 43, 1239, 1881, - 1600, 1251, 534, 532, 1330, 1882, 1600, 557, 1883, 532, - 558, 1887, 958, 1890, 44, 829, 3524, 1827, 1897, 1898, - 1904, 829, 3785, 1905, 1908, 3525, 1911, 1912, 1914, 1600, - 1915, 1916, 2379, 1917, 1919, 1597, 959, 1946, 45, 822, - 1920, 534, 534, 1933, 1829, 822, 1934, 1938, 534, 3526, - 2397, 2397, 46, 2021, -582, 2581, 3356, 1827, 2374, 1971, - 1973, 2584, 1241, 1976, 1979, 960, 1974, 1891, 3062, 2002, - 2010, 2011, 2018, 831, 2041, 2048, 2043, 2044, 3063, 831, - 2783, 534, 3132, 1826, 2054, 534, 2058, 2059, 884, 3510, - 534, 534, 2069, 3546, 2375, 2060, 2072, 2066, 1252, 1608, - 1615, 1597, 3365, 3366, 3253, 1620, 1892, 1828, 2073, 2075, - 2076, 2078, 1824, 3008, 2108, 2030, 534, 534, 2109, 2077, - 2787, 534, 1075, 3527, 927, 26, 27, 28, 928, -1899, - 2508, 1670, 2143, 2148, 2144, 2152, 3528, 534, 3546, 2155, - 534, 534, 534, 1893, 2158, 2157, 2160, 1828, 2181, 1253, - 2159, 1827, 2201, 2202, 2228, 2204, 2229, 2235, 534, 730, - 1254, 1829, 2247, 532, 2249, 534, 1650, 1894, 534, 1243, - 2248, 2250, 1255, 2251, -2180, -2180, -2180, 893, 1994, 1995, - 1996, 1997, 1998, 1999, 534, 2255, 1597, 2265, 534, 2277, - 532, 3546, 33, 2268, 2278, 2282, 1895, 2279, 534, 2280, - 2281, 1829, 829, 2299, 1256, 2303, 532, 2304, 892, 2314, - 532, 2307, 2319, 2318, 2791, 534, 534, 2320, 829, 2328, - 3562, 1589, 829, -657, 2346, 2347, 822, 2351, -657, 1824, - -1899, 38, 534, 2388, 534, 2352, 2389, 894, 2390, 2412, - 3342, 1828, 822, 1037, 2402, 2406, 822, 534, 1038, 2422, - 2423, 2426, 2424, 2425, 2427, 550, 1941, 2441, 2445, 1258, - 831, 730, 2704, 2446, 2448, 2487, 40, 2489, 2488, 1824, - 730, 730, 730, 2490, 2491, 2493, 831, 43, -1899, 893, - 831, 2342, 2342, 2342, 1259, 2506, 2007, 2497, 2514, 1982, - 2544, 2006, 2551, -1899, 44, 1829, 2552, 2553, -1899, 1597, - -657, 2557, 2573, -1899, 2561, 1261, 2562, 1039, 2583, 2585, - 2597, 2599, -1899, 1669, 2604, 534, 1670, -1899, 45, 551, - 1671, 1672, -582, 2605, 1330, -2180, -2180, -2180, 2606, 2627, - 2614, 2608, 3009, 2609, 1597, 2622, 1463, -582, 2610, 2615, - 2611, 2626, -582, 3529, 2623, 2629, 3530, 1642, 2628, -1899, - 2630, -657, 1680, 2631, 2632, 1899, 2633, 730, 2634, 1681, - 894, 2640, 1600, 1824, 1853, 2644, 2651, 1900, 2827, 1902, - -1899, 2654, 3499, 1903, 2652, 2659, 2683, 2666, 2675, 2676, - 1040, 2681, 2682, 2694, 14, 15, 1682, 2701, 1221, 2699, - 2702, 2708, 2710, -582, 2724, 553, 2725, 2711, -719, 2728, - 1597, 2721, 2732, 2722, 2733, 1798, 1597, 2736, 2738, 1589, - 2740, 2744, 2741, 2765, -582, 2767, 1041, 2782, 1600, 2818, - 2788, -1899, 534, 3712, -1899, 2780, 2789, 2819, 2824, 1597, - -1899, 23, 1042, 2781, 2790, 2802, 2803, 2817, 730, 1661, - 2837, 2841, 2838, 1043, 2845, 2842, 2852, 2864, 2844, 1954, - 2879, 2848, 1824, 2871, 1339, 2882, 2885, 2888, 2975, 2976, - 2889, -223, 2890, 2891, 2902, -582, 2905, 2903, 2916, 2906, - 2917, -1899, 2930, 1597, -582, 2941, 534, 2922, 1074, 1044, - 2948, 1982, 2938, 2969, 1683, 1983, 1984, 2949, 2967, 2970, - 1985, 1986, 1987, 2973, 2982, 2988, -1899, 2983, 532, 2999, - 1222, 1684, 2990, 2994, 3002, 3022, 1685, 555, 3026, 556, - 829, 3024, 3034, 3040, 2774, 895, 3067, 3000, 534, 3083, - 3181, 2163, 3186, 3020, 3047, 3551, 1045, 3553, 3041, 3154, - 1591, 3155, 3156, 1046, 1600, 558, 3190, 1774, 3160, 3164, - 3165, 884, 3561, 3180, 3202, 1688, 1463, 3203, 3207, 2405, - 3212, 534, 3218, 3213, 3238, 3241, 871, 3245, 534, 534, - 3249, 3259, 3260, 3303, 3263, 2985, 3264, 3289, 831, 3296, - 3299, 3300, 3310, 2809, 534, 1047, 884, 3690, 26, 27, - 28, 3563, 3312, 3565, 3321, -1899, 1597, 534, 1175, 3317, - 534, 3322, 534, 3323, 1048, 3329, -1899, 3331, 3330, 1600, - 534, 3335, 1691, 534, 534, 3339, 3651, 3340, 534, 534, - 3341, 3349, 3352, 3354, 3355, 534, -1899, 3359, -1899, -1899, - 3373, -2145, -2146, -2147, 550, 3374, 3390, 3392, 3371, -582, - 3028, 3655, 534, 3370, 3407, 3416, -2148, 3412, -2149, -2150, - 3417, -2151, 3375, 534, -2152, 33, 3011, -2153, -2154, -1411, - 3376, -2155, 3425, 3640, -2156, -1899, 35, -2158, -1899, -1899, - -1899, -2159, -2160, 534, -2161, 3635, -2162, 1049, -2163, -2164, - 3211, 2302, 1223, -2165, -2167, -2168, -2169, 2869, 3426, 2870, - 37, -2170, -2171, 2875, 38, 2878, -2172, -2173, 551, -2174, - -2175, -2176, -2177, -2178, 1989, 3437, -2179, 3443, 3378, 927, - -1364, 3379, 1194, 928, 3393, 1693, 730, 3388, 3409, 3135, - 730, 3394, 730, 534, 534, 3420, 3410, 2342, 1591, 40, - 3429, 3137, 3431, 2348, 3455, 3433, 3445, 3439, 534, 534, - 43, 3440, 1589, 3444, 3448, 3451, 3473, 3477, 3487, 3452, - 3471, 534, 3031, 3489, 986, 3032, 3472, 44, 884, 3484, - 3501, 3502, 1600, -1363, 3509, 3511, 3517, 3519, 3520, 3533, - 534, 3167, 3205, 3534, 3536, 3548, 3549, 1990, 1899, 3552, - 3555, 45, 3556, 3558, 3084, 3570, 3564, 3581, 3593, 3348, - 1900, 3583, 1902, 1463, 1463, 46, 1903, 3597, 2486, 1463, - 3600, 3141, 3142, 3143, 3144, 3195, 3146, 3147, 3148, 3149, - 3150, 3599, 3603, 3609, 3626, 3616, 3617, 3618, 3679, 884, - 3633, 3628, 3621, 3630, 3292, 534, 3634, 1694, 1600, 3637, - -2180, -2180, -2180, 554, 1698, 1699, 1700, 1701, 1702, 1703, - 1589, 1397, 3639, 3644, 3646, 534, 534, 3638, 534, 3648, - 3649, 3652, 534, 3344, 3662, 534, 3657, 3668, 3669, 3670, - 3675, 3676, 3695, 3714, 3677, 3717, 3685, 530, 541, 3719, - 3738, 3739, 3687, 565, 3689, 3735, 3748, 3715, 1597, 565, - 3716, 3762, 534, 819, 3767, 834, 555, 3781, 987, 837, - 565, 846, 3772, 3774, 846, 3776, 3786, 866, 866, 3799, - 3797, 866, 534, 3793, 565, 565, 3798, 534, 534, 3800, - 1197, 557, 534, 1600, 558, 2753, 3082, 534, 2354, 884, - 534, 534, 3086, 2763, 3577, 534, 1330, 3683, 3015, 534, - 3643, 3773, 3395, 534, 3362, 2499, 819, 819, 2866, 3666, - 3755, 1037, 3497, 3720, 1173, 534, 1038, 3713, 3749, 1813, - 3532, 78, 2743, 2102, 2770, 532, 3128, 2098, 2105, 2324, - 2100, 2101, 866, 2104, 3711, 929, 1249, 829, 2325, 866, - 565, 866, 866, 866, 3718, 3185, 3051, 3138, 3709, 1936, - 1589, 2739, 3708, 1667, 3779, 3481, 1250, 2858, 1896, 2860, - 534, 822, 3163, 2809, 2813, 1991, 1992, 1993, 534, 1994, - 1995, 1996, 1997, 1998, 1999, 1039, 2727, 1489, 3199, 3140, - 3421, 1735, 1487, 1488, 1397, 2713, 1669, 534, 2306, 1670, - 1778, 1777, 2893, 1671, 1672, 831, 1600, 3688, 3191, 1517, - 2723, 3629, 1251, 3073, 2274, 3554, 2305, 823, 2698, 1782, - 3248, 2589, 3447, 3074, 1454, 2887, 1074, 2886, 2919, 1982, - 3503, 1591, 2603, 1983, 1984, 1680, 3632, 2123, 1985, 1986, - 1987, 3135, -2180, 1438, 2124, 2125, 1440, 1444, 1445, 2126, - 2127, 2128, 1446, 3343, 3631, 2964, 1447, 2649, 1040, 2943, - 2716, 2439, 2619, 2673, 3470, 2671, 1589, 3302, 1448, 1682, - 2032, 1449, 2276, 1450, 2650, 2961, 1397, 3301, 3619, 1397, - 1397, 1001, 23, 3075, 3272, 1948, 1399, 730, 2400, 2596, - 3016, 2243, 1229, 2245, 1041, 2868, 2381, 0, 2827, 0, - 730, 0, 0, 0, 0, 0, 3398, 1600, 0, 1252, - 1042, 2827, 0, 1600, 3399, 0, 0, 0, 884, 0, - 1589, 1043, 0, 534, 534, 0, 534, 23, 0, 1591, - 0, 0, 1589, 0, 1592, 534, 1600, 0, 0, 23, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, - 0, 534, 0, 0, 3298, 0, 0, 1044, 0, 0, - 1253, 0, 1589, 0, 0, 0, 0, -2180, 0, 1589, - 0, 1254, 0, 0, 0, 0, 0, 0, 2050, 0, - 1600, 0, 0, 1255, -2180, 3076, 534, 0, 0, -2180, + 536, 934, 79, 53, 72, 887, 1194, 70, 1162, 732, + 71, 96, 1270, 1230, 1476, 101, 536, 57, 1290, 66, + 1861, 1056, 79, 84, 105, 2277, 983, 1862, 1402, 873, + 1533, 1385, 1348, 534, 731, 799, 1345, 1446, 1391, 1744, + 2244, 871, 900, 2039, 2014, 1845, 2513, 904, 2149, 534, + 793, 905, 1449, 2109, 2396, 1450, 1643, 533, 1469, 569, + 2767, 2077, 1834, 2161, 2405, 763, 985, 2089, 2200, 1448, + 536, 536, 1880, 832, 1828, 2179, 841, 2719, 2729, 3183, + 2328, 758, 2610, 1803, 50, 825, 2699, 51, 2701, 52, + 884, 884, 2177, 2178, 3180, 3150, 885, 2680, 3148, 2348, + 55, 2718, 1866, 534, 534, 924, 56, 915, 3197, 59, + 866, 1946, 1290, 2758, 60, 993, 1721, 61, 1657, 3201, + 2825, 1724, 2965, 1034, 2869, 2248, 64, 832, 832, 1059, + 67, 2423, 68, 1002, 1415, 69, 82, 2384, 83, 825, + 825, 2422, 85, 2427, 2591, 2592, -1428, 3038, 2392, 86, + 834, 3043, 2581, 2582, 2583, 2598, 1619, 1620, 920, 2602, + -505, 1080, 1080, 3280, 1269, 92, 1275, 2834, 1279, 3283, + 93, 95, -2021, 2202, 1633, 100, 104, 106, 2560, -926, + 548, 2818, -395, 836, -2021, -572, -1369, 1379, 1081, 2714, + 2032, 3635, -2169, -2169, -931, 2033, 1081, 2829, -931, -1369, + -1350, 1729, 1460, -2012, 834, 834, -2012, -1366, -1366, -934, + -2029, -2160, -2160, 2835, -576, 2125, 2843, 2339, -2178, -2178, + 3416, -934, -2155, -2155, 2806, 2297, -1370, 2809, 937, 2300, + -2029, -1370, 836, -893, -1367, -1367, 1080, -906, 1003, -921, + 2827, -1189, 836, 836, 1824, 3182, -535, 3756, 1653, -1189, + 3475, 2768, -2180, -2180, 938, 1864, 1395, 1878, 1725, 1395, + 836, 1080, 2630, 1290, 1080, 1677, 1281, 877, 1879, 2192, + 1679, 1991, 3507, 1824, 2972, 2192, 1993, 3350, 1804, 1219, + 901, 902, 1664, 895, 2797, 1846, -572, 3365, 1849, 1850, + 1370, 1028, 2419, 1752, -249, 1182, 2231, 3383, 549, -249, + 1222, 1687, 2885, 2887, 2232, 2890, 2714, 3592, 1624, 1379, + 1059, 552, 1766, 1493, 3172, -576, 2310, 2926, 1500, 3400, + 1503, 939, 1034, 2456, -1210, 1635, 3333, 3, 4, 1371, + -721, 2031, -1210, 897, 1181, 1689, 3403, 1284, 1179, 1528, + 2852, 836, 1026, 2241, 1991, 1881, 2570, 3520, 1992, 1993, + 940, 1409, 1035, 897, 896, 3705, 3420, 2761, 1802, 836, + 3639, 1767, 2769, 3712, 3729, 1002, 3551, 3202, 2083, 2031, + 1831, 1851, 3004, 1726, 3006, 553, 3552, 2528, 974, 3777, + 1798, 1799, 2896, -863, 854, 3481, 3796, 3483, 961, 2534, + 3347, 3053, 3361, 2272, 941, 1748, 1648, 3082, 3622, 3714, + 1215, 1209, 2571, 2303, 1812, 1856, 3359, 3698, 1629, 1677, + 3715, 1638, 2691, 1678, 1679, 2907, 3031, 2587, 3586, 2562, + 3587, 3318, 3716, 1677, 2333, 3603, 2567, 962, 3659, 1857, + 1768, 1520, 1677, 1639, 1012, 897, 1678, 1679, 3410, -2154, + -2154, 3382, 1885, 546, 3493, 1687, 891, 3775, 1958, 1927, + 891, 1991, 1041, 1080, 3425, 1992, 1993, 2588, 3411, 1687, + 1832, 3319, 1649, 1677, 2838, 1183, 3348, 1184, 1687, 1348, + 2603, 3494, 3083, 3052, 2603, 1763, 1560, 1217, 2273, 1689, + -572, 1769, 3360, 1521, 3032, 2334, 855, 2203, 1379, 1380, + 3604, 3717, 3737, 1689, 2762, 3685, 1036, 2242, 3688, 1727, + 1815, 3641, 1689, 1210, 2624, 3772, 3810, 1989, 1990, -576, + 1346, 3220, 1372, 2010, 1882, 3706, 2204, 965, 3564, 1852, + -863, 2304, 1243, 1246, 942, 2957, 3736, 1643, 1244, 3034, + 1853, 3518, 2758, 1689, 2758, 943, 1285, 3203, 892, 2897, + 3593, 3778, 892, 804, 3707, 3553, 3713, 1770, 1175, 3532, + 2572, 3621, -572, 3797, 949, 3502, 2612, 3362, 2420, 3084, + 1499, 1749, 1637, 557, 1884, 992, 1960, 3309, 3536, 3311, + 803, 1080, 944, 2649, 3512, 1833, 2811, 2812, 2813, 893, + 3233, -576, 1833, 1805, 2080, 3173, 1764, 2770, 3397, 2771, + 898, 560, 2819, 2820, 2200, 1009, 945, 3764, 732, 1668, + 2311, 1865, -475, 3210, 3770, 1384, 2023, 3508, 2119, 2606, + 1800, 1380, 969, 1619, 1620, -1189, 3193, 3422, 1871, 2772, + 878, 2411, 3492, 971, 2149, 946, 3351, 3496, 2404, 3724, + 2798, 1801, 1373, 1245, 2205, 1739, 932, 3165, 1633, 2408, + 933, 3760, 838, 983, 2207, 2926, 2040, 2193, 1948, 3211, + 3686, 1615, 550, 2682, 1952, 1381, 3187, 3757, 2652, 3580, + 3528, 835, 1747, 3224, 1817, 1751, 2839, 2805, 2064, 2703, + 2233, 2721, 3374, 3375, 2836, -572, 1862, -1428, 2984, 2940, + 2846, 2507, 2920, 2908, 2909, 2910, 2911, 2247, 2727, 1949, + 3150, -505, -505, 3148, 2894, -2021, 1638, 975, -1210, 2202, + 2131, 2132, -926, 1630, -576, 3730, 2821, -2021, 1638, -1369, + 1344, 1124, 1125, -395, -572, 1630, -572, -931, 1639, 1124, + 1125, 932, -1369, -1350, 3378, 933, -2012, 887, 1384, -2012, + 1639, 2297, 1640, -2029, 2732, 2810, 929, 1621, 2116, 1653, + 3718, 1956, 3499, -576, 1642, -576, 3464, 3500, 1653, -1370, + 1290, 2173, 1290, -2029, -1370, 1672, 1622, 2825, 3476, 1002, + 1627, 1001, -921, 2152, 3341, 1722, 2249, 1022, 1624, 79, + 1185, 1399, 1400, 3620, 1399, 1400, 3673, 1384, 1710, 887, + 1856, 2054, 799, 3687, 2008, 2029, 1818, 894, 1635, 932, + 1380, 2625, 3627, 1665, 1246, 2174, 3442, 3628, 998, 2533, + 1243, 2208, 536, 2071, 1857, 3025, 1244, 988, 2333, -249, + -249, 3765, 2209, 536, 1176, 3414, 2522, 116, 1040, 1079, + 536, 2523, 1991, 2107, 2404, 880, 1992, 1993, 3018, 107, + 2561, -2183, -2183, -2183, 2789, 534, 3444, 2707, 2771, 2339, + 904, 3720, 1638, 951, 905, 952, 534, 2790, 3062, 536, + 536, 3655, 3656, 534, 2005, 2006, 2007, 2008, 1248, 832, + 3417, 1766, 3332, 2708, 1639, 3721, 1180, 3415, 3095, 2755, + 832, 1231, 2568, 536, 3063, 3418, 2372, 832, 1640, 3005, + 108, 2524, 1024, 3696, 3019, 1180, 2375, 3180, 889, 2378, + 1249, 1177, 79, 53, 72, 2870, 110, 70, 3228, 3766, + 71, 96, 117, 1331, 1074, 101, 3700, 57, 1186, 66, + 1767, 1245, 1064, 84, 105, 3674, 111, 2568, 3334, 1707, + 1708, 1709, 1710, 1065, 3519, 536, 732, 884, 3767, 1337, + 536, 1947, 1195, 2371, 866, 866, 834, 866, 1824, 866, + 1705, 1706, 1707, 1708, 1709, 1710, 2541, 834, 3411, 1825, + 1215, 953, 874, 954, 834, 1216, 2195, 1505, 3401, 2003, + 2004, 2005, 2006, 2007, 2008, 1250, 2730, -741, 2824, 1824, + 2692, 1080, 3150, 2927, 50, 3148, 890, 51, 2497, 52, + 1827, 906, 1337, 3290, 2433, 3166, 3167, 2397, 2398, 2399, + 55, 2506, 111, 1001, 2575, 2508, 56, 887, 2510, 59, + 536, 536, 3675, 1391, 60, 79, 536, 61, 118, 536, + 536, 2936, 536, 536, 536, 536, 64, 3798, 799, 1347, + 67, 3050, 68, 1025, 907, 69, 82, 1217, 83, 536, + 1769, 536, 85, 2535, 2536, 2537, 2538, 2539, 2540, 86, + 536, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, + 2553, 3099, 2149, 2325, 909, 92, 3801, 536, 1337, 1607, + 93, 95, 534, 3505, 534, 100, 104, 106, 928, 1256, + 3159, 3419, 1246, 534, 2765, 3785, 3066, 1215, 1218, 536, + 1631, 1632, 887, 3792, 2975, 2370, 832, -659, 832, 1257, + 112, 932, -659, 2542, 1045, 933, 1770, 832, 825, 536, + 1024, 113, 2704, 2543, 2705, 2381, 3367, 927, 1999, 825, + 2388, 536, 536, 536, 3372, 536, 536, 1243, 732, 3069, + 836, 2373, 1900, 1244, 1668, 2014, 2376, 2039, 1862, 23, + 1046, 3488, 1247, 2215, 3802, 1258, 1248, 1638, 114, 959, + 2117, -2149, -2149, 1658, 3056, 1835, 1506, 2814, 2738, 2123, + 1243, 536, 2566, 3616, 1217, 1393, 1244, 1048, 1394, 1639, + 1947, 1901, -2021, 834, -659, 834, 112, 552, 1249, 536, + 536, 2521, 3300, 1642, 834, 2525, 1505, 113, 2527, 1754, + 1755, 932, 1761, 905, 905, 1665, 905, 3146, 930, 2672, + 1508, 115, -1414, 936, 1180, 2235, 1515, 1180, 3405, 2236, + 948, 2727, 1432, 1433, 1737, 1218, 536, 1738, 2799, 3537, + 536, 536, 960, 1836, 114, -659, -2150, -2150, 2673, 3599, + 536, 536, 536, 1953, 552, 536, 1954, 3436, 1245, 3803, + 2456, 553, 1259, 3057, 2667, 2668, 932, 1657, -1350, 1496, + 933, 1025, 884, 1250, 2628, 2022, 955, 2024, 2025, 1607, + 1512, 1904, 1169, 1170, 967, 1172, 2079, 1174, 2216, 2080, + 3396, 1245, 2853, 1863, 3538, 1513, -225, 115, 2105, 1518, + 972, 2106, 2864, 3539, 973, 2217, 26, 27, 28, 2416, + 2218, 3510, 2417, 1260, 958, 1337, 3073, 991, 553, 1052, + 1068, 1069, 1070, 976, 1261, 1073, 1337, 3540, 1613, 977, + 46, 2927, 3058, 1837, 3059, 978, 1262, 2516, 1507, 2554, + 2517, 979, 2555, 1045, 961, 3342, 2448, 2604, 2449, 2219, + 2605, 1337, 1438, 1439, 2670, 120, -2183, -2183, -2183, 547, + 2003, 2004, 2005, 2006, 2007, 2008, 3074, 762, 1263, 2607, + 989, 2674, 2605, 33, 2785, 2791, 2675, 2786, 2792, 1046, + 3391, 853, 1007, 962, 3075, 867, 2700, 1838, 1009, 2807, + 2867, 732, 2517, 2868, 1008, 1514, 556, 1908, 1010, 1909, + 732, 3541, 1013, 1911, 1912, 3276, 1048, 2933, 959, 2824, + 2605, 2576, 38, 2577, 3542, 1016, 1941, 1001, 3804, 1246, + 963, 1017, 2779, 1265, 2781, 79, 2934, 2937, 2939, 2106, + 2938, 2938, 3026, 3190, 732, 3027, 3191, 3268, 799, 536, + 3270, 836, 3272, 3192, 964, 1018, 2417, 40, 1266, 557, + 1005, 992, 1246, 3230, 998, 1833, 3231, 1019, 43, 1963, + 1950, 3291, 1951, 3386, 2106, 3229, 2275, 2921, 2578, 1268, + 2579, 2928, 1020, 965, 559, 2149, 895, 560, 2782, 1273, + 2784, 960, 1063, 1248, 3049, 1021, 3051, 2109, 3076, 536, + 536, -660, 1212, 2741, 1214, 536, -660, 536, 3077, 1022, + 2658, 1027, 536, 536, 536, 536, 557, 1171, 992, 2220, + 3794, 3437, 1277, 46, 2106, 1274, 1248, 536, 536, 3438, + 3470, 1029, 2605, 2106, 1061, 2870, 1066, 536, 3477, 536, + 534, 2080, 536, 2676, 560, -2151, -2151, 536, 1052, 536, + 536, 1071, 536, 1072, 2677, 2833, 536, 896, 1278, 3489, + -2152, -2152, 3490, 3086, 832, 3526, 838, 1507, 3191, 3093, + 534, 1075, 534, 14, 15, 534, 3739, 1074, -660, 3773, + 534, 3774, 1173, 534, 3527, 534, 1196, 2417, 1189, 534, + 1835, 3751, 3561, 1900, 832, 2106, 832, 3726, 3625, 832, + 1250, 3191, 3637, 1197, 832, 3638, 825, 832, 825, 832, + 1199, 825, 1607, 832, 3667, 3176, 825, 3668, 1203, 825, + 23, 825, -2156, -2156, 3085, 825, 3094, 2958, 2959, -660, + 1200, 3543, 1901, 1250, 3544, 1201, 2339, 1213, 897, 1204, + 3809, 834, 2456, 1206, 3711, 536, 536, 3638, 3618, -2157, + -2157, 79, 2108, 2112, 536, 536, 2110, 1401, 1836, 2111, + 2114, -2158, -2158, 536, 2115, 3805, 1232, 1444, 1235, 1902, + 536, 834, 3771, 834, 1237, 3638, 834, 536, 3808, 3806, + 1238, 834, 3490, 1239, 834, 1240, 834, 2944, 1657, -1902, + 834, 3582, 3583, 1903, -2159, -2159, 1242, 536, 732, -2161, + -2161, 116, 536, 2946, 2948, 536, 2947, 2949, 2950, 3463, + 1607, 536, 536, 536, 536, 536, 536, 536, 536, 732, + 2945, 1860, 1904, 536, 536, 536, -2162, -2162, 536, 1331, + 1241, 552, 536, -2163, -2163, 536, 536, 536, 536, 536, + 536, 536, 536, 536, 2189, 1254, 536, 2055, 1837, 2056, + -2164, -2164, 2058, 536, 1255, 1337, -1414, 2062, 534, 1272, + 2065, 1276, 2066, 2988, 2989, 1835, 2070, 26, 27, 28, + 2279, -2165, -2165, 536, 905, 1424, 1425, -2166, -2166, -2167, + -2167, -2168, -2168, -2170, -2170, -2171, -2171, 2298, 2299, 1670, + -1902, 1835, 1280, 898, 1282, 553, -2172, -2172, 536, 1673, + 1340, 1243, 1838, -2173, -2173, 1344, 2133, 1244, 1186, 536, + 536, -716, -716, 2134, 2135, 1343, 2880, 1349, 2136, 2137, + 2138, 1723, 3146, -2174, -2174, -2175, -2175, -2177, -2177, 1354, + 1728, 1243, 1366, 1836, 33, 1369, 2727, 1244, -1902, -2179, + -2179, -2182, -2182, 2173, 1368, 35, 2244, 1376, 1432, 1433, + 2203, 2432, 1607, -1902, 1887, 1888, -720, -720, -1902, 1836, + -719, -719, 732, -1902, 1434, 1435, 732, 1438, 1439, 37, + 1833, 1387, -1902, 38, 3214, 3215, 1377, -1902, 1383, 2204, + 1386, 1079, 3338, 3339, 1991, 3758, 3761, 2354, 1992, 1993, + 1392, 2360, 3093, 1994, 1995, 1996, 1867, 2872, 2874, 1410, + 2329, 3747, 3748, 3783, 3784, 1733, 1734, 732, 40, -1902, + 536, 2343, 1245, 2346, 2746, 2747, 2357, 1337, 1411, 43, + 536, 536, 1870, 1837, 1416, 1474, 2365, 1487, 2367, 552, + -1902, 1489, 1963, 1490, 3042, 1501, 44, 1497, 1872, 1509, + 1510, 2374, 1245, 1760, 1516, 1517, 2377, 1527, 1523, 1837, + 2382, 2383, 1529, 2385, 3184, 2389, 2390, 2497, 1607, 3153, + 45, 2083, 2498, 3171, 1609, -897, 3223, 2452, 1438, 1439, + 1610, 1612, -904, 1243, 46, 1621, 3225, 1838, 1625, 1244, + 732, -1902, 46, 557, -1902, 992, -741, 1337, 536, 3208, + -1902, -742, -894, 553, -895, 1636, 536, 2205, -898, 1637, + 3738, 3302, 2206, 1838, 3740, -896, 1644, 2207, 559, 1669, + 1659, 560, 1607, 1835, 1716, 1671, 536, 536, 1718, 536, + 1720, 1732, 1740, 1741, 1607, 536, 536, 536, 536, 536, + 536, -1902, 1745, 536, 536, 536, 536, 536, 536, 536, + 536, 536, 536, 1750, 3560, 1833, 1753, 1216, 536, 536, + 1788, 1218, 536, 1790, 1607, 1792, -1902, 23, 1808, 536, + 1810, 1607, 1829, 1830, 1840, 1841, 1842, 1962, 1847, 555, + 1854, 1833, 1855, 1246, 1997, 1877, 1859, 1869, 3799, 1890, + 1835, 1836, 1874, 536, 1245, 1891, 1079, 1896, 1899, 1991, + 536, 1906, 536, 1992, 1993, 2726, 536, 1998, 1994, 1995, + 1996, 1892, 1913, 1246, 3146, 1607, -585, 1907, 1914, 1607, + 1917, 536, 1923, 1337, 11, 1607, 874, 1920, 2727, 1921, + 1924, -585, 1925, 534, 1926, 1928, -585, 1929, 1346, 534, + 1942, 1947, 1943, 2324, 1955, -224, 1983, 1248, 2019, 1607, + 1980, 1982, 14, 15, 2796, -1902, 1985, 832, 1836, 2030, + 3358, 536, 536, 832, 3370, 2050, -1902, 1988, 536, 825, + 2011, 1404, 2020, 2379, 2208, 825, 2057, 1248, 2027, 1249, + 1999, 1837, 2052, 2067, 2053, 2209, -1902, -585, -1902, -1902, + 2063, 557, 1604, 558, 3379, 3380, 2068, 2069, 2075, 23, + 3524, 536, 1622, 2081, 2078, 536, 1615, 887, -585, 2380, + 536, 536, 1627, 3560, 26, 27, 28, 2082, 2084, 560, + 2085, 2800, 2088, 2086, 2119, -1902, 2118, 1080, -1902, -1902, + -1902, 1677, 2153, 1835, 834, 1838, 536, 536, 2158, 2154, + 834, 536, 2162, 2165, 2168, 2167, 2169, 3267, 1837, 2191, + 2039, 2211, 2170, 2214, 1250, 1246, 2238, 536, 3560, -585, + 536, 536, 536, 2212, 2239, 2257, 2245, 2259, -585, 2258, + 2260, 2261, 2265, 896, 2275, 2278, 2287, 2288, 536, 732, + 2289, 33, 2292, 2290, 1250, 536, -1904, 1657, 536, 2312, + 2291, 2313, 1042, 2309, 534, 2317, 2314, 1043, 2322, 1730, + 2326, 1836, 1838, 1833, 536, 2330, 2331, 2332, 536, 2340, + 2593, 3560, 1998, 1674, 897, 2386, 2596, 3435, 536, 1248, + 38, 534, 2358, 2139, 2140, 2141, 2359, 2142, 2143, 2144, + 2145, 2146, 2147, 2400, 2363, 536, 536, 534, 2364, 2401, + 2414, 534, 2418, 2424, 1404, 832, 26, 27, 28, 3576, + 2402, 2387, 536, 1950, 536, 40, 1044, 825, 2434, 2435, + 2438, 832, 2436, 2437, 2804, 832, 43, 2439, 2453, 536, + 1833, 2457, 2458, 825, 2501, 2460, 2499, 825, 2500, 2503, + 2502, 2505, 1604, 732, 3356, 1999, 2518, -1904, 2509, 2526, + 2016, 1837, 732, 732, 732, 1991, 2556, 2250, 2000, 2001, + 2002, 2015, 2003, 2004, 2005, 2006, 2007, 2008, 2563, 2564, + 2585, 2597, 2573, 33, 2565, 2569, 2609, 2354, 2354, 2354, + 2574, 46, 834, -585, 35, 2595, 1250, 2611, 1404, 1045, + 2616, 1404, 1404, 2617, 3020, -1904, 2618, 536, 834, 2627, + 2626, -585, 834, 2634, 2635, 1838, 1337, 23, 37, 2620, + -1904, 2638, 38, 2639, 2621, -1904, -585, 2622, 2640, 2623, + -1904, -585, 2643, 2641, 2642, 1046, 2644, 2645, 2656, -1904, + 2646, 2319, 2321, 39, -1904, 1676, 1862, 1649, 1677, 732, + 2652, 1047, 1678, 1679, 1607, 2666, 2663, 40, 2664, 1908, + 23, 1909, 1048, 932, 2671, 1911, 1912, 933, 43, 2687, + 3513, 2678, 2711, 2688, 2840, 2695, -1904, 2693, 2714, 2694, + 2706, 2713, -585, 1833, 1687, 44, 2737, 2740, 2716, 2720, + 2723, -2183, -722, 2722, 2733, 2734, 2736, -1904, 1049, 2744, + 2745, 3066, 2751, -585, 1805, 2748, 2749, 3067, 2391, 45, + 1607, 2757, 2754, 2778, 536, 2780, 2795, 2857, 1689, 1229, + 3068, 2753, 2793, 46, 2801, 2794, 2409, 2409, 2802, 1668, + 732, 2803, 2815, 2816, 2830, 2831, 2832, 2855, 2837, 2850, + 1959, 1961, 2854, 2851, 3069, 1050, 3070, 2858, -1904, 2861, + 2865, -1904, 1051, 1042, -585, 1963, 1833, -1904, 1043, 2877, + 2884, 2892, 2895, -585, 26, 27, 28, 2898, 536, 2901, + 2915, 2902, 2916, 2000, 2001, 2002, 2903, 2003, 2004, 2005, + 2006, 2007, 2008, 2918, 1596, 2904, 2929, 2930, 2919, 3565, + 2935, 3567, 2173, 2954, 1052, 3021, 2943, 2951, -1904, 2982, + 2983, 534, 2962, 2961, 2980, 2995, 2520, 26, 27, 28, + 536, 2986, 3003, 1053, 2996, 3001, -2183, 1044, 3007, 3012, + 3013, 3015, 3033, -1904, 3035, 832, 1607, 3575, 3037, 3039, + 3048, 33, 887, -2183, 3054, 3071, 1180, 3055, -2183, 3061, + 3081, 3168, 3169, 536, 3097, 3170, 3178, 3174, 3179, 3194, + 536, 536, 3195, 3200, 3204, 3216, 3217, 2417, 3226, 3221, + 3227, 3252, 3232, 3255, 3259, 3263, 536, 887, 3704, 3577, + 38, 3579, 3273, 3274, 33, 1604, 3277, -2183, 3278, 536, + 2998, 3317, 536, 874, 536, 3303, 1054, 3310, 3313, 3314, + 1045, 1607, 536, 3324, 3335, 536, 536, 3326, 3331, 3336, + 536, 536, 834, 3345, 3665, 40, 3072, 536, 3337, 3343, + 3349, 3073, -1904, 38, 3654, 3353, 43, 3669, -585, 3354, + 3363, 3041, 3355, -1904, 536, 3366, 1046, 3368, 3369, 3344, + 3373, -2148, 3384, 44, 1698, 536, -2149, -2150, 3385, 3024, + -2151, 3387, 1047, -1904, -2152, -1904, -1904, -2153, 40, -2154, + 3649, -2155, 3388, 1048, 3389, 536, -2156, 45, -2157, 43, + -2158, 3074, 3404, -2159, -2161, -2162, 3406, -2163, 2124, 3407, + -2164, 46, 3421, 1604, 1596, -2165, 44, -2166, 3390, 3075, + -2167, 2155, -1904, 2156, 3392, -1904, -1904, -1904, 932, 1049, + -2168, 1199, 933, -2170, -2171, -2172, -2173, -2174, -2175, 732, + 45, -2176, 3149, 732, -2177, 732, 536, 536, -2178, -2179, + -2180, 3044, 2175, -2181, 3022, 3046, 3424, -2182, -1367, 3426, + 3393, 536, 536, 3402, 2354, 3423, 3408, 3430, 3151, 3431, + 2360, 1256, 3434, 3439, 536, 3440, 1050, -2183, 3443, 3445, + 887, 3451, 3454, 1051, 3098, 1607, 3447, 3457, 3453, 3458, + 3459, 1257, 3466, 536, 3219, 3181, 3465, 3462, 3487, 3469, + 3485, 3155, 3156, 3157, 3158, 3486, 3160, 3161, 3162, 3163, + 3164, 3491, 3498, 1908, 3501, 1909, 532, 543, 3503, 1911, + 1912, 2498, 567, 3076, 3515, 1052, 3516, 3525, 567, 3209, + -1366, 3523, 822, 3077, 837, 3531, 3533, 1258, 840, 567, + 849, 887, 3534, 849, 1053, 3547, 869, 869, 536, 3550, + 869, 1607, 3548, 567, 567, 1604, 3562, 2296, 2296, 3563, + 3566, 3569, 3693, 3572, 3578, 3570, 3595, 3584, 536, 536, + 3597, 536, 3607, 3362, 3611, 536, 3613, 3614, 536, 3617, + 3623, 3630, 3631, 3635, 3640, 3632, 822, 822, 3642, -2183, + 3644, 3306, 3647, 3813, 3653, 3651, 1705, 1706, 1707, 1708, + 1709, 1710, 3648, 3652, 3660, 536, 3658, 3662, 3089, 3663, + 3666, 3671, 869, 3682, 3676, 3683, 3684, 1054, 3689, 917, + 869, 567, 869, 869, 869, 536, 3690, 3691, 3709, 1404, + 536, 536, 3728, 2787, 1259, 536, 1607, 3699, 3701, 1404, + 536, 887, 1404, 536, 536, 3731, 3703, 3733, 536, 1337, + 3749, 3752, 536, 3762, 3753, 3729, 536, 3730, 3786, 1676, + 3776, 1604, 1677, 3781, 3790, -1919, 1678, 1679, 536, 3788, + 3795, 1682, 1683, 1684, 3800, 79, 2108, 2112, 3812, 3807, + 2110, 3811, 1256, 2111, 2114, 1260, 934, 3814, 2115, 534, + 1202, 3096, 2822, 2766, 2366, 2776, 1261, 3100, 1687, 3591, + 3028, 1256, 1257, 3697, 3657, 1688, 3787, 3409, 1262, 2511, + 1178, 3376, 2879, 832, 536, 1604, 3680, 3769, 3511, 3727, + 3734, 1257, 536, 3763, 3546, 825, 1822, 1604, 2756, 2783, + 2336, 2337, 1689, 3199, 3725, 3732, 3065, 1596, 3152, 3723, + 1263, 536, 2752, 1945, 3722, 3793, 1079, 3495, 1258, 1991, + 1607, 2871, 2826, 1992, 1993, 2873, 1905, 1604, 1994, 1995, + 1996, 3213, 2739, 3177, 1604, 1494, -1919, 1258, 3087, 3154, + 1495, 1742, 2725, 2316, 3357, 2977, 1785, 2906, 3088, 3702, + 2735, 1524, 1784, 3643, 2284, 3149, 3568, 2315, 826, 2601, + 834, 1789, 2710, 3461, 1461, 1265, 2882, 3262, 2883, 2900, + 2899, 3517, 2888, 2932, 2891, 1445, 3646, 2615, 1604, 3645, + 2661, 1447, 1604, 1451, -1919, 1452, 1453, 1454, 1604, 1455, + 1266, 1456, 2956, 1457, 2685, 2728, 2631, 3484, 2683, -1919, + 1690, 732, 2974, 3316, -1919, 1596, 2662, 2041, 3089, -1919, + 2412, 1268, 1604, 3633, 732, 1259, 3315, 1691, -1919, 3286, + 1006, 1607, 1692, -1919, 2608, 2253, 2840, 1607, 1406, 3413, + 1957, 887, 3029, 552, 1259, 2255, 1496, 536, 536, 2840, + 536, 2619, 2881, 2393, 0, 1693, 1694, 0, 0, 536, + 1607, 1234, 0, 0, 0, -1919, 0, 0, -1414, 0, + 0, 1695, 0, 0, 0, 536, 1260, 0, 0, 0, + 0, 0, 0, 0, 0, 0, -1919, 1261, 0, 0, + 0, 0, 0, 0, 0, 1260, 0, 0, 0, 1262, + 0, 0, 0, 0, 1607, 0, 1261, 553, 0, 1079, + 536, 1696, 1991, 0, 1697, 73, 1992, 1993, 1262, 0, + 3090, 1994, 1995, 1996, 0, 0, 536, 536, 1698, 0, + 536, 1263, 536, 0, 0, 73, 0, -1919, 824, 0, + -1919, 0, 1998, 0, 0, 0, -1919, 0, 0, 0, + 1263, 0, 73, 0, 0, 0, 0, 1596, 0, 0, + 0, 886, 0, 554, 536, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1676, 0, 0, 1677, 0, 0, + 0, 1678, 1679, 555, 0, 0, 1265, -1919, 536, 0, + 0, 0, 824, 824, 903, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1265, 0, 0, 0, 0, + 2702, 1266, -1919, 1687, 0, 1999, 0, 3149, 73, 1676, + -2183, 0, 1677, 1966, 0, 0, 1678, 1679, 0, 0, + 1266, 1700, 1268, 0, 0, 0, 0, 0, 0, 0, + 869, 732, 556, 2731, 2731, 869, 3142, 1689, 3529, -224, + 0, 1268, 3514, 3091, 0, 567, 3092, 0, 1687, 0, + 0, 0, 0, 1596, 0, -2183, 2840, 536, 0, 0, + 1967, 0, 874, 3521, 3522, 0, 536, 0, 536, 0, + 536, 0, 0, 2822, 536, 0, 536, 0, 536, 0, + 0, 1968, 1689, 0, 0, 557, 0, 558, 3535, 0, + 534, -1919, 536, 0, 0, 0, 2451, 536, 536, 1969, + 0, 0, -1919, 2669, 1970, 0, 0, 1596, 3205, 536, + 559, 0, 0, 560, 832, 1998, 0, 23, 0, 1596, + 0, 3588, -1919, 3590, -1919, -1919, 732, 1971, 0, 0, + 1972, 536, 0, 1701, 0, -2183, 1702, 1703, 1704, 0, + 1705, 1706, 1707, 1708, 1709, 1710, 1973, 1604, 2679, 1596, + 0, 3600, -2183, 0, 0, 0, 1596, -2183, 1833, 0, + 0, -1919, 0, 0, -1919, -1919, -1919, 3626, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, + -2183, 0, 0, 0, 3629, 0, 0, 0, 1999, 536, + 0, 834, 0, 0, 0, 0, -2183, -2183, 0, 0, + 1596, 536, -2183, 1604, 1596, 0, 0, 0, 0, 0, + 1596, 0, 0, 0, 0, 0, 536, 0, 536, 0, + 0, 0, 0, 2000, 2001, 2002, 0, 2003, 2004, 2005, + 2006, 2007, 2008, 0, 1596, 0, 0, 536, 981, 567, + 567, -2183, 1974, 0, 0, 0, 0, 0, 0, 534, + 1975, 0, 0, 1698, 3312, 0, 0, 0, 0, 0, + 0, 536, 0, 0, 26, 27, 28, 0, 0, 0, + 0, 0, 1976, 832, 0, 0, 536, 0, 0, 0, + 0, 0, 1004, 543, 0, 0, 0, 0, 532, 3149, + 869, 732, 0, 0, 534, 0, 0, 0, 1698, 822, + 0, 0, 1977, 1031, 1031, 0, 822, 0, 0, 1031, + 1058, 0, 0, 0, 536, 0, 3600, 3735, 832, 1604, + 0, 0, 849, 849, 849, 0, 0, 849, 536, 536, + 536, 33, 0, 0, 3710, 1128, 1128, 849, 849, 536, + 849, 0, 849, 3750, 0, 0, 0, 534, 0, 0, + 834, 0, 0, 0, 869, 0, -2183, 0, 0, 0, + 567, 0, 0, 0, 536, 0, 0, 0, 2941, 2942, + 38, 832, 0, 869, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1604, 0, 0, 869, 837, 0, + 0, 0, 0, 0, 0, 834, 0, 0, 0, 0, + 0, -2183, 0, 0, 0, 40, 2000, 2001, 2002, 0, + 2003, 2004, 2005, 2006, 2007, 2008, 43, 0, 0, 3412, + 0, 0, 0, 536, 0, 0, 869, 1342, 0, 0, + 536, 0, 0, 44, 0, 0, 0, 1352, 0, 0, + 23, 869, 869, 869, 869, 869, 0, 0, 834, 0, + 23, 0, 0, 0, 0, 73, 0, 45, 1375, 735, + 0, 0, 0, 0, 0, 0, 0, 0, -2183, 0, + 0, 46, 0, 0, 0, 1705, 1706, 1707, 1708, 1709, + 1710, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1031, 1058, 0, 869, 0, 1604, 1468, 0, 0, 0, + 0, 0, 1031, 1031, 0, 0, 0, 0, 0, 567, + 0, 0, 0, -2183, 0, 822, 736, 822, 0, 0, + 1705, 1706, 1707, 1708, 1709, 1710, 822, 1676, 0, 0, + 1677, 0, 737, 0, 1678, 1679, 567, 0, 1604, 0, + 1404, 0, 0, 0, 0, 0, 0, 1676, 0, 0, + 1677, 0, 0, 1611, 1678, 1679, 0, 3482, 0, 1682, + 1683, 1684, 0, 0, 0, 0, 1687, 0, 0, 1596, + 0, 0, 0, -2183, 0, 0, 1685, 0, 1598, 0, + 0, 567, 738, 1599, 0, 3589, 1687, 26, 27, 28, + 0, 0, 739, 1688, 0, 0, 0, 26, 27, 28, + 1689, 0, 0, 0, 1604, 740, 567, 0, 0, 0, + 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1689, 0, 0, 0, 0, 1596, 0, 0, 0, 0, + 0, 3142, 0, 0, 0, 0, 0, 0, 1676, 742, + 0, 1677, 0, 0, 0, 1678, 1679, 0, 0, 0, + -2183, -2183, -2183, 0, 33, 0, 0, 0, 0, 1736, + 0, 0, 0, 0, 33, 0, 2987, 0, 0, 0, + 0, 3549, 73, 886, 0, 567, 567, 1687, 0, 0, + 0, 0, 869, 743, 1688, 869, 3269, 0, 744, 1604, + 0, 0, 0, 38, 0, 0, 0, 0, -2183, 0, + 0, 0, 0, 38, 0, 0, 1468, 1128, 1128, 0, + 0, 1689, 0, 0, 1011, -2183, 869, 0, 1690, 1809, + -2183, 869, 1821, 0, 0, 1023, 0, 0, 40, 0, + 0, 0, 1039, 0, 869, 1691, 0, 0, 40, 43, + 1692, 1596, 0, 0, 0, 0, 0, 0, 0, 43, + 0, 869, 0, 0, 555, 869, 44, 0, 0, -2183, + 745, 1873, 0, 1693, 1694, 1601, 44, 0, 1598, 0, + 0, 0, 3619, 1599, 0, 746, 0, 903, 0, 1695, + 45, 0, 0, 0, 0, 14, 15, 0, 0, 0, + 45, 0, 0, 0, 46, 73, 0, 0, 0, 0, + 0, 0, 0, 1604, 46, 0, 1596, 0, 0, 1690, + 747, 0, 0, 748, 0, 0, 1698, 0, 0, 1696, + 0, 0, 1697, 1893, 749, 869, 1691, 750, 0, 0, + 0, 1692, 23, 869, 0, 0, 1698, 1404, 1604, 1699, + 0, 0, 1404, 0, 0, 0, 1938, 751, 0, 0, + 0, 0, 0, 0, 0, 981, 0, 0, 0, 0, + 981, 752, 567, 567, 0, 567, 981, 0, 753, 754, + 1695, 0, 0, 0, 0, 0, 0, 0, 0, 755, + 0, 0, 0, 0, 0, 756, 0, 0, 0, 0, + 0, 1959, 1961, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1604, 0, 0, 0, 73, 0, + 1604, 0, 0, 3142, 757, 0, 0, 1596, 0, -2183, + 0, 0, 0, 0, 0, 0, 0, 1698, 0, 0, + 0, 0, 0, 1604, 0, 0, 0, 0, 0, 1700, + 0, 824, 0, 1023, 0, 1468, 1468, 0, 0, 0, + 0, 1468, 824, 532, 0, 1601, 0, 0, 0, 0, + 1596, 0, 0, 0, 0, 0, 1031, 0, 567, 2034, + 0, 0, 0, 1128, 1128, 0, 869, 1604, 0, 26, + 27, 28, 0, 822, 1042, 822, 0, 1614, 822, 1043, + 0, 0, 0, 822, 0, 1128, 822, 0, 822, 1626, + 0, 0, 822, 0, 567, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 534, 534, 1045, 0, 534, 0, 534, 0, - 0, 1046, 0, 1589, 0, 1256, 0, 1589, 1950, 1952, - 0, 0, 0, 1589, 0, 0, 0, 0, -2180, 26, - 27, 28, 0, 0, 0, 0, 0, 0, 0, 0, - 534, 0, 1989, 0, 0, 0, 1589, 0, 0, 0, - 0, 0, 0, 1047, 1074, 0, 0, 1982, 0, 0, - 0, 1983, 1984, 0, 534, 0, 1985, 1986, 1987, 1591, - 1258, 0, 1048, 0, 26, 27, 28, 0, 0, 0, - 0, 0, 0, 2965, 0, 1691, 26, 27, 28, 0, - 0, 0, 0, 3135, 0, 1259, 33, 1463, 1463, 1463, - 1463, 1463, 1463, 0, 0, 1463, 1463, 1463, 1463, 1463, - 1463, 1463, 1463, 1463, 1463, 1990, 1261, 730, 3077, 0, - 0, 3078, 1592, 3515, 0, 0, 2714, 0, 2827, 0, - 0, 0, 0, 0, 0, 38, 0, 1594, 0, 0, - 0, 33, 0, 534, 0, 1049, 0, 0, 0, 0, - 2051, 3500, 534, 33, 534, 0, 534, 0, 0, 0, - 534, 0, 534, 0, 534, 532, 0, 0, 0, 0, - 40, 23, 3507, 3508, 0, 1591, 0, 829, 534, 0, - 38, 43, 0, 534, 534, 0, 0, 0, -2180, 866, - 0, 0, 38, 0, 866, 534, 0, 3521, 44, 0, - 0, 0, 0, 0, 565, 0, 3574, 0, 3576, 0, - 0, 0, 730, 0, 0, 40, 0, 534, 0, 0, - 1037, 0, 45, 3586, 0, 1038, 43, 40, 0, 1591, - 0, 0, 0, 1824, 0, 831, 46, 0, 43, 0, - 0, 1591, 0, 44, 0, 0, 2114, 0, 0, 0, - 0, 0, 3612, 0, 0, 44, 3468, 0, 0, 2145, - 0, 2146, 0, 0, 0, 534, 0, 45, 0, 3615, - 1989, 1591, 0, 0, 0, 534, 1464, 0, 1591, 45, - 0, 46, 0, 0, 1039, 0, 0, 534, 0, 0, - 2165, 0, 0, 46, 0, 0, 0, 1463, 1463, 0, - -2180, 0, 534, 0, 534, 0, 3575, 1698, 1699, 1700, - 1701, 1702, 1703, 0, 0, 0, 0, 1595, 26, 27, - 28, 0, 1591, 534, 532, 0, 1591, 0, 0, 0, - 3128, 0, 1591, 1991, 1992, 1993, 829, 1994, 1995, 1996, - 1997, 1998, 1999, 1990, 2129, 2130, 2131, 534, 2132, 2133, - 2134, 2135, 2136, 2137, 0, 1591, 0, 1040, 0, 0, - 0, 0, 534, 0, 0, 0, 0, 0, 0, 532, - 3535, 0, 0, 0, 0, 3135, 0, 730, 0, 1589, - 0, 829, 0, 0, 0, 33, 0, 0, 3586, 0, - 0, 0, 0, 1041, 831, 2286, 2286, 976, 565, 565, - 534, 0, 3721, 0, 0, 0, 0, 0, 0, 1042, - 0, 0, 0, 1463, 534, 534, 534, 0, 0, 0, - 1043, 0, 532, 0, 38, 534, 0, 0, 3736, 0, - 0, 1249, 0, 0, 829, 1589, 0, 0, 0, 831, - 999, 541, 0, 3696, 0, 1592, 530, 0, 866, 0, - 534, 1250, 0, 0, 0, 0, 1044, 819, 0, 40, - 1594, 1026, 1026, 0, 819, 1397, 0, 1026, 1053, 0, - 43, 3605, 0, 0, 0, 1397, 0, 1224, 1397, 0, - 846, 846, 846, 0, 0, 846, 1464, 44, 0, 0, - 0, 0, 831, 1123, 1123, 846, 846, 1251, 846, 0, - 846, 0, 0, 1045, 0, 1595, 0, 0, 0, 534, - 1046, 45, 866, 0, 1669, 0, 534, 1670, 565, 0, - 0, 1671, 1672, 0, 0, 46, 0, 72, 0, 0, - 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1592, 0, 866, 834, 72, 0, 0, - 821, 1589, 1047, 1680, 1249, 0, 0, 0, 1594, 0, - -2180, 1991, 1992, 1993, 72, 1994, 1995, 1996, 1997, 1998, - 1999, 1048, 0, 883, 1250, 0, 0, 0, 1669, 0, - 0, 1670, 866, 1335, 0, 1671, 1672, 1682, 0, 0, - 1675, 1676, 1677, 1345, 1252, 0, 0, 866, 866, 866, - 866, 866, 0, 821, 821, 900, 0, 1678, 0, 0, - 0, 0, 3128, 0, 1368, 0, 1589, 1680, 0, 0, - 1251, 0, 0, 0, 1681, 0, 0, 0, 0, 72, + 0, 0, 0, 0, 0, 0, 1596, 0, 1655, 0, + 1700, -2183, 0, 0, 0, 0, 0, 0, 1705, 1706, + 1707, 1708, 1709, 1710, 0, 0, 33, 0, 1044, 0, + 0, 1701, 0, 0, 1702, 1703, 1704, 35, 1705, 1706, + 1707, 1708, 1709, 1710, 1676, 0, 0, 1677, 0, 0, + 0, 1678, 1679, 0, 0, 0, 0, 0, 0, 0, + 0, 37, 0, 0, 0, 38, 0, 0, 0, 0, + 1604, 1598, 0, 0, 0, 0, 1599, 1676, 0, 0, + 1677, 0, 0, 1687, 1678, 1679, 0, 0, 0, 0, + -2183, 1596, 0, 0, 0, 2148, 903, 903, 0, 903, + 40, 1045, 0, 0, 0, 1959, 1961, 0, 567, 1226, + 0, 43, 0, 0, 0, 1404, 1687, 1689, 0, 0, + 0, 0, 1701, -2183, 0, -2183, -2183, -2183, 44, 1705, + 1706, 1707, 1708, 1709, 1710, 0, 0, 1046, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1602, 981, 0, + 1689, 1468, 45, 1047, 1042, 0, 0, 0, 0, 1043, + 0, 0, 0, 0, 1048, 0, 46, 0, 0, 1598, + 0, 0, 0, 0, 1599, 0, 0, 0, 0, 1128, + 0, 0, 0, 2990, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2251, 0, 869, 0, 869, 0, + 1049, 0, 0, 0, 0, 1596, 0, 0, 0, 869, + 0, 0, 2267, 0, 0, -2183, 2991, 0, 1044, 0, + 0, 1227, 1079, 0, 1468, 1991, 0, 0, 0, 1992, + 1993, 0, -2183, 0, 1994, 1995, 1996, -2183, 0, 0, + 1596, 0, 0, 0, 0, 0, 0, 1050, -2183, 869, + 0, 567, 0, 0, 1051, 0, 0, 0, 1601, 0, + 0, 0, 0, 2318, 2320, -2183, 0, 0, 1603, 2323, + -2183, 0, 1821, 567, 0, 0, -2183, 0, 73, 0, + 0, 0, 0, 0, 567, 2344, 567, 1821, 0, 567, + 0, 1045, 0, 0, 0, 0, 1052, 0, 0, 567, + 0, 567, 0, 0, 0, 0, 1596, 0, 0, -2183, + 0, 1598, 1596, 981, 567, 1053, 1599, 0, 981, 567, + 0, 0, 0, 567, 567, 1821, 567, 1046, 567, 567, + 0, 0, 0, 1698, 0, 1596, 0, 1602, 0, 0, + 868, 0, 0, 1047, 876, 0, 0, 0, 0, 0, + 2413, 0, 1604, 0, 1048, 0, 1601, 0, 1352, 0, + 869, 869, 869, 869, 869, 869, 1698, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1596, + 2441, 0, 0, 0, 0, 0, 0, 0, 1054, 824, + 1049, 824, 0, 1228, 824, 0, 0, 1404, 0, 824, + 0, 2504, 824, 0, 824, 0, 914, 0, 824, 0, + 0, 2059, 0, 0, 919, 0, 922, 1598, 926, 0, + 0, 0, 1599, 0, 0, 1470, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, -2183, 1050, -2183, 0, + 0, 0, 0, 0, 1051, 0, 0, 0, 0, 0, + 0, 1468, 1468, 1468, 1468, 1468, 1468, 0, 1603, 1468, + 1468, 1468, 1468, 1468, 1468, 1468, 1468, 1468, 1468, -2183, + 0, 1598, 0, 0, 0, 0, 1599, 0, 0, 0, + 0, 0, 0, 1598, 0, 0, 1052, 0, 1599, 567, + 0, 0, 1596, 0, 73, 0, 0, 0, 1601, 0, + 0, 0, 0, 869, 0, 1053, 0, 0, 0, 0, + 0, 1999, 0, 1598, 0, 0, 822, 0, 1599, 0, + 1598, 0, 822, 0, 0, 1599, 0, 0, 567, 0, + 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, + 0, 0, 0, 2613, 2613, 0, 0, 0, -2183, 0, + 0, 0, 0, 0, 0, 1705, 1706, 1707, 1708, 1709, + 1710, 0, 0, 0, 1598, 568, 0, 0, 1598, 1599, + 0, 568, 0, 1599, 1598, 823, 0, 2199, 1054, 1599, + 0, -2183, 568, 2060, 0, 0, 0, 0, 1705, 1706, + 1707, 1708, 1709, 1710, 0, 0, 568, 568, 1598, 0, + 0, 0, 0, 1599, 1601, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, + 0, 0, 0, 567, 0, 0, 1781, 0, 567, 823, + 823, 0, 0, 0, 0, 1470, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, + 0, 0, 1468, 1468, 0, 0, 0, 0, 1601, 0, + 1602, 0, 0, 0, 568, 0, 0, 0, 0, 0, + 1601, 2148, 0, 0, 0, 0, 0, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1957, 0, 1049, 1253, 1026, 1053, 0, 866, - 550, 1682, 1461, 2667, 0, 0, 1254, 0, 1026, 1026, - 0, 0, 0, 0, 0, 565, 0, 0, 1255, 0, - 0, 819, 0, 819, 0, -1411, 0, 0, 0, 0, - 0, 0, 819, 1592, 0, -2180, 0, 0, 1591, 1958, - 0, 3075, 565, 0, 0, 0, 0, 0, 1594, 0, - 1256, 0, -2180, 0, 0, 0, 2607, -2180, 0, 1604, - 1959, 0, 0, 0, 551, 0, 0, 1252, 0, 0, - 0, 0, 0, 1464, 1464, 0, 1589, 0, 1960, 1464, - 0, 0, 0, 1961, 0, 0, 0, 565, 0, 0, - 0, 0, 0, 0, 1591, 0, -2180, 0, 0, 1683, - 0, 0, 0, 0, 0, 1258, 1962, 0, 0, 1963, - 0, 0, 565, 0, 0, 0, 1684, 0, 1253, 1589, - 552, 1685, 0, 0, 1669, 1964, 0, 1670, 0, 1254, - 1259, 1671, 1672, 0, 1463, 1463, 0, 0, 0, 1592, - 553, 1255, 0, 0, 1686, 1687, 0, 0, 0, 0, - 0, 1261, 0, 1691, 1594, 0, 0, 0, 0, 0, - 1688, 0, 0, 1680, 0, 1729, 0, 0, 0, 0, - -2180, 0, 0, 1256, 0, 0, 0, 0, 1595, 0, - 0, 565, 565, 0, 0, 1589, 0, 0, 866, 0, - 0, 866, 0, 1592, 0, 0, 0, 1682, 0, 554, - 1689, 0, 0, 1690, 0, 1592, -223, 0, 1594, 0, - 1591, 0, 1461, 1123, 1123, 2690, 0, 1691, 0, 0, - 1594, 1965, 866, 0, 0, 866, 1812, 0, 1258, 1966, - 0, 0, 0, 0, 0, 1592, 0, 0, 866, 0, - 0, 0, 1592, 0, 0, 0, 0, 0, 2719, 2719, - 1594, 1967, 555, 1259, 556, 866, -2180, 1594, 0, 866, - 0, 0, 0, 2974, 0, 1864, 0, 0, 0, 0, - 1589, 0, 0, 0, 1261, 1591, 1595, 557, 0, 0, - 558, 1968, 0, 0, 0, 0, 1592, 0, 0, 0, - 1592, 1596, 0, 0, 0, -2180, 1592, 0, 0, 0, - 0, 1594, 0, 0, 0, 1594, 0, 0, 0, 0, - 0, 1594, -2180, 0, 0, 0, 0, -2180, 0, 1592, - 1693, 0, 0, 0, 0, 2233, 0, 1884, 0, 866, - 0, 0, 0, 0, 1594, 0, 0, 866, 0, 0, - 0, 0, 0, 0, 0, 0, 1463, 0, 0, 0, - 1929, 0, 0, 0, 0, 0, -2180, 0, 0, 976, - 0, 0, 0, 0, 976, 0, 565, 565, -2180, 565, - 976, 0, 1464, 3242, 0, 1698, 1699, 1700, 1701, 1702, - 1703, 0, 0, 1589, 0, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, -1901, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1691, 0, 0, 1595, 0, 1589, 0, - 0, 0, 0, 3271, 0, 72, 0, 0, 1591, 0, - 0, 0, 1694, 0, 0, 1695, 1696, 1697, 0, 1698, - 1699, 1700, 1701, 1702, 1703, 0, 0, 0, 0, 1461, - 1461, 0, 0, 0, 0, 1461, 0, 530, 0, 0, + 1601, 0, 0, 0, 822, 0, 0, 1601, 0, 0, + 0, 0, 0, 0, 0, 0, 567, 0, 0, 0, + 822, 0, 0, 0, 822, 2267, 0, 0, 0, 2000, + 2001, 2002, 0, 2003, 2004, 2005, 2006, 2007, 2008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1026, 0, 565, 2025, 0, 0, 0, 1123, 1123, 1596, - 866, 0, 0, 0, 1589, 0, 0, 819, 0, 819, - 1589, 0, 819, 0, 1591, 0, 0, 819, 0, 1123, - 819, 0, 819, 0, -1901, 0, 819, 0, 565, 0, - 565, 0, 0, 1589, 0, 0, -2180, 0, 0, 1463, - 0, 0, 1595, 0, 0, 0, 0, 0, 0, 566, - 0, 0, 0, 0, 0, 566, 0, 0, 0, 820, - 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, - 0, 0, -1901, 0, 0, 0, 0, 1589, 0, 0, - 566, 566, 0, 0, 0, 0, 0, -1901, 0, 0, - 0, 0, -1901, 0, 0, 0, 1595, -1901, 0, 1591, - 0, 2927, 2929, 0, 0, 0, -1901, 0, 1595, 0, - 0, -1901, 820, 820, 0, 0, 0, 1464, 1464, 1464, - 1464, 1464, 1464, 0, 0, 1464, 1464, 1464, 1464, 1464, - 1464, 1464, 1464, 1464, 1464, 0, 0, 0, 1595, 2138, - 0, 0, 0, -1901, 0, 1595, 566, 0, -2180, 0, - 0, 0, 565, 0, 0, 1698, 1699, 1700, 1701, 1702, - 1703, 72, 883, 0, -1901, 0, 0, 0, 0, 0, - 0, 0, 1592, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1594, 0, 1595, - 1589, 0, 976, 1595, 0, 1461, 1463, 0, 0, 1595, - 0, 0, 0, 1006, 0, 0, 0, 0, 0, 0, - 0, 0, 1591, 0, 1018, -1901, 0, 0, -1901, 0, - 0, 1034, 1595, 1123, -1901, 0, 0, 0, 1592, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2241, 0, - 866, 0, 866, 1594, 0, 0, 0, 1591, 0, 0, - 0, 0, 0, 866, 0, 0, 2257, 0, 0, 0, - 0, 0, 0, 0, 0, -1901, 900, 0, 1461, 0, - 0, 0, 1397, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, - -1901, 0, 0, 866, 0, 565, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2308, 2310, 0, - 0, 0, 0, 1591, 1812, 565, 0, 1464, 1464, 1591, - 0, 0, 1596, 0, 0, 0, 565, 2332, 565, 1812, - 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 565, 1591, 565, 1592, 0, 0, 0, 0, 0, - 871, 0, 0, 0, 0, 976, 565, 0, 0, 1594, - 976, 565, 0, 0, 0, 565, 565, 1812, 565, 0, - 565, 565, 0, 0, 0, 0, 0, 0, 0, -1901, - 0, 0, 0, 0, 0, 0, 1591, 0, 0, 0, - -1901, 0, 2401, 0, 0, 72, 0, 0, 0, 0, - 1345, 0, 866, 866, 866, 866, 866, 866, 0, 1592, - -1901, 0, -1901, -1901, 0, 0, 0, 0, 3255, 0, - 1596, 0, 2429, 3560, 1594, 0, 0, 0, 821, 0, - 1018, 0, 0, 1464, 0, 0, 0, 0, 0, 821, - 0, 0, 0, 2492, 0, 0, 0, 0, 0, -1901, - 0, 0, -1901, -1901, -1901, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 865, 0, 0, 0, 873, - 0, 0, 1589, 0, 1607, 1669, 0, 0, 1670, 0, - 0, 0, 1671, 1672, 0, 0, 1619, 0, 0, 0, - 0, 0, 0, 1461, 1461, 1461, 1461, 1461, 1461, 1591, - 0, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - 1461, 0, 0, 0, 1680, 1648, 0, 0, 0, 1592, - 0, -2180, 0, 0, 0, 0, 0, 0, 0, 0, - 911, 565, 0, 0, 1594, 0, 0, 914, 0, 917, - 0, 921, 0, 0, 0, 866, 0, 0, 1682, 0, - 0, 0, 0, 0, 0, 1595, 0, 0, 819, 0, - 1596, 0, 1592, 0, 819, 0, 0, 0, 1397, 0, - 565, 0, 0, 1397, 0, 0, 565, 1594, 0, 0, - 0, 0, 0, 0, 0, 2601, 2601, 0, 0, 0, + 0, 1601, 0, 0, 0, 1601, 0, 0, 1602, 0, + 0, 1601, 0, 0, 0, 567, 0, 0, 1468, 0, + 1128, 567, 0, 0, 1596, 0, 0, 0, 0, 0, + 0, 1603, 0, 0, 0, 1601, 0, 1079, 0, 0, + 1991, 0, 0, 1893, 1992, 1993, 0, 0, 0, 1994, + 1995, 1996, 0, 0, 0, 0, 0, 1079, 73, 0, + 1991, 0, 0, 0, 1992, 1993, 2978, 0, 0, 1994, + 1995, 1996, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 3293, 1079, 0, 0, + 1991, 0, 0, 0, 1992, 1993, 0, 0, 0, 1994, + 1995, 1996, 0, 0, 950, 0, 0, 0, 0, 957, + 0, 0, 0, 0, 0, 0, 3294, 0, 0, 0, + 0, 0, 0, 0, 1470, 1470, 1893, 0, 0, 1603, + 1470, 0, 0, 869, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1352, 0, 0, 1893, 869, 869, 869, + 1602, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 567, 0, 869, 0, 0, 0, 869, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 566, 0, 0, 900, 900, 0, 900, 0, 0, 0, - 0, 1595, 0, 0, 2977, 0, 0, 0, 0, 0, - 0, 0, 1950, 1952, 0, 0, 0, 0, 1592, 0, + 0, 0, 0, 1598, 0, 0, 0, 0, 1599, 0, + 0, 0, 824, 0, 0, 0, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1594, 0, 0, -2180, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, - 0, 0, 0, -2180, 565, 0, 1596, 0, -2180, 565, - 0, 1669, 0, 0, 1670, 0, 0, 0, 1671, 1672, + 0, 0, 869, 0, 0, 0, 0, 0, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1461, 1461, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, -2180, 0, 0, - 1680, 0, 2138, 1592, 0, 0, 0, -2180, 1461, 0, - 1596, 0, 0, 0, 0, 0, 0, 1595, 1594, 0, - 0, 0, 1596, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1682, 819, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, - 0, 819, 1596, 0, 1691, 819, 2257, 0, 0, 1596, + 0, 0, 1893, 1893, 0, 1893, 0, 0, 0, 1598, + 1676, 0, 0, 1677, 1599, 0, 0, 1678, 1679, 0, + 0, 0, 0, 1998, 0, 0, 1602, 0, 0, 0, + 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, + 0, 1603, 1471, 1998, 0, 0, 0, 0, 0, 1687, + 0, 0, 2905, 0, 0, 0, -2183, 0, 0, 0, + 869, 869, 869, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1998, 567, 0, 1468, 0, 567, 0, + 1602, 0, 0, 1689, 567, 0, 0, 0, 0, 0, + 0, 0, 1602, 0, 0, 0, 1999, 0, 0, 0, + 0, 0, 0, 1655, 869, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1999, 0, 0, 2148, + 0, 0, 1602, 2199, 0, 1598, 0, 0, 0, 1602, + 1599, 567, 0, 0, 1015, 567, 0, 0, 568, 2689, + 1601, 0, 0, 0, 0, 0, 1999, 1603, 0, 3297, + 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1468, 1468, 0, 0, 0, 824, 0, 0, 0, + 824, 0, 0, 1602, 0, 0, 0, 1602, 0, 0, + 0, -2183, 0, 1602, 0, 3008, 0, 0, 0, 0, + 1598, 0, 3017, 2286, 0, 1599, 1601, 0, -2183, 2267, + 0, 1603, 0, -2183, 0, 0, 0, 1602, 0, 0, + 0, 0, 0, 1603, 0, 869, 0, 0, 0, 567, + 0, 1128, 0, 567, 567, 0, 0, 3045, 567, 0, + 0, 1233, 1893, 1821, 1893, 0, 1938, 0, 0, 0, + 0, 0, -2183, 1603, 0, 0, 0, 0, 0, 0, + 1603, 0, 1471, 0, 0, 0, 0, 567, 0, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1591, 0, 72, 0, 0, 0, 0, 0, 0, - 0, 0, 1595, 0, 566, 566, 0, 0, 0, 0, - 0, 0, 0, 0, 1464, 1464, 565, 0, 0, 1461, - 2978, 1123, 565, 1596, 0, 0, 0, 1596, 0, 0, - 0, 0, 0, 1596, 0, 0, 1592, 0, 0, 0, - 0, 0, 1884, 0, 0, 0, 0, 0, 0, 0, - 0, 1594, -2180, 0, 0, 0, 1596, 0, 0, 0, - 0, 0, 0, 820, 0, 0, 1950, 1952, 0, -2180, - 0, 1592, 0, 0, -2180, 0, 1397, -2180, 0, 0, - 0, 0, 0, 0, 0, 0, 1594, 0, 0, 0, + 1341, 0, 0, 0, 567, 567, 567, 567, 1821, 567, + 567, 567, 567, 567, 0, 1356, 1358, 1361, 1363, 1365, + 0, 0, 0, 0, 1603, 0, 0, 0, 1603, 1698, + 0, 1598, 0, 0, 1603, 0, 1599, 0, 0, 2441, + 0, 0, 1601, 0, 2000, 2001, 2002, 869, 2003, 2004, + 2005, 2006, 2007, 2008, 0, 0, 0, 0, 1603, 0, + 3198, 0, 568, 568, 2000, 2001, 2002, 1463, 2003, 2004, + 2005, 2006, 2007, 2008, 1598, 0, 0, 0, 0, 1599, + 0, 0, 0, 0, 0, 0, 1938, 0, 0, 2862, + 0, 0, 0, 1893, 2000, 2001, 2002, 0, 2003, 2004, + 2005, 2006, 2007, 2008, 1468, 0, 0, 1601, 0, 0, + 567, 0, 0, 0, 0, 0, 0, 869, 869, 869, + 869, 0, 823, 0, 0, 0, 0, 0, 0, 0, + 0, 1468, -2183, 0, 1468, 0, 0, 0, 567, 981, + 1598, 0, 0, 0, 0, 1599, 0, 3271, 0, 0, + 1470, 1470, 1470, 1470, 1470, 1470, 0, 0, 1470, 1470, + 1470, 1470, 1470, 1470, 1470, 1470, 1470, 1470, 0, 0, + 567, 0, 0, 0, 0, 0, 0, 3279, 567, 0, + 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, + 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 3307, 0, 822, 0, 0, 0, 0, 0, 1601, 0, + 0, 1471, 1471, 0, 1128, 1598, 0, 1471, 0, 0, + 1599, 3321, 0, 0, 0, 0, 2267, 0, 0, 0, + 0, 0, 2148, 0, -2183, 0, 0, 0, 0, 0, + 0, 1705, 1706, 1707, 1708, 1709, 1710, 1821, 0, 0, + 0, 1601, 0, 1893, 1418, 1419, 0, 0, 0, 0, + 0, 0, 1602, 0, 0, 0, 981, 567, 1468, 0, + 0, 0, 0, 0, 869, 0, 0, 0, 1655, 0, + 1807, 0, 0, 0, 0, 1811, 0, 0, 0, 0, + 0, 3377, 0, 0, 0, 1420, 1421, 0, 1843, 1422, + 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1655, 0, 0, 0, 1601, 1602, 1868, + 0, 0, 1488, 0, 0, 0, 0, 0, 823, 1598, + 823, 0, 0, 0, 1599, 0, 0, 0, 0, 823, + 73, 1470, 1470, 0, 0, 0, 0, 0, 0, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 821, 0, 821, 0, 0, 821, - 0, 0, 1595, -2180, 821, 0, 0, 821, 0, 821, - 0, 0, 0, 821, 566, 1884, 0, 0, 0, 0, - 0, 0, 866, 0, 0, 0, 0, 1592, 0, 0, - 0, 0, 1345, 1592, 0, 1884, 866, 866, 866, 0, - 0, 0, 1594, 0, 0, 1595, 0, 0, 1594, 565, - 0, 866, 0, 0, 0, 866, 1592, 0, 866, 0, - 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1594, 0, -1916, 0, 0, 0, 945, 0, -2180, - 0, 0, 952, 0, 0, 0, 1698, 1699, 1700, 1701, - 1702, 1703, 0, 0, 0, 0, 0, 0, 0, 72, - 1592, 866, 0, 0, 0, 0, 0, 976, 0, 0, - 0, 1595, 0, 0, 0, 1594, 1464, 0, 0, 0, - 0, 1884, 1884, 0, 1884, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1598, 0, 0, 3040, 0, 1599, + 0, 0, 0, 3394, 0, 0, 1424, 1425, 1893, 0, + 0, 0, 0, 1603, 0, 0, 0, 0, 0, 1898, + 0, 0, 0, 2441, 568, 0, 0, 1916, 0, 0, + 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, + 3432, 0, 0, 0, 0, 1468, 0, 0, 0, 0, + 1598, 0, 0, 0, 1602, 1599, 1598, 0, 0, 1603, + 0, 1599, 0, 1426, 1427, 1428, 1429, 1430, 1431, 1432, + 1433, 3448, 0, 1434, 1435, 0, 73, 1470, 567, 1598, + 0, 0, 0, 2243, 1599, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1464, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 530, 0, 0, 0, 0, 0, 0, - 0, 1481, 0, -2180, 0, 0, 0, 820, 0, 820, - 0, 2892, 0, 0, -1916, 0, 0, 0, 820, 866, - 866, 866, 2189, 0, 0, 0, 0, 0, 1519, 0, - 0, 0, 0, 565, 0, 1461, 1595, 565, 0, 0, - 0, 0, 1074, 565, 0, 1982, 0, 0, 0, 1983, - 1984, 0, 0, 1592, 1985, 1986, 1987, 0, 0, 0, - 0, 0, -1916, 866, 0, 0, 0, 0, 1594, 0, - 0, 3279, 0, 566, 0, 0, 0, -1916, 2138, 0, - 0, 0, -1916, 0, 0, 0, 0, -1916, 1397, 0, - 565, 0, 900, 0, 565, 0, -1916, 0, 1653, 0, - 0, -1916, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -2180, 0, 0, 0, 1596, - 1461, 1461, 1698, 1699, 1700, 1701, 1702, 1703, 0, 1464, - 0, 1669, 0, -1916, 1670, 0, 0, 0, 1671, 1672, - 0, 0, 0, 0, 2995, 0, 0, 0, 0, 1595, - 0, 3004, 0, 0, -1916, 0, 0, 0, 2257, 0, - 0, 0, 0, 0, 0, 0, 0, 566, 566, 0, - 1680, 0, 0, 0, 866, 1596, 1010, -2180, 565, 0, - 1123, 0, 565, 565, 1595, 0, 565, 0, 0, 0, - 1884, 1812, 1884, 0, 1929, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1682, -1916, 0, 0, -1916, 0, - 0, 0, 0, 0, -1916, 565, 0, 1812, 0, 0, + 0, 0, 3467, 0, 0, 0, 886, 0, 568, 568, + 73, 0, 0, 0, 0, 0, 0, 0, 0, 1602, + 3218, 0, 0, 1598, 0, 3479, 0, 0, 1599, 0, + 1471, 0, 0, 3008, 0, 0, 1601, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1436, 1437, + 0, 0, 0, 0, 567, 0, 0, 0, 0, 567, + 2048, 0, 0, 0, 0, 0, 0, 1655, 0, 0, + 0, 1601, 0, 0, 0, 1603, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 567, 0, 0, 0, 1438, + 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 567, 567, 0, 0, + 0, 0, 0, 0, 0, 1418, 1419, 0, 0, 0, + 0, 0, 0, 0, 869, 0, 3198, 0, 0, 0, + 1602, 567, 0, 0, 0, 0, 1598, 1601, 0, 0, + 1603, 1599, 0, 1601, 0, 0, 0, 0, 824, 0, + 869, 0, 0, 3557, 0, 0, 1420, 1421, 73, 0, + 1422, 1423, 0, 0, 0, 0, 1601, 886, 0, 0, + 0, 0, 1468, 1602, 1128, 0, 567, 1031, 0, 1031, + 0, 0, 0, 0, 567, 0, 0, 0, 0, 1440, + 1441, 0, 0, 0, 0, 568, 568, 0, 568, 0, + 0, 0, 0, 0, 0, 1128, 0, 0, 0, 3321, + 1601, 0, 0, 1442, 1443, 0, 0, 0, 0, 0, + 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1424, 1425, 1602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 565, 565, 565, 565, 1812, 565, 565, 565, - 565, 565, 0, 0, 0, 0, 0, 0, 0, 0, - 1595, 72, 0, 0, 0, -1916, 1595, 0, 1989, 0, - 0, 0, 0, 0, 0, 0, 0, 2429, 0, 0, - 3283, 0, 0, 1228, 0, 866, 0, 0, 0, 1595, - -1916, 0, 0, 0, 0, 0, 1464, 0, 3184, 0, - 0, 1596, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -2180, 0, 1074, 0, 0, 1982, 0, 0, - 1334, 1983, 1984, 0, 1929, 0, 1985, 1986, 1987, -2180, - 0, 1884, 0, 1595, -2180, 1349, 1351, 1354, 1356, 1358, - 0, 1990, 1461, 3280, 0, 1592, 0, 0, 565, 0, - 871, 0, 0, 0, 0, 866, 866, 866, 866, 0, - 1594, 0, 566, 566, 0, 566, 1596, 0, 0, 1461, - 0, 0, 1461, -2180, 0, 0, 565, 976, 0, -1916, - 0, 0, 0, 0, 0, 3257, 0, 1456, 0, 0, - -1916, 0, 0, 0, 0, 821, 0, 0, 0, 0, - 0, 821, 0, 0, 0, 0, 0, 0, 565, 0, - -1916, 0, -1916, -1916, 0, 3265, 565, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1074, 0, 0, 1982, - 1691, 0, 0, 1983, 1984, 0, 0, 0, 1985, 1986, - 1987, 0, 0, 0, 0, 0, 1595, 0, 3293, -1916, - 819, 0, -1916, -1916, -1916, 0, 0, 0, 1074, 0, - 0, 1982, 1123, 0, 0, 1983, 1984, 0, 566, 3307, - 1985, 1986, 1987, 0, 2257, 0, 1596, 0, 0, 0, - 2138, 0, 0, 820, 0, 820, 0, 0, 820, 0, - 0, 0, 0, 820, 1812, 0, 820, 0, 820, 0, - 1884, 0, 820, 0, 2063, 0, 2067, 0, 0, 0, - 0, 0, 0, 976, 565, 1461, 0, 0, 0, 1596, - 0, 866, 0, 1464, 0, 0, 0, 0, 0, 0, - 1989, 0, 0, -2180, 0, 1648, 0, 0, 3363, 0, + 869, 1603, 0, 0, 0, 0, 0, 1471, 1471, 1471, + 1471, 1471, 1471, 0, 0, 1471, 1471, 1471, 1471, 1471, + 1471, 1471, 1471, 1471, 1471, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, + 0, 0, 0, 0, 1603, 0, 0, 1128, 0, 0, + 2252, 568, 2254, 0, 1426, 1427, 1428, 1429, 1430, 1431, + 1432, 1433, 3557, 2263, 1434, 1435, 823, 0, 823, 0, + 0, 823, 0, 1601, 1602, 0, 823, 0, 0, 823, + 0, 823, 0, 0, 0, 823, 0, 2072, 0, 2076, + 73, 0, 3198, 0, 0, 0, 0, 735, 0, 0, + 1470, 1470, 0, 2301, 981, 981, 0, 3557, 981, 0, + 1603, 0, 0, 0, 0, 0, 2034, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1655, 0, 1436, + 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1243, 3198, 736, 0, 0, 0, 1244, 0, + 3557, 0, 0, 0, 0, 0, 1256, 0, 1598, 0, + 737, 0, 0, 1599, 0, 0, 0, 0, 0, 0, + 1438, 1439, 0, 0, 0, 0, 1257, 0, 1602, 0, + 0, 0, 0, 0, 0, 1603, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 981, 0, 1471, 1471, + 0, 2160, 0, 0, 2425, 2426, 2428, 2429, 2430, 2431, + 738, 2267, 0, 1602, 0, 0, 3321, 0, 0, 2267, + 739, 0, 1258, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 740, 0, 0, 0, 0, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2189, 0, 0, 0, 1991, - 1992, 1993, 0, 1994, 1995, 1996, 1997, 1998, 1999, 0, - 0, 2677, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 821, 0, 0, 1596, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 821, 0, - 0, 0, 821, 1990, 0, 0, 0, 0, 0, 0, - 1800, 0, 0, 1802, 0, 0, 0, 0, 0, 0, - 3380, 0, 0, 0, 0, 1884, 1834, 0, 2150, 0, + 0, 0, 0, 1245, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 742, 0, 0, + 1440, 1441, 0, 0, 0, 0, 0, 0, 0, 1602, + 0, 0, 1143, 1143, 0, 1602, 0, 0, 0, 73, + 0, 0, 0, 0, 1442, 1443, 0, 0, 0, 1603, + 0, 0, 0, 1470, 0, 0, 0, 0, 1602, 0, + 0, 743, 0, 0, 1471, 3554, 744, 0, 0, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2429, 0, 1989, 0, 0, -2180, 0, 1859, 0, 0, - 0, 0, 1698, 1699, 1700, 1701, 1702, 1703, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3418, 0, 0, - 1596, 0, 1461, 0, -2180, 0, 0, 0, 0, 0, + 3256, 0, 0, 0, 1603, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 568, 1601, 0, 2580, 0, 0, + 0, 0, 1602, 0, 0, 0, 0, 0, 1286, 0, + 0, 0, 1332, 1339, 0, 0, 568, 0, 0, 0, + 1260, 0, 73, 0, 73, 0, 0, 568, 0, 568, + 3285, 1261, 568, 0, 0, 0, 0, 0, 745, 0, + 0, 0, 568, 1262, 568, 0, 0, 0, 0, 0, + 1603, 0, 0, 746, 0, 0, 1603, 568, 0, 0, + 0, 0, 568, 0, 1246, 1390, 568, 568, 73, 568, + 0, 568, 568, 0, 0, 1263, 0, 0, 0, 1603, + 0, 0, 0, 1414, 0, 73, 0, 0, 747, 1462, + 0, 748, 1464, 0, 0, 1475, 1478, 1483, 1486, 0, + 0, 0, 749, 0, 0, 750, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1602, 0, 0, 0, 0, + 0, 0, 0, 1603, 1264, 751, 0, 1470, 1248, 0, + 1265, 0, 0, 0, 0, 0, 0, 0, 0, 752, + 1530, 1332, 0, 0, 0, 0, 0, 754, 0, 0, + 0, 0, 0, 0, 0, 1266, 0, 755, 0, 0, + 1267, 0, 1617, 756, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1268, 0, 0, 0, + 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 757, 0, 1645, 1646, 1647, 0, 1652, 1656, + 0, 1676, 0, 0, 1677, 0, 0, 0, 1678, 1679, + 1680, 1681, 0, 1682, 1683, 1684, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3434, 0, - 0, 0, 0, 0, 0, 565, 0, 0, 0, 0, - 0, 0, 565, 0, 0, 1990, 0, 1889, 0, 0, - 0, 0, 0, 0, 0, 1907, 0, 0, 1595, 3453, - 0, 0, 0, 0, 0, 0, 1669, 0, 11, 1670, - 0, 0, 0, 1671, 1672, 1673, 1674, 1990, 1675, 1676, - 1677, 0, 3465, 0, 0, 0, 0, 0, 0, 0, - 2995, 0, 0, 0, 0, 1678, 14, 15, 0, 0, - 0, 566, 0, 0, 0, 1680, 0, 0, 0, 0, - 0, 565, 1681, 1596, 0, 0, 565, 0, 0, 2849, - 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 566, 0, 566, 0, 0, 566, 0, 1682, - 0, 0, 565, 23, 0, 0, 0, 566, 1596, 566, - 0, 1991, 1992, 1993, 0, 1994, 1995, 1996, 1997, 1998, - 1999, 0, 566, 565, 565, 0, 0, 566, 0, 0, - 0, 566, 566, 0, 566, 0, 566, 566, 0, 0, - 0, 866, 0, 3184, 0, 0, 0, 0, 565, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, - 0, 0, 0, 0, 0, 0, 0, 866, 0, 0, - 3543, 0, 0, 0, 1596, 0, 0, 0, 0, 0, - 1596, 0, 0, 0, 0, 0, 0, 0, 0, 1461, - 0, 1123, 0, 565, 1026, 0, 1026, 1683, 0, 0, - 0, 565, 0, 1596, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1684, 0, 0, 0, 0, 1685, - 0, 0, 1123, 1991, 1992, 1993, 3307, 1994, 1995, 1996, - 1997, 1998, 1999, 0, 0, 0, 0, 866, 0, 0, - 0, 0, 1686, 1687, 0, 0, 0, 1596, 0, 0, - 26, 27, 28, 0, 0, 1991, 1992, 1993, 1688, 1994, - 1995, 1996, 1997, 1998, 1999, 0, 0, 866, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1648, 0, - 0, 1669, 0, 0, 1670, 0, 0, 0, 1671, 1672, - 1673, 1674, 0, 1675, 1676, 1677, 0, 566, 1689, 0, - 0, 1690, 0, 0, 0, 0, 3434, 0, 0, 0, - 1678, 0, 0, 1648, 1123, 1691, 0, 33, 1692, 0, - 1680, 0, 0, 0, 820, 0, 0, 1681, 35, 3543, - 820, 0, 0, 0, 0, 0, 2588, 0, 0, 0, - 72, 0, 2067, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 37, 0, 1682, 0, 38, 0, 0, 3184, - 1596, 0, 0, 0, 0, 0, 0, 3027, 0, 0, - 0, 976, 976, 0, 3543, 976, 0, 39, 0, 0, - 0, 0, 0, 2025, 0, 0, 0, 0, 0, 0, - 0, 40, 0, 0, 565, 0, 0, 0, 2242, 0, - 2244, 0, 43, 0, 0, 0, 0, 0, 0, 0, - 0, 2253, 0, 0, 0, 0, 0, 0, 1693, 44, - 3184, 0, 0, 0, 0, 2150, 0, 3543, 0, 0, - 566, 0, 0, 0, 0, 1653, 0, 0, 0, 0, - 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, - 0, 2291, 1683, 0, 0, 72, 0, 46, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1684, - 0, 0, 0, 976, 1685, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 883, 0, 0, 2257, 72, - 0, 0, 0, 3307, 0, 0, 2257, 1686, 1687, 3204, + 1685, 0, 568, 0, 1719, 1250, 1603, 0, 0, 0, + 1687, 0, 0, 0, 73, 0, 0, 1688, 0, 1243, + 0, 0, 1530, 1530, 0, 1244, 0, 0, 0, 823, + 0, 0, 0, 1256, 0, 823, 0, 0, 0, 0, + 0, 2600, 0, 0, 1689, 0, 0, 2076, 0, 0, + 0, 0, 0, 1257, 1470, 0, 0, 0, 0, 1759, + 0, 0, 0, 1775, 1780, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1143, 1143, 1675, 0, 0, 0, + 0, 1676, 0, 0, 1677, 0, 0, 0, 1678, 1679, + 1680, 1681, 0, 1682, 1683, 1684, 0, 0, 0, 1258, + 0, 0, 0, 0, 0, 0, 0, 1471, 1471, 0, + 1685, 0, 0, 0, 1686, 0, 0, 0, 0, 0, + 1687, 0, 0, 0, 0, 0, 0, 1688, 0, 0, + 1245, 2160, 0, 0, 0, 0, 568, 0, 1332, 0, + 0, 1660, 1690, 0, 0, 0, 0, 2828, 0, 1332, + 0, 0, 0, 0, 1689, 0, 0, 0, 0, 1691, + 0, 2844, 2845, 2847, 1692, 0, 0, 1602, 0, 0, + 0, 0, 0, 0, 1332, 0, 2860, 0, 0, 0, + 2863, 0, 0, 2866, 0, 0, 0, 1693, 1694, 0, + 0, 0, 0, 0, 0, 0, 1259, 0, 0, 0, + 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, + 0, 0, 0, 0, 0, 0, 2878, 0, 0, 568, + 0, 0, 0, 823, 0, 0, 0, 823, 0, 0, + 0, 0, 0, 1696, 0, 0, 1697, 1260, 0, 0, + 0, 0, 1690, 0, 0, 0, 0, 0, 1261, 0, + 1698, 0, 1981, 1699, 0, 0, 0, 0, 0, 1691, + 1262, 0, 0, 0, 1692, 0, 0, 0, 568, 0, + 0, 3574, 0, 0, 2743, 0, 0, 0, 1603, 0, + 0, 1246, 0, 0, 0, 0, 0, 1693, 1694, 0, + 0, 0, 1263, 0, 2912, 2913, 2914, 0, 0, 0, + 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, + 1471, 0, 0, 0, 0, 1483, 0, 1483, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 820, 0, 1688, 0, 0, 1138, 1138, 0, 0, - 0, 0, 0, 566, 0, 0, 0, 820, 0, 0, - 1694, 820, 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, - 1701, 1702, 1703, 0, 0, 0, 1648, 0, 0, 0, - 1975, 0, 0, 1689, 0, 0, 1690, 0, 0, 0, - 2413, 2414, 2416, 2417, 2418, 2419, 0, 0, 0, 0, - 1691, 0, 566, 1692, 0, 0, 0, 0, 2731, 0, - 0, 0, 0, -47, 0, 0, 0, 0, 0, 0, - 1279, 0, 0, 0, 1325, 1332, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 2, 0, 3, 4, - 0, 0, 0, 0, 0, 0, 0, 821, 0, 0, - 0, 5, 0, 0, 0, 0, 6, 72, 0, 0, - 0, 0, 1596, 0, 0, 7, 883, 1383, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, - 0, 0, 0, 0, 0, 1407, 0, 0, 9, 0, - 10, 1455, 0, 1693, 1457, 0, 0, 1468, 1471, 1476, - 1479, 0, 11, 0, 12, 0, 1669, 0, 0, 1670, - 0, 0, 0, 1671, 1672, 13, 0, 0, 1675, 1676, - 1677, 0, 0, 2568, 0, 0, 1669, 0, 0, 1670, - 14, 15, 16, 1671, 1672, 566, 0, 0, 1675, 1676, - 1677, 17, 1523, 1325, 0, 1680, 0, 18, 0, 0, - 0, 0, 1681, 0, 0, 0, 19, 0, 20, 21, - 0, 0, 0, 0, 1610, 1680, 0, 0, 0, 0, - 0, 0, 1681, 22, 0, 0, 0, 23, 0, 1682, - 0, 0, 0, 0, 1627, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1638, 1639, 1640, 1682, - 1645, 1649, 0, 24, 0, 1694, 0, 0, 1695, 1696, - 1697, 0, 1698, 1699, 1700, 1701, 1702, 1703, 0, -1521, - 0, 0, 2156, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1712, 0, 72, 0, - 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, - 0, 0, 0, 0, 1523, 1523, 0, 0, 0, 0, + 1143, 1143, 0, 0, 0, 0, 0, 1471, 0, 0, + 0, 2368, 0, 1696, 0, 1248, 1697, 1265, 0, 0, + 0, 0, 1143, 1700, 0, 0, 0, 0, 0, 0, + 1698, 0, 0, 1699, 0, 0, 0, 0, 0, 0, + 0, 0, 1266, 0, 0, 0, 0, 2369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1683, 0, 566, - 0, 0, 0, 566, 0, 1648, 0, 0, 0, 2063, - 0, 1752, 0, 0, 1684, 1768, 1773, 1683, 0, 1685, - 0, 0, 0, 0, 0, 0, 1138, 1138, 0, 0, - 0, 0, 0, 0, 1684, 0, 0, 0, 0, 1685, - 0, 0, 1686, 1687, 26, 27, 28, 0, 0, 0, - 0, 0, 29, 0, 0, 30, 2950, 0, 1688, 0, - 2150, 0, -2180, -2180, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1688, 0, - 0, 0, 0, 0, 0, 0, 31, 0, 1325, 0, - 0, 0, 0, 0, 0, 32, 0, 0, 1689, 1325, - 0, 1690, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 33, 0, 0, 0, 1691, 0, 0, 34, 0, - 0, -2180, 35, 0, 1325, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 36, 1691, 0, 0, 0, 0, - 0, 0, 0, 0, 566, 0, 37, 0, 566, 566, - 38, 0, 566, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, - 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 566, 0, 0, 0, 40, 0, 0, 41, 0, - 0, 42, 0, 3540, 0, 0, 43, 0, 566, 566, - 566, 566, 0, 566, 566, 566, 566, 566, 0, 0, - 0, 0, 1972, 44, 0, 0, 0, 0, 1693, 0, - 2815, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2831, 2832, 2834, 45, 1693, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, - 72, 46, 72, 2850, -47, 0, 2853, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1476, 0, 1476, 1476, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1138, 1138, 0, 0, 0, 0, 72, 0, 0, 2865, - 0, 0, 0, 0, 2063, 0, 0, 0, 0, 0, - 0, 0, 1138, 72, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1694, 0, 2588, 1695, 1696, 1697, 0, 1698, 1699, 1700, - 1701, 1702, 1703, 0, 0, 1410, 0, 833, 0, 0, - 1694, 0, 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, - 1701, 1702, 1703, 0, 2150, 0, 0, 0, 0, 0, - 0, 0, 2150, 0, 0, 0, 1668, 2899, 2900, 2901, - 0, 1669, 0, 0, 1670, 0, 0, 0, 1671, 1672, - 1673, 1674, 0, 1675, 1676, 1677, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 820, 0, 0, 0, - 1678, 1411, 1412, 0, 1679, 0, 0, 2112, 0, 0, - 1680, 0, 0, 0, 0, 0, 2116, 1681, 0, 0, + 0, 0, 0, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2147, 0, 0, 0, 0, 0, 72, 0, 0, 0, - 0, 0, 1413, 1414, 1682, 0, 1415, 1416, 0, 0, - 2161, 0, 0, 0, 0, 0, 0, 0, 2166, 0, - 566, 0, 72, 0, 2170, 2171, 2172, 2173, 2174, 2175, - 2176, 2177, 0, 0, 0, 0, 2186, 2187, 0, 0, - 0, 2200, 0, 0, 0, 2203, 0, 0, 2211, 2212, - 2213, 2214, 2215, 2216, 2217, 2218, 2219, 0, 0, 2220, - 0, 0, 0, 0, 0, 0, 1138, 0, 1325, 0, + 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3023, 1417, 1418, 0, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1669, 0, 1683, 1670, 0, 0, 0, 1671, 1672, 1673, - 1674, 0, 1675, 1676, 1677, 0, 0, 0, 0, 1684, - 0, 0, 1639, 1640, 1685, 0, 0, 0, 0, 1678, - 0, 0, 0, 2955, 0, 0, 0, 0, 0, 1680, - 0, 0, 0, 0, 0, 0, 1681, 1686, 1687, 0, - 1419, 1420, 1421, 1422, 1423, 1424, 1425, 1426, 0, 0, - 1427, 1428, 0, 1688, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3172, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 3036, + 0, 0, 1250, 0, 0, 0, 0, 2122, 0, 0, + 0, 0, 0, 1700, 0, 1701, 2126, 0, 1702, 1703, + 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, 0, 0, + 2157, 0, 0, 0, 1471, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1689, 0, 0, 1690, 0, 0, 0, - 0, 3442, 0, 0, 0, 0, 0, 0, 2588, 0, - 1691, 2383, 0, 1692, 0, 0, 0, 0, 1325, 0, - 0, 2394, 2395, 0, 0, 1429, 1430, 0, 0, 0, + 2171, 0, 0, 0, 0, 0, 0, 0, 2176, 0, + 0, 0, 0, 0, 2180, 2181, 2182, 2183, 2184, 2185, + 2186, 2187, 0, 0, 0, 0, 2196, 2197, 0, 0, + 0, 2210, 0, 0, 0, 2213, 0, 0, 2221, 2222, + 2223, 2224, 2225, 2226, 2227, 2228, 2229, 0, 0, 2230, + 0, 3186, 0, 0, 0, 0, 1143, 568, 1332, 0, + 0, 568, 0, 0, 0, 0, 0, 2072, 0, 0, + 0, 0, 0, 0, 0, 1701, 2256, 0, 1702, 1703, + 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3220, 3221, 3222, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1683, 0, 0, 0, 0, 1431, 1432, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 566, 1684, 0, - 0, 0, 566, 1685, 0, 0, 0, 0, 1383, 2500, - 0, 0, 0, 0, 0, 0, 0, 2502, 0, 0, - 0, 0, 0, 0, 0, 0, 1686, 1687, 566, 0, - 0, 0, 0, 1693, 0, 0, 0, 2518, 2519, 0, - 2520, 0, 1688, 0, 0, 0, 0, 0, 0, 566, - 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2546, - 2547, 0, 0, 2246, 566, 0, 0, 0, 0, 0, - 0, 0, 1689, 0, 0, 1690, 1433, 1434, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1691, - 0, 0, 1692, 0, 2572, 0, 0, 0, 0, 0, - 1435, 1436, 0, 2582, 0, 0, 0, 0, 0, 566, - 0, 0, 0, 0, 0, 0, 0, 2150, 0, 3350, - 0, 0, 1523, 0, 1325, 0, 733, 0, 0, 0, - 0, 0, 0, 0, 0, 1694, 0, 0, 1695, 1696, - 1697, 0, 1698, 1699, 1700, 1701, 1702, 1703, 0, 0, + 0, 0, 1646, 1647, 2963, 0, 0, 0, 2160, 0, + 0, 3234, 3235, 3236, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 2617, 0, 0, 0, 0, 0, 2624, 0, - 0, 0, 0, 0, 0, 1236, 0, 0, 0, 0, - 0, 1237, 0, 734, 0, 0, 0, 0, 0, 1249, - 0, 0, 1693, 0, 0, 0, 0, 0, 0, 735, - 0, 2636, 0, 0, 0, 2642, 0, 0, 0, 1250, - 2647, 2648, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1669, - 0, 0, 1670, 0, 0, 0, 1671, 1672, 1673, 1674, - 0, 1675, 1676, 1677, 0, 0, 0, 0, 0, 736, - 0, 0, 0, 0, 0, 1251, 0, 2669, 1678, 737, - 2672, 0, 2674, 0, 0, 0, 0, 0, 1680, 0, - 0, 0, 738, 0, 0, 1681, 0, 739, 2678, 0, - 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, + 0, 1471, 0, 0, 0, 1676, 0, 0, 1677, 0, + 0, 0, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1682, 0, 1694, 0, 740, 1695, 1696, 1697, - 566, 1698, 1699, 1700, 1701, 1702, 1703, 1669, 1752, 0, - 1670, 2156, 0, 0, 1671, 1672, 1673, 1674, 0, 1675, - 1676, 1677, 0, 0, 0, 1773, 2218, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1678, 0, 0, 0, - 741, 0, 1252, 0, 1138, 742, 1680, 0, 0, 0, - 0, 0, 0, 1681, 0, 0, 0, 2737, 0, 0, - 1669, 0, 0, 1670, 0, 0, 0, 1671, 1672, 1673, - 1674, 0, 1675, 1676, 1677, 0, 0, 0, 0, 0, - 1682, 0, 0, 0, 0, 0, 0, 0, 0, 1678, - 1683, 0, 0, 1253, 0, 0, 0, 0, 0, 1680, - 0, 0, 0, 0, 1254, 0, 1681, 1684, 0, 0, - 0, 553, 1685, 0, 0, 0, 1255, 743, 0, 0, - 0, 0, 0, 0, 0, 1640, 0, 0, 0, 0, - 0, 0, 744, 1682, 1325, 1686, 1687, 1239, 0, 3516, - 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, - 0, 1688, 0, 0, 1465, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3541, 0, 745, 0, 0, - 746, 0, 0, 0, 0, 0, 0, 0, 1683, 0, - 0, 747, 0, 0, 748, 0, 0, 0, 0, 0, - 0, 1689, 0, 0, 1690, 1684, 0, 1257, 0, 0, - 1685, 1241, 0, 1258, 749, 0, 0, 0, 1691, 0, - 0, 1692, 0, 0, 0, 0, 0, 0, 750, 0, - 0, 0, 0, 1686, 1687, 751, 752, 0, 1259, 0, - 0, 1683, 2863, 1260, 0, 3584, 753, 0, 0, 1688, - 0, 0, 754, 0, 0, 0, 0, 0, 1684, 1261, - 0, 0, 0, 1685, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, - 0, 755, 0, 0, 0, 0, 1686, 1687, 0, 1689, - 1706, 0, 1690, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1688, 0, 0, 0, 1691, 0, 0, 1692, - 0, 0, 0, 0, 0, 0, 0, 0, 1243, 0, - 0, 1693, 0, 0, 1669, 0, 0, 1670, 0, 0, - 0, 1671, 1672, 1673, 1674, 0, 1675, 1676, 1677, 1706, - 0, 0, 1689, 0, 0, 1690, 0, 0, 0, 0, - 0, 0, 0, 1678, 0, 0, 0, 0, 0, 1691, - 0, 0, 1692, 1680, 0, 0, 0, 0, 0, 0, - 1681, 2940, 0, 0, 1465, 0, 0, 0, 2942, 2116, + 0, 0, 0, 0, 1685, 0, 0, 0, 2968, 0, + 0, 0, 0, 0, 1687, 0, 0, 0, 0, 0, + 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 568, 0, 0, 0, 568, 568, 0, 0, + 0, 568, 0, 2395, 0, 0, 0, 0, 1689, 0, + 1332, 0, 0, 2406, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2951, 0, 0, 1682, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2963, 0, 1693, - 2966, 0, 2968, 0, 0, 0, 0, 0, 0, 0, - 2972, 0, 0, 0, 0, 0, 0, 0, 2979, 2980, - 0, 1706, 0, 1694, 0, 2987, 1695, 1696, 1697, 0, - 1698, 1699, 1700, 1701, 1702, 1703, 0, 0, 0, 0, - 2517, 0, 3001, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1693, 3017, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1706, 0, 0, - 0, 0, 0, 1138, 1706, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1683, 0, 0, 0, 0, + 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 568, 568, 568, + 568, 0, 568, 568, 568, 568, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1684, 0, 0, 0, 0, 1685, 0, 0, - 0, 1694, 0, 0, 1695, 1696, 1697, 0, 1698, 1699, - 1700, 1701, 1702, 1703, 1752, 1706, 0, 0, 2621, 0, - 1686, 1687, 0, 0, 0, 0, 0, 0, 2383, 2383, - 0, 0, 0, 0, 0, 0, 1688, 0, 0, 1706, - 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1694, 0, 0, 1695, 1696, 1697, - 3175, 1698, 1699, 1700, 1701, 1702, 1703, 0, 0, 0, - 0, 2635, 733, 0, 0, 0, 1689, 0, 0, 1690, - 0, 0, 0, 0, 0, 0, 0, 1706, 0, 1706, - 0, 1465, 1465, 1691, 2000, 0, 1692, 1465, 0, 0, - 1706, 0, 0, 1706, 0, 0, 0, 0, 1706, 0, - 0, 1706, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1236, 0, 0, 734, - 0, 0, 1237, 0, 0, 3236, 3237, 0, 3240, 1669, - 1249, 0, 1670, 0, 0, 735, 1671, 1672, 1673, 1674, - 0, 1675, 1676, 1677, 0, 1706, 0, 0, 0, 0, - 1250, 0, 0, 0, 0, 0, 0, 0, 1678, 0, - 0, 0, 3261, 0, 0, 0, 0, 0, 1680, 0, - 0, 0, 0, 0, 0, 1681, 0, 0, 0, 0, - 0, 0, 3270, 0, 0, 736, 1693, 3273, 3274, 0, - 0, 0, 3275, 0, 0, 737, 1251, 3278, 0, 0, - 3281, 3282, 1682, 0, 0, 2383, 1325, 0, 738, 3290, - 0, 0, 0, 739, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1138, 0, 1238, 0, 0, - 0, 0, 1706, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 740, 1669, 0, 0, 1670, 0, 0, 1706, - 1671, 1672, 1673, 1674, 0, 1675, 1676, 1677, 0, 0, - 1706, 1706, 1706, 0, 0, 0, 0, 1706, 0, 0, - 3338, 1706, 1678, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1680, 0, 0, 0, 741, 0, 0, 1681, - 0, 742, 0, 1252, 0, 0, 0, 3357, 1694, 0, - 1683, 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, 1702, - 1703, 0, 0, 0, 0, 2653, 1682, 1684, 0, 0, - 0, 0, 1685, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1706, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1253, 1686, 1687, 0, 0, 0, - 0, 0, 0, 0, 0, 1254, 0, 0, 0, 0, - 0, 1688, 0, 743, 0, 0, 0, 1255, 0, 0, - 0, 0, 0, 0, 1706, 0, 0, 0, 744, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1239, 0, - 1706, 0, 0, 0, 0, 1706, 0, 0, 0, 1256, - 0, 1689, 0, 0, 1690, 0, 0, 0, 0, 0, - 0, 0, 2000, 745, 1683, 0, 746, 0, 1691, 0, - 0, 1692, 0, 3414, 3415, 0, 2863, 747, 0, 0, - 748, 1684, 0, 0, 0, 0, 1685, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2356, 0, - 749, 1649, 1241, 0, 1258, 0, 0, 0, 0, 1686, - 1687, 0, 0, 0, 750, 0, 0, 0, 0, 0, - 0, 0, 752, 0, 0, 1688, 0, 0, 0, 1259, - 0, 0, 753, 0, 2357, 0, 2642, 0, 754, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 3364, 0, + 1390, 2512, 0, 0, 0, 0, 0, 0, 0, 2514, + 1676, 0, 0, 1677, 0, 0, 0, 1678, 1679, 1680, + 1681, 0, 1682, 1683, 1684, 0, 1690, 0, 0, 2530, + 2531, 0, 2532, 0, 0, 0, 0, 0, 0, 1685, + 0, 0, 0, 1691, 0, 0, 0, 0, 1692, 1687, + 0, 0, 0, 0, 0, 0, 1688, 0, 0, 0, + 0, 2558, 2559, 0, 0, 2256, 0, -48, 0, 0, + 0, 1693, 1694, 2072, 0, 0, 0, 0, 1471, 0, + 0, 0, 0, 1689, 0, 0, 0, 1695, 0, 0, + 1, 0, 0, 0, 0, 0, 2584, 0, 0, 0, + 2, 2600, 3, 4, 0, 2594, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, + 6, 0, 0, 0, 1530, 0, 1332, 1696, 0, 7, + 1697, 0, 0, 2160, 0, 0, 0, 0, 0, 0, + 0, 2160, 0, 8, 1698, 0, 0, 1699, 0, 0, + 0, 0, 9, 0, 10, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2629, 11, 0, 12, 0, + 0, 2636, 0, 0, 0, 823, 0, 0, 0, 13, + 0, 1690, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 14, 15, 16, 0, 1691, 0, + 0, 0, 0, 1692, 2648, 17, 0, 0, 2654, 0, + 0, 18, 0, 2659, 2660, 0, 0, 0, 0, 0, + 19, 0, 20, 21, 0, 0, 1693, 1694, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, + 568, 23, 1695, 0, 0, 0, 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1261, 0, 3457, 3458, 0, 0, 3459, 0, 1640, 0, - 0, 0, 0, 0, 0, 1689, 0, 755, 1690, 0, - 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1691, 0, 0, 1692, 1411, 1412, 0, 0, - 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1243, - 0, 0, 0, 0, 3495, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1413, 1414, 0, - 0, 1415, 1416, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1706, 0, 0, 0, 0, 0, - 0, 0, 2000, 2000, 0, 1465, 1465, 1465, 1465, 1465, - 1465, 0, 0, 1465, 1465, 1465, 1465, 1465, 1465, 1465, - 1465, 1465, 1465, 2000, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1694, 0, 1693, 1695, 1696, 1697, 0, - 1698, 1699, 1700, 1701, 1702, 1703, 0, 0, 0, 0, - 2880, 0, 0, 0, 0, 0, 3236, 0, 1417, 1418, - 3557, 0, 0, 0, 1138, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3567, 0, - 0, 0, 0, 2383, 2383, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1669, 1138, 0, 1670, 0, 0, - 0, 1671, 1672, 1673, 1674, 0, 1675, 1676, 1677, 0, - 0, 0, 0, 0, 0, 0, 0, 3595, 0, 0, - 0, 0, 0, 1678, 0, 1419, 1420, 1421, 1422, 1423, - 1424, 1425, 1426, 1680, 1706, 1427, 1428, 0, 1706, 0, - 1681, 0, 0, 0, 0, 0, 0, 1694, 0, 0, - 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, 1702, 1703, - 0, 0, 0, 0, 2947, 3620, 0, 1682, 0, 1706, - 0, 0, 0, 0, 0, 3236, 0, 0, 0, 0, - 0, 0, 0, 1706, 0, 0, 0, 1138, 1706, 0, - 0, 0, 1706, 1706, 1706, 1706, 1706, 1706, 1706, 1706, - 0, 0, 0, 0, 3647, 1465, 1465, 0, 1706, 1706, - 1429, 1430, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1706, 3667, 0, 1706, 0, 0, 0, 0, - 0, 0, 0, 1706, 1706, 1706, 1706, 1706, 1706, 1706, - 1706, 1706, 1706, 0, 0, 0, 0, 0, 0, 0, - 0, 1431, 1432, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1683, 0, 0, 1706, 0, + 2681, 0, 0, 2684, 0, 2686, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1684, 0, 0, 0, 0, 1685, 0, 0, - 0, 0, 0, 0, 0, 1669, 0, 0, 1670, 0, - 0, 0, 1671, 1672, 1673, 1674, 0, 1675, 1676, 1677, - 1686, 1687, 0, 0, 3730, 3730, 3730, 0, 0, 0, - 0, 1465, 0, 0, 1678, 3741, 1688, 0, 0, 0, - 0, 0, 0, 0, 1680, 0, 0, 0, 0, 0, - 0, 1681, 0, 0, 0, 0, 0, 0, 0, 0, - 3730, 1433, 1434, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1689, 0, 1682, 1690, - 0, 0, 0, 0, 0, 1435, 1436, 0, 0, 0, - 0, 0, 0, 1691, 0, 0, 1692, 0, 0, 0, - 0, 0, 0, 0, 0, 2080, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1706, 0, 0, 0, 3730, - 0, 0, 0, 0, 0, 0, 1706, 1706, 0, 0, + 0, 2690, 1696, -1524, 0, 1697, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1698, + 0, 0, 1699, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1759, 0, 0, 0, 0, 0, 0, 3530, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1780, 2228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3555, 0, 0, 1143, 0, 1701, + 0, 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, + 1709, 1710, 2750, 0, 0, 0, 2166, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 26, 27, + 28, 0, 0, 0, 0, 0, 29, 0, 0, 30, + 0, 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3598, 0, 0, 0, 0, 1676, + 31, 3456, 1677, 0, 0, 0, 1678, 1679, 2600, 32, + 1647, 1682, 1683, 1684, 0, 0, 0, 0, 0, 1332, + 0, 0, 0, 0, 3624, 33, 0, 0, 1685, 0, + 0, 0, 34, 1472, 1418, 1419, 35, 0, 1687, 0, + 0, 0, 0, 0, 0, 1688, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 38, 0, 0, 0, 0, 0, + 0, 0, 1689, 0, 0, 1420, 1421, 568, 0, 1422, + 1423, 0, 568, 0, 1701, 39, 0, 1702, 1703, 1704, + 0, 1705, 1706, 1707, 1708, 1709, 1710, 0, 0, 40, + 0, 2166, 41, 0, 0, 42, 0, 0, 568, 0, + 43, 0, 0, 0, 0, 0, 0, 2876, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 44, 0, 568, + 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1683, 0, 0, 0, + 0, 45, 0, 0, 568, 0, 1424, 1425, 0, 1713, + 0, 0, 0, 0, 0, 46, 0, 0, -48, 0, + 1690, 1676, 0, 0, 1677, 0, 0, 0, 1678, 1679, + 1680, 1681, 0, 1682, 1683, 1684, 0, 1691, 0, 0, + 0, 0, 1692, 0, 0, 0, 0, 0, 0, 568, + 1685, 0, 0, 0, 0, 0, 0, 2160, 1713, 0, + 1687, 0, 0, 0, 0, 1693, 1694, 1688, 0, 0, + 0, 0, 0, 1426, 1427, 1428, 1429, 1430, 1431, 1432, + 1433, 1695, 0, 1434, 1435, 0, 0, 0, 0, 0, + 0, 0, 0, 1472, 1689, 0, 2953, 0, 0, 0, + 0, 0, 1676, 2955, 2126, 1677, 0, 0, 0, 1678, + 1679, 1680, 1681, 0, 1682, 1683, 1684, 0, 0, 2964, + 0, 1696, 0, 0, 1697, 0, 0, 0, 0, 0, + 0, 1685, 2976, 0, 0, 2979, 0, 2981, 1698, 0, + 0, 1687, 0, 0, 0, 2985, 0, 0, 1688, 0, + 0, 0, 1713, 2992, 2993, 0, 0, 0, 1436, 1437, + 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1689, 0, 3014, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 3030, 0, + 0, 0, 1690, 0, 0, 0, 0, 0, 1713, 1438, + 1439, 0, 0, 0, 0, 1713, 0, 0, 1143, 1691, + 0, 0, 0, 0, 1692, 1676, 0, 0, 1677, 0, + 0, 0, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, + 0, 0, 0, 0, 0, 0, 0, 1693, 1694, 0, + 0, 1700, 0, 0, 1685, 0, 0, 0, 0, 0, + 568, 0, 0, 1695, 1687, 0, 1713, 0, 0, 0, + 1759, 1688, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1690, 2395, 2395, 0, 0, 0, 0, + 1713, 0, 0, 0, 0, 0, 0, 3175, 1689, 0, + 1691, 0, 0, 1696, 0, 1692, 1697, 0, 0, 1440, + 1441, 0, 0, 0, 0, 0, 3189, 0, 0, 0, + 1698, 0, 0, 1699, 0, 0, 0, 0, 1693, 1694, + 0, 0, 0, 1442, 1443, 0, 0, 0, 1713, 0, + 1713, 0, 1472, 1472, 1695, 2009, 0, 0, 1472, 0, + 0, 1713, 0, 2090, 1713, 0, 0, 0, 0, 1713, + 0, 0, 1713, 1701, 0, 0, 1702, 1703, 1704, 0, + 1705, 1706, 1707, 1708, 1709, 1710, 0, 0, 0, 0, + 0, 0, 0, 0, 1696, 0, 0, 1697, 0, 0, + 0, 3250, 3251, 0, 3254, 0, 1690, 0, 0, 0, + 0, 1698, 0, 0, 1699, 0, 1713, 0, 0, 0, + 0, 0, 0, 1691, 0, 0, 0, 0, 1692, 0, + 0, 0, 0, 1700, 0, 0, 0, 0, 3275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1684, 0, 0, 1693, 0, 1685, 0, + 0, 1693, 1694, 0, 0, 0, 0, 0, 3284, 0, + 0, 0, 0, 3287, 3288, 0, 0, 1695, 3289, 0, + 0, 0, 0, 3292, 0, 0, 3295, 3296, 0, 0, + 0, 2395, 1332, 0, 0, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1143, 0, 1713, 0, 0, 0, 1696, 0, 0, + 1697, 0, 0, 0, 1700, 0, 0, 0, 0, 0, + 1713, 0, 0, 0, 1698, 0, 0, 1699, 0, 0, + 0, 1713, 1713, 1713, 0, 0, 0, 0, 1713, 0, + 0, 0, 1713, 0, 0, 1701, 0, 3352, 1702, 1703, + 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, 0, 0, + 0, 0, 2529, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1686, 1687, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1706, 0, 1706, 0, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1706, 1706, 1706, 0, 0, 2000, 2000, 2000, 2000, 2000, - 2000, 0, 0, 0, 2000, 2000, 2000, 2000, 2000, 2000, - 2000, 2000, 2000, 2000, 0, 0, 0, 1689, 1706, 1706, - 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1691, 0, 0, 1692, 0, 0, - 0, 0, 0, 0, 1706, 0, 0, 0, 1694, 0, - 0, 1695, 1696, 1697, 1706, 1698, 1699, 1700, 1701, 1702, - 1703, 0, 0, 0, 0, 2960, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, - 0, 0, 1669, 0, 0, 1670, 1706, 0, 0, 1671, - 1672, 1673, 1674, 0, 1675, 1676, 1677, 0, 1706, 0, - 0, 0, 0, 0, 1706, 0, 0, 0, 0, 1706, - 1706, 1678, 0, 0, 0, 0, 0, 2000, 2000, 0, - 0, 1680, 0, 0, 0, 0, 0, 1693, 1681, 0, - 0, 1706, 1465, 1465, 1706, 1669, 1706, 0, 1670, 0, - 1706, 0, 1671, 1672, 1673, 1674, 0, 1675, 1676, 1677, - 0, 0, 0, 0, 0, 1682, 0, 0, 0, 0, - 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1680, 0, 0, 0, 0, 0, - 0, 1681, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1411, 1412, 0, 0, 1706, - 0, 0, 0, 0, 0, 0, 0, 0, 1682, 0, + 0, 0, 0, 0, 0, 0, 1701, 1700, 0, 1702, + 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, 1676, + 0, 0, 1677, 2633, 0, 1713, 1678, 1679, 1680, 1681, + 0, 1682, 1683, 1684, 0, 0, 0, 0, 0, 0, + 0, 1713, 0, 0, 0, 0, 1713, 0, 1685, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1687, 0, + 0, 0, 0, 2009, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 3428, 3429, 0, 2876, 0, 0, 0, 0, 0, 0, + 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1656, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1701, + 0, 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, + 1709, 1710, 1676, 0, 0, 1677, 2647, 0, 0, 1678, + 1679, 1680, 1681, 2654, 1682, 1683, 1684, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 3471, + 3472, 1685, 0, 3473, 0, 1647, 0, 0, 0, 0, + 0, 1687, 0, 0, 0, 0, 0, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1413, 1414, 0, 1694, - 1415, 1416, 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, - 1702, 1703, 0, 1683, 0, 0, 3033, 0, 0, 0, + 1690, 0, 0, 0, 0, 0, 0, 3497, 0, 0, + 0, 0, 0, 0, 0, 1689, 0, 1691, 0, 0, + 0, 0, 1692, 0, 0, 0, 0, 0, 0, 0, + 0, 3509, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1693, 1694, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1713, 0, 0, + 0, 1695, 0, 0, 0, 2009, 2009, 0, 1472, 1472, + 1472, 1472, 1472, 1472, 0, 0, 1472, 1472, 1472, 1472, + 1472, 1472, 1472, 1472, 1472, 1472, 2009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1684, 0, 0, 1669, 0, 1685, 1670, 0, 0, 0, - 1671, 1672, 1673, 1674, 0, 1675, 1676, 1677, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1686, 1687, - 0, 0, 1678, 0, 0, 0, 1683, 0, 0, 0, - 0, 0, 1680, 0, 1688, 0, 0, 1417, 1418, 1681, - 0, 0, 0, 1684, 0, 1706, 0, 1669, 1685, 0, - 1670, 0, 0, 0, 1671, 1672, 1673, 1674, 0, 1675, - 1676, 1677, 0, 0, 1465, 0, 1682, 0, 0, 0, - 0, 1686, 1687, 0, 1689, 0, 1678, 1690, 0, 0, - 0, 0, 0, 0, 0, 0, 1680, 1688, 0, 0, - 0, 1691, 0, 1681, 1692, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1419, 1420, 1421, 1422, 1423, 1424, - 1425, 1426, 0, 0, 1427, 1428, 0, 0, 0, 0, - 1682, 0, 1706, 0, 1706, 0, 0, 1689, 0, 0, - 1690, 0, 0, 1706, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1691, 1706, 0, 1692, 1706, 0, - 1706, 0, 0, 0, 1706, 0, 0, 2000, 2000, 0, - 0, 1706, 1706, 0, 1683, 0, 0, 0, 0, 1706, + 0, 1696, 0, 0, 1697, 0, 0, 0, 0, 0, + 0, 0, 0, 1690, 0, 0, 0, 0, 1698, 0, + 0, 1699, 0, 3250, 0, 0, 0, 3571, 0, 0, + 1691, 1143, 0, 0, 0, 1692, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 3581, 0, 0, 0, 0, + 2395, 2395, 0, 0, 0, 0, 0, 0, 1693, 1694, + 0, 0, 1143, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3609, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, + 0, 0, 1713, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1696, 0, 0, 1697, 0, 0, + 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1698, 3634, 1713, 1699, 0, 0, 0, 0, 0, + 0, 0, 3250, 0, 0, 0, 0, 1713, 0, 0, + 0, 0, 1713, 0, 1143, 0, 1713, 1713, 1713, 1713, + 1713, 1713, 1713, 1713, 0, 0, 0, 0, 0, 1472, + 1472, 3661, 1713, 1713, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1713, 0, 0, 1713, + 3681, 0, 0, 0, 0, 0, 0, 1713, 1713, 1713, + 1713, 1713, 1713, 1713, 1713, 1713, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1684, 0, 1706, 0, 0, 1685, 0, 0, 1429, - 1430, 0, 0, 0, 1693, 0, 0, 0, 0, 1706, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1686, - 1687, 0, 0, 0, 0, 0, 0, 0, 1683, 0, - 0, 0, 0, 0, 0, 1688, 0, 0, 0, 0, - 1431, 1432, 0, 0, 0, 1684, 0, 1465, 0, 0, - 1685, 0, 0, 0, 0, 0, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1686, 1687, 1689, 0, 0, 1690, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1688, - 0, 0, 1691, 0, 0, 1692, 0, 0, 0, 0, + 0, 0, 1713, 1701, 1700, 0, 1702, 1703, 1704, 0, + 1705, 1706, 1707, 1708, 1709, 1710, 0, 0, 0, 0, + 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1676, 0, 0, 1677, 0, 0, 0, 1678, 1679, + 1680, 1681, 0, 1682, 1683, 1684, 0, 0, 0, 0, + 0, 3744, 3744, 3744, 0, 1472, 0, 0, 0, 0, + 1685, 0, 3755, 0, 0, 0, 0, 0, 0, 0, + 1687, 0, 0, 0, 0, 0, 0, 1688, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1694, 0, 0, 1695, - 1696, 1697, 0, 1698, 1699, 1700, 1701, 1702, 1703, 1689, - 0, 0, 1690, 3284, 0, 0, 0, 0, 0, 0, - 1433, 1434, 0, 0, 0, 0, 1691, 0, 0, 1692, - 0, 0, 0, 1706, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1435, 1436, 0, 1706, 0, 1694, - 0, 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, - 1702, 1703, 0, 0, 0, 0, 3291, 0, 0, 0, - 0, 0, 0, 0, 0, 1693, 0, 0, 0, 0, - 0, 0, 0, 2000, 1465, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1689, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1701, 0, 0, 1702, + 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, 0, + 0, 0, 0, 2893, 0, 0, 0, 0, 0, 0, + 0, 1713, 0, 0, 0, 0, 3744, 0, 0, 0, + 0, 0, 1713, 1713, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1676, 0, 0, + 1677, 0, 0, 0, 1678, 1679, 1680, 1681, 0, 1682, + 1683, 1684, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1685, 0, 0, 0, + 0, 0, 1690, 0, 0, 0, 1687, 0, 0, 0, + 0, 0, 0, 1688, 0, 0, 0, 0, 0, 1691, + 0, 0, 0, 0, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1706, 1706, - 0, 0, 1706, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, - 0, 0, 0, 1706, 0, 0, 0, 0, 0, 0, - 0, 0, 1706, 0, 0, 1706, 1706, 1706, 0, 0, - 1706, 0, 0, 1706, 1706, 0, 0, 0, 0, 0, - 0, 0, 1706, 0, 0, 0, 0, 0, 0, 0, + 1689, 0, 0, 0, 0, 0, 0, 1693, 1694, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, + 1713, 0, 0, 1695, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1713, 1713, 1713, 0, + 0, 2009, 2009, 2009, 2009, 2009, 2009, 0, 0, 0, + 2009, 2009, 2009, 2009, 2009, 2009, 2009, 2009, 2009, 2009, + 0, 0, 0, 1696, 1713, 1713, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1694, 0, 0, - 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, 1702, 1703, - 0, 0, 0, 0, 3381, 0, 0, 0, 0, 0, - 1706, 0, 0, 0, 0, 0, 2000, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, + 1698, 0, 0, 1699, 0, 0, 0, 0, 0, 0, + 1713, 0, 0, 0, 1676, 0, 0, 1677, 1690, 0, + 1713, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, 0, + 0, 0, 0, 0, 0, 1691, 0, 0, 0, 0, + 1692, 0, 0, 1685, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1687, 0, 1713, 0, 0, 0, 0, + 1688, 0, 1713, 1693, 1694, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1713, 0, 0, 0, 0, 1695, + 1713, 0, 0, 0, 0, 1713, 1713, 1689, 0, 0, + 0, 0, 0, 2009, 2009, 0, 0, 0, 0, 0, + 0, 0, 0, 1700, 0, 0, 0, 1713, 1472, 1472, + 1713, 0, 1713, 0, 0, 0, 1713, 0, 0, 1696, + 1676, 0, 1697, 1677, 0, 0, 0, 1678, 1679, 1680, + 1681, 0, 1682, 1683, 1684, 0, 1698, 0, 0, 1699, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, + 0, 0, 0, 0, 0, 1676, 0, 0, 1677, 1687, + 0, 0, 1678, 1679, 1680, 1681, 1688, 1682, 1683, 1684, + 0, 0, 0, 0, 0, 0, 1713, 0, 0, 0, + 0, 0, 0, 0, 1685, 1690, 0, 0, 0, 0, + 0, 0, 0, 1689, 1687, 0, 0, 0, 0, 0, + 0, 1688, 1691, 0, 0, 0, 0, 1692, 0, 0, + 0, 0, 0, 0, 0, 1701, 0, 0, 1702, 1703, + 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, 1689, 0, + 1693, 1694, 2960, 0, 0, 0, 0, 0, 0, 1700, + 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1694, 0, 0, 1695, 1696, 1697, 0, 1698, 1699, - 1700, 1701, 1702, 1703, 0, 0, 0, 0, 3464, 0, + 0, 1676, 0, 0, 1677, 0, 0, 0, 1678, 1679, + 1680, 1681, 0, 1682, 1683, 1684, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1696, 0, 0, 1697, + 1685, 1690, 0, 0, 0, 0, 0, 0, 0, 0, + 1687, 0, 1713, 1698, 0, 0, 1699, 1688, 1691, 0, + 0, 0, 0, 1692, 0, 0, 0, 0, 0, 0, + 0, 1472, 0, 0, 0, 0, 1690, 0, 0, 0, + 0, 0, 0, 0, 1689, 0, 1693, 1694, 0, 0, + 0, 0, 0, 1691, 0, 0, 0, 0, 1692, 0, + 0, 1701, 1695, 0, 1702, 1703, 1704, 0, 1705, 1706, + 1707, 1708, 1709, 1710, 0, 0, 0, 0, 2973, 0, + 0, 1693, 1694, 0, 0, 0, 0, 0, 0, 1713, + 0, 1713, 0, 0, 0, 0, 0, 1695, 0, 0, + 1713, 0, 1696, 0, 0, 1697, 0, 0, 0, 0, + 0, 0, 1713, 0, 0, 1713, 1700, 1713, 0, 1698, + 0, 1713, 1699, 0, 2009, 2009, 0, 0, 1713, 1713, + 0, 0, 0, 0, 0, 0, 1713, 1696, 0, 0, + 1697, 0, 1690, 0, 0, 0, 0, 0, 0, 0, + 1713, 0, 0, 0, 1698, 0, 0, 1699, 0, 1691, + 0, 0, 0, 0, 1692, 0, 1713, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1676, 0, 0, + 1677, 0, 0, 0, 1678, 1679, 0, 1693, 1694, 1682, + 1683, 1684, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1695, 0, 1472, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1687, 0, 0, 0, + 0, 0, 1700, 1688, 0, 0, 0, 0, 1701, 0, + 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, + 1710, 0, 0, 1696, 0, 3047, 1697, 0, 0, 0, + 1689, 0, 0, 0, 0, 0, 0, 1700, 0, 0, + 1698, 0, 0, 1699, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1676, 0, 0, 1677, 0, 0, + 0, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1685, 0, 0, 0, 0, 0, 0, + 0, 1713, 0, 1687, 0, 0, 0, 0, 0, 0, + 1688, 0, 0, 0, 0, 1713, 0, 0, 0, 0, + 0, 0, 0, 0, 1701, 0, 0, 1702, 1703, 1704, + 0, 1705, 1706, 1707, 1708, 1709, 1710, 1689, 1690, 0, + 0, 3298, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2009, 1472, 1700, 0, 1691, 0, 0, 0, 1701, + 1692, 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, + 1709, 1710, 0, 0, 1876, 0, 1713, 1713, 0, 0, + 1713, 0, 0, -2183, -2183, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, + 0, 1713, 0, 0, 0, 0, 0, 0, 0, 0, + 1713, 0, 0, 1713, 1713, 1713, 0, 0, 1713, 0, + 0, 1713, 1713, 0, 0, 0, 0, 0, 0, 0, + 1713, 0, 0, 0, 0, 1690, 0, 0, 0, 0, + 0, 0, -2183, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1691, 0, 0, 0, 1698, 1692, 0, 0, + 0, 0, 0, 0, 0, 1701, 0, 0, 1702, 1703, + 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, 1713, 0, + 1693, 1694, 3305, 0, 2009, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1695, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1706, 1706, 0, 0, - 0, 0, 0, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, - 1706, 1706, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1696, 0, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1706, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1706, 0, 0, + 0, 0, 0, 1698, 1713, 1713, 1699, 0, 0, 1700, + 0, 2009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1713, 1713, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1713, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, + 0, 0, 0, 0, 0, 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1701, 0, 0, 1702, 1703, 1704, 0, 1705, 1706, + 1707, 1708, 1709, 1710, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1713, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, - 0, 0, 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 1077, - 0, 0, 131, 132, 133, 0, 134, 135, 136, 137, - 138, 139, 140, 141, 1078, 143, 1079, 1080, 0, 0, - 146, 147, 148, 149, 150, 1081, 802, 151, 152, 153, - 154, 1082, 1083, 157, 0, 158, 159, 160, 161, 803, - 0, 804, 1706, 1084, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 1706, 174, 175, 176, 177, 178, 179, - 0, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 1085, 192, 193, 1086, 195, 1087, 196, 0, - 197, 198, 199, 200, 201, 202, 14, 15, 203, 204, - 205, 206, 0, 0, 207, 208, 1088, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 1089, 224, 225, 226, 227, 228, 229, - 805, 1090, 231, 0, 232, 233, 1091, 235, 0, 236, - 0, 237, 238, 23, 239, 240, 241, 242, 243, 244, - 245, 246, 0, 1092, 1093, 249, 250, 0, 251, 252, + 1713, 0, 0, 0, 0, 0, 0, 0, 1701, 0, + 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, + 1710, 0, 0, 0, 0, 3395, 0, 1713, 0, 0, + 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1713, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 1082, 0, 0, + 133, 134, 135, 0, 136, 137, 138, 139, 140, 141, + 142, 143, 1083, 145, 1084, 1085, 0, 0, 148, 149, + 150, 151, 152, 1086, 805, 153, 154, 155, 156, 1087, + 1088, 159, 0, 160, 161, 162, 163, 806, 0, 807, + 1713, 1089, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 1713, 176, 177, 178, 179, 180, 181, 0, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 1090, 194, 195, 1091, 197, 1092, 198, 0, 199, 200, + 201, 202, 203, 204, 14, 15, 205, 206, 207, 208, + 0, 0, 209, 210, 1093, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 1094, 226, 227, 228, 229, 230, 231, 808, 1095, + 233, 0, 234, 235, 1096, 237, 0, 238, 0, 239, + 240, 23, 241, 242, 243, 244, 245, 246, 247, 248, + 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 273, 274, + 275, 276, 277, 278, 279, 1099, 1100, 0, 1101, 0, + 283, 284, 285, 286, 287, 288, 289, 290, 1102, 291, + 292, 293, 0, 0, 294, 295, 296, 297, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 1104, 330, 1105, 332, 333, 334, 335, 1106, 336, 337, + 338, 339, 1107, 810, 341, 1108, 343, 344, 345, 0, + 346, 347, 0, 0, 1109, 349, 350, 0, 0, 351, + 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 26, 27, + 28, 0, 370, 371, 813, 373, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 1110, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, + 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 33, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 35, 438, 439, 440, + 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 815, + 37, 0, 460, 461, 38, 462, 463, 464, 465, 466, + 467, 468, 469, 0, 470, 1113, 1114, 0, 0, 473, + 474, 816, 476, 817, 1115, 478, 479, 818, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 1116, 40, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 819, 1117, 500, 0, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 44, 511, 512, + 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 1118, + 0, 45, 0, 0, 0, 0, 1119, 1120, 1121, 0, + 0, 0, 0, 1122, 0, 1123, 3446, 0, 0, 0, + 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, 1078, 1079, + 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 1082, 0, 0, 133, 134, 135, 0, 136, 137, + 138, 139, 140, 141, 142, 143, 1083, 145, 1084, 1085, + 0, 0, 148, 149, 150, 151, 152, 1086, 805, 153, + 154, 155, 156, 1087, 1088, 159, 0, 160, 161, 162, + 163, 806, 0, 807, 0, 1089, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 1090, 194, 195, 1091, 197, 1092, + 198, 0, 199, 200, 201, 202, 203, 204, 14, 15, + 205, 206, 207, 208, 0, 0, 209, 210, 1093, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 1094, 226, 227, 228, 229, + 230, 231, 808, 1095, 233, 0, 234, 235, 1096, 237, + 0, 238, 0, 239, 240, 23, 241, 242, 243, 244, + 245, 246, 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 271, 272, 273, 274, 275, 276, 277, 1094, 1095, 0, - 1096, 0, 281, 282, 283, 284, 285, 286, 287, 288, - 1097, 289, 290, 291, 0, 0, 292, 293, 294, 295, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 1098, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 273, 274, 275, 276, 277, 278, 279, 1099, + 1100, 0, 1101, 0, 283, 284, 285, 286, 287, 288, + 289, 290, 1102, 291, 292, 293, 0, 0, 294, 295, + 296, 297, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 1099, 328, 1100, 330, 331, 332, 333, 1101, - 334, 335, 336, 337, 1102, 807, 339, 1103, 341, 342, - 343, 0, 344, 345, 0, 0, 1104, 347, 348, 0, - 0, 349, 350, 351, 352, 353, 354, 809, 356, 357, + 325, 326, 327, 328, 1104, 330, 1105, 332, 333, 334, + 335, 1106, 336, 337, 338, 339, 1107, 810, 341, 1108, + 343, 344, 345, 0, 346, 347, 0, 0, 1109, 349, + 350, 0, 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 26, 27, 28, 0, 368, 369, 810, 371, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 1105, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 409, - 410, 411, 412, 1106, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 33, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 35, 436, - 437, 438, 1107, 440, 0, 441, 442, 443, 444, 445, + 368, 369, 26, 27, 28, 0, 370, 371, 813, 373, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 1110, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 33, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 35, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 812, 37, 0, 458, 459, 38, 460, 461, 462, - 463, 464, 465, 466, 467, 0, 468, 1108, 1109, 0, - 0, 471, 472, 813, 474, 814, 1110, 476, 477, 815, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 1111, 40, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 816, 1112, 498, 0, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 44, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 456, 457, 458, 815, 37, 0, 460, 461, 38, 462, + 463, 464, 465, 466, 467, 468, 469, 0, 470, 1113, + 1114, 0, 0, 473, 474, 816, 476, 817, 1115, 478, + 479, 818, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 1116, 40, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 819, 1117, 500, 0, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 44, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 1113, 0, 45, 0, 0, 0, 0, 1114, 1115, - 1116, 0, 0, 0, 0, 1117, 0, 1118, 3432, 0, - 0, 0, 1119, 1120, 1121, 1122, 119, 1071, 833, 1072, - 1073, 1074, 1075, 1076, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 1077, 0, 0, 131, 132, 133, 0, - 134, 135, 136, 137, 138, 139, 140, 141, 1078, 143, - 1079, 1080, 0, 0, 146, 147, 148, 149, 150, 1081, - 802, 151, 152, 153, 154, 1082, 1083, 157, 0, 158, - 159, 160, 161, 803, 0, 804, 0, 1084, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 1085, 192, 193, 1086, - 195, 1087, 196, 0, 197, 198, 199, 200, 201, 202, - 14, 15, 203, 204, 205, 206, 0, 0, 207, 208, - 1088, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 1089, 224, 225, - 226, 227, 228, 229, 805, 1090, 231, 0, 232, 233, - 1091, 235, 0, 236, 0, 237, 238, 23, 239, 240, - 241, 242, 243, 244, 245, 246, 0, 1092, 1093, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 271, 272, 273, 274, 275, 276, - 277, 1094, 1095, 0, 1096, 0, 281, 282, 283, 284, - 285, 286, 287, 288, 1097, 289, 290, 291, 0, 0, - 292, 293, 294, 295, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 1098, 305, 306, 307, 308, 309, 310, + 529, 530, 531, 1118, 0, 45, 0, 0, 0, 0, + 1119, 1120, 1121, 0, 0, 0, 0, 1122, 0, 1123, + 0, 0, 0, 0, 1124, 1125, 1126, 1127, 121, 1076, + 836, 1077, 1078, 1079, 1080, 1081, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 1082, 0, 0, 133, 134, + 135, 0, 136, 137, 138, 139, 140, 141, 142, 143, + 1083, 145, 1084, 1085, 0, 0, 148, 149, 150, 151, + 152, 1086, 805, 153, 154, 155, 156, 1087, 1088, 159, + 0, 160, 161, 162, 163, 806, 0, 807, 0, 1089, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 1090, 194, + 195, 1091, 197, 1092, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 1093, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 1094, + 226, 227, 228, 229, 230, 231, 808, 1095, 233, 0, + 234, 235, 1096, 237, 0, 238, 0, 239, 240, 23, + 241, 242, 243, 244, 245, 246, 247, 248, 0, 1097, + 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 273, 274, 275, 276, + 277, 278, 279, 1099, 1100, 0, 1101, 0, 283, 284, + 285, 286, 287, 288, 289, 290, 1102, 291, 292, 293, + 0, 0, 294, 295, 296, 297, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 1099, 328, 1100, 330, - 331, 332, 333, 1101, 334, 335, 336, 337, 1102, 807, - 339, 1103, 341, 342, 343, 0, 344, 345, 0, 0, - 1104, 347, 348, 0, 0, 349, 350, 351, 352, 353, - 354, 809, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 26, 27, 28, 0, 368, 369, - 810, 371, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 1105, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 1104, 330, + 1105, 332, 333, 334, 335, 1106, 336, 337, 338, 339, + 1107, 810, 341, 1108, 343, 344, 345, 0, 346, 347, + 0, 0, 1109, 349, 350, 0, 0, 351, 352, 353, + 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 26, 27, 28, 0, + 370, 371, 813, 373, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 409, 410, 411, 412, 1106, 414, 415, + 407, 408, 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 33, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 35, 436, 437, 438, 1107, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 812, 37, 0, 458, 459, - 38, 460, 461, 462, 463, 464, 465, 466, 467, 0, - 468, 1108, 1109, 0, 0, 471, 472, 813, 474, 814, - 1110, 476, 477, 815, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 1111, 40, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 816, 1112, 498, 0, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 44, 509, 510, 511, 512, 513, 514, + 426, 427, 428, 33, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 438, 439, 440, 1112, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 457, 458, 815, 0, 0, + 460, 461, 38, 462, 463, 464, 465, 466, 467, 468, + 469, 0, 470, 1113, 1114, 0, 0, 473, 474, 816, + 476, 817, 1115, 478, 479, 818, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 1116, 40, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 819, 1117, + 500, 0, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 44, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 1113, 0, 45, 0, 0, - 0, 0, 1114, 1115, 1116, 0, 0, 0, 0, 1117, - 0, 1118, 0, 0, 0, 0, 1119, 1120, 1121, 1122, - 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, 0, 0, + 525, 526, 527, 528, 529, 530, 531, 1118, 0, 45, + 0, 0, 0, 0, 1119, 1120, 1121, 0, 0, 0, + 0, 1122, 0, 1123, 0, 0, 0, 0, 1124, 1125, + 1126, 1127, 1291, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 1077, 0, 0, - 131, 132, 133, 0, 134, 135, 136, 137, 138, 139, - 140, 141, 1078, 143, 1079, 1080, 0, 0, 146, 147, - 148, 149, 150, 1081, 802, 151, 152, 153, 154, 1082, - 1083, 157, 0, 158, 159, 160, 161, 803, 0, 804, - 0, 1084, 165, 166, 167, 168, 169, 170, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 1085, 192, 193, 1086, 195, 1087, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 1088, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 1089, 224, 225, 226, 227, 228, 229, 805, 1090, - 231, 0, 232, 233, 1091, 235, 0, 236, 0, 237, - 238, 23, 239, 240, 241, 242, 243, 244, 245, 246, - 0, 1092, 1093, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 271, 272, - 273, 274, 275, 276, 277, 1094, 1095, 0, 1096, 0, - 281, 282, 283, 284, 285, 286, 287, 288, 1097, 289, - 290, 291, 0, 0, 292, 293, 294, 295, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 1098, 305, 306, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 1292, 130, 131, 132, 0, 0, 0, 1293, 0, 1082, + 0, 0, 1294, 134, 135, 0, 1295, 137, 138, 1296, + 140, 141, 142, 143, 1083, 1297, 1084, 1085, 0, 1298, + 148, 149, 150, 151, 152, 1086, 805, 153, 154, 155, + 156, 1087, 1088, 159, 0, 160, 161, 162, 163, 806, + 0, 1299, 0, 1300, 167, 168, 169, 170, 171, 1301, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 1302, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 1090, 194, 195, 1091, 197, 1092, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 1303, 0, 209, 210, 1093, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 1094, 226, 227, 228, 229, 230, 231, + 808, 1095, 233, 0, 234, 235, 1096, 237, 0, 238, + 0, 239, 1304, 0, 1305, 242, 243, 1306, 1307, 246, + 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 1308, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 1309, 274, 275, 276, 277, 278, 279, 1099, 1100, 0, + 1101, 0, 283, 1310, 1311, 286, 1312, 288, 289, 290, + 1102, 291, 292, 293, 0, 0, 294, 1313, 296, 1314, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 1315, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 1099, 328, 1100, 330, 331, 332, 333, 1101, 334, 335, - 336, 337, 1102, 807, 339, 1103, 341, 342, 343, 0, - 344, 345, 0, 0, 1104, 347, 348, 0, 0, 349, - 350, 351, 352, 353, 354, 809, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 26, 27, - 28, 0, 368, 369, 810, 371, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 1105, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 409, 410, 411, - 412, 1106, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 33, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 436, 437, 438, - 1107, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 812, - 0, 0, 458, 459, 38, 460, 461, 462, 463, 464, - 465, 466, 467, 0, 468, 1108, 1109, 0, 0, 471, - 472, 813, 474, 814, 1110, 476, 477, 815, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 1111, 40, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 816, 1112, 498, 0, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 44, 509, 510, + 327, 328, 1104, 1316, 1105, 332, 333, 334, 335, 1106, + 336, 337, 1317, 339, 1107, 810, 341, 1108, 343, 344, + 345, 0, 346, 347, 0, 0, 1109, 349, 350, 0, + 897, 351, 352, 353, 1318, 355, 1319, 812, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 813, 1320, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 1110, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 1321, + 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 1322, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 1323, + 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 1324, + 458, 815, 0, 0, 460, 461, 0, 462, 1325, 464, + 465, 466, 467, 468, 469, 0, 470, 1113, 1114, 0, + 0, 473, 474, 816, 476, 817, 1115, 478, 479, 1326, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 1327, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 1117, 500, 1328, 501, 1329, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 1113, - 0, 45, 0, 0, 0, 0, 1114, 1115, 1116, 0, - 0, 0, 0, 1117, 0, 1118, 0, 0, 0, 0, - 1119, 1120, 1121, 1122, 1284, 1071, 833, 1072, 1073, 1074, - 1075, 1076, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 1285, 128, 129, 130, 0, 0, 0, 1286, - 0, 1077, 0, 0, 1287, 132, 133, 0, 1288, 135, - 136, 1289, 138, 139, 140, 141, 1078, 1290, 1079, 1080, - 0, 1291, 146, 147, 148, 149, 150, 1081, 802, 151, - 152, 153, 154, 1082, 1083, 157, 0, 158, 159, 160, - 161, 803, 0, 1292, 0, 1293, 165, 166, 167, 168, - 169, 1294, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 1295, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 1085, 192, 193, 1086, 195, 1087, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 1296, 0, 207, 208, 1088, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 1089, 224, 225, 226, 227, - 228, 229, 805, 1090, 231, 0, 232, 233, 1091, 235, - 0, 236, 0, 237, 1297, 0, 1298, 240, 241, 1299, - 1300, 244, 245, 246, 0, 1092, 1093, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 1301, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 1302, 272, 273, 274, 275, 276, 277, 1094, - 1095, 0, 1096, 0, 281, 1303, 1304, 284, 1305, 286, - 287, 288, 1097, 289, 290, 291, 0, 0, 292, 1306, - 294, 1307, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 1308, 305, 306, 307, 308, 309, 310, 311, 312, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 1118, 0, 0, 0, 0, 0, 0, 1119, 1120, + 1121, 0, 0, 0, 0, 1122, 0, 1123, 1330, 0, + 0, 0, 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, + 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 1082, 0, 0, 133, 134, 135, 0, + 136, 137, 138, 139, 140, 141, 142, 143, 1083, 145, + 1084, 1085, 1479, 0, 148, 149, 150, 151, 152, 1086, + 805, 153, 154, 155, 156, 1087, 1088, 159, 0, 160, + 161, 162, 163, 806, 0, 807, 0, 1089, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 1090, 194, 195, 1091, + 197, 1092, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 1093, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 1094, 226, 227, + 228, 229, 230, 231, 808, 1095, 233, 0, 234, 235, + 1096, 237, 0, 238, 0, 239, 240, 1480, 241, 242, + 243, 244, 245, 246, 247, 248, 0, 1097, 1098, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 273, 274, 275, 276, 277, 278, + 279, 1099, 1100, 0, 1101, 0, 283, 284, 285, 286, + 287, 288, 289, 290, 1102, 291, 292, 293, 0, 1481, + 294, 295, 296, 297, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 1099, 1309, 1100, 330, 331, 332, - 333, 1101, 334, 335, 1310, 337, 1102, 807, 339, 1103, - 341, 342, 343, 0, 344, 345, 0, 0, 1104, 347, - 348, 0, 894, 349, 350, 351, 1311, 353, 1312, 809, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 810, 1313, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 1105, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 1314, 410, 411, 412, 1106, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 1315, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 1316, 437, 438, 1107, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 1104, 330, 1105, 332, + 333, 334, 335, 1106, 336, 337, 338, 339, 1107, 810, + 341, 1108, 343, 344, 345, 0, 346, 347, 0, 0, + 1109, 349, 350, 0, 0, 351, 352, 353, 354, 355, + 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 813, 373, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 1317, 456, 812, 0, 0, 458, 459, 0, 460, - 1318, 462, 463, 464, 465, 466, 467, 0, 468, 1108, - 1109, 0, 0, 471, 472, 813, 474, 814, 1110, 476, - 477, 1319, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 1320, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 1112, 498, 1321, 499, 1322, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 457, 458, 815, 0, 0, 460, 461, + 0, 462, 463, 464, 465, 466, 467, 468, 469, 0, + 470, 1113, 1114, 0, 1482, 473, 474, 816, 476, 817, + 1115, 478, 479, 818, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 1116, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 1117, 500, 0, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 1113, 0, 0, 0, 0, 0, 0, - 1114, 1115, 1116, 0, 0, 0, 0, 1117, 0, 1118, - 1323, 0, 0, 0, 1119, 1120, 1121, 1122, 119, 1071, - 833, 1072, 1073, 1074, 1075, 1076, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 1077, 0, 0, 131, 132, - 133, 0, 134, 135, 136, 137, 138, 139, 140, 141, - 1078, 143, 1079, 1080, 1472, 0, 146, 147, 148, 149, - 150, 1081, 802, 151, 152, 153, 154, 1082, 1083, 157, - 0, 158, 159, 160, 161, 803, 0, 804, 0, 1084, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 1085, 192, - 193, 1086, 195, 1087, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 1088, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 1089, - 224, 225, 226, 227, 228, 229, 805, 1090, 231, 0, - 232, 233, 1091, 235, 0, 236, 0, 237, 238, 1473, - 239, 240, 241, 242, 243, 244, 245, 246, 0, 1092, - 1093, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 271, 272, 273, 274, - 275, 276, 277, 1094, 1095, 0, 1096, 0, 281, 282, - 283, 284, 285, 286, 287, 288, 1097, 289, 290, 291, - 0, 1474, 292, 293, 294, 295, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 1098, 305, 306, 307, 308, + 527, 528, 529, 530, 531, 1118, 0, 0, 0, 0, + 0, 0, 1119, 1120, 1121, 0, 0, 0, 0, 1122, + 0, 1123, 0, 0, 0, 0, 1124, 1125, 1126, 1127, + 1291, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 1293, 0, 1082, 0, 0, + 1294, 134, 135, 0, 1295, 137, 138, 1296, 140, 141, + 142, 143, 1083, 1297, 1084, 1085, 0, 1298, 148, 149, + 150, 151, 152, 1086, 805, 153, 154, 155, 156, 1087, + 1088, 159, 0, 160, 161, 162, 163, 806, 0, 1299, + 0, 1300, 167, 168, 169, 170, 171, 1301, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 1302, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 1090, 194, 195, 1091, 197, 1092, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 1093, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 1094, 226, 227, 228, 229, 230, 231, 808, 1095, + 233, 0, 234, 235, 1096, 237, 0, 238, 0, 239, + 1304, 0, 1305, 242, 243, 1306, 1307, 246, 247, 248, + 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 1308, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 1309, 274, + 275, 276, 277, 278, 279, 1099, 1100, 0, 1101, 0, + 283, 1310, 1311, 286, 1312, 288, 289, 290, 1102, 291, + 292, 293, 0, 0, 294, 1313, 296, 1314, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 1315, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 1099, 328, - 1100, 330, 331, 332, 333, 1101, 334, 335, 336, 337, - 1102, 807, 339, 1103, 341, 342, 343, 0, 344, 345, - 0, 0, 1104, 347, 348, 0, 0, 349, 350, 351, - 352, 353, 354, 809, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 810, 371, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 1105, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 409, 410, 411, 412, 1106, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 436, 437, 438, 1107, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 455, 456, 812, 0, 0, - 458, 459, 0, 460, 461, 462, 463, 464, 465, 466, - 467, 0, 468, 1108, 1109, 0, 1475, 471, 472, 813, - 474, 814, 1110, 476, 477, 815, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 1111, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 1112, - 498, 0, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 1104, 1316, 1105, 332, 333, 334, 335, 1106, 336, 337, + 1317, 339, 1107, 810, 341, 1108, 343, 344, 345, 0, + 346, 347, 0, 0, 1109, 349, 350, 0, 0, 351, + 352, 353, 1318, 355, 1319, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 813, 1320, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 1110, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 1321, 412, 413, + 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 1322, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 1323, 439, 440, + 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 1324, 458, 815, + 0, 0, 460, 461, 0, 462, 1325, 464, 465, 466, + 467, 468, 469, 0, 470, 1113, 1114, 0, 0, 473, + 474, 816, 476, 817, 1115, 478, 479, 1326, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 1327, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 1117, 500, 2403, 501, 1329, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 1113, 0, 0, - 0, 0, 0, 0, 1114, 1115, 1116, 0, 0, 0, - 0, 1117, 0, 1118, 0, 0, 0, 0, 1119, 1120, - 1121, 1122, 1284, 1071, 833, 1072, 1073, 1074, 1075, 1076, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 1286, 0, 1077, - 0, 0, 1287, 132, 133, 0, 1288, 135, 136, 1289, - 138, 139, 140, 141, 1078, 1290, 1079, 1080, 0, 1291, - 146, 147, 148, 149, 150, 1081, 802, 151, 152, 153, - 154, 1082, 1083, 157, 0, 158, 159, 160, 161, 803, - 0, 1292, 0, 1293, 165, 166, 167, 168, 169, 1294, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 1295, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 1085, 192, 193, 1086, 195, 1087, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 1088, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 1089, 224, 225, 226, 227, 228, 229, - 805, 1090, 231, 0, 232, 233, 1091, 235, 0, 236, - 0, 237, 1297, 0, 1298, 240, 241, 1299, 1300, 244, - 245, 246, 0, 1092, 1093, 249, 250, 0, 251, 252, - 253, 254, 255, 256, 257, 1301, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 1302, 272, 273, 274, 275, 276, 277, 1094, 1095, 0, - 1096, 0, 281, 1303, 1304, 284, 1305, 286, 287, 288, - 1097, 289, 290, 291, 0, 0, 292, 1306, 294, 1307, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 1308, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 1118, + 0, 0, 0, 0, 0, 0, 1119, 1120, 1121, 0, + 0, 0, 0, 1122, 0, 1123, 0, 0, 0, 0, + 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, 1078, 0, + 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 1082, 0, 0, 133, 134, 135, 0, 136, 137, + 138, 139, 140, 141, 142, 143, 1083, 145, 1084, 1085, + 0, 0, 148, 149, 150, 151, 152, 1086, 805, 153, + 154, 155, 156, 1087, 1088, 159, 0, 160, 161, 162, + 163, 806, 0, 807, 0, 166, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 1090, 194, 195, 1091, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 14, 15, + 205, 206, 207, 208, 0, 0, 209, 210, 1093, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 1094, 226, 227, 228, 229, + 230, 231, 808, 1095, 233, 0, 234, 235, 1096, 237, + 0, 238, 0, 239, 240, 23, 241, 242, 243, 244, + 245, 246, 247, 248, 0, 1097, 1098, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 273, 274, 275, 276, 277, 278, 279, 1099, + 1100, 0, 1101, 0, 283, 284, 285, 286, 287, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 295, + 296, 297, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 1099, 1309, 1100, 330, 331, 332, 333, 1101, - 334, 335, 1310, 337, 1102, 807, 339, 1103, 341, 342, - 343, 0, 344, 345, 0, 0, 1104, 347, 348, 0, - 0, 349, 350, 351, 1311, 353, 1312, 809, 356, 357, + 325, 326, 327, 328, 1104, 330, 1105, 332, 333, 334, + 335, 0, 336, 337, 338, 339, 1107, 810, 341, 1108, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 810, 1313, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 1105, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 1314, - 410, 411, 412, 1106, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 1315, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 1316, - 437, 438, 1107, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 1317, - 456, 812, 0, 0, 458, 459, 0, 460, 1318, 462, - 463, 464, 465, 466, 467, 0, 468, 1108, 1109, 0, - 0, 471, 472, 813, 474, 814, 1110, 476, 477, 1319, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 1320, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 1112, 498, 2391, 499, 1322, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 368, 369, 26, 27, 28, 0, 370, 371, 813, 373, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 1110, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 33, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 35, 438, 439, 440, 1112, 442, 0, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 457, 458, 815, 37, 0, 460, 461, 38, 462, + 463, 464, 465, 466, 467, 468, 469, 0, 470, 1113, + 1114, 0, 0, 473, 474, 816, 476, 817, 1115, 478, + 479, 818, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 40, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 819, 1117, 500, 0, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 44, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 1113, 0, 0, 0, 0, 0, 0, 1114, 1115, - 1116, 0, 0, 0, 0, 1117, 0, 1118, 0, 0, - 0, 0, 1119, 1120, 1121, 1122, 119, 1071, 833, 1072, - 1073, 0, 1075, 1076, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 1077, 0, 0, 131, 132, 133, 0, - 134, 135, 136, 137, 138, 139, 140, 141, 1078, 143, - 1079, 1080, 0, 0, 146, 147, 148, 149, 150, 1081, - 802, 151, 152, 153, 154, 1082, 1083, 157, 0, 158, - 159, 160, 161, 803, 0, 804, 0, 164, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 1085, 192, 193, 1086, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 14, 15, 203, 204, 205, 206, 0, 0, 207, 208, - 1088, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 1089, 224, 225, - 226, 227, 228, 229, 805, 1090, 231, 0, 232, 233, - 1091, 235, 0, 236, 0, 237, 238, 23, 239, 240, - 241, 242, 243, 244, 245, 246, 0, 1092, 1093, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 271, 272, 273, 274, 275, 276, - 277, 1094, 1095, 0, 1096, 0, 281, 282, 283, 284, - 285, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 293, 294, 295, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 1098, 305, 306, 307, 308, 309, 310, + 529, 530, 531, 0, 0, 45, 0, 0, 1291, 1076, + 836, 1077, 1078, 1079, 1080, 1081, 0, 1122, 0, 1123, + 0, 0, 0, 0, 1124, 1125, 1126, 1127, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 1293, 0, 1082, 0, 0, 1294, 134, + 135, 0, 1295, 137, 138, 1296, 140, 141, 142, 143, + 1083, 1297, 1084, 1085, 0, 1298, 148, 149, 150, 151, + 152, 1086, 805, 153, 154, 155, 156, 1087, 1088, 159, + 0, 160, 161, 162, 163, 806, 0, 1299, 0, 1300, + 167, 168, 169, 170, 171, 1301, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 1302, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 1090, 194, + 195, 1091, 197, 1092, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 1093, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 1094, + 226, 227, 228, 229, 230, 231, 808, 1095, 233, 0, + 234, 235, 1096, 237, 0, 238, 0, 239, 1304, 0, + 1305, 242, 243, 1306, 1307, 246, 247, 248, 0, 1097, + 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 1308, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 1309, 274, 275, 276, + 277, 278, 279, 1099, 1100, 0, 1101, 0, 283, 1310, + 1311, 286, 1312, 288, 289, 290, 1102, 291, 292, 293, + 0, 0, 294, 1313, 296, 1314, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 1315, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 1099, 328, 1100, 330, - 331, 332, 333, 0, 334, 335, 336, 337, 1102, 807, - 339, 1103, 341, 342, 343, 0, 344, 345, 0, 0, - 346, 347, 348, 0, 0, 349, 350, 351, 352, 353, - 354, 809, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 26, 27, 28, 0, 368, 369, - 810, 371, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 1105, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 1104, 1316, + 1105, 332, 333, 334, 335, 1106, 336, 337, 1317, 339, + 1107, 810, 341, 1108, 343, 344, 345, 0, 346, 347, + 0, 0, 1109, 349, 350, 0, 0, 351, 352, 353, + 1318, 355, 1319, 812, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 813, 1320, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 409, 410, 411, 412, 1106, 414, 415, + 407, 408, 0, 409, 410, 1321, 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 33, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 35, 436, 437, 438, 1107, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 812, 37, 0, 458, 459, - 38, 460, 461, 462, 463, 464, 465, 466, 467, 0, - 468, 1108, 1109, 0, 0, 471, 472, 813, 474, 814, - 1110, 476, 477, 815, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 40, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 816, 1112, 498, 0, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 44, 509, 510, 511, 512, 513, 514, + 426, 427, 428, 0, 1322, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 1323, 439, 440, 1112, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 1324, 458, 815, 0, 0, + 460, 461, 0, 462, 1325, 464, 465, 466, 467, 468, + 469, 0, 470, 1113, 1114, 0, 0, 473, 474, 816, + 476, 817, 1115, 478, 479, 1326, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 1327, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 1117, + 500, 0, 501, 1329, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 0, 0, 45, 0, 0, - 1284, 1071, 833, 1072, 1073, 1074, 1075, 1076, 0, 1117, - 0, 1118, 0, 0, 0, 0, 1119, 1120, 1121, 1122, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 1286, 0, 1077, 0, 0, - 1287, 132, 133, 0, 1288, 135, 136, 1289, 138, 139, - 140, 141, 1078, 1290, 1079, 1080, 0, 1291, 146, 147, - 148, 149, 150, 1081, 802, 151, 152, 153, 154, 1082, - 1083, 157, 0, 158, 159, 160, 161, 803, 0, 1292, - 0, 1293, 165, 166, 167, 168, 169, 1294, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 1295, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 1085, 192, 193, 1086, 195, 1087, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 1088, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 1089, 224, 225, 226, 227, 228, 229, 805, 1090, - 231, 0, 232, 233, 1091, 235, 0, 236, 0, 237, - 1297, 0, 1298, 240, 241, 1299, 1300, 244, 245, 246, - 0, 1092, 1093, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 1301, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 1302, 272, - 273, 274, 275, 276, 277, 1094, 1095, 0, 1096, 0, - 281, 1303, 1304, 284, 1305, 286, 287, 288, 1097, 289, - 290, 291, 0, 0, 292, 1306, 294, 1307, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 1308, 305, 306, + 525, 526, 527, 528, 529, 530, 531, 1118, 0, 0, + 0, 0, 0, 0, 1119, 1120, 1121, 0, 0, 0, + 0, 1122, 0, 1123, 3301, 0, 0, 0, 1124, 1125, + 1126, 1127, 1291, 1076, 836, 1077, 1078, 1079, 1080, 1081, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 1293, 0, 1082, + 0, 0, 1294, 134, 135, 0, 1295, 137, 138, 1296, + 140, 141, 142, 143, 1083, 1297, 1084, 1085, 0, 1298, + 148, 149, 150, 151, 152, 1086, 805, 153, 154, 155, + 156, 1087, 1088, 159, 0, 160, 161, 162, 163, 806, + 0, 1299, 0, 1300, 167, 168, 169, 170, 171, 1301, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 1302, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 1090, 194, 195, 1091, 197, 1092, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 1093, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 1094, 226, 227, 228, 229, 230, 231, + 808, 1095, 233, 0, 234, 235, 1096, 237, 0, 238, + 0, 239, 1304, 0, 1305, 242, 243, 1306, 1307, 246, + 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 1308, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 1309, 274, 275, 276, 277, 278, 279, 1099, 1100, 0, + 1101, 0, 283, 1310, 1311, 286, 1312, 288, 289, 290, + 1102, 291, 292, 293, 0, 0, 294, 1313, 296, 1314, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 1315, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 1099, 1309, 1100, 330, 331, 332, 333, 1101, 334, 335, - 1310, 337, 1102, 807, 339, 1103, 341, 342, 343, 0, - 344, 345, 0, 0, 1104, 347, 348, 0, 0, 349, - 350, 351, 1311, 353, 1312, 809, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 810, 1313, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 1105, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 1314, 410, 411, - 412, 1106, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 1315, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 1316, 437, 438, - 1107, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 1317, 456, 812, - 0, 0, 458, 459, 0, 460, 1318, 462, 463, 464, - 465, 466, 467, 0, 468, 1108, 1109, 0, 0, 471, - 472, 813, 474, 814, 1110, 476, 477, 1319, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 1320, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 1112, 498, 0, 499, 1322, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 327, 328, 1104, 1316, 1105, 332, 333, 334, 335, 1106, + 336, 337, 1317, 339, 1107, 810, 341, 1108, 343, 344, + 345, 0, 346, 347, 0, 0, 1109, 349, 350, 0, + 0, 351, 352, 353, 1318, 355, 1319, 812, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 813, 1320, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 1110, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 1321, + 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 1322, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 1323, + 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 1324, + 458, 815, 0, 0, 460, 461, 0, 462, 1325, 464, + 465, 466, 467, 468, 469, 0, 470, 1113, 1114, 0, + 0, 473, 474, 816, 476, 817, 1115, 478, 479, 1326, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 1327, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 1117, 500, 0, 501, 1329, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 1113, - 0, 0, 0, 0, 0, 0, 1114, 1115, 1116, 0, - 0, 0, 0, 1117, 0, 1118, 3287, 0, 0, 0, - 1119, 1120, 1121, 1122, 1284, 1071, 833, 1072, 1073, 1074, - 1075, 1076, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 1286, - 0, 1077, 0, 0, 1287, 132, 133, 0, 1288, 135, - 136, 1289, 138, 139, 140, 141, 1078, 1290, 1079, 1080, - 0, 1291, 146, 147, 148, 149, 150, 1081, 802, 151, - 152, 153, 154, 1082, 1083, 157, 0, 158, 159, 160, - 161, 803, 0, 1292, 0, 1293, 165, 166, 167, 168, - 169, 1294, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 1295, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 1085, 192, 193, 1086, 195, 1087, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 1088, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 1089, 224, 225, 226, 227, - 228, 229, 805, 1090, 231, 0, 232, 233, 1091, 235, - 0, 236, 0, 237, 1297, 0, 1298, 240, 241, 1299, - 1300, 244, 245, 246, 0, 1092, 1093, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 1301, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 1302, 272, 273, 274, 275, 276, 277, 1094, - 1095, 0, 1096, 0, 281, 1303, 1304, 284, 1305, 286, - 287, 288, 1097, 289, 290, 291, 0, 0, 292, 1306, - 294, 1307, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 1308, 305, 306, 307, 308, 309, 310, 311, 312, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 1118, 0, 0, 0, 0, 0, 0, 1119, 1120, + 1121, 0, 0, 0, 0, 1122, 0, 1123, 0, 0, + 0, 0, 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, + 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, -1196, 130, 131, 132, 0, 0, + 0, 0, -1196, 1082, 0, 0, 133, 134, 135, 0, + 136, 137, 138, 139, 140, 141, 142, 143, 1083, 145, + 1084, 1085, 0, 0, 148, 149, 150, 151, 152, 1086, + 805, 153, 154, 155, 156, 1087, 1088, 159, 0, 160, + 161, 162, 163, 806, 0, 807, 0, 1089, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 1090, 194, 195, 1091, + 197, 1092, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 1093, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 1094, 226, 227, + 228, 229, 230, 231, 808, 1095, 233, 0, 234, 235, + 1096, 237, 0, 238, 0, 239, 240, 0, 241, 242, + 243, 244, 245, 246, 247, 248, 0, 1097, 1098, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 273, 274, 275, 276, 277, 278, + 279, 1099, 1100, 0, 1101, 0, 283, 284, 285, 286, + 287, 288, 289, 290, 1102, 291, 292, 293, 0, 0, + 294, 295, 296, 297, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 1099, 1309, 1100, 330, 331, 332, - 333, 1101, 334, 335, 1310, 337, 1102, 807, 339, 1103, - 341, 342, 343, 0, 344, 345, 0, 0, 1104, 347, - 348, 0, 0, 349, 350, 351, 1311, 353, 1312, 809, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 810, 1313, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 1105, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 1314, 410, 411, 412, 1106, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 1315, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 1316, 437, 438, 1107, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 1104, 330, 1105, 332, + 333, 334, 335, 1106, 336, 337, 338, 339, 1107, 810, + 341, 1108, 343, 344, 345, 0, 346, 347, 0, 0, + 1109, 349, 350, 0, 0, 351, 352, 353, 354, 355, + 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 813, 373, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 438, 439, 440, 1112, 442, -1196, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 1317, 456, 812, 0, 0, 458, 459, 0, 460, - 1318, 462, 463, 464, 465, 466, 467, 0, 468, 1108, - 1109, 0, 0, 471, 472, 813, 474, 814, 1110, 476, - 477, 1319, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 1320, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 1112, 498, 0, 499, 1322, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 457, 458, 815, 0, 0, 460, 461, + 0, 462, 463, 464, 465, 466, 467, 468, 469, 0, + 470, 1113, 1114, 0, 0, 473, 474, 816, 476, 817, + 1115, 478, 479, 818, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 1116, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 1117, 500, 0, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 1113, 0, 0, 0, 0, 0, 0, - 1114, 1115, 1116, 0, 0, 0, 0, 1117, 0, 1118, - 0, 0, 0, 0, 1119, 1120, 1121, 1122, 119, 1071, - 833, 1072, 1073, 1074, 1075, 1076, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, -1193, 128, 129, 130, - 0, 0, 0, 0, -1193, 1077, 0, 0, 131, 132, - 133, 0, 134, 135, 136, 137, 138, 139, 140, 141, - 1078, 143, 1079, 1080, 0, 0, 146, 147, 148, 149, - 150, 1081, 802, 151, 152, 153, 154, 1082, 1083, 157, - 0, 158, 159, 160, 161, 803, 0, 804, 0, 1084, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 1085, 192, - 193, 1086, 195, 1087, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 1088, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 1089, - 224, 225, 226, 227, 228, 229, 805, 1090, 231, 0, - 232, 233, 1091, 235, 0, 236, 0, 237, 238, 0, - 239, 240, 241, 242, 243, 244, 245, 246, 0, 1092, - 1093, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 271, 272, 273, 274, - 275, 276, 277, 1094, 1095, 0, 1096, 0, 281, 282, - 283, 284, 285, 286, 287, 288, 1097, 289, 290, 291, - 0, 0, 292, 293, 294, 295, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 1098, 305, 306, 307, 308, + 527, 528, 529, 530, 531, 1118, 0, 0, 0, 0, + 0, 0, 1119, 1120, 1121, 0, 0, 0, 0, 1122, + 0, 1123, 0, 0, 0, 0, 1124, 1125, 1126, 1127, + 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 1756, 130, + 131, 132, 0, 0, 0, 0, 0, 1082, 0, 0, + 133, 134, 135, 0, 136, 137, 138, 139, 140, 141, + 142, 143, 1083, 145, 1084, 1085, 0, 0, 148, 149, + 150, 151, 152, 1086, 805, 153, 154, 155, 156, 1087, + 1088, 159, 0, 160, 161, 162, 163, 806, 0, 807, + 0, 1089, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 1090, 194, 195, 1091, 197, 1092, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 1093, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 1094, 226, 227, 228, 229, 230, 231, 808, 1095, + 233, 0, 234, 235, 1096, 237, 0, 238, 0, 239, + 240, 0, 241, 242, 243, 244, 245, 246, 247, 248, + 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 273, 274, + 275, 276, 277, 278, 279, 1099, 1100, 0, 1101, 0, + 283, 284, 285, 286, 287, 288, 289, 290, 1102, 291, + 292, 293, 0, 0, 294, 295, 296, 297, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 1099, 328, - 1100, 330, 331, 332, 333, 1101, 334, 335, 336, 337, - 1102, 807, 339, 1103, 341, 342, 343, 0, 344, 345, - 0, 0, 1104, 347, 348, 0, 0, 349, 350, 351, - 352, 353, 354, 809, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 810, 371, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 1105, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 409, 410, 411, 412, 1106, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 436, 437, 438, 1107, 440, - -1193, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 455, 456, 812, 0, 0, - 458, 459, 0, 460, 461, 462, 463, 464, 465, 466, - 467, 0, 468, 1108, 1109, 0, 0, 471, 472, 813, - 474, 814, 1110, 476, 477, 815, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 1111, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 1112, - 498, 0, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 1104, 330, 1105, 332, 333, 334, 335, 1106, 336, 337, + 338, 339, 1107, 810, 341, 1108, 343, 344, 345, 0, + 346, 347, 0, 0, 1109, 349, 350, 0, 0, 351, + 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 813, 373, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 1110, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, + 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 438, 439, 440, + 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 815, + 0, 0, 460, 461, 0, 462, 463, 464, 465, 466, + 467, 468, 469, 0, 470, 1113, 1114, 0, 0, 473, + 474, 816, 476, 817, 1115, 478, 479, 818, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 1116, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 1117, 500, 0, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 1113, 0, 0, - 0, 0, 0, 0, 1114, 1115, 1116, 0, 0, 0, - 0, 1117, 0, 1118, 0, 0, 0, 0, 1119, 1120, - 1121, 1122, 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 1749, 128, 129, 130, 0, 0, 0, 0, 0, 1077, - 0, 0, 131, 132, 133, 0, 134, 135, 136, 137, - 138, 139, 140, 141, 1078, 143, 1079, 1080, 0, 0, - 146, 147, 148, 149, 150, 1081, 802, 151, 152, 153, - 154, 1082, 1083, 157, 0, 158, 159, 160, 161, 803, - 0, 804, 0, 1084, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 1085, 192, 193, 1086, 195, 1087, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 1088, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 1089, 224, 225, 226, 227, 228, 229, - 805, 1090, 231, 0, 232, 233, 1091, 235, 0, 236, - 0, 237, 238, 0, 239, 240, 241, 242, 243, 244, - 245, 246, 0, 1092, 1093, 249, 250, 0, 251, 252, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 1118, + 0, 0, 0, 0, 0, 0, 1119, 1120, 1121, 0, + 0, 0, 0, 1122, 0, 1123, 0, 0, 0, 0, + 1124, 1125, 1126, 1127, 121, 1771, 836, 1077, 1078, 1079, + 1772, 1081, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 1773, 130, 131, 132, 0, 0, 0, 0, + 0, 1082, 0, 0, 133, 134, 135, 0, 136, 137, + 138, 139, 140, 141, 142, 143, 1083, 145, 1084, 1085, + 0, 0, 148, 149, 150, 151, 152, 1086, 805, 153, + 154, 155, 156, 1087, 1088, 159, 0, 160, 161, 162, + 163, 806, 0, 807, 0, 1089, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 1090, 194, 195, 1091, 197, 1092, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 1093, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 1094, 226, 227, 228, 229, + 230, 231, 808, 1095, 233, 0, 234, 235, 1096, 237, + 0, 238, 0, 239, 240, 0, 241, 242, 243, 244, + 245, 246, 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 271, 272, 273, 274, 275, 276, 277, 1094, 1095, 0, - 1096, 0, 281, 282, 283, 284, 285, 286, 287, 288, - 1097, 289, 290, 291, 0, 0, 292, 293, 294, 295, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 1098, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 273, 274, 275, 276, 277, 278, 279, 1099, + 1100, 0, 1101, 0, 283, 284, 285, 286, 287, 288, + 289, 290, 1102, 291, 292, 293, 0, 0, 294, 295, + 296, 297, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 1099, 328, 1100, 330, 331, 332, 333, 1101, - 334, 335, 336, 337, 1102, 807, 339, 1103, 341, 342, - 343, 0, 344, 345, 0, 0, 1104, 347, 348, 0, - 0, 349, 350, 351, 352, 353, 354, 809, 356, 357, + 325, 326, 327, 328, 1104, 330, 1105, 332, 333, 334, + 335, 1106, 336, 337, 338, 339, 1107, 810, 341, 1108, + 343, 344, 345, 0, 346, 347, 0, 0, 1109, 349, + 350, 0, 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 810, 371, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 1105, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 409, - 410, 411, 412, 1106, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 436, - 437, 438, 1107, 440, 0, 441, 442, 443, 444, 445, + 368, 369, 0, 0, 0, 0, 370, 371, 813, 373, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 1110, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 812, 0, 0, 458, 459, 0, 460, 461, 462, - 463, 464, 465, 466, 467, 0, 468, 1108, 1109, 0, - 0, 471, 472, 813, 474, 814, 1110, 476, 477, 815, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 1111, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 1112, 498, 0, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 456, 457, 458, 815, 0, 0, 460, 461, 0, 462, + 463, 464, 465, 466, 467, 468, 469, 0, 470, 1113, + 1114, 0, 0, 473, 474, 816, 476, 817, 1115, 478, + 479, 818, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 1116, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 1117, 500, 0, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 1113, 0, 0, 0, 0, 0, 0, 1114, 1115, - 1116, 0, 0, 0, 0, 1117, 0, 1118, 0, 0, - 0, 0, 1119, 1120, 1121, 1122, 119, 1764, 833, 1072, - 1073, 1074, 1765, 1076, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 1766, 128, 129, 130, 0, 0, - 0, 0, 0, 1077, 0, 0, 131, 132, 133, 0, - 134, 135, 136, 137, 138, 139, 140, 141, 1078, 143, - 1079, 1080, 0, 0, 146, 147, 148, 149, 150, 1081, - 802, 151, 152, 153, 154, 1082, 1083, 157, 0, 158, - 159, 160, 161, 803, 0, 804, 0, 1084, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 1085, 192, 193, 1086, - 195, 1087, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 1088, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 1089, 224, 225, - 226, 227, 228, 229, 805, 1090, 231, 0, 232, 233, - 1091, 235, 0, 236, 0, 237, 238, 0, 239, 240, - 241, 242, 243, 244, 245, 246, 0, 1092, 1093, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 271, 272, 273, 274, 275, 276, - 277, 1094, 1095, 0, 1096, 0, 281, 282, 283, 284, - 285, 286, 287, 288, 1097, 289, 290, 291, 0, 0, - 292, 293, 294, 295, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 1098, 305, 306, 307, 308, 309, 310, + 529, 530, 531, 1118, 0, 0, 0, 0, 0, 0, + 1119, 1120, 1121, 0, 0, 0, 0, 1122, 0, 1123, + 0, 0, 0, 0, 1124, 1125, 1126, 1127, 121, 1076, + 836, 1077, 1078, 1079, 1080, 1081, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 1082, 0, 0, 133, 134, + 135, 0, 136, 137, 138, 139, 140, 141, 142, 143, + 1083, 145, 1084, 1085, 0, 0, 148, 149, 150, 151, + 152, 1086, 805, 153, 154, 155, 156, 1087, 1088, 159, + 0, 160, 161, 162, 163, 806, 0, 807, 0, 1089, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 1090, 194, + 195, 1091, 197, 1092, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 1093, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 1094, + 226, 227, 228, 229, 230, 231, 808, 1095, 233, 0, + 234, 235, 1096, 237, 0, 238, 0, 239, 240, 1480, + 241, 242, 243, 244, 245, 246, 247, 248, 0, 1097, + 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 273, 274, 275, 276, + 277, 278, 279, 1099, 1100, 0, 1101, 0, 283, 284, + 285, 286, 287, 288, 289, 290, 1102, 291, 292, 293, + 0, 0, 294, 295, 296, 297, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 1099, 328, 1100, 330, - 331, 332, 333, 1101, 334, 335, 336, 337, 1102, 807, - 339, 1103, 341, 342, 343, 0, 344, 345, 0, 0, - 1104, 347, 348, 0, 0, 349, 350, 351, 352, 353, - 354, 809, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 810, 371, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 1105, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 1104, 330, + 1105, 332, 333, 334, 335, 1106, 336, 337, 338, 339, + 1107, 810, 341, 1108, 343, 344, 345, 0, 346, 347, + 0, 0, 1109, 349, 350, 0, 0, 351, 352, 353, + 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 813, 373, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 409, 410, 411, 412, 1106, 414, 415, + 407, 408, 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 436, 437, 438, 1107, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 812, 0, 0, 458, 459, - 0, 460, 461, 462, 463, 464, 465, 466, 467, 0, - 468, 1108, 1109, 0, 0, 471, 472, 813, 474, 814, - 1110, 476, 477, 815, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 1111, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 1112, 498, 0, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 438, 439, 440, 1112, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 457, 458, 815, 0, 0, + 460, 461, 0, 462, 463, 464, 465, 466, 467, 468, + 469, 0, 470, 1113, 1114, 0, 0, 473, 474, 816, + 476, 817, 1115, 478, 479, 818, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 1116, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 1117, + 500, 0, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 1113, 0, 0, 0, 0, - 0, 0, 1114, 1115, 1116, 0, 0, 0, 0, 1117, - 0, 1118, 0, 0, 0, 0, 1119, 1120, 1121, 1122, - 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, 0, 0, + 525, 526, 527, 528, 529, 530, 531, 1118, 0, 0, + 0, 0, 0, 0, 1119, 1120, 1121, 0, 0, 0, + 0, 1122, 0, 1123, 0, 0, 0, 0, 1124, 1125, + 1126, 1127, 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 1077, 0, 0, - 131, 132, 133, 0, 134, 135, 136, 137, 138, 139, - 140, 141, 1078, 143, 1079, 1080, 0, 0, 146, 147, - 148, 149, 150, 1081, 802, 151, 152, 153, 154, 1082, - 1083, 157, 0, 158, 159, 160, 161, 803, 0, 804, - 0, 1084, 165, 166, 167, 168, 169, 170, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 1085, 192, 193, 1086, 195, 1087, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 1088, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 1089, 224, 225, 226, 227, 228, 229, 805, 1090, - 231, 0, 232, 233, 1091, 235, 0, 236, 0, 237, - 238, 1473, 239, 240, 241, 242, 243, 244, 245, 246, - 0, 1092, 1093, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 271, 272, - 273, 274, 275, 276, 277, 1094, 1095, 0, 1096, 0, - 281, 282, 283, 284, 285, 286, 287, 288, 1097, 289, - 290, 291, 0, 0, 292, 293, 294, 295, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 1098, 305, 306, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 1082, + 0, 0, 133, 134, 135, 0, 136, 137, 138, 139, + 140, 141, 142, 143, 1083, 145, 1084, 1085, 0, 0, + 148, 149, 150, 151, 152, 1086, 805, 153, 154, 155, + 156, 1087, 1088, 159, 0, 160, 161, 162, 163, 806, + 0, 807, 0, 1089, 167, 168, 169, 170, 171, 172, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 1090, 194, 195, 1091, 197, 1092, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 1093, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 1094, 226, 227, 228, 229, 230, 231, + 808, 1095, 233, 0, 234, 235, 1096, 237, 0, 238, + 0, 239, 240, 0, 241, 242, 243, 244, 245, 246, + 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 273, 274, 275, 276, 277, 278, 279, 1099, 1100, 0, + 1101, 0, 283, 284, 285, 286, 287, 288, 289, 290, + 1102, 291, 292, 293, 0, 0, 294, 295, 296, 297, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 1099, 328, 1100, 330, 331, 332, 333, 1101, 334, 335, - 336, 337, 1102, 807, 339, 1103, 341, 342, 343, 0, - 344, 345, 0, 0, 1104, 347, 348, 0, 0, 349, - 350, 351, 352, 353, 354, 809, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 810, 371, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 1105, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 409, 410, 411, - 412, 1106, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 436, 437, 438, - 1107, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 812, - 0, 0, 458, 459, 0, 460, 461, 462, 463, 464, - 465, 466, 467, 0, 468, 1108, 1109, 0, 0, 471, - 472, 813, 474, 814, 1110, 476, 477, 815, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 1111, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 1112, 498, 0, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 327, 328, 1104, 330, 1105, 332, 333, 334, 335, 1106, + 336, 337, 338, 339, 1107, 810, 341, 1108, 343, 344, + 345, 0, 346, 347, 0, 0, 1109, 349, 350, 0, + 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 813, 373, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 1110, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 411, + 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 438, + 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, + 458, 815, 0, 0, 460, 461, 0, 462, 463, 464, + 465, 466, 467, 468, 469, 0, 470, 1113, 1114, 0, + 0, 473, 474, 816, 476, 817, 1115, 478, 479, 818, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 1116, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 1117, 500, 0, 501, 502, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 1113, - 0, 0, 0, 0, 0, 0, 1114, 1115, 1116, 0, - 0, 0, 0, 1117, 0, 1118, 0, 0, 0, 0, - 1119, 1120, 1121, 1122, 119, 1071, 833, 1072, 1073, 1074, - 1075, 1076, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 1077, 0, 0, 131, 132, 133, 0, 134, 135, - 136, 137, 138, 139, 140, 141, 1078, 143, 1079, 1080, - 0, 0, 146, 147, 148, 149, 150, 1081, 802, 151, - 152, 153, 154, 1082, 1083, 157, 0, 158, 159, 160, - 161, 803, 0, 804, 0, 1084, 165, 166, 167, 168, - 169, 170, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 1085, 192, 193, 1086, 195, 1087, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 1088, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 1089, 224, 225, 226, 227, - 228, 229, 805, 1090, 231, 0, 232, 233, 1091, 235, - 0, 236, 0, 237, 238, 0, 239, 240, 241, 242, - 243, 244, 245, 246, 0, 1092, 1093, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 271, 272, 273, 274, 275, 276, 277, 1094, - 1095, 0, 1096, 0, 281, 282, 283, 284, 285, 286, - 287, 288, 1097, 289, 290, 291, 0, 0, 292, 293, - 294, 295, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 1098, 305, 306, 307, 308, 309, 310, 311, 312, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 1118, 0, 0, 0, 0, 0, 0, 1119, 1120, + 1121, 0, 0, 0, 0, 1122, 0, 1123, 2151, 0, + 0, 0, 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, + 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, + 0, 0, 2808, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 1082, 0, 0, 133, 134, 135, 0, + 136, 137, 138, 139, 140, 141, 142, 143, 1083, 145, + 1084, 1085, 0, 0, 148, 149, 150, 151, 152, 1086, + 805, 153, 154, 155, 156, 1087, 1088, 159, 0, 160, + 161, 162, 163, 806, 0, 807, 0, 1089, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 1090, 194, 195, 1091, + 197, 1092, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 1093, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 1094, 226, 227, + 228, 229, 230, 231, 808, 1095, 233, 0, 234, 235, + 1096, 237, 0, 238, 0, 239, 240, 0, 241, 242, + 243, 244, 245, 246, 247, 248, 0, 1097, 1098, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 273, 274, 275, 276, 277, 278, + 279, 1099, 1100, 0, 1101, 0, 283, 284, 285, 286, + 287, 288, 289, 290, 1102, 291, 292, 293, 0, 0, + 294, 295, 296, 297, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 1099, 328, 1100, 330, 331, 332, - 333, 1101, 334, 335, 336, 337, 1102, 807, 339, 1103, - 341, 342, 343, 0, 344, 345, 0, 0, 1104, 347, - 348, 0, 0, 349, 350, 351, 352, 353, 354, 809, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 810, 371, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 1105, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 409, 410, 411, 412, 1106, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 436, 437, 438, 1107, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 1104, 330, 1105, 332, + 333, 334, 335, 1106, 336, 337, 338, 339, 1107, 810, + 341, 1108, 343, 344, 345, 0, 346, 347, 0, 0, + 1109, 349, 350, 0, 0, 351, 352, 353, 354, 355, + 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 813, 373, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 455, 456, 812, 0, 0, 458, 459, 0, 460, - 461, 462, 463, 464, 465, 466, 467, 0, 468, 1108, - 1109, 0, 0, 471, 472, 813, 474, 814, 1110, 476, - 477, 815, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 1111, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 1112, 498, 0, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 457, 458, 815, 0, 0, 460, 461, + 0, 462, 463, 464, 465, 466, 467, 468, 469, 0, + 470, 1113, 1114, 0, 0, 473, 474, 816, 476, 817, + 1115, 478, 479, 818, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 1116, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 1117, 500, 0, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 1113, 0, 0, 0, 0, 0, 0, - 1114, 1115, 1116, 0, 0, 0, 0, 1117, 0, 1118, - 2141, 0, 0, 0, 1119, 1120, 1121, 1122, 119, 1071, - 833, 1072, 1073, 1074, 1075, 1076, 0, 0, 0, 0, - 0, 0, 0, 0, 2795, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 1077, 0, 0, 131, 132, - 133, 0, 134, 135, 136, 137, 138, 139, 140, 141, - 1078, 143, 1079, 1080, 0, 0, 146, 147, 148, 149, - 150, 1081, 802, 151, 152, 153, 154, 1082, 1083, 157, - 0, 158, 159, 160, 161, 803, 0, 804, 0, 1084, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 1085, 192, - 193, 1086, 195, 1087, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 1088, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 1089, - 224, 225, 226, 227, 228, 229, 805, 1090, 231, 0, - 232, 233, 1091, 235, 0, 236, 0, 237, 238, 0, - 239, 240, 241, 242, 243, 244, 245, 246, 0, 1092, - 1093, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 271, 272, 273, 274, - 275, 276, 277, 1094, 1095, 0, 1096, 0, 281, 282, - 283, 284, 285, 286, 287, 288, 1097, 289, 290, 291, - 0, 0, 292, 293, 294, 295, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 1098, 305, 306, 307, 308, + 527, 528, 529, 530, 531, 1118, 0, 0, 0, 0, + 0, 0, 1119, 1120, 1121, 0, 0, 0, 0, 1122, + 0, 1123, 0, 0, 0, 0, 1124, 1125, 1126, 1127, + 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 1082, 0, 0, + 133, 134, 135, 0, 136, 137, 138, 139, 140, 141, + 142, 143, 1083, 145, 1084, 1085, 0, 0, 148, 149, + 150, 151, 152, 1086, 805, 153, 154, 155, 156, 1087, + 1088, 159, 0, 160, 161, 162, 163, 806, 0, 807, + 0, 1089, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 1090, 194, 195, 1091, 197, 1092, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 1093, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 1094, 226, 227, 228, 229, 230, 231, 808, 1095, + 233, 0, 234, 235, 1096, 237, 0, 238, 0, 239, + 240, 0, 241, 242, 243, 244, 245, 246, 247, 248, + 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 273, 274, + 275, 276, 277, 278, 279, 1099, 1100, 0, 1101, 0, + 283, 284, 285, 286, 287, 288, 289, 290, 1102, 291, + 292, 293, 0, 0, 294, 295, 296, 297, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 1099, 328, - 1100, 330, 331, 332, 333, 1101, 334, 335, 336, 337, - 1102, 807, 339, 1103, 341, 342, 343, 0, 344, 345, - 0, 0, 1104, 347, 348, 0, 0, 349, 350, 351, - 352, 353, 354, 809, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 810, 371, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 1105, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 409, 410, 411, 412, 1106, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 436, 437, 438, 1107, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 455, 456, 812, 0, 0, - 458, 459, 0, 460, 461, 462, 463, 464, 465, 466, - 467, 0, 468, 1108, 1109, 0, 0, 471, 472, 813, - 474, 814, 1110, 476, 477, 815, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 1111, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 1112, - 498, 0, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 1104, 330, 1105, 332, 333, 334, 335, 1106, 336, 337, + 338, 339, 1107, 810, 341, 1108, 343, 344, 345, 0, + 346, 347, 0, 0, 1109, 349, 350, 0, 0, 351, + 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 813, 373, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 1110, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, + 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 438, 439, 440, + 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 815, + 0, 0, 460, 461, 2875, 462, 463, 464, 465, 466, + 467, 468, 469, 0, 470, 1113, 1114, 0, 0, 473, + 474, 816, 476, 817, 1115, 478, 479, 818, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 1116, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 1117, 500, 0, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 1113, 0, 0, - 0, 0, 0, 0, 1114, 1115, 1116, 0, 0, 0, - 0, 1117, 0, 1118, 0, 0, 0, 0, 1119, 1120, - 1121, 1122, 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 1077, - 0, 0, 131, 132, 133, 0, 134, 135, 136, 137, - 138, 139, 140, 141, 1078, 143, 1079, 1080, 0, 0, - 146, 147, 148, 149, 150, 1081, 802, 151, 152, 153, - 154, 1082, 1083, 157, 0, 158, 159, 160, 161, 803, - 0, 804, 0, 1084, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 1085, 192, 193, 1086, 195, 1087, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 1088, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 1089, 224, 225, 226, 227, 228, 229, - 805, 1090, 231, 0, 232, 233, 1091, 235, 0, 236, - 0, 237, 238, 0, 239, 240, 241, 242, 243, 244, - 245, 246, 0, 1092, 1093, 249, 250, 0, 251, 252, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 1118, + 0, 0, 0, 0, 0, 0, 1119, 1120, 1121, 0, + 0, 0, 0, 1122, 0, 1123, 0, 0, 0, 0, + 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, 1078, 1079, + 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 1082, 0, 0, 133, 134, 135, 0, 136, 137, + 138, 139, 140, 141, 142, 143, 1083, 145, 1084, 1085, + 0, 0, 148, 149, 150, 151, 152, 1086, 805, 153, + 154, 155, 156, 1087, 1088, 159, 0, 160, 161, 162, + 163, 806, 0, 807, 0, 1089, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 1090, 194, 195, 1091, 197, 1092, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 1093, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 1094, 226, 227, 228, 229, + 230, 231, 808, 1095, 233, 0, 234, 235, 1096, 237, + 0, 238, 0, 239, 240, 0, 241, 242, 243, 244, + 245, 246, 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 271, 272, 273, 274, 275, 276, 277, 1094, 1095, 0, - 1096, 0, 281, 282, 283, 284, 285, 286, 287, 288, - 1097, 289, 290, 291, 0, 0, 292, 293, 294, 295, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 1098, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 273, 274, 275, 276, 277, 278, 279, 1099, + 1100, 0, 1101, 0, 283, 284, 285, 286, 287, 288, + 289, 290, 1102, 291, 292, 293, 0, 0, 294, 295, + 296, 297, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 1099, 328, 1100, 330, 331, 332, 333, 1101, - 334, 335, 336, 337, 1102, 807, 339, 1103, 341, 342, - 343, 0, 344, 345, 0, 0, 1104, 347, 348, 0, - 0, 349, 350, 351, 352, 353, 354, 809, 356, 357, + 325, 326, 327, 328, 1104, 330, 1105, 332, 333, 334, + 335, 1106, 336, 337, 338, 339, 1107, 810, 341, 1108, + 343, 344, 345, 0, 346, 347, 0, 0, 1109, 349, + 350, 0, 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 810, 371, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 1105, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 409, - 410, 411, 412, 1106, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 436, - 437, 438, 1107, 440, 0, 441, 442, 443, 444, 445, + 368, 369, 0, 0, 0, 0, 370, 371, 813, 373, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 1110, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 812, 0, 0, 458, 459, 2862, 460, 461, 462, - 463, 464, 465, 466, 467, 0, 468, 1108, 1109, 0, - 0, 471, 472, 813, 474, 814, 1110, 476, 477, 815, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 1111, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 1112, 498, 0, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 456, 457, 458, 815, 0, 0, 460, 461, 0, 462, + 463, 464, 465, 466, 467, 468, 469, 0, 470, 1113, + 1114, 0, 0, 473, 474, 816, 476, 817, 1115, 478, + 479, 818, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 1116, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 1117, 500, 0, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 0, 2999, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 1113, 0, 0, 0, 0, 0, 0, 1114, 1115, - 1116, 0, 0, 0, 0, 1117, 0, 1118, 0, 0, - 0, 0, 1119, 1120, 1121, 1122, 119, 1071, 833, 1072, - 1073, 1074, 1075, 1076, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 1077, 0, 0, 131, 132, 133, 0, - 134, 135, 136, 137, 138, 139, 140, 141, 1078, 143, - 1079, 1080, 0, 0, 146, 147, 148, 149, 150, 1081, - 802, 151, 152, 153, 154, 1082, 1083, 157, 0, 158, - 159, 160, 161, 803, 0, 804, 0, 1084, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 1085, 192, 193, 1086, - 195, 1087, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 1088, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 1089, 224, 225, - 226, 227, 228, 229, 805, 1090, 231, 0, 232, 233, - 1091, 235, 0, 236, 0, 237, 238, 0, 239, 240, - 241, 242, 243, 244, 245, 246, 0, 1092, 1093, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 271, 272, 273, 274, 275, 276, - 277, 1094, 1095, 0, 1096, 0, 281, 282, 283, 284, - 285, 286, 287, 288, 1097, 289, 290, 291, 0, 0, - 292, 293, 294, 295, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 1098, 305, 306, 307, 308, 309, 310, + 529, 530, 531, 1118, 0, 0, 0, 0, 0, 0, + 1119, 1120, 1121, 0, 0, 0, 0, 1122, 0, 1123, + 0, 0, 0, 0, 1124, 1125, 1126, 1127, 121, 1076, + 836, 1077, 1078, 1079, 1080, 1081, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 3238, 130, 131, 132, + 0, 0, 0, 0, 0, 1082, 0, 0, 133, 134, + 135, 0, 136, 137, 138, 139, 140, 141, 142, 143, + 1083, 145, 1084, 1085, 0, 0, 148, 149, 150, 151, + 152, 1086, 805, 153, 154, 155, 156, 1087, 1088, 159, + 0, 160, 161, 162, 163, 806, 0, 807, 0, 1089, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 182, 183, 3239, + 185, 186, 187, 188, 189, 190, 191, 192, 1090, 194, + 195, 1091, 197, 1092, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 1093, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 1094, + 226, 227, 228, 229, 230, 231, 808, 1095, 233, 0, + 234, 235, 1096, 237, 0, 238, 0, 239, 240, 0, + 241, 242, 243, 244, 245, 246, 247, 248, 0, 3240, + 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 273, 274, 275, 276, + 277, 278, 279, 1099, 1100, 0, 1101, 0, 283, 284, + 285, 286, 287, 288, 289, 290, 1102, 291, 292, 293, + 0, 0, 294, 295, 296, 297, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 1099, 328, 1100, 330, - 331, 332, 333, 1101, 334, 335, 336, 337, 1102, 807, - 339, 1103, 341, 342, 343, 0, 344, 345, 0, 0, - 1104, 347, 348, 0, 0, 349, 350, 351, 352, 353, - 354, 809, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 810, 371, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 1105, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 1104, 330, + 1105, 332, 333, 334, 335, 1106, 336, 337, 338, 339, + 1107, 810, 341, 1108, 343, 344, 345, 0, 346, 347, + 0, 0, 1109, 349, 350, 0, 0, 351, 352, 353, + 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 813, 373, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 409, 410, 411, 412, 1106, 414, 415, + 407, 408, 0, 409, 410, 411, 412, 413, 3241, 1111, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 436, 437, 438, 1107, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 812, 0, 0, 458, 459, - 0, 460, 461, 462, 463, 464, 465, 466, 467, 0, - 468, 1108, 1109, 0, 0, 471, 472, 813, 474, 814, - 1110, 476, 477, 815, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 1111, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 1112, 498, 0, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 0, - 2986, 508, 0, 0, 509, 510, 511, 512, 513, 514, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 438, 439, 440, 1112, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 457, 458, 815, 0, 0, + 460, 461, 0, 462, 463, 464, 465, 466, 467, 468, + 469, 0, 470, 1113, 1114, 0, 0, 473, 474, 816, + 476, 817, 1115, 478, 479, 818, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 1116, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 1117, + 500, 0, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 1113, 0, 0, 0, 0, - 0, 0, 1114, 1115, 1116, 0, 0, 0, 0, 1117, - 0, 1118, 0, 0, 0, 0, 1119, 1120, 1121, 1122, - 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 3224, 128, - 129, 130, 0, 0, 0, 0, 0, 1077, 0, 0, - 131, 132, 133, 0, 134, 135, 136, 137, 138, 139, - 140, 141, 1078, 143, 1079, 1080, 0, 0, 146, 147, - 148, 149, 150, 1081, 802, 151, 152, 153, 154, 1082, - 1083, 157, 0, 158, 159, 160, 161, 803, 0, 804, - 0, 1084, 165, 166, 167, 168, 169, 170, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 180, - 181, 3225, 183, 184, 185, 186, 187, 188, 189, 190, - 1085, 192, 193, 1086, 195, 1087, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 1088, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 1089, 224, 225, 226, 227, 228, 229, 805, 1090, - 231, 0, 232, 233, 1091, 235, 0, 236, 0, 237, - 238, 0, 239, 240, 241, 242, 243, 244, 245, 246, - 0, 3226, 1093, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 271, 272, - 273, 274, 275, 276, 277, 1094, 1095, 0, 1096, 0, - 281, 282, 283, 284, 285, 286, 287, 288, 1097, 289, - 290, 291, 0, 0, 292, 293, 294, 295, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 1098, 305, 306, + 525, 526, 527, 528, 529, 530, 531, 1118, 0, 0, + 0, 0, 0, 0, 1119, 1120, 1121, 0, 0, 0, + 0, 1122, 0, 3242, 0, 0, 0, 0, 1124, 1125, + 1126, 1127, 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, + 0, 0, 0, 0, 0, 0, 0, 0, 3474, 0, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 1082, + 0, 0, 133, 134, 135, 0, 136, 137, 138, 139, + 140, 141, 142, 143, 1083, 145, 1084, 1085, 0, 0, + 148, 149, 150, 151, 152, 1086, 805, 153, 154, 155, + 156, 1087, 1088, 159, 0, 160, 161, 162, 163, 806, + 0, 807, 0, 1089, 167, 168, 169, 170, 171, 172, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 1090, 194, 195, 1091, 197, 1092, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 1093, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 1094, 226, 227, 228, 229, 230, 231, + 808, 1095, 233, 0, 234, 235, 1096, 237, 0, 238, + 0, 239, 240, 0, 241, 242, 243, 244, 245, 246, + 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 273, 274, 275, 276, 277, 278, 279, 1099, 1100, 0, + 1101, 0, 283, 284, 285, 286, 287, 288, 289, 290, + 1102, 291, 292, 293, 0, 0, 294, 295, 296, 297, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 1099, 328, 1100, 330, 331, 332, 333, 1101, 334, 335, - 336, 337, 1102, 807, 339, 1103, 341, 342, 343, 0, - 344, 345, 0, 0, 1104, 347, 348, 0, 0, 349, - 350, 351, 352, 353, 354, 809, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 810, 371, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 1105, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 409, 410, 411, - 3227, 1106, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 436, 437, 438, - 1107, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 812, - 0, 0, 458, 459, 0, 460, 461, 462, 463, 464, - 465, 466, 467, 0, 468, 1108, 1109, 0, 0, 471, - 472, 813, 474, 814, 1110, 476, 477, 815, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 1111, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 1112, 498, 0, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 327, 328, 1104, 330, 1105, 332, 333, 334, 335, 1106, + 336, 337, 338, 339, 1107, 810, 341, 1108, 343, 344, + 345, 0, 346, 347, 0, 0, 1109, 349, 350, 0, + 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 813, 373, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 1110, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 411, + 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 438, + 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, + 458, 815, 0, 0, 460, 461, 0, 462, 463, 464, + 465, 466, 467, 468, 469, 0, 470, 1113, 1114, 0, + 0, 473, 474, 816, 476, 817, 1115, 478, 479, 818, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 1116, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 1117, 500, 0, 501, 502, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 1113, - 0, 0, 0, 0, 0, 0, 1114, 1115, 1116, 0, - 0, 0, 0, 1117, 0, 3228, 0, 0, 0, 0, - 1119, 1120, 1121, 1122, 119, 1071, 833, 1072, 1073, 1074, - 1075, 1076, 0, 0, 0, 0, 0, 0, 0, 0, - 3460, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 1077, 0, 0, 131, 132, 133, 0, 134, 135, - 136, 137, 138, 139, 140, 141, 1078, 143, 1079, 1080, - 0, 0, 146, 147, 148, 149, 150, 1081, 802, 151, - 152, 153, 154, 1082, 1083, 157, 0, 158, 159, 160, - 161, 803, 0, 804, 0, 1084, 165, 166, 167, 168, - 169, 170, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 1085, 192, 193, 1086, 195, 1087, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 1088, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 1089, 224, 225, 226, 227, - 228, 229, 805, 1090, 231, 0, 232, 233, 1091, 235, - 0, 236, 0, 237, 238, 0, 239, 240, 241, 242, - 243, 244, 245, 246, 0, 1092, 1093, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 271, 272, 273, 274, 275, 276, 277, 1094, - 1095, 0, 1096, 0, 281, 282, 283, 284, 285, 286, - 287, 288, 1097, 289, 290, 291, 0, 0, 292, 293, - 294, 295, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 1098, 305, 306, 307, 308, 309, 310, 311, 312, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 1118, 0, 0, 0, 0, 0, 0, 1119, 1120, + 1121, 0, 0, 0, 0, 1122, 0, 1123, 0, 0, + 0, 0, 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, + 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 1082, 0, 0, 133, 134, 135, 0, + 136, 137, 138, 139, 140, 141, 142, 143, 1083, 145, + 1084, 1085, 0, 0, 148, 149, 150, 151, 152, 1086, + 805, 153, 154, 155, 156, 1087, 1088, 159, 0, 160, + 161, 162, 163, 806, 0, 807, 0, 1089, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 1090, 194, 195, 1091, + 197, 1092, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 1093, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 1094, 226, 227, + 228, 229, 230, 231, 808, 1095, 233, 0, 234, 235, + 1096, 237, 0, 238, 0, 239, 240, 0, 241, 242, + 243, 244, 245, 246, 247, 248, 0, 1097, 1098, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 273, 274, 275, 276, 277, 278, + 279, 1099, 1100, 0, 1101, 0, 283, 284, 285, 286, + 287, 288, 289, 290, 1102, 291, 292, 293, 0, 0, + 294, 295, 296, 297, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 1099, 328, 1100, 330, 331, 332, - 333, 1101, 334, 335, 336, 337, 1102, 807, 339, 1103, - 341, 342, 343, 0, 344, 345, 0, 0, 1104, 347, - 348, 0, 0, 349, 350, 351, 352, 353, 354, 809, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 810, 371, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 1105, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 409, 410, 411, 412, 1106, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 436, 437, 438, 1107, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 1104, 330, 1105, 332, + 333, 334, 335, 1106, 336, 337, 338, 339, 1107, 810, + 341, 1108, 343, 344, 345, 0, 346, 347, 0, 0, + 1109, 349, 350, 0, 0, 351, 352, 353, 354, 355, + 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 813, 373, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 455, 456, 812, 0, 0, 458, 459, 0, 460, - 461, 462, 463, 464, 465, 466, 467, 0, 468, 1108, - 1109, 0, 0, 471, 472, 813, 474, 814, 1110, 476, - 477, 815, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 1111, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 1112, 498, 0, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 457, 458, 815, 0, 0, 460, 461, + 0, 462, 463, 464, 465, 466, 467, 468, 469, 0, + 470, 1113, 1114, 0, 0, 473, 474, 816, 476, 817, + 1115, 478, 479, 818, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 1116, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 1117, 500, 0, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 1113, 0, 0, 0, 0, 0, 0, - 1114, 1115, 1116, 0, 0, 0, 0, 1117, 0, 1118, - 0, 0, 0, 0, 1119, 1120, 1121, 1122, 119, 1071, - 833, 1072, 1073, 1074, 1075, 1076, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 1077, 0, 0, 131, 132, - 133, 0, 134, 135, 136, 137, 138, 139, 140, 141, - 1078, 143, 1079, 1080, 0, 0, 146, 147, 148, 149, - 150, 1081, 802, 151, 152, 153, 154, 1082, 1083, 157, - 0, 158, 159, 160, 161, 803, 0, 804, 0, 1084, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 1085, 192, - 193, 1086, 195, 1087, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 1088, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 1089, - 224, 225, 226, 227, 228, 229, 805, 1090, 231, 0, - 232, 233, 1091, 235, 0, 236, 0, 237, 238, 0, - 239, 240, 241, 242, 243, 244, 245, 246, 0, 1092, - 1093, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 271, 272, 273, 274, - 275, 276, 277, 1094, 1095, 0, 1096, 0, 281, 282, - 283, 284, 285, 286, 287, 288, 1097, 289, 290, 291, - 0, 0, 292, 293, 294, 295, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 1098, 305, 306, 307, 308, + 527, 528, 529, 530, 531, 1118, 0, 0, 0, 0, + 0, 0, 1119, 1120, 1121, 0, 0, 0, 0, 1122, + 0, 1123, 0, 0, 0, 0, 1124, 1125, 1126, 1127, + 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 1082, 0, 0, + 133, 134, 135, 0, 136, 137, 138, 139, 140, 141, + 142, 143, 1083, 145, 1084, 1085, 0, 0, 148, 149, + 150, 151, 152, 1086, 805, 153, 154, 155, 156, 1087, + 1088, 159, 0, 160, 161, 162, 163, 806, 0, 807, + 0, 1089, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 1090, 194, 195, 1091, 197, 1092, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 1093, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 1094, 226, 227, 228, 229, 230, 231, 808, 1095, + 233, 0, 234, 235, 1096, 237, 0, 238, 0, 239, + 240, 0, 241, 242, 243, 244, 245, 246, 247, 248, + 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 273, 274, + 275, 276, 277, 278, 279, 1099, 1100, 0, 1101, 0, + 283, 284, 285, 286, 287, 288, 289, 290, 1102, 291, + 292, 293, 0, 0, 294, 295, 296, 297, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 1099, 328, - 1100, 330, 331, 332, 333, 1101, 334, 335, 336, 337, - 1102, 807, 339, 1103, 341, 342, 343, 0, 344, 345, - 0, 0, 1104, 347, 348, 0, 0, 349, 350, 351, - 352, 353, 354, 809, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 810, 371, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 1105, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 409, 410, 411, 412, 1106, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 436, 437, 438, 1107, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 455, 456, 812, 0, 0, - 458, 459, 0, 460, 461, 462, 463, 464, 465, 466, - 467, 0, 468, 1108, 1109, 0, 0, 471, 472, 813, - 474, 814, 1110, 476, 477, 815, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 1111, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 1112, - 498, 0, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 1104, 330, 1105, 332, 333, 334, 335, 1106, 336, 337, + 338, 339, 1107, 810, 341, 1108, 343, 344, 345, 0, + 346, 347, 0, 0, 1109, 349, 350, 0, 0, 351, + 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 813, 373, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 1110, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, + 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 438, 439, 440, + 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 815, + 0, 0, 460, 461, 0, 462, 463, 464, 465, 466, + 467, 468, 469, 0, 470, 1113, 1114, 0, 0, 473, + 474, 816, 476, 817, 1115, 478, 479, 818, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 1116, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 1117, 500, 0, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 1113, 0, 0, - 0, 0, 0, 0, 1114, 1115, 1116, 0, 0, 0, - 0, 1117, 0, 1118, 0, 0, 0, 0, 1119, 1120, - 1121, 1122, 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 1077, - 0, 0, 131, 132, 133, 0, 134, 135, 136, 137, - 138, 139, 140, 141, 1078, 143, 1079, 1080, 0, 0, - 146, 147, 148, 149, 150, 1081, 802, 151, 152, 153, - 154, 1082, 1083, 157, 0, 158, 159, 160, 161, 803, - 0, 804, 0, 1084, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 1085, 192, 193, 1086, 195, 1087, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 1088, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 1089, 224, 225, 226, 227, 228, 229, - 805, 1090, 231, 0, 232, 233, 1091, 235, 0, 236, - 0, 237, 238, 0, 239, 240, 241, 242, 243, 244, - 245, 246, 0, 1092, 1093, 249, 250, 0, 251, 252, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 1118, + 0, 0, 0, 0, 0, 0, 1776, 1777, 1121, 0, + 0, 0, 0, 1122, 0, 1123, 0, 0, 0, 0, + 1124, 1125, 1126, 1127, 121, 2294, 836, 1077, 1078, 1079, + 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 1082, 0, 0, 133, 134, 135, 0, 136, 137, + 138, 139, 140, 141, 142, 143, 1083, 145, 1084, 1085, + 0, 0, 148, 149, 150, 151, 152, 1086, 805, 153, + 154, 155, 156, 1087, 1088, 159, 0, 160, 161, 162, + 163, 806, 0, 807, 0, 1089, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 1090, 194, 195, 1091, 197, 1092, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 1093, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 1094, 226, 227, 228, 229, + 230, 231, 808, 1095, 233, 0, 234, 235, 1096, 237, + 0, 238, 0, 239, 240, 0, 241, 242, 243, 244, + 245, 246, 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 271, 272, 273, 274, 275, 276, 277, 1094, 1095, 0, - 1096, 0, 281, 282, 283, 284, 285, 286, 287, 288, - 1097, 289, 290, 291, 0, 0, 292, 293, 294, 295, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 1098, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 273, 274, 275, 276, 277, 278, 279, 1099, + 1100, 0, 1101, 0, 283, 284, 285, 286, 287, 288, + 289, 290, 1102, 291, 292, 293, 0, 0, 294, 295, + 296, 297, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 1099, 328, 1100, 330, 331, 332, 333, 1101, - 334, 335, 336, 337, 1102, 807, 339, 1103, 341, 342, - 343, 0, 344, 345, 0, 0, 1104, 347, 348, 0, - 0, 349, 350, 351, 352, 353, 354, 809, 356, 357, + 325, 326, 327, 328, 1104, 330, 1105, 332, 333, 334, + 335, 1106, 336, 337, 338, 339, 1107, 810, 341, 1108, + 343, 344, 345, 0, 346, 347, 0, 0, 1109, 349, + 350, 0, 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 810, 371, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 1105, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 409, - 410, 411, 412, 1106, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 436, - 437, 438, 1107, 440, 0, 441, 442, 443, 444, 445, + 368, 369, 0, 0, 0, 0, 370, 371, 813, 373, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 1110, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 812, 0, 0, 458, 459, 0, 460, 461, 462, - 463, 464, 465, 466, 467, 0, 468, 1108, 1109, 0, - 0, 471, 472, 813, 474, 814, 1110, 476, 477, 815, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 1111, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 1112, 498, 0, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 456, 457, 458, 815, 0, 0, 460, 461, 0, 462, + 463, 464, 465, 466, 467, 468, 469, 0, 470, 1113, + 1114, 0, 0, 473, 474, 816, 476, 817, 1115, 478, + 479, 818, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 1116, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 1117, 500, 0, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 1113, 0, 0, 0, 0, 0, 0, 1769, 1770, - 1116, 0, 0, 0, 0, 1117, 0, 1118, 0, 0, - 0, 0, 1119, 1120, 1121, 1122, 119, 2284, 833, 1072, - 1073, 1074, 1075, 1076, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 1077, 0, 0, 131, 132, 133, 0, - 134, 135, 136, 137, 138, 139, 140, 141, 1078, 143, - 1079, 1080, 0, 0, 146, 147, 148, 149, 150, 1081, - 802, 151, 152, 153, 154, 1082, 1083, 157, 0, 158, - 159, 160, 161, 803, 0, 804, 0, 1084, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 1085, 192, 193, 1086, - 195, 1087, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 1088, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 1089, 224, 225, - 226, 227, 228, 229, 805, 1090, 231, 0, 232, 233, - 1091, 235, 0, 236, 0, 237, 238, 0, 239, 240, - 241, 242, 243, 244, 245, 246, 0, 1092, 1093, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 271, 272, 273, 274, 275, 276, - 277, 1094, 1095, 0, 1096, 0, 281, 282, 283, 284, - 285, 286, 287, 288, 1097, 289, 290, 291, 0, 0, - 292, 293, 294, 295, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 1098, 305, 306, 307, 308, 309, 310, + 529, 530, 531, 1118, 0, 0, 0, 0, 0, 0, + 1119, 1120, 1121, 0, 0, 0, 0, 1122, 0, 1123, + 0, 0, 0, 0, 1124, 1125, 1126, 1127, 121, 1076, + 836, 1077, 1078, 1079, 1080, 1081, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 1082, 0, 0, 133, 134, + 135, 0, 136, 137, 138, 139, 140, 141, 142, 143, + 1083, 145, 1084, 1085, 0, 0, 148, 149, 150, 151, + 152, 1086, 805, 153, 154, 155, 156, 1087, 1088, 159, + 0, 160, 161, 162, 163, 806, 0, 807, 0, 1089, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 1090, 194, + 195, 1091, 197, 1092, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 1093, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 1094, + 226, 227, 228, 229, 230, 231, 808, 1095, 233, 0, + 234, 235, 1096, 237, 0, 238, 0, 239, 240, 0, + 241, 242, 243, 244, 245, 246, 247, 248, 0, 1097, + 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 273, 274, 275, 276, + 277, 278, 279, 1099, 1100, 0, 1101, 0, 283, 284, + 285, 286, 287, 288, 289, 290, 1102, 291, 292, 293, + 0, 0, 294, 295, 296, 297, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 1099, 328, 1100, 330, - 331, 332, 333, 1101, 334, 335, 336, 337, 1102, 807, - 339, 1103, 341, 342, 343, 0, 344, 345, 0, 0, - 1104, 347, 348, 0, 0, 349, 350, 351, 352, 353, - 354, 809, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 810, 371, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 1105, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 1104, 330, + 1105, 332, 333, 334, 335, 1106, 336, 337, 338, 339, + 1107, 810, 341, 1108, 343, 344, 345, 0, 346, 347, + 0, 0, 1109, 349, 350, 0, 0, 351, 352, 353, + 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 813, 373, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 409, 410, 411, 412, 1106, 414, 415, + 407, 408, 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 436, 437, 438, 1107, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 812, 0, 0, 458, 459, - 0, 460, 461, 462, 463, 464, 465, 466, 467, 0, - 468, 1108, 1109, 0, 0, 471, 472, 813, 474, 814, - 1110, 476, 477, 815, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 1111, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 1112, 498, 0, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 438, 439, 440, 1112, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 457, 458, 815, 0, 0, + 460, 461, 0, 462, 463, 464, 465, 466, 467, 468, + 469, 0, 470, 1113, 1114, 0, 0, 473, 474, 816, + 476, 817, 1115, 478, 479, 818, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 1116, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 1117, + 500, 0, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 1113, 0, 0, 0, 0, - 0, 0, 1114, 1115, 1116, 0, 0, 0, 0, 1117, - 0, 1118, 0, 0, 0, 0, 1119, 1120, 1121, 1122, - 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, 0, 0, + 525, 526, 527, 528, 529, 530, 531, 1118, 0, 0, + 0, 0, 0, 0, 1119, 2394, 1121, 0, 0, 0, + 0, 1122, 0, 1123, 0, 0, 0, 0, 1124, 1125, + 1126, 1127, 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 1077, 0, 0, - 131, 132, 133, 0, 134, 135, 136, 137, 138, 139, - 140, 141, 1078, 143, 1079, 1080, 0, 0, 146, 147, - 148, 149, 150, 1081, 802, 151, 152, 153, 154, 1082, - 1083, 157, 0, 158, 159, 160, 161, 803, 0, 804, - 0, 1084, 165, 166, 167, 168, 169, 170, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 1085, 192, 193, 1086, 195, 1087, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 1088, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 1089, 224, 225, 226, 227, 228, 229, 805, 1090, - 231, 0, 232, 233, 1091, 235, 0, 236, 0, 237, - 238, 0, 239, 240, 241, 242, 243, 244, 245, 246, - 0, 1092, 1093, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 271, 272, - 273, 274, 275, 276, 277, 1094, 1095, 0, 1096, 0, - 281, 282, 283, 284, 285, 286, 287, 288, 1097, 289, - 290, 291, 0, 0, 292, 293, 294, 295, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 1098, 305, 306, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 1082, + 0, 0, 133, 134, 135, 0, 136, 137, 138, 139, + 140, 141, 142, 143, 1083, 145, 1084, 1085, 0, 0, + 148, 149, 150, 151, 152, 1086, 805, 153, 154, 155, + 156, 1087, 1088, 159, 0, 160, 161, 162, 163, 806, + 0, 807, 0, 1089, 167, 168, 169, 170, 171, 172, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 1090, 194, 195, 1091, 197, 1092, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 1093, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 1094, 226, 227, 228, 229, 230, 231, + 808, 1095, 233, 0, 234, 235, 1096, 237, 0, 238, + 0, 239, 240, 0, 241, 242, 243, 244, 245, 246, + 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 273, 274, 275, 276, 277, 278, 279, 1099, 1100, 0, + 1101, 0, 283, 284, 285, 286, 287, 288, 289, 290, + 1102, 291, 292, 293, 0, 0, 294, 295, 296, 297, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 1099, 328, 1100, 330, 331, 332, 333, 1101, 334, 335, - 336, 337, 1102, 807, 339, 1103, 341, 342, 343, 0, - 344, 345, 0, 0, 1104, 347, 348, 0, 0, 349, - 350, 351, 352, 353, 354, 809, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 810, 371, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 1105, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 409, 410, 411, - 412, 1106, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 436, 437, 438, - 1107, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 812, - 0, 0, 458, 459, 0, 460, 461, 462, 463, 464, - 465, 466, 467, 0, 468, 1108, 1109, 0, 0, 471, - 472, 813, 474, 814, 1110, 476, 477, 815, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 1111, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 1112, 498, 0, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 327, 328, 1104, 330, 1105, 332, 333, 334, 335, 1106, + 336, 337, 338, 339, 1107, 810, 341, 1108, 343, 344, + 345, 0, 346, 347, 0, 0, 1109, 349, 350, 0, + 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 813, 373, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 1110, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 411, + 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 438, + 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, + 458, 815, 0, 0, 460, 461, 0, 462, 463, 464, + 465, 466, 467, 468, 469, 0, 470, 1113, 1114, 0, + 0, 473, 474, 816, 476, 817, 1115, 478, 479, 818, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 1116, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 1117, 500, 0, 501, 502, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 1113, - 0, 0, 0, 0, 0, 0, 1114, 2382, 1116, 0, - 0, 0, 0, 1117, 0, 1118, 0, 0, 0, 0, - 1119, 1120, 1121, 1122, 119, 1071, 833, 1072, 1073, 1074, - 1075, 1076, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 1077, 0, 0, 131, 132, 133, 0, 134, 135, - 136, 137, 138, 139, 140, 141, 1078, 143, 1079, 1080, - 0, 0, 146, 147, 148, 149, 150, 1081, 802, 151, - 152, 153, 154, 1082, 1083, 157, 0, 158, 159, 160, - 161, 803, 0, 804, 0, 1084, 165, 166, 167, 168, - 169, 170, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 1085, 192, 193, 1086, 195, 1087, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 1088, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 1089, 224, 225, 226, 227, - 228, 229, 805, 1090, 231, 0, 232, 233, 1091, 235, - 0, 236, 0, 237, 238, 0, 239, 240, 241, 242, - 243, 244, 245, 246, 0, 1092, 1093, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 271, 272, 273, 274, 275, 276, 277, 1094, - 1095, 0, 1096, 0, 281, 282, 283, 284, 285, 286, - 287, 288, 1097, 289, 290, 291, 0, 0, 292, 293, - 294, 295, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 1098, 305, 306, 307, 308, 309, 310, 311, 312, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 1118, 0, 0, 0, 0, 0, 0, 1119, 1120, + 1121, 0, 0, 0, 0, 1122, 0, 2653, 0, 0, + 0, 0, 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, + 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 1082, 0, 0, 133, 134, 135, 0, + 136, 137, 138, 139, 140, 141, 142, 143, 1083, 145, + 1084, 1085, 0, 0, 148, 149, 150, 151, 152, 1086, + 805, 153, 154, 155, 156, 1087, 1088, 159, 0, 160, + 161, 162, 163, 806, 0, 807, 0, 1089, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 1090, 194, 195, 1091, + 197, 1092, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 1093, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 1094, 226, 227, + 228, 229, 230, 231, 808, 1095, 233, 0, 234, 235, + 1096, 237, 0, 238, 0, 239, 240, 0, 241, 242, + 243, 244, 245, 246, 247, 248, 0, 1097, 1098, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 273, 274, 275, 276, 277, 278, + 279, 1099, 1100, 0, 1101, 0, 283, 284, 285, 286, + 287, 288, 289, 290, 1102, 291, 292, 293, 0, 0, + 294, 295, 296, 297, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 1099, 328, 1100, 330, 331, 332, - 333, 1101, 334, 335, 336, 337, 1102, 807, 339, 1103, - 341, 342, 343, 0, 344, 345, 0, 0, 1104, 347, - 348, 0, 0, 349, 350, 351, 352, 353, 354, 809, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 810, 371, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 1105, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 409, 410, 411, 412, 1106, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 436, 437, 438, 1107, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 1104, 330, 1105, 332, + 333, 334, 335, 1106, 336, 337, 338, 339, 1107, 810, + 341, 1108, 343, 344, 345, 0, 346, 347, 0, 0, + 1109, 349, 350, 0, 0, 351, 352, 353, 354, 355, + 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 813, 373, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 455, 456, 812, 0, 0, 458, 459, 0, 460, - 461, 462, 463, 464, 465, 466, 467, 0, 468, 1108, - 1109, 0, 0, 471, 472, 813, 474, 814, 1110, 476, - 477, 815, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 1111, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 1112, 498, 0, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 457, 458, 815, 0, 0, 460, 461, + 0, 462, 463, 464, 465, 466, 467, 468, 469, 0, + 470, 1113, 1114, 0, 0, 473, 474, 816, 476, 817, + 1115, 478, 479, 818, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 1116, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 1117, 500, 0, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 1113, 0, 0, 0, 0, 0, 0, - 1114, 1115, 1116, 0, 0, 0, 0, 1117, 0, 2641, - 0, 0, 0, 0, 1119, 1120, 1121, 1122, 119, 1071, - 833, 1072, 1073, 1074, 1075, 1076, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 1077, 0, 0, 131, 132, - 133, 0, 134, 135, 136, 137, 138, 139, 140, 141, - 1078, 143, 1079, 1080, 0, 0, 146, 147, 148, 149, - 150, 1081, 802, 151, 152, 153, 154, 1082, 1083, 157, - 0, 158, 159, 160, 161, 803, 0, 804, 0, 1084, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 1085, 192, - 193, 1086, 195, 1087, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 1088, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 1089, - 224, 225, 226, 227, 228, 229, 805, 1090, 231, 0, - 232, 233, 1091, 235, 0, 236, 0, 237, 238, 0, - 239, 240, 241, 242, 243, 244, 245, 246, 0, 1092, - 1093, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 271, 272, 273, 274, - 275, 276, 277, 1094, 1095, 0, 1096, 0, 281, 282, - 283, 284, 285, 286, 287, 288, 1097, 289, 290, 291, - 0, 0, 292, 293, 294, 295, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 1098, 305, 306, 307, 308, + 527, 528, 529, 530, 531, 1118, 0, 0, 0, 0, + 0, 0, 1119, 3299, 1121, 0, 0, 0, 0, 1122, + 0, 1123, 0, 0, 0, 0, 1124, 1125, 1126, 1127, + 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 1082, 0, 0, + 133, 134, 135, 0, 136, 137, 138, 139, 140, 141, + 142, 143, 1083, 145, 1084, 1085, 0, 0, 148, 149, + 150, 151, 152, 1086, 805, 153, 154, 155, 156, 1087, + 1088, 159, 0, 160, 161, 162, 163, 806, 0, 807, + 0, 1089, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 182, + 183, 3239, 185, 186, 187, 188, 189, 190, 191, 192, + 1090, 194, 195, 1091, 197, 1092, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 1093, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 1094, 226, 227, 228, 229, 230, 231, 808, 1095, + 233, 0, 234, 235, 1096, 237, 0, 238, 0, 239, + 240, 0, 241, 242, 243, 244, 245, 246, 247, 248, + 0, 3240, 1098, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 273, 274, + 275, 276, 277, 278, 279, 1099, 1100, 0, 1101, 0, + 283, 284, 285, 286, 287, 288, 289, 290, 1102, 291, + 292, 293, 0, 0, 294, 295, 296, 297, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 1099, 328, - 1100, 330, 331, 332, 333, 1101, 334, 335, 336, 337, - 1102, 807, 339, 1103, 341, 342, 343, 0, 344, 345, - 0, 0, 1104, 347, 348, 0, 0, 349, 350, 351, - 352, 353, 354, 809, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 810, 371, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 1105, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 409, 410, 411, 412, 1106, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 436, 437, 438, 1107, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 455, 456, 812, 0, 0, - 458, 459, 0, 460, 461, 462, 463, 464, 465, 466, - 467, 0, 468, 1108, 1109, 0, 0, 471, 472, 813, - 474, 814, 1110, 476, 477, 815, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 1111, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 1112, - 498, 0, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 1104, 330, 1105, 332, 333, 334, 335, 1106, 336, 337, + 338, 339, 1107, 810, 341, 1108, 343, 344, 345, 0, + 346, 347, 0, 0, 1109, 349, 350, 0, 0, 351, + 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 813, 373, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 1110, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, + 3241, 1111, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 438, 439, 440, + 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 815, + 0, 0, 460, 461, 0, 462, 463, 464, 465, 466, + 467, 468, 469, 0, 470, 1113, 1114, 0, 0, 473, + 474, 816, 476, 817, 1115, 478, 479, 818, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 1116, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 1117, 500, 0, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 1113, 0, 0, - 0, 0, 0, 0, 1114, 3285, 1116, 0, 0, 0, - 0, 1117, 0, 1118, 0, 0, 0, 0, 1119, 1120, - 1121, 1122, 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 1077, - 0, 0, 131, 132, 133, 0, 134, 135, 136, 137, - 138, 139, 140, 141, 1078, 143, 1079, 1080, 0, 0, - 146, 147, 148, 149, 150, 1081, 802, 151, 152, 153, - 154, 1082, 1083, 157, 0, 158, 159, 160, 161, 803, - 0, 804, 0, 1084, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 180, 181, 3225, 183, 184, 185, 186, 187, 188, - 189, 190, 1085, 192, 193, 1086, 195, 1087, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 1088, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 1089, 224, 225, 226, 227, 228, 229, - 805, 1090, 231, 0, 232, 233, 1091, 235, 0, 236, - 0, 237, 238, 0, 239, 240, 241, 242, 243, 244, - 245, 246, 0, 3226, 1093, 249, 250, 0, 251, 252, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 1118, + 0, 0, 0, 0, 0, 0, 1119, 1120, 1121, 0, + 0, 0, 0, 1122, 0, 3242, 0, 0, 0, 0, + 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, 1078, 1079, + 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 1082, 0, 0, 133, 134, 135, 0, 136, 137, + 138, 139, 140, 141, 142, 3741, 1083, 145, 1084, 1085, + 0, 0, 148, 149, 150, 151, 152, 1086, 805, 153, + 154, 155, 156, 1087, 1088, 159, 0, 160, 161, 162, + 163, 806, 0, 807, 0, 1089, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 182, 183, 184, 3742, 186, 187, 188, + 189, 190, 191, 192, 1090, 194, 195, 1091, 197, 1092, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 1093, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 1094, 226, 227, 228, 229, + 230, 231, 808, 1095, 233, 0, 234, 235, 1096, 237, + 0, 238, 0, 239, 240, 0, 241, 242, 243, 244, + 245, 246, 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 271, 272, 273, 274, 275, 276, 277, 1094, 1095, 0, - 1096, 0, 281, 282, 283, 284, 285, 286, 287, 288, - 1097, 289, 290, 291, 0, 0, 292, 293, 294, 295, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 1098, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 273, 274, 275, 276, 277, 278, 279, 1099, + 1100, 0, 1101, 0, 283, 284, 285, 286, 287, 288, + 289, 290, 1102, 291, 292, 293, 0, 0, 294, 295, + 296, 297, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 1099, 328, 1100, 330, 331, 332, 333, 1101, - 334, 335, 336, 337, 1102, 807, 339, 1103, 341, 342, - 343, 0, 344, 345, 0, 0, 1104, 347, 348, 0, - 0, 349, 350, 351, 352, 353, 354, 809, 356, 357, + 325, 326, 327, 328, 1104, 330, 1105, 332, 333, 334, + 335, 1106, 336, 337, 338, 339, 1107, 810, 341, 1108, + 343, 344, 345, 0, 346, 347, 0, 0, 1109, 349, + 350, 0, 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 810, 371, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 1105, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 409, - 410, 411, 3227, 1106, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 436, - 437, 438, 1107, 440, 0, 441, 442, 443, 444, 445, + 368, 369, 0, 0, 0, 0, 370, 371, 813, 373, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 1110, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 812, 0, 0, 458, 459, 0, 460, 461, 462, - 463, 464, 465, 466, 467, 0, 468, 1108, 1109, 0, - 0, 471, 472, 813, 474, 814, 1110, 476, 477, 815, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 1111, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 1112, 498, 0, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 456, 457, 458, 815, 0, 0, 460, 461, 0, 462, + 463, 464, 465, 466, 467, 468, 469, 0, 470, 1113, + 1114, 0, 0, 473, 474, 816, 476, 817, 1115, 478, + 479, 818, 481, 482, 3743, 484, 485, 0, 0, 486, + 487, 488, 1116, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 1117, 500, 0, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 1113, 0, 0, 0, 0, 0, 0, 1114, 1115, - 1116, 0, 0, 0, 0, 1117, 0, 3228, 0, 0, - 0, 0, 1119, 1120, 1121, 1122, 119, 1071, 833, 1072, - 1073, 1074, 1075, 1076, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 1077, 0, 0, 131, 132, 133, 0, - 134, 135, 136, 137, 138, 139, 140, 3727, 1078, 143, - 1079, 1080, 0, 0, 146, 147, 148, 149, 150, 1081, - 802, 151, 152, 153, 154, 1082, 1083, 157, 0, 158, - 159, 160, 161, 803, 0, 804, 0, 1084, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 180, 181, 182, 3728, 184, - 185, 186, 187, 188, 189, 190, 1085, 192, 193, 1086, - 195, 1087, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 1088, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 1089, 224, 225, - 226, 227, 228, 229, 805, 1090, 231, 0, 232, 233, - 1091, 235, 0, 236, 0, 237, 238, 0, 239, 240, - 241, 242, 243, 244, 245, 246, 0, 1092, 1093, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 271, 272, 273, 274, 275, 276, - 277, 1094, 1095, 0, 1096, 0, 281, 282, 283, 284, - 285, 286, 287, 288, 1097, 289, 290, 291, 0, 0, - 292, 293, 294, 295, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 1098, 305, 306, 307, 308, 309, 310, + 529, 530, 531, 1118, 0, 0, 0, 0, 0, 0, + 1119, 1120, 1121, 0, 0, 0, 0, 1122, 0, 1123, + 0, 0, 0, 0, 1124, 1125, 1126, 1127, 121, 1076, + 836, 1077, 1078, 1079, 1080, 1081, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 1082, 0, 0, 133, 134, + 135, 0, 136, 137, 138, 139, 140, 141, 142, 143, + 1083, 145, 1084, 1085, 0, 0, 148, 149, 150, 151, + 152, 1086, 805, 153, 154, 155, 156, 1087, 1088, 159, + 0, 160, 161, 162, 163, 806, 0, 807, 0, 1089, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 182, 183, 184, + 3742, 186, 187, 188, 189, 190, 191, 192, 1090, 194, + 195, 1091, 197, 1092, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 1093, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 1094, + 226, 227, 228, 229, 230, 231, 808, 1095, 233, 0, + 234, 235, 1096, 237, 0, 238, 0, 239, 240, 0, + 241, 242, 243, 244, 245, 246, 247, 248, 0, 1097, + 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 273, 274, 275, 276, + 277, 278, 279, 1099, 1100, 0, 1101, 0, 283, 284, + 285, 286, 287, 288, 289, 290, 1102, 291, 292, 293, + 0, 0, 294, 295, 296, 297, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 1099, 328, 1100, 330, - 331, 332, 333, 1101, 334, 335, 336, 337, 1102, 807, - 339, 1103, 341, 342, 343, 0, 344, 345, 0, 0, - 1104, 347, 348, 0, 0, 349, 350, 351, 352, 353, - 354, 809, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 810, 371, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 1105, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 1104, 330, + 1105, 332, 333, 334, 335, 1106, 336, 337, 338, 339, + 1107, 810, 341, 1108, 343, 344, 345, 0, 346, 347, + 0, 0, 1109, 349, 350, 0, 0, 351, 352, 353, + 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 813, 373, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 409, 410, 411, 412, 1106, 414, 415, + 407, 408, 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 436, 437, 438, 1107, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 812, 0, 0, 458, 459, - 0, 460, 461, 462, 463, 464, 465, 466, 467, 0, - 468, 1108, 1109, 0, 0, 471, 472, 813, 474, 814, - 1110, 476, 477, 815, 479, 480, 3729, 482, 483, 0, - 0, 484, 485, 486, 1111, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 1112, 498, 0, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 438, 439, 440, 1112, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 457, 458, 815, 0, 0, + 460, 461, 0, 462, 463, 464, 465, 466, 467, 468, + 469, 0, 470, 1113, 1114, 0, 0, 473, 474, 816, + 476, 817, 1115, 478, 479, 818, 481, 482, 3743, 484, + 485, 0, 0, 486, 487, 488, 1116, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 1117, + 500, 0, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 1113, 0, 0, 0, 0, - 0, 0, 1114, 1115, 1116, 0, 0, 0, 0, 1117, - 0, 1118, 0, 0, 0, 0, 1119, 1120, 1121, 1122, - 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, 0, 0, + 525, 526, 527, 528, 529, 530, 531, 1118, 0, 0, + 0, 0, 0, 0, 1119, 1120, 1121, 0, 0, 0, + 0, 1122, 0, 1123, 0, 0, 0, 0, 1124, 1125, + 1126, 1127, 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 1077, 0, 0, - 131, 132, 133, 0, 134, 135, 136, 137, 138, 139, - 140, 141, 1078, 143, 1079, 1080, 0, 0, 146, 147, - 148, 149, 150, 1081, 802, 151, 152, 153, 154, 1082, - 1083, 157, 0, 158, 159, 160, 161, 803, 0, 804, - 0, 1084, 165, 166, 167, 168, 169, 170, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 180, - 181, 182, 3728, 184, 185, 186, 187, 188, 189, 190, - 1085, 192, 193, 1086, 195, 1087, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 1088, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 1089, 224, 225, 226, 227, 228, 229, 805, 1090, - 231, 0, 232, 233, 1091, 235, 0, 236, 0, 237, - 238, 0, 239, 240, 241, 242, 243, 244, 245, 246, - 0, 1092, 1093, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 271, 272, - 273, 274, 275, 276, 277, 1094, 1095, 0, 1096, 0, - 281, 282, 283, 284, 285, 286, 287, 288, 1097, 289, - 290, 291, 0, 0, 292, 293, 294, 295, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 1098, 305, 306, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 1082, + 0, 0, 133, 134, 135, 0, 136, 137, 138, 139, + 140, 141, 142, -2183, 1083, 145, 1084, 1085, 0, 0, + 148, 149, 150, 151, 152, 1086, 805, 153, 154, 155, + 156, 1087, 1088, 159, 0, 160, 161, 162, 163, 806, + 0, 807, 0, 1089, 167, 168, 169, 170, 171, 172, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 182, 183, 184, 3742, 186, 187, 188, 189, 190, + 191, 192, 1090, 194, 195, 1091, 197, 1092, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 1093, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 1094, 226, 227, 228, 229, 230, 231, + 808, 1095, 233, 0, 234, 235, 1096, 237, 0, 238, + 0, 239, 240, 0, 241, 242, 243, 244, -2183, 246, + 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, -2183, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 273, 274, 275, 276, 277, 278, 279, 1099, 1100, 0, + 1101, 0, 283, 0, 0, 286, 287, 288, 289, 290, + 1102, 291, 292, 293, 0, 0, 294, 295, 296, -2183, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 1099, 328, 1100, 330, 331, 332, 333, 1101, 334, 335, - 336, 337, 1102, 807, 339, 1103, 341, 342, 343, 0, - 344, 345, 0, 0, 1104, 347, 348, 0, 0, 349, - 350, 351, 352, 353, 354, 809, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 810, 371, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 1105, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 409, 410, 411, - 412, 1106, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 436, 437, 438, - 1107, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 812, - 0, 0, 458, 459, 0, 460, 461, 462, 463, 464, - 465, 466, 467, 0, 468, 1108, 1109, 0, 0, 471, - 472, 813, 474, 814, 1110, 476, 477, 815, 479, 480, - 3729, 482, 483, 0, 0, 484, 485, 486, 1111, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 1112, 498, 0, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 327, 328, 1104, 330, 1105, 332, 333, 334, 335, 0, + 336, 337, 0, 339, 1107, 810, 341, 1108, 343, 344, + 345, 0, 346, 347, 0, 0, 1109, 349, 350, 0, + 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 813, 373, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 1110, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 411, + 412, 413, 414, 1111, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, -2183, + 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, + 458, 815, 0, 0, 460, 461, 0, 462, 463, 464, + 465, 466, 467, 468, 469, 0, 470, 1113, 1114, 0, + 0, 473, 474, 816, 476, 817, 1115, 478, 479, 818, + 481, 482, 3743, 484, 485, 0, 0, 486, 487, 488, + 1116, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 1117, 500, 0, 501, 502, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 1113, - 0, 0, 0, 0, 0, 0, 1114, 1115, 1116, 0, - 0, 0, 0, 1117, 0, 1118, 0, 0, 0, 0, - 1119, 1120, 1121, 1122, 119, 1071, 833, 1072, 1073, 1074, - 1075, 1076, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 1077, 0, 0, 131, 132, 133, 0, 134, 135, - 136, 137, 138, 139, 140, -2180, 1078, 143, 1079, 1080, - 0, 0, 146, 147, 148, 149, 150, 1081, 802, 151, - 152, 153, 154, 1082, 1083, 157, 0, 158, 159, 160, - 161, 803, 0, 804, 0, 1084, 165, 166, 167, 168, - 169, 170, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 180, 181, 182, 3728, 184, 185, 186, - 187, 188, 189, 190, 1085, 192, 193, 1086, 195, 1087, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 1088, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 1089, 224, 225, 226, 227, - 228, 229, 805, 1090, 231, 0, 232, 233, 1091, 235, - 0, 236, 0, 237, 238, 0, 239, 240, 241, 242, - -2180, 244, 245, 246, 0, 1092, 1093, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, -2180, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 271, 272, 273, 274, 275, 276, 277, 1094, - 1095, 0, 1096, 0, 281, 0, 0, 284, 285, 286, - 287, 288, 1097, 289, 290, 291, 0, 0, 292, 293, - 294, -2180, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 1098, 305, 306, 307, 308, 309, 310, 311, 312, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, -2183, 0, 0, 0, 0, 0, 0, 1119, 1120, + 1121, 0, 0, 0, 0, 1122, 0, 1123, 0, 0, + 0, 0, 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, + 1078, 1079, 1080, 1081, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 1082, 0, 0, 133, 134, 135, 0, + 136, 137, 138, 139, 140, 141, 142, 143, 1083, 145, + 1084, 1085, 0, 0, 148, 149, 150, 151, 152, 1086, + 805, 153, 154, 155, 156, 1087, 1088, 159, 0, 160, + 161, 162, 163, 806, 0, 807, 0, 166, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 1090, 194, 195, 1091, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 1093, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 1094, 226, 227, + 228, 229, 230, 231, 808, 1095, 233, 0, 234, 235, + 1096, 237, 0, 238, 0, 239, 240, 0, 241, 242, + 243, 244, 245, 246, 247, 248, 0, 1097, 1098, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 273, 274, 275, 276, 277, 278, + 279, 1099, 1100, 0, 1101, 0, 283, 284, 285, 286, + 287, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 295, 296, 297, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 1099, 328, 1100, 330, 331, 332, - 333, 0, 334, 335, 0, 337, 1102, 807, 339, 1103, - 341, 342, 343, 0, 344, 345, 0, 0, 1104, 347, - 348, 0, 0, 349, 350, 351, 352, 353, 354, 809, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 810, 371, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 1105, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 409, 410, 411, 412, 1106, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, -2180, 437, 438, 1107, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 1104, 330, 1105, 332, + 333, 334, 335, 0, 336, 337, 338, 339, 1107, 810, + 341, 1108, 343, 344, 345, 0, 346, 347, 0, 0, + 1109, 349, 350, 0, 0, 351, 352, 353, 354, 355, + 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 813, 373, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 455, 456, 812, 0, 0, 458, 459, 0, 460, - 461, 462, 463, 464, 465, 466, 467, 0, 468, 1108, - 1109, 0, 0, 471, 472, 813, 474, 814, 1110, 476, - 477, 815, 479, 480, 3729, 482, 483, 0, 0, 484, - 485, 486, 1111, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 1112, 498, 0, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 457, 458, 815, 0, 0, 460, 461, + 0, 462, 463, 464, 465, 466, 467, 468, 469, 0, + 470, 1113, 1114, 0, 0, 473, 474, 816, 476, 817, + 1115, 478, 479, 818, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 1117, 500, 0, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, -2180, 0, 0, 0, 0, 0, 0, - 1114, 1115, 1116, 0, 0, 0, 0, 1117, 0, 1118, - 0, 0, 0, 0, 1119, 1120, 1121, 1122, 119, 1071, - 833, 1072, 1073, 1074, 1075, 1076, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 1077, 0, 0, 131, 132, - 133, 0, 134, 135, 136, 137, 138, 139, 140, 141, - 1078, 143, 1079, 1080, 0, 0, 146, 147, 148, 149, - 150, 1081, 802, 151, 152, 153, 154, 1082, 1083, 157, - 0, 158, 159, 160, 161, 803, 0, 804, 0, 164, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 1085, 192, - 193, 1086, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 1088, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 1089, - 224, 225, 226, 227, 228, 229, 805, 1090, 231, 0, - 232, 233, 1091, 235, 0, 236, 0, 237, 238, 0, - 239, 240, 241, 242, 243, 244, 245, 246, 0, 1092, - 1093, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 271, 272, 273, 274, - 275, 276, 277, 1094, 1095, 0, 1096, 0, 281, 282, - 283, 284, 285, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 293, 294, 295, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 1098, 305, 306, 307, 308, + 527, 528, 529, 530, 531, 0, 0, 0, 0, 0, + 0, 0, 1466, 1467, 0, 0, 0, 0, 0, 1122, + 0, 1123, 0, 0, 0, 0, 1124, 1125, 1126, 1127, + 121, 1076, 836, 1077, 1078, 0, 1080, 1081, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 1082, 0, 0, + 133, 134, 135, 0, 136, 137, 138, 139, 140, 141, + 142, 143, 1083, 145, 1084, 1085, 0, 0, 148, 149, + 150, 151, 152, 1086, 805, 153, 154, 155, 156, 1087, + 1088, 159, 0, 160, 161, 162, 163, 806, 0, 807, + 0, 166, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 1090, 194, 195, 1091, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 1093, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 1094, 226, 227, 228, 229, 230, 231, 808, 1095, + 233, 0, 234, 235, 1096, 237, 0, 238, 0, 239, + 240, 0, 241, 242, 243, 244, 245, 246, 247, 248, + 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 273, 274, + 275, 276, 277, 278, 279, 1099, 1100, 0, 1101, 0, + 283, 284, 285, 286, 287, 288, 289, 290, 0, 291, + 292, 293, 0, 0, 294, 295, 296, 297, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 1099, 328, - 1100, 330, 331, 332, 333, 0, 334, 335, 336, 337, - 1102, 807, 339, 1103, 341, 342, 343, 0, 344, 345, - 0, 0, 1104, 347, 348, 0, 0, 349, 350, 351, - 352, 353, 354, 809, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 810, 371, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 1105, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 409, 410, 411, 412, 1106, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 436, 437, 438, 1107, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 455, 456, 812, 0, 0, - 458, 459, 0, 460, 461, 462, 463, 464, 465, 466, - 467, 0, 468, 1108, 1109, 0, 0, 471, 472, 813, - 474, 814, 1110, 476, 477, 815, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 1112, - 498, 0, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 1104, 330, 1105, 332, 333, 334, 335, 0, 336, 337, + 338, 339, 1107, 810, 341, 1108, 343, 344, 345, 0, + 346, 347, 0, 0, 348, 349, 350, 0, 0, 351, + 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 813, 373, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 1110, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, + 414, 2280, 2281, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 438, 439, 440, + 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 815, + 0, 0, 460, 461, 0, 462, 463, 464, 465, 466, + 467, 468, 469, 0, 470, 1113, 1114, 0, 0, 473, + 474, 816, 476, 817, 1115, 478, 479, 818, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 0, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 1117, 500, 0, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 0, 0, 0, - 0, 0, 0, 0, 1459, 1460, 0, 0, 0, 0, - 0, 1117, 0, 1118, 0, 0, 0, 0, 1119, 1120, - 1121, 1122, 119, 1071, 833, 1072, 1073, 0, 1075, 1076, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 1077, - 0, 0, 131, 132, 133, 0, 134, 135, 136, 137, - 138, 139, 140, 141, 1078, 143, 1079, 1080, 0, 0, - 146, 147, 148, 149, 150, 1081, 802, 151, 152, 153, - 154, 1082, 1083, 157, 0, 158, 159, 160, 161, 803, - 0, 804, 0, 164, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 1085, 192, 193, 1086, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 1088, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 1089, 224, 225, 226, 227, 228, 229, - 805, 1090, 231, 0, 232, 233, 1091, 235, 0, 236, - 0, 237, 238, 0, 239, 240, 241, 242, 243, 244, - 245, 246, 0, 1092, 1093, 249, 250, 0, 251, 252, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 0, + 0, 0, 0, 0, 0, 0, 2282, 2283, 0, 0, + 0, 0, 0, 1122, 0, 1123, 0, 0, 0, 0, + 1124, 1125, 1126, 1127, 121, 1076, 836, 1077, 1078, 1079, + 1080, 1081, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 1082, 0, 0, 133, 134, 135, 0, 136, 137, + 138, 139, 140, 141, 142, 143, 1083, 145, 1084, 1085, + 0, 0, 148, 149, 150, 151, 152, 1086, 805, 153, + 154, 155, 156, 1087, 1088, 159, 0, 160, 161, 162, + 163, 806, 0, 807, 0, 166, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 1090, 194, 195, 1091, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 1093, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 1094, 226, 227, 228, 229, + 230, 231, 808, 1095, 233, 0, 234, 235, 1096, 237, + 0, 238, 0, 239, 240, 0, 241, 242, 243, 244, + 245, 246, 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 271, 272, 273, 274, 275, 276, 277, 1094, 1095, 0, - 1096, 0, 281, 282, 283, 284, 285, 286, 287, 288, - 0, 289, 290, 291, 0, 0, 292, 293, 294, 295, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 1098, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 273, 274, 275, 276, 277, 278, 279, 1099, + 1100, 0, 1101, 0, 283, 0, 285, 286, 287, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 295, + 296, 297, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 1099, 328, 1100, 330, 331, 332, 333, 0, - 334, 335, 336, 337, 1102, 807, 339, 1103, 341, 342, - 343, 0, 344, 345, 0, 0, 346, 347, 348, 0, - 0, 349, 350, 351, 352, 353, 354, 809, 356, 357, + 325, 326, 327, 328, 1104, 330, 1105, 332, 333, 334, + 335, 0, 336, 337, 338, 339, 1107, 810, 341, 1108, + 343, 344, 345, 0, 346, 347, 0, 0, 1109, 349, + 350, 0, 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 810, 371, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 1105, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 409, - 410, 411, 412, 2270, 2271, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 436, - 437, 438, 1107, 440, 0, 441, 442, 443, 444, 445, + 368, 369, 0, 0, 0, 0, 370, 371, 813, 373, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 1110, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 411, 412, 413, 414, 1111, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 812, 0, 0, 458, 459, 0, 460, 461, 462, - 463, 464, 465, 466, 467, 0, 468, 1108, 1109, 0, - 0, 471, 472, 813, 474, 814, 1110, 476, 477, 815, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 0, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 1112, 498, 0, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 456, 457, 458, 815, 0, 0, 460, 461, 0, 462, + 463, 464, 465, 466, 467, 468, 469, 0, 470, 1113, + 1114, 0, 0, 473, 474, 816, 476, 817, 1115, 478, + 479, 818, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 1117, 500, 0, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 0, 0, 0, 0, 0, 0, 0, 2272, 2273, - 0, 0, 0, 0, 0, 1117, 0, 1118, 0, 0, - 0, 0, 1119, 1120, 1121, 1122, 119, 1071, 833, 1072, - 1073, 1074, 1075, 1076, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 1077, 0, 0, 131, 132, 133, 0, - 134, 135, 136, 137, 138, 139, 140, 141, 1078, 143, - 1079, 1080, 0, 0, 146, 147, 148, 149, 150, 1081, - 802, 151, 152, 153, 154, 1082, 1083, 157, 0, 158, - 159, 160, 161, 803, 0, 804, 0, 164, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 1085, 192, 193, 1086, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 1088, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 1089, 224, 225, - 226, 227, 228, 229, 805, 1090, 231, 0, 232, 233, - 1091, 235, 0, 236, 0, 237, 238, 0, 239, 240, - 241, 242, 243, 244, 245, 246, 0, 1092, 1093, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 271, 272, 273, 274, 275, 276, - 277, 1094, 1095, 0, 1096, 0, 281, 0, 283, 284, - 285, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 293, 294, 295, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 1098, 305, 306, 307, 308, 309, 310, + 529, 530, 531, 0, 0, 0, 0, 0, 0, 0, + 1466, 1467, 0, 0, 0, 0, 0, 1122, 0, 1123, + 0, 0, 0, 0, 1124, 1125, 1126, 1127, 121, 1076, + 836, 1077, 1078, 0, 1080, 1081, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 1082, 0, 0, 133, 134, + 135, 0, 136, 137, 138, 139, 140, 141, 142, 143, + 1083, 145, 1084, 1085, 0, 0, 148, 149, 150, 151, + 152, 1086, 805, 153, 154, 155, 156, 1087, 1088, 159, + 0, 160, 161, 162, 163, 806, 0, 807, 0, 166, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 1090, 194, + 195, 1091, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 1093, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 1094, + 226, 227, 228, 229, 230, 231, 808, 1095, 233, 0, + 234, 235, 1096, 237, 0, 238, 0, 239, 240, 0, + 241, 242, 243, 244, 245, 246, 247, 248, 3260, 1097, + 1098, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 273, 274, 275, 276, + 277, 278, 279, 1099, 1100, 0, 1101, 0, 283, 284, + 285, 286, 287, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 295, 296, 297, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 1099, 328, 1100, 330, - 331, 332, 333, 0, 334, 335, 336, 337, 1102, 807, - 339, 1103, 341, 342, 343, 0, 344, 345, 0, 0, - 1104, 347, 348, 0, 0, 349, 350, 351, 352, 353, - 354, 809, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 810, 371, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 1105, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 1104, 330, + 1105, 332, 333, 334, 335, 0, 336, 337, 338, 339, + 1107, 810, 341, 1108, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 353, + 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 813, 373, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 409, 410, 411, 412, 1106, 414, 415, + 407, 408, 0, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 436, 437, 438, 1107, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 812, 0, 0, 458, 459, - 0, 460, 461, 462, 463, 464, 465, 466, 467, 0, - 468, 1108, 1109, 0, 0, 471, 472, 813, 474, 814, - 1110, 476, 477, 815, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 1112, 498, 0, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 438, 439, 440, 1112, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 457, 458, 815, 0, 0, + 460, 461, 0, 462, 463, 464, 465, 466, 467, 468, + 469, 0, 470, 1113, 1114, 0, 0, 473, 474, 816, + 476, 817, 1115, 478, 479, 818, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 0, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 1117, + 500, 0, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 0, 0, 0, 0, 0, - 0, 0, 1459, 1460, 0, 0, 0, 0, 0, 1117, - 0, 1118, 0, 0, 0, 0, 1119, 1120, 1121, 1122, - 119, 1071, 833, 1072, 1073, 0, 1075, 1076, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 1077, 0, 0, - 131, 132, 133, 0, 134, 135, 136, 137, 138, 139, - 140, 141, 1078, 143, 1079, 1080, 0, 0, 146, 147, - 148, 149, 150, 1081, 802, 151, 152, 153, 154, 1082, - 1083, 157, 0, 158, 159, 160, 161, 803, 0, 804, - 0, 164, 165, 166, 167, 168, 169, 170, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 1085, 192, 193, 1086, 195, 0, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 1088, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 1089, 224, 225, 226, 227, 228, 229, 805, 1090, - 231, 0, 232, 233, 1091, 235, 0, 236, 0, 237, - 238, 0, 239, 240, 241, 242, 243, 244, 245, 246, - 3246, 1092, 1093, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 271, 272, - 273, 274, 275, 276, 277, 1094, 1095, 0, 1096, 0, - 281, 282, 283, 284, 285, 286, 287, 288, 0, 289, - 290, 291, 0, 0, 292, 293, 294, 295, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 1098, 305, 306, + 525, 526, 527, 528, 529, 530, 531, 0, 0, 0, + 0, 0, 121, 1076, 836, 1077, 1078, 0, 1080, 1081, + 0, 1122, 0, 2922, 0, 0, 0, 0, 1124, 1125, + 1126, 1127, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 1082, + 0, 0, 133, 134, 135, 0, 136, 137, 138, 139, + 140, 141, 142, 143, 1083, 145, 1084, 1085, 0, 0, + 148, 149, 150, 151, 152, 1086, 805, 153, 154, 155, + 156, 1087, 1088, 159, 0, 160, 161, 162, 163, 806, + 0, 807, 0, 166, 167, 168, 169, 170, 171, 172, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 1090, 194, 195, 1091, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 1093, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 1094, 226, 227, 228, 229, 230, 231, + 808, 1095, 233, 0, 234, 235, 1096, 237, 0, 238, + 0, 239, 240, 0, 241, 242, 243, 244, 245, 246, + 247, 248, 0, 1097, 1098, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 273, 274, 275, 276, 277, 278, 279, 1099, 1100, 0, + 1101, 0, 283, 284, 285, 286, 287, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 295, 296, 297, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 1099, 328, 1100, 330, 331, 332, 333, 0, 334, 335, - 336, 337, 1102, 807, 339, 1103, 341, 342, 343, 0, - 344, 345, 0, 0, 346, 347, 348, 0, 0, 349, - 350, 351, 352, 353, 354, 809, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 810, 371, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 1105, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 409, 410, 411, + 327, 328, 1104, 330, 1105, 332, 333, 334, 335, 0, + 336, 337, 338, 339, 1107, 810, 341, 1108, 343, 344, + 345, 0, 346, 347, 0, 0, 348, 349, 350, 0, + 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 813, 373, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 1110, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 436, 437, 438, - 1107, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 812, - 0, 0, 458, 459, 0, 460, 461, 462, 463, 464, - 465, 466, 467, 0, 468, 1108, 1109, 0, 0, 471, - 472, 813, 474, 814, 1110, 476, 477, 815, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 0, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 1112, 498, 0, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 438, + 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, + 458, 815, 0, 0, 460, 461, 0, 462, 463, 464, + 465, 466, 467, 468, 469, 0, 470, 1113, 1114, 0, + 0, 473, 474, 816, 476, 817, 1115, 478, 479, 818, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 0, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 1117, 500, 0, 501, 502, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 0, - 0, 0, 0, 0, 119, 1071, 833, 1072, 1073, 0, - 1075, 1076, 0, 1117, 0, 2909, 0, 0, 0, 0, - 1119, 1120, 1121, 1122, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 1077, 0, 0, 131, 132, 133, 0, 134, 135, - 136, 137, 138, 139, 140, 141, 1078, 143, 1079, 1080, - 0, 0, 146, 147, 148, 149, 150, 1081, 802, 151, - 152, 153, 154, 1082, 1083, 157, 0, 158, 159, 160, - 161, 803, 0, 804, 0, 164, 165, 166, 167, 168, - 169, 170, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 1085, 192, 193, 1086, 195, 0, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 1088, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 1089, 224, 225, 226, 227, - 228, 229, 805, 1090, 231, 0, 232, 233, 1091, 235, - 0, 236, 0, 237, 238, 0, 239, 240, 241, 242, - 243, 244, 245, 246, 0, 1092, 1093, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 271, 272, 273, 274, 275, 276, 277, 1094, - 1095, 0, 1096, 0, 281, 282, 283, 284, 285, 286, - 287, 288, 0, 289, 290, 291, 0, 0, 292, 293, - 294, 295, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 1098, 305, 306, 307, 308, 309, 310, 311, 312, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 0, 0, 0, 0, 0, 121, 1076, 836, 1077, + 1078, 0, 1080, 1081, 0, 1122, 0, 2922, 0, 0, + 0, 0, 1124, 1125, 1126, 1127, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 1082, 0, 0, 133, 134, 135, 0, + 136, 137, 138, 139, 140, 141, 142, 143, 1083, 145, + 1084, 1085, 0, 0, 148, 149, 150, 151, 152, 1086, + 805, 153, 154, 155, 156, 1087, 1088, 159, 0, 160, + 161, 162, 163, 806, 0, 807, 0, 166, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 1090, 194, 195, 1091, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 1093, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 1094, 226, 227, + 228, 229, 230, 231, 808, 1095, 233, 0, 234, 235, + 1096, 237, 0, 238, 0, 239, 240, 0, 241, 242, + 243, 244, 245, 246, 247, 248, 0, 1097, 1098, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 273, 274, 275, 276, 277, 278, + 279, 1099, 1100, 0, 1101, 0, 283, 284, 285, 286, + 287, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 295, 296, 297, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 1099, 328, 1100, 330, 331, 332, - 333, 0, 334, 335, 336, 337, 1102, 807, 339, 1103, - 341, 342, 343, 0, 344, 345, 0, 0, 346, 347, - 348, 0, 0, 349, 350, 351, 352, 353, 354, 809, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 810, 371, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 1105, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 436, 437, 438, 1107, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 1104, 330, 1105, 332, + 333, 334, 335, 0, 336, 337, 338, 339, 1107, 810, + 341, 1108, 343, 344, 345, 0, 346, 347, 0, 0, + 348, 349, 350, 0, 0, 351, 352, 353, 354, 355, + 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 813, 373, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 1110, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 411, 412, 413, 414, 1111, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 438, 439, 440, 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 455, 456, 812, 0, 0, 458, 459, 0, 460, - 461, 462, 463, 464, 465, 466, 467, 0, 468, 1108, - 1109, 0, 0, 471, 472, 813, 474, 814, 1110, 476, - 477, 815, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 0, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 1112, 498, 0, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 457, 458, 815, 0, 0, 460, 461, + 0, 462, 463, 464, 465, 466, 467, 468, 469, 0, + 470, 1113, 1114, 0, 0, 473, 474, 816, 476, 817, + 1115, 478, 479, 818, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 1117, 500, 0, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 0, 0, 0, 0, 0, 119, 1071, - 833, 1072, 1073, 0, 1075, 1076, 0, 1117, 0, 2909, - 0, 0, 0, 0, 1119, 1120, 1121, 1122, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 1077, 0, 0, 131, 132, - 133, 0, 134, 135, 136, 137, 138, 139, 140, 141, - 1078, 143, 1079, 1080, 0, 0, 146, 147, 148, 149, - 150, 1081, 802, 151, 152, 153, 154, 1082, 1083, 157, - 0, 158, 159, 160, 161, 803, 0, 804, 0, 164, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 1085, 192, - 193, 1086, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 1088, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 1089, - 224, 225, 226, 227, 228, 229, 805, 1090, 231, 0, - 232, 233, 1091, 235, 0, 236, 0, 237, 238, 0, - 239, 240, 241, 242, 243, 244, 245, 246, 0, 1092, - 1093, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 271, 272, 273, 274, - 275, 276, 277, 1094, 1095, 0, 1096, 0, 281, 282, - 283, 284, 285, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 293, 294, 295, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 1098, 305, 306, 307, 308, + 527, 528, 529, 530, 531, 0, 0, 0, 0, 0, + 121, 1076, 836, 1077, 1078, 1079, 1080, 1081, 0, 1122, + 0, 1123, 0, 0, 0, 0, 1124, 1125, 1126, 1127, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 1082, 0, 0, + 133, 134, 135, 0, 136, 137, 138, 139, 140, 141, + 142, 0, 1083, 145, 1084, 1085, 0, 0, 148, 149, + 150, 151, 152, 1086, 805, 153, 154, 155, 156, 1087, + 1088, 159, 0, 160, 161, 162, 163, 806, 0, 807, + 0, 1089, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 1090, 194, 195, 1091, 197, 1092, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 1093, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 0, 220, 0, 221, 222, 223, + 224, 1094, 226, 227, 228, 229, 230, 231, 808, 1095, + 233, 0, 234, 235, 1096, 237, 0, 238, 0, 239, + 240, 0, 241, 242, 243, 244, 0, 246, 247, 248, + 0, 1097, 1098, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 0, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 273, 274, + 275, 276, 277, 278, 279, 1099, 1100, 0, 1101, 0, + 283, 0, 0, 286, 287, 288, 289, 290, 1102, 291, + 292, 293, 0, 0, 294, 295, 296, 0, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 1099, 328, - 1100, 330, 331, 332, 333, 0, 334, 335, 336, 337, - 1102, 807, 339, 1103, 341, 342, 343, 0, 344, 345, - 0, 0, 346, 347, 348, 0, 0, 349, 350, 351, - 352, 353, 354, 809, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 810, 371, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 1105, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 409, 410, 411, 412, 1106, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 436, 437, 438, 1107, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 455, 456, 812, 0, 0, - 458, 459, 0, 460, 461, 462, 463, 464, 465, 466, - 467, 0, 468, 1108, 1109, 0, 0, 471, 472, 813, - 474, 814, 1110, 476, 477, 815, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 1112, - 498, 0, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 1104, 330, 1105, 332, 333, 334, 335, 0, 336, 337, + 0, 339, 1107, 810, 341, 1108, 343, 344, 345, 0, + 346, 347, 0, 0, 1109, 349, 350, 0, 0, 351, + 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 813, 373, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 1110, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, + 414, 1111, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 0, 439, 440, + 1112, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 815, + 0, 0, 460, 461, 0, 462, 463, 464, 465, 466, + 467, 468, 469, 0, 470, 1113, 1114, 0, 0, 473, + 474, 816, 476, 817, 1115, 478, 479, 818, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 1116, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 1117, 500, 0, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 0, 0, 0, - 0, 0, 119, 1071, 833, 1072, 1073, 1074, 1075, 1076, - 0, 1117, 0, 1118, 0, 0, 0, 0, 1119, 1120, - 1121, 1122, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 1077, - 0, 0, 131, 132, 133, 0, 134, 135, 136, 137, - 138, 139, 140, 0, 1078, 143, 1079, 1080, 0, 0, - 146, 147, 148, 149, 150, 1081, 802, 151, 152, 153, - 154, 1082, 1083, 157, 0, 158, 159, 160, 161, 803, - 0, 804, 0, 1084, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 1085, 192, 193, 1086, 195, 1087, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 1088, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 0, 218, 0, 219, - 220, 221, 222, 1089, 224, 225, 226, 227, 228, 229, - 805, 1090, 231, 0, 232, 233, 1091, 235, 0, 236, - 0, 237, 238, 0, 239, 240, 241, 242, 0, 244, - 245, 246, 0, 1092, 1093, 249, 250, 0, 251, 252, - 253, 254, 255, 256, 257, 0, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 271, 272, 273, 274, 275, 276, 277, 1094, 1095, 0, - 1096, 0, 281, 0, 0, 284, 285, 286, 287, 288, - 1097, 289, 290, 291, 0, 0, 292, 293, 294, 0, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 1098, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 0, + 0, 0, 0, 0, 121, 0, 1119, 1120, 1121, 0, + 0, 1081, 0, 1122, 0, 1123, 0, 0, 0, 0, + 1124, 1125, 1126, 1127, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 1082, 0, 0, 133, 134, 135, 0, 136, 137, + 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 0, 0, 148, 149, 150, 151, 152, 1086, 805, 153, + 154, 155, 156, 157, 158, 159, 0, 160, 161, 162, + 163, 806, 0, 807, 0, 166, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 211, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 808, 0, 233, 0, 234, 235, 236, 237, + 0, 238, 0, 239, 240, 0, 241, 242, 243, 244, + 245, 246, 247, 248, 0, 249, 250, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 273, 274, 275, 276, 277, 278, 279, 280, + 281, 0, 282, 0, 283, 284, 285, 286, 287, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 295, + 296, 297, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 1103, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 1099, 328, 1100, 330, 331, 332, 333, 0, - 334, 335, 0, 337, 1102, 807, 339, 1103, 341, 342, - 343, 0, 344, 345, 0, 0, 1104, 347, 348, 0, - 0, 349, 350, 351, 352, 353, 354, 809, 356, 357, + 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, + 335, 0, 336, 337, 338, 339, 0, 810, 341, 342, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 810, 371, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 1105, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 409, - 410, 411, 412, 1106, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 0, - 437, 438, 1107, 440, 0, 441, 442, 443, 444, 445, + 368, 369, 0, 0, 0, 0, 370, 371, 813, 373, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 438, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 812, 0, 0, 458, 459, 0, 460, 461, 462, - 463, 464, 465, 466, 467, 0, 468, 1108, 1109, 0, - 0, 471, 472, 813, 474, 814, 1110, 476, 477, 815, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 1111, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 1112, 498, 0, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 456, 457, 458, 815, 0, 0, 460, 461, 0, 462, + 463, 464, 465, 466, 467, 468, 469, 0, 470, 471, + 472, 0, 0, 473, 474, 816, 476, 817, 0, 478, + 479, 818, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 499, 500, 0, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 0, 0, 0, 0, 0, 119, 0, 1114, 1115, - 1116, 0, 0, 1076, 0, 1117, 0, 1118, 0, 0, - 0, 0, 1119, 1120, 1121, 1122, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 1077, 0, 0, 131, 132, 133, 0, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, - 144, 145, 0, 0, 146, 147, 148, 149, 150, 1081, - 802, 151, 152, 153, 154, 155, 156, 157, 0, 158, - 159, 160, 161, 803, 0, 804, 0, 164, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 180, 181, 182, 183, 184, + 529, 530, 531, 0, 0, 0, 0, 0, 539, 2032, + 0, 0, 0, 0, 2033, 1081, 0, 1122, 0, 2198, + 0, 0, 0, 0, 1124, 1125, 1126, 1127, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, + 135, 0, 0, 137, 138, 0, 140, 141, 142, 143, + 144, 0, 146, 147, 0, 0, 148, 149, 150, 151, + 152, 0, 0, 153, 154, 155, 156, 157, 158, 159, + 0, 160, 161, 162, 163, 164, 0, 0, 0, 166, + 167, 168, 169, 170, 171, 0, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 209, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 805, 0, 231, 0, 232, 233, - 234, 235, 0, 236, 0, 237, 238, 0, 239, 240, - 241, 242, 243, 244, 245, 246, 0, 247, 248, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, 0, 280, 0, 281, 282, 283, 284, - 285, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 293, 294, 295, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 1098, 305, 306, 307, 308, 309, 310, + 195, 196, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 211, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, 231, 232, 0, 233, 0, + 234, 235, 236, 237, 0, 238, 0, 239, 0, 0, + 0, 242, 243, 540, 0, 246, 247, 248, 0, 249, + 250, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 0, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 0, 274, 275, 276, + 277, 278, 279, 280, 281, 0, 282, 0, 283, 0, + 0, 286, 0, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 0, 296, 0, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, - 331, 332, 333, 0, 334, 335, 336, 337, 0, 807, - 339, 340, 341, 342, 343, 0, 344, 345, 0, 0, - 346, 347, 348, 0, 0, 349, 350, 351, 352, 353, - 354, 809, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 810, 371, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 329, 0, + 331, 332, 333, 334, 335, 0, 336, 337, 0, 339, + 0, 340, 341, 342, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 353, + 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 372, 0, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 409, 410, 411, 412, 413, 414, 415, + 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 436, 437, 438, 439, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 812, 0, 0, 458, 459, - 0, 460, 461, 462, 463, 464, 465, 466, 467, 0, - 468, 469, 470, 0, 0, 471, 472, 813, 474, 814, - 0, 476, 477, 815, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 497, 498, 0, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 0, 439, 440, 441, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 542, 458, 459, 0, 0, + 460, 461, 0, 462, 0, 464, 465, 466, 467, 468, + 469, 0, 470, 471, 472, 0, 0, 473, 474, 475, + 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 0, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 499, + 500, 0, 501, 0, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 0, 0, 0, 0, 0, - 537, 2023, 0, 0, 0, 0, 2024, 1076, 0, 1117, - 0, 2188, 0, 0, 0, 0, 1119, 1120, 1121, 1122, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 132, 133, 0, 0, 135, 136, 0, 138, 139, - 140, 141, 142, 0, 144, 145, 0, 0, 146, 147, - 148, 149, 150, 0, 0, 151, 152, 153, 154, 155, - 156, 157, 0, 158, 159, 160, 161, 162, 0, 0, - 0, 164, 165, 166, 167, 168, 169, 0, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 0, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 0, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 209, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, 0, - 231, 0, 232, 233, 234, 235, 0, 236, 0, 237, - 0, 0, 0, 240, 241, 538, 0, 244, 245, 246, - 0, 247, 248, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 0, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 0, 272, - 273, 274, 275, 276, 277, 278, 279, 0, 280, 0, - 281, 0, 0, 284, 0, 286, 287, 288, 0, 289, - 290, 291, 0, 0, 292, 0, 294, 0, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 539, 305, 306, - 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, 0, 329, 330, 331, 332, 333, 0, 334, 335, - 0, 337, 0, 338, 339, 340, 341, 342, 343, 0, - 344, 345, 0, 0, 346, 347, 348, 0, 0, 349, - 350, 351, 0, 353, 0, 355, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 370, 0, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 0, 410, 411, - 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 0, 437, 438, - 439, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 540, 456, 457, - 0, 0, 458, 459, 0, 460, 0, 462, 463, 464, - 465, 466, 467, 0, 468, 469, 470, 0, 0, 471, - 472, 473, 474, 475, 0, 476, 477, 478, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 0, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 497, 498, 0, 499, 0, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, - 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 0, - 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1119, 1120, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 0, - 1025, 0, 0, 132, 133, 0, 0, 135, 136, 0, - 138, 139, 140, 141, 142, 0, 144, 145, 0, 0, - 146, 147, 148, 149, 150, 0, 0, 151, 152, 153, - 154, 155, 156, 157, 0, 158, 159, 160, 161, 162, - 0, 0, 0, 164, 165, 166, 167, 168, 169, 0, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 0, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 209, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, -582, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, 0, 231, -582, 232, 233, 234, 235, -582, 236, - 0, 237, 0, 0, 0, 240, 241, 538, 0, 244, - 245, 246, 0, 247, 248, 249, 250, 0, 251, 252, - 253, 254, 255, 256, 257, 0, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 0, 272, 273, 274, 275, 276, 277, 278, 279, -582, - 280, 0, 281, 0, 0, 284, 0, 286, 287, 288, - 0, 289, 290, 291, 0, 0, 292, 0, 294, 0, - -582, 296, 297, 298, 299, 300, 301, 302, 303, 539, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 0, 329, 330, 331, 332, 333, 0, - 334, 335, 0, 337, 0, 338, 339, 340, 341, 342, - 343, -582, 344, 345, 0, 0, 346, 347, 348, 0, - -582, 349, 350, 351, 0, 353, 0, 355, 356, 357, - 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 370, 0, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 0, - 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 0, - 437, 438, 439, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 540, - 456, 457, 0, 0, 458, 459, 0, 460, 0, 462, - 463, 464, 465, 466, 467, 0, 468, 469, 470, 0, - 0, 471, 472, 473, 474, 475, 0, 476, 477, 478, - 479, 480, 481, 482, 483, -582, 0, 484, 485, 486, - 0, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 497, 498, 0, 499, 0, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, - 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 537, 0, 562, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1194, 0, - 0, 120, 121, 122, 123, 124, 125, 126, 127, 0, - 128, 129, 130, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 132, 133, 0, 0, 135, 136, 0, 138, - 139, 140, 141, 142, 0, 144, 145, 0, 0, 146, - 147, 148, 149, 150, 0, 0, 151, 152, 153, 154, - 155, 156, 157, 0, 158, 159, 160, 161, 162, 0, - 0, 0, 164, 165, 166, 167, 168, 169, 0, 171, - 172, 173, 0, 174, 175, 176, 177, 178, 179, 0, - 0, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 0, 196, 0, 197, - 198, 199, 200, 201, 202, 0, 0, 203, 204, 205, - 206, 0, 0, 207, 208, 209, 210, 211, 0, 212, - 213, 214, 0, 215, 216, 217, 218, 0, 219, 220, - 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - 0, 231, 0, 232, 233, 234, 235, 0, 236, 0, - 237, 0, 0, 0, 240, 241, 538, 0, 244, 245, - 246, 0, 247, 248, 249, 250, 0, 251, 252, 253, - 254, 255, 256, 257, 0, 259, 260, 261, 262, 0, - 263, 264, 265, 266, 267, 268, 269, 0, 270, 0, - 272, 273, 274, 275, 276, 277, 278, 279, 0, 280, - 0, 281, 0, 0, 284, 0, 286, 287, 288, 0, - 289, 290, 291, 0, 0, 292, 0, 294, 0, 0, - 296, 297, 298, 299, 300, 301, 302, 303, 539, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, 0, 329, 330, 331, 332, 333, 0, 334, - 335, 0, 337, 0, 338, 339, 340, 341, 342, 343, - 0, 344, 345, 0, 0, 346, 347, 348, 0, 0, - 349, 350, 351, 0, 353, 0, 355, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 0, - 0, 0, 0, 368, 369, 370, 0, 372, 373, 374, - 375, 376, 377, 0, 378, 379, 380, 381, 382, 383, - 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 0, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 0, 407, 408, 0, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 0, 0, 427, 428, - 429, 430, 431, 432, 433, 434, 435, 0, 0, 437, - 438, 439, 440, 0, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 540, 456, - 457, 0, 0, 458, 459, 0, 460, 0, 462, 463, - 464, 465, 466, 467, 0, 468, 469, 470, 0, 0, - 471, 472, 473, 474, 475, 0, 476, 477, 478, 479, - 480, 481, 482, 483, 0, 0, 484, 485, 486, 0, - 0, 487, 488, 489, 490, 0, 491, 492, 493, 494, - 495, 496, 497, 498, 0, 499, 0, 501, 502, 503, - 504, 505, 506, 507, 0, 0, 508, 0, 0, 509, - 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, - 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, - 2449, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 2587, 3446, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 1286, 0, 0, 0, 0, - 2450, 132, 133, 0, 2451, 135, 136, 2452, 138, 139, - 140, 0, 0, 2453, 0, 0, 0, 1291, 146, 147, - 148, 149, 150, 0, 0, 151, 152, 153, 154, 0, - 0, 157, 0, 158, 159, 160, 161, 0, 0, 2454, - 0, 2455, 165, 166, 167, 168, 169, 2456, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 2457, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 0, 192, 193, 0, 195, 0, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 209, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 0, 224, 225, 226, 227, 228, 229, 0, 0, - 231, 0, 232, 233, 0, 235, 0, 236, 0, 237, - 2458, 0, 2459, 240, 241, 2460, 2461, 244, 245, 246, - 0, 0, 0, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 2462, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 2463, 0, - 273, 274, 275, 276, 277, 0, 0, 0, 0, 0, - 281, 2464, 2465, 284, 2466, 286, 287, 288, 0, 289, - 290, 291, 0, 0, 292, 2467, 294, 2468, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 2469, 305, 306, - 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 0, 2470, 0, 330, 331, 332, 0, 0, 334, 335, - 2471, 337, 0, 0, 339, 0, 341, 342, 343, 0, - 344, 345, 0, 0, 346, 347, 348, 0, 0, 349, - 350, 0, 2472, 353, 2473, 0, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 0, 2474, 372, 373, 0, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 0, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 2475, 410, 411, - 412, 0, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 1315, 427, 428, 429, - 430, 431, 432, 0, 434, 435, 0, 2476, 437, 438, - 0, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 2477, 456, 0, - 0, 0, 458, 459, 0, 460, 2478, 462, 463, 464, - 465, 466, 467, 0, 468, 0, 0, 0, 0, 471, - 472, 0, 474, 0, 0, 476, 477, 2479, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 2480, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 0, 0, 498, 0, 499, 2481, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, - 511, 512, 513, 514, 2449, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 526, 527, 528, 529, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 2482, 0, 0, 1286, - 0, 0, 0, 0, 2450, 132, 133, 0, 2451, 135, - 136, 2452, 138, 139, 140, 0, 0, 2453, 0, 0, - 0, 1291, 146, 147, 148, 149, 150, 0, 0, 151, - 152, 153, 154, 0, 0, 157, 0, 158, 159, 160, - 161, 0, 0, 2454, 0, 2455, 165, 166, 167, 168, - 169, 2456, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 2457, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 0, 192, 193, 0, 195, 0, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 209, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 0, 224, 225, 226, 227, - 228, 229, 0, 0, 231, 0, 232, 233, 0, 235, - 0, 236, 0, 237, 2458, 0, 2459, 240, 241, 2460, - 2461, 244, 245, 246, 0, 0, 0, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 2462, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 2463, 0, 273, 274, 275, 276, 277, 0, - 0, 0, 0, 0, 281, 2464, 2465, 284, 2466, 286, - 287, 288, 0, 289, 290, 291, 0, 0, 292, 2467, - 294, 2468, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 2469, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 0, 2470, 0, 330, 331, 332, - 0, 0, 334, 335, 2471, 337, 0, 0, 339, 0, - 341, 342, 343, 0, 344, 345, 0, 0, 346, 347, - 348, 0, 0, 349, 350, 0, 2472, 353, 2473, 0, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 0, 2474, - 372, 373, 0, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 0, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 2475, 410, 411, 412, 0, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 1315, 427, 428, 429, 430, 431, 432, 0, 434, 435, - 0, 2476, 437, 438, 0, 440, 0, 441, 442, 443, - 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 2477, 456, 0, 0, 0, 458, 459, 0, 460, - 2478, 462, 463, 464, 465, 466, 467, 0, 468, 0, - 0, 0, 0, 471, 472, 0, 474, 0, 0, 476, - 477, 2479, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 2480, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 0, 0, 498, 0, 499, 2481, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 0, 0, - 0, 0, 989, 0, 0, 0, 0, 0, 0, 526, - 527, 528, 529, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 3198, 128, 129, 130, 3, 4, 0, 573, 0, 0, - 0, 0, 578, 132, 133, 0, 580, 135, 136, 581, - 138, 139, 140, 582, 583, 584, 585, 586, 0, 588, - 146, 147, 148, 149, 150, 0, 0, 151, 152, 153, - 154, 591, 592, 157, 0, 158, 159, 160, 161, 594, - 0, 596, 0, 598, 165, 166, 167, 168, 169, 599, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 602, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 604, 192, 193, 605, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 14, 15, 203, 204, - 205, 206, 0, 0, 207, 208, 209, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 615, 224, 225, 226, 227, 228, 229, - 616, 0, 231, 0, 232, 233, 619, 235, 0, 236, - 0, 237, 622, 23, 624, 240, 241, 625, 626, 244, - 245, 246, 0, 628, 629, 249, 250, 0, 251, 252, - 253, 254, 255, 256, 257, 631, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 634, 635, 273, 274, 275, 276, 277, 636, 637, 0, - 639, 0, 281, 641, 642, 284, 643, 286, 287, 288, - 0, 289, 290, 291, 0, 0, 292, 647, 294, 648, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 650, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 651, 652, 653, 330, 331, 332, 654, 0, - 334, 335, 656, 337, 0, 658, 339, 659, 341, 342, - 343, 0, 344, 345, 0, 0, 346, 347, 348, 0, - 0, 349, 350, 665, 666, 353, 667, 668, 356, 357, - 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 26, 27, 28, 0, 368, 369, 673, 674, 372, 373, - 675, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 678, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 681, - 410, 411, 412, 682, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 33, 684, 427, - 428, 429, 430, 431, 432, 685, 434, 435, 35, 687, - 437, 438, 688, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 690, - 456, 691, 37, 0, 458, 459, 38, 460, 695, 462, - 463, 464, 465, 466, 467, 0, 468, 697, 698, 0, - 0, 471, 472, 701, 474, 702, 0, 476, 477, 704, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 707, 40, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 990, 711, 498, 0, 499, 713, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 44, - 509, 510, 511, 512, 513, 514, 718, 719, 720, 721, - 722, 723, 724, 725, 726, 727, 728, 526, 527, 528, - 529, 0, 119, 45, 562, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 0, - 0, 0, 131, 132, 133, 0, 134, 135, 136, 137, - 138, 139, 140, 141, 142, 143, 144, 145, 0, 0, - 146, 147, 148, 149, 150, 0, 802, 151, 152, 153, - 154, 155, 156, 157, 0, 158, 159, 160, 161, 803, - 0, 804, 0, 164, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 14, 15, 203, 204, - 205, 206, 0, 0, 207, 208, 209, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 805, 0, 231, 0, 232, 233, 234, 235, 0, 236, - 0, 237, 238, 23, 239, 240, 241, 242, 243, 244, - 245, 246, 0, 247, 248, 249, 250, 0, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, 0, - 280, 0, 281, 282, 283, 284, 285, 286, 287, 288, - 0, 289, 290, 291, 806, 0, 292, 293, 294, 295, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 328, 329, 330, 331, 332, 333, 0, - 334, 335, 336, 337, 0, 807, 339, 340, 341, 342, - 343, 0, 344, 345, 0, 808, 346, 347, 348, 0, - 0, 349, 350, 351, 352, 353, 354, 809, 356, 357, - 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 26, 27, 28, 0, 368, 369, 810, 371, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 409, - 410, 411, 412, 413, 811, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 33, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 35, 436, - 437, 438, 439, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 812, 37, 0, 458, 459, 38, 460, 461, 462, - 463, 464, 465, 466, 467, 0, 468, 469, 470, 0, - 0, 471, 472, 813, 474, 814, 0, 476, 477, 815, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 0, 40, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 816, 497, 498, 0, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 44, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, - 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 0, 119, 45, 562, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 0, - 0, 0, 131, 132, 133, 0, 134, 135, 136, 137, - 138, 139, 140, 141, 142, 143, 144, 145, 0, 0, - 146, 147, 148, 149, 150, 0, 802, 151, 152, 153, - 154, 155, 156, 157, 0, 158, 159, 160, 161, 803, - 0, 804, 0, 164, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 209, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 805, 0, 231, 0, 232, 233, 234, 235, 0, 236, - 0, 237, 238, 0, 239, 240, 241, 242, 243, 244, - 245, 246, 0, 247, 248, 249, 250, 0, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, 0, - 280, 0, 281, 282, 283, 284, 285, 286, 287, 288, - 0, 289, 290, 291, 806, 0, 292, 293, 294, 295, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 328, 329, 330, 331, 332, 333, 0, - 334, 335, 336, 337, 0, 807, 339, 340, 341, 342, - 343, 0, 344, 345, 0, 808, 346, 347, 348, 0, - 0, 349, 350, 351, 352, 353, 354, 809, 356, 357, - 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 810, 371, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 409, - 410, 411, 412, 413, 811, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 436, - 437, 438, 439, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 812, 0, 0, 458, 459, 0, 460, 461, 462, - 463, 464, 465, 466, 467, 0, 468, 469, 470, 0, - 0, 471, 472, 813, 474, 814, 0, 476, 477, 815, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 0, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 816, 497, 498, 0, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, - 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 119, 0, 562, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, - 0, 120, 121, 122, 123, 124, 125, 126, 127, 0, - 128, 129, 130, 0, 0, 0, 0, 0, 0, 0, - 0, 131, 132, 133, 0, 134, 135, 136, 137, 138, - 139, 140, 141, 142, 143, 144, 145, 0, 0, 146, - 147, 148, 149, 150, 0, 802, 151, 152, 153, 154, - 155, 156, 157, 0, 158, 159, 160, 161, 803, 0, - 804, 0, 164, 165, 166, 167, 168, 169, 170, 171, - 172, 173, 0, 174, 175, 176, 177, 178, 179, 0, - 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 0, 196, 0, 197, - 198, 199, 200, 201, 202, 0, 0, 203, 204, 205, - 206, 0, 0, 207, 208, 209, 210, 211, 0, 212, - 213, 214, 0, 215, 216, 217, 218, 0, 219, 220, - 221, 222, 223, 224, 225, 226, 227, 228, 229, 805, - 0, 231, 0, 232, 233, 234, 235, 0, 236, 0, - 237, 238, 0, 239, 240, 241, 242, 243, 244, 245, - 246, 0, 247, 248, 249, 250, 0, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 0, - 263, 264, 265, 266, 267, 268, 269, 0, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, 0, 280, - 0, 281, 282, 283, 284, 285, 286, 287, 288, 0, - 289, 290, 291, 0, 0, 292, 293, 294, 295, 0, - 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, 328, 329, 330, 331, 332, 333, 0, 334, - 335, 336, 337, 0, 807, 339, 340, 341, 342, 343, - 0, 344, 345, 0, 808, 346, 347, 348, 0, 0, - 349, 350, 351, 352, 353, 354, 809, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 0, - 0, 0, 0, 368, 369, 810, 371, 372, 373, 374, - 375, 376, 377, 0, 378, 379, 380, 381, 382, 383, - 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 0, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 0, 407, 408, 409, 410, - 411, 412, 413, 811, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 0, 0, 427, 428, - 429, 430, 431, 432, 433, 434, 435, 0, 436, 437, - 438, 439, 440, 0, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, - 812, 0, 0, 458, 459, 0, 460, 461, 462, 463, - 464, 465, 466, 467, 0, 468, 469, 470, 0, 0, - 471, 472, 813, 474, 814, 0, 476, 477, 815, 479, - 480, 481, 482, 483, 0, 0, 484, 485, 486, 0, - 0, 487, 488, 489, 490, 0, 491, 492, 493, 494, - 495, 496, 497, 498, 0, 499, 500, 501, 502, 503, - 504, 505, 506, 507, 0, 0, 508, 0, 0, 509, - 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, - 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, - 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 0, 0, 0, - 131, 132, 133, 0, 134, 135, 136, 137, 138, 139, - 140, 141, 142, 143, 144, 145, 0, 0, 146, 147, - 148, 149, 150, 0, 802, 151, 152, 153, 154, 155, - 156, 157, 0, 158, 159, 160, 161, 803, 0, 804, - 0, 164, 165, 166, 167, 168, 169, 170, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 0, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 209, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 805, 0, - 231, 0, 232, 233, 234, 235, 0, 236, 0, 237, - 238, 0, 239, 240, 241, 242, 243, 244, 245, 246, - 0, 247, 248, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 271, 272, - 273, 274, 275, 276, 277, 278, 279, 0, 280, 0, - 281, 282, 283, 284, 285, 286, 287, 288, 0, 289, - 290, 291, 0, 0, 292, 293, 294, 295, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, 328, 329, 330, 331, 332, 333, 0, 334, 335, - 336, 337, 0, 807, 339, 340, 341, 342, 343, 0, - 344, 345, 0, 0, 346, 347, 348, 0, 0, 349, - 350, 351, 352, 353, 354, 809, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 810, 371, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 409, 410, 411, - 412, 413, 811, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 436, 437, 438, - 439, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 812, - 0, 0, 458, 459, 0, 460, 461, 462, 463, 464, - 465, 466, 467, 0, 468, 469, 470, 0, 0, 471, - 472, 813, 474, 814, 0, 476, 477, 815, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 0, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 497, 498, 0, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, - 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 119, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 46, 0, 0, 0, 120, - 121, 122, 123, 124, 125, 126, 127, 0, 128, 129, - 130, 0, 0, 0, 0, 0, 0, 0, 0, 131, - 132, 133, 0, 134, 135, 136, 137, 138, 139, 140, - 141, 142, 143, 144, 145, 0, 0, 146, 147, 148, - 149, 150, 0, 802, 151, 152, 153, 154, 155, 156, - 157, 0, 158, 159, 160, 161, 803, 0, 804, 0, - 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, - 0, 174, 175, 176, 177, 178, 179, 0, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 0, 196, 0, 197, 198, 199, - 200, 201, 202, 0, 0, 203, 204, 205, 206, 0, - 0, 207, 208, 209, 210, 211, 0, 212, 213, 214, - 0, 215, 216, 217, 218, 0, 219, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 805, 0, 231, - 0, 232, 233, 234, 235, 0, 236, 0, 237, 238, - 0, 239, 240, 241, 242, 243, 244, 245, 246, 0, - 247, 248, 249, 250, 0, 251, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 0, 263, 264, - 265, 266, 267, 268, 269, 0, 270, 271, 272, 273, - 274, 275, 276, 277, 278, 279, 0, 280, 0, 281, - 282, 283, 284, 285, 286, 287, 288, 0, 289, 290, - 291, 0, 0, 292, 293, 294, 295, 0, 296, 297, - 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, - 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - 328, 329, 330, 331, 332, 333, 0, 334, 335, 336, - 337, 0, 807, 339, 340, 341, 342, 343, 0, 344, - 345, 0, 0, 346, 347, 348, 0, 0, 349, 350, - 351, 352, 353, 354, 809, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 0, 0, 0, - 0, 368, 369, 810, 371, 372, 373, 374, 375, 376, - 377, 0, 378, 379, 380, 381, 382, 383, 0, 384, - 385, 386, 387, 388, 389, 390, 391, 392, 393, 0, - 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 0, 407, 408, 409, 410, 411, 412, - 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 426, 0, 0, 427, 428, 429, 430, - 431, 432, 433, 434, 435, 0, 436, 437, 438, 439, - 440, 0, 441, 442, 443, 444, 445, 446, 447, 448, - 449, 450, 451, 452, 453, 454, 455, 456, 812, 0, - 0, 458, 459, 0, 460, 461, 462, 463, 464, 465, - 466, 467, 0, 468, 469, 470, 0, 0, 471, 472, - 813, 474, 814, 0, 476, 477, 815, 479, 480, 481, - 482, 483, 0, 0, 484, 485, 486, 0, 0, 487, - 488, 489, 490, 0, 491, 492, 493, 494, 495, 496, - 497, 498, 0, 499, 500, 501, 502, 503, 504, 505, - 506, 507, 0, 0, 508, 0, 0, 509, 510, 511, - 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, - 522, 523, 524, 525, 526, 527, 528, 529, 537, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3542, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, - 133, 0, 0, 135, 136, 0, 138, 139, 140, 141, - 142, 0, 144, 145, 0, 0, 146, 147, 148, 149, - 150, 0, 0, 151, 152, 153, 154, 155, 156, 157, - 0, 158, 159, 160, 161, 162, 0, 0, 0, 164, - 165, 166, 167, 168, 169, 0, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 0, 181, 182, + 525, 526, 527, 528, 529, 530, 531, 0, 0, 0, + 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1124, 1125, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 0, 1030, 0, + 0, 134, 135, 0, 0, 137, 138, 0, 140, 141, + 142, 143, 144, 0, 146, 147, 0, 0, 148, 149, + 150, 151, 152, 0, 0, 153, 154, 155, 156, 157, + 158, 159, 0, 160, 161, 162, 163, 164, 0, 0, + 0, 166, 167, 168, 169, 170, 171, 0, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 14, 15, 203, 204, 205, 206, 0, 0, - 207, 208, 209, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, 0, 231, 0, - 232, 233, 234, 235, 0, 236, 0, 237, 0, 23, - 0, 240, 241, 538, 0, 244, 245, 246, 0, 247, - 248, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 0, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 0, 272, 273, 274, - 275, 276, 277, 278, 279, 0, 280, 0, 281, 0, - 0, 284, 0, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 0, 294, 0, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 539, 305, 306, 307, 308, + 193, 194, 195, 196, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 211, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, -585, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 0, + 233, -585, 234, 235, 236, 237, -585, 238, 0, 239, + 0, 0, 0, 242, 243, 540, 0, 246, 247, 248, + 0, 249, 250, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 0, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 0, 274, + 275, 276, 277, 278, 279, 280, 281, -585, 282, 0, + 283, 0, 0, 286, 0, 288, 289, 290, 0, 291, + 292, 293, 0, 0, 294, 0, 296, 0, -585, 298, + 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, 0, - 329, 330, 331, 332, 333, 0, 334, 335, 0, 337, - 0, 338, 339, 340, 341, 342, 343, 0, 344, 345, - 0, 0, 346, 347, 348, 0, 0, 349, 350, 351, - 0, 353, 0, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 26, 27, 28, 0, - 368, 369, 370, 0, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 0, 410, 411, 412, 413, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 0, 331, 332, 333, 334, 335, 0, 336, 337, + 0, 339, 0, 340, 341, 342, 343, 344, 345, -585, + 346, 347, 0, 0, 348, 349, 350, 0, -585, 351, + 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 372, 0, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 33, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 35, 0, 437, 438, 439, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 540, 456, 457, 37, 0, - 458, 459, 38, 460, 0, 462, 463, 464, 465, 466, - 467, 0, 468, 469, 470, 0, 0, 471, 472, 473, - 474, 475, 0, 476, 477, 478, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 40, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 816, 497, - 498, 0, 499, 0, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 44, 509, 510, 511, 512, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 0, 439, 440, + 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 542, 458, 459, + 0, 0, 460, 461, 0, 462, 0, 464, 465, 466, + 467, 468, 469, 0, 470, 471, 472, 0, 0, 473, + 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, + 483, 484, 485, -585, 0, 486, 487, 488, 0, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 499, 500, 0, 501, 0, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 0, 537, 45, - 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 46, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 907, 128, 129, 130, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, - 133, 0, 0, 135, 136, 0, 138, 139, 140, 141, - 142, 0, 144, 145, 0, 0, 146, 147, 148, 149, - 150, 0, 0, 151, 152, 153, 154, 155, 156, 157, - 0, 158, 159, 160, 161, 162, 0, 0, 0, 164, - 165, 166, 167, 168, 169, 0, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 0, 181, 182, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 539, + 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1199, 0, 0, 122, + 123, 124, 125, 126, 127, 128, 129, 0, 130, 131, + 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 134, 135, 0, 0, 137, 138, 0, 140, 141, 142, + 143, 144, 0, 146, 147, 0, 0, 148, 149, 150, + 151, 152, 0, 0, 153, 154, 155, 156, 157, 158, + 159, 0, 160, 161, 162, 163, 164, 0, 0, 0, + 166, 167, 168, 169, 170, 171, 0, 173, 174, 175, + 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, + 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, 0, 198, 0, 199, 200, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 0, + 0, 209, 210, 211, 212, 213, 0, 214, 215, 216, + 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 0, 233, + 0, 234, 235, 236, 237, 0, 238, 0, 239, 0, + 0, 0, 242, 243, 540, 0, 246, 247, 248, 0, + 249, 250, 251, 252, 0, 253, 254, 255, 256, 257, + 258, 259, 0, 261, 262, 263, 264, 0, 265, 266, + 267, 268, 269, 270, 271, 0, 272, 0, 274, 275, + 276, 277, 278, 279, 280, 281, 0, 282, 0, 283, + 0, 0, 286, 0, 288, 289, 290, 0, 291, 292, + 293, 0, 0, 294, 0, 296, 0, 0, 298, 299, + 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, + 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, + 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 0, 331, 332, 333, 334, 335, 0, 336, 337, 0, + 339, 0, 340, 341, 342, 343, 344, 345, 0, 346, + 347, 0, 0, 348, 349, 350, 0, 0, 351, 352, + 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 0, 0, 0, + 0, 370, 371, 372, 0, 374, 375, 376, 377, 378, + 379, 0, 380, 381, 382, 383, 384, 385, 0, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 0, + 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, + 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 426, 427, 428, 0, 0, 429, 430, 431, 432, + 433, 434, 435, 436, 437, 0, 0, 439, 440, 441, + 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, + 451, 452, 453, 454, 455, 456, 542, 458, 459, 0, + 0, 460, 461, 0, 462, 0, 464, 465, 466, 467, + 468, 469, 0, 470, 471, 472, 0, 0, 473, 474, + 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, + 484, 485, 0, 0, 486, 487, 488, 0, 0, 489, + 490, 491, 492, 0, 493, 494, 495, 496, 497, 498, + 499, 500, 0, 501, 0, 503, 504, 505, 506, 507, + 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, + 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, + 524, 525, 526, 527, 528, 529, 530, 531, 2461, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2599, 3460, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 1293, 0, 0, 0, 0, 2462, 134, + 135, 0, 2463, 137, 138, 2464, 140, 141, 142, 0, + 0, 2465, 0, 0, 0, 1298, 148, 149, 150, 151, + 152, 0, 0, 153, 154, 155, 156, 0, 0, 159, + 0, 160, 161, 162, 163, 0, 0, 2466, 0, 2467, + 167, 168, 169, 170, 171, 2468, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 2469, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 0, 194, + 195, 0, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 211, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 0, + 226, 227, 228, 229, 230, 231, 0, 0, 233, 0, + 234, 235, 0, 237, 0, 238, 0, 239, 2470, 0, + 2471, 242, 243, 2472, 2473, 246, 247, 248, 0, 0, + 0, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 2474, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 2475, 0, 275, 276, + 277, 278, 279, 0, 0, 0, 0, 0, 283, 2476, + 2477, 286, 2478, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 2479, 296, 2480, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 2481, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + 321, 322, 323, 324, 325, 326, 327, 328, 0, 2482, + 0, 332, 333, 334, 0, 0, 336, 337, 2483, 339, + 0, 0, 341, 0, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 0, + 2484, 355, 2485, 0, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 0, 2486, 374, 375, 0, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 0, 392, 393, 394, 395, 0, 396, + 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 0, 409, 410, 2487, 412, 413, 414, 0, + 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, + 426, 427, 428, 0, 1322, 429, 430, 431, 432, 433, + 434, 0, 436, 437, 0, 2488, 439, 440, 0, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 2489, 458, 0, 0, 0, + 460, 461, 0, 462, 2490, 464, 465, 466, 467, 468, + 469, 0, 470, 0, 0, 0, 0, 473, 474, 0, + 476, 0, 0, 478, 479, 2491, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 2492, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 0, 0, + 500, 0, 501, 2493, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, + 515, 516, 2461, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 528, 529, 530, 531, 0, 0, 0, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 2494, 0, 0, 1293, 0, 0, + 0, 0, 2462, 134, 135, 0, 2463, 137, 138, 2464, + 140, 141, 142, 0, 0, 2465, 0, 0, 0, 1298, + 148, 149, 150, 151, 152, 0, 0, 153, 154, 155, + 156, 0, 0, 159, 0, 160, 161, 162, 163, 0, + 0, 2466, 0, 2467, 167, 168, 169, 170, 171, 2468, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 2469, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 0, 194, 195, 0, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 211, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 0, 226, 227, 228, 229, 230, 231, + 0, 0, 233, 0, 234, 235, 0, 237, 0, 238, + 0, 239, 2470, 0, 2471, 242, 243, 2472, 2473, 246, + 247, 248, 0, 0, 0, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 2474, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 2475, 0, 275, 276, 277, 278, 279, 0, 0, 0, + 0, 0, 283, 2476, 2477, 286, 2478, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 2479, 296, 2480, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 2481, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 327, 328, 0, 2482, 0, 332, 333, 334, 0, 0, + 336, 337, 2483, 339, 0, 0, 341, 0, 343, 344, + 345, 0, 346, 347, 0, 0, 348, 349, 350, 0, + 0, 351, 352, 0, 2484, 355, 2485, 0, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 0, 2486, 374, 375, + 0, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 0, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 2487, + 412, 413, 414, 0, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 1322, 429, + 430, 431, 432, 433, 434, 0, 436, 437, 0, 2488, + 439, 440, 0, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 2489, + 458, 0, 0, 0, 460, 461, 0, 462, 2490, 464, + 465, 466, 467, 468, 469, 0, 470, 0, 0, 0, + 0, 473, 474, 0, 476, 0, 0, 478, 479, 2491, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 2492, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 0, 0, 500, 0, 501, 2493, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, + 511, 512, 513, 514, 515, 516, 0, 0, 0, 0, + 994, 0, 0, 0, 0, 0, 0, 528, 529, 530, + 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 3212, 130, + 131, 132, 3, 4, 0, 575, 0, 0, 0, 0, + 580, 134, 135, 0, 582, 137, 138, 583, 140, 141, + 142, 584, 585, 586, 587, 588, 0, 590, 148, 149, + 150, 151, 152, 0, 0, 153, 154, 155, 156, 593, + 594, 159, 0, 160, 161, 162, 163, 596, 0, 598, + 0, 600, 167, 168, 169, 170, 171, 601, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 604, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 209, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, 0, 231, 0, - 232, 233, 234, 235, 0, 236, 0, 237, 0, 23, - 0, 240, 241, 538, 0, 244, 245, 246, 0, 247, - 248, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 0, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 0, 272, 273, 274, - 275, 276, 277, 278, 279, 0, 280, 0, 281, 0, - 0, 284, 0, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 0, 294, 0, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 539, 305, 306, 307, 308, + 606, 194, 195, 607, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 14, 15, 205, 206, 207, 208, + 0, 0, 209, 210, 211, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 617, 226, 227, 228, 229, 230, 231, 618, 0, + 233, 0, 234, 235, 621, 237, 0, 238, 0, 239, + 624, 23, 626, 242, 243, 627, 628, 246, 247, 248, + 0, 630, 631, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 633, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 636, 637, + 275, 276, 277, 278, 279, 638, 639, 0, 641, 0, + 283, 643, 644, 286, 645, 288, 289, 290, 0, 291, + 292, 293, 0, 0, 294, 649, 296, 650, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 652, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, 0, - 329, 330, 331, 332, 333, 0, 334, 335, 0, 337, - 0, 338, 339, 340, 341, 342, 343, 0, 344, 345, - 0, 0, 346, 347, 348, 0, 0, 349, 350, 351, - 0, 353, 0, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 26, 27, 28, 0, - 368, 369, 370, 0, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 0, 410, 411, 412, 413, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 33, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 0, 437, 438, 439, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 540, 456, 457, 0, 0, - 458, 459, 38, 908, 0, 462, 463, 464, 465, 466, - 467, 0, 468, 909, 470, 0, 0, 910, 472, 473, - 474, 475, 0, 476, 477, 478, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 40, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 816, 497, - 498, 0, 499, 0, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 44, 509, 510, 511, 512, - 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 0, 537, 45, - 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 46, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, - 133, 0, 0, 135, 136, 0, 138, 139, 140, 141, - 142, 0, 144, 145, 0, 0, 146, 147, 148, 149, - 150, 0, 0, 151, 152, 153, 154, 155, 156, 157, - 0, 158, 159, 160, 161, 162, 0, 0, 0, 164, - 165, 166, 167, 168, 169, 0, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 0, 181, 182, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 653, 654, 655, 332, 333, 334, 656, 0, 336, 337, + 658, 339, 0, 660, 341, 661, 343, 344, 345, 0, + 346, 347, 0, 0, 348, 349, 350, 0, 0, 351, + 352, 667, 668, 355, 669, 670, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 26, 27, + 28, 0, 370, 371, 675, 676, 374, 375, 677, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 680, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 683, 412, 413, + 414, 684, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 33, 686, 429, 430, 431, + 432, 433, 434, 687, 436, 437, 35, 689, 439, 440, + 690, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 692, 458, 693, + 37, 0, 460, 461, 38, 462, 697, 464, 465, 466, + 467, 468, 469, 0, 470, 699, 700, 0, 0, 473, + 474, 703, 476, 704, 0, 478, 479, 706, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 709, 40, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 995, 713, 500, 0, 501, 715, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 44, 511, 512, + 513, 514, 515, 516, 720, 721, 722, 723, 724, 725, + 726, 727, 728, 729, 730, 528, 529, 530, 531, 0, + 121, 45, 564, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 0, 0, 0, + 133, 134, 135, 0, 136, 137, 138, 139, 140, 141, + 142, 143, 144, 145, 146, 147, 0, 0, 148, 149, + 150, 151, 152, 0, 805, 153, 154, 155, 156, 157, + 158, 159, 0, 160, 161, 162, 163, 806, 0, 807, + 0, 166, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 209, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, 0, 231, 0, - 232, 233, 234, 235, 0, 236, 0, 237, 0, 23, - 0, 240, 241, 538, 0, 244, 245, 246, 0, 247, - 248, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 0, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 0, 272, 273, 274, - 275, 276, 277, 278, 279, 0, 280, 0, 281, 0, - 0, 284, 0, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 0, 294, 0, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 539, 305, 306, 307, 308, + 193, 194, 195, 196, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 14, 15, 205, 206, 207, 208, + 0, 0, 209, 210, 211, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 808, 0, + 233, 0, 234, 235, 236, 237, 0, 238, 0, 239, + 240, 23, 241, 242, 243, 244, 245, 246, 247, 248, + 0, 249, 250, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 273, 274, + 275, 276, 277, 278, 279, 280, 281, 0, 282, 0, + 283, 284, 285, 286, 287, 288, 289, 290, 0, 291, + 292, 293, 809, 0, 294, 295, 296, 297, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, 0, - 329, 330, 331, 332, 333, 0, 334, 335, 0, 337, - 0, 338, 339, 340, 341, 342, 343, 0, 344, 345, - 0, 0, 346, 347, 348, 0, 0, 349, 350, 351, - 0, 353, 0, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 26, 27, 28, 0, - 368, 369, 370, 0, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 0, 410, 411, 412, 413, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 33, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 0, 437, 438, 439, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 540, 456, 457, 0, 0, - 458, 459, 38, 460, 0, 462, 463, 464, 465, 466, - 467, 0, 468, 469, 470, 0, 0, 471, 472, 473, - 474, 475, 0, 476, 477, 478, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 40, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 816, 497, - 498, 0, 499, 0, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 44, 509, 510, 511, 512, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 330, 331, 332, 333, 334, 335, 0, 336, 337, + 338, 339, 0, 810, 341, 342, 343, 344, 345, 0, + 346, 347, 0, 811, 348, 349, 350, 0, 0, 351, + 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 26, 27, + 28, 0, 370, 371, 813, 373, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, + 414, 415, 814, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 33, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 35, 438, 439, 440, + 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 815, + 37, 0, 460, 461, 38, 462, 463, 464, 465, 466, + 467, 468, 469, 0, 470, 471, 472, 0, 0, 473, + 474, 816, 476, 817, 0, 478, 479, 818, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 0, 40, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 819, 499, 500, 0, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 44, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 0, 537, 45, - 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 46, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, - 133, 0, 0, 135, 136, 0, 138, 139, 140, 141, - 142, 0, 144, 145, 0, 0, 146, 147, 148, 149, - 150, 0, 0, 151, 152, 153, 154, 155, 156, 157, - 0, 158, 159, 160, 161, 162, 0, 0, 0, 164, - 165, 166, 167, 168, 169, 0, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 0, 181, 182, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 0, + 121, 45, 564, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 820, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 0, 0, 0, + 133, 134, 135, 0, 136, 137, 138, 139, 140, 141, + 142, 143, 144, 145, 146, 147, 0, 0, 148, 149, + 150, 151, 152, 0, 805, 153, 154, 155, 156, 157, + 158, 159, 0, 160, 161, 162, 163, 806, 0, 807, + 0, 166, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 209, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, 0, 231, 0, - 232, 233, 234, 235, 0, 236, 0, 237, 0, 0, - 0, 240, 241, 538, 0, 244, 245, 246, 0, 247, - 248, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 0, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 0, 272, 273, 274, - 275, 276, 277, 278, 279, 0, 280, 0, 281, 0, - 0, 284, 0, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 0, 294, 0, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 539, 305, 306, 307, 308, + 193, 194, 195, 196, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 211, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 808, 0, + 233, 0, 234, 235, 236, 237, 0, 238, 0, 239, + 240, 0, 241, 242, 243, 244, 245, 246, 247, 248, + 0, 249, 250, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 273, 274, + 275, 276, 277, 278, 279, 280, 281, 0, 282, 0, + 283, 284, 285, 286, 287, 288, 289, 290, 0, 291, + 292, 293, 809, 0, 294, 295, 296, 297, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, 0, - 329, 330, 331, 332, 333, 0, 334, 335, 0, 337, - 0, 338, 339, 340, 341, 342, 343, 0, 344, 345, - 0, 0, 346, 347, 348, 0, 0, 349, 350, 351, - 0, 353, 0, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 370, 0, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 0, 410, 411, 412, 413, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 0, 437, 438, 439, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 540, 456, 457, 0, 0, - 458, 459, 0, 460, 0, 462, 463, 464, 465, 466, - 467, 0, 468, 469, 470, 0, 0, 471, 472, 473, - 474, 475, 0, 476, 477, 478, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 497, - 498, 0, 499, 0, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 330, 331, 332, 333, 334, 335, 0, 336, 337, + 338, 339, 0, 810, 341, 342, 343, 344, 345, 0, + 346, 347, 0, 811, 348, 349, 350, 0, 0, 351, + 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 813, 373, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 411, 412, 413, + 414, 415, 814, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 438, 439, 440, + 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 815, + 0, 0, 460, 461, 0, 462, 463, 464, 465, 466, + 467, 468, 469, 0, 470, 471, 472, 0, 0, 473, + 474, 816, 476, 817, 0, 478, 479, 818, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 0, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 819, 499, 500, 0, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 537, 0, 562, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1009, 0, 0, 0, 120, 121, 122, - 123, 124, 125, 126, 127, 0, 128, 129, 130, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 132, 133, - 0, 0, 135, 136, 0, 138, 139, 140, 141, 142, - 0, 144, 145, 0, 0, 146, 147, 148, 149, 150, - 0, 0, 151, 152, 153, 154, 155, 156, 157, 0, - 158, 159, 160, 161, 162, 0, 0, 0, 164, 165, - 166, 167, 168, 169, 0, 171, 172, 173, 0, 174, - 175, 176, 177, 178, 179, 0, 0, 181, 182, 183, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 121, + 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 820, 0, 0, 0, 122, + 123, 124, 125, 126, 127, 128, 129, 0, 130, 131, + 132, 0, 0, 0, 0, 0, 0, 0, 0, 133, + 134, 135, 0, 136, 137, 138, 139, 140, 141, 142, + 143, 144, 145, 146, 147, 0, 0, 148, 149, 150, + 151, 152, 0, 805, 153, 154, 155, 156, 157, 158, + 159, 0, 160, 161, 162, 163, 806, 0, 807, 0, + 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, + 0, 176, 177, 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 0, 196, 0, 197, 198, 199, 200, 201, - 202, 0, 0, 203, 204, 205, 206, 0, 0, 207, - 208, 209, 210, 211, 0, 212, 213, 214, 0, 215, - 216, 217, 218, 0, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 0, 231, 0, 232, - 233, 234, 235, 0, 236, 0, 237, 0, 0, 0, - 240, 241, 538, 0, 244, 245, 246, 0, 247, 248, - 249, 250, 0, 251, 252, 253, 254, 255, 256, 257, - 0, 259, 260, 261, 262, 0, 263, 264, 265, 266, - 267, 268, 269, 0, 270, 0, 272, 273, 274, 275, - 276, 277, 278, 279, 0, 280, 0, 281, 0, 0, - 284, 0, 286, 287, 288, 0, 289, 290, 291, 0, - 0, 292, 0, 294, 0, 0, 296, 297, 298, 299, - 300, 301, 302, 303, 539, 305, 306, 307, 308, 309, + 194, 195, 196, 197, 0, 198, 0, 199, 200, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 0, + 0, 209, 210, 211, 212, 213, 0, 214, 215, 216, + 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 808, 0, 233, + 0, 234, 235, 236, 237, 0, 238, 0, 239, 240, + 0, 241, 242, 243, 244, 245, 246, 247, 248, 0, + 249, 250, 251, 252, 0, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, + 267, 268, 269, 270, 271, 0, 272, 273, 274, 275, + 276, 277, 278, 279, 280, 281, 0, 282, 0, 283, + 284, 285, 286, 287, 288, 289, 290, 0, 291, 292, + 293, 0, 0, 294, 295, 296, 297, 0, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, 0, 329, - 330, 331, 332, 333, 0, 334, 335, 0, 337, 0, - 338, 339, 340, 341, 342, 343, 0, 344, 345, 0, - 0, 346, 347, 348, 0, 0, 349, 350, 351, 0, - 353, 0, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 0, 0, 0, 0, 368, - 369, 370, 0, 372, 373, 374, 375, 376, 377, 0, - 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, - 387, 388, 389, 390, 391, 392, 393, 0, 394, 395, + 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 330, 331, 332, 333, 334, 335, 0, 336, 337, 338, + 339, 0, 810, 341, 342, 343, 344, 345, 0, 346, + 347, 0, 811, 348, 349, 350, 0, 0, 351, 352, + 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 0, 0, 0, + 0, 370, 371, 813, 373, 374, 375, 376, 377, 378, + 379, 0, 380, 381, 382, 383, 384, 385, 0, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 0, 407, 408, 0, 410, 411, 412, 413, 414, - 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 0, 0, 427, 428, 429, 430, 431, 432, - 433, 434, 435, 0, 0, 437, 438, 439, 440, 0, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 540, 456, 457, 0, 0, 458, - 459, 0, 460, 0, 462, 463, 464, 465, 466, 467, - 0, 468, 469, 470, 0, 0, 471, 472, 473, 474, - 475, 0, 476, 477, 478, 479, 480, 481, 482, 483, - 0, 0, 484, 485, 486, 0, 0, 487, 488, 489, - 490, 0, 491, 492, 493, 494, 495, 496, 497, 498, - 0, 499, 0, 501, 502, 503, 504, 505, 506, 507, - 0, 0, 508, 0, 0, 509, 510, 511, 512, 513, + 406, 407, 408, 0, 409, 410, 411, 412, 413, 414, + 415, 814, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 426, 427, 428, 0, 0, 429, 430, 431, 432, + 433, 434, 435, 436, 437, 0, 438, 439, 440, 441, + 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, + 451, 452, 453, 454, 455, 456, 457, 458, 815, 0, + 0, 460, 461, 0, 462, 463, 464, 465, 466, 467, + 468, 469, 0, 470, 471, 472, 0, 0, 473, 474, + 816, 476, 817, 0, 478, 479, 818, 481, 482, 483, + 484, 485, 0, 0, 486, 487, 488, 0, 0, 489, + 490, 491, 492, 0, 493, 494, 495, 496, 497, 498, + 499, 500, 0, 501, 502, 503, 504, 505, 506, 507, + 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, - 524, 525, 526, 527, 528, 529, 537, 0, 562, 0, + 524, 525, 526, 527, 528, 529, 530, 531, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1518, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 132, 133, 0, - 0, 135, 136, 0, 138, 139, 140, 141, 142, 0, - 144, 145, 0, 0, 146, 147, 148, 149, 150, 0, - 0, 151, 152, 153, 154, 155, 156, 157, 0, 158, - 159, 160, 161, 162, 0, 0, 0, 164, 165, 166, - 167, 168, 169, 0, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 0, 181, 182, 183, 184, + 0, 0, 0, 0, 1038, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 0, 0, 0, 133, 134, + 135, 0, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 0, 0, 148, 149, 150, 151, + 152, 0, 805, 153, 154, 155, 156, 157, 158, 159, + 0, 160, 161, 162, 163, 806, 0, 807, 0, 166, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 209, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, 0, 231, 0, 232, 233, - 234, 235, 0, 236, 0, 237, 0, 0, 0, 240, - 241, 538, 0, 244, 245, 246, 0, 247, 248, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 0, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 0, 272, 273, 274, 275, 276, - 277, 278, 279, 0, 280, 0, 281, 0, 0, 284, - 0, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 0, 294, 0, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 539, 305, 306, 307, 308, 309, 310, + 195, 196, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 211, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, 231, 808, 0, 233, 0, + 234, 235, 236, 237, 0, 238, 0, 239, 240, 0, + 241, 242, 243, 244, 245, 246, 247, 248, 0, 249, + 250, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 273, 274, 275, 276, + 277, 278, 279, 280, 281, 0, 282, 0, 283, 284, + 285, 286, 287, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 295, 296, 297, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 0, 329, 330, - 331, 332, 333, 0, 334, 335, 0, 337, 0, 338, - 339, 340, 341, 342, 343, 0, 344, 345, 0, 0, - 346, 347, 348, 0, 0, 349, 350, 351, 0, 353, - 0, 355, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 370, 0, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, + 331, 332, 333, 334, 335, 0, 336, 337, 338, 339, + 0, 810, 341, 342, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 353, + 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 813, 373, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 0, 410, 411, 412, 413, 414, 415, - 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 0, 437, 438, 439, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 540, 456, 457, 0, 0, 458, 459, - 0, 460, 0, 462, 463, 464, 465, 466, 467, 0, - 468, 469, 470, 0, 0, 471, 472, 473, 474, 475, - 0, 476, 477, 478, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 497, 498, 0, - 499, 0, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, + 407, 408, 0, 409, 410, 411, 412, 413, 414, 415, + 814, 417, 418, 419, 420, 421, 422, 423, 424, 425, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 438, 439, 440, 441, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 457, 458, 815, 0, 0, + 460, 461, 0, 462, 463, 464, 465, 466, 467, 468, + 469, 0, 470, 471, 472, 0, 0, 473, 474, 816, + 476, 817, 0, 478, 479, 818, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 0, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 499, + 500, 0, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 537, 0, 562, 0, 0, + 525, 526, 527, 528, 529, 530, 531, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2149, 0, 0, 0, 120, 121, 122, 123, 124, - 125, 126, 127, 0, 128, 129, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 132, 133, 0, 0, - 135, 136, 0, 138, 139, 140, 141, 142, 0, 144, - 145, 0, 0, 146, 147, 148, 149, 150, 0, 0, - 151, 152, 153, 154, 155, 156, 157, 0, 158, 159, - 160, 161, 162, 0, 0, 0, 164, 165, 166, 167, - 168, 169, 0, 171, 172, 173, 0, 174, 175, 176, - 177, 178, 179, 0, 0, 181, 182, 183, 184, 185, + 0, 0, 0, 46, 0, 0, 0, 122, 123, 124, + 125, 126, 127, 128, 129, 0, 130, 131, 132, 0, + 0, 0, 0, 0, 0, 0, 0, 133, 134, 135, + 0, 136, 137, 138, 139, 140, 141, 142, 143, 144, + 145, 146, 147, 0, 0, 148, 149, 150, 151, 152, + 0, 805, 153, 154, 155, 156, 157, 158, 159, 0, + 160, 161, 162, 163, 806, 0, 807, 0, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, 0, 176, + 177, 178, 179, 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 0, 196, 0, 197, 198, 199, 200, 201, 202, 0, - 0, 203, 204, 205, 206, 0, 0, 207, 208, 209, - 210, 211, 0, 212, 213, 214, 0, 215, 216, 217, - 218, 0, 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 0, 231, 0, 232, 233, 234, - 235, 0, 236, 0, 237, 0, 0, 0, 240, 241, - 538, 0, 244, 245, 246, 0, 247, 248, 249, 250, - 0, 251, 252, 253, 254, 255, 256, 257, 0, 259, - 260, 261, 262, 0, 263, 264, 265, 266, 267, 268, - 269, 0, 270, 0, 272, 273, 274, 275, 276, 277, - 278, 279, 0, 280, 0, 281, 0, 0, 284, 0, - 286, 287, 288, 0, 289, 290, 291, 0, 0, 292, - 0, 294, 0, 0, 296, 297, 298, 299, 300, 301, - 302, 303, 539, 305, 306, 307, 308, 309, 310, 311, + 196, 197, 0, 198, 0, 199, 200, 201, 202, 203, + 204, 0, 0, 205, 206, 207, 208, 0, 0, 209, + 210, 211, 212, 213, 0, 214, 215, 216, 0, 217, + 218, 219, 220, 0, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, 231, 808, 0, 233, 0, 234, + 235, 236, 237, 0, 238, 0, 239, 240, 0, 241, + 242, 243, 244, 245, 246, 247, 248, 0, 249, 250, + 251, 252, 0, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, + 269, 270, 271, 0, 272, 273, 274, 275, 276, 277, + 278, 279, 280, 281, 0, 282, 0, 283, 284, 285, + 286, 287, 288, 289, 290, 0, 291, 292, 293, 0, + 0, 294, 295, 296, 297, 0, 298, 299, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 0, 329, 330, 331, - 332, 333, 0, 334, 335, 0, 337, 0, 338, 339, - 340, 341, 342, 343, 0, 344, 345, 0, 0, 346, - 347, 348, 0, 0, 349, 350, 351, 0, 353, 0, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 0, 0, 0, 0, 368, 369, 370, - 0, 372, 373, 374, 375, 376, 377, 0, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 0, 394, 395, 396, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 0, - 407, 408, 0, 410, 411, 412, 413, 414, 415, 416, + 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, + 332, 333, 334, 335, 0, 336, 337, 338, 339, 0, + 810, 341, 342, 343, 344, 345, 0, 346, 347, 0, + 0, 348, 349, 350, 0, 0, 351, 352, 353, 354, + 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 0, 0, 0, 0, 370, + 371, 813, 373, 374, 375, 376, 377, 378, 379, 0, + 380, 381, 382, 383, 384, 385, 0, 386, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 0, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 0, 0, 427, 428, 429, 430, 431, 432, 433, 434, - 435, 0, 0, 437, 438, 439, 440, 0, 441, 442, + 427, 428, 0, 0, 429, 430, 431, 432, 433, 434, + 435, 436, 437, 0, 438, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, - 453, 454, 540, 456, 457, 0, 0, 458, 459, 0, - 460, 0, 462, 463, 464, 465, 466, 467, 0, 468, - 469, 470, 0, 0, 471, 472, 473, 474, 475, 0, - 476, 477, 478, 479, 480, 481, 482, 483, 0, 0, - 484, 485, 486, 0, 0, 487, 488, 489, 490, 0, - 491, 492, 493, 494, 495, 496, 497, 498, 0, 499, - 0, 501, 502, 503, 504, 505, 506, 507, 0, 0, - 508, 0, 0, 509, 510, 511, 512, 513, 514, 515, + 453, 454, 455, 456, 457, 458, 815, 0, 0, 460, + 461, 0, 462, 463, 464, 465, 466, 467, 468, 469, + 0, 470, 471, 472, 0, 0, 473, 474, 816, 476, + 817, 0, 478, 479, 818, 481, 482, 483, 484, 485, + 0, 0, 486, 487, 488, 0, 0, 489, 490, 491, + 492, 0, 493, 494, 495, 496, 497, 498, 499, 500, + 0, 501, 502, 503, 504, 505, 506, 507, 508, 509, + 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, - 526, 527, 528, 529, 537, 0, 562, 0, 0, 0, + 526, 527, 528, 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2296, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 132, 133, 0, 0, 135, - 136, 0, 138, 139, 140, 141, 142, 0, 144, 145, - 0, 0, 146, 147, 148, 149, 150, 0, 0, 151, - 152, 153, 154, 155, 156, 157, 0, 158, 159, 160, - 161, 162, 0, 0, 0, 164, 165, 166, 167, 168, - 169, 0, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 0, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 0, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 209, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 0, 231, 0, 232, 233, 234, 235, - 0, 236, 0, 237, 0, 0, 0, 240, 241, 538, - 0, 244, 245, 246, 0, 247, 248, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 0, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 0, 272, 273, 274, 275, 276, 277, 278, - 279, 0, 280, 0, 281, 0, 0, 284, 0, 286, - 287, 288, 0, 289, 290, 291, 0, 0, 292, 0, - 294, 0, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 539, 305, 306, 307, 308, 309, 310, 311, 312, + 0, 0, 3556, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 134, 135, 0, + 0, 137, 138, 0, 140, 141, 142, 143, 144, 0, + 146, 147, 0, 0, 148, 149, 150, 151, 152, 0, + 0, 153, 154, 155, 156, 157, 158, 159, 0, 160, + 161, 162, 163, 164, 0, 0, 0, 166, 167, 168, + 169, 170, 171, 0, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 14, 15, 205, 206, 207, 208, 0, 0, 209, 210, + 211, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 0, 233, 0, 234, 235, + 236, 237, 0, 238, 0, 239, 0, 23, 0, 242, + 243, 540, 0, 246, 247, 248, 0, 249, 250, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 0, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 0, 274, 275, 276, 277, 278, + 279, 280, 281, 0, 282, 0, 283, 0, 0, 286, + 0, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 0, 296, 0, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, 0, 329, 330, 331, 332, - 333, 0, 334, 335, 0, 337, 0, 338, 339, 340, - 341, 342, 343, 0, 344, 345, 0, 0, 346, 347, - 348, 0, 0, 349, 350, 351, 0, 353, 0, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 370, 0, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 0, 410, 411, 412, 413, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 0, 437, 438, 439, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 329, 0, 331, 332, + 333, 334, 335, 0, 336, 337, 0, 339, 0, 340, + 341, 342, 343, 344, 345, 0, 346, 347, 0, 0, + 348, 349, 350, 0, 0, 351, 352, 353, 0, 355, + 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 26, 27, 28, 0, 370, 371, + 372, 0, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 33, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 35, 0, 439, 440, 441, 442, 0, 443, + 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 542, 458, 459, 37, 0, 460, 461, + 38, 462, 0, 464, 465, 466, 467, 468, 469, 0, + 470, 471, 472, 0, 0, 473, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 40, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 819, 499, 500, 0, + 501, 0, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 44, 511, 512, 513, 514, 515, 516, + 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, + 527, 528, 529, 530, 531, 0, 539, 45, 564, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 46, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 910, 130, 131, 132, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 134, 135, 0, + 0, 137, 138, 0, 140, 141, 142, 143, 144, 0, + 146, 147, 0, 0, 148, 149, 150, 151, 152, 0, + 0, 153, 154, 155, 156, 157, 158, 159, 0, 160, + 161, 162, 163, 164, 0, 0, 0, 166, 167, 168, + 169, 170, 171, 0, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 211, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 0, 233, 0, 234, 235, + 236, 237, 0, 238, 0, 239, 0, 23, 0, 242, + 243, 540, 0, 246, 247, 248, 0, 249, 250, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 0, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 0, 274, 275, 276, 277, 278, + 279, 280, 281, 0, 282, 0, 283, 0, 0, 286, + 0, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 0, 296, 0, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, + 323, 324, 325, 326, 327, 328, 329, 0, 331, 332, + 333, 334, 335, 0, 336, 337, 0, 339, 0, 340, + 341, 342, 343, 344, 345, 0, 346, 347, 0, 0, + 348, 349, 350, 0, 0, 351, 352, 353, 0, 355, + 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 26, 27, 28, 0, 370, 371, + 372, 0, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 33, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, + 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 542, 458, 459, 0, 0, 460, 461, + 38, 911, 0, 464, 465, 466, 467, 468, 469, 0, + 470, 912, 472, 0, 0, 913, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 40, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 819, 499, 500, 0, + 501, 0, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 44, 511, 512, 513, 514, 515, 516, + 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, + 527, 528, 529, 530, 531, 0, 539, 45, 564, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 46, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 134, 135, 0, + 0, 137, 138, 0, 140, 141, 142, 143, 144, 0, + 146, 147, 0, 0, 148, 149, 150, 151, 152, 0, + 0, 153, 154, 155, 156, 157, 158, 159, 0, 160, + 161, 162, 163, 164, 0, 0, 0, 166, 167, 168, + 169, 170, 171, 0, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 211, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 0, 233, 0, 234, 235, + 236, 237, 0, 238, 0, 239, 0, 23, 0, 242, + 243, 540, 0, 246, 247, 248, 0, 249, 250, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 0, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 0, 274, 275, 276, 277, 278, + 279, 280, 281, 0, 282, 0, 283, 0, 0, 286, + 0, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 0, 296, 0, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, + 323, 324, 325, 326, 327, 328, 329, 0, 331, 332, + 333, 334, 335, 0, 336, 337, 0, 339, 0, 340, + 341, 342, 343, 344, 345, 0, 346, 347, 0, 0, + 348, 349, 350, 0, 0, 351, 352, 353, 0, 355, + 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 26, 27, 28, 0, 370, 371, + 372, 0, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 33, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, + 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 542, 458, 459, 0, 0, 460, 461, + 38, 462, 0, 464, 465, 466, 467, 468, 469, 0, + 470, 471, 472, 0, 0, 473, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 40, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 819, 499, 500, 0, + 501, 0, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 44, 511, 512, 513, 514, 515, 516, + 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, + 527, 528, 529, 530, 531, 0, 539, 45, 564, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 46, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 134, 135, 0, + 0, 137, 138, 0, 140, 141, 142, 143, 144, 0, + 146, 147, 0, 0, 148, 149, 150, 151, 152, 0, + 0, 153, 154, 155, 156, 157, 158, 159, 0, 160, + 161, 162, 163, 164, 0, 0, 0, 166, 167, 168, + 169, 170, 171, 0, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 211, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 0, 233, 0, 234, 235, + 236, 237, 0, 238, 0, 239, 0, 0, 0, 242, + 243, 540, 0, 246, 247, 248, 0, 249, 250, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 0, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 0, 274, 275, 276, 277, 278, + 279, 280, 281, 0, 282, 0, 283, 0, 0, 286, + 0, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 0, 296, 0, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, + 323, 324, 325, 326, 327, 328, 329, 0, 331, 332, + 333, 334, 335, 0, 336, 337, 0, 339, 0, 340, + 341, 342, 343, 344, 345, 0, 346, 347, 0, 0, + 348, 349, 350, 0, 0, 351, 352, 353, 0, 355, + 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 372, 0, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 540, 456, 457, 0, 0, 458, 459, 0, 460, - 0, 462, 463, 464, 465, 466, 467, 0, 468, 469, - 470, 0, 0, 471, 472, 473, 474, 475, 0, 476, - 477, 478, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 0, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 497, 498, 0, 499, 0, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 542, 458, 459, 0, 0, 460, 461, + 0, 462, 0, 464, 465, 466, 467, 468, 469, 0, + 470, 471, 472, 0, 0, 473, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 499, 500, 0, + 501, 0, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 537, 0, 562, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2587, - 0, 0, 0, 120, 121, 122, 123, 124, 125, 126, - 127, 0, 128, 129, 130, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 133, 0, 0, 135, 136, - 0, 138, 139, 140, 141, 142, 0, 144, 145, 0, - 0, 146, 147, 148, 149, 150, 0, 0, 151, 152, - 153, 154, 155, 156, 157, 0, 158, 159, 160, 161, - 162, 0, 0, 0, 164, 165, 166, 167, 168, 169, - 0, 171, 172, 173, 0, 174, 175, 176, 177, 178, - 179, 0, 0, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 0, 196, - 0, 197, 198, 199, 200, 201, 202, 0, 0, 203, - 204, 205, 206, 0, 0, 207, 208, 209, 210, 211, - 0, 212, 213, 214, 0, 215, 216, 217, 218, 0, - 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, 0, 231, 0, 232, 233, 234, 235, 0, - 236, 0, 237, 0, 0, 0, 240, 241, 538, 0, - 244, 245, 246, 0, 247, 248, 249, 250, 0, 251, - 252, 253, 254, 255, 256, 257, 0, 259, 260, 261, - 262, 0, 263, 264, 265, 266, 267, 268, 269, 0, - 270, 0, 272, 273, 274, 275, 276, 277, 278, 279, - 0, 280, 0, 281, 0, 0, 284, 0, 286, 287, - 288, 0, 289, 290, 291, 0, 0, 292, 0, 294, - 0, 0, 296, 297, 298, 299, 300, 301, 302, 303, - 539, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 527, 528, 529, 530, 531, 539, 0, 564, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1014, 0, 0, 0, 122, 123, 124, 125, 126, + 127, 128, 129, 0, 130, 131, 132, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 134, 135, 0, 0, + 137, 138, 0, 140, 141, 142, 143, 144, 0, 146, + 147, 0, 0, 148, 149, 150, 151, 152, 0, 0, + 153, 154, 155, 156, 157, 158, 159, 0, 160, 161, + 162, 163, 164, 0, 0, 0, 166, 167, 168, 169, + 170, 171, 0, 173, 174, 175, 0, 176, 177, 178, + 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 0, 198, 0, 199, 200, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 0, 0, 209, 210, 211, + 212, 213, 0, 214, 215, 216, 0, 217, 218, 219, + 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 0, 233, 0, 234, 235, 236, + 237, 0, 238, 0, 239, 0, 0, 0, 242, 243, + 540, 0, 246, 247, 248, 0, 249, 250, 251, 252, + 0, 253, 254, 255, 256, 257, 258, 259, 0, 261, + 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, + 271, 0, 272, 0, 274, 275, 276, 277, 278, 279, + 280, 281, 0, 282, 0, 283, 0, 0, 286, 0, + 288, 289, 290, 0, 291, 292, 293, 0, 0, 294, + 0, 296, 0, 0, 298, 299, 300, 301, 302, 303, + 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 327, 0, 329, 330, 331, 332, 333, - 0, 334, 335, 0, 337, 0, 338, 339, 340, 341, - 342, 343, 0, 344, 345, 0, 0, 346, 347, 348, - 0, 0, 349, 350, 351, 0, 353, 0, 355, 356, + 324, 325, 326, 327, 328, 329, 0, 331, 332, 333, + 334, 335, 0, 336, 337, 0, 339, 0, 340, 341, + 342, 343, 344, 345, 0, 346, 347, 0, 0, 348, + 349, 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 0, 0, 0, 0, 368, 369, 370, 0, 372, - 373, 374, 375, 376, 377, 0, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 0, 394, 395, 396, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 0, 407, 408, - 0, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 426, 0, 0, - 427, 428, 429, 430, 431, 432, 433, 434, 435, 0, - 0, 437, 438, 439, 440, 0, 441, 442, 443, 444, + 367, 368, 369, 0, 0, 0, 0, 370, 371, 372, + 0, 374, 375, 376, 377, 378, 379, 0, 380, 381, + 382, 383, 384, 385, 0, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 0, + 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 0, 0, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 540, 456, 457, 0, 0, 458, 459, 0, 460, 0, - 462, 463, 464, 465, 466, 467, 0, 468, 469, 470, - 0, 0, 471, 472, 473, 474, 475, 0, 476, 477, - 478, 479, 480, 481, 482, 483, 0, 0, 484, 485, - 486, 0, 0, 487, 488, 489, 490, 0, 491, 492, - 493, 494, 495, 496, 497, 498, 0, 499, 0, 501, - 502, 503, 504, 505, 506, 507, 0, 0, 508, 0, - 0, 509, 510, 511, 512, 513, 514, 515, 516, 517, + 455, 456, 542, 458, 459, 0, 0, 460, 461, 0, + 462, 0, 464, 465, 466, 467, 468, 469, 0, 470, + 471, 472, 0, 0, 473, 474, 475, 476, 477, 0, + 478, 479, 480, 481, 482, 483, 484, 485, 0, 0, + 486, 487, 488, 0, 0, 489, 490, 491, 492, 0, + 493, 494, 495, 496, 497, 498, 499, 500, 0, 501, + 0, 503, 504, 505, 506, 507, 508, 509, 0, 0, + 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, - 528, 529, 537, 0, 562, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2730, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 132, 133, 0, 0, 135, 136, 0, - 138, 139, 140, 141, 142, 0, 144, 145, 0, 0, - 146, 147, 148, 149, 150, 0, 0, 151, 152, 153, - 154, 155, 156, 157, 0, 158, 159, 160, 161, 162, - 0, 0, 0, 164, 165, 166, 167, 168, 169, 0, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 0, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 209, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, 0, 231, 0, 232, 233, 234, 235, 0, 236, - 0, 237, 0, 0, 0, 240, 241, 538, 0, 244, - 245, 246, 0, 247, 248, 249, 250, 0, 251, 252, - 253, 254, 255, 256, 257, 0, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 0, 272, 273, 274, 275, 276, 277, 278, 279, 0, - 280, 0, 281, 0, 0, 284, 0, 286, 287, 288, - 0, 289, 290, 291, 0, 0, 292, 0, 294, 0, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 539, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 528, 529, 530, 531, 539, 0, 564, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1525, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 134, 135, 0, 0, 137, + 138, 0, 140, 141, 142, 143, 144, 0, 146, 147, + 0, 0, 148, 149, 150, 151, 152, 0, 0, 153, + 154, 155, 156, 157, 158, 159, 0, 160, 161, 162, + 163, 164, 0, 0, 0, 166, 167, 168, 169, 170, + 171, 0, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 211, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 232, 0, 233, 0, 234, 235, 236, 237, + 0, 238, 0, 239, 0, 0, 0, 242, 243, 540, + 0, 246, 247, 248, 0, 249, 250, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 0, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 0, 274, 275, 276, 277, 278, 279, 280, + 281, 0, 282, 0, 283, 0, 0, 286, 0, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 0, + 296, 0, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 0, 329, 330, 331, 332, 333, 0, - 334, 335, 0, 337, 0, 338, 339, 340, 341, 342, - 343, 0, 344, 345, 0, 0, 346, 347, 348, 0, - 0, 349, 350, 351, 0, 353, 0, 355, 356, 357, + 325, 326, 327, 328, 329, 0, 331, 332, 333, 334, + 335, 0, 336, 337, 0, 339, 0, 340, 341, 342, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 370, 0, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 0, - 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 0, - 437, 438, 439, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 540, - 456, 457, 0, 0, 458, 459, 0, 460, 0, 462, - 463, 464, 465, 466, 467, 0, 468, 469, 470, 0, - 0, 471, 472, 473, 474, 475, 0, 476, 477, 478, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 0, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 497, 498, 0, 499, 0, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 368, 369, 0, 0, 0, 0, 370, 371, 372, 0, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 542, 458, 459, 0, 0, 460, 461, 0, 462, + 0, 464, 465, 466, 467, 468, 469, 0, 470, 471, + 472, 0, 0, 473, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 499, 500, 0, 501, 0, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 537, 0, 562, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2956, 0, 0, - 0, 120, 121, 122, 123, 124, 125, 126, 127, 0, - 128, 129, 130, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 132, 133, 0, 0, 135, 136, 0, 138, - 139, 140, 141, 142, 0, 144, 145, 0, 0, 146, - 147, 148, 149, 150, 0, 0, 151, 152, 153, 154, - 155, 156, 157, 0, 158, 159, 160, 161, 162, 0, - 0, 0, 164, 165, 166, 167, 168, 169, 0, 171, - 172, 173, 0, 174, 175, 176, 177, 178, 179, 0, - 0, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 0, 196, 0, 197, - 198, 199, 200, 201, 202, 0, 0, 203, 204, 205, - 206, 0, 0, 207, 208, 209, 210, 211, 0, 212, - 213, 214, 0, 215, 216, 217, 218, 0, 219, 220, + 529, 530, 531, 539, 0, 564, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2159, + 0, 0, 0, 122, 123, 124, 125, 126, 127, 128, + 129, 0, 130, 131, 132, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 134, 135, 0, 0, 137, 138, + 0, 140, 141, 142, 143, 144, 0, 146, 147, 0, + 0, 148, 149, 150, 151, 152, 0, 0, 153, 154, + 155, 156, 157, 158, 159, 0, 160, 161, 162, 163, + 164, 0, 0, 0, 166, 167, 168, 169, 170, 171, + 0, 173, 174, 175, 0, 176, 177, 178, 179, 180, + 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 0, 198, + 0, 199, 200, 201, 202, 203, 204, 0, 0, 205, + 206, 207, 208, 0, 0, 209, 210, 211, 212, 213, + 0, 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - 0, 231, 0, 232, 233, 234, 235, 0, 236, 0, - 237, 0, 0, 0, 240, 241, 538, 0, 244, 245, - 246, 0, 247, 248, 249, 250, 0, 251, 252, 253, - 254, 255, 256, 257, 0, 259, 260, 261, 262, 0, - 263, 264, 265, 266, 267, 268, 269, 0, 270, 0, - 272, 273, 274, 275, 276, 277, 278, 279, 0, 280, - 0, 281, 0, 0, 284, 0, 286, 287, 288, 0, - 289, 290, 291, 0, 0, 292, 0, 294, 0, 0, - 296, 297, 298, 299, 300, 301, 302, 303, 539, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 231, 232, 0, 233, 0, 234, 235, 236, 237, 0, + 238, 0, 239, 0, 0, 0, 242, 243, 540, 0, + 246, 247, 248, 0, 249, 250, 251, 252, 0, 253, + 254, 255, 256, 257, 258, 259, 0, 261, 262, 263, + 264, 0, 265, 266, 267, 268, 269, 270, 271, 0, + 272, 0, 274, 275, 276, 277, 278, 279, 280, 281, + 0, 282, 0, 283, 0, 0, 286, 0, 288, 289, + 290, 0, 291, 292, 293, 0, 0, 294, 0, 296, + 0, 0, 298, 299, 300, 301, 302, 303, 304, 305, + 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, 0, 329, 330, 331, 332, 333, 0, 334, - 335, 0, 337, 0, 338, 339, 340, 341, 342, 343, - 0, 344, 345, 0, 0, 346, 347, 348, 0, 0, - 349, 350, 351, 0, 353, 0, 355, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 0, - 0, 0, 0, 368, 369, 370, 0, 372, 373, 374, - 375, 376, 377, 0, 378, 379, 380, 381, 382, 383, - 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 0, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 0, 407, 408, 0, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 0, 0, 427, 428, - 429, 430, 431, 432, 433, 434, 435, 0, 0, 437, - 438, 439, 440, 0, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 540, 456, - 457, 0, 0, 458, 459, 0, 460, 0, 462, 463, - 464, 465, 466, 467, 0, 468, 469, 470, 0, 0, - 471, 472, 473, 474, 475, 0, 476, 477, 478, 479, - 480, 481, 482, 483, 0, 0, 484, 485, 486, 0, - 0, 487, 488, 489, 490, 0, 491, 492, 493, 494, - 495, 496, 497, 498, 0, 499, 0, 501, 502, 503, - 504, 505, 506, 507, 0, 0, 508, 0, 0, 509, - 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, + 326, 327, 328, 329, 0, 331, 332, 333, 334, 335, + 0, 336, 337, 0, 339, 0, 340, 341, 342, 343, + 344, 345, 0, 346, 347, 0, 0, 348, 349, 350, + 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 0, 0, 0, 0, 370, 371, 372, 0, 374, + 375, 376, 377, 378, 379, 0, 380, 381, 382, 383, + 384, 385, 0, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 0, 409, 410, + 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 426, 427, 428, 0, 0, + 429, 430, 431, 432, 433, 434, 435, 436, 437, 0, + 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 542, 458, 459, 0, 0, 460, 461, 0, 462, 0, + 464, 465, 466, 467, 468, 469, 0, 470, 471, 472, + 0, 0, 473, 474, 475, 476, 477, 0, 478, 479, + 480, 481, 482, 483, 484, 485, 0, 0, 486, 487, + 488, 0, 0, 489, 490, 491, 492, 0, 493, 494, + 495, 496, 497, 498, 499, 500, 0, 501, 0, 503, + 504, 505, 506, 507, 508, 509, 0, 0, 510, 0, + 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, - 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3441, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 132, 133, 0, 0, 135, 136, 0, 138, 139, - 140, 141, 142, 0, 144, 145, 0, 0, 146, 147, - 148, 149, 150, 0, 0, 151, 152, 153, 154, 155, - 156, 157, 0, 158, 159, 160, 161, 162, 0, 0, - 0, 164, 165, 166, 167, 168, 169, 0, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 0, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 0, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 209, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, 0, - 231, 0, 232, 233, 234, 235, 0, 236, 0, 237, - 0, 0, 0, 240, 241, 538, 0, 244, 245, 246, - 0, 247, 248, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 0, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 0, 272, - 273, 274, 275, 276, 277, 278, 279, 0, 280, 0, - 281, 0, 0, 284, 0, 286, 287, 288, 0, 289, - 290, 291, 0, 0, 292, 0, 294, 0, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 539, 305, 306, + 530, 531, 539, 0, 564, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2306, 0, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 134, 135, 0, 0, 137, 138, 0, + 140, 141, 142, 143, 144, 0, 146, 147, 0, 0, + 148, 149, 150, 151, 152, 0, 0, 153, 154, 155, + 156, 157, 158, 159, 0, 160, 161, 162, 163, 164, + 0, 0, 0, 166, 167, 168, 169, 170, 171, 0, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 211, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 0, 233, 0, 234, 235, 236, 237, 0, 238, + 0, 239, 0, 0, 0, 242, 243, 540, 0, 246, + 247, 248, 0, 249, 250, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 0, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 0, 274, 275, 276, 277, 278, 279, 280, 281, 0, + 282, 0, 283, 0, 0, 286, 0, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 0, 296, 0, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, 0, 329, 330, 331, 332, 333, 0, 334, 335, - 0, 337, 0, 338, 339, 340, 341, 342, 343, 0, - 344, 345, 0, 0, 346, 347, 348, 0, 0, 349, - 350, 351, 0, 353, 0, 355, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 370, 0, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 0, 410, 411, + 327, 328, 329, 0, 331, 332, 333, 334, 335, 0, + 336, 337, 0, 339, 0, 340, 341, 342, 343, 344, + 345, 0, 346, 347, 0, 0, 348, 349, 350, 0, + 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 372, 0, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 0, 437, 438, - 439, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 540, 456, 457, - 0, 0, 458, 459, 0, 460, 0, 462, 463, 464, - 465, 466, 467, 0, 468, 469, 470, 0, 0, 471, - 472, 473, 474, 475, 0, 476, 477, 478, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 0, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 497, 498, 0, 499, 0, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 0, + 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 542, + 458, 459, 0, 0, 460, 461, 0, 462, 0, 464, + 465, 466, 467, 468, 469, 0, 470, 471, 472, 0, + 0, 473, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 0, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 499, 500, 0, 501, 0, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 537, - 0, 0, 0, 0, 0, 0, 0, 0, 3777, 0, - 0, 0, 0, 0, 0, 2256, 0, 0, 0, 120, - 121, 122, 123, 124, 125, 126, 127, 0, 128, 129, - 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 132, 133, 0, 0, 135, 136, 0, 138, 139, 140, - 141, 142, 0, 144, 145, 0, 0, 146, 147, 148, - 149, 150, 0, 0, 151, 152, 153, 154, 155, 156, - 157, 0, 158, 159, 160, 161, 162, 0, 0, 0, - 164, 165, 166, 167, 168, 169, 0, 171, 172, 173, - 0, 174, 175, 176, 177, 178, 179, 0, 0, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 0, 196, 0, 197, 198, 199, - 200, 201, 202, 0, 0, 203, 204, 205, 206, 0, - 0, 207, 208, 209, 210, 211, 0, 212, 213, 214, - 0, 215, 216, 217, 218, 0, 219, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, 0, 231, - 0, 232, 233, 234, 235, 0, 236, 0, 237, 0, - 0, 0, 240, 241, 538, 0, 244, 245, 246, 0, - 247, 248, 249, 250, 0, 251, 252, 253, 254, 255, - 256, 257, 0, 259, 260, 261, 262, 0, 263, 264, - 265, 266, 267, 268, 269, 0, 270, 0, 272, 273, - 274, 275, 276, 277, 278, 279, 0, 280, 0, 281, - 0, 0, 284, 0, 286, 287, 288, 0, 289, 290, - 291, 0, 0, 292, 0, 294, 0, 0, 296, 297, - 298, 299, 300, 301, 302, 303, 539, 305, 306, 307, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2599, 0, 0, + 0, 122, 123, 124, 125, 126, 127, 128, 129, 0, + 130, 131, 132, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 134, 135, 0, 0, 137, 138, 0, 140, + 141, 142, 143, 144, 0, 146, 147, 0, 0, 148, + 149, 150, 151, 152, 0, 0, 153, 154, 155, 156, + 157, 158, 159, 0, 160, 161, 162, 163, 164, 0, + 0, 0, 166, 167, 168, 169, 170, 171, 0, 173, + 174, 175, 0, 176, 177, 178, 179, 180, 181, 0, + 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 0, 198, 0, 199, + 200, 201, 202, 203, 204, 0, 0, 205, 206, 207, + 208, 0, 0, 209, 210, 211, 212, 213, 0, 214, + 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, + 0, 233, 0, 234, 235, 236, 237, 0, 238, 0, + 239, 0, 0, 0, 242, 243, 540, 0, 246, 247, + 248, 0, 249, 250, 251, 252, 0, 253, 254, 255, + 256, 257, 258, 259, 0, 261, 262, 263, 264, 0, + 265, 266, 267, 268, 269, 270, 271, 0, 272, 0, + 274, 275, 276, 277, 278, 279, 280, 281, 0, 282, + 0, 283, 0, 0, 286, 0, 288, 289, 290, 0, + 291, 292, 293, 0, 0, 294, 0, 296, 0, 0, + 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - 0, 329, 330, 331, 332, 333, 0, 334, 335, 0, - 337, 0, 338, 339, 340, 341, 342, 343, 0, 344, - 345, 0, 0, 346, 347, 348, 0, 0, 349, 350, - 351, 0, 353, 0, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 0, 0, 0, - 0, 368, 369, 370, 0, 372, 373, 374, 375, 376, - 377, 0, 378, 379, 380, 381, 382, 383, 0, 384, - 385, 386, 387, 388, 389, 390, 391, 392, 393, 0, - 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 0, 407, 408, 0, 410, 411, 412, + 328, 329, 0, 331, 332, 333, 334, 335, 0, 336, + 337, 0, 339, 0, 340, 341, 342, 343, 344, 345, + 0, 346, 347, 0, 0, 348, 349, 350, 0, 0, + 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 0, + 0, 0, 0, 370, 371, 372, 0, 374, 375, 376, + 377, 378, 379, 0, 380, 381, 382, 383, 384, 385, + 0, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 426, 0, 0, 427, 428, 429, 430, - 431, 432, 433, 434, 435, 0, 0, 437, 438, 439, - 440, 0, 441, 442, 443, 444, 445, 446, 447, 448, - 449, 450, 451, 452, 453, 454, 540, 456, 457, 0, - 0, 458, 459, 0, 460, 0, 462, 463, 464, 465, - 466, 467, 0, 468, 469, 470, 0, 0, 471, 472, - 473, 474, 475, 0, 476, 477, 478, 479, 480, 481, - 482, 483, 0, 0, 484, 485, 486, 0, 0, 487, - 488, 489, 490, 0, 491, 492, 493, 494, 495, 496, - 497, 498, 0, 499, 0, 501, 502, 503, 504, 505, - 506, 507, 0, 0, 508, 0, 0, 509, 510, 511, + 423, 424, 425, 426, 427, 428, 0, 0, 429, 430, + 431, 432, 433, 434, 435, 436, 437, 0, 0, 439, + 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 456, 542, 458, + 459, 0, 0, 460, 461, 0, 462, 0, 464, 465, + 466, 467, 468, 469, 0, 470, 471, 472, 0, 0, + 473, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 485, 0, 0, 486, 487, 488, 0, + 0, 489, 490, 491, 492, 0, 493, 494, 495, 496, + 497, 498, 499, 500, 0, 501, 0, 503, 504, 505, + 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, - 522, 523, 524, 525, 526, 527, 528, 529, 537, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2256, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, - 133, 0, 0, 135, 136, 0, 138, 139, 140, 141, - 142, 0, 144, 145, 0, 0, 146, 147, 148, 149, - 150, 0, 0, 151, 152, 153, 154, 155, 156, 157, - 0, 158, 159, 160, 161, 162, 0, 0, 0, 164, - 165, 166, 167, 168, 169, 0, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 0, 181, 182, + 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, + 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2742, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 134, 135, 0, 0, 137, 138, 0, 140, 141, + 142, 143, 144, 0, 146, 147, 0, 0, 148, 149, + 150, 151, 152, 0, 0, 153, 154, 155, 156, 157, + 158, 159, 0, 160, 161, 162, 163, 164, 0, 0, + 0, 166, 167, 168, 169, 170, 171, 0, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 209, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, 0, 231, 0, - 232, 233, 234, 235, 0, 236, 0, 237, 0, 0, - 0, 240, 241, 538, 0, 244, 245, 246, 0, 247, - 248, 249, 250, 0, 251, 252, 253, 254, 255, 256, - 257, 0, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 0, 272, 273, 274, - 275, 276, 277, 278, 279, 0, 280, 0, 281, 0, - 0, 284, 0, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 0, 294, 0, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 539, 305, 306, 307, 308, + 193, 194, 195, 196, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 211, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 0, + 233, 0, 234, 235, 236, 237, 0, 238, 0, 239, + 0, 0, 0, 242, 243, 540, 0, 246, 247, 248, + 0, 249, 250, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 0, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 0, 274, + 275, 276, 277, 278, 279, 280, 281, 0, 282, 0, + 283, 0, 0, 286, 0, 288, 289, 290, 0, 291, + 292, 293, 0, 0, 294, 0, 296, 0, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, 0, - 329, 330, 331, 332, 333, 0, 334, 335, 0, 337, - 0, 338, 339, 340, 341, 342, 343, 0, 344, 345, - 0, 0, 346, 347, 348, 0, 0, 349, 350, 351, - 0, 353, 0, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 370, 0, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 0, 410, 411, 412, 413, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 0, 331, 332, 333, 334, 335, 0, 336, 337, + 0, 339, 0, 340, 341, 342, 343, 344, 345, 0, + 346, 347, 0, 0, 348, 349, 350, 0, 0, 351, + 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 372, 0, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 0, 437, 438, 439, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 540, 456, 457, 0, 0, - 458, 459, 0, 460, 0, 462, 463, 464, 465, 466, - 467, 0, 468, 469, 470, 0, 0, 471, 472, 473, - 474, 475, 0, 476, 477, 478, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 497, - 498, 0, 499, 0, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 0, 439, 440, + 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 542, 458, 459, + 0, 0, 460, 461, 0, 462, 0, 464, 465, 466, + 467, 468, 469, 0, 470, 471, 472, 0, 0, 473, + 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 0, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 499, 500, 0, 501, 0, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 3087, 1388, 833, - 0, 0, 2123, 1075, 0, 0, 0, 0, 0, 2124, - 2125, 0, 0, 3292, 2126, 2127, 2128, 120, 121, 122, - 123, 124, 125, 126, 127, 569, 128, 129, 130, 570, - 571, 572, 3088, 574, 575, 576, 577, 3089, 132, 133, - 579, 3090, 135, 136, 3091, 138, 139, 140, 0, 1532, - 3092, 1534, 1535, 587, 3093, 146, 147, 148, 149, 150, - 589, 590, 151, 152, 153, 154, 1537, 1538, 157, 593, - 158, 159, 160, 161, 0, 595, 3094, 597, 3095, 165, - 166, 167, 168, 169, 3096, 171, 172, 173, 600, 174, - 175, 176, 177, 178, 179, 601, 3097, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 1543, 192, 193, - 1544, 195, 606, 196, 607, 197, 198, 199, 200, 201, - 202, 608, 609, 203, 204, 205, 206, 610, 611, 207, - 208, 1088, 210, 211, 612, 212, 213, 214, 613, 215, - 216, 217, 218, 614, 219, 220, 221, 222, 0, 224, - 225, 226, 227, 228, 229, 0, 617, 231, 618, 232, - 233, 1545, 235, 620, 236, 621, 237, 3098, 623, 3099, - 240, 241, 2460, 3100, 244, 245, 246, 627, 0, 0, - 249, 250, 630, 251, 252, 253, 254, 255, 256, 257, - 3101, 259, 260, 261, 262, 632, 263, 264, 265, 266, - 267, 268, 269, 633, 270, 3102, 0, 273, 274, 275, - 276, 277, 1551, 1552, 638, 1553, 640, 281, 3103, 3104, - 284, 3105, 286, 287, 288, 644, 289, 290, 291, 645, - 646, 292, 3106, 294, 3107, 649, 296, 297, 298, 299, - 300, 301, 302, 303, 3108, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 1560, 3109, 1562, - 330, 331, 332, 3110, 655, 334, 335, 3111, 337, 657, - 0, 339, 1564, 341, 342, 343, 660, 344, 345, 661, - 662, 3112, 347, 348, 663, 664, 349, 350, 0, 3113, - 353, 3114, 0, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 669, 670, 671, 672, 368, - 369, 0, 3115, 372, 373, 0, 375, 376, 377, 676, - 378, 379, 380, 381, 382, 383, 677, 384, 385, 386, - 387, 388, 1568, 390, 391, 392, 393, 679, 394, 395, - 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 680, 407, 408, 3116, 410, 411, 412, 1570, 414, - 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 683, 3117, 427, 428, 429, 430, 431, 432, - 3118, 434, 435, 686, 3119, 437, 438, 1574, 440, 689, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 3120, 456, 0, 692, 693, 458, - 459, 694, 460, 3121, 462, 463, 464, 465, 466, 467, - 696, 468, 1577, 1578, 699, 700, 471, 472, 0, 474, - 0, 703, 476, 477, 3122, 479, 480, 481, 482, 483, - 3123, 706, 484, 485, 486, 3124, 708, 487, 488, 489, - 490, 709, 491, 492, 493, 494, 495, 0, 1582, 498, - 712, 499, 3125, 501, 502, 503, 504, 505, 506, 507, - 714, 715, 508, 716, 717, 509, 510, 511, 512, 513, - 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 526, 527, 528, 529, 0, 537, 0, 2129, - 2130, 2131, 2123, 3126, 3127, 2134, 2135, 2136, 2137, 2124, - 2125, 0, 0, 0, 2126, 2127, 2128, 120, 121, 122, - 123, 124, 125, 126, 127, 0, 128, 129, 130, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 132, 133, - 0, 0, 135, 136, 0, 138, 139, 140, 141, 142, - 0, 144, 145, 0, 0, 146, 147, 148, 149, 150, - 0, 0, 151, 152, 153, 154, 155, 156, 157, 0, - 158, 159, 160, 161, 162, 0, 0, 0, 164, 165, - 166, 167, 168, 169, 0, 171, 172, 173, 0, 174, - 175, 176, 177, 178, 179, 0, 0, 181, 182, 183, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 539, + 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2969, 0, 0, 0, 122, + 123, 124, 125, 126, 127, 128, 129, 0, 130, 131, + 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 134, 135, 0, 0, 137, 138, 0, 140, 141, 142, + 143, 144, 0, 146, 147, 0, 0, 148, 149, 150, + 151, 152, 0, 0, 153, 154, 155, 156, 157, 158, + 159, 0, 160, 161, 162, 163, 164, 0, 0, 0, + 166, 167, 168, 169, 170, 171, 0, 173, 174, 175, + 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 0, 196, 0, 197, 198, 199, 200, 201, - 202, 0, 0, 203, 204, 205, 206, 0, 0, 207, - 208, 209, 210, 211, 0, 212, 213, 214, 0, 215, - 216, 217, 218, 0, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 0, 231, 0, 232, - 233, 234, 235, 0, 236, 0, 237, 0, 0, 0, - 240, 241, 538, 0, 244, 245, 246, 0, 247, 248, - 249, 250, 0, 251, 252, 253, 254, 255, 256, 257, - 0, 259, 260, 261, 262, 0, 263, 264, 265, 266, - 267, 268, 269, 0, 270, 0, 272, 273, 274, 275, - 276, 277, 278, 279, 0, 280, 0, 281, 0, 0, - 284, 0, 286, 287, 288, 0, 289, 290, 291, 0, - 0, 292, 0, 294, 0, 0, 296, 297, 298, 299, - 300, 301, 302, 303, 539, 305, 306, 307, 308, 309, + 194, 195, 196, 197, 0, 198, 0, 199, 200, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 0, + 0, 209, 210, 211, 212, 213, 0, 214, 215, 216, + 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 0, 233, + 0, 234, 235, 236, 237, 0, 238, 0, 239, 0, + 0, 0, 242, 243, 540, 0, 246, 247, 248, 0, + 249, 250, 251, 252, 0, 253, 254, 255, 256, 257, + 258, 259, 0, 261, 262, 263, 264, 0, 265, 266, + 267, 268, 269, 270, 271, 0, 272, 0, 274, 275, + 276, 277, 278, 279, 280, 281, 0, 282, 0, 283, + 0, 0, 286, 0, 288, 289, 290, 0, 291, 292, + 293, 0, 0, 294, 0, 296, 0, 0, 298, 299, + 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, 0, 329, - 330, 331, 332, 333, 0, 334, 335, 0, 337, 0, - 338, 339, 340, 341, 342, 343, 0, 344, 345, 0, - 0, 346, 347, 348, 0, 0, 349, 350, 351, 0, - 353, 0, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 0, 0, 0, 0, 368, - 369, 370, 0, 372, 373, 374, 375, 376, 377, 0, - 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, - 387, 388, 389, 390, 391, 392, 393, 0, 394, 395, + 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 0, 331, 332, 333, 334, 335, 0, 336, 337, 0, + 339, 0, 340, 341, 342, 343, 344, 345, 0, 346, + 347, 0, 0, 348, 349, 350, 0, 0, 351, 352, + 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 0, 0, 0, + 0, 370, 371, 372, 0, 374, 375, 376, 377, 378, + 379, 0, 380, 381, 382, 383, 384, 385, 0, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 0, 407, 408, 0, 410, 411, 412, 413, 414, + 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 0, 0, 427, 428, 429, 430, 431, 432, - 433, 434, 435, 0, 0, 437, 438, 439, 440, 0, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 540, 456, 457, 0, 0, 458, - 459, 0, 460, 0, 462, 463, 464, 465, 466, 467, - 0, 468, 469, 470, 0, 0, 471, 472, 473, 474, - 475, 0, 476, 477, 478, 479, 480, 481, 482, 483, - 0, 0, 484, 485, 486, 0, 0, 487, 488, 489, - 490, 0, 491, 492, 493, 494, 495, 496, 497, 498, - 0, 499, 0, 501, 502, 503, 504, 505, 506, 507, - 0, 0, 508, 0, 0, 509, 510, 511, 512, 513, + 425, 426, 427, 428, 0, 0, 429, 430, 431, 432, + 433, 434, 435, 436, 437, 0, 0, 439, 440, 441, + 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, + 451, 452, 453, 454, 455, 456, 542, 458, 459, 0, + 0, 460, 461, 0, 462, 0, 464, 465, 466, 467, + 468, 469, 0, 470, 471, 472, 0, 0, 473, 474, + 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, + 484, 485, 0, 0, 486, 487, 488, 0, 0, 489, + 490, 491, 492, 0, 493, 494, 495, 496, 497, 498, + 499, 500, 0, 501, 0, 503, 504, 505, 506, 507, + 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, - 524, 525, 526, 527, 528, 529, 0, 0, 0, 2129, - 2130, 2131, 0, 2132, 2133, 2134, 2135, 2136, 2137, 1669, - 0, 0, 1670, 0, 0, 0, 1671, 1672, 1673, 1674, - 0, 1675, 1676, 1677, 0, 0, 0, 1669, 0, 0, - 1670, 0, 0, 0, 1671, 1672, 1673, 1674, 1678, 1675, - 1676, 1677, 0, 0, 0, 0, 0, 0, 1680, 0, - 0, 0, 0, 0, 0, 1681, 1678, 0, 0, 0, - 0, 0, 1669, 0, 0, 1670, 1680, 0, 0, 1671, - 1672, 1673, 1674, 1681, 1675, 1676, 1677, 0, 0, 0, - 1669, 0, 1682, 1670, 0, 0, 0, 1671, 1672, 1673, - 1674, 1678, 1675, 1676, 1677, 0, 0, 0, 0, 0, - 1682, 1680, 0, 0, 0, 0, 0, 0, 1681, 1678, - 0, 0, 0, 0, 0, 1669, 0, 0, 1670, 1680, - 0, 0, 1671, 1672, 1673, 1674, 1681, 1675, 1676, 1677, - 0, 0, 0, 0, 0, 1682, 0, 0, 0, 0, - 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, - 0, 0, 0, 1682, 1680, 0, 0, 1669, 0, 0, - 1670, 1681, 0, 0, 1671, 1672, 1673, 1674, 0, 1675, - 1676, 1677, 0, 0, 0, 0, 0, 0, 0, 0, - 1683, 0, 0, 0, 0, 0, 1678, 0, 1682, 0, - 0, 0, 0, 0, 0, 0, 1680, 1684, 1683, 0, - 0, 0, 1685, 1681, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1684, 0, 0, 0, 0, - 1685, 0, 0, 0, 0, 1686, 1687, 0, 0, 0, - 1682, 0, 0, 1683, 0, 0, 0, 0, 0, 0, - 0, 1688, 0, 1686, 1687, 0, 0, 0, 0, 0, - 1684, 1683, 0, 0, 0, 1685, 0, 0, 0, 1688, - 0, 0, 0, 0, 0, 0, 0, 0, 1684, 0, - 0, 0, 0, 1685, 0, 0, 0, 0, 1686, 1687, - 0, 1689, 0, 0, 1690, 0, 1683, 0, 0, 0, - 0, 0, 0, 0, 1688, 0, 1686, 1687, 1691, 1689, - 0, 1692, 1690, 1684, 0, 0, 0, 0, 1685, 0, - 0, 0, 1688, 0, 0, 0, 1691, 0, 0, 1692, - 0, 0, 0, 0, 0, 0, 0, 0, 1683, 0, - 0, 1686, 1687, 0, 1689, 0, 0, 1690, 0, 0, - 0, 0, 0, 0, 0, 1684, 0, 1688, 0, 0, - 1685, 1691, 1689, 0, 1692, 1690, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1691, - 0, 0, 1692, 1686, 1687, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1689, 0, 1688, - 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1693, 0, 0, 1691, 0, 0, 1692, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, - 0, 0, 0, 0, 0, 0, 0, 1669, 0, 1689, - 1670, 0, 1690, 0, 1671, 1672, 1673, 1674, 0, 1675, - 1676, 1677, 0, 0, 0, 0, 1691, 0, 0, 1692, - 0, 0, 0, 0, 1693, 0, 1678, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1680, 0, 0, 0, - 0, 0, 1693, 1681, 0, 0, 0, 0, 0, 0, + 524, 525, 526, 527, 528, 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3455, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, + 135, 0, 0, 137, 138, 0, 140, 141, 142, 143, + 144, 0, 146, 147, 0, 0, 148, 149, 150, 151, + 152, 0, 0, 153, 154, 155, 156, 157, 158, 159, + 0, 160, 161, 162, 163, 164, 0, 0, 0, 166, + 167, 168, 169, 170, 171, 0, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 211, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, 231, 232, 0, 233, 0, + 234, 235, 236, 237, 0, 238, 0, 239, 0, 0, + 0, 242, 243, 540, 0, 246, 247, 248, 0, 249, + 250, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 0, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 0, 274, 275, 276, + 277, 278, 279, 280, 281, 0, 282, 0, 283, 0, + 0, 286, 0, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 0, 296, 0, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + 321, 322, 323, 324, 325, 326, 327, 328, 329, 0, + 331, 332, 333, 334, 335, 0, 336, 337, 0, 339, + 0, 340, 341, 342, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 353, + 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 372, 0, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, + 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, + 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 0, 439, 440, 441, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 542, 458, 459, 0, 0, + 460, 461, 0, 462, 0, 464, 465, 466, 467, 468, + 469, 0, 470, 471, 472, 0, 0, 473, 474, 475, + 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 0, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 499, + 500, 0, 501, 0, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, + 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, + 525, 526, 527, 528, 529, 530, 531, 539, 0, 0, + 0, 0, 0, 0, 0, 0, 3791, 0, 0, 0, + 0, 0, 0, 2266, 0, 0, 0, 122, 123, 124, + 125, 126, 127, 128, 129, 0, 130, 131, 132, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 134, 135, + 0, 0, 137, 138, 0, 140, 141, 142, 143, 144, + 0, 146, 147, 0, 0, 148, 149, 150, 151, 152, + 0, 0, 153, 154, 155, 156, 157, 158, 159, 0, + 160, 161, 162, 163, 164, 0, 0, 0, 166, 167, + 168, 169, 170, 171, 0, 173, 174, 175, 0, 176, + 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, + 196, 197, 0, 198, 0, 199, 200, 201, 202, 203, + 204, 0, 0, 205, 206, 207, 208, 0, 0, 209, + 210, 211, 212, 213, 0, 214, 215, 216, 0, 217, + 218, 219, 220, 0, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, 231, 232, 0, 233, 0, 234, + 235, 236, 237, 0, 238, 0, 239, 0, 0, 0, + 242, 243, 540, 0, 246, 247, 248, 0, 249, 250, + 251, 252, 0, 253, 254, 255, 256, 257, 258, 259, + 0, 261, 262, 263, 264, 0, 265, 266, 267, 268, + 269, 270, 271, 0, 272, 0, 274, 275, 276, 277, + 278, 279, 280, 281, 0, 282, 0, 283, 0, 0, + 286, 0, 288, 289, 290, 0, 291, 292, 293, 0, + 0, 294, 0, 296, 0, 0, 298, 299, 300, 301, + 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, + 322, 323, 324, 325, 326, 327, 328, 329, 0, 331, + 332, 333, 334, 335, 0, 336, 337, 0, 339, 0, + 340, 341, 342, 343, 344, 345, 0, 346, 347, 0, + 0, 348, 349, 350, 0, 0, 351, 352, 353, 0, + 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 0, 0, 0, 0, 370, + 371, 372, 0, 374, 375, 376, 377, 378, 379, 0, + 380, 381, 382, 383, 384, 385, 0, 386, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, + 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, + 427, 428, 0, 0, 429, 430, 431, 432, 433, 434, + 435, 436, 437, 0, 0, 439, 440, 441, 442, 0, + 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, + 453, 454, 455, 456, 542, 458, 459, 0, 0, 460, + 461, 0, 462, 0, 464, 465, 466, 467, 468, 469, + 0, 470, 471, 472, 0, 0, 473, 474, 475, 476, + 477, 0, 478, 479, 480, 481, 482, 483, 484, 485, + 0, 0, 486, 487, 488, 0, 0, 489, 490, 491, + 492, 0, 493, 494, 495, 496, 497, 498, 499, 500, + 0, 501, 0, 503, 504, 505, 506, 507, 508, 509, + 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, + 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, + 526, 527, 528, 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1682, 0, 0, 0, 0, 0, 0, 1693, 0, 0, - 0, 0, 0, 1694, 0, 0, 1695, 1696, 1697, 0, - 1698, 1699, 1700, 1701, 1702, 1703, 0, 0, 0, 0, - 3492, 1694, 0, 0, 1695, 1696, 1697, 0, 1698, 1699, - 1700, 1701, 1702, 1703, 0, 0, 0, 0, 3596, 1693, + 0, 0, 2266, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 134, 135, 0, + 0, 137, 138, 0, 140, 141, 142, 143, 144, 0, + 146, 147, 0, 0, 148, 149, 150, 151, 152, 0, + 0, 153, 154, 155, 156, 157, 158, 159, 0, 160, + 161, 162, 163, 164, 0, 0, 0, 166, 167, 168, + 169, 170, 171, 0, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 211, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 0, 233, 0, 234, 235, + 236, 237, 0, 238, 0, 239, 0, 0, 0, 242, + 243, 540, 0, 246, 247, 248, 0, 249, 250, 251, + 252, 0, 253, 254, 255, 256, 257, 258, 259, 0, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 0, 274, 275, 276, 277, 278, + 279, 280, 281, 0, 282, 0, 283, 0, 0, 286, + 0, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 0, 296, 0, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, + 323, 324, 325, 326, 327, 328, 329, 0, 331, 332, + 333, 334, 335, 0, 336, 337, 0, 339, 0, 340, + 341, 342, 343, 344, 345, 0, 346, 347, 0, 0, + 348, 349, 350, 0, 0, 351, 352, 353, 0, 355, + 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 372, 0, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, + 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 542, 458, 459, 0, 0, 460, 461, + 0, 462, 0, 464, 465, 466, 467, 468, 469, 0, + 470, 471, 472, 0, 0, 473, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 499, 500, 0, + 501, 0, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, + 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, + 527, 528, 529, 530, 531, 3101, 1395, 836, 0, 0, + 2133, 1080, 0, 0, 0, 0, 0, 2134, 2135, 0, + 0, 3306, 2136, 2137, 2138, 122, 123, 124, 125, 126, + 127, 128, 129, 571, 130, 131, 132, 572, 573, 574, + 3102, 576, 577, 578, 579, 3103, 134, 135, 581, 3104, + 137, 138, 3105, 140, 141, 142, 0, 1539, 3106, 1541, + 1542, 589, 3107, 148, 149, 150, 151, 152, 591, 592, + 153, 154, 155, 156, 1544, 1545, 159, 595, 160, 161, + 162, 163, 0, 597, 3108, 599, 3109, 167, 168, 169, + 170, 171, 3110, 173, 174, 175, 602, 176, 177, 178, + 179, 180, 181, 603, 3111, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 1550, 194, 195, 1551, 197, + 608, 198, 609, 199, 200, 201, 202, 203, 204, 610, + 611, 205, 206, 207, 208, 612, 613, 209, 210, 1093, + 212, 213, 614, 214, 215, 216, 615, 217, 218, 219, + 220, 616, 221, 222, 223, 224, 0, 226, 227, 228, + 229, 230, 231, 0, 619, 233, 620, 234, 235, 1552, + 237, 622, 238, 623, 239, 3112, 625, 3113, 242, 243, + 2472, 3114, 246, 247, 248, 629, 0, 0, 251, 252, + 632, 253, 254, 255, 256, 257, 258, 259, 3115, 261, + 262, 263, 264, 634, 265, 266, 267, 268, 269, 270, + 271, 635, 272, 3116, 0, 275, 276, 277, 278, 279, + 1558, 1559, 640, 1560, 642, 283, 3117, 3118, 286, 3119, + 288, 289, 290, 646, 291, 292, 293, 647, 648, 294, + 3120, 296, 3121, 651, 298, 299, 300, 301, 302, 303, + 304, 305, 3122, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, 328, 1567, 3123, 1569, 332, 333, + 334, 3124, 657, 336, 337, 3125, 339, 659, 0, 341, + 1571, 343, 344, 345, 662, 346, 347, 663, 664, 3126, + 349, 350, 665, 666, 351, 352, 0, 3127, 355, 3128, + 0, 358, 359, 360, 361, 362, 363, 364, 365, 366, + 367, 368, 369, 671, 672, 673, 674, 370, 371, 0, + 3129, 374, 375, 0, 377, 378, 379, 678, 380, 381, + 382, 383, 384, 385, 679, 386, 387, 388, 389, 390, + 1575, 392, 393, 394, 395, 681, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 682, + 409, 410, 3130, 412, 413, 414, 1577, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 685, 3131, 429, 430, 431, 432, 433, 434, 3132, 436, + 437, 688, 3133, 439, 440, 1581, 442, 691, 443, 444, + 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 3134, 458, 0, 694, 695, 460, 461, 696, + 462, 3135, 464, 465, 466, 467, 468, 469, 698, 470, + 1584, 1585, 701, 702, 473, 474, 0, 476, 0, 705, + 478, 479, 3136, 481, 482, 483, 484, 485, 3137, 708, + 486, 487, 488, 3138, 710, 489, 490, 491, 492, 711, + 493, 494, 495, 496, 497, 0, 1589, 500, 714, 501, + 3139, 503, 504, 505, 506, 507, 508, 509, 716, 717, + 510, 718, 719, 511, 512, 513, 514, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1694, 0, 0, 1695, - 1696, 1697, 0, 1698, 1699, 1700, 1701, 1702, 1703, 0, - 0, 0, 0, 3656, 1694, 0, 0, 1695, 1696, 1697, - 0, 1698, 1699, 1700, 1701, 1702, 1703, 0, 1683, 0, - 0, 3678, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1684, 0, 0, 0, 1694, - 1685, 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, - 1702, 1703, 0, 0, 1867, 0, 0, 0, 1669, 0, - 0, 1670, 0, 1686, 1687, 1671, 1672, 1673, 1674, 0, - 1675, 1676, 1677, 0, 0, 0, 0, 0, 0, 1688, - 0, 1694, 0, 0, 1695, 1696, 1697, 1678, 1698, 1699, - 1700, 1701, 1702, 1703, 0, 0, 2981, 1680, 0, 0, - 0, 0, 0, 0, 1681, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1689, - 0, 0, 1690, 0, 0, 0, 0, 0, 0, 0, - 0, 1682, 0, 0, 0, 0, 1691, 0, 0, 1692, + 528, 529, 530, 531, 0, 539, 0, 2139, 2140, 2141, + 2133, 3140, 3141, 2144, 2145, 2146, 2147, 2134, 2135, 0, + 0, 0, 2136, 2137, 2138, 122, 123, 124, 125, 126, + 127, 128, 129, 0, 130, 131, 132, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 134, 135, 0, 0, + 137, 138, 0, 140, 141, 142, 143, 144, 0, 146, + 147, 0, 0, 148, 149, 150, 151, 152, 0, 0, + 153, 154, 155, 156, 157, 158, 159, 0, 160, 161, + 162, 163, 164, 0, 0, 0, 166, 167, 168, 169, + 170, 171, 0, 173, 174, 175, 0, 176, 177, 178, + 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 0, 198, 0, 199, 200, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 0, 0, 209, 210, 211, + 212, 213, 0, 214, 215, 216, 0, 217, 218, 219, + 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 0, 233, 0, 234, 235, 236, + 237, 0, 238, 0, 239, 0, 0, 0, 242, 243, + 540, 0, 246, 247, 248, 0, 249, 250, 251, 252, + 0, 253, 254, 255, 256, 257, 258, 259, 0, 261, + 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, + 271, 0, 272, 0, 274, 275, 276, 277, 278, 279, + 280, 281, 0, 282, 0, 283, 0, 0, 286, 0, + 288, 289, 290, 0, 291, 292, 293, 0, 0, 294, + 0, 296, 0, 0, 298, 299, 300, 301, 302, 303, + 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, 328, 329, 0, 331, 332, 333, + 334, 335, 0, 336, 337, 0, 339, 0, 340, 341, + 342, 343, 344, 345, 0, 346, 347, 0, 0, 348, + 349, 350, 0, 0, 351, 352, 353, 0, 355, 0, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + 367, 368, 369, 0, 0, 0, 0, 370, 371, 372, + 0, 374, 375, 376, 377, 378, 379, 0, 380, 381, + 382, 383, 384, 385, 0, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 0, + 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 0, 0, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, + 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 542, 458, 459, 0, 0, 460, 461, 0, + 462, 0, 464, 465, 466, 467, 468, 469, 0, 470, + 471, 472, 0, 0, 473, 474, 475, 476, 477, 0, + 478, 479, 480, 481, 482, 483, 484, 485, 0, 0, + 486, 487, 488, 0, 0, 489, 490, 491, 492, 0, + 493, 494, 495, 496, 497, 498, 499, 500, 0, 501, + 0, 503, 504, 505, 506, 507, 508, 509, 0, 0, + 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, + 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, + 528, 529, 530, 531, 0, 0, 0, 2139, 2140, 2141, + 0, 2142, 2143, 2144, 2145, 2146, 2147, 1676, 0, 0, + 1677, 0, 0, 0, 1678, 1679, 1680, 1681, 0, 1682, + 1683, 1684, 0, 0, 0, 1676, 0, 0, 1677, 0, + 0, 0, 1678, 1679, 1680, 1681, 1685, 1682, 1683, 1684, + 0, 0, 0, 0, 0, 0, 1687, 0, 0, 0, + 0, 0, 0, 1688, 1685, 0, 0, 0, 0, 0, + 1676, 0, 0, 1677, 1687, 0, 0, 1678, 1679, 1680, + 1681, 1688, 1682, 1683, 1684, 0, 0, 0, 1676, 0, + 1689, 1677, 0, 0, 0, 1678, 1679, 1680, 1681, 1685, + 1682, 1683, 1684, 0, 0, 0, 0, 0, 1689, 1687, + 0, 0, 0, 0, 0, 0, 1688, 1685, 0, 0, + 0, 0, 0, 1676, 0, 0, 1677, 1687, 0, 0, + 1678, 1679, 1680, 1681, 1688, 1682, 1683, 1684, 0, 0, + 0, 0, 0, 1689, 0, 0, 0, 0, 0, 0, + 0, 0, 1685, 0, 0, 0, 0, 0, 0, 0, + 0, 1689, 1687, 0, 0, 1676, 0, 0, 1677, 1688, + 0, 0, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, + 0, 0, 0, 0, 0, 0, 0, 0, 1690, 0, + 0, 0, 0, 0, 1685, 0, 1689, 0, 0, 0, + 0, 0, 0, 0, 1687, 1691, 1690, 0, 0, 0, + 1692, 1688, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1691, 0, 0, 0, 0, 1692, 0, + 0, 0, 0, 1693, 1694, 0, 0, 0, 1689, 0, + 0, 1690, 0, 0, 0, 0, 0, 0, 0, 1695, + 0, 1693, 1694, 0, 0, 0, 0, 0, 1691, 1690, + 0, 0, 0, 1692, 0, 0, 0, 1695, 0, 0, + 0, 0, 0, 0, 0, 0, 1691, 0, 0, 0, + 0, 1692, 0, 0, 0, 0, 1693, 1694, 0, 1696, + 0, 0, 1697, 0, 1690, 0, 0, 0, 0, 0, + 0, 0, 1695, 0, 1693, 1694, 1698, 1696, 0, 1699, + 1697, 1691, 0, 0, 0, 0, 1692, 0, 0, 0, + 1695, 0, 0, 0, 1698, 0, 0, 1699, 0, 0, + 0, 0, 0, 0, 0, 0, 1690, 0, 0, 1693, + 1694, 0, 1696, 0, 0, 1697, 0, 0, 0, 0, + 0, 0, 0, 1691, 0, 1695, 0, 0, 1692, 1698, + 1696, 0, 1699, 1697, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1698, 0, 0, + 1699, 1693, 1694, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1696, 0, 1695, 1697, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, + 0, 0, 1698, 0, 0, 1699, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1700, 0, 0, + 0, 0, 0, 0, 0, 1676, 0, 1696, 1677, 0, + 1697, 0, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, + 0, 0, 0, 0, 1698, 0, 0, 1699, 0, 0, + 0, 0, 1700, 0, 1685, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1687, 0, 0, 0, 0, 0, + 1700, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1689, 0, + 0, 0, 0, 0, 0, 1700, 0, 0, 0, 0, + 0, 1701, 0, 0, 1702, 1703, 1704, 0, 1705, 1706, + 1707, 1708, 1709, 1710, 0, 0, 0, 0, 3478, 1701, + 0, 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, + 1709, 1710, 0, 0, 0, 0, 3506, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1701, 0, 0, 1702, 1703, 1704, + 0, 1705, 1706, 1707, 1708, 1709, 1710, 0, 0, 0, + 0, 3610, 1701, 0, 0, 1702, 1703, 1704, 0, 1705, + 1706, 1707, 1708, 1709, 1710, 0, 1690, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1691, 0, 0, 0, 1701, 1692, 0, + 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, + 0, 0, 0, 0, 3692, 0, 1676, 0, 0, 1677, + 0, 1693, 1694, 1678, 1679, 1680, 1681, 0, 1682, 1683, + 1684, 0, 0, 0, 0, 0, 0, 1695, 0, 1701, + 0, 0, 1702, 1703, 1704, 1685, 1705, 1706, 1707, 1708, + 1709, 1710, 0, 0, 2994, 1687, 0, 0, 0, 0, + 0, 0, 1688, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1696, 0, 0, + 1697, 0, 0, 0, 0, 0, 0, 0, 0, 1689, + 0, 0, 0, 0, 1698, 0, 0, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1683, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, - 0, 0, 0, 0, 0, 0, 1684, 0, 0, 0, - 0, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1686, 1687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1690, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1700, 0, 0, + 0, 0, 0, 0, 1691, 0, 0, 0, 0, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1689, 0, 0, 1690, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1691, 0, 0, - 1692, 1694, 0, 0, 1695, 1696, 1697, 0, 1698, 1699, - 1700, 1701, 1702, 1703, 0, 0, 3454, 0, 0, 0, + 0, 0, 1693, 1694, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1696, 0, + 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1698, 0, 0, 1699, 1701, + 0, 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, + 1709, 1710, 0, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1700, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9092,4369 +9095,4314 @@ static const yytype_int16 yytable[] = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, - 0, 0, 1694, 0, 0, 1695, 1696, 1697, 0, 1698, - 1699, 1700, 1701, 1702, 1703, 0, 0, 3636, 120, 121, - 122, 123, 124, 125, 126, 127, 569, 128, 129, 130, - 570, 571, 572, 573, 574, 575, 576, 577, 578, 132, - 133, 579, 580, 135, 136, 581, 138, 139, 140, 582, - 583, 584, 585, 586, 587, 588, 146, 147, 148, 149, - 150, 589, 590, 151, 152, 153, 154, 591, 592, 157, - 593, 158, 159, 160, 161, 594, 595, 596, 597, 598, - 165, 166, 167, 168, 169, 599, 171, 172, 173, 600, - 174, 175, 176, 177, 178, 179, 601, 602, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 604, 192, - 193, 605, 195, 606, 196, 607, 197, 198, 199, 200, - 201, 202, 608, 609, 203, 204, 205, 206, 610, 611, - 207, 208, 209, 210, 211, 612, 212, 213, 214, 613, - 215, 216, 217, 218, 614, 219, 220, 221, 222, 615, - 224, 225, 226, 227, 228, 229, 616, 617, 231, 618, - 232, 233, 619, 235, 620, 236, 621, 237, 622, 623, - 624, 240, 241, 625, 626, 244, 245, 246, 627, 628, - 629, 249, 250, 630, 251, 252, 253, 254, 255, 256, - 257, 631, 259, 260, 261, 262, 632, 263, 264, 265, - 266, 267, 268, 269, 633, 270, 634, 635, 273, 274, - 275, 276, 277, 636, 637, 638, 639, 640, 281, 641, - 642, 284, 643, 286, 287, 288, 644, 289, 290, 291, - 645, 646, 292, 647, 294, 648, 649, 296, 297, 298, - 299, 300, 301, 302, 303, 650, 305, 306, 307, 308, - 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 651, 652, - 653, 330, 331, 332, 654, 655, 334, 335, 656, 337, - 657, 658, 339, 659, 341, 342, 343, 660, 344, 345, - 661, 662, 346, 347, 348, 663, 664, 349, 350, 665, - 666, 353, 667, 668, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 669, 670, 671, 672, - 368, 369, 673, 674, 372, 373, 675, 375, 376, 377, - 676, 378, 379, 380, 381, 382, 383, 677, 384, 385, - 386, 387, 388, 678, 390, 391, 392, 393, 679, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 680, 407, 408, 681, 410, 411, 412, 682, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 683, 684, 427, 428, 429, 430, 431, - 432, 685, 434, 435, 686, 687, 437, 438, 688, 440, - 689, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 690, 456, 691, 692, 693, - 458, 459, 694, 460, 695, 462, 463, 464, 465, 466, - 467, 696, 468, 697, 698, 699, 700, 471, 472, 701, - 474, 702, 703, 476, 477, 704, 479, 480, 481, 482, - 483, 705, 706, 484, 485, 486, 707, 708, 487, 488, - 489, 490, 709, 491, 492, 493, 494, 495, 710, 711, - 498, 712, 499, 713, 501, 502, 503, 504, 505, 506, - 507, 714, 715, 508, 716, 717, 509, 510, 511, 512, - 513, 514, 718, 719, 720, 721, 722, 723, 724, 725, - 726, 727, 728, 526, 527, 528, 529, 537, 0, 0, - 0, 0, 0, 0, 0, 0, 2162, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 120, 121, 122, - 123, 124, 125, 126, 127, 0, 128, 129, 130, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 132, 133, - 0, 0, 135, 136, 0, 138, 139, 140, 141, 142, - 0, 144, 145, 0, 0, 146, 147, 148, 149, 150, - 0, 0, 151, 152, 153, 154, 155, 156, 157, 0, - 158, 159, 160, 161, 162, 0, 0, 0, 164, 165, - 166, 167, 168, 169, 0, 171, 172, 173, 0, 174, - 175, 176, 177, 178, 179, 0, 0, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 0, 196, 0, 197, 198, 199, 200, 201, - 202, 0, 0, 203, 204, 205, 206, 0, 0, 207, - 208, 209, 210, 211, 0, 212, 213, 214, 0, 215, - 216, 217, 218, 0, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 0, 231, 0, 232, - 233, 234, 235, 0, 236, 0, 237, 0, 0, 0, - 240, 241, 538, 0, 244, 245, 246, 0, 247, 248, - 249, 250, 0, 251, 252, 253, 254, 255, 256, 257, - 0, 259, 260, 261, 262, 0, 263, 264, 265, 266, - 267, 268, 269, 0, 270, 0, 272, 273, 274, 275, - 276, 277, 278, 279, 0, 280, 0, 281, 0, 0, - 284, 0, 286, 287, 288, 0, 289, 290, 291, 0, - 0, 292, 0, 294, 0, 0, 296, 297, 298, 299, - 300, 301, 302, 303, 539, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, 0, 329, - 330, 331, 332, 333, 0, 334, 335, 0, 337, 0, - 338, 339, 340, 341, 342, 343, 0, 344, 345, 0, - 0, 346, 347, 348, 0, 0, 349, 350, 351, 0, - 353, 0, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 0, 0, 0, 0, 368, - 369, 370, 0, 372, 373, 374, 375, 376, 377, 0, - 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, - 387, 388, 389, 390, 391, 392, 393, 0, 394, 395, - 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 0, 407, 408, 0, 410, 411, 412, 413, 414, - 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 0, 0, 427, 428, 429, 430, 431, 432, - 433, 434, 435, 0, 0, 437, 438, 439, 440, 0, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 540, 456, 457, 0, 0, 458, - 459, 0, 460, 0, 462, 463, 464, 465, 466, 467, - 0, 468, 469, 470, 0, 0, 471, 472, 473, 474, - 475, 0, 476, 477, 478, 479, 480, 481, 482, 483, - 0, 0, 484, 485, 486, 0, 0, 487, 488, 489, - 490, 0, 491, 492, 493, 494, 495, 496, 497, 498, - 0, 499, 0, 501, 502, 503, 504, 505, 506, 507, - 0, 0, 508, 0, 0, 509, 510, 511, 512, 513, - 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, - 524, 525, 526, 527, 528, 529, 537, 0, 0, 0, - 0, 0, 0, 0, 0, 2873, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 132, 133, 0, - 0, 135, 136, 0, 138, 139, 140, 141, 142, 0, - 144, 145, 0, 0, 146, 147, 148, 149, 150, 0, - 0, 151, 152, 153, 154, 155, 156, 157, 0, 158, - 159, 160, 161, 162, 0, 0, 0, 164, 165, 166, - 167, 168, 169, 0, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 0, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 209, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, 0, 231, 0, 232, 233, - 234, 235, 0, 236, 0, 237, 0, 0, 0, 240, - 241, 538, 0, 244, 245, 246, 0, 247, 248, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 0, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 0, 272, 273, 274, 275, 276, - 277, 278, 279, 0, 280, 0, 281, 0, 0, 284, - 0, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 0, 294, 0, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 539, 305, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 0, 329, 330, - 331, 332, 333, 0, 334, 335, 0, 337, 0, 338, - 339, 340, 341, 342, 343, 0, 344, 345, 0, 0, - 346, 347, 348, 0, 0, 349, 350, 351, 0, 353, - 0, 355, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 370, 0, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 0, 394, 395, 396, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 0, 410, 411, 412, 413, 414, 415, - 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 0, 437, 438, 439, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 540, 456, 457, 0, 0, 458, 459, - 0, 460, 0, 462, 463, 464, 465, 466, 467, 0, - 468, 469, 470, 0, 0, 471, 472, 473, 474, 475, - 0, 476, 477, 478, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 497, 498, 0, - 499, 0, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, - 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 989, 1388, 833, 0, 0, - 0, 1075, 0, 0, 2876, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 120, 121, 122, 123, 124, - 125, 126, 127, 0, 128, 129, 130, 0, 0, 0, - 573, 0, 0, 0, 0, 578, 132, 133, 0, 580, - 135, 136, 581, 138, 139, 140, 582, 583, 584, 585, - 586, 0, 588, 146, 147, 148, 149, 150, 0, 0, - 151, 152, 153, 154, 591, 592, 157, 0, 158, 159, - 160, 161, 594, 0, 596, 0, 598, 165, 166, 167, - 168, 169, 599, 171, 172, 173, 0, 174, 175, 176, - 177, 178, 179, 0, 602, 181, 182, 183, 184, 185, - 186, 187, 188, 189, 190, 604, 192, 193, 605, 195, - 0, 196, 0, 197, 198, 199, 200, 201, 202, 0, - 0, 203, 204, 205, 206, 0, 0, 207, 208, 209, - 210, 211, 0, 212, 213, 214, 0, 215, 216, 217, - 218, 0, 219, 220, 221, 222, 615, 224, 225, 226, - 227, 228, 229, 616, 1389, 231, 0, 232, 233, 619, - 235, 0, 236, 0, 237, 622, 0, 624, 240, 241, - 625, 626, 244, 245, 246, 0, 628, 629, 249, 250, - 0, 251, 252, 253, 254, 255, 256, 257, 631, 259, - 260, 261, 262, 0, 263, 264, 265, 266, 267, 268, - 269, 0, 270, 634, 635, 273, 274, 275, 276, 277, - 636, 637, 0, 639, 0, 281, 641, 642, 284, 643, - 286, 287, 288, 0, 289, 290, 291, 0, 0, 292, - 647, 294, 648, 0, 296, 297, 298, 299, 300, 301, - 302, 303, 650, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 651, 652, 653, 330, 331, - 332, 654, 0, 334, 335, 656, 337, 0, 658, 339, - 659, 341, 342, 343, 0, 344, 345, 1390, 0, 346, - 347, 348, 0, 0, 349, 350, 665, 666, 353, 667, - 668, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 0, 0, 0, 0, 368, 369, 673, - 674, 372, 373, 675, 375, 376, 377, 0, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, - 678, 390, 391, 392, 393, 0, 394, 395, 396, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 0, - 407, 408, 681, 410, 411, 412, 682, 414, 415, 416, - 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 0, 684, 427, 428, 429, 430, 431, 432, 685, 434, - 435, 0, 687, 437, 438, 688, 440, 0, 441, 442, - 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, - 453, 454, 690, 456, 691, 0, 0, 458, 459, 0, - 460, 695, 462, 463, 464, 465, 466, 467, 0, 468, - 697, 698, 0, 0, 471, 472, 701, 474, 702, 1391, - 476, 477, 704, 479, 480, 481, 482, 483, 0, 0, - 484, 485, 486, 707, 0, 487, 488, 489, 490, 0, - 491, 492, 493, 494, 495, 710, 711, 498, 0, 499, - 713, 501, 502, 503, 504, 505, 506, 507, 0, 0, - 508, 0, 0, 509, 510, 511, 512, 513, 514, 718, - 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, - 526, 527, 528, 529, 0, 0, 1669, 0, 0, 1670, - 0, 1392, 1393, 1671, 1672, 1673, 1674, 0, 1675, 1676, - 1677, 0, 0, 0, 1669, 0, 0, 1670, 0, 0, - 0, 1671, 1672, 1673, 1674, 1678, 1675, 1676, 1677, 0, - 2262, 0, 0, 0, 0, 1680, 0, 0, 0, 0, - 0, 0, 1681, 1678, 0, 0, 0, 0, 0, 1669, - 0, 0, 1670, 1680, 0, 0, 1671, 1672, 1673, 1674, - 1681, 1675, 1676, 1677, 0, 0, 0, 1669, 0, 1682, - 1670, 0, 0, 0, 1671, 1672, 1673, 1674, 1678, 1675, - 1676, 1677, 0, 0, 0, 0, 0, 1682, 1680, 0, - 0, 0, 0, 0, 0, 1681, 1678, 0, 0, 0, - 1970, 0, 0, 0, 0, 0, 1680, 0, 0, 0, - 0, 0, 0, 1681, 0, 2263, 0, 0, 0, 0, - 0, 0, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1682, 0, 0, 0, 1669, 0, 0, 1670, 0, 0, - 0, 1671, 1672, 1673, 1674, 0, 1675, 1676, 1677, 0, - 0, 0, 0, 0, 0, 0, 0, 1683, 0, 0, - 0, 0, 0, 1678, 0, 2006, 0, 0, 0, 0, - 2007, 0, 0, 1680, 1684, 1683, 0, 0, 0, 1685, - 1681, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1684, 0, 0, 0, 0, 1685, 0, 3765, - 0, 0, 1686, 1687, 0, 0, 0, 1682, 0, 0, - 1683, 0, 0, 0, 0, 0, 0, 0, 1688, 0, - 1686, 1687, 0, 0, 0, 0, 0, 1684, 1683, 0, - 0, 0, 1685, 0, 0, 0, 1688, 0, 0, 0, - 0, 0, 0, 0, 0, 1684, 0, 0, 0, 0, - 1685, 0, 0, 0, 0, 1686, 1687, 0, 1689, 0, - 0, 1690, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1688, 0, 1686, 1687, 1691, 1689, 0, 1692, 1690, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1688, - 0, 0, 0, 1691, 0, 0, 1692, 0, 0, 0, - 0, 0, 0, 0, 0, 1683, 0, 0, 0, 0, - 0, 1689, 0, 0, 1690, 0, 0, 0, 0, 0, - 0, 0, 1684, 0, 0, 0, 0, 1685, 1691, 1689, - 0, 1692, 1690, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1691, 0, 0, 1692, - 1686, 1687, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3766, 0, 0, 0, 0, 1688, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1669, 0, 0, 1670, 0, 1693, 0, 1671, 1672, - 1673, 1674, 0, 1675, 1676, 1677, 1689, 0, 0, 1690, + 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, + 1701, 0, 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, + 1708, 1709, 1710, 0, 0, 3650, 122, 123, 124, 125, + 126, 127, 128, 129, 571, 130, 131, 132, 572, 573, + 574, 575, 576, 577, 578, 579, 580, 134, 135, 581, + 582, 137, 138, 583, 140, 141, 142, 584, 585, 586, + 587, 588, 589, 590, 148, 149, 150, 151, 152, 591, + 592, 153, 154, 155, 156, 593, 594, 159, 595, 160, + 161, 162, 163, 596, 597, 598, 599, 600, 167, 168, + 169, 170, 171, 601, 173, 174, 175, 602, 176, 177, + 178, 179, 180, 181, 603, 604, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 606, 194, 195, 607, + 197, 608, 198, 609, 199, 200, 201, 202, 203, 204, + 610, 611, 205, 206, 207, 208, 612, 613, 209, 210, + 211, 212, 213, 614, 214, 215, 216, 615, 217, 218, + 219, 220, 616, 221, 222, 223, 224, 617, 226, 227, + 228, 229, 230, 231, 618, 619, 233, 620, 234, 235, + 621, 237, 622, 238, 623, 239, 624, 625, 626, 242, + 243, 627, 628, 246, 247, 248, 629, 630, 631, 251, + 252, 632, 253, 254, 255, 256, 257, 258, 259, 633, + 261, 262, 263, 264, 634, 265, 266, 267, 268, 269, + 270, 271, 635, 272, 636, 637, 275, 276, 277, 278, + 279, 638, 639, 640, 641, 642, 283, 643, 644, 286, + 645, 288, 289, 290, 646, 291, 292, 293, 647, 648, + 294, 649, 296, 650, 651, 298, 299, 300, 301, 302, + 303, 304, 305, 652, 307, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, + 323, 324, 325, 326, 327, 328, 653, 654, 655, 332, + 333, 334, 656, 657, 336, 337, 658, 339, 659, 660, + 341, 661, 343, 344, 345, 662, 346, 347, 663, 664, + 348, 349, 350, 665, 666, 351, 352, 667, 668, 355, + 669, 670, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 671, 672, 673, 674, 370, 371, + 675, 676, 374, 375, 677, 377, 378, 379, 678, 380, + 381, 382, 383, 384, 385, 679, 386, 387, 388, 389, + 390, 680, 392, 393, 394, 395, 681, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 682, 409, 410, 683, 412, 413, 414, 684, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 685, 686, 429, 430, 431, 432, 433, 434, 687, + 436, 437, 688, 689, 439, 440, 690, 442, 691, 443, + 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 692, 458, 693, 694, 695, 460, 461, + 696, 462, 697, 464, 465, 466, 467, 468, 469, 698, + 470, 699, 700, 701, 702, 473, 474, 703, 476, 704, + 705, 478, 479, 706, 481, 482, 483, 484, 485, 707, + 708, 486, 487, 488, 709, 710, 489, 490, 491, 492, + 711, 493, 494, 495, 496, 497, 712, 713, 500, 714, + 501, 715, 503, 504, 505, 506, 507, 508, 509, 716, + 717, 510, 718, 719, 511, 512, 513, 514, 515, 516, + 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, + 730, 528, 529, 530, 531, 539, 0, 0, 0, 0, + 0, 0, 0, 0, 2172, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 122, 123, 124, 125, 126, + 127, 128, 129, 0, 130, 131, 132, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 134, 135, 0, 0, + 137, 138, 0, 140, 141, 142, 143, 144, 0, 146, + 147, 0, 0, 148, 149, 150, 151, 152, 0, 0, + 153, 154, 155, 156, 157, 158, 159, 0, 160, 161, + 162, 163, 164, 0, 0, 0, 166, 167, 168, 169, + 170, 171, 0, 173, 174, 175, 0, 176, 177, 178, + 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 0, 198, 0, 199, 200, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 0, 0, 209, 210, 211, + 212, 213, 0, 214, 215, 216, 0, 217, 218, 219, + 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 0, 233, 0, 234, 235, 236, + 237, 0, 238, 0, 239, 0, 0, 0, 242, 243, + 540, 0, 246, 247, 248, 0, 249, 250, 251, 252, + 0, 253, 254, 255, 256, 257, 258, 259, 0, 261, + 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, + 271, 0, 272, 0, 274, 275, 276, 277, 278, 279, + 280, 281, 0, 282, 0, 283, 0, 0, 286, 0, + 288, 289, 290, 0, 291, 292, 293, 0, 0, 294, + 0, 296, 0, 0, 298, 299, 300, 301, 302, 303, + 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, 328, 329, 0, 331, 332, 333, + 334, 335, 0, 336, 337, 0, 339, 0, 340, 341, + 342, 343, 344, 345, 0, 346, 347, 0, 0, 348, + 349, 350, 0, 0, 351, 352, 353, 0, 355, 0, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + 367, 368, 369, 0, 0, 0, 0, 370, 371, 372, + 0, 374, 375, 376, 377, 378, 379, 0, 380, 381, + 382, 383, 384, 385, 0, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 0, + 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 0, 0, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, + 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 542, 458, 459, 0, 0, 460, 461, 0, + 462, 0, 464, 465, 466, 467, 468, 469, 0, 470, + 471, 472, 0, 0, 473, 474, 475, 476, 477, 0, + 478, 479, 480, 481, 482, 483, 484, 485, 0, 0, + 486, 487, 488, 0, 0, 489, 490, 491, 492, 0, + 493, 494, 495, 496, 497, 498, 499, 500, 0, 501, + 0, 503, 504, 505, 506, 507, 508, 509, 0, 0, + 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, + 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, + 528, 529, 530, 531, 539, 0, 0, 0, 0, 0, + 0, 0, 0, 2886, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 134, 135, 0, 0, 137, + 138, 0, 140, 141, 142, 143, 144, 0, 146, 147, + 0, 0, 148, 149, 150, 151, 152, 0, 0, 153, + 154, 155, 156, 157, 158, 159, 0, 160, 161, 162, + 163, 164, 0, 0, 0, 166, 167, 168, 169, 170, + 171, 0, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 211, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 232, 0, 233, 0, 234, 235, 236, 237, + 0, 238, 0, 239, 0, 0, 0, 242, 243, 540, + 0, 246, 247, 248, 0, 249, 250, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 0, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 0, 274, 275, 276, 277, 278, 279, 280, + 281, 0, 282, 0, 283, 0, 0, 286, 0, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 0, + 296, 0, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, 328, 329, 0, 331, 332, 333, 334, + 335, 0, 336, 337, 0, 339, 0, 340, 341, 342, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, + 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, + 368, 369, 0, 0, 0, 0, 370, 371, 372, 0, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 542, 458, 459, 0, 0, 460, 461, 0, 462, + 0, 464, 465, 466, 467, 468, 469, 0, 470, 471, + 472, 0, 0, 473, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 499, 500, 0, 501, 0, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, + 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, + 529, 530, 531, 994, 1395, 836, 0, 0, 0, 1080, + 0, 0, 2889, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 122, 123, 124, 125, 126, 127, 128, + 129, 0, 130, 131, 132, 0, 0, 0, 575, 0, + 0, 0, 0, 580, 134, 135, 0, 582, 137, 138, + 583, 140, 141, 142, 584, 585, 586, 587, 588, 0, + 590, 148, 149, 150, 151, 152, 0, 0, 153, 154, + 155, 156, 593, 594, 159, 0, 160, 161, 162, 163, + 596, 0, 598, 0, 600, 167, 168, 169, 170, 171, + 601, 173, 174, 175, 0, 176, 177, 178, 179, 180, + 181, 0, 604, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 606, 194, 195, 607, 197, 0, 198, + 0, 199, 200, 201, 202, 203, 204, 0, 0, 205, + 206, 207, 208, 0, 0, 209, 210, 211, 212, 213, + 0, 214, 215, 216, 0, 217, 218, 219, 220, 0, + 221, 222, 223, 224, 617, 226, 227, 228, 229, 230, + 231, 618, 1396, 233, 0, 234, 235, 621, 237, 0, + 238, 0, 239, 624, 0, 626, 242, 243, 627, 628, + 246, 247, 248, 0, 630, 631, 251, 252, 0, 253, + 254, 255, 256, 257, 258, 259, 633, 261, 262, 263, + 264, 0, 265, 266, 267, 268, 269, 270, 271, 0, + 272, 636, 637, 275, 276, 277, 278, 279, 638, 639, + 0, 641, 0, 283, 643, 644, 286, 645, 288, 289, + 290, 0, 291, 292, 293, 0, 0, 294, 649, 296, + 650, 0, 298, 299, 300, 301, 302, 303, 304, 305, + 652, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, + 326, 327, 328, 653, 654, 655, 332, 333, 334, 656, + 0, 336, 337, 658, 339, 0, 660, 341, 661, 343, + 344, 345, 0, 346, 347, 1397, 0, 348, 349, 350, + 0, 0, 351, 352, 667, 668, 355, 669, 670, 358, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 0, 0, 0, 0, 370, 371, 675, 676, 374, + 375, 677, 377, 378, 379, 0, 380, 381, 382, 383, + 384, 385, 0, 386, 387, 388, 389, 390, 680, 392, + 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 0, 409, 410, + 683, 412, 413, 414, 684, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 426, 427, 428, 0, 686, + 429, 430, 431, 432, 433, 434, 687, 436, 437, 0, + 689, 439, 440, 690, 442, 0, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 692, 458, 693, 0, 0, 460, 461, 0, 462, 697, + 464, 465, 466, 467, 468, 469, 0, 470, 699, 700, + 0, 0, 473, 474, 703, 476, 704, 1398, 478, 479, + 706, 481, 482, 483, 484, 485, 0, 0, 486, 487, + 488, 709, 0, 489, 490, 491, 492, 0, 493, 494, + 495, 496, 497, 712, 713, 500, 0, 501, 715, 503, + 504, 505, 506, 507, 508, 509, 0, 0, 510, 0, + 0, 511, 512, 513, 514, 515, 516, 720, 721, 722, + 723, 724, 725, 726, 727, 728, 729, 730, 528, 529, + 530, 531, 0, 0, 1676, 0, 0, 1677, 0, 1399, + 1400, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, 0, + 0, 0, 1676, 0, 0, 1677, 0, 0, 0, 1678, + 1679, 1680, 1681, 1685, 1682, 1683, 1684, 0, 2272, 0, + 0, 0, 0, 1687, 0, 0, 0, 0, 0, 0, + 1688, 1685, 0, 0, 0, 0, 0, 1676, 0, 0, + 1677, 1687, 0, 0, 1678, 1679, 1680, 1681, 1688, 1682, + 1683, 1684, 0, 0, 0, 1676, 0, 1689, 1677, 0, + 0, 0, 1678, 1679, 1680, 1681, 1685, 1682, 1683, 1684, + 0, 0, 0, 0, 0, 1689, 1687, 0, 0, 0, + 0, 0, 0, 1688, 1685, 0, 0, 0, 1979, 0, + 0, 0, 0, 0, 1687, 0, 0, 0, 0, 0, + 0, 1688, 0, 2273, 0, 0, 0, 0, 0, 0, + 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1689, 0, + 0, 0, 1676, 0, 0, 1677, 0, 0, 0, 1678, + 1679, 1680, 1681, 0, 1682, 1683, 1684, 0, 0, 0, + 0, 0, 0, 0, 0, 1690, 0, 0, 0, 0, + 0, 1685, 0, 2015, 0, 0, 0, 0, 2016, 0, + 0, 1687, 1691, 1690, 0, 0, 0, 1692, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1678, 0, 0, 1691, 2012, 0, 1692, 0, 0, 0, - 1680, 1693, 0, 0, 0, 0, 0, 1681, 0, 0, - 0, 0, 0, 0, 0, 2266, 0, 0, 0, 1693, - 1977, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1682, 0, 0, 0, 0, 0, + 1691, 0, 0, 0, 0, 1692, 0, 3779, 0, 0, + 1693, 1694, 0, 0, 0, 1689, 0, 0, 1690, 0, + 0, 0, 0, 0, 0, 0, 1695, 0, 1693, 1694, + 0, 0, 0, 0, 0, 1691, 1690, 0, 0, 0, + 1692, 0, 0, 0, 1695, 0, 0, 0, 0, 0, + 0, 0, 0, 1691, 0, 0, 0, 0, 1692, 0, + 0, 0, 0, 1693, 1694, 0, 1696, 0, 0, 1697, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, + 0, 1693, 1694, 1698, 1696, 0, 1699, 1697, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, + 0, 1698, 0, 0, 1699, 0, 0, 0, 0, 0, + 0, 0, 0, 1690, 0, 0, 0, 0, 0, 1696, + 0, 0, 1697, 0, 0, 0, 0, 0, 0, 0, + 1691, 0, 0, 0, 0, 1692, 1698, 1696, 0, 1699, + 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1698, 0, 0, 1699, 1693, 1694, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, + 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1700, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1676, + 0, 0, 1677, 0, 1700, 0, 1678, 1679, 1680, 1681, + 0, 1682, 1683, 1684, 1696, 0, 0, 1697, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, + 0, 1698, 2021, 0, 1699, 0, 0, 0, 1687, 1700, + 0, 0, 0, 0, 0, 1688, 0, 0, 0, 0, + 0, 0, 0, 2276, 0, 0, 0, 1700, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1694, 0, 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, - 1701, 1702, 1703, 0, 0, 0, 0, 0, 1694, 0, - 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, 1702, - 1703, 0, 0, 0, 0, 0, 1693, 0, 1669, 0, - 0, 1670, 0, 0, 0, 1671, 1672, 1673, 1674, 0, - 1675, 1676, 1677, 1694, 0, 0, 1695, 1696, 1697, 0, - 1698, 1699, 1700, 1701, 1702, 1703, 0, 1678, 0, 0, - 0, 1694, 1683, 0, 1695, 1696, 1697, 1680, 1698, 1699, - 1700, 1701, 1702, 1703, 1681, 0, 0, 0, 0, 1684, - 1669, 0, 0, 1670, 1685, 0, 0, 1671, 1672, 1673, - 1674, 0, 1675, 1676, 1677, 0, 0, 0, 0, 0, - 0, 1682, 0, 0, 0, 0, 0, 1686, 1687, 1678, - 0, 0, 0, 2019, 0, 0, 0, 0, 0, 1680, - 0, 0, 0, 1688, 0, 0, 1681, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1694, 0, - 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, 1702, - 1703, 0, 0, 1682, 0, 0, 0, 0, 0, 0, - 0, 1669, 0, 1689, 1670, 0, 1690, 0, 1671, 1672, - 1673, 1674, 0, 1675, 1676, 1677, 0, 0, 0, 0, - 1691, 0, 0, 1692, 0, 0, 0, 0, 0, 0, - 1678, 0, 0, 0, 2017, 0, 0, 0, 0, 1683, - 1680, 0, 0, 0, 0, 0, 0, 1681, 0, 0, - 0, 0, 0, 0, 0, 0, 1684, 0, 0, 0, - 0, 1685, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1682, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1686, 1687, 0, 0, 0, 0, - 0, 1683, 0, 0, 0, 0, 0, 0, 0, 0, - 1688, 0, 0, 0, 0, 0, 0, 0, 1684, 0, - 0, 0, 0, 1685, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1693, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1686, 1687, 0, 0, - 1689, 0, 0, 1690, 0, 0, 0, 0, 0, 0, - 0, 0, 1688, 0, 0, 0, 0, 1691, 0, 0, - 1692, 0, 2154, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1683, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1684, - 0, 0, 1689, 0, 1685, 1690, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1691, - 0, 0, 1692, 0, 0, 0, 0, 1686, 1687, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1701, 0, + 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, + 1710, 0, 0, 0, 0, 0, 1701, 0, 0, 1702, + 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, 0, + 0, 0, 0, 0, 1700, 0, 1676, 0, 0, 1677, + 0, 0, 0, 1678, 1679, 1680, 1681, 0, 1682, 1683, + 1684, 1701, 0, 0, 1702, 1703, 1704, 0, 1705, 1706, + 1707, 1708, 1709, 1710, 0, 1685, 0, 0, 0, 1701, + 1690, 0, 1702, 1703, 1704, 1687, 1705, 1706, 1707, 1708, + 1709, 1710, 1688, 0, 0, 0, 0, 1691, 1676, 0, + 0, 1677, 1692, 0, 0, 1678, 1679, 1680, 1681, 0, + 1682, 1683, 1684, 0, 0, 0, 0, 0, 0, 1689, + 0, 0, 0, 0, 0, 1693, 1694, 1685, 0, 0, + 0, 2028, 0, 0, 0, 0, 0, 1687, 0, 0, + 0, 1695, 0, 0, 1688, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1701, 0, 0, 1702, + 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, 0, + 0, 1689, 0, 0, 0, 0, 0, 0, 0, 1676, + 0, 1696, 1677, 0, 1697, 0, 1678, 1679, 1680, 1681, + 0, 1682, 1683, 1684, 0, 0, 0, 0, 1698, 0, + 0, 1699, 0, 0, 0, 0, 0, 0, 1685, 0, + 0, 0, 2026, 0, 0, 0, 0, 1690, 1687, 0, + 0, 0, 0, 0, 0, 1688, 0, 0, 0, 0, + 0, 0, 0, 0, 1691, 0, 0, 0, 0, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1688, 0, 1694, 0, 0, 1695, 1696, - 1697, 0, 1698, 1699, 1700, 1701, 1702, 1703, 0, 0, - 0, 0, 0, 0, 0, 1669, 0, 0, 1670, 0, - 1693, 0, 1671, 1672, 1673, 1674, 2625, 1675, 1676, 1677, - 0, 0, 0, 1689, 0, 0, 1690, 0, 0, 0, - 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, - 1691, 0, 0, 1692, 1680, 0, 0, 0, 0, 0, - 0, 1681, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1693, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1682, 0, - 0, 0, 1669, 0, 0, 1670, 0, 0, 0, 1671, - 1672, 1673, 1674, 0, 1675, 1676, 1677, 0, 0, 0, + 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1693, 1694, 0, 0, 0, 0, 0, 1690, + 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, + 0, 0, 0, 0, 0, 0, 1691, 0, 0, 0, + 0, 1692, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1693, 1694, 0, 0, 1696, 0, + 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, + 1695, 0, 0, 0, 0, 1698, 0, 0, 1699, 0, + 2164, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1691, 0, 0, + 1696, 0, 1692, 1697, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1698, 0, 0, + 1699, 0, 0, 0, 0, 1693, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1678, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1680, 1694, 0, 0, 1695, 1696, 1697, 1681, 1698, - 1699, 1700, 1701, 1702, 1703, 0, 0, 0, 0, 0, - 0, 0, 0, 1693, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1682, 0, 0, 0, 0, + 0, 1695, 0, 1701, 0, 0, 1702, 1703, 1704, 0, + 1705, 1706, 1707, 1708, 1709, 1710, 0, 0, 0, 0, + 0, 0, 0, 1676, 0, 0, 1677, 0, 1700, 0, + 1678, 1679, 1680, 1681, 2637, 1682, 1683, 1684, 0, 0, + 0, 1696, 0, 0, 1697, 0, 0, 0, 0, 0, + 0, 0, 1685, 0, 0, 0, 0, 0, 1698, 0, + 0, 1699, 1687, 0, 0, 0, 0, 0, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1694, 0, 1683, 1695, 1696, 1697, - 0, 1698, 1699, 1700, 1701, 1702, 1703, 0, 0, 0, - 0, 0, 0, 1684, 0, 0, 1669, 0, 1685, 1670, - 0, 0, 0, 1671, 1672, 1673, 1674, 0, 1675, 1676, - 1677, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1686, 1687, 0, 0, 1678, 0, 0, 0, 2955, - 0, 0, 0, 0, 0, 1680, 0, 1688, 0, 0, - 0, 0, 1681, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1683, 0, 1694, 0, 0, 1695, 1696, - 1697, 0, 1698, 1699, 1700, 1701, 1702, 1703, 0, 1682, - 1684, 0, 0, 0, 0, 1685, 1669, 1689, 0, 1670, - 1690, 0, 0, 1671, 1672, 1673, 1674, 0, 1675, 1676, - 1677, 0, 0, 0, 1691, 0, 0, 1692, 1686, 1687, - 0, 0, 0, 0, 0, 1678, 0, 0, 0, 0, - 0, 0, 0, 0, 1688, 1680, 0, 0, 0, 0, - 0, 0, 1681, 0, 0, 0, 0, 0, 0, 0, + 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1689, 0, 0, 0, + 1676, 0, 0, 1677, 0, 0, 0, 1678, 1679, 1680, + 1681, 0, 1682, 1683, 1684, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1687, + 1701, 0, 0, 1702, 1703, 1704, 1688, 1705, 1706, 1707, + 1708, 1709, 1710, 0, 0, 0, 0, 0, 0, 0, + 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1682, - 0, 0, 0, 0, 1689, 0, 0, 1690, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1683, 0, 0, - 0, 1691, 0, 0, 1692, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1684, 0, 0, 0, 0, 1685, - 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, + 0, 0, 1701, 0, 1690, 1702, 1703, 1704, 0, 1705, + 1706, 1707, 1708, 1709, 1710, 0, 0, 0, 0, 0, + 0, 1691, 0, 0, 1676, 0, 1692, 1677, 0, 0, + 0, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, + 1694, 0, 0, 1685, 0, 0, 0, 2968, 0, 0, + 0, 0, 0, 1687, 0, 1695, 0, 0, 0, 0, + 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1690, 0, 1701, 0, 0, 1702, 1703, 1704, 0, + 1705, 1706, 1707, 1708, 1709, 1710, 0, 1689, 1691, 0, + 0, 0, 0, 1692, 1676, 1696, 0, 1677, 1697, 0, + 0, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, 0, + 0, 0, 1698, 0, 0, 1699, 1693, 1694, 0, 0, + 0, 0, 0, 1685, 0, 0, 0, 0, 0, 0, + 0, 0, 1695, 1687, 0, 0, 0, 0, 0, 0, + 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1686, 1687, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1688, 0, + 0, 0, 0, 0, 0, 0, 0, 1689, 0, 0, + 0, 0, 1696, 0, 0, 1697, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1690, 0, 0, 0, 1698, + 0, 0, 1699, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1691, 0, 0, 0, 0, 1692, 0, 0, + 0, 0, 0, 0, 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1683, 0, 0, + 1693, 1694, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1684, 0, 0, 0, 1689, 1685, - 0, 1690, 0, 0, 1693, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1691, 0, 0, 1692, 0, - 0, 0, 1686, 1687, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1688, 1694, - 2939, 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, - 1702, 1703, 0, 0, 0, 0, 0, 1669, 0, 0, - 1670, 0, 0, 0, 1671, 1672, 1673, 1674, 0, 1675, - 1676, 1677, 0, 0, 0, 0, 0, 0, 1689, 0, - 0, 1690, 0, 0, 0, 0, 1678, 0, 0, 0, - 0, 0, 0, 0, 0, 1691, 1680, 0, 1692, 0, - 0, 0, 0, 1681, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1694, 0, 1693, 1695, - 1696, 1697, 0, 1698, 1699, 1700, 1701, 1702, 1703, 0, - 1682, 0, 0, 0, 0, 1669, 0, 0, 1670, 0, - 0, 0, 1671, 1672, 1673, 1674, 0, 1675, 1676, 1677, - 0, 0, 0, 0, 1669, 0, 0, 1670, 0, 0, - 0, 1671, 1672, 0, 1678, 0, 1675, 1676, 1677, 0, - 0, 0, 0, 0, 1680, 0, 0, 0, 0, 0, - 0, 1681, 0, 1678, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1680, 0, 0, 0, 0, 1693, 0, - 1681, 0, 0, 0, 0, 0, 0, 0, 1682, 0, + 0, 0, 0, 0, 0, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1682, 1683, 0, - 1694, 0, 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, - 1701, 1702, 1703, 0, 0, 1684, 0, 0, 0, 0, - 1685, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1691, 0, 0, 0, 1696, 1692, 0, 1697, + 0, 0, 1700, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1698, 0, 0, 1699, 0, 0, 0, + 1693, 1694, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1695, 1701, 2952, 0, + 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, 1710, + 0, 0, 0, 0, 0, 1676, 0, 0, 1677, 0, + 0, 0, 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, + 0, 0, 0, 0, 0, 0, 1696, 0, 0, 1697, + 0, 0, 0, 0, 1685, 0, 0, 0, 0, 0, + 0, 0, 0, 1698, 1687, 0, 1699, 0, 0, 0, + 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1701, 0, 1700, 1702, 1703, 1704, + 0, 1705, 1706, 1707, 1708, 1709, 1710, 0, 1689, 0, + 0, 0, 0, 1676, 0, 0, 1677, 0, 0, 0, + 1678, 1679, 1680, 1681, 0, 1682, 1683, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1880, 1687, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1688, + 0, 0, 1685, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1687, 0, 0, 0, 0, 0, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1683, 0, 0, 0, - 1694, 0, 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, - 1701, 1702, 1703, 1684, 0, 1683, 0, 0, 1685, 1689, - 0, 0, 1690, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1684, 0, 0, 0, 1691, 1685, 0, 1692, - 0, 1686, 1687, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1688, 0, 0, - 1686, 1687, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1688, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1700, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1689, 0, 0, - 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1691, 0, 1689, 1692, 0, 1690, + 0, 0, 0, 0, 0, 0, 1690, 0, 1701, 0, + 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, + 1710, 0, 0, 1691, 0, 0, 0, 0, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1691, 0, 0, 1692, 0, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1889, 1694, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1690, 0, 0, 0, 1701, 0, + 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, 1709, + 1710, 1691, 0, 0, 0, 0, 1692, 1696, 0, 0, + 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1698, 0, 0, 1699, 0, 1693, + 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1696, 0, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1698, 0, 0, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1694, 0, 0, 1695, 1696, 1697, 0, 1698, 1699, - 1700, 1701, 1702, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1694, - 0, 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, - 2283, 1703, 568, 0, 2178, 0, 0, 0, 1694, 0, - 0, 1695, 1696, 1697, 0, 1698, 1699, 1700, 1701, 1702, - 1703, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 569, 128, 129, 130, 570, 571, 572, 573, 574, 575, - 576, 577, 578, 132, 133, 579, 580, 135, 136, 581, - 138, 139, 140, 582, 583, 584, 585, 586, 587, 588, - 146, 147, 148, 149, 150, 589, 590, 151, 152, 153, - 154, 591, 592, 157, 593, 158, 159, 160, 161, 594, - 595, 596, 597, 598, 165, 166, 167, 168, 169, 599, - 171, 172, 173, 600, 174, 175, 176, 177, 178, 179, - 601, 602, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 604, 192, 193, 605, 195, 606, 196, 607, - 197, 198, 199, 200, 201, 202, 608, 609, 203, 204, - 205, 206, 610, 611, 207, 208, 209, 210, 211, 612, - 212, 213, 214, 613, 215, 216, 217, 218, 614, 219, - 220, 221, 222, 615, 224, 225, 226, 227, 228, 229, - 616, 617, 231, 618, 232, 233, 619, 235, 620, 236, - 621, 237, 622, 623, 624, 240, 241, 625, 626, 244, - 245, 246, 627, 628, 629, 249, 250, 630, 251, 252, - 253, 254, 255, 256, 257, 631, 259, 260, 261, 262, - 632, 263, 264, 265, 266, 267, 268, 269, 633, 270, - 634, 635, 273, 274, 275, 276, 277, 636, 637, 638, - 639, 640, 281, 641, 642, 284, 643, 286, 287, 288, - 644, 289, 290, 291, 645, 646, 292, 647, 294, 648, - 649, 296, 297, 298, 299, 300, 301, 302, 303, 650, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 651, 652, 653, 330, 331, 332, 654, 655, - 334, 335, 656, 337, 657, 658, 339, 659, 341, 342, - 343, 660, 344, 345, 661, 662, 346, 347, 348, 663, - 664, 349, 350, 665, 666, 353, 667, 668, 356, 357, - 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 669, 670, 671, 672, 368, 369, 673, 674, 372, 373, - 675, 375, 376, 377, 676, 378, 379, 380, 381, 382, - 383, 677, 384, 385, 386, 387, 388, 678, 390, 391, - 392, 393, 679, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 680, 407, 408, 681, - 410, 411, 412, 682, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 683, 684, 427, - 428, 429, 430, 431, 432, 685, 434, 435, 686, 687, - 437, 438, 688, 440, 689, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 690, - 456, 691, 692, 693, 458, 459, 694, 460, 695, 462, - 463, 464, 465, 466, 467, 696, 468, 697, 698, 699, - 700, 471, 472, 701, 474, 702, 703, 476, 477, 704, - 479, 480, 481, 482, 483, 705, 706, 484, 485, 486, - 707, 708, 487, 488, 489, 490, 709, 491, 492, 493, - 494, 495, 710, 711, 498, 712, 499, 713, 501, 502, - 503, 504, 505, 506, 507, 714, 715, 508, 716, 717, - 509, 510, 511, 512, 513, 514, 718, 719, 720, 721, - 722, 723, 724, 725, 726, 727, 728, 526, 527, 528, - 529, 568, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1700, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1701, + 0, 0, 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, + 1709, 1710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 120, 121, 122, 123, 124, 125, 126, 127, 569, - 128, 129, 130, 570, 571, 572, 573, 574, 575, 576, - 577, 578, 132, 133, 579, 580, 135, 136, 581, 138, - 139, 140, 582, 583, 584, 585, 586, 587, 588, 146, - 147, 148, 149, 150, 589, 590, 151, 152, 153, 154, - 591, 592, 157, 593, 158, 159, 160, 161, 594, 595, - 596, 597, 598, 165, 166, 167, 168, 169, 599, 171, - 172, 173, 600, 174, 175, 176, 177, 178, 179, 601, - 602, 181, 182, 183, 184, 185, 186, 603, 188, 189, - 190, 604, 192, 193, 605, 195, 606, 196, 607, 197, - 198, 199, 200, 201, 202, 608, 609, 203, 204, 205, - 206, 610, 611, 207, 208, 209, 210, 211, 612, 212, - 213, 214, 613, 215, 216, 217, 218, 614, 219, 220, - 221, 222, 615, 224, 225, 226, 227, 228, 229, 616, - 617, 231, 618, 232, 233, 619, 235, 620, 236, 621, - 237, 622, 623, 624, 240, 241, 625, 626, 244, 245, - 246, 627, 628, 629, 249, 250, 630, 251, 252, 253, - 254, 255, 256, 257, 631, 259, 260, 261, 262, 632, - 263, 264, 265, 266, 267, 268, 269, 633, 270, 634, - 635, 273, 274, 275, 276, 277, 636, 637, 638, 639, - 640, 281, 641, 642, 284, 643, 286, 287, 288, 644, - 289, 290, 291, 645, 646, 292, 647, 294, 648, 649, - 296, 297, 298, 299, 300, 301, 302, 303, 650, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 651, 652, 653, 330, 331, 332, 654, 655, 334, - 335, 656, 337, 657, 658, 339, 659, 341, 342, 343, - 660, 344, 345, 661, 662, 346, 347, 348, 663, 664, - 349, 350, 665, 666, 353, 667, 668, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 669, - 670, 671, 672, 368, 369, 673, 674, 372, 373, 675, - 375, 376, 377, 676, 378, 379, 380, 381, 382, 383, - 677, 384, 385, 386, 387, 388, 678, 390, 391, 392, - 393, 679, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 680, 407, 408, 681, 410, - 411, 412, 682, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 683, 684, 427, 428, - 429, 430, 431, 432, 685, 434, 435, 686, 687, 437, - 438, 688, 440, 689, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 690, 456, - 691, 692, 693, 458, 459, 694, 460, 695, 462, 463, - 464, 465, 466, 467, 696, 468, 697, 698, 699, 700, - 471, 472, 701, 474, 702, 703, 476, 477, 704, 479, - 480, 481, 482, 483, 705, 706, 484, 485, 486, 707, - 708, 487, 488, 489, 490, 709, 491, 492, 493, 494, - 495, 710, 711, 498, 712, 499, 713, 501, 502, 503, - 504, 505, 506, 507, 714, 715, 508, 716, 717, 509, - 510, 511, 512, 513, 514, 718, 719, 720, 721, 722, - 723, 724, 725, 726, 727, 728, 526, 527, 528, 529, - 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 569, 128, - 129, 130, 570, 571, 572, 573, 574, 575, 576, 577, - 578, 132, 133, 579, 580, 135, 136, 581, 138, 139, - 140, 582, 583, 584, 585, 586, 587, 588, 146, 147, - 148, 149, 150, 589, 590, 151, 152, 153, 154, 591, - 592, 157, 593, 158, 159, 160, 161, 594, 595, 596, - 597, 598, 165, 166, 167, 168, 169, 599, 171, 172, - 173, 600, 174, 175, 176, 177, 178, 179, 601, 602, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 604, 192, 193, 605, 195, 606, 196, 607, 197, 198, - 199, 200, 201, 202, 608, 609, 203, 204, 205, 206, - 610, 611, 207, 208, 209, 210, 211, 612, 212, 213, - 214, 613, 215, 216, 217, 218, 614, 219, 220, 221, - 222, 615, 224, 225, 226, 227, 228, 229, 616, 617, - 231, 618, 232, 233, 619, 235, 620, 236, 621, 237, - 622, 623, 624, 240, 241, 625, 626, 244, 245, 246, - 627, 628, 629, 249, 250, 630, 251, 252, 253, 254, - 255, 965, 257, 631, 259, 260, 261, 262, 632, 263, - 264, 265, 266, 267, 268, 269, 633, 270, 634, 635, - 273, 274, 275, 276, 277, 636, 637, 638, 639, 640, - 281, 641, 642, 284, 643, 286, 287, 288, 644, 289, - 290, 291, 645, 646, 292, 647, 294, 648, 649, 296, - 297, 298, 299, 300, 301, 302, 303, 650, 305, 306, - 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 651, 652, 653, 330, 331, 332, 654, 655, 334, 335, - 656, 337, 657, 658, 339, 659, 341, 342, 343, 660, - 344, 345, 661, 662, 346, 347, 348, 663, 664, 349, - 350, 665, 666, 353, 667, 668, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 669, 670, - 671, 672, 368, 369, 673, 674, 372, 373, 675, 375, - 376, 377, 676, 378, 379, 380, 381, 382, 383, 677, - 384, 385, 386, 387, 388, 678, 390, 391, 392, 393, - 679, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 680, 407, 408, 681, 410, 411, - 412, 682, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 683, 684, 427, 428, 429, - 430, 431, 432, 685, 434, 435, 686, 687, 437, 438, - 688, 440, 689, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 690, 456, 691, - 692, 693, 458, 459, 694, 460, 695, 462, 463, 464, - 465, 466, 467, 696, 468, 697, 698, 699, 700, 471, - 472, 701, 474, 702, 703, 476, 477, 704, 479, 480, - 481, 482, 483, 705, 706, 484, 485, 486, 707, 708, - 487, 488, 489, 490, 709, 491, 492, 493, 494, 495, - 710, 711, 498, 712, 499, 713, 501, 502, 503, 504, - 505, 506, 507, 714, 715, 508, 716, 717, 509, 510, - 511, 512, 513, 514, 718, 719, 720, 721, 722, 723, - 724, 725, 726, 727, 728, 526, 527, 528, 529, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, - 121, 122, 123, 124, 125, 126, 127, 569, 128, 129, - 130, 570, 571, 572, 573, 574, 575, 576, 577, 578, - 132, 133, 579, 580, 135, 136, 581, 138, 139, 140, - 582, 583, 584, 585, 586, 587, 588, 146, 147, 148, - 149, 150, 589, 590, 151, 152, 153, 154, 591, 592, - 157, 593, 158, 159, 160, 161, 594, 595, 596, 597, - 598, 165, 166, 167, 168, 169, 599, 171, 172, 173, - 600, 174, 175, 176, 177, 178, 179, 601, 602, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 604, - 192, 193, 605, 195, 606, 196, 607, 197, 198, 199, - 200, 201, 202, 608, 609, 203, 204, 205, 206, 610, - 611, 207, 208, 209, 210, 211, 612, 212, 213, 214, - 613, 215, 216, 217, 218, 614, 219, 220, 221, 222, - 615, 224, 225, 226, 227, 228, 229, 616, 617, 231, - 618, 232, 233, 619, 235, 620, 236, 621, 237, 622, - 623, 624, 240, 241, 625, 626, 244, 245, 246, 627, - 628, 629, 249, 250, 630, 251, 252, 253, 254, 255, - 256, 257, 631, 259, 260, 261, 262, 632, 263, 264, - 265, 266, 267, 268, 269, 633, 270, 634, 635, 273, - 274, 275, 276, 277, 636, 637, 638, 639, 640, 281, - 641, 642, 284, 643, 286, 287, 288, 644, 289, 290, - 291, 645, 646, 292, 647, 294, 648, 649, 296, 297, - 298, 299, 300, 301, 302, 303, 650, 305, 306, 307, - 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, - 318, 319, 320, 321, 322, 323, 324, 325, 326, 651, - 652, 653, 330, 331, 332, 654, 655, 334, 335, 656, - 337, 657, 658, 339, 659, 341, 342, 343, 660, 344, - 345, 661, 662, 346, 347, 348, 663, 664, 349, 350, - 665, 666, 353, 667, 668, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 669, 670, 671, - 672, 368, 369, 673, 674, 372, 373, 675, 375, 376, - 377, 676, 378, 379, 380, 381, 382, 383, 677, 384, - 385, 386, 387, 388, 678, 390, 391, 392, 393, 679, - 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 680, 407, 408, 681, 410, 411, 412, - 682, 414, 415, 416, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 426, 683, 684, 427, 428, 429, 430, - 431, 432, 685, 434, 435, 686, 687, 437, 438, 688, - 440, 689, 441, 442, 443, 444, 445, 446, 447, 448, - 449, 450, 451, 452, 453, 454, 690, 456, 691, 692, - 693, 458, 459, 694, 460, 695, 462, 463, 464, 465, - 466, 467, 696, 468, 697, 698, 699, 700, 471, 472, - 701, 474, 702, 703, 476, 477, 704, 479, 480, 481, - 482, 483, 705, 706, 484, 485, 486, 707, 708, 487, - 488, 489, 490, 709, 491, 492, 493, 494, 495, 710, - 711, 498, 712, 499, 713, 501, 502, 503, 504, 505, - 506, 507, 714, 715, 508, 716, 717, 509, 510, 511, - 512, 513, 514, 718, 719, 720, 721, 722, 723, 724, - 725, 726, 727, 728, 526, 527, 528, 529, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 120, 121, - 122, 123, 2337, 125, 126, 127, 569, 128, 129, 130, - 570, 571, 572, 573, 574, 575, 576, 577, 578, 132, - 133, 579, 580, 135, 136, 581, 138, 139, 140, 582, - 583, 584, 585, 586, 587, 588, 146, 147, 148, 149, - 150, 589, 590, 151, 152, 153, 154, 591, 592, 157, - 593, 158, 159, 160, 161, 594, 595, 596, 597, 598, - 165, 166, 167, 168, 169, 599, 171, 172, 173, 600, - 174, 175, 176, 177, 178, 179, 601, 602, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 604, 192, - 193, 605, 195, 606, 196, 607, 197, 198, 199, 200, - 201, 202, 608, 609, 203, 204, 205, 206, 610, 611, - 207, 208, 209, 2338, 211, 612, 212, 213, 214, 613, - 215, 216, 217, 218, 614, 219, 220, 221, 222, 615, - 224, 225, 226, 227, 228, 229, 616, 617, 231, 618, - 232, 233, 619, 235, 620, 236, 621, 237, 622, 623, - 624, 240, 241, 625, 626, 244, 245, 246, 627, 628, - 629, 249, 250, 630, 251, 252, 253, 254, 255, 256, - 257, 631, 259, 260, 261, 262, 632, 263, 264, 265, - 266, 267, 268, 269, 633, 270, 634, 635, 273, 274, - 275, 276, 277, 636, 637, 638, 639, 640, 281, 641, - 642, 284, 643, 286, 287, 288, 644, 289, 290, 291, - 645, 646, 292, 647, 294, 648, 649, 296, 297, 298, - 299, 300, 301, 302, 303, 650, 305, 306, 307, 308, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 570, 0, 2188, 0, 0, 0, 0, 1701, 0, 0, + 1702, 1703, 1704, 0, 1705, 1706, 1707, 1708, 2293, 1710, + 122, 123, 124, 125, 126, 127, 128, 129, 571, 130, + 131, 132, 572, 573, 574, 575, 576, 577, 578, 579, + 580, 134, 135, 581, 582, 137, 138, 583, 140, 141, + 142, 584, 585, 586, 587, 588, 589, 590, 148, 149, + 150, 151, 152, 591, 592, 153, 154, 155, 156, 593, + 594, 159, 595, 160, 161, 162, 163, 596, 597, 598, + 599, 600, 167, 168, 169, 170, 171, 601, 173, 174, + 175, 602, 176, 177, 178, 179, 180, 181, 603, 604, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 606, 194, 195, 607, 197, 608, 198, 609, 199, 200, + 201, 202, 203, 204, 610, 611, 205, 206, 207, 208, + 612, 613, 209, 210, 211, 212, 213, 614, 214, 215, + 216, 615, 217, 218, 219, 220, 616, 221, 222, 223, + 224, 617, 226, 227, 228, 229, 230, 231, 618, 619, + 233, 620, 234, 235, 621, 237, 622, 238, 623, 239, + 624, 625, 626, 242, 243, 627, 628, 246, 247, 248, + 629, 630, 631, 251, 252, 632, 253, 254, 255, 256, + 257, 258, 259, 633, 261, 262, 263, 264, 634, 265, + 266, 267, 268, 269, 270, 271, 635, 272, 636, 637, + 275, 276, 277, 278, 279, 638, 639, 640, 641, 642, + 283, 643, 644, 286, 645, 288, 289, 290, 646, 291, + 292, 293, 647, 648, 294, 649, 296, 650, 651, 298, + 299, 300, 301, 302, 303, 304, 305, 652, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 651, 652, - 653, 330, 331, 332, 654, 655, 334, 335, 656, 337, - 657, 658, 339, 659, 341, 342, 343, 660, 344, 345, - 661, 662, 346, 347, 348, 663, 664, 349, 350, 665, - 666, 353, 667, 668, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 669, 670, 671, 672, - 368, 369, 673, 674, 372, 373, 675, 375, 376, 377, - 676, 378, 379, 380, 381, 382, 383, 677, 384, 385, - 386, 387, 388, 678, 390, 391, 392, 393, 679, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 680, 407, 408, 681, 410, 411, 412, 682, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 683, 684, 427, 428, 429, 430, 431, - 2339, 685, 434, 435, 686, 687, 437, 438, 688, 440, - 689, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 690, 456, 691, 692, 693, - 458, 459, 694, 460, 695, 462, 463, 464, 465, 466, - 467, 696, 468, 697, 698, 699, 700, 471, 472, 701, - 474, 702, 703, 476, 477, 704, 479, 480, 481, 482, - 483, 705, 706, 484, 485, 486, 707, 708, 487, 488, - 489, 490, 709, 491, 492, 493, 494, 495, 710, 711, - 498, 712, 499, 713, 501, 502, 503, 504, 505, 506, - 507, 714, 715, 508, 716, 717, 509, 510, 511, 512, - 513, 514, 718, 719, 720, 721, 722, 723, 724, 725, - 726, 727, 728, 526, 527, 528, 529, 989, 0, 0, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 653, 654, 655, 332, 333, 334, 656, 657, 336, 337, + 658, 339, 659, 660, 341, 661, 343, 344, 345, 662, + 346, 347, 663, 664, 348, 349, 350, 665, 666, 351, + 352, 667, 668, 355, 669, 670, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 671, 672, + 673, 674, 370, 371, 675, 676, 374, 375, 677, 377, + 378, 379, 678, 380, 381, 382, 383, 384, 385, 679, + 386, 387, 388, 389, 390, 680, 392, 393, 394, 395, + 681, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 682, 409, 410, 683, 412, 413, + 414, 684, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 685, 686, 429, 430, 431, + 432, 433, 434, 687, 436, 437, 688, 689, 439, 440, + 690, 442, 691, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 692, 458, 693, + 694, 695, 460, 461, 696, 462, 697, 464, 465, 466, + 467, 468, 469, 698, 470, 699, 700, 701, 702, 473, + 474, 703, 476, 704, 705, 478, 479, 706, 481, 482, + 483, 484, 485, 707, 708, 486, 487, 488, 709, 710, + 489, 490, 491, 492, 711, 493, 494, 495, 496, 497, + 712, 713, 500, 714, 501, 715, 503, 504, 505, 506, + 507, 508, 509, 716, 717, 510, 718, 719, 511, 512, + 513, 514, 515, 516, 720, 721, 722, 723, 724, 725, + 726, 727, 728, 729, 730, 528, 529, 530, 531, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 120, 121, 122, - 123, 124, 125, 126, 127, 0, 128, 129, 130, 3, - 4, 0, 573, 0, 0, 0, 0, 578, 132, 133, - 0, 580, 135, 136, 581, 138, 139, 140, 582, 583, - 584, 585, 586, 0, 588, 146, 147, 148, 149, 150, - 0, 0, 151, 152, 153, 154, 591, 592, 157, 0, - 158, 159, 160, 161, 594, 0, 596, 0, 598, 165, - 166, 167, 168, 169, 599, 171, 172, 173, 0, 174, - 175, 176, 177, 178, 179, 0, 602, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 604, 192, 193, - 605, 195, 0, 196, 0, 197, 198, 199, 200, 201, - 202, 0, 0, 203, 204, 205, 206, 0, 0, 207, - 208, 209, 210, 211, 0, 212, 213, 214, 0, 215, - 216, 217, 218, 0, 219, 220, 221, 222, 615, 224, - 225, 226, 227, 228, 229, 616, 0, 231, 0, 232, - 233, 619, 235, 0, 236, 0, 237, 622, 0, 624, - 240, 241, 625, 626, 244, 245, 246, 0, 628, 629, - 249, 250, 0, 251, 252, 253, 254, 255, 256, 257, - 631, 259, 260, 261, 262, 0, 263, 264, 265, 266, - 267, 268, 269, 0, 270, 634, 635, 273, 274, 275, - 276, 277, 636, 637, 0, 639, 0, 281, 641, 642, - 284, 643, 286, 287, 288, 0, 289, 290, 291, 0, - 0, 292, 647, 294, 648, 0, 296, 297, 298, 299, - 300, 301, 302, 303, 650, 305, 306, 307, 308, 309, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, + 123, 124, 125, 126, 127, 128, 129, 571, 130, 131, + 132, 572, 573, 574, 575, 576, 577, 578, 579, 580, + 134, 135, 581, 582, 137, 138, 583, 140, 141, 142, + 584, 585, 586, 587, 588, 589, 590, 148, 149, 150, + 151, 152, 591, 592, 153, 154, 155, 156, 593, 594, + 159, 595, 160, 161, 162, 163, 596, 597, 598, 599, + 600, 167, 168, 169, 170, 171, 601, 173, 174, 175, + 602, 176, 177, 178, 179, 180, 181, 603, 604, 183, + 184, 185, 186, 187, 188, 605, 190, 191, 192, 606, + 194, 195, 607, 197, 608, 198, 609, 199, 200, 201, + 202, 203, 204, 610, 611, 205, 206, 207, 208, 612, + 613, 209, 210, 211, 212, 213, 614, 214, 215, 216, + 615, 217, 218, 219, 220, 616, 221, 222, 223, 224, + 617, 226, 227, 228, 229, 230, 231, 618, 619, 233, + 620, 234, 235, 621, 237, 622, 238, 623, 239, 624, + 625, 626, 242, 243, 627, 628, 246, 247, 248, 629, + 630, 631, 251, 252, 632, 253, 254, 255, 256, 257, + 258, 259, 633, 261, 262, 263, 264, 634, 265, 266, + 267, 268, 269, 270, 271, 635, 272, 636, 637, 275, + 276, 277, 278, 279, 638, 639, 640, 641, 642, 283, + 643, 644, 286, 645, 288, 289, 290, 646, 291, 292, + 293, 647, 648, 294, 649, 296, 650, 651, 298, 299, + 300, 301, 302, 303, 304, 305, 652, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 651, 652, 653, - 330, 331, 332, 654, 0, 334, 335, 656, 337, 0, - 658, 339, 659, 341, 342, 343, 0, 344, 345, 0, - 0, 346, 347, 348, 0, 0, 349, 350, 665, 666, - 353, 667, 668, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 0, 0, 0, 0, 368, - 369, 673, 674, 372, 373, 675, 375, 376, 377, 0, - 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, - 387, 388, 678, 390, 391, 392, 393, 0, 394, 395, + 320, 321, 322, 323, 324, 325, 326, 327, 328, 653, + 654, 655, 332, 333, 334, 656, 657, 336, 337, 658, + 339, 659, 660, 341, 661, 343, 344, 345, 662, 346, + 347, 663, 664, 348, 349, 350, 665, 666, 351, 352, + 667, 668, 355, 669, 670, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 671, 672, 673, + 674, 370, 371, 675, 676, 374, 375, 677, 377, 378, + 379, 678, 380, 381, 382, 383, 384, 385, 679, 386, + 387, 388, 389, 390, 680, 392, 393, 394, 395, 681, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 0, 407, 408, 681, 410, 411, 412, 682, 414, - 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 0, 684, 427, 428, 429, 430, 431, 432, - 685, 434, 435, 0, 687, 437, 438, 688, 440, 0, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 690, 456, 691, 0, 0, 458, - 459, 0, 460, 695, 462, 463, 464, 465, 466, 467, - 0, 468, 697, 698, 0, 0, 471, 472, 701, 474, - 702, 0, 476, 477, 704, 479, 480, 481, 482, 483, - 0, 0, 484, 485, 486, 707, 0, 487, 488, 489, - 490, 0, 491, 492, 493, 494, 495, 710, 711, 498, - 0, 499, 713, 501, 502, 503, 504, 505, 506, 507, - 0, 0, 508, 0, 0, 509, 510, 511, 512, 513, - 514, 718, 719, 720, 721, 722, 723, 724, 725, 726, - 727, 728, 526, 527, 528, 529, 119, 0, 0, 0, + 406, 407, 408, 682, 409, 410, 683, 412, 413, 414, + 684, 416, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 426, 427, 428, 685, 686, 429, 430, 431, 432, + 433, 434, 687, 436, 437, 688, 689, 439, 440, 690, + 442, 691, 443, 444, 445, 446, 447, 448, 449, 450, + 451, 452, 453, 454, 455, 456, 692, 458, 693, 694, + 695, 460, 461, 696, 462, 697, 464, 465, 466, 467, + 468, 469, 698, 470, 699, 700, 701, 702, 473, 474, + 703, 476, 704, 705, 478, 479, 706, 481, 482, 483, + 484, 485, 707, 708, 486, 487, 488, 709, 710, 489, + 490, 491, 492, 711, 493, 494, 495, 496, 497, 712, + 713, 500, 714, 501, 715, 503, 504, 505, 506, 507, + 508, 509, 716, 717, 510, 718, 719, 511, 512, 513, + 514, 515, 516, 720, 721, 722, 723, 724, 725, 726, + 727, 728, 729, 730, 528, 529, 530, 531, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 0, 0, 0, 131, 132, 133, 0, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, - 144, 145, 0, 0, 146, 147, 148, 149, 150, 0, - 802, 151, 152, 153, 154, 155, 156, 157, 0, 158, - 159, 160, 161, 803, 0, 804, 0, 164, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 209, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 805, 0, 231, 0, 232, 233, - 234, 235, 0, 236, 0, 237, 238, 0, 239, 240, - 241, 242, 243, 244, 245, 246, 0, 247, 248, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, 0, 280, 0, 281, 282, 283, 284, - 285, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 293, 294, 295, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 571, 130, 131, 132, + 572, 573, 574, 575, 576, 577, 578, 579, 580, 134, + 135, 581, 582, 137, 138, 583, 140, 141, 142, 584, + 585, 586, 587, 588, 589, 590, 148, 149, 150, 151, + 152, 591, 592, 153, 154, 155, 156, 593, 594, 159, + 595, 160, 161, 162, 163, 596, 597, 598, 599, 600, + 167, 168, 169, 170, 171, 601, 173, 174, 175, 602, + 176, 177, 178, 179, 180, 181, 603, 604, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 606, 194, + 195, 607, 197, 608, 198, 609, 199, 200, 201, 202, + 203, 204, 610, 611, 205, 206, 207, 208, 612, 613, + 209, 210, 211, 212, 213, 614, 214, 215, 216, 615, + 217, 218, 219, 220, 616, 221, 222, 223, 224, 617, + 226, 227, 228, 229, 230, 231, 618, 619, 233, 620, + 234, 235, 621, 237, 622, 238, 623, 239, 624, 625, + 626, 242, 243, 627, 628, 246, 247, 248, 629, 630, + 631, 251, 252, 632, 253, 254, 255, 256, 257, 970, + 259, 633, 261, 262, 263, 264, 634, 265, 266, 267, + 268, 269, 270, 271, 635, 272, 636, 637, 275, 276, + 277, 278, 279, 638, 639, 640, 641, 642, 283, 643, + 644, 286, 645, 288, 289, 290, 646, 291, 292, 293, + 647, 648, 294, 649, 296, 650, 651, 298, 299, 300, + 301, 302, 303, 304, 305, 652, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, - 331, 332, 333, 0, 334, 335, 336, 337, 0, 807, - 339, 340, 341, 342, 343, 0, 344, 345, 0, 0, - 346, 347, 348, 0, 0, 349, 350, 351, 352, 353, - 354, 809, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 810, 371, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 653, 654, + 655, 332, 333, 334, 656, 657, 336, 337, 658, 339, + 659, 660, 341, 661, 343, 344, 345, 662, 346, 347, + 663, 664, 348, 349, 350, 665, 666, 351, 352, 667, + 668, 355, 669, 670, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 671, 672, 673, 674, + 370, 371, 675, 676, 374, 375, 677, 377, 378, 379, + 678, 380, 381, 382, 383, 384, 385, 679, 386, 387, + 388, 389, 390, 680, 392, 393, 394, 395, 681, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 409, 410, 411, 412, 413, 414, 415, + 407, 408, 682, 409, 410, 683, 412, 413, 414, 684, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 436, 437, 438, 439, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 812, 0, 0, 458, 459, - 0, 460, 461, 462, 463, 464, 465, 466, 467, 0, - 468, 469, 470, 0, 0, 471, 472, 813, 474, 814, - 0, 476, 477, 815, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 497, 498, 0, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, - 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 119, 0, 0, 0, 0, + 426, 427, 428, 685, 686, 429, 430, 431, 432, 433, + 434, 687, 436, 437, 688, 689, 439, 440, 690, 442, + 691, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 692, 458, 693, 694, 695, + 460, 461, 696, 462, 697, 464, 465, 466, 467, 468, + 469, 698, 470, 699, 700, 701, 702, 473, 474, 703, + 476, 704, 705, 478, 479, 706, 481, 482, 483, 484, + 485, 707, 708, 486, 487, 488, 709, 710, 489, 490, + 491, 492, 711, 493, 494, 495, 496, 497, 712, 713, + 500, 714, 501, 715, 503, 504, 505, 506, 507, 508, + 509, 716, 717, 510, 718, 719, 511, 512, 513, 514, + 515, 516, 720, 721, 722, 723, 724, 725, 726, 727, + 728, 729, 730, 528, 529, 530, 531, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 120, 121, 122, 123, 124, - 125, 126, 127, 0, 128, 129, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 131, 132, 133, 0, 134, - 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, - 145, 0, 0, 146, 147, 148, 149, 150, 0, 0, - 151, 152, 153, 154, 155, 156, 157, 0, 158, 159, - 160, 161, 162, 0, 163, 0, 164, 165, 166, 167, - 168, 169, 170, 171, 172, 173, 0, 174, 175, 176, - 177, 178, 179, 0, 180, 181, 182, 183, 184, 185, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 0, 196, 0, 197, 198, 199, 200, 201, 202, 0, - 0, 203, 204, 205, 206, 0, 0, 207, 208, 209, - 210, 211, 0, 212, 213, 214, 0, 215, 216, 217, - 218, 0, 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 0, 231, 0, 232, 233, 234, - 235, 0, 236, 0, 237, 238, 0, 239, 240, 241, - 242, 243, 244, 245, 246, 0, 247, 248, 249, 250, - 0, 251, 252, 253, 254, 255, 256, 257, 258, 259, - 260, 261, 262, 0, 263, 264, 265, 266, 267, 268, - 269, 0, 270, 271, 272, 273, 274, 275, 276, 277, - 278, 279, 0, 280, 0, 281, 282, 283, 284, 285, - 286, 287, 288, 0, 289, 290, 291, 0, 0, 292, - 293, 294, 295, 0, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 0, 0, 0, 0, 0, 0, 0, 122, 123, 124, + 125, 126, 127, 128, 129, 571, 130, 131, 132, 572, + 573, 574, 575, 576, 577, 578, 579, 580, 134, 135, + 581, 582, 137, 138, 583, 140, 141, 142, 584, 585, + 586, 587, 588, 589, 590, 148, 149, 150, 151, 152, + 591, 592, 153, 154, 155, 156, 593, 594, 159, 595, + 160, 161, 162, 163, 596, 597, 598, 599, 600, 167, + 168, 169, 170, 171, 601, 173, 174, 175, 602, 176, + 177, 178, 179, 180, 181, 603, 604, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 606, 194, 195, + 607, 197, 608, 198, 609, 199, 200, 201, 202, 203, + 204, 610, 611, 205, 206, 207, 208, 612, 613, 209, + 210, 211, 212, 213, 614, 214, 215, 216, 615, 217, + 218, 219, 220, 616, 221, 222, 223, 224, 617, 226, + 227, 228, 229, 230, 231, 618, 619, 233, 620, 234, + 235, 621, 237, 622, 238, 623, 239, 624, 625, 626, + 242, 243, 627, 628, 246, 247, 248, 629, 630, 631, + 251, 252, 632, 253, 254, 255, 256, 257, 258, 259, + 633, 261, 262, 263, 264, 634, 265, 266, 267, 268, + 269, 270, 271, 635, 272, 636, 637, 275, 276, 277, + 278, 279, 638, 639, 640, 641, 642, 283, 643, 644, + 286, 645, 288, 289, 290, 646, 291, 292, 293, 647, + 648, 294, 649, 296, 650, 651, 298, 299, 300, 301, + 302, 303, 304, 305, 652, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, - 332, 333, 0, 334, 335, 336, 337, 0, 338, 339, - 340, 341, 342, 343, 0, 344, 345, 0, 0, 346, - 347, 348, 0, 0, 349, 350, 351, 352, 353, 354, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 0, 0, 0, 0, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 0, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 0, 394, 395, 396, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 0, - 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 322, 323, 324, 325, 326, 327, 328, 653, 654, 655, + 332, 333, 334, 656, 657, 336, 337, 658, 339, 659, + 660, 341, 661, 343, 344, 345, 662, 346, 347, 663, + 664, 348, 349, 350, 665, 666, 351, 352, 667, 668, + 355, 669, 670, 358, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 671, 672, 673, 674, 370, + 371, 675, 676, 374, 375, 677, 377, 378, 379, 678, + 380, 381, 382, 383, 384, 385, 679, 386, 387, 388, + 389, 390, 680, 392, 393, 394, 395, 681, 396, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 682, 409, 410, 683, 412, 413, 414, 684, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 0, 0, 427, 428, 429, 430, 431, 432, 433, 434, - 435, 0, 436, 437, 438, 439, 440, 0, 441, 442, + 427, 428, 685, 686, 429, 430, 431, 432, 433, 434, + 687, 436, 437, 688, 689, 439, 440, 690, 442, 691, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, - 453, 454, 455, 456, 457, 0, 0, 458, 459, 0, - 460, 461, 462, 463, 464, 465, 466, 467, 0, 468, - 469, 470, 0, 0, 471, 472, 473, 474, 475, 0, - 476, 477, 478, 479, 480, 481, 482, 483, 0, 0, - 484, 485, 486, 0, 0, 487, 488, 489, 490, 0, - 491, 492, 493, 494, 495, 496, 497, 498, 0, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 0, 0, - 508, 0, 0, 509, 510, 511, 512, 513, 514, 515, - 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, - 526, 527, 528, 529, 537, 0, 0, 0, 0, 0, + 453, 454, 455, 456, 692, 458, 693, 694, 695, 460, + 461, 696, 462, 697, 464, 465, 466, 467, 468, 469, + 698, 470, 699, 700, 701, 702, 473, 474, 703, 476, + 704, 705, 478, 479, 706, 481, 482, 483, 484, 485, + 707, 708, 486, 487, 488, 709, 710, 489, 490, 491, + 492, 711, 493, 494, 495, 496, 497, 712, 713, 500, + 714, 501, 715, 503, 504, 505, 506, 507, 508, 509, + 716, 717, 510, 718, 719, 511, 512, 513, 514, 515, + 516, 720, 721, 722, 723, 724, 725, 726, 727, 728, + 729, 730, 528, 529, 530, 531, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 132, 133, 0, 0, 135, - 136, 0, 138, 139, 140, 141, 142, 0, 144, 145, - 0, 0, 146, 147, 148, 149, 150, 0, 0, 151, - 152, 153, 154, 155, 156, 157, 1803, 158, 159, 160, - 161, 162, 0, 0, 1804, 164, 165, 166, 167, 168, - 169, 0, 171, 172, 173, 1805, 174, 175, 176, 177, - 178, 179, 0, 0, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 0, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 209, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 0, 231, 0, 232, 233, 234, 235, - 0, 236, 1806, 237, 0, 0, 0, 240, 241, 538, - 0, 244, 245, 246, 0, 247, 248, 249, 250, 0, - 251, 252, 253, 254, 255, 1807, 257, 0, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 0, 272, 273, 274, 275, 276, 277, 278, - 279, 0, 280, 0, 281, 0, 0, 284, 0, 286, - 287, 288, 0, 289, 290, 291, 0, 0, 292, 0, - 294, 0, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 539, 305, 306, 307, 308, 309, 310, 311, 312, + 0, 0, 0, 0, 0, 0, 122, 123, 124, 125, + 2349, 127, 128, 129, 571, 130, 131, 132, 572, 573, + 574, 575, 576, 577, 578, 579, 580, 134, 135, 581, + 582, 137, 138, 583, 140, 141, 142, 584, 585, 586, + 587, 588, 589, 590, 148, 149, 150, 151, 152, 591, + 592, 153, 154, 155, 156, 593, 594, 159, 595, 160, + 161, 162, 163, 596, 597, 598, 599, 600, 167, 168, + 169, 170, 171, 601, 173, 174, 175, 602, 176, 177, + 178, 179, 180, 181, 603, 604, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 606, 194, 195, 607, + 197, 608, 198, 609, 199, 200, 201, 202, 203, 204, + 610, 611, 205, 206, 207, 208, 612, 613, 209, 210, + 211, 2350, 213, 614, 214, 215, 216, 615, 217, 218, + 219, 220, 616, 221, 222, 223, 224, 617, 226, 227, + 228, 229, 230, 231, 618, 619, 233, 620, 234, 235, + 621, 237, 622, 238, 623, 239, 624, 625, 626, 242, + 243, 627, 628, 246, 247, 248, 629, 630, 631, 251, + 252, 632, 253, 254, 255, 256, 257, 258, 259, 633, + 261, 262, 263, 264, 634, 265, 266, 267, 268, 269, + 270, 271, 635, 272, 636, 637, 275, 276, 277, 278, + 279, 638, 639, 640, 641, 642, 283, 643, 644, 286, + 645, 288, 289, 290, 646, 291, 292, 293, 647, 648, + 294, 649, 296, 650, 651, 298, 299, 300, 301, 302, + 303, 304, 305, 652, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, 0, 329, 330, 331, 332, - 333, 0, 334, 335, 0, 337, 0, 338, 339, 340, - 341, 342, 343, 0, 344, 345, 0, 0, 346, 347, - 348, 0, 0, 349, 350, 351, 0, 353, 0, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 370, 0, - 372, 373, 374, 375, 376, 377, 1808, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 0, 410, 411, 412, 413, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 0, 437, 438, 439, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 653, 654, 655, 332, + 333, 334, 656, 657, 336, 337, 658, 339, 659, 660, + 341, 661, 343, 344, 345, 662, 346, 347, 663, 664, + 348, 349, 350, 665, 666, 351, 352, 667, 668, 355, + 669, 670, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 671, 672, 673, 674, 370, 371, + 675, 676, 374, 375, 677, 377, 378, 379, 678, 380, + 381, 382, 383, 384, 385, 679, 386, 387, 388, 389, + 390, 680, 392, 393, 394, 395, 681, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 682, 409, 410, 683, 412, 413, 414, 684, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 685, 686, 429, 430, 431, 432, 433, 2351, 687, + 436, 437, 688, 689, 439, 440, 690, 442, 691, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 540, 456, 457, 0, 0, 458, 459, 0, 460, - 0, 462, 463, 464, 465, 466, 467, 0, 468, 469, - 470, 0, 0, 471, 472, 473, 474, 475, 0, 476, - 477, 478, 479, 480, 481, 482, 483, 0, 1809, 484, - 485, 486, 0, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 497, 498, 0, 499, 0, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, - 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 537, 0, 0, 0, 0, 0, 0, + 454, 455, 456, 692, 458, 693, 694, 695, 460, 461, + 696, 462, 697, 464, 465, 466, 467, 468, 469, 698, + 470, 699, 700, 701, 702, 473, 474, 703, 476, 704, + 705, 478, 479, 706, 481, 482, 483, 484, 485, 707, + 708, 486, 487, 488, 709, 710, 489, 490, 491, 492, + 711, 493, 494, 495, 496, 497, 712, 713, 500, 714, + 501, 715, 503, 504, 505, 506, 507, 508, 509, 716, + 717, 510, 718, 719, 511, 512, 513, 514, 515, 516, + 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, + 730, 528, 529, 530, 531, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 120, 121, 122, 123, 124, 125, 126, - 127, 0, 128, 129, 130, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 133, 0, 0, 135, 136, - 0, 138, 139, 140, 141, 142, 0, 144, 145, 0, - 0, 146, 147, 148, 149, 150, 0, 0, 151, 152, - 153, 154, 155, 156, 157, 1803, 158, 159, 160, 161, - 162, 0, 0, 0, 164, 165, 166, 167, 168, 169, - 0, 171, 172, 173, 1805, 174, 175, 176, 177, 178, - 179, 0, 0, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 0, 196, - 0, 197, 198, 199, 200, 201, 202, 0, 0, 203, - 204, 205, 206, 0, 0, 207, 208, 209, 210, 211, - 0, 212, 213, 214, 0, 215, 216, 217, 218, 0, - 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, 0, 231, 0, 232, 233, 234, 235, 0, - 236, 1806, 237, 0, 0, 0, 240, 241, 538, 0, - 244, 245, 246, 0, 247, 248, 249, 250, 0, 251, - 252, 253, 254, 255, 256, 257, 0, 259, 260, 261, - 262, 0, 263, 264, 265, 266, 267, 268, 269, 0, - 270, 0, 272, 273, 274, 275, 276, 277, 278, 279, - 0, 280, 0, 281, 0, 0, 284, 0, 286, 287, - 288, 0, 289, 290, 291, 0, 0, 292, 0, 294, - 2428, 0, 296, 297, 298, 299, 300, 301, 302, 303, - 539, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 0, 0, 0, 0, 0, 122, 123, 124, 125, 126, + 127, 128, 129, 0, 130, 131, 132, 3, 4, 0, + 575, 0, 0, 0, 0, 580, 134, 135, 0, 582, + 137, 138, 583, 140, 141, 142, 584, 585, 586, 587, + 588, 0, 590, 148, 149, 150, 151, 152, 0, 0, + 153, 154, 155, 156, 593, 594, 159, 0, 160, 161, + 162, 163, 596, 0, 598, 0, 600, 167, 168, 169, + 170, 171, 601, 173, 174, 175, 0, 176, 177, 178, + 179, 180, 181, 0, 604, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 606, 194, 195, 607, 197, + 0, 198, 0, 199, 200, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 0, 0, 209, 210, 211, + 212, 213, 0, 214, 215, 216, 0, 217, 218, 219, + 220, 0, 221, 222, 223, 224, 617, 226, 227, 228, + 229, 230, 231, 618, 0, 233, 0, 234, 235, 621, + 237, 0, 238, 0, 239, 624, 0, 626, 242, 243, + 627, 628, 246, 247, 248, 0, 630, 631, 251, 252, + 0, 253, 254, 255, 256, 257, 258, 259, 633, 261, + 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, + 271, 0, 272, 636, 637, 275, 276, 277, 278, 279, + 638, 639, 0, 641, 0, 283, 643, 644, 286, 645, + 288, 289, 290, 0, 291, 292, 293, 0, 0, 294, + 649, 296, 650, 0, 298, 299, 300, 301, 302, 303, + 304, 305, 652, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 327, 0, 329, 330, 331, 332, 333, - 0, 334, 335, 0, 337, 0, 338, 339, 340, 341, - 342, 343, 0, 344, 345, 0, 0, 346, 347, 348, - 0, 0, 349, 350, 351, 0, 353, 0, 355, 356, - 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 0, 0, 0, 0, 368, 369, 370, 0, 372, - 373, 374, 375, 376, 377, 1808, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 0, 394, 395, 396, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 0, 407, 408, - 0, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 426, 0, 0, - 427, 428, 429, 430, 431, 432, 433, 434, 435, 0, - 0, 437, 438, 439, 440, 0, 441, 442, 443, 444, + 324, 325, 326, 327, 328, 653, 654, 655, 332, 333, + 334, 656, 0, 336, 337, 658, 339, 0, 660, 341, + 661, 343, 344, 345, 0, 346, 347, 0, 0, 348, + 349, 350, 0, 0, 351, 352, 667, 668, 355, 669, + 670, 358, 359, 360, 361, 362, 363, 364, 365, 366, + 367, 368, 369, 0, 0, 0, 0, 370, 371, 675, + 676, 374, 375, 677, 377, 378, 379, 0, 380, 381, + 382, 383, 384, 385, 0, 386, 387, 388, 389, 390, + 680, 392, 393, 394, 395, 0, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 0, + 409, 410, 683, 412, 413, 414, 684, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 0, 686, 429, 430, 431, 432, 433, 434, 687, 436, + 437, 0, 689, 439, 440, 690, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 540, 456, 457, 0, 0, 458, 459, 0, 460, 0, - 462, 463, 464, 465, 466, 467, 0, 468, 469, 470, - 0, 0, 471, 472, 473, 474, 475, 0, 476, 477, - 478, 479, 480, 481, 482, 483, 0, 1809, 484, 485, - 486, 0, 0, 487, 488, 489, 490, 0, 491, 492, - 493, 494, 495, 496, 497, 498, 0, 499, 0, 501, - 502, 503, 504, 505, 506, 507, 0, 0, 508, 0, - 0, 509, 510, 511, 512, 513, 514, 515, 516, 517, - 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, - 528, 529, 1527, 0, 0, 0, 0, 0, 0, 0, + 455, 456, 692, 458, 693, 0, 0, 460, 461, 0, + 462, 697, 464, 465, 466, 467, 468, 469, 0, 470, + 699, 700, 0, 0, 473, 474, 703, 476, 704, 0, + 478, 479, 706, 481, 482, 483, 484, 485, 0, 0, + 486, 487, 488, 709, 0, 489, 490, 491, 492, 0, + 493, 494, 495, 496, 497, 712, 713, 500, 0, 501, + 715, 503, 504, 505, 506, 507, 508, 509, 0, 0, + 510, 0, 0, 511, 512, 513, 514, 515, 516, 720, + 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, + 528, 529, 530, 531, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 1528, 0, 0, - -861, 0, 1529, 132, 133, 0, 1530, 135, 136, 1531, - 138, 139, 140, 0, 1532, 1533, 1534, 1535, 0, 1536, - 146, 147, 148, 149, 150, 0, 0, 151, 152, 153, - 154, 1537, 1538, 157, 0, 158, 159, 160, 161, 0, - 0, 1539, 0, 1540, 165, 166, 167, 168, 169, 1541, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 1542, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 1543, 192, 193, 1544, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 1088, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 0, 224, 225, 226, 227, 228, 229, - 0, 0, 231, 0, 232, 233, 1545, 235, 0, 236, - 0, 237, 1546, 0, 1547, 240, 241, -861, 1548, 244, - 245, 246, 0, 0, 0, 249, 250, 0, 251, 252, - 253, 254, 255, 256, 257, 1549, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 1550, 0, 273, 274, 275, 276, 277, 1551, 1552, 0, - 1553, 0, 281, 1554, 1555, 284, 1556, 286, 287, 288, - 0, 289, 290, 291, 0, 0, 292, 1557, 294, 1558, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 1559, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 0, 0, 0, 133, 134, 135, 0, 136, 137, + 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 0, 0, 148, 149, 150, 151, 152, 0, 805, 153, + 154, 155, 156, 157, 158, 159, 0, 160, 161, 162, + 163, 806, 0, 807, 0, 166, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 211, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 808, 0, 233, 0, 234, 235, 236, 237, + 0, 238, 0, 239, 240, 0, 241, 242, 243, 244, + 245, 246, 247, 248, 0, 249, 250, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 273, 274, 275, 276, 277, 278, 279, 280, + 281, 0, 282, 0, 283, 284, 285, 286, 287, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 295, + 296, 297, 0, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 1560, 1561, 1562, 330, 331, 332, 0, 0, - 334, 335, 1563, 337, 0, 0, 339, 1564, 341, 342, - 343, 0, 344, 345, 0, 0, 346, 347, 348, 0, - 0, 349, 350, 0, 1565, 353, 1566, 0, 356, 357, + 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, + 335, 0, 336, 337, 338, 339, 0, 810, 341, 342, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 353, 354, 355, 356, 812, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 0, 1567, 372, 373, - 0, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 1568, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 1569, - 410, 411, 412, 1570, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 1571, 427, - 428, 429, 430, 431, 432, 1572, 434, 435, 0, 1573, - 437, 438, 1574, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 1575, - 456, 0, 0, 0, 458, 459, 0, 460, 1576, 462, - 463, 464, 465, 466, 467, 0, 468, 1577, 1578, 0, - 0, 471, 472, 0, 474, 0, 0, 476, 477, 1579, - 479, 480, 481, 482, 483, 1580, 0, 484, 485, 486, - 1581, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 0, 1582, 498, 0, 499, 1583, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 537, 0, 562, 0, - 0, 0, 0, 0, 0, 0, 0, 526, 527, 528, - 529, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 3, 4, - 0, 0, 0, 0, 0, 0, 0, 132, 133, 0, - 0, 135, 136, 0, 138, 139, 140, 141, 142, 0, - 144, 145, 0, 0, 146, 147, 148, 149, 150, 0, - 0, 151, 152, 153, 154, 155, 156, 157, 0, 158, - 159, 160, 161, 162, 0, 0, 0, 164, 165, 166, - 167, 168, 169, 0, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 0, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 209, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, 0, 231, 0, 232, 233, - 234, 235, 0, 236, 0, 237, 0, 0, 0, 240, - 241, 538, 0, 244, 245, 246, 0, 247, 248, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 0, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 0, 272, 273, 274, 275, 276, - 277, 278, 279, 0, 280, 0, 281, 0, 0, 284, - 0, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 0, 294, 0, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 539, 305, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 0, 329, 330, - 331, 332, 333, 0, 334, 335, 0, 337, 0, 338, - 339, 340, 341, 342, 343, 0, 344, 345, 0, 0, - 346, 347, 348, 0, 0, 349, 350, 351, 0, 353, - 0, 355, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 370, 0, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 0, 394, 395, 396, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 0, 410, 411, 412, 413, 414, 415, - 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 0, 437, 438, 439, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 540, 456, 457, 0, 0, 458, 459, - 0, 460, 0, 462, 463, 464, 465, 466, 467, 0, - 468, 469, 470, 0, 0, 471, 472, 473, 474, 475, - 0, 476, 477, 478, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 497, 498, 0, - 499, 0, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, - 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 537, 0, 562, 0, 0, + 368, 369, 0, 0, 0, 0, 370, 371, 813, 373, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 438, 439, 440, 441, 442, 0, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 457, 458, 815, 0, 0, 460, 461, 0, 462, + 463, 464, 465, 466, 467, 468, 469, 0, 470, 471, + 472, 0, 0, 473, 474, 816, 476, 817, 0, 478, + 479, 818, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 499, 500, 0, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, + 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, + 529, 530, 531, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 120, 121, 122, 123, 124, - 125, 126, 127, 563, 128, 129, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 132, 133, 0, 0, - 135, 136, 0, 138, 139, 140, 141, 142, 0, 144, - 145, 0, 0, 146, 147, 148, 149, 150, 0, 0, - 151, 152, 153, 154, 155, 156, 157, 0, 158, 159, - 160, 161, 162, 0, 0, 0, 164, 165, 166, 167, - 168, 169, 0, 171, 172, 173, 0, 174, 175, 176, - 177, 178, 179, 0, 0, 181, 182, 183, 184, 185, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 0, 196, 0, 197, 198, 199, 200, 201, 202, 0, - 0, 203, 204, 205, 206, 0, 0, 207, 208, 209, - 210, 211, 0, 212, 213, 214, 0, 215, 216, 217, - 218, 0, 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 0, 231, 0, 232, 233, 234, - 235, 0, 236, 0, 237, 0, 0, 0, 240, 241, - 538, 0, 244, 245, 246, 0, 247, 248, 249, 250, - 0, 251, 252, 253, 254, 255, 256, 257, 0, 259, - 260, 261, 262, 0, 263, 264, 265, 266, 267, 268, - 269, 0, 270, 0, 272, 273, 274, 275, 276, 277, - 278, 279, 0, 280, 0, 281, 0, 0, 284, 0, - 286, 287, 288, 0, 289, 290, 291, 0, 0, 292, - 0, 294, 0, 0, 296, 297, 298, 299, 300, 301, - 302, 303, 539, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 0, 329, 330, 331, - 332, 333, 0, 334, 335, 0, 337, 0, 338, 339, - 340, 341, 342, 343, 0, 344, 345, 0, 0, 346, - 347, 348, 0, 0, 349, 350, 351, 0, 353, 0, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 0, 0, 0, 0, 368, 369, 370, - 0, 372, 373, 374, 564, 376, 377, 0, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 0, 394, 395, 396, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 0, - 407, 408, 0, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 0, 0, 427, 428, 429, 430, 431, 432, 433, 434, - 435, 0, 0, 437, 438, 439, 440, 0, 441, 442, - 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, - 453, 454, 540, 456, 457, 0, 0, 458, 459, 0, - 460, 0, 462, 463, 464, 465, 466, 467, 0, 468, - 469, 470, 0, 0, 471, 472, 473, 474, 475, 0, - 476, 477, 478, 479, 480, 481, 482, 483, 0, 0, - 484, 485, 486, 0, 0, 487, 488, 489, 490, 0, - 491, 492, 493, 494, 495, 496, 497, 498, 0, 499, - 0, 501, 502, 503, 504, 505, 506, 507, 0, 0, - 508, 0, 0, 509, 510, 511, 512, 513, 514, 515, - 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, - 526, 527, 528, 529, 537, 0, 562, 0, 0, 0, + 0, 0, 0, 122, 123, 124, 125, 126, 127, 128, + 129, 0, 130, 131, 132, 0, 0, 0, 0, 0, + 0, 0, 0, 133, 134, 135, 0, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 147, 0, + 0, 148, 149, 150, 151, 152, 0, 0, 153, 154, + 155, 156, 157, 158, 159, 0, 160, 161, 162, 163, + 164, 0, 165, 0, 166, 167, 168, 169, 170, 171, + 172, 173, 174, 175, 0, 176, 177, 178, 179, 180, + 181, 0, 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 0, 198, + 0, 199, 200, 201, 202, 203, 204, 0, 0, 205, + 206, 207, 208, 0, 0, 209, 210, 211, 212, 213, + 0, 214, 215, 216, 0, 217, 218, 219, 220, 0, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + 231, 232, 0, 233, 0, 234, 235, 236, 237, 0, + 238, 0, 239, 240, 0, 241, 242, 243, 244, 245, + 246, 247, 248, 0, 249, 250, 251, 252, 0, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 0, 265, 266, 267, 268, 269, 270, 271, 0, + 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, + 0, 282, 0, 283, 284, 285, 286, 287, 288, 289, + 290, 0, 291, 292, 293, 0, 0, 294, 295, 296, + 297, 0, 298, 299, 300, 301, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, + 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, + 0, 336, 337, 338, 339, 0, 340, 341, 342, 343, + 344, 345, 0, 346, 347, 0, 0, 348, 349, 350, + 0, 0, 351, 352, 353, 354, 355, 356, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 0, 0, 0, 0, 370, 371, 372, 373, 374, + 375, 376, 377, 378, 379, 0, 380, 381, 382, 383, + 384, 385, 0, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 0, 409, 410, + 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 426, 427, 428, 0, 0, + 429, 430, 431, 432, 433, 434, 435, 436, 437, 0, + 438, 439, 440, 441, 442, 0, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 457, 458, 459, 0, 0, 460, 461, 0, 462, 463, + 464, 465, 466, 467, 468, 469, 0, 470, 471, 472, + 0, 0, 473, 474, 475, 476, 477, 0, 478, 479, + 480, 481, 482, 483, 484, 485, 0, 0, 486, 487, + 488, 0, 0, 489, 490, 491, 492, 0, 493, 494, + 495, 496, 497, 498, 499, 500, 0, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 0, 0, 510, 0, + 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, + 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, + 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 132, 133, 0, 0, 135, - 136, 0, 138, 139, 140, 141, 142, 0, 144, 145, - 0, 0, 146, 147, 148, 149, 150, 0, 0, 151, - 152, 153, 154, 155, 156, 157, 0, 158, 159, 160, - 161, 162, 0, 0, 0, 164, 165, 166, 167, 168, - 169, 0, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 0, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 0, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 209, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 0, 231, 0, 232, 233, 234, 235, - 0, 236, 0, 237, 0, 0, 0, 240, 241, 538, - 0, 244, 245, 246, 0, 247, 248, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 0, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 0, 272, 273, 274, 275, 276, 277, 278, - 279, 0, 280, 0, 281, 0, 0, 284, 0, 286, - 287, 288, 0, 289, 290, 291, 0, 0, 292, 0, - 294, 0, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 539, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, 0, 329, 330, 331, 332, - 333, 0, 334, 335, 0, 337, 0, 338, 339, 340, - 341, 342, 343, 0, 344, 345, 0, 808, 346, 347, - 348, 0, 0, 349, 350, 351, 0, 353, 0, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 370, 0, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 0, 410, 411, 412, 413, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 0, 437, 438, 439, 440, 0, 441, 442, 443, - 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 540, 456, 457, 0, 0, 458, 459, 0, 460, - 0, 462, 463, 464, 465, 466, 467, 0, 468, 469, - 470, 0, 0, 471, 472, 473, 474, 475, 0, 476, - 477, 478, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 0, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 497, 498, 0, 499, 0, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, - 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 537, 0, 562, 0, 0, 0, 0, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 134, 135, 0, 0, 137, 138, 0, + 140, 141, 142, 143, 144, 0, 146, 147, 0, 0, + 148, 149, 150, 151, 152, 0, 0, 153, 154, 155, + 156, 157, 158, 159, 1812, 160, 161, 162, 163, 164, + 0, 0, 1813, 166, 167, 168, 169, 170, 171, 0, + 173, 174, 175, 1814, 176, 177, 178, 179, 180, 181, + 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 211, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 0, 233, 0, 234, 235, 236, 237, 0, 238, + 1815, 239, 0, 0, 0, 242, 243, 540, 0, 246, + 247, 248, 0, 249, 250, 251, 252, 0, 253, 254, + 255, 256, 257, 1816, 259, 0, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 0, 274, 275, 276, 277, 278, 279, 280, 281, 0, + 282, 0, 283, 0, 0, 286, 0, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 0, 296, 0, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 541, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 327, 328, 329, 0, 331, 332, 333, 334, 335, 0, + 336, 337, 0, 339, 0, 340, 341, 342, 343, 344, + 345, 0, 346, 347, 0, 0, 348, 349, 350, 0, + 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 372, 0, 374, 375, + 376, 377, 378, 379, 1817, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 0, + 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 0, + 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 542, + 458, 459, 0, 0, 460, 461, 0, 462, 0, 464, + 465, 466, 467, 468, 469, 0, 470, 471, 472, 0, + 0, 473, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 485, 0, 1818, 486, 487, 488, + 0, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 499, 500, 0, 501, 0, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, + 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 120, 121, 122, 123, 124, 125, 126, - 127, 0, 128, 129, 130, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 133, 0, 0, 135, 136, - 0, 138, 139, 140, 141, 142, 0, 144, 145, 0, - 0, 146, 147, 148, 149, 150, 0, 0, 151, 152, - 153, 154, 155, 156, 157, 0, 158, 159, 160, 161, - 162, 0, 0, 0, 164, 165, 166, 167, 168, 169, - 0, 171, 172, 173, 0, 174, 175, 176, 177, 178, - 179, 0, 0, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 0, 196, - 0, 197, 198, 199, 200, 201, 202, 0, 0, 203, - 204, 205, 206, 0, 0, 207, 208, 209, 210, 211, - 0, 212, 213, 214, 0, 215, 216, 217, 218, 0, - 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, 0, 231, 0, 232, 233, 234, 235, 0, - 236, 0, 237, 0, 0, 0, 240, 241, 538, 0, - 244, 245, 246, 0, 247, 248, 249, 250, 0, 251, - 252, 253, 254, 255, 918, 257, 0, 259, 260, 261, - 262, 0, 263, 264, 265, 266, 267, 268, 269, 0, - 270, 0, 272, 273, 274, 275, 276, 277, 278, 279, - 0, 280, 0, 281, 0, 0, 284, 0, 286, 287, - 288, 0, 289, 290, 291, 0, 0, 292, 0, 294, - 0, 0, 296, 297, 298, 299, 300, 301, 302, 303, - 539, 305, 306, 307, 308, 309, 310, 311, 312, 313, - 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 327, 0, 329, 330, 331, 332, 333, - 0, 334, 335, 0, 337, 0, 338, 339, 340, 341, - 342, 343, 0, 344, 345, 0, 808, 346, 347, 348, - 0, 0, 349, 350, 351, 0, 353, 0, 355, 356, - 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 0, 0, 0, 0, 368, 369, 370, 0, 372, - 373, 374, 375, 376, 377, 0, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 0, 394, 395, 396, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 0, 407, 408, - 0, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 426, 0, 0, - 427, 428, 429, 430, 431, 432, 433, 434, 435, 0, - 0, 437, 438, 439, 440, 0, 441, 442, 443, 444, - 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 540, 456, 457, 0, 0, 458, 459, 0, 460, 0, - 462, 463, 464, 465, 466, 467, 0, 468, 469, 470, - 0, 0, 471, 472, 473, 474, 475, 0, 476, 477, - 478, 479, 480, 481, 482, 483, 0, 0, 484, 485, - 486, 0, 0, 487, 488, 489, 490, 0, 491, 492, - 493, 494, 495, 496, 497, 498, 0, 499, 0, 501, - 502, 503, 504, 505, 506, 507, 0, 0, 508, 0, - 0, 509, 510, 511, 512, 513, 514, 515, 516, 517, - 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, - 528, 529, 537, 0, 562, 0, 0, 0, 0, 0, + 0, 122, 123, 124, 125, 126, 127, 128, 129, 0, + 130, 131, 132, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 134, 135, 0, 0, 137, 138, 0, 140, + 141, 142, 143, 144, 0, 146, 147, 0, 0, 148, + 149, 150, 151, 152, 0, 0, 153, 154, 155, 156, + 157, 158, 159, 1812, 160, 161, 162, 163, 164, 0, + 0, 0, 166, 167, 168, 169, 170, 171, 0, 173, + 174, 175, 1814, 176, 177, 178, 179, 180, 181, 0, + 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 0, 198, 0, 199, + 200, 201, 202, 203, 204, 0, 0, 205, 206, 207, + 208, 0, 0, 209, 210, 211, 212, 213, 0, 214, + 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, + 0, 233, 0, 234, 235, 236, 237, 0, 238, 1815, + 239, 0, 0, 0, 242, 243, 540, 0, 246, 247, + 248, 0, 249, 250, 251, 252, 0, 253, 254, 255, + 256, 257, 258, 259, 0, 261, 262, 263, 264, 0, + 265, 266, 267, 268, 269, 270, 271, 0, 272, 0, + 274, 275, 276, 277, 278, 279, 280, 281, 0, 282, + 0, 283, 0, 0, 286, 0, 288, 289, 290, 0, + 291, 292, 293, 0, 0, 294, 0, 296, 2440, 0, + 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, + 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, + 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, + 328, 329, 0, 331, 332, 333, 334, 335, 0, 336, + 337, 0, 339, 0, 340, 341, 342, 343, 344, 345, + 0, 346, 347, 0, 0, 348, 349, 350, 0, 0, + 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 0, + 0, 0, 0, 370, 371, 372, 0, 374, 375, 376, + 377, 378, 379, 1817, 380, 381, 382, 383, 384, 385, + 0, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, + 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, + 423, 424, 425, 426, 427, 428, 0, 0, 429, 430, + 431, 432, 433, 434, 435, 436, 437, 0, 0, 439, + 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 456, 542, 458, + 459, 0, 0, 460, 461, 0, 462, 0, 464, 465, + 466, 467, 468, 469, 0, 470, 471, 472, 0, 0, + 473, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 485, 0, 1818, 486, 487, 488, 0, + 0, 489, 490, 491, 492, 0, 493, 494, 495, 496, + 497, 498, 499, 500, 0, 501, 0, 503, 504, 505, + 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, + 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, + 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, + 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 963, 128, 129, 130, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 132, 133, 0, 0, 135, 136, 0, - 138, 139, 140, 141, 142, 0, 144, 145, 0, 0, - 146, 147, 148, 149, 150, 0, 0, 151, 152, 153, - 154, 155, 156, 157, 0, 158, 159, 160, 161, 162, - 0, 0, 0, 164, 165, 166, 167, 168, 169, 0, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 0, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 209, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, 0, 231, 0, 232, 233, 234, 235, 0, 236, - 0, 237, 0, 0, 0, 240, 241, 538, 0, 244, - 245, 246, 0, 247, 248, 249, 250, 0, 251, 252, - 253, 254, 255, 256, 257, 0, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 0, 272, 273, 274, 275, 276, 277, 278, 279, 0, - 280, 0, 281, 0, 0, 284, 0, 286, 287, 288, - 0, 289, 290, 291, 0, 0, 292, 0, 294, 0, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 539, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 1535, 0, 0, -864, 0, + 1536, 134, 135, 0, 1537, 137, 138, 1538, 140, 141, + 142, 0, 1539, 1540, 1541, 1542, 0, 1543, 148, 149, + 150, 151, 152, 0, 0, 153, 154, 155, 156, 1544, + 1545, 159, 0, 160, 161, 162, 163, 0, 0, 1546, + 0, 1547, 167, 168, 169, 170, 171, 1548, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 1549, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 1550, 194, 195, 1551, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 1093, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 0, 226, 227, 228, 229, 230, 231, 0, 0, + 233, 0, 234, 235, 1552, 237, 0, 238, 0, 239, + 1553, 0, 1554, 242, 243, -864, 1555, 246, 247, 248, + 0, 0, 0, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 1556, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 1557, 0, + 275, 276, 277, 278, 279, 1558, 1559, 0, 1560, 0, + 283, 1561, 1562, 286, 1563, 288, 289, 290, 0, 291, + 292, 293, 0, 0, 294, 1564, 296, 1565, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 1566, 307, 308, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 1567, 1568, 1569, 332, 333, 334, 0, 0, 336, 337, + 1570, 339, 0, 0, 341, 1571, 343, 344, 345, 0, + 346, 347, 0, 0, 348, 349, 350, 0, 0, 351, + 352, 0, 1572, 355, 1573, 0, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 0, 1574, 374, 375, 0, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 1575, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 1576, 412, 413, + 414, 1577, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 1578, 429, 430, 431, + 432, 433, 434, 1579, 436, 437, 0, 1580, 439, 440, + 1581, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 1582, 458, 0, + 0, 0, 460, 461, 0, 462, 1583, 464, 465, 466, + 467, 468, 469, 0, 470, 1584, 1585, 0, 0, 473, + 474, 0, 476, 0, 0, 478, 479, 1586, 481, 482, + 483, 484, 485, 1587, 0, 486, 487, 488, 1588, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 0, 1589, 500, 0, 501, 1590, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, + 513, 514, 515, 516, 539, 0, 564, 0, 0, 0, + 0, 0, 0, 0, 0, 528, 529, 530, 531, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 3, 4, 0, 0, + 0, 0, 0, 0, 0, 134, 135, 0, 0, 137, + 138, 0, 140, 141, 142, 143, 144, 0, 146, 147, + 0, 0, 148, 149, 150, 151, 152, 0, 0, 153, + 154, 155, 156, 157, 158, 159, 0, 160, 161, 162, + 163, 164, 0, 0, 0, 166, 167, 168, 169, 170, + 171, 0, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 211, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 232, 0, 233, 0, 234, 235, 236, 237, + 0, 238, 0, 239, 0, 0, 0, 242, 243, 540, + 0, 246, 247, 248, 0, 249, 250, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 0, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 0, 274, 275, 276, 277, 278, 279, 280, + 281, 0, 282, 0, 283, 0, 0, 286, 0, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 0, + 296, 0, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 0, 329, 330, 331, 332, 333, 0, - 334, 335, 0, 337, 0, 338, 339, 340, 341, 342, - 343, 0, 344, 345, 0, 0, 346, 347, 348, 0, - 0, 349, 350, 351, 0, 353, 0, 355, 356, 357, + 325, 326, 327, 328, 329, 0, 331, 332, 333, 334, + 335, 0, 336, 337, 0, 339, 0, 340, 341, 342, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 370, 0, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 0, - 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 0, - 437, 438, 439, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 540, - 456, 457, 0, 0, 458, 459, 0, 460, 0, 462, - 463, 464, 465, 466, 467, 0, 468, 469, 470, 0, - 0, 471, 472, 473, 474, 475, 0, 476, 477, 478, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 0, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 497, 498, 0, 499, 0, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 368, 369, 0, 0, 0, 0, 370, 371, 372, 0, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 542, 458, 459, 0, 0, 460, 461, 0, 462, + 0, 464, 465, 466, 467, 468, 469, 0, 470, 471, + 472, 0, 0, 473, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 499, 500, 0, 501, 0, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 537, 0, 562, 0, 0, 0, 0, 0, 0, + 529, 530, 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 120, 121, 122, 123, 124, 125, 126, 127, 0, - 128, 129, 130, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 132, 133, 0, 0, 135, 136, 0, 138, - 139, 140, 141, 142, 0, 144, 145, 0, 0, 146, - 147, 148, 149, 150, 0, 0, 151, 152, 153, 154, - 155, 156, 157, 0, 158, 159, 160, 161, 162, 0, - 0, 0, 164, 165, 166, 167, 168, 169, 0, 171, - 172, 173, 0, 174, 175, 176, 177, 178, 179, 0, - 0, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 0, 196, 0, 197, - 198, 199, 200, 201, 202, 0, 0, 203, 204, 205, - 206, 0, 0, 207, 208, 209, 210, 211, 0, 212, - 213, 214, 0, 215, 216, 217, 218, 0, 219, 220, - 221, 222, 223, 224, 225, 226, 227, 1202, 229, 230, - 0, 231, 0, 232, 233, 234, 235, 0, 236, 0, - 237, 0, 0, 0, 240, 241, 538, 0, 244, 245, - 246, 0, 247, 248, 249, 250, 0, 251, 252, 253, - 254, 255, 256, 257, 0, 259, 260, 261, 262, 0, - 263, 264, 265, 266, 267, 268, 269, 0, 270, 0, - 272, 273, 274, 275, 276, 277, 278, 279, 0, 280, - 0, 281, 0, 0, 284, 0, 286, 287, 288, 0, - 289, 290, 291, 0, 0, 292, 0, 294, 0, 0, - 296, 297, 298, 299, 300, 301, 302, 303, 539, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 0, 0, 0, 122, 123, 124, 125, 126, 127, 128, + 129, 565, 130, 131, 132, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 134, 135, 0, 0, 137, 138, + 0, 140, 141, 142, 143, 144, 0, 146, 147, 0, + 0, 148, 149, 150, 151, 152, 0, 0, 153, 154, + 155, 156, 157, 158, 159, 0, 160, 161, 162, 163, + 164, 0, 0, 0, 166, 167, 168, 169, 170, 171, + 0, 173, 174, 175, 0, 176, 177, 178, 179, 180, + 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 0, 198, + 0, 199, 200, 201, 202, 203, 204, 0, 0, 205, + 206, 207, 208, 0, 0, 209, 210, 211, 212, 213, + 0, 214, 215, 216, 0, 217, 218, 219, 220, 0, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + 231, 232, 0, 233, 0, 234, 235, 236, 237, 0, + 238, 0, 239, 0, 0, 0, 242, 243, 540, 0, + 246, 247, 248, 0, 249, 250, 251, 252, 0, 253, + 254, 255, 256, 257, 258, 259, 0, 261, 262, 263, + 264, 0, 265, 266, 267, 268, 269, 270, 271, 0, + 272, 0, 274, 275, 276, 277, 278, 279, 280, 281, + 0, 282, 0, 283, 0, 0, 286, 0, 288, 289, + 290, 0, 291, 292, 293, 0, 0, 294, 0, 296, + 0, 0, 298, 299, 300, 301, 302, 303, 304, 305, + 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, 0, 329, 330, 331, 332, 333, 0, 334, - 335, 0, 337, 0, 338, 339, 340, 341, 342, 343, - 0, 344, 345, 0, 808, 346, 347, 348, 0, 0, - 349, 350, 351, 0, 353, 0, 355, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 0, - 0, 0, 0, 368, 369, 370, 0, 372, 373, 374, - 375, 376, 377, 0, 378, 379, 380, 381, 382, 383, - 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 0, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 0, 407, 408, 0, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 0, 0, 427, 428, - 429, 430, 431, 432, 433, 434, 435, 0, 0, 437, - 438, 439, 440, 0, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 540, 456, - 457, 0, 0, 458, 459, 0, 460, 0, 462, 463, - 464, 465, 466, 467, 0, 468, 469, 470, 0, 0, - 471, 472, 473, 474, 475, 0, 476, 477, 478, 479, - 480, 481, 482, 483, 0, 0, 484, 485, 486, 0, - 0, 487, 488, 489, 490, 0, 491, 492, 493, 494, - 495, 496, 497, 498, 0, 499, 0, 501, 502, 503, - 504, 505, 506, 507, 0, 0, 508, 0, 0, 509, - 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, + 326, 327, 328, 329, 0, 331, 332, 333, 334, 335, + 0, 336, 337, 0, 339, 0, 340, 341, 342, 343, + 344, 345, 0, 346, 347, 0, 0, 348, 349, 350, + 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 0, 0, 0, 0, 370, 371, 372, 0, 374, + 375, 376, 566, 378, 379, 0, 380, 381, 382, 383, + 384, 385, 0, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 0, 409, 410, + 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 426, 427, 428, 0, 0, + 429, 430, 431, 432, 433, 434, 435, 436, 437, 0, + 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 542, 458, 459, 0, 0, 460, 461, 0, 462, 0, + 464, 465, 466, 467, 468, 469, 0, 470, 471, 472, + 0, 0, 473, 474, 475, 476, 477, 0, 478, 479, + 480, 481, 482, 483, 484, 485, 0, 0, 486, 487, + 488, 0, 0, 489, 490, 491, 492, 0, 493, 494, + 495, 496, 497, 498, 499, 500, 0, 501, 0, 503, + 504, 505, 506, 507, 508, 509, 0, 0, 510, 0, + 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, - 1527, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 530, 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 1528, 0, 0, 0, 0, - 1529, 132, 133, 0, 1530, 135, 136, 1531, 138, 139, - 140, 0, 1532, 1533, 1534, 1535, 0, 1536, 146, 147, - 148, 149, 150, 0, 0, 151, 152, 153, 154, 1537, - 1538, 157, 0, 158, 159, 160, 161, 0, 0, 1539, - 0, 1540, 165, 166, 167, 168, 169, 1541, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 1542, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 1543, 192, 193, 1544, 195, 0, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 1088, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 0, 224, 225, 226, 227, 228, 229, 0, 0, - 231, 0, 232, 233, 1545, 235, 0, 236, 0, 237, - 1546, 0, 1547, 240, 241, 0, 1548, 244, 245, 246, - 0, 0, 0, 249, 250, 0, 251, 252, 253, 254, - 255, 256, 257, 1549, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 1550, 0, - 273, 274, 275, 276, 277, 1551, 1552, 0, 1553, 0, - 281, 1554, 1555, 284, 1556, 286, 287, 288, 0, 289, - 290, 291, 0, 0, 292, 1557, 294, 1558, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 1559, 305, 306, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 134, 135, 0, 0, 137, 138, 0, + 140, 141, 142, 143, 144, 0, 146, 147, 0, 0, + 148, 149, 150, 151, 152, 0, 0, 153, 154, 155, + 156, 157, 158, 159, 0, 160, 161, 162, 163, 164, + 0, 0, 0, 166, 167, 168, 169, 170, 171, 0, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 211, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 0, 233, 0, 234, 235, 236, 237, 0, 238, + 0, 239, 0, 0, 0, 242, 243, 540, 0, 246, + 247, 248, 0, 249, 250, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 0, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 0, 274, 275, 276, 277, 278, 279, 280, 281, 0, + 282, 0, 283, 0, 0, 286, 0, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 0, 296, 0, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 1560, 1561, 1562, 330, 331, 332, 0, 0, 334, 335, - 1563, 337, 0, 0, 339, 1564, 341, 342, 343, 0, - 344, 345, 0, 0, 346, 347, 348, 0, 0, 349, - 350, 0, 1565, 353, 1566, 0, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 0, 1567, 372, 373, 0, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 1568, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 1569, 410, 411, - 412, 1570, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 1571, 427, 428, 429, - 430, 431, 432, 1572, 434, 435, 0, 1573, 437, 438, - 1574, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 1575, 456, 0, - 0, 0, 458, 459, 0, 460, 1576, 462, 463, 464, - 465, 466, 467, 0, 468, 1577, 1578, 0, 0, 471, - 472, 0, 474, 0, 0, 476, 477, 1579, 479, 480, - 481, 482, 483, 1580, 0, 484, 485, 486, 1581, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 0, 1582, 498, 0, 499, 1583, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, - 511, 512, 513, 514, 537, 0, 562, 0, 0, 0, - 0, 0, 0, 0, 0, 526, 527, 528, 529, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 132, 133, 0, 0, 135, - 136, 0, 138, 139, 140, 141, 142, 0, 144, 145, - 0, 0, 146, 147, 148, 149, 150, 0, 0, 151, - 152, 153, 154, 155, 156, 157, 0, 158, 159, 160, - 161, 162, 0, 0, 0, 164, 165, 166, 167, 168, - 169, 0, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 0, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 0, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 209, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 0, 231, 0, 232, 233, 234, 235, - 0, 236, 0, 237, 0, 0, 0, 240, 241, 538, - 0, 2033, 245, 246, 0, 247, 248, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 0, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 0, 272, 273, 274, 275, 276, 277, 278, - 279, 0, 280, 0, 281, 0, 0, 284, 0, 286, - 287, 288, 0, 289, 290, 291, 0, 0, 292, 0, - 294, 0, 0, 296, 297, 2034, 299, 300, 301, 302, - 303, 539, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, 0, 329, 330, 331, 332, - 333, 0, 334, 335, 0, 337, 0, 338, 339, 340, - 341, 342, 343, 0, 344, 345, 0, 0, 346, 347, - 348, 0, 0, 349, 350, 351, 0, 353, 0, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 370, 0, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 0, 410, 411, 412, 413, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 0, 437, 438, 439, 440, 0, 441, 442, 443, - 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 540, 456, 457, 0, 0, 458, 459, 2035, 460, - 0, 462, 463, 2036, 465, 2037, 467, 0, 468, 469, - 470, 0, 0, 471, 472, 473, 474, 475, 0, 476, - 477, 478, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 2038, 0, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 497, 498, 0, 499, 0, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, - 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 1527, 0, 0, 0, 0, 0, 0, + 327, 328, 329, 0, 331, 332, 333, 334, 335, 0, + 336, 337, 0, 339, 0, 340, 341, 342, 343, 344, + 345, 0, 346, 347, 0, 811, 348, 349, 350, 0, + 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 372, 0, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 0, + 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 0, + 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 542, + 458, 459, 0, 0, 460, 461, 0, 462, 0, 464, + 465, 466, 467, 468, 469, 0, 470, 471, 472, 0, + 0, 473, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 0, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 499, 500, 0, 501, 0, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, + 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 120, 121, 122, 123, 124, 125, 126, - 127, 0, 128, 129, 130, 0, 0, 0, 1528, 0, - 0, 0, 0, 1529, 132, 133, 0, 1530, 135, 136, - 1531, 138, 139, 140, 0, 1532, 1533, 1534, 1535, 0, - 1536, 146, 147, 148, 149, 150, 0, 0, 151, 152, - 153, 154, 1537, 1538, 157, 0, 158, 159, 160, 161, - 0, 0, 1539, 0, 1540, 165, 166, 167, 168, 169, - 1541, 171, 172, 173, 0, 174, 175, 176, 177, 178, - 179, 0, 1542, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 1543, 192, 193, 1544, 195, 0, 196, - 0, 197, 198, 199, 200, 201, 202, 0, 0, 203, - 204, 205, 206, 0, 0, 207, 208, 1088, 210, 211, - 0, 212, 213, 214, 0, 2442, 216, 217, 218, 0, - 219, 220, 221, 222, 0, 224, 225, 226, 227, 228, - 229, 0, 0, 231, 0, 232, 233, 1545, 235, 0, - 236, 0, 237, 1546, 0, 1547, 240, 241, 0, 1548, - 244, 245, 246, 0, 0, 0, 249, 250, 0, 251, - 252, 253, 254, 255, 256, 257, 1549, 259, 260, 261, - 262, 0, 263, 264, 265, 266, 267, 268, 269, 0, - 270, 1550, 0, 273, 274, 275, 276, 277, 1551, 1552, - 0, 1553, 0, 281, 1554, 1555, 284, 1556, 286, 287, - 288, 0, 289, 290, 291, 0, 0, 292, 1557, 294, - 1558, 0, 296, 297, 298, 299, 300, 301, 302, 303, - 1559, 305, 306, 307, 308, 309, 310, 311, 312, 313, - 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 1560, 1561, 1562, 330, 331, 332, 0, - 0, 334, 335, 1563, 337, 0, 0, 339, 1564, 341, - 342, 343, 0, 344, 345, 0, 0, 346, 347, 348, - 0, 0, 349, 350, 0, 1565, 353, 1566, 0, 356, - 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 0, 0, 0, 0, 368, 369, 0, 1567, 372, - 373, 0, 375, 376, 377, 0, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 1568, 390, - 391, 392, 393, 0, 394, 395, 396, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 0, 407, 408, - 1569, 410, 411, 412, 1570, 414, 415, 416, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 426, 0, 1571, - 427, 428, 429, 430, 431, 432, 1572, 434, 435, 0, - 1573, 437, 438, 1574, 440, 0, 441, 442, 443, 444, - 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 1575, 456, 0, 0, 0, 458, 459, 0, 460, 1576, - 462, 463, 464, 465, 466, 467, 0, 468, 1577, 1578, - 0, 0, 471, 472, 0, 474, 0, 0, 476, 477, - 1579, 479, 480, 481, 482, 483, 1580, 0, 484, 485, - 486, 1581, 0, 487, 488, 489, 490, 0, 491, 492, - 493, 494, 495, 0, 1582, 498, 0, 499, 1583, 501, - 502, 503, 504, 505, 506, 507, 0, 0, 508, 0, - 0, 509, 510, 511, 512, 513, 514, 537, 0, 562, - 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, - 528, 529, 0, 0, 0, 0, 0, 120, 121, 122, - 123, 124, 125, 126, 127, 0, 128, 129, 130, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 132, 133, - 0, 0, 135, 136, 0, 138, 139, 140, 141, 142, - 0, 144, 145, 0, 0, 146, 147, 148, 149, 150, - 0, 0, 151, 152, 153, 154, 155, 156, 157, 0, - 158, 159, 160, 161, 162, 0, 0, 0, 164, 165, - 166, 167, 168, 169, 0, 171, 172, 173, 0, 174, - 175, 176, 177, 178, 179, 0, 0, 181, 182, 183, + 0, 122, 123, 124, 125, 126, 127, 128, 129, 0, + 130, 131, 132, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 134, 135, 0, 0, 137, 138, 0, 140, + 141, 142, 143, 144, 0, 146, 147, 0, 0, 148, + 149, 150, 151, 152, 0, 0, 153, 154, 155, 156, + 157, 158, 159, 0, 160, 161, 162, 163, 164, 0, + 0, 0, 166, 167, 168, 169, 170, 171, 0, 173, + 174, 175, 0, 176, 177, 178, 179, 180, 181, 0, + 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 0, 198, 0, 199, + 200, 201, 202, 203, 204, 0, 0, 205, 206, 207, + 208, 0, 0, 209, 210, 211, 212, 213, 0, 214, + 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, + 0, 233, 0, 234, 235, 236, 237, 0, 238, 0, + 239, 0, 0, 0, 242, 243, 540, 0, 246, 247, + 248, 0, 249, 250, 251, 252, 0, 253, 254, 255, + 256, 257, 923, 259, 0, 261, 262, 263, 264, 0, + 265, 266, 267, 268, 269, 270, 271, 0, 272, 0, + 274, 275, 276, 277, 278, 279, 280, 281, 0, 282, + 0, 283, 0, 0, 286, 0, 288, 289, 290, 0, + 291, 292, 293, 0, 0, 294, 0, 296, 0, 0, + 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, + 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, + 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, + 328, 329, 0, 331, 332, 333, 334, 335, 0, 336, + 337, 0, 339, 0, 340, 341, 342, 343, 344, 345, + 0, 346, 347, 0, 811, 348, 349, 350, 0, 0, + 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 0, + 0, 0, 0, 370, 371, 372, 0, 374, 375, 376, + 377, 378, 379, 0, 380, 381, 382, 383, 384, 385, + 0, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, + 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, + 423, 424, 425, 426, 427, 428, 0, 0, 429, 430, + 431, 432, 433, 434, 435, 436, 437, 0, 0, 439, + 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 456, 542, 458, + 459, 0, 0, 460, 461, 0, 462, 0, 464, 465, + 466, 467, 468, 469, 0, 470, 471, 472, 0, 0, + 473, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 485, 0, 0, 486, 487, 488, 0, + 0, 489, 490, 491, 492, 0, 493, 494, 495, 496, + 497, 498, 499, 500, 0, 501, 0, 503, 504, 505, + 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, + 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, + 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, + 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 968, 130, + 131, 132, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 134, 135, 0, 0, 137, 138, 0, 140, 141, + 142, 143, 144, 0, 146, 147, 0, 0, 148, 149, + 150, 151, 152, 0, 0, 153, 154, 155, 156, 157, + 158, 159, 0, 160, 161, 162, 163, 164, 0, 0, + 0, 166, 167, 168, 169, 170, 171, 0, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 0, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 193, 194, 195, 196, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 211, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 0, + 233, 0, 234, 235, 236, 237, 0, 238, 0, 239, + 0, 0, 0, 242, 243, 540, 0, 246, 247, 248, + 0, 249, 250, 251, 252, 0, 253, 254, 255, 256, + 257, 258, 259, 0, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 0, 274, + 275, 276, 277, 278, 279, 280, 281, 0, 282, 0, + 283, 0, 0, 286, 0, 288, 289, 290, 0, 291, + 292, 293, 0, 0, 294, 0, 296, 0, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 0, 331, 332, 333, 334, 335, 0, 336, 337, + 0, 339, 0, 340, 341, 342, 343, 344, 345, 0, + 346, 347, 0, 0, 348, 349, 350, 0, 0, 351, + 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 372, 0, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 0, 439, 440, + 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 542, 458, 459, + 0, 0, 460, 461, 0, 462, 0, 464, 465, 466, + 467, 468, 469, 0, 470, 471, 472, 0, 0, 473, + 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 0, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 499, 500, 0, 501, 0, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, + 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 539, + 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, + 123, 124, 125, 126, 127, 128, 129, 0, 130, 131, + 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 134, 135, 0, 0, 137, 138, 0, 140, 141, 142, + 143, 144, 0, 146, 147, 0, 0, 148, 149, 150, + 151, 152, 0, 0, 153, 154, 155, 156, 157, 158, + 159, 0, 160, 161, 162, 163, 164, 0, 0, 0, + 166, 167, 168, 169, 170, 171, 0, 173, 174, 175, + 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 0, 196, 0, 197, 198, 199, 200, 201, - 202, 0, 0, 203, 204, 205, 206, 0, 0, 207, - 208, 209, 210, 211, 0, 212, 213, 214, 0, 215, - 216, 217, 218, 0, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 0, 231, 0, 232, - 233, 234, 235, 0, 236, 0, 237, 0, 0, 0, - 240, 241, 538, 0, 244, 245, 246, 0, 247, 248, - 249, 250, 0, 251, 252, 253, 254, 255, 256, 257, - 0, 259, 260, 261, 262, 0, 263, 264, 265, 266, - 267, 268, 269, 0, 270, 0, 272, 273, 274, 275, - 276, 277, 278, 279, 0, 280, 0, 281, 0, 0, - 284, 0, 286, 287, 288, 0, 289, 290, 291, 0, - 0, 292, 0, 294, 0, 0, 296, 297, 298, 299, - 300, 301, 302, 303, 539, 305, 306, 307, 308, 309, + 194, 195, 196, 197, 0, 198, 0, 199, 200, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 0, + 0, 209, 210, 211, 212, 213, 0, 214, 215, 216, + 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 1207, 231, 232, 0, 233, + 0, 234, 235, 236, 237, 0, 238, 0, 239, 0, + 0, 0, 242, 243, 540, 0, 246, 247, 248, 0, + 249, 250, 251, 252, 0, 253, 254, 255, 256, 257, + 258, 259, 0, 261, 262, 263, 264, 0, 265, 266, + 267, 268, 269, 270, 271, 0, 272, 0, 274, 275, + 276, 277, 278, 279, 280, 281, 0, 282, 0, 283, + 0, 0, 286, 0, 288, 289, 290, 0, 291, 292, + 293, 0, 0, 294, 0, 296, 0, 0, 298, 299, + 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, 0, 329, - 330, 331, 332, 333, 0, 334, 335, 0, 337, 0, - 338, 339, 340, 341, 342, 343, 0, 344, 345, 0, - 0, 346, 347, 348, 0, 0, 349, 350, 351, 0, - 353, 0, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 0, 0, 0, 0, 368, - 369, 370, 0, 372, 373, 374, 375, 376, 377, 0, - 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, - 387, 388, 389, 390, 391, 392, 393, 0, 394, 395, + 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 0, 331, 332, 333, 334, 335, 0, 336, 337, 0, + 339, 0, 340, 341, 342, 343, 344, 345, 0, 346, + 347, 0, 811, 348, 349, 350, 0, 0, 351, 352, + 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 0, 0, 0, + 0, 370, 371, 372, 0, 374, 375, 376, 377, 378, + 379, 0, 380, 381, 382, 383, 384, 385, 0, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 0, 407, 408, 0, 410, 411, 412, 413, 414, + 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 0, 0, 427, 428, 429, 430, 431, 432, - 433, 434, 435, 0, 0, 437, 438, 439, 440, 0, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 540, 456, 457, 0, 0, 458, - 459, 0, 460, 0, 462, 463, 464, 465, 466, 467, - 0, 468, 469, 470, 0, 0, 471, 472, 473, 474, - 475, 0, 476, 477, 478, 479, 480, 481, 482, 483, - 0, 0, 484, 485, 486, 0, 0, 487, 488, 489, - 490, 0, 491, 492, 493, 494, 495, 496, 497, 498, - 0, 499, 0, 501, 502, 503, 504, 505, 506, 507, - 0, 0, 508, 0, 0, 509, 510, 511, 512, 513, + 425, 426, 427, 428, 0, 0, 429, 430, 431, 432, + 433, 434, 435, 436, 437, 0, 0, 439, 440, 441, + 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, + 451, 452, 453, 454, 455, 456, 542, 458, 459, 0, + 0, 460, 461, 0, 462, 0, 464, 465, 466, 467, + 468, 469, 0, 470, 471, 472, 0, 0, 473, 474, + 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, + 484, 485, 0, 0, 486, 487, 488, 0, 0, 489, + 490, 491, 492, 0, 493, 494, 495, 496, 497, 498, + 499, 500, 0, 501, 0, 503, 504, 505, 506, 507, + 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, - 524, 525, 526, 527, 528, 529, 537, 0, 833, 0, + 524, 525, 526, 527, 528, 529, 530, 531, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 132, 133, 0, - 0, 135, 136, 0, 138, 139, 140, 141, 142, 0, - 144, 145, 0, 0, 146, 147, 148, 149, 150, 0, - 0, 151, 152, 153, 154, 155, 156, 157, 0, 158, - 159, 160, 161, 162, 0, 0, 0, 164, 165, 166, - 167, 168, 169, 0, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 0, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 209, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, 0, 231, 0, 232, 233, - 234, 235, 0, 236, 0, 237, 0, 0, 0, 240, - 241, 538, 0, 244, 245, 246, 0, 247, 248, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 0, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 0, 272, 273, 274, 275, 276, - 277, 278, 279, 0, 280, 0, 281, 0, 0, 284, - 0, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 0, 294, 0, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 539, 305, 306, 307, 308, 309, 310, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 1535, 0, 0, 0, 0, 1536, 134, + 135, 0, 1537, 137, 138, 1538, 140, 141, 142, 0, + 1539, 1540, 1541, 1542, 0, 1543, 148, 149, 150, 151, + 152, 0, 0, 153, 154, 155, 156, 1544, 1545, 159, + 0, 160, 161, 162, 163, 0, 0, 1546, 0, 1547, + 167, 168, 169, 170, 171, 1548, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 1549, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 1550, 194, + 195, 1551, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 1093, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 0, + 226, 227, 228, 229, 230, 231, 0, 0, 233, 0, + 234, 235, 1552, 237, 0, 238, 0, 239, 1553, 0, + 1554, 242, 243, 0, 1555, 246, 247, 248, 0, 0, + 0, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 1556, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 1557, 0, 275, 276, + 277, 278, 279, 1558, 1559, 0, 1560, 0, 283, 1561, + 1562, 286, 1563, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 1564, 296, 1565, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 1566, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 0, 329, 330, - 331, 332, 333, 0, 334, 335, 0, 337, 0, 338, - 339, 340, 341, 342, 343, 0, 344, 345, 0, 0, - 346, 347, 348, 0, 0, 349, 350, 351, 0, 353, - 0, 355, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 370, 0, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 1567, 1568, + 1569, 332, 333, 334, 0, 0, 336, 337, 1570, 339, + 0, 0, 341, 1571, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 0, + 1572, 355, 1573, 0, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 0, 1574, 374, 375, 0, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 1575, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 0, 410, 411, 412, 413, 414, 415, + 407, 408, 0, 409, 410, 1576, 412, 413, 414, 1577, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 0, 437, 438, 439, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 540, 456, 457, 0, 0, 458, 459, - 0, 460, 0, 462, 463, 464, 465, 466, 467, 0, - 468, 469, 470, 0, 0, 471, 472, 473, 474, 475, - 0, 476, 477, 478, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 497, 498, 0, - 499, 0, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, - 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 537, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 120, 121, 122, 123, 124, - 125, 126, 127, 839, 128, 129, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 132, 133, 0, 0, - 135, 136, 0, 138, 139, 140, 141, 142, 0, 144, - 145, 0, 0, 146, 147, 148, 149, 150, 0, 0, - 151, 152, 153, 154, 155, 156, 157, 0, 158, 159, - 160, 161, 162, 0, 0, 0, 164, 165, 166, 167, - 168, 169, 0, 171, 172, 173, 0, 174, 175, 176, - 177, 178, 179, 0, 0, 181, 182, 183, 184, 185, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 0, 196, 0, 197, 198, 199, 200, 201, 202, 0, - 0, 203, 204, 205, 206, 0, 0, 207, 208, 209, - 210, 211, 0, 212, 213, 214, 0, 215, 216, 217, - 218, 0, 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 0, 231, 0, 232, 233, 234, - 235, 0, 236, 0, 237, 0, 0, 0, 240, 241, - 538, 0, 840, 245, 246, 0, 247, 248, 249, 250, - 0, 251, 252, 253, 254, 255, 256, 257, 0, 259, - 260, 261, 262, 0, 263, 264, 265, 266, 267, 268, - 269, 0, 270, 0, 272, 273, 274, 275, 276, 277, - 278, 279, 0, 280, 0, 281, 0, 0, 284, 0, - 286, 287, 288, 0, 289, 290, 291, 0, 0, 292, - 0, 294, 0, 0, 296, 297, 841, 299, 300, 301, - 302, 303, 539, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 0, 329, 330, 331, - 332, 333, 0, 334, 335, 0, 337, 0, 338, 339, - 340, 341, 342, 343, 0, 344, 345, 0, 0, 346, - 347, 348, 0, 0, 349, 350, 351, 0, 353, 0, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 0, 0, 0, 0, 368, 369, 370, - 0, 372, 373, 374, 375, 376, 377, 0, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 0, 394, 395, 396, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 0, - 407, 408, 0, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 0, 0, 427, 428, 429, 430, 842, 432, 433, 434, - 435, 0, 0, 437, 438, 439, 440, 0, 441, 442, - 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, - 453, 454, 540, 456, 457, 0, 0, 458, 459, 0, - 460, 0, 462, 463, 464, 465, 466, 467, 0, 468, - 843, 470, 0, 0, 844, 472, 473, 474, 475, 0, - 476, 477, 478, 479, 480, 481, 482, 483, 0, 0, - 484, 485, 486, 0, 0, 487, 488, 489, 490, 0, - 491, 492, 493, 494, 495, 496, 497, 845, 0, 499, - 0, 501, 502, 503, 504, 505, 506, 507, 0, 0, - 508, 0, 0, 509, 510, 511, 512, 513, 514, 515, - 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, - 526, 527, 528, 529, 537, 0, 562, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 132, 133, 0, 0, 135, - 136, 0, 138, 139, 140, 141, 142, 0, 144, 145, - 0, 0, 146, 147, 148, 149, 150, 0, 0, 151, - 152, 153, 154, 155, 156, 157, 0, 158, 159, 160, - 161, 162, 0, 0, 0, 164, 165, 166, 167, 168, - 169, 0, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 0, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 0, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 209, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 0, 231, 0, 232, 233, 234, 235, - 0, 236, 0, 237, 0, 0, 0, 240, 241, 538, - 0, 244, 245, 246, 0, 247, 248, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 0, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 0, 272, 273, 274, 275, 276, 277, 278, - 279, 0, 280, 0, 281, 0, 0, 284, 0, 286, - 287, 288, 0, 289, 290, 291, 0, 0, 292, 0, - 294, 0, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 539, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, 0, 329, 330, 331, 332, - 333, 0, 334, 335, 0, 337, 0, 338, 339, 340, - 341, 342, 343, 0, 344, 345, 0, 0, 346, 347, - 348, 0, 0, 349, 350, 351, 0, 353, 0, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 370, 0, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, - 390, 391, 878, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 0, 410, 411, 412, 413, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 0, 437, 438, 439, 440, 0, 441, 442, 443, - 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 540, 456, 457, 0, 0, 458, 459, 0, 460, - 0, 462, 463, 464, 465, 466, 467, 0, 468, 469, - 470, 0, 0, 471, 472, 473, 474, 475, 0, 476, - 477, 478, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 0, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 497, 498, 0, 499, 0, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, - 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 537, 0, 562, 0, 0, 0, 0, + 426, 427, 428, 0, 1578, 429, 430, 431, 432, 433, + 434, 1579, 436, 437, 0, 1580, 439, 440, 1581, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 1582, 458, 0, 0, 0, + 460, 461, 0, 462, 1583, 464, 465, 466, 467, 468, + 469, 0, 470, 1584, 1585, 0, 0, 473, 474, 0, + 476, 0, 0, 478, 479, 1586, 481, 482, 483, 484, + 485, 1587, 0, 486, 487, 488, 1588, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 0, 1589, + 500, 0, 501, 1590, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, + 515, 516, 539, 0, 564, 0, 0, 0, 0, 0, + 0, 0, 0, 528, 529, 530, 531, 0, 0, 0, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 134, 135, 0, 0, 137, 138, 0, + 140, 141, 142, 143, 144, 0, 146, 147, 0, 0, + 148, 149, 150, 151, 152, 0, 0, 153, 154, 155, + 156, 157, 158, 159, 0, 160, 161, 162, 163, 164, + 0, 0, 0, 166, 167, 168, 169, 170, 171, 0, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 211, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 0, 233, 0, 234, 235, 236, 237, 0, 238, + 0, 239, 0, 0, 0, 242, 243, 540, 0, 2042, + 247, 248, 0, 249, 250, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 0, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 0, 274, 275, 276, 277, 278, 279, 280, 281, 0, + 282, 0, 283, 0, 0, 286, 0, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 0, 296, 0, + 0, 298, 299, 2043, 301, 302, 303, 304, 305, 541, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 327, 328, 329, 0, 331, 332, 333, 334, 335, 0, + 336, 337, 0, 339, 0, 340, 341, 342, 343, 344, + 345, 0, 346, 347, 0, 0, 348, 349, 350, 0, + 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 372, 0, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 0, + 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 0, + 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 542, + 458, 459, 0, 0, 460, 461, 2044, 462, 0, 464, + 465, 2045, 467, 2046, 469, 0, 470, 471, 472, 0, + 0, 473, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 2047, + 0, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 499, 500, 0, 501, 0, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, + 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 120, 121, 122, 123, 124, 125, 126, - 127, 0, 128, 129, 130, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 133, 0, 0, 135, 136, - 0, 138, 139, 140, 141, 142, 0, 144, 145, 0, - 0, 146, 147, 148, 149, 150, 0, 0, 151, 152, - 153, 154, 155, 156, 157, 0, 158, 159, 160, 161, - 162, 0, 0, 0, 164, 165, 166, 167, 168, 169, - 0, 171, 172, 173, 0, 174, 175, 176, 177, 178, - 179, 0, 0, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 0, 196, - 0, 197, 198, 199, 200, 201, 202, 0, 0, 203, - 204, 205, 206, 0, 0, 207, 208, 209, 210, 211, - 0, 212, 213, 214, 0, 215, 216, 217, 218, 0, - 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, 0, 231, 0, 232, 233, 234, 235, 0, - 236, 0, 237, 0, 0, 0, 240, 241, 538, 0, - 244, 245, 246, 0, 247, 248, 249, 250, 0, 251, - 252, 253, 254, 255, 913, 257, 0, 259, 260, 261, - 262, 0, 263, 264, 265, 266, 267, 268, 269, 0, - 270, 0, 272, 273, 274, 275, 276, 277, 278, 279, - 0, 280, 0, 281, 0, 0, 284, 0, 286, 287, - 288, 0, 289, 290, 291, 0, 0, 292, 0, 294, - 0, 0, 296, 297, 298, 299, 300, 301, 302, 303, - 539, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 0, 122, 123, 124, 125, 126, 127, 128, 129, 0, + 130, 131, 132, 0, 0, 0, 1535, 0, 0, 0, + 0, 1536, 134, 135, 0, 1537, 137, 138, 1538, 140, + 141, 142, 0, 1539, 1540, 1541, 1542, 0, 1543, 148, + 149, 150, 151, 152, 0, 0, 153, 154, 155, 156, + 1544, 1545, 159, 0, 160, 161, 162, 163, 0, 0, + 1546, 0, 1547, 167, 168, 169, 170, 171, 1548, 173, + 174, 175, 0, 176, 177, 178, 179, 180, 181, 0, + 1549, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 1550, 194, 195, 1551, 197, 0, 198, 0, 199, + 200, 201, 202, 203, 204, 0, 0, 205, 206, 207, + 208, 0, 0, 209, 210, 1093, 212, 213, 0, 214, + 215, 216, 0, 2454, 218, 219, 220, 0, 221, 222, + 223, 224, 0, 226, 227, 228, 229, 230, 231, 0, + 0, 233, 0, 234, 235, 1552, 237, 0, 238, 0, + 239, 1553, 0, 1554, 242, 243, 0, 1555, 246, 247, + 248, 0, 0, 0, 251, 252, 0, 253, 254, 255, + 256, 257, 258, 259, 1556, 261, 262, 263, 264, 0, + 265, 266, 267, 268, 269, 270, 271, 0, 272, 1557, + 0, 275, 276, 277, 278, 279, 1558, 1559, 0, 1560, + 0, 283, 1561, 1562, 286, 1563, 288, 289, 290, 0, + 291, 292, 293, 0, 0, 294, 1564, 296, 1565, 0, + 298, 299, 300, 301, 302, 303, 304, 305, 1566, 307, + 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, + 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, + 328, 1567, 1568, 1569, 332, 333, 334, 0, 0, 336, + 337, 1570, 339, 0, 0, 341, 1571, 343, 344, 345, + 0, 346, 347, 0, 0, 348, 349, 350, 0, 0, + 351, 352, 0, 1572, 355, 1573, 0, 358, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 0, + 0, 0, 0, 370, 371, 0, 1574, 374, 375, 0, + 377, 378, 379, 0, 380, 381, 382, 383, 384, 385, + 0, 386, 387, 388, 389, 390, 1575, 392, 393, 394, + 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 0, 409, 410, 1576, 412, + 413, 414, 1577, 416, 417, 418, 419, 420, 421, 422, + 423, 424, 425, 426, 427, 428, 0, 1578, 429, 430, + 431, 432, 433, 434, 1579, 436, 437, 0, 1580, 439, + 440, 1581, 442, 0, 443, 444, 445, 446, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 456, 1582, 458, + 0, 0, 0, 460, 461, 0, 462, 1583, 464, 465, + 466, 467, 468, 469, 0, 470, 1584, 1585, 0, 0, + 473, 474, 0, 476, 0, 0, 478, 479, 1586, 481, + 482, 483, 484, 485, 1587, 0, 486, 487, 488, 1588, + 0, 489, 490, 491, 492, 0, 493, 494, 495, 496, + 497, 0, 1589, 500, 0, 501, 1590, 503, 504, 505, + 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, + 512, 513, 514, 515, 516, 539, 0, 564, 0, 0, + 0, 0, 0, 0, 0, 0, 528, 529, 530, 531, + 0, 0, 0, 0, 0, 122, 123, 124, 125, 126, + 127, 128, 129, 0, 130, 131, 132, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 134, 135, 0, 0, + 137, 138, 0, 140, 141, 142, 143, 144, 0, 146, + 147, 0, 0, 148, 149, 150, 151, 152, 0, 0, + 153, 154, 155, 156, 157, 158, 159, 0, 160, 161, + 162, 163, 164, 0, 0, 0, 166, 167, 168, 169, + 170, 171, 0, 173, 174, 175, 0, 176, 177, 178, + 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 0, 198, 0, 199, 200, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 0, 0, 209, 210, 211, + 212, 213, 0, 214, 215, 216, 0, 217, 218, 219, + 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 0, 233, 0, 234, 235, 236, + 237, 0, 238, 0, 239, 0, 0, 0, 242, 243, + 540, 0, 246, 247, 248, 0, 249, 250, 251, 252, + 0, 253, 254, 255, 256, 257, 258, 259, 0, 261, + 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, + 271, 0, 272, 0, 274, 275, 276, 277, 278, 279, + 280, 281, 0, 282, 0, 283, 0, 0, 286, 0, + 288, 289, 290, 0, 291, 292, 293, 0, 0, 294, + 0, 296, 0, 0, 298, 299, 300, 301, 302, 303, + 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 327, 0, 329, 330, 331, 332, 333, - 0, 334, 335, 0, 337, 0, 338, 339, 340, 341, - 342, 343, 0, 344, 345, 0, 0, 346, 347, 348, - 0, 0, 349, 350, 351, 0, 353, 0, 355, 356, + 324, 325, 326, 327, 328, 329, 0, 331, 332, 333, + 334, 335, 0, 336, 337, 0, 339, 0, 340, 341, + 342, 343, 344, 345, 0, 346, 347, 0, 0, 348, + 349, 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 0, 0, 0, 0, 368, 369, 370, 0, 372, - 373, 374, 375, 376, 377, 0, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 0, 394, 395, 396, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 0, 407, 408, - 0, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 426, 0, 0, - 427, 428, 429, 430, 431, 432, 433, 434, 435, 0, - 0, 437, 438, 439, 440, 0, 441, 442, 443, 444, + 367, 368, 369, 0, 0, 0, 0, 370, 371, 372, + 0, 374, 375, 376, 377, 378, 379, 0, 380, 381, + 382, 383, 384, 385, 0, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 0, + 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 0, 0, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 540, 456, 457, 0, 0, 458, 459, 0, 460, 0, - 462, 463, 464, 465, 466, 467, 0, 468, 469, 470, - 0, 0, 471, 472, 473, 474, 475, 0, 476, 477, - 478, 479, 480, 481, 482, 483, 0, 0, 484, 485, - 486, 0, 0, 487, 488, 489, 490, 0, 491, 492, - 493, 494, 495, 496, 497, 498, 0, 499, 0, 501, - 502, 503, 504, 505, 506, 507, 0, 0, 508, 0, - 0, 509, 510, 511, 512, 513, 514, 515, 516, 517, + 455, 456, 542, 458, 459, 0, 0, 460, 461, 0, + 462, 0, 464, 465, 466, 467, 468, 469, 0, 470, + 471, 472, 0, 0, 473, 474, 475, 476, 477, 0, + 478, 479, 480, 481, 482, 483, 484, 485, 0, 0, + 486, 487, 488, 0, 0, 489, 490, 491, 492, 0, + 493, 494, 495, 496, 497, 498, 499, 500, 0, 501, + 0, 503, 504, 505, 506, 507, 508, 509, 0, 0, + 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, - 528, 529, 537, 0, 562, 0, 0, 0, 0, 0, + 528, 529, 530, 531, 539, 0, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 132, 133, 0, 0, 135, 136, 0, - 138, 139, 140, 141, 142, 0, 144, 145, 0, 0, - 146, 147, 148, 149, 150, 0, 0, 151, 152, 153, - 154, 155, 156, 157, 0, 158, 159, 160, 161, 162, - 0, 0, 0, 164, 165, 166, 167, 168, 169, 0, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 0, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 209, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, 0, 231, 0, 232, 233, 234, 235, 0, 236, - 0, 237, 0, 0, 0, 240, 241, 538, 0, 244, - 245, 246, 0, 247, 248, 249, 250, 0, 251, 252, - 253, 254, 255, 916, 257, 0, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 0, 272, 273, 274, 275, 276, 277, 278, 279, 0, - 280, 0, 281, 0, 0, 284, 0, 286, 287, 288, - 0, 289, 290, 291, 0, 0, 292, 0, 294, 0, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 539, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 134, 135, 0, 0, 137, + 138, 0, 140, 141, 142, 143, 144, 0, 146, 147, + 0, 0, 148, 149, 150, 151, 152, 0, 0, 153, + 154, 155, 156, 157, 158, 159, 0, 160, 161, 162, + 163, 164, 0, 0, 0, 166, 167, 168, 169, 170, + 171, 0, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 211, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 232, 0, 233, 0, 234, 235, 236, 237, + 0, 238, 0, 239, 0, 0, 0, 242, 243, 540, + 0, 246, 247, 248, 0, 249, 250, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 0, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 0, 274, 275, 276, 277, 278, 279, 280, + 281, 0, 282, 0, 283, 0, 0, 286, 0, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 0, + 296, 0, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 0, 329, 330, 331, 332, 333, 0, - 334, 335, 0, 337, 0, 338, 339, 340, 341, 342, - 343, 0, 344, 345, 0, 0, 346, 347, 348, 0, - 0, 349, 350, 351, 0, 353, 0, 355, 356, 357, + 325, 326, 327, 328, 329, 0, 331, 332, 333, 334, + 335, 0, 336, 337, 0, 339, 0, 340, 341, 342, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 370, 0, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 0, - 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 0, - 437, 438, 439, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 540, - 456, 457, 0, 0, 458, 459, 0, 460, 0, 462, - 463, 464, 465, 466, 467, 0, 468, 469, 470, 0, - 0, 471, 472, 473, 474, 475, 0, 476, 477, 478, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 0, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 497, 498, 0, 499, 0, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 368, 369, 0, 0, 0, 0, 370, 371, 372, 0, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 542, 458, 459, 0, 0, 460, 461, 0, 462, + 0, 464, 465, 466, 467, 468, 469, 0, 470, 471, + 472, 0, 0, 473, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 499, 500, 0, 501, 0, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 537, 0, 562, 0, 0, 0, 0, 0, 0, + 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 120, 121, 122, 123, 124, 125, 126, 127, 0, - 128, 129, 130, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 132, 133, 0, 0, 135, 136, 0, 138, - 139, 140, 141, 142, 0, 144, 145, 0, 0, 146, - 147, 148, 149, 150, 0, 0, 151, 152, 153, 154, - 155, 156, 157, 0, 158, 159, 160, 161, 162, 0, - 0, 0, 164, 165, 166, 167, 168, 169, 0, 171, - 172, 173, 0, 174, 175, 176, 177, 178, 179, 0, - 0, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 0, 196, 0, 197, - 198, 199, 200, 201, 202, 0, 0, 203, 204, 205, - 206, 0, 0, 207, 208, 209, 210, 211, 0, 212, - 213, 214, 0, 215, 216, 217, 218, 0, 219, 220, + 0, 0, 0, 122, 123, 124, 125, 126, 127, 128, + 129, 842, 130, 131, 132, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 134, 135, 0, 0, 137, 138, + 0, 140, 141, 142, 143, 144, 0, 146, 147, 0, + 0, 148, 149, 150, 151, 152, 0, 0, 153, 154, + 155, 156, 157, 158, 159, 0, 160, 161, 162, 163, + 164, 0, 0, 0, 166, 167, 168, 169, 170, 171, + 0, 173, 174, 175, 0, 176, 177, 178, 179, 180, + 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 0, 198, + 0, 199, 200, 201, 202, 203, 204, 0, 0, 205, + 206, 207, 208, 0, 0, 209, 210, 211, 212, 213, + 0, 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - 0, 231, 0, 232, 233, 234, 235, 0, 236, 0, - 237, 0, 0, 0, 240, 241, 538, 0, 244, 245, - 246, 0, 247, 248, 249, 250, 0, 251, 252, 253, - 254, 255, 920, 257, 0, 259, 260, 261, 262, 0, - 263, 264, 265, 266, 267, 268, 269, 0, 270, 0, - 272, 273, 274, 275, 276, 277, 278, 279, 0, 280, - 0, 281, 0, 0, 284, 0, 286, 287, 288, 0, - 289, 290, 291, 0, 0, 292, 0, 294, 0, 0, - 296, 297, 298, 299, 300, 301, 302, 303, 539, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 231, 232, 0, 233, 0, 234, 235, 236, 237, 0, + 238, 0, 239, 0, 0, 0, 242, 243, 540, 0, + 843, 247, 248, 0, 249, 250, 251, 252, 0, 253, + 254, 255, 256, 257, 258, 259, 0, 261, 262, 263, + 264, 0, 265, 266, 267, 268, 269, 270, 271, 0, + 272, 0, 274, 275, 276, 277, 278, 279, 280, 281, + 0, 282, 0, 283, 0, 0, 286, 0, 288, 289, + 290, 0, 291, 292, 293, 0, 0, 294, 0, 296, + 0, 0, 298, 299, 844, 301, 302, 303, 304, 305, + 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, 0, 329, 330, 331, 332, 333, 0, 334, - 335, 0, 337, 0, 338, 339, 340, 341, 342, 343, - 0, 344, 345, 0, 0, 346, 347, 348, 0, 0, - 349, 350, 351, 0, 353, 0, 355, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 0, - 0, 0, 0, 368, 369, 370, 0, 372, 373, 374, - 375, 376, 377, 0, 378, 379, 380, 381, 382, 383, - 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 0, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 0, 407, 408, 0, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 0, 0, 427, 428, - 429, 430, 431, 432, 433, 434, 435, 0, 0, 437, - 438, 439, 440, 0, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 540, 456, - 457, 0, 0, 458, 459, 0, 460, 0, 462, 463, - 464, 465, 466, 467, 0, 468, 469, 470, 0, 0, - 471, 472, 473, 474, 475, 0, 476, 477, 478, 479, - 480, 481, 482, 483, 0, 0, 484, 485, 486, 0, - 0, 487, 488, 489, 490, 0, 491, 492, 493, 494, - 495, 496, 497, 498, 0, 499, 0, 501, 502, 503, - 504, 505, 506, 507, 0, 0, 508, 0, 0, 509, - 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, + 326, 327, 328, 329, 0, 331, 332, 333, 334, 335, + 0, 336, 337, 0, 339, 0, 340, 341, 342, 343, + 344, 345, 0, 346, 347, 0, 0, 348, 349, 350, + 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 0, 0, 0, 0, 370, 371, 372, 0, 374, + 375, 376, 377, 378, 379, 0, 380, 381, 382, 383, + 384, 385, 0, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 0, 409, 410, + 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 426, 427, 428, 0, 0, + 429, 430, 431, 432, 845, 434, 435, 436, 437, 0, + 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 542, 458, 459, 0, 0, 460, 461, 0, 462, 0, + 464, 465, 466, 467, 468, 469, 0, 470, 846, 472, + 0, 0, 847, 474, 475, 476, 477, 0, 478, 479, + 480, 481, 482, 483, 484, 485, 0, 0, 486, 487, + 488, 0, 0, 489, 490, 491, 492, 0, 493, 494, + 495, 496, 497, 498, 499, 848, 0, 501, 0, 503, + 504, 505, 506, 507, 508, 509, 0, 0, 510, 0, + 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, - 537, 0, 562, 0, 0, 0, 0, 0, 0, 0, + 530, 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 132, 133, 0, 0, 135, 136, 0, 138, 139, - 140, 141, 142, 0, 144, 145, 0, 0, 146, 147, - 148, 149, 150, 0, 0, 151, 152, 153, 154, 155, - 156, 157, 0, 158, 159, 160, 161, 162, 0, 0, - 0, 164, 165, 166, 167, 168, 169, 0, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 0, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 0, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 209, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, 0, - 231, 0, 232, 233, 234, 235, 0, 236, 0, 237, - 0, 0, 0, 240, 241, 538, 0, 244, 245, 246, - 0, 247, 248, 249, 250, 0, 251, 252, 253, 254, - 255, 951, 257, 0, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 0, 272, - 273, 274, 275, 276, 277, 278, 279, 0, 280, 0, - 281, 0, 0, 284, 0, 286, 287, 288, 0, 289, - 290, 291, 0, 0, 292, 0, 294, 0, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 539, 305, 306, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 134, 135, 0, 0, 137, 138, 0, + 140, 141, 142, 143, 144, 0, 146, 147, 0, 0, + 148, 149, 150, 151, 152, 0, 0, 153, 154, 155, + 156, 157, 158, 159, 0, 160, 161, 162, 163, 164, + 0, 0, 0, 166, 167, 168, 169, 170, 171, 0, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 211, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 0, 233, 0, 234, 235, 236, 237, 0, 238, + 0, 239, 0, 0, 0, 242, 243, 540, 0, 246, + 247, 248, 0, 249, 250, 251, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 0, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 0, 274, 275, 276, 277, 278, 279, 280, 281, 0, + 282, 0, 283, 0, 0, 286, 0, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 0, 296, 0, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, 0, 329, 330, 331, 332, 333, 0, 334, 335, - 0, 337, 0, 338, 339, 340, 341, 342, 343, 0, - 344, 345, 0, 0, 346, 347, 348, 0, 0, 349, - 350, 351, 0, 353, 0, 355, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 370, 0, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 0, 410, 411, + 327, 328, 329, 0, 331, 332, 333, 334, 335, 0, + 336, 337, 0, 339, 0, 340, 341, 342, 343, 344, + 345, 0, 346, 347, 0, 0, 348, 349, 350, 0, + 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 372, 0, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 391, 392, 393, + 881, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 0, 437, 438, - 439, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 540, 456, 457, - 0, 0, 458, 459, 0, 460, 0, 462, 463, 464, - 465, 466, 467, 0, 468, 469, 470, 0, 0, 471, - 472, 473, 474, 475, 0, 476, 477, 478, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 0, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 497, 498, 0, 499, 0, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 0, + 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 542, + 458, 459, 0, 0, 460, 461, 0, 462, 0, 464, + 465, 466, 467, 468, 469, 0, 470, 471, 472, 0, + 0, 473, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 0, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 499, 500, 0, 501, 0, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 537, - 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, - 121, 122, 123, 124, 125, 126, 127, 0, 128, 129, - 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 132, 133, 0, 0, 135, 136, 0, 138, 139, 140, - 141, 142, 0, 144, 145, 0, 0, 146, 147, 148, - 149, 150, 0, 0, 151, 152, 153, 154, 155, 156, - 157, 0, 158, 159, 160, 161, 162, 0, 0, 0, - 164, 165, 166, 167, 168, 169, 0, 171, 172, 173, - 0, 174, 175, 176, 177, 178, 179, 0, 0, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 0, 196, 0, 197, 198, 199, - 200, 201, 202, 0, 0, 203, 204, 205, 206, 0, - 0, 207, 208, 209, 210, 211, 0, 212, 213, 214, - 0, 215, 216, 217, 218, 0, 219, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, 0, 231, - 0, 232, 233, 234, 235, 0, 236, 0, 237, 0, - 0, 0, 240, 241, 538, 0, 244, 245, 246, 0, - 247, 248, 249, 250, 0, 251, 252, 253, 254, 255, - 979, 257, 0, 259, 260, 261, 262, 0, 263, 264, - 265, 266, 267, 268, 269, 0, 270, 0, 272, 273, - 274, 275, 276, 277, 278, 279, 0, 280, 0, 281, - 0, 0, 284, 0, 286, 287, 288, 0, 289, 290, - 291, 0, 0, 292, 0, 294, 0, 0, 296, 297, - 298, 299, 300, 301, 302, 303, 539, 305, 306, 307, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 122, 123, 124, 125, 126, 127, 128, 129, 0, + 130, 131, 132, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 134, 135, 0, 0, 137, 138, 0, 140, + 141, 142, 143, 144, 0, 146, 147, 0, 0, 148, + 149, 150, 151, 152, 0, 0, 153, 154, 155, 156, + 157, 158, 159, 0, 160, 161, 162, 163, 164, 0, + 0, 0, 166, 167, 168, 169, 170, 171, 0, 173, + 174, 175, 0, 176, 177, 178, 179, 180, 181, 0, + 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 0, 198, 0, 199, + 200, 201, 202, 203, 204, 0, 0, 205, 206, 207, + 208, 0, 0, 209, 210, 211, 212, 213, 0, 214, + 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, + 0, 233, 0, 234, 235, 236, 237, 0, 238, 0, + 239, 0, 0, 0, 242, 243, 540, 0, 246, 247, + 248, 0, 249, 250, 251, 252, 0, 253, 254, 255, + 256, 257, 918, 259, 0, 261, 262, 263, 264, 0, + 265, 266, 267, 268, 269, 270, 271, 0, 272, 0, + 274, 275, 276, 277, 278, 279, 280, 281, 0, 282, + 0, 283, 0, 0, 286, 0, 288, 289, 290, 0, + 291, 292, 293, 0, 0, 294, 0, 296, 0, 0, + 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - 0, 329, 330, 331, 332, 333, 0, 334, 335, 0, - 337, 0, 338, 339, 340, 341, 342, 343, 0, 344, - 345, 0, 0, 346, 347, 348, 0, 0, 349, 350, - 351, 0, 353, 0, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 0, 0, 0, - 0, 368, 369, 370, 0, 372, 373, 374, 375, 376, - 377, 0, 378, 379, 380, 381, 382, 383, 0, 384, - 385, 386, 387, 388, 389, 390, 391, 392, 393, 0, - 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 0, 407, 408, 0, 410, 411, 412, + 328, 329, 0, 331, 332, 333, 334, 335, 0, 336, + 337, 0, 339, 0, 340, 341, 342, 343, 344, 345, + 0, 346, 347, 0, 0, 348, 349, 350, 0, 0, + 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 0, + 0, 0, 0, 370, 371, 372, 0, 374, 375, 376, + 377, 378, 379, 0, 380, 381, 382, 383, 384, 385, + 0, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 426, 0, 0, 427, 428, 429, 430, - 431, 432, 433, 434, 435, 0, 0, 437, 438, 439, - 440, 0, 441, 442, 443, 444, 445, 446, 447, 448, - 449, 450, 451, 452, 453, 454, 540, 456, 457, 0, - 0, 458, 459, 0, 460, 0, 462, 463, 464, 465, - 466, 467, 0, 468, 469, 470, 0, 0, 471, 472, - 473, 474, 475, 0, 476, 477, 478, 479, 480, 481, - 482, 483, 0, 0, 484, 485, 486, 0, 0, 487, - 488, 489, 490, 0, 491, 492, 493, 494, 495, 496, - 497, 498, 0, 499, 0, 501, 502, 503, 504, 505, - 506, 507, 0, 0, 508, 0, 0, 509, 510, 511, + 423, 424, 425, 426, 427, 428, 0, 0, 429, 430, + 431, 432, 433, 434, 435, 436, 437, 0, 0, 439, + 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 456, 542, 458, + 459, 0, 0, 460, 461, 0, 462, 0, 464, 465, + 466, 467, 468, 469, 0, 470, 471, 472, 0, 0, + 473, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 485, 0, 0, 486, 487, 488, 0, + 0, 489, 490, 491, 492, 0, 493, 494, 495, 496, + 497, 498, 499, 500, 0, 501, 0, 503, 504, 505, + 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, - 522, 523, 524, 525, 526, 527, 528, 529, 537, 0, - 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, - 133, 0, 0, 135, 136, 0, 138, 139, 140, 141, - 142, 0, 144, 145, 0, 0, 146, 147, 148, 149, - 150, 0, 0, 151, 152, 153, 154, 155, 156, 157, - 0, 158, 159, 160, 161, 162, 0, 0, 0, 164, - 165, 166, 167, 168, 169, 0, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 0, 181, 182, + 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, + 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 134, 135, 0, 0, 137, 138, 0, 140, 141, + 142, 143, 144, 0, 146, 147, 0, 0, 148, 149, + 150, 151, 152, 0, 0, 153, 154, 155, 156, 157, + 158, 159, 0, 160, 161, 162, 163, 164, 0, 0, + 0, 166, 167, 168, 169, 170, 171, 0, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 209, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, 0, 231, 0, - 232, 233, 234, 235, 0, 236, 0, 237, 0, 0, - 0, 240, 241, 538, 0, 244, 245, 246, 0, 247, - 248, 249, 250, 0, 251, 252, 253, 254, 255, 982, - 257, 0, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 0, 272, 273, 274, - 275, 276, 277, 278, 279, 0, 280, 0, 281, 0, - 0, 284, 0, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 0, 294, 0, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 539, 305, 306, 307, 308, + 193, 194, 195, 196, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 211, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 0, + 233, 0, 234, 235, 236, 237, 0, 238, 0, 239, + 0, 0, 0, 242, 243, 540, 0, 246, 247, 248, + 0, 249, 250, 251, 252, 0, 253, 254, 255, 256, + 257, 921, 259, 0, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 0, 274, + 275, 276, 277, 278, 279, 280, 281, 0, 282, 0, + 283, 0, 0, 286, 0, 288, 289, 290, 0, 291, + 292, 293, 0, 0, 294, 0, 296, 0, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, 0, - 329, 330, 331, 332, 333, 0, 334, 335, 0, 337, - 0, 338, 339, 340, 341, 342, 343, 0, 344, 345, - 0, 0, 346, 347, 348, 0, 0, 349, 350, 351, - 0, 353, 0, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 370, 0, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 0, 410, 411, 412, 413, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 0, 331, 332, 333, 334, 335, 0, 336, 337, + 0, 339, 0, 340, 341, 342, 343, 344, 345, 0, + 346, 347, 0, 0, 348, 349, 350, 0, 0, 351, + 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 372, 0, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 0, 437, 438, 439, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 540, 456, 457, 0, 0, - 458, 459, 0, 460, 0, 462, 463, 464, 465, 466, - 467, 0, 468, 469, 470, 0, 0, 471, 472, 473, - 474, 475, 0, 476, 477, 478, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 497, - 498, 0, 499, 0, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 0, 439, 440, + 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 542, 458, 459, + 0, 0, 460, 461, 0, 462, 0, 464, 465, 466, + 467, 468, 469, 0, 470, 471, 472, 0, 0, 473, + 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 0, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 499, 500, 0, 501, 0, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 537, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 120, 121, 122, - 123, 124, 125, 126, 127, 0, 128, 129, 130, 0, - 0, 0, 0, 0, 0, 1025, 0, 0, 132, 133, - 0, 0, 135, 136, 0, 138, 139, 140, 141, 142, - 0, 144, 145, 0, 0, 146, 147, 148, 149, 150, - 0, 0, 151, 152, 153, 154, 155, 156, 157, 0, - 158, 159, 160, 161, 162, 0, 0, 0, 164, 165, - 166, 167, 168, 169, 0, 171, 172, 173, 0, 174, - 175, 176, 177, 178, 179, 0, 0, 181, 182, 183, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 539, + 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, + 123, 124, 125, 126, 127, 128, 129, 0, 130, 131, + 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 134, 135, 0, 0, 137, 138, 0, 140, 141, 142, + 143, 144, 0, 146, 147, 0, 0, 148, 149, 150, + 151, 152, 0, 0, 153, 154, 155, 156, 157, 158, + 159, 0, 160, 161, 162, 163, 164, 0, 0, 0, + 166, 167, 168, 169, 170, 171, 0, 173, 174, 175, + 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 0, 196, 0, 197, 198, 199, 200, 201, - 202, 0, 0, 203, 204, 205, 206, 0, 0, 207, - 208, 209, 210, 211, 0, 212, 213, 214, 0, 215, - 216, 217, 218, 0, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 0, 231, 0, 232, - 233, 234, 235, 0, 236, 0, 237, 0, 0, 0, - 240, 241, 538, 0, 244, 245, 246, 0, 247, 248, - 249, 250, 0, 251, 252, 253, 254, 255, 256, 257, - 0, 259, 260, 261, 262, 0, 263, 264, 265, 266, - 267, 268, 269, 0, 270, 0, 272, 273, 274, 275, - 276, 277, 278, 279, 0, 280, 0, 281, 0, 0, - 284, 0, 286, 287, 288, 0, 289, 290, 291, 0, - 0, 292, 0, 294, 0, 0, 296, 297, 298, 299, - 300, 301, 302, 303, 539, 305, 306, 307, 308, 309, + 194, 195, 196, 197, 0, 198, 0, 199, 200, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 0, + 0, 209, 210, 211, 212, 213, 0, 214, 215, 216, + 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 0, 233, + 0, 234, 235, 236, 237, 0, 238, 0, 239, 0, + 0, 0, 242, 243, 540, 0, 246, 247, 248, 0, + 249, 250, 251, 252, 0, 253, 254, 255, 256, 257, + 925, 259, 0, 261, 262, 263, 264, 0, 265, 266, + 267, 268, 269, 270, 271, 0, 272, 0, 274, 275, + 276, 277, 278, 279, 280, 281, 0, 282, 0, 283, + 0, 0, 286, 0, 288, 289, 290, 0, 291, 292, + 293, 0, 0, 294, 0, 296, 0, 0, 298, 299, + 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, 0, 329, - 330, 331, 332, 333, 0, 334, 335, 0, 337, 0, - 338, 339, 340, 341, 342, 343, 0, 344, 345, 0, - 0, 346, 347, 348, 0, 0, 349, 350, 351, 0, - 353, 0, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 0, 0, 0, 0, 368, - 369, 370, 0, 372, 373, 374, 375, 376, 377, 0, - 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, - 387, 388, 389, 390, 391, 392, 393, 0, 394, 395, + 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 0, 331, 332, 333, 334, 335, 0, 336, 337, 0, + 339, 0, 340, 341, 342, 343, 344, 345, 0, 346, + 347, 0, 0, 348, 349, 350, 0, 0, 351, 352, + 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 0, 0, 0, + 0, 370, 371, 372, 0, 374, 375, 376, 377, 378, + 379, 0, 380, 381, 382, 383, 384, 385, 0, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 0, 407, 408, 0, 410, 411, 412, 413, 414, + 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 0, 0, 427, 428, 429, 430, 431, 432, - 433, 434, 435, 0, 0, 437, 438, 439, 440, 0, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 540, 456, 457, 0, 0, 458, - 459, 0, 460, 0, 462, 463, 464, 465, 466, 467, - 0, 468, 469, 470, 0, 0, 471, 472, 473, 474, - 475, 0, 476, 477, 478, 479, 480, 481, 482, 483, - 0, 0, 484, 485, 486, 0, 0, 487, 488, 489, - 490, 0, 491, 492, 493, 494, 495, 496, 497, 498, - 0, 499, 0, 501, 502, 503, 504, 505, 506, 507, - 0, 0, 508, 0, 0, 509, 510, 511, 512, 513, + 425, 426, 427, 428, 0, 0, 429, 430, 431, 432, + 433, 434, 435, 436, 437, 0, 0, 439, 440, 441, + 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, + 451, 452, 453, 454, 455, 456, 542, 458, 459, 0, + 0, 460, 461, 0, 462, 0, 464, 465, 466, 467, + 468, 469, 0, 470, 471, 472, 0, 0, 473, 474, + 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, + 484, 485, 0, 0, 486, 487, 488, 0, 0, 489, + 490, 491, 492, 0, 493, 494, 495, 496, 497, 498, + 499, 500, 0, 501, 0, 503, 504, 505, 506, 507, + 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, - 524, 525, 526, 527, 528, 529, 537, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 0, 1052, 0, 0, 132, 133, 0, - 0, 135, 136, 0, 138, 139, 140, 141, 142, 0, - 144, 145, 0, 0, 146, 147, 148, 149, 150, 0, - 0, 151, 152, 153, 154, 155, 156, 157, 0, 158, - 159, 160, 161, 162, 0, 0, 0, 164, 165, 166, - 167, 168, 169, 0, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 0, 181, 182, 183, 184, + 524, 525, 526, 527, 528, 529, 530, 531, 539, 0, + 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, + 135, 0, 0, 137, 138, 0, 140, 141, 142, 143, + 144, 0, 146, 147, 0, 0, 148, 149, 150, 151, + 152, 0, 0, 153, 154, 155, 156, 157, 158, 159, + 0, 160, 161, 162, 163, 164, 0, 0, 0, 166, + 167, 168, 169, 170, 171, 0, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 209, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, 0, 231, 0, 232, 233, - 234, 235, 0, 236, 0, 237, 0, 0, 0, 240, - 241, 538, 0, 244, 245, 246, 0, 247, 248, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 0, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 0, 272, 273, 274, 275, 276, - 277, 278, 279, 0, 280, 0, 281, 0, 0, 284, - 0, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 0, 294, 0, 0, 296, 297, 298, 299, 300, - 301, 302, 303, 539, 305, 306, 307, 308, 309, 310, + 195, 196, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 211, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, 231, 232, 0, 233, 0, + 234, 235, 236, 237, 0, 238, 0, 239, 0, 0, + 0, 242, 243, 540, 0, 246, 247, 248, 0, 249, + 250, 251, 252, 0, 253, 254, 255, 256, 257, 956, + 259, 0, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 0, 274, 275, 276, + 277, 278, 279, 280, 281, 0, 282, 0, 283, 0, + 0, 286, 0, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 0, 296, 0, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 0, 329, 330, - 331, 332, 333, 0, 334, 335, 0, 337, 0, 338, - 339, 340, 341, 342, 343, 0, 344, 345, 0, 0, - 346, 347, 348, 0, 0, 349, 350, 351, 0, 353, - 0, 355, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 370, 0, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 329, 0, + 331, 332, 333, 334, 335, 0, 336, 337, 0, 339, + 0, 340, 341, 342, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 353, + 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 372, 0, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 0, 410, 411, 412, 413, 414, 415, + 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 0, 0, 437, 438, 439, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 540, 456, 457, 0, 0, 458, 459, - 0, 460, 0, 462, 463, 464, 465, 466, 467, 0, - 468, 469, 470, 0, 0, 471, 472, 473, 474, 475, - 0, 476, 477, 478, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 497, 498, 0, - 499, 0, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 0, 439, 440, 441, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 542, 458, 459, 0, 0, + 460, 461, 0, 462, 0, 464, 465, 466, 467, 468, + 469, 0, 470, 471, 472, 0, 0, 473, 474, 475, + 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 0, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 499, + 500, 0, 501, 0, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 537, 0, 0, 0, 0, + 525, 526, 527, 528, 529, 530, 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 120, 121, 122, 123, 124, - 125, 126, 127, 839, 128, 129, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 132, 133, 0, 0, - 135, 136, 0, 138, 139, 140, 141, 142, 0, 144, - 145, 0, 0, 146, 147, 148, 149, 150, 0, 0, - 151, 152, 153, 154, 155, 156, 157, 0, 158, 159, - 160, 161, 162, 0, 0, 0, 164, 165, 166, 167, - 168, 169, 0, 171, 172, 173, 0, 174, 175, 176, - 177, 178, 179, 0, 0, 181, 182, 183, 184, 185, + 0, 0, 0, 0, 0, 0, 0, 122, 123, 124, + 125, 126, 127, 128, 129, 0, 130, 131, 132, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 134, 135, + 0, 0, 137, 138, 0, 140, 141, 142, 143, 144, + 0, 146, 147, 0, 0, 148, 149, 150, 151, 152, + 0, 0, 153, 154, 155, 156, 157, 158, 159, 0, + 160, 161, 162, 163, 164, 0, 0, 0, 166, 167, + 168, 169, 170, 171, 0, 173, 174, 175, 0, 176, + 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 0, 196, 0, 197, 198, 199, 200, 201, 202, 0, - 0, 203, 204, 205, 206, 0, 0, 207, 208, 209, - 210, 211, 0, 212, 213, 214, 0, 215, 216, 217, - 218, 0, 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 0, 231, 0, 232, 233, 234, - 235, 0, 236, 0, 237, 0, 0, 0, 240, 241, - 538, 0, 244, 245, 246, 0, 247, 248, 249, 250, - 0, 251, 252, 253, 254, 255, 256, 257, 0, 259, - 260, 261, 262, 0, 263, 264, 265, 266, 267, 268, - 269, 0, 270, 0, 272, 273, 274, 275, 276, 277, - 278, 279, 0, 280, 0, 281, 0, 0, 284, 0, - 286, 287, 288, 0, 289, 290, 291, 0, 0, 292, - 0, 294, 0, 0, 296, 297, 298, 299, 300, 301, - 302, 303, 539, 305, 306, 307, 308, 309, 310, 311, + 196, 197, 0, 198, 0, 199, 200, 201, 202, 203, + 204, 0, 0, 205, 206, 207, 208, 0, 0, 209, + 210, 211, 212, 213, 0, 214, 215, 216, 0, 217, + 218, 219, 220, 0, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, 231, 232, 0, 233, 0, 234, + 235, 236, 237, 0, 238, 0, 239, 0, 0, 0, + 242, 243, 540, 0, 246, 247, 248, 0, 249, 250, + 251, 252, 0, 253, 254, 255, 256, 257, 984, 259, + 0, 261, 262, 263, 264, 0, 265, 266, 267, 268, + 269, 270, 271, 0, 272, 0, 274, 275, 276, 277, + 278, 279, 280, 281, 0, 282, 0, 283, 0, 0, + 286, 0, 288, 289, 290, 0, 291, 292, 293, 0, + 0, 294, 0, 296, 0, 0, 298, 299, 300, 301, + 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 0, 329, 330, 331, - 332, 333, 0, 334, 335, 0, 337, 0, 338, 339, - 340, 341, 342, 343, 0, 344, 345, 0, 0, 346, - 347, 348, 0, 0, 349, 350, 351, 0, 353, 0, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 0, 0, 0, 0, 368, 369, 370, - 0, 372, 373, 374, 375, 376, 377, 0, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 0, 394, 395, 396, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 0, - 407, 408, 0, 410, 411, 412, 413, 414, 415, 416, + 322, 323, 324, 325, 326, 327, 328, 329, 0, 331, + 332, 333, 334, 335, 0, 336, 337, 0, 339, 0, + 340, 341, 342, 343, 344, 345, 0, 346, 347, 0, + 0, 348, 349, 350, 0, 0, 351, 352, 353, 0, + 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 0, 0, 0, 0, 370, + 371, 372, 0, 374, 375, 376, 377, 378, 379, 0, + 380, 381, 382, 383, 384, 385, 0, 386, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 0, 0, 427, 428, 429, 430, 431, 432, 433, 434, - 435, 0, 0, 437, 438, 439, 440, 0, 441, 442, + 427, 428, 0, 0, 429, 430, 431, 432, 433, 434, + 435, 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, - 453, 454, 540, 456, 457, 0, 0, 458, 459, 0, - 460, 0, 462, 463, 464, 465, 466, 467, 0, 468, - 843, 470, 0, 0, 844, 472, 473, 474, 475, 0, - 476, 477, 478, 479, 480, 481, 482, 483, 0, 0, - 484, 485, 486, 0, 0, 487, 488, 489, 490, 0, - 491, 492, 493, 494, 495, 496, 497, 498, 0, 499, - 0, 501, 502, 503, 504, 505, 506, 507, 0, 0, - 508, 0, 0, 509, 510, 511, 512, 513, 514, 515, + 453, 454, 455, 456, 542, 458, 459, 0, 0, 460, + 461, 0, 462, 0, 464, 465, 466, 467, 468, 469, + 0, 470, 471, 472, 0, 0, 473, 474, 475, 476, + 477, 0, 478, 479, 480, 481, 482, 483, 484, 485, + 0, 0, 486, 487, 488, 0, 0, 489, 490, 491, + 492, 0, 493, 494, 495, 496, 497, 498, 499, 500, + 0, 501, 0, 503, 504, 505, 506, 507, 508, 509, + 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, - 526, 527, 528, 529, 537, 0, 562, 0, 0, 0, + 526, 527, 528, 529, 530, 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 132, 133, 0, 0, 135, - 136, 0, 138, 139, 140, 141, 142, 0, 144, 145, - 0, 0, 146, 147, 148, 149, 150, 0, 0, 151, - 152, 153, 154, 155, 156, 157, 0, 158, 159, 160, - 161, 162, 0, 0, 0, 164, 165, 166, 167, 168, - 169, 0, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 0, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 0, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 209, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 0, 231, 0, 232, 233, 234, 235, - 0, 236, 0, 237, 0, 0, 0, 240, 241, 538, - 0, 244, 245, 246, 0, 247, 248, 249, 250, 0, - 251, 252, 253, 254, 255, 1348, 257, 0, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 0, 272, 273, 274, 275, 276, 277, 278, - 279, 0, 280, 0, 281, 0, 0, 284, 0, 286, - 287, 288, 0, 289, 290, 291, 0, 0, 292, 0, - 294, 0, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 539, 305, 306, 307, 308, 309, 310, 311, 312, + 0, 0, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 134, 135, 0, + 0, 137, 138, 0, 140, 141, 142, 143, 144, 0, + 146, 147, 0, 0, 148, 149, 150, 151, 152, 0, + 0, 153, 154, 155, 156, 157, 158, 159, 0, 160, + 161, 162, 163, 164, 0, 0, 0, 166, 167, 168, + 169, 170, 171, 0, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 211, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 0, 233, 0, 234, 235, + 236, 237, 0, 238, 0, 239, 0, 0, 0, 242, + 243, 540, 0, 246, 247, 248, 0, 249, 250, 251, + 252, 0, 253, 254, 255, 256, 257, 987, 259, 0, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 0, 274, 275, 276, 277, 278, + 279, 280, 281, 0, 282, 0, 283, 0, 0, 286, + 0, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 0, 296, 0, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, 0, 329, 330, 331, 332, - 333, 0, 334, 335, 0, 337, 0, 338, 339, 340, - 341, 342, 343, 0, 344, 345, 0, 0, 346, 347, - 348, 0, 0, 349, 350, 351, 0, 353, 0, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 370, 0, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 0, 410, 411, 412, 413, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 0, 437, 438, 439, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 329, 0, 331, 332, + 333, 334, 335, 0, 336, 337, 0, 339, 0, 340, + 341, 342, 343, 344, 345, 0, 346, 347, 0, 0, + 348, 349, 350, 0, 0, 351, 352, 353, 0, 355, + 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 372, 0, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 540, 456, 457, 0, 0, 458, 459, 0, 460, - 0, 462, 463, 464, 465, 466, 467, 0, 468, 469, - 470, 0, 0, 471, 472, 473, 474, 475, 0, 476, - 477, 478, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 0, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 497, 498, 0, 499, 0, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 542, 458, 459, 0, 0, 460, 461, + 0, 462, 0, 464, 465, 466, 467, 468, 469, 0, + 470, 471, 472, 0, 0, 473, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 499, 500, 0, + 501, 0, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 537, 0, 562, 0, 0, 0, 0, + 527, 528, 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 120, 121, 122, 123, 124, 125, 126, - 127, 0, 128, 129, 130, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 133, 0, 0, 135, 136, - 0, 138, 139, 140, 141, 142, 0, 144, 145, 0, - 0, 146, 147, 148, 149, 150, 0, 0, 151, 152, - 153, 154, 155, 156, 157, 0, 158, 159, 160, 161, - 162, 0, 0, 0, 164, 165, 166, 167, 168, 169, - 0, 171, 172, 173, 0, 174, 175, 176, 177, 178, - 179, 0, 0, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 0, 196, - 0, 197, 198, 199, 200, 201, 202, 0, 0, 203, - 204, 205, 206, 0, 0, 207, 208, 209, 210, 211, - 0, 212, 213, 214, 0, 215, 216, 217, 218, 0, - 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, 0, 231, 0, 232, 233, 234, 235, 0, - 236, 0, 237, 0, 0, 0, 240, 241, 538, 0, - 244, 245, 246, 0, 247, 248, 249, 250, 0, 251, - 252, 253, 254, 255, 1350, 257, 0, 259, 260, 261, - 262, 0, 263, 264, 265, 266, 267, 268, 269, 0, - 270, 0, 272, 273, 274, 275, 276, 277, 278, 279, - 0, 280, 0, 281, 0, 0, 284, 0, 286, 287, - 288, 0, 289, 290, 291, 0, 0, 292, 0, 294, - 0, 0, 296, 297, 298, 299, 300, 301, 302, 303, - 539, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 0, 0, 0, 0, 0, 122, 123, 124, 125, 126, + 127, 128, 129, 0, 130, 131, 132, 0, 0, 0, + 0, 0, 0, 1030, 0, 0, 134, 135, 0, 0, + 137, 138, 0, 140, 141, 142, 143, 144, 0, 146, + 147, 0, 0, 148, 149, 150, 151, 152, 0, 0, + 153, 154, 155, 156, 157, 158, 159, 0, 160, 161, + 162, 163, 164, 0, 0, 0, 166, 167, 168, 169, + 170, 171, 0, 173, 174, 175, 0, 176, 177, 178, + 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 0, 198, 0, 199, 200, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 0, 0, 209, 210, 211, + 212, 213, 0, 214, 215, 216, 0, 217, 218, 219, + 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 0, 233, 0, 234, 235, 236, + 237, 0, 238, 0, 239, 0, 0, 0, 242, 243, + 540, 0, 246, 247, 248, 0, 249, 250, 251, 252, + 0, 253, 254, 255, 256, 257, 258, 259, 0, 261, + 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, + 271, 0, 272, 0, 274, 275, 276, 277, 278, 279, + 280, 281, 0, 282, 0, 283, 0, 0, 286, 0, + 288, 289, 290, 0, 291, 292, 293, 0, 0, 294, + 0, 296, 0, 0, 298, 299, 300, 301, 302, 303, + 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 327, 0, 329, 330, 331, 332, 333, - 0, 334, 335, 0, 337, 0, 338, 339, 340, 341, - 342, 343, 0, 344, 345, 0, 0, 346, 347, 348, - 0, 0, 349, 350, 351, 0, 353, 0, 355, 356, + 324, 325, 326, 327, 328, 329, 0, 331, 332, 333, + 334, 335, 0, 336, 337, 0, 339, 0, 340, 341, + 342, 343, 344, 345, 0, 346, 347, 0, 0, 348, + 349, 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 0, 0, 0, 0, 368, 369, 370, 0, 372, - 373, 374, 375, 376, 377, 0, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 0, 394, 395, 396, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 0, 407, 408, - 0, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 426, 0, 0, - 427, 428, 429, 430, 431, 432, 433, 434, 435, 0, - 0, 437, 438, 439, 440, 0, 441, 442, 443, 444, + 367, 368, 369, 0, 0, 0, 0, 370, 371, 372, + 0, 374, 375, 376, 377, 378, 379, 0, 380, 381, + 382, 383, 384, 385, 0, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 0, + 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 0, 0, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 540, 456, 457, 0, 0, 458, 459, 0, 460, 0, - 462, 463, 464, 465, 466, 467, 0, 468, 469, 470, - 0, 0, 471, 472, 473, 474, 475, 0, 476, 477, - 478, 479, 480, 481, 482, 483, 0, 0, 484, 485, - 486, 0, 0, 487, 488, 489, 490, 0, 491, 492, - 493, 494, 495, 496, 497, 498, 0, 499, 0, 501, - 502, 503, 504, 505, 506, 507, 0, 0, 508, 0, - 0, 509, 510, 511, 512, 513, 514, 515, 516, 517, + 455, 456, 542, 458, 459, 0, 0, 460, 461, 0, + 462, 0, 464, 465, 466, 467, 468, 469, 0, 470, + 471, 472, 0, 0, 473, 474, 475, 476, 477, 0, + 478, 479, 480, 481, 482, 483, 484, 485, 0, 0, + 486, 487, 488, 0, 0, 489, 490, 491, 492, 0, + 493, 494, 495, 496, 497, 498, 499, 500, 0, 501, + 0, 503, 504, 505, 506, 507, 508, 509, 0, 0, + 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, - 528, 529, 537, 0, 562, 0, 0, 0, 0, 0, + 528, 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 132, 133, 0, 0, 135, 136, 0, - 138, 139, 140, 141, 142, 0, 144, 145, 0, 0, - 146, 147, 148, 149, 150, 0, 0, 151, 152, 153, - 154, 155, 156, 157, 0, 158, 159, 160, 161, 162, - 0, 0, 0, 164, 165, 166, 167, 168, 169, 0, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 0, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 209, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, 0, 231, 0, 232, 233, 234, 235, 0, 236, - 0, 237, 0, 0, 0, 240, 241, 538, 0, 244, - 245, 246, 0, 247, 248, 249, 250, 0, 251, 252, - 253, 254, 255, 1353, 257, 0, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 0, 272, 273, 274, 275, 276, 277, 278, 279, 0, - 280, 0, 281, 0, 0, 284, 0, 286, 287, 288, - 0, 289, 290, 291, 0, 0, 292, 0, 294, 0, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 539, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 0, 1057, 0, 0, 134, 135, 0, 0, 137, + 138, 0, 140, 141, 142, 143, 144, 0, 146, 147, + 0, 0, 148, 149, 150, 151, 152, 0, 0, 153, + 154, 155, 156, 157, 158, 159, 0, 160, 161, 162, + 163, 164, 0, 0, 0, 166, 167, 168, 169, 170, + 171, 0, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 211, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 232, 0, 233, 0, 234, 235, 236, 237, + 0, 238, 0, 239, 0, 0, 0, 242, 243, 540, + 0, 246, 247, 248, 0, 249, 250, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 0, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 0, 274, 275, 276, 277, 278, 279, 280, + 281, 0, 282, 0, 283, 0, 0, 286, 0, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 0, + 296, 0, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 0, 329, 330, 331, 332, 333, 0, - 334, 335, 0, 337, 0, 338, 339, 340, 341, 342, - 343, 0, 344, 345, 0, 0, 346, 347, 348, 0, - 0, 349, 350, 351, 0, 353, 0, 355, 356, 357, + 325, 326, 327, 328, 329, 0, 331, 332, 333, 334, + 335, 0, 336, 337, 0, 339, 0, 340, 341, 342, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 370, 0, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 0, - 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 0, - 437, 438, 439, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 540, - 456, 457, 0, 0, 458, 459, 0, 460, 0, 462, - 463, 464, 465, 466, 467, 0, 468, 469, 470, 0, - 0, 471, 472, 473, 474, 475, 0, 476, 477, 478, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 0, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 497, 498, 0, 499, 0, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 368, 369, 0, 0, 0, 0, 370, 371, 372, 0, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 542, 458, 459, 0, 0, 460, 461, 0, 462, + 0, 464, 465, 466, 467, 468, 469, 0, 470, 471, + 472, 0, 0, 473, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 499, 500, 0, 501, 0, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 537, 0, 562, 0, 0, 0, 0, 0, 0, + 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 120, 121, 122, 123, 124, 125, 126, 127, 0, - 128, 129, 130, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 132, 133, 0, 0, 135, 136, 0, 138, - 139, 140, 141, 142, 0, 144, 145, 0, 0, 146, - 147, 148, 149, 150, 0, 0, 151, 152, 153, 154, - 155, 156, 157, 0, 158, 159, 160, 161, 162, 0, - 0, 0, 164, 165, 166, 167, 168, 169, 0, 171, - 172, 173, 0, 174, 175, 176, 177, 178, 179, 0, - 0, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 0, 196, 0, 197, - 198, 199, 200, 201, 202, 0, 0, 203, 204, 205, - 206, 0, 0, 207, 208, 209, 210, 211, 0, 212, - 213, 214, 0, 215, 216, 217, 218, 0, 219, 220, + 0, 0, 0, 122, 123, 124, 125, 126, 127, 128, + 129, 842, 130, 131, 132, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 134, 135, 0, 0, 137, 138, + 0, 140, 141, 142, 143, 144, 0, 146, 147, 0, + 0, 148, 149, 150, 151, 152, 0, 0, 153, 154, + 155, 156, 157, 158, 159, 0, 160, 161, 162, 163, + 164, 0, 0, 0, 166, 167, 168, 169, 170, 171, + 0, 173, 174, 175, 0, 176, 177, 178, 179, 180, + 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 0, 198, + 0, 199, 200, 201, 202, 203, 204, 0, 0, 205, + 206, 207, 208, 0, 0, 209, 210, 211, 212, 213, + 0, 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - 0, 231, 0, 232, 233, 234, 235, 0, 236, 0, - 237, 0, 0, 0, 240, 241, 538, 0, 244, 245, - 246, 0, 247, 248, 249, 250, 0, 251, 252, 253, - 254, 255, 1355, 257, 0, 259, 260, 261, 262, 0, - 263, 264, 265, 266, 267, 268, 269, 0, 270, 0, - 272, 273, 274, 275, 276, 277, 278, 279, 0, 280, - 0, 281, 0, 0, 284, 0, 286, 287, 288, 0, - 289, 290, 291, 0, 0, 292, 0, 294, 0, 0, - 296, 297, 298, 299, 300, 301, 302, 303, 539, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 231, 232, 0, 233, 0, 234, 235, 236, 237, 0, + 238, 0, 239, 0, 0, 0, 242, 243, 540, 0, + 246, 247, 248, 0, 249, 250, 251, 252, 0, 253, + 254, 255, 256, 257, 258, 259, 0, 261, 262, 263, + 264, 0, 265, 266, 267, 268, 269, 270, 271, 0, + 272, 0, 274, 275, 276, 277, 278, 279, 280, 281, + 0, 282, 0, 283, 0, 0, 286, 0, 288, 289, + 290, 0, 291, 292, 293, 0, 0, 294, 0, 296, + 0, 0, 298, 299, 300, 301, 302, 303, 304, 305, + 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, 0, 329, 330, 331, 332, 333, 0, 334, - 335, 0, 337, 0, 338, 339, 340, 341, 342, 343, - 0, 344, 345, 0, 0, 346, 347, 348, 0, 0, - 349, 350, 351, 0, 353, 0, 355, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 0, - 0, 0, 0, 368, 369, 370, 0, 372, 373, 374, - 375, 376, 377, 0, 378, 379, 380, 381, 382, 383, - 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 0, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 0, 407, 408, 0, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 0, 0, 427, 428, - 429, 430, 431, 432, 433, 434, 435, 0, 0, 437, - 438, 439, 440, 0, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 540, 456, - 457, 0, 0, 458, 459, 0, 460, 0, 462, 463, - 464, 465, 466, 467, 0, 468, 469, 470, 0, 0, - 471, 472, 473, 474, 475, 0, 476, 477, 478, 479, - 480, 481, 482, 483, 0, 0, 484, 485, 486, 0, - 0, 487, 488, 489, 490, 0, 491, 492, 493, 494, - 495, 496, 497, 498, 0, 499, 0, 501, 502, 503, - 504, 505, 506, 507, 0, 0, 508, 0, 0, 509, - 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, + 326, 327, 328, 329, 0, 331, 332, 333, 334, 335, + 0, 336, 337, 0, 339, 0, 340, 341, 342, 343, + 344, 345, 0, 346, 347, 0, 0, 348, 349, 350, + 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 0, 0, 0, 0, 370, 371, 372, 0, 374, + 375, 376, 377, 378, 379, 0, 380, 381, 382, 383, + 384, 385, 0, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 0, 409, 410, + 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 426, 427, 428, 0, 0, + 429, 430, 431, 432, 433, 434, 435, 436, 437, 0, + 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 542, 458, 459, 0, 0, 460, 461, 0, 462, 0, + 464, 465, 466, 467, 468, 469, 0, 470, 846, 472, + 0, 0, 847, 474, 475, 476, 477, 0, 478, 479, + 480, 481, 482, 483, 484, 485, 0, 0, 486, 487, + 488, 0, 0, 489, 490, 491, 492, 0, 493, 494, + 495, 496, 497, 498, 499, 500, 0, 501, 0, 503, + 504, 505, 506, 507, 508, 509, 0, 0, 510, 0, + 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, - 537, 0, 562, 0, 0, 0, 0, 0, 0, 0, + 530, 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 132, 133, 0, 0, 135, 136, 0, 138, 139, - 140, 141, 142, 0, 144, 145, 0, 0, 146, 147, - 148, 149, 150, 0, 0, 151, 152, 153, 154, 155, - 156, 157, 0, 158, 159, 160, 161, 162, 0, 0, - 0, 164, 165, 166, 167, 168, 169, 0, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 0, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 0, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 209, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, 0, - 231, 0, 232, 233, 234, 235, 0, 236, 0, 237, - 0, 0, 0, 240, 241, 538, 0, 244, 245, 246, - 0, 247, 248, 249, 250, 0, 251, 252, 253, 254, - 255, 1357, 257, 0, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 0, 272, - 273, 274, 275, 276, 277, 278, 279, 0, 280, 0, - 281, 0, 0, 284, 0, 286, 287, 288, 0, 289, - 290, 291, 0, 0, 292, 0, 294, 0, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 539, 305, 306, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 134, 135, 0, 0, 137, 138, 0, + 140, 141, 142, 143, 144, 0, 146, 147, 0, 0, + 148, 149, 150, 151, 152, 0, 0, 153, 154, 155, + 156, 157, 158, 159, 0, 160, 161, 162, 163, 164, + 0, 0, 0, 166, 167, 168, 169, 170, 171, 0, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 211, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 0, 233, 0, 234, 235, 236, 237, 0, 238, + 0, 239, 0, 0, 0, 242, 243, 540, 0, 246, + 247, 248, 0, 249, 250, 251, 252, 0, 253, 254, + 255, 256, 257, 1355, 259, 0, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 0, 274, 275, 276, 277, 278, 279, 280, 281, 0, + 282, 0, 283, 0, 0, 286, 0, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 0, 296, 0, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, 0, 329, 330, 331, 332, 333, 0, 334, 335, - 0, 337, 0, 338, 339, 340, 341, 342, 343, 0, - 344, 345, 0, 0, 346, 347, 348, 0, 0, 349, - 350, 351, 0, 353, 0, 355, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 370, 0, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 0, 410, 411, + 327, 328, 329, 0, 331, 332, 333, 334, 335, 0, + 336, 337, 0, 339, 0, 340, 341, 342, 343, 344, + 345, 0, 346, 347, 0, 0, 348, 349, 350, 0, + 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 372, 0, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 0, 437, 438, - 439, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 540, 456, 457, - 0, 0, 458, 459, 0, 460, 0, 462, 463, 464, - 465, 466, 467, 0, 468, 469, 470, 0, 0, 471, - 472, 473, 474, 475, 0, 476, 477, 478, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 0, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 497, 498, 0, 499, 0, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 0, + 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 542, + 458, 459, 0, 0, 460, 461, 0, 462, 0, 464, + 465, 466, 467, 468, 469, 0, 470, 471, 472, 0, + 0, 473, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 0, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 499, 500, 0, 501, 0, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 537, - 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, - 121, 122, 123, 124, 125, 126, 127, 0, 128, 129, - 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 132, 133, 0, 0, 135, 136, 0, 138, 139, 140, - 141, 142, 0, 144, 145, 0, 0, 146, 147, 148, - 149, 150, 0, 0, 151, 152, 153, 154, 155, 156, - 157, 0, 158, 159, 160, 161, 162, 0, 0, 0, - 164, 165, 166, 167, 168, 169, 0, 171, 172, 173, - 0, 174, 175, 176, 177, 178, 179, 0, 0, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 0, 196, 0, 197, 198, 199, - 200, 201, 202, 0, 0, 203, 204, 205, 206, 0, - 0, 207, 208, 209, 210, 211, 0, 212, 213, 214, - 0, 215, 216, 217, 218, 0, 219, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, 0, 231, - 0, 232, 233, 234, 235, 0, 236, 0, 237, 0, - 0, 0, 240, 241, 538, 0, 244, 245, 246, 0, - 247, 248, 249, 250, 0, 251, 252, 253, 254, 255, - 2333, 257, 0, 259, 260, 261, 262, 0, 263, 264, - 265, 266, 267, 268, 269, 0, 270, 0, 272, 273, - 274, 275, 276, 277, 278, 279, 0, 280, 0, 281, - 0, 0, 284, 0, 286, 287, 288, 0, 289, 290, - 291, 0, 0, 292, 0, 294, 0, 0, 296, 297, - 298, 299, 300, 301, 302, 303, 539, 305, 306, 307, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 122, 123, 124, 125, 126, 127, 128, 129, 0, + 130, 131, 132, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 134, 135, 0, 0, 137, 138, 0, 140, + 141, 142, 143, 144, 0, 146, 147, 0, 0, 148, + 149, 150, 151, 152, 0, 0, 153, 154, 155, 156, + 157, 158, 159, 0, 160, 161, 162, 163, 164, 0, + 0, 0, 166, 167, 168, 169, 170, 171, 0, 173, + 174, 175, 0, 176, 177, 178, 179, 180, 181, 0, + 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 0, 198, 0, 199, + 200, 201, 202, 203, 204, 0, 0, 205, 206, 207, + 208, 0, 0, 209, 210, 211, 212, 213, 0, 214, + 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, + 0, 233, 0, 234, 235, 236, 237, 0, 238, 0, + 239, 0, 0, 0, 242, 243, 540, 0, 246, 247, + 248, 0, 249, 250, 251, 252, 0, 253, 254, 255, + 256, 257, 1357, 259, 0, 261, 262, 263, 264, 0, + 265, 266, 267, 268, 269, 270, 271, 0, 272, 0, + 274, 275, 276, 277, 278, 279, 280, 281, 0, 282, + 0, 283, 0, 0, 286, 0, 288, 289, 290, 0, + 291, 292, 293, 0, 0, 294, 0, 296, 0, 0, + 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - 0, 329, 330, 331, 332, 333, 0, 334, 335, 0, - 337, 0, 338, 339, 340, 341, 342, 343, 0, 344, - 345, 0, 0, 346, 347, 348, 0, 0, 349, 350, - 351, 0, 353, 0, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 0, 0, 0, - 0, 368, 369, 370, 0, 372, 373, 374, 375, 376, - 377, 0, 378, 379, 380, 381, 382, 383, 0, 384, - 385, 386, 387, 388, 389, 390, 391, 392, 393, 0, - 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 0, 407, 408, 0, 410, 411, 412, + 328, 329, 0, 331, 332, 333, 334, 335, 0, 336, + 337, 0, 339, 0, 340, 341, 342, 343, 344, 345, + 0, 346, 347, 0, 0, 348, 349, 350, 0, 0, + 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 0, + 0, 0, 0, 370, 371, 372, 0, 374, 375, 376, + 377, 378, 379, 0, 380, 381, 382, 383, 384, 385, + 0, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 426, 0, 0, 427, 428, 429, 430, - 431, 432, 433, 434, 435, 0, 0, 437, 438, 439, - 440, 0, 441, 442, 443, 444, 445, 446, 447, 448, - 449, 450, 451, 452, 453, 454, 540, 456, 457, 0, - 0, 458, 459, 0, 460, 0, 462, 463, 464, 465, - 466, 467, 0, 468, 469, 470, 0, 0, 471, 472, - 473, 474, 475, 0, 476, 477, 478, 479, 480, 481, - 482, 483, 0, 0, 484, 485, 486, 0, 0, 487, - 488, 489, 490, 0, 491, 492, 493, 494, 495, 496, - 497, 498, 0, 499, 0, 501, 502, 503, 504, 505, - 506, 507, 0, 0, 508, 0, 0, 509, 510, 511, + 423, 424, 425, 426, 427, 428, 0, 0, 429, 430, + 431, 432, 433, 434, 435, 436, 437, 0, 0, 439, + 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 456, 542, 458, + 459, 0, 0, 460, 461, 0, 462, 0, 464, 465, + 466, 467, 468, 469, 0, 470, 471, 472, 0, 0, + 473, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 485, 0, 0, 486, 487, 488, 0, + 0, 489, 490, 491, 492, 0, 493, 494, 495, 496, + 497, 498, 499, 500, 0, 501, 0, 503, 504, 505, + 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, - 522, 523, 524, 525, 526, 527, 528, 529, 537, 0, - 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, - 133, 0, 0, 135, 136, 0, 138, 139, 140, 141, - 142, 0, 144, 145, 0, 0, 146, 147, 148, 149, - 150, 0, 0, 151, 152, 153, 154, 155, 156, 157, - 0, 158, 159, 160, 161, 162, 0, 0, 0, 164, - 165, 166, 167, 168, 169, 0, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 0, 181, 182, + 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, + 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 134, 135, 0, 0, 137, 138, 0, 140, 141, + 142, 143, 144, 0, 146, 147, 0, 0, 148, 149, + 150, 151, 152, 0, 0, 153, 154, 155, 156, 157, + 158, 159, 0, 160, 161, 162, 163, 164, 0, 0, + 0, 166, 167, 168, 169, 170, 171, 0, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 209, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, 0, 231, 0, - 232, 233, 234, 235, 0, 236, 0, 237, 0, 0, - 0, 240, 241, 538, 0, 244, 245, 246, 0, 247, - 248, 249, 250, 0, 251, 252, 253, 254, 255, 3171, - 257, 0, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 0, 272, 273, 274, - 275, 276, 277, 278, 279, 0, 280, 0, 281, 0, - 0, 284, 0, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 0, 294, 0, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 539, 305, 306, 307, 308, + 193, 194, 195, 196, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 211, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 0, + 233, 0, 234, 235, 236, 237, 0, 238, 0, 239, + 0, 0, 0, 242, 243, 540, 0, 246, 247, 248, + 0, 249, 250, 251, 252, 0, 253, 254, 255, 256, + 257, 1360, 259, 0, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 0, 274, + 275, 276, 277, 278, 279, 280, 281, 0, 282, 0, + 283, 0, 0, 286, 0, 288, 289, 290, 0, 291, + 292, 293, 0, 0, 294, 0, 296, 0, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, 0, - 329, 330, 331, 332, 333, 0, 334, 335, 0, 337, - 0, 338, 339, 340, 341, 342, 343, 0, 344, 345, - 0, 0, 346, 347, 348, 0, 0, 349, 350, 351, - 0, 353, 0, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 370, 0, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 0, 410, 411, 412, 413, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 0, 331, 332, 333, 334, 335, 0, 336, 337, + 0, 339, 0, 340, 341, 342, 343, 344, 345, 0, + 346, 347, 0, 0, 348, 349, 350, 0, 0, 351, + 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 372, 0, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 0, 437, 438, 439, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 540, 456, 457, 0, 0, - 458, 459, 0, 460, 0, 462, 463, 464, 465, 466, - 467, 0, 468, 469, 470, 0, 0, 471, 472, 473, - 474, 475, 0, 476, 477, 478, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 497, - 498, 0, 499, 0, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 0, 439, 440, + 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 542, 458, 459, + 0, 0, 460, 461, 0, 462, 0, 464, 465, 466, + 467, 468, 469, 0, 470, 471, 472, 0, 0, 473, + 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 0, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 499, 500, 0, 501, 0, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 537, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 120, 121, 122, - 123, 124, 125, 126, 127, 0, 128, 129, 130, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 132, 133, - 0, 0, 135, 136, 0, 138, 139, 140, 141, 142, - 0, 144, 145, 0, 0, 146, 147, 148, 149, 150, - 0, 0, 151, 152, 153, 154, 155, 156, 157, 0, - 158, 159, 160, 161, 162, 0, 0, 0, 164, 165, - 166, 167, 168, 169, 0, 171, 172, 173, 0, 174, - 175, 176, 177, 178, 179, 0, 0, 181, 182, 183, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 539, + 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, + 123, 124, 125, 126, 127, 128, 129, 0, 130, 131, + 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 134, 135, 0, 0, 137, 138, 0, 140, 141, 142, + 143, 144, 0, 146, 147, 0, 0, 148, 149, 150, + 151, 152, 0, 0, 153, 154, 155, 156, 157, 158, + 159, 0, 160, 161, 162, 163, 164, 0, 0, 0, + 166, 167, 168, 169, 170, 171, 0, 173, 174, 175, + 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 0, 196, 0, 197, 198, 199, 200, 201, - 202, 0, 0, 203, 204, 205, 206, 0, 0, 207, - 208, 209, 210, 211, 0, 212, 213, 214, 0, 215, - 216, 217, 218, 0, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 0, 231, 0, 232, - 233, 234, 235, 0, 236, 0, 237, 0, 0, 0, - 240, 241, 538, 0, 244, 245, 246, 0, 247, 248, - 249, 250, 0, 251, 252, 253, 254, 255, 256, 257, - 0, 259, 260, 261, 262, 0, 263, 264, 265, 266, - 267, 268, 269, 0, 270, 0, 272, 273, 274, 275, - 276, 277, 278, 279, 0, 280, 0, 281, 0, 0, - 284, 0, 286, 287, 288, 0, 289, 290, 291, 0, - 0, 292, 0, 294, 0, 0, 296, 297, 298, 299, - 300, 301, 302, 303, 539, 305, 306, 307, 308, 309, + 194, 195, 196, 197, 0, 198, 0, 199, 200, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 0, + 0, 209, 210, 211, 212, 213, 0, 214, 215, 216, + 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 0, 233, + 0, 234, 235, 236, 237, 0, 238, 0, 239, 0, + 0, 0, 242, 243, 540, 0, 246, 247, 248, 0, + 249, 250, 251, 252, 0, 253, 254, 255, 256, 257, + 1362, 259, 0, 261, 262, 263, 264, 0, 265, 266, + 267, 268, 269, 270, 271, 0, 272, 0, 274, 275, + 276, 277, 278, 279, 280, 281, 0, 282, 0, 283, + 0, 0, 286, 0, 288, 289, 290, 0, 291, 292, + 293, 0, 0, 294, 0, 296, 0, 0, 298, 299, + 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, 0, 329, - 330, 331, 332, 333, 0, 334, 335, 0, 337, 0, - 338, 339, 340, 341, 342, 343, 0, 344, 345, 0, - 0, 346, 347, 348, 0, 0, 349, 350, 351, 0, - 353, 0, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 0, 0, 0, 0, 368, - 369, 370, 0, 372, 373, 374, 375, 376, 377, 0, - 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, - 387, 388, 389, 390, 391, 392, 393, 0, 394, 395, + 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 0, 331, 332, 333, 334, 335, 0, 336, 337, 0, + 339, 0, 340, 341, 342, 343, 344, 345, 0, 346, + 347, 0, 0, 348, 349, 350, 0, 0, 351, 352, + 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 0, 0, 0, + 0, 370, 371, 372, 0, 374, 375, 376, 377, 378, + 379, 0, 380, 381, 382, 383, 384, 385, 0, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 0, 407, 408, 0, 410, 411, 412, 413, 414, + 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 0, 0, 427, 428, 429, 430, 431, 432, - 433, 434, 435, 0, 0, 437, 438, 439, 440, 0, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 540, 456, 457, 0, 0, 458, - 459, 0, 460, 0, 462, 463, 464, 465, 466, 467, - 0, 468, 469, 470, 0, 0, 471, 472, 473, 474, - 475, 0, 476, 477, 478, 479, 480, 481, 482, 483, - 0, 0, 484, 485, 486, 0, 0, 487, 488, 489, - 490, 0, 491, 492, 493, 494, 495, 496, 497, 498, - 0, 499, 0, 501, 502, 503, 504, 505, 506, 507, - 0, 0, 508, 0, 0, 509, 510, 511, 512, 513, + 425, 426, 427, 428, 0, 0, 429, 430, 431, 432, + 433, 434, 435, 436, 437, 0, 0, 439, 440, 441, + 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, + 451, 452, 453, 454, 455, 456, 542, 458, 459, 0, + 0, 460, 461, 0, 462, 0, 464, 465, 466, 467, + 468, 469, 0, 470, 471, 472, 0, 0, 473, 474, + 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, + 484, 485, 0, 0, 486, 487, 488, 0, 0, 489, + 490, 491, 492, 0, 493, 494, 495, 496, 497, 498, + 499, 500, 0, 501, 0, 503, 504, 505, 506, 507, + 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, - 524, 525, 526, 527, 528, 529, 537, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 120, 121, 122, 123, - 124, 125, 126, 127, 0, 128, 129, 130, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 132, 133, 0, - 0, 135, 136, 0, 138, 139, 140, 141, 142, 0, - 144, 145, 0, 0, 146, 147, 148, 149, 150, 0, - 0, 151, 152, 153, 154, 155, 156, 157, 0, 158, - 159, 160, 161, 162, 0, 0, 0, 164, 165, 166, - 167, 168, 169, 0, 171, 172, 173, 0, 174, 175, - 176, 177, 178, 179, 0, 0, 181, 182, 183, 184, + 524, 525, 526, 527, 528, 529, 530, 531, 539, 0, + 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, + 135, 0, 0, 137, 138, 0, 140, 141, 142, 143, + 144, 0, 146, 147, 0, 0, 148, 149, 150, 151, + 152, 0, 0, 153, 154, 155, 156, 157, 158, 159, + 0, 160, 161, 162, 163, 164, 0, 0, 0, 166, + 167, 168, 169, 170, 171, 0, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 0, 196, 0, 197, 198, 199, 200, 201, 202, - 0, 0, 203, 204, 205, 206, 0, 0, 207, 208, - 209, 210, 211, 0, 212, 213, 214, 0, 215, 216, - 217, 218, 0, 219, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, 0, 231, 0, 232, 233, - 234, 235, 0, 236, 0, 237, 0, 0, 0, 240, - 241, 538, 0, 855, 245, 246, 0, 247, 248, 249, - 250, 0, 251, 252, 253, 254, 255, 256, 257, 0, - 259, 260, 261, 262, 0, 263, 264, 265, 266, 267, - 268, 269, 0, 270, 0, 272, 273, 274, 275, 276, - 277, 278, 279, 0, 280, 0, 281, 0, 0, 284, - 0, 286, 287, 288, 0, 289, 290, 291, 0, 0, - 292, 0, 294, 0, 0, 296, 297, 856, 299, 300, - 301, 302, 303, 539, 305, 306, 307, 308, 309, 310, + 195, 196, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 211, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, 231, 232, 0, 233, 0, + 234, 235, 236, 237, 0, 238, 0, 239, 0, 0, + 0, 242, 243, 540, 0, 246, 247, 248, 0, 249, + 250, 251, 252, 0, 253, 254, 255, 256, 257, 1364, + 259, 0, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 0, 274, 275, 276, + 277, 278, 279, 280, 281, 0, 282, 0, 283, 0, + 0, 286, 0, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 0, 296, 0, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 0, 329, 330, - 331, 332, 333, 0, 334, 335, 0, 337, 0, 338, - 339, 340, 341, 342, 343, 0, 344, 345, 0, 0, - 346, 347, 348, 0, 0, 349, 350, 351, 0, 353, - 0, 355, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 0, 0, 0, 0, 368, 369, - 370, 0, 372, 373, 374, 375, 376, 377, 0, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 0, 394, 395, 396, + 321, 322, 323, 324, 325, 326, 327, 328, 329, 0, + 331, 332, 333, 334, 335, 0, 336, 337, 0, 339, + 0, 340, 341, 342, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 353, + 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 372, 0, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 0, 407, 408, 0, 410, 411, 412, 413, 414, 415, - 416, 417, 857, 419, 420, 421, 422, 423, 424, 425, - 426, 0, 0, 427, 428, 429, 430, 858, 432, 433, - 434, 435, 0, 0, 437, 438, 439, 440, 0, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 540, 456, 457, 0, 0, 458, 459, - 0, 460, 0, 462, 463, 464, 465, 466, 467, 0, - 468, 859, 470, 0, 0, 471, 472, 473, 474, 475, - 0, 476, 477, 478, 479, 480, 481, 482, 483, 0, - 0, 484, 485, 486, 0, 0, 487, 488, 489, 490, - 0, 491, 492, 493, 494, 495, 496, 497, 860, 0, - 499, 0, 501, 502, 503, 504, 505, 506, 507, 0, - 0, 508, 0, 0, 509, 510, 511, 512, 513, 514, + 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, + 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 0, 0, 439, 440, 441, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 542, 458, 459, 0, 0, + 460, 461, 0, 462, 0, 464, 465, 466, 467, 468, + 469, 0, 470, 471, 472, 0, 0, 473, 474, 475, + 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 0, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 499, + 500, 0, 501, 0, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, - 525, 526, 527, 528, 529, 537, 0, 0, 0, 0, + 525, 526, 527, 528, 529, 530, 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 120, 121, 122, 123, 124, - 125, 126, 127, 0, 128, 129, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 132, 133, 0, 0, - 135, 136, 0, 138, 139, 140, 141, 142, 0, 144, - 145, 0, 0, 146, 147, 148, 149, 150, 0, 0, - 151, 152, 153, 154, 155, 156, 157, 0, 158, 159, - 160, 161, 162, 0, 0, 0, 164, 165, 166, 167, - 168, 169, 0, 171, 172, 173, 0, 174, 175, 176, - 177, 178, 179, 0, 0, 181, 182, 183, 184, 185, + 0, 0, 0, 0, 0, 0, 0, 122, 123, 124, + 125, 126, 127, 128, 129, 0, 130, 131, 132, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 134, 135, + 0, 0, 137, 138, 0, 140, 141, 142, 143, 144, + 0, 146, 147, 0, 0, 148, 149, 150, 151, 152, + 0, 0, 153, 154, 155, 156, 157, 158, 159, 0, + 160, 161, 162, 163, 164, 0, 0, 0, 166, 167, + 168, 169, 170, 171, 0, 173, 174, 175, 0, 176, + 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 0, 196, 0, 197, 198, 199, 200, 201, 202, 0, - 0, 203, 204, 205, 206, 0, 0, 207, 208, 209, - 210, 211, 0, 212, 213, 214, 0, 215, 216, 217, - 218, 0, 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 0, 231, 0, 232, 233, 234, - 235, 0, 236, 0, 237, 0, 0, 0, 240, 241, - 538, 0, 244, 245, 246, 0, 247, 248, 249, 250, - 0, 251, 252, 253, 254, 255, 975, 257, 0, 259, - 260, 261, 262, 0, 263, 264, 265, 266, 267, 268, - 269, 0, 270, 0, 272, 273, 274, 275, 276, 277, - 278, 279, 0, 280, 0, 281, 0, 0, 284, 0, - 286, 287, 288, 0, 289, 290, 291, 0, 0, 292, - 0, 294, 0, 0, 296, 297, 298, 299, 300, 301, - 302, 303, 539, 305, 306, 307, 308, 309, 310, 311, + 196, 197, 0, 198, 0, 199, 200, 201, 202, 203, + 204, 0, 0, 205, 206, 207, 208, 0, 0, 209, + 210, 211, 212, 213, 0, 214, 215, 216, 0, 217, + 218, 219, 220, 0, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, 231, 232, 0, 233, 0, 234, + 235, 236, 237, 0, 238, 0, 239, 0, 0, 0, + 242, 243, 540, 0, 246, 247, 248, 0, 249, 250, + 251, 252, 0, 253, 254, 255, 256, 257, 2345, 259, + 0, 261, 262, 263, 264, 0, 265, 266, 267, 268, + 269, 270, 271, 0, 272, 0, 274, 275, 276, 277, + 278, 279, 280, 281, 0, 282, 0, 283, 0, 0, + 286, 0, 288, 289, 290, 0, 291, 292, 293, 0, + 0, 294, 0, 296, 0, 0, 298, 299, 300, 301, + 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 0, 329, 330, 331, - 332, 333, 0, 334, 335, 0, 337, 0, 338, 339, - 340, 341, 342, 343, 0, 344, 345, 0, 0, 346, - 347, 348, 0, 0, 349, 350, 351, 0, 353, 0, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 0, 0, 0, 0, 368, 369, 370, - 0, 372, 373, 374, 375, 376, 377, 0, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 0, 394, 395, 396, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 0, - 407, 408, 0, 410, 411, 412, 413, 414, 415, 416, + 322, 323, 324, 325, 326, 327, 328, 329, 0, 331, + 332, 333, 334, 335, 0, 336, 337, 0, 339, 0, + 340, 341, 342, 343, 344, 345, 0, 346, 347, 0, + 0, 348, 349, 350, 0, 0, 351, 352, 353, 0, + 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 0, 0, 0, 0, 370, + 371, 372, 0, 374, 375, 376, 377, 378, 379, 0, + 380, 381, 382, 383, 384, 385, 0, 386, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 0, 0, 427, 428, 429, 430, 431, 432, 433, 434, - 435, 0, 0, 437, 438, 439, 440, 0, 441, 442, + 427, 428, 0, 0, 429, 430, 431, 432, 433, 434, + 435, 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, - 453, 454, 540, 456, 457, 0, 0, 458, 459, 0, - 460, 0, 462, 463, 464, 465, 466, 467, 0, 468, - 469, 470, 0, 0, 471, 472, 473, 474, 475, 0, - 476, 477, 478, 479, 480, 481, 482, 483, 0, 0, - 484, 485, 486, 0, 0, 487, 488, 489, 490, 0, - 491, 492, 493, 494, 495, 496, 497, 498, 0, 499, - 0, 501, 502, 503, 504, 505, 506, 507, 0, 0, - 508, 0, 0, 509, 510, 511, 512, 513, 514, 515, + 453, 454, 455, 456, 542, 458, 459, 0, 0, 460, + 461, 0, 462, 0, 464, 465, 466, 467, 468, 469, + 0, 470, 471, 472, 0, 0, 473, 474, 475, 476, + 477, 0, 478, 479, 480, 481, 482, 483, 484, 485, + 0, 0, 486, 487, 488, 0, 0, 489, 490, 491, + 492, 0, 493, 494, 495, 496, 497, 498, 499, 500, + 0, 501, 0, 503, 504, 505, 506, 507, 508, 509, + 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, - 526, 527, 528, 529, 537, 0, 0, 0, 0, 0, + 526, 527, 528, 529, 530, 531, 539, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 121, 122, 123, 124, 125, - 126, 127, 0, 128, 129, 130, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 132, 133, 0, 0, 135, - 136, 0, 138, 139, 140, 141, 142, 0, 144, 145, - 0, 0, 146, 147, 148, 149, 150, 0, 0, 151, - 152, 153, 154, 155, 156, 157, 0, 158, 159, 160, - 161, 162, 0, 0, 0, 164, 165, 166, 167, 168, - 169, 0, 171, 172, 173, 0, 174, 175, 176, 177, - 178, 179, 0, 0, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 0, - 196, 0, 197, 198, 199, 200, 201, 202, 0, 0, - 203, 204, 205, 206, 0, 0, 207, 208, 209, 210, - 211, 0, 212, 213, 214, 0, 215, 216, 217, 218, - 0, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 0, 231, 0, 232, 233, 234, 235, - 0, 236, 0, 237, 0, 0, 0, 240, 241, 538, - 0, 244, 245, 246, 0, 247, 248, 249, 250, 0, - 251, 252, 253, 254, 255, 256, 257, 0, 259, 260, - 261, 262, 0, 263, 264, 265, 266, 267, 268, 269, - 0, 270, 0, 272, 273, 274, 275, 276, 277, 278, - 279, 0, 280, 0, 281, 0, 0, 284, 0, 286, - 287, 288, 0, 289, 290, 291, 0, 0, 292, 0, - 294, 0, 0, 296, 297, 298, 299, 300, 301, 302, - 303, 539, 305, 306, 307, 308, 309, 310, 311, 312, + 0, 0, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 134, 135, 0, + 0, 137, 138, 0, 140, 141, 142, 143, 144, 0, + 146, 147, 0, 0, 148, 149, 150, 151, 152, 0, + 0, 153, 154, 155, 156, 157, 158, 159, 0, 160, + 161, 162, 163, 164, 0, 0, 0, 166, 167, 168, + 169, 170, 171, 0, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 211, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 0, 233, 0, 234, 235, + 236, 237, 0, 238, 0, 239, 0, 0, 0, 242, + 243, 540, 0, 246, 247, 248, 0, 249, 250, 251, + 252, 0, 253, 254, 255, 256, 257, 3185, 259, 0, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 0, 274, 275, 276, 277, 278, + 279, 280, 281, 0, 282, 0, 283, 0, 0, 286, + 0, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 0, 296, 0, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, 0, 329, 330, 331, 332, - 333, 0, 334, 335, 0, 337, 0, 338, 339, 340, - 341, 342, 343, 0, 344, 345, 0, 0, 346, 347, - 348, 0, 0, 349, 350, 351, 0, 353, 0, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 0, 0, 0, 0, 368, 369, 370, 0, - 372, 373, 374, 375, 376, 377, 0, 378, 379, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, - 390, 391, 392, 393, 0, 394, 395, 396, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 0, 407, - 408, 0, 410, 411, 412, 413, 414, 415, 416, 417, - 857, 419, 420, 421, 422, 423, 424, 425, 426, 0, - 0, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 0, 0, 437, 438, 439, 440, 0, 441, 442, 443, + 323, 324, 325, 326, 327, 328, 329, 0, 331, 332, + 333, 334, 335, 0, 336, 337, 0, 339, 0, 340, + 341, 342, 343, 344, 345, 0, 346, 347, 0, 0, + 348, 349, 350, 0, 0, 351, 352, 353, 0, 355, + 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 372, 0, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, - 454, 540, 456, 457, 0, 0, 458, 459, 0, 460, - 0, 462, 463, 464, 465, 466, 467, 0, 468, 859, - 470, 0, 0, 471, 472, 473, 474, 475, 0, 476, - 477, 478, 479, 480, 481, 482, 483, 0, 0, 484, - 485, 486, 0, 0, 487, 488, 489, 490, 0, 491, - 492, 493, 494, 495, 496, 497, 498, 0, 499, 0, - 501, 502, 503, 504, 505, 506, 507, 0, 0, 508, - 0, 0, 509, 510, 511, 512, 513, 514, 515, 516, + 454, 455, 456, 542, 458, 459, 0, 0, 460, 461, + 0, 462, 0, 464, 465, 466, 467, 468, 469, 0, + 470, 471, 472, 0, 0, 473, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 499, 500, 0, + 501, 0, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, - 527, 528, 529, 537, 0, 0, 0, 0, 0, 0, + 527, 528, 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 120, 121, 122, 123, 124, 125, 126, - 127, 0, 128, 129, 130, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 133, 0, 0, 135, 136, - 0, 138, 139, 140, 141, 142, 0, 144, 145, 0, - 0, 146, 147, 148, 149, 150, 0, 0, 151, 152, - 153, 154, 155, 156, 157, 0, 158, 159, 160, 161, - 162, 0, 0, 0, 164, 165, 166, 167, 168, 169, - 0, 171, 172, 173, 0, 174, 175, 176, 177, 178, - 179, 0, 0, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 0, 196, - 0, 197, 198, 199, 200, 201, 202, 0, 0, 203, - 204, 205, 206, 0, 0, 207, 208, 209, 210, 211, - 0, 212, 213, 214, 0, 215, 216, 217, 218, 0, - 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, 0, 231, 0, 232, 233, 234, 235, 0, - 236, 0, 237, 0, 0, 0, 240, 241, 538, 0, - 244, 245, 246, 0, 247, 248, 249, 250, 0, 251, - 252, 253, 254, 255, 1344, 257, 0, 259, 260, 261, - 262, 0, 263, 264, 265, 266, 267, 268, 269, 0, - 270, 0, 272, 273, 274, 275, 276, 277, 278, 279, - 0, 280, 0, 281, 0, 0, 284, 0, 286, 287, - 288, 0, 289, 290, 291, 0, 0, 292, 0, 294, - 0, 0, 296, 297, 298, 299, 300, 301, 302, 303, - 539, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 0, 0, 0, 0, 0, 122, 123, 124, 125, 126, + 127, 128, 129, 0, 130, 131, 132, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 134, 135, 0, 0, + 137, 138, 0, 140, 141, 142, 143, 144, 0, 146, + 147, 0, 0, 148, 149, 150, 151, 152, 0, 0, + 153, 154, 155, 156, 157, 158, 159, 0, 160, 161, + 162, 163, 164, 0, 0, 0, 166, 167, 168, 169, + 170, 171, 0, 173, 174, 175, 0, 176, 177, 178, + 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 0, 198, 0, 199, 200, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 0, 0, 209, 210, 211, + 212, 213, 0, 214, 215, 216, 0, 217, 218, 219, + 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 0, 233, 0, 234, 235, 236, + 237, 0, 238, 0, 239, 0, 0, 0, 242, 243, + 540, 0, 246, 247, 248, 0, 249, 250, 251, 252, + 0, 253, 254, 255, 256, 257, 258, 259, 0, 261, + 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, + 271, 0, 272, 0, 274, 275, 276, 277, 278, 279, + 280, 281, 0, 282, 0, 283, 0, 0, 286, 0, + 288, 289, 290, 0, 291, 292, 293, 0, 0, 294, + 0, 296, 0, 0, 298, 299, 300, 301, 302, 303, + 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 327, 0, 329, 330, 331, 332, 333, - 0, 334, 335, 0, 337, 0, 338, 339, 340, 341, - 342, 343, 0, 344, 345, 0, 0, 346, 347, 348, - 0, 0, 349, 350, 351, 0, 353, 0, 355, 356, + 324, 325, 326, 327, 328, 329, 0, 331, 332, 333, + 334, 335, 0, 336, 337, 0, 339, 0, 340, 341, + 342, 343, 344, 345, 0, 346, 347, 0, 0, 348, + 349, 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 0, 0, 0, 0, 368, 369, 370, 0, 372, - 373, 374, 375, 376, 377, 0, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 0, 394, 395, 396, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 0, 407, 408, - 0, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 426, 0, 0, - 427, 428, 429, 430, 431, 432, 433, 434, 435, 0, - 0, 437, 438, 439, 440, 0, 441, 442, 443, 444, + 367, 368, 369, 0, 0, 0, 0, 370, 371, 372, + 0, 374, 375, 376, 377, 378, 379, 0, 380, 381, + 382, 383, 384, 385, 0, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 0, + 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 0, 0, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 540, 456, 457, 0, 0, 458, 459, 0, 460, 0, - 462, 463, 464, 465, 466, 467, 0, 468, 469, 470, - 0, 0, 471, 472, 473, 474, 475, 0, 476, 477, - 478, 479, 480, 481, 482, 483, 0, 0, 484, 485, - 486, 0, 0, 487, 488, 489, 490, 0, 491, 492, - 493, 494, 495, 496, 497, 498, 0, 499, 0, 501, - 502, 503, 504, 505, 506, 507, 0, 0, 508, 0, - 0, 509, 510, 511, 512, 513, 514, 515, 516, 517, + 455, 456, 542, 458, 459, 0, 0, 460, 461, 0, + 462, 0, 464, 465, 466, 467, 468, 469, 0, 470, + 471, 472, 0, 0, 473, 474, 475, 476, 477, 0, + 478, 479, 480, 481, 482, 483, 484, 485, 0, 0, + 486, 487, 488, 0, 0, 489, 490, 491, 492, 0, + 493, 494, 495, 496, 497, 498, 499, 500, 0, 501, + 0, 503, 504, 505, 506, 507, 508, 509, 0, 0, + 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, - 528, 529, 537, 0, 0, 0, 0, 0, 0, 0, + 528, 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 121, 122, 123, 124, 125, 126, 127, - 0, 128, 129, 130, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 132, 133, 0, 0, 135, 136, 0, - 138, 139, 140, 141, 142, 0, 144, 145, 0, 0, - 146, 147, 148, 149, 150, 0, 0, 151, 152, 153, - 154, 155, 156, 157, 0, 158, 159, 160, 161, 162, - 0, 0, 0, 164, 165, 166, 167, 168, 169, 0, - 171, 172, 173, 0, 174, 175, 176, 177, 178, 179, - 0, 0, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 0, 196, 0, - 197, 198, 199, 200, 201, 202, 0, 0, 203, 204, - 205, 206, 0, 0, 207, 208, 209, 210, 211, 0, - 212, 213, 214, 0, 215, 216, 217, 218, 0, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, 0, 231, 0, 232, 233, 234, 235, 0, 236, - 0, 237, 0, 0, 0, 240, 241, 538, 0, 244, - 245, 246, 0, 247, 248, 249, 250, 0, 251, 252, - 253, 254, 255, 1367, 257, 0, 259, 260, 261, 262, - 0, 263, 264, 265, 266, 267, 268, 269, 0, 270, - 0, 272, 273, 274, 275, 276, 277, 278, 279, 0, - 280, 0, 281, 0, 0, 284, 0, 286, 287, 288, - 0, 289, 290, 291, 0, 0, 292, 0, 294, 0, - 0, 296, 297, 298, 299, 300, 301, 302, 303, 539, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 134, 135, 0, 0, 137, + 138, 0, 140, 141, 142, 143, 144, 0, 146, 147, + 0, 0, 148, 149, 150, 151, 152, 0, 0, 153, + 154, 155, 156, 157, 158, 159, 0, 160, 161, 162, + 163, 164, 0, 0, 0, 166, 167, 168, 169, 170, + 171, 0, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 211, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 232, 0, 233, 0, 234, 235, 236, 237, + 0, 238, 0, 239, 0, 0, 0, 242, 243, 540, + 0, 858, 247, 248, 0, 249, 250, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 0, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 0, 274, 275, 276, 277, 278, 279, 280, + 281, 0, 282, 0, 283, 0, 0, 286, 0, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 0, + 296, 0, 0, 298, 299, 859, 301, 302, 303, 304, + 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 0, 329, 330, 331, 332, 333, 0, - 334, 335, 0, 337, 0, 338, 339, 340, 341, 342, - 343, 0, 344, 345, 0, 0, 346, 347, 348, 0, - 0, 349, 350, 351, 0, 353, 0, 355, 356, 357, + 325, 326, 327, 328, 329, 0, 331, 332, 333, 334, + 335, 0, 336, 337, 0, 339, 0, 340, 341, 342, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 0, 0, 0, 0, 368, 369, 370, 0, 372, 373, - 374, 375, 376, 377, 0, 378, 379, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 0, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 0, 407, 408, 0, - 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 0, 0, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 0, 0, - 437, 438, 439, 440, 0, 441, 442, 443, 444, 445, - 446, 447, 448, 449, 450, 451, 452, 453, 454, 540, - 456, 457, 0, 0, 458, 459, 0, 460, 0, 462, - 463, 464, 465, 466, 467, 0, 468, 469, 470, 0, - 0, 471, 472, 473, 474, 475, 0, 476, 477, 478, - 479, 480, 481, 482, 483, 0, 0, 484, 485, 486, - 0, 0, 487, 488, 489, 490, 0, 491, 492, 493, - 494, 495, 496, 497, 498, 0, 499, 0, 501, 502, - 503, 504, 505, 506, 507, 0, 0, 508, 0, 0, - 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, + 368, 369, 0, 0, 0, 0, 370, 371, 372, 0, + 374, 375, 376, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, + 860, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 0, 429, 430, 431, 432, 861, 434, 435, 436, 437, + 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 542, 458, 459, 0, 0, 460, 461, 0, 462, + 0, 464, 465, 466, 467, 468, 469, 0, 470, 862, + 472, 0, 0, 473, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 0, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 498, 499, 863, 0, 501, 0, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, - 529, 537, 0, 0, 0, 0, 0, 0, 0, 0, + 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 120, 121, 122, 123, 124, 125, 126, 127, 0, - 128, 129, 130, 0, 0, 0, 0, 0, 0, 1728, - 0, 0, 132, 133, 0, 0, 135, 136, 0, 138, - 139, 140, 141, 142, 0, 144, 145, 0, 0, 146, - 147, 148, 149, 150, 0, 0, 151, 152, 153, 154, - 155, 156, 157, 0, 158, 159, 160, 161, 162, 0, - 0, 0, 164, 165, 166, 167, 168, 169, 0, 171, - 172, 173, 0, 174, 175, 176, 177, 178, 179, 0, - 0, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 0, 196, 0, 197, - 198, 199, 200, 201, 202, 0, 0, 203, 204, 205, - 206, 0, 0, 207, 208, 209, 210, 211, 0, 212, - 213, 214, 0, 215, 216, 217, 218, 0, 219, 220, + 0, 0, 0, 122, 123, 124, 125, 126, 127, 128, + 129, 0, 130, 131, 132, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 134, 135, 0, 0, 137, 138, + 0, 140, 141, 142, 143, 144, 0, 146, 147, 0, + 0, 148, 149, 150, 151, 152, 0, 0, 153, 154, + 155, 156, 157, 158, 159, 0, 160, 161, 162, 163, + 164, 0, 0, 0, 166, 167, 168, 169, 170, 171, + 0, 173, 174, 175, 0, 176, 177, 178, 179, 180, + 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 0, 198, + 0, 199, 200, 201, 202, 203, 204, 0, 0, 205, + 206, 207, 208, 0, 0, 209, 210, 211, 212, 213, + 0, 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - 0, 231, 0, 232, 233, 234, 235, 0, 236, 0, - 237, 0, 0, 0, 240, 241, 538, 0, 244, 245, - 246, 0, 247, 248, 249, 250, 0, 251, 252, 253, - 254, 255, 256, 257, 0, 259, 260, 261, 262, 0, - 263, 264, 265, 266, 267, 268, 269, 0, 270, 0, - 272, 273, 274, 275, 276, 277, 278, 279, 0, 280, - 0, 281, 0, 0, 284, 0, 286, 287, 288, 0, - 289, 290, 291, 0, 0, 292, 0, 294, 0, 0, - 296, 297, 298, 299, 300, 301, 302, 303, 539, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 231, 232, 0, 233, 0, 234, 235, 236, 237, 0, + 238, 0, 239, 0, 0, 0, 242, 243, 540, 0, + 246, 247, 248, 0, 249, 250, 251, 252, 0, 253, + 254, 255, 256, 257, 916, 259, 0, 261, 262, 263, + 264, 0, 265, 266, 267, 268, 269, 270, 271, 0, + 272, 0, 274, 275, 276, 277, 278, 279, 280, 281, + 0, 282, 0, 283, 0, 0, 286, 0, 288, 289, + 290, 0, 291, 292, 293, 0, 0, 294, 0, 296, + 0, 0, 298, 299, 300, 301, 302, 303, 304, 305, + 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, 0, 329, 330, 331, 332, 333, 0, 334, - 335, 0, 337, 0, 338, 339, 340, 341, 342, 343, - 0, 344, 345, 0, 0, 346, 347, 348, 0, 0, - 349, 350, 351, 0, 353, 0, 355, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 0, - 0, 0, 0, 368, 369, 370, 0, 372, 373, 374, - 375, 376, 377, 0, 378, 379, 380, 381, 382, 383, - 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 0, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 0, 407, 408, 0, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 0, 0, 427, 428, - 429, 430, 431, 0, 433, 434, 435, 0, 0, 437, - 438, 439, 440, 0, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 540, 456, - 457, 0, 0, 458, 459, 0, 460, 0, 462, 463, - 464, 465, 466, 467, 0, 468, 469, 470, 0, 0, - 471, 472, 473, 474, 475, 0, 476, 477, 478, 479, - 480, 481, 482, 483, 0, 0, 484, 485, 486, 0, - 0, 487, 488, 489, 490, 0, 491, 492, 493, 494, - 495, 496, 497, 498, 0, 499, 0, 501, 502, 503, - 504, 505, 506, 507, 0, 0, 508, 0, 0, 509, - 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, + 326, 327, 328, 329, 0, 331, 332, 333, 334, 335, + 0, 336, 337, 0, 339, 0, 340, 341, 342, 343, + 344, 345, 0, 346, 347, 0, 0, 348, 349, 350, + 0, 0, 351, 352, 353, 0, 355, 0, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 0, 0, 0, 0, 370, 371, 372, 0, 374, + 375, 376, 377, 378, 379, 0, 380, 381, 382, 383, + 384, 385, 0, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 0, 409, 410, + 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 426, 427, 428, 0, 0, + 429, 430, 431, 432, 433, 434, 435, 436, 437, 0, + 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 542, 458, 459, 0, 0, 460, 461, 0, 462, 0, + 464, 465, 466, 467, 468, 469, 0, 470, 471, 472, + 0, 0, 473, 474, 475, 476, 477, 0, 478, 479, + 480, 481, 482, 483, 484, 485, 0, 0, 486, 487, + 488, 0, 0, 489, 490, 491, 492, 0, 493, 494, + 495, 496, 497, 498, 499, 500, 0, 501, 0, 503, + 504, 505, 506, 507, 508, 509, 0, 0, 510, 0, + 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, - 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 121, 122, 123, 124, 125, 126, 127, 0, 128, - 129, 130, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 132, 133, 0, 0, 135, 136, 0, 138, 139, - 140, 141, 142, 0, 144, 145, 0, 0, 146, 147, - 148, 149, 150, 0, 0, 151, 152, 153, 154, 155, - 156, 157, 0, 158, 159, 160, 161, 162, 0, 0, - 0, 164, 165, 166, 167, 168, 169, 0, 171, 172, - 173, 0, 174, 175, 176, 177, 178, 179, 0, 0, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 0, 196, 0, 197, 198, - 199, 200, 201, 202, 0, 0, 203, 204, 205, 206, - 0, 0, 207, 208, 209, 210, 211, 0, 212, 213, - 214, 0, 215, 216, 217, 218, 0, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, 0, - 231, 0, 232, 233, 234, 235, 0, 236, 0, 237, - 0, 0, 0, 240, 241, 538, 0, 244, 245, 246, - 0, 247, 248, 249, 250, 0, 251, 252, 253, 254, - 255, 1928, 257, 0, 259, 260, 261, 262, 0, 263, - 264, 265, 266, 267, 268, 269, 0, 270, 0, 272, - 273, 274, 275, 276, 277, 278, 279, 0, 280, 0, - 281, 0, 0, 284, 0, 286, 287, 288, 0, 289, - 290, 291, 0, 0, 292, 0, 294, 0, 0, 296, - 297, 298, 299, 300, 301, 302, 303, 539, 305, 306, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 134, 135, 0, 0, 137, 138, 0, + 140, 141, 142, 143, 144, 0, 146, 147, 0, 0, + 148, 149, 150, 151, 152, 0, 0, 153, 154, 155, + 156, 157, 158, 159, 0, 160, 161, 162, 163, 164, + 0, 0, 0, 166, 167, 168, 169, 170, 171, 0, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 211, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 0, 233, 0, 234, 235, 236, 237, 0, 238, + 0, 239, 0, 0, 0, 242, 243, 540, 0, 246, + 247, 248, 0, 249, 250, 251, 252, 0, 253, 254, + 255, 256, 257, 980, 259, 0, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 0, 274, 275, 276, 277, 278, 279, 280, 281, 0, + 282, 0, 283, 0, 0, 286, 0, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 0, 296, 0, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, 0, 329, 330, 331, 332, 333, 0, 334, 335, - 0, 337, 0, 338, 339, 340, 341, 342, 343, 0, - 344, 345, 0, 0, 346, 347, 348, 0, 0, 349, - 350, 351, 0, 353, 0, 355, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 0, 0, - 0, 0, 368, 369, 370, 0, 372, 373, 374, 375, - 376, 377, 0, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 0, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 0, 407, 408, 0, 410, 411, + 327, 328, 329, 0, 331, 332, 333, 334, 335, 0, + 336, 337, 0, 339, 0, 340, 341, 342, 343, 344, + 345, 0, 346, 347, 0, 0, 348, 349, 350, 0, + 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 372, 0, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 0, 0, 427, 428, 429, - 430, 431, 432, 433, 434, 435, 0, 0, 437, 438, - 439, 440, 0, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 540, 456, 457, - 0, 0, 458, 459, 0, 460, 0, 462, 463, 464, - 465, 466, 467, 0, 468, 469, 470, 0, 0, 471, - 472, 473, 474, 475, 0, 476, 477, 478, 479, 480, - 481, 482, 483, 0, 0, 484, 485, 486, 0, 0, - 487, 488, 489, 490, 0, 491, 492, 493, 494, 495, - 496, 497, 498, 0, 499, 0, 501, 502, 503, 504, - 505, 506, 507, 0, 0, 508, 0, 0, 509, 510, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 0, + 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 542, + 458, 459, 0, 0, 460, 461, 0, 462, 0, 464, + 465, 466, 467, 468, 469, 0, 470, 471, 472, 0, + 0, 473, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 0, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 499, 500, 0, 501, 0, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, - 521, 522, 523, 524, 525, 526, 527, 528, 529, 537, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, - 121, 122, 123, 124, 125, 126, 127, 0, 128, 129, - 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 132, 133, 0, 0, 135, 136, 0, 138, 139, 140, - 141, 142, 0, 144, 145, 0, 0, 146, 147, 148, - 149, 150, 0, 0, 151, 152, 153, 154, 155, 156, - 157, 0, 158, 159, 160, 161, 162, 0, 0, 0, - 164, 165, 166, 167, 168, 169, 0, 171, 172, 173, - 0, 174, 175, 176, 177, 178, 179, 0, 0, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 0, 196, 0, 197, 198, 199, - 200, 201, 202, 0, 0, 203, 204, 205, 206, 0, - 0, 207, 208, 209, 210, 211, 0, 212, 213, 214, - 0, 215, 216, 217, 218, 0, 219, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, 0, 231, - 0, 232, 233, 234, 235, 0, 236, 0, 237, 0, - 0, 0, 240, 241, 538, 0, 244, 245, 246, 0, - 247, 248, 249, 250, 0, 251, 252, 253, 254, 255, - 2315, 257, 0, 259, 260, 261, 262, 0, 263, 264, - 265, 266, 267, 268, 269, 0, 270, 0, 272, 273, - 274, 275, 276, 277, 278, 279, 0, 280, 0, 281, - 0, 0, 284, 0, 286, 287, 288, 0, 289, 290, - 291, 0, 0, 292, 0, 294, 0, 0, 296, 297, - 298, 299, 300, 301, 302, 303, 539, 305, 306, 307, + 0, 122, 123, 124, 125, 126, 127, 128, 129, 0, + 130, 131, 132, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 134, 135, 0, 0, 137, 138, 0, 140, + 141, 142, 143, 144, 0, 146, 147, 0, 0, 148, + 149, 150, 151, 152, 0, 0, 153, 154, 155, 156, + 157, 158, 159, 0, 160, 161, 162, 163, 164, 0, + 0, 0, 166, 167, 168, 169, 170, 171, 0, 173, + 174, 175, 0, 176, 177, 178, 179, 180, 181, 0, + 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 0, 198, 0, 199, + 200, 201, 202, 203, 204, 0, 0, 205, 206, 207, + 208, 0, 0, 209, 210, 211, 212, 213, 0, 214, + 215, 216, 0, 217, 218, 219, 220, 0, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, + 0, 233, 0, 234, 235, 236, 237, 0, 238, 0, + 239, 0, 0, 0, 242, 243, 540, 0, 246, 247, + 248, 0, 249, 250, 251, 252, 0, 253, 254, 255, + 256, 257, 258, 259, 0, 261, 262, 263, 264, 0, + 265, 266, 267, 268, 269, 270, 271, 0, 272, 0, + 274, 275, 276, 277, 278, 279, 280, 281, 0, 282, + 0, 283, 0, 0, 286, 0, 288, 289, 290, 0, + 291, 292, 293, 0, 0, 294, 0, 296, 0, 0, + 298, 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - 0, 329, 330, 331, 332, 333, 0, 334, 335, 0, - 337, 0, 338, 339, 340, 341, 342, 343, 0, 344, - 345, 0, 0, 346, 347, 348, 0, 0, 349, 350, - 351, 0, 353, 0, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 0, 0, 0, - 0, 368, 369, 370, 0, 372, 373, 374, 375, 376, - 377, 0, 378, 379, 380, 381, 382, 383, 0, 384, - 385, 386, 387, 388, 389, 390, 391, 392, 393, 0, - 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 0, 407, 408, 0, 410, 411, 412, - 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 426, 0, 0, 427, 428, 429, 430, - 431, 432, 433, 434, 435, 0, 0, 437, 438, 439, - 440, 0, 441, 442, 443, 444, 445, 446, 447, 448, - 449, 450, 451, 452, 453, 454, 540, 456, 457, 0, - 0, 458, 459, 0, 460, 0, 462, 463, 464, 465, - 466, 467, 0, 468, 469, 470, 0, 0, 471, 472, - 473, 474, 475, 0, 476, 477, 478, 479, 480, 481, - 482, 483, 0, 0, 484, 485, 486, 0, 0, 487, - 488, 489, 490, 0, 491, 492, 493, 494, 495, 496, - 497, 498, 0, 499, 0, 501, 502, 503, 504, 505, - 506, 507, 0, 0, 508, 0, 0, 509, 510, 511, + 328, 329, 0, 331, 332, 333, 334, 335, 0, 336, + 337, 0, 339, 0, 340, 341, 342, 343, 344, 345, + 0, 346, 347, 0, 0, 348, 349, 350, 0, 0, + 351, 352, 353, 0, 355, 0, 357, 358, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 0, + 0, 0, 0, 370, 371, 372, 0, 374, 375, 376, + 377, 378, 379, 0, 380, 381, 382, 383, 384, 385, + 0, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 0, 409, 410, 0, 412, + 413, 414, 415, 416, 417, 418, 419, 860, 421, 422, + 423, 424, 425, 426, 427, 428, 0, 0, 429, 430, + 431, 432, 433, 434, 435, 436, 437, 0, 0, 439, + 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 456, 542, 458, + 459, 0, 0, 460, 461, 0, 462, 0, 464, 465, + 466, 467, 468, 469, 0, 470, 862, 472, 0, 0, + 473, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 485, 0, 0, 486, 487, 488, 0, + 0, 489, 490, 491, 492, 0, 493, 494, 495, 496, + 497, 498, 499, 500, 0, 501, 0, 503, 504, 505, + 506, 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, - 522, 523, 524, 525, 526, 527, 528, 529, 537, 0, + 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, + 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 120, 121, - 122, 123, 124, 125, 126, 127, 0, 128, 129, 130, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, - 133, 0, 0, 135, 136, 0, 138, 139, 140, 141, - 142, 0, 144, 145, 0, 0, 146, 147, 148, 149, - 150, 0, 0, 151, 152, 153, 154, 155, 156, 157, - 0, 158, 159, 160, 161, 162, 0, 0, 0, 164, - 165, 166, 167, 168, 169, 0, 171, 172, 173, 0, - 174, 175, 176, 177, 178, 179, 0, 0, 181, 182, + 122, 123, 124, 125, 126, 127, 128, 129, 0, 130, + 131, 132, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 134, 135, 0, 0, 137, 138, 0, 140, 141, + 142, 143, 144, 0, 146, 147, 0, 0, 148, 149, + 150, 151, 152, 0, 0, 153, 154, 155, 156, 157, + 158, 159, 0, 160, 161, 162, 163, 164, 0, 0, + 0, 166, 167, 168, 169, 170, 171, 0, 173, 174, + 175, 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 0, 196, 0, 197, 198, 199, 200, - 201, 202, 0, 0, 203, 204, 205, 206, 0, 0, - 207, 208, 209, 210, 211, 0, 212, 213, 214, 0, - 215, 216, 217, 218, 0, 219, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, 0, 231, 0, - 232, 233, 234, 235, 0, 236, 0, 237, 0, 0, - 0, 240, 241, 538, 0, 244, 245, 246, 0, 247, - 248, 249, 250, 0, 251, 252, 253, 254, 255, 2335, - 257, 0, 259, 260, 261, 262, 0, 263, 264, 265, - 266, 267, 268, 269, 0, 270, 0, 272, 273, 274, - 275, 276, 277, 278, 279, 0, 280, 0, 281, 0, - 0, 284, 0, 286, 287, 288, 0, 289, 290, 291, - 0, 0, 292, 0, 294, 0, 0, 296, 297, 298, - 299, 300, 301, 302, 303, 539, 305, 306, 307, 308, + 193, 194, 195, 196, 197, 0, 198, 0, 199, 200, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 0, 0, 209, 210, 211, 212, 213, 0, 214, 215, + 216, 0, 217, 218, 219, 220, 0, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 0, + 233, 0, 234, 235, 236, 237, 0, 238, 0, 239, + 0, 0, 0, 242, 243, 540, 0, 246, 247, 248, + 0, 249, 250, 251, 252, 0, 253, 254, 255, 256, + 257, 1351, 259, 0, 261, 262, 263, 264, 0, 265, + 266, 267, 268, 269, 270, 271, 0, 272, 0, 274, + 275, 276, 277, 278, 279, 280, 281, 0, 282, 0, + 283, 0, 0, 286, 0, 288, 289, 290, 0, 291, + 292, 293, 0, 0, 294, 0, 296, 0, 0, 298, + 299, 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, 0, - 329, 330, 331, 332, 333, 0, 334, 335, 0, 337, - 0, 338, 339, 340, 341, 342, 343, 0, 344, 345, - 0, 0, 346, 347, 348, 0, 0, 349, 350, 351, - 0, 353, 0, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 0, 0, 0, 0, - 368, 369, 370, 0, 372, 373, 374, 375, 376, 377, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 0, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 0, 407, 408, 0, 410, 411, 412, 413, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 0, 331, 332, 333, 334, 335, 0, 336, 337, + 0, 339, 0, 340, 341, 342, 343, 344, 345, 0, + 346, 347, 0, 0, 348, 349, 350, 0, 0, 351, + 352, 353, 0, 355, 0, 357, 358, 359, 360, 361, + 362, 363, 364, 365, 366, 367, 368, 369, 0, 0, + 0, 0, 370, 371, 372, 0, 374, 375, 376, 377, + 378, 379, 0, 380, 381, 382, 383, 384, 385, 0, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 426, 0, 0, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 0, 0, 437, 438, 439, 440, - 0, 441, 442, 443, 444, 445, 446, 447, 448, 449, - 450, 451, 452, 453, 454, 540, 456, 457, 0, 0, - 458, 459, 0, 460, 0, 462, 463, 464, 465, 466, - 467, 0, 468, 469, 470, 0, 0, 471, 472, 473, - 474, 475, 0, 476, 477, 478, 479, 480, 481, 482, - 483, 0, 0, 484, 485, 486, 0, 0, 487, 488, - 489, 490, 0, 491, 492, 493, 494, 495, 496, 497, - 498, 0, 499, 0, 501, 502, 503, 504, 505, 506, - 507, 0, 0, 508, 0, 0, 509, 510, 511, 512, + 424, 425, 426, 427, 428, 0, 0, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 0, 0, 439, 440, + 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 542, 458, 459, + 0, 0, 460, 461, 0, 462, 0, 464, 465, 466, + 467, 468, 469, 0, 470, 471, 472, 0, 0, 473, + 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, + 483, 484, 485, 0, 0, 486, 487, 488, 0, 0, + 489, 490, 491, 492, 0, 493, 494, 495, 496, 497, + 498, 499, 500, 0, 501, 0, 503, 504, 505, 506, + 507, 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, - 523, 524, 525, 526, 527, 528, 529, 3367, 0, 0, + 523, 524, 525, 526, 527, 528, 529, 530, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 120, 121, 122, - 123, 124, 125, 126, 127, 0, 128, 129, 130, 0, - 0, 0, 3088, 0, 0, 0, 0, 3089, 132, 133, - 0, 3090, 135, 136, 3091, 138, 139, 140, 0, 1532, - 3092, 1534, 1535, 0, 3093, 146, 147, 148, 149, 150, - 0, 0, 151, 152, 153, 154, 1537, 1538, 157, 0, - 158, 159, 160, 161, 0, 0, 3094, 0, 3095, 165, - 166, 167, 168, 169, 3096, 171, 172, 173, 0, 174, - 175, 176, 177, 178, 179, 0, 3097, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 1543, 192, 193, - 1544, 195, 0, 196, 0, 197, 198, 199, 200, 201, - 202, 0, 0, 203, 204, 205, 206, 0, 0, 207, - 208, 1088, 210, 211, 0, 212, 213, 214, 0, 215, - 216, 217, 218, 0, 219, 220, 221, 222, 0, 224, - 225, 226, 227, 228, 229, 0, 0, 231, 0, 232, - 233, 1545, 235, 0, 236, 0, 237, 3098, 0, 3099, - 240, 241, 2460, 3100, 244, 245, 246, 0, 0, 0, - 249, 250, 0, 251, 252, 253, 254, 255, 256, 257, - 3101, 259, 260, 261, 262, 0, 263, 264, 265, 266, - 267, 268, 269, 0, 270, 3102, 0, 273, 274, 275, - 276, 277, 1551, 1552, 0, 1553, 0, 281, 3103, 3104, - 284, 3105, 286, 287, 288, 0, 289, 290, 291, 0, - 0, 292, 3106, 294, 3107, 0, 296, 297, 298, 299, - 300, 301, 302, 303, 2469, 305, 306, 307, 308, 309, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, + 123, 124, 125, 126, 127, 128, 129, 0, 130, 131, + 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 134, 135, 0, 0, 137, 138, 0, 140, 141, 142, + 143, 144, 0, 146, 147, 0, 0, 148, 149, 150, + 151, 152, 0, 0, 153, 154, 155, 156, 157, 158, + 159, 0, 160, 161, 162, 163, 164, 0, 0, 0, + 166, 167, 168, 169, 170, 171, 0, 173, 174, 175, + 0, 176, 177, 178, 179, 180, 181, 0, 0, 183, + 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, 0, 198, 0, 199, 200, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 0, + 0, 209, 210, 211, 212, 213, 0, 214, 215, 216, + 0, 217, 218, 219, 220, 0, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 0, 233, + 0, 234, 235, 236, 237, 0, 238, 0, 239, 0, + 0, 0, 242, 243, 540, 0, 246, 247, 248, 0, + 249, 250, 251, 252, 0, 253, 254, 255, 256, 257, + 1374, 259, 0, 261, 262, 263, 264, 0, 265, 266, + 267, 268, 269, 270, 271, 0, 272, 0, 274, 275, + 276, 277, 278, 279, 280, 281, 0, 282, 0, 283, + 0, 0, 286, 0, 288, 289, 290, 0, 291, 292, + 293, 0, 0, 294, 0, 296, 0, 0, 298, 299, + 300, 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 1560, 3109, 1562, - 330, 331, 332, 0, 0, 334, 335, 3111, 337, 0, - 0, 339, 1564, 341, 342, 343, 0, 344, 345, 0, - 0, 346, 347, 348, 0, 0, 349, 350, 0, 3113, - 353, 3114, 0, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 0, 0, 0, 0, 368, - 369, 0, 3115, 372, 373, 0, 375, 376, 377, 0, - 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, - 387, 388, 1568, 390, 391, 392, 393, 0, 394, 395, + 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, + 0, 331, 332, 333, 334, 335, 0, 336, 337, 0, + 339, 0, 340, 341, 342, 343, 344, 345, 0, 346, + 347, 0, 0, 348, 349, 350, 0, 0, 351, 352, + 353, 0, 355, 0, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 0, 0, 0, + 0, 370, 371, 372, 0, 374, 375, 376, 377, 378, + 379, 0, 380, 381, 382, 383, 384, 385, 0, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 0, 407, 408, 3116, 410, 411, 412, 0, 414, + 406, 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 0, 3117, 427, 428, 429, 430, 431, 432, - 0, 434, 435, 0, 3119, 437, 438, 1574, 440, 0, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 2477, 456, 0, 0, 0, 458, - 459, 0, 460, 3121, 462, 463, 464, 465, 466, 467, - 0, 468, 1577, 1578, 0, 0, 471, 472, 0, 474, - 0, 0, 476, 477, 3122, 479, 480, 481, 482, 483, - 0, 0, 484, 485, 486, 3124, 0, 487, 488, 489, - 490, 0, 491, 492, 493, 494, 495, 0, 1582, 498, - 0, 499, 3125, 501, 502, 503, 504, 505, 506, 507, - 0, 0, 508, 0, 0, 509, 510, 511, 512, 513, - 514, 1835, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 526, 527, 528, 529, 0, 0, 0, 0, - 0, 120, 121, 122, 123, 124, 125, 126, 127, 0, - 128, 129, 130, 0, 0, 0, 1528, 0, 0, 0, - 0, 1529, 132, 133, 0, 1530, 135, 136, 1531, 138, - 139, 140, 0, 1532, 1533, 1534, 1535, 0, 1536, 146, - 147, 148, 149, 150, 0, 0, 151, 152, 153, 154, - 1537, 1538, 157, 0, 158, 159, 160, 161, 0, 0, - 1539, 0, 1540, 165, 166, 167, 168, 169, 1541, 171, - 172, 173, 0, 174, 175, 176, 177, 178, 179, 0, - 1542, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 1543, 192, 193, 1544, 195, 0, 196, 0, 197, - 198, 199, 200, 201, 202, 0, 0, 203, 204, 205, - 206, 0, 0, 207, 208, 1088, 210, 211, 0, 212, - 213, 214, 0, 215, 216, 217, 218, 0, 219, 220, - 221, 222, 0, 224, 225, 226, 227, 228, 229, 0, - 0, 231, 0, 232, 233, 1545, 235, 0, 236, 0, - 237, 1546, 0, 1547, 240, 241, 0, 1548, 244, 245, - 246, 0, 0, 0, 249, 250, 0, 251, 252, 253, - 254, 255, 256, 257, 1549, 259, 260, 261, 262, 0, - 263, 264, 265, 266, 267, 268, 269, 0, 270, 1550, - 0, 273, 274, 275, 276, 277, 1551, 1552, 0, 1553, - 0, 281, 1554, 1555, 284, 1556, 286, 287, 288, 0, - 289, 290, 291, 0, 0, 292, 1557, 294, 1558, 0, - 296, 297, 298, 299, 300, 301, 302, 303, 0, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 1560, 1561, 1562, 330, 331, 332, 0, 0, 334, - 335, 1563, 337, 0, 0, 339, 1564, 341, 342, 343, - 0, 344, 345, 0, 0, 346, 347, 348, 0, 0, - 349, 350, 0, 1565, 353, 1566, 0, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 0, - 0, 0, 0, 368, 369, 0, 1567, 372, 373, 0, - 375, 376, 377, 0, 378, 379, 380, 381, 382, 383, - 0, 384, 385, 386, 387, 388, 1568, 390, 391, 392, - 393, 0, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 0, 407, 408, 1569, 410, - 411, 412, 0, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 0, 1571, 427, 428, - 429, 430, 431, 432, 0, 434, 435, 0, 1573, 437, - 438, 1574, 440, 0, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 0, 456, - 0, 0, 0, 458, 459, 0, 460, 1576, 462, 463, - 464, 465, 466, 467, 0, 468, 1577, 1578, 0, 0, - 471, 472, 0, 474, 0, 0, 476, 477, 1579, 479, - 480, 481, 482, 483, 0, 0, 484, 485, 486, 1581, - 0, 487, 488, 489, 490, 0, 491, 492, 493, 494, - 495, 0, 1582, 498, 0, 499, 1583, 501, 502, 503, - 504, 505, 506, 507, 0, 0, 508, 0, 0, 509, - 510, 511, 512, 513, 514, 537, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 526, 527, 528, 529, - 0, 0, 0, 0, 0, 120, 121, 122, 123, 124, - 125, 126, 127, 0, 128, 129, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 132, 133, 0, 0, - 135, 136, 0, 138, 139, 140, 141, 142, 0, 144, - 145, 0, 0, 146, 147, 148, 149, 150, 0, 0, - 151, 152, 153, 154, 155, 156, 157, 0, 158, 159, - 160, 161, 162, 0, 0, 0, 164, 165, 166, 167, - 168, 169, 0, 171, 172, 173, 0, 174, 175, 176, - 177, 178, 179, 0, 0, 181, 182, 183, 184, 185, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 0, 196, 0, 197, 198, 199, 200, 201, 202, 0, - 0, 203, 204, 205, 206, 0, 0, 207, 208, 209, - 210, 211, 0, 212, 213, 214, 0, 215, 216, 217, - 218, 0, 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 0, 231, 0, 232, 233, 234, - 235, 0, 236, 0, 237, 0, 0, 0, 240, 241, - 538, 0, 244, 245, 246, 0, 247, 248, 0, 250, - 0, 251, 252, 253, 254, 255, 256, 257, 0, 259, - 260, 261, 262, 0, 263, 264, 265, 266, 267, 268, - 269, 0, 270, 0, 272, 273, 274, 275, 276, 277, - 278, 279, 0, 280, 0, 281, 0, 0, 284, 0, - 286, 287, 288, 0, 289, 290, 291, 0, 0, 292, - 0, 294, 0, 0, 296, 297, 298, 299, 300, 301, - 302, 303, 539, 305, 306, 307, 308, 309, 310, 311, + 425, 426, 427, 428, 0, 0, 429, 430, 431, 432, + 433, 434, 435, 436, 437, 0, 0, 439, 440, 441, + 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, + 451, 452, 453, 454, 455, 456, 542, 458, 459, 0, + 0, 460, 461, 0, 462, 0, 464, 465, 466, 467, + 468, 469, 0, 470, 471, 472, 0, 0, 473, 474, + 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, + 484, 485, 0, 0, 486, 487, 488, 0, 0, 489, + 490, 491, 492, 0, 493, 494, 495, 496, 497, 498, + 499, 500, 0, 501, 0, 503, 504, 505, 506, 507, + 508, 509, 0, 0, 510, 0, 0, 511, 512, 513, + 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, + 524, 525, 526, 527, 528, 529, 530, 531, 539, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 0, 0, 0, 1735, 0, 0, 134, + 135, 0, 0, 137, 138, 0, 140, 141, 142, 143, + 144, 0, 146, 147, 0, 0, 148, 149, 150, 151, + 152, 0, 0, 153, 154, 155, 156, 157, 158, 159, + 0, 160, 161, 162, 163, 164, 0, 0, 0, 166, + 167, 168, 169, 170, 171, 0, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 0, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 211, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, 231, 232, 0, 233, 0, + 234, 235, 236, 237, 0, 238, 0, 239, 0, 0, + 0, 242, 243, 540, 0, 246, 247, 248, 0, 249, + 250, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 0, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 0, 274, 275, 276, + 277, 278, 279, 280, 281, 0, 282, 0, 283, 0, + 0, 286, 0, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 0, 296, 0, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 541, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + 321, 322, 323, 324, 325, 326, 327, 328, 329, 0, + 331, 332, 333, 334, 335, 0, 336, 337, 0, 339, + 0, 340, 341, 342, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 353, + 0, 355, 0, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 372, 0, 374, 375, 376, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 0, 396, + 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 0, 409, 410, 0, 412, 413, 414, 415, + 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, + 426, 427, 428, 0, 0, 429, 430, 431, 432, 433, + 0, 435, 436, 437, 0, 0, 439, 440, 441, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 542, 458, 459, 0, 0, + 460, 461, 0, 462, 0, 464, 465, 466, 467, 468, + 469, 0, 470, 471, 472, 0, 0, 473, 474, 475, + 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 0, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 498, 499, + 500, 0, 501, 0, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, + 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, + 525, 526, 527, 528, 529, 530, 531, 539, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 122, 123, 124, + 125, 126, 127, 128, 129, 0, 130, 131, 132, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 134, 135, + 0, 0, 137, 138, 0, 140, 141, 142, 143, 144, + 0, 146, 147, 0, 0, 148, 149, 150, 151, 152, + 0, 0, 153, 154, 155, 156, 157, 158, 159, 0, + 160, 161, 162, 163, 164, 0, 0, 0, 166, 167, + 168, 169, 170, 171, 0, 173, 174, 175, 0, 176, + 177, 178, 179, 180, 181, 0, 0, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, + 196, 197, 0, 198, 0, 199, 200, 201, 202, 203, + 204, 0, 0, 205, 206, 207, 208, 0, 0, 209, + 210, 211, 212, 213, 0, 214, 215, 216, 0, 217, + 218, 219, 220, 0, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, 231, 232, 0, 233, 0, 234, + 235, 236, 237, 0, 238, 0, 239, 0, 0, 0, + 242, 243, 540, 0, 246, 247, 248, 0, 249, 250, + 251, 252, 0, 253, 254, 255, 256, 257, 1937, 259, + 0, 261, 262, 263, 264, 0, 265, 266, 267, 268, + 269, 270, 271, 0, 272, 0, 274, 275, 276, 277, + 278, 279, 280, 281, 0, 282, 0, 283, 0, 0, + 286, 0, 288, 289, 290, 0, 291, 292, 293, 0, + 0, 294, 0, 296, 0, 0, 298, 299, 300, 301, + 302, 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 0, 329, 330, 331, - 332, 333, 0, 334, 335, 0, 337, 0, 338, 339, - 340, 341, 342, 343, 0, 344, 345, 0, 0, 346, - 347, 348, 0, 0, 349, 350, 351, 0, 353, 0, - 355, 356, 357, 358, 359, 360, 361, 0, 363, 364, - 365, 366, 367, 0, 0, 0, 0, 368, 369, 370, - 0, 372, 373, 374, 375, 376, 377, 0, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 0, 388, - 389, 390, 391, 392, 393, 0, 394, 395, 396, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 0, - 407, 408, 0, 410, 411, 412, 413, 0, 415, 416, + 322, 323, 324, 325, 326, 327, 328, 329, 0, 331, + 332, 333, 334, 335, 0, 336, 337, 0, 339, 0, + 340, 341, 342, 343, 344, 345, 0, 346, 347, 0, + 0, 348, 349, 350, 0, 0, 351, 352, 353, 0, + 355, 0, 357, 358, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 0, 0, 0, 0, 370, + 371, 372, 0, 374, 375, 376, 377, 378, 379, 0, + 380, 381, 382, 383, 384, 385, 0, 386, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 0, 396, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 0, 0, 427, 428, 429, 430, 431, 432, 433, 434, - 435, 0, 0, 437, 438, 439, 440, 0, 441, 442, + 427, 428, 0, 0, 429, 430, 431, 432, 433, 434, + 435, 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, - 453, 454, 540, 456, 457, 0, 0, 458, 459, 0, - 460, 0, 462, 463, 464, 465, 466, 467, 0, 468, - 469, 470, 0, 0, 471, 472, 473, 474, 475, 0, - 476, 477, 478, 479, 480, 481, 482, 483, 0, 0, - 484, 485, 486, 0, 0, 487, 488, 489, 490, 0, - 491, 492, 493, 494, 495, 496, 497, 498, 0, 499, - 0, 501, 502, 503, 504, 505, 506, 507, 0, 0, - 508, 0, 0, 509, 510, 511, 512, 513, 514, 515, + 453, 454, 455, 456, 542, 458, 459, 0, 0, 460, + 461, 0, 462, 0, 464, 465, 466, 467, 468, 469, + 0, 470, 471, 472, 0, 0, 473, 474, 475, 476, + 477, 0, 478, 479, 480, 481, 482, 483, 484, 485, + 0, 0, 486, 487, 488, 0, 0, 489, 490, 491, + 492, 0, 493, 494, 495, 496, 497, 498, 499, 500, + 0, 501, 0, 503, 504, 505, 506, 507, 508, 509, + 0, 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, - 526, 527, 528, 529, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2, 0, 3, 4, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 2, 0, 6, 0, 0, 0, 0, 0, - 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, - 0, 0, 6, 0, 0, 0, 0, 8, 0, 0, - 0, 7, 0, 0, 0, 0, 0, 0, 10, 0, - 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, - 11, 0, 762, 0, 0, 0, 10, 0, 0, 0, - 0, 0, 0, 13, 0, 0, 0, 0, 11, 0, - 762, 0, 0, 0, 0, 0, 0, 0, 14, 15, - 0, 13, 0, 0, 0, 0, 0, 0, 0, 763, - 0, 0, 0, 0, 0, 18, 14, 15, 0, 0, - 0, 0, 0, 0, 19, 0, 0, 763, 0, 0, - 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, - 0, 22, 19, 0, 0, 23, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, - 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, + 526, 527, 528, 529, 530, 531, 539, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 122, 123, 124, 125, + 126, 127, 128, 129, 0, 130, 131, 132, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 134, 135, 0, + 0, 137, 138, 0, 140, 141, 142, 143, 144, 0, + 146, 147, 0, 0, 148, 149, 150, 151, 152, 0, + 0, 153, 154, 155, 156, 157, 158, 159, 0, 160, + 161, 162, 163, 164, 0, 0, 0, 166, 167, 168, + 169, 170, 171, 0, 173, 174, 175, 0, 176, 177, + 178, 179, 180, 181, 0, 0, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 0, 198, 0, 199, 200, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 0, 0, 209, 210, + 211, 212, 213, 0, 214, 215, 216, 0, 217, 218, + 219, 220, 0, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 0, 233, 0, 234, 235, + 236, 237, 0, 238, 0, 239, 0, 0, 0, 242, + 243, 540, 0, 246, 247, 248, 0, 249, 250, 251, + 252, 0, 253, 254, 255, 256, 257, 2327, 259, 0, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 0, 272, 0, 274, 275, 276, 277, 278, + 279, 280, 281, 0, 282, 0, 283, 0, 0, 286, + 0, 288, 289, 290, 0, 291, 292, 293, 0, 0, + 294, 0, 296, 0, 0, 298, 299, 300, 301, 302, + 303, 304, 305, 541, 307, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, + 323, 324, 325, 326, 327, 328, 329, 0, 331, 332, + 333, 334, 335, 0, 336, 337, 0, 339, 0, 340, + 341, 342, 343, 344, 345, 0, 346, 347, 0, 0, + 348, 349, 350, 0, 0, 351, 352, 353, 0, 355, + 0, 357, 358, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 0, 0, 0, 370, 371, + 372, 0, 374, 375, 376, 377, 378, 379, 0, 380, + 381, 382, 383, 384, 385, 0, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 0, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 0, 409, 410, 0, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 0, 0, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 0, 0, 439, 440, 441, 442, 0, 443, + 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 542, 458, 459, 0, 0, 460, 461, + 0, 462, 0, 464, 465, 466, 467, 468, 469, 0, + 470, 471, 472, 0, 0, 473, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 485, 0, + 0, 486, 487, 488, 0, 0, 489, 490, 491, 492, + 0, 493, 494, 495, 496, 497, 498, 499, 500, 0, + 501, 0, 503, 504, 505, 506, 507, 508, 509, 0, + 0, 510, 0, 0, 511, 512, 513, 514, 515, 516, + 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, + 527, 528, 529, 530, 531, 539, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 122, 123, 124, 125, 126, + 127, 128, 129, 0, 130, 131, 132, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 134, 135, 0, 0, + 137, 138, 0, 140, 141, 142, 143, 144, 0, 146, + 147, 0, 0, 148, 149, 150, 151, 152, 0, 0, + 153, 154, 155, 156, 157, 158, 159, 0, 160, 161, + 162, 163, 164, 0, 0, 0, 166, 167, 168, 169, + 170, 171, 0, 173, 174, 175, 0, 176, 177, 178, + 179, 180, 181, 0, 0, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 0, 198, 0, 199, 200, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 0, 0, 209, 210, 211, + 212, 213, 0, 214, 215, 216, 0, 217, 218, 219, + 220, 0, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 0, 233, 0, 234, 235, 236, + 237, 0, 238, 0, 239, 0, 0, 0, 242, 243, + 540, 0, 246, 247, 248, 0, 249, 250, 251, 252, + 0, 253, 254, 255, 256, 257, 2347, 259, 0, 261, + 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, + 271, 0, 272, 0, 274, 275, 276, 277, 278, 279, + 280, 281, 0, 282, 0, 283, 0, 0, 286, 0, + 288, 289, 290, 0, 291, 292, 293, 0, 0, 294, + 0, 296, 0, 0, 298, 299, 300, 301, 302, 303, + 304, 305, 541, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, 328, 329, 0, 331, 332, 333, + 334, 335, 0, 336, 337, 0, 339, 0, 340, 341, + 342, 343, 344, 345, 0, 346, 347, 0, 0, 348, + 349, 350, 0, 0, 351, 352, 353, 0, 355, 0, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + 367, 368, 369, 0, 0, 0, 0, 370, 371, 372, + 0, 374, 375, 376, 377, 378, 379, 0, 380, 381, + 382, 383, 384, 385, 0, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 0, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 0, + 409, 410, 0, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 0, 0, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 0, 0, 439, 440, 441, 442, 0, 443, 444, + 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 542, 458, 459, 0, 0, 460, 461, 0, + 462, 0, 464, 465, 466, 467, 468, 469, 0, 470, + 471, 472, 0, 0, 473, 474, 475, 476, 477, 0, + 478, 479, 480, 481, 482, 483, 484, 485, 0, 0, + 486, 487, 488, 0, 0, 489, 490, 491, 492, 0, + 493, 494, 495, 496, 497, 498, 499, 500, 0, 501, + 0, 503, 504, 505, 506, 507, 508, 509, 0, 0, + 510, 0, 0, 511, 512, 513, 514, 515, 516, 517, + 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, + 528, 529, 530, 531, 3381, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 122, 123, 124, 125, 126, 127, + 128, 129, 0, 130, 131, 132, 0, 0, 0, 3102, + 0, 0, 0, 0, 3103, 134, 135, 0, 3104, 137, + 138, 3105, 140, 141, 142, 0, 1539, 3106, 1541, 1542, + 0, 3107, 148, 149, 150, 151, 152, 0, 0, 153, + 154, 155, 156, 1544, 1545, 159, 0, 160, 161, 162, + 163, 0, 0, 3108, 0, 3109, 167, 168, 169, 170, + 171, 3110, 173, 174, 175, 0, 176, 177, 178, 179, + 180, 181, 0, 3111, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 1550, 194, 195, 1551, 197, 0, + 198, 0, 199, 200, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 0, 0, 209, 210, 1093, 212, + 213, 0, 214, 215, 216, 0, 217, 218, 219, 220, + 0, 221, 222, 223, 224, 0, 226, 227, 228, 229, + 230, 231, 0, 0, 233, 0, 234, 235, 1552, 237, + 0, 238, 0, 239, 3112, 0, 3113, 242, 243, 2472, + 3114, 246, 247, 248, 0, 0, 0, 251, 252, 0, + 253, 254, 255, 256, 257, 258, 259, 3115, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 0, 272, 3116, 0, 275, 276, 277, 278, 279, 1558, + 1559, 0, 1560, 0, 283, 3117, 3118, 286, 3119, 288, + 289, 290, 0, 291, 292, 293, 0, 0, 294, 3120, + 296, 3121, 0, 298, 299, 300, 301, 302, 303, 304, + 305, 2481, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, 328, 1567, 3123, 1569, 332, 333, 334, + 0, 0, 336, 337, 3125, 339, 0, 0, 341, 1571, + 343, 344, 345, 0, 346, 347, 0, 0, 348, 349, + 350, 0, 0, 351, 352, 0, 3127, 355, 3128, 0, + 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, + 368, 369, 0, 0, 0, 0, 370, 371, 0, 3129, + 374, 375, 0, 377, 378, 379, 0, 380, 381, 382, + 383, 384, 385, 0, 386, 387, 388, 389, 390, 1575, + 392, 393, 394, 395, 0, 396, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 0, 409, + 410, 3130, 412, 413, 414, 0, 416, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 426, 427, 428, 0, + 3131, 429, 430, 431, 432, 433, 434, 0, 436, 437, + 0, 3133, 439, 440, 1581, 442, 0, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 2489, 458, 0, 0, 0, 460, 461, 0, 462, + 3135, 464, 465, 466, 467, 468, 469, 0, 470, 1584, + 1585, 0, 0, 473, 474, 0, 476, 0, 0, 478, + 479, 3136, 481, 482, 483, 484, 485, 0, 0, 486, + 487, 488, 3138, 0, 489, 490, 491, 492, 0, 493, + 494, 495, 496, 497, 0, 1589, 500, 0, 501, 3139, + 503, 504, 505, 506, 507, 508, 509, 0, 0, 510, + 0, 0, 511, 512, 513, 514, 515, 516, 1844, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, + 529, 530, 531, 0, 0, 0, 0, 0, 122, 123, + 124, 125, 126, 127, 128, 129, 0, 130, 131, 132, + 0, 0, 0, 1535, 0, 0, 0, 0, 1536, 134, + 135, 0, 1537, 137, 138, 1538, 140, 141, 142, 0, + 1539, 1540, 1541, 1542, 0, 1543, 148, 149, 150, 151, + 152, 0, 0, 153, 154, 155, 156, 1544, 1545, 159, + 0, 160, 161, 162, 163, 0, 0, 1546, 0, 1547, + 167, 168, 169, 170, 171, 1548, 173, 174, 175, 0, + 176, 177, 178, 179, 180, 181, 0, 1549, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 1550, 194, + 195, 1551, 197, 0, 198, 0, 199, 200, 201, 202, + 203, 204, 0, 0, 205, 206, 207, 208, 0, 0, + 209, 210, 1093, 212, 213, 0, 214, 215, 216, 0, + 217, 218, 219, 220, 0, 221, 222, 223, 224, 0, + 226, 227, 228, 229, 230, 231, 0, 0, 233, 0, + 234, 235, 1552, 237, 0, 238, 0, 239, 1553, 0, + 1554, 242, 243, 0, 1555, 246, 247, 248, 0, 0, + 0, 251, 252, 0, 253, 254, 255, 256, 257, 258, + 259, 1556, 261, 262, 263, 264, 0, 265, 266, 267, + 268, 269, 270, 271, 0, 272, 1557, 0, 275, 276, + 277, 278, 279, 1558, 1559, 0, 1560, 0, 283, 1561, + 1562, 286, 1563, 288, 289, 290, 0, 291, 292, 293, + 0, 0, 294, 1564, 296, 1565, 0, 298, 299, 300, + 301, 302, 303, 304, 305, 0, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + 321, 322, 323, 324, 325, 326, 327, 328, 1567, 1568, + 1569, 332, 333, 334, 0, 0, 336, 337, 1570, 339, + 0, 0, 341, 1571, 343, 344, 345, 0, 346, 347, + 0, 0, 348, 349, 350, 0, 0, 351, 352, 0, + 1572, 355, 1573, 0, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 0, 0, 0, 0, + 370, 371, 0, 1574, 374, 375, 0, 377, 378, 379, + 0, 380, 381, 382, 383, 384, 385, 0, 386, 387, + 388, 389, 390, 1575, 392, 393, 394, 395, 0, 396, + 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 0, 409, 410, 1576, 412, 413, 414, 0, + 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, + 426, 427, 428, 0, 1578, 429, 430, 431, 432, 433, + 434, 0, 436, 437, 0, 1580, 439, 440, 1581, 442, + 0, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 0, 458, 0, 0, 0, + 460, 461, 0, 462, 1583, 464, 465, 466, 467, 468, + 469, 0, 470, 1584, 1585, 0, 0, 473, 474, 0, + 476, 0, 0, 478, 479, 1586, 481, 482, 483, 484, + 485, 0, 0, 486, 487, 488, 1588, 0, 489, 490, + 491, 492, 0, 493, 494, 495, 496, 497, 0, 1589, + 500, 0, 501, 1590, 503, 504, 505, 506, 507, 508, + 509, 0, 0, 510, 0, 0, 511, 512, 513, 514, + 515, 516, 539, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 528, 529, 530, 531, 0, 0, 0, + 0, 0, 122, 123, 124, 125, 126, 127, 128, 129, + 0, 130, 131, 132, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 134, 135, 0, 0, 137, 138, 0, + 140, 141, 142, 143, 144, 0, 146, 147, 0, 0, + 148, 149, 150, 151, 152, 0, 0, 153, 154, 155, + 156, 157, 158, 159, 0, 160, 161, 162, 163, 164, + 0, 0, 0, 166, 167, 168, 169, 170, 171, 0, + 173, 174, 175, 0, 176, 177, 178, 179, 180, 181, + 0, 0, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, 0, 198, 0, + 199, 200, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 0, 0, 209, 210, 211, 212, 213, 0, + 214, 215, 216, 0, 217, 218, 219, 220, 0, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 0, 233, 0, 234, 235, 236, 237, 0, 238, + 0, 239, 0, 0, 0, 242, 243, 540, 0, 246, + 247, 248, 0, 249, 250, 0, 252, 0, 253, 254, + 255, 256, 257, 258, 259, 0, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 0, 272, + 0, 274, 275, 276, 277, 278, 279, 280, 281, 0, + 282, 0, 283, 0, 0, 286, 0, 288, 289, 290, + 0, 291, 292, 293, 0, 0, 294, 0, 296, 0, + 0, 298, 299, 300, 301, 302, 303, 304, 305, 541, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 327, 328, 329, 0, 331, 332, 333, 334, 335, 0, + 336, 337, 0, 339, 0, 340, 341, 342, 343, 344, + 345, 0, 346, 347, 0, 0, 348, 349, 350, 0, + 0, 351, 352, 353, 0, 355, 0, 357, 358, 359, + 360, 361, 362, 363, 0, 365, 366, 367, 368, 369, + 0, 0, 0, 0, 370, 371, 372, 0, 374, 375, + 376, 377, 378, 379, 0, 380, 381, 382, 383, 384, + 385, 0, 386, 387, 388, 0, 390, 391, 392, 393, + 394, 395, 0, 396, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 0, 409, 410, 0, + 412, 413, 414, 415, 0, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 0, 0, 429, + 430, 431, 432, 433, 434, 435, 436, 437, 0, 0, + 439, 440, 441, 442, 0, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 542, + 458, 459, 0, 0, 460, 461, 0, 462, 0, 464, + 465, 466, 467, 468, 469, 0, 470, 471, 472, 0, + 0, 473, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 485, 0, 0, 486, 487, 488, + 0, 0, 489, 490, 491, 492, 0, 493, 494, 495, + 496, 497, 498, 499, 500, 0, 501, 0, 503, 504, + 505, 506, 507, 508, 509, 0, 0, 510, 0, 0, + 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, + 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, + 531, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 764, 0, 3, 4, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, + 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, + 0, 0, 0, 0, 8, 0, 0, 0, 7, 0, + 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, + 0, 0, 8, 0, 0, 0, 0, 11, 0, 765, + 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 11, 0, 765, 0, 0, + 0, 0, 0, 0, 0, 14, 15, 0, 13, 0, + 0, 0, 0, 0, 0, 0, 766, 0, 0, 0, + 0, 0, 18, 14, 15, 0, 0, 0, 0, 0, + 0, 19, 0, 0, 766, 0, 0, 0, 0, 0, + 18, 0, 0, 0, 0, 0, 0, 0, 22, 19, + 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, + 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, -1521, 0, 0, + 0, 0, -1524, 0, 0, 0, 0, 0, 0, 0, + 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -1521, 0, 0, 0, 0, - 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, + 27, 28, 0, 0, 0, 0, 0, 29, 0, 0, + 30, 0, 0, 0, 0, 0, 0, 26, 27, 28, + 0, 0, 0, 0, 0, 29, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, + 32, 0, 0, 0, 0, 0, 0, 0, 0, 31, + 0, 0, 0, 0, 0, 0, 33, 0, 32, 0, + 0, 0, 0, 34, 0, 0, 0, 35, 0, 0, + 0, 0, 0, 0, 33, 0, 0, 0, 0, 36, + 0, 34, 0, 0, 0, 35, 0, 0, 0, 0, + 0, 37, 0, 0, 0, 38, 0, 36, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, + 0, 0, 0, 38, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 26, 27, 28, 0, 0, 0, 0, 0, - 29, 0, 0, 30, 0, 0, 0, 0, 0, 0, - 26, 27, 28, 0, 0, 0, 0, 0, 29, 0, - 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, - 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, - 0, 0, 31, 0, 0, 0, 0, 0, 0, 33, - 0, 32, 0, 0, 0, 0, 34, 0, 0, 0, - 35, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 0, 36, 0, 34, 0, 0, 0, 35, 0, - 0, 0, 0, 0, 37, 0, 0, 0, 38, 0, - 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 37, 0, 0, 0, 38, 0, 0, 39, + 40, 0, 0, 0, 39, 0, 42, 0, 0, 0, + 0, 43, 0, 0, 0, 0, 767, 0, 40, 0, + 0, 0, 0, 0, 42, 0, 0, 0, 44, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 40, 0, 0, 0, 39, 0, 42, - 0, 0, 0, 0, 43, 0, 0, 0, 0, 764, - 0, 40, 0, 0, 0, 0, 0, 42, 0, 0, - 0, 44, 43, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, - 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, - 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 46 + 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, + 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 768, 0, 0, 0, + 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 46 }; static const yytype_int16 yycheck[] = { - 7, 0, 530, 0, 757, 0, 46, 0, 0, 16, - 0, 0, 1014, 0, 853, 917, 23, 0, 74, 0, - 16, 20, 7, 0, 827, 1483, 1261, 945, 20, 1192, - 1629, 1060, 1263, 877, 7, 1249, 929, 984, 23, 995, - 37, 1467, 950, 901, 20, 1713, 1005, 78, 1005, 1602, - 23, 78, 1005, 1868, 1953, 2331, 1005, 1876, 980, 758, - 1241, 1109, 2267, 1520, 1642, 1685, 1804, 2073, 1238, 76, - 77, 2257, 1321, 38, 23, 1370, 1752, 1666, 1667, 0, - 1819, 0, 13, 2820, 2236, 1228, 2238, 0, 19, 17, - 2817, 76, 77, 2776, 2776, 0, 1266, 0, 0, 30, - 0, 2074, 0, 76, 77, 102, 45, 0, 23, 2202, - 0, 2837, 1157, 44, 45, 0, 1118, 1162, 1857, 2322, - 2256, 2844, 764, 2641, 2046, 2047, 2485, 76, 77, 0, - 1023, 765, 1717, 34, 917, 2057, 919, 2728, 921, 2061, - 0, 2732, 822, 76, 77, 0, 2405, 112, 828, 0, - 0, 0, 0, 0, 0, 0, 0, 1082, 1083, 1003, - 1670, 76, 77, 1893, 0, 1863, 1890, 5, 5, 1685, - 5, 0, 1086, 81, 1889, 1100, 0, 1770, 3, 110, - 5, 0, 2418, 2956, 41, 10, 2036, 2037, 2038, 1103, - 5, 0, 1772, 9, 5, 5, 64, 5, 13, 14, - 5, 64, 4, 1006, 9, 5, 5, 9, 10, 9, - 5, 2418, 1168, 5, 13, 14, 5, 5, 13, 14, - 5, 60, 2425, 5, 13, 14, 5, 5, 13, 14, - 2411, 5, 5, 5, 11, 19, 5, 13, 14, 16, - 2413, 2380, 3183, 790, 2819, 13, 14, 3, 4, 5, - 46, 83, 175, 9, 4, 25, 1285, 75, 174, 9, - 923, 2008, 94, 46, 27, 140, 11, 1296, 3370, 56, - 47, 16, 19, 1117, 9, 896, 19, 101, 31, 118, - 19, 1174, 101, 125, 1202, 83, 39, 4, 105, 193, - 3167, 819, 9, 174, 1250, 3062, 94, 1253, 1254, 3016, - 1810, 11, 296, 3, 81, 15, 16, 174, 2511, 2512, - 3187, 2514, 38, 5, 60, 174, 120, 3164, 3134, 1482, - 229, 293, 133, 175, 11, 201, 896, 1007, 15, 16, - 2585, 295, 91, 64, 301, 35, 36, 1061, 866, 1019, - 3515, 229, 123, 817, 11, 296, 301, 117, 15, 16, - 2991, 162, 2993, 171, 109, 281, 998, 1520, 1572, 108, - 47, 995, 118, 120, 118, 2687, 183, 2689, 31, 278, - 388, 130, 186, 316, 1029, 138, 39, 397, 168, 1034, - 47, 1036, 31, 1227, 1223, 1224, 123, 174, 374, 195, - 278, 3546, 31, 3295, 81, 3297, 42, 284, 1097, 3625, - 1055, 31, 166, 11, 41, 65, 1916, 15, 16, 39, - 2225, 2563, 123, 164, 81, 75, 11, 118, 133, 121, - 3579, 107, 467, 253, 3467, 41, 3469, 138, 121, 442, - 11, 271, 108, 517, 215, 3118, 523, 3612, 1356, 194, - 3615, 3178, 183, 530, 1168, 54, 530, 3727, 109, 175, - 280, 123, 47, 3180, 321, 322, 323, 3194, 76, 347, - 1970, 278, 314, 406, 872, 221, 467, 2743, 1326, 280, - 34, 295, 1982, 173, 285, 175, 295, 1385, 215, 290, - 493, 127, 123, 528, 133, 211, 81, 327, 244, 398, - 241, 427, 193, 134, 133, 197, 60, 3723, 366, 2744, - 81, 227, 2012, 366, 197, 146, 3786, 3550, 528, 2019, - 398, 2716, 238, 277, 532, 402, 195, 335, 3677, 256, - 295, 388, 3399, 3678, 440, 1249, 3428, 528, 285, 278, - 1578, 285, 346, 108, 528, 2628, 299, 2740, 464, 2742, - 330, 178, 528, 367, 495, 300, 3721, 3394, 367, 401, - 3652, 548, 3368, 2063, 313, 464, 528, 2067, 364, 1033, - 427, 375, 178, 3330, 528, 532, 2385, 2386, 2387, 255, - 179, 2149, 2894, 3514, 215, 290, 464, 532, 3219, 280, - 266, 440, 3349, 3067, 244, 528, 2096, 196, 2208, 414, - 415, 3075, 201, 285, 3311, 462, 603, 1125, 479, 3316, - 363, 517, 365, 2202, 1225, 528, 473, 603, 440, 2066, - 1273, 486, 1537, 1538, 530, 3190, 1369, 2853, 526, 2864, - 437, 1374, 530, 334, 528, 467, 1875, 1380, 424, 1473, - 1544, 240, 395, 564, 473, 366, 447, 1562, 1185, 457, - 477, 424, 440, 1938, 370, 2826, 2853, 458, 1487, 395, - 1564, 2240, 405, 2792, 492, 1225, 532, 2912, 2831, 2581, - 2666, 528, 3388, 389, 532, 469, 490, 2266, 2871, 532, - 1509, 490, 3613, 530, 433, 2260, 514, 1376, 3451, 1910, - 2273, 2379, 361, 464, 492, 1599, 1715, 3370, 3370, 528, - 528, 528, 2208, 528, 2274, 531, 532, 2427, 2422, 1624, - 1625, 536, 531, 528, 2419, 529, 514, 531, 533, 534, - 529, 527, 531, 528, 2564, 2565, 2566, 2567, 528, 528, - 528, 536, 531, 528, 3083, 765, 467, 464, 528, 528, - 3321, 533, 534, 528, 526, 3326, 528, 3614, 530, 528, - 528, 528, 405, 528, 1386, 3263, 528, 524, 532, 528, - 528, 427, 528, 1387, 528, 528, 528, 3033, 765, 528, - 528, 1605, 1606, 519, 520, 764, 1659, 1660, 1661, 519, - 520, 1615, 764, 490, 2521, 405, 3513, 817, 478, 524, - 527, 3040, 2704, 526, 519, 520, 1630, 530, 527, 765, - 1198, 400, 464, 1201, 3520, 166, 109, 514, 3521, 806, - 1480, 64, 519, 520, 379, 13, 14, 490, 164, 528, - 817, 521, 522, 523, 524, 84, 1606, 824, 1662, 3572, - 3573, 806, 2071, 464, 280, 1615, 439, 426, 759, 285, - 0, 514, 817, 806, 521, 522, 523, 524, 869, 824, - 1495, 432, 869, 434, 817, 901, 853, 854, 1572, 150, - 365, 824, 519, 520, 521, 522, 523, 524, 2021, 824, - 1012, 319, 1517, 234, 532, 75, 479, 466, 817, 2688, - 877, 151, 164, 248, 3627, 931, 531, 169, 3014, 2885, - 395, 150, 3609, 1839, 817, 241, 3623, 133, 887, 345, - 887, 806, 887, 1849, 887, 887, 1852, 887, 887, 875, - 887, 202, 817, 2066, 887, 478, 887, 872, 118, 824, - 887, 519, 520, 521, 522, 523, 524, 375, 417, 2429, - 927, 928, 83, 203, 931, 932, 891, 3020, 216, 1682, - 6, 2226, 1834, 94, 10, 361, 292, 254, 27, 878, - 1978, 316, 18, 3670, 33, 2403, 1670, 878, 2921, 241, - 492, 2924, 205, 2926, 855, 856, 32, 858, 280, 860, - 36, 169, 3165, 280, 2970, 2585, 887, 517, 887, 3652, - 3652, 2009, 514, 1920, 887, 2485, 529, 984, 528, 532, - 2795, 2796, 887, 482, 887, 887, 2031, 887, 995, 887, - 1898, 60, 120, 1033, 887, 1002, 1003, 887, 1945, 998, - 292, 1008, 887, 2741, 1011, 1012, 998, 1014, 1015, 1016, - 1017, 1869, 1870, 1871, 1936, 278, 887, 149, 1940, 1802, - 3757, 1943, 950, 492, 1031, 180, 1033, 887, 2767, 2628, - 800, 406, 887, 2328, 280, 1042, 887, 887, 887, 887, - 887, 887, 887, 887, 290, 514, 1031, 2786, 1033, 138, - 38, 1834, 1059, 1060, 1061, 27, 3332, 1042, 1031, 528, - 1033, 33, 832, 4, 2653, 523, 407, 528, 9, 1042, - 280, 1854, 83, 27, 1081, 285, 1859, 806, 1118, 33, - 347, 209, 1031, 94, 1033, 460, 1810, 857, 220, 244, - 431, 2601, 1845, 1042, 1101, 824, 1038, 1850, 1031, 2242, - 1033, 2244, 1044, 366, 2562, 174, 1113, 1114, 1115, 1042, - 1117, 1118, 351, 1120, 3738, 351, 1031, 2543, 1033, 180, - 1648, 3307, 3746, 492, 1120, 335, 379, 1042, 2359, 432, - 4, 434, 395, 2296, 164, 9, 906, 390, 27, 169, - 191, 192, 2391, 353, 33, 514, 1153, 72, 73, 523, - 278, 390, 280, 528, 390, 5, 530, 855, 856, 528, - 858, 528, 860, 248, 1171, 1172, 138, 2982, 526, 301, - 528, 133, 530, 2017, 427, 1206, 1207, 2776, 1209, 1206, - 1207, 526, 1209, 244, 138, 530, 1961, 175, 427, 4, - 1965, 427, 1916, 1968, 9, 526, 109, 2367, 526, 530, - 162, 1208, 530, 38, 361, 1212, 1213, 3213, 43, 386, - 299, 241, 3488, 264, 265, 1222, 1223, 1224, 529, 386, - 1227, 532, 1043, 211, 379, 995, 1047, 480, 205, 529, - 2232, 316, 532, 1198, 529, 1005, 1201, 532, 257, 227, - 529, 480, 1249, 532, 480, 529, 1970, 457, 532, 138, - 238, 445, 321, 322, 323, 2436, 529, 467, 1982, 532, - 1191, 427, 292, 3162, 2108, 2446, 2776, 102, 8, 529, - 359, 11, 532, 480, 363, 15, 16, 432, 1285, 434, - 20, 21, 22, 8, 529, 383, 11, 532, 2012, 1296, - 15, 16, 2912, 386, 379, 2019, 3034, 264, 265, 1472, - 1265, 1474, 1475, 13, 14, 460, 395, 1459, 1460, 2819, - 13, 14, 133, 1465, 1321, 526, 528, 528, 280, 388, - 248, 406, 47, 285, 456, 13, 14, 299, 290, 54, - 490, 528, 492, 384, 385, 83, 13, 14, 3157, 2063, - 175, 162, 528, 2067, 2188, 299, 169, 529, 1343, 2073, - 532, 432, 528, 434, 1361, 2813, 81, 1127, 427, 528, - 1343, 109, 1343, 1370, 528, 1361, 1343, 1137, 13, 14, - 2948, 432, 2096, 434, 528, 460, 211, 13, 14, 2237, - 1387, 470, 370, 529, 13, 14, 532, 1386, 316, 1159, - 174, 363, 227, 462, 1386, 13, 14, 1404, 1168, 460, - 521, 389, 1409, 238, 473, 359, 13, 14, 1404, 363, - 299, 1387, 2334, 529, 2336, 528, 532, 384, 385, 528, - 2583, 3020, 147, 395, 2587, 840, 841, 842, 176, 529, - 845, 2888, 532, 529, 529, 19, 532, 532, 27, 274, - 528, 395, 529, 528, 33, 532, 1377, 3000, 1379, 13, - 14, 379, 1459, 1460, 179, 2154, 528, 205, 1465, 528, - 1467, 201, 2301, 528, 285, 1472, 1473, 1474, 1475, 290, - 359, 196, 529, 3749, 363, 532, 201, 432, 406, 434, - 1487, 1488, 1467, 514, 224, 447, 321, 2337, 529, 2339, - 1497, 532, 1499, 328, 1467, 1502, 458, 511, 470, 2417, - 1507, 249, 1509, 1510, 1274, 1512, 395, 529, 529, 1516, - 532, 532, 1497, 226, 1499, 240, 470, 1502, 2740, 3724, - 2742, 3726, 1507, 2758, 1497, 1510, 1499, 1512, 2759, 1502, - 3682, 1516, 460, 529, 1507, 370, 532, 1510, 529, 1512, - 893, 532, 895, 1516, 529, 3697, 530, 532, 1497, 138, - 1499, 295, 1467, 1502, 389, 511, 529, 297, 1507, 532, - 173, 1510, 532, 1512, 1497, 1572, 1499, 1516, 529, 1502, - 528, 532, 297, 3083, 1507, 2810, 2757, 1510, 2759, 1512, - 3785, 470, 1497, 1516, 1499, 2633, 2634, 1502, 174, 5, - 27, 529, 1507, 529, 532, 1510, 33, 1512, 1605, 1606, - 528, 1516, 529, 1602, 300, 1602, 529, 1614, 1615, 1602, - 1602, 3510, 1602, 1602, 529, 1602, 1623, 532, 529, 3771, - 60, 532, 529, 1630, 222, 532, 447, 462, 13, 14, - 1637, 379, 3784, 1403, 428, 529, 528, 458, 532, 2641, - 174, 529, 390, 511, 532, 3460, 3461, 13, 14, 2608, - 1657, 1658, 2609, 2610, 2611, 1662, 2609, 2610, 1665, 1715, - 2609, 13, 14, 1670, 1671, 1672, 1673, 1674, 1675, 1676, - 1677, 1678, 1679, 226, 3184, 400, 1683, 1684, 1685, 427, - 3190, 1688, 154, 1679, 529, 1692, 359, 532, 1695, 1696, - 1697, 1698, 1699, 1700, 1701, 1702, 1703, 13, 14, 1706, - 1685, 138, 154, 451, 529, 2429, 1713, 532, 1715, 154, - 299, 127, 128, 529, 109, 529, 532, 529, 532, 529, - 532, 154, 532, 154, 1755, 427, 1733, 8, 1755, 41, - 11, 529, 480, 529, 15, 16, 532, 13, 14, 20, - 21, 22, 13, 14, 2497, 528, 75, 3257, 13, 14, - 280, 1758, 81, 13, 14, 1731, 13, 14, 174, 13, - 14, 2485, 1769, 1770, 480, 94, 13, 14, 13, 14, - 359, 3370, 374, 375, 363, 515, 516, 517, 154, 519, - 520, 521, 522, 523, 524, 90, 27, 512, 494, 118, - 180, 120, 33, 154, 519, 520, 521, 522, 523, 524, - 13, 14, 530, 1810, 3472, 154, 395, 149, 13, 14, - 13, 14, 295, 1820, 2707, 205, 154, 1824, 13, 14, - 361, 176, 164, 440, 1820, 374, 375, 169, 1824, 374, - 375, 1983, 1984, 1985, 1986, 1987, 1988, 374, 375, 1991, - 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 528, - 3081, 268, 269, 528, 244, 384, 385, 528, 1865, 529, - 3370, 1868, 299, 13, 14, 528, 472, 473, 1875, 1865, - 429, 1878, 1879, 47, 1805, 3703, 3704, 2601, 220, 222, - 209, 470, 2487, 2488, 305, 1816, 227, 1818, 3692, 3693, - 1821, 2730, 248, 528, 249, 3733, 3734, 138, 528, 241, - 1831, 227, 1833, 227, 3118, 321, 322, 323, 2855, 1916, - 300, 2308, 2309, 1920, 302, 1846, 2824, 1914, 41, 239, - 1851, 5, 359, 528, 1855, 1856, 363, 1858, 5, 1860, - 1861, 1938, 2850, 1171, 1172, 528, 2780, 331, 1945, 1946, - 528, 528, 2666, 2801, 5, 2867, 528, 1954, 5, 528, - 292, 280, 176, 1723, 2983, 5, 285, 528, 395, 301, - 316, 5, 528, 1970, 9, 528, 528, 1974, 1975, 491, - 1977, 307, 388, 105, 532, 1982, 1983, 1984, 1985, 1986, - 1987, 1988, 118, 399, 1991, 1992, 1993, 1994, 1995, 1996, - 1997, 1998, 1999, 2000, 3420, 41, 351, 532, 529, 2006, - 2007, 467, 222, 2010, 3680, 2012, 335, 423, 3684, 248, - 2017, 427, 2019, 395, 169, 61, 297, 292, 169, 60, - 290, 31, 239, 379, 353, 249, 528, 1797, 1798, 440, - 94, 2183, 2184, 470, 2041, 390, 532, 440, 174, 60, - 60, 2048, 432, 2050, 434, 271, 462, 2054, 440, 248, - 406, 440, 2776, 3652, 528, 19, 2063, 473, 299, 224, - 2067, 107, 2069, 2048, 2071, 440, 2073, 457, 440, 2054, - 460, 490, 427, 386, 490, 2048, 86, 316, 154, 101, - 280, 2054, 3758, 201, 280, 95, 280, 528, 41, 2096, - 280, 41, 1862, 280, 280, 2819, 451, 13, 514, 2048, - 528, 2108, 2109, 154, 460, 2054, 174, 530, 2115, 119, - 1880, 1881, 528, 528, 456, 2048, 3072, 316, 359, 529, - 529, 2054, 363, 529, 529, 480, 174, 351, 457, 529, - 532, 529, 529, 2048, 487, 227, 529, 528, 467, 2054, - 379, 2148, 3652, 248, 227, 2152, 287, 287, 2188, 3384, - 2157, 2158, 532, 3579, 395, 528, 530, 528, 204, 477, - 528, 2885, 3084, 3085, 2917, 528, 390, 406, 528, 528, - 528, 40, 528, 309, 528, 3633, 2183, 2184, 486, 530, - 379, 2188, 9, 193, 526, 321, 322, 323, 530, 0, - 1960, 11, 438, 528, 438, 361, 206, 2204, 3624, 527, - 2207, 2208, 2209, 427, 19, 532, 537, 406, 438, 255, - 532, 316, 285, 528, 183, 440, 165, 174, 2225, 2226, - 266, 460, 532, 2208, 229, 2232, 3228, 451, 2235, 470, - 529, 41, 278, 467, 515, 516, 517, 220, 519, 520, - 521, 522, 523, 524, 2251, 467, 2970, 532, 2255, 398, - 2235, 3677, 388, 271, 229, 532, 480, 296, 2265, 319, - 319, 460, 2235, 532, 310, 183, 2251, 222, 149, 528, - 2255, 529, 280, 229, 379, 2282, 2283, 229, 2251, 530, - 3443, 1061, 2255, 164, 60, 60, 2235, 60, 169, 528, - 101, 427, 2299, 287, 2301, 60, 287, 301, 529, 480, - 3053, 406, 2251, 38, 340, 293, 2255, 2314, 43, 154, - 3, 154, 528, 528, 154, 180, 2247, 154, 154, 365, - 2235, 2328, 2255, 490, 154, 532, 462, 41, 532, 528, - 2337, 2338, 2339, 280, 295, 3, 2251, 473, 149, 220, - 2255, 2337, 2338, 2339, 390, 41, 174, 295, 60, 11, - 41, 169, 529, 164, 490, 460, 529, 529, 169, 3083, - 241, 529, 183, 174, 528, 411, 528, 102, 528, 169, - 3, 40, 183, 8, 3, 2382, 11, 188, 514, 244, - 15, 16, 149, 526, 2391, 20, 21, 22, 526, 537, - 529, 440, 528, 440, 3118, 529, 1012, 164, 440, 527, - 440, 532, 169, 413, 529, 529, 416, 150, 530, 220, - 529, 292, 47, 511, 511, 2410, 529, 2424, 529, 54, - 301, 530, 2429, 528, 3655, 359, 529, 2410, 2424, 2410, - 241, 511, 3350, 2410, 529, 174, 157, 528, 440, 528, - 175, 528, 528, 528, 127, 128, 81, 41, 183, 487, - 60, 532, 517, 220, 532, 320, 247, 513, 296, 60, - 3184, 296, 60, 464, 271, 479, 3190, 440, 280, 1249, - 528, 205, 154, 154, 241, 154, 211, 440, 2485, 41, - 440, 292, 2489, 529, 295, 528, 440, 41, 361, 3213, - 301, 174, 227, 528, 440, 529, 528, 528, 2505, 3027, - 529, 293, 532, 238, 490, 295, 41, 154, 2439, 2505, - 529, 528, 528, 285, 379, 174, 528, 528, 2670, 2671, - 529, 386, 529, 529, 60, 292, 14, 188, 169, 529, - 81, 342, 529, 3257, 301, 144, 2543, 526, 8, 274, - 532, 11, 529, 174, 179, 15, 16, 529, 529, 528, - 20, 21, 22, 529, 19, 306, 367, 528, 2543, 254, - 295, 196, 532, 529, 367, 296, 201, 432, 183, 434, - 2543, 529, 154, 532, 2344, 456, 178, 528, 2585, 451, - 41, 3474, 87, 528, 528, 3429, 321, 3431, 529, 529, - 1061, 529, 529, 328, 2601, 460, 41, 1213, 530, 528, - 528, 2641, 3441, 528, 41, 240, 1222, 41, 467, 532, - 174, 2618, 529, 528, 489, 529, 427, 201, 2625, 2626, - 529, 527, 527, 473, 532, 2681, 529, 529, 2543, 529, - 529, 532, 295, 2403, 2641, 370, 2676, 3639, 321, 322, - 323, 3444, 486, 3446, 60, 456, 3370, 2654, 2613, 517, - 2657, 529, 2659, 529, 389, 529, 467, 490, 529, 2666, - 2667, 207, 297, 2670, 2671, 118, 3584, 41, 2675, 2676, - 528, 229, 89, 194, 285, 2682, 487, 285, 489, 490, - 440, 530, 530, 530, 180, 440, 41, 280, 529, 456, - 2730, 3593, 2699, 517, 108, 427, 530, 490, 530, 530, - 295, 530, 527, 2710, 530, 388, 2703, 530, 530, 205, - 527, 530, 9, 3571, 530, 526, 399, 530, 529, 530, - 531, 530, 530, 2730, 530, 3564, 530, 462, 530, 530, - 2882, 201, 467, 530, 530, 530, 530, 2507, 360, 2509, - 423, 530, 530, 2513, 427, 2515, 530, 530, 244, 530, - 530, 530, 530, 530, 224, 342, 530, 60, 529, 526, - 530, 529, 529, 530, 528, 400, 2773, 529, 529, 2776, - 2777, 528, 2779, 2780, 2781, 528, 532, 2773, 1249, 462, - 528, 2777, 528, 2779, 194, 532, 201, 532, 2795, 2796, - 473, 529, 1572, 529, 527, 532, 472, 92, 353, 529, - 529, 2808, 2733, 528, 300, 2736, 532, 490, 2848, 529, - 41, 154, 2819, 530, 529, 125, 154, 41, 529, 375, - 2827, 2818, 2862, 375, 41, 529, 528, 297, 2823, 528, - 41, 514, 532, 467, 2765, 315, 528, 37, 528, 285, - 2823, 253, 2823, 1459, 1460, 528, 2823, 252, 2855, 1465, - 467, 2782, 2783, 2784, 2785, 2852, 2787, 2788, 2789, 2790, - 2791, 193, 451, 528, 9, 75, 298, 75, 3621, 2909, - 377, 529, 81, 529, 528, 2882, 529, 512, 2885, 527, - 515, 516, 517, 379, 519, 520, 521, 522, 523, 524, - 1670, 995, 60, 94, 134, 2902, 2903, 527, 2905, 436, - 60, 517, 2909, 3055, 295, 2912, 278, 451, 41, 528, - 298, 298, 472, 207, 528, 295, 529, 7, 8, 295, - 395, 281, 529, 13, 529, 529, 436, 123, 3652, 19, - 464, 374, 2939, 23, 150, 25, 432, 26, 434, 29, - 30, 31, 473, 521, 34, 473, 37, 37, 38, 3793, - 374, 41, 2959, 528, 44, 45, 303, 2964, 2965, 529, - 887, 457, 2969, 2970, 460, 2329, 2760, 2974, 1832, 3009, - 2977, 2978, 2773, 2332, 3476, 2982, 2983, 3624, 2708, 2986, - 3575, 3743, 3177, 2990, 3077, 1945, 76, 77, 2492, 3599, - 3716, 38, 3343, 3665, 864, 3002, 43, 3656, 3709, 1236, - 3401, 3000, 2320, 3000, 2338, 2990, 2776, 3000, 3000, 1810, - 3000, 3000, 102, 3000, 3654, 3543, 41, 2990, 1810, 109, - 110, 111, 112, 113, 3663, 2838, 2750, 2779, 3651, 1369, - 1810, 2317, 3650, 1137, 3748, 3315, 61, 2487, 1341, 2488, - 3047, 2990, 2813, 2813, 2409, 515, 516, 517, 3055, 519, - 520, 521, 522, 523, 524, 102, 2299, 2990, 2855, 2781, - 3212, 1191, 1030, 1030, 1168, 2265, 8, 3074, 1794, 11, - 1216, 1215, 2562, 15, 16, 2990, 3083, 3633, 2848, 1051, - 2282, 3552, 107, 108, 1758, 3433, 1793, 23, 2251, 1218, - 2912, 2060, 3252, 118, 1007, 2543, 8, 2542, 2593, 11, - 3364, 1572, 2076, 15, 16, 47, 3556, 8, 20, 21, - 22, 3118, 54, 1005, 15, 16, 1005, 1005, 1005, 20, - 21, 22, 1005, 3054, 3555, 37, 1005, 2159, 175, 2626, - 2266, 153, 2110, 2208, 3300, 2205, 1916, 3002, 1005, 81, - 1488, 1005, 1758, 1005, 2162, 2650, 1250, 3000, 3541, 1253, - 1254, 801, 174, 178, 2961, 1387, 995, 3164, 1881, 2069, - 2709, 1731, 906, 1732, 211, 2505, 1865, -1, 3164, -1, - 3177, -1, -1, -1, -1, -1, 153, 3184, -1, 204, - 227, 3177, -1, 3190, 3181, -1, -1, -1, 3228, -1, - 1970, 238, -1, 3200, 3201, -1, 3203, 174, -1, 1670, - -1, -1, 1982, -1, 1061, 3212, 3213, -1, -1, 174, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1061, - -1, 3228, -1, -1, 2994, -1, -1, 274, -1, -1, - 255, -1, 2012, -1, -1, -1, -1, 179, -1, 2019, - -1, 266, -1, -1, -1, -1, -1, -1, 295, -1, - 3257, -1, -1, 278, 196, 280, 3263, -1, -1, 201, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 3279, 3280, 321, -1, 3283, -1, 3285, -1, - -1, 328, -1, 2063, -1, 310, -1, 2067, 1392, 1393, - -1, -1, -1, 2073, -1, -1, -1, -1, 240, 321, - 322, 323, -1, -1, -1, -1, -1, -1, -1, -1, - 3317, -1, 224, -1, -1, -1, 2096, -1, -1, -1, - -1, -1, -1, 370, 8, -1, -1, 11, -1, -1, - -1, 15, 16, -1, 3341, -1, 20, 21, 22, 1810, - 365, -1, 389, -1, 321, 322, 323, -1, -1, -1, - -1, -1, -1, 37, -1, 297, 321, 322, 323, -1, - -1, -1, -1, 3370, -1, 390, 388, 1983, 1984, 1985, - 1986, 1987, 1988, -1, -1, 1991, 1992, 1993, 1994, 1995, - 1996, 1997, 1998, 1999, 2000, 297, 411, 3394, 413, -1, - -1, 416, 1249, 3390, -1, -1, 297, -1, 3394, -1, - -1, -1, -1, -1, -1, 427, -1, 1249, -1, -1, - -1, 388, -1, 3420, -1, 462, -1, -1, -1, -1, - 467, 3352, 3429, 388, 3431, -1, 3433, -1, -1, -1, - 3437, -1, 3439, -1, 3441, 3420, -1, -1, -1, -1, - 462, 174, 3373, 3374, -1, 1916, -1, 3420, 3455, -1, - 427, 473, -1, 3460, 3461, -1, -1, -1, 400, 549, - -1, -1, 427, -1, 554, 3472, -1, 3398, 490, -1, - -1, -1, -1, -1, 564, -1, 3473, -1, 3475, -1, - -1, -1, 3489, -1, -1, 462, -1, 3494, -1, -1, - 38, -1, 514, 3489, -1, 43, 473, 462, -1, 1970, - -1, -1, -1, 528, -1, 3420, 528, -1, 473, -1, - -1, 1982, -1, 490, -1, -1, 1620, -1, -1, -1, - -1, -1, 3519, -1, -1, 490, 3296, -1, -1, 1633, - -1, 1635, -1, -1, -1, 3542, -1, 514, -1, 3536, - 224, 2012, -1, -1, -1, 3552, 1012, -1, 2019, 514, - -1, 528, -1, -1, 102, -1, -1, 3564, -1, -1, - 1664, -1, -1, 528, -1, -1, -1, 2183, 2184, -1, - 512, -1, 3579, -1, 3581, -1, 309, 519, 520, 521, - 522, 523, 524, -1, -1, -1, -1, 1061, 321, 322, - 323, -1, 2063, 3600, 3579, -1, 2067, -1, -1, -1, - 3370, -1, 2073, 515, 516, 517, 3579, 519, 520, 521, - 522, 523, 524, 297, 515, 516, 517, 3624, 519, 520, - 521, 522, 523, 524, -1, 2096, -1, 175, -1, -1, - -1, -1, 3639, -1, -1, -1, -1, -1, -1, 3624, - 3410, -1, -1, -1, -1, 3652, -1, 3654, -1, 2429, - -1, 3624, -1, -1, -1, 388, -1, -1, 3654, -1, - -1, -1, -1, 211, 3579, 1769, 1770, 757, 758, 759, - 3677, -1, 3669, -1, -1, -1, -1, -1, -1, 227, - -1, -1, -1, 2299, 3691, 3692, 3693, -1, -1, -1, - 238, -1, 3677, -1, 427, 3702, -1, -1, 3695, -1, - -1, 41, -1, -1, 3677, 2485, -1, -1, -1, 3624, - 800, 801, -1, 3644, -1, 1572, 806, -1, 808, -1, - 3727, 61, -1, -1, -1, -1, 274, 817, -1, 462, - 1572, 821, 822, -1, 824, 1839, -1, 827, 828, -1, - 473, 3511, -1, -1, -1, 1849, -1, 295, 1852, -1, - 840, 841, 842, -1, -1, 845, 1222, 490, -1, -1, - -1, -1, 3677, 853, 854, 855, 856, 107, 858, -1, - 860, -1, -1, 321, -1, 1249, -1, -1, -1, 3786, - 328, 514, 872, -1, 8, -1, 3793, 11, 878, -1, - -1, 15, 16, -1, -1, 528, -1, 0, -1, -1, - -1, 891, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1670, -1, 905, 906, 20, -1, -1, - 23, 2601, 370, 47, 41, -1, -1, -1, 1670, -1, - 54, 515, 516, 517, 37, 519, 520, 521, 522, 523, - 524, 389, -1, 46, 61, -1, -1, -1, 8, -1, - -1, 11, 942, 943, -1, 15, 16, 81, -1, -1, - 20, 21, 22, 953, 204, -1, -1, 957, 958, 959, - 960, 961, -1, 76, 77, 78, -1, 37, -1, -1, - -1, -1, 3652, -1, 974, -1, 2666, 47, -1, -1, - 107, -1, -1, -1, 54, -1, -1, -1, -1, 102, + 7, 532, 0, 0, 0, 46, 880, 0, 856, 16, + 0, 0, 922, 904, 1019, 0, 23, 0, 934, 0, + 1268, 830, 20, 0, 0, 1759, 759, 1270, 1000, 38, + 1065, 985, 955, 7, 16, 20, 950, 1010, 989, 1197, + 1720, 37, 75, 1490, 1474, 1256, 1962, 79, 1636, 23, + 20, 79, 1010, 1609, 1877, 1010, 1114, 7, 1017, 13, + 2343, 1527, 1248, 1649, 1885, 19, 760, 1594, 1692, 1010, + 77, 78, 1328, 23, 1245, 1677, 30, 2267, 2277, 2833, + 1813, 17, 2082, 1233, 0, 23, 2246, 0, 2248, 0, + 44, 45, 1673, 1674, 2830, 2789, 45, 2212, 2789, 1828, + 0, 2266, 1273, 77, 78, 114, 0, 103, 2850, 0, + 34, 1377, 1028, 2334, 0, 767, 1162, 0, 1123, 2857, + 2417, 1167, 2653, 825, 2497, 1724, 0, 77, 78, 831, + 0, 1899, 0, 768, 1008, 0, 0, 1866, 0, 77, + 78, 1898, 0, 1902, 2055, 2056, 0, 2740, 1872, 0, + 23, 2744, 2045, 2046, 2047, 2066, 1087, 1088, 112, 2070, + 0, 9, 9, 2969, 922, 0, 924, 2430, 926, 41, + 0, 0, 5, 1692, 1105, 0, 0, 0, 2017, 5, + 56, 3, 0, 5, 5, 0, 5, 64, 10, 60, + 4, 81, 13, 14, 5, 9, 10, 2425, 9, 5, + 5, 1173, 1011, 5, 77, 78, 5, 13, 14, 5, + 5, 13, 14, 2430, 0, 19, 2437, 1819, 13, 14, + 3197, 5, 13, 14, 2392, 1777, 5, 19, 83, 1779, + 5, 5, 5, 5, 13, 14, 9, 5, 793, 5, + 2423, 31, 5, 5, 83, 2832, 125, 118, 1122, 39, + 19, 27, 13, 14, 109, 94, 4, 1292, 3, 4, + 5, 9, 140, 1179, 9, 11, 928, 175, 1303, 46, + 16, 11, 193, 83, 201, 46, 16, 120, 174, 899, + 77, 78, 19, 149, 94, 1257, 101, 3076, 1260, 1261, + 91, 822, 174, 1207, 4, 3, 31, 3148, 174, 9, + 899, 47, 2523, 2524, 39, 2526, 60, 295, 1091, 64, + 1012, 180, 229, 1034, 108, 101, 183, 2597, 1039, 3178, + 1041, 176, 1024, 1925, 31, 1108, 3029, 35, 36, 130, + 296, 1489, 39, 301, 875, 81, 3181, 296, 869, 1060, + 293, 5, 820, 374, 11, 195, 319, 3384, 15, 16, + 205, 1003, 388, 301, 220, 186, 3201, 41, 1232, 5, + 3560, 278, 138, 281, 123, 1000, 65, 108, 1579, 1527, + 316, 107, 2699, 118, 2701, 244, 75, 1979, 108, 168, + 1228, 1229, 133, 41, 31, 3309, 118, 3311, 351, 1991, + 121, 2757, 120, 42, 249, 195, 84, 118, 3529, 123, + 164, 31, 375, 284, 75, 490, 121, 3639, 1102, 11, + 134, 492, 2235, 15, 16, 2575, 166, 439, 3481, 2021, + 3483, 426, 146, 11, 467, 417, 2028, 390, 3593, 514, + 347, 150, 11, 514, 809, 301, 15, 16, 3192, 13, + 14, 3132, 1333, 295, 253, 47, 123, 3741, 4, 1363, + 123, 11, 827, 9, 3208, 15, 16, 479, 3194, 47, + 406, 466, 150, 11, 427, 173, 197, 175, 47, 1392, + 2072, 280, 193, 2756, 2076, 166, 221, 241, 127, 81, + 295, 398, 197, 202, 234, 528, 133, 133, 64, 366, + 482, 215, 3692, 81, 178, 3626, 532, 528, 3629, 244, + 171, 3564, 81, 133, 2106, 3737, 3800, 1466, 1467, 295, + 379, 2877, 313, 1472, 364, 346, 162, 480, 3442, 255, + 178, 402, 27, 299, 379, 2640, 3691, 1585, 33, 2728, + 266, 3382, 2753, 81, 2755, 390, 495, 278, 215, 290, + 528, 330, 215, 76, 375, 244, 464, 464, 174, 3408, + 523, 3528, 367, 285, 550, 3344, 2083, 285, 440, 280, + 1038, 361, 528, 432, 532, 434, 4, 3004, 3413, 3006, + 109, 9, 427, 2159, 3363, 528, 2397, 2398, 2399, 256, + 2907, 367, 528, 479, 532, 379, 277, 363, 467, 365, + 456, 460, 414, 415, 2218, 169, 451, 64, 605, 1130, + 467, 440, 473, 2866, 3735, 532, 1480, 528, 486, 2075, + 1230, 366, 566, 1544, 1545, 405, 2844, 3204, 1280, 395, + 528, 285, 3325, 605, 2212, 480, 469, 3330, 1884, 3666, + 440, 1230, 433, 138, 280, 1190, 526, 2805, 1569, 285, + 530, 395, 25, 1376, 290, 2925, 1494, 424, 1381, 2866, + 3627, 477, 528, 424, 1387, 532, 2839, 528, 530, 3465, + 3402, 109, 1203, 2884, 335, 1206, 2434, 2391, 1516, 2250, + 405, 2270, 519, 520, 2431, 490, 1919, 531, 2678, 527, + 2439, 1947, 2593, 2576, 2577, 2578, 2579, 1722, 2276, 1383, + 3384, 531, 532, 3384, 2533, 528, 492, 427, 405, 2218, + 1631, 1632, 528, 536, 490, 464, 528, 528, 492, 528, + 528, 533, 534, 531, 529, 536, 531, 528, 514, 533, + 534, 526, 528, 528, 3097, 530, 528, 768, 532, 528, + 514, 2283, 528, 528, 2284, 527, 119, 528, 1612, 1613, + 464, 1393, 3335, 529, 528, 531, 3277, 3340, 1622, 528, + 1666, 1667, 1668, 528, 528, 528, 528, 3054, 527, 1394, + 528, 768, 528, 1637, 3047, 528, 528, 528, 1551, 767, + 478, 519, 520, 3527, 519, 520, 175, 532, 524, 820, + 490, 1502, 767, 3628, 524, 1487, 457, 464, 1571, 526, + 366, 464, 3534, 530, 299, 1669, 3233, 3535, 768, 1987, + 27, 447, 809, 1524, 514, 2716, 33, 761, 467, 519, + 520, 278, 458, 820, 440, 151, 280, 109, 827, 8, + 827, 285, 11, 1606, 2080, 528, 15, 16, 271, 442, + 2018, 20, 21, 22, 517, 809, 397, 254, 365, 2441, + 872, 407, 492, 432, 872, 434, 820, 530, 34, 856, + 857, 3586, 3587, 827, 521, 522, 523, 524, 363, 809, + 123, 229, 3027, 280, 514, 431, 875, 203, 395, 528, + 820, 904, 2030, 880, 60, 138, 1848, 827, 528, 2700, + 493, 345, 820, 3637, 327, 894, 1858, 3623, 0, 1861, + 395, 517, 890, 890, 890, 2497, 109, 890, 2898, 366, + 890, 890, 194, 936, 530, 890, 3641, 890, 878, 890, + 278, 138, 517, 890, 890, 314, 205, 2075, 3033, 521, + 522, 523, 524, 528, 523, 932, 933, 881, 395, 936, + 937, 530, 881, 1843, 858, 859, 809, 861, 83, 863, + 519, 520, 521, 522, 523, 524, 133, 820, 3684, 94, + 164, 432, 427, 434, 827, 169, 1689, 38, 3179, 519, + 520, 521, 522, 523, 524, 470, 4, 528, 2415, 83, + 2236, 9, 3666, 2597, 890, 3666, 531, 890, 1929, 890, + 94, 532, 989, 2983, 1907, 2808, 2809, 1878, 1879, 1880, + 890, 1945, 205, 1000, 2040, 1949, 890, 1038, 1952, 890, + 1007, 1008, 401, 1954, 890, 1003, 1013, 890, 300, 1016, + 1017, 2613, 1019, 1020, 1021, 1022, 890, 3771, 1003, 955, + 890, 2754, 890, 820, 478, 890, 890, 241, 890, 1036, + 398, 1038, 890, 1992, 1993, 1994, 1995, 1996, 1997, 890, + 1047, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, + 2009, 2780, 2640, 1811, 216, 890, 105, 1064, 1065, 1066, + 890, 890, 1036, 3346, 1038, 890, 890, 890, 361, 41, + 2799, 334, 299, 1047, 2340, 3752, 75, 164, 292, 1086, + 72, 73, 1123, 3760, 2665, 1843, 1036, 164, 1038, 61, + 379, 526, 169, 280, 175, 530, 464, 1047, 1036, 1106, + 1038, 390, 2252, 290, 2254, 1863, 3081, 280, 297, 1047, + 1868, 1118, 1119, 1120, 3089, 1122, 1123, 27, 1125, 118, + 5, 1854, 351, 33, 1655, 2555, 1859, 2574, 2371, 174, + 211, 3321, 359, 54, 183, 107, 363, 492, 427, 176, + 1613, 13, 14, 1125, 120, 248, 227, 2403, 2306, 1622, + 27, 1158, 2026, 523, 241, 529, 33, 238, 532, 514, + 530, 390, 528, 1036, 241, 1038, 379, 180, 395, 1176, + 1177, 1970, 2995, 528, 1047, 1974, 38, 390, 1977, 1211, + 1212, 526, 1214, 1211, 1212, 530, 1214, 2789, 347, 133, + 1043, 480, 205, 528, 1203, 526, 1049, 1206, 427, 530, + 109, 2789, 264, 265, 529, 292, 1213, 532, 2379, 31, + 1217, 1218, 249, 316, 427, 292, 13, 14, 162, 3502, + 1227, 1228, 1229, 529, 180, 1232, 532, 3227, 138, 278, + 2832, 244, 204, 209, 2193, 2194, 526, 2242, 528, 1036, + 530, 1038, 1196, 470, 2118, 1479, 361, 1481, 1482, 1256, + 1047, 480, 858, 859, 205, 861, 529, 863, 179, 532, + 3176, 138, 2448, 1272, 86, 1048, 386, 480, 529, 1052, + 257, 532, 2458, 95, 445, 196, 321, 322, 323, 529, + 201, 280, 532, 255, 386, 1292, 285, 300, 244, 370, + 843, 844, 845, 427, 266, 848, 1303, 119, 526, 480, + 528, 2925, 278, 406, 280, 383, 278, 529, 389, 529, + 532, 386, 532, 175, 351, 3048, 490, 529, 492, 240, + 532, 1328, 384, 385, 2198, 6, 515, 516, 517, 10, + 519, 520, 521, 522, 523, 524, 335, 18, 310, 529, + 528, 285, 532, 388, 529, 529, 290, 532, 532, 211, + 3171, 32, 528, 390, 353, 36, 2247, 460, 169, 529, + 529, 1368, 532, 532, 528, 227, 379, 1350, 528, 1350, + 1377, 193, 528, 1350, 1350, 2961, 238, 529, 176, 2826, + 532, 432, 427, 434, 206, 528, 1368, 1394, 437, 299, + 427, 528, 2346, 365, 2348, 1393, 529, 529, 529, 532, + 532, 532, 529, 529, 1411, 532, 532, 2934, 1393, 1416, + 2937, 5, 2939, 529, 451, 174, 532, 462, 390, 432, + 803, 434, 299, 529, 1394, 528, 532, 528, 473, 1411, + 1384, 529, 1386, 529, 532, 2901, 532, 2595, 432, 411, + 434, 2599, 528, 480, 457, 3033, 149, 460, 2349, 359, + 2351, 249, 835, 363, 2753, 528, 2755, 3013, 457, 1466, + 1467, 164, 896, 2311, 898, 1472, 169, 1474, 467, 528, + 2164, 521, 1479, 1480, 1481, 1482, 432, 860, 434, 400, + 3763, 529, 359, 528, 532, 395, 363, 1494, 1495, 529, + 529, 19, 532, 532, 514, 3097, 528, 1504, 529, 1506, + 1474, 532, 1509, 447, 460, 13, 14, 1514, 370, 1516, + 1517, 511, 1519, 226, 458, 2429, 1523, 220, 395, 529, + 13, 14, 532, 2771, 1474, 529, 909, 389, 532, 2772, + 1504, 295, 1506, 127, 128, 1509, 3696, 530, 241, 3738, + 1514, 3740, 511, 1517, 529, 1519, 532, 532, 173, 1523, + 248, 3711, 529, 351, 1504, 532, 1506, 529, 529, 1509, + 470, 532, 529, 528, 1514, 532, 1504, 1517, 1506, 1519, + 529, 1509, 1579, 1523, 529, 2823, 1514, 532, 174, 1517, + 174, 1519, 13, 14, 2770, 1523, 2772, 2645, 2646, 292, + 529, 413, 390, 470, 416, 529, 3198, 60, 301, 300, + 3799, 1474, 3204, 222, 529, 1612, 1613, 532, 3524, 13, + 14, 1609, 1609, 1609, 1621, 1622, 1609, 1000, 316, 1609, + 1609, 13, 14, 1630, 1609, 3785, 528, 1010, 428, 427, + 1637, 1504, 529, 1506, 174, 532, 1509, 1644, 3798, 529, + 511, 1514, 532, 226, 1517, 154, 1519, 2620, 2653, 0, + 1523, 3474, 3475, 451, 13, 14, 154, 1664, 1665, 13, + 14, 109, 1669, 2621, 2622, 1672, 2621, 2622, 2623, 3271, + 1677, 1678, 1679, 1680, 1681, 1682, 1683, 1684, 1685, 1686, + 2621, 379, 480, 1690, 1691, 1692, 13, 14, 1695, 1722, + 359, 180, 1699, 13, 14, 1702, 1703, 1704, 1705, 1706, + 1707, 1708, 1709, 1710, 1686, 359, 1713, 1504, 406, 1506, + 13, 14, 1509, 1720, 154, 1722, 205, 1514, 1692, 154, + 1517, 154, 1519, 2682, 2683, 248, 1523, 321, 322, 323, + 1762, 13, 14, 1740, 1762, 191, 192, 13, 14, 13, + 14, 13, 14, 13, 14, 13, 14, 374, 375, 1132, + 101, 248, 154, 456, 41, 244, 13, 14, 1765, 1142, + 427, 27, 460, 13, 14, 528, 8, 33, 1738, 1776, + 1777, 374, 375, 15, 16, 529, 2509, 280, 20, 21, + 22, 1164, 3384, 13, 14, 13, 14, 13, 14, 480, + 1173, 27, 90, 316, 388, 494, 3384, 33, 149, 13, + 14, 13, 14, 2719, 154, 399, 3486, 154, 264, 265, + 133, 300, 1819, 164, 13, 14, 374, 375, 169, 316, + 374, 375, 1829, 174, 268, 269, 1833, 384, 385, 423, + 528, 295, 183, 427, 13, 14, 530, 188, 154, 162, + 154, 8, 472, 473, 11, 3717, 3718, 1829, 15, 16, + 361, 1833, 3095, 20, 21, 22, 379, 2499, 2500, 440, + 1814, 3706, 3707, 3747, 3748, 1176, 1177, 1874, 462, 220, + 1877, 1825, 138, 1827, 2318, 2319, 1830, 1884, 528, 473, + 1887, 1888, 379, 406, 528, 528, 1840, 529, 1842, 180, + 241, 528, 1874, 429, 2742, 47, 490, 222, 1281, 227, + 305, 1855, 138, 60, 528, 227, 1860, 528, 227, 406, + 1864, 1865, 302, 1867, 2837, 1869, 1870, 2868, 1925, 2793, + 514, 3132, 1929, 2814, 41, 5, 2880, 1923, 384, 385, + 239, 528, 5, 27, 528, 528, 2895, 460, 331, 33, + 1947, 292, 528, 432, 295, 434, 528, 1954, 1955, 2863, + 301, 528, 5, 244, 5, 528, 1963, 280, 5, 528, + 3694, 2996, 285, 460, 3698, 5, 528, 290, 457, 528, + 9, 460, 1979, 248, 491, 528, 1983, 1984, 307, 1986, + 532, 105, 532, 529, 1991, 1992, 1993, 1994, 1995, 1996, + 1997, 342, 467, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 222, 3434, 528, 395, 169, 2015, 2016, + 169, 292, 2019, 290, 2021, 60, 367, 174, 239, 2026, + 440, 2028, 528, 440, 94, 532, 440, 1410, 60, 320, + 60, 528, 440, 299, 201, 19, 271, 440, 3772, 224, + 248, 316, 528, 2050, 138, 440, 8, 490, 386, 11, + 2057, 154, 2059, 15, 16, 297, 2063, 224, 20, 21, + 22, 440, 280, 299, 3666, 2072, 149, 101, 201, 2076, + 280, 2078, 41, 2080, 99, 2082, 427, 280, 3666, 528, + 280, 164, 41, 2057, 280, 280, 169, 528, 379, 2063, + 154, 530, 174, 359, 13, 386, 174, 363, 532, 2106, + 529, 529, 127, 128, 379, 456, 529, 2057, 316, 528, + 3069, 2118, 2119, 2063, 3086, 487, 467, 529, 2125, 2057, + 529, 1000, 529, 359, 447, 2063, 227, 363, 529, 395, + 297, 406, 529, 287, 528, 458, 487, 220, 489, 490, + 227, 432, 1066, 434, 3098, 3099, 287, 528, 528, 174, + 3398, 2158, 528, 530, 532, 2162, 477, 2198, 241, 395, + 2167, 2168, 528, 3593, 321, 322, 323, 528, 528, 460, + 528, 379, 40, 530, 486, 526, 528, 9, 529, 530, + 531, 11, 438, 248, 2057, 460, 2193, 2194, 528, 438, + 2063, 2198, 361, 527, 19, 532, 532, 2930, 406, 438, + 3647, 285, 537, 440, 470, 299, 183, 2214, 3638, 292, + 2217, 2218, 2219, 528, 165, 532, 174, 229, 301, 529, + 41, 467, 467, 220, 532, 271, 398, 229, 2235, 2236, + 296, 388, 532, 319, 470, 2242, 0, 3242, 2245, 201, + 319, 183, 38, 532, 2218, 529, 222, 43, 359, 1173, + 528, 316, 460, 528, 2261, 229, 280, 229, 2265, 530, + 2057, 3691, 224, 1142, 301, 359, 2063, 3226, 2275, 363, + 427, 2245, 60, 515, 516, 517, 60, 519, 520, 521, + 522, 523, 524, 287, 60, 2292, 2293, 2261, 60, 287, + 340, 2265, 293, 480, 1173, 2245, 321, 322, 323, 3457, + 529, 395, 2309, 2257, 2311, 462, 102, 2245, 154, 3, + 154, 2261, 528, 528, 379, 2265, 473, 154, 154, 2326, + 528, 154, 490, 2261, 41, 154, 532, 2265, 532, 295, + 280, 3, 1256, 2340, 3067, 297, 41, 101, 295, 60, + 174, 406, 2349, 2350, 2351, 11, 41, 1730, 515, 516, + 517, 169, 519, 520, 521, 522, 523, 524, 529, 529, + 183, 169, 528, 388, 529, 529, 3, 2349, 2350, 2351, + 528, 528, 2245, 456, 399, 528, 470, 40, 1257, 175, + 3, 1260, 1261, 526, 118, 149, 526, 2394, 2261, 527, + 529, 149, 2265, 529, 529, 460, 2403, 174, 423, 440, + 164, 532, 427, 537, 440, 169, 164, 440, 530, 440, + 174, 169, 511, 529, 529, 211, 511, 529, 359, 183, + 529, 1804, 1805, 448, 188, 8, 3669, 150, 11, 2436, + 530, 227, 15, 16, 2441, 511, 529, 462, 529, 2422, + 174, 2422, 238, 526, 174, 2422, 2422, 530, 473, 440, + 3364, 528, 487, 528, 2436, 157, 220, 528, 60, 528, + 528, 41, 220, 528, 47, 490, 247, 60, 2265, 532, + 513, 54, 296, 517, 296, 464, 532, 241, 274, 60, + 271, 75, 280, 241, 479, 440, 440, 81, 1871, 514, + 2497, 205, 154, 154, 2501, 154, 440, 2451, 81, 295, + 94, 528, 528, 528, 440, 528, 1889, 1890, 440, 3040, + 2517, 440, 529, 528, 528, 41, 41, 295, 361, 529, + 1399, 1400, 293, 532, 118, 321, 120, 490, 292, 528, + 41, 295, 328, 38, 292, 2517, 528, 301, 43, 154, + 285, 529, 174, 301, 321, 322, 323, 528, 2555, 528, + 60, 529, 188, 515, 516, 517, 529, 519, 520, 521, + 522, 523, 524, 14, 1066, 529, 169, 81, 529, 3443, + 526, 3445, 3488, 144, 370, 309, 529, 529, 342, 174, + 528, 2555, 529, 532, 529, 19, 1969, 321, 322, 323, + 2597, 529, 532, 389, 528, 306, 179, 102, 529, 254, + 528, 367, 528, 367, 296, 2555, 2613, 3455, 529, 183, + 154, 388, 2653, 196, 532, 209, 2625, 529, 201, 528, + 178, 529, 529, 2630, 451, 529, 528, 530, 528, 528, + 2637, 2638, 41, 87, 41, 41, 41, 532, 174, 467, + 528, 489, 529, 529, 201, 529, 2653, 2688, 3653, 3458, + 427, 3460, 527, 527, 388, 1579, 532, 240, 529, 2666, + 2693, 473, 2669, 427, 2671, 529, 462, 529, 529, 532, + 175, 2678, 2679, 295, 60, 2682, 2683, 486, 517, 529, + 2687, 2688, 2555, 490, 3598, 462, 280, 2694, 529, 529, + 207, 285, 456, 427, 3585, 118, 473, 3607, 456, 41, + 229, 2742, 528, 467, 2711, 89, 211, 194, 285, 529, + 285, 530, 517, 490, 297, 2722, 530, 530, 529, 2715, + 530, 440, 227, 487, 530, 489, 490, 530, 462, 530, + 3578, 530, 440, 238, 527, 2742, 530, 514, 530, 473, + 530, 335, 41, 530, 530, 530, 280, 530, 1627, 528, + 530, 528, 108, 1677, 1256, 530, 490, 530, 527, 353, + 530, 1640, 526, 1642, 529, 529, 530, 531, 526, 274, + 530, 529, 530, 530, 530, 530, 530, 530, 530, 2786, + 514, 530, 2789, 2790, 530, 2792, 2793, 2794, 530, 530, + 530, 2745, 1671, 530, 528, 2749, 532, 530, 530, 490, + 529, 2808, 2809, 529, 2786, 529, 528, 427, 2790, 295, + 2792, 41, 528, 9, 2821, 360, 321, 400, 528, 528, + 2861, 342, 529, 328, 2778, 2832, 532, 60, 532, 529, + 201, 61, 529, 2840, 2875, 2831, 532, 527, 472, 194, + 529, 2795, 2796, 2797, 2798, 532, 2800, 2801, 2802, 2803, + 2804, 92, 529, 2836, 353, 2836, 7, 8, 528, 2836, + 2836, 2868, 13, 457, 41, 370, 154, 125, 19, 2865, + 530, 529, 23, 467, 25, 154, 41, 107, 29, 30, + 31, 2922, 529, 34, 389, 375, 37, 38, 2895, 41, + 41, 2898, 375, 44, 45, 1819, 529, 1776, 1777, 528, + 528, 41, 3635, 467, 528, 532, 37, 315, 2915, 2916, + 253, 2918, 528, 285, 252, 2922, 193, 467, 2925, 451, + 528, 75, 298, 81, 9, 75, 77, 78, 529, 512, + 529, 528, 377, 3807, 60, 527, 519, 520, 521, 522, + 523, 524, 529, 527, 134, 2952, 94, 436, 178, 60, + 517, 278, 103, 451, 295, 41, 528, 462, 298, 110, + 111, 112, 113, 114, 115, 2972, 298, 528, 472, 1848, + 2977, 2978, 207, 2356, 204, 2982, 2983, 529, 529, 1858, + 2987, 3022, 1861, 2990, 2991, 295, 529, 295, 2995, 2996, + 529, 395, 2999, 436, 281, 123, 3003, 464, 473, 8, + 374, 1925, 11, 150, 473, 0, 15, 16, 3015, 521, + 26, 20, 21, 22, 37, 3013, 3013, 3013, 303, 528, + 3013, 374, 41, 3013, 3013, 255, 3557, 529, 3013, 3003, + 890, 2773, 2415, 2341, 1841, 2344, 266, 2786, 47, 3490, + 2720, 41, 61, 3638, 3589, 54, 3757, 3191, 278, 1954, + 867, 3091, 2504, 3003, 3061, 1979, 3613, 3730, 3357, 3670, + 3679, 61, 3069, 3723, 3415, 3003, 1243, 1991, 2332, 2350, + 1819, 1819, 81, 2851, 3668, 3677, 2763, 1579, 2792, 3665, + 310, 3088, 2329, 1376, 3664, 3762, 8, 3329, 107, 11, + 3097, 2499, 2421, 15, 16, 2500, 1348, 2021, 20, 21, + 22, 2868, 2309, 2826, 2028, 1035, 101, 107, 108, 2794, + 1035, 1196, 2275, 1801, 3068, 37, 1221, 2574, 118, 3647, + 2292, 1056, 1220, 3566, 1765, 3132, 3447, 1800, 23, 2069, + 3003, 1223, 2261, 3266, 1012, 365, 2519, 2925, 2521, 2555, + 2554, 3378, 2525, 2605, 2527, 1010, 3570, 2085, 2072, 3569, + 2169, 1010, 2076, 1010, 149, 1010, 1010, 1010, 2082, 1010, + 390, 1010, 2638, 1010, 2218, 2276, 2120, 3314, 2215, 164, + 179, 3178, 2662, 3015, 169, 1677, 2172, 1495, 178, 174, + 1890, 411, 2106, 3555, 3191, 204, 3013, 196, 183, 2974, + 804, 3198, 201, 188, 2078, 1738, 3178, 3204, 1000, 3195, + 1394, 3242, 2721, 180, 204, 1739, 3003, 3214, 3215, 3191, + 3217, 2090, 2517, 1874, -1, 224, 225, -1, -1, 3226, + 3227, 909, -1, -1, -1, 220, -1, -1, 205, -1, + -1, 240, -1, -1, -1, 3242, 255, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 241, 266, -1, -1, + -1, -1, -1, -1, -1, 255, -1, -1, -1, 278, + -1, -1, -1, -1, 3271, -1, 266, 244, -1, 8, + 3277, 280, 11, -1, 283, 0, 15, 16, 278, -1, + 280, 20, 21, 22, -1, -1, 3293, 3294, 297, -1, + 3297, 310, 3299, -1, -1, 20, -1, 292, 23, -1, + 295, -1, 224, -1, -1, -1, 301, -1, -1, -1, + 310, -1, 37, -1, -1, -1, -1, 1819, -1, -1, + -1, 46, -1, 300, 3331, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 8, -1, -1, 11, -1, -1, + -1, 15, 16, 320, -1, -1, 365, 342, 3355, -1, + -1, -1, 77, 78, 79, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 365, -1, -1, -1, -1, + 2249, 390, 367, 47, -1, 297, -1, 3384, 103, 8, + 54, -1, 11, 56, -1, -1, 15, 16, -1, -1, + 390, 400, 411, -1, -1, -1, -1, -1, -1, -1, + 551, 3408, 379, 2282, 2283, 556, 2789, 81, 3404, 386, + -1, 411, 3366, 413, -1, 566, 416, -1, 47, -1, + -1, -1, -1, 1925, -1, 54, 3408, 3434, -1, -1, + 103, -1, 427, 3387, 3388, -1, 3443, -1, 3445, -1, + 3447, -1, -1, 2826, 3451, -1, 3453, -1, 3455, -1, + -1, 124, 81, -1, -1, 432, -1, 434, 3412, -1, + 3434, 456, 3469, -1, -1, -1, 153, 3474, 3475, 142, + -1, -1, 467, 147, 147, -1, -1, 1979, 2861, 3486, + 457, -1, -1, 460, 3434, 224, -1, 174, -1, 1991, + -1, 3487, 487, 3489, 489, 490, 3503, 170, -1, -1, + 173, 3508, -1, 512, -1, 179, 515, 516, 517, -1, + 519, 520, 521, 522, 523, 524, 189, 2441, 147, 2021, + -1, 3503, 196, -1, -1, -1, 2028, 201, 528, -1, + -1, 526, -1, -1, 529, 530, 531, 3533, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 3556, + 179, -1, -1, -1, 3550, -1, -1, -1, 297, 3566, + -1, 3434, -1, -1, -1, -1, 240, 196, -1, -1, + 2072, 3578, 201, 2497, 2076, -1, -1, -1, -1, -1, + 2082, -1, -1, -1, -1, -1, 3593, -1, 3595, -1, + -1, -1, -1, 515, 516, 517, -1, 519, 520, 521, + 522, 523, 524, -1, 2106, -1, -1, 3614, 759, 760, + 761, 240, 285, -1, -1, -1, -1, -1, -1, 3593, + 293, -1, -1, 297, 3007, -1, -1, -1, -1, -1, + -1, 3638, -1, -1, 321, 322, 323, -1, -1, -1, + -1, -1, 315, 3593, -1, -1, 3653, -1, -1, -1, + -1, -1, 803, 804, -1, -1, -1, -1, 809, 3666, + 811, 3668, -1, -1, 3638, -1, -1, -1, 297, 820, + -1, -1, 345, 824, 825, -1, 827, -1, -1, 830, + 831, -1, -1, -1, 3691, -1, 3668, 3683, 3638, 2613, + -1, -1, 843, 844, 845, -1, -1, 848, 3705, 3706, + 3707, 388, -1, -1, 3658, 856, 857, 858, 859, 3716, + 861, -1, 863, 3709, -1, -1, -1, 3691, -1, -1, + 3593, -1, -1, -1, 875, -1, 400, -1, -1, -1, + 881, -1, -1, -1, 3741, -1, -1, -1, 2617, 2618, + 427, 3691, -1, 894, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 2678, -1, -1, 908, 909, -1, + -1, -1, -1, -1, -1, 3638, -1, -1, -1, -1, + -1, 400, -1, -1, -1, 462, 515, 516, 517, -1, + 519, 520, 521, 522, 523, 524, 473, -1, -1, 153, + -1, -1, -1, 3800, -1, -1, 947, 948, -1, -1, + 3807, -1, -1, 490, -1, -1, -1, 958, -1, -1, + 174, 962, 963, 964, 965, 966, -1, -1, 3691, -1, + 174, -1, -1, -1, -1, 550, -1, 514, 979, 25, + -1, -1, -1, -1, -1, -1, -1, -1, 512, -1, + -1, 528, -1, -1, -1, 519, 520, 521, 522, 523, + 524, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1011, 1012, -1, 1014, -1, 2789, 1017, -1, -1, -1, + -1, -1, 1023, 1024, -1, -1, -1, -1, -1, 1030, + -1, -1, -1, 512, -1, 1036, 82, 1038, -1, -1, + 519, 520, 521, 522, 523, 524, 1047, 8, -1, -1, + 11, -1, 98, -1, 15, 16, 1057, -1, 2832, -1, + 2789, -1, -1, -1, -1, -1, -1, 8, -1, -1, + 11, -1, -1, 1074, 15, 16, -1, 3310, -1, 20, + 21, 22, -1, -1, -1, -1, 47, -1, -1, 2441, + -1, -1, -1, 54, -1, -1, 37, -1, 1066, -1, + -1, 1102, 148, 1066, -1, 309, 47, 321, 322, 323, + -1, -1, 158, 54, -1, -1, -1, 321, 322, 323, + 81, -1, -1, -1, 2898, 171, 1127, -1, -1, -1, + 176, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 81, -1, -1, -1, -1, 2497, -1, -1, -1, -1, + -1, 3384, -1, -1, -1, -1, -1, -1, 8, 205, + -1, 11, -1, -1, -1, 15, 16, -1, -1, -1, + 20, 21, 22, -1, 388, -1, -1, -1, -1, 1180, + -1, -1, -1, -1, 388, -1, 147, -1, -1, -1, + -1, 3424, 767, 768, -1, 1196, 1197, 47, -1, -1, + -1, -1, 1203, 249, 54, 1206, 2935, -1, 254, 2983, + -1, -1, -1, 427, -1, -1, -1, -1, 179, -1, + -1, -1, -1, 427, -1, -1, 1227, 1228, 1229, -1, + -1, 81, -1, -1, 809, 196, 1237, -1, 179, 1240, + 201, 1242, 1243, -1, -1, 820, -1, -1, 462, -1, + -1, -1, 827, -1, 1255, 196, -1, -1, 462, 473, + 201, 2613, -1, -1, -1, -1, -1, -1, -1, 473, + -1, 1272, -1, -1, 320, 1276, 490, -1, -1, 240, + 326, 1282, -1, 224, 225, 1066, 490, -1, 1256, -1, + -1, -1, 3525, 1256, -1, 341, -1, 872, -1, 240, + 514, -1, -1, -1, -1, 127, 128, -1, -1, -1, + 514, -1, -1, -1, 528, 890, -1, -1, -1, -1, + -1, -1, -1, 3097, 528, -1, 2678, -1, -1, 179, + 376, -1, -1, 379, -1, -1, 297, -1, -1, 280, + -1, -1, 283, 1344, 390, 1346, 196, 393, -1, -1, + -1, 201, 174, 1354, -1, -1, 297, 3086, 3132, 300, + -1, -1, 3091, -1, -1, -1, 1367, 413, -1, -1, + -1, -1, -1, -1, -1, 1376, -1, -1, -1, -1, + 1381, 427, 1383, 1384, -1, 1386, 1387, -1, 434, 435, + 240, -1, -1, -1, -1, -1, -1, -1, -1, 445, + -1, -1, -1, -1, -1, 451, -1, -1, -1, -1, + -1, 3140, 3141, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 3198, -1, -1, -1, 1003, -1, + 3204, -1, -1, 3666, 480, -1, -1, 2789, -1, 400, + -1, -1, -1, -1, -1, -1, -1, 297, -1, -1, + -1, -1, -1, 3227, -1, -1, -1, -1, -1, 400, + -1, 1036, -1, 1038, -1, 1466, 1467, -1, -1, -1, + -1, 1472, 1047, 1474, -1, 1256, -1, -1, -1, -1, + 2832, -1, -1, -1, -1, -1, 1487, -1, 1489, 1490, + -1, -1, -1, 1494, 1495, -1, 1497, 3271, -1, 321, + 322, 323, -1, 1504, 38, 1506, -1, 1082, 1509, 43, + -1, -1, -1, 1514, -1, 1516, 1517, -1, 1519, 1094, + -1, -1, 1523, -1, 1525, -1, 1527, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 56, -1, 462, 255, 1006, 1007, -1, 1009, - 180, 81, 1012, 147, -1, -1, 266, -1, 1018, 1019, - -1, -1, -1, -1, -1, 1025, -1, -1, 278, -1, - -1, 1031, -1, 1033, -1, 205, -1, -1, -1, -1, - -1, -1, 1042, 1810, -1, 179, -1, -1, 2429, 103, - -1, 178, 1052, -1, -1, -1, -1, -1, 1810, -1, - 310, -1, 196, -1, -1, -1, 2080, 201, -1, 1069, - 124, -1, -1, -1, 244, -1, -1, 204, -1, -1, - -1, -1, -1, 1459, 1460, -1, 2776, -1, 142, 1465, - -1, -1, -1, 147, -1, -1, -1, 1097, -1, -1, - -1, -1, -1, -1, 2485, -1, 240, -1, -1, 179, - -1, -1, -1, -1, -1, 365, 170, -1, -1, 173, - -1, -1, 1122, -1, -1, -1, 196, -1, 255, 2819, - 300, 201, -1, -1, 8, 189, -1, 11, -1, 266, - 390, 15, 16, -1, 2670, 2671, -1, -1, -1, 1916, - 320, 278, -1, -1, 224, 225, -1, -1, -1, -1, - -1, 411, -1, 297, 1916, -1, -1, -1, -1, -1, - 240, -1, -1, 47, -1, 1175, -1, -1, -1, -1, - 54, -1, -1, 310, -1, -1, -1, -1, 1572, -1, - -1, 1191, 1192, -1, -1, 2885, -1, -1, 1198, -1, - -1, 1201, -1, 1970, -1, -1, -1, 81, -1, 379, - 280, -1, -1, 283, -1, 1982, 386, -1, 1970, -1, - 2601, -1, 1222, 1223, 1224, 2239, -1, 297, -1, -1, - 1982, 285, 1232, -1, -1, 1235, 1236, -1, 365, 293, - -1, -1, -1, -1, -1, 2012, -1, -1, 1248, -1, - -1, -1, 2019, -1, -1, -1, -1, -1, 2272, 2273, - 2012, 315, 432, 390, 434, 1265, 400, 2019, -1, 1269, - -1, -1, -1, 147, -1, 1275, -1, -1, -1, -1, - 2970, -1, -1, -1, 411, 2666, 1670, 457, -1, -1, - 460, 345, -1, -1, -1, -1, 2063, -1, -1, -1, - 2067, 1061, -1, -1, -1, 179, 2073, -1, -1, -1, - -1, 2063, -1, -1, -1, 2067, -1, -1, -1, -1, - -1, 2073, 196, -1, -1, -1, -1, 201, -1, 2096, - 400, -1, -1, -1, -1, 1711, -1, 1337, -1, 1339, - -1, -1, -1, -1, 2096, -1, -1, 1347, -1, -1, - -1, -1, -1, -1, -1, -1, 2882, -1, -1, -1, - 1360, -1, -1, -1, -1, -1, 240, -1, -1, 1369, - -1, -1, -1, -1, 1374, -1, 1376, 1377, 512, 1379, - 1380, -1, 1758, 2909, -1, 519, 520, 521, 522, 523, - 524, -1, -1, 3083, -1, 2776, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 2898, -1, 1123, -1, + 400, 512, -1, -1, -1, -1, -1, -1, 519, 520, + 521, 522, 523, 524, -1, -1, 388, -1, 102, -1, + -1, 512, -1, -1, 515, 516, 517, 399, 519, 520, + 521, 522, 523, 524, 8, -1, -1, 11, -1, -1, + -1, 15, 16, -1, -1, -1, -1, -1, -1, -1, + -1, 423, -1, -1, -1, 427, -1, -1, -1, -1, + 3384, 1579, -1, -1, -1, -1, 1579, 8, -1, -1, + 11, -1, -1, 47, 15, 16, -1, -1, -1, -1, + 54, 2983, -1, -1, -1, 1636, 1211, 1212, -1, 1214, + 462, 175, -1, -1, -1, 3374, 3375, -1, 1649, 183, + -1, 473, -1, -1, -1, 3384, 47, 81, -1, -1, + -1, -1, 512, 54, -1, 515, 516, 517, 490, 519, + 520, 521, 522, 523, 524, -1, -1, 211, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 1066, 1689, -1, + 81, 1692, 514, 227, 38, -1, -1, -1, -1, 43, + -1, -1, -1, -1, 238, -1, 528, -1, -1, 1677, + -1, -1, -1, -1, 1677, -1, -1, -1, -1, 1720, + -1, -1, -1, 147, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1735, -1, 1737, -1, 1739, -1, + 274, -1, -1, -1, -1, 3097, -1, -1, -1, 1750, + -1, -1, 1753, -1, -1, 179, 147, -1, 102, -1, + -1, 295, 8, -1, 1765, 11, -1, -1, -1, 15, + 16, -1, 196, -1, 20, 21, 22, 201, -1, -1, + 3132, -1, -1, -1, -1, -1, -1, 321, 179, 1790, + -1, 1792, -1, -1, 328, -1, -1, -1, 1579, -1, + -1, -1, -1, 1804, 1805, 196, -1, -1, 1066, 1810, + 201, -1, 1813, 1814, -1, -1, 240, -1, 1393, -1, + -1, -1, -1, -1, 1825, 1826, 1827, 1828, -1, 1830, + -1, 175, -1, -1, -1, -1, 370, -1, -1, 1840, + -1, 1842, -1, -1, -1, -1, 3198, -1, -1, 240, + -1, 1819, 3204, 1854, 1855, 389, 1819, -1, 1859, 1860, + -1, -1, -1, 1864, 1865, 1866, 1867, 211, 1869, 1870, + -1, -1, -1, 297, -1, 3227, -1, 1256, -1, -1, + 37, -1, -1, 227, 41, -1, -1, -1, -1, -1, + 1891, -1, 3666, -1, 238, -1, 1677, -1, 1899, -1, + 1901, 1902, 1903, 1904, 1905, 1906, 297, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 3271, + 1921, -1, -1, -1, -1, -1, -1, -1, 462, 1504, + 274, 1506, -1, 467, 1509, -1, -1, 3666, -1, 1514, + -1, 1942, 1517, -1, 1519, -1, 103, -1, 1523, -1, + -1, 295, -1, -1, 111, -1, 113, 1925, 115, -1, + -1, -1, 1925, -1, -1, 1017, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 400, 321, 224, -1, + -1, -1, -1, -1, 328, -1, -1, -1, -1, -1, + -1, 1992, 1993, 1994, 1995, 1996, 1997, -1, 1256, 2000, + 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 400, + -1, 1979, -1, -1, -1, -1, 1979, -1, -1, -1, + -1, -1, -1, 1991, -1, -1, 370, -1, 1991, 2030, + -1, -1, 3384, -1, 1609, -1, -1, -1, 1819, -1, + -1, -1, -1, 2044, -1, 389, -1, -1, -1, -1, + -1, 297, -1, 2021, -1, -1, 2057, -1, 2021, -1, + 2028, -1, 2063, -1, -1, 2028, -1, -1, 2069, -1, + -1, -1, -1, -1, 2075, -1, -1, -1, -1, -1, + -1, -1, -1, 2084, 2085, -1, -1, -1, 512, -1, + -1, -1, -1, -1, -1, 519, 520, 521, 522, 523, + 524, -1, -1, -1, 2072, 13, -1, -1, 2076, 2072, + -1, 19, -1, 2076, 2082, 23, -1, 1692, 462, 2082, + -1, 512, 30, 467, -1, -1, -1, -1, 519, 520, + 521, 522, 523, 524, -1, -1, 44, 45, 2106, -1, + -1, -1, -1, 2106, 1925, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 2159, -1, + -1, -1, -1, 2164, -1, -1, 1218, -1, 2169, 77, + 78, -1, -1, -1, -1, 1227, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 1762, -1, -1, + -1, -1, 2193, 2194, -1, -1, -1, -1, 1979, -1, + 1579, -1, -1, -1, 112, -1, -1, -1, -1, -1, + 1991, 2212, -1, -1, -1, -1, -1, 2218, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 297, -1, -1, 1810, -1, 3118, -1, - -1, -1, -1, 2959, -1, 548, -1, -1, 2819, -1, - -1, -1, 512, -1, -1, 515, 516, 517, -1, 519, - 520, 521, 522, 523, 524, -1, -1, -1, -1, 1459, - 1460, -1, -1, -1, -1, 1465, -1, 1467, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1480, -1, 1482, 1483, -1, -1, -1, 1487, 1488, 1249, - 1490, -1, -1, -1, 3184, -1, -1, 1497, -1, 1499, - 3190, -1, 1502, -1, 2885, -1, -1, 1507, -1, 1509, - 1510, -1, 1512, -1, 101, -1, 1516, -1, 1518, -1, - 1520, -1, -1, 3213, -1, -1, 400, -1, -1, 3055, - -1, -1, 1916, -1, -1, -1, -1, -1, -1, 13, - -1, -1, -1, -1, -1, 19, -1, -1, -1, 23, - -1, -1, -1, -1, -1, -1, 30, -1, -1, -1, - -1, -1, 149, -1, -1, -1, -1, 3257, -1, -1, - 44, 45, -1, -1, -1, -1, -1, 164, -1, -1, - -1, -1, 169, -1, -1, -1, 1970, 174, -1, 2970, - -1, 2605, 2606, -1, -1, -1, 183, -1, 1982, -1, - -1, 188, 76, 77, -1, -1, -1, 1983, 1984, 1985, - 1986, 1987, 1988, -1, -1, 1991, 1992, 1993, 1994, 1995, - 1996, 1997, 1998, 1999, 2000, -1, -1, -1, 2012, 1629, - -1, -1, -1, 220, -1, 2019, 110, -1, 512, -1, - -1, -1, 1642, -1, -1, 519, 520, 521, 522, 523, - 524, 764, 765, -1, 241, -1, -1, -1, -1, -1, - -1, -1, 2429, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 2429, -1, 2063, - 3370, -1, 1682, 2067, -1, 1685, 3212, -1, -1, 2073, - -1, -1, -1, 806, -1, -1, -1, -1, -1, -1, - -1, -1, 3083, -1, 817, 292, -1, -1, 295, -1, - -1, 824, 2096, 1713, 301, -1, -1, -1, 2485, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1728, -1, - 1730, -1, 1732, 2485, -1, -1, -1, 3118, -1, -1, - -1, -1, -1, 1743, -1, -1, 1746, -1, -1, -1, - -1, -1, -1, -1, -1, 342, 869, -1, 1758, -1, - -1, -1, 2776, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 887, -1, -1, -1, -1, -1, - 367, -1, -1, 1783, -1, 1785, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 1797, 1798, -1, - -1, -1, -1, 3184, 1804, 1805, -1, 2183, 2184, 3190, - -1, -1, 1572, -1, -1, -1, 1816, 1817, 1818, 1819, - -1, 1821, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1831, 3213, 1833, 2601, -1, -1, -1, -1, -1, - 427, -1, -1, -1, -1, 1845, 1846, -1, -1, 2601, - 1850, 1851, -1, -1, -1, 1855, 1856, 1857, 1858, -1, - 1860, 1861, -1, -1, -1, -1, -1, -1, -1, 456, - -1, -1, -1, -1, -1, -1, 3257, -1, -1, -1, - 467, -1, 1882, -1, -1, 998, -1, -1, -1, -1, - 1890, -1, 1892, 1893, 1894, 1895, 1896, 1897, -1, 2666, - 487, -1, 489, 490, -1, -1, -1, -1, 2922, -1, - 1670, -1, 1912, 3439, 2666, -1, -1, -1, 1031, -1, - 1033, -1, -1, 2299, -1, -1, -1, -1, -1, 1042, - -1, -1, -1, 1933, -1, -1, -1, -1, -1, 526, - -1, -1, 529, 530, 531, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 37, -1, -1, -1, 41, - -1, -1, 3652, -1, 1077, 8, -1, -1, 11, -1, - -1, -1, 15, 16, -1, -1, 1089, -1, -1, -1, - -1, -1, -1, 1983, 1984, 1985, 1986, 1987, 1988, 3370, - -1, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, - 2000, -1, -1, -1, 47, 1118, -1, -1, -1, 2776, - -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, - 102, 2021, -1, -1, 2776, -1, -1, 109, -1, 111, - -1, 113, -1, -1, -1, 2035, -1, -1, 81, -1, - -1, -1, -1, -1, -1, 2429, -1, -1, 2048, -1, - 1810, -1, 2819, -1, 2054, -1, -1, -1, 3072, -1, - 2060, -1, -1, 3077, -1, -1, 2066, 2819, -1, -1, - -1, -1, -1, -1, -1, 2075, 2076, -1, -1, -1, + 2021, -1, -1, -1, 2245, -1, -1, 2028, -1, -1, + -1, -1, -1, -1, -1, -1, 2257, -1, -1, -1, + 2261, -1, -1, -1, 2265, 2266, -1, -1, -1, 515, + 516, 517, -1, 519, 520, 521, 522, 523, 524, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 564, -1, -1, 1206, 1207, -1, 1209, -1, -1, -1, - -1, 2485, -1, -1, 147, -1, -1, -1, -1, -1, - -1, -1, 3126, 3127, -1, -1, -1, -1, 2885, -1, + -1, 2072, -1, -1, -1, 2076, -1, -1, 1677, -1, + -1, 2082, -1, -1, -1, 2306, -1, -1, 2309, -1, + 2311, 2312, -1, -1, 3666, -1, -1, -1, -1, -1, + -1, 1579, -1, -1, -1, 2106, -1, 8, -1, -1, + 11, -1, -1, 2334, 15, 16, -1, -1, -1, 20, + 21, 22, -1, -1, -1, -1, -1, 8, 1923, -1, + 11, -1, -1, -1, 15, 16, 37, -1, -1, 20, + 21, 22, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 37, 8, -1, -1, + 11, -1, -1, -1, 15, 16, -1, -1, -1, 20, + 21, 22, -1, -1, 551, -1, -1, -1, -1, 556, + -1, -1, -1, -1, -1, -1, 37, -1, -1, -1, + -1, -1, -1, -1, 1466, 1467, 2417, -1, -1, 1677, + 1472, -1, -1, 2424, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 2434, -1, -1, 2437, 2438, 2439, 2440, + 1819, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 2451, -1, 2453, -1, -1, -1, 2457, -1, -1, 2460, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 2885, -1, -1, 179, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 2149, - -1, -1, -1, 196, 2154, -1, 1916, -1, 201, 2159, - -1, 8, -1, -1, 11, -1, -1, -1, 15, 16, + -1, -1, -1, 2441, -1, -1, -1, -1, 2441, -1, + -1, -1, 2057, -1, -1, -1, -1, -1, 2063, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 2183, 2184, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 240, -1, -1, - 47, -1, 2202, 2970, -1, -1, -1, 54, 2208, -1, - 1970, -1, -1, -1, -1, -1, -1, 2601, 2970, -1, - -1, -1, 1982, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 81, 2235, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 2247, -1, -1, - -1, 2251, 2012, -1, 297, 2255, 2256, -1, -1, 2019, + -1, -1, 2503, -1, -1, -1, -1, -1, 2509, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 3652, -1, 1386, -1, -1, -1, -1, -1, -1, - -1, -1, 2666, -1, 758, 759, -1, -1, -1, -1, - -1, -1, -1, -1, 2670, 2671, 2296, -1, -1, 2299, - 147, 2301, 2302, 2063, -1, -1, -1, 2067, -1, -1, - -1, -1, -1, 2073, -1, -1, 3083, -1, -1, -1, - -1, -1, 2322, -1, -1, -1, -1, -1, -1, -1, - -1, 3083, 179, -1, -1, -1, 2096, -1, -1, -1, - -1, -1, -1, 817, -1, -1, 3360, 3361, -1, 196, - -1, 3118, -1, -1, 201, -1, 3370, 400, -1, -1, - -1, -1, -1, -1, -1, -1, 3118, -1, -1, -1, + -1, -1, 2523, 2524, -1, 2526, -1, -1, -1, 2497, + 8, -1, -1, 11, 2497, -1, -1, 15, 16, -1, + -1, -1, -1, 224, -1, -1, 1925, -1, -1, -1, + -1, -1, -1, -1, 2555, -1, -1, -1, -1, -1, + -1, 1819, 1017, 224, -1, -1, -1, -1, -1, 47, + -1, -1, 2573, -1, -1, -1, 54, -1, -1, -1, + 2581, 2582, 2583, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 224, 2595, -1, 2597, -1, 2599, -1, + 1979, -1, -1, 81, 2605, -1, -1, -1, -1, -1, + -1, -1, 1991, -1, -1, -1, 297, -1, -1, -1, + -1, -1, -1, 2198, 2625, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 297, -1, -1, 2640, + -1, -1, 2021, 2218, -1, 2613, -1, -1, -1, 2028, + 2613, 2652, -1, -1, 811, 2656, -1, -1, 566, 2234, + 2441, -1, -1, -1, -1, -1, 297, 1925, -1, 147, + 2245, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 2682, 2683, -1, -1, -1, 2261, -1, -1, -1, + 2265, -1, -1, 2072, -1, -1, -1, 2076, -1, -1, + -1, 179, -1, 2082, -1, 2706, -1, -1, -1, -1, + 2678, -1, 2713, 1765, -1, 2678, 2497, -1, 196, 2720, + -1, 1979, -1, 201, -1, -1, -1, 2106, -1, -1, + -1, -1, -1, 1991, -1, 2736, -1, -1, -1, 2740, + -1, 2742, -1, 2744, 2745, -1, -1, 2748, 2749, -1, + -1, 908, 2753, 2754, 2755, -1, 2757, -1, -1, -1, + -1, -1, 240, 2021, -1, -1, -1, -1, -1, -1, + 2028, -1, 1227, -1, -1, -1, -1, 2778, -1, 2780, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1497, -1, 1499, -1, -1, 1502, - -1, -1, 2776, 240, 1507, -1, -1, 1510, -1, 1512, - -1, -1, -1, 1516, 878, 2405, -1, -1, -1, -1, - -1, -1, 2412, -1, -1, -1, -1, 3184, -1, -1, - -1, -1, 2422, 3190, -1, 2425, 2426, 2427, 2428, -1, - -1, -1, 3184, -1, -1, 2819, -1, -1, 3190, 2439, - -1, 2441, -1, -1, -1, 2445, 3213, -1, 2448, -1, - 297, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 3213, -1, 0, -1, -1, -1, 549, -1, 512, - -1, -1, 554, -1, -1, -1, 519, 520, 521, 522, - 523, 524, -1, -1, -1, -1, -1, -1, -1, 1602, - 3257, 2491, -1, -1, -1, -1, -1, 2497, -1, -1, - -1, 2885, -1, -1, -1, 3257, 2882, -1, -1, -1, - -1, 2511, 2512, -1, 2514, -1, -1, -1, -1, -1, + 947, -1, -1, -1, 2795, 2796, 2797, 2798, 2799, 2800, + 2801, 2802, 2803, 2804, -1, 962, 963, 964, 965, 966, + -1, -1, -1, -1, 2072, -1, -1, -1, 2076, 297, + -1, 2789, -1, -1, 2082, -1, 2789, -1, -1, 2830, + -1, -1, 2613, -1, 515, 516, 517, 2838, 519, 520, + 521, 522, 523, 524, -1, -1, -1, -1, 2106, -1, + 2851, -1, 760, 761, 515, 516, 517, 1014, 519, 520, + 521, 522, 523, 524, 2832, -1, -1, -1, -1, 2832, + -1, -1, -1, -1, -1, -1, 2877, -1, -1, 2454, + -1, -1, -1, 2884, 515, 516, 517, -1, 519, 520, + 521, 522, 523, 524, 2895, -1, -1, 2678, -1, -1, + 2901, -1, -1, -1, -1, -1, -1, 2908, 2909, 2910, + 2911, -1, 820, -1, -1, -1, -1, -1, -1, -1, + -1, 2922, 400, -1, 2925, -1, -1, -1, 2929, 2930, + 2898, -1, -1, -1, -1, 2898, -1, 2938, -1, -1, + 1992, 1993, 1994, 1995, 1996, 1997, -1, -1, 2000, 2001, + 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, -1, -1, + 2961, -1, -1, -1, -1, -1, -1, 2968, 2969, -1, + -1, -1, -1, 881, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 3, -1, + 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 3001, -1, 3003, -1, -1, -1, -1, -1, 2789, -1, + -1, 1466, 1467, -1, 3015, 2983, -1, 1472, -1, -1, + 2983, 3022, -1, -1, -1, -1, 3027, -1, -1, -1, + -1, -1, 3033, -1, 512, -1, -1, -1, -1, -1, + -1, 519, 520, 521, 522, 523, 524, 3048, -1, -1, + -1, 2832, -1, 3054, 69, 70, -1, -1, -1, -1, + -1, -1, 2441, -1, -1, -1, 3067, 3068, 3069, -1, + -1, -1, -1, -1, 3075, -1, -1, -1, 2653, -1, + 1237, -1, -1, -1, -1, 1242, -1, -1, -1, -1, + -1, 3092, -1, -1, -1, 110, 111, -1, 1255, 114, + 115, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 2688, -1, -1, -1, 2898, 2497, 1276, + -1, -1, 1030, -1, -1, -1, -1, -1, 1036, 3097, + 1038, -1, -1, -1, 3097, -1, -1, -1, -1, 1047, + 2715, 2193, 2194, -1, -1, -1, -1, -1, -1, 1057, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 2909, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 2543, -1, -1, -1, -1, -1, -1, - -1, 1025, -1, 400, -1, -1, -1, 1031, -1, 1033, - -1, 2561, -1, -1, 101, -1, -1, -1, 1042, 2569, - 2570, 2571, 1685, -1, -1, -1, -1, -1, 1052, -1, - -1, -1, -1, 2583, -1, 2585, 2970, 2587, -1, -1, - -1, -1, 8, 2593, -1, 11, -1, -1, -1, 15, - 16, -1, -1, 3370, 20, 21, 22, -1, -1, -1, - -1, -1, 149, 2613, -1, -1, -1, -1, 3370, -1, - -1, 37, -1, 1097, -1, -1, -1, 164, 2628, -1, - -1, -1, 169, -1, -1, -1, -1, 174, 3652, -1, - 2640, -1, 1755, -1, 2644, -1, 183, -1, 1122, -1, - -1, 188, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 512, -1, -1, -1, 2429, - 2670, 2671, 519, 520, 521, 522, 523, 524, -1, 3055, - -1, 8, -1, 220, 11, -1, -1, -1, 15, 16, - -1, -1, -1, -1, 2694, -1, -1, -1, -1, 3083, - -1, 2701, -1, -1, 241, -1, -1, -1, 2708, -1, - -1, -1, -1, -1, -1, -1, -1, 1191, 1192, -1, - 47, -1, -1, -1, 2724, 2485, 808, 54, 2728, -1, - 2730, -1, 2732, 2733, 3118, -1, 2736, -1, -1, -1, - 2740, 2741, 2742, -1, 2744, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 81, 292, -1, -1, 295, -1, - -1, -1, -1, -1, 301, 2765, -1, 2767, -1, -1, + -1, -1, -1, -1, 3132, -1, -1, 2742, -1, 3132, + -1, -1, -1, 3174, -1, -1, 191, 192, 3179, -1, + -1, -1, -1, 2441, -1, -1, -1, -1, -1, 1346, + -1, -1, -1, 3194, 1102, -1, -1, 1354, -1, -1, + -1, -1, 2983, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1127, + 3221, -1, -1, -1, -1, 3226, -1, -1, -1, -1, + 3198, -1, -1, -1, 2613, 3198, 3204, -1, -1, 2497, + -1, 3204, -1, 258, 259, 260, 261, 262, 263, 264, + 265, 3252, -1, 268, 269, -1, 2831, 2309, 3259, 3227, + -1, -1, -1, 1718, 3227, 3266, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 2782, 2783, 2784, 2785, 2786, 2787, 2788, 2789, - 2790, 2791, -1, -1, -1, -1, -1, -1, -1, -1, - 3184, 1914, -1, -1, -1, 342, 3190, -1, 224, -1, - -1, -1, -1, -1, -1, -1, -1, 2817, -1, -1, - 147, -1, -1, 905, -1, 2825, -1, -1, -1, 3213, - 367, -1, -1, -1, -1, -1, 3212, -1, 2838, -1, - -1, 2601, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 179, -1, 8, -1, -1, 11, -1, -1, - 942, 15, 16, -1, 2864, -1, 20, 21, 22, 196, - -1, 2871, -1, 3257, 201, 957, 958, 959, 960, 961, - -1, 297, 2882, 37, -1, 3652, -1, -1, 2888, -1, - 427, -1, -1, -1, -1, 2895, 2896, 2897, 2898, -1, - 3652, -1, 1376, 1377, -1, 1379, 2666, -1, -1, 2909, - -1, -1, 2912, 240, -1, -1, 2916, 2917, -1, 456, - -1, -1, -1, -1, -1, 2925, -1, 1009, -1, -1, - 467, -1, -1, -1, -1, 2048, -1, -1, -1, -1, - -1, 2054, -1, -1, -1, -1, -1, -1, 2948, -1, - 487, -1, 489, 490, -1, 2955, 2956, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 8, -1, -1, 11, - 297, -1, -1, 15, 16, -1, -1, -1, 20, 21, - 22, -1, -1, -1, -1, -1, 3370, -1, 2988, 526, - 2990, -1, 529, 530, 531, -1, -1, -1, 8, -1, - -1, 11, 3002, -1, -1, 15, 16, -1, 1482, 3009, - 20, 21, 22, -1, 3014, -1, 2776, -1, -1, -1, - 3020, -1, -1, 1497, -1, 1499, -1, -1, 1502, -1, - -1, -1, -1, 1507, 3034, -1, 1510, -1, 1512, -1, - 3040, -1, 1516, -1, 1518, -1, 1520, -1, -1, -1, - -1, -1, -1, 3053, 3054, 3055, -1, -1, -1, 2819, - -1, 3061, -1, 3439, -1, -1, -1, -1, -1, -1, - 224, -1, -1, 400, -1, 2188, -1, -1, 3078, -1, + -1, -1, 3283, -1, -1, -1, 2861, -1, 1196, 1197, + 2865, -1, -1, -1, -1, -1, -1, -1, -1, 2678, + 2875, -1, -1, 3271, -1, 3306, -1, -1, 3271, -1, + 1765, -1, -1, 3314, -1, -1, 3097, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 343, 344, + -1, -1, -1, -1, 3335, -1, -1, -1, -1, 3340, + 1497, -1, -1, -1, -1, -1, -1, 2922, -1, -1, + -1, 3132, -1, -1, -1, 2613, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 3366, -1, -1, -1, 384, + 385, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 3387, 3388, -1, -1, + -1, -1, -1, -1, -1, 69, 70, -1, -1, -1, + -1, -1, -1, -1, 3405, -1, 3407, -1, -1, -1, + 2789, 3412, -1, -1, -1, -1, 3384, 3198, -1, -1, + 2678, 3384, -1, 3204, -1, -1, -1, -1, 3003, -1, + 3431, -1, -1, 3434, -1, -1, 110, 111, 3013, -1, + 114, 115, -1, -1, -1, -1, 3227, 3022, -1, -1, + -1, -1, 3453, 2832, 3455, -1, 3457, 3458, -1, 3460, + -1, -1, -1, -1, 3465, -1, -1, -1, -1, 484, + 485, -1, -1, -1, -1, 1383, 1384, -1, 1386, -1, + -1, -1, -1, -1, -1, 3486, -1, -1, -1, 3490, + 3271, -1, -1, 508, 509, -1, -1, -1, -1, -1, + 3501, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 191, 192, 2898, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 2208, -1, -1, -1, 515, - 516, 517, -1, 519, 520, 521, 522, 523, 524, -1, - -1, 2224, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 2235, -1, -1, 2885, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 2251, -1, - -1, -1, 2255, 297, -1, -1, -1, -1, -1, -1, - 1232, -1, -1, 1235, -1, -1, -1, -1, -1, -1, - 3160, -1, -1, -1, -1, 3165, 1248, -1, 1642, -1, + 3531, 2789, -1, -1, -1, -1, -1, 1992, 1993, 1994, + 1995, 1996, 1997, -1, -1, 2000, 2001, 2002, 2003, 2004, + 2005, 2006, 2007, 2008, 2009, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 3570, + -1, -1, -1, -1, 2832, -1, -1, 3578, -1, -1, + 1737, 1489, 1739, -1, 258, 259, 260, 261, 262, 263, + 264, 265, 3593, 1750, 268, 269, 1504, -1, 1506, -1, + -1, 1509, -1, 3384, 2983, -1, 1514, -1, -1, 1517, + -1, 1519, -1, -1, -1, 1523, -1, 1525, -1, 1527, + 3195, -1, 3623, -1, -1, -1, -1, 25, -1, -1, + 2682, 2683, -1, 1790, 3635, 3636, -1, 3638, 3639, -1, + 2898, -1, -1, -1, -1, -1, 3647, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3658, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3242, -1, 343, + 344, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 27, 3684, 82, -1, -1, -1, 33, -1, + 3691, -1, -1, -1, -1, -1, 41, -1, 3666, -1, + 98, -1, -1, 3666, -1, -1, -1, -1, -1, -1, + 384, 385, -1, -1, -1, -1, 61, -1, 3097, -1, + -1, -1, -1, -1, -1, 2983, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 3737, -1, 2193, 2194, + -1, 1649, -1, -1, 1901, 1902, 1903, 1904, 1905, 1906, + 148, 3752, -1, 3132, -1, -1, 3757, -1, -1, 3760, + 158, -1, 107, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 171, -1, -1, -1, -1, 176, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 3180, -1, 224, -1, -1, 512, -1, 1269, -1, -1, - -1, -1, 519, 520, 521, 522, 523, 524, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 3207, -1, -1, - 2970, -1, 3212, -1, 224, -1, -1, -1, -1, -1, + -1, -1, -1, 138, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 205, -1, -1, + 484, 485, -1, -1, -1, -1, -1, -1, -1, 3198, + -1, -1, 856, 857, -1, 3204, -1, -1, -1, 3404, + -1, -1, -1, -1, 508, 509, -1, -1, -1, 3097, + -1, -1, -1, 2895, -1, -1, -1, -1, 3227, -1, + -1, 249, -1, -1, 2309, 3430, 254, -1, -1, 204, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 3238, -1, - -1, -1, -1, -1, -1, 3245, -1, -1, -1, -1, - -1, -1, 3252, -1, -1, 297, -1, 1339, -1, -1, - -1, -1, -1, -1, -1, 1347, -1, -1, 3652, 3269, - -1, -1, -1, -1, -1, -1, 8, -1, 99, 11, - -1, -1, -1, 15, 16, 17, 18, 297, 20, 21, - 22, -1, 3292, -1, -1, -1, -1, -1, -1, -1, - 3300, -1, -1, -1, -1, 37, 127, 128, -1, -1, - -1, 1785, -1, -1, -1, 47, -1, -1, -1, -1, - -1, 3321, 54, 3083, -1, -1, 3326, -1, -1, 2442, - -1, 1805, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1816, -1, 1818, -1, -1, 1821, -1, 81, - -1, -1, 3352, 174, -1, -1, -1, 1831, 3118, 1833, - -1, 515, 516, 517, -1, 519, 520, 521, 522, 523, - 524, -1, 1846, 3373, 3374, -1, -1, 1851, -1, -1, - -1, 1855, 1856, -1, 1858, -1, 1860, 1861, -1, -1, - -1, 3391, -1, 3393, -1, -1, -1, -1, 3398, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1490, -1, - -1, -1, -1, -1, -1, -1, -1, 3417, -1, -1, - 3420, -1, -1, -1, 3184, -1, -1, -1, -1, -1, - 3190, -1, -1, -1, -1, -1, -1, -1, -1, 3439, - -1, 3441, -1, 3443, 3444, -1, 3446, 179, -1, -1, - -1, 3451, -1, 3213, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 196, -1, -1, -1, -1, 201, - -1, -1, 3472, 515, 516, 517, 3476, 519, 520, 521, - 522, 523, 524, -1, -1, -1, -1, 3487, -1, -1, - -1, -1, 224, 225, -1, -1, -1, 3257, -1, -1, - 321, 322, 323, -1, -1, 515, 516, 517, 240, 519, - 520, 521, 522, 523, 524, -1, -1, 3517, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 2641, -1, + 2922, -1, -1, -1, 3132, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1792, 3666, -1, 2044, -1, -1, + -1, -1, 3271, -1, -1, -1, -1, -1, 932, -1, + -1, -1, 936, 937, -1, -1, 1814, -1, -1, -1, + 255, -1, 3487, -1, 3489, -1, -1, 1825, -1, 1827, + 2972, 266, 1830, -1, -1, -1, -1, -1, 326, -1, + -1, -1, 1840, 278, 1842, -1, -1, -1, -1, -1, + 3198, -1, -1, 341, -1, -1, 3204, 1855, -1, -1, + -1, -1, 1860, -1, 299, 989, 1864, 1865, 3533, 1867, + -1, 1869, 1870, -1, -1, 310, -1, -1, -1, 3227, + -1, -1, -1, 1007, -1, 3550, -1, -1, 376, 1013, + -1, 379, 1016, -1, -1, 1019, 1020, 1021, 1022, -1, + -1, -1, 390, -1, -1, 393, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 3384, -1, -1, -1, -1, + -1, -1, -1, 3271, 359, 413, -1, 3069, 363, -1, + 365, -1, -1, -1, -1, -1, -1, -1, -1, 427, + 1064, 1065, -1, -1, -1, -1, -1, 435, -1, -1, + -1, -1, -1, -1, -1, 390, -1, 445, -1, -1, + 395, -1, 1086, 451, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 411, -1, -1, -1, + -1, -1, 1106, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 480, -1, 1118, 1119, 1120, -1, 1122, 1123, -1, 8, -1, -1, 11, -1, -1, -1, 15, 16, - 17, 18, -1, 20, 21, 22, -1, 2021, 280, -1, - -1, 283, -1, -1, -1, -1, 3556, -1, -1, -1, - 37, -1, -1, 2676, 3564, 297, -1, 388, 300, -1, - 47, -1, -1, -1, 2048, -1, -1, 54, 399, 3579, - 2054, -1, -1, -1, -1, -1, 2060, -1, -1, -1, - 2703, -1, 2066, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 423, -1, 81, -1, 427, -1, -1, 3609, - 3370, -1, -1, -1, -1, -1, -1, 2730, -1, -1, - -1, 3621, 3622, -1, 3624, 3625, -1, 448, -1, -1, - -1, -1, -1, 3633, -1, -1, -1, -1, -1, -1, - -1, 462, -1, -1, 3644, -1, -1, -1, 1730, -1, - 1732, -1, 473, -1, -1, -1, -1, -1, -1, -1, - -1, 1743, -1, -1, -1, -1, -1, -1, 400, 490, - 3670, -1, -1, -1, -1, 2149, -1, 3677, -1, -1, - 2154, -1, -1, -1, -1, 2159, -1, -1, -1, -1, - -1, -1, -1, 514, -1, -1, -1, -1, -1, -1, - -1, 1783, 179, -1, -1, 2818, -1, 528, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 196, - -1, -1, -1, 3723, 201, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 2848, -1, -1, 3738, 2852, - -1, -1, -1, 3743, -1, -1, 3746, 224, 225, 2862, + 17, 18, -1, 20, 21, 22, -1, -1, 3683, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 2235, -1, 240, -1, -1, 853, 854, -1, -1, - -1, -1, -1, 2247, -1, -1, -1, 2251, -1, -1, - 512, 2255, -1, 515, 516, 517, -1, 519, 520, 521, - 522, 523, 524, -1, -1, -1, 2909, -1, -1, -1, - 532, -1, -1, 280, -1, -1, 283, -1, -1, -1, - 1892, 1893, 1894, 1895, 1896, 1897, -1, -1, -1, -1, - 297, -1, 2296, 300, -1, -1, -1, -1, 2302, -1, - -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, - 927, -1, -1, -1, 931, 932, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 23, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 33, -1, 35, 36, - -1, -1, -1, -1, -1, -1, -1, 2990, -1, -1, - -1, 48, -1, -1, -1, -1, 53, 3000, -1, -1, - -1, -1, 3652, -1, -1, 62, 3009, 984, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 76, - -1, -1, -1, -1, -1, 1002, -1, -1, 85, -1, - 87, 1008, -1, 400, 1011, -1, -1, 1014, 1015, 1016, - 1017, -1, 99, -1, 101, -1, 8, -1, -1, 11, - -1, -1, -1, 15, 16, 112, -1, -1, 20, 21, - 22, -1, -1, 2035, -1, -1, 8, -1, -1, 11, - 127, 128, 129, 15, 16, 2439, -1, -1, 20, 21, - 22, 138, 1059, 1060, -1, 47, -1, 144, -1, -1, - -1, -1, 54, -1, -1, -1, 153, -1, 155, 156, - -1, -1, -1, -1, 1081, 47, -1, -1, -1, -1, - -1, -1, 54, 170, -1, -1, -1, 174, -1, 81, - -1, -1, -1, -1, 1101, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 1113, 1114, 1115, 81, - 1117, 1118, -1, 200, -1, 512, -1, -1, 515, 516, - 517, -1, 519, 520, 521, 522, 523, 524, -1, 216, - -1, -1, 529, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 1153, -1, 3181, -1, - -1, -1, -1, -1, -1, -1, 243, -1, -1, -1, - -1, -1, -1, -1, 1171, 1172, -1, -1, -1, -1, + 37, -1, 2030, -1, 1158, 470, 3384, -1, -1, -1, + 47, -1, -1, -1, 3709, -1, -1, 54, -1, 27, + -1, -1, 1176, 1177, -1, 33, -1, -1, -1, 2057, + -1, -1, -1, 41, -1, 2063, -1, -1, -1, -1, + -1, 2069, -1, -1, 81, -1, -1, 2075, -1, -1, + -1, -1, -1, 61, 3226, -1, -1, -1, -1, 1213, + -1, -1, -1, 1217, 1218, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1228, 1229, 3, -1, -1, -1, + -1, 8, -1, -1, 11, -1, -1, -1, 15, 16, + 17, 18, -1, 20, 21, 22, -1, -1, -1, 107, + -1, -1, -1, -1, -1, -1, -1, 2682, 2683, -1, + 37, -1, -1, -1, 41, -1, -1, -1, -1, -1, + 47, -1, -1, -1, -1, -1, -1, 54, -1, -1, + 138, 2159, -1, -1, -1, -1, 2164, -1, 1292, -1, + -1, 2169, 179, -1, -1, -1, -1, 2424, -1, 1303, + -1, -1, -1, -1, 81, -1, -1, -1, -1, 196, + -1, 2438, 2439, 2440, 201, -1, -1, 3666, -1, -1, + -1, -1, -1, -1, 1328, -1, 2453, -1, -1, -1, + 2457, -1, -1, 2460, -1, -1, -1, 224, 225, -1, + -1, -1, -1, -1, -1, -1, 204, -1, -1, -1, + -1, -1, -1, 240, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 2245, -1, -1, + -1, -1, -1, -1, -1, -1, 2503, -1, -1, 2257, + -1, -1, -1, 2261, -1, -1, -1, 2265, -1, -1, + -1, -1, -1, 280, -1, -1, 283, 255, -1, -1, + -1, -1, 179, -1, -1, -1, -1, -1, 266, -1, + 297, -1, 1416, 300, -1, -1, -1, -1, -1, 196, + 278, -1, -1, -1, 201, -1, -1, -1, 2306, -1, + -1, 3453, -1, -1, 2312, -1, -1, -1, 3666, -1, + -1, 299, -1, -1, -1, -1, -1, 224, 225, -1, + -1, -1, 310, -1, 2581, 2582, 2583, -1, -1, -1, + -1, -1, -1, 240, -1, -1, -1, -1, -1, -1, + 2895, -1, -1, -1, -1, 1479, -1, 1481, 1482, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 179, -1, 2583, - -1, -1, -1, 2587, -1, 3228, -1, -1, -1, 2593, - -1, 1208, -1, -1, 196, 1212, 1213, 179, -1, 201, - -1, -1, -1, -1, -1, -1, 1223, 1224, -1, -1, - -1, -1, -1, -1, 196, -1, -1, -1, -1, 201, - -1, -1, 224, 225, 321, 322, 323, -1, -1, -1, - -1, -1, 329, -1, -1, 332, 2640, -1, 240, -1, - 2644, -1, 224, 225, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 240, -1, - -1, -1, -1, -1, -1, -1, 363, -1, 1285, -1, - -1, -1, -1, -1, -1, 372, -1, -1, 280, 1296, - -1, 283, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 388, -1, -1, -1, 297, -1, -1, 395, -1, - -1, 283, 399, -1, 1321, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 411, 297, -1, -1, -1, -1, - -1, -1, -1, -1, 2728, -1, 423, -1, 2732, 2733, - 427, -1, 2736, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 3390, -1, -1, - -1, 448, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 2765, -1, -1, -1, 462, -1, -1, 465, -1, - -1, 468, -1, 3416, -1, -1, 473, -1, 2782, 2783, - 2784, 2785, -1, 2787, 2788, 2789, 2790, 2791, -1, -1, - -1, -1, 1409, 490, -1, -1, -1, -1, 400, -1, - 2412, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 2426, 2427, 2428, 514, 400, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 2441, - 3473, 528, 3475, 2445, 531, -1, 2448, -1, -1, -1, + 1494, 1495, -1, -1, -1, -1, -1, 2922, -1, -1, + -1, 359, -1, 280, -1, 363, 283, 365, -1, -1, + -1, -1, 1516, 400, -1, -1, -1, -1, -1, -1, + 297, -1, -1, 300, -1, -1, -1, -1, -1, -1, + -1, -1, 390, -1, -1, -1, -1, 395, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1472, -1, 1474, 1475, -1, + -1, -1, -1, 411, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1487, 1488, -1, -1, -1, -1, 3519, -1, -1, 2491, - -1, -1, -1, -1, 2888, -1, -1, -1, -1, -1, - -1, -1, 1509, 3536, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 2451, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 512, -1, 2916, 515, 516, 517, -1, 519, 520, 521, - 522, 523, 524, -1, -1, 3, -1, 5, -1, -1, - 512, -1, -1, 515, 516, 517, -1, 519, 520, 521, - 522, 523, 524, -1, 2948, -1, -1, -1, -1, -1, - -1, -1, 2956, -1, -1, -1, 3, 2569, 2570, 2571, - -1, 8, -1, -1, 11, -1, -1, -1, 15, 16, - 17, 18, -1, 20, 21, 22, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 2990, -1, -1, -1, - 37, 69, 70, -1, 41, -1, -1, 1614, -1, -1, - 47, -1, -1, -1, -1, -1, 1623, 54, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1637, -1, -1, -1, -1, -1, 3669, -1, -1, -1, - -1, -1, 110, 111, 81, -1, 114, 115, -1, -1, - 1657, -1, -1, -1, -1, -1, -1, -1, 1665, -1, - 3054, -1, 3695, -1, 1671, 1672, 1673, 1674, 1675, 1676, - 1677, 1678, -1, -1, -1, -1, 1683, 1684, -1, -1, - -1, 1688, -1, -1, -1, 1692, -1, -1, 1695, 1696, - 1697, 1698, 1699, 1700, 1701, 1702, 1703, -1, -1, 1706, - -1, -1, -1, -1, -1, -1, 1713, -1, 1715, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 2736, + -1, -1, 470, -1, -1, -1, -1, 1621, -1, -1, + -1, -1, -1, 400, -1, 512, 1630, -1, 515, 516, + 517, -1, 519, 520, 521, 522, 523, 524, -1, -1, + 1644, -1, -1, -1, 3069, 532, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 2724, 191, 192, -1, 1733, -1, -1, -1, + 1664, -1, -1, -1, -1, -1, -1, -1, 1672, -1, + -1, -1, -1, -1, 1678, 1679, 1680, 1681, 1682, 1683, + 1684, 1685, -1, -1, -1, -1, 1690, 1691, -1, -1, + -1, 1695, -1, -1, -1, 1699, -1, -1, 1702, 1703, + 1704, 1705, 1706, 1707, 1708, 1709, 1710, -1, -1, 1713, + -1, 2838, -1, -1, -1, -1, 1720, 2595, 1722, -1, + -1, 2599, -1, -1, -1, -1, -1, 2605, -1, -1, + -1, -1, -1, -1, -1, 512, 1740, -1, 515, 516, + 517, -1, 519, 520, 521, 522, 523, 524, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 8, -1, 179, 11, -1, -1, -1, 15, 16, 17, - 18, -1, 20, 21, 22, -1, -1, -1, -1, 196, - -1, -1, 1769, 1770, 201, -1, -1, -1, -1, 37, - -1, -1, -1, 41, -1, -1, -1, -1, -1, 47, - -1, -1, -1, -1, -1, -1, 54, 224, 225, -1, - 258, 259, 260, 261, 262, 263, 264, 265, -1, -1, - 268, 269, -1, 240, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 81, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 2825, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 280, -1, -1, 283, -1, -1, -1, - -1, 3245, -1, -1, -1, -1, -1, -1, 3252, -1, - 297, 1868, -1, 300, -1, -1, -1, -1, 1875, -1, - -1, 1878, 1879, -1, -1, 343, 344, -1, -1, -1, + -1, -1, 1776, 1777, 2652, -1, -1, -1, 2656, -1, + -1, 2908, 2909, 2910, 2911, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 2895, 2896, 2897, 2898, -1, -1, -1, + -1, 3226, -1, -1, -1, 8, -1, -1, 11, -1, + -1, -1, 15, 16, 17, 18, -1, 20, 21, 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 179, -1, -1, -1, -1, 384, 385, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 3321, 196, -1, - -1, -1, 3326, 201, -1, -1, -1, -1, 1945, 1946, - -1, -1, -1, -1, -1, -1, -1, 1954, -1, -1, - -1, -1, -1, -1, -1, -1, 224, 225, 3352, -1, - -1, -1, -1, 400, -1, -1, -1, 1974, 1975, -1, - 1977, -1, 240, -1, -1, -1, -1, -1, -1, 3373, - 3374, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 2006, - 2007, -1, -1, 2010, 3398, -1, -1, -1, -1, -1, - -1, -1, 280, -1, -1, 283, 484, 485, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 297, - -1, -1, 300, -1, 2041, -1, -1, -1, -1, -1, - 508, 509, -1, 2050, -1, -1, -1, -1, -1, 3443, - -1, -1, -1, -1, -1, -1, -1, 3451, -1, 3061, - -1, -1, 2069, -1, 2071, -1, 25, -1, -1, -1, - -1, -1, -1, -1, -1, 512, -1, -1, 515, 516, - 517, -1, 519, 520, 521, 522, 523, 524, -1, -1, + -1, -1, -1, -1, 37, -1, -1, -1, 41, -1, + -1, -1, -1, -1, 47, -1, -1, -1, -1, -1, + -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 2740, -1, -1, -1, 2744, 2745, -1, -1, + -1, 2749, -1, 1877, -1, -1, -1, -1, 81, -1, + 1884, -1, -1, 1887, 1888, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 2109, -1, -1, -1, -1, -1, 2115, -1, - -1, -1, -1, -1, -1, 27, -1, -1, -1, -1, - -1, 33, -1, 82, -1, -1, -1, -1, -1, 41, - -1, -1, 400, -1, -1, -1, -1, -1, -1, 98, - -1, 2148, -1, -1, -1, 2152, -1, -1, -1, 61, - 2157, 2158, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 8, - -1, -1, 11, -1, -1, -1, 15, 16, 17, 18, - -1, 20, 21, 22, -1, -1, -1, -1, -1, 148, - -1, -1, -1, -1, -1, 107, -1, 2204, 37, 158, - 2207, -1, 2209, -1, -1, -1, -1, -1, 47, -1, - -1, -1, 171, -1, -1, 54, -1, 176, 2225, -1, - -1, -1, -1, -1, -1, -1, 138, -1, -1, -1, + 2778, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 2795, 2796, 2797, + 2798, -1, 2800, 2801, 2802, 2803, 2804, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 81, -1, 512, -1, 205, 515, 516, 517, - 3644, 519, 520, 521, 522, 523, 524, 8, 2265, -1, - 11, 529, -1, -1, 15, 16, 17, 18, -1, 20, - 21, 22, -1, -1, -1, 2282, 2283, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 37, -1, -1, -1, - 249, -1, 204, -1, 2301, 254, 47, -1, -1, -1, - -1, -1, -1, 54, -1, -1, -1, 2314, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 3075, -1, + 1954, 1955, -1, -1, -1, -1, -1, -1, -1, 1963, 8, -1, -1, 11, -1, -1, -1, 15, 16, 17, - 18, -1, 20, 21, 22, -1, -1, -1, -1, -1, - 81, -1, -1, -1, -1, -1, -1, -1, -1, 37, - 179, -1, -1, 255, -1, -1, -1, -1, -1, 47, - -1, -1, -1, -1, 266, -1, 54, 196, -1, -1, - -1, 320, 201, -1, -1, -1, 278, 326, -1, -1, - -1, -1, -1, -1, -1, 2382, -1, -1, -1, -1, - -1, -1, 341, 81, 2391, 224, 225, 299, -1, 3391, - -1, -1, -1, -1, -1, -1, -1, -1, 310, -1, - -1, 240, -1, -1, 1012, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 3417, -1, 376, -1, -1, - 379, -1, -1, -1, -1, -1, -1, -1, 179, -1, - -1, 390, -1, -1, 393, -1, -1, -1, -1, -1, - -1, 280, -1, -1, 283, 196, -1, 359, -1, -1, - 201, 363, -1, 365, 413, -1, -1, -1, 297, -1, - -1, 300, -1, -1, -1, -1, -1, -1, 427, -1, - -1, -1, -1, 224, 225, 434, 435, -1, 390, -1, - -1, 179, 2489, 395, -1, 3487, 445, -1, -1, 240, - -1, -1, 451, -1, -1, -1, -1, -1, 196, 411, - -1, -1, -1, 201, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 3517, -1, -1, -1, -1, - -1, 480, -1, -1, -1, -1, 224, 225, -1, 280, - 1138, -1, 283, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 240, -1, -1, -1, 297, -1, -1, 300, - -1, -1, -1, -1, -1, -1, -1, -1, 470, -1, - -1, 400, -1, -1, 8, -1, -1, 11, -1, -1, - -1, 15, 16, 17, 18, -1, 20, 21, 22, 1187, - -1, -1, 280, -1, -1, 283, -1, -1, -1, -1, - -1, -1, -1, 37, -1, -1, -1, -1, -1, 297, - -1, -1, 300, 47, -1, -1, -1, -1, -1, -1, - 54, 2618, -1, -1, 1222, -1, -1, -1, 2625, 2626, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 2641, -1, -1, 81, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 2654, -1, 400, - 2657, -1, 2659, -1, -1, -1, -1, -1, -1, -1, - 2667, -1, -1, -1, -1, -1, -1, -1, 2675, 2676, - -1, 1279, -1, 512, -1, 2682, 515, 516, 517, -1, - 519, 520, 521, 522, 523, 524, -1, -1, -1, -1, - 529, -1, 2699, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 400, 2710, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 1325, -1, -1, - -1, -1, -1, 2730, 1332, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 179, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 196, -1, -1, -1, -1, 201, -1, -1, - -1, 512, -1, -1, 515, 516, 517, -1, 519, 520, - 521, 522, 523, 524, 2781, 1383, -1, -1, 529, -1, - 224, 225, -1, -1, -1, -1, -1, -1, 2795, 2796, - -1, -1, -1, -1, -1, -1, 240, -1, -1, 1407, - -1, 2808, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 512, -1, -1, 515, 516, 517, - 2827, 519, 520, 521, 522, 523, 524, -1, -1, -1, - -1, 529, 25, -1, -1, -1, 280, -1, -1, 283, - -1, -1, -1, -1, -1, -1, -1, 1455, -1, 1457, - -1, 1459, 1460, 297, 1462, -1, 300, 1465, -1, -1, - 1468, -1, -1, 1471, -1, -1, -1, -1, 1476, -1, - -1, 1479, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 27, -1, -1, 82, - -1, -1, 33, -1, -1, 2902, 2903, -1, 2905, 8, - 41, -1, 11, -1, -1, 98, 15, 16, 17, 18, - -1, 20, 21, 22, -1, 1523, -1, -1, -1, -1, - 61, -1, -1, -1, -1, -1, -1, -1, 37, -1, - -1, -1, 2939, -1, -1, -1, -1, -1, 47, -1, - -1, -1, -1, -1, -1, 54, -1, -1, -1, -1, - -1, -1, 2959, -1, -1, 148, 400, 2964, 2965, -1, - -1, -1, 2969, -1, -1, 158, 107, 2974, -1, -1, - 2977, 2978, 81, -1, -1, 2982, 2983, -1, 171, 2986, - -1, -1, -1, 176, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 3002, -1, 138, -1, -1, - -1, -1, 1610, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 205, 8, -1, -1, 11, -1, -1, 1627, - 15, 16, 17, 18, -1, 20, 21, 22, -1, -1, - 1638, 1639, 1640, -1, -1, -1, -1, 1645, -1, -1, - 3047, 1649, 37, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 47, -1, -1, -1, 249, -1, -1, 54, - -1, 254, -1, 204, -1, -1, -1, 3074, 512, -1, - 179, 515, 516, 517, -1, 519, 520, 521, 522, 523, - 524, -1, -1, -1, -1, 529, 81, 196, -1, -1, - -1, -1, 201, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1712, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 255, 224, 225, -1, -1, -1, - -1, -1, -1, -1, -1, 266, -1, -1, -1, -1, - -1, 240, -1, 326, -1, -1, -1, 278, -1, -1, - -1, -1, -1, -1, 1752, -1, -1, -1, 341, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 299, -1, - 1768, -1, -1, -1, -1, 1773, -1, -1, -1, 310, - -1, 280, -1, -1, 283, -1, -1, -1, -1, -1, - -1, -1, 1790, 376, 179, -1, 379, -1, 297, -1, - -1, 300, -1, 3200, 3201, -1, 3203, 390, -1, -1, - 393, 196, -1, -1, -1, -1, 201, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 359, -1, - 413, 3228, 363, -1, 365, -1, -1, -1, -1, 224, - 225, -1, -1, -1, 427, -1, -1, -1, -1, -1, - -1, -1, 435, -1, -1, 240, -1, -1, -1, 390, - -1, -1, 445, -1, 395, -1, 3263, -1, 451, -1, + 18, -1, 20, 21, 22, -1, 179, -1, -1, 1983, + 1984, -1, 1986, -1, -1, -1, -1, -1, -1, 37, + -1, -1, -1, 196, -1, -1, -1, -1, 201, 47, + -1, -1, -1, -1, -1, -1, 54, -1, -1, -1, + -1, 2015, 2016, -1, -1, 2019, -1, 0, -1, -1, + -1, 224, 225, 2901, -1, -1, -1, -1, 3453, -1, + -1, -1, -1, 81, -1, -1, -1, 240, -1, -1, + 23, -1, -1, -1, -1, -1, 2050, -1, -1, -1, + 33, 2929, 35, 36, -1, 2059, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 48, -1, -1, -1, -1, + 53, -1, -1, -1, 2078, -1, 2080, 280, -1, 62, + 283, -1, -1, 2961, -1, -1, -1, -1, -1, -1, + -1, 2969, -1, 76, 297, -1, -1, 300, -1, -1, + -1, -1, 85, -1, 87, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 2119, 99, -1, 101, -1, + -1, 2125, -1, -1, -1, 3003, -1, -1, -1, 112, + -1, 179, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 127, 128, 129, -1, 196, -1, + -1, -1, -1, 201, 2158, 138, -1, -1, 2162, -1, + -1, 144, -1, 2167, 2168, -1, -1, -1, -1, -1, + 153, -1, 155, 156, -1, -1, 224, 225, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 170, -1, -1, + 3068, 174, 240, -1, -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 411, -1, 3279, 3280, -1, -1, 3283, -1, 3285, -1, - -1, -1, -1, -1, -1, 280, -1, 480, 283, -1, - -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 297, -1, -1, 300, 69, 70, -1, -1, - 3317, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 470, - -1, -1, -1, -1, 3341, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 110, 111, -1, - -1, 114, 115, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1972, -1, -1, -1, -1, -1, - -1, -1, 1980, 1981, -1, 1983, 1984, 1985, 1986, 1987, - 1988, -1, -1, 1991, 1992, 1993, 1994, 1995, 1996, 1997, - 1998, 1999, 2000, 2001, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 512, -1, 400, 515, 516, 517, -1, - 519, 520, 521, 522, 523, 524, -1, -1, -1, -1, - 529, -1, -1, -1, -1, -1, 3433, -1, 191, 192, - 3437, -1, -1, -1, 3441, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 3455, -1, - -1, -1, -1, 3460, 3461, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 8, 3472, -1, 11, -1, -1, - -1, 15, 16, 17, 18, -1, 20, 21, 22, -1, - -1, -1, -1, -1, -1, -1, -1, 3494, -1, -1, - -1, -1, -1, 37, -1, 258, 259, 260, 261, 262, - 263, 264, 265, 47, 2112, 268, 269, -1, 2116, -1, - 54, -1, -1, -1, -1, -1, -1, 512, -1, -1, - 515, 516, 517, -1, 519, 520, 521, 522, 523, 524, - -1, -1, -1, -1, 529, 3542, -1, 81, -1, 2147, - -1, -1, -1, -1, -1, 3552, -1, -1, -1, -1, - -1, -1, -1, 2161, -1, -1, -1, 3564, 2166, -1, - -1, -1, 2170, 2171, 2172, 2173, 2174, 2175, 2176, 2177, - -1, -1, -1, -1, 3581, 2183, 2184, -1, 2186, 2187, - 343, 344, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 2200, 3600, -1, 2203, -1, -1, -1, -1, - -1, -1, -1, 2211, 2212, 2213, 2214, 2215, 2216, 2217, - 2218, 2219, 2220, -1, -1, -1, -1, -1, -1, -1, - -1, 384, 385, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 179, -1, -1, 2246, -1, + 2214, -1, -1, 2217, -1, 2219, -1, 200, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 196, -1, -1, -1, -1, 201, -1, -1, - -1, -1, -1, -1, -1, 8, -1, -1, 11, -1, - -1, -1, 15, 16, 17, 18, -1, 20, 21, 22, - 224, 225, -1, -1, 3691, 3692, 3693, -1, -1, -1, - -1, 2299, -1, -1, 37, 3702, 240, -1, -1, -1, - -1, -1, -1, -1, 47, -1, -1, -1, -1, -1, - -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, - 3727, 484, 485, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 280, -1, 81, 283, - -1, -1, -1, -1, -1, 508, 509, -1, -1, -1, - -1, -1, -1, 297, -1, -1, 300, -1, -1, -1, - -1, -1, -1, -1, -1, 528, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 2383, -1, -1, -1, 3786, - -1, -1, -1, -1, -1, -1, 2394, 2395, -1, -1, + -1, 2235, 280, 216, -1, 283, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 297, + -1, -1, 300, -1, -1, -1, -1, -1, -1, -1, + 243, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 2275, -1, -1, -1, -1, -1, -1, 3405, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 2292, 2293, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 3431, -1, -1, 2311, -1, 512, + -1, -1, 515, 516, 517, -1, 519, 520, 521, 522, + 523, 524, 2326, -1, -1, -1, 529, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 321, 322, + 323, -1, -1, -1, -1, -1, 329, -1, -1, 332, + -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 3501, -1, -1, -1, -1, 8, + 363, 3259, 11, -1, -1, -1, 15, 16, 3266, 372, + 2394, 20, 21, 22, -1, -1, -1, -1, -1, 2403, + -1, -1, -1, -1, 3531, 388, -1, -1, 37, -1, + -1, -1, 395, 1017, 69, 70, 399, -1, 47, -1, + -1, -1, -1, -1, -1, 54, -1, -1, 411, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 423, -1, -1, -1, 427, -1, -1, -1, -1, -1, + -1, -1, 81, -1, -1, 110, 111, 3335, -1, 114, + 115, -1, 3340, -1, 512, 448, -1, 515, 516, 517, + -1, 519, 520, 521, 522, 523, 524, -1, -1, 462, + -1, 529, 465, -1, -1, 468, -1, -1, 3366, -1, + 473, -1, -1, -1, -1, -1, -1, 2501, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 490, -1, 3387, + 3388, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 179, -1, -1, -1, + -1, 514, -1, -1, 3412, -1, 191, 192, -1, 1143, + -1, -1, -1, -1, -1, 528, -1, -1, 531, -1, + 179, 8, -1, -1, 11, -1, -1, -1, 15, 16, + 17, 18, -1, 20, 21, 22, -1, 196, -1, -1, + -1, -1, 201, -1, -1, -1, -1, -1, -1, 3457, + 37, -1, -1, -1, -1, -1, -1, 3465, 1192, -1, + 47, -1, -1, -1, -1, 224, 225, 54, -1, -1, + -1, -1, -1, 258, 259, 260, 261, 262, 263, 264, + 265, 240, -1, 268, 269, -1, -1, -1, -1, -1, + -1, -1, -1, 1227, 81, -1, 2630, -1, -1, -1, + -1, -1, 8, 2637, 2638, 11, -1, -1, -1, 15, + 16, 17, 18, -1, 20, 21, 22, -1, -1, 2653, + -1, 280, -1, -1, 283, -1, -1, -1, -1, -1, + -1, 37, 2666, -1, -1, 2669, -1, 2671, 297, -1, + -1, 47, -1, -1, -1, 2679, -1, -1, 54, -1, + -1, -1, 1286, 2687, 2688, -1, -1, -1, 343, 344, + 2694, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 81, -1, 2711, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 2722, -1, + -1, -1, 179, -1, -1, -1, -1, -1, 1332, 384, + 385, -1, -1, -1, -1, 1339, -1, -1, 2742, 196, + -1, -1, -1, -1, 201, 8, -1, -1, 11, -1, + -1, -1, 15, 16, 17, 18, -1, 20, 21, 22, + -1, -1, -1, -1, -1, -1, -1, 224, 225, -1, + -1, 400, -1, -1, 37, -1, -1, -1, -1, -1, + 3658, -1, -1, 240, 47, -1, 1390, -1, -1, -1, + 2794, 54, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 179, 2808, 2809, -1, -1, -1, -1, + 1414, -1, -1, -1, -1, -1, -1, 2821, 81, -1, + 196, -1, -1, 280, -1, 201, 283, -1, -1, 484, + 485, -1, -1, -1, -1, -1, 2840, -1, -1, -1, + 297, -1, -1, 300, -1, -1, -1, -1, 224, 225, + -1, -1, -1, 508, 509, -1, -1, -1, 1462, -1, + 1464, -1, 1466, 1467, 240, 1469, -1, -1, 1472, -1, + -1, 1475, -1, 528, 1478, -1, -1, -1, -1, 1483, + -1, -1, 1486, 512, -1, -1, 515, 516, 517, -1, + 519, 520, 521, 522, 523, 524, -1, -1, -1, -1, + -1, -1, -1, -1, 280, -1, -1, 283, -1, -1, + -1, 2915, 2916, -1, 2918, -1, 179, -1, -1, -1, + -1, 297, -1, -1, 300, -1, 1530, -1, -1, -1, + -1, -1, -1, 196, -1, -1, -1, -1, 201, -1, + -1, -1, -1, 400, -1, -1, -1, -1, 2952, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 196, -1, -1, 400, -1, 201, -1, + -1, 224, 225, -1, -1, -1, -1, -1, 2972, -1, + -1, -1, -1, 2977, 2978, -1, -1, 240, 2982, -1, + -1, -1, -1, 2987, -1, -1, 2990, 2991, -1, -1, + -1, 2995, 2996, -1, -1, 2999, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 3015, -1, 1617, -1, -1, -1, 280, -1, -1, + 283, -1, -1, -1, 400, -1, -1, -1, -1, -1, + 1634, -1, -1, -1, 297, -1, -1, 300, -1, -1, + -1, 1645, 1646, 1647, -1, -1, -1, -1, 1652, -1, + -1, -1, 1656, -1, -1, 512, -1, 3061, 515, 516, + 517, -1, 519, 520, 521, 522, 523, 524, -1, -1, + -1, -1, 529, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 3088, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 224, 225, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 2500, -1, 2502, -1, -1, 240, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 2518, 2519, 2520, -1, -1, 2523, 2524, 2525, 2526, 2527, - 2528, -1, -1, -1, 2532, 2533, 2534, 2535, 2536, 2537, - 2538, 2539, 2540, 2541, -1, -1, -1, 280, 2546, 2547, - 283, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 297, -1, -1, 300, -1, -1, - -1, -1, -1, -1, 2572, -1, -1, -1, 512, -1, - -1, 515, 516, 517, 2582, 519, 520, 521, 522, 523, - 524, -1, -1, -1, -1, 529, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 1719, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 2617, - -1, -1, 8, -1, -1, 11, 2624, -1, -1, 15, - 16, 17, 18, -1, 20, 21, 22, -1, 2636, -1, - -1, -1, -1, -1, 2642, -1, -1, -1, -1, 2647, - 2648, 37, -1, -1, -1, -1, -1, 2655, 2656, -1, - -1, 47, -1, -1, -1, -1, -1, 400, 54, -1, - -1, 2669, 2670, 2671, 2672, 8, 2674, -1, 11, -1, - 2678, -1, 15, 16, 17, 18, -1, 20, 21, 22, - -1, -1, -1, -1, -1, 81, -1, -1, -1, -1, - -1, -1, -1, -1, 37, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 47, -1, -1, -1, -1, -1, - -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 69, 70, -1, -1, 2737, - -1, -1, -1, -1, -1, -1, -1, -1, 81, -1, + -1, -1, -1, -1, -1, -1, 512, 400, -1, 515, + 516, 517, -1, 519, 520, 521, 522, 523, 524, 8, + -1, -1, 11, 529, -1, 1759, 15, 16, 17, 18, + -1, 20, 21, 22, -1, -1, -1, -1, -1, -1, + -1, 1775, -1, -1, -1, -1, 1780, -1, 37, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 47, -1, + -1, -1, -1, 1797, -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 3214, 3215, -1, 3217, -1, -1, -1, -1, -1, -1, + -1, -1, 81, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 3242, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 512, + -1, -1, 515, 516, 517, -1, 519, 520, 521, 522, + 523, 524, 8, -1, -1, 11, 529, -1, -1, 15, + 16, 17, 18, 3277, 20, 21, 22, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 3293, + 3294, 37, -1, 3297, -1, 3299, -1, -1, -1, -1, + -1, 47, -1, -1, -1, -1, -1, -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 110, 111, -1, 512, - 114, 115, 515, 516, 517, -1, 519, 520, 521, 522, - 523, 524, -1, 179, -1, -1, 529, -1, -1, -1, + 179, -1, -1, -1, -1, -1, -1, 3331, -1, -1, + -1, -1, -1, -1, -1, 81, -1, 196, -1, -1, + -1, -1, 201, -1, -1, -1, -1, -1, -1, -1, + -1, 3355, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 224, 225, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 1981, -1, -1, + -1, 240, -1, -1, -1, 1989, 1990, -1, 1992, 1993, + 1994, 1995, 1996, 1997, -1, -1, 2000, 2001, 2002, 2003, + 2004, 2005, 2006, 2007, 2008, 2009, 2010, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 196, -1, -1, 8, -1, 201, 11, -1, -1, -1, - 15, 16, 17, 18, -1, 20, 21, 22, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 224, 225, - -1, -1, 37, -1, -1, -1, 179, -1, -1, -1, - -1, -1, 47, -1, 240, -1, -1, 191, 192, 54, - -1, -1, -1, 196, -1, 2863, -1, 8, 201, -1, - 11, -1, -1, -1, 15, 16, 17, 18, -1, 20, - 21, 22, -1, -1, 2882, -1, 81, -1, -1, -1, - -1, 224, 225, -1, 280, -1, 37, 283, -1, -1, - -1, -1, -1, -1, -1, -1, 47, 240, -1, -1, - -1, 297, -1, 54, 300, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 258, 259, 260, 261, 262, 263, - 264, 265, -1, -1, 268, 269, -1, -1, -1, -1, - 81, -1, 2940, -1, 2942, -1, -1, 280, -1, -1, - 283, -1, -1, 2951, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 297, 2963, -1, 300, 2966, -1, - 2968, -1, -1, -1, 2972, -1, -1, 2975, 2976, -1, - -1, 2979, 2980, -1, 179, -1, -1, -1, -1, 2987, + -1, 280, -1, -1, 283, -1, -1, -1, -1, -1, + -1, -1, -1, 179, -1, -1, -1, -1, 297, -1, + -1, 300, -1, 3447, -1, -1, -1, 3451, -1, -1, + 196, 3455, -1, -1, -1, 201, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 3469, -1, -1, -1, -1, + 3474, 3475, -1, -1, -1, -1, -1, -1, 224, 225, + -1, -1, 3486, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 240, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 3508, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 2122, -1, + -1, -1, 2126, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 280, -1, -1, 283, -1, -1, + -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 297, 3556, 2157, 300, -1, -1, -1, -1, -1, + -1, -1, 3566, -1, -1, -1, -1, 2171, -1, -1, + -1, -1, 2176, -1, 3578, -1, 2180, 2181, 2182, 2183, + 2184, 2185, 2186, 2187, -1, -1, -1, -1, -1, 2193, + 2194, 3595, 2196, 2197, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 2210, -1, -1, 2213, + 3614, -1, -1, -1, -1, -1, -1, 2221, 2222, 2223, + 2224, 2225, 2226, 2227, 2228, 2229, 2230, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 196, -1, 3001, -1, -1, 201, -1, -1, 343, - 344, -1, -1, -1, 400, -1, -1, -1, -1, 3017, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 224, - 225, -1, -1, -1, -1, -1, -1, -1, 179, -1, - -1, -1, -1, -1, -1, 240, -1, -1, -1, -1, - 384, 385, -1, -1, -1, 196, -1, 3055, -1, -1, - 201, -1, -1, -1, -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 224, 225, 280, -1, -1, 283, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 240, - -1, -1, 297, -1, -1, 300, -1, -1, -1, -1, + -1, -1, 2256, 512, 400, -1, 515, 516, 517, -1, + 519, 520, 521, 522, 523, 524, -1, -1, -1, -1, + 529, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 8, -1, -1, 11, -1, -1, -1, 15, 16, + 17, 18, -1, 20, 21, 22, -1, -1, -1, -1, + -1, 3705, 3706, 3707, -1, 2309, -1, -1, -1, -1, + 37, -1, 3716, -1, -1, -1, -1, -1, -1, -1, + 47, -1, -1, -1, -1, -1, -1, 54, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3741, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 81, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 512, -1, -1, 515, - 516, 517, -1, 519, 520, 521, 522, 523, 524, 280, - -1, -1, 283, 529, -1, -1, -1, -1, -1, -1, - 484, 485, -1, -1, -1, -1, 297, -1, -1, 300, - -1, -1, -1, 3161, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 508, 509, -1, 3175, -1, 512, - -1, -1, 515, 516, 517, -1, 519, 520, 521, 522, - 523, 524, -1, -1, -1, -1, 529, -1, -1, -1, - -1, -1, -1, -1, -1, 400, -1, -1, -1, -1, - -1, -1, -1, 3211, 3212, -1, -1, -1, -1, -1, + 516, 517, -1, 519, 520, 521, 522, 523, 524, -1, + -1, -1, -1, 529, -1, -1, -1, -1, -1, -1, + -1, 2395, -1, -1, -1, -1, 3800, -1, -1, -1, + -1, -1, 2406, 2407, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 8, -1, -1, + 11, -1, -1, -1, 15, 16, 17, 18, -1, 20, + 21, 22, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 37, -1, -1, -1, + -1, -1, 179, -1, -1, -1, 47, -1, -1, -1, + -1, -1, -1, 54, -1, -1, -1, -1, -1, 196, + -1, -1, -1, -1, 201, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 3236, 3237, - -1, -1, 3240, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 400, - -1, -1, -1, 3261, -1, -1, -1, -1, -1, -1, - -1, -1, 3270, -1, -1, 3273, 3274, 3275, -1, -1, - 3278, -1, -1, 3281, 3282, -1, -1, -1, -1, -1, - -1, -1, 3290, -1, -1, -1, -1, -1, -1, -1, + 81, -1, -1, -1, -1, -1, -1, 224, 225, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 2512, -1, + 2514, -1, -1, 240, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 2530, 2531, 2532, -1, + -1, 2535, 2536, 2537, 2538, 2539, 2540, -1, -1, -1, + 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, + -1, -1, -1, 280, 2558, 2559, 283, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 512, -1, -1, - 515, 516, 517, -1, 519, 520, 521, 522, 523, 524, - -1, -1, -1, -1, 529, -1, -1, -1, -1, -1, - 3338, -1, -1, -1, -1, -1, 3344, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 3357, + 297, -1, -1, 300, -1, -1, -1, -1, -1, -1, + 2584, -1, -1, -1, 8, -1, -1, 11, 179, -1, + 2594, 15, 16, 17, 18, -1, 20, 21, 22, -1, + -1, -1, -1, -1, -1, 196, -1, -1, -1, -1, + 201, -1, -1, 37, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 47, -1, 2629, -1, -1, -1, -1, + 54, -1, 2636, 224, 225, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 2648, -1, -1, -1, -1, 240, + 2654, -1, -1, -1, -1, 2659, 2660, 81, -1, -1, + -1, -1, -1, 2667, 2668, -1, -1, -1, -1, -1, + -1, -1, -1, 400, -1, -1, -1, 2681, 2682, 2683, + 2684, -1, 2686, -1, -1, -1, 2690, -1, -1, 280, + 8, -1, 283, 11, -1, -1, -1, 15, 16, 17, + 18, -1, 20, 21, 22, -1, 297, -1, -1, 300, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 37, + -1, -1, -1, -1, -1, 8, -1, -1, 11, 47, + -1, -1, 15, 16, 17, 18, 54, 20, 21, 22, + -1, -1, -1, -1, -1, -1, 2750, -1, -1, -1, + -1, -1, -1, -1, 37, 179, -1, -1, -1, -1, + -1, -1, -1, 81, 47, -1, -1, -1, -1, -1, + -1, 54, 196, -1, -1, -1, -1, 201, -1, -1, + -1, -1, -1, -1, -1, 512, -1, -1, 515, 516, + 517, -1, 519, 520, 521, 522, 523, 524, 81, -1, + 224, 225, 529, -1, -1, -1, -1, -1, -1, 400, + -1, -1, -1, -1, -1, -1, 240, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 512, -1, -1, 515, 516, 517, -1, 519, 520, + -1, 8, -1, -1, 11, -1, -1, -1, 15, 16, + 17, 18, -1, 20, 21, 22, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 280, -1, -1, 283, + 37, 179, -1, -1, -1, -1, -1, -1, -1, -1, + 47, -1, 2876, 297, -1, -1, 300, 54, 196, -1, + -1, -1, -1, 201, -1, -1, -1, -1, -1, -1, + -1, 2895, -1, -1, -1, -1, 179, -1, -1, -1, + -1, -1, -1, -1, 81, -1, 224, 225, -1, -1, + -1, -1, -1, 196, -1, -1, -1, -1, 201, -1, + -1, 512, 240, -1, 515, 516, 517, -1, 519, 520, 521, 522, 523, 524, -1, -1, -1, -1, 529, -1, + -1, 224, 225, -1, -1, -1, -1, -1, -1, 2953, + -1, 2955, -1, -1, -1, -1, -1, 240, -1, -1, + 2964, -1, 280, -1, -1, 283, -1, -1, -1, -1, + -1, -1, 2976, -1, -1, 2979, 400, 2981, -1, 297, + -1, 2985, 300, -1, 2988, 2989, -1, -1, 2992, 2993, + -1, -1, -1, -1, -1, -1, 3000, 280, -1, -1, + 283, -1, 179, -1, -1, -1, -1, -1, -1, -1, + 3014, -1, -1, -1, 297, -1, -1, 300, -1, 196, + -1, -1, -1, -1, 201, -1, 3030, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 8, -1, -1, + 11, -1, -1, -1, 15, 16, -1, 224, 225, 20, + 21, 22, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 240, -1, 3069, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 47, -1, -1, -1, + -1, -1, 400, 54, -1, -1, -1, -1, 512, -1, + -1, 515, 516, 517, -1, 519, 520, 521, 522, 523, + 524, -1, -1, 280, -1, 529, 283, -1, -1, -1, + 81, -1, -1, -1, -1, -1, -1, 400, -1, -1, + 297, -1, -1, 300, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 8, -1, -1, 11, -1, -1, + -1, 15, 16, 17, 18, -1, 20, 21, 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 37, -1, -1, -1, -1, -1, -1, + -1, 3175, -1, 47, -1, -1, -1, -1, -1, -1, + 54, -1, -1, -1, -1, 3189, -1, -1, -1, -1, + -1, -1, -1, -1, 512, -1, -1, 515, 516, 517, + -1, 519, 520, 521, 522, 523, 524, 81, 179, -1, + -1, 529, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 3225, 3226, 400, -1, 196, -1, -1, -1, 512, + 201, -1, 515, 516, 517, -1, 519, 520, 521, 522, + 523, 524, -1, -1, 527, -1, 3250, 3251, -1, -1, + 3254, -1, -1, 224, 225, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 240, + -1, 3275, -1, -1, -1, -1, -1, -1, -1, -1, + 3284, -1, -1, 3287, 3288, 3289, -1, -1, 3292, -1, + -1, 3295, 3296, -1, -1, -1, -1, -1, -1, -1, + 3304, -1, -1, -1, -1, 179, -1, -1, -1, -1, + -1, -1, 283, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 196, -1, -1, -1, 297, 201, -1, -1, + -1, -1, -1, -1, -1, 512, -1, -1, 515, 516, + 517, -1, 519, 520, 521, 522, 523, 524, 3352, -1, + 224, 225, 529, -1, 3358, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 240, 3371, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 3414, 3415, -1, -1, - -1, -1, -1, 3421, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 3457, - 3458, 3459, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 280, -1, -1, 283, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 3483, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 3495, -1, -1, + -1, -1, -1, 297, 3428, 3429, 300, -1, -1, 400, + -1, 3435, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3471, 3472, 3473, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 3497, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 3509, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 3557, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 3567, + -1, -1, -1, -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 512, -1, -1, 515, 516, 517, -1, 519, 520, + 521, 522, 523, 524, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3571, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3581, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 3595, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 3609, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 3620, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 3647, - -1, -1, 3, 4, 5, 6, 7, 8, 9, 10, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 3667, - -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, - -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, - -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, - 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, - -1, 82, 3730, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, 3741, 95, 96, 97, 98, 99, 100, - -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, 118, 119, -1, - 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, - 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, - 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, 162, 163, -1, 165, 166, 167, 168, -1, 170, - -1, 172, 173, 174, 175, 176, 177, 178, 179, 180, - 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, - 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, - -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, - 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, - 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, - 231, 232, 233, 234, -1, -1, 237, 238, 239, 240, - -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, - 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, - -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, -1, 325, 326, 327, 328, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, - 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, - 381, 382, 383, 384, 385, 386, 387, 388, -1, 390, - 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, -1, 425, 426, 427, 428, 429, 430, - 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, - -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, - 461, 462, 463, 464, 465, 466, -1, 468, 469, 470, - 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, - 481, 482, 483, 484, 485, -1, -1, 488, -1, 490, - 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, 512, -1, 514, -1, -1, -1, -1, 519, 520, - 521, -1, -1, -1, -1, 526, -1, 528, 529, -1, - -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, - 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, - 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, - -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, - 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, -1, -1, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, - 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, - 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, - 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, 118, 119, -1, 121, 122, 123, 124, 125, 126, - 127, 128, 129, 130, 131, 132, -1, -1, 135, 136, - 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, - 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, - 167, 168, -1, 170, -1, 172, 173, 174, 175, 176, - 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, - 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, - 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, - 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, - 217, 218, 219, -1, 221, -1, 223, 224, 225, 226, - 227, 228, 229, 230, 231, 232, 233, 234, -1, -1, - 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, - 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, - 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 321, 322, 323, -1, 325, 326, - 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, - 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, - 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, - 387, 388, -1, 390, 391, 392, 393, 394, 395, 396, - 397, 398, 399, 400, 401, 402, 403, 404, -1, 406, - 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, 428, 429, 430, 431, 432, 433, 434, 435, -1, - 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, - -1, 458, 459, 460, 461, 462, 463, 464, 465, 466, - -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, - -1, 488, -1, 490, 491, 492, 493, 494, 495, 496, - 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 508, 509, 510, 511, 512, -1, 514, -1, -1, - -1, -1, 519, 520, 521, -1, -1, -1, -1, 526, - -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, + 3634, -1, -1, -1, -1, -1, -1, -1, 512, -1, + -1, 515, 516, 517, -1, 519, 520, 521, 522, 523, + 524, -1, -1, -1, -1, 529, -1, 3661, -1, -1, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3681, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, - -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, - 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, + 3744, 84, 85, 86, 87, 88, 89, 90, 91, 92, + 93, 3755, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, -1, 121, 122, - 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, + 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, @@ -13481,10 +13429,10 @@ static const yytype_int16 yycheck[] = 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, -1, 390, 391, 392, - 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, + 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - -1, -1, 425, 426, 427, 428, 429, 430, 431, 432, + 423, -1, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, 461, 462, @@ -13494,25 +13442,25 @@ static const yytype_int16 yycheck[] = 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, -1, 514, -1, -1, -1, -1, 519, 520, 521, -1, - -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, + -1, -1, -1, 526, -1, 528, 529, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, - 29, 30, 31, 32, 33, 34, -1, -1, -1, 38, + 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - -1, 60, 61, 62, 63, 64, 65, 66, 67, 68, + -1, -1, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, - 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, - 129, 130, 131, 132, 133, -1, 135, 136, 137, 138, + 119, -1, 121, 122, 123, 124, 125, 126, 127, 128, + 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, 168, - -1, 170, -1, 172, 173, -1, 175, 176, 177, 178, + -1, 170, -1, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, @@ -13525,36 +13473,36 @@ static const yytype_int16 yycheck[] = 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, - 299, -1, 301, 302, 303, 304, 305, 306, 307, 308, + 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, -1, -1, -1, -1, 325, 326, 327, 328, + 319, 320, 321, 322, 323, -1, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, - 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, - -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, + 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, + -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 399, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, + 419, 420, 421, 422, 423, -1, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, - 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, - 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, + 459, 460, 461, 462, 463, 464, 465, 466, -1, 468, + 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, - -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, + -1, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, - 509, 510, 511, 512, -1, -1, -1, -1, -1, -1, + 509, 510, 511, 512, -1, 514, -1, -1, -1, -1, 519, 520, 521, -1, -1, -1, -1, 526, -1, 528, - 529, -1, -1, -1, 533, 534, 535, 536, 3, 4, + -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 58, 59, -1, 61, 62, 63, 64, + 55, 56, 57, 58, -1, -1, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, @@ -13572,7 +13520,7 @@ static const yytype_int16 yycheck[] = 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, - -1, 236, 237, 238, 239, 240, -1, 242, 243, 244, + -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, @@ -13580,32 +13528,32 @@ static const yytype_int16 yycheck[] = 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, + 315, 316, 317, 318, 319, 320, 321, 322, 323, -1, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, - 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, + 385, 386, 387, 388, -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, - 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, - 435, -1, 437, 438, 439, -1, 441, 442, 443, 444, + 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, + 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 455, -1, -1, 458, 459, 460, 461, -1, 463, 464, + 455, -1, -1, 458, 459, 460, 461, 462, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, - 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, + 485, -1, -1, 488, -1, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, -1, -1, + 505, 506, 507, 508, 509, 510, 511, 512, -1, 514, -1, -1, -1, -1, 519, 520, 521, -1, -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, - -1, 32, 33, 34, -1, -1, -1, 38, -1, 40, + 31, 32, 33, 34, -1, -1, -1, 38, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, -1, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, @@ -13615,7 +13563,7 @@ static const yytype_int16 yycheck[] = -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, - 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, + 131, 132, 133, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, 168, -1, 170, @@ -13632,7 +13580,7 @@ static const yytype_int16 yycheck[] = 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, - -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, @@ -13654,21 +13602,21 @@ static const yytype_int16 yycheck[] = 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, -1, -1, -1, -1, -1, -1, 519, 520, - 521, -1, -1, -1, -1, 526, -1, 528, -1, -1, + 521, -1, -1, -1, -1, 526, -1, 528, 529, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, - 7, -1, 9, 10, -1, -1, -1, -1, -1, -1, + 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, -1, -1, 61, 62, 63, 64, 65, 66, + 57, 58, 59, -1, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, - 127, 128, 129, 130, 131, 132, -1, -1, 135, 136, + 117, 118, 119, -1, 121, 122, 123, 124, 125, 126, + -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, @@ -13678,37 +13626,39 @@ static const yytype_int16 yycheck[] = 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, 224, 225, 226, - 227, 228, 229, 230, -1, 232, 233, 234, -1, -1, + 227, 228, 229, 230, 231, 232, 233, 234, -1, 236, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, -1, 281, 282, 283, 284, 285, 286, + 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 321, 322, 323, -1, 325, 326, + 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, - 387, 388, -1, 390, 391, 392, 393, 394, 395, 396, - 397, 398, 399, 400, 401, 402, 403, 404, -1, 406, + 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, + 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, 428, 429, 430, 431, 432, 433, 434, 435, -1, - 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, + 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, + -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, + 437, 438, 439, -1, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, - -1, 458, 459, 460, -1, 462, 463, 464, 465, 466, + -1, 458, 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, - -1, 488, -1, 490, 491, 492, 493, 494, 495, 496, + -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 508, 509, 510, 511, -1, -1, 514, -1, -1, - 3, 4, 5, 6, 7, 8, 9, 10, -1, 526, + 507, 508, 509, 510, 511, 512, -1, -1, -1, -1, + -1, -1, 519, 520, 521, -1, -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, + 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, 38, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, @@ -13754,72 +13704,70 @@ static const yytype_int16 yycheck[] = 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, - 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, + 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, -1, -1, -1, -1, -1, -1, 519, 520, 521, -1, - -1, -1, -1, 526, -1, 528, 529, -1, -1, -1, - 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, + -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, + 533, 534, 535, 536, 3, 4, 5, 6, 7, -1, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, - 29, 30, -1, 32, 33, 34, -1, -1, -1, 38, + 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - -1, 60, 61, 62, 63, 64, 65, 66, 67, 68, + -1, -1, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, - 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, + 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, + 119, -1, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, 168, - -1, 170, -1, 172, 173, -1, 175, 176, 177, 178, + -1, 170, -1, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, 224, 225, 226, 227, 228, - 229, 230, 231, 232, 233, 234, -1, -1, 237, 238, + 229, 230, -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, - 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, + 279, -1, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, -1, -1, -1, -1, 325, 326, 327, 328, + 319, 320, 321, 322, 323, -1, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, - 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, - -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, + 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, + -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 399, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, + 419, 420, 421, 422, 423, -1, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, - 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, + 459, 460, -1, 462, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, - -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, + -1, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, - 509, 510, 511, 512, -1, -1, -1, -1, -1, -1, - 519, 520, 521, -1, -1, -1, -1, 526, -1, 528, - -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, - 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - -1, -1, -1, -1, 39, 40, -1, -1, 43, 44, + 509, 510, 511, -1, -1, 514, -1, -1, 3, 4, + 5, 6, 7, 8, 9, 10, -1, 526, -1, 528, + -1, -1, -1, -1, 533, 534, 535, 536, 23, 24, + 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, + -1, -1, -1, 38, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 58, -1, -1, 61, 62, 63, 64, + 55, 56, 57, 58, -1, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, @@ -13852,9 +13800,9 @@ static const yytype_int16 yycheck[] = 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, - 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, + 385, 386, 387, -1, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, + -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, @@ -13866,13 +13814,13 @@ static const yytype_int16 yycheck[] = 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, -1, -1, -1, -1, -1, -1, 519, 520, 521, -1, -1, -1, - -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, + -1, 526, -1, 528, 529, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, - 31, 32, 33, 34, -1, -1, -1, -1, -1, 40, + -1, 32, 33, 34, -1, -1, -1, 38, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, + 51, 52, 53, 54, 55, 56, 57, 58, -1, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, @@ -13905,7 +13853,7 @@ static const yytype_int16 yycheck[] = 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, - 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, + 381, 382, 383, 384, 385, 386, 387, -1, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, @@ -13924,7 +13872,7 @@ static const yytype_int16 yycheck[] = 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, -1, -1, - -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, + -1, -1, 39, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, @@ -13960,7 +13908,7 @@ static const yytype_int16 yycheck[] = -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, - 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, + 397, 398, -1, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, @@ -13976,7 +13924,7 @@ static const yytype_int16 yycheck[] = -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, 61, 62, @@ -13991,7 +13939,7 @@ static const yytype_int16 yycheck[] = 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, - 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, + 173, -1, 175, 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, @@ -14030,7 +13978,7 @@ static const yytype_int16 yycheck[] = 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, - 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, + 29, 30, 31, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, 61, 62, 63, 64, 65, 66, 67, 68, @@ -14080,9 +14028,9 @@ static const yytype_int16 yycheck[] = 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, -1, -1, -1, -1, -1, -1, 519, 520, 521, -1, -1, -1, -1, 526, -1, 528, - 529, -1, -1, -1, 533, 534, 535, 536, 3, 4, + -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, - -1, -1, -1, -1, 19, -1, -1, -1, 23, 24, + -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, @@ -14097,7 +14045,7 @@ static const yytype_int16 yycheck[] = 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, - 165, 166, 167, 168, -1, 170, -1, 172, 173, -1, + 165, 166, 167, 168, -1, 170, -1, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, @@ -14176,7 +14124,7 @@ static const yytype_int16 yycheck[] = 391, 392, 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, -1, -1, 425, 426, 427, 428, 429, 430, + 421, 422, -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, @@ -14186,10 +14134,10 @@ static const yytype_int16 yycheck[] = 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, -1, -1, -1, -1, -1, -1, 519, 520, - 521, -1, -1, -1, -1, 526, -1, 528, -1, -1, + 521, -1, -1, -1, -1, 526, -1, 528, 529, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, + -1, -1, 19, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, @@ -14236,14 +14184,14 @@ static const yytype_int16 yycheck[] = -1, 458, 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, - 487, 488, -1, -1, 491, 492, 493, 494, 495, 496, + -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, -1, -1, -1, -1, -1, -1, 519, 520, 521, -1, -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, 61, 62, @@ -14283,7 +14231,7 @@ static const yytype_int16 yycheck[] = 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, + -1, -1, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, 461, -1, @@ -14296,7 +14244,7 @@ static const yytype_int16 yycheck[] = -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, - 19, -1, -1, -1, 23, 24, 25, 26, 27, 28, + -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, @@ -14342,7 +14290,7 @@ static const yytype_int16 yycheck[] = 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, - 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, + 479, 480, 481, 482, 483, 484, 485, -1, 487, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, -1, -1, -1, -1, -1, -1, @@ -14350,7 +14298,7 @@ static const yytype_int16 yycheck[] = -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, - 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, 61, 62, 63, 64, @@ -14402,7 +14350,7 @@ static const yytype_int16 yycheck[] = -1, -1, -1, -1, 519, 520, 521, -1, -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 19, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, @@ -14850,13 +14798,13 @@ static const yytype_int16 yycheck[] = 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, - 219, -1, 221, -1, 223, -1, -1, 226, 227, 228, + 219, -1, 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, - 279, -1, 281, 282, -1, 284, 285, 286, 287, 288, + 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, @@ -14893,7 +14841,7 @@ static const yytype_int16 yycheck[] = 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, + 115, 116, 117, 118, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, @@ -14904,12 +14852,12 @@ static const yytype_int16 yycheck[] = 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, 224, - 225, 226, 227, 228, 229, 230, -1, 232, 233, 234, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, -1, 281, 282, 283, 284, + 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, @@ -14927,15 +14875,15 @@ static const yytype_int16 yycheck[] = 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, + 455, -1, -1, 458, 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, -1, -1, -1, - -1, -1, -1, -1, 519, 520, -1, -1, -1, -1, + 505, 506, 507, 508, 509, 510, 511, 512, -1, -1, + -1, -1, -1, -1, 519, 520, 521, -1, -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, - 535, 536, 3, 4, 5, 6, 7, -1, 9, 10, + 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, @@ -14946,7 +14894,7 @@ static const yytype_int16 yycheck[] = -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, + 111, 112, 113, 114, 115, 116, 117, 118, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, @@ -14957,13 +14905,13 @@ static const yytype_int16 yycheck[] = 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, - 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, - -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, + 221, -1, 223, -1, -1, 226, 227, 228, 229, 230, + 231, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, -1, - 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, + 281, 282, -1, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, @@ -14981,13 +14929,13 @@ static const yytype_int16 yycheck[] = 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, - -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, + 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, -1, -1, -1, -1, -1, -1, -1, 519, 520, - -1, -1, -1, -1, -1, 526, -1, 528, -1, -1, + 511, 512, -1, -1, -1, -1, -1, -1, 519, 520, + 521, -1, -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, @@ -15010,7 +14958,7 @@ static const yytype_int16 yycheck[] = 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, - 217, 218, 219, -1, 221, -1, 223, -1, 225, 226, + 217, 218, 219, -1, 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, @@ -15060,7 +15008,7 @@ static const yytype_int16 yycheck[] = 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, 173, -1, 175, 176, 177, 178, 179, 180, 181, 182, - 183, 184, 185, 186, 187, -1, 189, 190, 191, 192, + -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, @@ -15093,9 +15041,11 @@ static const yytype_int16 yycheck[] = 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, -1, - -1, -1, -1, -1, 3, 4, 5, 6, 7, -1, - 9, 10, -1, 526, -1, 528, -1, -1, -1, -1, - 533, 534, 535, 536, 23, 24, 25, 26, 27, 28, + -1, -1, -1, -1, -1, -1, 519, 520, -1, -1, + -1, -1, -1, 526, -1, 528, -1, -1, -1, -1, + 533, 534, 535, 536, 3, 4, 5, 6, 7, 8, + 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, @@ -15115,7 +15065,7 @@ static const yytype_int16 yycheck[] = 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, - 219, -1, 221, -1, 223, 224, 225, 226, 227, 228, + 219, -1, 221, -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, @@ -15144,9 +15094,11 @@ static const yytype_int16 yycheck[] = 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, - 509, 510, 511, -1, -1, -1, -1, -1, 3, 4, - 5, 6, 7, -1, 9, 10, -1, 526, -1, 528, - -1, -1, -1, -1, 533, 534, 535, 536, 23, 24, + 509, 510, 511, -1, -1, -1, -1, -1, -1, -1, + 519, 520, -1, -1, -1, -1, -1, 526, -1, 528, + -1, -1, -1, -1, 533, 534, 535, 536, 3, 4, + 5, 6, 7, -1, 9, 10, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, @@ -15162,7 +15114,7 @@ static const yytype_int16 yycheck[] = 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, 173, -1, - 175, 176, 177, 178, 179, 180, 181, 182, -1, 184, + 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, @@ -15196,35 +15148,35 @@ static const yytype_int16 yycheck[] = 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, -1, -1, -1, - -1, -1, 3, 4, 5, 6, 7, 8, 9, 10, + -1, -1, 3, 4, 5, 6, 7, -1, 9, 10, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, - 51, 52, 53, -1, 55, 56, 57, 58, -1, -1, + 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, 118, 119, -1, + 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, - 141, 142, 143, -1, 145, 146, -1, 148, -1, 150, + 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, 168, -1, 170, - -1, 172, 173, -1, 175, 176, 177, 178, -1, 180, + -1, 172, 173, -1, 175, 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, - 191, 192, 193, 194, 195, -1, 197, 198, 199, 200, + 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, - 221, -1, 223, -1, -1, 226, 227, 228, 229, 230, - 231, 232, 233, 234, -1, -1, 237, 238, 239, -1, + 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, + -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, -1, - 281, 282, -1, 284, 285, 286, 287, 288, 289, 290, + 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, @@ -15235,20 +15187,20 @@ static const yytype_int16 yycheck[] = 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, - 391, 392, 393, 394, 395, 396, 397, 398, -1, -1, + 391, 392, 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, - 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, + -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, -1, -1, -1, -1, -1, 3, -1, 519, 520, - 521, -1, -1, 10, -1, 526, -1, 528, -1, -1, + 511, -1, -1, -1, -1, -1, 3, 4, 5, 6, + 7, -1, 9, 10, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, 43, 44, 45, -1, @@ -15263,7 +15215,7 @@ static const yytype_int16 yycheck[] = -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, + 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, 173, -1, 175, 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, @@ -15275,7 +15227,7 @@ static const yytype_int16 yycheck[] = 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, -1, 281, 282, 283, 284, -1, 286, + 277, 278, 279, -1, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, @@ -15292,471 +15244,472 @@ static const yytype_int16 yycheck[] = 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, - -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, + 447, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, -1, -1, -1, -1, -1, - 3, 4, -1, -1, -1, -1, 9, 10, -1, 526, + 3, 4, 5, 6, 7, 8, 9, 10, -1, 526, -1, 528, -1, -1, -1, -1, 533, 534, 535, 536, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, - 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, - 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, - 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, - 73, 74, -1, 76, 77, 78, 79, 80, -1, -1, - -1, 84, 85, 86, 87, 88, 89, -1, 91, 92, - 93, -1, 95, 96, 97, 98, 99, 100, -1, -1, + 33, 34, -1, -1, -1, -1, -1, 40, -1, -1, + 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, + 53, -1, 55, 56, 57, 58, -1, -1, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, + 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, + -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, + 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, + 113, 114, 115, 116, 117, 118, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, - 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, - 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, + 143, -1, 145, 146, -1, 148, -1, 150, 151, 152, + 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, - -1, -1, -1, 176, 177, 178, -1, 180, 181, 182, + 173, -1, 175, 176, 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, 197, 198, 199, 200, -1, 202, - 203, 204, 205, 206, 207, 208, -1, 210, -1, 212, + 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, - 223, -1, -1, 226, -1, 228, 229, 230, -1, 232, - 233, 234, -1, -1, 237, -1, 239, -1, -1, 242, + 223, -1, -1, 226, 227, 228, 229, 230, 231, 232, + 233, 234, -1, -1, 237, 238, 239, -1, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, - 273, -1, 275, 276, 277, 278, 279, -1, 281, 282, - -1, 284, -1, 286, 287, 288, 289, 290, 291, -1, + 273, 274, 275, 276, 277, 278, 279, -1, 281, 282, + -1, 284, 285, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, - 303, 304, -1, 306, -1, 308, 309, 310, 311, 312, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, - -1, -1, 325, 326, 327, -1, 329, 330, 331, 332, + -1, -1, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, -1, 368, 369, -1, 371, 372, + 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - -1, -1, 425, 426, -1, 428, -1, 430, 431, 432, + -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, - 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, - 453, 454, 455, -1, -1, 458, 459, 460, -1, -1, + 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, + 453, 454, 455, -1, -1, 458, 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, - 473, 474, 475, -1, 477, -1, 479, 480, 481, 482, + 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, -1, - -1, -1, 3, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 533, 534, 23, 24, 25, 26, 27, 28, 29, 30, - -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, - 41, -1, -1, 44, 45, -1, -1, 48, 49, -1, - 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, - 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, - 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, - -1, -1, -1, 84, 85, 86, 87, 88, 89, -1, - 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, - -1, -1, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, - 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, - 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, - 141, 142, 143, -1, 145, 146, 147, 148, 149, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, -1, 163, 164, 165, 166, 167, 168, 169, 170, - -1, 172, -1, -1, -1, 176, 177, 178, -1, 180, - 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, - 191, 192, 193, 194, 195, -1, 197, 198, 199, 200, - -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, - -1, 212, 213, 214, 215, 216, 217, 218, 219, 220, - 221, -1, 223, -1, -1, 226, -1, 228, 229, 230, - -1, 232, 233, 234, -1, -1, 237, -1, 239, -1, - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, -1, 275, 276, 277, 278, 279, -1, - 281, 282, -1, 284, -1, 286, 287, 288, 289, 290, - 291, 292, 293, 294, -1, -1, 297, 298, 299, -1, - 301, 302, 303, 304, -1, 306, -1, 308, 309, 310, - 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - -1, -1, -1, -1, 325, 326, 327, -1, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, - 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, -1, 368, 369, -1, - 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, - 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, - 391, 392, 393, 394, 395, 396, 397, 398, -1, -1, - 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, -1, -1, 425, 426, -1, 428, -1, 430, - 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, - -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, - 451, 452, 453, 454, 455, 456, -1, 458, 459, 460, - -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, - 471, 472, 473, 474, 475, -1, 477, -1, 479, 480, - 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, - 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, 3, -1, 5, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 529, -1, - -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, - 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, - 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, - 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, - 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, - -1, -1, 84, 85, 86, 87, 88, 89, -1, 91, - 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, - -1, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, - 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, - 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, - 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, - -1, 163, -1, 165, 166, 167, 168, -1, 170, -1, - 172, -1, -1, -1, 176, 177, 178, -1, 180, 181, - 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, - 192, 193, 194, 195, -1, 197, 198, 199, 200, -1, - 202, 203, 204, 205, 206, 207, 208, -1, 210, -1, - 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, - -1, 223, -1, -1, 226, -1, 228, 229, 230, -1, - 232, 233, 234, -1, -1, 237, -1, 239, -1, -1, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, -1, 275, 276, 277, 278, 279, -1, 281, - 282, -1, 284, -1, 286, 287, 288, 289, 290, 291, - -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, - 302, 303, 304, -1, 306, -1, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, - -1, -1, -1, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, - 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, -1, 368, 369, -1, 371, - 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, - 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, - 392, 393, 394, 395, 396, 397, 398, -1, -1, 401, - 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, -1, -1, 425, 426, -1, 428, -1, 430, 431, - 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, - 442, 443, 444, 445, 446, -1, 448, 449, 450, 451, - 452, 453, 454, 455, -1, -1, 458, 459, 460, -1, - -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, - 472, 473, 474, 475, -1, 477, -1, 479, 480, 481, - 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, - 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, - 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 528, 529, -1, -1, - 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, - 33, 34, -1, -1, -1, 38, -1, -1, -1, -1, - 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, - 53, -1, -1, 56, -1, -1, -1, 60, 61, 62, - 63, 64, 65, -1, -1, 68, 69, 70, 71, -1, - -1, 74, -1, 76, 77, 78, 79, -1, -1, 82, - -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, - 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, - 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - -1, 114, 115, -1, 117, -1, 119, -1, 121, 122, - 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, - -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, - 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, - 153, -1, 155, 156, 157, 158, 159, 160, -1, -1, - 163, -1, 165, 166, -1, 168, -1, 170, -1, 172, - 173, -1, 175, 176, 177, 178, 179, 180, 181, 182, - -1, -1, -1, 186, 187, -1, 189, 190, 191, 192, - 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, - 203, 204, 205, 206, 207, 208, -1, 210, 211, -1, - 213, 214, 215, 216, 217, -1, -1, -1, -1, -1, - 223, 224, 225, 226, 227, 228, 229, 230, -1, 232, - 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, - 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, - -1, 274, -1, 276, 277, 278, -1, -1, 281, 282, - 283, 284, -1, -1, 287, -1, 289, 290, 291, -1, - 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, - 303, -1, 305, 306, 307, -1, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, - -1, -1, 325, 326, -1, 328, 329, 330, -1, 332, - 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, - 343, 344, 345, 346, 347, -1, 349, 350, 351, 352, - -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, - 373, -1, 375, 376, 377, 378, 379, 380, 381, 382, - 383, 384, 385, 386, 387, -1, 389, 390, 391, 392, - 393, 394, 395, -1, 397, 398, -1, 400, 401, 402, - -1, 404, -1, 406, 407, 408, 409, 410, 411, 412, - 413, 414, 415, 416, 417, 418, 419, 420, 421, -1, - -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, - 433, 434, 435, -1, 437, -1, -1, -1, -1, 442, - 443, -1, 445, -1, -1, 448, 449, 450, 451, 452, - 453, 454, 455, -1, -1, 458, 459, 460, 461, -1, - 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, - -1, -1, 475, -1, 477, 478, 479, 480, 481, 482, - 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, - 493, 494, 495, 496, 3, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 508, 509, 510, 511, -1, - -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, - 29, 30, -1, 32, 33, 34, 529, -1, -1, 38, - -1, -1, -1, -1, 43, 44, 45, -1, 47, 48, - 49, 50, 51, 52, 53, -1, -1, 56, -1, -1, - -1, 60, 61, 62, 63, 64, 65, -1, -1, 68, - 69, 70, 71, -1, -1, 74, -1, 76, 77, 78, - 79, -1, -1, 82, -1, 84, 85, 86, 87, 88, + -1, -1, -1, -1, 3, -1, 519, 520, 521, -1, + -1, 10, -1, 526, -1, 528, -1, -1, -1, -1, + 533, 534, 535, 536, 23, 24, 25, 26, 27, 28, + 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, + -1, 40, -1, -1, 43, 44, 45, -1, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + -1, -1, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, + 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, -1, 114, 115, -1, 117, -1, + 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, - -1, 150, 151, 152, 153, -1, 155, 156, 157, 158, - 159, 160, -1, -1, 163, -1, 165, 166, -1, 168, + -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, 161, -1, 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, 173, -1, 175, 176, 177, 178, - 179, 180, 181, 182, -1, -1, -1, 186, 187, -1, + 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, - -1, 210, 211, -1, 213, 214, 215, 216, 217, -1, - -1, -1, -1, -1, 223, 224, 225, 226, 227, 228, + -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, + 219, -1, 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, - 269, 270, 271, 272, -1, 274, -1, 276, 277, 278, - -1, -1, 281, 282, 283, 284, -1, -1, 287, -1, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, -1, 281, 282, 283, 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, - 299, -1, -1, 302, 303, -1, 305, 306, 307, -1, + 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, -1, -1, -1, -1, 325, 326, -1, 328, - 329, 330, -1, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, -1, 343, 344, 345, 346, 347, -1, + 319, 320, -1, -1, -1, -1, 325, 326, 327, 328, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, - 369, 370, 371, 372, 373, -1, 375, 376, 377, 378, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, - 389, 390, 391, 392, 393, 394, 395, -1, 397, 398, - -1, 400, 401, 402, -1, 404, -1, 406, 407, 408, + -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, + -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, -1, -1, -1, 425, 426, -1, 428, - 429, 430, 431, 432, 433, 434, 435, -1, 437, -1, - -1, -1, -1, 442, 443, -1, 445, -1, -1, 448, + 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, + 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, + 439, -1, -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, - 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, - 469, 470, 471, 472, -1, -1, 475, -1, 477, 478, + 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, + 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, - -1, -1, 491, 492, 493, 494, 495, 496, -1, -1, - -1, -1, 3, -1, -1, -1, -1, -1, -1, 508, - 509, 510, 511, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, + 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 510, 511, -1, -1, -1, -1, -1, 3, 4, + -1, -1, -1, -1, 9, 10, -1, 526, -1, 528, + -1, -1, -1, -1, 533, 534, 535, 536, 23, 24, + 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, + 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, + 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, + 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, + -1, 76, 77, 78, 79, 80, -1, -1, -1, 84, + 85, 86, 87, 88, 89, -1, 91, 92, 93, -1, + 95, 96, 97, 98, 99, 100, -1, -1, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, + 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, + 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, + 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, + 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, + 165, 166, 167, 168, -1, 170, -1, 172, -1, -1, + -1, 176, 177, 178, -1, 180, 181, 182, -1, 184, + 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, + 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, + 205, 206, 207, 208, -1, 210, -1, 212, 213, 214, + 215, 216, 217, 218, 219, -1, 221, -1, 223, -1, + -1, 226, -1, 228, 229, 230, -1, 232, 233, 234, + -1, -1, 237, -1, 239, -1, -1, 242, 243, 244, + 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, + 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, + -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, + -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, + -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, + 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, + -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, + 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, + 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, + 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, + 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, + 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, + 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, + -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, + 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, + 425, 426, -1, 428, -1, 430, 431, 432, 433, 434, + 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, + 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, + 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, + 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, + 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, + 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, + 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, + 505, 506, 507, 508, 509, 510, 511, -1, -1, -1, + 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 533, 534, + 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, + 33, 34, -1, -1, -1, -1, -1, -1, 41, -1, + -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, + 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, + 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, + 73, 74, -1, 76, 77, 78, 79, 80, -1, -1, + -1, 84, 85, 86, 87, 88, 89, -1, 91, 92, + 93, -1, 95, 96, 97, 98, 99, 100, -1, -1, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, + 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, + -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, + 143, -1, 145, 146, 147, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, + 163, 164, 165, 166, 167, 168, 169, 170, -1, 172, + -1, -1, -1, 176, 177, 178, -1, 180, 181, 182, + -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, + 193, 194, 195, -1, 197, 198, 199, 200, -1, 202, + 203, 204, 205, 206, 207, 208, -1, 210, -1, 212, + 213, 214, 215, 216, 217, 218, 219, 220, 221, -1, + 223, -1, -1, 226, -1, 228, 229, 230, -1, 232, + 233, 234, -1, -1, 237, -1, 239, -1, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, + 273, -1, 275, 276, 277, 278, 279, -1, 281, 282, + -1, 284, -1, 286, 287, 288, 289, 290, 291, 292, + 293, 294, -1, -1, 297, 298, 299, -1, 301, 302, + 303, 304, -1, 306, -1, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, + -1, -1, 325, 326, 327, -1, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, + 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, + -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, -1, 368, 369, -1, 371, 372, + 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, + 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, + 393, 394, 395, 396, 397, 398, -1, -1, 401, 402, + 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, + 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, + -1, -1, 425, 426, -1, 428, -1, 430, 431, 432, + 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, + 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, + 453, 454, 455, 456, -1, 458, 459, 460, -1, -1, + 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, + 473, 474, 475, -1, 477, -1, 479, 480, 481, 482, + 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, + 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 529, -1, -1, 23, + 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, + 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, + 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, + 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, + 74, -1, 76, 77, 78, 79, 80, -1, -1, -1, + 84, 85, 86, 87, 88, 89, -1, 91, 92, 93, + -1, 95, 96, 97, 98, 99, 100, -1, -1, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, + 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, + -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, + -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, + -1, 165, 166, 167, 168, -1, 170, -1, 172, -1, + -1, -1, 176, 177, 178, -1, 180, 181, 182, -1, + 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, + 194, 195, -1, 197, 198, 199, 200, -1, 202, 203, + 204, 205, 206, 207, 208, -1, 210, -1, 212, 213, + 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, + -1, -1, 226, -1, 228, 229, 230, -1, 232, 233, + 234, -1, -1, 237, -1, 239, -1, -1, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + -1, 275, 276, 277, 278, 279, -1, 281, 282, -1, + 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, + 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, + 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, + -1, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, + 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, + 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, -1, 368, 369, -1, 371, 372, 373, + 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, + 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, + 394, 395, 396, 397, 398, -1, -1, 401, 402, 403, + 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, + -1, 425, 426, -1, 428, -1, 430, 431, 432, 433, + 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, + 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, + 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, + 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, + 474, 475, -1, 477, -1, 479, 480, 481, 482, 483, + 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, + 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 528, 529, -1, -1, 23, 24, + 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, + -1, -1, -1, 38, -1, -1, -1, -1, 43, 44, + 45, -1, 47, 48, 49, 50, 51, 52, 53, -1, + -1, 56, -1, -1, -1, 60, 61, 62, 63, 64, + 65, -1, -1, 68, 69, 70, 71, -1, -1, 74, + -1, 76, 77, 78, 79, -1, -1, 82, -1, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, + 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, -1, 114, + 115, -1, 117, -1, 119, -1, 121, 122, 123, 124, + 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, + 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, + 145, 146, 147, 148, -1, 150, 151, 152, 153, -1, + 155, 156, 157, 158, 159, 160, -1, -1, 163, -1, + 165, 166, -1, 168, -1, 170, -1, 172, 173, -1, + 175, 176, 177, 178, 179, 180, 181, 182, -1, -1, + -1, 186, 187, -1, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, + 205, 206, 207, 208, -1, 210, 211, -1, 213, 214, + 215, 216, 217, -1, -1, -1, -1, -1, 223, 224, + 225, 226, 227, 228, 229, 230, -1, 232, 233, 234, + -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, + 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, -1, 274, + -1, 276, 277, 278, -1, -1, 281, 282, 283, 284, + -1, -1, 287, -1, 289, 290, 291, -1, 293, 294, + -1, -1, 297, 298, 299, -1, -1, 302, 303, -1, + 305, 306, 307, -1, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, + 325, 326, -1, 328, 329, 330, -1, 332, 333, 334, + -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, + 345, 346, 347, -1, 349, 350, 351, 352, -1, 354, + 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, + 365, 366, -1, 368, 369, 370, 371, 372, 373, -1, + 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, + 385, 386, 387, -1, 389, 390, 391, 392, 393, 394, + 395, -1, 397, 398, -1, 400, 401, 402, -1, 404, + -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, + 415, 416, 417, 418, 419, 420, 421, -1, -1, -1, + 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, + 435, -1, 437, -1, -1, -1, -1, 442, 443, -1, + 445, -1, -1, 448, 449, 450, 451, 452, 453, 454, + 455, -1, -1, 458, 459, 460, 461, -1, 463, 464, + 465, 466, -1, 468, 469, 470, 471, 472, -1, -1, + 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, + 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, + 495, 496, 3, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 508, 509, 510, 511, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, - 529, 32, 33, 34, 35, 36, -1, 38, -1, -1, + -1, 32, 33, 34, 529, -1, -1, 38, -1, -1, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, -1, 60, + 51, 52, 53, -1, -1, 56, -1, -1, -1, 60, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, - 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, - -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, - -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, - 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, - 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, - 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, -1, 163, -1, 165, 166, 167, 168, -1, 170, - -1, 172, 173, 174, 175, 176, 177, 178, 179, 180, - 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, - 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, - -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, - 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, - 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, - -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, - -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, -1, - 281, 282, 283, 284, -1, 286, 287, 288, 289, 290, - 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, - -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, -1, 325, 326, 327, 328, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, - 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, - 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, -1, 425, 426, 427, 428, 429, 430, - 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, - -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, - 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, - 461, 462, 463, 464, 465, 466, -1, 468, 469, 470, - 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, - 481, 482, 483, 484, 485, -1, -1, 488, -1, 490, - 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, -1, 3, 514, 5, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 528, -1, -1, - -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, - -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, - -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, - 61, 62, 63, 64, 65, -1, 67, 68, 69, 70, - 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, - -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, - -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, - 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, - 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, - 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, -1, 163, -1, 165, 166, 167, 168, -1, 170, - -1, 172, 173, 174, 175, 176, 177, 178, 179, 180, - 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, - 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, - -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, - 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, - 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, - -1, 232, 233, 234, 235, -1, 237, 238, 239, 240, - -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, -1, - 281, 282, 283, 284, -1, 286, 287, 288, 289, 290, - 291, -1, 293, 294, -1, 296, 297, 298, 299, -1, - -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, -1, 325, 326, 327, 328, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, - 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, - 381, 382, 383, 384, 385, 386, 387, 388, -1, 390, - 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, -1, 425, 426, 427, 428, 429, 430, - 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, - -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, - 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, - -1, 462, 463, 464, 465, 466, -1, 468, 469, 470, - 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, - 481, 482, 483, 484, 485, -1, -1, 488, -1, 490, - 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, -1, 3, 514, 5, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 528, -1, -1, - -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, - -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, - -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, - 61, 62, 63, 64, 65, -1, 67, 68, 69, 70, - 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, + 71, -1, -1, 74, -1, 76, 77, 78, 79, -1, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, + 111, 112, -1, 114, 115, -1, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, -1, 163, -1, 165, 166, 167, 168, -1, 170, + 151, 152, 153, -1, 155, 156, 157, 158, 159, 160, + -1, -1, 163, -1, 165, 166, -1, 168, -1, 170, -1, 172, 173, -1, 175, 176, 177, 178, 179, 180, - 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, + 181, 182, -1, -1, -1, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, - 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, - 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, - -1, 232, 233, 234, 235, -1, 237, 238, 239, 240, + 211, -1, 213, 214, 215, 216, 217, -1, -1, -1, + -1, -1, 223, 224, 225, 226, 227, 228, 229, 230, + -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, -1, - 281, 282, 283, 284, -1, 286, 287, 288, 289, 290, - 291, -1, 293, 294, -1, 296, 297, 298, 299, -1, - -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 271, 272, -1, 274, -1, 276, 277, 278, -1, -1, + 281, 282, 283, 284, -1, -1, 287, -1, 289, 290, + 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, + -1, 302, 303, -1, 305, 306, 307, -1, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - -1, -1, -1, -1, 325, 326, 327, 328, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, + -1, -1, -1, -1, 325, 326, -1, 328, 329, 330, + -1, 332, 333, 334, -1, 336, 337, 338, 339, 340, + 341, -1, 343, 344, 345, 346, 347, -1, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, - 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, - 391, 392, 393, 394, 395, 396, 397, 398, -1, 400, - 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, + 371, 372, 373, -1, 375, 376, 377, 378, 379, 380, + 381, 382, 383, 384, 385, 386, 387, -1, 389, 390, + 391, 392, 393, 394, 395, -1, 397, 398, -1, 400, + 401, 402, -1, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, -1, -1, 425, 426, -1, 428, 429, 430, - 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, - -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, + 421, -1, -1, -1, 425, 426, -1, 428, 429, 430, + 431, 432, 433, 434, 435, -1, 437, -1, -1, -1, + -1, 442, 443, -1, 445, -1, -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, - -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, - 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, + 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, + 471, 472, -1, -1, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, - 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, 3, -1, 5, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 528, -1, -1, - -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, - 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, -1, -1, 61, - 62, 63, 64, 65, -1, 67, 68, 69, 70, 71, - 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, - 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, - 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, - 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, - 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, - 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, - -1, 163, -1, 165, 166, 167, 168, -1, 170, -1, - 172, 173, -1, 175, 176, 177, 178, 179, 180, 181, - 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, - 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, - 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, - 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, - -1, 223, 224, 225, 226, 227, 228, 229, 230, -1, - 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, -1, 281, - 282, 283, 284, -1, 286, 287, 288, 289, 290, 291, - -1, 293, 294, -1, 296, 297, 298, 299, -1, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, - -1, -1, -1, 325, 326, 327, 328, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, - 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, - 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, - 392, 393, 394, 395, 396, 397, 398, -1, 400, 401, - 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, -1, -1, 425, 426, -1, 428, 429, 430, 431, - 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, - 442, 443, 444, 445, 446, -1, 448, 449, 450, 451, - 452, 453, 454, 455, -1, -1, 458, 459, 460, -1, - -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, - 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, - 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, - 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 528, -1, -1, -1, + 491, 492, 493, 494, 495, 496, -1, -1, -1, -1, + 3, -1, -1, -1, -1, -1, -1, 508, 509, 510, + 511, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 23, 24, 25, 26, 27, 28, 29, 30, 529, 32, + 33, 34, 35, 36, -1, 38, -1, -1, -1, -1, + 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, 58, -1, 60, 61, 62, + 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, + 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, + -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, + 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, + 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, + 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, + 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, + 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, + 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, + -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, + 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, + 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, + 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, + 223, 224, 225, 226, 227, 228, 229, 230, -1, 232, + 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, + 273, 274, 275, 276, 277, 278, 279, -1, 281, 282, + 283, 284, -1, 286, 287, 288, 289, 290, 291, -1, + 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, + 323, -1, 325, 326, 327, 328, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, + 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, + -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, + 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, + 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, + 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, + 423, -1, 425, 426, 427, 428, 429, 430, 431, 432, + 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, + 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, + 453, 454, 455, -1, -1, 458, 459, 460, 461, 462, + 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, + 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, -1, -1, 488, -1, 490, 491, 492, + 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 510, 511, -1, + 3, 514, 5, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 528, -1, -1, -1, -1, + 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, + 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, + 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, 58, -1, -1, 61, 62, + 63, 64, 65, -1, 67, 68, 69, 70, 71, 72, + 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, + -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, + 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, + 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, + 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, + 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, + 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, + 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, + -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, + 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, + 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, + 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, + 223, 224, 225, 226, 227, 228, 229, 230, -1, 232, + 233, 234, 235, -1, 237, 238, 239, 240, -1, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, + 273, 274, 275, 276, 277, 278, 279, -1, 281, 282, + 283, 284, -1, 286, 287, 288, 289, 290, 291, -1, + 293, 294, -1, 296, 297, 298, 299, -1, -1, 302, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, + 323, -1, 325, 326, 327, 328, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, + 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, + -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, + 383, 384, 385, 386, 387, 388, -1, 390, 391, 392, + 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, + 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, + 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, + 423, -1, 425, 426, 427, 428, 429, 430, 431, 432, + 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, + 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, + 453, 454, 455, -1, -1, 458, 459, 460, -1, 462, + 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, + 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, -1, -1, 488, -1, 490, 491, 492, + 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 510, 511, -1, + 3, 514, 5, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 528, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, @@ -15778,13 +15731,13 @@ static const yytype_int16 yycheck[] = 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, -1, 232, - 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, + 233, 234, 235, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, -1, 281, 282, 283, 284, -1, 286, 287, 288, 289, 290, 291, -1, - 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, + 293, 294, -1, 296, 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, 328, 329, 330, 331, 332, @@ -15806,7 +15759,7 @@ static const yytype_int16 yycheck[] = 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 528, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, 43, @@ -15827,293 +15780,293 @@ static const yytype_int16 yycheck[] = 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, 213, - 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, - 224, 225, 226, 227, 228, 229, 230, -1, 232, 233, - 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, - 274, 275, 276, 277, 278, 279, -1, 281, 282, 283, - 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, - 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, - 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, - 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, - -1, 325, 326, 327, 328, 329, 330, 331, 332, 333, - 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, - 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, - 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, - 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, - 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, - 394, 395, 396, 397, 398, -1, 400, 401, 402, 403, - 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, - 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, - -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, - 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, - 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, - 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, - 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, - 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, - 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, - 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, - 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 528, -1, -1, -1, 23, 24, - 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, - 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, - 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, - 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, - -1, 76, 77, 78, 79, 80, -1, -1, -1, 84, - 85, 86, 87, 88, 89, -1, 91, 92, 93, -1, - 95, 96, 97, 98, 99, 100, -1, -1, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, - 125, 126, 127, 128, 129, 130, 131, 132, -1, -1, - 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, - 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, - 165, 166, 167, 168, -1, 170, -1, 172, -1, 174, - -1, 176, 177, 178, -1, 180, 181, 182, -1, 184, - 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, - 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, - 205, 206, 207, 208, -1, 210, -1, 212, 213, 214, - 215, 216, 217, 218, 219, -1, 221, -1, 223, -1, - -1, 226, -1, 228, 229, 230, -1, 232, 233, 234, - -1, -1, 237, -1, 239, -1, -1, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, - 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, - -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, - -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, - -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, -1, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, - 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, - 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, - 385, 386, 387, 388, -1, 390, 391, 392, 393, 394, - 395, 396, 397, 398, 399, -1, 401, 402, 403, 404, - -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 417, 418, 419, 420, 421, 422, 423, -1, - 425, 426, 427, 428, -1, 430, 431, 432, 433, 434, - 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, - 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, - 455, -1, -1, 458, 459, 460, -1, 462, 463, 464, - 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, - 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, - 485, -1, -1, 488, -1, 490, 491, 492, 493, 494, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, -1, 3, 514, - 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 528, -1, -1, -1, -1, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, - 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, - 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, - 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, - -1, 76, 77, 78, 79, 80, -1, -1, -1, 84, - 85, 86, 87, 88, 89, -1, 91, 92, 93, -1, - 95, 96, 97, 98, 99, 100, -1, -1, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, - 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, - 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, - 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, - 165, 166, 167, 168, -1, 170, -1, 172, -1, 174, - -1, 176, 177, 178, -1, 180, 181, 182, -1, 184, - 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, - 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, - 205, 206, 207, 208, -1, 210, -1, 212, 213, 214, - 215, 216, 217, 218, 219, -1, 221, -1, 223, -1, - -1, 226, -1, 228, 229, 230, -1, 232, 233, 234, - -1, -1, 237, -1, 239, -1, -1, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, - 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, - -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, - -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, - -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, -1, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, - 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, - 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, - 385, 386, 387, 388, -1, 390, 391, 392, 393, 394, - 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, - -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, - 425, 426, 427, 428, -1, 430, 431, 432, 433, 434, - 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, - 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, - 455, -1, -1, 458, 459, 460, -1, 462, 463, 464, - 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, - 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, - 485, -1, -1, 488, -1, 490, 491, 492, 493, 494, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, -1, 3, 514, - 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 528, -1, -1, -1, -1, 23, 24, - 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, - 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, - 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, - 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, - -1, 76, 77, 78, 79, 80, -1, -1, -1, 84, - 85, 86, 87, 88, 89, -1, 91, 92, 93, -1, - 95, 96, 97, 98, 99, 100, -1, -1, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, - 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, - 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, - 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, - 165, 166, 167, 168, -1, 170, -1, 172, -1, 174, - -1, 176, 177, 178, -1, 180, 181, 182, -1, 184, - 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, - 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, - 205, 206, 207, 208, -1, 210, -1, 212, 213, 214, - 215, 216, 217, 218, 219, -1, 221, -1, 223, -1, - -1, 226, -1, 228, 229, 230, -1, 232, 233, 234, - -1, -1, 237, -1, 239, -1, -1, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, - 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, - -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, - -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, - -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, -1, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, - 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, - 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, - 385, 386, 387, 388, -1, 390, 391, 392, 393, 394, - 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, - -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, - 425, 426, 427, 428, -1, 430, 431, 432, 433, 434, - 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, - 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, - 455, -1, -1, 458, 459, 460, -1, 462, 463, 464, - 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, - 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, - 485, -1, -1, 488, -1, 490, 491, 492, 493, 494, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, -1, 3, 514, - 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 528, -1, -1, -1, -1, 23, 24, + 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, + 224, 225, 226, 227, 228, 229, 230, -1, 232, 233, + 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, -1, 281, 282, 283, + 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, + 294, -1, 296, 297, 298, 299, -1, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, + -1, 325, 326, 327, 328, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, + 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, + 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, + 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, + 394, 395, 396, 397, 398, -1, 400, 401, 402, 403, + 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, + -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, + 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, + 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, + 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, + 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, + 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, + 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 528, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, - 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, - 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, - 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, - -1, 76, 77, 78, 79, 80, -1, -1, -1, 84, - 85, 86, 87, 88, 89, -1, 91, 92, 93, -1, - 95, 96, 97, 98, 99, 100, -1, -1, 103, 104, + -1, -1, -1, -1, -1, -1, -1, -1, 43, 44, + 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, -1, -1, 61, 62, 63, 64, + 65, -1, 67, 68, 69, 70, 71, 72, 73, 74, + -1, 76, 77, 78, 79, 80, -1, 82, -1, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, + 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, - 165, 166, 167, 168, -1, 170, -1, 172, -1, -1, - -1, 176, 177, 178, -1, 180, 181, 182, -1, 184, + 165, 166, 167, 168, -1, 170, -1, 172, 173, -1, + 175, 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, - 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, - 205, 206, 207, 208, -1, 210, -1, 212, 213, 214, - 215, 216, 217, 218, 219, -1, 221, -1, 223, -1, - -1, 226, -1, 228, 229, 230, -1, 232, 233, 234, - -1, -1, 237, -1, 239, -1, -1, 242, 243, 244, + 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, + 205, 206, 207, 208, -1, 210, 211, 212, 213, 214, + 215, 216, 217, 218, 219, -1, 221, -1, 223, 224, + 225, 226, 227, 228, 229, 230, -1, 232, 233, 234, + -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, - 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, 279, -1, 281, 282, 283, 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, - -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, + 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, + 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, + 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, - 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, + 395, 396, 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, - 425, 426, -1, 428, -1, 430, 431, 432, 433, 434, + 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, - 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, + 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 3, -1, 5, + 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 528, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, - -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, - -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, - -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, - 76, 77, 78, 79, 80, -1, -1, -1, 84, 85, - 86, 87, 88, 89, -1, 91, 92, 93, -1, 95, - 96, 97, 98, 99, 100, -1, -1, 103, 104, 105, + -1, -1, -1, -1, -1, -1, -1, 43, 44, 45, + -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, -1, -1, 61, 62, 63, 64, 65, + -1, 67, 68, 69, 70, 71, 72, 73, 74, -1, + 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, + 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, + 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, 165, - 166, 167, 168, -1, 170, -1, 172, -1, -1, -1, - 176, 177, 178, -1, 180, 181, 182, -1, 184, 185, + 166, 167, 168, -1, 170, -1, 172, 173, -1, 175, + 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, - -1, 197, 198, 199, 200, -1, 202, 203, 204, 205, - 206, 207, 208, -1, 210, -1, 212, 213, 214, 215, - 216, 217, 218, 219, -1, 221, -1, 223, -1, -1, - 226, -1, 228, 229, 230, -1, 232, 233, 234, -1, - -1, 237, -1, 239, -1, -1, 242, 243, 244, 245, + 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, + 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, -1, 221, -1, 223, 224, 225, + 226, 227, 228, 229, 230, -1, 232, 233, 234, -1, + -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, -1, 275, - 276, 277, 278, 279, -1, 281, 282, -1, 284, -1, + 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, + 276, 277, 278, 279, -1, 281, 282, 283, 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, - -1, 297, 298, 299, -1, -1, 302, 303, 304, -1, - 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, + -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, - 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, + 326, 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, + 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, 395, - 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, + 396, 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, - 426, -1, 428, -1, 430, 431, 432, 433, 434, 435, + 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, - -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, + -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, - 506, 507, 508, 509, 510, 511, 3, -1, 5, -1, + 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 528, -1, -1, -1, 23, 24, 25, 26, + 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, + -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, + 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, + -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, + 77, 78, 79, 80, -1, -1, -1, 84, 85, 86, + 87, 88, 89, -1, 91, 92, 93, -1, 95, 96, + 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, + 127, 128, 129, 130, 131, 132, -1, -1, 135, 136, + 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, + 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, + 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, + 167, 168, -1, 170, -1, 172, -1, 174, -1, 176, + 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, + 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, + 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, + 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, + 217, 218, 219, -1, 221, -1, 223, -1, -1, 226, + -1, 228, 229, 230, -1, 232, 233, 234, -1, -1, + 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, + 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, + 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, + 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, + -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, -1, 325, 326, + 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, + 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, + 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, + 387, 388, -1, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 399, -1, 401, 402, 403, 404, -1, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, + 427, 428, -1, 430, 431, 432, 433, 434, 435, -1, + 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, + -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, + -1, 458, 459, 460, -1, 462, 463, 464, 465, 466, + -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, + 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, + -1, 488, -1, 490, 491, 492, 493, 494, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, -1, 3, 514, 5, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 528, -1, -1, -1, -1, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, + -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, + 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, + -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, + 77, 78, 79, 80, -1, -1, -1, 84, 85, 86, + 87, 88, 89, -1, 91, 92, 93, -1, 95, 96, + 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, + -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, + 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, + 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, + 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, + 167, 168, -1, 170, -1, 172, -1, 174, -1, 176, + 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, + 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, + 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, + 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, + 217, 218, 219, -1, 221, -1, 223, -1, -1, 226, + -1, 228, 229, 230, -1, 232, 233, 234, -1, -1, + 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, + 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, + 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, + 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, + -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, -1, 325, 326, + 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, + 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, + 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, + 387, 388, -1, 390, 391, 392, 393, 394, 395, 396, + 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, + 427, 428, -1, 430, 431, 432, 433, 434, 435, -1, + 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, + -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, + -1, 458, 459, 460, -1, 462, 463, 464, 465, 466, + -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, + 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, + -1, 488, -1, 490, 491, 492, 493, 494, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, -1, 3, 514, 5, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 528, -1, -1, -1, -1, 23, 24, 25, 26, + 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, + -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, + 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, + -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, + 77, 78, 79, 80, -1, -1, -1, 84, 85, 86, + 87, 88, 89, -1, 91, 92, 93, -1, 95, 96, + 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, + -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, + 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, + 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, + 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, + 167, 168, -1, 170, -1, 172, -1, 174, -1, 176, + 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, + 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, + 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, + 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, + 217, 218, 219, -1, 221, -1, 223, -1, -1, 226, + -1, 228, 229, 230, -1, 232, 233, 234, -1, -1, + 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, + 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, + 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, + 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, + -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, -1, 325, 326, + 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, + 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, + 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, + 387, 388, -1, 390, 391, 392, 393, 394, 395, 396, + 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, + 427, 428, -1, 430, 431, 432, 433, 434, 435, -1, + 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, + -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, + -1, 458, 459, 460, -1, 462, 463, 464, 465, 466, + -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, + 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, + -1, 488, -1, 490, 491, 492, 493, 494, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, -1, 3, 514, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 528, -1, -1, -1, 23, 24, 25, 26, + -1, 528, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, @@ -16417,7 +16370,7 @@ static const yytype_int16 yycheck[] = 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 528, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, @@ -16456,389 +16409,122 @@ static const yytype_int16 yycheck[] = 363, 364, 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, - 393, 394, 395, 396, 397, 398, -1, -1, 401, 402, - 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, - 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - -1, -1, 425, 426, -1, 428, -1, 430, 431, 432, - 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, - 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, - 453, 454, 455, -1, -1, 458, 459, 460, -1, -1, - 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, - 473, 474, 475, -1, 477, -1, 479, 480, 481, 482, - 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, - 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, - -1, -1, -1, -1, -1, -1, -1, -1, 521, -1, - -1, -1, -1, -1, -1, 528, -1, -1, -1, 23, - 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, - 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, - 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, - 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, - 74, -1, 76, 77, 78, 79, 80, -1, -1, -1, - 84, 85, 86, 87, 88, 89, -1, 91, 92, 93, - -1, 95, 96, 97, 98, 99, 100, -1, -1, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, - 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, - -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, - -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, - 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, - -1, 165, 166, 167, 168, -1, 170, -1, 172, -1, - -1, -1, 176, 177, 178, -1, 180, 181, 182, -1, - 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, - 194, 195, -1, 197, 198, 199, 200, -1, 202, 203, - 204, 205, 206, 207, 208, -1, 210, -1, 212, 213, - 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, - -1, -1, 226, -1, 228, 229, 230, -1, 232, 233, - 234, -1, -1, 237, -1, 239, -1, -1, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, - -1, 275, 276, 277, 278, 279, -1, 281, 282, -1, - 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, - 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, - 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, - 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, - -1, 325, 326, 327, -1, 329, 330, 331, 332, 333, - 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, - 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, - 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, - 364, 365, 366, -1, 368, 369, -1, 371, 372, 373, - 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, - 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, - 394, 395, 396, 397, 398, -1, -1, 401, 402, 403, - 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, - 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, - -1, 425, 426, -1, 428, -1, 430, 431, 432, 433, - 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, - 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, - 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, - 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, - 474, 475, -1, 477, -1, 479, 480, 481, 482, 483, - 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, - 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, - 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 528, -1, -1, -1, 23, 24, - 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, - 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, - 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, - 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, - -1, 76, 77, 78, 79, 80, -1, -1, -1, 84, - 85, 86, 87, 88, 89, -1, 91, 92, 93, -1, - 95, 96, 97, 98, 99, 100, -1, -1, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, - 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, - 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, - 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, - 165, 166, 167, 168, -1, 170, -1, 172, -1, -1, - -1, 176, 177, 178, -1, 180, 181, 182, -1, 184, - 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, - 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, - 205, 206, 207, 208, -1, 210, -1, 212, 213, 214, - 215, 216, 217, 218, 219, -1, 221, -1, 223, -1, - -1, 226, -1, 228, 229, 230, -1, 232, 233, 234, - -1, -1, 237, -1, 239, -1, -1, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, - 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, - -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, - -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, - -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, - 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, - 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, - 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, - 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, - 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, - -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, - 425, 426, -1, 428, -1, 430, 431, 432, 433, 434, - 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, - 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, - 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, - 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, - 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, - 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 3, 4, 5, - -1, -1, 8, 9, -1, -1, -1, -1, -1, 15, - 16, -1, -1, 528, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, - 46, 47, 48, 49, 50, 51, 52, 53, -1, 55, - 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, - 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, - 76, 77, 78, 79, -1, 81, 82, 83, 84, 85, - 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, - 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, - 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, - 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, - 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 150, 151, 152, 153, -1, 155, - 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, - 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, - 176, 177, 178, 179, 180, 181, 182, 183, -1, -1, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, - 206, 207, 208, 209, 210, 211, -1, 213, 214, 215, - 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, - 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, - 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, - -1, 287, 288, 289, 290, 291, 292, 293, 294, 295, - 296, 297, 298, 299, 300, 301, 302, 303, -1, 305, - 306, 307, -1, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, -1, 328, 329, 330, -1, 332, 333, 334, 335, - 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, - 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, - 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, - 416, 417, 418, 419, 420, 421, -1, 423, 424, 425, - 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 443, -1, 445, - -1, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, - 466, 467, 468, 469, 470, 471, 472, -1, 474, 475, - 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, - 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, - 496, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 508, 509, 510, 511, -1, 3, -1, 515, - 516, 517, 8, 519, 520, 521, 522, 523, 524, 15, - 16, -1, -1, -1, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, - -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, - -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, - -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, - 76, 77, 78, 79, 80, -1, -1, -1, 84, 85, - 86, 87, 88, 89, -1, 91, 92, 93, -1, 95, - 96, 97, 98, 99, 100, -1, -1, 103, 104, 105, - 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, - 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, - 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, - 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, - 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, - 156, 157, 158, 159, 160, 161, -1, 163, -1, 165, - 166, 167, 168, -1, 170, -1, 172, -1, -1, -1, - 176, 177, 178, -1, 180, 181, 182, -1, 184, 185, - 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, - -1, 197, 198, 199, 200, -1, 202, 203, 204, 205, - 206, 207, 208, -1, 210, -1, 212, 213, 214, 215, - 216, 217, 218, 219, -1, 221, -1, 223, -1, -1, - 226, -1, 228, 229, 230, -1, 232, 233, 234, -1, - -1, 237, -1, 239, -1, -1, 242, 243, 244, 245, - 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, -1, 275, - 276, 277, 278, 279, -1, 281, 282, -1, 284, -1, - 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, - -1, 297, 298, 299, -1, -1, 302, 303, 304, -1, - 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, - 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, - 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, - 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, - 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, - 386, 387, -1, -1, 390, 391, 392, 393, 394, 395, - 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, - 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, - 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, - 426, -1, 428, -1, 430, 431, 432, 433, 434, 435, - -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, - 446, -1, 448, 449, 450, 451, 452, 453, 454, 455, - -1, -1, 458, 459, 460, -1, -1, 463, 464, 465, - 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, - -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, - -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, - 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, - 506, 507, 508, 509, 510, 511, -1, -1, -1, 515, - 516, 517, -1, 519, 520, 521, 522, 523, 524, 8, - -1, -1, 11, -1, -1, -1, 15, 16, 17, 18, - -1, 20, 21, 22, -1, -1, -1, 8, -1, -1, - 11, -1, -1, -1, 15, 16, 17, 18, 37, 20, - 21, 22, -1, -1, -1, -1, -1, -1, 47, -1, - -1, -1, -1, -1, -1, 54, 37, -1, -1, -1, - -1, -1, 8, -1, -1, 11, 47, -1, -1, 15, - 16, 17, 18, 54, 20, 21, 22, -1, -1, -1, - 8, -1, 81, 11, -1, -1, -1, 15, 16, 17, - 18, 37, 20, 21, 22, -1, -1, -1, -1, -1, - 81, 47, -1, -1, -1, -1, -1, -1, 54, 37, - -1, -1, -1, -1, -1, 8, -1, -1, 11, 47, - -1, -1, 15, 16, 17, 18, 54, 20, 21, 22, - -1, -1, -1, -1, -1, 81, -1, -1, -1, -1, - -1, -1, -1, -1, 37, -1, -1, -1, -1, -1, - -1, -1, -1, 81, 47, -1, -1, 8, -1, -1, - 11, 54, -1, -1, 15, 16, 17, 18, -1, 20, - 21, 22, -1, -1, -1, -1, -1, -1, -1, -1, - 179, -1, -1, -1, -1, -1, 37, -1, 81, -1, - -1, -1, -1, -1, -1, -1, 47, 196, 179, -1, - -1, -1, 201, 54, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 196, -1, -1, -1, -1, - 201, -1, -1, -1, -1, 224, 225, -1, -1, -1, - 81, -1, -1, 179, -1, -1, -1, -1, -1, -1, - -1, 240, -1, 224, 225, -1, -1, -1, -1, -1, - 196, 179, -1, -1, -1, 201, -1, -1, -1, 240, - -1, -1, -1, -1, -1, -1, -1, -1, 196, -1, - -1, -1, -1, 201, -1, -1, -1, -1, 224, 225, - -1, 280, -1, -1, 283, -1, 179, -1, -1, -1, - -1, -1, -1, -1, 240, -1, 224, 225, 297, 280, - -1, 300, 283, 196, -1, -1, -1, -1, 201, -1, - -1, -1, 240, -1, -1, -1, 297, -1, -1, 300, - -1, -1, -1, -1, -1, -1, -1, -1, 179, -1, - -1, 224, 225, -1, 280, -1, -1, 283, -1, -1, - -1, -1, -1, -1, -1, 196, -1, 240, -1, -1, - 201, 297, 280, -1, 300, 283, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 297, - -1, -1, 300, 224, 225, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 280, -1, 240, - 283, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 400, -1, -1, 297, -1, -1, 300, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 400, - -1, -1, -1, -1, -1, -1, -1, 8, -1, 280, - 11, -1, 283, -1, 15, 16, 17, 18, -1, 20, - 21, 22, -1, -1, -1, -1, 297, -1, -1, 300, - -1, -1, -1, -1, 400, -1, 37, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 47, -1, -1, -1, - -1, -1, 400, 54, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 81, -1, -1, -1, -1, -1, -1, 400, -1, -1, - -1, -1, -1, 512, -1, -1, 515, 516, 517, -1, - 519, 520, 521, 522, 523, 524, -1, -1, -1, -1, - 529, 512, -1, -1, 515, 516, 517, -1, 519, 520, - 521, 522, 523, 524, -1, -1, -1, -1, 529, 400, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 512, -1, -1, 515, - 516, 517, -1, 519, 520, 521, 522, 523, 524, -1, - -1, -1, -1, 529, 512, -1, -1, 515, 516, 517, - -1, 519, 520, 521, 522, 523, 524, -1, 179, -1, - -1, 529, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 196, -1, -1, -1, 512, - 201, -1, 515, 516, 517, -1, 519, 520, 521, 522, - 523, 524, -1, -1, 527, -1, -1, -1, 8, -1, - -1, 11, -1, 224, 225, 15, 16, 17, 18, -1, - 20, 21, 22, -1, -1, -1, -1, -1, -1, 240, - -1, 512, -1, -1, 515, 516, 517, 37, 519, 520, - 521, 522, 523, 524, -1, -1, 527, 47, -1, -1, - -1, -1, -1, -1, 54, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 280, - -1, -1, 283, -1, -1, -1, -1, -1, -1, -1, - -1, 81, -1, -1, -1, -1, 297, -1, -1, 300, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 179, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 400, - -1, -1, -1, -1, -1, -1, 196, -1, -1, -1, - -1, 201, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 224, 225, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 240, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 280, -1, -1, 283, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 297, -1, -1, - 300, 512, -1, -1, 515, 516, 517, -1, 519, 520, - 521, 522, 523, 524, -1, -1, 527, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 393, 394, 395, 396, 397, 398, -1, -1, 401, 402, + 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, + 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, + -1, -1, 425, 426, -1, 428, -1, 430, 431, 432, + 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, + 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, + 453, 454, 455, -1, -1, 458, 459, 460, -1, -1, + 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, + 473, 474, 475, -1, 477, -1, 479, 480, 481, 482, + 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, + 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 528, -1, -1, -1, 23, + 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, + 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, + 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, + 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, + 74, -1, 76, 77, 78, 79, 80, -1, -1, -1, + 84, 85, 86, 87, 88, 89, -1, 91, 92, 93, + -1, 95, 96, 97, 98, 99, 100, -1, -1, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, + 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, + -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, + -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, + -1, 165, 166, 167, 168, -1, 170, -1, 172, -1, + -1, -1, 176, 177, 178, -1, 180, 181, 182, -1, + 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, + 194, 195, -1, 197, 198, 199, 200, -1, 202, 203, + 204, 205, 206, 207, 208, -1, 210, -1, 212, 213, + 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, + -1, -1, 226, -1, 228, 229, 230, -1, 232, 233, + 234, -1, -1, 237, -1, 239, -1, -1, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + -1, 275, 276, 277, 278, 279, -1, 281, 282, -1, + 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, + 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, + 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, + -1, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, + 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, + 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, -1, 368, 369, -1, 371, 372, 373, + 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, + 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, + 394, 395, 396, 397, 398, -1, -1, 401, 402, 403, + 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, + -1, 425, 426, -1, 428, -1, 430, 431, 432, 433, + 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, + 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, + 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, + 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, + 474, 475, -1, 477, -1, 479, 480, 481, 482, 483, + 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, + 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 3, -1, - -1, -1, 512, -1, -1, 515, 516, 517, -1, 519, - 520, 521, 522, 523, 524, -1, -1, 527, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, - 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, - 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, - 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, - 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + -1, -1, -1, -1, 528, -1, -1, -1, 23, 24, + 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, + 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, + 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, + 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, + -1, 76, 77, 78, 79, 80, -1, -1, -1, 84, + 85, 86, 87, 88, 89, -1, 91, 92, 93, -1, + 95, 96, 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, - 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, - 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, - 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, - 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, - 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, - 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, + 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, + 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, + 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, + 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, + 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, + 165, 166, 167, 168, -1, 170, -1, 172, -1, -1, + -1, 176, 177, 178, -1, 180, 181, 182, -1, 184, + 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, + 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, + 205, 206, 207, 208, -1, 210, -1, 212, 213, 214, + 215, 216, 217, 218, 219, -1, 221, -1, 223, -1, + -1, 226, -1, 228, 229, 230, -1, 232, 233, 234, + -1, -1, 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, - 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, - 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, + 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, + 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, + -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, + -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, + -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, + 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, + -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, + 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, + 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, - 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, - 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, - 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, - 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, - 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, - 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, - 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, + 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, + 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, + -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, + 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, + 425, 426, -1, 428, -1, 430, 431, 432, 433, 434, + 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, + 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, + 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, + 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, + 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, + 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, 521, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, + -1, -1, -1, 528, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, @@ -16888,8 +16574,8 @@ static const yytype_int16 yycheck[] = -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, - -1, -1, -1, -1, -1, 521, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 528, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, @@ -16899,405 +16585,670 @@ static const yytype_int16 yycheck[] = 87, 88, 89, -1, 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, - -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, - 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, - 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, - 167, 168, -1, 170, -1, 172, -1, -1, -1, 176, - 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, - 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, - 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, - 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, - 217, 218, 219, -1, 221, -1, 223, -1, -1, 226, - -1, 228, 229, 230, -1, 232, 233, 234, -1, -1, - 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, + 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, + -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, + 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, + 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, + 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, + 167, 168, -1, 170, -1, 172, -1, -1, -1, 176, + 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, + 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, + 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, + 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, + 217, 218, 219, -1, 221, -1, 223, -1, -1, 226, + -1, 228, 229, 230, -1, 232, 233, 234, -1, -1, + 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, + 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, + 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, + 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, + -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, + 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, + 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, + 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, + 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, + 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, + -1, 428, -1, 430, 431, 432, 433, 434, 435, -1, + 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, + -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, + -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, + -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, + 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, + -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 3, 4, 5, -1, -1, + 8, 9, -1, -1, -1, -1, -1, 15, 16, -1, + -1, 528, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, + 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, -1, 55, 56, 57, + 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, + 78, 79, -1, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, + 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 148, 149, 150, 151, 152, 153, -1, 155, 156, 157, + 158, 159, 160, -1, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, + 178, 179, 180, 181, 182, 183, -1, -1, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, -1, 213, 214, 215, 216, 217, + 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, + 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 298, 299, 300, 301, 302, 303, -1, 305, 306, 307, + -1, 309, 310, 311, 312, 313, 314, 315, 316, 317, + 318, 319, 320, 321, 322, 323, 324, 325, 326, -1, + 328, 329, 330, -1, 332, 333, 334, 335, 336, 337, + 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, -1, 423, 424, 425, 426, 427, + 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, -1, 445, -1, 447, + 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, + 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, + 468, 469, 470, 471, 472, -1, 474, 475, 476, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 508, 509, 510, 511, -1, 3, -1, 515, 516, 517, + 8, 519, 520, 521, 522, 523, 524, 15, 16, -1, + -1, -1, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, + 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, + 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, + 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, + 78, 79, 80, -1, -1, -1, 84, 85, 86, 87, + 88, 89, -1, 91, 92, 93, -1, 95, 96, 97, + 98, 99, 100, -1, -1, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, + -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, + 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, + 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 160, 161, -1, 163, -1, 165, 166, 167, + 168, -1, 170, -1, 172, -1, -1, -1, 176, 177, + 178, -1, 180, 181, 182, -1, 184, 185, 186, 187, + -1, 189, 190, 191, 192, 193, 194, 195, -1, 197, + 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, + 208, -1, 210, -1, 212, 213, 214, 215, 216, 217, + 218, 219, -1, 221, -1, 223, -1, -1, 226, -1, + 228, 229, 230, -1, 232, 233, 234, -1, -1, 237, + -1, 239, -1, -1, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + 268, 269, 270, 271, 272, 273, -1, 275, 276, 277, + 278, 279, -1, 281, 282, -1, 284, -1, 286, 287, + 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, + 298, 299, -1, -1, 302, 303, 304, -1, 306, -1, + 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, + 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, + 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, + 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, + 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, + 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, + -1, -1, 390, 391, 392, 393, 394, 395, 396, 397, + 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, + 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, + 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, + 428, -1, 430, 431, 432, 433, 434, 435, -1, 437, + 438, 439, -1, -1, 442, 443, 444, 445, 446, -1, + 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, + 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, + 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, + -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, + 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, + 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, + 508, 509, 510, 511, -1, -1, -1, 515, 516, 517, + -1, 519, 520, 521, 522, 523, 524, 8, -1, -1, + 11, -1, -1, -1, 15, 16, 17, 18, -1, 20, + 21, 22, -1, -1, -1, 8, -1, -1, 11, -1, + -1, -1, 15, 16, 17, 18, 37, 20, 21, 22, + -1, -1, -1, -1, -1, -1, 47, -1, -1, -1, + -1, -1, -1, 54, 37, -1, -1, -1, -1, -1, + 8, -1, -1, 11, 47, -1, -1, 15, 16, 17, + 18, 54, 20, 21, 22, -1, -1, -1, 8, -1, + 81, 11, -1, -1, -1, 15, 16, 17, 18, 37, + 20, 21, 22, -1, -1, -1, -1, -1, 81, 47, + -1, -1, -1, -1, -1, -1, 54, 37, -1, -1, + -1, -1, -1, 8, -1, -1, 11, 47, -1, -1, + 15, 16, 17, 18, 54, 20, 21, 22, -1, -1, + -1, -1, -1, 81, -1, -1, -1, -1, -1, -1, + -1, -1, 37, -1, -1, -1, -1, -1, -1, -1, + -1, 81, 47, -1, -1, 8, -1, -1, 11, 54, + -1, -1, 15, 16, 17, 18, -1, 20, 21, 22, + -1, -1, -1, -1, -1, -1, -1, -1, 179, -1, + -1, -1, -1, -1, 37, -1, 81, -1, -1, -1, + -1, -1, -1, -1, 47, 196, 179, -1, -1, -1, + 201, 54, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 196, -1, -1, -1, -1, 201, -1, + -1, -1, -1, 224, 225, -1, -1, -1, 81, -1, + -1, 179, -1, -1, -1, -1, -1, -1, -1, 240, + -1, 224, 225, -1, -1, -1, -1, -1, 196, 179, + -1, -1, -1, 201, -1, -1, -1, 240, -1, -1, + -1, -1, -1, -1, -1, -1, 196, -1, -1, -1, + -1, 201, -1, -1, -1, -1, 224, 225, -1, 280, + -1, -1, 283, -1, 179, -1, -1, -1, -1, -1, + -1, -1, 240, -1, 224, 225, 297, 280, -1, 300, + 283, 196, -1, -1, -1, -1, 201, -1, -1, -1, + 240, -1, -1, -1, 297, -1, -1, 300, -1, -1, + -1, -1, -1, -1, -1, -1, 179, -1, -1, 224, + 225, -1, 280, -1, -1, 283, -1, -1, -1, -1, + -1, -1, -1, 196, -1, 240, -1, -1, 201, 297, + 280, -1, 300, 283, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 297, -1, -1, + 300, 224, 225, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 280, -1, 240, 283, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 400, + -1, -1, 297, -1, -1, 300, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 400, -1, -1, + -1, -1, -1, -1, -1, 8, -1, 280, 11, -1, + 283, -1, 15, 16, 17, 18, -1, 20, 21, 22, + -1, -1, -1, -1, 297, -1, -1, 300, -1, -1, + -1, -1, 400, -1, 37, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 47, -1, -1, -1, -1, -1, + 400, 54, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 81, -1, + -1, -1, -1, -1, -1, 400, -1, -1, -1, -1, + -1, 512, -1, -1, 515, 516, 517, -1, 519, 520, + 521, 522, 523, 524, -1, -1, -1, -1, 529, 512, + -1, -1, 515, 516, 517, -1, 519, 520, 521, 522, + 523, 524, -1, -1, -1, -1, 529, 400, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 512, -1, -1, 515, 516, 517, + -1, 519, 520, 521, 522, 523, 524, -1, -1, -1, + -1, 529, 512, -1, -1, 515, 516, 517, -1, 519, + 520, 521, 522, 523, 524, -1, 179, -1, -1, 529, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 196, -1, -1, -1, 512, 201, -1, + 515, 516, 517, -1, 519, 520, 521, 522, 523, 524, + -1, -1, -1, -1, 529, -1, 8, -1, -1, 11, + -1, 224, 225, 15, 16, 17, 18, -1, 20, 21, + 22, -1, -1, -1, -1, -1, -1, 240, -1, 512, + -1, -1, 515, 516, 517, 37, 519, 520, 521, 522, + 523, 524, -1, -1, 527, 47, -1, -1, -1, -1, + -1, -1, 54, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 280, -1, -1, + 283, -1, -1, -1, -1, -1, -1, -1, -1, 81, + -1, -1, -1, -1, 297, -1, -1, 300, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 179, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 400, -1, -1, + -1, -1, -1, -1, 196, -1, -1, -1, -1, 201, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 224, 225, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 240, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 280, -1, + -1, 283, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 297, -1, -1, 300, 512, + -1, -1, 515, 516, 517, -1, 519, 520, 521, 522, + 523, 524, -1, -1, 527, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 400, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 3, -1, -1, -1, + 512, -1, -1, 515, 516, 517, -1, 519, 520, 521, + 522, 523, 524, -1, -1, 527, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, + 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, + 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, + 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, + 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, + 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, + 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, + 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, + 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, - 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, - 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, - 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, - -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, - 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, + 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, + 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, + 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, + 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, + 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, + 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, - 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, - 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, - -1, 428, -1, 430, 431, 432, 433, 434, 435, -1, - 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, - -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, - -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, - -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, - 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, - -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, + 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, + 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, + 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, + 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, + 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 508, 509, 510, 511, 3, 4, 5, -1, -1, - -1, 9, -1, -1, 521, -1, -1, -1, -1, -1, + 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, + -1, -1, -1, -1, 521, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, - 38, -1, -1, -1, -1, 43, 44, 45, -1, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, -1, 60, 61, 62, 63, 64, 65, -1, -1, + -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, + 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, + 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, - 78, 79, 80, -1, 82, -1, 84, 85, 86, 87, - 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, - 98, 99, 100, -1, 102, 103, 104, 105, 106, 107, + 78, 79, 80, -1, -1, -1, 84, 85, 86, 87, + 88, 89, -1, 91, 92, 93, -1, 95, 96, 97, + 98, 99, 100, -1, -1, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, - 158, 159, 160, 161, 162, 163, -1, 165, 166, 167, - 168, -1, 170, -1, 172, 173, -1, 175, 176, 177, - 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, - -1, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 158, 159, 160, 161, -1, 163, -1, 165, 166, 167, + 168, -1, 170, -1, 172, -1, -1, -1, 176, 177, + 178, -1, 180, 181, 182, -1, 184, 185, 186, 187, + -1, 189, 190, 191, 192, 193, 194, 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, - 208, -1, 210, 211, 212, 213, 214, 215, 216, 217, - 218, 219, -1, 221, -1, 223, 224, 225, 226, 227, + 208, -1, 210, -1, 212, 213, 214, 215, 216, 217, + 218, 219, -1, 221, -1, 223, -1, -1, 226, -1, 228, 229, 230, -1, 232, 233, 234, -1, -1, 237, - 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, + -1, 239, -1, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, - 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, - 278, 279, -1, 281, 282, 283, 284, -1, 286, 287, - 288, 289, 290, 291, -1, 293, 294, 295, -1, 297, - 298, 299, -1, -1, 302, 303, 304, 305, 306, 307, + 268, 269, 270, 271, 272, 273, -1, 275, 276, 277, + 278, 279, -1, 281, 282, -1, 284, -1, 286, 287, + 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, + 298, 299, -1, -1, 302, 303, 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, - 328, 329, 330, 331, 332, 333, 334, -1, 336, 337, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, - 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, - -1, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, + -1, -1, 390, 391, 392, 393, 394, 395, 396, 397, + 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, - 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, - 438, 439, -1, -1, 442, 443, 444, 445, 446, 447, + 428, -1, 430, 431, 432, 433, 434, 435, -1, 437, + 438, 439, -1, -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, - 458, 459, 460, 461, -1, 463, 464, 465, 466, -1, + 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, - 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, + -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, - 508, 509, 510, 511, -1, -1, 8, -1, -1, 11, - -1, 519, 520, 15, 16, 17, 18, -1, 20, 21, - 22, -1, -1, -1, 8, -1, -1, 11, -1, -1, - -1, 15, 16, 17, 18, 37, 20, 21, 22, -1, - 42, -1, -1, -1, -1, 47, -1, -1, -1, -1, - -1, -1, 54, 37, -1, -1, -1, -1, -1, 8, - -1, -1, 11, 47, -1, -1, 15, 16, 17, 18, - 54, 20, 21, 22, -1, -1, -1, 8, -1, 81, - 11, -1, -1, -1, 15, 16, 17, 18, 37, 20, - 21, 22, -1, -1, -1, -1, -1, 81, 47, -1, - -1, -1, -1, -1, -1, 54, 37, -1, -1, -1, - 41, -1, -1, -1, -1, -1, 47, -1, -1, -1, - -1, -1, -1, 54, -1, 127, -1, -1, -1, -1, - -1, -1, 81, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 81, -1, -1, -1, 8, -1, -1, 11, -1, -1, - -1, 15, 16, 17, 18, -1, 20, 21, 22, -1, - -1, -1, -1, -1, -1, -1, -1, 179, -1, -1, - -1, -1, -1, 37, -1, 169, -1, -1, -1, -1, - 174, -1, -1, 47, 196, 179, -1, -1, -1, 201, - 54, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 196, -1, -1, -1, -1, 201, -1, 168, - -1, -1, 224, 225, -1, -1, -1, 81, -1, -1, - 179, -1, -1, -1, -1, -1, -1, -1, 240, -1, - 224, 225, -1, -1, -1, -1, -1, 196, 179, -1, - -1, -1, 201, -1, -1, -1, 240, -1, -1, -1, - -1, -1, -1, -1, -1, 196, -1, -1, -1, -1, - 201, -1, -1, -1, -1, 224, 225, -1, 280, -1, - -1, 283, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 240, -1, 224, 225, 297, 280, -1, 300, 283, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 240, - -1, -1, -1, 297, -1, -1, 300, -1, -1, -1, - -1, -1, -1, -1, -1, 179, -1, -1, -1, -1, - -1, 280, -1, -1, 283, -1, -1, -1, -1, -1, - -1, -1, 196, -1, -1, -1, -1, 201, 297, 280, - -1, 300, 283, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 297, -1, -1, 300, - 224, 225, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 330, -1, -1, -1, -1, 240, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 400, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 8, -1, -1, 11, -1, 400, -1, 15, 16, - 17, 18, -1, 20, 21, 22, 280, -1, -1, 283, + 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, + -1, -1, -1, 521, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, + 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, + 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, + -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, + 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, + 79, 80, -1, -1, -1, 84, 85, 86, 87, 88, + 89, -1, 91, 92, 93, -1, 95, 96, 97, 98, + 99, 100, -1, -1, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, + 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, + 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, + 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, + -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, 161, -1, 163, -1, 165, 166, 167, 168, + -1, 170, -1, 172, -1, -1, -1, 176, 177, 178, + -1, 180, 181, 182, -1, 184, 185, 186, 187, -1, + 189, 190, 191, 192, 193, 194, 195, -1, 197, 198, + 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, + -1, 210, -1, 212, 213, 214, 215, 216, 217, 218, + 219, -1, 221, -1, 223, -1, -1, 226, -1, 228, + 229, 230, -1, 232, 233, 234, -1, -1, 237, -1, + 239, -1, -1, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, + 279, -1, 281, 282, -1, 284, -1, 286, 287, 288, + 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, + 299, -1, -1, 302, 303, 304, -1, 306, -1, 308, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 320, -1, -1, -1, -1, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, + 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, + 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, + 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, + -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, + -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, + -1, 430, 431, 432, 433, 434, 435, -1, 437, 438, + 439, -1, -1, 442, 443, 444, 445, 446, -1, 448, + 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, + 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, + 469, 470, 471, 472, 473, 474, 475, -1, 477, -1, + 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, + -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, + 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 510, 511, 3, 4, 5, -1, -1, -1, 9, + -1, -1, 521, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, + 30, -1, 32, 33, 34, -1, -1, -1, 38, -1, + -1, -1, -1, 43, 44, 45, -1, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, -1, + 60, 61, 62, 63, 64, 65, -1, -1, 68, 69, + 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, + 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, + 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, + -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, + 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, + -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, -1, 165, 166, 167, 168, -1, + 170, -1, 172, 173, -1, 175, 176, 177, 178, 179, + 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, + 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, + -1, 221, -1, 223, 224, 225, 226, 227, 228, 229, + 230, -1, 232, 233, 234, -1, -1, 237, 238, 239, + 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, + -1, 281, 282, 283, 284, -1, 286, 287, 288, 289, + 290, 291, -1, 293, 294, 295, -1, 297, 298, 299, + -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, + 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, + 320, -1, -1, -1, -1, 325, 326, 327, 328, 329, + 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, + 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, + 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, + 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, + 380, 381, 382, 383, 384, 385, 386, 387, -1, 389, + 390, 391, 392, 393, 394, 395, 396, 397, 398, -1, + 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, + 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, + 420, 421, 422, -1, -1, 425, 426, -1, 428, 429, + 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, + -1, -1, 442, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, + 460, 461, -1, 463, 464, 465, 466, -1, 468, 469, + 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, + 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, + -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, + 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, + 510, 511, -1, -1, 8, -1, -1, 11, -1, 519, + 520, 15, 16, 17, 18, -1, 20, 21, 22, -1, + -1, -1, 8, -1, -1, 11, -1, -1, -1, 15, + 16, 17, 18, 37, 20, 21, 22, -1, 42, -1, + -1, -1, -1, 47, -1, -1, -1, -1, -1, -1, + 54, 37, -1, -1, -1, -1, -1, 8, -1, -1, + 11, 47, -1, -1, 15, 16, 17, 18, 54, 20, + 21, 22, -1, -1, -1, 8, -1, 81, 11, -1, + -1, -1, 15, 16, 17, 18, 37, 20, 21, 22, + -1, -1, -1, -1, -1, 81, 47, -1, -1, -1, + -1, -1, -1, 54, 37, -1, -1, -1, 41, -1, + -1, -1, -1, -1, 47, -1, -1, -1, -1, -1, + -1, 54, -1, 127, -1, -1, -1, -1, -1, -1, + 81, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 81, -1, + -1, -1, 8, -1, -1, 11, -1, -1, -1, 15, + 16, 17, 18, -1, 20, 21, 22, -1, -1, -1, + -1, -1, -1, -1, -1, 179, -1, -1, -1, -1, + -1, 37, -1, 169, -1, -1, -1, -1, 174, -1, + -1, 47, 196, 179, -1, -1, -1, 201, 54, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 37, -1, -1, 297, 41, -1, 300, -1, -1, -1, - 47, 400, -1, -1, -1, -1, -1, 54, -1, -1, - -1, -1, -1, -1, -1, 467, -1, -1, -1, 400, - 324, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 81, -1, -1, -1, -1, -1, + 196, -1, -1, -1, -1, 201, -1, 168, -1, -1, + 224, 225, -1, -1, -1, 81, -1, -1, 179, -1, + -1, -1, -1, -1, -1, -1, 240, -1, 224, 225, + -1, -1, -1, -1, -1, 196, 179, -1, -1, -1, + 201, -1, -1, -1, 240, -1, -1, -1, -1, -1, + -1, -1, -1, 196, -1, -1, -1, -1, 201, -1, + -1, -1, -1, 224, 225, -1, 280, -1, -1, 283, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 240, + -1, 224, 225, 297, 280, -1, 300, 283, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 240, -1, -1, + -1, 297, -1, -1, 300, -1, -1, -1, -1, -1, + -1, -1, -1, 179, -1, -1, -1, -1, -1, 280, + -1, -1, 283, -1, -1, -1, -1, -1, -1, -1, + 196, -1, -1, -1, -1, 201, 297, 280, -1, 300, + 283, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 297, -1, -1, 300, 224, 225, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 330, + -1, -1, -1, -1, 240, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 400, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 8, + -1, -1, 11, -1, 400, -1, 15, 16, 17, 18, + -1, 20, 21, 22, 280, -1, -1, 283, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 37, -1, + -1, 297, 41, -1, 300, -1, -1, -1, 47, 400, + -1, -1, -1, -1, -1, 54, -1, -1, -1, -1, + -1, -1, -1, 467, -1, -1, -1, 400, 324, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 81, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 512, -1, -1, 515, 516, 517, -1, 519, 520, 521, - 522, 523, 524, -1, -1, -1, -1, -1, 512, -1, - -1, 515, 516, 517, -1, 519, 520, 521, 522, 523, - 524, -1, -1, -1, -1, -1, 400, -1, 8, -1, - -1, 11, -1, -1, -1, 15, 16, 17, 18, -1, - 20, 21, 22, 512, -1, -1, 515, 516, 517, -1, - 519, 520, 521, 522, 523, 524, -1, 37, -1, -1, - -1, 512, 179, -1, 515, 516, 517, 47, 519, 520, - 521, 522, 523, 524, 54, -1, -1, -1, -1, 196, - 8, -1, -1, 11, 201, -1, -1, 15, 16, 17, - 18, -1, 20, 21, 22, -1, -1, -1, -1, -1, - -1, 81, -1, -1, -1, -1, -1, 224, 225, 37, - -1, -1, -1, 41, -1, -1, -1, -1, -1, 47, - -1, -1, -1, 240, -1, -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 512, -1, -1, 515, 516, 517, -1, 519, 520, 521, 522, 523, - 524, -1, -1, 81, -1, -1, -1, -1, -1, -1, - -1, 8, -1, 280, 11, -1, 283, -1, 15, 16, - 17, 18, -1, 20, 21, 22, -1, -1, -1, -1, - 297, -1, -1, 300, -1, -1, -1, -1, -1, -1, - 37, -1, -1, -1, 174, -1, -1, -1, -1, 179, - 47, -1, -1, -1, -1, -1, -1, 54, -1, -1, + 524, -1, -1, -1, -1, -1, 512, -1, -1, 515, + 516, 517, -1, 519, 520, 521, 522, 523, 524, -1, + -1, -1, -1, -1, 400, -1, 8, -1, -1, 11, + -1, -1, -1, 15, 16, 17, 18, -1, 20, 21, + 22, 512, -1, -1, 515, 516, 517, -1, 519, 520, + 521, 522, 523, 524, -1, 37, -1, -1, -1, 512, + 179, -1, 515, 516, 517, 47, 519, 520, 521, 522, + 523, 524, 54, -1, -1, -1, -1, 196, 8, -1, + -1, 11, 201, -1, -1, 15, 16, 17, 18, -1, + 20, 21, 22, -1, -1, -1, -1, -1, -1, 81, + -1, -1, -1, -1, -1, 224, 225, 37, -1, -1, + -1, 41, -1, -1, -1, -1, -1, 47, -1, -1, + -1, 240, -1, -1, 54, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 512, -1, -1, 515, + 516, 517, -1, 519, 520, 521, 522, 523, 524, -1, + -1, 81, -1, -1, -1, -1, -1, -1, -1, 8, + -1, 280, 11, -1, 283, -1, 15, 16, 17, 18, + -1, 20, 21, 22, -1, -1, -1, -1, 297, -1, + -1, 300, -1, -1, -1, -1, -1, -1, 37, -1, + -1, -1, 174, -1, -1, -1, -1, 179, 47, -1, + -1, -1, -1, -1, -1, 54, -1, -1, -1, -1, + -1, -1, -1, -1, 196, -1, -1, -1, -1, 201, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 81, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 224, 225, -1, -1, -1, -1, -1, 179, + -1, -1, -1, -1, -1, -1, -1, -1, 240, -1, -1, -1, -1, -1, -1, -1, 196, -1, -1, -1, -1, 201, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 81, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 224, 225, -1, -1, -1, -1, - -1, 179, -1, -1, -1, -1, -1, -1, -1, -1, - 240, -1, -1, -1, -1, -1, -1, -1, 196, -1, - -1, -1, -1, 201, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 224, 225, -1, -1, - 280, -1, -1, 283, -1, -1, -1, -1, -1, -1, - -1, -1, 240, -1, -1, -1, -1, 297, -1, -1, - 300, -1, 169, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 179, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 196, - -1, -1, 280, -1, 201, 283, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 297, - -1, -1, 300, -1, -1, -1, -1, 224, 225, -1, + -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 224, 225, -1, -1, 280, -1, + -1, 283, -1, -1, -1, -1, -1, -1, -1, -1, + 240, -1, -1, -1, -1, 297, -1, -1, 300, -1, + 169, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 179, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 196, -1, -1, + 280, -1, 201, 283, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 297, -1, -1, + 300, -1, -1, -1, -1, 224, 225, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 240, -1, 512, -1, -1, 515, 516, - 517, -1, 519, 520, 521, 522, 523, 524, -1, -1, - -1, -1, -1, -1, -1, 8, -1, -1, 11, -1, - 400, -1, 15, 16, 17, 18, 19, 20, 21, 22, - -1, -1, -1, 280, -1, -1, 283, -1, -1, -1, - -1, -1, -1, -1, 37, -1, -1, -1, -1, -1, - 297, -1, -1, 300, 47, -1, -1, -1, -1, -1, - -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 81, -1, - -1, -1, 8, -1, -1, 11, -1, -1, -1, 15, - 16, 17, 18, -1, 20, 21, 22, -1, -1, -1, + -1, 240, -1, 512, -1, -1, 515, 516, 517, -1, + 519, 520, 521, 522, 523, 524, -1, -1, -1, -1, + -1, -1, -1, 8, -1, -1, 11, -1, 400, -1, + 15, 16, 17, 18, 19, 20, 21, 22, -1, -1, + -1, 280, -1, -1, 283, -1, -1, -1, -1, -1, + -1, -1, 37, -1, -1, -1, -1, -1, 297, -1, + -1, 300, 47, -1, -1, -1, -1, -1, -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 37, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 47, 512, -1, -1, 515, 516, 517, 54, 519, - 520, 521, 522, 523, 524, -1, -1, -1, -1, -1, - -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 81, -1, -1, -1, -1, + 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 81, -1, -1, -1, + 8, -1, -1, 11, -1, -1, -1, 15, 16, 17, + 18, -1, 20, 21, 22, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 37, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 47, + 512, -1, -1, 515, 516, 517, 54, 519, 520, 521, + 522, 523, 524, -1, -1, -1, -1, -1, -1, -1, + -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 81, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 512, -1, 179, 515, 516, 517, - -1, 519, 520, 521, 522, 523, 524, -1, -1, -1, - -1, -1, -1, 196, -1, -1, 8, -1, 201, 11, - -1, -1, -1, 15, 16, 17, 18, -1, 20, 21, - 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 224, 225, -1, -1, 37, -1, -1, -1, 41, - -1, -1, -1, -1, -1, 47, -1, 240, -1, -1, - -1, -1, 54, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 179, -1, 512, -1, -1, 515, 516, - 517, -1, 519, 520, 521, 522, 523, 524, -1, 81, - 196, -1, -1, -1, -1, 201, 8, 280, -1, 11, - 283, -1, -1, 15, 16, 17, 18, -1, 20, 21, - 22, -1, -1, -1, 297, -1, -1, 300, 224, 225, - -1, -1, -1, -1, -1, 37, -1, -1, -1, -1, - -1, -1, -1, -1, 240, 47, -1, -1, -1, -1, - -1, -1, 54, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 512, -1, 179, 515, 516, 517, -1, 519, + 520, 521, 522, 523, 524, -1, -1, -1, -1, -1, + -1, 196, -1, -1, 8, -1, 201, 11, -1, -1, + -1, 15, 16, 17, 18, -1, 20, 21, 22, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 224, + 225, -1, -1, 37, -1, -1, -1, 41, -1, -1, + -1, -1, -1, 47, -1, 240, -1, -1, -1, -1, + 54, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 179, -1, 512, -1, -1, 515, 516, 517, -1, + 519, 520, 521, 522, 523, 524, -1, 81, 196, -1, + -1, -1, -1, 201, 8, 280, -1, 11, 283, -1, + -1, 15, 16, 17, 18, -1, 20, 21, 22, -1, + -1, -1, 297, -1, -1, 300, 224, 225, -1, -1, + -1, -1, -1, 37, -1, -1, -1, -1, -1, -1, + -1, -1, 240, 47, -1, -1, -1, -1, -1, -1, + 54, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 81, - -1, -1, -1, -1, 280, -1, -1, 283, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 179, -1, -1, - -1, 297, -1, -1, 300, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 196, -1, -1, -1, -1, 201, - -1, -1, -1, -1, -1, -1, -1, 400, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 81, -1, -1, + -1, -1, 280, -1, -1, 283, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 179, -1, -1, -1, 297, + -1, -1, 300, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 196, -1, -1, -1, -1, 201, -1, -1, + -1, -1, -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 224, 225, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 240, -1, + 224, 225, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 240, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 179, -1, -1, + -1, -1, -1, -1, -1, 179, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 196, -1, -1, -1, 280, 201, - -1, 283, -1, -1, 400, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 297, -1, -1, 300, -1, - -1, -1, 224, 225, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 240, 512, - 436, -1, 515, 516, 517, -1, 519, 520, 521, 522, - 523, 524, -1, -1, -1, -1, -1, 8, -1, -1, - 11, -1, -1, -1, 15, 16, 17, 18, -1, 20, - 21, 22, -1, -1, -1, -1, -1, -1, 280, -1, - -1, 283, -1, -1, -1, -1, 37, -1, -1, -1, - -1, -1, -1, -1, -1, 297, 47, -1, 300, -1, - -1, -1, -1, 54, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 512, -1, 400, 515, - 516, 517, -1, 519, 520, 521, 522, 523, 524, -1, - 81, -1, -1, -1, -1, 8, -1, -1, 11, -1, + -1, -1, 196, -1, -1, -1, 280, 201, -1, 283, + -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 297, -1, -1, 300, -1, -1, -1, + 224, 225, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 240, 512, 436, -1, + 515, 516, 517, -1, 519, 520, 521, 522, 523, 524, + -1, -1, -1, -1, -1, 8, -1, -1, 11, -1, -1, -1, 15, 16, 17, 18, -1, 20, 21, 22, - -1, -1, -1, -1, 8, -1, -1, 11, -1, -1, - -1, 15, 16, -1, 37, -1, 20, 21, 22, -1, - -1, -1, -1, -1, 47, -1, -1, -1, -1, -1, - -1, 54, -1, 37, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 47, -1, -1, -1, -1, 400, -1, - 54, -1, -1, -1, -1, -1, -1, -1, 81, -1, + -1, -1, -1, -1, -1, -1, 280, -1, -1, 283, + -1, -1, -1, -1, 37, -1, -1, -1, -1, -1, + -1, -1, -1, 297, 47, -1, 300, -1, -1, -1, + -1, 54, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 512, -1, 400, 515, 516, 517, + -1, 519, 520, 521, 522, 523, 524, -1, 81, -1, + -1, -1, -1, 8, -1, -1, 11, -1, -1, -1, + 15, 16, 17, 18, -1, 20, 21, 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 81, 179, -1, - 512, -1, -1, 515, 516, 517, -1, 519, 520, 521, - 522, 523, 524, -1, -1, 196, -1, -1, -1, -1, - 201, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 37, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 47, -1, -1, -1, -1, -1, -1, 54, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 400, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 81, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 179, -1, 512, -1, + -1, 515, 516, 517, -1, 519, 520, 521, 522, 523, + 524, -1, -1, 196, -1, -1, -1, -1, 201, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 224, 225, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 240, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 179, -1, -1, -1, - 512, -1, -1, 515, 516, 517, -1, 519, 520, 521, - 522, 523, 524, 196, -1, 179, -1, -1, 201, 280, - -1, -1, 283, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 196, -1, -1, -1, 297, 201, -1, 300, -1, 224, 225, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 240, -1, -1, - 224, 225, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 240, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 280, -1, -1, + -1, -1, -1, -1, 179, -1, -1, -1, 512, -1, + -1, 515, 516, 517, -1, 519, 520, 521, 522, 523, + 524, 196, -1, -1, -1, -1, 201, 280, -1, -1, 283, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 297, -1, 280, 300, -1, 283, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 297, -1, -1, 300, -1, -1, 400, + -1, -1, -1, -1, 297, -1, -1, 300, -1, 224, + 225, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 240, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 280, -1, -1, 283, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 297, -1, -1, 300, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 512, -1, -1, 515, 516, 517, -1, 519, 520, - 521, 522, 523, 524, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 400, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 512, -1, -1, 515, 516, 517, -1, 519, 520, 521, 522, - 523, 524, 3, -1, 5, -1, -1, -1, 512, -1, - -1, 515, 516, 517, -1, 519, 520, 521, 522, 523, - 524, -1, 23, 24, 25, 26, 27, 28, 29, 30, - 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, - 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, - 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, - 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, - 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, - 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, - 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, - 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, - 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, - 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, - 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, - 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, - 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, - 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, - 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, - 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, - 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, - 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, + 523, 524, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, - 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, - 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, - 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, - 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, - 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, - 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, - 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, - 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, - 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, - 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, - 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, - 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 3, -1, 5, -1, -1, -1, -1, 512, -1, -1, + 515, 516, 517, -1, 519, 520, 521, 522, 523, 524, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, @@ -17451,112 +17402,112 @@ static const yytype_int16 yycheck[] = 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, - 26, 27, 28, 29, 30, -1, 32, 33, 34, 35, - 36, -1, 38, -1, -1, -1, -1, 43, 44, 45, - -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, 58, -1, 60, 61, 62, 63, 64, 65, - -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, - 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, - 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, - 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, + 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, + 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, - 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, - 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, - 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, - 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, - 156, 157, 158, 159, 160, 161, -1, 163, -1, 165, - 166, 167, 168, -1, 170, -1, 172, 173, -1, 175, - 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, - 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, - 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, - 206, 207, 208, -1, 210, 211, 212, 213, 214, 215, - 216, 217, 218, 219, -1, 221, -1, 223, 224, 225, - 226, 227, 228, 229, 230, -1, 232, 233, 234, -1, - -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, + 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, + 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, + 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, + 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, + 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, + 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, - 276, 277, 278, 279, -1, 281, 282, 283, 284, -1, - 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, - -1, 297, 298, 299, -1, -1, 302, 303, 304, 305, + 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, - 326, 327, 328, 329, 330, 331, 332, 333, 334, -1, - 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, - 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, + 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, + 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, + 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, + 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, + 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, - 386, 387, -1, 389, 390, 391, 392, 393, 394, 395, - 396, 397, 398, -1, 400, 401, 402, 403, 404, -1, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, - 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, - 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, - -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, - 446, -1, 448, 449, 450, 451, 452, 453, 454, 455, - -1, -1, 458, 459, 460, 461, -1, 463, 464, 465, - 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, - -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, - -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, + 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, + 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, + 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, + 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, - 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, 45, -1, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, - 67, 68, 69, 70, 71, 72, 73, 74, -1, 76, - 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, - 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, - 97, 98, 99, 100, -1, 102, 103, 104, 105, 106, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, + 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, + 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, - -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, - 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, - 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, - 167, 168, -1, 170, -1, 172, 173, -1, 175, 176, - 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, - 187, -1, 189, 190, 191, 192, 193, 194, 195, 196, - 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, - 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, - 217, 218, 219, -1, 221, -1, 223, 224, 225, 226, - 227, 228, 229, 230, -1, 232, 233, 234, -1, -1, - 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, + 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, + 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, + 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, + 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, + 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, + 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, + 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, -1, 281, 282, 283, 284, -1, 286, - 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, - 297, 298, 299, -1, -1, 302, 303, 304, 305, 306, + 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, + 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, + 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, - 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, - 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, + 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, + 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, - 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, - 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, - -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, - 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, - -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, - -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, - -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, - -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, + 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, + 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, + 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, + 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, + 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, - 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, - -1, -1, -1, -1, -1, 43, 44, 45, -1, 47, + 28, 29, 30, -1, 32, 33, 34, 35, 36, -1, + 38, -1, -1, -1, -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, + 58, -1, 60, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, 96, 97, @@ -17589,14 +17540,14 @@ static const yytype_int16 yycheck[] = 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, - -1, -1, 390, 391, 392, 393, 394, 395, 396, 397, + -1, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, - 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, + 458, 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, @@ -17605,50 +17556,50 @@ static const yytype_int16 yycheck[] = -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, - 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, - -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, - 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - 79, 80, -1, -1, 83, 84, 85, 86, 87, 88, - 89, -1, 91, 92, 93, 94, 95, 96, 97, 98, - 99, 100, -1, -1, 103, 104, 105, 106, 107, 108, + -1, -1, -1, -1, 43, 44, 45, -1, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + -1, -1, 61, 62, 63, 64, 65, -1, 67, 68, + 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, + 79, 80, -1, 82, -1, 84, 85, 86, 87, 88, + 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, + 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, 167, 168, - -1, 170, 171, 172, -1, -1, -1, 176, 177, 178, - -1, 180, 181, 182, -1, 184, 185, 186, 187, -1, - 189, 190, 191, 192, 193, 194, 195, -1, 197, 198, + -1, 170, -1, 172, 173, -1, 175, 176, 177, 178, + 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, - -1, 210, -1, 212, 213, 214, 215, 216, 217, 218, - 219, -1, 221, -1, 223, -1, -1, 226, -1, 228, - 229, 230, -1, 232, 233, 234, -1, -1, 237, -1, - 239, -1, -1, 242, 243, 244, 245, 246, 247, 248, + -1, 210, 211, 212, 213, 214, 215, 216, 217, 218, + 219, -1, 221, -1, 223, 224, 225, 226, 227, 228, + 229, 230, -1, 232, 233, 234, -1, -1, 237, 238, + 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, - 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, - 279, -1, 281, 282, -1, 284, -1, 286, 287, 288, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, -1, 281, 282, 283, 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, - 299, -1, -1, 302, 303, 304, -1, 306, -1, 308, + 299, -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, -1, -1, -1, -1, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, + 319, 320, -1, -1, -1, -1, 325, 326, 327, 328, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, - 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, - -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, + -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, - -1, 430, 431, 432, 433, 434, 435, -1, 437, 438, + 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, -1, 448, - 449, 450, 451, 452, 453, 454, 455, -1, 457, 458, + 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, - 469, 470, 471, 472, 473, 474, 475, -1, 477, -1, + 469, 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, @@ -17656,208 +17607,208 @@ static const yytype_int16 yycheck[] = -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, - -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, + -1, -1, -1, 43, 44, 45, -1, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, - 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, - 80, -1, -1, -1, 84, 85, 86, 87, 88, 89, - -1, 91, 92, 93, 94, 95, 96, 97, 98, 99, - 100, -1, -1, 103, 104, 105, 106, 107, 108, 109, + 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, + 80, -1, 82, -1, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, + 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, 167, 168, -1, - 170, 171, 172, -1, -1, -1, 176, 177, 178, -1, + 170, -1, 172, 173, -1, 175, 176, 177, 178, 179, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, - 190, 191, 192, 193, 194, 195, -1, 197, 198, 199, + 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, - 210, -1, 212, 213, 214, 215, 216, 217, 218, 219, - -1, 221, -1, 223, -1, -1, 226, -1, 228, 229, - 230, -1, 232, 233, 234, -1, -1, 237, -1, 239, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, + -1, 221, -1, 223, 224, 225, 226, 227, 228, 229, + 230, -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, - 270, 271, 272, 273, -1, 275, 276, 277, 278, 279, - -1, 281, 282, -1, 284, -1, 286, 287, 288, 289, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, + -1, 281, 282, 283, 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, - -1, -1, 302, 303, 304, -1, 306, -1, 308, 309, + -1, -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, -1, -1, -1, -1, 325, 326, 327, -1, 329, - 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, + 320, -1, -1, -1, -1, 325, 326, 327, 328, 329, + 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, - -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, + 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, -1, - -1, 401, 402, 403, 404, -1, 406, 407, 408, 409, + 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, 422, -1, -1, 425, 426, -1, 428, -1, + 420, 421, 422, -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, -1, 448, 449, - 450, 451, 452, 453, 454, 455, -1, 457, 458, 459, + 450, 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, 469, - 470, 471, 472, 473, 474, 475, -1, 477, -1, 479, + 470, 471, 472, 473, 474, 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, - -1, 32, 33, 34, -1, -1, -1, 38, -1, -1, - 41, -1, 43, 44, 45, -1, 47, 48, 49, 50, - 51, 52, 53, -1, 55, 56, 57, 58, -1, 60, + -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, + 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, - 71, 72, 73, 74, -1, 76, 77, 78, 79, -1, - -1, 82, -1, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, - -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + -1, -1, 83, 84, 85, 86, 87, 88, 89, -1, + 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, + -1, -1, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, - 151, 152, 153, -1, 155, 156, 157, 158, 159, 160, - -1, -1, 163, -1, 165, 166, 167, 168, -1, 170, - -1, 172, 173, -1, 175, 176, 177, 178, 179, 180, - 181, 182, -1, -1, -1, 186, 187, -1, 189, 190, - 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, + 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, + 161, -1, 163, -1, 165, 166, 167, 168, -1, 170, + 171, 172, -1, -1, -1, 176, 177, 178, -1, 180, + 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, + 191, 192, 193, 194, 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, - 211, -1, 213, 214, 215, 216, 217, 218, 219, -1, - 221, -1, 223, 224, 225, 226, 227, 228, 229, 230, - -1, 232, 233, 234, -1, -1, 237, 238, 239, 240, + -1, 212, 213, 214, 215, 216, 217, 218, 219, -1, + 221, -1, 223, -1, -1, 226, -1, 228, 229, 230, + -1, 232, 233, 234, -1, -1, 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 278, -1, -1, - 281, 282, 283, 284, -1, -1, 287, 288, 289, 290, + 271, 272, 273, -1, 275, 276, 277, 278, 279, -1, + 281, 282, -1, 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, - -1, 302, 303, -1, 305, 306, 307, -1, 309, 310, + -1, 302, 303, 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, - -1, -1, -1, -1, 325, 326, -1, 328, 329, 330, - -1, 332, 333, 334, -1, 336, 337, 338, 339, 340, + -1, -1, -1, -1, 325, 326, 327, -1, 329, 330, + 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, - 361, 362, 363, 364, 365, 366, -1, 368, 369, 370, + 361, 362, 363, 364, 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, - 381, 382, 383, 384, 385, 386, 387, -1, 389, 390, - 391, 392, 393, 394, 395, 396, 397, 398, -1, 400, + 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, + 391, 392, 393, 394, 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, - 421, -1, -1, -1, 425, 426, -1, 428, 429, 430, + 421, 422, -1, -1, 425, 426, -1, 428, -1, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, - -1, 442, 443, -1, 445, -1, -1, 448, 449, 450, - 451, 452, 453, 454, 455, 456, -1, 458, 459, 460, - 461, -1, 463, 464, 465, 466, -1, 468, 469, 470, - 471, 472, -1, 474, 475, -1, 477, 478, 479, 480, + -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, + 451, 452, 453, 454, 455, -1, 457, 458, 459, 460, + -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, + 471, 472, 473, 474, 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, - 491, 492, 493, 494, 495, 496, 3, -1, 5, -1, - -1, -1, -1, -1, -1, -1, -1, 508, 509, 510, - 511, -1, -1, -1, -1, -1, 23, 24, 25, 26, - 27, 28, 29, 30, -1, 32, 33, 34, 35, 36, - -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, - -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, - 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, - -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, - 77, 78, 79, 80, -1, -1, -1, 84, 85, 86, - 87, 88, 89, -1, 91, 92, 93, -1, 95, 96, - 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, - -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, - 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, - 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, - 167, 168, -1, 170, -1, 172, -1, -1, -1, 176, - 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, - 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, - 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, - 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, - 217, 218, 219, -1, 221, -1, 223, -1, -1, 226, - -1, 228, 229, 230, -1, 232, 233, 234, -1, -1, - 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, - 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, - 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, - 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, - -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, - 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, - 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, - 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, - 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, - 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, - 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, - -1, 428, -1, 430, 431, 432, 433, 434, 435, -1, - 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, - -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, - -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, - -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, - 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, - -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, - 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 508, 509, 510, 511, 3, -1, 5, -1, -1, + 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, + 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, - 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, - 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, - 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, - 78, 79, 80, -1, -1, -1, 84, 85, 86, 87, - 88, 89, -1, 91, 92, 93, -1, 95, 96, 97, - 98, 99, 100, -1, -1, 103, 104, 105, 106, 107, - 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, - -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, - 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, - 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, - 158, 159, 160, 161, -1, 163, -1, 165, 166, 167, - 168, -1, 170, -1, 172, -1, -1, -1, 176, 177, - 178, -1, 180, 181, 182, -1, 184, 185, 186, 187, - -1, 189, 190, 191, 192, 193, 194, 195, -1, 197, - 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, - 208, -1, 210, -1, 212, 213, 214, 215, 216, 217, - 218, 219, -1, 221, -1, 223, -1, -1, 226, -1, - 228, 229, 230, -1, 232, 233, 234, -1, -1, 237, - -1, 239, -1, -1, 242, 243, 244, 245, 246, 247, - 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, - 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, - 268, 269, 270, 271, 272, 273, -1, 275, 276, 277, - 278, 279, -1, 281, 282, -1, 284, -1, 286, 287, - 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, - 298, 299, -1, -1, 302, 303, 304, -1, 306, -1, - 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, - 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, - -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, - 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, - 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, - 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, - 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, - 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, - -1, -1, 390, 391, 392, 393, 394, 395, 396, 397, - 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, - 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, - 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, - 428, -1, 430, 431, 432, 433, 434, 435, -1, 437, - 438, 439, -1, -1, 442, 443, 444, 445, 446, -1, - 448, 449, 450, 451, 452, 453, 454, 455, -1, -1, - 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, - 468, 469, 470, 471, 472, 473, 474, 475, -1, 477, - -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, - 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, - 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, - 508, 509, 510, 511, 3, -1, 5, -1, -1, -1, + -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, + 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, + 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, + 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 80, -1, + -1, -1, 84, 85, 86, 87, 88, 89, -1, 91, + 92, 93, 94, 95, 96, 97, 98, 99, 100, -1, + -1, 103, 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, + 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, + 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, + 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, + -1, 163, -1, 165, 166, 167, 168, -1, 170, 171, + 172, -1, -1, -1, 176, 177, 178, -1, 180, 181, + 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, + 192, 193, 194, 195, -1, 197, 198, 199, 200, -1, + 202, 203, 204, 205, 206, 207, 208, -1, 210, -1, + 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, + -1, 223, -1, -1, 226, -1, 228, 229, 230, -1, + 232, 233, 234, -1, -1, 237, -1, 239, 240, -1, + 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, + 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, + 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, -1, 275, 276, 277, 278, 279, -1, 281, + 282, -1, 284, -1, 286, 287, 288, 289, 290, 291, + -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, + 302, 303, 304, -1, 306, -1, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, + -1, -1, -1, 325, 326, 327, -1, 329, 330, 331, + 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, + -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, + 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, + 362, 363, 364, 365, 366, -1, 368, 369, -1, 371, + 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, + 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, + 392, 393, 394, 395, 396, 397, 398, -1, -1, 401, + 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, + 422, -1, -1, 425, 426, -1, 428, -1, 430, 431, + 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, + 442, 443, 444, 445, 446, -1, 448, 449, 450, 451, + 452, 453, 454, 455, -1, 457, 458, 459, 460, -1, + -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, + 472, 473, 474, 475, -1, 477, -1, 479, 480, 481, + 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, + 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, + 33, 34, -1, -1, -1, 38, -1, -1, 41, -1, + 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, + 53, -1, 55, 56, 57, 58, -1, 60, 61, 62, + 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, + 73, 74, -1, 76, 77, 78, 79, -1, -1, 82, + -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, + 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, + 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, + -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, + 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, + 153, -1, 155, 156, 157, 158, 159, 160, -1, -1, + 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, + 173, -1, 175, 176, 177, 178, 179, 180, 181, 182, + -1, -1, -1, 186, 187, -1, 189, 190, 191, 192, + 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, + 203, 204, 205, 206, 207, 208, -1, 210, 211, -1, + 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, + 223, 224, 225, 226, 227, 228, 229, 230, -1, 232, + 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, + 273, 274, 275, 276, 277, 278, -1, -1, 281, 282, + 283, 284, -1, -1, 287, 288, 289, 290, 291, -1, + 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, + 303, -1, 305, 306, 307, -1, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, + -1, -1, 325, 326, -1, 328, 329, 330, -1, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, + 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, + -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, + 383, 384, 385, 386, 387, -1, 389, 390, 391, 392, + 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, + 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, + 413, 414, 415, 416, 417, 418, 419, 420, 421, -1, + -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, + 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, + 443, -1, 445, -1, -1, 448, 449, 450, 451, 452, + 453, 454, 455, 456, -1, 458, 459, 460, 461, -1, + 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, + -1, 474, 475, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, + 493, 494, 495, 496, 3, -1, 5, -1, -1, -1, + -1, -1, -1, -1, -1, 508, 509, 510, 511, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, - 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, + 29, 30, -1, 32, 33, 34, 35, 36, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, @@ -17883,7 +17834,7 @@ static const yytype_int16 yycheck[] = 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, 287, 288, - 289, 290, 291, -1, 293, 294, -1, 296, 297, 298, + 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, -1, @@ -17908,7 +17859,7 @@ static const yytype_int16 yycheck[] = 509, 510, 511, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, - 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, + 30, 31, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, @@ -17934,7 +17885,7 @@ static const yytype_int16 yycheck[] = 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, 287, 288, 289, - 290, 291, -1, 293, 294, -1, 296, 297, 298, 299, + 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, -1, 329, @@ -17959,7 +17910,7 @@ static const yytype_int16 yycheck[] = 510, 511, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, - 31, 32, 33, 34, -1, -1, -1, -1, -1, -1, + -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, @@ -17985,7 +17936,7 @@ static const yytype_int16 yycheck[] = 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, 287, 288, 289, 290, - 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, + 291, -1, 293, 294, -1, 296, 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, -1, 329, 330, @@ -18058,261 +18009,261 @@ static const yytype_int16 yycheck[] = 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, - 33, 34, -1, -1, -1, 38, -1, -1, -1, -1, - 43, 44, 45, -1, 47, 48, 49, 50, 51, 52, - 53, -1, 55, 56, 57, 58, -1, 60, 61, 62, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, + 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, - 73, 74, -1, 76, 77, 78, 79, -1, -1, 82, - -1, 84, 85, 86, 87, 88, 89, 90, 91, 92, - 93, -1, 95, 96, 97, 98, 99, 100, -1, 102, + 73, 74, -1, 76, 77, 78, 79, 80, -1, -1, + -1, 84, 85, 86, 87, 88, 89, -1, 91, 92, + 93, -1, 95, 96, 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, 152, - 153, -1, 155, 156, 157, 158, 159, 160, -1, -1, + 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, - 173, -1, 175, 176, 177, -1, 179, 180, 181, 182, - -1, -1, -1, 186, 187, -1, 189, 190, 191, 192, - 193, 194, 195, 196, 197, 198, 199, 200, -1, 202, - 203, 204, 205, 206, 207, 208, -1, 210, 211, -1, + -1, -1, -1, 176, 177, 178, -1, 180, 181, 182, + -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, + 193, 194, 195, -1, 197, 198, 199, 200, -1, 202, + 203, 204, 205, 206, 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, 217, 218, 219, -1, 221, -1, - 223, 224, 225, 226, 227, 228, 229, 230, -1, 232, - 233, 234, -1, -1, 237, 238, 239, 240, -1, 242, + 223, -1, -1, 226, -1, 228, 229, 230, -1, 232, + 233, 234, -1, -1, 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, - 273, 274, 275, 276, 277, 278, -1, -1, 281, 282, - 283, 284, -1, -1, 287, 288, 289, 290, 291, -1, + 273, -1, 275, 276, 277, 278, 279, -1, 281, 282, + -1, 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, - 303, -1, 305, 306, 307, -1, 309, 310, 311, 312, + 303, 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, - -1, -1, 325, 326, -1, 328, 329, 330, -1, 332, + -1, -1, 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, -1, 368, 369, 370, 371, 372, + 363, 364, 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, - 383, 384, 385, 386, 387, -1, 389, 390, 391, 392, - 393, 394, 395, 396, 397, 398, -1, 400, 401, 402, + 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, + 393, 394, 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, - 413, 414, 415, 416, 417, 418, 419, 420, 421, -1, - -1, -1, 425, 426, -1, 428, 429, 430, 431, 432, + 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, + -1, -1, 425, 426, -1, 428, -1, 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, 442, - 443, -1, 445, -1, -1, 448, 449, 450, 451, 452, - 453, 454, 455, 456, -1, 458, 459, 460, 461, -1, + 443, 444, 445, 446, -1, 448, 449, 450, 451, 452, + 453, 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, 472, - -1, 474, 475, -1, 477, 478, 479, 480, 481, 482, + 473, 474, 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, - 493, 494, 495, 496, 3, -1, 5, -1, -1, -1, - -1, -1, -1, -1, -1, 508, 509, 510, 511, -1, - -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, - 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, - 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, - -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, - 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, - 79, 80, -1, -1, -1, 84, 85, 86, 87, 88, - 89, -1, 91, 92, 93, -1, 95, 96, 97, 98, - 99, 100, -1, -1, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, - 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, - 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, - 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, - -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, - 159, 160, 161, -1, 163, -1, 165, 166, 167, 168, - -1, 170, -1, 172, -1, -1, -1, 176, 177, 178, - -1, 180, 181, 182, -1, 184, 185, 186, 187, -1, - 189, 190, 191, 192, 193, 194, 195, -1, 197, 198, - 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, - -1, 210, -1, 212, 213, 214, 215, 216, 217, 218, - 219, -1, 221, -1, 223, -1, -1, 226, -1, 228, - 229, 230, -1, 232, 233, 234, -1, -1, 237, -1, - 239, -1, -1, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, - 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, - 279, -1, 281, 282, -1, 284, -1, 286, 287, 288, - 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, - 299, -1, -1, 302, 303, 304, -1, 306, -1, 308, - 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, -1, -1, -1, -1, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, - 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, - 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, - 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, - 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, - -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, - -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, - 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, - 419, 420, 421, 422, -1, -1, 425, 426, 427, 428, - -1, 430, 431, 432, 433, 434, 435, -1, 437, 438, - 439, -1, -1, 442, 443, 444, 445, 446, -1, 448, - 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, - 459, 460, -1, -1, 463, 464, 465, 466, -1, 468, - 469, 470, 471, 472, 473, 474, 475, -1, 477, -1, - 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, - -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, - 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, - 30, -1, 32, 33, 34, -1, -1, -1, 38, -1, - -1, -1, -1, 43, 44, 45, -1, 47, 48, 49, - 50, 51, 52, 53, -1, 55, 56, 57, 58, -1, - 60, 61, 62, 63, 64, 65, -1, -1, 68, 69, - 70, 71, 72, 73, 74, -1, 76, 77, 78, 79, - -1, -1, 82, -1, 84, 85, 86, 87, 88, 89, - 90, 91, 92, 93, -1, 95, 96, 97, 98, 99, - 100, -1, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, - -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, - 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, - -1, 141, 142, 143, -1, 145, 146, 147, 148, -1, - 150, 151, 152, 153, -1, 155, 156, 157, 158, 159, - 160, -1, -1, 163, -1, 165, 166, 167, 168, -1, - 170, -1, 172, 173, -1, 175, 176, 177, -1, 179, - 180, 181, 182, -1, -1, -1, 186, 187, -1, 189, - 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, - 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, - 210, 211, -1, 213, 214, 215, 216, 217, 218, 219, - -1, 221, -1, 223, 224, 225, 226, 227, 228, 229, - 230, -1, 232, 233, 234, -1, -1, 237, 238, 239, - 240, -1, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, - 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, - 270, 271, 272, 273, 274, 275, 276, 277, 278, -1, - -1, 281, 282, 283, 284, -1, -1, 287, 288, 289, - 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, - -1, -1, 302, 303, -1, 305, 306, 307, -1, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, -1, -1, -1, -1, 325, 326, -1, 328, 329, - 330, -1, 332, 333, 334, -1, 336, 337, 338, 339, - 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, - 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, -1, 368, 369, - 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, - 380, 381, 382, 383, 384, 385, 386, 387, -1, 389, - 390, 391, 392, 393, 394, 395, 396, 397, 398, -1, - 400, 401, 402, 403, 404, -1, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, -1, -1, -1, 425, 426, -1, 428, 429, - 430, 431, 432, 433, 434, 435, -1, 437, 438, 439, - -1, -1, 442, 443, -1, 445, -1, -1, 448, 449, - 450, 451, 452, 453, 454, 455, 456, -1, 458, 459, - 460, 461, -1, 463, 464, 465, 466, -1, 468, 469, - 470, 471, 472, -1, 474, 475, -1, 477, 478, 479, - 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, - -1, 491, 492, 493, 494, 495, 496, 3, -1, 5, - -1, -1, -1, -1, -1, -1, -1, -1, 508, 509, - 510, 511, -1, -1, -1, -1, -1, 23, 24, 25, - 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, - -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, - -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, - -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, - 76, 77, 78, 79, 80, -1, -1, -1, 84, 85, - 86, 87, 88, 89, -1, 91, 92, 93, -1, 95, - 96, 97, 98, 99, 100, -1, -1, 103, 104, 105, - 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, - 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, - 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, - 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, - 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, - 156, 157, 158, 159, 160, 161, -1, 163, -1, 165, - 166, 167, 168, -1, 170, -1, 172, -1, -1, -1, - 176, 177, 178, -1, 180, 181, 182, -1, 184, 185, - 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, - -1, 197, 198, 199, 200, -1, 202, 203, 204, 205, - 206, 207, 208, -1, 210, -1, 212, 213, 214, 215, - 216, 217, 218, 219, -1, 221, -1, 223, -1, -1, - 226, -1, 228, 229, 230, -1, 232, 233, 234, -1, - -1, 237, -1, 239, -1, -1, 242, 243, 244, 245, - 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, -1, 275, - 276, 277, 278, 279, -1, 281, 282, -1, 284, -1, - 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, - -1, 297, 298, 299, -1, -1, 302, 303, 304, -1, - 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, - 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, - 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, - 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, - 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, - 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, - 386, 387, -1, -1, 390, 391, 392, 393, 394, 395, - 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, - 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, - 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, - 426, -1, 428, -1, 430, 431, 432, 433, 434, 435, - -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, - 446, -1, 448, 449, 450, 451, 452, 453, 454, 455, - -1, -1, 458, 459, 460, -1, -1, 463, 464, 465, - 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, - -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, - -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, - 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, - 506, 507, 508, 509, 510, 511, 3, -1, 5, -1, + 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, + 503, 504, 505, 506, 507, 508, 509, 510, 511, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, + 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, + 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, + 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, + 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, + 74, -1, 76, 77, 78, 79, 80, -1, -1, -1, + 84, 85, 86, 87, 88, 89, -1, 91, 92, 93, + -1, 95, 96, 97, 98, 99, 100, -1, -1, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, + 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, + -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, + -1, 145, 146, 147, 148, -1, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, + -1, 165, 166, 167, 168, -1, 170, -1, 172, -1, + -1, -1, 176, 177, 178, -1, 180, 181, 182, -1, + 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, + 194, 195, -1, 197, 198, 199, 200, -1, 202, 203, + 204, 205, 206, 207, 208, -1, 210, -1, 212, 213, + 214, 215, 216, 217, 218, 219, -1, 221, -1, 223, + -1, -1, 226, -1, 228, 229, 230, -1, 232, 233, + 234, -1, -1, 237, -1, 239, -1, -1, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + -1, 275, 276, 277, 278, 279, -1, 281, 282, -1, + 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, + 294, -1, 296, 297, 298, 299, -1, -1, 302, 303, + 304, -1, 306, -1, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, + -1, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, + 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, + 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, -1, 368, 369, -1, 371, 372, 373, + 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, + 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, + 394, 395, 396, 397, 398, -1, -1, 401, 402, 403, + 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, + -1, 425, 426, -1, 428, -1, 430, 431, 432, 433, + 434, 435, -1, 437, 438, 439, -1, -1, 442, 443, + 444, 445, 446, -1, 448, 449, 450, 451, 452, 453, + 454, 455, -1, -1, 458, 459, 460, -1, -1, 463, + 464, 465, 466, -1, 468, 469, 470, 471, 472, 473, + 474, 475, -1, 477, -1, 479, 480, 481, 482, 483, + 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, + 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, - 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, - -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, - 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, - -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, - 77, 78, 79, 80, -1, -1, -1, 84, 85, 86, - 87, 88, 89, -1, 91, 92, 93, -1, 95, 96, - 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, - -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, - 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, - 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, - 167, 168, -1, 170, -1, 172, -1, -1, -1, 176, - 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, - 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, - 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, - 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, - 217, 218, 219, -1, 221, -1, 223, -1, -1, 226, - -1, 228, 229, 230, -1, 232, 233, 234, -1, -1, - 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, - 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, - 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, - 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, - -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, - 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, - 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, - 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, - 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, - 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, - 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, - -1, 428, -1, 430, 431, 432, 433, 434, 435, -1, - 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, - -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, - -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, - -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, - 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, - -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, - 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, + 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, + -1, -1, -1, 38, -1, -1, -1, -1, 43, 44, + 45, -1, 47, 48, 49, 50, 51, 52, 53, -1, + 55, 56, 57, 58, -1, 60, 61, 62, 63, 64, + 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, + -1, 76, 77, 78, 79, -1, -1, 82, -1, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, + 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, + 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, + 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, + 145, 146, 147, 148, -1, 150, 151, 152, 153, -1, + 155, 156, 157, 158, 159, 160, -1, -1, 163, -1, + 165, 166, 167, 168, -1, 170, -1, 172, 173, -1, + 175, 176, 177, -1, 179, 180, 181, 182, -1, -1, + -1, 186, 187, -1, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, + 205, 206, 207, 208, -1, 210, 211, -1, 213, 214, + 215, 216, 217, 218, 219, -1, 221, -1, 223, 224, + 225, 226, 227, 228, 229, 230, -1, 232, 233, 234, + -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, + 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, -1, -1, 281, 282, 283, 284, + -1, -1, 287, 288, 289, 290, 291, -1, 293, 294, + -1, -1, 297, 298, 299, -1, -1, 302, 303, -1, + 305, 306, 307, -1, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, + 325, 326, -1, 328, 329, 330, -1, 332, 333, 334, + -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, + 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, + 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, + 365, 366, -1, 368, 369, 370, 371, 372, 373, 374, + 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, + 385, 386, 387, -1, 389, 390, 391, 392, 393, 394, + 395, 396, 397, 398, -1, 400, 401, 402, 403, 404, + -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, + 415, 416, 417, 418, 419, 420, 421, -1, -1, -1, + 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, + 435, -1, 437, 438, 439, -1, -1, 442, 443, -1, + 445, -1, -1, 448, 449, 450, 451, 452, 453, 454, + 455, 456, -1, 458, 459, 460, 461, -1, 463, 464, + 465, 466, -1, 468, 469, 470, 471, 472, -1, 474, + 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, + 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, + 495, 496, 3, -1, 5, -1, -1, -1, -1, -1, + -1, -1, -1, 508, 509, 510, 511, -1, -1, -1, + -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, + -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, + 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, + 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, + 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, + -1, -1, -1, 84, 85, 86, 87, 88, 89, -1, + 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, + -1, -1, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, + 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, + 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, + 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, + 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, + 161, -1, 163, -1, 165, 166, 167, 168, -1, 170, + -1, 172, -1, -1, -1, 176, 177, 178, -1, 180, + 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, + 191, 192, 193, 194, 195, -1, 197, 198, 199, 200, + -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, + -1, 212, 213, 214, 215, 216, 217, 218, 219, -1, + 221, -1, 223, -1, -1, 226, -1, 228, 229, 230, + -1, 232, 233, 234, -1, -1, 237, -1, 239, -1, + -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, + 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, -1, 275, 276, 277, 278, 279, -1, + 281, 282, -1, 284, -1, 286, 287, 288, 289, 290, + 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, + -1, 302, 303, 304, -1, 306, -1, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + -1, -1, -1, -1, 325, 326, 327, -1, 329, 330, + 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, + 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, + 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, + 361, 362, 363, 364, 365, 366, -1, 368, 369, -1, + 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, + 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, + 391, 392, 393, 394, 395, 396, 397, 398, -1, -1, + 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, + 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, -1, -1, 425, 426, 427, 428, -1, 430, + 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, + -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, + 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, + -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, + 471, 472, 473, 474, 475, -1, 477, -1, 479, 480, + 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, + 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, + 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, + 32, 33, 34, -1, -1, -1, 38, -1, -1, -1, + -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, + 52, 53, -1, 55, 56, 57, 58, -1, 60, 61, + 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, + 72, 73, 74, -1, 76, 77, 78, 79, -1, -1, + 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, + 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, + 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, + 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, + 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, + 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, + 152, 153, -1, 155, 156, 157, 158, 159, 160, -1, + -1, 163, -1, 165, 166, 167, 168, -1, 170, -1, + 172, 173, -1, 175, 176, 177, -1, 179, 180, 181, + 182, -1, -1, -1, 186, 187, -1, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, + 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, + -1, 213, 214, 215, 216, 217, 218, 219, -1, 221, + -1, 223, 224, 225, 226, 227, 228, 229, 230, -1, + 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, + 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, + 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, + 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 275, 276, 277, 278, -1, -1, 281, + 282, 283, 284, -1, -1, 287, 288, 289, 290, 291, + -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, + 302, 303, -1, 305, 306, 307, -1, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, + -1, -1, -1, 325, 326, -1, 328, 329, 330, -1, + 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, + -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, + 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, + 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, + 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, + 382, 383, 384, 385, 386, 387, -1, 389, 390, 391, + 392, 393, 394, 395, 396, 397, 398, -1, 400, 401, + 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, + -1, -1, -1, 425, 426, -1, 428, 429, 430, 431, + 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, + 442, 443, -1, 445, -1, -1, 448, 449, 450, 451, + 452, 453, 454, 455, 456, -1, 458, 459, 460, 461, + -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, + 472, -1, 474, 475, -1, 477, 478, 479, 480, 481, + 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, + 492, 493, 494, 495, 496, 3, -1, 5, -1, -1, + -1, -1, -1, -1, -1, -1, 508, 509, 510, 511, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, -1, -1, -1, + 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, @@ -18411,10 +18362,10 @@ static const yytype_int16 yycheck[] = 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, - 509, 510, 511, 3, -1, 5, -1, -1, -1, -1, + 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, - 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, + 30, 31, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, @@ -18716,11 +18667,11 @@ static const yytype_int16 yycheck[] = 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, + 505, 506, 507, 508, 509, 510, 511, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, - -1, -1, -1, -1, -1, 41, -1, -1, 44, 45, + -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, @@ -18767,11 +18718,11 @@ static const yytype_int16 yycheck[] = -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, - 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, + 506, 507, 508, 509, 510, 511, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, - -1, -1, -1, -1, 41, -1, -1, 44, 45, -1, + -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, @@ -18821,8 +18772,8 @@ static const yytype_int16 yycheck[] = 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, + 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, + -1, -1, -1, 41, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, @@ -18869,11 +18820,11 @@ static const yytype_int16 yycheck[] = -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, - 508, 509, 510, 511, 3, -1, 5, -1, -1, -1, + 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, + -1, -1, 41, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, @@ -18920,10 +18871,10 @@ static const yytype_int16 yycheck[] = 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, - 509, 510, 511, 3, -1, 5, -1, -1, -1, -1, + 509, 510, 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, - 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, + 30, 31, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, @@ -19225,7 +19176,7 @@ static const yytype_int16 yycheck[] = 475, -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 3, -1, -1, + 505, 506, 507, 508, 509, 510, 511, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, @@ -19276,7 +19227,7 @@ static const yytype_int16 yycheck[] = -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, - 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, + 506, 507, 508, 509, 510, 511, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, @@ -19534,7 +19485,7 @@ static const yytype_int16 yycheck[] = 511, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, - 32, 33, 34, -1, -1, -1, -1, -1, -1, 41, + 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, @@ -19570,7 +19521,7 @@ static const yytype_int16 yycheck[] = 362, 363, 364, 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, - 392, 393, 394, -1, 396, 397, 398, -1, -1, 401, + 392, 393, 394, 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, -1, 430, 431, @@ -19687,7 +19638,7 @@ static const yytype_int16 yycheck[] = -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, + -1, -1, -1, -1, -1, -1, 41, -1, -1, 44, 45, -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, @@ -19723,7 +19674,7 @@ static const yytype_int16 yycheck[] = 365, 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, - 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, + -1, 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, -1, 428, -1, 430, 431, 432, 433, 434, @@ -19738,103 +19689,106 @@ static const yytype_int16 yycheck[] = -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, - -1, -1, 38, -1, -1, -1, -1, 43, 44, 45, - -1, 47, 48, 49, 50, 51, 52, 53, -1, 55, - 56, 57, 58, -1, 60, 61, 62, 63, 64, 65, + -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, + -1, -1, 48, 49, -1, 51, 52, 53, 54, 55, + -1, 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, -1, - 76, 77, 78, 79, -1, -1, 82, -1, 84, 85, - 86, 87, 88, 89, 90, 91, 92, 93, -1, 95, - 96, 97, 98, 99, 100, -1, 102, 103, 104, 105, + 76, 77, 78, 79, 80, -1, -1, -1, 84, 85, + 86, 87, 88, 89, -1, 91, 92, 93, -1, 95, + 96, 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, 145, - 146, 147, 148, -1, 150, 151, 152, 153, -1, 155, - 156, 157, 158, 159, 160, -1, -1, 163, -1, 165, - 166, 167, 168, -1, 170, -1, 172, 173, -1, 175, - 176, 177, 178, 179, 180, 181, 182, -1, -1, -1, + 146, 147, 148, -1, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, 161, -1, 163, -1, 165, + 166, 167, 168, -1, 170, -1, 172, -1, -1, -1, + 176, 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, - 196, 197, 198, 199, 200, -1, 202, 203, 204, 205, - 206, 207, 208, -1, 210, 211, -1, 213, 214, 215, - 216, 217, 218, 219, -1, 221, -1, 223, 224, 225, - 226, 227, 228, 229, 230, -1, 232, 233, 234, -1, - -1, 237, 238, 239, 240, -1, 242, 243, 244, 245, + -1, 197, 198, 199, 200, -1, 202, 203, 204, 205, + 206, 207, 208, -1, 210, -1, 212, 213, 214, 215, + 216, 217, 218, 219, -1, 221, -1, 223, -1, -1, + 226, -1, 228, 229, 230, -1, 232, 233, 234, -1, + -1, 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, - 276, 277, 278, -1, -1, 281, 282, 283, 284, -1, - -1, 287, 288, 289, 290, 291, -1, 293, 294, -1, - -1, 297, 298, 299, -1, -1, 302, 303, -1, 305, - 306, 307, -1, 309, 310, 311, 312, 313, 314, 315, + 266, 267, 268, 269, 270, 271, 272, 273, -1, 275, + 276, 277, 278, 279, -1, 281, 282, -1, 284, -1, + 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, + -1, 297, 298, 299, -1, -1, 302, 303, 304, -1, + 306, -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, - 326, -1, 328, 329, 330, -1, 332, 333, 334, -1, + 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, - 366, -1, 368, 369, 370, 371, 372, 373, -1, 375, + 366, -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, - 386, 387, -1, 389, 390, 391, 392, 393, 394, 395, - -1, 397, 398, -1, 400, 401, 402, 403, 404, -1, + 386, 387, -1, -1, 390, 391, 392, 393, 394, 395, + 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, - 416, 417, 418, 419, 420, 421, -1, -1, -1, 425, - 426, -1, 428, 429, 430, 431, 432, 433, 434, 435, - -1, 437, 438, 439, -1, -1, 442, 443, -1, 445, - -1, -1, 448, 449, 450, 451, 452, 453, 454, 455, - -1, -1, 458, 459, 460, 461, -1, 463, 464, 465, - 466, -1, 468, 469, 470, 471, 472, -1, 474, 475, - -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 416, 417, 418, 419, 420, 421, 422, -1, -1, 425, + 426, -1, 428, -1, 430, 431, 432, 433, 434, 435, + -1, 437, 438, 439, -1, -1, 442, 443, 444, 445, + 446, -1, 448, 449, 450, 451, 452, 453, 454, 455, + -1, -1, 458, 459, 460, -1, -1, 463, 464, 465, + 466, -1, 468, 469, 470, 471, 472, 473, 474, 475, + -1, 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, - 496, 3, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 508, 509, 510, 511, -1, -1, -1, -1, - -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, - 32, 33, 34, -1, -1, -1, 38, -1, -1, -1, - -1, 43, 44, 45, -1, 47, 48, 49, 50, 51, - 52, 53, -1, 55, 56, 57, 58, -1, 60, 61, - 62, 63, 64, 65, -1, -1, 68, 69, 70, 71, - 72, 73, 74, -1, 76, 77, 78, 79, -1, -1, - 82, -1, 84, 85, 86, 87, 88, 89, 90, 91, - 92, 93, -1, 95, 96, 97, 98, 99, 100, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, -1, 119, -1, 121, - 122, 123, 124, 125, 126, -1, -1, 129, 130, 131, - 132, -1, -1, 135, 136, 137, 138, 139, -1, 141, - 142, 143, -1, 145, 146, 147, 148, -1, 150, 151, - 152, 153, -1, 155, 156, 157, 158, 159, 160, -1, - -1, 163, -1, 165, 166, 167, 168, -1, 170, -1, - 172, 173, -1, 175, 176, 177, -1, 179, 180, 181, - 182, -1, -1, -1, 186, 187, -1, 189, 190, 191, - 192, 193, 194, 195, 196, 197, 198, 199, 200, -1, - 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, - -1, 213, 214, 215, 216, 217, 218, 219, -1, 221, - -1, 223, 224, 225, 226, 227, 228, 229, 230, -1, - 232, 233, 234, -1, -1, 237, 238, 239, 240, -1, - 242, 243, 244, 245, 246, 247, 248, 249, -1, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, -1, -1, 281, - 282, 283, 284, -1, -1, 287, 288, 289, 290, 291, - -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, - 302, 303, -1, 305, 306, 307, -1, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, - -1, -1, -1, 325, 326, -1, 328, 329, 330, -1, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, - 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, -1, 368, 369, 370, 371, - 372, 373, -1, 375, 376, 377, 378, 379, 380, 381, - 382, 383, 384, 385, 386, 387, -1, 389, 390, 391, - 392, 393, 394, 395, -1, 397, 398, -1, 400, 401, - 402, 403, 404, -1, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 417, 418, 419, -1, 421, - -1, -1, -1, 425, 426, -1, 428, 429, 430, 431, - 432, 433, 434, 435, -1, 437, 438, 439, -1, -1, - 442, 443, -1, 445, -1, -1, 448, 449, 450, 451, - 452, 453, 454, 455, -1, -1, 458, 459, 460, 461, - -1, 463, 464, 465, 466, -1, 468, 469, 470, 471, - 472, -1, 474, 475, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, - 492, 493, 494, 495, 496, 3, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 508, 509, 510, 511, + 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, + 506, 507, 508, 509, 510, 511, 3, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, + 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, + -1, 48, 49, -1, 51, 52, 53, 54, 55, -1, + 57, 58, -1, -1, 61, 62, 63, 64, 65, -1, + -1, 68, 69, 70, 71, 72, 73, 74, -1, 76, + 77, 78, 79, 80, -1, -1, -1, 84, 85, 86, + 87, 88, 89, -1, 91, 92, 93, -1, 95, 96, + 97, 98, 99, 100, -1, -1, 103, 104, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + 117, -1, 119, -1, 121, 122, 123, 124, 125, 126, + -1, -1, 129, 130, 131, 132, -1, -1, 135, 136, + 137, 138, 139, -1, 141, 142, 143, -1, 145, 146, + 147, 148, -1, 150, 151, 152, 153, 154, 155, 156, + 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, + 167, 168, -1, 170, -1, 172, -1, -1, -1, 176, + 177, 178, -1, 180, 181, 182, -1, 184, 185, 186, + 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, + 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, + 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, + 217, 218, 219, -1, 221, -1, 223, -1, -1, 226, + -1, 228, 229, 230, -1, 232, 233, 234, -1, -1, + 237, -1, 239, -1, -1, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, + 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, + 277, 278, 279, -1, 281, 282, -1, 284, -1, 286, + 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, + 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, + -1, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, + 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, + 337, 338, 339, 340, 341, -1, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + -1, 368, 369, -1, 371, 372, 373, 374, 375, 376, + 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, + 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, + 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 417, 418, 419, 420, 421, 422, -1, -1, 425, 426, + -1, 428, -1, 430, 431, 432, 433, 434, 435, -1, + 437, 438, 439, -1, -1, 442, 443, 444, 445, 446, + -1, 448, 449, 450, 451, 452, 453, 454, 455, -1, + -1, 458, 459, 460, -1, -1, 463, 464, 465, 466, + -1, 468, 469, 470, 471, 472, 473, 474, 475, -1, + 477, -1, 479, 480, 481, 482, 483, 484, 485, -1, + -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 3, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, 44, 45, -1, -1, @@ -19851,7 +19805,7 @@ static const yytype_int16 yycheck[] = 148, -1, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, -1, 163, -1, 165, 166, 167, 168, -1, 170, -1, 172, -1, -1, -1, 176, 177, - 178, -1, 180, 181, 182, -1, 184, 185, -1, 187, + 178, -1, 180, 181, 182, -1, 184, 185, 186, 187, -1, 189, 190, 191, 192, 193, 194, 195, -1, 197, 198, 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, -1, 212, 213, 214, 215, 216, 217, @@ -19864,13 +19818,13 @@ static const yytype_int16 yycheck[] = 278, 279, -1, 281, 282, -1, 284, -1, 286, 287, 288, 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, -1, 302, 303, 304, -1, 306, -1, - 308, 309, 310, 311, 312, 313, 314, -1, 316, 317, + 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, - 338, 339, 340, 341, -1, 343, 344, 345, -1, 347, + 338, 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, -1, - 368, 369, -1, 371, 372, 373, 374, -1, 376, 377, + 368, 369, -1, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, 391, 392, 393, 394, 395, 396, 397, 398, -1, -1, 401, 402, 403, 404, -1, 406, 407, @@ -19884,59 +19838,209 @@ static const yytype_int16 yycheck[] = -1, 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, - 508, 509, 510, 511, 23, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 33, -1, 35, 36, -1, -1, - -1, -1, 23, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 33, -1, 53, -1, -1, -1, -1, -1, - -1, -1, -1, 62, -1, -1, -1, -1, -1, -1, - -1, -1, 53, -1, -1, -1, -1, 76, -1, -1, - -1, 62, -1, -1, -1, -1, -1, -1, 87, -1, - -1, -1, -1, -1, -1, 76, -1, -1, -1, -1, - 99, -1, 101, -1, -1, -1, 87, -1, -1, -1, - -1, -1, -1, 112, -1, -1, -1, -1, 99, -1, - 101, -1, -1, -1, -1, -1, -1, -1, 127, 128, - -1, 112, -1, -1, -1, -1, -1, -1, -1, 138, - -1, -1, -1, -1, -1, 144, 127, 128, -1, -1, - -1, -1, -1, -1, 153, -1, -1, 138, -1, -1, - -1, -1, -1, 144, -1, -1, -1, -1, -1, -1, - -1, 170, 153, -1, -1, 174, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 170, - -1, -1, -1, 174, -1, -1, -1, -1, -1, -1, + 508, 509, 510, 511, 3, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 23, 24, 25, 26, 27, 28, + 29, 30, -1, 32, 33, 34, -1, -1, -1, 38, + -1, -1, -1, -1, 43, 44, 45, -1, 47, 48, + 49, 50, 51, 52, 53, -1, 55, 56, 57, 58, + -1, 60, 61, 62, 63, 64, 65, -1, -1, 68, + 69, 70, 71, 72, 73, 74, -1, 76, 77, 78, + 79, -1, -1, 82, -1, 84, 85, 86, 87, 88, + 89, 90, 91, 92, 93, -1, 95, 96, 97, 98, + 99, 100, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, -1, + 119, -1, 121, 122, 123, 124, 125, 126, -1, -1, + 129, 130, 131, 132, -1, -1, 135, 136, 137, 138, + 139, -1, 141, 142, 143, -1, 145, 146, 147, 148, + -1, 150, 151, 152, 153, -1, 155, 156, 157, 158, + 159, 160, -1, -1, 163, -1, 165, 166, 167, 168, + -1, 170, -1, 172, 173, -1, 175, 176, 177, 178, + 179, 180, 181, 182, -1, -1, -1, 186, 187, -1, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + 199, 200, -1, 202, 203, 204, 205, 206, 207, 208, + -1, 210, 211, -1, 213, 214, 215, 216, 217, 218, + 219, -1, 221, -1, 223, 224, 225, 226, 227, 228, + 229, 230, -1, 232, 233, 234, -1, -1, 237, 238, + 239, 240, -1, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + -1, -1, 281, 282, 283, 284, -1, -1, 287, 288, + 289, 290, 291, -1, 293, 294, -1, -1, 297, 298, + 299, -1, -1, 302, 303, -1, 305, 306, 307, -1, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 320, -1, -1, -1, -1, 325, 326, -1, 328, + 329, 330, -1, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, -1, 343, 344, 345, 346, 347, 348, + 349, 350, 351, 352, -1, 354, 355, 356, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, -1, 368, + 369, 370, 371, 372, 373, -1, 375, 376, 377, 378, + 379, 380, 381, 382, 383, 384, 385, 386, 387, -1, + 389, 390, 391, 392, 393, 394, 395, -1, 397, 398, + -1, 400, 401, 402, 403, 404, -1, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, -1, -1, -1, 425, 426, -1, 428, + 429, 430, 431, 432, 433, 434, 435, -1, 437, 438, + 439, -1, -1, 442, 443, -1, 445, -1, -1, 448, + 449, 450, 451, 452, 453, 454, 455, -1, -1, 458, + 459, 460, 461, -1, 463, 464, 465, 466, -1, 468, + 469, 470, 471, 472, -1, 474, 475, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, -1, -1, 488, + -1, -1, 491, 492, 493, 494, 495, 496, 3, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 508, + 509, 510, 511, -1, -1, -1, -1, -1, 23, 24, + 25, 26, 27, 28, 29, 30, -1, 32, 33, 34, + -1, -1, -1, 38, -1, -1, -1, -1, 43, 44, + 45, -1, 47, 48, 49, 50, 51, 52, 53, -1, + 55, 56, 57, 58, -1, 60, 61, 62, 63, 64, + 65, -1, -1, 68, 69, 70, 71, 72, 73, 74, + -1, 76, 77, 78, 79, -1, -1, 82, -1, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, -1, + 95, 96, 97, 98, 99, 100, -1, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + 115, 116, 117, -1, 119, -1, 121, 122, 123, 124, + 125, 126, -1, -1, 129, 130, 131, 132, -1, -1, + 135, 136, 137, 138, 139, -1, 141, 142, 143, -1, + 145, 146, 147, 148, -1, 150, 151, 152, 153, -1, + 155, 156, 157, 158, 159, 160, -1, -1, 163, -1, + 165, 166, 167, 168, -1, 170, -1, 172, 173, -1, + 175, 176, 177, -1, 179, 180, 181, 182, -1, -1, + -1, 186, 187, -1, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, -1, 202, 203, 204, + 205, 206, 207, 208, -1, 210, 211, -1, 213, 214, + 215, 216, 217, 218, 219, -1, 221, -1, 223, 224, + 225, 226, 227, 228, 229, 230, -1, 232, 233, 234, + -1, -1, 237, 238, 239, 240, -1, 242, 243, 244, + 245, 246, 247, 248, 249, -1, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, -1, -1, 281, 282, 283, 284, + -1, -1, 287, 288, 289, 290, 291, -1, 293, 294, + -1, -1, 297, 298, 299, -1, -1, 302, 303, -1, + 305, 306, 307, -1, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, -1, -1, -1, -1, + 325, 326, -1, 328, 329, 330, -1, 332, 333, 334, + -1, 336, 337, 338, 339, 340, 341, -1, 343, 344, + 345, 346, 347, 348, 349, 350, 351, 352, -1, 354, + 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, + 365, 366, -1, 368, 369, 370, 371, 372, 373, -1, + 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, + 385, 386, 387, -1, 389, 390, 391, 392, 393, 394, + 395, -1, 397, 398, -1, 400, 401, 402, 403, 404, + -1, 406, 407, 408, 409, 410, 411, 412, 413, 414, + 415, 416, 417, 418, 419, -1, 421, -1, -1, -1, + 425, 426, -1, 428, 429, 430, 431, 432, 433, 434, + 435, -1, 437, 438, 439, -1, -1, 442, 443, -1, + 445, -1, -1, 448, 449, 450, 451, 452, 453, 454, + 455, -1, -1, 458, 459, 460, 461, -1, 463, 464, + 465, 466, -1, 468, 469, 470, 471, 472, -1, 474, + 475, -1, 477, 478, 479, 480, 481, 482, 483, 484, + 485, -1, -1, 488, -1, -1, 491, 492, 493, 494, + 495, 496, 3, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 508, 509, 510, 511, -1, -1, -1, + -1, -1, 23, 24, 25, 26, 27, 28, 29, 30, + -1, 32, 33, 34, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 44, 45, -1, -1, 48, 49, -1, + 51, 52, 53, 54, 55, -1, 57, 58, -1, -1, + 61, 62, 63, 64, 65, -1, -1, 68, 69, 70, + 71, 72, 73, 74, -1, 76, 77, 78, 79, 80, + -1, -1, -1, 84, 85, 86, 87, 88, 89, -1, + 91, 92, 93, -1, 95, 96, 97, 98, 99, 100, + -1, -1, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, 117, -1, 119, -1, + 121, 122, 123, 124, 125, 126, -1, -1, 129, 130, + 131, 132, -1, -1, 135, 136, 137, 138, 139, -1, + 141, 142, 143, -1, 145, 146, 147, 148, -1, 150, + 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, + 161, -1, 163, -1, 165, 166, 167, 168, -1, 170, + -1, 172, -1, -1, -1, 176, 177, 178, -1, 180, + 181, 182, -1, 184, 185, -1, 187, -1, 189, 190, + 191, 192, 193, 194, 195, -1, 197, 198, 199, 200, + -1, 202, 203, 204, 205, 206, 207, 208, -1, 210, + -1, 212, 213, 214, 215, 216, 217, 218, 219, -1, + 221, -1, 223, -1, -1, 226, -1, 228, 229, 230, + -1, 232, 233, 234, -1, -1, 237, -1, 239, -1, + -1, 242, 243, 244, 245, 246, 247, 248, 249, 250, + 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, -1, 275, 276, 277, 278, 279, -1, + 281, 282, -1, 284, -1, 286, 287, 288, 289, 290, + 291, -1, 293, 294, -1, -1, 297, 298, 299, -1, + -1, 302, 303, 304, -1, 306, -1, 308, 309, 310, + 311, 312, 313, 314, -1, 316, 317, 318, 319, 320, + -1, -1, -1, -1, 325, 326, 327, -1, 329, 330, + 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, + 341, -1, 343, 344, 345, -1, 347, 348, 349, 350, + 351, 352, -1, 354, 355, 356, 357, 358, 359, 360, + 361, 362, 363, 364, 365, 366, -1, 368, 369, -1, + 371, 372, 373, 374, -1, 376, 377, 378, 379, 380, + 381, 382, 383, 384, 385, 386, 387, -1, -1, 390, + 391, 392, 393, 394, 395, 396, 397, 398, -1, -1, + 401, 402, 403, 404, -1, 406, 407, 408, 409, 410, + 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, -1, -1, 425, 426, -1, 428, -1, 430, + 431, 432, 433, 434, 435, -1, 437, 438, 439, -1, + -1, 442, 443, 444, 445, 446, -1, 448, 449, 450, + 451, 452, 453, 454, 455, -1, -1, 458, 459, 460, + -1, -1, 463, 464, 465, 466, -1, 468, 469, 470, + 471, 472, 473, 474, 475, -1, 477, -1, 479, 480, + 481, 482, 483, 484, 485, -1, -1, 488, -1, -1, + 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, + 511, 23, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 33, -1, 35, 36, -1, -1, -1, -1, 23, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 33, + -1, 53, -1, -1, -1, -1, -1, -1, -1, -1, + 62, -1, -1, -1, -1, -1, -1, -1, -1, 53, + -1, -1, -1, -1, 76, -1, -1, -1, 62, -1, + -1, -1, -1, -1, -1, 87, -1, -1, -1, -1, + -1, -1, 76, -1, -1, -1, -1, 99, -1, 101, + -1, -1, -1, 87, -1, -1, -1, -1, -1, -1, + 112, -1, -1, -1, -1, 99, -1, 101, -1, -1, + -1, -1, -1, -1, -1, 127, 128, -1, 112, -1, + -1, -1, -1, -1, -1, -1, 138, -1, -1, -1, + -1, -1, 144, 127, 128, -1, -1, -1, -1, -1, + -1, 153, -1, -1, 138, -1, -1, -1, -1, -1, + 144, -1, -1, -1, -1, -1, -1, -1, 170, 153, + -1, -1, 174, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 170, -1, -1, -1, + 174, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 216, -1, -1, + -1, -1, -1, -1, 216, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 216, -1, -1, -1, -1, - -1, -1, -1, -1, 243, -1, -1, -1, -1, -1, + -1, -1, 216, -1, -1, -1, -1, -1, -1, -1, + -1, 243, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 243, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 243, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 321, + 322, 323, -1, -1, -1, -1, -1, 329, -1, -1, + 332, -1, -1, -1, -1, -1, -1, 321, 322, 323, + -1, -1, -1, -1, -1, 329, -1, -1, 332, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 321, 322, 323, -1, -1, -1, -1, -1, - 329, -1, -1, 332, -1, -1, -1, -1, -1, -1, - 321, 322, 323, -1, -1, -1, -1, -1, 329, -1, - -1, 332, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 363, -1, -1, -1, -1, -1, - -1, -1, -1, 372, -1, -1, -1, -1, -1, -1, - -1, -1, 363, -1, -1, -1, -1, -1, -1, 388, - -1, 372, -1, -1, -1, -1, 395, -1, -1, -1, - 399, -1, -1, -1, -1, -1, -1, 388, -1, -1, - -1, -1, 411, -1, 395, -1, -1, -1, 399, -1, - -1, -1, -1, -1, 423, -1, -1, -1, 427, -1, - 411, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 423, -1, -1, -1, 427, -1, -1, 448, + -1, 363, -1, -1, -1, -1, -1, -1, -1, -1, + 372, -1, -1, -1, -1, -1, -1, -1, -1, 363, + -1, -1, -1, -1, -1, -1, 388, -1, 372, -1, + -1, -1, -1, 395, -1, -1, -1, 399, -1, -1, + -1, -1, -1, -1, 388, -1, -1, -1, -1, 411, + -1, 395, -1, -1, -1, 399, -1, -1, -1, -1, + -1, 423, -1, -1, -1, 427, -1, 411, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 423, + -1, -1, -1, 427, -1, -1, 448, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 462, -1, -1, -1, 448, -1, 468, - -1, -1, -1, -1, 473, -1, -1, -1, -1, 478, - -1, 462, -1, -1, -1, -1, -1, 468, -1, -1, - -1, 490, 473, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 490, - -1, -1, -1, -1, -1, 514, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 528, - -1, -1, -1, 514, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 528 + 462, -1, -1, -1, 448, -1, 468, -1, -1, -1, + -1, 473, -1, -1, -1, -1, 478, -1, 462, -1, + -1, -1, -1, -1, 468, -1, -1, -1, 490, 473, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 490, -1, -1, -1, + -1, -1, 514, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 528, -1, -1, -1, + 514, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 528 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing @@ -19949,381 +20053,382 @@ static const yytype_uint16 yystos[] = 332, 363, 372, 388, 395, 399, 411, 423, 427, 448, 462, 465, 468, 473, 490, 514, 528, 539, 540, 541, 542, 557, 566, 568, 573, 589, 593, 594, 596, 603, - 604, 608, 615, 617, 620, 621, 671, 677, 688, 699, - 700, 713, 714, 715, 716, 718, 720, 721, 725, 785, - 786, 966, 969, 972, 979, 980, 982, 985, 986, 987, - 994, 998, 1004, 1005, 1008, 1013, 1017, 1018, 1019, 1026, - 1029, 1030, 1031, 1034, 1035, 1037, 442, 493, 618, 205, - 379, 390, 427, 480, 109, 194, 300, 1020, 618, 3, - 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, - 34, 43, 44, 45, 47, 48, 49, 50, 51, 52, - 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, - 65, 68, 69, 70, 71, 72, 73, 74, 76, 77, - 78, 79, 80, 82, 84, 85, 86, 87, 88, 89, - 90, 91, 92, 93, 95, 96, 97, 98, 99, 100, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, 119, 121, 122, 123, - 124, 125, 126, 129, 130, 131, 132, 135, 136, 137, - 138, 139, 141, 142, 143, 145, 146, 147, 148, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, 163, 165, 166, 167, 168, 170, 172, 173, 175, - 176, 177, 178, 179, 180, 181, 182, 184, 185, 186, - 187, 189, 190, 191, 192, 193, 194, 195, 196, 197, - 198, 199, 200, 202, 203, 204, 205, 206, 207, 208, - 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - 221, 223, 224, 225, 226, 227, 228, 229, 230, 232, - 233, 234, 237, 238, 239, 240, 242, 243, 244, 245, - 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, - 276, 277, 278, 279, 281, 282, 283, 284, 286, 287, - 288, 289, 290, 291, 293, 294, 297, 298, 299, 302, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, 325, 326, - 327, 328, 329, 330, 331, 332, 333, 334, 336, 337, - 338, 339, 340, 341, 343, 344, 345, 346, 347, 348, - 349, 350, 351, 352, 354, 355, 356, 357, 358, 359, - 360, 361, 362, 363, 364, 365, 366, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, - 381, 382, 383, 384, 385, 386, 387, 390, 391, 392, - 393, 394, 395, 396, 397, 398, 400, 401, 402, 403, - 404, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 417, 418, 419, 420, 421, 422, 425, 426, - 428, 429, 430, 431, 432, 433, 434, 435, 437, 438, - 439, 442, 443, 444, 445, 446, 448, 449, 450, 451, - 452, 453, 454, 455, 458, 459, 460, 463, 464, 465, - 466, 468, 469, 470, 471, 472, 473, 474, 475, 477, - 478, 479, 480, 481, 482, 483, 484, 485, 488, 491, - 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, - 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 559, 867, 955, 959, 1040, 1041, 1042, 3, 178, 250, - 420, 559, 981, 1040, 295, 618, 56, 174, 528, 710, - 180, 244, 300, 320, 379, 432, 434, 457, 460, 598, - 669, 978, 5, 31, 332, 559, 560, 954, 3, 31, - 35, 36, 37, 38, 39, 40, 41, 42, 43, 46, - 47, 50, 54, 55, 56, 57, 58, 59, 60, 66, - 67, 72, 73, 75, 80, 81, 82, 83, 84, 90, - 94, 101, 102, 109, 113, 116, 118, 120, 127, 128, - 133, 134, 140, 144, 149, 154, 161, 162, 164, 167, - 169, 171, 173, 174, 175, 178, 179, 183, 184, 185, - 188, 196, 201, 209, 211, 212, 218, 219, 220, 221, - 222, 224, 225, 227, 231, 235, 236, 238, 240, 241, - 250, 273, 274, 275, 279, 280, 283, 285, 286, 288, - 292, 295, 296, 300, 301, 304, 305, 307, 308, 321, - 322, 323, 324, 327, 328, 331, 335, 342, 348, 353, - 367, 370, 374, 388, 389, 396, 399, 400, 403, 405, - 420, 422, 423, 424, 427, 429, 436, 438, 439, 440, - 441, 444, 446, 447, 450, 456, 457, 461, 462, 467, - 473, 474, 476, 478, 486, 487, 489, 490, 497, 498, - 499, 500, 501, 502, 503, 504, 505, 506, 507, 565, - 1040, 1044, 1046, 25, 82, 98, 148, 158, 171, 176, - 205, 249, 254, 326, 341, 376, 379, 390, 393, 413, - 427, 434, 435, 445, 451, 480, 598, 672, 673, 676, - 618, 954, 101, 138, 478, 528, 542, 557, 566, 568, - 589, 593, 594, 603, 604, 608, 617, 621, 671, 677, - 688, 699, 700, 713, 966, 969, 972, 979, 980, 990, - 994, 998, 1004, 1008, 1013, 1026, 1029, 1034, 1035, 1037, - 109, 76, 67, 80, 82, 161, 235, 286, 296, 308, - 327, 375, 422, 444, 446, 450, 473, 528, 558, 559, - 560, 714, 786, 788, 790, 791, 801, 808, 809, 867, - 869, 870, 109, 5, 559, 561, 1006, 559, 954, 31, - 180, 244, 394, 438, 442, 475, 559, 1027, 1028, 1033, - 618, 31, 133, 737, 738, 180, 244, 379, 394, 438, - 475, 999, 1000, 1033, 618, 558, 559, 560, 713, 725, - 808, 427, 734, 558, 175, 528, 1010, 528, 351, 726, - 727, 954, 726, 714, 715, 1029, 0, 531, 123, 215, - 256, 464, 149, 220, 301, 456, 740, 741, 791, 791, - 714, 716, 718, 532, 478, 988, 216, 31, 428, 438, - 442, 558, 713, 194, 558, 954, 194, 558, 194, 808, - 194, 558, 280, 361, 561, 347, 619, 526, 530, 562, - 563, 528, 83, 109, 176, 205, 249, 379, 390, 427, - 451, 480, 984, 109, 713, 558, 432, 434, 432, 434, - 361, 194, 558, 386, 176, 249, 351, 390, 427, 451, - 480, 695, 205, 31, 954, 194, 565, 257, 445, 108, - 427, 427, 480, 383, 386, 194, 559, 674, 961, 194, - 951, 954, 194, 954, 528, 607, 300, 434, 990, 3, - 473, 991, 993, 994, 996, 997, 1040, 1044, 988, 559, - 561, 981, 528, 528, 169, 528, 714, 809, 528, 528, - 558, 528, 528, 174, 528, 528, 528, 528, 714, 786, - 791, 801, 521, 562, 19, 41, 559, 802, 803, 802, - 388, 532, 717, 528, 714, 808, 809, 38, 43, 102, - 175, 211, 227, 238, 274, 321, 328, 370, 389, 462, - 805, 803, 41, 559, 802, 804, 514, 813, 561, 517, - 528, 528, 967, 1028, 1028, 1028, 511, 226, 1028, 530, - 295, 4, 6, 7, 8, 9, 10, 40, 55, 57, - 58, 66, 72, 73, 84, 113, 116, 118, 137, 154, - 162, 167, 184, 185, 218, 219, 221, 231, 250, 273, - 275, 280, 285, 288, 297, 348, 374, 403, 438, 439, - 447, 461, 474, 512, 519, 520, 521, 526, 528, 533, - 534, 535, 536, 559, 561, 714, 775, 825, 828, 831, - 832, 833, 835, 836, 837, 838, 840, 841, 857, 859, - 860, 861, 862, 863, 864, 865, 866, 867, 868, 870, - 871, 886, 887, 898, 920, 926, 934, 935, 936, 955, - 956, 957, 933, 935, 999, 999, 561, 999, 511, 999, - 174, 440, 517, 619, 562, 808, 1014, 3, 173, 175, - 478, 994, 1009, 1011, 173, 1012, 559, 857, 904, 905, - 726, 532, 528, 963, 529, 529, 529, 541, 174, 300, - 576, 222, 159, 1014, 31, 133, 735, 735, 60, 735, - 164, 169, 241, 292, 746, 748, 749, 778, 780, 781, - 782, 183, 295, 467, 295, 740, 741, 528, 558, 1006, - 428, 1032, 174, 511, 226, 154, 27, 33, 138, 299, - 359, 363, 395, 470, 551, 554, 555, 359, 154, 41, - 61, 107, 204, 255, 266, 278, 310, 359, 365, 390, - 395, 411, 554, 609, 612, 154, 359, 395, 554, 154, - 359, 395, 554, 154, 1020, 41, 1021, 296, 495, 857, - 927, 564, 565, 563, 3, 31, 38, 43, 47, 50, - 56, 60, 82, 84, 90, 102, 133, 173, 175, 178, - 179, 196, 211, 224, 225, 227, 238, 240, 250, 274, - 283, 305, 307, 328, 370, 389, 400, 420, 429, 450, - 461, 476, 478, 529, 741, 857, 907, 908, 958, 964, - 1040, 1045, 857, 427, 558, 559, 529, 528, 658, 379, - 598, 669, 280, 970, 194, 559, 597, 480, 194, 558, - 194, 558, 1039, 194, 558, 194, 558, 194, 558, 90, - 975, 154, 494, 91, 130, 313, 433, 194, 559, 154, - 530, 962, 64, 366, 532, 675, 154, 532, 675, 154, - 295, 605, 606, 857, 964, 361, 529, 532, 4, 162, - 295, 447, 519, 520, 561, 611, 614, 957, 989, 991, - 992, 995, 990, 440, 528, 707, 709, 857, 905, 528, - 3, 69, 70, 110, 111, 114, 115, 191, 192, 258, - 259, 260, 261, 262, 263, 264, 265, 268, 269, 343, - 344, 384, 385, 484, 485, 508, 509, 561, 843, 844, - 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, - 855, 910, 911, 803, 804, 857, 558, 857, 912, 519, - 520, 559, 858, 859, 887, 898, 914, 528, 857, 904, - 915, 857, 59, 174, 236, 441, 857, 905, 918, 857, - 529, 560, 528, 429, 755, 756, 756, 737, 738, 791, - 222, 732, 801, 756, 47, 760, 756, 227, 38, 227, - 389, 805, 227, 305, 806, 791, 806, 227, 805, 528, - 227, 806, 227, 150, 202, 793, 227, 760, 528, 560, - 528, 756, 302, 857, 1001, 1003, 907, 3, 38, 43, - 47, 50, 55, 56, 57, 58, 60, 72, 73, 82, - 84, 90, 102, 113, 116, 167, 173, 175, 179, 196, - 211, 218, 219, 221, 224, 225, 227, 238, 240, 250, - 273, 274, 275, 283, 288, 305, 307, 328, 348, 370, - 374, 389, 396, 400, 403, 420, 429, 438, 439, 450, - 456, 461, 474, 478, 819, 821, 822, 824, 826, 828, - 830, 832, 833, 834, 836, 837, 840, 841, 909, 960, - 1040, 1043, 41, 239, 559, 528, 526, 714, 477, 839, - 857, 924, 839, 839, 528, 528, 827, 827, 331, 714, - 528, 829, 951, 536, 72, 73, 839, 857, 827, 528, - 528, 492, 514, 528, 842, 528, 842, 528, 857, 857, - 857, 84, 150, 937, 941, 857, 905, 906, 714, 857, - 904, 565, 9, 560, 888, 889, 890, 19, 530, 562, - 928, 562, 528, 561, 528, 528, 561, 957, 3, 8, - 11, 15, 16, 17, 18, 20, 21, 22, 37, 41, - 47, 54, 81, 179, 196, 201, 224, 225, 240, 280, - 283, 297, 300, 400, 512, 515, 516, 517, 519, 520, - 521, 522, 523, 524, 896, 897, 898, 900, 931, 491, - 872, 307, 857, 532, 732, 528, 561, 732, 3, 118, - 244, 561, 611, 841, 1002, 105, 1003, 1003, 41, 559, - 529, 532, 988, 532, 529, 727, 951, 952, 467, 728, - 1014, 195, 361, 222, 1014, 658, 395, 716, 716, 31, - 742, 743, 857, 60, 716, 736, 166, 277, 766, 229, - 278, 347, 398, 464, 4, 9, 31, 761, 857, 519, - 520, 762, 763, 857, 859, 778, 779, 749, 748, 746, - 747, 169, 781, 290, 783, 60, 722, 723, 724, 794, - 858, 935, 935, 746, 778, 905, 963, 174, 479, 1007, - 558, 239, 558, 75, 83, 94, 171, 194, 335, 457, - 549, 550, 559, 638, 665, 83, 94, 567, 94, 567, - 528, 440, 316, 406, 528, 636, 248, 316, 406, 460, - 636, 94, 532, 440, 558, 3, 824, 611, 60, 613, - 611, 611, 107, 255, 266, 60, 440, 490, 514, 610, - 271, 379, 610, 612, 808, 94, 440, 567, 379, 558, - 440, 379, 1020, 561, 559, 528, 1025, 527, 19, 907, - 907, 908, 195, 364, 739, 532, 740, 741, 13, 14, - 224, 224, 440, 440, 559, 657, 662, 490, 708, 558, - 386, 351, 390, 427, 451, 480, 695, 154, 101, 594, - 621, 971, 972, 1035, 280, 201, 599, 558, 280, 595, - 609, 280, 528, 658, 41, 280, 41, 280, 658, 280, - 528, 689, 690, 691, 692, 693, 694, 696, 194, 559, - 652, 976, 565, 154, 174, 616, 674, 564, 530, 961, - 951, 954, 954, 961, 529, 532, 13, 990, 996, 4, - 957, 4, 957, 561, 565, 1023, 1024, 56, 103, 124, - 142, 147, 170, 173, 189, 285, 293, 315, 345, 711, - 41, 529, 857, 529, 174, 532, 529, 324, 913, 529, - 858, 858, 11, 15, 16, 20, 21, 22, 201, 224, - 297, 515, 516, 517, 519, 520, 521, 522, 523, 524, - 898, 858, 529, 810, 811, 869, 169, 174, 916, 917, - 532, 529, 41, 918, 905, 918, 918, 174, 529, 41, - 802, 528, 952, 4, 9, 559, 750, 751, 753, 754, - 862, 935, 933, 180, 244, 427, 432, 434, 460, 558, - 733, 487, 814, 529, 528, 756, 791, 791, 227, 791, - 295, 467, 807, 791, 227, 935, 791, 791, 287, 287, - 528, 791, 756, 560, 815, 816, 528, 560, 815, 532, - 529, 532, 530, 528, 824, 528, 528, 530, 40, 823, - 528, 843, 844, 845, 846, 847, 848, 849, 850, 851, - 852, 853, 854, 855, 856, 529, 532, 827, 568, 573, - 699, 700, 713, 968, 1013, 1029, 905, 906, 528, 486, - 921, 922, 857, 906, 957, 19, 857, 891, 892, 893, - 894, 839, 839, 8, 15, 16, 20, 21, 22, 515, - 516, 517, 519, 520, 521, 522, 523, 524, 559, 896, - 901, 529, 905, 438, 438, 957, 957, 857, 528, 528, - 560, 938, 361, 945, 169, 527, 529, 532, 19, 532, - 537, 857, 521, 563, 905, 957, 857, 856, 856, 821, - 857, 857, 857, 857, 857, 857, 857, 857, 5, 565, - 965, 438, 46, 424, 932, 961, 857, 857, 528, 714, - 863, 919, 926, 133, 162, 280, 285, 290, 447, 458, - 857, 285, 528, 857, 440, 54, 179, 196, 201, 240, - 400, 857, 857, 857, 857, 857, 857, 857, 857, 857, - 857, 31, 39, 405, 895, 526, 530, 930, 183, 165, - 873, 374, 528, 887, 936, 174, 787, 907, 787, 528, - 561, 559, 558, 1009, 558, 1017, 857, 532, 529, 229, - 41, 467, 1016, 558, 570, 467, 528, 559, 575, 585, - 586, 588, 42, 127, 744, 532, 467, 744, 271, 716, - 374, 375, 519, 520, 763, 765, 859, 398, 229, 296, - 319, 319, 532, 523, 4, 764, 957, 764, 374, 375, - 765, 558, 950, 284, 402, 784, 528, 952, 953, 532, - 183, 467, 201, 183, 222, 779, 747, 529, 559, 561, - 559, 561, 359, 554, 528, 194, 550, 954, 229, 280, - 229, 467, 528, 641, 648, 649, 820, 821, 530, 547, - 548, 954, 559, 194, 954, 194, 549, 27, 138, 395, - 546, 553, 565, 632, 646, 954, 60, 60, 565, 640, - 661, 60, 60, 954, 551, 954, 359, 395, 554, 609, - 611, 961, 954, 611, 961, 954, 611, 359, 395, 554, - 954, 954, 549, 954, 359, 395, 554, 954, 954, 561, - 1021, 1024, 520, 857, 927, 740, 740, 740, 287, 287, - 529, 476, 908, 739, 857, 857, 285, 561, 983, 285, - 983, 559, 340, 706, 529, 532, 293, 174, 440, 702, - 970, 597, 480, 558, 558, 1039, 558, 558, 558, 558, - 300, 669, 154, 3, 528, 528, 154, 154, 240, 559, - 638, 650, 653, 656, 666, 668, 490, 492, 643, 153, - 713, 154, 145, 590, 821, 154, 490, 977, 154, 3, - 43, 47, 50, 56, 82, 84, 90, 102, 173, 175, - 178, 179, 196, 211, 224, 225, 227, 238, 240, 250, - 274, 283, 305, 307, 328, 370, 400, 420, 429, 450, - 461, 478, 529, 697, 698, 964, 1040, 532, 532, 41, - 280, 295, 559, 3, 675, 564, 675, 295, 675, 605, - 857, 707, 857, 1022, 529, 532, 41, 705, 561, 705, - 280, 285, 345, 705, 60, 705, 821, 529, 857, 857, - 857, 916, 821, 858, 858, 858, 858, 858, 858, 133, - 280, 290, 858, 858, 858, 858, 858, 858, 858, 858, - 858, 858, 529, 532, 41, 812, 857, 857, 917, 916, - 821, 529, 529, 529, 905, 821, 952, 529, 319, 375, - 523, 528, 528, 732, 432, 434, 432, 434, 558, 734, - 734, 734, 857, 183, 767, 439, 479, 758, 759, 807, - 807, 791, 857, 528, 791, 169, 807, 528, 560, 798, - 807, 821, 529, 532, 815, 529, 1001, 3, 909, 40, - 823, 559, 818, 818, 3, 526, 526, 957, 440, 440, - 440, 440, 821, 464, 529, 527, 905, 857, 140, 922, - 923, 529, 529, 529, 857, 19, 532, 537, 530, 529, - 529, 511, 511, 529, 529, 529, 857, 938, 939, 940, - 530, 528, 857, 942, 359, 949, 951, 857, 857, 888, - 941, 529, 529, 529, 511, 858, 858, 147, 905, 174, - 133, 162, 285, 290, 447, 458, 528, 147, 901, 857, - 424, 932, 857, 919, 857, 440, 528, 714, 857, 927, - 564, 528, 528, 157, 874, 788, 789, 814, 740, 814, - 957, 856, 963, 963, 528, 254, 280, 731, 789, 487, - 1015, 41, 60, 571, 791, 581, 588, 928, 532, 787, - 517, 513, 745, 743, 297, 896, 899, 745, 4, 957, - 765, 296, 464, 762, 532, 247, 952, 722, 60, 935, - 528, 560, 60, 271, 1007, 1007, 440, 857, 280, 665, - 528, 154, 528, 641, 205, 662, 663, 622, 41, 178, - 631, 659, 564, 548, 622, 27, 138, 363, 365, 395, - 543, 544, 545, 555, 556, 154, 675, 154, 675, 632, - 646, 632, 529, 532, 561, 625, 517, 530, 529, 532, - 528, 528, 440, 379, 94, 440, 567, 379, 440, 440, - 440, 379, 1021, 1025, 529, 19, 19, 527, 739, 739, - 739, 908, 529, 528, 701, 3, 414, 415, 528, 561, - 712, 862, 657, 706, 599, 558, 595, 528, 41, 41, - 658, 692, 694, 970, 361, 427, 597, 565, 601, 602, - 662, 558, 558, 1039, 558, 648, 649, 529, 532, 293, - 636, 293, 295, 635, 954, 490, 1038, 558, 528, 714, - 558, 636, 41, 558, 529, 532, 820, 821, 690, 696, - 693, 696, 427, 857, 154, 558, 616, 961, 1023, 561, - 561, 285, 662, 521, 662, 561, 521, 662, 561, 529, - 529, 917, 174, 133, 290, 528, 813, 810, 528, 529, - 529, 529, 559, 751, 814, 734, 734, 734, 734, 558, - 558, 558, 60, 188, 776, 14, 529, 807, 952, 528, - 795, 796, 797, 860, 863, 952, 169, 81, 817, 816, - 529, 529, 526, 821, 529, 532, 529, 957, 527, 957, - 529, 844, 846, 847, 848, 847, 848, 848, 529, 436, - 857, 144, 857, 891, 901, 842, 842, 529, 532, 529, - 560, 857, 942, 943, 944, 41, 528, 938, 946, 201, - 529, 945, 856, 857, 37, 37, 857, 529, 857, 174, - 528, 909, 857, 529, 147, 858, 858, 147, 147, 857, - 857, 527, 19, 528, 929, 741, 487, 857, 306, 878, - 532, 767, 739, 767, 529, 559, 729, 730, 925, 254, - 528, 857, 367, 579, 559, 271, 327, 118, 309, 528, - 569, 713, 807, 529, 532, 575, 1015, 857, 166, 234, - 528, 745, 296, 558, 529, 953, 183, 714, 715, 935, - 953, 954, 954, 529, 154, 663, 550, 663, 622, 652, - 532, 529, 120, 209, 278, 280, 647, 528, 34, 60, - 670, 659, 75, 81, 94, 118, 120, 209, 280, 285, - 335, 353, 457, 467, 627, 628, 642, 178, 118, 193, - 280, 636, 610, 108, 118, 178, 280, 413, 416, 612, - 636, 395, 545, 451, 954, 549, 553, 3, 38, 43, - 47, 50, 56, 60, 82, 84, 90, 102, 173, 175, - 179, 196, 211, 224, 225, 227, 238, 240, 250, 274, - 279, 283, 297, 305, 307, 328, 370, 389, 396, 400, - 420, 429, 450, 456, 461, 478, 519, 520, 561, 611, - 623, 664, 821, 899, 958, 1040, 1046, 565, 661, 905, - 742, 954, 954, 954, 954, 549, 954, 954, 954, 954, - 954, 1025, 927, 927, 529, 529, 529, 740, 108, 379, - 530, 857, 610, 712, 528, 528, 656, 713, 590, 977, - 669, 194, 558, 599, 600, 857, 529, 532, 529, 595, - 528, 41, 645, 643, 559, 653, 87, 607, 108, 278, - 41, 561, 591, 592, 658, 713, 692, 694, 529, 698, - 13, 14, 41, 41, 714, 715, 652, 467, 974, 675, - 662, 858, 174, 528, 909, 815, 529, 532, 529, 767, - 558, 558, 558, 558, 31, 104, 184, 373, 528, 768, - 769, 770, 771, 772, 773, 774, 857, 857, 489, 875, - 857, 529, 859, 902, 903, 201, 183, 792, 796, 529, - 798, 799, 800, 961, 823, 957, 823, 559, 823, 527, - 527, 857, 938, 532, 529, 559, 946, 947, 948, 41, - 857, 859, 949, 857, 857, 857, 909, 529, 857, 37, - 37, 857, 857, 147, 529, 520, 927, 529, 907, 529, - 857, 529, 528, 559, 879, 776, 529, 776, 561, 529, - 532, 968, 934, 473, 426, 466, 580, 559, 574, 584, - 295, 577, 486, 683, 685, 686, 687, 517, 588, 579, - 901, 60, 529, 529, 472, 473, 719, 622, 550, 529, - 529, 490, 655, 121, 197, 207, 120, 469, 857, 118, - 41, 528, 961, 954, 858, 121, 197, 120, 285, 229, - 558, 655, 89, 670, 194, 285, 611, 857, 670, 285, - 519, 520, 614, 559, 820, 675, 675, 3, 958, 962, - 517, 529, 529, 440, 440, 527, 527, 739, 529, 529, - 559, 529, 707, 467, 703, 704, 602, 662, 529, 1038, - 41, 427, 280, 528, 528, 601, 977, 656, 153, 713, - 151, 203, 635, 123, 138, 334, 1038, 108, 590, 529, - 532, 977, 490, 1036, 857, 857, 427, 295, 559, 973, - 528, 858, 909, 529, 529, 9, 360, 757, 776, 528, - 397, 528, 529, 532, 559, 876, 877, 342, 777, 532, - 529, 528, 560, 60, 529, 201, 529, 799, 527, 821, - 942, 532, 529, 559, 527, 194, 529, 857, 857, 857, - 19, 19, 527, 529, 529, 559, 880, 875, 561, 875, - 925, 529, 532, 472, 928, 529, 532, 92, 579, 253, - 280, 687, 579, 857, 529, 953, 953, 353, 655, 528, - 644, 622, 529, 193, 528, 857, 280, 628, 655, 658, - 954, 41, 154, 817, 962, 523, 623, 954, 954, 529, - 610, 125, 529, 529, 643, 713, 558, 154, 602, 41, - 529, 954, 1038, 31, 86, 95, 119, 193, 206, 413, - 416, 639, 639, 375, 375, 561, 41, 65, 75, 244, - 714, 558, 528, 559, 578, 587, 869, 529, 529, 528, - 875, 905, 528, 905, 770, 41, 532, 857, 467, 752, - 859, 935, 952, 803, 528, 803, 946, 857, 927, 927, - 315, 881, 777, 777, 713, 309, 713, 574, 295, 528, - 572, 37, 678, 253, 558, 622, 565, 651, 654, 417, - 482, 629, 630, 528, 624, 857, 529, 252, 667, 193, - 467, 552, 523, 451, 707, 561, 977, 635, 1036, 528, - 558, 529, 713, 643, 607, 713, 75, 298, 75, 974, - 857, 81, 582, 529, 532, 582, 9, 777, 529, 769, - 529, 879, 877, 377, 529, 935, 527, 527, 527, 60, - 740, 752, 752, 580, 94, 587, 134, 857, 436, 60, - 684, 658, 517, 529, 532, 609, 529, 278, 637, 175, - 314, 401, 295, 633, 634, 660, 624, 857, 451, 41, - 528, 1036, 635, 1038, 1036, 298, 298, 528, 529, 961, - 583, 961, 977, 578, 583, 529, 752, 529, 754, 529, - 904, 186, 346, 375, 882, 472, 954, 529, 281, 464, - 123, 134, 146, 215, 464, 681, 407, 431, 678, 667, - 623, 654, 529, 630, 207, 123, 464, 295, 660, 295, - 633, 713, 587, 582, 744, 814, 744, 54, 105, 453, - 857, 883, 884, 883, 883, 529, 713, 814, 395, 281, - 682, 857, 118, 528, 571, 679, 395, 571, 436, 634, - 64, 278, 366, 395, 626, 626, 1036, 529, 583, 745, - 745, 884, 374, 168, 330, 168, 330, 150, 885, 885, - 885, 586, 473, 584, 521, 680, 473, 521, 586, 681, - 622, 26, 118, 285, 977, 744, 37, 105, 183, 278, - 437, 814, 529, 528, 814, 745, 884, 374, 303, 905, - 529 + 604, 608, 615, 616, 618, 621, 622, 672, 678, 689, + 700, 701, 714, 715, 716, 717, 719, 721, 722, 726, + 786, 787, 967, 970, 973, 980, 981, 983, 986, 987, + 988, 995, 999, 1005, 1006, 1009, 1014, 1018, 1019, 1020, + 1027, 1030, 1031, 1032, 1035, 1036, 1038, 442, 493, 619, + 109, 205, 379, 390, 427, 480, 109, 194, 300, 1021, + 619, 3, 23, 24, 25, 26, 27, 28, 29, 30, + 32, 33, 34, 43, 44, 45, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 61, 62, + 63, 64, 65, 68, 69, 70, 71, 72, 73, 74, + 76, 77, 78, 79, 80, 82, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 95, 96, 97, 98, + 99, 100, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 119, 121, + 122, 123, 124, 125, 126, 129, 130, 131, 132, 135, + 136, 137, 138, 139, 141, 142, 143, 145, 146, 147, + 148, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, 161, 163, 165, 166, 167, 168, 170, 172, + 173, 175, 176, 177, 178, 179, 180, 181, 182, 184, + 185, 186, 187, 189, 190, 191, 192, 193, 194, 195, + 196, 197, 198, 199, 200, 202, 203, 204, 205, 206, + 207, 208, 210, 211, 212, 213, 214, 215, 216, 217, + 218, 219, 221, 223, 224, 225, 226, 227, 228, 229, + 230, 232, 233, 234, 237, 238, 239, 240, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 281, 282, 283, 284, + 286, 287, 288, 289, 290, 291, 293, 294, 297, 298, + 299, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, + 336, 337, 338, 339, 340, 341, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, 354, 355, 356, 357, + 358, 359, 360, 361, 362, 363, 364, 365, 366, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + 379, 380, 381, 382, 383, 384, 385, 386, 387, 390, + 391, 392, 393, 394, 395, 396, 397, 398, 400, 401, + 402, 403, 404, 406, 407, 408, 409, 410, 411, 412, + 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, + 425, 426, 428, 429, 430, 431, 432, 433, 434, 435, + 437, 438, 439, 442, 443, 444, 445, 446, 448, 449, + 450, 451, 452, 453, 454, 455, 458, 459, 460, 463, + 464, 465, 466, 468, 469, 470, 471, 472, 473, 474, + 475, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 488, 491, 492, 493, 494, 495, 496, 497, 498, 499, + 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, + 510, 511, 559, 868, 956, 960, 1041, 1042, 1043, 3, + 178, 250, 420, 559, 982, 1041, 295, 619, 56, 174, + 528, 711, 180, 244, 300, 320, 379, 432, 434, 457, + 460, 598, 670, 979, 5, 31, 332, 559, 560, 955, + 3, 31, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 46, 47, 50, 54, 55, 56, 57, 58, 59, + 60, 66, 67, 72, 73, 75, 80, 81, 82, 83, + 84, 90, 94, 101, 102, 109, 113, 116, 118, 120, + 127, 128, 133, 134, 140, 144, 149, 154, 161, 162, + 164, 167, 169, 171, 173, 174, 175, 178, 179, 183, + 184, 185, 188, 196, 201, 209, 211, 212, 218, 219, + 220, 221, 222, 224, 225, 227, 231, 235, 236, 238, + 240, 241, 250, 273, 274, 275, 279, 280, 283, 285, + 286, 288, 292, 295, 296, 300, 301, 304, 305, 307, + 308, 321, 322, 323, 324, 327, 328, 331, 335, 342, + 348, 353, 367, 370, 374, 388, 389, 396, 399, 400, + 403, 405, 420, 422, 423, 424, 427, 429, 436, 438, + 439, 440, 441, 444, 446, 447, 450, 456, 457, 461, + 462, 467, 473, 474, 476, 478, 486, 487, 489, 490, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 565, 1041, 1045, 1047, 25, 82, 98, 148, 158, + 171, 176, 205, 249, 254, 326, 341, 376, 379, 390, + 393, 413, 427, 434, 435, 445, 451, 480, 598, 673, + 674, 677, 619, 955, 33, 101, 138, 478, 528, 542, + 557, 566, 568, 589, 593, 594, 603, 604, 608, 618, + 622, 672, 678, 689, 700, 701, 714, 967, 970, 973, + 980, 981, 991, 995, 999, 1005, 1009, 1014, 1027, 1030, + 1035, 1036, 1038, 109, 76, 67, 80, 82, 161, 235, + 286, 296, 308, 327, 375, 422, 444, 446, 450, 473, + 528, 558, 559, 560, 715, 787, 789, 791, 792, 802, + 809, 810, 868, 870, 871, 109, 5, 559, 561, 1007, + 559, 955, 31, 180, 244, 394, 438, 442, 475, 559, + 1028, 1029, 1034, 619, 31, 133, 738, 739, 180, 244, + 379, 394, 438, 475, 1000, 1001, 1034, 619, 558, 559, + 560, 714, 726, 809, 427, 735, 558, 175, 528, 1011, + 528, 351, 727, 728, 955, 727, 715, 716, 1030, 0, + 531, 123, 215, 256, 464, 149, 220, 301, 456, 741, + 742, 792, 792, 715, 717, 719, 532, 478, 989, 216, + 31, 428, 438, 442, 558, 714, 194, 559, 194, 558, + 955, 194, 558, 194, 809, 194, 558, 280, 361, 561, + 347, 620, 526, 530, 562, 563, 528, 83, 109, 176, + 205, 249, 379, 390, 427, 451, 480, 985, 109, 714, + 558, 432, 434, 432, 434, 361, 194, 558, 386, 176, + 249, 351, 390, 427, 451, 480, 696, 205, 31, 955, + 194, 565, 257, 445, 108, 427, 427, 480, 383, 386, + 194, 559, 675, 962, 194, 952, 955, 194, 955, 528, + 607, 300, 434, 991, 3, 473, 992, 994, 995, 997, + 998, 1041, 1045, 989, 559, 561, 982, 528, 528, 169, + 528, 715, 810, 528, 528, 558, 528, 528, 174, 528, + 528, 528, 528, 715, 787, 792, 802, 521, 562, 19, + 41, 559, 803, 804, 803, 388, 532, 718, 528, 715, + 809, 810, 38, 43, 102, 175, 211, 227, 238, 274, + 321, 328, 370, 389, 462, 806, 804, 41, 559, 803, + 805, 514, 814, 561, 517, 528, 528, 968, 1029, 1029, + 1029, 511, 226, 1029, 530, 295, 4, 6, 7, 8, + 9, 10, 40, 55, 57, 58, 66, 72, 73, 84, + 113, 116, 118, 137, 154, 162, 167, 184, 185, 218, + 219, 221, 231, 250, 273, 275, 280, 285, 288, 297, + 348, 374, 403, 438, 439, 447, 461, 474, 512, 519, + 520, 521, 526, 528, 533, 534, 535, 536, 559, 561, + 715, 776, 826, 829, 832, 833, 834, 836, 837, 838, + 839, 841, 842, 858, 860, 861, 862, 863, 864, 865, + 866, 867, 868, 869, 871, 872, 887, 888, 899, 921, + 927, 935, 936, 937, 956, 957, 958, 934, 936, 1000, + 1000, 561, 1000, 511, 1000, 174, 440, 517, 620, 562, + 809, 1015, 3, 173, 175, 478, 995, 1010, 1012, 173, + 1013, 559, 858, 905, 906, 727, 532, 528, 964, 529, + 529, 529, 541, 174, 300, 576, 222, 159, 1015, 31, + 133, 736, 736, 60, 736, 164, 169, 241, 292, 747, + 749, 750, 779, 781, 782, 783, 183, 295, 467, 295, + 741, 742, 528, 558, 1007, 428, 1033, 174, 511, 226, + 154, 359, 154, 27, 33, 138, 299, 359, 363, 395, + 470, 551, 554, 555, 359, 154, 41, 61, 107, 204, + 255, 266, 278, 310, 359, 365, 390, 395, 411, 554, + 609, 612, 154, 359, 395, 554, 154, 359, 395, 554, + 154, 1021, 41, 1022, 296, 495, 858, 928, 564, 565, + 563, 3, 31, 38, 43, 47, 50, 56, 60, 82, + 84, 90, 102, 133, 173, 175, 178, 179, 196, 211, + 224, 225, 227, 238, 240, 250, 274, 283, 305, 307, + 328, 370, 389, 400, 420, 429, 450, 461, 476, 478, + 529, 742, 858, 908, 909, 959, 965, 1041, 1046, 858, + 427, 558, 559, 529, 528, 659, 379, 598, 670, 280, + 971, 194, 559, 597, 480, 194, 558, 194, 558, 1040, + 194, 558, 194, 558, 194, 558, 90, 976, 154, 494, + 91, 130, 313, 433, 194, 559, 154, 530, 963, 64, + 366, 532, 676, 154, 532, 676, 154, 295, 605, 606, + 858, 965, 361, 529, 532, 4, 162, 295, 447, 519, + 520, 561, 611, 614, 958, 990, 992, 993, 996, 991, + 440, 528, 708, 710, 858, 906, 528, 3, 69, 70, + 110, 111, 114, 115, 191, 192, 258, 259, 260, 261, + 262, 263, 264, 265, 268, 269, 343, 344, 384, 385, + 484, 485, 508, 509, 561, 844, 845, 846, 847, 848, + 849, 850, 851, 852, 853, 854, 855, 856, 911, 912, + 804, 805, 858, 558, 858, 913, 519, 520, 559, 859, + 860, 888, 899, 915, 528, 858, 905, 916, 858, 59, + 174, 236, 441, 858, 906, 919, 858, 529, 560, 528, + 429, 756, 757, 757, 738, 739, 792, 222, 733, 802, + 757, 47, 761, 757, 227, 38, 227, 389, 806, 227, + 305, 807, 792, 807, 227, 806, 528, 227, 807, 227, + 150, 202, 794, 227, 761, 528, 560, 528, 757, 302, + 858, 1002, 1004, 908, 3, 38, 43, 47, 50, 55, + 56, 57, 58, 60, 72, 73, 82, 84, 90, 102, + 113, 116, 167, 173, 175, 179, 196, 211, 218, 219, + 221, 224, 225, 227, 238, 240, 250, 273, 274, 275, + 283, 288, 305, 307, 328, 348, 370, 374, 389, 396, + 400, 403, 420, 429, 438, 439, 450, 456, 461, 474, + 478, 820, 822, 823, 825, 827, 829, 831, 833, 834, + 835, 837, 838, 841, 842, 910, 961, 1041, 1044, 41, + 239, 559, 528, 526, 715, 477, 840, 858, 925, 840, + 840, 528, 528, 828, 828, 331, 715, 528, 830, 952, + 536, 72, 73, 840, 858, 828, 528, 528, 492, 514, + 528, 843, 528, 843, 528, 858, 858, 858, 84, 150, + 938, 942, 858, 906, 907, 715, 858, 905, 565, 9, + 560, 889, 890, 891, 19, 530, 562, 929, 562, 528, + 561, 528, 528, 561, 958, 3, 8, 11, 15, 16, + 17, 18, 20, 21, 22, 37, 41, 47, 54, 81, + 179, 196, 201, 224, 225, 240, 280, 283, 297, 300, + 400, 512, 515, 516, 517, 519, 520, 521, 522, 523, + 524, 897, 898, 899, 901, 932, 491, 873, 307, 858, + 532, 733, 528, 561, 733, 3, 118, 244, 561, 611, + 842, 1003, 105, 1004, 1004, 41, 559, 529, 532, 989, + 532, 529, 728, 952, 953, 467, 729, 1015, 195, 361, + 222, 1015, 659, 395, 717, 717, 31, 743, 744, 858, + 60, 717, 737, 166, 277, 767, 229, 278, 347, 398, + 464, 4, 9, 31, 762, 858, 519, 520, 763, 764, + 858, 860, 779, 780, 750, 749, 747, 748, 169, 782, + 290, 784, 60, 723, 724, 725, 795, 859, 936, 936, + 747, 779, 906, 964, 174, 479, 1008, 558, 239, 559, + 440, 558, 75, 83, 94, 171, 194, 335, 457, 549, + 550, 559, 639, 666, 83, 94, 567, 94, 567, 528, + 440, 316, 406, 528, 637, 248, 316, 406, 460, 637, + 94, 532, 440, 558, 3, 825, 611, 60, 613, 611, + 611, 107, 255, 266, 60, 440, 490, 514, 610, 271, + 379, 610, 612, 809, 94, 440, 567, 379, 558, 440, + 379, 1021, 561, 559, 528, 1026, 527, 19, 908, 908, + 909, 195, 364, 740, 532, 741, 742, 13, 14, 224, + 224, 440, 440, 559, 658, 663, 490, 709, 558, 386, + 351, 390, 427, 451, 480, 696, 154, 101, 594, 622, + 972, 973, 1036, 280, 201, 599, 558, 280, 595, 609, + 280, 528, 659, 41, 280, 41, 280, 659, 280, 528, + 690, 691, 692, 693, 694, 695, 697, 194, 559, 653, + 977, 565, 154, 174, 617, 675, 564, 530, 962, 952, + 955, 955, 962, 529, 532, 13, 991, 997, 4, 958, + 4, 958, 561, 565, 1024, 1025, 56, 103, 124, 142, + 147, 170, 173, 189, 285, 293, 315, 345, 712, 41, + 529, 858, 529, 174, 532, 529, 324, 914, 529, 859, + 859, 11, 15, 16, 20, 21, 22, 201, 224, 297, + 515, 516, 517, 519, 520, 521, 522, 523, 524, 899, + 859, 529, 811, 812, 870, 169, 174, 917, 918, 532, + 529, 41, 919, 906, 919, 919, 174, 529, 41, 803, + 528, 953, 4, 9, 559, 751, 752, 754, 755, 863, + 936, 934, 180, 244, 427, 432, 434, 460, 558, 734, + 487, 815, 529, 528, 757, 792, 792, 227, 792, 295, + 467, 808, 792, 227, 936, 792, 792, 287, 287, 528, + 792, 757, 560, 816, 817, 528, 560, 816, 532, 529, + 532, 530, 528, 825, 528, 528, 530, 824, 40, 824, + 528, 844, 845, 846, 847, 848, 849, 850, 851, 852, + 853, 854, 855, 856, 857, 529, 532, 828, 568, 573, + 700, 701, 714, 969, 1014, 1030, 906, 907, 528, 486, + 922, 923, 858, 907, 958, 19, 858, 892, 893, 894, + 895, 840, 840, 8, 15, 16, 20, 21, 22, 515, + 516, 517, 519, 520, 521, 522, 523, 524, 559, 897, + 902, 529, 906, 438, 438, 958, 958, 858, 528, 528, + 560, 939, 361, 946, 169, 527, 529, 532, 19, 532, + 537, 858, 521, 563, 906, 958, 858, 857, 857, 822, + 858, 858, 858, 858, 858, 858, 858, 858, 5, 565, + 966, 438, 46, 424, 933, 962, 858, 858, 528, 715, + 864, 920, 927, 133, 162, 280, 285, 290, 447, 458, + 858, 285, 528, 858, 440, 54, 179, 196, 201, 240, + 400, 858, 858, 858, 858, 858, 858, 858, 858, 858, + 858, 31, 39, 405, 896, 526, 530, 931, 183, 165, + 874, 374, 528, 888, 937, 174, 788, 908, 788, 528, + 561, 559, 558, 1010, 558, 1018, 858, 532, 529, 229, + 41, 467, 1017, 558, 570, 467, 528, 559, 575, 585, + 586, 588, 42, 127, 745, 532, 467, 745, 271, 717, + 374, 375, 519, 520, 764, 766, 860, 398, 229, 296, + 319, 319, 532, 523, 4, 765, 958, 765, 374, 375, + 766, 558, 951, 284, 402, 785, 528, 953, 954, 532, + 183, 467, 201, 183, 222, 780, 748, 529, 559, 561, + 559, 561, 359, 559, 359, 554, 528, 194, 550, 955, + 229, 280, 229, 467, 528, 642, 649, 650, 821, 822, + 530, 547, 548, 955, 559, 194, 955, 194, 549, 27, + 138, 395, 546, 553, 565, 633, 647, 955, 60, 60, + 565, 641, 662, 60, 60, 955, 551, 955, 359, 395, + 554, 609, 611, 962, 955, 611, 962, 955, 611, 359, + 395, 554, 955, 955, 549, 955, 359, 395, 554, 955, + 955, 561, 1022, 1025, 520, 858, 928, 741, 741, 741, + 287, 287, 529, 476, 909, 740, 858, 858, 285, 561, + 984, 285, 984, 559, 340, 707, 529, 532, 293, 174, + 440, 703, 971, 597, 480, 558, 558, 1040, 558, 558, + 558, 558, 300, 670, 154, 3, 528, 528, 154, 154, + 240, 559, 639, 651, 654, 657, 667, 669, 490, 492, + 644, 153, 714, 154, 145, 590, 822, 154, 490, 978, + 154, 3, 43, 47, 50, 56, 82, 84, 90, 102, + 173, 175, 178, 179, 196, 211, 224, 225, 227, 238, + 240, 250, 274, 283, 305, 307, 328, 370, 400, 420, + 429, 450, 461, 478, 529, 698, 699, 965, 1041, 532, + 532, 41, 280, 295, 559, 3, 676, 564, 676, 295, + 676, 605, 858, 708, 858, 1023, 529, 532, 41, 706, + 561, 706, 280, 285, 345, 706, 60, 706, 822, 529, + 858, 858, 858, 917, 822, 859, 859, 859, 859, 859, + 859, 133, 280, 290, 859, 859, 859, 859, 859, 859, + 859, 859, 859, 859, 529, 532, 41, 813, 858, 858, + 918, 917, 822, 529, 529, 529, 906, 822, 953, 529, + 319, 375, 523, 528, 528, 733, 432, 434, 432, 434, + 558, 735, 735, 735, 858, 183, 768, 439, 479, 759, + 760, 808, 808, 792, 858, 528, 792, 169, 808, 528, + 560, 799, 808, 822, 529, 532, 816, 529, 1002, 3, + 910, 40, 824, 559, 819, 819, 3, 526, 526, 958, + 440, 440, 440, 440, 822, 464, 529, 527, 906, 858, + 140, 923, 924, 529, 529, 529, 858, 19, 532, 537, + 530, 529, 529, 511, 511, 529, 529, 529, 858, 939, + 940, 941, 530, 528, 858, 943, 359, 950, 952, 858, + 858, 889, 942, 529, 529, 529, 511, 859, 859, 147, + 906, 174, 133, 162, 285, 290, 447, 458, 528, 147, + 902, 858, 424, 933, 858, 920, 858, 440, 528, 715, + 858, 928, 564, 528, 528, 157, 875, 789, 790, 815, + 741, 815, 958, 857, 964, 964, 528, 254, 280, 732, + 790, 487, 1016, 41, 60, 571, 792, 581, 588, 929, + 532, 788, 517, 513, 746, 744, 297, 897, 900, 746, + 4, 958, 766, 296, 464, 763, 532, 247, 953, 723, + 60, 936, 528, 560, 60, 271, 1008, 1008, 440, 440, + 858, 280, 666, 528, 154, 528, 642, 205, 663, 664, + 623, 41, 178, 632, 660, 564, 548, 623, 27, 138, + 363, 365, 395, 543, 544, 545, 555, 556, 154, 676, + 154, 676, 633, 647, 633, 529, 532, 561, 626, 517, + 530, 529, 532, 528, 528, 440, 379, 94, 440, 567, + 379, 440, 440, 440, 379, 1022, 1026, 529, 19, 19, + 527, 740, 740, 740, 909, 529, 528, 702, 3, 414, + 415, 528, 561, 713, 863, 658, 707, 599, 558, 595, + 528, 41, 41, 659, 693, 695, 971, 361, 427, 597, + 565, 601, 602, 663, 558, 558, 1040, 558, 649, 650, + 529, 532, 293, 637, 293, 295, 636, 955, 490, 1039, + 558, 528, 715, 558, 637, 41, 558, 529, 532, 821, + 822, 691, 697, 694, 697, 427, 858, 154, 558, 617, + 962, 1024, 561, 561, 285, 663, 521, 663, 561, 521, + 663, 561, 529, 529, 918, 174, 133, 290, 528, 814, + 811, 528, 529, 529, 529, 559, 752, 815, 735, 735, + 735, 735, 558, 558, 558, 60, 188, 777, 14, 529, + 808, 953, 528, 796, 797, 798, 861, 864, 953, 169, + 81, 818, 817, 529, 529, 526, 822, 529, 532, 529, + 527, 958, 958, 529, 845, 847, 848, 849, 848, 849, + 849, 529, 436, 858, 144, 858, 892, 902, 843, 843, + 529, 532, 529, 560, 858, 943, 944, 945, 41, 528, + 939, 947, 201, 529, 946, 857, 858, 37, 37, 858, + 529, 858, 174, 528, 910, 858, 529, 147, 859, 859, + 147, 147, 858, 858, 527, 19, 528, 930, 742, 487, + 858, 306, 879, 532, 768, 740, 768, 529, 559, 730, + 731, 926, 254, 528, 858, 367, 579, 559, 271, 327, + 118, 309, 528, 569, 714, 808, 529, 532, 575, 1016, + 858, 166, 234, 528, 746, 296, 558, 529, 954, 183, + 715, 716, 936, 954, 955, 559, 955, 529, 154, 664, + 550, 664, 623, 653, 532, 529, 120, 209, 278, 280, + 648, 528, 34, 60, 671, 660, 75, 81, 94, 118, + 120, 209, 280, 285, 335, 353, 457, 467, 628, 629, + 643, 178, 118, 193, 280, 637, 610, 108, 118, 178, + 280, 413, 416, 612, 637, 395, 545, 451, 955, 549, + 553, 3, 38, 43, 47, 50, 56, 60, 82, 84, + 90, 102, 173, 175, 179, 196, 211, 224, 225, 227, + 238, 240, 250, 274, 279, 283, 297, 305, 307, 328, + 370, 389, 396, 400, 420, 429, 450, 456, 461, 478, + 519, 520, 561, 611, 624, 665, 822, 900, 959, 1041, + 1047, 565, 662, 906, 743, 955, 955, 955, 955, 549, + 955, 955, 955, 955, 955, 1026, 928, 928, 529, 529, + 529, 741, 108, 379, 530, 858, 610, 713, 528, 528, + 657, 714, 590, 978, 670, 194, 558, 599, 600, 858, + 529, 532, 529, 595, 528, 41, 646, 644, 559, 654, + 87, 607, 108, 278, 41, 561, 591, 592, 659, 714, + 693, 695, 529, 699, 13, 14, 41, 41, 715, 716, + 653, 467, 975, 676, 663, 859, 174, 528, 910, 816, + 529, 532, 529, 768, 558, 558, 558, 558, 31, 104, + 184, 373, 528, 769, 770, 771, 772, 773, 774, 775, + 858, 858, 489, 876, 858, 529, 860, 903, 904, 201, + 183, 793, 797, 529, 799, 800, 801, 962, 824, 958, + 824, 559, 824, 527, 527, 858, 939, 532, 529, 559, + 947, 948, 949, 41, 858, 860, 950, 858, 858, 858, + 910, 529, 858, 37, 37, 858, 858, 147, 529, 520, + 928, 529, 908, 529, 858, 529, 528, 559, 880, 777, + 529, 777, 561, 529, 532, 969, 935, 473, 426, 466, + 580, 559, 574, 584, 295, 577, 486, 684, 686, 687, + 688, 517, 588, 579, 902, 60, 529, 529, 472, 473, + 720, 623, 550, 529, 529, 490, 656, 121, 197, 207, + 120, 469, 858, 118, 41, 528, 962, 955, 859, 121, + 197, 120, 285, 229, 558, 656, 89, 671, 194, 285, + 611, 858, 671, 285, 519, 520, 614, 559, 821, 676, + 676, 3, 959, 963, 517, 529, 529, 440, 440, 527, + 527, 740, 529, 529, 559, 529, 708, 467, 704, 705, + 602, 663, 529, 1039, 41, 427, 280, 528, 528, 601, + 978, 657, 153, 714, 151, 203, 636, 123, 138, 334, + 1039, 108, 590, 529, 532, 978, 490, 1037, 858, 858, + 427, 295, 559, 974, 528, 859, 910, 529, 529, 9, + 360, 758, 777, 528, 397, 528, 529, 532, 559, 877, + 878, 342, 778, 532, 529, 528, 560, 60, 529, 201, + 529, 800, 527, 822, 943, 532, 529, 559, 527, 194, + 529, 858, 858, 858, 19, 19, 527, 529, 529, 559, + 881, 876, 561, 876, 926, 529, 532, 472, 929, 529, + 532, 92, 579, 253, 280, 688, 579, 858, 529, 954, + 954, 353, 656, 528, 645, 623, 529, 193, 528, 858, + 280, 629, 656, 659, 955, 41, 154, 818, 963, 523, + 624, 955, 955, 529, 610, 125, 529, 529, 644, 714, + 558, 154, 602, 41, 529, 955, 1039, 31, 86, 95, + 119, 193, 206, 413, 416, 640, 640, 375, 375, 561, + 41, 65, 75, 244, 715, 558, 528, 559, 578, 587, + 870, 529, 529, 528, 876, 906, 528, 906, 771, 41, + 532, 858, 467, 753, 860, 936, 953, 804, 528, 804, + 947, 858, 928, 928, 315, 882, 778, 778, 714, 309, + 714, 574, 295, 528, 572, 37, 679, 253, 558, 623, + 565, 652, 655, 417, 482, 630, 631, 528, 625, 858, + 529, 252, 668, 193, 467, 552, 523, 451, 708, 561, + 978, 636, 1037, 528, 558, 529, 714, 644, 607, 714, + 75, 298, 75, 975, 858, 81, 582, 529, 532, 582, + 9, 778, 529, 770, 529, 880, 878, 377, 529, 936, + 527, 527, 527, 60, 741, 753, 753, 580, 94, 587, + 134, 858, 436, 60, 685, 659, 517, 529, 532, 609, + 529, 278, 638, 175, 314, 401, 295, 634, 635, 661, + 625, 858, 451, 41, 528, 1037, 636, 1039, 1037, 298, + 298, 528, 529, 962, 583, 962, 978, 578, 583, 529, + 753, 529, 755, 529, 905, 186, 346, 375, 883, 472, + 955, 529, 281, 464, 123, 134, 146, 215, 464, 682, + 407, 431, 679, 668, 624, 655, 529, 631, 207, 123, + 464, 295, 661, 295, 634, 714, 587, 582, 745, 815, + 745, 54, 105, 453, 858, 884, 885, 884, 884, 529, + 714, 815, 395, 281, 683, 858, 118, 528, 571, 680, + 395, 571, 436, 635, 64, 278, 366, 395, 627, 627, + 1037, 529, 583, 746, 746, 885, 374, 168, 330, 168, + 330, 150, 886, 886, 886, 586, 473, 584, 521, 681, + 473, 521, 586, 682, 623, 26, 118, 285, 978, 745, + 37, 105, 183, 278, 437, 815, 529, 528, 815, 746, + 885, 374, 303, 906, 529 }; #define yyerrok (yyerrstatus = 0) @@ -21166,14 +21271,14 @@ YYLTYPE yylloc; switch (yyn) { case 2: -#line 522 "third_party/libpg_query/grammar/grammar.y" +#line 523 "third_party/libpg_query/grammar/grammar.y" { pg_yyget_extra(yyscanner)->parsetree = (yyvsp[(1) - (1)].list); ;} break; case 3: -#line 538 "third_party/libpg_query/grammar/grammar.y" +#line 539 "third_party/libpg_query/grammar/grammar.y" { if ((yyvsp[(1) - (3)].list) != NIL) { @@ -21188,7 +21293,7 @@ YYLTYPE yylloc; break; case 4: -#line 550 "third_party/libpg_query/grammar/grammar.y" +#line 551 "third_party/libpg_query/grammar/grammar.y" { if ((yyvsp[(1) - (1)].node) != NULL) (yyval.list) = list_make1(makeRawStmt((yyvsp[(1) - (1)].node), 0)); @@ -21197,12 +21302,12 @@ YYLTYPE yylloc; ;} break; - case 47: -#line 601 "third_party/libpg_query/grammar/grammar.y" + case 48: +#line 603 "third_party/libpg_query/grammar/grammar.y" { (yyval.node) = NULL; ;} break; - case 48: + case 49: #line 10 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableStmt *n = makeNode(PGAlterTableStmt); @@ -21214,7 +21319,7 @@ YYLTYPE yylloc; ;} break; - case 49: + case 50: #line 19 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableStmt *n = makeNode(PGAlterTableStmt); @@ -21226,7 +21331,7 @@ YYLTYPE yylloc; ;} break; - case 50: + case 51: #line 28 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableStmt *n = makeNode(PGAlterTableStmt); @@ -21238,7 +21343,7 @@ YYLTYPE yylloc; ;} break; - case 51: + case 52: #line 37 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableStmt *n = makeNode(PGAlterTableStmt); @@ -21250,7 +21355,7 @@ YYLTYPE yylloc; ;} break; - case 52: + case 53: #line 46 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableStmt *n = makeNode(PGAlterTableStmt); @@ -21262,7 +21367,7 @@ YYLTYPE yylloc; ;} break; - case 53: + case 54: #line 55 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableStmt *n = makeNode(PGAlterTableStmt); @@ -21274,7 +21379,7 @@ YYLTYPE yylloc; ;} break; - case 54: + case 55: #line 64 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableStmt *n = makeNode(PGAlterTableStmt); @@ -21286,7 +21391,7 @@ YYLTYPE yylloc; ;} break; - case 55: + case 56: #line 73 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableStmt *n = makeNode(PGAlterTableStmt); @@ -21298,41 +21403,41 @@ YYLTYPE yylloc; ;} break; - case 56: + case 57: #line 86 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 57: + case 58: #line 88 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].defelt)); ;} break; - case 58: + case 59: #line 93 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.node) = (yyvsp[(3) - (3)].node); ;} break; - case 59: + case 60: #line 94 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.node) = NULL; ;} break; - case 60: + case 61: #line 100 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.defelt) = makeDefElem("restart", NULL, (yylsp[(1) - (1)])); ;} break; - case 61: + case 62: #line 104 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.defelt) = makeDefElem("restart", (PGNode *)(yyvsp[(3) - (3)].value), (yylsp[(1) - (3)])); ;} break; - case 62: + case 63: #line 108 "third_party/libpg_query/grammar/statements/alter_table.y" { if (strcmp((yyvsp[(2) - (2)].defelt)->defname, "as") == 0 || @@ -21346,55 +21451,55 @@ YYLTYPE yylloc; ;} break; - case 63: + case 64: #line 119 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.defelt) = makeDefElem("generated", (PGNode *) makeInteger((yyvsp[(3) - (3)].ival)), (yylsp[(1) - (3)])); ;} break; - case 64: + case 65: #line 127 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 65: + case 66: #line 131 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].defelt)); ;} break; - case 66: + case 67: #line 138 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].str)); ;} break; - case 67: + case 68: #line 139 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].str)); ;} break; - case 68: + case 69: #line 144 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.str) = (yyvsp[(2) - (2)].str); ;} break; - case 69: + case 70: #line 150 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].str)); ;} break; - case 70: + case 71: #line 151 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = list_concat(list_make1((yyvsp[(1) - (2)].str)), (yyvsp[(2) - (2)].list)); ;} break; - case 71: + case 72: #line 157 "third_party/libpg_query/grammar/statements/alter_table.y" { PGColumnDef *n = (PGColumnDef *) (yyvsp[(2) - (2)].node); @@ -21403,7 +21508,7 @@ YYLTYPE yylloc; ;} break; - case 72: + case 73: #line 164 "third_party/libpg_query/grammar/statements/alter_table.y" { PGColumnDef *n = (PGColumnDef *) (yyvsp[(2) - (2)].node); @@ -21412,7 +21517,7 @@ YYLTYPE yylloc; ;} break; - case 73: + case 74: #line 174 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21423,7 +21528,7 @@ YYLTYPE yylloc; ;} break; - case 74: + case 75: #line 183 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21434,7 +21539,7 @@ YYLTYPE yylloc; ;} break; - case 75: + case 76: #line 192 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21445,7 +21550,7 @@ YYLTYPE yylloc; ;} break; - case 76: + case 77: #line 201 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21456,7 +21561,7 @@ YYLTYPE yylloc; ;} break; - case 77: + case 78: #line 210 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21466,7 +21571,7 @@ YYLTYPE yylloc; ;} break; - case 78: + case 79: #line 218 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21476,7 +21581,7 @@ YYLTYPE yylloc; ;} break; - case 79: + case 80: #line 226 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21486,7 +21591,7 @@ YYLTYPE yylloc; ;} break; - case 80: + case 81: #line 234 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21496,7 +21601,7 @@ YYLTYPE yylloc; ;} break; - case 81: + case 82: #line 242 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21507,7 +21612,7 @@ YYLTYPE yylloc; ;} break; - case 82: + case 83: #line 251 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21517,7 +21622,7 @@ YYLTYPE yylloc; ;} break; - case 83: + case 84: #line 259 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21527,7 +21632,7 @@ YYLTYPE yylloc; ;} break; - case 84: + case 85: #line 267 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21538,7 +21643,7 @@ YYLTYPE yylloc; ;} break; - case 85: + case 86: #line 276 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21549,7 +21654,7 @@ YYLTYPE yylloc; ;} break; - case 86: + case 87: #line 285 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21560,7 +21665,7 @@ YYLTYPE yylloc; ;} break; - case 87: + case 88: #line 294 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21571,7 +21676,7 @@ YYLTYPE yylloc; ;} break; - case 88: + case 89: #line 303 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21590,7 +21695,7 @@ YYLTYPE yylloc; ;} break; - case 89: + case 90: #line 320 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21601,7 +21706,7 @@ YYLTYPE yylloc; ;} break; - case 90: + case 91: #line 329 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21612,7 +21717,7 @@ YYLTYPE yylloc; ;} break; - case 91: + case 92: #line 338 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21623,7 +21728,7 @@ YYLTYPE yylloc; ;} break; - case 92: + case 93: #line 347 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21635,7 +21740,7 @@ YYLTYPE yylloc; ;} break; - case 93: + case 94: #line 357 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21647,7 +21752,7 @@ YYLTYPE yylloc; ;} break; - case 94: + case 95: #line 370 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21664,7 +21769,7 @@ YYLTYPE yylloc; ;} break; - case 95: + case 96: #line 385 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21675,7 +21780,7 @@ YYLTYPE yylloc; ;} break; - case 96: + case 97: #line 394 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21685,7 +21790,7 @@ YYLTYPE yylloc; ;} break; - case 97: + case 98: #line 402 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21702,7 +21807,7 @@ YYLTYPE yylloc; ;} break; - case 98: + case 99: #line 417 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21712,7 +21817,7 @@ YYLTYPE yylloc; ;} break; - case 99: + case 100: #line 425 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21724,7 +21829,7 @@ YYLTYPE yylloc; ;} break; - case 100: + case 101: #line 435 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21736,7 +21841,7 @@ YYLTYPE yylloc; ;} break; - case 101: + case 102: #line 445 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21745,7 +21850,7 @@ YYLTYPE yylloc; ;} break; - case 102: + case 103: #line 452 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21754,7 +21859,7 @@ YYLTYPE yylloc; ;} break; - case 103: + case 104: #line 459 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21764,7 +21869,7 @@ YYLTYPE yylloc; ;} break; - case 104: + case 105: #line 467 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21774,7 +21879,7 @@ YYLTYPE yylloc; ;} break; - case 105: + case 106: #line 474 "third_party/libpg_query/grammar/statements/alter_table.y" { PGAlterTableCmd *n = makeNode(PGAlterTableCmd); @@ -21784,24 +21889,24 @@ YYLTYPE yylloc; ;} break; - case 106: + case 107: #line 484 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 107: + case 108: #line 485 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.node) = NULL; ;} break; - case 108: + case 109: #line 491 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.defelt) = (yyvsp[(1) - (1)].defelt); ;} break; - case 109: + case 110: #line 495 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.defelt) = (yyvsp[(2) - (2)].defelt); @@ -21809,7 +21914,7 @@ YYLTYPE yylloc; ;} break; - case 110: + case 111: #line 500 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.defelt) = (yyvsp[(2) - (2)].defelt); @@ -21817,44 +21922,44 @@ YYLTYPE yylloc; ;} break; - case 111: + case 112: #line 505 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.defelt) = makeDefElemExtended(NULL, (yyvsp[(2) - (2)].str), NULL, DEFELEM_DROP, (yylsp[(2) - (2)])); ;} break; - case 112: + case 113: #line 512 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 113: + case 114: #line 513 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 114: + case 115: #line 518 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 115: + case 116: #line 522 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.ival) = 1; ;} break; - case 116: + case 117: #line 523 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.ival) = 0; ;} break; - case 117: + case 118: #line 524 "third_party/libpg_query/grammar/statements/alter_table.y" { (yyval.ival) = 0; ;} break; - case 118: + case 119: #line 8 "third_party/libpg_query/grammar/statements/deallocate.y" { PGDeallocateStmt *n = makeNode(PGDeallocateStmt); @@ -21863,7 +21968,7 @@ YYLTYPE yylloc; ;} break; - case 119: + case 120: #line 14 "third_party/libpg_query/grammar/statements/deallocate.y" { PGDeallocateStmt *n = makeNode(PGDeallocateStmt); @@ -21872,7 +21977,7 @@ YYLTYPE yylloc; ;} break; - case 120: + case 121: #line 20 "third_party/libpg_query/grammar/statements/deallocate.y" { PGDeallocateStmt *n = makeNode(PGDeallocateStmt); @@ -21881,7 +21986,7 @@ YYLTYPE yylloc; ;} break; - case 121: + case 122: #line 26 "third_party/libpg_query/grammar/statements/deallocate.y" { PGDeallocateStmt *n = makeNode(PGDeallocateStmt); @@ -21890,14 +21995,14 @@ YYLTYPE yylloc; ;} break; - case 122: + case 123: #line 10 "third_party/libpg_query/grammar/statements/common.y" { (yyval.range) = makeRangeVar(NULL, (yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 123: + case 124: #line 14 "third_party/libpg_query/grammar/statements/common.y" { check_qualified_name((yyvsp[(2) - (2)].list), yyscanner); @@ -21926,79 +22031,79 @@ YYLTYPE yylloc; ;} break; - case 124: + case 125: #line 44 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 125: + case 126: #line 45 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 126: + case 127: #line 46 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 127: + case 128: #line 50 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 128: + case 129: #line 51 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 129: + case 130: #line 55 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 130: + case 131: #line 59 "third_party/libpg_query/grammar/statements/common.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 131: + case 132: #line 60 "third_party/libpg_query/grammar/statements/common.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 132: + case 133: #line 65 "third_party/libpg_query/grammar/statements/common.y" { (yyval.node) = (PGNode *) makeString((yyvsp[(2) - (2)].str)); ;} break; - case 133: + case 134: #line 70 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 134: + case 135: #line 75 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 135: + case 136: #line 76 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 136: + case 137: #line 77 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 137: + case 138: #line 78 "third_party/libpg_query/grammar/statements/common.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 138: + case 139: #line 7 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22010,7 +22115,7 @@ YYLTYPE yylloc; ;} break; - case 139: + case 140: #line 16 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22023,7 +22128,7 @@ YYLTYPE yylloc; ;} break; - case 140: + case 141: #line 26 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22036,7 +22141,7 @@ YYLTYPE yylloc; ;} break; - case 141: + case 142: #line 36 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22049,7 +22154,7 @@ YYLTYPE yylloc; ;} break; - case 142: + case 143: #line 46 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22062,7 +22167,7 @@ YYLTYPE yylloc; ;} break; - case 143: + case 144: #line 56 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22075,7 +22180,7 @@ YYLTYPE yylloc; ;} break; - case 144: + case 145: #line 66 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22088,7 +22193,7 @@ YYLTYPE yylloc; ;} break; - case 145: + case 146: #line 76 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22101,7 +22206,7 @@ YYLTYPE yylloc; ;} break; - case 146: + case 147: #line 86 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22114,7 +22219,7 @@ YYLTYPE yylloc; ;} break; - case 147: + case 148: #line 96 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22128,7 +22233,7 @@ YYLTYPE yylloc; ;} break; - case 148: + case 149: #line 107 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22142,7 +22247,7 @@ YYLTYPE yylloc; ;} break; - case 149: + case 150: #line 118 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22155,7 +22260,7 @@ YYLTYPE yylloc; ;} break; - case 150: + case 151: #line 128 "third_party/libpg_query/grammar/statements/rename.y" { PGRenameStmt *n = makeNode(PGRenameStmt); @@ -22168,17 +22273,17 @@ YYLTYPE yylloc; ;} break; - case 151: + case 152: #line 140 "third_party/libpg_query/grammar/statements/rename.y" { (yyval.ival) = COLUMN; ;} break; - case 152: + case 153: #line 141 "third_party/libpg_query/grammar/statements/rename.y" { (yyval.ival) = 0; ;} break; - case 153: + case 154: #line 11 "third_party/libpg_query/grammar/statements/insert.y" { (yyvsp[(7) - (9)].istmt)->relation = (yyvsp[(5) - (9)].range); @@ -22191,7 +22296,7 @@ YYLTYPE yylloc; ;} break; - case 154: + case 155: #line 24 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.istmt) = makeNode(PGInsertStmt); @@ -22200,7 +22305,7 @@ YYLTYPE yylloc; ;} break; - case 155: + case 156: #line 30 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.istmt) = makeNode(PGInsertStmt); @@ -22210,7 +22315,7 @@ YYLTYPE yylloc; ;} break; - case 156: + case 157: #line 37 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.istmt) = makeNode(PGInsertStmt); @@ -22219,7 +22324,7 @@ YYLTYPE yylloc; ;} break; - case 157: + case 158: #line 43 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.istmt) = makeNode(PGInsertStmt); @@ -22229,7 +22334,7 @@ YYLTYPE yylloc; ;} break; - case 158: + case 159: #line 50 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.istmt) = makeNode(PGInsertStmt); @@ -22238,14 +22343,14 @@ YYLTYPE yylloc; ;} break; - case 159: + case 160: #line 60 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.range) = (yyvsp[(1) - (1)].range); ;} break; - case 160: + case 161: #line 64 "third_party/libpg_query/grammar/statements/insert.y" { (yyvsp[(1) - (3)].range)->alias = makeAlias((yyvsp[(3) - (3)].str), NIL); @@ -22253,22 +22358,22 @@ YYLTYPE yylloc; ;} break; - case 161: + case 162: #line 71 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.bynameorposition) = PG_INSERT_BY_NAME; ;} break; - case 162: + case 163: #line 72 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.bynameorposition) = PG_INSERT_BY_POSITION; ;} break; - case 163: + case 164: #line 73 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.bynameorposition) = PG_INSERT_BY_POSITION; ;} break; - case 164: + case 165: #line 78 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.infer) = makeNode(PGInferClause); @@ -22279,7 +22384,7 @@ YYLTYPE yylloc; ;} break; - case 165: + case 166: #line 87 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.infer) = makeNode(PGInferClause); @@ -22290,24 +22395,24 @@ YYLTYPE yylloc; ;} break; - case 166: + case 167: #line 95 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.infer) = NULL; ;} break; - case 167: + case 168: #line 102 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.with) = (yyvsp[(1) - (1)].with); ;} break; - case 168: + case 169: #line 103 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.with) = NULL; ;} break; - case 169: + case 170: #line 109 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.target) = makeNode(PGResTarget); @@ -22318,7 +22423,7 @@ YYLTYPE yylloc; ;} break; - case 170: + case 171: #line 121 "third_party/libpg_query/grammar/statements/insert.y" { (yyvsp[(1) - (3)].target)->val = (PGNode *) (yyvsp[(3) - (3)].node); @@ -22326,7 +22431,7 @@ YYLTYPE yylloc; ;} break; - case 171: + case 172: #line 126 "third_party/libpg_query/grammar/statements/insert.y" { int ncolumns = list_length((yyvsp[(2) - (5)].list)); @@ -22350,28 +22455,28 @@ YYLTYPE yylloc; ;} break; - case 172: + case 173: #line 151 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.onconflictshorthand) = PG_ONCONFLICT_ALIAS_REPLACE; ;} break; - case 173: + case 174: #line 156 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.onconflictshorthand) = PG_ONCONFLICT_ALIAS_IGNORE; ;} break; - case 174: + case 175: #line 160 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.onconflictshorthand) = PG_ONCONFLICT_ALIAS_NONE; ;} break; - case 175: + case 176: #line 167 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.onconflict) = makeNode(PGOnConflictClause); @@ -22383,7 +22488,7 @@ YYLTYPE yylloc; ;} break; - case 176: + case 177: #line 177 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.onconflict) = makeNode(PGOnConflictClause); @@ -22395,14 +22500,14 @@ YYLTYPE yylloc; ;} break; - case 177: + case 178: #line 186 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.onconflict) = NULL; ;} break; - case 178: + case 179: #line 193 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.ielem) = makeNode(PGIndexElem); @@ -22416,7 +22521,7 @@ YYLTYPE yylloc; ;} break; - case 179: + case 180: #line 204 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.ielem) = makeNode(PGIndexElem); @@ -22430,7 +22535,7 @@ YYLTYPE yylloc; ;} break; - case 180: + case 181: #line 215 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.ielem) = makeNode(PGIndexElem); @@ -22444,97 +22549,97 @@ YYLTYPE yylloc; ;} break; - case 181: + case 182: #line 229 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 182: + case 183: #line 230 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = NIL; ;} break; - case 183: + case 184: #line 236 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.override) = PG_OVERRIDING_USER_VALUE; ;} break; - case 184: + case 185: #line 237 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.override) = OVERRIDING_SYSTEM_VALUE; ;} break; - case 185: + case 186: #line 242 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].target)); ;} break; - case 186: + case 187: #line 243 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list),(yyvsp[(3) - (3)].target)); ;} break; - case 187: + case 188: #line 249 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 188: + case 189: #line 250 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = NIL; ;} break; - case 189: + case 190: #line 254 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 190: + case 191: #line 255 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = NIL; ;} break; - case 191: + case 192: #line 261 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].target)); ;} break; - case 192: + case 193: #line 263 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].target)); ;} break; - case 193: + case 194: #line 268 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 194: + case 195: #line 269 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = list_concat((yyvsp[(1) - (3)].list),(yyvsp[(3) - (3)].list)); ;} break; - case 195: + case 196: #line 273 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 196: + case 197: #line 274 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 197: + case 198: #line 277 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].ielem)); ;} break; - case 198: + case 199: #line 278 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].ielem)); ;} break; - case 199: + case 200: #line 284 "third_party/libpg_query/grammar/statements/insert.y" { (yyval.target) = makeNode(PGResTarget); @@ -22545,7 +22650,7 @@ YYLTYPE yylloc; ;} break; - case 200: + case 201: #line 8 "third_party/libpg_query/grammar/statements/create_type.y" { PGCreateTypeStmt *n = (PGCreateTypeStmt *) (yyvsp[(6) - (6)].node); @@ -22556,7 +22661,7 @@ YYLTYPE yylloc; ;} break; - case 201: + case 202: #line 16 "third_party/libpg_query/grammar/statements/create_type.y" { PGCreateTypeStmt *n = (PGCreateTypeStmt *) (yyvsp[(9) - (9)].node); @@ -22567,7 +22672,7 @@ YYLTYPE yylloc; ;} break; - case 202: + case 203: #line 24 "third_party/libpg_query/grammar/statements/create_type.y" { PGCreateTypeStmt *n = (PGCreateTypeStmt *) (yyvsp[(8) - (8)].node); @@ -22578,7 +22683,7 @@ YYLTYPE yylloc; ;} break; - case 203: + case 204: #line 35 "third_party/libpg_query/grammar/statements/create_type.y" { PGCreateTypeStmt *n = makeNode(PGCreateTypeStmt); @@ -22589,7 +22694,7 @@ YYLTYPE yylloc; ;} break; - case 204: + case 205: #line 43 "third_party/libpg_query/grammar/statements/create_type.y" { PGCreateTypeStmt *n = makeNode(PGCreateTypeStmt); @@ -22600,7 +22705,7 @@ YYLTYPE yylloc; ;} break; - case 205: + case 206: #line 51 "third_party/libpg_query/grammar/statements/create_type.y" { PGCreateTypeStmt *n = makeNode(PGCreateTypeStmt); @@ -22617,31 +22722,31 @@ YYLTYPE yylloc; ;} break; - case 206: + case 207: #line 68 "third_party/libpg_query/grammar/statements/create_type.y" { (yyval.list) = (yyvsp[(1) - (1)].list);;} break; - case 207: + case 208: #line 69 "third_party/libpg_query/grammar/statements/create_type.y" {(yyval.list) = NIL;;} break; - case 208: + case 209: #line 73 "third_party/libpg_query/grammar/statements/create_type.y" { (yyval.list) = list_make1(makeStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)]))); ;} break; - case 209: + case 210: #line 77 "third_party/libpg_query/grammar/statements/create_type.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), makeStringConst((yyvsp[(3) - (3)].str), (yylsp[(3) - (3)]))); ;} break; - case 210: + case 211: #line 8 "third_party/libpg_query/grammar/statements/pragma.y" { PGPragmaStmt *n = makeNode(PGPragmaStmt); @@ -22651,7 +22756,7 @@ YYLTYPE yylloc; ;} break; - case 211: + case 212: #line 15 "third_party/libpg_query/grammar/statements/pragma.y" { PGPragmaStmt *n = makeNode(PGPragmaStmt); @@ -22662,7 +22767,7 @@ YYLTYPE yylloc; ;} break; - case 212: + case 213: #line 23 "third_party/libpg_query/grammar/statements/pragma.y" { PGPragmaStmt *n = makeNode(PGPragmaStmt); @@ -22673,7 +22778,7 @@ YYLTYPE yylloc; ;} break; - case 213: + case 214: #line 10 "third_party/libpg_query/grammar/statements/create_sequence.y" { PGCreateSeqStmt *n = makeNode(PGCreateSeqStmt); @@ -22686,7 +22791,7 @@ YYLTYPE yylloc; ;} break; - case 214: + case 215: #line 20 "third_party/libpg_query/grammar/statements/create_sequence.y" { PGCreateSeqStmt *n = makeNode(PGCreateSeqStmt); @@ -22699,7 +22804,7 @@ YYLTYPE yylloc; ;} break; - case 215: + case 216: #line 30 "third_party/libpg_query/grammar/statements/create_sequence.y" { PGCreateSeqStmt *n = makeNode(PGCreateSeqStmt); @@ -22712,17 +22817,17 @@ YYLTYPE yylloc; ;} break; - case 216: + case 217: #line 42 "third_party/libpg_query/grammar/statements/create_sequence.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 217: + case 218: #line 43 "third_party/libpg_query/grammar/statements/create_sequence.y" { (yyval.list) = NIL; ;} break; - case 218: + case 219: #line 8 "third_party/libpg_query/grammar/statements/create_secret.y" { PGCreateSecretStmt *n = makeNode(PGCreateSecretStmt); @@ -22735,7 +22840,7 @@ YYLTYPE yylloc; ;} break; - case 219: + case 220: #line 18 "third_party/libpg_query/grammar/statements/create_secret.y" { PGCreateSecretStmt *n = makeNode(PGCreateSecretStmt); @@ -22748,7 +22853,7 @@ YYLTYPE yylloc; ;} break; - case 220: + case 221: #line 28 "third_party/libpg_query/grammar/statements/create_secret.y" { PGCreateSecretStmt *n = makeNode(PGCreateSecretStmt); @@ -22761,68 +22866,68 @@ YYLTYPE yylloc; ;} break; - case 221: + case 222: #line 40 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.str) = NULL; ;} break; - case 222: + case 223: #line 41 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 223: + case 224: #line 45 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.str) = pstrdup("default"); ;} break; - case 224: + case 225: #line 46 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.str) = pstrdup("temporary"); ;} break; - case 225: + case 226: #line 47 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.str) = pstrdup("persistent"); ;} break; - case 226: + case 227: #line 51 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.str) = pstrdup(""); ;} break; - case 227: + case 228: #line 52 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.str) = (yyvsp[(2) - (2)].str); ;} break; - case 228: + case 229: #line 57 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.node) = (PGNode *) (yyvsp[(1) - (1)].node); ;} break; - case 229: + case 230: #line 62 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (2)].str), (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 230: + case 231: #line 69 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 231: + case 232: #line 73 "third_party/libpg_query/grammar/statements/create_secret.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].defelt)); ;} break; - case 232: + case 233: #line 8 "third_party/libpg_query/grammar/statements/update_extensions.y" { PGUpdateExtensionsStmt *n = makeNode(PGUpdateExtensionsStmt); @@ -22840,7 +22945,7 @@ YYLTYPE yylloc; ;} break; - case 233: + case 234: #line 8 "third_party/libpg_query/grammar/statements/execute.y" { PGExecuteStmt *n = makeNode(PGExecuteStmt); @@ -22850,7 +22955,7 @@ YYLTYPE yylloc; ;} break; - case 234: + case 235: #line 16 "third_party/libpg_query/grammar/statements/execute.y" { PGCreateTableAsStmt *ctas = makeNode(PGCreateTableAsStmt); @@ -22869,7 +22974,7 @@ YYLTYPE yylloc; ;} break; - case 235: + case 236: #line 33 "third_party/libpg_query/grammar/statements/execute.y" { PGCreateTableAsStmt *ctas = makeNode(PGCreateTableAsStmt); @@ -22888,14 +22993,14 @@ YYLTYPE yylloc; ;} break; - case 236: + case 237: #line 52 "third_party/libpg_query/grammar/statements/execute.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 237: + case 238: #line 56 "third_party/libpg_query/grammar/statements/execute.y" { PGNamedArgExpr *na = makeNode(PGNamedArgExpr); @@ -22907,31 +23012,31 @@ YYLTYPE yylloc; ;} break; - case 238: + case 239: #line 66 "third_party/libpg_query/grammar/statements/execute.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 239: + case 240: #line 70 "third_party/libpg_query/grammar/statements/execute.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 240: + case 241: #line 75 "third_party/libpg_query/grammar/statements/execute.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 241: + case 242: #line 76 "third_party/libpg_query/grammar/statements/execute.y" { (yyval.list) = NIL; ;} break; - case 242: + case 243: #line 10 "third_party/libpg_query/grammar/statements/alter_sequence.y" { PGAlterSeqStmt *n = makeNode(PGAlterSeqStmt); @@ -22942,7 +23047,7 @@ YYLTYPE yylloc; ;} break; - case 243: + case 244: #line 18 "third_party/libpg_query/grammar/statements/alter_sequence.y" { PGAlterSeqStmt *n = makeNode(PGAlterSeqStmt); @@ -22953,42 +23058,42 @@ YYLTYPE yylloc; ;} break; - case 244: + case 245: #line 29 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 245: + case 246: #line 30 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].defelt)); ;} break; - case 246: + case 247: #line 34 "third_party/libpg_query/grammar/statements/alter_sequence.y" {;} break; - case 247: + case 248: #line 35 "third_party/libpg_query/grammar/statements/alter_sequence.y" {;} break; - case 248: + case 249: #line 36 "third_party/libpg_query/grammar/statements/alter_sequence.y" {;} break; - case 249: + case 250: #line 41 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.value) = makeFloat((yyvsp[(1) - (1)].str)); ;} break; - case 250: + case 251: #line 42 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.value) = makeFloat((yyvsp[(2) - (2)].str)); ;} break; - case 251: + case 252: #line 44 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.value) = makeFloat((yyvsp[(2) - (2)].str)); @@ -22996,82 +23101,82 @@ YYLTYPE yylloc; ;} break; - case 252: + case 253: #line 48 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.value) = makeInteger((yyvsp[(1) - (1)].ival)); ;} break; - case 253: + case 254: #line 53 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("as", (PGNode *)(yyvsp[(2) - (2)].typnam), (yylsp[(1) - (2)])); ;} break; - case 254: + case 255: #line 57 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("cache", (PGNode *)(yyvsp[(2) - (2)].value), (yylsp[(1) - (2)])); ;} break; - case 255: + case 256: #line 61 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("cycle", (PGNode *)makeInteger(true), (yylsp[(1) - (1)])); ;} break; - case 256: + case 257: #line 65 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("cycle", (PGNode *)makeInteger(false), (yylsp[(1) - (2)])); ;} break; - case 257: + case 258: #line 69 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("increment", (PGNode *)(yyvsp[(3) - (3)].value), (yylsp[(1) - (3)])); ;} break; - case 258: + case 259: #line 73 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("maxvalue", (PGNode *)(yyvsp[(2) - (2)].value), (yylsp[(1) - (2)])); ;} break; - case 259: + case 260: #line 77 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("minvalue", (PGNode *)(yyvsp[(2) - (2)].value), (yylsp[(1) - (2)])); ;} break; - case 260: + case 261: #line 81 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("maxvalue", NULL, (yylsp[(1) - (2)])); ;} break; - case 261: + case 262: #line 85 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("minvalue", NULL, (yylsp[(1) - (2)])); ;} break; - case 262: + case 263: #line 89 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("owned_by", (PGNode *)(yyvsp[(3) - (3)].list), (yylsp[(1) - (3)])); ;} break; - case 263: + case 264: #line 93 "third_party/libpg_query/grammar/statements/alter_sequence.y" { /* not documented, only used by pg_dump */ @@ -23079,53 +23184,77 @@ YYLTYPE yylloc; ;} break; - case 264: + case 265: #line 98 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("start", (PGNode *)(yyvsp[(3) - (3)].value), (yylsp[(1) - (3)])); ;} break; - case 265: + case 266: #line 102 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("restart", NULL, (yylsp[(1) - (1)])); ;} break; - case 266: + case 267: #line 106 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.defelt) = makeDefElem("restart", (PGNode *)(yyvsp[(3) - (3)].value), (yylsp[(1) - (3)])); ;} break; - case 267: + case 268: #line 112 "third_party/libpg_query/grammar/statements/alter_sequence.y" {;} break; - case 268: + case 269: #line 113 "third_party/libpg_query/grammar/statements/alter_sequence.y" {;} break; - case 269: + case 270: #line 117 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.ival) = (yyvsp[(1) - (1)].ival); ;} break; - case 270: + case 271: #line 118 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.ival) = + (yyvsp[(2) - (2)].ival); ;} break; - case 271: + case 272: #line 119 "third_party/libpg_query/grammar/statements/alter_sequence.y" { (yyval.ival) = - (yyvsp[(2) - (2)].ival); ;} break; - case 272: + case 273: +#line 8 "third_party/libpg_query/grammar/statements/alter_database.y" + { + PGAlterDatabaseStmt *n = makeNode(PGAlterDatabaseStmt); + n->dbname = (yyvsp[(3) - (6)].str); + n->new_name = (yyvsp[(6) - (6)].str); + n->alter_type = PG_ALTER_DATABASE_RENAME; + n->missing_ok = false; + (yyval.node) = (PGNode *)n; + ;} + break; + + case 274: +#line 17 "third_party/libpg_query/grammar/statements/alter_database.y" + { + PGAlterDatabaseStmt *n = makeNode(PGAlterDatabaseStmt); + n->dbname = (yyvsp[(5) - (8)].str); + n->new_name = (yyvsp[(8) - (8)].str); + n->alter_type = PG_ALTER_DATABASE_RENAME; + n->missing_ok = true; + (yyval.node) = (PGNode *)n; + ;} + break; + + case 275: #line 8 "third_party/libpg_query/grammar/statements/drop_secret.y" { PGDropSecretStmt *n = makeNode(PGDropSecretStmt); @@ -23137,7 +23266,7 @@ YYLTYPE yylloc; ;} break; - case 273: + case 276: #line 17 "third_party/libpg_query/grammar/statements/drop_secret.y" { PGDropSecretStmt *n = makeNode(PGDropSecretStmt); @@ -23149,17 +23278,17 @@ YYLTYPE yylloc; ;} break; - case 274: + case 277: #line 28 "third_party/libpg_query/grammar/statements/drop_secret.y" { (yyval.str) = pstrdup(""); ;} break; - case 275: + case 278: #line 29 "third_party/libpg_query/grammar/statements/drop_secret.y" { (yyval.str) = (yyvsp[(2) - (2)].str); ;} break; - case 276: + case 279: #line 3 "third_party/libpg_query/grammar/statements/transaction.y" { PGTransactionStmt *n = makeNode(PGTransactionStmt); @@ -23170,7 +23299,7 @@ YYLTYPE yylloc; ;} break; - case 277: + case 280: #line 11 "third_party/libpg_query/grammar/statements/transaction.y" { PGTransactionStmt *n = makeNode(PGTransactionStmt); @@ -23180,7 +23309,7 @@ YYLTYPE yylloc; ;} break; - case 278: + case 281: #line 18 "third_party/libpg_query/grammar/statements/transaction.y" { PGTransactionStmt *n = makeNode(PGTransactionStmt); @@ -23190,7 +23319,7 @@ YYLTYPE yylloc; ;} break; - case 279: + case 282: #line 25 "third_party/libpg_query/grammar/statements/transaction.y" { PGTransactionStmt *n = makeNode(PGTransactionStmt); @@ -23201,7 +23330,7 @@ YYLTYPE yylloc; ;} break; - case 280: + case 283: #line 33 "third_party/libpg_query/grammar/statements/transaction.y" { PGTransactionStmt *n = makeNode(PGTransactionStmt); @@ -23212,7 +23341,7 @@ YYLTYPE yylloc; ;} break; - case 281: + case 284: #line 41 "third_party/libpg_query/grammar/statements/transaction.y" { PGTransactionStmt *n = makeNode(PGTransactionStmt); @@ -23223,37 +23352,37 @@ YYLTYPE yylloc; ;} break; - case 282: + case 285: #line 51 "third_party/libpg_query/grammar/statements/transaction.y" {;} break; - case 283: + case 286: #line 52 "third_party/libpg_query/grammar/statements/transaction.y" {;} break; - case 284: + case 287: #line 53 "third_party/libpg_query/grammar/statements/transaction.y" {;} break; - case 285: + case 288: #line 57 "third_party/libpg_query/grammar/statements/transaction.y" { (yyval.transactiontype) = PG_TRANS_TYPE_READ_ONLY; ;} break; - case 286: + case 289: #line 58 "third_party/libpg_query/grammar/statements/transaction.y" { (yyval.transactiontype) = PG_TRANS_TYPE_READ_WRITE; ;} break; - case 287: + case 290: #line 59 "third_party/libpg_query/grammar/statements/transaction.y" { (yyval.transactiontype) = PG_TRANS_TYPE_DEFAULT; ;} break; - case 288: + case 291: #line 3 "third_party/libpg_query/grammar/statements/use.y" { PGUseStmt *n = makeNode(PGUseStmt); @@ -23262,7 +23391,7 @@ YYLTYPE yylloc; ;} break; - case 289: + case 292: #line 9 "third_party/libpg_query/grammar/statements/create.y" { PGCreateStmt *n = makeNode(PGCreateStmt); @@ -23278,7 +23407,7 @@ YYLTYPE yylloc; ;} break; - case 290: + case 293: #line 24 "third_party/libpg_query/grammar/statements/create.y" { PGCreateStmt *n = makeNode(PGCreateStmt); @@ -23294,7 +23423,7 @@ YYLTYPE yylloc; ;} break; - case 291: + case 294: #line 39 "third_party/libpg_query/grammar/statements/create.y" { PGCreateStmt *n = makeNode(PGCreateStmt); @@ -23310,12 +23439,12 @@ YYLTYPE yylloc; ;} break; - case 292: + case 295: #line 56 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = 0; ;} break; - case 293: + case 296: #line 58 "third_party/libpg_query/grammar/statements/create.y" { /* @@ -23342,77 +23471,77 @@ YYLTYPE yylloc; ;} break; - case 294: + case 297: #line 84 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (PGNode *)(yyvsp[(1) - (1)].typnam); ;} break; - case 295: + case 298: #line 85 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (PGNode *)makeString(pstrdup((yyvsp[(1) - (1)].keyword))); ;} break; - case 296: + case 299: #line 86 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (PGNode *)(yyvsp[(1) - (1)].list); ;} break; - case 297: + case 300: #line 87 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (PGNode *)(yyvsp[(1) - (1)].value); ;} break; - case 298: + case 301: #line 88 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (PGNode *)makeString((yyvsp[(1) - (1)].str)); ;} break; - case 299: + case 302: #line 89 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (PGNode *)makeString(pstrdup((yyvsp[(1) - (1)].keyword))); ;} break; - case 300: + case 303: #line 93 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 301: + case 304: #line 94 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = NIL; ;} break; - case 302: + case 305: #line 99 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (PGNode *) makeString((yyvsp[(1) - (1)].str)); ;} break; - case 303: + case 306: #line 104 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_FKCONSTR_ACTION_NOACTION; ;} break; - case 304: + case 307: #line 105 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_FKCONSTR_ACTION_RESTRICT; ;} break; - case 305: + case 308: #line 106 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_FKCONSTR_ACTION_CASCADE; ;} break; - case 306: + case 309: #line 107 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_FKCONSTR_ACTION_SETNULL; ;} break; - case 307: + case 310: #line 108 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_FKCONSTR_ACTION_SETDEFAULT; ;} break; - case 308: + case 311: #line 114 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = castNode(PGConstraint, (yyvsp[(3) - (3)].node)); @@ -23422,17 +23551,17 @@ YYLTYPE yylloc; ;} break; - case 309: + case 312: #line 120 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 310: + case 313: #line 121 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 311: + case 314: #line 123 "third_party/libpg_query/grammar/statements/create.y" { /* @@ -23448,7 +23577,7 @@ YYLTYPE yylloc; ;} break; - case 312: + case 315: #line 140 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23458,7 +23587,7 @@ YYLTYPE yylloc; ;} break; - case 313: + case 316: #line 147 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23468,7 +23597,7 @@ YYLTYPE yylloc; ;} break; - case 314: + case 317: #line 154 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23481,7 +23610,7 @@ YYLTYPE yylloc; ;} break; - case 315: + case 318: #line 164 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23494,7 +23623,7 @@ YYLTYPE yylloc; ;} break; - case 316: + case 319: #line 174 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23509,7 +23638,7 @@ YYLTYPE yylloc; ;} break; - case 317: + case 320: #line 186 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23520,7 +23649,7 @@ YYLTYPE yylloc; ;} break; - case 318: + case 321: #line 194 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23532,7 +23661,7 @@ YYLTYPE yylloc; ;} break; - case 319: + case 322: #line 203 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23550,27 +23679,27 @@ YYLTYPE yylloc; ;} break; - case 320: + case 323: #line 220 "third_party/libpg_query/grammar/statements/create.y" { (yyval.constr) = PG_CONSTR_GENERATED_VIRTUAL; ;} break; - case 321: + case 324: #line 221 "third_party/libpg_query/grammar/statements/create.y" { (yyval.constr) = PG_CONSTR_GENERATED_STORED; ;} break; - case 322: + case 325: #line 225 "third_party/libpg_query/grammar/statements/create.y" { (yyval.constr) = (yyvsp[(1) - (1)].constr); ;} break; - case 323: + case 326: #line 226 "third_party/libpg_query/grammar/statements/create.y" { (yyval.constr) = PG_CONSTR_GENERATED_VIRTUAL; ;} break; - case 324: + case 327: #line 231 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23582,7 +23711,7 @@ YYLTYPE yylloc; ;} break; - case 325: + case 328: #line 240 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23608,7 +23737,7 @@ YYLTYPE yylloc; ;} break; - case 326: + case 329: #line 263 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23621,79 +23750,79 @@ YYLTYPE yylloc; ;} break; - case 327: + case 330: #line 277 "third_party/libpg_query/grammar/statements/create.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (2)].str), (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 328: + case 331: #line 283 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = (yyvsp[(3) - (3)].ival); ;} break; - case 329: + case 332: #line 289 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = ((yyvsp[(1) - (1)].ival) << 8) | (PG_FKCONSTR_ACTION_NOACTION & 0xFF); ;} break; - case 330: + case 333: #line 291 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = (PG_FKCONSTR_ACTION_NOACTION << 8) | ((yyvsp[(1) - (1)].ival) & 0xFF); ;} break; - case 331: + case 334: #line 293 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = ((yyvsp[(1) - (2)].ival) << 8) | ((yyvsp[(2) - (2)].ival) & 0xFF); ;} break; - case 332: + case 335: #line 295 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = ((yyvsp[(2) - (2)].ival) << 8) | ((yyvsp[(1) - (2)].ival) & 0xFF); ;} break; - case 333: + case 336: #line 297 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = (PG_FKCONSTR_ACTION_NOACTION << 8) | (PG_FKCONSTR_ACTION_NOACTION & 0xFF); ;} break; - case 334: + case 337: #line 300 "third_party/libpg_query/grammar/statements/create.y" { (yyval.oncommit) = ONCOMMIT_DROP; ;} break; - case 335: + case 338: #line 301 "third_party/libpg_query/grammar/statements/create.y" { (yyval.oncommit) = PG_ONCOMMIT_DELETE_ROWS; ;} break; - case 336: + case 339: #line 302 "third_party/libpg_query/grammar/statements/create.y" { (yyval.oncommit) = PG_ONCOMMIT_PRESERVE_ROWS; ;} break; - case 337: + case 340: #line 303 "third_party/libpg_query/grammar/statements/create.y" { (yyval.oncommit) = PG_ONCOMMIT_NOOP; ;} break; - case 338: + case 341: #line 308 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 339: + case 342: #line 312 "third_party/libpg_query/grammar/statements/create.y" { (yyval.boolean) = true; ;} break; - case 340: + case 343: #line 313 "third_party/libpg_query/grammar/statements/create.y" { (yyval.boolean) = false; ;} break; - case 341: + case 344: #line 319 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = castNode(PGConstraint, (yyvsp[(3) - (3)].node)); @@ -23703,67 +23832,67 @@ YYLTYPE yylloc; ;} break; - case 342: + case 345: #line 325 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 343: + case 346: #line 330 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_CREATE_TABLE_LIKE_COMMENTS; ;} break; - case 344: + case 347: #line 331 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_CREATE_TABLE_LIKE_CONSTRAINTS; ;} break; - case 345: + case 348: #line 332 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_CREATE_TABLE_LIKE_DEFAULTS; ;} break; - case 346: + case 349: #line 333 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_CREATE_TABLE_LIKE_IDENTITY; ;} break; - case 347: + case 350: #line 334 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_CREATE_TABLE_LIKE_INDEXES; ;} break; - case 348: + case 351: #line 335 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_CREATE_TABLE_LIKE_STATISTICS; ;} break; - case 349: + case 352: #line 336 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_CREATE_TABLE_LIKE_STORAGE; ;} break; - case 350: + case 353: #line 337 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_CREATE_TABLE_LIKE_ALL; ;} break; - case 351: + case 354: #line 343 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 352: + case 355: #line 344 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].defelt)); ;} break; - case 353: + case 356: #line 348 "third_party/libpg_query/grammar/statements/create.y" { (yyval.str) = (yyvsp[(3) - (3)].str); ;} break; - case 354: + case 357: #line 354 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23773,7 +23902,7 @@ YYLTYPE yylloc; ;} break; - case 355: + case 358: #line 361 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23783,7 +23912,7 @@ YYLTYPE yylloc; ;} break; - case 356: + case 359: #line 368 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23793,7 +23922,7 @@ YYLTYPE yylloc; ;} break; - case 357: + case 360: #line 375 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -23803,82 +23932,82 @@ YYLTYPE yylloc; ;} break; - case 358: + case 361: #line 386 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 359: + case 362: #line 387 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = list_make1(makeDefElem("oids", (PGNode *) makeInteger(true), (yylsp[(1) - (2)]))); ;} break; - case 360: + case 363: #line 388 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = list_make1(makeDefElem("oids", (PGNode *) makeInteger(false), (yylsp[(1) - (2)]))); ;} break; - case 361: + case 364: #line 389 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = NIL; ;} break; - case 362: + case 365: #line 393 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 363: + case 366: #line 398 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = (yyvsp[(1) - (3)].ival) | (yyvsp[(3) - (3)].ival); ;} break; - case 364: + case 367: #line 399 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = (yyvsp[(1) - (3)].ival) & ~(yyvsp[(3) - (3)].ival); ;} break; - case 365: + case 368: #line 400 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = 0; ;} break; - case 366: + case 369: #line 405 "third_party/libpg_query/grammar/statements/create.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 367: + case 370: #line 410 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = CAS_NOT_DEFERRABLE; ;} break; - case 368: + case 371: #line 411 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = CAS_DEFERRABLE; ;} break; - case 369: + case 372: #line 412 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = CAS_INITIALLY_IMMEDIATE; ;} break; - case 370: + case 373: #line 413 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = CAS_INITIALLY_DEFERRED; ;} break; - case 371: + case 374: #line 414 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = CAS_NOT_VALID; ;} break; - case 372: + case 375: #line 415 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = CAS_NO_INHERIT; ;} break; - case 373: + case 376: #line 421 "third_party/libpg_query/grammar/statements/create.y" { PGColumnDef *n = makeNode(PGColumnDef); @@ -23898,7 +24027,7 @@ YYLTYPE yylloc; ;} break; - case 374: + case 377: #line 441 "third_party/libpg_query/grammar/statements/create.y" { PGColumnDef *n = makeNode(PGColumnDef); @@ -23925,7 +24054,7 @@ YYLTYPE yylloc; ;} break; - case 375: + case 378: #line 467 "third_party/libpg_query/grammar/statements/create.y" { PGColumnDef *n = (PGColumnDef *) (yyvsp[(2) - (2)].node); @@ -23935,7 +24064,7 @@ YYLTYPE yylloc; ;} break; - case 376: + case 379: #line 475 "third_party/libpg_query/grammar/statements/create.y" { PGColumnDef *n = (PGColumnDef *) (yyvsp[(2) - (2)].node); @@ -23945,122 +24074,122 @@ YYLTYPE yylloc; ;} break; - case 377: + case 380: #line 484 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 378: + case 381: #line 485 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].defelt)); ;} break; - case 379: + case 382: #line 489 "third_party/libpg_query/grammar/statements/create.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 380: + case 383: #line 493 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 381: + case 384: #line 494 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 382: + case 385: #line 495 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 383: + case 386: #line 500 "third_party/libpg_query/grammar/statements/create.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (3)].str), (PGNode *) (yyvsp[(3) - (3)].node), (yylsp[(1) - (3)])); ;} break; - case 384: + case 387: #line 504 "third_party/libpg_query/grammar/statements/create.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (1)].str), NULL, (yylsp[(1) - (1)])); ;} break; - case 385: + case 388: #line 511 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 386: + case 389: #line 512 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = NIL; ;} break; - case 387: + case 390: #line 517 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 388: + case 391: #line 518 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 389: + case 392: #line 519 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = NIL; ;} break; - case 390: + case 393: #line 524 "third_party/libpg_query/grammar/statements/create.y" { (yyval.node) = (PGNode *) makeString((yyvsp[(1) - (1)].str)); ;} break; - case 391: + case 394: #line 531 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 392: + case 395: #line 532 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = NIL; ;} break; - case 393: + case 396: #line 537 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 394: + case 397: #line 538 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = NIL; ;} break; - case 395: + case 398: #line 542 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = (yyvsp[(3) - (3)].ival); ;} break; - case 396: + case 399: #line 548 "third_party/libpg_query/grammar/statements/create.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (3)].str), (PGNode *) (yyvsp[(3) - (3)].node), (yylsp[(1) - (3)])); ;} break; - case 397: + case 400: #line 552 "third_party/libpg_query/grammar/statements/create.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (1)].str), NULL, (yylsp[(1) - (1)])); ;} break; - case 398: + case 401: #line 556 "third_party/libpg_query/grammar/statements/create.y" { (yyval.defelt) = makeDefElemExtended((yyvsp[(1) - (5)].str), (yyvsp[(3) - (5)].str), (PGNode *) (yyvsp[(5) - (5)].node), @@ -24068,39 +24197,39 @@ YYLTYPE yylloc; ;} break; - case 399: + case 402: #line 561 "third_party/libpg_query/grammar/statements/create.y" { (yyval.defelt) = makeDefElemExtended((yyvsp[(1) - (3)].str), (yyvsp[(3) - (3)].str), NULL, PG_DEFELEM_UNSPEC, (yylsp[(1) - (3)])); ;} break; - case 400: + case 403: #line 568 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 401: + case 404: #line 569 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 402: + case 405: #line 573 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 403: + case 406: #line 574 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 404: + case 407: #line 578 "third_party/libpg_query/grammar/statements/create.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 405: + case 408: #line 580 "third_party/libpg_query/grammar/statements/create.y" { (yyval.typnam) = makeTypeNameFromNameList(lcons(makeString((yyvsp[(1) - (4)].str)), (yyvsp[(2) - (4)].list))); @@ -24109,7 +24238,7 @@ YYLTYPE yylloc; ;} break; - case 406: + case 409: #line 586 "third_party/libpg_query/grammar/statements/create.y" { (yyval.typnam) = makeTypeNameFromNameList(lcons(makeString((yyvsp[(2) - (5)].str)), (yyvsp[(3) - (5)].list))); @@ -24119,7 +24248,7 @@ YYLTYPE yylloc; ;} break; - case 407: + case 410: #line 597 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -24135,7 +24264,7 @@ YYLTYPE yylloc; ;} break; - case 408: + case 411: #line 611 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -24151,7 +24280,7 @@ YYLTYPE yylloc; ;} break; - case 409: + case 412: #line 624 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -24168,7 +24297,7 @@ YYLTYPE yylloc; ;} break; - case 410: + case 413: #line 639 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -24184,7 +24313,7 @@ YYLTYPE yylloc; ;} break; - case 411: + case 414: #line 652 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -24201,7 +24330,7 @@ YYLTYPE yylloc; ;} break; - case 412: + case 415: #line 667 "third_party/libpg_query/grammar/statements/create.y" { PGConstraint *n = makeNode(PGConstraint); @@ -24222,28 +24351,28 @@ YYLTYPE yylloc; ;} break; - case 413: + case 416: #line 689 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 414: + case 417: #line 693 "third_party/libpg_query/grammar/statements/create.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 415: + case 418: #line 700 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_FKCONSTR_MATCH_FULL; ;} break; - case 416: + case 419: #line 704 "third_party/libpg_query/grammar/statements/create.y" { ereport(ERROR, @@ -24254,21 +24383,21 @@ YYLTYPE yylloc; ;} break; - case 417: + case 420: #line 712 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_FKCONSTR_MATCH_SIMPLE; ;} break; - case 418: + case 421: #line 716 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_FKCONSTR_MATCH_SIMPLE; ;} break; - case 419: + case 422: #line 724 "third_party/libpg_query/grammar/statements/create.y" { PGTableLikeClause *n = makeNode(PGTableLikeClause); @@ -24278,27 +24407,27 @@ YYLTYPE yylloc; ;} break; - case 420: + case 423: #line 733 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_RELPERSISTENCE_TEMP; ;} break; - case 421: + case 424: #line 734 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_RELPERSISTENCE_TEMP; ;} break; - case 422: + case 425: #line 735 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_RELPERSISTENCE_TEMP; ;} break; - case 423: + case 426: #line 736 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_RELPERSISTENCE_TEMP; ;} break; - case 424: + case 427: #line 738 "third_party/libpg_query/grammar/statements/create.y" { ereport(PGWARNING, @@ -24308,7 +24437,7 @@ YYLTYPE yylloc; ;} break; - case 425: + case 428: #line 745 "third_party/libpg_query/grammar/statements/create.y" { ereport(PGWARNING, @@ -24318,27 +24447,27 @@ YYLTYPE yylloc; ;} break; - case 426: + case 429: #line 751 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_RELPERSISTENCE_UNLOGGED; ;} break; - case 427: + case 430: #line 752 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = RELPERSISTENCE_PERMANENT; ;} break; - case 428: + case 431: #line 757 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = PG_ATTRIBUTE_IDENTITY_ALWAYS; ;} break; - case 429: + case 432: #line 758 "third_party/libpg_query/grammar/statements/create.y" { (yyval.ival) = ATTRIBUTE_IDENTITY_BY_DEFAULT; ;} break; - case 430: + case 433: #line 10 "third_party/libpg_query/grammar/statements/drop.y" { PGDropStmt *n = makeNode(PGDropStmt); @@ -24351,7 +24480,7 @@ YYLTYPE yylloc; ;} break; - case 431: + case 434: #line 20 "third_party/libpg_query/grammar/statements/drop.y" { PGDropStmt *n = makeNode(PGDropStmt); @@ -24364,7 +24493,7 @@ YYLTYPE yylloc; ;} break; - case 432: + case 435: #line 30 "third_party/libpg_query/grammar/statements/drop.y" { PGDropStmt *n = makeNode(PGDropStmt); @@ -24377,7 +24506,7 @@ YYLTYPE yylloc; ;} break; - case 433: + case 436: #line 40 "third_party/libpg_query/grammar/statements/drop.y" { PGDropStmt *n = makeNode(PGDropStmt); @@ -24390,7 +24519,7 @@ YYLTYPE yylloc; ;} break; - case 434: + case 437: #line 50 "third_party/libpg_query/grammar/statements/drop.y" { PGDropStmt *n = makeNode(PGDropStmt); @@ -24403,7 +24532,7 @@ YYLTYPE yylloc; ;} break; - case 435: + case 438: #line 60 "third_party/libpg_query/grammar/statements/drop.y" { PGDropStmt *n = makeNode(PGDropStmt); @@ -24416,167 +24545,167 @@ YYLTYPE yylloc; ;} break; - case 436: + case 439: #line 73 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_TABLE; ;} break; - case 437: + case 440: #line 74 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_SEQUENCE; ;} break; - case 438: + case 441: #line 75 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_FUNCTION; ;} break; - case 439: + case 442: #line 76 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_FUNCTION; ;} break; - case 440: + case 443: #line 77 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_TABLE_MACRO; ;} break; - case 441: + case 444: #line 78 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_VIEW; ;} break; - case 442: + case 445: #line 79 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_MATVIEW; ;} break; - case 443: + case 446: #line 80 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_INDEX; ;} break; - case 444: + case 447: #line 81 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_FOREIGN_TABLE; ;} break; - case 445: + case 448: #line 82 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_COLLATION; ;} break; - case 446: + case 449: #line 83 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_CONVERSION; ;} break; - case 447: + case 450: #line 84 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_SCHEMA; ;} break; - case 448: + case 451: #line 85 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_STATISTIC_EXT; ;} break; - case 449: + case 452: #line 86 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_TSPARSER; ;} break; - case 450: + case 453: #line 87 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_TSDICTIONARY; ;} break; - case 451: + case 454: #line 88 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_TSTEMPLATE; ;} break; - case 452: + case 455: #line 89 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_TSCONFIGURATION; ;} break; - case 453: + case 456: #line 90 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_TYPE; ;} break; - case 454: + case 457: #line 95 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_ACCESS_METHOD; ;} break; - case 455: + case 458: #line 96 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_EVENT_TRIGGER; ;} break; - case 456: + case 459: #line 97 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_EXTENSION; ;} break; - case 457: + case 460: #line 98 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_FDW; ;} break; - case 458: + case 461: #line 99 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_PUBLICATION; ;} break; - case 459: + case 462: #line 100 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_FOREIGN_SERVER; ;} break; - case 460: + case 463: #line 105 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} break; - case 461: + case 464: #line 106 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; - case 462: + case 465: #line 111 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.dbehavior) = PG_DROP_CASCADE; ;} break; - case 463: + case 466: #line 112 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.dbehavior) = PG_DROP_RESTRICT; ;} break; - case 464: + case 467: #line 113 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.dbehavior) = PG_DROP_RESTRICT; /* default */ ;} break; - case 465: + case 468: #line 118 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_POLICY; ;} break; - case 466: + case 469: #line 119 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_RULE; ;} break; - case 467: + case 470: #line 120 "third_party/libpg_query/grammar/statements/drop.y" { (yyval.objtype) = PG_OBJECT_TRIGGER; ;} break; - case 468: + case 471: #line 13 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMergeIntoStmt *n = makeNode(PGMergeIntoStmt); @@ -24593,27 +24722,27 @@ YYLTYPE yylloc; ;} break; - case 469: + case 472: #line 29 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 470: + case 473: #line 30 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.node) = NULL; ;} break; - case 471: + case 474: #line 34 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 472: + case 475: #line 35 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.list) = NULL; ;} break; - case 475: + case 478: #line 44 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = makeNode(PGMatchAction); @@ -24624,7 +24753,7 @@ YYLTYPE yylloc; ;} break; - case 476: + case 479: #line 52 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = makeNode(PGMatchAction); @@ -24634,7 +24763,7 @@ YYLTYPE yylloc; ;} break; - case 477: + case 480: #line 59 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = makeNode(PGMatchAction); @@ -24644,7 +24773,7 @@ YYLTYPE yylloc; ;} break; - case 478: + case 481: #line 66 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = makeNode(PGMatchAction); @@ -24653,7 +24782,7 @@ YYLTYPE yylloc; ;} break; - case 479: + case 482: #line 72 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = makeNode(PGMatchAction); @@ -24665,7 +24794,7 @@ YYLTYPE yylloc; ;} break; - case 480: + case 483: #line 81 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = makeNode(PGMatchAction); @@ -24675,7 +24804,7 @@ YYLTYPE yylloc; ;} break; - case 481: + case 484: #line 88 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = makeNode(PGMatchAction); @@ -24686,7 +24815,7 @@ YYLTYPE yylloc; ;} break; - case 482: + case 485: #line 96 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = makeNode(PGMatchAction); @@ -24695,7 +24824,7 @@ YYLTYPE yylloc; ;} break; - case 483: + case 486: #line 102 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = makeNode(PGMatchAction); @@ -24705,17 +24834,17 @@ YYLTYPE yylloc; ;} break; - case 484: + case 487: #line 111 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 485: + case 488: #line 112 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.node) = NULL; ;} break; - case 486: + case 489: #line 117 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = (PGMatchAction *) (yyvsp[(5) - (5)].node); @@ -24725,22 +24854,22 @@ YYLTYPE yylloc; ;} break; - case 487: + case 490: #line 126 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.mergeaction) = MERGE_ACTION_WHEN_NOT_MATCHED_BY_SOURCE; ;} break; - case 488: + case 491: #line 127 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.mergeaction) = MERGE_ACTION_WHEN_NOT_MATCHED_BY_TARGET; ;} break; - case 489: + case 492: #line 128 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.mergeaction) = MERGE_ACTION_WHEN_NOT_MATCHED_BY_TARGET; ;} break; - case 490: + case 493: #line 133 "third_party/libpg_query/grammar/statements/merge_into.y" { PGMatchAction *n = (PGMatchAction *) (yyvsp[(7) - (7)].node); @@ -24750,17 +24879,17 @@ YYLTYPE yylloc; ;} break; - case 493: + case 496: #line 146 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 494: + case 497: #line 147 "third_party/libpg_query/grammar/statements/merge_into.y" { (yyval.list) = list_concat(list_make1((yyvsp[(1) - (2)].node)), (yyvsp[(2) - (2)].list)); ;} break; - case 495: + case 498: #line 8 "third_party/libpg_query/grammar/statements/create_function.y" { PGCreateFunctionStmt *n = makeNode(PGCreateFunctionStmt); @@ -24772,7 +24901,7 @@ YYLTYPE yylloc; ;} break; - case 496: + case 499: #line 17 "third_party/libpg_query/grammar/statements/create_function.y" { PGCreateFunctionStmt *n = makeNode(PGCreateFunctionStmt); @@ -24784,7 +24913,7 @@ YYLTYPE yylloc; ;} break; - case 497: + case 500: #line 26 "third_party/libpg_query/grammar/statements/create_function.y" { PGCreateFunctionStmt *n = makeNode(PGCreateFunctionStmt); @@ -24796,7 +24925,7 @@ YYLTYPE yylloc; ;} break; - case 498: + case 501: #line 35 "third_party/libpg_query/grammar/statements/create_function.y" { PGCreateFunctionStmt *n = makeNode(PGCreateFunctionStmt); @@ -24808,7 +24937,7 @@ YYLTYPE yylloc; ;} break; - case 499: + case 502: #line 44 "third_party/libpg_query/grammar/statements/create_function.y" { PGCreateFunctionStmt *n = makeNode(PGCreateFunctionStmt); @@ -24820,7 +24949,7 @@ YYLTYPE yylloc; ;} break; - case 500: + case 503: #line 53 "third_party/libpg_query/grammar/statements/create_function.y" { PGCreateFunctionStmt *n = makeNode(PGCreateFunctionStmt); @@ -24832,7 +24961,7 @@ YYLTYPE yylloc; ;} break; - case 501: + case 504: #line 65 "third_party/libpg_query/grammar/statements/create_function.y" { PGFunctionDefinition *n = makeNode(PGFunctionDefinition); @@ -24842,7 +24971,7 @@ YYLTYPE yylloc; ;} break; - case 502: + case 505: #line 75 "third_party/libpg_query/grammar/statements/create_function.y" { PGFunctionDefinition *n = makeNode(PGFunctionDefinition); @@ -24852,28 +24981,28 @@ YYLTYPE yylloc; ;} break; - case 503: + case 506: #line 85 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 504: + case 507: #line 89 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 505: + case 508: #line 96 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 507: + case 510: #line 104 "third_party/libpg_query/grammar/statements/create_function.y" { PGFunctionDefinition *n = makeNode(PGFunctionDefinition); @@ -24883,56 +25012,56 @@ YYLTYPE yylloc; ;} break; - case 508: + case 511: #line 114 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 509: + case 512: #line 118 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 512: + case 515: #line 131 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = NIL; ;} break; - case 513: + case 516: #line 135 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = (yyvsp[(2) - (4)].list); ;} break; - case 514: + case 517: #line 139 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 515: + case 518: #line 146 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 516: + case 519: #line 150 "third_party/libpg_query/grammar/statements/create_function.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 517: + case 520: #line 157 "third_party/libpg_query/grammar/statements/create_function.y" { PGFunctionParameter *n = makeNode(PGFunctionParameter); @@ -24943,7 +25072,7 @@ YYLTYPE yylloc; ;} break; - case 518: + case 521: #line 165 "third_party/libpg_query/grammar/statements/create_function.y" { PGFunctionParameter *n = makeNode(PGFunctionParameter); @@ -24954,7 +25083,7 @@ YYLTYPE yylloc; ;} break; - case 519: + case 522: #line 173 "third_party/libpg_query/grammar/statements/create_function.y" { PGFunctionParameter *n = makeNode(PGFunctionParameter); @@ -24965,7 +25094,7 @@ YYLTYPE yylloc; ;} break; - case 520: + case 523: #line 12 "third_party/libpg_query/grammar/statements/update.y" { PGUpdateStmt *n = makeNode(PGUpdateStmt); @@ -24979,7 +25108,7 @@ YYLTYPE yylloc; ;} break; - case 521: + case 524: #line 3 "third_party/libpg_query/grammar/statements/copy.y" { PGCopyStmt *n = makeNode(PGCopyStmt); @@ -25010,7 +25139,7 @@ YYLTYPE yylloc; ;} break; - case 522: + case 525: #line 31 "third_party/libpg_query/grammar/statements/copy.y" { PGCopyStmt *n = makeNode(PGCopyStmt); @@ -25032,7 +25161,7 @@ YYLTYPE yylloc; ;} break; - case 523: + case 526: #line 50 "third_party/libpg_query/grammar/statements/copy.y" { PGCopyDatabaseStmt *n = makeNode(PGCopyDatabaseStmt); @@ -25043,287 +25172,287 @@ YYLTYPE yylloc; ;} break; - case 524: + case 527: #line 61 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.conststr) = NULL; ;} break; - case 525: + case 528: #line 62 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.conststr) = "schema"; ;} break; - case 526: + case 529: #line 63 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.conststr) = "data"; ;} break; - case 527: + case 530: #line 67 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.boolean) = true; ;} break; - case 528: + case 531: #line 68 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.boolean) = false; ;} break; - case 529: + case 532: #line 74 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("delimiter", (PGNode *)makeString((yyvsp[(3) - (3)].str)), (yylsp[(2) - (3)])); ;} break; - case 530: + case 533: #line 77 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = NULL; ;} break; - case 531: + case 534: #line 94 "third_party/libpg_query/grammar/statements/copy.y" {;} break; - case 532: + case 535: #line 95 "third_party/libpg_query/grammar/statements/copy.y" {;} break; - case 533: + case 536: #line 99 "third_party/libpg_query/grammar/statements/copy.y" {;} break; - case 534: + case 537: #line 100 "third_party/libpg_query/grammar/statements/copy.y" {;} break; - case 535: + case 538: #line 105 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.boolean) = true; ;} break; - case 536: + case 539: #line 106 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.boolean) = false; ;} break; - case 537: + case 540: #line 110 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 538: + case 541: #line 111 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 539: + case 542: #line 116 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("oids", NULL, (yylsp[(1) - (2)])); ;} break; - case 540: + case 543: #line 119 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = NULL; ;} break; - case 541: + case 544: #line 124 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].defelt)); ;} break; - case 542: + case 545: #line 125 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.list) = NIL; ;} break; - case 543: + case 546: #line 131 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("format", (PGNode *)makeStringConst("binary", (yylsp[(1) - (1)])), (yylsp[(1) - (1)])); ;} break; - case 544: + case 547: #line 134 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = NULL; ;} break; - case 545: + case 548: #line 140 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("format", (PGNode *)makeStringConst("binary", (yylsp[(1) - (1)])), (yylsp[(1) - (1)])); ;} break; - case 546: + case 549: #line 144 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("oids", NULL, (yylsp[(1) - (1)])); ;} break; - case 547: + case 550: #line 148 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("freeze", NULL, (yylsp[(1) - (1)])); ;} break; - case 548: + case 551: #line 152 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("delimiter", (PGNode *)makeStringConst((yyvsp[(3) - (3)].str), (yylsp[(3) - (3)])), (yylsp[(1) - (3)])); ;} break; - case 549: + case 552: #line 156 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("null", (PGNode *)makeStringConst((yyvsp[(3) - (3)].str), (yylsp[(3) - (3)])), (yylsp[(1) - (3)])); ;} break; - case 550: + case 553: #line 160 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("format", (PGNode *)makeStringConst("csv", (yylsp[(1) - (1)])), (yylsp[(1) - (1)])); ;} break; - case 551: + case 554: #line 164 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("header", NULL, (yylsp[(1) - (1)])); ;} break; - case 552: + case 555: #line 168 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("quote", (PGNode *)makeStringConst((yyvsp[(3) - (3)].str), (yylsp[(3) - (3)])), (yylsp[(1) - (3)])); ;} break; - case 553: + case 556: #line 172 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("escape", (PGNode *)makeStringConst((yyvsp[(3) - (3)].str), (yylsp[(3) - (3)])), (yylsp[(1) - (3)])); ;} break; - case 554: + case 557: #line 176 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("force_quote", (PGNode *)(yyvsp[(3) - (3)].list), (yylsp[(1) - (3)])); ;} break; - case 555: + case 558: #line 180 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("force_quote", (PGNode *)makeNode(PGAStar), (yylsp[(1) - (3)])); ;} break; - case 556: + case 559: #line 184 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("partition_by", (PGNode *)(yyvsp[(3) - (3)].list), (yylsp[(1) - (3)])); ;} break; - case 557: + case 560: #line 188 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("partition_by", (PGNode *)makeNode(PGAStar), (yylsp[(1) - (3)])); ;} break; - case 558: + case 561: #line 192 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("force_not_null", (PGNode *)(yyvsp[(4) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 559: + case 562: #line 196 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("force_null", (PGNode *)(yyvsp[(3) - (3)].list), (yylsp[(1) - (3)])); ;} break; - case 560: + case 563: #line 200 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("encoding", (PGNode *)makeStringConst((yyvsp[(2) - (2)].str), (yylsp[(2) - (2)])), (yylsp[(1) - (2)])); ;} break; - case 561: + case 564: #line 211 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.node) = makeStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 562: + case 565: #line 212 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.node) = makeStringConst("/dev/stdin", (yylsp[(1) - (1)])); ;} break; - case 563: + case 566: #line 213 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.node) = makeStringConst("/dev/stdout", (yylsp[(1) - (1)])); ;} break; - case 564: + case 567: #line 214 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.node) = makeStringConst(psprintf("%s.%s", (yyvsp[(1) - (3)].str), (yyvsp[(3) - (3)].str)), (yylsp[(1) - (3)])); ;} break; - case 565: + case 568: #line 215 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.node) = makeStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 566: + case 569: #line 216 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 567: + case 570: #line 217 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 570: + case 573: #line 52 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 571: + case 574: #line 53 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 572: + case 575: #line 55 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 573: + case 576: #line 72 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 574: + case 577: #line 74 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(1) - (2)].node), (yyvsp[(2) - (2)].list), NIL, @@ -25333,7 +25462,7 @@ YYLTYPE yylloc; ;} break; - case 575: + case 578: #line 81 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(1) - (4)].node), (yyvsp[(2) - (4)].list), (yyvsp[(3) - (4)].list), @@ -25344,7 +25473,7 @@ YYLTYPE yylloc; ;} break; - case 576: + case 579: #line 89 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(1) - (4)].node), (yyvsp[(2) - (4)].list), (yyvsp[(4) - (4)].list), @@ -25355,7 +25484,7 @@ YYLTYPE yylloc; ;} break; - case 577: + case 580: #line 97 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(2) - (2)].node), NULL, NIL, @@ -25366,7 +25495,7 @@ YYLTYPE yylloc; ;} break; - case 578: + case 581: #line 105 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(2) - (3)].node), (yyvsp[(3) - (3)].list), NIL, @@ -25377,7 +25506,7 @@ YYLTYPE yylloc; ;} break; - case 579: + case 582: #line 113 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(2) - (5)].node), (yyvsp[(3) - (5)].list), (yyvsp[(4) - (5)].list), @@ -25388,7 +25517,7 @@ YYLTYPE yylloc; ;} break; - case 580: + case 583: #line 121 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(2) - (5)].node), (yyvsp[(3) - (5)].list), (yyvsp[(5) - (5)].list), @@ -25399,24 +25528,24 @@ YYLTYPE yylloc; ;} break; - case 581: + case 584: #line 131 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 582: + case 585: #line 132 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 583: + case 586: #line 160 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 584: + case 587: #line 164 "third_party/libpg_query/grammar/statements/select.y" { PGAStar *star = makeNode(PGAStar); @@ -25424,7 +25553,7 @@ YYLTYPE yylloc; ;} break; - case 585: + case 588: #line 175 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -25441,7 +25570,7 @@ YYLTYPE yylloc; ;} break; - case 586: + case 589: #line 191 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -25459,7 +25588,7 @@ YYLTYPE yylloc; ;} break; - case 587: + case 590: #line 208 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -25477,7 +25606,7 @@ YYLTYPE yylloc; ;} break; - case 588: + case 591: #line 226 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -25496,12 +25625,12 @@ YYLTYPE yylloc; ;} break; - case 589: + case 592: #line 241 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 590: + case 593: #line 243 "third_party/libpg_query/grammar/statements/select.y" { /* same as SELECT * FROM relation_expr */ @@ -25523,35 +25652,35 @@ YYLTYPE yylloc; ;} break; - case 591: + case 594: #line 262 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSetOp(PG_SETOP_UNION_BY_NAME, (yyvsp[(3) - (5)].boolean), (yyvsp[(1) - (5)].node), (yyvsp[(5) - (5)].node)); ;} break; - case 592: + case 595: #line 266 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSetOp(PG_SETOP_UNION, (yyvsp[(3) - (4)].boolean), (yyvsp[(1) - (4)].node), (yyvsp[(4) - (4)].node)); ;} break; - case 593: + case 596: #line 270 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSetOp(PG_SETOP_INTERSECT, (yyvsp[(3) - (4)].boolean), (yyvsp[(1) - (4)].node), (yyvsp[(4) - (4)].node)); ;} break; - case 594: + case 597: #line 274 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSetOp(PG_SETOP_EXCEPT, (yyvsp[(3) - (4)].boolean), (yyvsp[(1) - (4)].node), (yyvsp[(4) - (4)].node)); ;} break; - case 595: + case 598: #line 278 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -25564,7 +25693,7 @@ YYLTYPE yylloc; ;} break; - case 596: + case 599: #line 288 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -25578,7 +25707,7 @@ YYLTYPE yylloc; ;} break; - case 597: + case 600: #line 299 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -25591,7 +25720,7 @@ YYLTYPE yylloc; ;} break; - case 598: + case 601: #line 309 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -25603,7 +25732,7 @@ YYLTYPE yylloc; ;} break; - case 599: + case 602: #line 318 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -25617,7 +25746,7 @@ YYLTYPE yylloc; ;} break; - case 600: + case 603: #line 329 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -25631,7 +25760,7 @@ YYLTYPE yylloc; ;} break; - case 601: + case 604: #line 340 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -25646,7 +25775,7 @@ YYLTYPE yylloc; ;} break; - case 602: + case 605: #line 352 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -25664,7 +25793,7 @@ YYLTYPE yylloc; ;} break; - case 603: + case 606: #line 367 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -25682,7 +25811,7 @@ YYLTYPE yylloc; ;} break; - case 610: + case 613: #line 397 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -25691,7 +25820,7 @@ YYLTYPE yylloc; ;} break; - case 611: + case 614: #line 403 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -25701,32 +25830,32 @@ YYLTYPE yylloc; ;} break; - case 612: + case 615: #line 409 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 613: + case 616: #line 413 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 614: + case 617: #line 414 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 615: + case 618: #line 418 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 616: + case 619: #line 419 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 617: + case 620: #line 434 "third_party/libpg_query/grammar/statements/select.y" { (yyval.with) = makeNode(PGWithClause); @@ -25736,7 +25865,7 @@ YYLTYPE yylloc; ;} break; - case 618: + case 621: #line 441 "third_party/libpg_query/grammar/statements/select.y" { (yyval.with) = makeNode(PGWithClause); @@ -25746,7 +25875,7 @@ YYLTYPE yylloc; ;} break; - case 619: + case 622: #line 448 "third_party/libpg_query/grammar/statements/select.y" { (yyval.with) = makeNode(PGWithClause); @@ -25756,17 +25885,17 @@ YYLTYPE yylloc; ;} break; - case 620: + case 623: #line 457 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 621: + case 624: #line 458 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 622: + case 625: #line 462 "third_party/libpg_query/grammar/statements/select.y" { PGCommonTableExpr *n = makeNode(PGCommonTableExpr); @@ -25780,52 +25909,52 @@ YYLTYPE yylloc; ;} break; - case 623: + case 626: #line 475 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(4) - (5)].list); ;} break; - case 624: + case 627: #line 476 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(NIL); ;} break; - case 625: + case 628: #line 480 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 626: + case 629: #line 481 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 627: + case 630: #line 485 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 628: + case 631: #line 486 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 629: + case 632: #line 490 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ctematerialize) = PGCTEMaterializeAlways; ;} break; - case 630: + case 633: #line 491 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ctematerialize) = PGCTEMaterializeNever; ;} break; - case 631: + case 634: #line 492 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ctematerialize) = PGCTEMaterializeDefault; ;} break; - case 632: + case 635: #line 497 "third_party/libpg_query/grammar/statements/select.y" { (yyval.into) = makeNode(PGIntoClause); @@ -25838,12 +25967,12 @@ YYLTYPE yylloc; ;} break; - case 633: + case 636: #line 507 "third_party/libpg_query/grammar/statements/select.y" { (yyval.into) = NULL; ;} break; - case 634: + case 637: #line 516 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(3) - (3)].range); @@ -25851,7 +25980,7 @@ YYLTYPE yylloc; ;} break; - case 635: + case 638: #line 521 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(3) - (3)].range); @@ -25859,7 +25988,7 @@ YYLTYPE yylloc; ;} break; - case 636: + case 639: #line 526 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(4) - (4)].range); @@ -25867,7 +25996,7 @@ YYLTYPE yylloc; ;} break; - case 637: + case 640: #line 531 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(4) - (4)].range); @@ -25875,7 +26004,7 @@ YYLTYPE yylloc; ;} break; - case 638: + case 641: #line 536 "third_party/libpg_query/grammar/statements/select.y" { ereport(PGWARNING, @@ -25886,7 +26015,7 @@ YYLTYPE yylloc; ;} break; - case 639: + case 642: #line 544 "third_party/libpg_query/grammar/statements/select.y" { ereport(PGWARNING, @@ -25897,7 +26026,7 @@ YYLTYPE yylloc; ;} break; - case 640: + case 643: #line 552 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(3) - (3)].range); @@ -25905,7 +26034,7 @@ YYLTYPE yylloc; ;} break; - case 641: + case 644: #line 557 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(2) - (2)].range); @@ -25913,7 +26042,7 @@ YYLTYPE yylloc; ;} break; - case 642: + case 645: #line 562 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(1) - (1)].range); @@ -25921,87 +26050,87 @@ YYLTYPE yylloc; ;} break; - case 643: + case 646: #line 568 "third_party/libpg_query/grammar/statements/select.y" {;} break; - case 644: + case 647: #line 569 "third_party/libpg_query/grammar/statements/select.y" {;} break; - case 645: + case 648: #line 573 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 646: + case 649: #line 574 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 647: + case 650: #line 575 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 648: + case 651: #line 579 "third_party/libpg_query/grammar/statements/select.y" { ;} break; - case 649: + case 652: #line 586 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(NIL); ;} break; - case 650: + case 653: #line 587 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(4) - (5)].list); ;} break; - case 651: + case 654: #line 591 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL;;} break; - case 652: + case 655: #line 592 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 653: + case 656: #line 596 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ignorenulls) = PG_IGNORE_NULLS;;} break; - case 654: + case 657: #line 597 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ignorenulls) = PG_RESPECT_NULLS;;} break; - case 655: + case 658: #line 598 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ignorenulls) = PG_DEFAULT_NULLS; ;} break; - case 656: + case 659: #line 602 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list);;} break; - case 657: + case 660: #line 603 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 658: + case 661: #line 607 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 659: + case 662: #line 609 "third_party/libpg_query/grammar/statements/select.y" { PGSortBy *sort = makeNode(PGSortBy); @@ -26017,17 +26146,17 @@ YYLTYPE yylloc; ;} break; - case 660: + case 663: #line 624 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].sortby)); ;} break; - case 661: + case 664: #line 625 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].sortby)); ;} break; - case 662: + case 665: #line 629 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortby) = makeNode(PGSortBy); @@ -26039,7 +26168,7 @@ YYLTYPE yylloc; ;} break; - case 663: + case 666: #line 638 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortby) = makeNode(PGSortBy); @@ -26051,72 +26180,72 @@ YYLTYPE yylloc; ;} break; - case 664: + case 667: #line 648 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortorder) = PG_SORTBY_ASC; ;} break; - case 665: + case 668: #line 649 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortorder) = PG_SORTBY_DESC; ;} break; - case 666: + case 669: #line 650 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortorder) = PG_SORTBY_DEFAULT; ;} break; - case 667: + case 670: #line 653 "third_party/libpg_query/grammar/statements/select.y" { (yyval.nullorder) = PG_SORTBY_NULLS_FIRST; ;} break; - case 668: + case 671: #line 654 "third_party/libpg_query/grammar/statements/select.y" { (yyval.nullorder) = PG_SORTBY_NULLS_LAST; ;} break; - case 669: + case 672: #line 655 "third_party/libpg_query/grammar/statements/select.y" { (yyval.nullorder) = PG_SORTBY_NULLS_DEFAULT; ;} break; - case 670: + case 673: #line 659 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make3((yyvsp[(2) - (2)].node), (yyvsp[(1) - (2)].node), NULL); ;} break; - case 671: + case 674: #line 660 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make3((yyvsp[(1) - (2)].node), (yyvsp[(2) - (2)].node), (yyvsp[(1) - (2)].node)); ;} break; - case 672: + case 675: #line 661 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make3(NULL, (yyvsp[(1) - (1)].node), NULL); ;} break; - case 673: + case 676: #line 662 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make3((yyvsp[(1) - (1)].node), NULL, (yyvsp[(1) - (1)].node)); ;} break; - case 674: + case 677: #line 666 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 675: + case 678: #line 667 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make3(NULL,NULL,NULL); ;} break; - case 676: + case 679: #line 672 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 677: + case 680: #line 674 "third_party/libpg_query/grammar/statements/select.y" { /* Disabled because it was too confusing, bjm 2002-02-18 */ @@ -26128,91 +26257,91 @@ YYLTYPE yylloc; ;} break; - case 678: + case 681: #line 690 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(3) - (5)].node); ;} break; - case 679: + case 682: #line 692 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntConst(1, -1); ;} break; - case 680: + case 683: #line 697 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 681: + case 684: #line 700 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 682: + case 685: #line 705 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeFloatConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 683: + case 686: #line 709 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntConst((yyvsp[(1) - (1)].ival), (yylsp[(1) - (1)])); ;} break; - case 685: + case 688: #line 720 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize((yyvsp[(1) - (2)].node), true); ;} break; - case 686: + case 689: #line 724 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize((yyvsp[(1) - (2)].node), true); ;} break; - case 687: + case 690: #line 728 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize((yyvsp[(1) - (1)].node), false); ;} break; - case 688: + case 691: #line 732 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize((yyvsp[(1) - (2)].node), false); ;} break; - case 689: + case 692: #line 739 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(3) - (3)].node); ;} break; - case 690: + case 693: #line 743 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 691: + case 694: #line 750 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 692: + case 695: #line 751 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = NULL; ;} break; - case 693: + case 696: #line 756 "third_party/libpg_query/grammar/statements/select.y" { int seed = (yyvsp[(5) - (5)].ival); @@ -26220,21 +26349,21 @@ YYLTYPE yylloc; ;} break; - case 694: + case 697: #line 761 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleOptions((yyvsp[(1) - (1)].node), NULL, NULL, (yylsp[(1) - (1)])); ;} break; - case 695: + case 698: #line 765 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleOptions((yyvsp[(1) - (4)].node), (yyvsp[(3) - (4)].str), NULL, (yylsp[(1) - (4)])); ;} break; - case 696: + case 699: #line 769 "third_party/libpg_query/grammar/statements/select.y" { int seed = (yyvsp[(5) - (6)].ival); @@ -26242,44 +26371,44 @@ YYLTYPE yylloc; ;} break; - case 697: + case 700: #line 777 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 698: + case 701: #line 783 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 699: + case 702: #line 784 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 700: + case 703: #line 789 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = (yyvsp[(3) - (4)].ival); ;} break; - case 701: + case 704: #line 790 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = -1; ;} break; - case 702: + case 705: #line 795 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "TIMESTAMP"; ;} break; - case 703: + case 706: #line 796 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "VERSION"; ;} break; - case 704: + case 707: #line 801 "third_party/libpg_query/grammar/statements/select.y" { PGAtClause *n = makeNode(PGAtClause); @@ -26289,22 +26418,22 @@ YYLTYPE yylloc; ;} break; - case 705: + case 708: #line 810 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(3) - (4)].node); ;} break; - case 706: + case 709: #line 811 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 707: + case 710: #line 816 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 708: + case 711: #line 818 "third_party/libpg_query/grammar/statements/select.y" { /* LIMIT ALL is represented as a NULL constant */ @@ -26312,77 +26441,77 @@ YYLTYPE yylloc; ;} break; - case 709: + case 712: #line 823 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeLimitPercent((yyvsp[(1) - (2)].node)); ;} break; - case 710: + case 713: #line 825 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeLimitPercent(makeFloatConst((yyvsp[(1) - (2)].str),(yylsp[(1) - (2)]))); ;} break; - case 711: + case 714: #line 827 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeLimitPercent(makeIntConst((yyvsp[(1) - (2)].ival),(yylsp[(1) - (2)]))); ;} break; - case 712: + case 715: #line 831 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 713: + case 716: #line 851 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 714: + case 717: #line 853 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 715: + case 718: #line 855 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = doNegate((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 716: + case 719: #line 859 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntConst((yyvsp[(1) - (1)].ival),(yylsp[(1) - (1)])); ;} break; - case 717: + case 720: #line 860 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeFloatConst((yyvsp[(1) - (1)].str),(yylsp[(1) - (1)])); ;} break; - case 718: + case 721: #line 864 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 719: + case 722: #line 865 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 720: + case 723: #line 868 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 721: + case 724: #line 869 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 722: + case 725: #line 894 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 723: + case 726: #line 896 "third_party/libpg_query/grammar/statements/select.y" { PGNode *node = (PGNode *) makeGroupingSet(GROUPING_SET_ALL, NIL, (yylsp[(3) - (3)])); @@ -26390,145 +26519,145 @@ YYLTYPE yylloc; ;} break; - case 724: + case 727: #line 900 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 725: + case 728: #line 904 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 726: + case 729: #line 905 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list),(yyvsp[(3) - (3)].node)); ;} break; - case 727: + case 730: #line 909 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 728: + case 731: #line 910 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 729: + case 732: #line 914 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 730: + case 733: #line 915 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 731: + case 734: #line 916 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 732: + case 735: #line 917 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 733: + case 736: #line 918 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 734: + case 737: #line 923 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeGroupingSet(GROUPING_SET_EMPTY, NIL, (yylsp[(1) - (2)])); ;} break; - case 735: + case 738: #line 936 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeGroupingSet(GROUPING_SET_ROLLUP, (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 736: + case 739: #line 943 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeGroupingSet(GROUPING_SET_CUBE, (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 737: + case 740: #line 950 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeGroupingSet(GROUPING_SET_SETS, (yyvsp[(4) - (5)].list), (yylsp[(1) - (5)])); ;} break; - case 738: + case 741: #line 956 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 739: + case 742: #line 957 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 740: + case 743: #line 961 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 741: + case 744: #line 962 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 742: + case 745: #line 966 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 743: + case 746: #line 967 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 744: + case 747: #line 971 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 745: + case 748: #line 972 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 746: + case 749: #line 976 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 747: + case 750: #line 977 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 748: + case 751: #line 981 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 749: + case 752: #line 982 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 750: + case 753: #line 987 "third_party/libpg_query/grammar/statements/select.y" { PGLockingClause *n = makeNode(PGLockingClause); @@ -26539,52 +26668,52 @@ YYLTYPE yylloc; ;} break; - case 751: + case 754: #line 997 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockstrength) = LCS_FORUPDATE; ;} break; - case 752: + case 755: #line 998 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockstrength) = PG_LCS_FORNOKEYUPDATE; ;} break; - case 753: + case 756: #line 999 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockstrength) = PG_LCS_FORSHARE; ;} break; - case 754: + case 757: #line 1000 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockstrength) = PG_LCS_FORKEYSHARE; ;} break; - case 755: + case 758: #line 1004 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 756: + case 759: #line 1005 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 757: + case 760: #line 1010 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockwaitpolicy) = LockWaitError; ;} break; - case 758: + case 761: #line 1011 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockwaitpolicy) = PGLockWaitSkip; ;} break; - case 759: + case 762: #line 1012 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockwaitpolicy) = PGLockWaitBlock; ;} break; - case 760: + case 763: #line 1022 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -26593,7 +26722,7 @@ YYLTYPE yylloc; ;} break; - case 761: + case 764: #line 1028 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = (PGSelectStmt *) (yyvsp[(1) - (5)].node); @@ -26602,47 +26731,47 @@ YYLTYPE yylloc; ;} break; - case 762: + case 765: #line 1036 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 763: + case 766: #line 1037 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (2)].node); ;} break; - case 764: + case 767: #line 1050 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 765: + case 768: #line 1051 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 766: + case 769: #line 1055 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 767: + case 770: #line 1056 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 768: + case 771: #line 1060 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 769: + case 772: #line 1061 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 770: + case 773: #line 1066 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = makeNode(PGAlias); @@ -26650,7 +26779,7 @@ YYLTYPE yylloc; ;} break; - case 771: + case 774: #line 1077 "third_party/libpg_query/grammar/statements/select.y" { (yyvsp[(1) - (4)].range)->at_clause = (yyvsp[(3) - (4)].node); @@ -26660,7 +26789,7 @@ YYLTYPE yylloc; ;} break; - case 772: + case 775: #line 1084 "third_party/libpg_query/grammar/statements/select.y" { (yyvsp[(2) - (4)].range)->at_clause = (yyvsp[(3) - (4)].node); @@ -26670,7 +26799,7 @@ YYLTYPE yylloc; ;} break; - case 773: + case 776: #line 1091 "third_party/libpg_query/grammar/statements/select.y" { PGRangeFunction *n = (PGRangeFunction *) (yyvsp[(1) - (3)].node); @@ -26681,7 +26810,7 @@ YYLTYPE yylloc; ;} break; - case 774: + case 777: #line 1099 "third_party/libpg_query/grammar/statements/select.y" { PGRangeFunction *n = (PGRangeFunction *) (yyvsp[(2) - (3)].node); @@ -26691,7 +26820,7 @@ YYLTYPE yylloc; ;} break; - case 775: + case 778: #line 1107 "third_party/libpg_query/grammar/statements/select.y" { PGRangeSubselect *n = makeNode(PGRangeSubselect); @@ -26703,7 +26832,7 @@ YYLTYPE yylloc; ;} break; - case 776: + case 779: #line 1117 "third_party/libpg_query/grammar/statements/select.y" { PGRangeFunction *n = (PGRangeFunction *) (yyvsp[(2) - (3)].node); @@ -26714,7 +26843,7 @@ YYLTYPE yylloc; ;} break; - case 777: + case 780: #line 1125 "third_party/libpg_query/grammar/statements/select.y" { PGRangeSubselect *n = makeNode(PGRangeSubselect); @@ -26726,7 +26855,7 @@ YYLTYPE yylloc; ;} break; - case 778: + case 781: #line 1134 "third_party/libpg_query/grammar/statements/select.y" { PGRangeSubselect *n = makeNode(PGRangeSubselect); @@ -26738,7 +26867,7 @@ YYLTYPE yylloc; ;} break; - case 779: + case 782: #line 1143 "third_party/libpg_query/grammar/statements/select.y" { PGRangeSubselect *n = makeNode(PGRangeSubselect); @@ -26750,14 +26879,14 @@ YYLTYPE yylloc; ;} break; - case 780: + case 783: #line 1152 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) (yyvsp[(1) - (1)].jexpr); ;} break; - case 781: + case 784: #line 1156 "third_party/libpg_query/grammar/statements/select.y" { (yyvsp[(2) - (4)].jexpr)->alias = (yyvsp[(4) - (4)].alias); @@ -26765,7 +26894,7 @@ YYLTYPE yylloc; ;} break; - case 782: + case 785: #line 1161 "third_party/libpg_query/grammar/statements/select.y" { (yyvsp[(3) - (4)].jexpr)->alias = (yyvsp[(1) - (4)].alias); @@ -26773,7 +26902,7 @@ YYLTYPE yylloc; ;} break; - case 783: + case 786: #line 1166 "third_party/libpg_query/grammar/statements/select.y" { PGPivotExpr *n = makeNode(PGPivotExpr); @@ -26787,7 +26916,7 @@ YYLTYPE yylloc; ;} break; - case 784: + case 787: #line 1177 "third_party/libpg_query/grammar/statements/select.y" { PGPivotExpr *n = makeNode(PGPivotExpr); @@ -26801,32 +26930,32 @@ YYLTYPE yylloc; ;} break; - case 785: + case 788: #line 1190 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 786: + case 789: #line 1191 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 787: + case 790: #line 1194 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 788: + case 791: #line 1195 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 789: + case 792: #line 1196 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 790: + case 793: #line 1200 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -26836,7 +26965,7 @@ YYLTYPE yylloc; ;} break; - case 791: + case 794: #line 1208 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -26846,22 +26975,22 @@ YYLTYPE yylloc; ;} break; - case 792: + case 795: #line 1217 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 793: + case 796: #line 1218 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 794: + case 797: #line 1219 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 795: + case 798: #line 1223 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -26871,7 +27000,7 @@ YYLTYPE yylloc; ;} break; - case 796: + case 799: #line 1231 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -26881,31 +27010,31 @@ YYLTYPE yylloc; ;} break; - case 797: + case 800: #line 1240 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 798: + case 801: #line 1244 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 799: + case 802: #line 1250 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 800: + case 803: #line 1251 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 801: + case 804: #line 1256 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -26915,28 +27044,28 @@ YYLTYPE yylloc; ;} break; - case 802: + case 805: #line 1265 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 803: + case 806: #line 1269 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 804: + case 807: #line 1294 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jexpr) = (yyvsp[(2) - (3)].jexpr); ;} break; - case 805: + case 808: #line 1298 "third_party/libpg_query/grammar/statements/select.y" { /* CROSS JOIN is same as unqualified inner join */ @@ -26952,7 +27081,7 @@ YYLTYPE yylloc; ;} break; - case 806: + case 809: #line 1311 "third_party/libpg_query/grammar/statements/select.y" { PGJoinExpr *n = makeNode(PGJoinExpr); @@ -26969,7 +27098,7 @@ YYLTYPE yylloc; ;} break; - case 807: + case 810: #line 1325 "third_party/libpg_query/grammar/statements/select.y" { /* letting join_type reduce to empty doesn't work */ @@ -26987,7 +27116,7 @@ YYLTYPE yylloc; ;} break; - case 808: + case 811: #line 1340 "third_party/libpg_query/grammar/statements/select.y" { PGJoinExpr *n = makeNode(PGJoinExpr); @@ -27002,7 +27131,7 @@ YYLTYPE yylloc; ;} break; - case 809: + case 812: #line 1352 "third_party/libpg_query/grammar/statements/select.y" { /* letting join_type reduce to empty doesn't work */ @@ -27018,7 +27147,7 @@ YYLTYPE yylloc; ;} break; - case 810: + case 813: #line 1365 "third_party/libpg_query/grammar/statements/select.y" { PGJoinExpr *n = makeNode(PGJoinExpr); @@ -27035,7 +27164,7 @@ YYLTYPE yylloc; ;} break; - case 811: + case 814: #line 1379 "third_party/libpg_query/grammar/statements/select.y" { PGJoinExpr *n = makeNode(PGJoinExpr); @@ -27052,7 +27181,7 @@ YYLTYPE yylloc; ;} break; - case 812: + case 815: #line 1393 "third_party/libpg_query/grammar/statements/select.y" { /* POSITIONAL JOIN is a coordinated scan */ @@ -27068,7 +27197,7 @@ YYLTYPE yylloc; ;} break; - case 813: + case 816: #line 1406 "third_party/libpg_query/grammar/statements/select.y" { /* ANTI JOIN is a filter */ @@ -27086,7 +27215,7 @@ YYLTYPE yylloc; ;} break; - case 814: + case 817: #line 1421 "third_party/libpg_query/grammar/statements/select.y" { /* SEMI JOIN is also a filter */ @@ -27105,7 +27234,7 @@ YYLTYPE yylloc; ;} break; - case 815: + case 818: #line 1440 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = makeNode(PGAlias); @@ -27114,7 +27243,7 @@ YYLTYPE yylloc; ;} break; - case 816: + case 819: #line 1446 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = makeNode(PGAlias); @@ -27122,7 +27251,7 @@ YYLTYPE yylloc; ;} break; - case 817: + case 820: #line 1451 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = makeNode(PGAlias); @@ -27131,7 +27260,7 @@ YYLTYPE yylloc; ;} break; - case 818: + case 821: #line 1457 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = makeNode(PGAlias); @@ -27139,31 +27268,31 @@ YYLTYPE yylloc; ;} break; - case 819: + case 822: #line 1463 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = (yyvsp[(1) - (1)].alias); ;} break; - case 820: + case 823: #line 1464 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = NULL; ;} break; - case 821: + case 824: #line 1473 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (1)].alias), NIL); ;} break; - case 822: + case 825: #line 1477 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(NULL, (yyvsp[(3) - (4)].list)); ;} break; - case 823: + case 826: #line 1481 "third_party/libpg_query/grammar/statements/select.y" { PGAlias *a = makeNode(PGAlias); @@ -27172,7 +27301,7 @@ YYLTYPE yylloc; ;} break; - case 824: + case 827: #line 1487 "third_party/libpg_query/grammar/statements/select.y" { PGAlias *a = makeNode(PGAlias); @@ -27181,64 +27310,64 @@ YYLTYPE yylloc; ;} break; - case 825: + case 828: #line 1493 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(NULL, NIL); ;} break; - case 826: + case 829: #line 1498 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_FULL; ;} break; - case 827: + case 830: #line 1499 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_LEFT; ;} break; - case 828: + case 831: #line 1500 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_RIGHT; ;} break; - case 829: + case 832: #line 1501 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_SEMI; ;} break; - case 830: + case 833: #line 1502 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_ANTI; ;} break; - case 831: + case 834: #line 1503 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_INNER; ;} break; - case 832: + case 835: #line 1507 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 833: + case 836: #line 1508 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 834: + case 837: #line 1520 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) (yyvsp[(3) - (4)].list); ;} break; - case 835: + case 838: #line 1521 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 836: + case 839: #line 1527 "third_party/libpg_query/grammar/statements/select.y" { /* inheritance query, implicitly */ @@ -27248,7 +27377,7 @@ YYLTYPE yylloc; ;} break; - case 837: + case 840: #line 1534 "third_party/libpg_query/grammar/statements/select.y" { /* inheritance query, explicitly */ @@ -27258,7 +27387,7 @@ YYLTYPE yylloc; ;} break; - case 838: + case 841: #line 1541 "third_party/libpg_query/grammar/statements/select.y" { /* no inheritance */ @@ -27268,7 +27397,7 @@ YYLTYPE yylloc; ;} break; - case 839: + case 842: #line 1548 "third_party/libpg_query/grammar/statements/select.y" { /* no inheritance, SQL99-style syntax */ @@ -27278,7 +27407,7 @@ YYLTYPE yylloc; ;} break; - case 840: + case 843: #line 1580 "third_party/libpg_query/grammar/statements/select.y" { PGRangeFunction *n = makeNode(PGRangeFunction); @@ -27292,7 +27421,7 @@ YYLTYPE yylloc; ;} break; - case 841: + case 844: #line 1591 "third_party/libpg_query/grammar/statements/select.y" { PGRangeFunction *n = makeNode(PGRangeFunction); @@ -27306,66 +27435,66 @@ YYLTYPE yylloc; ;} break; - case 842: + case 845: #line 1604 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (2)].node), (yyvsp[(2) - (2)].list)); ;} break; - case 843: + case 846: #line 1608 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} break; - case 844: + case 847: #line 1609 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; - case 845: + case 848: #line 1612 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 846: + case 849: #line 1613 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 847: + case 850: #line 1616 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 848: + case 851: #line 1617 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 849: + case 852: #line 1622 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 850: + case 853: #line 1623 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 851: + case 854: #line 1629 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 852: + case 855: #line 1633 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 853: + case 856: #line 1639 "third_party/libpg_query/grammar/statements/select.y" { PGColumnDef *n = makeNode(PGColumnDef); @@ -27386,7 +27515,7 @@ YYLTYPE yylloc; ;} break; - case 854: + case 857: #line 1660 "third_party/libpg_query/grammar/statements/select.y" { PGCollateClause *n = makeNode(PGCollateClause); @@ -27397,36 +27526,36 @@ YYLTYPE yylloc; ;} break; - case 855: + case 858: #line 1667 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 856: + case 859: #line 1681 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(list_make2(makeString((yyvsp[(1) - (2)].str)), (yyvsp[(2) - (2)].typnam))); ;} break; - case 857: + case 860: #line 1684 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (4)].list), list_make2(makeString((yyvsp[(3) - (4)].str)), (yyvsp[(4) - (4)].typnam))); ;} break; - case 860: + case 863: #line 1691 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 861: + case 864: #line 1692 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = NULL; ;} break; - case 862: + case 865: #line 1695 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (2)].typnam); @@ -27434,7 +27563,7 @@ YYLTYPE yylloc; ;} break; - case 863: + case 866: #line 1700 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(2) - (3)].typnam); @@ -27443,7 +27572,7 @@ YYLTYPE yylloc; ;} break; - case 864: + case 867: #line 1707 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (5)].typnam); @@ -27451,7 +27580,7 @@ YYLTYPE yylloc; ;} break; - case 865: + case 868: #line 1712 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(2) - (6)].typnam); @@ -27460,7 +27589,7 @@ YYLTYPE yylloc; ;} break; - case 866: + case 869: #line 1718 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (2)].typnam); @@ -27468,7 +27597,7 @@ YYLTYPE yylloc; ;} break; - case 867: + case 870: #line 1723 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(2) - (3)].typnam); @@ -27477,15 +27606,16 @@ YYLTYPE yylloc; ;} break; - case 868: + case 871: #line 1729 "third_party/libpg_query/grammar/statements/select.y" { - (yyval.typnam) = makeTypeNameFromNameList((yyvsp[(1) - (1)].list)); + (yyval.typnam) = makeTypeNameFromNameList((yyvsp[(1) - (2)].list)); + (yyval.typnam)->arrayBounds = (yyvsp[(2) - (2)].list); ;} break; - case 869: -#line 1733 "third_party/libpg_query/grammar/statements/select.y" + case 872: +#line 1734 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("struct"); (yyval.typnam)->arrayBounds = (yyvsp[(5) - (5)].list); @@ -27494,8 +27624,8 @@ YYLTYPE yylloc; ;} break; - case 870: -#line 1740 "third_party/libpg_query/grammar/statements/select.y" + case 873: +#line 1741 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("map"); (yyval.typnam)->arrayBounds = (yyvsp[(5) - (5)].list); @@ -27504,8 +27634,8 @@ YYLTYPE yylloc; ;} break; - case 871: -#line 1747 "third_party/libpg_query/grammar/statements/select.y" + case 874: +#line 1748 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("union"); (yyval.typnam)->arrayBounds = (yyvsp[(5) - (5)].list); @@ -27514,66 +27644,66 @@ YYLTYPE yylloc; ;} break; - case 872: -#line 1756 "third_party/libpg_query/grammar/statements/select.y" + case 875: +#line 1757 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(makeString((yyvsp[(1) - (3)].str)), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 873: -#line 1757 "third_party/libpg_query/grammar/statements/select.y" + case 876: +#line 1758 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 874: -#line 1762 "third_party/libpg_query/grammar/statements/select.y" + case 877: +#line 1763 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), makeInteger(-1)); ;} break; - case 875: -#line 1764 "third_party/libpg_query/grammar/statements/select.y" + case 878: +#line 1765 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (4)].list), makeInteger((yyvsp[(3) - (4)].ival))); ;} break; - case 876: -#line 1766 "third_party/libpg_query/grammar/statements/select.y" + case 879: +#line 1767 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 877: -#line 1770 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} - break; - - case 878: + case 880: #line 1771 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 879: + case 881: #line 1772 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 880: + case 882: #line 1773 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 881: + case 883: #line 1774 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 882: -#line 1776 "third_party/libpg_query/grammar/statements/select.y" + case 884: +#line 1775 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} + break; + + case 885: +#line 1777 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (2)].typnam); (yyval.typnam)->typmods = (yyvsp[(2) - (2)].list); ;} break; - case 883: -#line 1781 "third_party/libpg_query/grammar/statements/select.y" + case 886: +#line 1782 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (4)].typnam); (yyval.typnam)->typmods = list_make2(makeIntConst(INTERVAL_FULL_RANGE, -1), @@ -27581,28 +27711,28 @@ YYLTYPE yylloc; ;} break; - case 884: -#line 1800 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} - break; - - case 885: + case 887: #line 1801 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 886: + case 888: #line 1802 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 887: + case 889: #line 1803 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 888: -#line 1815 "third_party/libpg_query/grammar/statements/select.y" + case 890: +#line 1804 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} + break; + + case 891: +#line 1816 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = makeTypeName((yyvsp[(1) - (2)].str)); (yyval.typnam)->typmods = (yyvsp[(2) - (2)].list); @@ -27610,74 +27740,74 @@ YYLTYPE yylloc; ;} break; - case 889: -#line 1828 "third_party/libpg_query/grammar/statements/select.y" + case 892: +#line 1829 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 890: -#line 1829 "third_party/libpg_query/grammar/statements/select.y" + case 893: +#line 1830 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 891: -#line 1836 "third_party/libpg_query/grammar/statements/select.y" + case 894: +#line 1837 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("int4"); (yyval.typnam)->location = (yylsp[(1) - (1)]); ;} break; - case 892: -#line 1841 "third_party/libpg_query/grammar/statements/select.y" + case 895: +#line 1842 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("int4"); (yyval.typnam)->location = (yylsp[(1) - (1)]); ;} break; - case 893: -#line 1846 "third_party/libpg_query/grammar/statements/select.y" + case 896: +#line 1847 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("int2"); (yyval.typnam)->location = (yylsp[(1) - (1)]); ;} break; - case 894: -#line 1851 "third_party/libpg_query/grammar/statements/select.y" + case 897: +#line 1852 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("int8"); (yyval.typnam)->location = (yylsp[(1) - (1)]); ;} break; - case 895: -#line 1856 "third_party/libpg_query/grammar/statements/select.y" + case 898: +#line 1857 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("float4"); (yyval.typnam)->location = (yylsp[(1) - (1)]); ;} break; - case 896: -#line 1861 "third_party/libpg_query/grammar/statements/select.y" + case 899: +#line 1862 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(2) - (2)].typnam); (yyval.typnam)->location = (yylsp[(1) - (2)]); ;} break; - case 897: -#line 1866 "third_party/libpg_query/grammar/statements/select.y" + case 900: +#line 1867 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("float8"); (yyval.typnam)->location = (yylsp[(1) - (2)]); ;} break; - case 898: -#line 1871 "third_party/libpg_query/grammar/statements/select.y" + case 901: +#line 1872 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("numeric"); (yyval.typnam)->typmods = (yyvsp[(2) - (2)].list); @@ -27685,8 +27815,8 @@ YYLTYPE yylloc; ;} break; - case 899: -#line 1877 "third_party/libpg_query/grammar/statements/select.y" + case 902: +#line 1878 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("numeric"); (yyval.typnam)->typmods = (yyvsp[(2) - (2)].list); @@ -27694,8 +27824,8 @@ YYLTYPE yylloc; ;} break; - case 900: -#line 1883 "third_party/libpg_query/grammar/statements/select.y" + case 903: +#line 1884 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("numeric"); (yyval.typnam)->typmods = (yyvsp[(2) - (2)].list); @@ -27703,16 +27833,16 @@ YYLTYPE yylloc; ;} break; - case 901: -#line 1889 "third_party/libpg_query/grammar/statements/select.y" + case 904: +#line 1890 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("bool"); (yyval.typnam)->location = (yylsp[(1) - (1)]); ;} break; - case 902: -#line 1896 "third_party/libpg_query/grammar/statements/select.y" + case 905: +#line 1897 "third_party/libpg_query/grammar/statements/select.y" { /* * Check FLOAT() precision limits assuming IEEE floating @@ -27735,44 +27865,44 @@ YYLTYPE yylloc; ;} break; - case 903: -#line 1917 "third_party/libpg_query/grammar/statements/select.y" + case 906: +#line 1918 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("float4"); ;} break; - case 904: -#line 1927 "third_party/libpg_query/grammar/statements/select.y" + case 907: +#line 1928 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 905: -#line 1931 "third_party/libpg_query/grammar/statements/select.y" + case 908: +#line 1932 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 906: -#line 1939 "third_party/libpg_query/grammar/statements/select.y" + case 909: +#line 1940 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 907: -#line 1943 "third_party/libpg_query/grammar/statements/select.y" + case 910: +#line 1944 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); (yyval.typnam)->typmods = NIL; ;} break; - case 908: -#line 1951 "third_party/libpg_query/grammar/statements/select.y" + case 911: +#line 1952 "third_party/libpg_query/grammar/statements/select.y" { const char *typname; @@ -27783,8 +27913,8 @@ YYLTYPE yylloc; ;} break; - case 909: -#line 1963 "third_party/libpg_query/grammar/statements/select.y" + case 912: +#line 1964 "third_party/libpg_query/grammar/statements/select.y" { /* bit defaults to bit(1), varbit to no limit */ if ((yyvsp[(2) - (2)].boolean)) @@ -27800,29 +27930,29 @@ YYLTYPE yylloc; ;} break; - case 910: -#line 1984 "third_party/libpg_query/grammar/statements/select.y" + case 913: +#line 1985 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 911: -#line 1988 "third_party/libpg_query/grammar/statements/select.y" + case 914: +#line 1989 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 912: -#line 1994 "third_party/libpg_query/grammar/statements/select.y" + case 915: +#line 1995 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 913: -#line 1998 "third_party/libpg_query/grammar/statements/select.y" + case 916: +#line 1999 "third_party/libpg_query/grammar/statements/select.y" { /* Length was not specified so allow to be unrestricted. * This handles problems with fixed-length (bpchar) strings @@ -27835,8 +27965,8 @@ YYLTYPE yylloc; ;} break; - case 914: -#line 2011 "third_party/libpg_query/grammar/statements/select.y" + case 917: +#line 2012 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName((yyvsp[(1) - (4)].conststr)); (yyval.typnam)->typmods = list_make1(makeIntConst((yyvsp[(3) - (4)].ival), (yylsp[(3) - (4)]))); @@ -27844,8 +27974,8 @@ YYLTYPE yylloc; ;} break; - case 915: -#line 2019 "third_party/libpg_query/grammar/statements/select.y" + case 918: +#line 2020 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName((yyvsp[(1) - (1)].conststr)); /* char defaults to char(1), varchar to no limit */ @@ -27855,48 +27985,48 @@ YYLTYPE yylloc; ;} break; - case 916: -#line 2029 "third_party/libpg_query/grammar/statements/select.y" + case 919: +#line 2030 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(2) - (2)].boolean) ? "varchar": "bpchar"; ;} break; - case 917: -#line 2031 "third_party/libpg_query/grammar/statements/select.y" + case 920: +#line 2032 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(2) - (2)].boolean) ? "varchar": "bpchar"; ;} break; - case 918: -#line 2033 "third_party/libpg_query/grammar/statements/select.y" + case 921: +#line 2034 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "varchar"; ;} break; - case 919: -#line 2035 "third_party/libpg_query/grammar/statements/select.y" + case 922: +#line 2036 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(3) - (3)].boolean) ? "varchar": "bpchar"; ;} break; - case 920: -#line 2037 "third_party/libpg_query/grammar/statements/select.y" + case 923: +#line 2038 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(3) - (3)].boolean) ? "varchar": "bpchar"; ;} break; - case 921: -#line 2039 "third_party/libpg_query/grammar/statements/select.y" + case 924: +#line 2040 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(2) - (2)].boolean) ? "varchar": "bpchar"; ;} break; - case 922: -#line 2043 "third_party/libpg_query/grammar/statements/select.y" + case 925: +#line 2044 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 923: -#line 2044 "third_party/libpg_query/grammar/statements/select.y" + case 926: +#line 2045 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 924: -#line 2052 "third_party/libpg_query/grammar/statements/select.y" + case 927: +#line 2053 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(5) - (5)].boolean)) (yyval.typnam) = SystemTypeName("timestamptz"); @@ -27907,8 +28037,8 @@ YYLTYPE yylloc; ;} break; - case 925: -#line 2061 "third_party/libpg_query/grammar/statements/select.y" + case 928: +#line 2062 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(2) - (2)].boolean)) (yyval.typnam) = SystemTypeName("timestamptz"); @@ -27918,8 +28048,8 @@ YYLTYPE yylloc; ;} break; - case 926: -#line 2069 "third_party/libpg_query/grammar/statements/select.y" + case 929: +#line 2070 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(5) - (5)].boolean)) (yyval.typnam) = SystemTypeName("timetz"); @@ -27930,8 +28060,8 @@ YYLTYPE yylloc; ;} break; - case 927: -#line 2078 "third_party/libpg_query/grammar/statements/select.y" + case 930: +#line 2079 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(2) - (2)].boolean)) (yyval.typnam) = SystemTypeName("timetz"); @@ -27941,112 +28071,112 @@ YYLTYPE yylloc; ;} break; - case 928: -#line 2089 "third_party/libpg_query/grammar/statements/select.y" + case 931: +#line 2090 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("interval"); (yyval.typnam)->location = (yylsp[(1) - (1)]); ;} break; - case 929: -#line 2096 "third_party/libpg_query/grammar/statements/select.y" + case 932: +#line 2097 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 930: -#line 2097 "third_party/libpg_query/grammar/statements/select.y" + case 933: +#line 2098 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 931: -#line 2098 "third_party/libpg_query/grammar/statements/select.y" + case 934: +#line 2099 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 958: -#line 2142 "third_party/libpg_query/grammar/statements/select.y" + case 961: +#line 2143 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(YEAR), (yylsp[(1) - (1)]))); ;} break; - case 959: -#line 2144 "third_party/libpg_query/grammar/statements/select.y" + case 962: +#line 2145 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MONTH), (yylsp[(1) - (1)]))); ;} break; - case 960: -#line 2146 "third_party/libpg_query/grammar/statements/select.y" + case 963: +#line 2147 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DAY), (yylsp[(1) - (1)]))); ;} break; - case 961: -#line 2148 "third_party/libpg_query/grammar/statements/select.y" + case 964: +#line 2149 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(HOUR), (yylsp[(1) - (1)]))); ;} break; - case 962: -#line 2150 "third_party/libpg_query/grammar/statements/select.y" + case 965: +#line 2151 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MINUTE), (yylsp[(1) - (1)]))); ;} break; - case 963: -#line 2152 "third_party/libpg_query/grammar/statements/select.y" + case 966: +#line 2153 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(SECOND), (yylsp[(1) - (1)]))); ;} break; - case 964: -#line 2154 "third_party/libpg_query/grammar/statements/select.y" + case 967: +#line 2155 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MILLISECOND), (yylsp[(1) - (1)]))); ;} break; - case 965: -#line 2156 "third_party/libpg_query/grammar/statements/select.y" + case 968: +#line 2157 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MICROSECOND), (yylsp[(1) - (1)]))); ;} break; - case 966: -#line 2158 "third_party/libpg_query/grammar/statements/select.y" + case 969: +#line 2159 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(WEEK), (yylsp[(1) - (1)]))); ;} break; - case 967: -#line 2160 "third_party/libpg_query/grammar/statements/select.y" + case 970: +#line 2161 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(QUARTER), (yylsp[(1) - (1)]))); ;} break; - case 968: -#line 2162 "third_party/libpg_query/grammar/statements/select.y" + case 971: +#line 2163 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DECADE), (yylsp[(1) - (1)]))); ;} break; - case 969: -#line 2164 "third_party/libpg_query/grammar/statements/select.y" + case 972: +#line 2165 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(CENTURY), (yylsp[(1) - (1)]))); ;} break; - case 970: -#line 2166 "third_party/libpg_query/grammar/statements/select.y" + case 973: +#line 2167 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MILLENNIUM), (yylsp[(1) - (1)]))); ;} break; - case 971: -#line 2168 "third_party/libpg_query/grammar/statements/select.y" + case 974: +#line 2169 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(YEAR) | INTERVAL_MASK(MONTH), (yylsp[(1) - (3)]))); ;} break; - case 972: -#line 2173 "third_party/libpg_query/grammar/statements/select.y" + case 975: +#line 2174 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DAY) | INTERVAL_MASK(HOUR), (yylsp[(1) - (3)]))); ;} break; - case 973: -#line 2178 "third_party/libpg_query/grammar/statements/select.y" + case 976: +#line 2179 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DAY) | INTERVAL_MASK(HOUR) | @@ -28054,8 +28184,8 @@ YYLTYPE yylloc; ;} break; - case 974: -#line 2184 "third_party/libpg_query/grammar/statements/select.y" + case 977: +#line 2185 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DAY) | INTERVAL_MASK(HOUR) | @@ -28064,16 +28194,16 @@ YYLTYPE yylloc; ;} break; - case 975: -#line 2191 "third_party/libpg_query/grammar/statements/select.y" + case 978: +#line 2192 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(HOUR) | INTERVAL_MASK(MINUTE), (yylsp[(1) - (3)]))); ;} break; - case 976: -#line 2196 "third_party/libpg_query/grammar/statements/select.y" + case 979: +#line 2197 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(HOUR) | INTERVAL_MASK(MINUTE) | @@ -28081,31 +28211,31 @@ YYLTYPE yylloc; ;} break; - case 977: -#line 2202 "third_party/libpg_query/grammar/statements/select.y" + case 980: +#line 2203 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MINUTE) | INTERVAL_MASK(SECOND), (yylsp[(1) - (3)]))); ;} break; - case 978: -#line 2207 "third_party/libpg_query/grammar/statements/select.y" + case 981: +#line 2208 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 979: -#line 2238 "third_party/libpg_query/grammar/statements/select.y" + case 982: +#line 2239 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 980: -#line 2241 "third_party/libpg_query/grammar/statements/select.y" + case 983: +#line 2242 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeTypeCast((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].typnam), 0, (yylsp[(2) - (3)])); ;} break; - case 981: -#line 2243 "third_party/libpg_query/grammar/statements/select.y" + case 984: +#line 2244 "third_party/libpg_query/grammar/statements/select.y" { PGCollateClause *n = makeNode(PGCollateClause); n->arg = (yyvsp[(1) - (3)].node); @@ -28115,8 +28245,8 @@ YYLTYPE yylloc; ;} break; - case 982: -#line 2251 "third_party/libpg_query/grammar/statements/select.y" + case 985: +#line 2252 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("timezone"), list_make2((yyvsp[(5) - (5)].node), (yyvsp[(1) - (5)].node)), @@ -28124,139 +28254,139 @@ YYLTYPE yylloc; ;} break; - case 983: -#line 2266 "third_party/libpg_query/grammar/statements/select.y" + case 986: +#line 2267 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 984: -#line 2268 "third_party/libpg_query/grammar/statements/select.y" + case 987: +#line 2269 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = doNegate((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 985: -#line 2270 "third_party/libpg_query/grammar/statements/select.y" + case 988: +#line 2271 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 986: -#line 2272 "third_party/libpg_query/grammar/statements/select.y" + case 989: +#line 2273 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "-", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 987: -#line 2274 "third_party/libpg_query/grammar/statements/select.y" + case 990: +#line 2275 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "*", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 988: -#line 2276 "third_party/libpg_query/grammar/statements/select.y" + case 991: +#line 2277 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "/", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 989: -#line 2278 "third_party/libpg_query/grammar/statements/select.y" + case 992: +#line 2279 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "//", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 990: -#line 2280 "third_party/libpg_query/grammar/statements/select.y" + case 993: +#line 2281 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "%", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 991: -#line 2282 "third_party/libpg_query/grammar/statements/select.y" + case 994: +#line 2283 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "^", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 992: -#line 2284 "third_party/libpg_query/grammar/statements/select.y" + case 995: +#line 2285 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "**", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 993: -#line 2286 "third_party/libpg_query/grammar/statements/select.y" + case 996: +#line 2287 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 994: -#line 2288 "third_party/libpg_query/grammar/statements/select.y" + case 997: +#line 2289 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, ">", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 995: -#line 2290 "third_party/libpg_query/grammar/statements/select.y" + case 998: +#line 2291 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 996: -#line 2292 "third_party/libpg_query/grammar/statements/select.y" + case 999: +#line 2293 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 997: -#line 2294 "third_party/libpg_query/grammar/statements/select.y" + case 1000: +#line 2295 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, ">=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 998: -#line 2296 "third_party/libpg_query/grammar/statements/select.y" + case 1001: +#line 2297 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<>", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 999: -#line 2299 "third_party/libpg_query/grammar/statements/select.y" + case 1002: +#line 2300 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(2) - (3)].list), (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1000: -#line 2301 "third_party/libpg_query/grammar/statements/select.y" + case 1003: +#line 2302 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(1) - (2)].list), NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 1001: -#line 2303 "third_party/libpg_query/grammar/statements/select.y" + case 1004: +#line 2304 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(2) - (2)].list), (yyvsp[(1) - (2)].node), NULL, (yylsp[(2) - (2)])); ;} break; - case 1002: -#line 2306 "third_party/libpg_query/grammar/statements/select.y" + case 1005: +#line 2307 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeAndExpr((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1003: -#line 2308 "third_party/libpg_query/grammar/statements/select.y" + case 1006: +#line 2309 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeOrExpr((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1004: -#line 2310 "third_party/libpg_query/grammar/statements/select.y" + case 1007: +#line 2311 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeNotExpr((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 1005: -#line 2312 "third_party/libpg_query/grammar/statements/select.y" + case 1008: +#line 2313 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeNotExpr((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 1006: -#line 2314 "third_party/libpg_query/grammar/statements/select.y" + case 1009: +#line 2315 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_GLOB, "~~~", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1007: -#line 2319 "third_party/libpg_query/grammar/statements/select.y" + case 1010: +#line 2320 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_LIKE, "~~", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1008: -#line 2324 "third_party/libpg_query/grammar/statements/select.y" + case 1011: +#line 2325 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("like_escape"), list_make3((yyvsp[(1) - (5)].node), (yyvsp[(3) - (5)].node), (yyvsp[(5) - (5)].node)), @@ -28265,16 +28395,16 @@ YYLTYPE yylloc; ;} break; - case 1009: -#line 2331 "third_party/libpg_query/grammar/statements/select.y" + case 1012: +#line 2332 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_LIKE, "!~~", (yyvsp[(1) - (4)].node), (yyvsp[(4) - (4)].node), (yylsp[(2) - (4)])); ;} break; - case 1010: -#line 2336 "third_party/libpg_query/grammar/statements/select.y" + case 1013: +#line 2337 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("not_like_escape"), list_make3((yyvsp[(1) - (6)].node), (yyvsp[(4) - (6)].node), (yyvsp[(6) - (6)].node)), @@ -28283,16 +28413,16 @@ YYLTYPE yylloc; ;} break; - case 1011: -#line 2343 "third_party/libpg_query/grammar/statements/select.y" + case 1014: +#line 2344 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_ILIKE, "~~*", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1012: -#line 2348 "third_party/libpg_query/grammar/statements/select.y" + case 1015: +#line 2349 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("ilike_escape"), list_make3((yyvsp[(1) - (5)].node), (yyvsp[(3) - (5)].node), (yyvsp[(5) - (5)].node)), @@ -28301,16 +28431,16 @@ YYLTYPE yylloc; ;} break; - case 1013: -#line 2355 "third_party/libpg_query/grammar/statements/select.y" + case 1016: +#line 2356 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_ILIKE, "!~~*", (yyvsp[(1) - (4)].node), (yyvsp[(4) - (4)].node), (yylsp[(2) - (4)])); ;} break; - case 1014: -#line 2360 "third_party/libpg_query/grammar/statements/select.y" + case 1017: +#line 2361 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("not_ilike_escape"), list_make3((yyvsp[(1) - (6)].node), (yyvsp[(4) - (6)].node), (yyvsp[(6) - (6)].node)), @@ -28319,8 +28449,8 @@ YYLTYPE yylloc; ;} break; - case 1015: -#line 2368 "third_party/libpg_query/grammar/statements/select.y" + case 1018: +#line 2369 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("similar_escape"), list_make2((yyvsp[(4) - (4)].node), makeNullAConst(-1)), @@ -28330,8 +28460,8 @@ YYLTYPE yylloc; ;} break; - case 1016: -#line 2376 "third_party/libpg_query/grammar/statements/select.y" + case 1019: +#line 2377 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("similar_escape"), list_make2((yyvsp[(4) - (6)].node), (yyvsp[(6) - (6)].node)), @@ -28341,8 +28471,8 @@ YYLTYPE yylloc; ;} break; - case 1017: -#line 2384 "third_party/libpg_query/grammar/statements/select.y" + case 1020: +#line 2385 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("similar_escape"), list_make2((yyvsp[(5) - (5)].node), makeNullAConst(-1)), @@ -28352,8 +28482,8 @@ YYLTYPE yylloc; ;} break; - case 1018: -#line 2392 "third_party/libpg_query/grammar/statements/select.y" + case 1021: +#line 2393 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("similar_escape"), list_make2((yyvsp[(5) - (7)].node), (yyvsp[(7) - (7)].node)), @@ -28363,8 +28493,8 @@ YYLTYPE yylloc; ;} break; - case 1019: -#line 2410 "third_party/libpg_query/grammar/statements/select.y" + case 1022: +#line 2411 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); n->arg = (PGExpr *) (yyvsp[(1) - (3)].node); @@ -28374,8 +28504,8 @@ YYLTYPE yylloc; ;} break; - case 1020: -#line 2418 "third_party/libpg_query/grammar/statements/select.y" + case 1023: +#line 2419 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); n->arg = (PGExpr *) (yyvsp[(1) - (2)].node); @@ -28385,8 +28515,8 @@ YYLTYPE yylloc; ;} break; - case 1021: -#line 2426 "third_party/libpg_query/grammar/statements/select.y" + case 1024: +#line 2427 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); n->arg = (PGExpr *) (yyvsp[(1) - (4)].node); @@ -28396,8 +28526,8 @@ YYLTYPE yylloc; ;} break; - case 1022: -#line 2434 "third_party/libpg_query/grammar/statements/select.y" + case 1025: +#line 2435 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); n->arg = (PGExpr *) (yyvsp[(1) - (3)].node); @@ -28407,8 +28537,8 @@ YYLTYPE yylloc; ;} break; - case 1023: -#line 2442 "third_party/libpg_query/grammar/statements/select.y" + case 1026: +#line 2443 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); n->arg = (PGExpr *) (yyvsp[(1) - (2)].node); @@ -28418,8 +28548,8 @@ YYLTYPE yylloc; ;} break; - case 1024: -#line 2450 "third_party/libpg_query/grammar/statements/select.y" + case 1027: +#line 2451 "third_party/libpg_query/grammar/statements/select.y" { PGLambdaFunction *n = makeNode(PGLambdaFunction); n->lhs = (yyvsp[(2) - (4)].list); @@ -28429,8 +28559,8 @@ YYLTYPE yylloc; ;} break; - case 1025: -#line 2458 "third_party/libpg_query/grammar/statements/select.y" + case 1028: +#line 2459 "third_party/libpg_query/grammar/statements/select.y" { PGSingleArrowFunction *n = makeNode(PGSingleArrowFunction); n->lhs = (yyvsp[(1) - (3)].node); @@ -28440,15 +28570,15 @@ YYLTYPE yylloc; ;} break; - case 1026: -#line 2466 "third_party/libpg_query/grammar/statements/select.y" + case 1029: +#line 2467 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "->>", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1027: -#line 2470 "third_party/libpg_query/grammar/statements/select.y" + case 1030: +#line 2471 "third_party/libpg_query/grammar/statements/select.y" { if (list_length((yyvsp[(1) - (3)].list)) != 2) ereport(ERROR, @@ -28466,8 +28596,8 @@ YYLTYPE yylloc; ;} break; - case 1028: -#line 2486 "third_party/libpg_query/grammar/statements/select.y" + case 1031: +#line 2487 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); b->arg = (PGExpr *) (yyvsp[(1) - (3)].node); @@ -28477,8 +28607,8 @@ YYLTYPE yylloc; ;} break; - case 1029: -#line 2494 "third_party/libpg_query/grammar/statements/select.y" + case 1032: +#line 2495 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); b->arg = (PGExpr *) (yyvsp[(1) - (4)].node); @@ -28488,8 +28618,8 @@ YYLTYPE yylloc; ;} break; - case 1030: -#line 2502 "third_party/libpg_query/grammar/statements/select.y" + case 1033: +#line 2503 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); b->arg = (PGExpr *) (yyvsp[(1) - (3)].node); @@ -28499,8 +28629,8 @@ YYLTYPE yylloc; ;} break; - case 1031: -#line 2510 "third_party/libpg_query/grammar/statements/select.y" + case 1034: +#line 2511 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); b->arg = (PGExpr *) (yyvsp[(1) - (4)].node); @@ -28510,8 +28640,8 @@ YYLTYPE yylloc; ;} break; - case 1032: -#line 2518 "third_party/libpg_query/grammar/statements/select.y" + case 1035: +#line 2519 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); b->arg = (PGExpr *) (yyvsp[(1) - (3)].node); @@ -28521,8 +28651,8 @@ YYLTYPE yylloc; ;} break; - case 1033: -#line 2526 "third_party/libpg_query/grammar/statements/select.y" + case 1036: +#line 2527 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); b->arg = (PGExpr *) (yyvsp[(1) - (4)].node); @@ -28532,36 +28662,36 @@ YYLTYPE yylloc; ;} break; - case 1034: -#line 2534 "third_party/libpg_query/grammar/statements/select.y" + case 1037: +#line 2535 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_DISTINCT, "=", (yyvsp[(1) - (5)].node), (yyvsp[(5) - (5)].node), (yylsp[(2) - (5)])); ;} break; - case 1035: -#line 2538 "third_party/libpg_query/grammar/statements/select.y" + case 1038: +#line 2539 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NOT_DISTINCT, "=", (yyvsp[(1) - (6)].node), (yyvsp[(6) - (6)].node), (yylsp[(2) - (6)])); ;} break; - case 1036: -#line 2542 "third_party/libpg_query/grammar/statements/select.y" + case 1039: +#line 2543 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OF, "=", (yyvsp[(1) - (6)].node), (PGNode *) (yyvsp[(5) - (6)].list), (yylsp[(2) - (6)])); ;} break; - case 1037: -#line 2546 "third_party/libpg_query/grammar/statements/select.y" + case 1040: +#line 2547 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OF, "<>", (yyvsp[(1) - (7)].node), (PGNode *) (yyvsp[(6) - (7)].list), (yylsp[(2) - (7)])); ;} break; - case 1038: -#line 2550 "third_party/libpg_query/grammar/statements/select.y" + case 1041: +#line 2551 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_BETWEEN, "BETWEEN", @@ -28571,8 +28701,8 @@ YYLTYPE yylloc; ;} break; - case 1039: -#line 2558 "third_party/libpg_query/grammar/statements/select.y" + case 1042: +#line 2559 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NOT_BETWEEN, "NOT BETWEEN", @@ -28582,8 +28712,8 @@ YYLTYPE yylloc; ;} break; - case 1040: -#line 2566 "third_party/libpg_query/grammar/statements/select.y" + case 1043: +#line 2567 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_BETWEEN_SYM, "BETWEEN SYMMETRIC", @@ -28593,8 +28723,8 @@ YYLTYPE yylloc; ;} break; - case 1041: -#line 2574 "third_party/libpg_query/grammar/statements/select.y" + case 1044: +#line 2575 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NOT_BETWEEN_SYM, "NOT BETWEEN SYMMETRIC", @@ -28604,8 +28734,8 @@ YYLTYPE yylloc; ;} break; - case 1042: -#line 2582 "third_party/libpg_query/grammar/statements/select.y" + case 1045: +#line 2583 "third_party/libpg_query/grammar/statements/select.y" { /* in_expr returns a PGSubLink or a list of a_exprs */ if (IsA((yyvsp[(3) - (3)].node), PGSubLink)) @@ -28627,8 +28757,8 @@ YYLTYPE yylloc; ;} break; - case 1043: -#line 2602 "third_party/libpg_query/grammar/statements/select.y" + case 1046: +#line 2603 "third_party/libpg_query/grammar/statements/select.y" { /* in_expr returns a PGSubLink or a list of a_exprs */ if (IsA((yyvsp[(4) - (4)].node), PGSubLink)) @@ -28652,8 +28782,8 @@ YYLTYPE yylloc; ;} break; - case 1044: -#line 2624 "third_party/libpg_query/grammar/statements/select.y" + case 1047: +#line 2625 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); n->subLinkType = (yyvsp[(3) - (4)].subquerytype); @@ -28666,8 +28796,8 @@ YYLTYPE yylloc; ;} break; - case 1045: -#line 2635 "third_party/libpg_query/grammar/statements/select.y" + case 1048: +#line 2636 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(3) - (6)].subquerytype) == PG_ANY_SUBLINK) (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP_ANY, (yyvsp[(2) - (6)].list), (yyvsp[(1) - (6)].node), (yyvsp[(5) - (6)].node), (yylsp[(2) - (6)])); @@ -28676,8 +28806,8 @@ YYLTYPE yylloc; ;} break; - case 1046: -#line 2642 "third_party/libpg_query/grammar/statements/select.y" + case 1049: +#line 2643 "third_party/libpg_query/grammar/statements/select.y" { /* * The SQL spec only allows DEFAULT in "contextually typed @@ -28693,8 +28823,8 @@ YYLTYPE yylloc; ;} break; - case 1047: -#line 2656 "third_party/libpg_query/grammar/statements/select.y" + case 1050: +#line 2657 "third_party/libpg_query/grammar/statements/select.y" { PGAStar *star = makeNode(PGAStar); star->expr = (yyvsp[(3) - (4)].node); @@ -28704,16 +28834,16 @@ YYLTYPE yylloc; ;} break; - case 1048: -#line 2664 "third_party/libpg_query/grammar/statements/select.y" + case 1051: +#line 2665 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("unpack"), list_make1((yyvsp[(3) - (4)].node)), (yylsp[(1) - (4)])); (yyval.node) = (PGNode *) n; ;} break; - case 1049: -#line 2669 "third_party/libpg_query/grammar/statements/select.y" + case 1052: +#line 2670 "third_party/libpg_query/grammar/statements/select.y" { PGAStar *star = makeNode(PGAStar); star->expr = (yyvsp[(4) - (5)].node); @@ -28725,8 +28855,8 @@ YYLTYPE yylloc; ;} break; - case 1050: -#line 2679 "third_party/libpg_query/grammar/statements/select.y" + case 1053: +#line 2680 "third_party/libpg_query/grammar/statements/select.y" { PGAStar *star = makeNode(PGAStar); star->except_list = (yyvsp[(2) - (4)].list); @@ -28737,8 +28867,8 @@ YYLTYPE yylloc; ;} break; - case 1051: -#line 2688 "third_party/libpg_query/grammar/statements/select.y" + case 1054: +#line 2689 "third_party/libpg_query/grammar/statements/select.y" { PGAStar *star = makeNode(PGAStar); star->relation = (yyvsp[(1) - (6)].str); @@ -28750,141 +28880,141 @@ YYLTYPE yylloc; ;} break; - case 1052: -#line 2709 "third_party/libpg_query/grammar/statements/select.y" + case 1055: +#line 2710 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1053: -#line 2711 "third_party/libpg_query/grammar/statements/select.y" + case 1056: +#line 2712 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeTypeCast((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].typnam), 0, (yylsp[(2) - (3)])); ;} break; - case 1054: -#line 2713 "third_party/libpg_query/grammar/statements/select.y" + case 1057: +#line 2714 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 1055: -#line 2715 "third_party/libpg_query/grammar/statements/select.y" + case 1058: +#line 2716 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = doNegate((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 1056: -#line 2717 "third_party/libpg_query/grammar/statements/select.y" + case 1059: +#line 2718 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1057: -#line 2719 "third_party/libpg_query/grammar/statements/select.y" + case 1060: +#line 2720 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "-", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1058: -#line 2721 "third_party/libpg_query/grammar/statements/select.y" + case 1061: +#line 2722 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "*", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1059: -#line 2723 "third_party/libpg_query/grammar/statements/select.y" + case 1062: +#line 2724 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "/", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1060: -#line 2725 "third_party/libpg_query/grammar/statements/select.y" + case 1063: +#line 2726 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "//", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1061: -#line 2727 "third_party/libpg_query/grammar/statements/select.y" + case 1064: +#line 2728 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "%", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1062: -#line 2729 "third_party/libpg_query/grammar/statements/select.y" + case 1065: +#line 2730 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "^", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1063: -#line 2731 "third_party/libpg_query/grammar/statements/select.y" + case 1066: +#line 2732 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "**", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1064: -#line 2733 "third_party/libpg_query/grammar/statements/select.y" + case 1067: +#line 2734 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1065: -#line 2735 "third_party/libpg_query/grammar/statements/select.y" + case 1068: +#line 2736 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, ">", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1066: -#line 2737 "third_party/libpg_query/grammar/statements/select.y" + case 1069: +#line 2738 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1067: -#line 2739 "third_party/libpg_query/grammar/statements/select.y" + case 1070: +#line 2740 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1068: -#line 2741 "third_party/libpg_query/grammar/statements/select.y" + case 1071: +#line 2742 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, ">=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1069: -#line 2743 "third_party/libpg_query/grammar/statements/select.y" + case 1072: +#line 2744 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<>", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1070: -#line 2745 "third_party/libpg_query/grammar/statements/select.y" + case 1073: +#line 2746 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(2) - (3)].list), (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 1071: -#line 2747 "third_party/libpg_query/grammar/statements/select.y" + case 1074: +#line 2748 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(1) - (2)].list), NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 1072: -#line 2749 "third_party/libpg_query/grammar/statements/select.y" + case 1075: +#line 2750 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(2) - (2)].list), (yyvsp[(1) - (2)].node), NULL, (yylsp[(2) - (2)])); ;} break; - case 1073: -#line 2751 "third_party/libpg_query/grammar/statements/select.y" + case 1076: +#line 2752 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_DISTINCT, "=", (yyvsp[(1) - (5)].node), (yyvsp[(5) - (5)].node), (yylsp[(2) - (5)])); ;} break; - case 1074: -#line 2755 "third_party/libpg_query/grammar/statements/select.y" + case 1077: +#line 2756 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NOT_DISTINCT, "=", (yyvsp[(1) - (6)].node), (yyvsp[(6) - (6)].node), (yylsp[(2) - (6)])); ;} break; - case 1075: -#line 2759 "third_party/libpg_query/grammar/statements/select.y" + case 1078: +#line 2760 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OF, "=", (yyvsp[(1) - (6)].node), (PGNode *) (yyvsp[(5) - (6)].list), (yylsp[(2) - (6)])); ;} break; - case 1076: -#line 2763 "third_party/libpg_query/grammar/statements/select.y" + case 1079: +#line 2764 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OF, "<>", (yyvsp[(1) - (7)].node), (PGNode *) (yyvsp[(6) - (7)].list), (yylsp[(2) - (7)])); ;} break; - case 1078: -#line 2778 "third_party/libpg_query/grammar/statements/select.y" + case 1081: +#line 2779 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(2) - (2)].list)) { @@ -28898,18 +29028,18 @@ YYLTYPE yylloc; ;} break; - case 1079: -#line 2791 "third_party/libpg_query/grammar/statements/select.y" + case 1082: +#line 2792 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1080: -#line 2792 "third_party/libpg_query/grammar/statements/select.y" + case 1083: +#line 2793 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1081: -#line 2794 "third_party/libpg_query/grammar/statements/select.y" + case 1084: +#line 2795 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); n->subLinkType = PG_EXPR_SUBLINK; @@ -28922,8 +29052,8 @@ YYLTYPE yylloc; ;} break; - case 1082: -#line 2805 "third_party/libpg_query/grammar/statements/select.y" + case 1085: +#line 2806 "third_party/libpg_query/grammar/statements/select.y" { /* * Because the select_with_parens nonterminal is designed @@ -28949,8 +29079,8 @@ YYLTYPE yylloc; ;} break; - case 1083: -#line 2829 "third_party/libpg_query/grammar/statements/select.y" + case 1086: +#line 2830 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); n->subLinkType = PG_EXISTS_SUBLINK; @@ -28963,8 +29093,8 @@ YYLTYPE yylloc; ;} break; - case 1084: -#line 2840 "third_party/libpg_query/grammar/statements/select.y" + case 1087: +#line 2841 "third_party/libpg_query/grammar/statements/select.y" { PGGroupingFunc *g = makeNode(PGGroupingFunc); g->args = (yyvsp[(3) - (4)].list); @@ -28973,37 +29103,37 @@ YYLTYPE yylloc; ;} break; - case 1085: -#line 2850 "third_party/libpg_query/grammar/statements/select.y" + case 1088: +#line 2851 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 1086: -#line 2854 "third_party/libpg_query/grammar/statements/select.y" + case 1089: +#line 2855 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1087: -#line 2857 "third_party/libpg_query/grammar/statements/select.y" + case 1090: +#line 2858 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("row"), (yyvsp[(1) - (1)].list), (yylsp[(1) - (1)])); (yyval.node) = (PGNode *) n; ;} break; - case 1088: -#line 2865 "third_party/libpg_query/grammar/statements/select.y" + case 1091: +#line 2866 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeParamRef(0, (yylsp[(1) - (1)])); ;} break; - case 1089: -#line 2869 "third_party/libpg_query/grammar/statements/select.y" + case 1092: +#line 2870 "third_party/libpg_query/grammar/statements/select.y" { PGParamRef *p = makeNode(PGParamRef); p->number = (yyvsp[(1) - (1)].ival); @@ -29012,15 +29142,15 @@ YYLTYPE yylloc; ;} break; - case 1090: -#line 2876 "third_party/libpg_query/grammar/statements/select.y" + case 1093: +#line 2877 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeNamedParamRef((yyvsp[(2) - (2)].str), (yylsp[(1) - (2)])); ;} break; - case 1098: -#line 2890 "third_party/libpg_query/grammar/statements/select.y" + case 1101: +#line 2891 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); n->subLinkType = PG_ARRAY_SUBLINK; @@ -29033,8 +29163,8 @@ YYLTYPE yylloc; ;} break; - case 1099: -#line 2900 "third_party/libpg_query/grammar/statements/select.y" + case 1102: +#line 2901 "third_party/libpg_query/grammar/statements/select.y" { PGList *func_name = list_make1(makeString("construct_array")); PGFuncCall *n = makeFuncCall(func_name, (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); @@ -29042,8 +29172,8 @@ YYLTYPE yylloc; ;} break; - case 1100: -#line 2906 "third_party/libpg_query/grammar/statements/select.y" + case 1103: +#line 2907 "third_party/libpg_query/grammar/statements/select.y" { PGPositionalReference *n = makeNode(PGPositionalReference); n->position = (yyvsp[(2) - (2)].ival); @@ -29052,24 +29182,24 @@ YYLTYPE yylloc; ;} break; - case 1101: -#line 2914 "third_party/libpg_query/grammar/statements/select.y" + case 1104: +#line 2915 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("list_value"), (yyvsp[(2) - (3)].list), (yylsp[(2) - (3)])); (yyval.node) = (PGNode *) n; ;} break; - case 1102: -#line 2921 "third_party/libpg_query/grammar/statements/select.y" + case 1105: +#line 2922 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *f = makeFuncCall(SystemFuncName("struct_pack"), (yyvsp[(2) - (3)].list), (yylsp[(2) - (3)])); (yyval.node) = (PGNode *) f; ;} break; - case 1103: -#line 2928 "third_party/libpg_query/grammar/statements/select.y" + case 1106: +#line 2929 "third_party/libpg_query/grammar/statements/select.y" { PGList *key_list = NULL; PGList *value_list = NULL; @@ -29088,15 +29218,15 @@ YYLTYPE yylloc; ;} break; - case 1104: -#line 2948 "third_party/libpg_query/grammar/statements/select.y" + case 1107: +#line 2949 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall((yyvsp[(1) - (3)].list), NIL, (yylsp[(1) - (3)])); ;} break; - case 1105: -#line 2952 "third_party/libpg_query/grammar/statements/select.y" + case 1108: +#line 2953 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (5)].list), NIL, (yylsp[(1) - (5)])); n->agg_order = (yyvsp[(3) - (5)].list); @@ -29105,8 +29235,8 @@ YYLTYPE yylloc; ;} break; - case 1106: -#line 2959 "third_party/libpg_query/grammar/statements/select.y" + case 1109: +#line 2960 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (6)].list), (yyvsp[(3) - (6)].list), (yylsp[(1) - (6)])); n->agg_order = (yyvsp[(4) - (6)].list); @@ -29115,8 +29245,8 @@ YYLTYPE yylloc; ;} break; - case 1107: -#line 2966 "third_party/libpg_query/grammar/statements/select.y" + case 1110: +#line 2967 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (7)].list), list_make1((yyvsp[(4) - (7)].node)), (yylsp[(1) - (7)])); n->func_variadic = true; @@ -29126,8 +29256,8 @@ YYLTYPE yylloc; ;} break; - case 1108: -#line 2974 "third_party/libpg_query/grammar/statements/select.y" + case 1111: +#line 2975 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (9)].list), lappend((yyvsp[(3) - (9)].list), (yyvsp[(6) - (9)].node)), (yylsp[(1) - (9)])); n->func_variadic = true; @@ -29137,8 +29267,8 @@ YYLTYPE yylloc; ;} break; - case 1109: -#line 2982 "third_party/libpg_query/grammar/statements/select.y" + case 1112: +#line 2983 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (7)].list), (yyvsp[(4) - (7)].list), (yylsp[(1) - (7)])); n->agg_order = (yyvsp[(5) - (7)].list); @@ -29151,8 +29281,8 @@ YYLTYPE yylloc; ;} break; - case 1110: -#line 2993 "third_party/libpg_query/grammar/statements/select.y" + case 1113: +#line 2994 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (7)].list), (yyvsp[(4) - (7)].list), (yylsp[(1) - (7)])); n->agg_order = (yyvsp[(5) - (7)].list); @@ -29162,8 +29292,8 @@ YYLTYPE yylloc; ;} break; - case 1111: -#line 3013 "third_party/libpg_query/grammar/statements/select.y" + case 1114: +#line 3014 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = (PGFuncCall *) (yyvsp[(1) - (5)].node); /* @@ -29201,23 +29331,23 @@ YYLTYPE yylloc; ;} break; - case 1112: -#line 3049 "third_party/libpg_query/grammar/statements/select.y" + case 1115: +#line 3050 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1113: -#line 3059 "third_party/libpg_query/grammar/statements/select.y" + case 1116: +#line 3060 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1114: -#line 3060 "third_party/libpg_query/grammar/statements/select.y" + case 1117: +#line 3061 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1115: -#line 3068 "third_party/libpg_query/grammar/statements/select.y" + case 1118: +#line 3069 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("pg_collation_for"), list_make1((yyvsp[(4) - (5)].node)), @@ -29225,25 +29355,25 @@ YYLTYPE yylloc; ;} break; - case 1116: -#line 3074 "third_party/libpg_query/grammar/statements/select.y" + case 1119: +#line 3075 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeTypeCast((yyvsp[(3) - (6)].node), (yyvsp[(5) - (6)].typnam), 0, (yylsp[(1) - (6)])); ;} break; - case 1117: -#line 3076 "third_party/libpg_query/grammar/statements/select.y" + case 1120: +#line 3077 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeTypeCast((yyvsp[(3) - (6)].node), (yyvsp[(5) - (6)].typnam), 1, (yylsp[(1) - (6)])); ;} break; - case 1118: -#line 3078 "third_party/libpg_query/grammar/statements/select.y" + case 1121: +#line 3079 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("date_part"), (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 1119: -#line 3082 "third_party/libpg_query/grammar/statements/select.y" + case 1122: +#line 3083 "third_party/libpg_query/grammar/statements/select.y" { /* overlay(A PLACING B FROM C FOR D) is converted to * overlay(A, B, C, D) @@ -29254,16 +29384,16 @@ YYLTYPE yylloc; ;} break; - case 1120: -#line 3091 "third_party/libpg_query/grammar/statements/select.y" + case 1123: +#line 3092 "third_party/libpg_query/grammar/statements/select.y" { /* position(A in B) is converted to position_inverse(A, B) */ (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("__internal_position_operator"), (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 1121: -#line 3096 "third_party/libpg_query/grammar/statements/select.y" + case 1124: +#line 3097 "third_party/libpg_query/grammar/statements/select.y" { /* substring(A from B for C) is converted to * substring(A, B, C) - thomas 2000-11-28 @@ -29272,8 +29402,8 @@ YYLTYPE yylloc; ;} break; - case 1122: -#line 3103 "third_party/libpg_query/grammar/statements/select.y" + case 1125: +#line 3104 "third_party/libpg_query/grammar/statements/select.y" { /* TREAT(expr AS target) converts expr of a particular type to target, * which is defined to be a subtype of the original expression. @@ -29290,8 +29420,8 @@ YYLTYPE yylloc; ;} break; - case 1123: -#line 3118 "third_party/libpg_query/grammar/statements/select.y" + case 1126: +#line 3119 "third_party/libpg_query/grammar/statements/select.y" { /* various trim expressions are defined in SQL * - thomas 1997-07-19 @@ -29300,36 +29430,36 @@ YYLTYPE yylloc; ;} break; - case 1124: -#line 3125 "third_party/libpg_query/grammar/statements/select.y" + case 1127: +#line 3126 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("ltrim"), (yyvsp[(4) - (5)].list), (yylsp[(1) - (5)])); ;} break; - case 1125: -#line 3129 "third_party/libpg_query/grammar/statements/select.y" + case 1128: +#line 3130 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("rtrim"), (yyvsp[(4) - (5)].list), (yylsp[(1) - (5)])); ;} break; - case 1126: -#line 3133 "third_party/libpg_query/grammar/statements/select.y" + case 1129: +#line 3134 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("trim"), (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 1127: -#line 3137 "third_party/libpg_query/grammar/statements/select.y" + case 1130: +#line 3138 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NULLIF, "=", (yyvsp[(3) - (6)].node), (yyvsp[(5) - (6)].node), (yylsp[(1) - (6)])); ;} break; - case 1128: -#line 3141 "third_party/libpg_query/grammar/statements/select.y" + case 1131: +#line 3142 "third_party/libpg_query/grammar/statements/select.y" { PGCoalesceExpr *c = makeNode(PGCoalesceExpr); c->args = (yyvsp[(3) - (4)].list); @@ -29338,8 +29468,8 @@ YYLTYPE yylloc; ;} break; - case 1129: -#line 3151 "third_party/libpg_query/grammar/statements/select.y" + case 1132: +#line 3152 "third_party/libpg_query/grammar/statements/select.y" { PGLambdaFunction *lambda = makeNode(PGLambdaFunction); lambda->lhs = (yyvsp[(4) - (7)].list); @@ -29350,8 +29480,8 @@ YYLTYPE yylloc; ;} break; - case 1130: -#line 3160 "third_party/libpg_query/grammar/statements/select.y" + case 1133: +#line 3161 "third_party/libpg_query/grammar/statements/select.y" { PGLambdaFunction *lambda = makeNode(PGLambdaFunction); lambda->lhs = (yyvsp[(4) - (9)].list); @@ -29368,63 +29498,63 @@ YYLTYPE yylloc; ;} break; - case 1131: -#line 3181 "third_party/libpg_query/grammar/statements/select.y" + case 1134: +#line 3182 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(4) - (5)].list); ;} break; - case 1132: -#line 3182 "third_party/libpg_query/grammar/statements/select.y" + case 1135: +#line 3183 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1133: -#line 3186 "third_party/libpg_query/grammar/statements/select.y" + case 1136: +#line 3187 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(4) - (5)].node); ;} break; - case 1134: -#line 3187 "third_party/libpg_query/grammar/statements/select.y" + case 1137: +#line 3188 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(3) - (4)].node); ;} break; - case 1135: -#line 3188 "third_party/libpg_query/grammar/statements/select.y" + case 1138: +#line 3189 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 1136: -#line 3192 "third_party/libpg_query/grammar/statements/select.y" + case 1139: +#line 3193 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 1137: -#line 3193 "third_party/libpg_query/grammar/statements/select.y" + case 1140: +#line 3194 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 1138: -#line 3200 "third_party/libpg_query/grammar/statements/select.y" + case 1141: +#line 3201 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 1139: -#line 3201 "third_party/libpg_query/grammar/statements/select.y" + case 1142: +#line 3202 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1140: -#line 3205 "third_party/libpg_query/grammar/statements/select.y" + case 1143: +#line 3206 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].windef)); ;} break; - case 1141: -#line 3207 "third_party/libpg_query/grammar/statements/select.y" + case 1144: +#line 3208 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].windef)); ;} break; - case 1142: -#line 3212 "third_party/libpg_query/grammar/statements/select.y" + case 1145: +#line 3213 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(3) - (3)].windef); n->name = (yyvsp[(1) - (3)].str); @@ -29432,13 +29562,13 @@ YYLTYPE yylloc; ;} break; - case 1143: -#line 3220 "third_party/libpg_query/grammar/statements/select.y" + case 1146: +#line 3221 "third_party/libpg_query/grammar/statements/select.y" { (yyval.windef) = (yyvsp[(2) - (2)].windef); ;} break; - case 1144: -#line 3222 "third_party/libpg_query/grammar/statements/select.y" + case 1147: +#line 3223 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); n->name = (yyvsp[(2) - (2)].str); @@ -29453,13 +29583,13 @@ YYLTYPE yylloc; ;} break; - case 1145: -#line 3235 "third_party/libpg_query/grammar/statements/select.y" + case 1148: +#line 3236 "third_party/libpg_query/grammar/statements/select.y" { (yyval.windef) = NULL; ;} break; - case 1146: -#line 3240 "third_party/libpg_query/grammar/statements/select.y" + case 1149: +#line 3241 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); n->name = NULL; @@ -29475,28 +29605,28 @@ YYLTYPE yylloc; ;} break; - case 1147: -#line 3265 "third_party/libpg_query/grammar/statements/select.y" + case 1150: +#line 3266 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1148: -#line 3266 "third_party/libpg_query/grammar/statements/select.y" + case 1151: +#line 3267 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = NULL; ;} break; - case 1149: -#line 3269 "third_party/libpg_query/grammar/statements/select.y" + case 1152: +#line 3270 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 1150: -#line 3270 "third_party/libpg_query/grammar/statements/select.y" + case 1153: +#line 3271 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1151: -#line 3279 "third_party/libpg_query/grammar/statements/select.y" + case 1154: +#line 3280 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(2) - (3)].windef); @@ -29506,8 +29636,8 @@ YYLTYPE yylloc; ;} break; - case 1152: -#line 3287 "third_party/libpg_query/grammar/statements/select.y" + case 1155: +#line 3288 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(2) - (3)].windef); @@ -29517,8 +29647,8 @@ YYLTYPE yylloc; ;} break; - case 1153: -#line 3295 "third_party/libpg_query/grammar/statements/select.y" + case 1156: +#line 3296 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(2) - (3)].windef); @@ -29528,8 +29658,8 @@ YYLTYPE yylloc; ;} break; - case 1154: -#line 3303 "third_party/libpg_query/grammar/statements/select.y" + case 1157: +#line 3304 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -29540,8 +29670,8 @@ YYLTYPE yylloc; ;} break; - case 1155: -#line 3314 "third_party/libpg_query/grammar/statements/select.y" + case 1158: +#line 3315 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(1) - (1)].windef); @@ -29561,8 +29691,8 @@ YYLTYPE yylloc; ;} break; - case 1156: -#line 3332 "third_party/libpg_query/grammar/statements/select.y" + case 1159: +#line 3333 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n1 = (yyvsp[(2) - (4)].windef); PGWindowDef *n2 = (yyvsp[(4) - (4)].windef); @@ -29602,8 +29732,8 @@ YYLTYPE yylloc; ;} break; - case 1157: -#line 3378 "third_party/libpg_query/grammar/statements/select.y" + case 1160: +#line 3379 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -29614,8 +29744,8 @@ YYLTYPE yylloc; ;} break; - case 1158: -#line 3387 "third_party/libpg_query/grammar/statements/select.y" + case 1161: +#line 3388 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -29626,8 +29756,8 @@ YYLTYPE yylloc; ;} break; - case 1159: -#line 3396 "third_party/libpg_query/grammar/statements/select.y" + case 1162: +#line 3397 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -29638,8 +29768,8 @@ YYLTYPE yylloc; ;} break; - case 1160: -#line 3405 "third_party/libpg_query/grammar/statements/select.y" + case 1163: +#line 3406 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -29650,8 +29780,8 @@ YYLTYPE yylloc; ;} break; - case 1161: -#line 3414 "third_party/libpg_query/grammar/statements/select.y" + case 1164: +#line 3415 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -29662,53 +29792,53 @@ YYLTYPE yylloc; ;} break; - case 1162: -#line 3425 "third_party/libpg_query/grammar/statements/select.y" + case 1165: +#line 3426 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = FRAMEOPTION_EXCLUDE_CURRENT_ROW; ;} break; - case 1163: -#line 3426 "third_party/libpg_query/grammar/statements/select.y" + case 1166: +#line 3427 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = FRAMEOPTION_EXCLUDE_GROUP; ;} break; - case 1164: -#line 3427 "third_party/libpg_query/grammar/statements/select.y" + case 1167: +#line 3428 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = FRAMEOPTION_EXCLUDE_TIES; ;} break; - case 1165: -#line 3428 "third_party/libpg_query/grammar/statements/select.y" + case 1168: +#line 3429 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 1166: -#line 3429 "third_party/libpg_query/grammar/statements/select.y" + case 1169: +#line 3430 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 1167: -#line 3443 "third_party/libpg_query/grammar/statements/select.y" + case 1170: +#line 3444 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1168: -#line 3444 "third_party/libpg_query/grammar/statements/select.y" + case 1171: +#line 3445 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1169: -#line 3447 "third_party/libpg_query/grammar/statements/select.y" + case 1172: +#line 3448 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list);;} break; - case 1170: -#line 3448 "third_party/libpg_query/grammar/statements/select.y" + case 1173: +#line 3449 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(2) - (5)].list), (yyvsp[(4) - (5)].node)); ;} break; - case 1171: -#line 3452 "third_party/libpg_query/grammar/statements/select.y" + case 1174: +#line 3453 "third_party/libpg_query/grammar/statements/select.y" { PGNamedArgExpr *na = makeNode(PGNamedArgExpr); na->name = (yyvsp[(1) - (3)].str); @@ -29719,321 +29849,321 @@ YYLTYPE yylloc; ;} break; - case 1172: -#line 3462 "third_party/libpg_query/grammar/statements/select.y" + case 1175: +#line 3463 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1173: -#line 3463 "third_party/libpg_query/grammar/statements/select.y" + case 1176: +#line 3464 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 1174: -#line 3467 "third_party/libpg_query/grammar/statements/select.y" + case 1177: +#line 3468 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1175: -#line 3468 "third_party/libpg_query/grammar/statements/select.y" + case 1178: +#line 3469 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1176: -#line 3473 "third_party/libpg_query/grammar/statements/select.y" + case 1179: +#line 3474 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node)); ;} break; - case 1177: -#line 3479 "third_party/libpg_query/grammar/statements/select.y" + case 1180: +#line 3480 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} break; - case 1178: -#line 3480 "third_party/libpg_query/grammar/statements/select.y" + case 1181: +#line 3481 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; - case 1179: -#line 3485 "third_party/libpg_query/grammar/statements/select.y" + case 1182: +#line 3486 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1180: -#line 3486 "third_party/libpg_query/grammar/statements/select.y" + case 1183: +#line 3487 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1181: -#line 3491 "third_party/libpg_query/grammar/statements/select.y" + case 1184: +#line 3492 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1182: -#line 3492 "third_party/libpg_query/grammar/statements/select.y" + case 1185: +#line 3493 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1183: -#line 3495 "third_party/libpg_query/grammar/statements/select.y" + case 1186: +#line 3496 "third_party/libpg_query/grammar/statements/select.y" { (yyval.subquerytype) = PG_ANY_SUBLINK; ;} break; - case 1184: -#line 3496 "third_party/libpg_query/grammar/statements/select.y" + case 1187: +#line 3497 "third_party/libpg_query/grammar/statements/select.y" { (yyval.subquerytype) = PG_ANY_SUBLINK; ;} break; - case 1185: -#line 3497 "third_party/libpg_query/grammar/statements/select.y" + case 1188: +#line 3498 "third_party/libpg_query/grammar/statements/select.y" { (yyval.subquerytype) = PG_ALL_SUBLINK; ;} break; - case 1186: -#line 3500 "third_party/libpg_query/grammar/statements/select.y" + case 1189: +#line 3501 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1187: -#line 3501 "third_party/libpg_query/grammar/statements/select.y" + case 1190: +#line 3502 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) (yyvsp[(1) - (1)].conststr); ;} break; - case 1188: -#line 3504 "third_party/libpg_query/grammar/statements/select.y" + case 1191: +#line 3505 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "+"; ;} break; - case 1189: -#line 3505 "third_party/libpg_query/grammar/statements/select.y" + case 1192: +#line 3506 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "-"; ;} break; - case 1190: -#line 3506 "third_party/libpg_query/grammar/statements/select.y" + case 1193: +#line 3507 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "*"; ;} break; - case 1191: -#line 3507 "third_party/libpg_query/grammar/statements/select.y" + case 1194: +#line 3508 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "/"; ;} break; - case 1192: -#line 3508 "third_party/libpg_query/grammar/statements/select.y" + case 1195: +#line 3509 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "//"; ;} break; - case 1193: -#line 3509 "third_party/libpg_query/grammar/statements/select.y" + case 1196: +#line 3510 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "%"; ;} break; - case 1194: -#line 3510 "third_party/libpg_query/grammar/statements/select.y" + case 1197: +#line 3511 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "^"; ;} break; - case 1195: -#line 3511 "third_party/libpg_query/grammar/statements/select.y" + case 1198: +#line 3512 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "**"; ;} break; - case 1196: -#line 3512 "third_party/libpg_query/grammar/statements/select.y" + case 1199: +#line 3513 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "<"; ;} break; - case 1197: -#line 3513 "third_party/libpg_query/grammar/statements/select.y" + case 1200: +#line 3514 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = ">"; ;} break; - case 1198: -#line 3514 "third_party/libpg_query/grammar/statements/select.y" + case 1201: +#line 3515 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "="; ;} break; - case 1199: -#line 3515 "third_party/libpg_query/grammar/statements/select.y" + case 1202: +#line 3516 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "<="; ;} break; - case 1200: -#line 3516 "third_party/libpg_query/grammar/statements/select.y" + case 1203: +#line 3517 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = ">="; ;} break; - case 1201: -#line 3517 "third_party/libpg_query/grammar/statements/select.y" + case 1204: +#line 3518 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "<>"; ;} break; - case 1202: -#line 3521 "third_party/libpg_query/grammar/statements/select.y" + case 1205: +#line 3522 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1203: -#line 3523 "third_party/libpg_query/grammar/statements/select.y" + case 1206: +#line 3524 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1204: -#line 3528 "third_party/libpg_query/grammar/statements/select.y" + case 1207: +#line 3529 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1205: -#line 3530 "third_party/libpg_query/grammar/statements/select.y" + case 1208: +#line 3531 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1206: -#line 3535 "third_party/libpg_query/grammar/statements/select.y" + case 1209: +#line 3536 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1207: -#line 3537 "third_party/libpg_query/grammar/statements/select.y" + case 1210: +#line 3538 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1208: -#line 3539 "third_party/libpg_query/grammar/statements/select.y" + case 1211: +#line 3540 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString("~~")); ;} break; - case 1209: -#line 3541 "third_party/libpg_query/grammar/statements/select.y" + case 1212: +#line 3542 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString("!~~")); ;} break; - case 1210: -#line 3543 "third_party/libpg_query/grammar/statements/select.y" + case 1213: +#line 3544 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString("~~~")); ;} break; - case 1211: -#line 3545 "third_party/libpg_query/grammar/statements/select.y" + case 1214: +#line 3546 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString("!~~~")); ;} break; - case 1212: -#line 3547 "third_party/libpg_query/grammar/statements/select.y" + case 1215: +#line 3548 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString("~~*")); ;} break; - case 1213: -#line 3549 "third_party/libpg_query/grammar/statements/select.y" + case 1216: +#line 3550 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString("!~~*")); ;} break; - case 1214: -#line 3563 "third_party/libpg_query/grammar/statements/select.y" + case 1217: +#line 3564 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1215: -#line 3565 "third_party/libpg_query/grammar/statements/select.y" + case 1218: +#line 3566 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lcons(makeString((yyvsp[(1) - (3)].str)), (yyvsp[(3) - (3)].list)); ;} break; - case 1216: -#line 3570 "third_party/libpg_query/grammar/statements/select.y" + case 1219: +#line 3571 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1217: -#line 3574 "third_party/libpg_query/grammar/statements/select.y" + case 1220: +#line 3575 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 1218: -#line 3581 "third_party/libpg_query/grammar/statements/select.y" + case 1221: +#line 3582 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1219: -#line 3586 "third_party/libpg_query/grammar/statements/select.y" + case 1222: +#line 3587 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1220: -#line 3592 "third_party/libpg_query/grammar/statements/select.y" + case 1223: +#line 3593 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1221: -#line 3596 "third_party/libpg_query/grammar/statements/select.y" + case 1224: +#line 3597 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 1222: -#line 3603 "third_party/libpg_query/grammar/statements/select.y" + case 1225: +#line 3604 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1223: -#line 3608 "third_party/libpg_query/grammar/statements/select.y" + case 1226: +#line 3609 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1224: -#line 3615 "third_party/libpg_query/grammar/statements/select.y" + case 1227: +#line 3616 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1225: -#line 3619 "third_party/libpg_query/grammar/statements/select.y" + case 1228: +#line 3620 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1226: -#line 3628 "third_party/libpg_query/grammar/statements/select.y" + case 1229: +#line 3629 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1227: -#line 3632 "third_party/libpg_query/grammar/statements/select.y" + case 1230: +#line 3633 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 1228: -#line 3638 "third_party/libpg_query/grammar/statements/select.y" + case 1231: +#line 3639 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1229: -#line 3642 "third_party/libpg_query/grammar/statements/select.y" + case 1232: +#line 3643 "third_party/libpg_query/grammar/statements/select.y" { PGNamedArgExpr *na = makeNode(PGNamedArgExpr); na->name = (yyvsp[(1) - (3)].str); @@ -30044,8 +30174,8 @@ YYLTYPE yylloc; ;} break; - case 1230: -#line 3651 "third_party/libpg_query/grammar/statements/select.y" + case 1233: +#line 3652 "third_party/libpg_query/grammar/statements/select.y" { PGNamedArgExpr *na = makeNode(PGNamedArgExpr); na->name = (yyvsp[(1) - (3)].str); @@ -30056,156 +30186,156 @@ YYLTYPE yylloc; ;} break; - case 1231: -#line 3661 "third_party/libpg_query/grammar/statements/select.y" + case 1234: +#line 3662 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].typnam)); ;} break; - case 1232: -#line 3662 "third_party/libpg_query/grammar/statements/select.y" + case 1235: +#line 3663 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].typnam)); ;} break; - case 1233: -#line 3667 "third_party/libpg_query/grammar/statements/select.y" + case 1236: +#line 3668 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(makeStringConst((yyvsp[(1) - (3)].str), (yylsp[(1) - (3)])), (yyvsp[(3) - (3)].node)); ;} break; - case 1234: -#line 3670 "third_party/libpg_query/grammar/statements/select.y" + case 1237: +#line 3671 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1235: -#line 3677 "third_party/libpg_query/grammar/statements/select.y" + case 1238: +#line 3678 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1236: -#line 3678 "third_party/libpg_query/grammar/statements/select.y" + case 1239: +#line 3679 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "year"; ;} break; - case 1237: -#line 3679 "third_party/libpg_query/grammar/statements/select.y" + case 1240: +#line 3680 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "month"; ;} break; - case 1238: -#line 3680 "third_party/libpg_query/grammar/statements/select.y" + case 1241: +#line 3681 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "day"; ;} break; - case 1239: -#line 3681 "third_party/libpg_query/grammar/statements/select.y" + case 1242: +#line 3682 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "hour"; ;} break; - case 1240: -#line 3682 "third_party/libpg_query/grammar/statements/select.y" + case 1243: +#line 3683 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "minute"; ;} break; - case 1241: -#line 3683 "third_party/libpg_query/grammar/statements/select.y" + case 1244: +#line 3684 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "second"; ;} break; - case 1242: -#line 3684 "third_party/libpg_query/grammar/statements/select.y" + case 1245: +#line 3685 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "millisecond"; ;} break; - case 1243: -#line 3685 "third_party/libpg_query/grammar/statements/select.y" + case 1246: +#line 3686 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "microsecond"; ;} break; - case 1244: -#line 3686 "third_party/libpg_query/grammar/statements/select.y" + case 1247: +#line 3687 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "week"; ;} break; - case 1245: -#line 3687 "third_party/libpg_query/grammar/statements/select.y" + case 1248: +#line 3688 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "quarter"; ;} break; - case 1246: -#line 3688 "third_party/libpg_query/grammar/statements/select.y" + case 1249: +#line 3689 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "decade"; ;} break; - case 1247: -#line 3689 "third_party/libpg_query/grammar/statements/select.y" + case 1250: +#line 3690 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "century"; ;} break; - case 1248: -#line 3690 "third_party/libpg_query/grammar/statements/select.y" + case 1251: +#line 3691 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "millennium"; ;} break; - case 1249: -#line 3691 "third_party/libpg_query/grammar/statements/select.y" + case 1252: +#line 3692 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1250: -#line 3702 "third_party/libpg_query/grammar/statements/select.y" + case 1253: +#line 3703 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make4((yyvsp[(1) - (4)].node), (yyvsp[(2) - (4)].node), (yyvsp[(3) - (4)].node), (yyvsp[(4) - (4)].node)); ;} break; - case 1251: -#line 3706 "third_party/libpg_query/grammar/statements/select.y" + case 1254: +#line 3707 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make3((yyvsp[(1) - (3)].node), (yyvsp[(2) - (3)].node), (yyvsp[(3) - (3)].node)); ;} break; - case 1252: -#line 3713 "third_party/libpg_query/grammar/statements/select.y" + case 1255: +#line 3714 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1253: -#line 3719 "third_party/libpg_query/grammar/statements/select.y" + case 1256: +#line 3720 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node)); ;} break; - case 1254: -#line 3720 "third_party/libpg_query/grammar/statements/select.y" + case 1257: +#line 3721 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1255: -#line 3737 "third_party/libpg_query/grammar/statements/select.y" + case 1258: +#line 3738 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make3((yyvsp[(1) - (3)].node), (yyvsp[(2) - (3)].node), (yyvsp[(3) - (3)].node)); ;} break; - case 1256: -#line 3741 "third_party/libpg_query/grammar/statements/select.y" + case 1259: +#line 3742 "third_party/libpg_query/grammar/statements/select.y" { /* not legal per SQL99, but might as well allow it */ (yyval.list) = list_make3((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yyvsp[(2) - (3)].node)); ;} break; - case 1257: -#line 3746 "third_party/libpg_query/grammar/statements/select.y" + case 1260: +#line 3747 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (2)].node), (yyvsp[(2) - (2)].node)); ;} break; - case 1258: -#line 3750 "third_party/libpg_query/grammar/statements/select.y" + case 1261: +#line 3751 "third_party/libpg_query/grammar/statements/select.y" { /* * Since there are no cases where this syntax allows @@ -30222,45 +30352,45 @@ YYLTYPE yylloc; ;} break; - case 1259: -#line 3765 "third_party/libpg_query/grammar/statements/select.y" + case 1262: +#line 3766 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1260: -#line 3769 "third_party/libpg_query/grammar/statements/select.y" + case 1263: +#line 3770 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1261: -#line 3773 "third_party/libpg_query/grammar/statements/select.y" + case 1264: +#line 3774 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1262: -#line 3776 "third_party/libpg_query/grammar/statements/select.y" + case 1265: +#line 3777 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1263: -#line 3779 "third_party/libpg_query/grammar/statements/select.y" + case 1266: +#line 3780 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(3) - (3)].list), (yyvsp[(1) - (3)].node)); ;} break; - case 1264: -#line 3780 "third_party/libpg_query/grammar/statements/select.y" + case 1267: +#line 3781 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 1265: -#line 3781 "third_party/libpg_query/grammar/statements/select.y" + case 1268: +#line 3782 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1266: -#line 3785 "third_party/libpg_query/grammar/statements/select.y" + case 1269: +#line 3786 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); n->subselect = (yyvsp[(1) - (1)].node); @@ -30269,18 +30399,18 @@ YYLTYPE yylloc; ;} break; - case 1267: -#line 3791 "third_party/libpg_query/grammar/statements/select.y" + case 1270: +#line 3792 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *)(yyvsp[(2) - (3)].list); ;} break; - case 1269: -#line 3793 "third_party/libpg_query/grammar/statements/select.y" + case 1272: +#line 3794 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *)(yyvsp[(1) - (1)].node); ;} break; - case 1270: -#line 3804 "third_party/libpg_query/grammar/statements/select.y" + case 1273: +#line 3805 "third_party/libpg_query/grammar/statements/select.y" { PGCaseExpr *c = makeNode(PGCaseExpr); c->casetype = InvalidOid; /* not analyzed yet */ @@ -30292,18 +30422,18 @@ YYLTYPE yylloc; ;} break; - case 1271: -#line 3817 "third_party/libpg_query/grammar/statements/select.y" + case 1274: +#line 3818 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1272: -#line 3818 "third_party/libpg_query/grammar/statements/select.y" + case 1275: +#line 3819 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 1273: -#line 3823 "third_party/libpg_query/grammar/statements/select.y" + case 1276: +#line 3824 "third_party/libpg_query/grammar/statements/select.y" { PGCaseWhen *w = makeNode(PGCaseWhen); w->expr = (PGExpr *) (yyvsp[(2) - (4)].node); @@ -30313,49 +30443,49 @@ YYLTYPE yylloc; ;} break; - case 1274: -#line 3833 "third_party/libpg_query/grammar/statements/select.y" + case 1277: +#line 3834 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1275: -#line 3834 "third_party/libpg_query/grammar/statements/select.y" + case 1278: +#line 3835 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 1276: -#line 3837 "third_party/libpg_query/grammar/statements/select.y" + case 1279: +#line 3838 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1277: -#line 3838 "third_party/libpg_query/grammar/statements/select.y" + case 1280: +#line 3839 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 1278: -#line 3847 "third_party/libpg_query/grammar/statements/select.y" + case 1281: +#line 3848 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeColumnRef((yyvsp[(1) - (1)].str), NIL, (yylsp[(1) - (1)]), yyscanner); ;} break; - case 1279: -#line 3853 "third_party/libpg_query/grammar/statements/select.y" + case 1282: +#line 3854 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeColumnRef((yyvsp[(1) - (1)].str), NIL, (yylsp[(1) - (1)]), yyscanner); ;} break; - case 1280: -#line 3857 "third_party/libpg_query/grammar/statements/select.y" + case 1283: +#line 3858 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeColumnRef((yyvsp[(1) - (2)].str), (yyvsp[(2) - (2)].list), (yylsp[(1) - (2)]), yyscanner); ;} break; - case 1281: -#line 3864 "third_party/libpg_query/grammar/statements/select.y" + case 1284: +#line 3865 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = false; @@ -30365,8 +30495,8 @@ YYLTYPE yylloc; ;} break; - case 1282: -#line 3872 "third_party/libpg_query/grammar/statements/select.y" + case 1285: +#line 3873 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -30376,8 +30506,8 @@ YYLTYPE yylloc; ;} break; - case 1283: -#line 3879 "third_party/libpg_query/grammar/statements/select.y" + case 1286: +#line 3880 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -30388,8 +30518,8 @@ YYLTYPE yylloc; ;} break; - case 1284: -#line 3887 "third_party/libpg_query/grammar/statements/select.y" + case 1287: +#line 3888 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -30399,43 +30529,43 @@ YYLTYPE yylloc; ;} break; - case 1285: -#line 3897 "third_party/libpg_query/grammar/statements/select.y" + case 1288: +#line 3898 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1286: -#line 3898 "third_party/libpg_query/grammar/statements/select.y" + case 1289: +#line 3899 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 1287: -#line 3903 "third_party/libpg_query/grammar/statements/select.y" + case 1290: +#line 3904 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1288: -#line 3904 "third_party/libpg_query/grammar/statements/select.y" + case 1291: +#line 3905 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 1289: -#line 3908 "third_party/libpg_query/grammar/statements/select.y" + case 1292: +#line 3909 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1290: -#line 3909 "third_party/libpg_query/grammar/statements/select.y" + case 1293: +#line 3910 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(NULL); ;} break; - case 1291: -#line 3910 "third_party/libpg_query/grammar/statements/select.y" + case 1294: +#line 3911 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 1292: -#line 3915 "third_party/libpg_query/grammar/statements/select.y" + case 1295: +#line 3916 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(3) - (3)].list)) { PGFuncCall *n = makeFuncCall(list_make1(makeString((yyvsp[(2) - (3)].str))), (yyvsp[(3) - (3)].list)->head->data.ptr_value ? (yyvsp[(3) - (3)].list) : NULL, (yylsp[(2) - (3)])); @@ -30446,8 +30576,8 @@ YYLTYPE yylloc; ;} break; - case 1293: -#line 3924 "third_party/libpg_query/grammar/statements/select.y" + case 1296: +#line 3925 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = false; @@ -30457,8 +30587,8 @@ YYLTYPE yylloc; ;} break; - case 1294: -#line 3932 "third_party/libpg_query/grammar/statements/select.y" + case 1297: +#line 3933 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -30468,8 +30598,8 @@ YYLTYPE yylloc; ;} break; - case 1295: -#line 3939 "third_party/libpg_query/grammar/statements/select.y" + case 1298: +#line 3940 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -30480,8 +30610,8 @@ YYLTYPE yylloc; ;} break; - case 1296: -#line 3948 "third_party/libpg_query/grammar/statements/select.y" + case 1299: +#line 3949 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -30491,48 +30621,48 @@ YYLTYPE yylloc; ;} break; - case 1297: -#line 3963 "third_party/libpg_query/grammar/statements/select.y" + case 1300: +#line 3964 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1298: -#line 3964 "third_party/libpg_query/grammar/statements/select.y" + case 1301: +#line 3965 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 1301: -#line 3980 "third_party/libpg_query/grammar/statements/select.y" + case 1304: +#line 3981 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1302: -#line 3981 "third_party/libpg_query/grammar/statements/select.y" + case 1305: +#line 3982 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1303: -#line 3985 "third_party/libpg_query/grammar/statements/select.y" + case 1306: +#line 3986 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].target)); ;} break; - case 1304: -#line 3986 "third_party/libpg_query/grammar/statements/select.y" + case 1307: +#line 3987 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].target)); ;} break; - case 1305: -#line 3990 "third_party/libpg_query/grammar/statements/select.y" + case 1308: +#line 3991 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1306: -#line 3991 "third_party/libpg_query/grammar/statements/select.y" + case 1309: +#line 3992 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1307: -#line 3995 "third_party/libpg_query/grammar/statements/select.y" + case 1310: +#line 3996 "third_party/libpg_query/grammar/statements/select.y" { (yyval.target) = makeNode(PGResTarget); (yyval.target)->name = (yyvsp[(3) - (3)].str); @@ -30542,8 +30672,8 @@ YYLTYPE yylloc; ;} break; - case 1308: -#line 4011 "third_party/libpg_query/grammar/statements/select.y" + case 1311: +#line 4012 "third_party/libpg_query/grammar/statements/select.y" { (yyval.target) = makeNode(PGResTarget); (yyval.target)->name = (yyvsp[(2) - (2)].str); @@ -30553,8 +30683,8 @@ YYLTYPE yylloc; ;} break; - case 1309: -#line 4019 "third_party/libpg_query/grammar/statements/select.y" + case 1312: +#line 4020 "third_party/libpg_query/grammar/statements/select.y" { (yyval.target) = makeNode(PGResTarget); (yyval.target)->name = NULL; @@ -30564,8 +30694,8 @@ YYLTYPE yylloc; ;} break; - case 1310: -#line 4027 "third_party/libpg_query/grammar/statements/select.y" + case 1313: +#line 4028 "third_party/libpg_query/grammar/statements/select.y" { (yyval.target) = makeNode(PGResTarget); (yyval.target)->name = (yyvsp[(1) - (3)].str); @@ -30575,214 +30705,214 @@ YYLTYPE yylloc; ;} break; - case 1311: -#line 4036 "third_party/libpg_query/grammar/statements/select.y" + case 1314: +#line 4037 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1312: -#line 4037 "third_party/libpg_query/grammar/statements/select.y" + case 1315: +#line 4038 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(2) - (2)].list)); ;} break; - case 1313: -#line 4042 "third_party/libpg_query/grammar/statements/select.y" + case 1316: +#line 4043 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].str)); ;} break; - case 1314: -#line 4046 "third_party/libpg_query/grammar/statements/select.y" + case 1317: +#line 4047 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].str)); ;} break; - case 1315: -#line 4052 "third_party/libpg_query/grammar/statements/select.y" + case 1318: +#line 4053 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} break; - case 1316: -#line 4054 "third_party/libpg_query/grammar/statements/select.y" + case 1319: +#line 4055 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; - case 1317: -#line 4058 "third_party/libpg_query/grammar/statements/select.y" + case 1320: +#line 4059 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1318: -#line 4059 "third_party/libpg_query/grammar/statements/select.y" + case 1321: +#line 4060 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1319: -#line 4063 "third_party/libpg_query/grammar/statements/select.y" + case 1322: +#line 4064 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1320: -#line 4064 "third_party/libpg_query/grammar/statements/select.y" + case 1323: +#line 4065 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1321: -#line 4067 "third_party/libpg_query/grammar/statements/select.y" + case 1324: +#line 4068 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (3)].node), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 1322: -#line 4071 "third_party/libpg_query/grammar/statements/select.y" + case 1325: +#line 4072 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} break; - case 1323: -#line 4072 "third_party/libpg_query/grammar/statements/select.y" + case 1326: +#line 4073 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; - case 1324: -#line 4076 "third_party/libpg_query/grammar/statements/select.y" + case 1327: +#line 4077 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1325: -#line 4077 "third_party/libpg_query/grammar/statements/select.y" + case 1328: +#line 4078 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1326: -#line 4080 "third_party/libpg_query/grammar/statements/select.y" + case 1329: +#line 4081 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1327: -#line 4081 "third_party/libpg_query/grammar/statements/select.y" + case 1330: +#line 4082 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(2) - (2)].list)); ;} break; - case 1328: -#line 4082 "third_party/libpg_query/grammar/statements/select.y" + case 1331: +#line 4083 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1329: -#line 4085 "third_party/libpg_query/grammar/statements/select.y" + case 1332: +#line 4086 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].str)); ;} break; - case 1330: -#line 4089 "third_party/libpg_query/grammar/statements/select.y" + case 1333: +#line 4090 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} break; - case 1331: -#line 4090 "third_party/libpg_query/grammar/statements/select.y" + case 1334: +#line 4091 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; - case 1332: -#line 4094 "third_party/libpg_query/grammar/statements/select.y" + case 1335: +#line 4095 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1333: -#line 4095 "third_party/libpg_query/grammar/statements/select.y" + case 1336: +#line 4096 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1334: -#line 4097 "third_party/libpg_query/grammar/statements/select.y" + case 1337: +#line 4098 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1335: -#line 4098 "third_party/libpg_query/grammar/statements/select.y" + case 1338: +#line 4099 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(2) - (2)].list)); ;} break; - case 1336: -#line 4099 "third_party/libpg_query/grammar/statements/select.y" + case 1339: +#line 4100 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1337: -#line 4109 "third_party/libpg_query/grammar/statements/select.y" + case 1340: +#line 4110 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].range)); ;} break; - case 1338: -#line 4110 "third_party/libpg_query/grammar/statements/select.y" + case 1341: +#line 4111 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].range)); ;} break; - case 1339: -#line 4115 "third_party/libpg_query/grammar/statements/select.y" + case 1342: +#line 4116 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1340: -#line 4117 "third_party/libpg_query/grammar/statements/select.y" + case 1343: +#line 4118 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 1341: -#line 4122 "third_party/libpg_query/grammar/statements/select.y" + case 1344: +#line 4123 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1342: -#line 4123 "third_party/libpg_query/grammar/statements/select.y" + case 1345: +#line 4124 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1343: -#line 4127 "third_party/libpg_query/grammar/statements/select.y" + case 1346: +#line 4128 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1344: -#line 4128 "third_party/libpg_query/grammar/statements/select.y" + case 1347: +#line 4129 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 1345: -#line 4131 "third_party/libpg_query/grammar/statements/select.y" + case 1348: +#line 4132 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1346: -#line 4143 "third_party/libpg_query/grammar/statements/select.y" + case 1349: +#line 4144 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1347: -#line 4146 "third_party/libpg_query/grammar/statements/select.y" + case 1350: +#line 4147 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = check_func_name(lcons(makeString((yyvsp[(1) - (2)].str)), (yyvsp[(2) - (2)].list)), yyscanner); ;} break; - case 1348: -#line 4157 "third_party/libpg_query/grammar/statements/select.y" + case 1351: +#line 4158 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntConst((yyvsp[(1) - (1)].ival), (yylsp[(1) - (1)])); ;} break; - case 1349: -#line 4161 "third_party/libpg_query/grammar/statements/select.y" + case 1352: +#line 4162 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeFloatConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1350: -#line 4165 "third_party/libpg_query/grammar/statements/select.y" + case 1353: +#line 4166 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(2) - (2)].list)) { @@ -30796,15 +30926,15 @@ YYLTYPE yylloc; ;} break; - case 1351: -#line 4177 "third_party/libpg_query/grammar/statements/select.y" + case 1354: +#line 4178 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeBitStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1352: -#line 4181 "third_party/libpg_query/grammar/statements/select.y" + case 1355: +#line 4182 "third_party/libpg_query/grammar/statements/select.y" { /* This is a bit constant per SQL99: * Without Feature F511, "BIT data type", @@ -30815,8 +30945,8 @@ YYLTYPE yylloc; ;} break; - case 1353: -#line 4190 "third_party/libpg_query/grammar/statements/select.y" + case 1356: +#line 4191 "third_party/libpg_query/grammar/statements/select.y" { /* generic type 'literal' syntax */ PGTypeName *t = makeTypeNameFromNameList((yyvsp[(1) - (2)].list)); @@ -30825,8 +30955,8 @@ YYLTYPE yylloc; ;} break; - case 1354: -#line 4197 "third_party/libpg_query/grammar/statements/select.y" + case 1357: +#line 4198 "third_party/libpg_query/grammar/statements/select.y" { /* generic syntax with a type modifier */ PGTypeName *t = makeTypeNameFromNameList((yyvsp[(1) - (7)].list)); @@ -30866,146 +30996,146 @@ YYLTYPE yylloc; ;} break; - case 1355: -#line 4235 "third_party/libpg_query/grammar/statements/select.y" + case 1358: +#line 4236 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeStringConstCast((yyvsp[(2) - (2)].str), (yylsp[(2) - (2)]), (yyvsp[(1) - (2)].typnam)); ;} break; - case 1356: -#line 4239 "third_party/libpg_query/grammar/statements/select.y" + case 1359: +#line 4240 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntervalNode((yyvsp[(3) - (5)].node), (yylsp[(3) - (5)]), (yyvsp[(5) - (5)].list)); ;} break; - case 1357: -#line 4243 "third_party/libpg_query/grammar/statements/select.y" + case 1360: +#line 4244 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntervalNode((yyvsp[(2) - (3)].ival), (yylsp[(2) - (3)]), (yyvsp[(3) - (3)].list)); ;} break; - case 1358: -#line 4247 "third_party/libpg_query/grammar/statements/select.y" + case 1361: +#line 4248 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntervalNode((yyvsp[(2) - (3)].str), (yylsp[(2) - (3)]), (yyvsp[(3) - (3)].list)); ;} break; - case 1359: -#line 4251 "third_party/libpg_query/grammar/statements/select.y" + case 1362: +#line 4252 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeBoolAConst(true, (yylsp[(1) - (1)])); ;} break; - case 1360: -#line 4255 "third_party/libpg_query/grammar/statements/select.y" + case 1363: +#line 4256 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeBoolAConst(false, (yylsp[(1) - (1)])); ;} break; - case 1361: -#line 4259 "third_party/libpg_query/grammar/statements/select.y" + case 1364: +#line 4260 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeNullAConst((yylsp[(1) - (1)])); ;} break; - case 1362: -#line 4264 "third_party/libpg_query/grammar/statements/select.y" + case 1365: +#line 4265 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = (yyvsp[(1) - (1)].ival); ;} break; - case 1363: -#line 4281 "third_party/libpg_query/grammar/statements/select.y" + case 1366: +#line 4282 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1364: -#line 4282 "third_party/libpg_query/grammar/statements/select.y" + case 1367: +#line 4283 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1365: -#line 4283 "third_party/libpg_query/grammar/statements/select.y" + case 1368: +#line 4284 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1366: -#line 4286 "third_party/libpg_query/grammar/statements/select.y" + case 1369: +#line 4287 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1367: -#line 4287 "third_party/libpg_query/grammar/statements/select.y" + case 1370: +#line 4288 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1368: -#line 4288 "third_party/libpg_query/grammar/statements/select.y" + case 1371: +#line 4289 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1369: -#line 4291 "third_party/libpg_query/grammar/statements/select.y" + case 1372: +#line 4292 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1370: -#line 4292 "third_party/libpg_query/grammar/statements/select.y" + case 1373: +#line 4293 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1371: -#line 4293 "third_party/libpg_query/grammar/statements/select.y" + case 1374: +#line 4294 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1372: -#line 4296 "third_party/libpg_query/grammar/statements/select.y" + case 1375: +#line 4297 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1373: -#line 4297 "third_party/libpg_query/grammar/statements/select.y" + case 1376: +#line 4298 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lcons(makeString((yyvsp[(1) - (2)].str)), (yyvsp[(2) - (2)].list)); ;} break; - case 1374: -#line 4301 "third_party/libpg_query/grammar/statements/select.y" + case 1377: +#line 4302 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(2) - (2)].str))); ;} break; - case 1375: -#line 4303 "third_party/libpg_query/grammar/statements/select.y" + case 1378: +#line 4304 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 1376: -#line 4307 "third_party/libpg_query/grammar/statements/select.y" + case 1379: +#line 4308 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 1377: -#line 4308 "third_party/libpg_query/grammar/statements/select.y" + case 1380: +#line 4309 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1379: -#line 4315 "third_party/libpg_query/grammar/statements/select.y" + case 1382: +#line 4316 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1380: -#line 4316 "third_party/libpg_query/grammar/statements/select.y" + case 1383: +#line 4317 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1381: + case 1384: #line 8 "third_party/libpg_query/grammar/statements/prepare.y" { PGPrepareStmt *n = makeNode(PGPrepareStmt); @@ -31016,17 +31146,17 @@ YYLTYPE yylloc; ;} break; - case 1382: + case 1385: #line 18 "third_party/libpg_query/grammar/statements/prepare.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 1383: + case 1386: #line 19 "third_party/libpg_query/grammar/statements/prepare.y" { (yyval.list) = NIL; ;} break; - case 1390: + case 1393: #line 8 "third_party/libpg_query/grammar/statements/create_schema.y" { PGCreateSchemaStmt *n = makeNode(PGCreateSchemaStmt); @@ -31048,7 +31178,7 @@ YYLTYPE yylloc; ;} break; - case 1391: + case 1394: #line 27 "third_party/libpg_query/grammar/statements/create_schema.y" { PGCreateSchemaStmt *n = makeNode(PGCreateSchemaStmt); @@ -31075,7 +31205,7 @@ YYLTYPE yylloc; ;} break; - case 1392: + case 1395: #line 51 "third_party/libpg_query/grammar/statements/create_schema.y" { PGCreateSchemaStmt *n = makeNode(PGCreateSchemaStmt); @@ -31097,7 +31227,7 @@ YYLTYPE yylloc; ;} break; - case 1393: + case 1396: #line 74 "third_party/libpg_query/grammar/statements/create_schema.y" { if ((yyloc) < 0) /* see comments for YYLLOC_DEFAULT */ @@ -31106,12 +31236,12 @@ YYLTYPE yylloc; ;} break; - case 1394: + case 1397: #line 80 "third_party/libpg_query/grammar/statements/create_schema.y" { (yyval.list) = NIL; ;} break; - case 1399: + case 1402: #line 11 "third_party/libpg_query/grammar/statements/index.y" { PGIndexStmt *n = makeNode(PGIndexStmt); @@ -31137,7 +31267,7 @@ YYLTYPE yylloc; ;} break; - case 1400: + case 1403: #line 36 "third_party/libpg_query/grammar/statements/index.y" { PGIndexStmt *n = makeNode(PGIndexStmt); @@ -31163,62 +31293,62 @@ YYLTYPE yylloc; ;} break; - case 1401: + case 1404: #line 62 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1402: + case 1405: #line 66 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = (yyvsp[(2) - (2)].str); ;} break; - case 1403: + case 1406: #line 67 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = (char*) DEFAULT_INDEX_TYPE; ;} break; - case 1404: + case 1407: #line 72 "third_party/libpg_query/grammar/statements/index.y" { (yyval.boolean) = true; ;} break; - case 1405: + case 1408: #line 73 "third_party/libpg_query/grammar/statements/index.y" { (yyval.boolean) = false; ;} break; - case 1406: + case 1409: #line 78 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1407: + case 1410: #line 79 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = NULL; ;} break; - case 1408: + case 1411: #line 83 "third_party/libpg_query/grammar/statements/index.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 1409: + case 1412: #line 84 "third_party/libpg_query/grammar/statements/index.y" { (yyval.list) = NIL; ;} break; - case 1410: + case 1413: #line 89 "third_party/libpg_query/grammar/statements/index.y" { (yyval.boolean) = true; ;} break; - case 1411: + case 1414: #line 90 "third_party/libpg_query/grammar/statements/index.y" { (yyval.boolean) = false; ;} break; - case 1412: + case 1415: #line 8 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -31230,7 +31360,7 @@ YYLTYPE yylloc; ;} break; - case 1413: + case 1416: #line 17 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -31242,7 +31372,7 @@ YYLTYPE yylloc; ;} break; - case 1414: + case 1417: #line 26 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -31254,7 +31384,7 @@ YYLTYPE yylloc; ;} break; - case 1415: + case 1418: #line 35 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -31266,7 +31396,7 @@ YYLTYPE yylloc; ;} break; - case 1416: + case 1419: #line 44 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -31278,7 +31408,7 @@ YYLTYPE yylloc; ;} break; - case 1417: + case 1420: #line 53 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -31290,7 +31420,7 @@ YYLTYPE yylloc; ;} break; - case 1418: + case 1421: #line 6 "third_party/libpg_query/grammar/statements/checkpoint.y" { PGCheckPointStmt *n = makeNode(PGCheckPointStmt); @@ -31300,7 +31430,7 @@ YYLTYPE yylloc; ;} break; - case 1419: + case 1422: #line 13 "third_party/libpg_query/grammar/statements/checkpoint.y" { PGCheckPointStmt *n = makeNode(PGCheckPointStmt); @@ -31310,17 +31440,17 @@ YYLTYPE yylloc; ;} break; - case 1420: + case 1423: #line 22 "third_party/libpg_query/grammar/statements/checkpoint.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1421: + case 1424: #line 23 "third_party/libpg_query/grammar/statements/checkpoint.y" { (yyval.str) = NULL; ;} break; - case 1422: + case 1425: #line 8 "third_party/libpg_query/grammar/statements/comment_on.y" { PGCommentOnStmt *n = makeNode(PGCommentOnStmt); @@ -31331,7 +31461,7 @@ YYLTYPE yylloc; ;} break; - case 1423: + case 1426: #line 16 "third_party/libpg_query/grammar/statements/comment_on.y" { PGCommentOnStmt *n = makeNode(PGCommentOnStmt); @@ -31342,67 +31472,67 @@ YYLTYPE yylloc; ;} break; - case 1424: + case 1427: #line 26 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.node) = makeStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1425: + case 1428: #line 27 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.node) = makeNullAConst((yylsp[(1) - (1)])); ;} break; - case 1426: + case 1429: #line 30 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_TABLE; ;} break; - case 1427: + case 1430: #line 31 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_SEQUENCE; ;} break; - case 1428: + case 1431: #line 32 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_FUNCTION; ;} break; - case 1429: + case 1432: #line 33 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_FUNCTION; ;} break; - case 1430: + case 1433: #line 34 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_TABLE_MACRO; ;} break; - case 1431: + case 1434: #line 35 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_VIEW; ;} break; - case 1432: + case 1435: #line 36 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_DATABASE; ;} break; - case 1433: + case 1436: #line 37 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_INDEX; ;} break; - case 1434: + case 1437: #line 38 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_SCHEMA; ;} break; - case 1435: + case 1438: #line 39 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_TYPE; ;} break; - case 1436: + case 1439: #line 8 "third_party/libpg_query/grammar/statements/export.y" { PGExportStmt *n = makeNode(PGExportStmt); @@ -31416,7 +31546,7 @@ YYLTYPE yylloc; ;} break; - case 1437: + case 1440: #line 20 "third_party/libpg_query/grammar/statements/export.y" { PGExportStmt *n = makeNode(PGExportStmt); @@ -31430,7 +31560,7 @@ YYLTYPE yylloc; ;} break; - case 1438: + case 1441: #line 34 "third_party/libpg_query/grammar/statements/export.y" { PGImportStmt *n = makeNode(PGImportStmt); @@ -31439,7 +31569,7 @@ YYLTYPE yylloc; ;} break; - case 1439: + case 1442: #line 10 "third_party/libpg_query/grammar/statements/explain.y" { PGExplainStmt *n = makeNode(PGExplainStmt); @@ -31449,7 +31579,7 @@ YYLTYPE yylloc; ;} break; - case 1440: + case 1443: #line 17 "third_party/libpg_query/grammar/statements/explain.y" { PGExplainStmt *n = makeNode(PGExplainStmt); @@ -31462,7 +31592,7 @@ YYLTYPE yylloc; ;} break; - case 1441: + case 1444: #line 27 "third_party/libpg_query/grammar/statements/explain.y" { PGExplainStmt *n = makeNode(PGExplainStmt); @@ -31472,7 +31602,7 @@ YYLTYPE yylloc; ;} break; - case 1442: + case 1445: #line 34 "third_party/libpg_query/grammar/statements/explain.y" { PGExplainStmt *n = makeNode(PGExplainStmt); @@ -31482,118 +31612,118 @@ YYLTYPE yylloc; ;} break; - case 1443: + case 1446: #line 44 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.boolean) = true; ;} break; - case 1444: + case 1447: #line 45 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.boolean) = false; ;} break; - case 1445: + case 1448: #line 50 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.node) = (PGNode *) makeString((yyvsp[(1) - (1)].str)); ;} break; - case 1446: + case 1449: #line 51 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.node) = (PGNode *) (yyvsp[(1) - (1)].value); ;} break; - case 1447: + case 1450: #line 52 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.node) = NULL; ;} break; - case 1480: + case 1483: #line 92 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1481: + case 1484: #line 93 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1482: + case 1485: #line 94 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1483: + case 1486: #line 99 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1484: + case 1487: #line 100 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1485: + case 1488: #line 106 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 1486: + case 1489: #line 110 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].defelt)); ;} break; - case 1487: + case 1490: #line 117 "third_party/libpg_query/grammar/statements/explain.y" {;} break; - case 1488: + case 1491: #line 118 "third_party/libpg_query/grammar/statements/explain.y" {;} break; - case 1489: + case 1492: #line 123 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (char*) "true"; ;} break; - case 1490: + case 1493: #line 124 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (char*) "false"; ;} break; - case 1491: + case 1494: #line 125 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (char*) "on"; ;} break; - case 1492: + case 1495: #line 131 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1493: + case 1496: #line 137 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (2)].str), (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 1494: + case 1497: #line 144 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1495: + case 1498: #line 145 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (char*) "analyze"; ;} break; - case 1496: + case 1499: #line 11 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = (yyvsp[(2) - (2)].vsetstmt); @@ -31602,7 +31732,7 @@ YYLTYPE yylloc; ;} break; - case 1497: + case 1500: #line 17 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = (yyvsp[(3) - (3)].vsetstmt); @@ -31611,7 +31741,7 @@ YYLTYPE yylloc; ;} break; - case 1498: + case 1501: #line 23 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = (yyvsp[(3) - (3)].vsetstmt); @@ -31620,7 +31750,7 @@ YYLTYPE yylloc; ;} break; - case 1499: + case 1502: #line 29 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = (yyvsp[(3) - (3)].vsetstmt); @@ -31629,7 +31759,7 @@ YYLTYPE yylloc; ;} break; - case 1500: + case 1503: #line 35 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = (yyvsp[(3) - (3)].vsetstmt); @@ -31638,12 +31768,12 @@ YYLTYPE yylloc; ;} break; - case 1501: + case 1504: #line 44 "third_party/libpg_query/grammar/statements/variable_set.y" {(yyval.vsetstmt) = (yyvsp[(1) - (1)].vsetstmt);;} break; - case 1502: + case 1505: #line 46 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -31653,7 +31783,7 @@ YYLTYPE yylloc; ;} break; - case 1503: + case 1506: #line 54 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -31667,7 +31797,7 @@ YYLTYPE yylloc; ;} break; - case 1504: + case 1507: #line 65 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -31678,7 +31808,7 @@ YYLTYPE yylloc; ;} break; - case 1505: + case 1508: #line 77 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -31689,7 +31819,7 @@ YYLTYPE yylloc; ;} break; - case 1506: + case 1509: #line 85 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -31700,26 +31830,26 @@ YYLTYPE yylloc; ;} break; - case 1507: + case 1510: #line 96 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1508: + case 1511: #line 102 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = makeStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1509: + case 1512: #line 106 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = makeStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1510: + case 1513: #line 110 "third_party/libpg_query/grammar/statements/variable_set.y" { PGTypeName *t = (yyvsp[(1) - (3)].typnam); @@ -31737,7 +31867,7 @@ YYLTYPE yylloc; ;} break; - case 1511: + case 1514: #line 125 "third_party/libpg_query/grammar/statements/variable_set.y" { PGTypeName *t = (yyvsp[(1) - (5)].typnam); @@ -31747,32 +31877,32 @@ YYLTYPE yylloc; ;} break; - case 1512: + case 1515: #line 131 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = makeAConst((yyvsp[(1) - (1)].value), (yylsp[(1) - (1)])); ;} break; - case 1513: + case 1516: #line 132 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = NULL; ;} break; - case 1514: + case 1517: #line 133 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = NULL; ;} break; - case 1515: + case 1518: #line 137 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1516: + case 1519: #line 138 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 1517: + case 1520: #line 8 "third_party/libpg_query/grammar/statements/load.y" { PGLoadStmt *n = makeNode(PGLoadStmt); @@ -31785,7 +31915,7 @@ YYLTYPE yylloc; ;} break; - case 1518: + case 1521: #line 17 "third_party/libpg_query/grammar/statements/load.y" { PGLoadStmt *n = makeNode(PGLoadStmt); @@ -31798,7 +31928,7 @@ YYLTYPE yylloc; ;} break; - case 1519: + case 1522: #line 26 "third_party/libpg_query/grammar/statements/load.y" { PGLoadStmt *n = makeNode(PGLoadStmt); @@ -31811,7 +31941,7 @@ YYLTYPE yylloc; ;} break; - case 1520: + case 1523: #line 35 "third_party/libpg_query/grammar/statements/load.y" { PGLoadStmt *n = makeNode(PGLoadStmt); @@ -31824,42 +31954,42 @@ YYLTYPE yylloc; ;} break; - case 1521: + case 1524: #line 46 "third_party/libpg_query/grammar/statements/load.y" { (yyval.loadinstalltype) = PG_LOAD_TYPE_INSTALL; ;} break; - case 1522: + case 1525: #line 47 "third_party/libpg_query/grammar/statements/load.y" { (yyval.loadinstalltype) = PG_LOAD_TYPE_FORCE_INSTALL; ;} break; - case 1523: + case 1526: #line 49 "third_party/libpg_query/grammar/statements/load.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1524: + case 1527: #line 50 "third_party/libpg_query/grammar/statements/load.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1525: + case 1528: #line 53 "third_party/libpg_query/grammar/statements/load.y" { (yyval.str) = NULL; ;} break; - case 1526: + case 1529: #line 54 "third_party/libpg_query/grammar/statements/load.y" { (yyval.str) = (yyvsp[(2) - (2)].str); ;} break; - case 1527: + case 1530: #line 55 "third_party/libpg_query/grammar/statements/load.y" { (yyval.str) = (yyvsp[(2) - (2)].str); ;} break; - case 1528: + case 1531: #line 9 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -31876,7 +32006,7 @@ YYLTYPE yylloc; ;} break; - case 1529: + case 1532: #line 23 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -31893,7 +32023,7 @@ YYLTYPE yylloc; ;} break; - case 1530: + case 1533: #line 37 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = (PGVacuumStmt *) (yyvsp[(5) - (5)].node); @@ -31908,7 +32038,7 @@ YYLTYPE yylloc; ;} break; - case 1531: + case 1534: #line 49 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -31919,7 +32049,7 @@ YYLTYPE yylloc; ;} break; - case 1532: + case 1535: #line 57 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -31932,27 +32062,27 @@ YYLTYPE yylloc; ;} break; - case 1533: + case 1536: #line 70 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = PG_VACOPT_ANALYZE; ;} break; - case 1534: + case 1537: #line 71 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = PG_VACOPT_VERBOSE; ;} break; - case 1535: + case 1538: #line 72 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = PG_VACOPT_FREEZE; ;} break; - case 1536: + case 1539: #line 73 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = PG_VACOPT_FULL; ;} break; - case 1537: + case 1540: #line 75 "third_party/libpg_query/grammar/statements/vacuum.y" { if (strcmp((yyvsp[(1) - (1)].str), "disable_page_skipping") == 0) @@ -31965,37 +32095,37 @@ YYLTYPE yylloc; ;} break; - case 1538: + case 1541: #line 87 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.boolean) = true; ;} break; - case 1539: + case 1542: #line 88 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.boolean) = false; ;} break; - case 1540: + case 1543: #line 93 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = (yyvsp[(1) - (1)].ival); ;} break; - case 1541: + case 1544: #line 94 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = (yyvsp[(1) - (3)].ival) | (yyvsp[(3) - (3)].ival); ;} break; - case 1542: + case 1545: #line 98 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.boolean) = true; ;} break; - case 1543: + case 1546: #line 99 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.boolean) = false; ;} break; - case 1544: + case 1547: #line 9 "third_party/libpg_query/grammar/statements/delete.y" { PGDeleteStmt *n = makeNode(PGDeleteStmt); @@ -32008,7 +32138,7 @@ YYLTYPE yylloc; ;} break; - case 1545: + case 1548: #line 19 "third_party/libpg_query/grammar/statements/delete.y" { PGDeleteStmt *n = makeNode(PGDeleteStmt); @@ -32021,14 +32151,14 @@ YYLTYPE yylloc; ;} break; - case 1546: + case 1549: #line 32 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.range) = (yyvsp[(1) - (1)].range); ;} break; - case 1547: + case 1550: #line 36 "third_party/libpg_query/grammar/statements/delete.y" { PGAlias *alias = makeNode(PGAlias); @@ -32038,7 +32168,7 @@ YYLTYPE yylloc; ;} break; - case 1548: + case 1551: #line 43 "third_party/libpg_query/grammar/statements/delete.y" { PGAlias *alias = makeNode(PGAlias); @@ -32048,27 +32178,27 @@ YYLTYPE yylloc; ;} break; - case 1549: + case 1552: #line 53 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1550: + case 1553: #line 54 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.node) = NULL; ;} break; - case 1551: + case 1554: #line 60 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 1552: + case 1555: #line 61 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.list) = NIL; ;} break; - case 1553: + case 1556: #line 10 "third_party/libpg_query/grammar/statements/analyze.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -32081,7 +32211,7 @@ YYLTYPE yylloc; ;} break; - case 1554: + case 1557: #line 20 "third_party/libpg_query/grammar/statements/analyze.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -32094,7 +32224,7 @@ YYLTYPE yylloc; ;} break; - case 1555: + case 1558: #line 8 "third_party/libpg_query/grammar/statements/attach.y" { PGAttachStmt *n = makeNode(PGAttachStmt); @@ -32106,7 +32236,7 @@ YYLTYPE yylloc; ;} break; - case 1556: + case 1559: #line 17 "third_party/libpg_query/grammar/statements/attach.y" { PGAttachStmt *n = makeNode(PGAttachStmt); @@ -32118,7 +32248,7 @@ YYLTYPE yylloc; ;} break; - case 1557: + case 1560: #line 26 "third_party/libpg_query/grammar/statements/attach.y" { PGAttachStmt *n = makeNode(PGAttachStmt); @@ -32130,7 +32260,7 @@ YYLTYPE yylloc; ;} break; - case 1558: + case 1561: #line 38 "third_party/libpg_query/grammar/statements/attach.y" { PGDetachStmt *n = makeNode(PGDetachStmt); @@ -32140,7 +32270,7 @@ YYLTYPE yylloc; ;} break; - case 1559: + case 1562: #line 45 "third_party/libpg_query/grammar/statements/attach.y" { PGDetachStmt *n = makeNode(PGDetachStmt); @@ -32150,7 +32280,7 @@ YYLTYPE yylloc; ;} break; - case 1560: + case 1563: #line 52 "third_party/libpg_query/grammar/statements/attach.y" { PGDetachStmt *n = makeNode(PGDetachStmt); @@ -32160,72 +32290,72 @@ YYLTYPE yylloc; ;} break; - case 1561: + case 1564: #line 60 "third_party/libpg_query/grammar/statements/attach.y" {;} break; - case 1562: + case 1565: #line 61 "third_party/libpg_query/grammar/statements/attach.y" {;} break; - case 1563: + case 1566: #line 65 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.str) = (yyvsp[(2) - (2)].str); ;} break; - case 1564: + case 1567: #line 66 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.str) = NULL; ;} break; - case 1565: + case 1568: #line 77 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.node) = (PGNode *) (yyvsp[(1) - (1)].node); ;} break; - case 1566: + case 1569: #line 78 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.node) = NULL; ;} break; - case 1567: + case 1570: #line 83 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (2)].str), (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 1568: + case 1571: #line 90 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 1569: + case 1572: #line 94 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].defelt)); ;} break; - case 1570: + case 1573: #line 101 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 1571: + case 1574: #line 105 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.list) = NULL; ;} break; - case 1572: + case 1575: #line 3 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyvsp[(2) - (2)].vsetstmt)->scope = VAR_SET_SCOPE_DEFAULT; @@ -32233,7 +32363,7 @@ YYLTYPE yylloc; ;} break; - case 1573: + case 1576: #line 8 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyvsp[(3) - (3)].vsetstmt)->scope = VAR_SET_SCOPE_LOCAL; @@ -32241,7 +32371,7 @@ YYLTYPE yylloc; ;} break; - case 1574: + case 1577: #line 13 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyvsp[(3) - (3)].vsetstmt)->scope = VAR_SET_SCOPE_SESSION; @@ -32249,7 +32379,7 @@ YYLTYPE yylloc; ;} break; - case 1575: + case 1578: #line 18 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyvsp[(3) - (3)].vsetstmt)->scope = VAR_SET_SCOPE_GLOBAL; @@ -32257,7 +32387,7 @@ YYLTYPE yylloc; ;} break; - case 1576: + case 1579: #line 23 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyvsp[(3) - (3)].vsetstmt)->scope = VAR_SET_SCOPE_VARIABLE; @@ -32265,7 +32395,7 @@ YYLTYPE yylloc; ;} break; - case 1577: + case 1580: #line 32 "third_party/libpg_query/grammar/statements/variable_reset.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -32275,7 +32405,7 @@ YYLTYPE yylloc; ;} break; - case 1578: + case 1581: #line 39 "third_party/libpg_query/grammar/statements/variable_reset.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -32284,12 +32414,12 @@ YYLTYPE yylloc; ;} break; - case 1579: + case 1582: #line 48 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyval.vsetstmt) = (yyvsp[(1) - (1)].vsetstmt); ;} break; - case 1580: + case 1583: #line 50 "third_party/libpg_query/grammar/statements/variable_reset.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -32299,7 +32429,7 @@ YYLTYPE yylloc; ;} break; - case 1581: + case 1584: #line 57 "third_party/libpg_query/grammar/statements/variable_reset.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -32309,7 +32439,7 @@ YYLTYPE yylloc; ;} break; - case 1582: + case 1585: #line 3 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowSelectStmt *n = makeNode(PGVariableShowSelectStmt); @@ -32320,7 +32450,7 @@ YYLTYPE yylloc; ;} break; - case 1583: + case 1586: #line 10 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowSelectStmt *n = makeNode(PGVariableShowSelectStmt); @@ -32331,7 +32461,7 @@ YYLTYPE yylloc; ;} break; - case 1584: + case 1587: #line 18 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -32341,7 +32471,7 @@ YYLTYPE yylloc; ;} break; - case 1585: + case 1588: #line 25 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -32352,7 +32482,7 @@ YYLTYPE yylloc; ;} break; - case 1586: + case 1589: #line 33 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -32362,7 +32492,7 @@ YYLTYPE yylloc; ;} break; - case 1587: + case 1590: #line 40 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -32372,7 +32502,7 @@ YYLTYPE yylloc; ;} break; - case 1588: + case 1591: #line 47 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -32382,7 +32512,7 @@ YYLTYPE yylloc; ;} break; - case 1589: + case 1592: #line 54 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -32392,7 +32522,7 @@ YYLTYPE yylloc; ;} break; - case 1590: + case 1593: #line 61 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -32402,17 +32532,17 @@ YYLTYPE yylloc; ;} break; - case 1597: + case 1600: #line 75 "third_party/libpg_query/grammar/statements/variable_show.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1598: + case 1601: #line 77 "third_party/libpg_query/grammar/statements/variable_show.y" { (yyval.str) = psprintf("%s.%s", (yyvsp[(1) - (3)].str), (yyvsp[(3) - (3)].str)); ;} break; - case 1599: + case 1602: #line 7 "third_party/libpg_query/grammar/statements/call.y" { PGCallStmt *n = makeNode(PGCallStmt); @@ -32421,7 +32551,7 @@ YYLTYPE yylloc; ;} break; - case 1600: + case 1603: #line 10 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -32436,7 +32566,7 @@ YYLTYPE yylloc; ;} break; - case 1601: + case 1604: #line 23 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -32451,7 +32581,7 @@ YYLTYPE yylloc; ;} break; - case 1602: + case 1605: #line 36 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -32466,7 +32596,7 @@ YYLTYPE yylloc; ;} break; - case 1603: + case 1606: #line 49 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -32486,7 +32616,7 @@ YYLTYPE yylloc; ;} break; - case 1604: + case 1607: #line 67 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -32506,27 +32636,27 @@ YYLTYPE yylloc; ;} break; - case 1605: + case 1608: #line 87 "third_party/libpg_query/grammar/statements/view.y" { (yyval.viewcheckoption) = CASCADED_CHECK_OPTION; ;} break; - case 1606: + case 1609: #line 88 "third_party/libpg_query/grammar/statements/view.y" { (yyval.viewcheckoption) = CASCADED_CHECK_OPTION; ;} break; - case 1607: + case 1610: #line 89 "third_party/libpg_query/grammar/statements/view.y" { (yyval.viewcheckoption) = PG_LOCAL_CHECK_OPTION; ;} break; - case 1608: + case 1611: #line 90 "third_party/libpg_query/grammar/statements/view.y" { (yyval.viewcheckoption) = PG_NO_CHECK_OPTION; ;} break; - case 1609: + case 1612: #line 12 "third_party/libpg_query/grammar/statements/create_as.y" { PGCreateTableAsStmt *ctas = makeNode(PGCreateTableAsStmt); @@ -32542,7 +32672,7 @@ YYLTYPE yylloc; ;} break; - case 1610: + case 1613: #line 25 "third_party/libpg_query/grammar/statements/create_as.y" { PGCreateTableAsStmt *ctas = makeNode(PGCreateTableAsStmt); @@ -32558,7 +32688,7 @@ YYLTYPE yylloc; ;} break; - case 1611: + case 1614: #line 38 "third_party/libpg_query/grammar/statements/create_as.y" { PGCreateTableAsStmt *ctas = makeNode(PGCreateTableAsStmt); @@ -32574,22 +32704,22 @@ YYLTYPE yylloc; ;} break; - case 1612: + case 1615: #line 54 "third_party/libpg_query/grammar/statements/create_as.y" { (yyval.boolean) = true; ;} break; - case 1613: + case 1616: #line 55 "third_party/libpg_query/grammar/statements/create_as.y" { (yyval.boolean) = false; ;} break; - case 1614: + case 1617: #line 56 "third_party/libpg_query/grammar/statements/create_as.y" { (yyval.boolean) = true; ;} break; - case 1615: + case 1618: #line 62 "third_party/libpg_query/grammar/statements/create_as.y" { (yyval.into) = makeNode(PGIntoClause); @@ -32604,7 +32734,7 @@ YYLTYPE yylloc; /* Line 1267 of yacc.c. */ -#line 32608 "third_party/libpg_query/grammar/grammar_out.cpp" +#line 32738 "third_party/libpg_query/grammar/grammar_out.cpp" default: break; } YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); diff --git a/third_party/mbedtls/README.md b/third_party/mbedtls/README.md new file mode 100644 index 000000000000..b2d54b811cfc --- /dev/null +++ b/third_party/mbedtls/README.md @@ -0,0 +1,18 @@ +In order to bump mbedtls, we can run the helper script to download a new version and copy over existing files: + +``` +python inline_mbedtls.py +``` + +We then need to make it work in C++ and as part of our amalgamation. We have a diff available that fixes these issues for the current version. It is possible / likely this does not map 1-1 to the new version, so rejects might need to be handled. + +``` +git apply inline_mbedtls.diff --reject +``` + +It is recommended to test regular compilation, as well as the amalgamation path: + +``` +python scripts/amalgamation.py +clang++ -std=c++11 -Isrc/amalgamation src/amalgamation/duckdb.cpp +``` diff --git a/third_party/mbedtls/VERSION b/third_party/mbedtls/VERSION index 77a069e39b86..c3f6bdc3de94 100644 --- a/third_party/mbedtls/VERSION +++ b/third_party/mbedtls/VERSION @@ -1 +1 @@ -3.6.2 \ No newline at end of file +3.6.4 \ No newline at end of file diff --git a/third_party/mbedtls/include/des_alt.h b/third_party/mbedtls/include/des_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/des_alt.h +++ b/third_party/mbedtls/include/des_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/aes_alt.h b/third_party/mbedtls/include/mbedtls/aes_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/aes_alt.h +++ b/third_party/mbedtls/include/mbedtls/aes_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/aria_alt.h b/third_party/mbedtls/include/mbedtls/aria_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/aria_alt.h +++ b/third_party/mbedtls/include/mbedtls/aria_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/block_cipher.h b/third_party/mbedtls/include/mbedtls/block_cipher.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/block_cipher.h +++ b/third_party/mbedtls/include/mbedtls/block_cipher.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/build_info.h b/third_party/mbedtls/include/mbedtls/build_info.h index d91d2964b6ae..e18e823c15e4 100644 --- a/third_party/mbedtls/include/mbedtls/build_info.h +++ b/third_party/mbedtls/include/mbedtls/build_info.h @@ -26,16 +26,16 @@ */ #define MBEDTLS_VERSION_MAJOR 3 #define MBEDTLS_VERSION_MINOR 6 -#define MBEDTLS_VERSION_PATCH 2 +#define MBEDTLS_VERSION_PATCH 4 /** * The single version number has the following structure: * MMNNPP00 * Major version | Minor version | Patch version */ -#define MBEDTLS_VERSION_NUMBER 0x03060200 -#define MBEDTLS_VERSION_STRING "3.6.2" -#define MBEDTLS_VERSION_STRING_FULL "Mbed TLS 3.6.2" +#define MBEDTLS_VERSION_NUMBER 0x03060400 +#define MBEDTLS_VERSION_STRING "3.6.4" +#define MBEDTLS_VERSION_STRING_FULL "Mbed TLS 3.6.4" /* Macros for build-time platform detection */ diff --git a/third_party/mbedtls/include/mbedtls/camellia_alt.h b/third_party/mbedtls/include/mbedtls/camellia_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/camellia_alt.h +++ b/third_party/mbedtls/include/mbedtls/camellia_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/ccm_alt.h b/third_party/mbedtls/include/mbedtls/ccm_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/ccm_alt.h +++ b/third_party/mbedtls/include/mbedtls/ccm_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/chacha20.h b/third_party/mbedtls/include/mbedtls/chacha20.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/chacha20.h +++ b/third_party/mbedtls/include/mbedtls/chacha20.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/chachapoly.h b/third_party/mbedtls/include/mbedtls/chachapoly.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/chachapoly.h +++ b/third_party/mbedtls/include/mbedtls/chachapoly.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/cmac.h b/third_party/mbedtls/include/mbedtls/cmac.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/cmac.h +++ b/third_party/mbedtls/include/mbedtls/cmac.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/config_psa.h b/third_party/mbedtls/include/mbedtls/config_psa.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/config_psa.h +++ b/third_party/mbedtls/include/mbedtls/config_psa.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/ecdsa.h b/third_party/mbedtls/include/mbedtls/ecdsa.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/ecdsa.h +++ b/third_party/mbedtls/include/mbedtls/ecdsa.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/gcm_alt.h b/third_party/mbedtls/include/mbedtls/gcm_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/gcm_alt.h +++ b/third_party/mbedtls/include/mbedtls/gcm_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/mbedtls_config.h b/third_party/mbedtls/include/mbedtls/mbedtls_config.h index 33d55af9a8ea..65943829c75f 100644 --- a/third_party/mbedtls/include/mbedtls/mbedtls_config.h +++ b/third_party/mbedtls/include/mbedtls/mbedtls_config.h @@ -22,4 +22,8 @@ #define MBEDTLS_PLATFORM_C #define MBEDTLS_RSA_C #define MBEDTLS_SHA1_C -#define MBEDTLS_SHA256_C \ No newline at end of file +#define MBEDTLS_SHA256_C +#define MBEDTLS_CIPHER_MODE_CTR +#define MBEDTLS_CIPHER_MODE_CBC +#define MBEDTLS_CIPHER_MODE_WITH_PADDING +#define MBEDTLS_CIPHER_PADDING_PKCS7 \ No newline at end of file diff --git a/third_party/mbedtls/include/mbedtls/md5.h b/third_party/mbedtls/include/mbedtls/md5.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/md5.h +++ b/third_party/mbedtls/include/mbedtls/md5.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/nist_kw.h b/third_party/mbedtls/include/mbedtls/nist_kw.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/nist_kw.h +++ b/third_party/mbedtls/include/mbedtls/nist_kw.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/pkcs12.h b/third_party/mbedtls/include/mbedtls/pkcs12.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/pkcs12.h +++ b/third_party/mbedtls/include/mbedtls/pkcs12.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/pkcs5.h b/third_party/mbedtls/include/mbedtls/pkcs5.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/pkcs5.h +++ b/third_party/mbedtls/include/mbedtls/pkcs5.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/psa_util.h b/third_party/mbedtls/include/mbedtls/psa_util.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/psa_util.h +++ b/third_party/mbedtls/include/mbedtls/psa_util.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/ripemd160.h b/third_party/mbedtls/include/mbedtls/ripemd160.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/ripemd160.h +++ b/third_party/mbedtls/include/mbedtls/ripemd160.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/sha3.h b/third_party/mbedtls/include/mbedtls/sha3.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/sha3.h +++ b/third_party/mbedtls/include/mbedtls/sha3.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/threading.h b/third_party/mbedtls/include/mbedtls/threading.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/threading.h +++ b/third_party/mbedtls/include/mbedtls/threading.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls/timing.h b/third_party/mbedtls/include/mbedtls/timing.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/mbedtls/timing.h +++ b/third_party/mbedtls/include/mbedtls/timing.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/mbedtls_wrapper.hpp b/third_party/mbedtls/include/mbedtls_wrapper.hpp index dc81abba848c..d9f8111d8095 100644 --- a/third_party/mbedtls/include/mbedtls_wrapper.hpp +++ b/third_party/mbedtls/include/mbedtls_wrapper.hpp @@ -19,6 +19,8 @@ typedef struct mbedtls_cipher_info_t mbedtls_cipher_info_t; namespace duckdb_mbedtls { + + class MbedTlsWrapper { public: static void ComputeSha256Hash(const char *in, size_t in_len, char *out); @@ -64,7 +66,7 @@ class MbedTlsWrapper { class AESStateMBEDTLS : public duckdb::EncryptionState { public: - DUCKDB_API explicit AESStateMBEDTLS(duckdb::const_data_ptr_t key = nullptr, duckdb::idx_t key_len = 0); + DUCKDB_API explicit AESStateMBEDTLS(duckdb::EncryptionTypes::CipherType cipher_p, duckdb::idx_t key_len); DUCKDB_API ~AESStateMBEDTLS() override; public: @@ -84,16 +86,15 @@ class AESStateMBEDTLS : public duckdb::EncryptionState { DUCKDB_API void InitializeInternal(duckdb::const_data_ptr_t iv, duckdb::idx_t iv_len, duckdb::const_data_ptr_t aad, duckdb::idx_t aad_len); private: - Mode mode; - Cipher cipher = GCM; + duckdb::EncryptionTypes::Mode mode; duckdb::unique_ptr context; }; class AESStateMBEDTLSFactory : public duckdb::EncryptionUtil { public: - duckdb::shared_ptr CreateEncryptionState(duckdb::const_data_ptr_t key = nullptr, duckdb::idx_t key_len = 0) const override { - return duckdb::make_shared_ptr(key, key_len); + duckdb::shared_ptr CreateEncryptionState(duckdb::EncryptionTypes::CipherType cipher_p, duckdb::idx_t key_len = 0) const override { + return duckdb::make_shared_ptr(cipher_p, key_len); } ~AESStateMBEDTLSFactory() override {} // diff --git a/third_party/mbedtls/include/platform_alt.h b/third_party/mbedtls/include/platform_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/platform_alt.h +++ b/third_party/mbedtls/include/platform_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/psa/build_info.h b/third_party/mbedtls/include/psa/build_info.h new file mode 100644 index 000000000000..3ee6cd7b1bca --- /dev/null +++ b/third_party/mbedtls/include/psa/build_info.h @@ -0,0 +1,20 @@ +/** + * \file psa/build_info.h + * + * \brief Build-time PSA configuration info + * + * Include this file if you need to depend on the + * configuration options defined in mbedtls_config.h or MBEDTLS_CONFIG_FILE + * in PSA cryptography core specific files. + */ +/* + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ + +#ifndef PSA_CRYPTO_BUILD_INFO_H +#define PSA_CRYPTO_BUILD_INFO_H + +#include "mbedtls/build_info.h" + +#endif /* PSA_CRYPTO_BUILD_INFO_H */ diff --git a/third_party/mbedtls/include/psa/crypto.h b/third_party/mbedtls/include/psa/crypto.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/psa/crypto.h +++ b/third_party/mbedtls/include/psa/crypto.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/psa/crypto_config.h b/third_party/mbedtls/include/psa/crypto_config.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/psa/crypto_config.h +++ b/third_party/mbedtls/include/psa/crypto_config.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/psa/crypto_se_driver.h b/third_party/mbedtls/include/psa/crypto_se_driver.h new file mode 100644 index 000000000000..b262fa10a3a2 --- /dev/null +++ b/third_party/mbedtls/include/psa/crypto_se_driver.h @@ -0,0 +1 @@ +// dummy file diff --git a/third_party/mbedtls/include/rsa_alt.h b/third_party/mbedtls/include/rsa_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/rsa_alt.h +++ b/third_party/mbedtls/include/rsa_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/sha1_alt.h b/third_party/mbedtls/include/sha1_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/sha1_alt.h +++ b/third_party/mbedtls/include/sha1_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/sha256_alt.h b/third_party/mbedtls/include/sha256_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/sha256_alt.h +++ b/third_party/mbedtls/include/sha256_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/sha512_alt.h b/third_party/mbedtls/include/sha512_alt.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/sha512_alt.h +++ b/third_party/mbedtls/include/sha512_alt.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/include/ssl_misc.h b/third_party/mbedtls/include/ssl_misc.h index 6b63dc03793e..ae22b408f869 100644 --- a/third_party/mbedtls/include/ssl_misc.h +++ b/third_party/mbedtls/include/ssl_misc.h @@ -1 +1 @@ -// dummy file to make amalgamantion happy +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/inline_mbedtls.diff b/third_party/mbedtls/inline_mbedtls.diff new file mode 100644 index 000000000000..3489fb44f907 --- /dev/null +++ b/third_party/mbedtls/inline_mbedtls.diff @@ -0,0 +1,584 @@ +diff --git a/third_party/mbedtls/VERSION b/third_party/mbedtls/VERSION +index 77a069e39b..c3f6bdc3de 100644 +--- a/third_party/mbedtls/VERSION ++++ b/third_party/mbedtls/VERSION +@@ -1 +1 @@ +-3.6.2 +\ No newline at end of file ++3.6.4 +\ No newline at end of file +diff --git a/third_party/mbedtls/library/aes.cpp b/third_party/mbedtls/library/aes.cpp +index b1a5c3ed10..00d64c6173 100644 +--- a/third_party/mbedtls/library/aes.cpp ++++ b/third_party/mbedtls/library/aes.cpp +@@ -52,7 +52,6 @@ + #include "aesce.h" + #endif + +-#include "mbedtls/platform.h" + #include "ctr.h" + + /* +diff --git a/third_party/mbedtls/library/asn1parse.cpp b/third_party/mbedtls/library/asn1parse.cpp +index e33fdf71da..8db38d8921 100644 +--- a/third_party/mbedtls/library/asn1parse.cpp ++++ b/third_party/mbedtls/library/asn1parse.cpp +@@ -315,7 +315,7 @@ static int asn1_get_sequence_of_cb(void *ctx, + cb_ctx->cur; + + if (cur->buf.p != NULL) { +- cur->next = ++ cur->next = (mbedtls_asn1_sequence *) + mbedtls_calloc(1, sizeof(mbedtls_asn1_sequence)); + + if (cur->next == NULL) { +diff --git a/third_party/mbedtls/library/asn1write.cpp b/third_party/mbedtls/library/asn1write.cpp +index 97f9db039b..431c7ba8bb 100644 +--- a/third_party/mbedtls/library/asn1write.cpp ++++ b/third_party/mbedtls/library/asn1write.cpp +@@ -391,7 +391,7 @@ mbedtls_asn1_named_data *mbedtls_asn1_store_named_data( + } + + cur->oid.len = oid_len; +- cur->oid.p = mbedtls_calloc(1, oid_len); ++ cur->oid.p = (unsigned char *) mbedtls_calloc(1, oid_len); + if (cur->oid.p == NULL) { + mbedtls_free(cur); + return NULL; +@@ -401,7 +401,7 @@ mbedtls_asn1_named_data *mbedtls_asn1_store_named_data( + + cur->val.len = val_len; + if (val_len != 0) { +- cur->val.p = mbedtls_calloc(1, val_len); ++ cur->val.p = (unsigned char *) mbedtls_calloc(1, val_len); + if (cur->val.p == NULL) { + mbedtls_free(cur->oid.p); + mbedtls_free(cur); +@@ -421,13 +421,13 @@ mbedtls_asn1_named_data *mbedtls_asn1_store_named_data( + * Preserve old data until the allocation succeeded, to leave list in + * a consistent state in case allocation fails. + */ +- void *p = mbedtls_calloc(1, val_len); ++ void *p = (unsigned char *) mbedtls_calloc(1, val_len); + if (p == NULL) { + return NULL; + } + + mbedtls_free(cur->val.p); +- cur->val.p = p; ++ cur->val.p = (unsigned char *) p; + cur->val.len = val_len; + } + +diff --git a/third_party/mbedtls/library/bignum.cpp b/third_party/mbedtls/library/bignum.cpp +index 424490951d..3e743b0291 100644 +--- a/third_party/mbedtls/library/bignum.cpp ++++ b/third_party/mbedtls/library/bignum.cpp +@@ -88,7 +88,7 @@ int mbedtls_mpi_lt_mpi_ct(const mbedtls_mpi *X, + /* This array is used to conditionally swap the pointers in const time */ + void * const p[2] = { X->p, Y->p }; + size_t i = mbedtls_ct_size_if_else_0(X_is_negative, 1); +- mbedtls_ct_condition_t lt = mbedtls_mpi_core_lt_ct(p[i], p[i ^ 1], X->n); ++ mbedtls_ct_condition_t lt = mbedtls_mpi_core_lt_ct((const mbedtls_mpi_uint *) p[i], (const mbedtls_mpi_uint *) p[i ^ 1], X->n); + + /* + * Store in result iff the signs are the same (i.e., iff different_sign == false). If +diff --git a/third_party/mbedtls/library/bignum_core.cpp b/third_party/mbedtls/library/bignum_core.cpp +index 88582c2d38..c8bc21c5f9 100644 +--- a/third_party/mbedtls/library/bignum_core.cpp ++++ b/third_party/mbedtls/library/bignum_core.cpp +@@ -19,7 +19,6 @@ + + #include "bignum_core.h" + #include "bn_mul.h" +-#include "constant_time_internal.h" + + size_t mbedtls_mpi_core_clz(mbedtls_mpi_uint a) + { +diff --git a/third_party/mbedtls/library/cipher.cpp b/third_party/mbedtls/library/cipher.cpp +index 2ae01dd84d..8d473a7597 100644 +--- a/third_party/mbedtls/library/cipher.cpp ++++ b/third_party/mbedtls/library/cipher.cpp +@@ -1451,7 +1451,7 @@ static int mbedtls_cipher_aead_encrypt(mbedtls_cipher_context_t *ctx, + #if defined(MBEDTLS_GCM_C) + if (MBEDTLS_MODE_GCM == ((mbedtls_cipher_mode_t) ctx->cipher_info->mode)) { + *olen = ilen; +- return mbedtls_gcm_crypt_and_tag(ctx->cipher_ctx, MBEDTLS_GCM_ENCRYPT, ++ return mbedtls_gcm_crypt_and_tag((mbedtls_gcm_context *) ctx->cipher_ctx, MBEDTLS_GCM_ENCRYPT, + ilen, iv, iv_len, ad, ad_len, + input, output, tag_len, tag); + } +@@ -1531,7 +1531,7 @@ static int mbedtls_cipher_aead_decrypt(mbedtls_cipher_context_t *ctx, + int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; + + *olen = ilen; +- ret = mbedtls_gcm_auth_decrypt(ctx->cipher_ctx, ilen, ++ ret = mbedtls_gcm_auth_decrypt((mbedtls_gcm_context *) ctx->cipher_ctx, ilen, + iv, iv_len, ad, ad_len, + tag, tag_len, input, output); + +diff --git a/third_party/mbedtls/library/cipher_wrap.cpp b/third_party/mbedtls/library/cipher_wrap.cpp +index b4c21ee395..005e99d313 100644 +--- a/third_party/mbedtls/library/cipher_wrap.cpp ++++ b/third_party/mbedtls/library/cipher_wrap.cpp +@@ -131,7 +131,7 @@ static void *gcm_ctx_alloc(void) + + static void gcm_ctx_free(void *ctx) + { +- mbedtls_gcm_free(ctx); ++ mbedtls_gcm_free((mbedtls_gcm_context *) ctx); + mbedtls_free(ctx); + } + #endif /* MBEDTLS_GCM_C */ +@@ -246,7 +246,7 @@ static int aes_setkey_enc_wrap(void *ctx, const unsigned char *key, + + static void *aes_ctx_alloc(void) + { +- mbedtls_aes_context *aes = mbedtls_calloc(1, sizeof(mbedtls_aes_context)); ++ mbedtls_aes_context *aes = (mbedtls_aes_context *) mbedtls_calloc(1, sizeof(mbedtls_aes_context)); + + if (aes == NULL) { + return NULL; +diff --git a/third_party/mbedtls/library/constant_time.cpp b/third_party/mbedtls/library/constant_time.cpp +index d212ddfd81..285f963b3c 100644 +--- a/third_party/mbedtls/library/constant_time.cpp ++++ b/third_party/mbedtls/library/constant_time.cpp +@@ -150,7 +150,7 @@ int mbedtls_ct_memcmp_partial(const void *a, + + void mbedtls_ct_memmove_left(void *start, size_t total, size_t offset) + { +- volatile unsigned char *buf = start; ++ volatile unsigned char *buf = (unsigned char *) start; + for (size_t i = 0; i < total; i++) { + mbedtls_ct_condition_t no_op = mbedtls_ct_uint_gt(total - offset, i); + /* The first `total - offset` passes are a no-op. The last +diff --git a/third_party/mbedtls/library/md.cpp b/third_party/mbedtls/library/md.cpp +index c95846aa04..00addd62c3 100644 +--- a/third_party/mbedtls/library/md.cpp ++++ b/third_party/mbedtls/library/md.cpp +@@ -283,7 +283,7 @@ void mbedtls_md_free(mbedtls_md_context_t *ctx) + #endif + #if defined(MBEDTLS_SHA1_C) + case MBEDTLS_MD_SHA1: +- mbedtls_sha1_free(ctx->md_ctx); ++ mbedtls_sha1_free((mbedtls_sha1_context *) ctx->md_ctx); + break; + #endif + #if defined(MBEDTLS_SHA224_C) +@@ -293,7 +293,7 @@ void mbedtls_md_free(mbedtls_md_context_t *ctx) + #endif + #if defined(MBEDTLS_SHA256_C) + case MBEDTLS_MD_SHA256: +- mbedtls_sha256_free(ctx->md_ctx); ++ mbedtls_sha256_free((mbedtls_sha256_context *) ctx->md_ctx); + break; + #endif + #if defined(MBEDTLS_SHA384_C) +@@ -368,7 +368,7 @@ int mbedtls_md_clone(mbedtls_md_context_t *dst, + #endif + #if defined(MBEDTLS_SHA1_C) + case MBEDTLS_MD_SHA1: +- mbedtls_sha1_clone(dst->md_ctx, src->md_ctx); ++ mbedtls_sha1_clone((mbedtls_sha1_context *) dst->md_ctx, (mbedtls_sha1_context *) src->md_ctx); + break; + #endif + #if defined(MBEDTLS_SHA224_C) +@@ -378,7 +378,7 @@ int mbedtls_md_clone(mbedtls_md_context_t *dst, + #endif + #if defined(MBEDTLS_SHA256_C) + case MBEDTLS_MD_SHA256: +- mbedtls_sha256_clone(dst->md_ctx, src->md_ctx); ++ mbedtls_sha256_clone((mbedtls_sha256_context *) dst->md_ctx, (mbedtls_sha256_context *) src->md_ctx); + break; + #endif + #if defined(MBEDTLS_SHA384_C) +@@ -411,7 +411,7 @@ int mbedtls_md_clone(mbedtls_md_context_t *dst, + ctx->md_ctx = mbedtls_calloc(1, sizeof(mbedtls_##type##_context)); \ + if (ctx->md_ctx == NULL) \ + return MBEDTLS_ERR_MD_ALLOC_FAILED; \ +- mbedtls_##type##_init(ctx->md_ctx); \ ++ mbedtls_##type##_init((mbedtls_##type##_context *) ctx->md_ctx); \ + } \ + while (0) + +@@ -535,7 +535,7 @@ int mbedtls_md_starts(mbedtls_md_context_t *ctx) + #endif + #if defined(MBEDTLS_SHA1_C) + case MBEDTLS_MD_SHA1: +- return mbedtls_sha1_starts(ctx->md_ctx); ++ return mbedtls_sha1_starts((mbedtls_sha1_context *)ctx->md_ctx); + #endif + #if defined(MBEDTLS_SHA224_C) + case MBEDTLS_MD_SHA224: +@@ -543,7 +543,7 @@ int mbedtls_md_starts(mbedtls_md_context_t *ctx) + #endif + #if defined(MBEDTLS_SHA256_C) + case MBEDTLS_MD_SHA256: +- return mbedtls_sha256_starts(ctx->md_ctx, 0); ++ return mbedtls_sha256_starts((mbedtls_sha256_context *)ctx->md_ctx, 0); + #endif + #if defined(MBEDTLS_SHA384_C) + case MBEDTLS_MD_SHA384: +@@ -594,7 +594,7 @@ int mbedtls_md_update(mbedtls_md_context_t *ctx, const unsigned char *input, siz + #endif + #if defined(MBEDTLS_SHA1_C) + case MBEDTLS_MD_SHA1: +- return mbedtls_sha1_update(ctx->md_ctx, input, ilen); ++ return mbedtls_sha1_update((mbedtls_sha1_context *)ctx->md_ctx, input, ilen); + #endif + #if defined(MBEDTLS_SHA224_C) + case MBEDTLS_MD_SHA224: +@@ -602,7 +602,7 @@ int mbedtls_md_update(mbedtls_md_context_t *ctx, const unsigned char *input, siz + #endif + #if defined(MBEDTLS_SHA256_C) + case MBEDTLS_MD_SHA256: +- return mbedtls_sha256_update(ctx->md_ctx, input, ilen); ++ return mbedtls_sha256_update((mbedtls_sha256_context *)ctx->md_ctx, input, ilen); + #endif + #if defined(MBEDTLS_SHA384_C) + case MBEDTLS_MD_SHA384: +@@ -652,7 +652,7 @@ int mbedtls_md_finish(mbedtls_md_context_t *ctx, unsigned char *output) + #endif + #if defined(MBEDTLS_SHA1_C) + case MBEDTLS_MD_SHA1: +- return mbedtls_sha1_finish(ctx->md_ctx, output); ++ return mbedtls_sha1_finish((mbedtls_sha1_context *)ctx->md_ctx, output); + #endif + #if defined(MBEDTLS_SHA224_C) + case MBEDTLS_MD_SHA224: +@@ -660,7 +660,7 @@ int mbedtls_md_finish(mbedtls_md_context_t *ctx, unsigned char *output) + #endif + #if defined(MBEDTLS_SHA256_C) + case MBEDTLS_MD_SHA256: +- return mbedtls_sha256_finish(ctx->md_ctx, output); ++ return mbedtls_sha256_finish((mbedtls_sha256_context *)ctx->md_ctx, output); + #endif + #if defined(MBEDTLS_SHA384_C) + case MBEDTLS_MD_SHA384: +diff --git a/third_party/mbedtls/library/oid.cpp b/third_party/mbedtls/library/oid.cpp +index 1d6b1eb866..fdaa52deeb 100644 +--- a/third_party/mbedtls/library/oid.cpp ++++ b/third_party/mbedtls/library/oid.cpp +@@ -1071,7 +1071,7 @@ int mbedtls_oid_from_numeric_string(mbedtls_asn1_buf *oid, + size_t bytes_per_subidentifier = (((sizeof(unsigned int) * 8) - 1) / 7) + + 1; + size_t max_possible_bytes = num_dots * bytes_per_subidentifier; +- oid->p = mbedtls_calloc(max_possible_bytes, 1); ++ oid->p = (unsigned char *) mbedtls_calloc(max_possible_bytes, 1); + if (oid->p == NULL) { + return MBEDTLS_ERR_ASN1_ALLOC_FAILED; + } +@@ -1142,7 +1142,7 @@ int mbedtls_oid_from_numeric_string(mbedtls_asn1_buf *oid, + } + + encoded_len = (size_t) (out_ptr - oid->p); +- resized_mem = mbedtls_calloc(encoded_len, 1); ++ resized_mem = (unsigned char *) mbedtls_calloc(encoded_len, 1); + if (resized_mem == NULL) { + ret = MBEDTLS_ERR_ASN1_ALLOC_FAILED; + goto error; +diff --git a/third_party/mbedtls/library/pem.cpp b/third_party/mbedtls/library/pem.cpp +index 119fd59e12..f2d06b1d22 100644 +--- a/third_party/mbedtls/library/pem.cpp ++++ b/third_party/mbedtls/library/pem.cpp +@@ -419,7 +419,7 @@ int mbedtls_pem_read_buffer(mbedtls_pem_context *ctx, const char *header, const + return MBEDTLS_ERR_PEM_BAD_INPUT_DATA; + } + +- if ((buf = mbedtls_calloc(1, len)) == NULL) { ++ if ((buf = (unsigned char *) mbedtls_calloc(1, len)) == NULL) { + return MBEDTLS_ERR_PEM_ALLOC_FAILED; + } + +diff --git a/third_party/mbedtls/library/pk_wrap.cpp b/third_party/mbedtls/library/pk_wrap.cpp +index 19196b559a..380144ae3d 100644 +--- a/third_party/mbedtls/library/pk_wrap.cpp ++++ b/third_party/mbedtls/library/pk_wrap.cpp +@@ -29,7 +29,6 @@ + #if defined(MBEDTLS_USE_PSA_CRYPTO) + #include "psa_util_internal.h" + #include "psa/crypto.h" +-#include "mbedtls/psa_util.h" + + #if defined(MBEDTLS_RSA_C) + #include "pkwrite.h" +diff --git a/third_party/mbedtls/library/pkparse.cpp b/third_party/mbedtls/library/pkparse.cpp +index 4f6ee13986..6f478f0120 100644 +--- a/third_party/mbedtls/library/pkparse.cpp ++++ b/third_party/mbedtls/library/pkparse.cpp +@@ -471,7 +471,7 @@ static int pk_parse_key_rfc8410_der(mbedtls_pk_context *pk, + static int pk_get_pk_alg(unsigned char **p, + const unsigned char *end, + mbedtls_pk_type_t *pk_alg, mbedtls_asn1_buf *params, +- mbedtls_ecp_group_id *ec_grp_id) ++ void *ec_grp_id) + { + int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; + mbedtls_asn1_buf alg_oid; +@@ -485,7 +485,7 @@ static int pk_get_pk_alg(unsigned char **p, + ret = mbedtls_oid_get_pk_alg(&alg_oid, pk_alg); + #if defined(MBEDTLS_PK_HAVE_ECC_KEYS) + if (ret == MBEDTLS_ERR_OID_NOT_FOUND) { +- ret = mbedtls_oid_get_ec_grp_algid(&alg_oid, ec_grp_id); ++ ret = mbedtls_oid_get_ec_grp_algid(&alg_oid, reinterpret_cast(ec_grp_id)); + if (ret == 0) { + *pk_alg = MBEDTLS_PK_ECKEY; + } +@@ -521,7 +521,7 @@ int mbedtls_pk_parse_subpubkey(unsigned char **p, const unsigned char *end, + size_t len; + mbedtls_asn1_buf alg_params; + mbedtls_pk_type_t pk_alg = MBEDTLS_PK_NONE; +- mbedtls_ecp_group_id ec_grp_id = MBEDTLS_ECP_DP_NONE; ++ int ec_grp_id = 0; + const mbedtls_pk_info_t *pk_info; + + if ((ret = mbedtls_asn1_get_tag(p, end, &len, +@@ -750,7 +750,7 @@ static int pk_parse_key_pkcs8_unencrypted_der( + unsigned char *p = (unsigned char *) key; + unsigned char *end = p + keylen; + mbedtls_pk_type_t pk_alg = MBEDTLS_PK_NONE; +- mbedtls_ecp_group_id ec_grp_id = MBEDTLS_ECP_DP_NONE; ++ int ec_grp_id = 0; + const mbedtls_pk_info_t *pk_info; + + #if !defined(MBEDTLS_PK_HAVE_ECC_KEYS) +diff --git a/third_party/mbedtls/library/rsa.cpp b/third_party/mbedtls/library/rsa.cpp +index 557faaf363..63faf1c8cc 100644 +--- a/third_party/mbedtls/library/rsa.cpp ++++ b/third_party/mbedtls/library/rsa.cpp +@@ -2458,12 +2458,12 @@ int mbedtls_rsa_rsassa_pkcs1_v15_sign(mbedtls_rsa_context *ctx, + * temporary buffer and check it before returning it. + */ + +- sig_try = mbedtls_calloc(1, ctx->len); ++ sig_try = (unsigned char *) mbedtls_calloc(1, ctx->len); + if (sig_try == NULL) { + return MBEDTLS_ERR_MPI_ALLOC_FAILED; + } + +- verif = mbedtls_calloc(1, ctx->len); ++ verif = (unsigned char *) mbedtls_calloc(1, ctx->len); + if (verif == NULL) { + mbedtls_free(sig_try); + return MBEDTLS_ERR_MPI_ALLOC_FAILED; +@@ -2692,8 +2692,8 @@ int mbedtls_rsa_rsassa_pkcs1_v15_verify(mbedtls_rsa_context *ctx, + * Prepare expected PKCS1 v1.5 encoding of hash. + */ + +- if ((encoded = mbedtls_calloc(1, sig_len)) == NULL || +- (encoded_expected = mbedtls_calloc(1, sig_len)) == NULL) { ++ if ((encoded = (unsigned char *) mbedtls_calloc(1, sig_len)) == NULL || ++ (encoded_expected = (unsigned char *) mbedtls_calloc(1, sig_len)) == NULL) { + ret = MBEDTLS_ERR_MPI_ALLOC_FAILED; + goto cleanup; + } +diff --git a/third_party/mbedtls/library/sha1.cpp b/third_party/mbedtls/library/sha1.cpp +index dfbe481f39..dacfe263f3 100644 +--- a/third_party/mbedtls/library/sha1.cpp ++++ b/third_party/mbedtls/library/sha1.cpp +@@ -88,7 +88,7 @@ int mbedtls_internal_sha1_process(mbedtls_sha1_context *ctx, + + #define S(x, n) (((x) << (n)) | (((x) & 0xFFFFFFFF) >> (32 - (n)))) + +-#define R(t) \ ++#define SHA1R(t) \ + ( \ + local.temp = local.W[((t) - 3) & 0x0F] ^ \ + local.W[((t) - 8) & 0x0F] ^ \ +@@ -97,7 +97,7 @@ int mbedtls_internal_sha1_process(mbedtls_sha1_context *ctx, + (local.W[(t) & 0x0F] = S(local.temp, 1)) \ + ) + +-#define P(a, b, c, d, e, x) \ ++#define SHA1P(a, b, c, d, e, x) \ + do \ + { \ + (e) += S((a), 5) + F((b), (c), (d)) + K + (x); \ +@@ -113,26 +113,26 @@ int mbedtls_internal_sha1_process(mbedtls_sha1_context *ctx, + #define F(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) + #define K 0x5A827999 + +- P(local.A, local.B, local.C, local.D, local.E, local.W[0]); +- P(local.E, local.A, local.B, local.C, local.D, local.W[1]); +- P(local.D, local.E, local.A, local.B, local.C, local.W[2]); +- P(local.C, local.D, local.E, local.A, local.B, local.W[3]); +- P(local.B, local.C, local.D, local.E, local.A, local.W[4]); +- P(local.A, local.B, local.C, local.D, local.E, local.W[5]); +- P(local.E, local.A, local.B, local.C, local.D, local.W[6]); +- P(local.D, local.E, local.A, local.B, local.C, local.W[7]); +- P(local.C, local.D, local.E, local.A, local.B, local.W[8]); +- P(local.B, local.C, local.D, local.E, local.A, local.W[9]); +- P(local.A, local.B, local.C, local.D, local.E, local.W[10]); +- P(local.E, local.A, local.B, local.C, local.D, local.W[11]); +- P(local.D, local.E, local.A, local.B, local.C, local.W[12]); +- P(local.C, local.D, local.E, local.A, local.B, local.W[13]); +- P(local.B, local.C, local.D, local.E, local.A, local.W[14]); +- P(local.A, local.B, local.C, local.D, local.E, local.W[15]); +- P(local.E, local.A, local.B, local.C, local.D, R(16)); +- P(local.D, local.E, local.A, local.B, local.C, R(17)); +- P(local.C, local.D, local.E, local.A, local.B, R(18)); +- P(local.B, local.C, local.D, local.E, local.A, R(19)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, local.W[0]); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, local.W[1]); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, local.W[2]); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, local.W[3]); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, local.W[4]); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, local.W[5]); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, local.W[6]); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, local.W[7]); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, local.W[8]); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, local.W[9]); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, local.W[10]); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, local.W[11]); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, local.W[12]); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, local.W[13]); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, local.W[14]); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, local.W[15]); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(16)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(17)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(18)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(19)); + + #undef K + #undef F +@@ -140,26 +140,26 @@ int mbedtls_internal_sha1_process(mbedtls_sha1_context *ctx, + #define F(x, y, z) ((x) ^ (y) ^ (z)) + #define K 0x6ED9EBA1 + +- P(local.A, local.B, local.C, local.D, local.E, R(20)); +- P(local.E, local.A, local.B, local.C, local.D, R(21)); +- P(local.D, local.E, local.A, local.B, local.C, R(22)); +- P(local.C, local.D, local.E, local.A, local.B, R(23)); +- P(local.B, local.C, local.D, local.E, local.A, R(24)); +- P(local.A, local.B, local.C, local.D, local.E, R(25)); +- P(local.E, local.A, local.B, local.C, local.D, R(26)); +- P(local.D, local.E, local.A, local.B, local.C, R(27)); +- P(local.C, local.D, local.E, local.A, local.B, R(28)); +- P(local.B, local.C, local.D, local.E, local.A, R(29)); +- P(local.A, local.B, local.C, local.D, local.E, R(30)); +- P(local.E, local.A, local.B, local.C, local.D, R(31)); +- P(local.D, local.E, local.A, local.B, local.C, R(32)); +- P(local.C, local.D, local.E, local.A, local.B, R(33)); +- P(local.B, local.C, local.D, local.E, local.A, R(34)); +- P(local.A, local.B, local.C, local.D, local.E, R(35)); +- P(local.E, local.A, local.B, local.C, local.D, R(36)); +- P(local.D, local.E, local.A, local.B, local.C, R(37)); +- P(local.C, local.D, local.E, local.A, local.B, R(38)); +- P(local.B, local.C, local.D, local.E, local.A, R(39)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(20)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(21)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(22)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(23)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(24)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(25)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(26)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(27)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(28)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(29)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(30)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(31)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(32)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(33)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(34)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(35)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(36)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(37)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(38)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(39)); + + #undef K + #undef F +@@ -167,26 +167,26 @@ int mbedtls_internal_sha1_process(mbedtls_sha1_context *ctx, + #define F(x, y, z) (((x) & (y)) | ((z) & ((x) | (y)))) + #define K 0x8F1BBCDC + +- P(local.A, local.B, local.C, local.D, local.E, R(40)); +- P(local.E, local.A, local.B, local.C, local.D, R(41)); +- P(local.D, local.E, local.A, local.B, local.C, R(42)); +- P(local.C, local.D, local.E, local.A, local.B, R(43)); +- P(local.B, local.C, local.D, local.E, local.A, R(44)); +- P(local.A, local.B, local.C, local.D, local.E, R(45)); +- P(local.E, local.A, local.B, local.C, local.D, R(46)); +- P(local.D, local.E, local.A, local.B, local.C, R(47)); +- P(local.C, local.D, local.E, local.A, local.B, R(48)); +- P(local.B, local.C, local.D, local.E, local.A, R(49)); +- P(local.A, local.B, local.C, local.D, local.E, R(50)); +- P(local.E, local.A, local.B, local.C, local.D, R(51)); +- P(local.D, local.E, local.A, local.B, local.C, R(52)); +- P(local.C, local.D, local.E, local.A, local.B, R(53)); +- P(local.B, local.C, local.D, local.E, local.A, R(54)); +- P(local.A, local.B, local.C, local.D, local.E, R(55)); +- P(local.E, local.A, local.B, local.C, local.D, R(56)); +- P(local.D, local.E, local.A, local.B, local.C, R(57)); +- P(local.C, local.D, local.E, local.A, local.B, R(58)); +- P(local.B, local.C, local.D, local.E, local.A, R(59)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(40)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(41)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(42)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(43)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(44)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(45)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(46)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(47)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(48)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(49)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(50)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(51)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(52)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(53)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(54)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(55)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(56)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(57)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(58)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(59)); + + #undef K + #undef F +@@ -194,26 +194,26 @@ int mbedtls_internal_sha1_process(mbedtls_sha1_context *ctx, + #define F(x, y, z) ((x) ^ (y) ^ (z)) + #define K 0xCA62C1D6 + +- P(local.A, local.B, local.C, local.D, local.E, R(60)); +- P(local.E, local.A, local.B, local.C, local.D, R(61)); +- P(local.D, local.E, local.A, local.B, local.C, R(62)); +- P(local.C, local.D, local.E, local.A, local.B, R(63)); +- P(local.B, local.C, local.D, local.E, local.A, R(64)); +- P(local.A, local.B, local.C, local.D, local.E, R(65)); +- P(local.E, local.A, local.B, local.C, local.D, R(66)); +- P(local.D, local.E, local.A, local.B, local.C, R(67)); +- P(local.C, local.D, local.E, local.A, local.B, R(68)); +- P(local.B, local.C, local.D, local.E, local.A, R(69)); +- P(local.A, local.B, local.C, local.D, local.E, R(70)); +- P(local.E, local.A, local.B, local.C, local.D, R(71)); +- P(local.D, local.E, local.A, local.B, local.C, R(72)); +- P(local.C, local.D, local.E, local.A, local.B, R(73)); +- P(local.B, local.C, local.D, local.E, local.A, R(74)); +- P(local.A, local.B, local.C, local.D, local.E, R(75)); +- P(local.E, local.A, local.B, local.C, local.D, R(76)); +- P(local.D, local.E, local.A, local.B, local.C, R(77)); +- P(local.C, local.D, local.E, local.A, local.B, R(78)); +- P(local.B, local.C, local.D, local.E, local.A, R(79)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(60)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(61)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(62)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(63)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(64)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(65)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(66)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(67)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(68)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(69)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(70)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(71)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(72)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(73)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(74)); ++ SHA1P(local.A, local.B, local.C, local.D, local.E, SHA1R(75)); ++ SHA1P(local.E, local.A, local.B, local.C, local.D, SHA1R(76)); ++ SHA1P(local.D, local.E, local.A, local.B, local.C, SHA1R(77)); ++ SHA1P(local.C, local.D, local.E, local.A, local.B, SHA1R(78)); ++ SHA1P(local.B, local.C, local.D, local.E, local.A, SHA1R(79)); + + #undef K + #undef F diff --git a/third_party/mbedtls/inline_mbedtls.py b/third_party/mbedtls/inline_mbedtls.py new file mode 100644 index 000000000000..1992bca71c3e --- /dev/null +++ b/third_party/mbedtls/inline_mbedtls.py @@ -0,0 +1,50 @@ +import os +version = '3.6.4' + +# os.system(f'wget https://github.com/Mbed-TLS/mbedtls/archive/refs/tags/mbedtls-{version}.tar.gz') +# os.system(f'tar xvf mbedtls-{version}.tar.gz') + +directories = ['include', 'library'] +source_dir = f'mbedtls-mbedtls-{version}' +target_dir = '.' +extensions = ['.h', '.hpp', '.c', '.cpp'] + +class FileToCopy: + def __init__(self, source, target): + self.source_file = source + self.target_file = target + +def get_copy_list(source_dir, target_dir): + result = [] + for file in os.listdir(source_dir): + is_source_file = False + for ext in extensions: + if file.endswith(ext): + is_source_file = True + if not is_source_file: + continue + target_file_name = file + if target_file_name.endswith('.c'): + target_file_name = target_file_name[:-2] + '.cpp' + source_file = os.path.join(source_dir, file) + target_file = os.path.join(target_dir, target_file_name) + if os.path.isdir(source_file): + result += get_copy_list(source_file, target_file) + if not os.path.isfile(target_file): + continue + # check if this is a dummy file + with open(target_file, 'r') as f: + text = f.read() + if '// dummy file' in text: + continue + print(target_file) + result.append(FileToCopy(source_file, target_file)) + return result + + +copy_list = [] +for directory in directories: + copy_list += get_copy_list(os.path.join(source_dir, directory), os.path.join(target_dir, directory)) + +for file in copy_list: + os.system(f'cp {file.source_file} {file.target_file}') \ No newline at end of file diff --git a/third_party/mbedtls/library/aes.cpp b/third_party/mbedtls/library/aes.cpp index c9196d1c9efc..00d64c617324 100644 --- a/third_party/mbedtls/library/aes.cpp +++ b/third_party/mbedtls/library/aes.cpp @@ -1161,7 +1161,7 @@ typedef unsigned char mbedtls_be128[16]; #if defined(MBEDTLS_AESCE_C) || defined(MBEDTLS_AESNI_C) MBEDTLS_OPTIMIZE_FOR_PERFORMANCE #endif -inline void mbedtls_gf128mul_x_ble(unsigned char r[16], +static inline void mbedtls_gf128mul_x_ble(unsigned char r[16], const unsigned char x[16]) { uint64_t a, b, ra, rb; diff --git a/third_party/mbedtls/library/aesce.h b/third_party/mbedtls/library/aesce.h index 456385123cf7..a14d085efa28 100644 --- a/third_party/mbedtls/library/aesce.h +++ b/third_party/mbedtls/library/aesce.h @@ -1 +1,136 @@ -// dummy \ No newline at end of file +/** + * \file aesce.h + * + * \brief Support hardware AES acceleration on Armv8-A processors with + * the Armv8-A Cryptographic Extension. + * + * \warning These functions are only for internal use by other library + * functions; you must not call them directly. + */ +/* + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ +#ifndef MBEDTLS_AESCE_H +#define MBEDTLS_AESCE_H + +#include "mbedtls/build_info.h" +#include "common.h" + +#include "mbedtls/aes.h" + + +#if defined(MBEDTLS_AESCE_C) \ + && defined(MBEDTLS_ARCH_IS_ARMV8_A) && defined(MBEDTLS_HAVE_NEON_INTRINSICS) \ + && (defined(MBEDTLS_COMPILER_IS_GCC) || defined(__clang__) || defined(MSC_VER)) + +/* MBEDTLS_AESCE_HAVE_CODE is defined if we have a suitable target platform, and a + * potentially suitable compiler (compiler version & flags are not checked when defining + * this). */ +#define MBEDTLS_AESCE_HAVE_CODE + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__linux__) && !defined(MBEDTLS_AES_USE_HARDWARE_ONLY) + +extern signed char mbedtls_aesce_has_support_result; + +/** + * \brief Internal function to detect the crypto extension in CPUs. + * + * \return 1 if CPU has support for the feature, 0 otherwise + */ +int mbedtls_aesce_has_support_impl(void); + +#define MBEDTLS_AESCE_HAS_SUPPORT() (mbedtls_aesce_has_support_result == -1 ? \ + mbedtls_aesce_has_support_impl() : \ + mbedtls_aesce_has_support_result) + +#else /* defined(__linux__) && !defined(MBEDTLS_AES_USE_HARDWARE_ONLY) */ + +/* If we are not on Linux, we can't detect support so assume that it's supported. + * Similarly, assume support if MBEDTLS_AES_USE_HARDWARE_ONLY is set. + */ +#define MBEDTLS_AESCE_HAS_SUPPORT() 1 + +#endif /* defined(__linux__) && !defined(MBEDTLS_AES_USE_HARDWARE_ONLY) */ + +/** + * \brief Internal AES-ECB block encryption and decryption + * + * \warning This assumes that the context specifies either 10, 12 or 14 + * rounds and will behave incorrectly if this is not the case. + * + * \param ctx AES context + * \param mode MBEDTLS_AES_ENCRYPT or MBEDTLS_AES_DECRYPT + * \param input 16-byte input block + * \param output 16-byte output block + * + * \return 0 on success (cannot fail) + */ +int mbedtls_aesce_crypt_ecb(mbedtls_aes_context *ctx, + int mode, + const unsigned char input[16], + unsigned char output[16]); + +/** + * \brief Internal GCM multiplication: c = a * b in GF(2^128) + * + * \note This function is only for internal use by other library + * functions; you must not call it directly. + * + * \param c Result + * \param a First operand + * \param b Second operand + * + * \note Both operands and result are bit strings interpreted as + * elements of GF(2^128) as per the GCM spec. + */ +void mbedtls_aesce_gcm_mult(unsigned char c[16], + const unsigned char a[16], + const unsigned char b[16]); + + +#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT) +/** + * \brief Internal round key inversion. This function computes + * decryption round keys from the encryption round keys. + * + * \param invkey Round keys for the equivalent inverse cipher + * \param fwdkey Original round keys (for encryption) + * \param nr Number of rounds (that is, number of round keys minus one) + */ +void mbedtls_aesce_inverse_key(unsigned char *invkey, + const unsigned char *fwdkey, + int nr); +#endif /* !MBEDTLS_BLOCK_CIPHER_NO_DECRYPT */ + +/** + * \brief Internal key expansion for encryption + * + * \param rk Destination buffer where the round keys are written + * \param key Encryption key + * \param bits Key size in bits (must be 128, 192 or 256) + * + * \return 0 if successful, or MBEDTLS_ERR_AES_INVALID_KEY_LENGTH + */ +int mbedtls_aesce_setkey_enc(unsigned char *rk, + const unsigned char *key, + size_t bits); + +#ifdef __cplusplus +} +#endif + +#else + +#if defined(MBEDTLS_AES_USE_HARDWARE_ONLY) && defined(MBEDTLS_ARCH_IS_ARMV8_A) +#error "AES hardware acceleration not supported on this platform / compiler" +#endif + +#endif /* MBEDTLS_AESCE_C && MBEDTLS_ARCH_IS_ARMV8_A && MBEDTLS_HAVE_NEON_INTRINSICS && + (MBEDTLS_COMPILER_IS_GCC || __clang__ || MSC_VER) */ + +#endif /* MBEDTLS_AESCE_H */ diff --git a/third_party/mbedtls/library/alignment.h b/third_party/mbedtls/library/alignment.h index 3b15148d68dc..a17001dd91b1 100644 --- a/third_party/mbedtls/library/alignment.h +++ b/third_party/mbedtls/library/alignment.h @@ -112,7 +112,7 @@ typedef struct { #elif defined(__GNUC__) __attribute__((always_inline)) #endif -inline uint16_t mbedtls_get_unaligned_uint16(const void *p) +static inline uint16_t mbedtls_get_unaligned_uint16(const void *p) { uint16_t r; #if defined(UINT_UNALIGNED) @@ -139,7 +139,7 @@ inline uint16_t mbedtls_get_unaligned_uint16(const void *p) #elif defined(__GNUC__) __attribute__((always_inline)) #endif -inline void mbedtls_put_unaligned_uint16(void *p, uint16_t x) +static inline void mbedtls_put_unaligned_uint16(void *p, uint16_t x) { #if defined(UINT_UNALIGNED) mbedtls_uint16_unaligned_t *p16 = (mbedtls_uint16_unaligned_t *) p; @@ -164,7 +164,7 @@ inline void mbedtls_put_unaligned_uint16(void *p, uint16_t x) #elif defined(__GNUC__) __attribute__((always_inline)) #endif -inline uint32_t mbedtls_get_unaligned_uint32(const void *p) +static inline uint32_t mbedtls_get_unaligned_uint32(const void *p) { uint32_t r; #if defined(UINT_UNALIGNED) @@ -191,7 +191,7 @@ inline uint32_t mbedtls_get_unaligned_uint32(const void *p) #elif defined(__GNUC__) __attribute__((always_inline)) #endif -inline void mbedtls_put_unaligned_uint32(void *p, uint32_t x) +static inline void mbedtls_put_unaligned_uint32(void *p, uint32_t x) { #if defined(UINT_UNALIGNED) mbedtls_uint32_unaligned_t *p32 = (mbedtls_uint32_unaligned_t *) p; @@ -216,7 +216,7 @@ inline void mbedtls_put_unaligned_uint32(void *p, uint32_t x) #elif defined(__GNUC__) __attribute__((always_inline)) #endif -inline uint64_t mbedtls_get_unaligned_uint64(const void *p) +static inline uint64_t mbedtls_get_unaligned_uint64(const void *p) { uint64_t r; #if defined(UINT_UNALIGNED) @@ -243,7 +243,7 @@ inline uint64_t mbedtls_get_unaligned_uint64(const void *p) #elif defined(__GNUC__) __attribute__((always_inline)) #endif -inline void mbedtls_put_unaligned_uint64(void *p, uint64_t x) +static inline void mbedtls_put_unaligned_uint64(void *p, uint64_t x) { #if defined(UINT_UNALIGNED) mbedtls_uint64_unaligned_t *p64 = (mbedtls_uint64_unaligned_t *) p; @@ -341,7 +341,7 @@ inline void mbedtls_put_unaligned_uint64(void *p, uint64_t x) * similar instruction. */ #if !defined(MBEDTLS_BSWAP16) -inline uint16_t mbedtls_bswap16(uint16_t x) +static inline uint16_t mbedtls_bswap16(uint16_t x) { return (x & 0x00ff) << 8 | @@ -351,7 +351,7 @@ inline uint16_t mbedtls_bswap16(uint16_t x) #endif /* !defined(MBEDTLS_BSWAP16) */ #if !defined(MBEDTLS_BSWAP32) -inline uint32_t mbedtls_bswap32(uint32_t x) +static inline uint32_t mbedtls_bswap32(uint32_t x) { return (x & 0x000000ff) << 24 | @@ -363,7 +363,7 @@ inline uint32_t mbedtls_bswap32(uint32_t x) #endif /* !defined(MBEDTLS_BSWAP32) */ #if !defined(MBEDTLS_BSWAP64) -inline uint64_t mbedtls_bswap64(uint64_t x) +static inline uint64_t mbedtls_bswap64(uint64_t x) { return (x & 0x00000000000000ffULL) << 56 | diff --git a/third_party/mbedtls/library/asn1parse.cpp b/third_party/mbedtls/library/asn1parse.cpp index 2e30f6bf09b2..8db38d89213c 100644 --- a/third_party/mbedtls/library/asn1parse.cpp +++ b/third_party/mbedtls/library/asn1parse.cpp @@ -315,7 +315,7 @@ static int asn1_get_sequence_of_cb(void *ctx, cb_ctx->cur; if (cur->buf.p != NULL) { - cur->next = (struct mbedtls_asn1_sequence *) + cur->next = (mbedtls_asn1_sequence *) mbedtls_calloc(1, sizeof(mbedtls_asn1_sequence)); if (cur->next == NULL) { diff --git a/third_party/mbedtls/library/asn1write.cpp b/third_party/mbedtls/library/asn1write.cpp index 42bc7fa2db40..431c7ba8bbc7 100644 --- a/third_party/mbedtls/library/asn1write.cpp +++ b/third_party/mbedtls/library/asn1write.cpp @@ -90,7 +90,9 @@ int mbedtls_asn1_write_raw_buffer(unsigned char **p, const unsigned char *start, len = size; (*p) -= len; - memcpy(*p, buf, len); + if (len != 0) { + memcpy(*p, buf, len); + } return (int) len; } @@ -412,19 +414,20 @@ mbedtls_asn1_named_data *mbedtls_asn1_store_named_data( } else if (val_len == 0) { mbedtls_free(cur->val.p); cur->val.p = NULL; + cur->val.len = 0; } else if (cur->val.len != val_len) { /* * Enlarge existing value buffer if needed * Preserve old data until the allocation succeeded, to leave list in * a consistent state in case allocation fails. */ - void *p = mbedtls_calloc(1, val_len); + void *p = (unsigned char *) mbedtls_calloc(1, val_len); if (p == NULL) { return NULL; } mbedtls_free(cur->val.p); - cur->val.p = (unsigned char *)p; + cur->val.p = (unsigned char *) p; cur->val.len = val_len; } diff --git a/third_party/mbedtls/library/base64.cpp b/third_party/mbedtls/library/base64.cpp index 9677dee5b295..388fa9f388e5 100644 --- a/third_party/mbedtls/library/base64.cpp +++ b/third_party/mbedtls/library/base64.cpp @@ -14,6 +14,7 @@ #include "mbedtls/base64.h" #include "base64_internal.h" #include "constant_time_internal.h" +#include "mbedtls/error.h" #include @@ -183,49 +184,72 @@ int mbedtls_base64_decode(unsigned char *dst, size_t dlen, size_t *olen, n++; } - if (n == 0) { - *olen = 0; - return 0; + /* In valid base64, the number of digits (n-equals) is always of the form + * 4*k, 4*k+2 or *4k+3. Also, the number n of digits plus the number of + * equal signs at the end is always a multiple of 4. */ + if ((n - equals) % 4 == 1) { + return MBEDTLS_ERR_BASE64_INVALID_CHARACTER; + } + if (n % 4 != 0) { + return MBEDTLS_ERR_BASE64_INVALID_CHARACTER; } - /* The following expression is to calculate the following formula without - * risk of integer overflow in n: - * n = ( ( n * 6 ) + 7 ) >> 3; + /* We've determined that the input is valid, and that it contains + * exactly k blocks of digits-or-equals, with n = 4 * k, + * and equals only present at the end of the last block if at all. + * Now we can calculate the length of the output. + * + * Each block of 4 digits in the input map to 3 bytes of output. + * For the last block: + * - abcd (where abcd are digits) is a full 3-byte block; + * - abc= means 1 byte less than a full 3-byte block of output; + * - ab== means 2 bytes less than a full 3-byte block of output; + * - a==== and ==== is rejected above. */ - n = (6 * (n >> 3)) + ((6 * (n & 0x7) + 7) >> 3); - n -= equals; - - if (dst == NULL || dlen < n) { - *olen = n; + *olen = (n / 4) * 3 - equals; + + /* If the output buffer is too small, signal this and stop here. + * Also, as documented, stop here if `dst` is null, independently of + * `dlen`. + * + * There is an edge case when the output is empty: in this case, + * `dlen == 0` with `dst == NULL` is valid (on some platforms, + * `malloc(0)` returns `NULL`). Since the call is valid, we return + * 0 in this case. + */ + if ((*olen != 0 && dst == NULL) || dlen < *olen) { return MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL; } - equals = 0; for (x = 0, p = dst; i > 0; i--, src++) { if (*src == '\r' || *src == '\n' || *src == ' ') { continue; } - - x = x << 6; if (*src == '=') { - ++equals; - } else { - x |= mbedtls_ct_base64_dec_value(*src); + /* We already know from the first loop that equal signs are + * only at the end. */ + break; } + x = x << 6; + x |= mbedtls_ct_base64_dec_value(*src); if (++accumulated_digits == 4) { accumulated_digits = 0; *p++ = MBEDTLS_BYTE_2(x); - if (equals <= 1) { - *p++ = MBEDTLS_BYTE_1(x); - } - if (equals <= 0) { - *p++ = MBEDTLS_BYTE_0(x); - } + *p++ = MBEDTLS_BYTE_1(x); + *p++ = MBEDTLS_BYTE_0(x); } } + if (accumulated_digits == 3) { + *p++ = MBEDTLS_BYTE_2(x << 6); + *p++ = MBEDTLS_BYTE_1(x << 6); + } else if (accumulated_digits == 2) { + *p++ = MBEDTLS_BYTE_2(x << 12); + } - *olen = (size_t) (p - dst); + if (*olen != (size_t) (p - dst)) { + return MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; + } return 0; } diff --git a/third_party/mbedtls/library/base64_internal.h b/third_party/mbedtls/library/base64_internal.h index 456385123cf7..a09bd2377714 100644 --- a/third_party/mbedtls/library/base64_internal.h +++ b/third_party/mbedtls/library/base64_internal.h @@ -1 +1,45 @@ -// dummy \ No newline at end of file +/** + * \file base64_internal.h + * + * \brief RFC 1521 base64 encoding/decoding: interfaces for invasive testing + */ +/* + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ + +#ifndef MBEDTLS_BASE64_INTERNAL +#define MBEDTLS_BASE64_INTERNAL + +#include "common.h" + +#if defined(MBEDTLS_TEST_HOOKS) + +/** Given a value in the range 0..63, return the corresponding Base64 digit. + * + * The implementation assumes that letters are consecutive (e.g. ASCII + * but not EBCDIC). + * + * \param value A value in the range 0..63. + * + * \return A base64 digit converted from \p value. + */ +unsigned char mbedtls_ct_base64_enc_char(unsigned char value); + +/** Given a Base64 digit, return its value. + * + * If c is not a Base64 digit ('A'..'Z', 'a'..'z', '0'..'9', '+' or '/'), + * return -1. + * + * The implementation assumes that letters are consecutive (e.g. ASCII + * but not EBCDIC). + * + * \param c A base64 digit. + * + * \return The value of the base64 digit \p c. + */ +signed char mbedtls_ct_base64_dec_value(unsigned char c); + +#endif /* MBEDTLS_TEST_HOOKS */ + +#endif /* MBEDTLS_BASE64_INTERNAL */ diff --git a/third_party/mbedtls/library/bignum.cpp b/third_party/mbedtls/library/bignum.cpp index 1c0f55c8e76d..3e743b0291d8 100644 --- a/third_party/mbedtls/library/bignum.cpp +++ b/third_party/mbedtls/library/bignum.cpp @@ -45,7 +45,7 @@ * (MPI sign is the field s in mbedtls_mpi. It is unsigned short and only 1 and -1 are valid * values.) */ -inline signed short mbedtls_ct_mpi_sign_if(mbedtls_ct_condition_t cond, +static inline signed short mbedtls_ct_mpi_sign_if(mbedtls_ct_condition_t cond, signed short sign1, signed short sign2) { return (signed short) mbedtls_ct_uint_if(cond, sign1 + 1, sign2 + 1) - 1; @@ -88,7 +88,7 @@ int mbedtls_mpi_lt_mpi_ct(const mbedtls_mpi *X, /* This array is used to conditionally swap the pointers in const time */ void * const p[2] = { X->p, Y->p }; size_t i = mbedtls_ct_size_if_else_0(X_is_negative, 1); - mbedtls_ct_condition_t lt = mbedtls_mpi_core_lt_ct((const mbedtls_mpi_uint *)p[i], (const mbedtls_mpi_uint *)p[i ^ 1], X->n); + mbedtls_ct_condition_t lt = mbedtls_mpi_core_lt_ct((const mbedtls_mpi_uint *) p[i], (const mbedtls_mpi_uint *) p[i ^ 1], X->n); /* * Store in result iff the signs are the same (i.e., iff different_sign == false). If @@ -355,7 +355,7 @@ void mbedtls_mpi_swap(mbedtls_mpi *X, mbedtls_mpi *Y) memcpy(Y, &T, sizeof(mbedtls_mpi)); } -inline mbedtls_mpi_uint mpi_sint_abs(mbedtls_mpi_sint z) +static inline mbedtls_mpi_uint mpi_sint_abs(mbedtls_mpi_sint z) { if (z >= 0) { return z; diff --git a/third_party/mbedtls/library/bignum_core.cpp b/third_party/mbedtls/library/bignum_core.cpp index b0413ff06498..c8bc21c5f98e 100644 --- a/third_party/mbedtls/library/bignum_core.cpp +++ b/third_party/mbedtls/library/bignum_core.cpp @@ -746,8 +746,8 @@ static void exp_mod_precompute_window(const mbedtls_mpi_uint *A, } #if defined(MBEDTLS_TEST_HOOKS) && !defined(MBEDTLS_THREADING_C) -// Set to a default that is neither MBEDTLS_MPI_IS_PUBLIC nor MBEDTLS_MPI_IS_SECRET -int mbedtls_mpi_optionally_safe_codepath = MBEDTLS_MPI_IS_PUBLIC + MBEDTLS_MPI_IS_SECRET + 1; +void (*mbedtls_safe_codepath_hook)(void) = NULL; +void (*mbedtls_unsafe_codepath_hook)(void) = NULL; #endif /* @@ -757,7 +757,7 @@ int mbedtls_mpi_optionally_safe_codepath = MBEDTLS_MPI_IS_PUBLIC + MBEDTLS_MPI_I * Warning! If the parameter E_public has MBEDTLS_MPI_IS_PUBLIC as its value, * this function is not constant time with respect to the exponent (parameter E). */ -inline void exp_mod_calc_first_bit_optionally_safe(const mbedtls_mpi_uint *E, +static inline void exp_mod_calc_first_bit_optionally_safe(const mbedtls_mpi_uint *E, size_t E_limbs, int E_public, size_t *E_limb_index, @@ -780,7 +780,9 @@ inline void exp_mod_calc_first_bit_optionally_safe(const mbedtls_mpi_uint *E, *E_bit_index = E_bits % biL; #if defined(MBEDTLS_TEST_HOOKS) && !defined(MBEDTLS_THREADING_C) - mbedtls_mpi_optionally_safe_codepath = MBEDTLS_MPI_IS_PUBLIC; + if (mbedtls_unsafe_codepath_hook != NULL) { + mbedtls_unsafe_codepath_hook(); + } #endif } else { /* @@ -790,9 +792,8 @@ inline void exp_mod_calc_first_bit_optionally_safe(const mbedtls_mpi_uint *E, *E_limb_index = E_limbs; *E_bit_index = 0; #if defined(MBEDTLS_TEST_HOOKS) && !defined(MBEDTLS_THREADING_C) - // Only mark the codepath safe if there wasn't an unsafe codepath before - if (mbedtls_mpi_optionally_safe_codepath != MBEDTLS_MPI_IS_PUBLIC) { - mbedtls_mpi_optionally_safe_codepath = MBEDTLS_MPI_IS_SECRET; + if (mbedtls_safe_codepath_hook != NULL) { + mbedtls_safe_codepath_hook(); } #endif } @@ -803,7 +804,7 @@ inline void exp_mod_calc_first_bit_optionally_safe(const mbedtls_mpi_uint *E, * not constant time with respect to the window parameter and consequently the exponent of the * exponentiation (parameter E of mbedtls_mpi_core_exp_mod_optionally_safe). */ -inline void exp_mod_table_lookup_optionally_safe(mbedtls_mpi_uint *Wselect, +static inline void exp_mod_table_lookup_optionally_safe(mbedtls_mpi_uint *Wselect, mbedtls_mpi_uint *Wtable, size_t AN_limbs, size_t welem, mbedtls_mpi_uint window, @@ -812,7 +813,9 @@ inline void exp_mod_table_lookup_optionally_safe(mbedtls_mpi_uint *Wselect, if (window_public == MBEDTLS_MPI_IS_PUBLIC) { memcpy(Wselect, Wtable + window * AN_limbs, AN_limbs * ciL); #if defined(MBEDTLS_TEST_HOOKS) && !defined(MBEDTLS_THREADING_C) - mbedtls_mpi_optionally_safe_codepath = MBEDTLS_MPI_IS_PUBLIC; + if (mbedtls_unsafe_codepath_hook != NULL) { + mbedtls_unsafe_codepath_hook(); + } #endif } else { /* Select Wtable[window] without leaking window through @@ -820,9 +823,8 @@ inline void exp_mod_table_lookup_optionally_safe(mbedtls_mpi_uint *Wselect, mbedtls_mpi_core_ct_uint_table_lookup(Wselect, Wtable, AN_limbs, welem, window); #if defined(MBEDTLS_TEST_HOOKS) && !defined(MBEDTLS_THREADING_C) - // Only mark the codepath safe if there wasn't an unsafe codepath before - if (mbedtls_mpi_optionally_safe_codepath != MBEDTLS_MPI_IS_PUBLIC) { - mbedtls_mpi_optionally_safe_codepath = MBEDTLS_MPI_IS_SECRET; + if (mbedtls_safe_codepath_hook != NULL) { + mbedtls_safe_codepath_hook(); } #endif } @@ -856,8 +858,8 @@ static void mbedtls_mpi_core_exp_mod_optionally_safe(mbedtls_mpi_uint *X, /* We'll process the bits of E from most significant * (limb_index=E_limbs-1, E_bit_index=biL-1) to least significant * (limb_index=0, E_bit_index=0). */ - size_t E_limb_index; - size_t E_bit_index; + size_t E_limb_index = E_limbs; + size_t E_bit_index = 0; exp_mod_calc_first_bit_optionally_safe(E, E_limbs, E_public, &E_limb_index, &E_bit_index); diff --git a/third_party/mbedtls/library/bignum_core.h b/third_party/mbedtls/library/bignum_core.h index d29250ce642c..264ee6355069 100644 --- a/third_party/mbedtls/library/bignum_core.h +++ b/third_party/mbedtls/library/bignum_core.h @@ -70,9 +70,7 @@ #include "common.h" -#if defined(MBEDTLS_BIGNUM_C) #include "mbedtls/bignum.h" -#endif #include "constant_time_internal.h" @@ -106,10 +104,17 @@ * } else { * // safe path * } - * not the other way round, in order to prevent misuse. (This is, if a value - * other than the two below is passed, default to the safe path.) */ + * not the other way round, in order to prevent misuse. (That is, if a value + * other than the two below is passed, default to the safe path.) + * + * The value of MBEDTLS_MPI_IS_PUBLIC is chosen in a way that is unlikely to happen by accident, but + * which can be used as an immediate value in a Thumb2 comparison (for code size). */ #define MBEDTLS_MPI_IS_PUBLIC 0x2a2a2a2a #define MBEDTLS_MPI_IS_SECRET 0 +#if defined(MBEDTLS_TEST_HOOKS) && !defined(MBEDTLS_THREADING_C) +// Default value for testing that is neither MBEDTLS_MPI_IS_PUBLIC nor MBEDTLS_MPI_IS_SECRET +#define MBEDTLS_MPI_IS_TEST 1 +#endif /** Count leading zero bits in a given integer. * @@ -737,7 +742,7 @@ mbedtls_ct_condition_t mbedtls_mpi_core_check_zero_ct(const mbedtls_mpi_uint *A, * \return The number of limbs of working memory required by * `mbedtls_mpi_core_montmul()` (or other similar function). */ -inline size_t mbedtls_mpi_core_montmul_working_limbs(size_t AN_limbs) +static inline size_t mbedtls_mpi_core_montmul_working_limbs(size_t AN_limbs) { return 2 * AN_limbs + 1; } @@ -817,17 +822,4 @@ void mbedtls_mpi_core_from_mont_rep(mbedtls_mpi_uint *X, mbedtls_mpi_uint mm, mbedtls_mpi_uint *T); -/* - * Can't define thread local variables with our abstraction layer: do nothing if threading is on. - */ -#if defined(MBEDTLS_TEST_HOOKS) && !defined(MBEDTLS_THREADING_C) -extern int mbedtls_mpi_optionally_safe_codepath; - -inline void mbedtls_mpi_optionally_safe_codepath_reset(void) -{ - // Set to a default that is neither MBEDTLS_MPI_IS_PUBLIC nor MBEDTLS_MPI_IS_SECRET - mbedtls_mpi_optionally_safe_codepath = MBEDTLS_MPI_IS_PUBLIC + MBEDTLS_MPI_IS_SECRET + 1; -} -#endif - #endif /* MBEDTLS_BIGNUM_CORE_H */ diff --git a/third_party/mbedtls/library/block_cipher_internal.h b/third_party/mbedtls/library/block_cipher_internal.h index 456385123cf7..c57338b751e0 100644 --- a/third_party/mbedtls/library/block_cipher_internal.h +++ b/third_party/mbedtls/library/block_cipher_internal.h @@ -1 +1,99 @@ -// dummy \ No newline at end of file +/** + * \file block_cipher_internal.h + * + * \brief Lightweight abstraction layer for block ciphers with 128 bit blocks, + * for use by the GCM and CCM modules. + */ +/* + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ +#ifndef MBEDTLS_BLOCK_CIPHER_INTERNAL_H +#define MBEDTLS_BLOCK_CIPHER_INTERNAL_H + +#include "mbedtls/build_info.h" + +#include "mbedtls/cipher.h" + +#include "mbedtls/block_cipher.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \brief Initialize the context. + * This must be the first API call before using the context. + * + * \param ctx The context to initialize. + */ +static inline void mbedtls_block_cipher_init(mbedtls_block_cipher_context_t *ctx) +{ + memset(ctx, 0, sizeof(*ctx)); +} + +/** + * \brief Set the block cipher to use with this context. + * This must be called after mbedtls_block_cipher_init(). + * + * \param ctx The context to set up. + * \param cipher_id The identifier of the cipher to use. + * This must be either AES, ARIA or Camellia. + * Warning: this is a ::mbedtls_cipher_id_t, + * not a ::mbedtls_block_cipher_id_t! + * + * \retval \c 0 on success. + * \retval #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA if \p cipher_id was + * invalid. + */ +int mbedtls_block_cipher_setup(mbedtls_block_cipher_context_t *ctx, + mbedtls_cipher_id_t cipher_id); + +/** + * \brief Set the key into the context. + * + * \param ctx The context to configure. + * \param key The buffer holding the key material. + * \param key_bitlen The size of the key in bits. + * + * \retval \c 0 on success. + * \retval #MBEDTLS_ERR_CIPHER_INVALID_CONTEXT if the context was not + * properly set up before calling this function. + * \retval One of #MBEDTLS_ERR_AES_INVALID_KEY_LENGTH, + * #MBEDTLS_ERR_ARIA_BAD_INPUT_DATA, + * #MBEDTLS_ERR_CAMELLIA_BAD_INPUT_DATA if \p key_bitlen is + * invalid. + */ +int mbedtls_block_cipher_setkey(mbedtls_block_cipher_context_t *ctx, + const unsigned char *key, + unsigned key_bitlen); + +/** + * \brief Encrypt one block (16 bytes) with the configured key. + * + * \param ctx The context holding the key. + * \param input The buffer holding the input block. Must be 16 bytes. + * \param output The buffer to which the output block will be written. + * Must be writable and 16 bytes long. + * This must either not overlap with \p input, or be equal. + * + * \retval \c 0 on success. + * \retval #MBEDTLS_ERR_CIPHER_INVALID_CONTEXT if the context was not + * properly set up before calling this function. + * \retval Another negative value if encryption failed. + */ +int mbedtls_block_cipher_encrypt(mbedtls_block_cipher_context_t *ctx, + const unsigned char input[16], + unsigned char output[16]); +/** + * \brief Clear the context. + * + * \param ctx The context to clear. + */ +void mbedtls_block_cipher_free(mbedtls_block_cipher_context_t *ctx); + +#ifdef __cplusplus +} +#endif + +#endif /* MBEDTLS_BLOCK_CIPHER_INTERNAL_H */ diff --git a/third_party/mbedtls/library/check_crypto_config.h b/third_party/mbedtls/library/check_crypto_config.h index 456385123cf7..6469e9f43939 100644 --- a/third_party/mbedtls/library/check_crypto_config.h +++ b/third_party/mbedtls/library/check_crypto_config.h @@ -1 +1,141 @@ -// dummy \ No newline at end of file +/** + * \file check_crypto_config.h + * + * \brief Consistency checks for PSA configuration options + */ +/* + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ + +/* + * It is recommended to include this file from your crypto_config.h + * in order to catch dependency issues early. + */ + +#ifndef MBEDTLS_CHECK_CRYPTO_CONFIG_H +#define MBEDTLS_CHECK_CRYPTO_CONFIG_H + +#if defined(PSA_WANT_ALG_CCM) && \ + !(defined(PSA_WANT_KEY_TYPE_AES) || \ + defined(PSA_WANT_KEY_TYPE_CAMELLIA)) +#error "PSA_WANT_ALG_CCM defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_ALG_CMAC) && \ + !(defined(PSA_WANT_KEY_TYPE_AES) || \ + defined(PSA_WANT_KEY_TYPE_CAMELLIA) || \ + defined(PSA_WANT_KEY_TYPE_DES)) +#error "PSA_WANT_ALG_CMAC defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_ALG_DETERMINISTIC_ECDSA) && \ + !(defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) || \ + defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY)) +#error "PSA_WANT_ALG_DETERMINISTIC_ECDSA defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_ALG_ECDSA) && \ + !(defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) || \ + defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY)) +#error "PSA_WANT_ALG_ECDSA defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_ALG_GCM) && \ + !(defined(PSA_WANT_KEY_TYPE_AES) || \ + defined(PSA_WANT_KEY_TYPE_CAMELLIA)) +#error "PSA_WANT_ALG_GCM defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_ALG_RSA_PKCS1V15_CRYPT) && \ + !(defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) || \ + defined(PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY)) +#error "PSA_WANT_ALG_RSA_PKCS1V15_CRYPT defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_ALG_RSA_PKCS1V15_SIGN) && \ + !(defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) || \ + defined(PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY)) +#error "PSA_WANT_ALG_RSA_PKCS1V15_SIGN defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_ALG_RSA_OAEP) && \ + !(defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) || \ + defined(PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY)) +#error "PSA_WANT_ALG_RSA_OAEP defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_ALG_RSA_PSS) && \ + !(defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) || \ + defined(PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY)) +#error "PSA_WANT_ALG_RSA_PSS defined, but not all prerequisites" +#endif + +#if (defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) || \ + defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT) || \ + defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT) || \ + defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE) || \ + defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE)) && \ + !defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY) +#error "PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_xxx defined, but not all prerequisites" +#endif + +#if (defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) || \ + defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT) || \ + defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT) || \ + defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE)) && \ + !defined(PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY) +#error "PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_xxx defined, but not all prerequisites" +#endif + +#if (defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC) || \ + defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT) || \ + defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT) || \ + defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE)) && \ + !defined(PSA_WANT_KEY_TYPE_DH_PUBLIC_KEY) +#error "PSA_WANT_KEY_TYPE_DH_KEY_PAIR_xxx defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR) +#if defined(MBEDTLS_DEPRECATED_REMOVED) +#error "PSA_WANT_KEY_TYPE_ECC_KEY_PAIR is deprecated and will be removed in a \ + future version of Mbed TLS. Please switch to new PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_xxx \ + symbols, where xxx can be: USE, IMPORT, EXPORT, GENERATE, DERIVE" +#elif defined(MBEDTLS_DEPRECATED_WARNING) +#warning "PSA_WANT_KEY_TYPE_ECC_KEY_PAIR is deprecated and will be removed in a \ + future version of Mbed TLS. Please switch to new PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_xxx \ + symbols, where xxx can be: USE, IMPORT, EXPORT, GENERATE, DERIVE" +#endif /* MBEDTLS_DEPRECATED_WARNING */ +#endif /* PSA_WANT_KEY_TYPE_ECC_KEY_PAIR */ + +#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR) +#if defined(MBEDTLS_DEPRECATED_REMOVED) +#error "PSA_WANT_KEY_TYPE_RSA_KEY_PAIR is deprecated and will be removed in a \ + future version of Mbed TLS. Please switch to new PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_xxx \ + symbols, where xxx can be: USE, IMPORT, EXPORT, GENERATE, DERIVE" +#elif defined(MBEDTLS_DEPRECATED_WARNING) +#warning "PSA_WANT_KEY_TYPE_RSA_KEY_PAIR is deprecated and will be removed in a \ + future version of Mbed TLS. Please switch to new PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_xxx \ + symbols, where xxx can be: USE, IMPORT, EXPORT, GENERATE, DERIVE" +#endif /* MBEDTLS_DEPRECATED_WARNING */ +#endif /* PSA_WANT_KEY_TYPE_RSA_KEY_PAIR */ + +#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE) +#error "PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE defined, but feature is not supported" +#endif + +#if defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE) +#error "PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE defined, but feature is not supported" +#endif + +#if defined(MBEDTLS_SSL_PROTO_TLS1_2) && defined(MBEDTLS_USE_PSA_CRYPTO) && \ + !(defined(PSA_WANT_ALG_SHA_1) || defined(PSA_WANT_ALG_SHA_256) || defined(PSA_WANT_ALG_SHA_512)) +#error "MBEDTLS_SSL_PROTO_TLS1_2 defined, but not all prerequisites" +#endif + +#if defined(PSA_WANT_ALG_TLS12_ECJPAKE_TO_PMS) && \ + !defined(PSA_WANT_ALG_SHA_256) +#error "PSA_WANT_ALG_TLS12_ECJPAKE_TO_PMS defined, but not all prerequisites" +#endif + +#endif /* MBEDTLS_CHECK_CRYPTO_CONFIG_H */ diff --git a/third_party/mbedtls/library/cipher.cpp b/third_party/mbedtls/library/cipher.cpp index ea5d302df0a1..8d473a75974e 100644 --- a/third_party/mbedtls/library/cipher.cpp +++ b/third_party/mbedtls/library/cipher.cpp @@ -14,6 +14,7 @@ #if defined(MBEDTLS_CIPHER_C) #include "mbedtls/cipher.h" +#include "cipher_invasive.h" #include "cipher_wrap.h" #include "mbedtls/platform_util.h" #include "mbedtls/error.h" @@ -55,7 +56,7 @@ static int supported_init = 0; -inline const mbedtls_cipher_base_t *mbedtls_cipher_get_base( +static inline const mbedtls_cipher_base_t *mbedtls_cipher_get_base( const mbedtls_cipher_info_t *info) { return mbedtls_cipher_base_lookup_table[info->base_idx]; @@ -133,7 +134,7 @@ const mbedtls_cipher_info_t *mbedtls_cipher_info_from_values( } #if defined(MBEDTLS_USE_PSA_CRYPTO) && !defined(MBEDTLS_DEPRECATED_REMOVED) -inline psa_key_type_t mbedtls_psa_translate_cipher_type( +static inline psa_key_type_t mbedtls_psa_translate_cipher_type( mbedtls_cipher_type_t cipher) { switch (cipher) { @@ -174,7 +175,7 @@ inline psa_key_type_t mbedtls_psa_translate_cipher_type( } } -inline psa_algorithm_t mbedtls_psa_translate_cipher_mode( +static inline psa_algorithm_t mbedtls_psa_translate_cipher_mode( mbedtls_cipher_mode_t mode, size_t taglen) { switch (mode) { @@ -838,8 +839,14 @@ static void add_pkcs_padding(unsigned char *output, size_t output_len, } } -static int get_pkcs_padding(unsigned char *input, size_t input_len, - size_t *data_len) +/* + * Get the length of the PKCS7 padding. + * + * Note: input_len must be the block size of the cipher. + */ +MBEDTLS_STATIC_TESTABLE int mbedtls_get_pkcs_padding(unsigned char *input, + size_t input_len, + size_t *data_len) { size_t i, pad_idx; unsigned char padding_len; @@ -849,10 +856,6 @@ static int get_pkcs_padding(unsigned char *input, size_t input_len, } padding_len = input[input_len - 1]; - if (padding_len == 0 || padding_len > input_len) { - return MBEDTLS_ERR_CIPHER_INVALID_PADDING; - } - *data_len = input_len - padding_len; mbedtls_ct_condition_t bad = mbedtls_ct_uint_gt(padding_len, input_len); bad = mbedtls_ct_bool_or(bad, mbedtls_ct_uint_eq(padding_len, 0)); @@ -866,6 +869,9 @@ static int get_pkcs_padding(unsigned char *input, size_t input_len, bad = mbedtls_ct_bool_or(bad, mbedtls_ct_bool_and(in_padding, different)); } + /* If the padding is invalid, set the output length to 0 */ + *data_len = mbedtls_ct_if(bad, 0, input_len - padding_len); + return mbedtls_ct_error_if_else_0(bad, MBEDTLS_ERR_CIPHER_INVALID_PADDING); } #endif /* MBEDTLS_CIPHER_PADDING_PKCS7 */ @@ -1144,7 +1150,7 @@ int mbedtls_cipher_set_padding_mode(mbedtls_cipher_context_t *ctx, #if defined(MBEDTLS_CIPHER_PADDING_PKCS7) case MBEDTLS_PADDING_PKCS7: ctx->add_padding = add_pkcs_padding; - ctx->get_padding = get_pkcs_padding; + ctx->get_padding = mbedtls_get_pkcs_padding; break; #endif #if defined(MBEDTLS_CIPHER_PADDING_ONE_AND_ZEROS) @@ -1445,7 +1451,7 @@ static int mbedtls_cipher_aead_encrypt(mbedtls_cipher_context_t *ctx, #if defined(MBEDTLS_GCM_C) if (MBEDTLS_MODE_GCM == ((mbedtls_cipher_mode_t) ctx->cipher_info->mode)) { *olen = ilen; - return mbedtls_gcm_crypt_and_tag((mbedtls_gcm_context *)ctx->cipher_ctx, MBEDTLS_GCM_ENCRYPT, + return mbedtls_gcm_crypt_and_tag((mbedtls_gcm_context *) ctx->cipher_ctx, MBEDTLS_GCM_ENCRYPT, ilen, iv, iv_len, ad, ad_len, input, output, tag_len, tag); } @@ -1525,7 +1531,7 @@ static int mbedtls_cipher_aead_decrypt(mbedtls_cipher_context_t *ctx, int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED; *olen = ilen; - ret = mbedtls_gcm_auth_decrypt((mbedtls_gcm_context *)ctx->cipher_ctx, ilen, + ret = mbedtls_gcm_auth_decrypt((mbedtls_gcm_context *) ctx->cipher_ctx, ilen, iv, iv_len, ad, ad_len, tag, tag_len, input, output); diff --git a/third_party/mbedtls/library/cipher_invasive.h b/third_party/mbedtls/library/cipher_invasive.h new file mode 100644 index 000000000000..6756e4f0756f --- /dev/null +++ b/third_party/mbedtls/library/cipher_invasive.h @@ -0,0 +1 @@ +// dummy file \ No newline at end of file diff --git a/third_party/mbedtls/library/cipher_wrap.cpp b/third_party/mbedtls/library/cipher_wrap.cpp index de0d502ea5bb..712920c80a7a 100644 --- a/third_party/mbedtls/library/cipher_wrap.cpp +++ b/third_party/mbedtls/library/cipher_wrap.cpp @@ -58,9 +58,60 @@ #include "mbedtls/platform.h" -enum mbedtls_cipher_base_index : uint8_t { - MBEDTLS_CIPHER_BASE_INDEX_AES = 0, - MBEDTLS_CIPHER_BASE_INDEX_GCM_AES = 1, +enum mbedtls_cipher_base_index { +#if defined(MBEDTLS_AES_C) + MBEDTLS_CIPHER_BASE_INDEX_AES, +#endif +#if defined(MBEDTLS_ARIA_C) + MBEDTLS_CIPHER_BASE_INDEX_ARIA, +#endif +#if defined(MBEDTLS_CAMELLIA_C) + MBEDTLS_CIPHER_BASE_INDEX_CAMELLIA, +#endif +#if defined(MBEDTLS_CIPHER_HAVE_CCM_AES_VIA_LEGACY_OR_USE_PSA) + MBEDTLS_CIPHER_BASE_INDEX_CCM_AES, +#endif +#if defined(MBEDTLS_CCM_C) && defined(MBEDTLS_ARIA_C) + MBEDTLS_CIPHER_BASE_INDEX_CCM_ARIA, +#endif +#if defined(MBEDTLS_CCM_C) && defined(MBEDTLS_CAMELLIA_C) + MBEDTLS_CIPHER_BASE_INDEX_CCM_CAMELLIA, +#endif +#if defined(MBEDTLS_CHACHA20_C) + MBEDTLS_CIPHER_BASE_INDEX_CHACHA20_BASE, +#endif +#if defined(MBEDTLS_CHACHAPOLY_C) + MBEDTLS_CIPHER_BASE_INDEX_CHACHAPOLY_BASE, +#endif +#if defined(MBEDTLS_DES_C) + MBEDTLS_CIPHER_BASE_INDEX_DES_EDE3, +#endif +#if defined(MBEDTLS_DES_C) + MBEDTLS_CIPHER_BASE_INDEX_DES_EDE, +#endif +#if defined(MBEDTLS_DES_C) + MBEDTLS_CIPHER_BASE_INDEX_DES, +#endif +#if defined(MBEDTLS_CIPHER_HAVE_GCM_AES_VIA_LEGACY_OR_USE_PSA) + MBEDTLS_CIPHER_BASE_INDEX_GCM_AES, +#endif +#if defined(MBEDTLS_GCM_C) && defined(MBEDTLS_ARIA_C) + MBEDTLS_CIPHER_BASE_INDEX_GCM_ARIA, +#endif +#if defined(MBEDTLS_GCM_C) && defined(MBEDTLS_CAMELLIA_C) + MBEDTLS_CIPHER_BASE_INDEX_GCM_CAMELLIA, +#endif +#if defined(MBEDTLS_NIST_KW_C) + MBEDTLS_CIPHER_BASE_INDEX_KW_AES, +#endif +#if defined(MBEDTLS_CIPHER_NULL_CIPHER) + MBEDTLS_CIPHER_BASE_INDEX_NULL_BASE, +#endif +#if defined(MBEDTLS_CIPHER_MODE_XTS) && defined(MBEDTLS_AES_C) + MBEDTLS_CIPHER_BASE_INDEX_XTS_AES, +#endif + /* Prevent compile failure due to empty enum */ + MBEDTLS_CIPHER_BASE_PREVENT_EMPTY_ENUM }; #if defined(MBEDTLS_GCM_C) && \ @@ -80,7 +131,7 @@ static void *gcm_ctx_alloc(void) static void gcm_ctx_free(void *ctx) { - mbedtls_gcm_free((mbedtls_gcm_context *)ctx); + mbedtls_gcm_free((mbedtls_gcm_context *) ctx); mbedtls_free(ctx); } #endif /* MBEDTLS_GCM_C */ @@ -195,7 +246,7 @@ static int aes_setkey_enc_wrap(void *ctx, const unsigned char *key, static void *aes_ctx_alloc(void) { - mbedtls_aes_context *aes = (mbedtls_aes_context *)mbedtls_calloc(1, sizeof(mbedtls_aes_context)); + mbedtls_aes_context *aes = (mbedtls_aes_context *) mbedtls_calloc(1, sizeof(mbedtls_aes_context)); if (aes == NULL) { return NULL; @@ -777,7 +828,7 @@ static int camellia_setkey_enc_wrap(void *ctx, const unsigned char *key, static void *camellia_ctx_alloc(void) { mbedtls_camellia_context *ctx; - ctx = (mbedtls_camellia_context *)mbedtls_calloc(1, sizeof(mbedtls_camellia_context)); + ctx = mbedtls_calloc(1, sizeof(mbedtls_camellia_context)); if (ctx == NULL) { return NULL; @@ -1195,7 +1246,7 @@ static int aria_setkey_enc_wrap(void *ctx, const unsigned char *key, static void *aria_ctx_alloc(void) { mbedtls_aria_context *ctx; - ctx = (mbedtls_aria_context *)mbedtls_calloc(1, sizeof(mbedtls_aria_context)); + ctx = mbedtls_calloc(1, sizeof(mbedtls_aria_context)); if (ctx == NULL) { return NULL; @@ -2374,9 +2425,58 @@ const mbedtls_cipher_definition_t mbedtls_cipher_definitions[] = sizeof(mbedtls_cipher_definitions[0])) int mbedtls_cipher_supported[NUM_CIPHERS]; -const mbedtls_cipher_base_t *mbedtls_cipher_base_lookup_table[] = { - /* MBEDTLS_CIPHER_BASE_INDEX_AES */ &aes_info, - /* MBEDTLS_CIPHER_BASE_INDEX_GCM_AES */ &gcm_aes_info, +const mbedtls_cipher_base_t * const mbedtls_cipher_base_lookup_table[] = { +#if defined(MBEDTLS_AES_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_AES] = */ &aes_info, +#endif +#if defined(MBEDTLS_ARIA_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_ARIA] = */ &aria_info, +#endif +#if defined(MBEDTLS_CAMELLIA_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_CAMELLIA] = */ &camellia_info, +#endif +#if defined(MBEDTLS_CIPHER_HAVE_CCM_AES_VIA_LEGACY_OR_USE_PSA) + /* [MBEDTLS_CIPHER_BASE_INDEX_CCM_AES] = */ &ccm_aes_info, +#endif +#if defined(MBEDTLS_CCM_C) && defined(MBEDTLS_ARIA_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_CCM_ARIA] = */ &ccm_aria_info, +#endif +#if defined(MBEDTLS_CCM_C) && defined(MBEDTLS_CAMELLIA_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_CCM_CAMELLIA] = */ &ccm_camellia_info, +#endif +#if defined(MBEDTLS_CHACHA20_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_CHACHA20_BASE] = */ &chacha20_base_info, +#endif +#if defined(MBEDTLS_CHACHAPOLY_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_CHACHAPOLY_BASE] = */ &chachapoly_base_info, +#endif +#if defined(MBEDTLS_DES_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_DES_EDE3] = */ &des_ede3_info, +#endif +#if defined(MBEDTLS_DES_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_DES_EDE] = */ &des_ede_info, +#endif +#if defined(MBEDTLS_DES_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_DES] = */ &des_info, +#endif +#if defined(MBEDTLS_CIPHER_HAVE_GCM_AES_VIA_LEGACY_OR_USE_PSA) + /* [MBEDTLS_CIPHER_BASE_INDEX_GCM_AES] = */ &gcm_aes_info, +#endif +#if defined(MBEDTLS_GCM_C) && defined(MBEDTLS_ARIA_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_GCM_ARIA] = */ &gcm_aria_info, +#endif +#if defined(MBEDTLS_GCM_C) && defined(MBEDTLS_CAMELLIA_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_GCM_CAMELLIA] =*/ &gcm_camellia_info, +#endif +#if defined(MBEDTLS_NIST_KW_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_KW_AES] = */ &kw_aes_info, +#endif +#if defined(MBEDTLS_CIPHER_NULL_CIPHER) + /* [MBEDTLS_CIPHER_BASE_INDEX_NULL_BASE] = */ &null_base_info, +#endif +#if defined(MBEDTLS_CIPHER_MODE_XTS) && defined(MBEDTLS_AES_C) + /* [MBEDTLS_CIPHER_BASE_INDEX_XTS_AES] = */ &xts_aes_info +#endif }; #endif /* MBEDTLS_CIPHER_C */ diff --git a/third_party/mbedtls/library/cipher_wrap.h b/third_party/mbedtls/library/cipher_wrap.h index f22915120dda..9564c5efe1c0 100644 --- a/third_party/mbedtls/library/cipher_wrap.h +++ b/third_party/mbedtls/library/cipher_wrap.h @@ -169,7 +169,7 @@ extern const mbedtls_cipher_definition_t mbedtls_cipher_definitions[]; extern int mbedtls_cipher_supported[]; -extern const mbedtls_cipher_base_t *mbedtls_cipher_base_lookup_table[]; +extern const mbedtls_cipher_base_t * const mbedtls_cipher_base_lookup_table[]; #ifdef __cplusplus } diff --git a/third_party/mbedtls/library/common.h b/third_party/mbedtls/library/common.h index 1e2976b83bc7..50f2a29a7237 100644 --- a/third_party/mbedtls/library/common.h +++ b/third_party/mbedtls/library/common.h @@ -135,7 +135,7 @@ void mbedtls_zeroize_and_free(void *buf, size_t len); * Note that this is only a valid pointer if the size of the * buffer is at least \p n + 1. */ -inline unsigned char *mbedtls_buffer_offset( +static inline unsigned char *mbedtls_buffer_offset( unsigned char *p, size_t n) { return p == NULL ? NULL : p + n; @@ -152,7 +152,7 @@ inline unsigned char *mbedtls_buffer_offset( * Note that this is only a valid pointer if the size of the * buffer is at least \p n + 1. */ -inline const unsigned char *mbedtls_buffer_offset_const( +static inline const unsigned char *mbedtls_buffer_offset_const( const unsigned char *p, size_t n) { return p == NULL ? NULL : p + n; @@ -183,7 +183,7 @@ __attribute__((always_inline)) * the result is not used immediately (e.g., in AES-CTR), mbedtls_xor() may be faster. * For targets without SIMD support, they will behave the same. */ -inline void mbedtls_xor(unsigned char *r, +static inline void mbedtls_xor(unsigned char *r, const unsigned char *a, const unsigned char *b, size_t n) @@ -265,7 +265,7 @@ __attribute__((always_inline)) * the result is not used immediately (e.g., in AES-CTR), mbedtls_xor() may be faster. * For targets without SIMD support, they will behave the same. */ -inline void mbedtls_xor_no_simd(unsigned char *r, +static inline void mbedtls_xor_no_simd(unsigned char *r, const unsigned char *a, const unsigned char *b, size_t n) @@ -434,4 +434,20 @@ inline void mbedtls_xor_no_simd(unsigned char *r, # define MBEDTLS_MAYBE_UNUSED #endif +/* GCC >= 15 has a warning 'unterminated-string-initialization' which complains if you initialize + * a string into an array without space for a terminating NULL character. In some places in the + * codebase this behaviour is intended, so we add the macro MBEDTLS_ATTRIBUTE_UNTERMINATED_STRING + * to suppress the warning in these places. + */ +#if defined(__has_attribute) +#if __has_attribute(nonstring) +#define MBEDTLS_HAS_ATTRIBUTE_NONSTRING +#endif /* __has_attribute(nonstring) */ +#endif /* __has_attribute */ +#if defined(MBEDTLS_HAS_ATTRIBUTE_NONSTRING) +#define MBEDTLS_ATTRIBUTE_UNTERMINATED_STRING __attribute__((nonstring)) +#else +#define MBEDTLS_ATTRIBUTE_UNTERMINATED_STRING +#endif /* MBEDTLS_HAS_ATTRIBUTE_NONSTRING */ + #endif /* MBEDTLS_LIBRARY_COMMON_H */ diff --git a/third_party/mbedtls/library/constant_time.cpp b/third_party/mbedtls/library/constant_time.cpp index e648a78a28c2..285f963b3c41 100644 --- a/third_party/mbedtls/library/constant_time.cpp +++ b/third_party/mbedtls/library/constant_time.cpp @@ -46,7 +46,7 @@ volatile mbedtls_ct_uint_t mbedtls_ct_zero = 0; /* We check pointer sizes to avoid issues with them not matching register size requirements */ #define MBEDTLS_EFFICIENT_UNALIGNED_VOLATILE_ACCESS -inline uint32_t mbedtls_get_unaligned_volatile_uint32(volatile const unsigned char *p) +static inline uint32_t mbedtls_get_unaligned_volatile_uint32(volatile const unsigned char *p) { /* This is UB, even where it's safe: * return *((volatile uint32_t*)p); diff --git a/third_party/mbedtls/library/constant_time_impl.h b/third_party/mbedtls/library/constant_time_impl.h index f63176c16bcd..aeaeecb7de39 100644 --- a/third_party/mbedtls/library/constant_time_impl.h +++ b/third_party/mbedtls/library/constant_time_impl.h @@ -17,12 +17,12 @@ #endif /* - * To improve readability of constant_time_internal.h, the inline + * To improve readability of constant_time_internal.h, the static inline * definitions are here, and constant_time_internal.h has only the declarations. * * This results in duplicate declarations of the form: - * inline void f(); // from constant_time_internal.h - * inline void f() { ... } // from constant_time_impl.h + * static inline void f(); // from constant_time_internal.h + * static inline void f() { ... } // from constant_time_impl.h * when constant_time_internal.h is included. * * This appears to behave as if the declaration-without-definition was not present @@ -36,24 +36,9 @@ #pragma GCC diagnostic ignored "-Wredundant-decls" #endif -/* Disable asm under Memsan because it confuses Memsan and generates false errors. - * - * We also disable under Valgrind by default, because it's more useful - * for Valgrind to test the plain C implementation. MBEDTLS_TEST_CONSTANT_FLOW_ASM //no-check-names - * may be set to permit building asm under Valgrind. - */ -#if defined(MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN) || \ - (defined(MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND) && !defined(MBEDTLS_TEST_CONSTANT_FLOW_ASM)) //no-check-names -#define MBEDTLS_CT_NO_ASM -#elif defined(__has_feature) -#if __has_feature(memory_sanitizer) -#define MBEDTLS_CT_NO_ASM -#endif -#endif - /* armcc5 --gnu defines __GNUC__ but doesn't support GNU's extended asm */ #if defined(MBEDTLS_HAVE_ASM) && defined(__GNUC__) && (!defined(__ARMCC_VERSION) || \ - __ARMCC_VERSION >= 6000000) && !defined(MBEDTLS_CT_NO_ASM) + __ARMCC_VERSION >= 6000000) #define MBEDTLS_CT_ASM #if (defined(__arm__) || defined(__thumb__) || defined(__thumb2__)) #define MBEDTLS_CT_ARM_ASM @@ -97,7 +82,7 @@ extern volatile mbedtls_ct_uint_t mbedtls_ct_zero; * there is no way for the compiler to ever know anything about * the value of an mbedtls_ct_condition_t. */ -inline mbedtls_ct_uint_t mbedtls_ct_compiler_opaque(mbedtls_ct_uint_t x) +static inline mbedtls_ct_uint_t mbedtls_ct_compiler_opaque(mbedtls_ct_uint_t x) { #if defined(MBEDTLS_CT_ASM) asm volatile ("" : [x] "+r" (x) :); @@ -132,7 +117,7 @@ inline mbedtls_ct_uint_t mbedtls_ct_compiler_opaque(mbedtls_ct_uint_t x) #endif /* Convert a number into a condition in constant time. */ -inline mbedtls_ct_condition_t mbedtls_ct_bool(mbedtls_ct_uint_t x) +static inline mbedtls_ct_condition_t mbedtls_ct_bool(mbedtls_ct_uint_t x) { /* * Define mask-generation code that, as far as possible, will not use branches or conditional instructions. @@ -217,7 +202,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool(mbedtls_ct_uint_t x) #endif } -inline mbedtls_ct_uint_t mbedtls_ct_if(mbedtls_ct_condition_t condition, +static inline mbedtls_ct_uint_t mbedtls_ct_if(mbedtls_ct_condition_t condition, mbedtls_ct_uint_t if1, mbedtls_ct_uint_t if0) { @@ -283,7 +268,7 @@ inline mbedtls_ct_uint_t mbedtls_ct_if(mbedtls_ct_condition_t condition, #endif } -inline mbedtls_ct_condition_t mbedtls_ct_uint_lt(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y) +static inline mbedtls_ct_condition_t mbedtls_ct_uint_lt(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y) { #if defined(MBEDTLS_CT_AARCH64_ASM) && (defined(MBEDTLS_CT_SIZE_32) || defined(MBEDTLS_CT_SIZE_64)) uint64_t s1; @@ -394,7 +379,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_uint_lt(mbedtls_ct_uint_t x, mbedtls_ct #endif } -inline mbedtls_ct_condition_t mbedtls_ct_uint_ne(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y) +static inline mbedtls_ct_condition_t mbedtls_ct_uint_ne(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y) { /* diff = 0 if x == y, non-zero otherwise */ const mbedtls_ct_uint_t diff = mbedtls_ct_compiler_opaque(x) ^ mbedtls_ct_compiler_opaque(y); @@ -403,7 +388,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_uint_ne(mbedtls_ct_uint_t x, mbedtls_ct return mbedtls_ct_bool(diff); } -inline unsigned char mbedtls_ct_uchar_in_range_if(unsigned char low, +static inline unsigned char mbedtls_ct_uchar_in_range_if(unsigned char low, unsigned char high, unsigned char c, unsigned char t) @@ -423,21 +408,21 @@ inline unsigned char mbedtls_ct_uchar_in_range_if(unsigned char low, * Everything below here is trivial wrapper functions */ -inline size_t mbedtls_ct_size_if(mbedtls_ct_condition_t condition, +static inline size_t mbedtls_ct_size_if(mbedtls_ct_condition_t condition, size_t if1, size_t if0) { return (size_t) mbedtls_ct_if(condition, (mbedtls_ct_uint_t) if1, (mbedtls_ct_uint_t) if0); } -inline unsigned mbedtls_ct_uint_if(mbedtls_ct_condition_t condition, +static inline unsigned mbedtls_ct_uint_if(mbedtls_ct_condition_t condition, unsigned if1, unsigned if0) { return (unsigned) mbedtls_ct_if(condition, (mbedtls_ct_uint_t) if1, (mbedtls_ct_uint_t) if0); } -inline mbedtls_ct_condition_t mbedtls_ct_bool_if(mbedtls_ct_condition_t condition, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_if(mbedtls_ct_condition_t condition, mbedtls_ct_condition_t if1, mbedtls_ct_condition_t if0) { @@ -447,7 +432,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool_if(mbedtls_ct_condition_t conditio #if defined(MBEDTLS_BIGNUM_C) -inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if(mbedtls_ct_condition_t condition, +static inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if(mbedtls_ct_condition_t condition, mbedtls_mpi_uint if1, mbedtls_mpi_uint if0) { @@ -458,17 +443,17 @@ inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if(mbedtls_ct_condition_t condition, #endif -inline size_t mbedtls_ct_size_if_else_0(mbedtls_ct_condition_t condition, size_t if1) +static inline size_t mbedtls_ct_size_if_else_0(mbedtls_ct_condition_t condition, size_t if1) { return (size_t) (condition & if1); } -inline unsigned mbedtls_ct_uint_if_else_0(mbedtls_ct_condition_t condition, unsigned if1) +static inline unsigned mbedtls_ct_uint_if_else_0(mbedtls_ct_condition_t condition, unsigned if1) { return (unsigned) (condition & if1); } -inline mbedtls_ct_condition_t mbedtls_ct_bool_if_else_0(mbedtls_ct_condition_t condition, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_if_else_0(mbedtls_ct_condition_t condition, mbedtls_ct_condition_t if1) { return (mbedtls_ct_condition_t) (condition & if1); @@ -476,7 +461,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool_if_else_0(mbedtls_ct_condition_t c #if defined(MBEDTLS_BIGNUM_C) -inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if_else_0(mbedtls_ct_condition_t condition, +static inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if_else_0(mbedtls_ct_condition_t condition, mbedtls_mpi_uint if1) { return (mbedtls_mpi_uint) (condition & if1); @@ -484,7 +469,7 @@ inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if_else_0(mbedtls_ct_condition_t con #endif /* MBEDTLS_BIGNUM_C */ -inline int mbedtls_ct_error_if(mbedtls_ct_condition_t condition, int if1, int if0) +static inline int mbedtls_ct_error_if(mbedtls_ct_condition_t condition, int if1, int if0) { /* Coverting int -> uint -> int here is safe, because we require if1 and if0 to be * in the range -32767..0, and we require 32-bit int and uint types. @@ -496,54 +481,54 @@ inline int mbedtls_ct_error_if(mbedtls_ct_condition_t condition, int if1, int if (mbedtls_ct_uint_t) (-if0))); } -inline int mbedtls_ct_error_if_else_0(mbedtls_ct_condition_t condition, int if1) +static inline int mbedtls_ct_error_if_else_0(mbedtls_ct_condition_t condition, int if1) { return -((int) (condition & (-if1))); } -inline mbedtls_ct_condition_t mbedtls_ct_uint_eq(mbedtls_ct_uint_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_uint_eq(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y) { return ~mbedtls_ct_uint_ne(x, y); } -inline mbedtls_ct_condition_t mbedtls_ct_uint_gt(mbedtls_ct_uint_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_uint_gt(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y) { return mbedtls_ct_uint_lt(y, x); } -inline mbedtls_ct_condition_t mbedtls_ct_uint_ge(mbedtls_ct_uint_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_uint_ge(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y) { return ~mbedtls_ct_uint_lt(x, y); } -inline mbedtls_ct_condition_t mbedtls_ct_uint_le(mbedtls_ct_uint_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_uint_le(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y) { return ~mbedtls_ct_uint_gt(x, y); } -inline mbedtls_ct_condition_t mbedtls_ct_bool_ne(mbedtls_ct_condition_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_ne(mbedtls_ct_condition_t x, mbedtls_ct_condition_t y) { return (mbedtls_ct_condition_t) (x ^ y); } -inline mbedtls_ct_condition_t mbedtls_ct_bool_and(mbedtls_ct_condition_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_and(mbedtls_ct_condition_t x, mbedtls_ct_condition_t y) { return (mbedtls_ct_condition_t) (x & y); } -inline mbedtls_ct_condition_t mbedtls_ct_bool_or(mbedtls_ct_condition_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_or(mbedtls_ct_condition_t x, mbedtls_ct_condition_t y) { return (mbedtls_ct_condition_t) (x | y); } -inline mbedtls_ct_condition_t mbedtls_ct_bool_not(mbedtls_ct_condition_t x) +static inline mbedtls_ct_condition_t mbedtls_ct_bool_not(mbedtls_ct_condition_t x) { return (mbedtls_ct_condition_t) (~x); } diff --git a/third_party/mbedtls/library/constant_time_internal.h b/third_party/mbedtls/library/constant_time_internal.h index 92a026fcd2e0..61a5c6d4e952 100644 --- a/third_party/mbedtls/library/constant_time_internal.h +++ b/third_party/mbedtls/library/constant_time_internal.h @@ -58,7 +58,7 @@ * (this has been observed to be constant-time on latest gcc, clang and MSVC * as of May 2023). * - * For readability, the inline definitions are separated out into + * For readability, the static inline definitions are separated out into * constant_time_impl.h. */ @@ -96,7 +96,7 @@ typedef int32_t mbedtls_ct_int_t; * \return MBEDTLS_CT_TRUE if \p x != 0, or MBEDTLS_CT_FALSE if \p x == 0 * */ -inline mbedtls_ct_condition_t mbedtls_ct_bool(mbedtls_ct_uint_t x); +static inline mbedtls_ct_condition_t mbedtls_ct_bool(mbedtls_ct_uint_t x); /** Boolean "not equal" operation. * @@ -109,7 +109,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool(mbedtls_ct_uint_t x); * * \return MBEDTLS_CT_TRUE if \p x != \p y, otherwise MBEDTLS_CT_FALSE. */ -inline mbedtls_ct_condition_t mbedtls_ct_uint_ne(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y); +static inline mbedtls_ct_condition_t mbedtls_ct_uint_ne(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y); /** Boolean "equals" operation. * @@ -122,7 +122,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_uint_ne(mbedtls_ct_uint_t x, mbedtls_ct * * \return MBEDTLS_CT_TRUE if \p x == \p y, otherwise MBEDTLS_CT_FALSE. */ -inline mbedtls_ct_condition_t mbedtls_ct_uint_eq(mbedtls_ct_uint_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_uint_eq(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y); /** Boolean "less than" operation. @@ -136,7 +136,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_uint_eq(mbedtls_ct_uint_t x, * * \return MBEDTLS_CT_TRUE if \p x < \p y, otherwise MBEDTLS_CT_FALSE. */ -inline mbedtls_ct_condition_t mbedtls_ct_uint_lt(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y); +static inline mbedtls_ct_condition_t mbedtls_ct_uint_lt(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y); /** Boolean "greater than" operation. * @@ -149,7 +149,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_uint_lt(mbedtls_ct_uint_t x, mbedtls_ct * * \return MBEDTLS_CT_TRUE if \p x > \p y, otherwise MBEDTLS_CT_FALSE. */ -inline mbedtls_ct_condition_t mbedtls_ct_uint_gt(mbedtls_ct_uint_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_uint_gt(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y); /** Boolean "greater or equal" operation. @@ -164,7 +164,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_uint_gt(mbedtls_ct_uint_t x, * \return MBEDTLS_CT_TRUE if \p x >= \p y, * otherwise MBEDTLS_CT_FALSE. */ -inline mbedtls_ct_condition_t mbedtls_ct_uint_ge(mbedtls_ct_uint_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_uint_ge(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y); /** Boolean "less than or equal" operation. @@ -179,7 +179,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_uint_ge(mbedtls_ct_uint_t x, * \return MBEDTLS_CT_TRUE if \p x <= \p y, * otherwise MBEDTLS_CT_FALSE. */ -inline mbedtls_ct_condition_t mbedtls_ct_uint_le(mbedtls_ct_uint_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_uint_le(mbedtls_ct_uint_t x, mbedtls_ct_uint_t y); /** Boolean not-equals operation. @@ -197,7 +197,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_uint_le(mbedtls_ct_uint_t x, * \return MBEDTLS_CT_TRUE if \p x != \p y, * otherwise MBEDTLS_CT_FALSE. */ -inline mbedtls_ct_condition_t mbedtls_ct_bool_ne(mbedtls_ct_condition_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_ne(mbedtls_ct_condition_t x, mbedtls_ct_condition_t y); /** Boolean "and" operation. @@ -212,7 +212,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool_ne(mbedtls_ct_condition_t x, * \return MBEDTLS_CT_TRUE if \p x && \p y, * otherwise MBEDTLS_CT_FALSE. */ -inline mbedtls_ct_condition_t mbedtls_ct_bool_and(mbedtls_ct_condition_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_and(mbedtls_ct_condition_t x, mbedtls_ct_condition_t y); /** Boolean "or" operation. @@ -227,7 +227,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool_and(mbedtls_ct_condition_t x, * \return MBEDTLS_CT_TRUE if \p x || \p y, * otherwise MBEDTLS_CT_FALSE. */ -inline mbedtls_ct_condition_t mbedtls_ct_bool_or(mbedtls_ct_condition_t x, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_or(mbedtls_ct_condition_t x, mbedtls_ct_condition_t y); /** Boolean "not" operation. @@ -240,7 +240,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool_or(mbedtls_ct_condition_t x, * * \return MBEDTLS_CT_FALSE if \p x, otherwise MBEDTLS_CT_TRUE. */ -inline mbedtls_ct_condition_t mbedtls_ct_bool_not(mbedtls_ct_condition_t x); +static inline mbedtls_ct_condition_t mbedtls_ct_bool_not(mbedtls_ct_condition_t x); /* ============================================================================ @@ -259,7 +259,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool_not(mbedtls_ct_condition_t x); * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise \c if0. */ -inline size_t mbedtls_ct_size_if(mbedtls_ct_condition_t condition, +static inline size_t mbedtls_ct_size_if(mbedtls_ct_condition_t condition, size_t if1, size_t if0); @@ -275,7 +275,7 @@ inline size_t mbedtls_ct_size_if(mbedtls_ct_condition_t condition, * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise \c if0. */ -inline unsigned mbedtls_ct_uint_if(mbedtls_ct_condition_t condition, +static inline unsigned mbedtls_ct_uint_if(mbedtls_ct_condition_t condition, unsigned if1, unsigned if0); @@ -291,7 +291,7 @@ inline unsigned mbedtls_ct_uint_if(mbedtls_ct_condition_t condition, * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise \c if0. */ -inline mbedtls_ct_condition_t mbedtls_ct_bool_if(mbedtls_ct_condition_t condition, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_if(mbedtls_ct_condition_t condition, mbedtls_ct_condition_t if1, mbedtls_ct_condition_t if0); @@ -309,7 +309,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool_if(mbedtls_ct_condition_t conditio * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise \c if0. */ -inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if(mbedtls_ct_condition_t condition, \ +static inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if(mbedtls_ct_condition_t condition, \ mbedtls_mpi_uint if1, \ mbedtls_mpi_uint if0); @@ -329,7 +329,7 @@ inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if(mbedtls_ct_condition_t condition, * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise 0. */ -inline unsigned mbedtls_ct_uint_if_else_0(mbedtls_ct_condition_t condition, unsigned if1); +static inline unsigned mbedtls_ct_uint_if_else_0(mbedtls_ct_condition_t condition, unsigned if1); /** Choose between an mbedtls_ct_condition_t and 0. * @@ -345,7 +345,7 @@ inline unsigned mbedtls_ct_uint_if_else_0(mbedtls_ct_condition_t condition, unsi * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise 0. */ -inline mbedtls_ct_condition_t mbedtls_ct_bool_if_else_0(mbedtls_ct_condition_t condition, +static inline mbedtls_ct_condition_t mbedtls_ct_bool_if_else_0(mbedtls_ct_condition_t condition, mbedtls_ct_condition_t if1); /** Choose between a size_t value and 0. @@ -362,7 +362,7 @@ inline mbedtls_ct_condition_t mbedtls_ct_bool_if_else_0(mbedtls_ct_condition_t c * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise 0. */ -inline size_t mbedtls_ct_size_if_else_0(mbedtls_ct_condition_t condition, size_t if1); +static inline size_t mbedtls_ct_size_if_else_0(mbedtls_ct_condition_t condition, size_t if1); #if defined(MBEDTLS_BIGNUM_C) @@ -380,7 +380,7 @@ inline size_t mbedtls_ct_size_if_else_0(mbedtls_ct_condition_t condition, size_t * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise 0. */ -inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if_else_0(mbedtls_ct_condition_t condition, +static inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if_else_0(mbedtls_ct_condition_t condition, mbedtls_mpi_uint if1); #endif @@ -394,7 +394,7 @@ inline mbedtls_mpi_uint mbedtls_ct_mpi_uint_if_else_0(mbedtls_ct_condition_t con * * \return \p t if \p low <= \p c <= \p high, 0 otherwise. */ -inline unsigned char mbedtls_ct_uchar_in_range_if(unsigned char low, +static inline unsigned char mbedtls_ct_uchar_in_range_if(unsigned char low, unsigned char high, unsigned char c, unsigned char t); @@ -411,7 +411,7 @@ inline unsigned char mbedtls_ct_uchar_in_range_if(unsigned char low, * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise \c if0. */ -inline int mbedtls_ct_error_if(mbedtls_ct_condition_t condition, int if1, int if0); +static inline int mbedtls_ct_error_if(mbedtls_ct_condition_t condition, int if1, int if0); /** Choose between an error value and 0. The error value must be in the range [-32767..0]. * @@ -427,7 +427,7 @@ inline int mbedtls_ct_error_if(mbedtls_ct_condition_t condition, int if1, int if * * \return \c if1 if \p condition == MBEDTLS_CT_TRUE, otherwise 0. */ -inline int mbedtls_ct_error_if_else_0(mbedtls_ct_condition_t condition, int if1); +static inline int mbedtls_ct_error_if_else_0(mbedtls_ct_condition_t condition, int if1); /* ============================================================================ * Block memory operations @@ -573,7 +573,7 @@ int mbedtls_ct_memcmp_partial(const void *a, #endif -/* Include the implementation of inline functions above. */ +/* Include the implementation of static inline functions above. */ #include "constant_time_impl.h" #endif /* MBEDTLS_CONSTANT_TIME_INTERNAL_H */ diff --git a/third_party/mbedtls/library/ctr.h b/third_party/mbedtls/library/ctr.h index 98dce68dc4c2..aa48fb9e707a 100644 --- a/third_party/mbedtls/library/ctr.h +++ b/third_party/mbedtls/library/ctr.h @@ -1 +1,35 @@ -//ct \ No newline at end of file +/** + * \file ctr.h + * + * \brief This file contains common functionality for counter algorithms. + * + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ + +#ifndef MBEDTLS_CTR_H +#define MBEDTLS_CTR_H + +#include "common.h" + +/** + * \brief Increment a big-endian 16-byte value. + * This is quite performance-sensitive for AES-CTR and CTR-DRBG. + * + * \param n A 16-byte value to be incremented. + */ +static inline void mbedtls_ctr_increment_counter(uint8_t n[16]) +{ + // The 32-bit version seems to perform about the same as a 64-bit version + // on 64-bit architectures, so no need to define a 64-bit version. + for (int i = 3;; i--) { + uint32_t x = MBEDTLS_GET_UINT32_BE(n, i << 2); + x += 1; + MBEDTLS_PUT_UINT32_BE(x, n, i << 2); + if (x != 0 || i == 0) { + break; + } + } +} + +#endif /* MBEDTLS_CTR_H */ diff --git a/third_party/mbedtls/library/gcm.cpp b/third_party/mbedtls/library/gcm.cpp index f4384cb04277..5dfac2349c82 100644 --- a/third_party/mbedtls/library/gcm.cpp +++ b/third_party/mbedtls/library/gcm.cpp @@ -55,7 +55,7 @@ void mbedtls_gcm_init(mbedtls_gcm_context *ctx) memset(ctx, 0, sizeof(mbedtls_gcm_context)); } -inline void gcm_set_acceleration(mbedtls_gcm_context *ctx) +static inline void gcm_set_acceleration(mbedtls_gcm_context *ctx) { #if defined(MBEDTLS_GCM_LARGE_TABLE) ctx->acceleration = MBEDTLS_GCM_ACC_LARGETABLE; @@ -77,7 +77,7 @@ inline void gcm_set_acceleration(mbedtls_gcm_context *ctx) #endif } -inline void gcm_gen_table_rightshift(uint64_t dst[2], const uint64_t src[2]) +static inline void gcm_gen_table_rightshift(uint64_t dst[2], const uint64_t src[2]) { uint8_t *u8Dst = (uint8_t *) dst; uint8_t *u8Src = (uint8_t *) src; diff --git a/third_party/mbedtls/library/md.cpp b/third_party/mbedtls/library/md.cpp index 10c6844adaf3..00addd62c389 100644 --- a/third_party/mbedtls/library/md.cpp +++ b/third_party/mbedtls/library/md.cpp @@ -283,17 +283,17 @@ void mbedtls_md_free(mbedtls_md_context_t *ctx) #endif #if defined(MBEDTLS_SHA1_C) case MBEDTLS_MD_SHA1: - mbedtls_sha1_free((mbedtls_sha1_context *)ctx->md_ctx); + mbedtls_sha1_free((mbedtls_sha1_context *) ctx->md_ctx); break; #endif #if defined(MBEDTLS_SHA224_C) case MBEDTLS_MD_SHA224: - mbedtls_sha256_free((mbedtls_sha256_context *)ctx->md_ctx); + mbedtls_sha256_free(ctx->md_ctx); break; #endif #if defined(MBEDTLS_SHA256_C) case MBEDTLS_MD_SHA256: - mbedtls_sha256_free((mbedtls_sha256_context *)ctx->md_ctx); + mbedtls_sha256_free((mbedtls_sha256_context *) ctx->md_ctx); break; #endif #if defined(MBEDTLS_SHA384_C) @@ -368,17 +368,17 @@ int mbedtls_md_clone(mbedtls_md_context_t *dst, #endif #if defined(MBEDTLS_SHA1_C) case MBEDTLS_MD_SHA1: - mbedtls_sha1_clone((mbedtls_sha1_context *)dst->md_ctx, (mbedtls_sha1_context *)src->md_ctx); + mbedtls_sha1_clone((mbedtls_sha1_context *) dst->md_ctx, (mbedtls_sha1_context *) src->md_ctx); break; #endif #if defined(MBEDTLS_SHA224_C) case MBEDTLS_MD_SHA224: - mbedtls_sha256_clone((mbedtls_sha256_context *)dst->md_ctx, (mbedtls_sha256_context *)src->md_ctx); + mbedtls_sha256_clone(dst->md_ctx, src->md_ctx); break; #endif #if defined(MBEDTLS_SHA256_C) case MBEDTLS_MD_SHA256: - mbedtls_sha256_clone((mbedtls_sha256_context *)dst->md_ctx, (mbedtls_sha256_context *)src->md_ctx); + mbedtls_sha256_clone((mbedtls_sha256_context *) dst->md_ctx, (mbedtls_sha256_context *) src->md_ctx); break; #endif #if defined(MBEDTLS_SHA384_C) @@ -408,10 +408,10 @@ int mbedtls_md_clone(mbedtls_md_context_t *dst, #define ALLOC(type) \ do { \ - ctx->md_ctx = (mbedtls_##type##_context*) mbedtls_calloc(1, sizeof(mbedtls_##type##_context)); \ + ctx->md_ctx = mbedtls_calloc(1, sizeof(mbedtls_##type##_context)); \ if (ctx->md_ctx == NULL) \ return MBEDTLS_ERR_MD_ALLOC_FAILED; \ - mbedtls_##type##_init((mbedtls_##type##_context*)ctx->md_ctx); \ + mbedtls_##type##_init((mbedtls_##type##_context *) ctx->md_ctx); \ } \ while (0) @@ -539,7 +539,7 @@ int mbedtls_md_starts(mbedtls_md_context_t *ctx) #endif #if defined(MBEDTLS_SHA224_C) case MBEDTLS_MD_SHA224: - return mbedtls_sha256_starts((mbedtls_sha256_context *)ctx->md_ctx, 1); + return mbedtls_sha256_starts(ctx->md_ctx, 1); #endif #if defined(MBEDTLS_SHA256_C) case MBEDTLS_MD_SHA256: @@ -598,7 +598,7 @@ int mbedtls_md_update(mbedtls_md_context_t *ctx, const unsigned char *input, siz #endif #if defined(MBEDTLS_SHA224_C) case MBEDTLS_MD_SHA224: - return mbedtls_sha256_update((mbedtls_sha256_context *)ctx->md_ctx, input, ilen); + return mbedtls_sha256_update(ctx->md_ctx, input, ilen); #endif #if defined(MBEDTLS_SHA256_C) case MBEDTLS_MD_SHA256: @@ -656,7 +656,7 @@ int mbedtls_md_finish(mbedtls_md_context_t *ctx, unsigned char *output) #endif #if defined(MBEDTLS_SHA224_C) case MBEDTLS_MD_SHA224: - return mbedtls_sha256_finish((mbedtls_sha256_context *)ctx->md_ctx, output); + return mbedtls_sha256_finish(ctx->md_ctx, output); #endif #if defined(MBEDTLS_SHA256_C) case MBEDTLS_MD_SHA256: diff --git a/third_party/mbedtls/library/md_psa.h b/third_party/mbedtls/library/md_psa.h index 6756e4f0756f..ae22b408f869 100644 --- a/third_party/mbedtls/library/md_psa.h +++ b/third_party/mbedtls/library/md_psa.h @@ -1 +1 @@ -// dummy file \ No newline at end of file +// dummy file to make amalgamation happy diff --git a/third_party/mbedtls/library/oid.cpp b/third_party/mbedtls/library/oid.cpp index 7f2a71de1c6d..fdaa52deeb71 100644 --- a/third_party/mbedtls/library/oid.cpp +++ b/third_party/mbedtls/library/oid.cpp @@ -1071,7 +1071,7 @@ int mbedtls_oid_from_numeric_string(mbedtls_asn1_buf *oid, size_t bytes_per_subidentifier = (((sizeof(unsigned int) * 8) - 1) / 7) + 1; size_t max_possible_bytes = num_dots * bytes_per_subidentifier; - oid->p = (unsigned char*) mbedtls_calloc(max_possible_bytes, 1); + oid->p = (unsigned char *) mbedtls_calloc(max_possible_bytes, 1); if (oid->p == NULL) { return MBEDTLS_ERR_ASN1_ALLOC_FAILED; } @@ -1142,7 +1142,7 @@ int mbedtls_oid_from_numeric_string(mbedtls_asn1_buf *oid, } encoded_len = (size_t) (out_ptr - oid->p); - resized_mem = (unsigned char*) mbedtls_calloc(encoded_len, 1); + resized_mem = (unsigned char *) mbedtls_calloc(encoded_len, 1); if (resized_mem == NULL) { ret = MBEDTLS_ERR_ASN1_ALLOC_FAILED; goto error; diff --git a/third_party/mbedtls/library/pem.cpp b/third_party/mbedtls/library/pem.cpp index d663fa242d2d..f2d06b1d22e9 100644 --- a/third_party/mbedtls/library/pem.cpp +++ b/third_party/mbedtls/library/pem.cpp @@ -243,7 +243,10 @@ static int pem_aes_decrypt(unsigned char aes_iv[16], unsigned int keylen, #if defined(MBEDTLS_DES_C) || defined(MBEDTLS_AES_C) static int pem_check_pkcs_padding(unsigned char *input, size_t input_len, size_t *data_len) { - /* input_len > 0 is guaranteed by mbedtls_pem_read_buffer(). */ + /* input_len > 0 is not guaranteed by mbedtls_pem_read_buffer(). */ + if (input_len < 1) { + return MBEDTLS_ERR_PEM_INVALID_DATA; + } size_t pad_len = input[input_len - 1]; size_t i; @@ -416,7 +419,7 @@ int mbedtls_pem_read_buffer(mbedtls_pem_context *ctx, const char *header, const return MBEDTLS_ERR_PEM_BAD_INPUT_DATA; } - if ((buf = (unsigned char *)mbedtls_calloc(1, len)) == NULL) { + if ((buf = (unsigned char *) mbedtls_calloc(1, len)) == NULL) { return MBEDTLS_ERR_PEM_ALLOC_FAILED; } diff --git a/third_party/mbedtls/library/pk.cpp b/third_party/mbedtls/library/pk.cpp index a0ff59cee85d..51f0c240885f 100644 --- a/third_party/mbedtls/library/pk.cpp +++ b/third_party/mbedtls/library/pk.cpp @@ -35,10 +35,6 @@ #include #include -#define PSA_EXPORT_KEY_PAIR_OR_PUBLIC_MAX_SIZE \ - (PSA_EXPORT_KEY_PAIR_MAX_SIZE > PSA_EXPORT_PUBLIC_KEY_MAX_SIZE) ? \ - PSA_EXPORT_KEY_PAIR_MAX_SIZE : PSA_EXPORT_PUBLIC_KEY_MAX_SIZE - /* * Initialise a mbedtls_pk_context */ @@ -994,7 +990,7 @@ int mbedtls_pk_copy_public_from_psa(mbedtls_svc_key_id_t key_id, /* * Helper for mbedtls_pk_sign and mbedtls_pk_verify */ -inline int pk_hashlen_helper(mbedtls_md_type_t md_alg, size_t *hash_len) +static inline int pk_hashlen_helper(mbedtls_md_type_t md_alg, size_t *hash_len) { if (*hash_len != 0) { return 0; diff --git a/third_party/mbedtls/library/pk_internal.h b/third_party/mbedtls/library/pk_internal.h index d1baf5f31490..e86a3a09d2c5 100644 --- a/third_party/mbedtls/library/pk_internal.h +++ b/third_party/mbedtls/library/pk_internal.h @@ -57,7 +57,7 @@ * usage of the returned pointer explicit. Of course the returned value is * const or non-const accordingly. */ -inline const mbedtls_ecp_keypair *mbedtls_pk_ec_ro(const mbedtls_pk_context pk) +static inline const mbedtls_ecp_keypair *mbedtls_pk_ec_ro(const mbedtls_pk_context pk) { switch (mbedtls_pk_get_type(&pk)) { case MBEDTLS_PK_ECKEY: @@ -69,7 +69,7 @@ inline const mbedtls_ecp_keypair *mbedtls_pk_ec_ro(const mbedtls_pk_context pk) } } -inline mbedtls_ecp_keypair *mbedtls_pk_ec_rw(const mbedtls_pk_context pk) +static inline mbedtls_ecp_keypair *mbedtls_pk_ec_rw(const mbedtls_pk_context pk) { switch (mbedtls_pk_get_type(&pk)) { case MBEDTLS_PK_ECKEY: @@ -83,7 +83,7 @@ inline mbedtls_ecp_keypair *mbedtls_pk_ec_rw(const mbedtls_pk_context pk) #endif /* MBEDTLS_PK_HAVE_ECC_KEYS && !MBEDTLS_PK_USE_PSA_EC_DATA */ #if defined(MBEDTLS_PK_HAVE_ECC_KEYS) -inline mbedtls_ecp_group_id mbedtls_pk_get_ec_group_id(const mbedtls_pk_context *pk) +static inline mbedtls_ecp_group_id mbedtls_pk_get_ec_group_id(const mbedtls_pk_context *pk) { mbedtls_ecp_group_id id; @@ -121,7 +121,7 @@ inline mbedtls_ecp_group_id mbedtls_pk_get_ec_group_id(const mbedtls_pk_context #define MBEDTLS_PK_IS_RFC8410_GROUP_ID(id) \ ((id == MBEDTLS_ECP_DP_CURVE25519) || (id == MBEDTLS_ECP_DP_CURVE448)) -inline int mbedtls_pk_is_rfc8410(const mbedtls_pk_context *pk) +static inline int mbedtls_pk_is_rfc8410(const mbedtls_pk_context *pk) { mbedtls_ecp_group_id id = mbedtls_pk_get_ec_group_id(pk); diff --git a/third_party/mbedtls/library/pk_wrap.cpp b/third_party/mbedtls/library/pk_wrap.cpp index 2ac558b0f444..4283e6457028 100644 --- a/third_party/mbedtls/library/pk_wrap.cpp +++ b/third_party/mbedtls/library/pk_wrap.cpp @@ -489,28 +489,26 @@ static void rsa_debug(mbedtls_pk_context *pk, mbedtls_pk_debug_item *items) } const mbedtls_pk_info_t mbedtls_rsa_info = { - /* type */ MBEDTLS_PK_RSA, - /* name */ "RSA", - /* get_bitlen */ rsa_get_bitlen, - /* can_do */ rsa_can_do, - /* verify_func */ rsa_verify_wrap, - /* sign_func */ rsa_sign_wrap, + /* .type = */ MBEDTLS_PK_RSA, + /* .name = */ "RSA", + /* .get_bitlen = */ rsa_get_bitlen, + /* .can_do = */ rsa_can_do, + /* .verify_func = */ rsa_verify_wrap, + /* .sign_func = */ rsa_sign_wrap, #if defined(MBEDTLS_ECDSA_C) && defined(MBEDTLS_ECP_RESTARTABLE) - /* verify_rs_func */ NULL, - /* sign_rs_func */ NULL, - /* rs_alloc_func */ NULL, - /* rs_free_func */ NULL, + /* .verify_rs_func = */ NULL, + /* .sign_rs_func = */ NULL, #endif /* MBEDTLS_ECDSA_C && MBEDTLS_ECP_RESTARTABLE */ - /* decrypt_func */ rsa_decrypt_wrap, - /* encrypt_func */ rsa_encrypt_wrap, - /* check_pair_func */ rsa_check_pair_wrap, - /* ctx_alloc_func */ rsa_alloc_wrap, - /* ctx_free_func */ rsa_free_wrap, + /* .decrypt_func = */ rsa_decrypt_wrap, + /* .encrypt_func = */ rsa_encrypt_wrap, + /* .check_pair_func = */ rsa_check_pair_wrap, + /* .ctx_alloc_func = */ rsa_alloc_wrap, + /* .ctx_free_func = */ rsa_free_wrap, #if defined(MBEDTLS_ECDSA_C) && defined(MBEDTLS_ECP_RESTARTABLE) - /* rs_alloc_func */ NULL, - /* rs_free_func */ NULL, + /* .rs_alloc_func = */ NULL, + /* .rs_free_func = */ NULL, #endif - /* debug_func */ rsa_debug + /* .debug_func = */ rsa_debug }; #endif /* MBEDTLS_RSA_C */ diff --git a/third_party/mbedtls/library/pkwrite.h b/third_party/mbedtls/library/pkwrite.h index ab0c01417a74..01dc3d2f0ffa 100644 --- a/third_party/mbedtls/library/pkwrite.h +++ b/third_party/mbedtls/library/pkwrite.h @@ -1 +1,121 @@ -// \ No newline at end of file +/** + * \file pkwrite.h + * + * \brief Internal defines shared by the PK write module + */ +/* + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ + +#ifndef MBEDTLS_PK_WRITE_H +#define MBEDTLS_PK_WRITE_H + +#include "mbedtls/build_info.h" + +#include "mbedtls/pk.h" + +#if defined(MBEDTLS_USE_PSA_CRYPTO) +#include "psa/crypto.h" +#endif /* MBEDTLS_USE_PSA_CRYPTO */ + +/* + * Max sizes of key per types. Shown as tag + len (+ content). + */ + +#if defined(MBEDTLS_RSA_C) +/* + * RSA public keys: + * SubjectPublicKeyInfo ::= SEQUENCE { 1 + 3 + * algorithm AlgorithmIdentifier, 1 + 1 (sequence) + * + 1 + 1 + 9 (rsa oid) + * + 1 + 1 (params null) + * subjectPublicKey BIT STRING } 1 + 3 + (1 + below) + * RSAPublicKey ::= SEQUENCE { 1 + 3 + * modulus INTEGER, -- n 1 + 3 + MPI_MAX + 1 + * publicExponent INTEGER -- e 1 + 3 + MPI_MAX + 1 + * } + */ +#define MBEDTLS_PK_RSA_PUB_DER_MAX_BYTES (38 + 2 * MBEDTLS_MPI_MAX_SIZE) + +/* + * RSA private keys: + * RSAPrivateKey ::= SEQUENCE { 1 + 3 + * version Version, 1 + 1 + 1 + * modulus INTEGER, 1 + 3 + MPI_MAX + 1 + * publicExponent INTEGER, 1 + 3 + MPI_MAX + 1 + * privateExponent INTEGER, 1 + 3 + MPI_MAX + 1 + * prime1 INTEGER, 1 + 3 + MPI_MAX / 2 + 1 + * prime2 INTEGER, 1 + 3 + MPI_MAX / 2 + 1 + * exponent1 INTEGER, 1 + 3 + MPI_MAX / 2 + 1 + * exponent2 INTEGER, 1 + 3 + MPI_MAX / 2 + 1 + * coefficient INTEGER, 1 + 3 + MPI_MAX / 2 + 1 + * otherPrimeInfos OtherPrimeInfos OPTIONAL 0 (not supported) + * } + */ +#define MBEDTLS_MPI_MAX_SIZE_2 (MBEDTLS_MPI_MAX_SIZE / 2 + \ + MBEDTLS_MPI_MAX_SIZE % 2) +#define MBEDTLS_PK_RSA_PRV_DER_MAX_BYTES (47 + 3 * MBEDTLS_MPI_MAX_SIZE \ + + 5 * MBEDTLS_MPI_MAX_SIZE_2) + +#else /* MBEDTLS_RSA_C */ + +#define MBEDTLS_PK_RSA_PUB_DER_MAX_BYTES 0 +#define MBEDTLS_PK_RSA_PRV_DER_MAX_BYTES 0 + +#endif /* MBEDTLS_RSA_C */ + +#if defined(MBEDTLS_PK_HAVE_ECC_KEYS) + +/* Find the maximum number of bytes necessary to store an EC point. When USE_PSA + * is defined this means looking for the maximum between PSA and built-in + * supported curves. */ +#if defined(MBEDTLS_USE_PSA_CRYPTO) +#define MBEDTLS_PK_MAX_ECC_BYTES (PSA_BITS_TO_BYTES(PSA_VENDOR_ECC_MAX_CURVE_BITS) > \ + MBEDTLS_ECP_MAX_BYTES ? \ + PSA_BITS_TO_BYTES(PSA_VENDOR_ECC_MAX_CURVE_BITS) : \ + MBEDTLS_ECP_MAX_BYTES) +#else /* MBEDTLS_USE_PSA_CRYPTO */ +#define MBEDTLS_PK_MAX_ECC_BYTES MBEDTLS_ECP_MAX_BYTES +#endif /* MBEDTLS_USE_PSA_CRYPTO */ + +/* + * EC public keys: + * SubjectPublicKeyInfo ::= SEQUENCE { 1 + 2 + * algorithm AlgorithmIdentifier, 1 + 1 (sequence) + * + 1 + 1 + 7 (ec oid) + * + 1 + 1 + 9 (namedCurve oid) + * subjectPublicKey BIT STRING 1 + 2 + 1 [1] + * + 1 (point format) [1] + * + 2 * ECP_MAX (coords) [1] + * } + */ +#define MBEDTLS_PK_ECP_PUB_DER_MAX_BYTES (30 + 2 * MBEDTLS_PK_MAX_ECC_BYTES) + +/* + * EC private keys: + * ECPrivateKey ::= SEQUENCE { 1 + 2 + * version INTEGER , 1 + 1 + 1 + * privateKey OCTET STRING, 1 + 1 + ECP_MAX + * parameters [0] ECParameters OPTIONAL, 1 + 1 + (1 + 1 + 9) + * publicKey [1] BIT STRING OPTIONAL 1 + 2 + [1] above + * } + */ +#define MBEDTLS_PK_ECP_PRV_DER_MAX_BYTES (29 + 3 * MBEDTLS_PK_MAX_ECC_BYTES) + +#else /* MBEDTLS_PK_HAVE_ECC_KEYS */ + +#define MBEDTLS_PK_ECP_PUB_DER_MAX_BYTES 0 +#define MBEDTLS_PK_ECP_PRV_DER_MAX_BYTES 0 + +#endif /* MBEDTLS_PK_HAVE_ECC_KEYS */ + +/* Define the maximum available public key DER length based on the supported + * key types (EC and/or RSA). */ +#if (MBEDTLS_PK_RSA_PUB_DER_MAX_BYTES > MBEDTLS_PK_ECP_PUB_DER_MAX_BYTES) +#define MBEDTLS_PK_WRITE_PUBKEY_MAX_SIZE MBEDTLS_PK_RSA_PUB_DER_MAX_BYTES +#else +#define MBEDTLS_PK_WRITE_PUBKEY_MAX_SIZE MBEDTLS_PK_ECP_PUB_DER_MAX_BYTES +#endif + +#endif /* MBEDTLS_PK_WRITE_H */ diff --git a/third_party/mbedtls/library/psa_crypto_core.h b/third_party/mbedtls/library/psa_crypto_core.h index 456385123cf7..c3c0770142a6 100644 --- a/third_party/mbedtls/library/psa_crypto_core.h +++ b/third_party/mbedtls/library/psa_crypto_core.h @@ -1 +1,995 @@ -// dummy \ No newline at end of file +/* + * PSA crypto core internal interfaces + */ +/* + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ + +#ifndef PSA_CRYPTO_CORE_H +#define PSA_CRYPTO_CORE_H + +/* + * Include the build-time configuration information header. Here, we do not + * include `"mbedtls/build_info.h"` directly but `"psa/build_info.h"`, which + * is basically just an alias to it. This is to ease the maintenance of the + * TF-PSA-Crypto repository which has a different build system and + * configuration. + */ +#include "psa/build_info.h" + +#include "psa/crypto.h" +#include "psa/crypto_se_driver.h" +#if defined(MBEDTLS_THREADING_C) +#include "mbedtls/threading.h" +#endif + +/** + * Tell if PSA is ready for this cipher. + * + * \note For now, only checks the state of the driver subsystem, + * not the algorithm. Might do more in the future. + * + * \param cipher_alg The cipher algorithm (ignored for now). + * + * \return 1 if the driver subsytem is ready, 0 otherwise. + */ +int psa_can_do_cipher(psa_key_type_t key_type, psa_algorithm_t cipher_alg); + +typedef enum { + PSA_SLOT_EMPTY = 0, + PSA_SLOT_FILLING, + PSA_SLOT_FULL, + PSA_SLOT_PENDING_DELETION, +} psa_key_slot_state_t; + +/** The data structure representing a key slot, containing key material + * and metadata for one key. + */ +typedef struct { + /* This field is accessed in a lot of places. Putting it first + * reduces the code size. */ + psa_key_attributes_t attr; + + /* + * The current state of the key slot, as described in + * docs/architecture/psa-thread-safety/psa-thread-safety.md. + * + * Library functions can modify the state of a key slot by calling + * psa_key_slot_state_transition. + * + * The state variable is used to help determine whether library functions + * which operate on the slot succeed. For example, psa_finish_key_creation, + * which transfers the state of a slot from PSA_SLOT_FILLING to + * PSA_SLOT_FULL, must fail with error code PSA_ERROR_CORRUPTION_DETECTED + * if the state of the slot is not PSA_SLOT_FILLING. + * + * Library functions which traverse the array of key slots only consider + * slots that are in a suitable state for the function. + * For example, psa_get_and_lock_key_slot_in_memory, which finds a slot + * containing a given key ID, will only check slots whose state variable is + * PSA_SLOT_FULL. + */ + psa_key_slot_state_t state; + +#if defined(MBEDTLS_PSA_KEY_STORE_DYNAMIC) + /* The index of the slice containing this slot. + * This field must be filled if the slot contains a key + * (including keys being created or destroyed), and can be either + * filled or 0 when the slot is free. + * + * In most cases, the slice index can be deduced from the key identifer. + * We keep it in a separate field for robustness (it reduces the chance + * that a coding mistake in the key store will result in accessing the + * wrong slice), and also so that it's available even on code paths + * during creation or destruction where the key identifier might not be + * filled in. + * */ + uint8_t slice_index; +#endif /* MBEDTLS_PSA_KEY_STORE_DYNAMIC */ + + union { + struct { + /* The index of the next slot in the free list for this + * slice, relative * to the next array element. + * + * That is, 0 means the next slot, 1 means the next slot + * but one, etc. -1 would mean the slot itself. -2 means + * the previous slot, etc. + * + * If this is beyond the array length, the free list ends with the + * current element. + * + * The reason for this strange encoding is that 0 means the next + * element. This way, when we allocate a slice and initialize it + * to all-zero, the slice is ready for use, with a free list that + * consists of all the slots in order. + */ + int32_t next_free_relative_to_next; + } free; + + struct { + /* + * Number of functions registered as reading the material in the key slot. + * + * Library functions must not write directly to registered_readers + * + * A function must call psa_register_read(slot) before reading + * the current contents of the slot for an operation. + * They then must call psa_unregister_read(slot) once they have + * finished reading the current contents of the slot. If the key + * slot mutex is not held (when mutexes are enabled), this call + * must be done via a call to + * psa_unregister_read_under_mutex(slot). + * A function must call psa_key_slot_has_readers(slot) to check if + * the slot is in use for reading. + * + * This counter is used to prevent resetting the key slot while + * the library may access it. For example, such control is needed + * in the following scenarios: + * . In case of key slot starvation, all key slots contain the + * description of a key, and the library asks for the + * description of a persistent key not present in the + * key slots, the key slots currently accessed by the + * library cannot be reclaimed to free a key slot to load + * the persistent key. + * . In case of a multi-threaded application where one thread + * asks to close or purge or destroy a key while it is in use + * by the library through another thread. */ + size_t registered_readers; + } occupied; + } var; + + /* Dynamically allocated key data buffer. + * Format as specified in psa_export_key(). */ + struct key_data { +#if defined(MBEDTLS_PSA_STATIC_KEY_SLOTS) + uint8_t data[MBEDTLS_PSA_STATIC_KEY_SLOT_BUFFER_SIZE]; +#else + uint8_t *data; +#endif + size_t bytes; + } key; +} psa_key_slot_t; + +#if defined(MBEDTLS_THREADING_C) + +/** Perform a mutex operation and return immediately upon failure. + * + * Returns PSA_ERROR_SERVICE_FAILURE if the operation fails + * and status was PSA_SUCCESS. + * + * Assumptions: + * psa_status_t status exists. + * f is a mutex operation which returns 0 upon success. + */ +#define PSA_THREADING_CHK_RET(f) \ + do \ + { \ + if ((f) != 0) { \ + if (status == PSA_SUCCESS) { \ + return PSA_ERROR_SERVICE_FAILURE; \ + } \ + return status; \ + } \ + } while (0); + +/** Perform a mutex operation and goto exit on failure. + * + * Sets status to PSA_ERROR_SERVICE_FAILURE if status was PSA_SUCCESS. + * + * Assumptions: + * psa_status_t status exists. + * Label exit: exists. + * f is a mutex operation which returns 0 upon success. + */ +#define PSA_THREADING_CHK_GOTO_EXIT(f) \ + do \ + { \ + if ((f) != 0) { \ + if (status == PSA_SUCCESS) { \ + status = PSA_ERROR_SERVICE_FAILURE; \ + } \ + goto exit; \ + } \ + } while (0); +#endif + +/** Test whether a key slot has any registered readers. + * If multi-threading is enabled, the caller must hold the + * global key slot mutex. + * + * \param[in] slot The key slot to test. + * + * \return 1 if the slot has any registered readers, 0 otherwise. + */ +static inline int psa_key_slot_has_readers(const psa_key_slot_t *slot) +{ + return slot->var.occupied.registered_readers > 0; +} + +#if defined(MBEDTLS_PSA_CRYPTO_SE_C) +/** Get the SE slot number of a key from the key slot storing its description. + * + * \param[in] slot The key slot to query. This must be a key slot storing + * the description of a key of a dynamically registered + * secure element, otherwise the behaviour is undefined. + */ +static inline psa_key_slot_number_t psa_key_slot_get_slot_number( + const psa_key_slot_t *slot) +{ + return *((psa_key_slot_number_t *) (slot->key.data)); +} +#endif + +/** Completely wipe a slot in memory, including its policy. + * + * Persistent storage is not affected. + * Sets the slot's state to PSA_SLOT_EMPTY. + * If multi-threading is enabled, the caller must hold the + * global key slot mutex. + * + * \param[in,out] slot The key slot to wipe. + * + * \retval #PSA_SUCCESS + * The slot has been successfully wiped. + * \retval #PSA_ERROR_CORRUPTION_DETECTED + * The slot's state was PSA_SLOT_FULL or PSA_SLOT_PENDING_DELETION, and + * the amount of registered readers was not equal to 1. Or, + * the slot's state was PSA_SLOT_EMPTY. Or, + * the slot's state was PSA_SLOT_FILLING, and the amount + * of registered readers was not equal to 0. + */ +psa_status_t psa_wipe_key_slot(psa_key_slot_t *slot); + +/** Try to allocate a buffer to an empty key slot. + * + * \param[in,out] slot Key slot to attach buffer to. + * \param[in] buffer_length Requested size of the buffer. + * + * \retval #PSA_SUCCESS + * The buffer has been successfully allocated. + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY + * Not enough memory was available for allocation. + * \retval #PSA_ERROR_ALREADY_EXISTS + * Trying to allocate a buffer to a non-empty key slot. + */ +psa_status_t psa_allocate_buffer_to_slot(psa_key_slot_t *slot, + size_t buffer_length); + +/** Wipe key data from a slot. Preserves metadata such as the policy. */ +psa_status_t psa_remove_key_data_from_memory(psa_key_slot_t *slot); + +/** Copy key data (in export format) into an empty key slot. + * + * This function assumes that the slot does not contain + * any key material yet. On failure, the slot content is unchanged. + * + * \param[in,out] slot Key slot to copy the key into. + * \param[in] data Buffer containing the key material. + * \param data_length Size of the key buffer. + * + * \retval #PSA_SUCCESS + * The key has been copied successfully. + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY + * Not enough memory was available for allocation of the + * copy buffer. + * \retval #PSA_ERROR_ALREADY_EXISTS + * There was other key material already present in the slot. + */ +psa_status_t psa_copy_key_material_into_slot(psa_key_slot_t *slot, + const uint8_t *data, + size_t data_length); + +/** Convert an Mbed TLS error code to a PSA error code + * + * \note This function is provided solely for the convenience of + * Mbed TLS and may be removed at any time without notice. + * + * \param ret An Mbed TLS-thrown error code + * + * \return The corresponding PSA error code + */ +psa_status_t mbedtls_to_psa_error(int ret); + +/** Import a key in binary format. + * + * \note The signature of this function is that of a PSA driver + * import_key entry point. This function behaves as an import_key + * entry point as defined in the PSA driver interface specification for + * transparent drivers. + * + * \param[in] attributes The attributes for the key to import. + * \param[in] data The buffer containing the key data in import + * format. + * \param[in] data_length Size of the \p data buffer in bytes. + * \param[out] key_buffer The buffer to contain the key data in output + * format upon successful return. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in bytes. This + * size is greater or equal to \p data_length. + * \param[out] key_buffer_length The length of the data written in \p + * key_buffer in bytes. + * \param[out] bits The key size in number of bits. + * + * \retval #PSA_SUCCESS The key was imported successfully. + * \retval #PSA_ERROR_INVALID_ARGUMENT + * The key data is not correctly formatted. + * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription + */ +psa_status_t psa_import_key_into_slot( + const psa_key_attributes_t *attributes, + const uint8_t *data, size_t data_length, + uint8_t *key_buffer, size_t key_buffer_size, + size_t *key_buffer_length, size_t *bits); + +/** Export a key in binary format + * + * \note The signature of this function is that of a PSA driver export_key + * entry point. This function behaves as an export_key entry point as + * defined in the PSA driver interface specification. + * + * \param[in] attributes The attributes for the key to export. + * \param[in] key_buffer Material or context of the key to export. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in bytes. + * \param[out] data Buffer where the key data is to be written. + * \param[in] data_size Size of the \p data buffer in bytes. + * \param[out] data_length On success, the number of bytes written in + * \p data + * + * \retval #PSA_SUCCESS The key was exported successfully. + * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription + * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription + * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription + * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription + * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + */ +psa_status_t psa_export_key_internal( + const psa_key_attributes_t *attributes, + const uint8_t *key_buffer, size_t key_buffer_size, + uint8_t *data, size_t data_size, size_t *data_length); + +/** Export a public key or the public part of a key pair in binary format. + * + * \note The signature of this function is that of a PSA driver + * export_public_key entry point. This function behaves as an + * export_public_key entry point as defined in the PSA driver interface + * specification. + * + * \param[in] attributes The attributes for the key to export. + * \param[in] key_buffer Material or context of the key to export. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in bytes. + * \param[out] data Buffer where the key data is to be written. + * \param[in] data_size Size of the \p data buffer in bytes. + * \param[out] data_length On success, the number of bytes written in + * \p data + * + * \retval #PSA_SUCCESS The public key was exported successfully. + * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription + * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription + * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription + * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription + * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + */ +psa_status_t psa_export_public_key_internal( + const psa_key_attributes_t *attributes, + const uint8_t *key_buffer, size_t key_buffer_size, + uint8_t *data, size_t data_size, size_t *data_length); + +/** Whether a key custom production parameters structure is the default. + * + * Calls to a key generation driver with non-default custom production parameters + * require a driver supporting custom production parameters. + * + * \param[in] custom The key custom production parameters to check. + * \param custom_data_length Size of the associated variable-length data + * in bytes. + */ +int psa_custom_key_parameters_are_default( + const psa_custom_key_parameters_t *custom, + size_t custom_data_length); + +/** + * \brief Generate a key. + * + * \note The signature of the function is that of a PSA driver generate_key + * entry point. + * + * \param[in] attributes The attributes for the key to generate. + * \param[in] custom Custom parameters for the key generation. + * \param[in] custom_data Variable-length data associated with \c custom. + * \param custom_data_length Length of `custom_data` in bytes. + * \param[out] key_buffer Buffer where the key data is to be written. + * \param[in] key_buffer_size Size of \p key_buffer in bytes. + * \param[out] key_buffer_length On success, the number of bytes written in + * \p key_buffer. + * + * \retval #PSA_SUCCESS + * The key was generated successfully. + * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription + * \retval #PSA_ERROR_NOT_SUPPORTED + * Key size in bits or type not supported. + * \retval #PSA_ERROR_BUFFER_TOO_SMALL + * The size of \p key_buffer is too small. + */ +psa_status_t psa_generate_key_internal(const psa_key_attributes_t *attributes, + const psa_custom_key_parameters_t *custom, + const uint8_t *custom_data, + size_t custom_data_length, + uint8_t *key_buffer, + size_t key_buffer_size, + size_t *key_buffer_length); + +/** Sign a message with a private key. For hash-and-sign algorithms, + * this includes the hashing step. + * + * \note The signature of this function is that of a PSA driver + * sign_message entry point. This function behaves as a sign_message + * entry point as defined in the PSA driver interface specification for + * transparent drivers. + * + * \note This function will call the driver for psa_sign_hash + * and go through driver dispatch again. + * + * \param[in] attributes The attributes of the key to use for the + * operation. + * \param[in] key_buffer The buffer containing the key context. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in bytes. + * \param[in] alg A signature algorithm that is compatible with + * the type of the key. + * \param[in] input The input message to sign. + * \param[in] input_length Size of the \p input buffer in bytes. + * \param[out] signature Buffer where the signature is to be written. + * \param[in] signature_size Size of the \p signature buffer in bytes. + * \param[out] signature_length On success, the number of bytes + * that make up the returned signature value. + * + * \retval #PSA_SUCCESS \emptydescription + * \retval #PSA_ERROR_BUFFER_TOO_SMALL + * The size of the \p signature buffer is too small. You can + * determine a sufficient buffer size by calling + * #PSA_SIGN_OUTPUT_SIZE(\c key_type, \c key_bits, \p alg) + * where \c key_type and \c key_bits are the type and bit-size + * respectively of the key. + * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription + * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription + */ +psa_status_t psa_sign_message_builtin( + const psa_key_attributes_t *attributes, + const uint8_t *key_buffer, size_t key_buffer_size, + psa_algorithm_t alg, const uint8_t *input, size_t input_length, + uint8_t *signature, size_t signature_size, size_t *signature_length); + +/** Verify the signature of a message with a public key, using + * a hash-and-sign verification algorithm. + * + * \note The signature of this function is that of a PSA driver + * verify_message entry point. This function behaves as a verify_message + * entry point as defined in the PSA driver interface specification for + * transparent drivers. + * + * \note This function will call the driver for psa_verify_hash + * and go through driver dispatch again. + * + * \param[in] attributes The attributes of the key to use for the + * operation. + * \param[in] key_buffer The buffer containing the key context. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in bytes. + * \param[in] alg A signature algorithm that is compatible with + * the type of the key. + * \param[in] input The message whose signature is to be verified. + * \param[in] input_length Size of the \p input buffer in bytes. + * \param[in] signature Buffer containing the signature to verify. + * \param[in] signature_length Size of the \p signature buffer in bytes. + * + * \retval #PSA_SUCCESS + * The signature is valid. + * \retval #PSA_ERROR_INVALID_SIGNATURE + * The calculation was performed successfully, but the passed + * signature is not a valid signature. + * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription + * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + */ +psa_status_t psa_verify_message_builtin( + const psa_key_attributes_t *attributes, + const uint8_t *key_buffer, size_t key_buffer_size, + psa_algorithm_t alg, const uint8_t *input, size_t input_length, + const uint8_t *signature, size_t signature_length); + +/** Sign an already-calculated hash with a private key. + * + * \note The signature of this function is that of a PSA driver + * sign_hash entry point. This function behaves as a sign_hash + * entry point as defined in the PSA driver interface specification for + * transparent drivers. + * + * \param[in] attributes The attributes of the key to use for the + * operation. + * \param[in] key_buffer The buffer containing the key context. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in bytes. + * \param[in] alg A signature algorithm that is compatible with + * the type of the key. + * \param[in] hash The hash or message to sign. + * \param[in] hash_length Size of the \p hash buffer in bytes. + * \param[out] signature Buffer where the signature is to be written. + * \param[in] signature_size Size of the \p signature buffer in bytes. + * \param[out] signature_length On success, the number of bytes + * that make up the returned signature value. + * + * \retval #PSA_SUCCESS \emptydescription + * \retval #PSA_ERROR_BUFFER_TOO_SMALL + * The size of the \p signature buffer is too small. You can + * determine a sufficient buffer size by calling + * #PSA_SIGN_OUTPUT_SIZE(\c key_type, \c key_bits, \p alg) + * where \c key_type and \c key_bits are the type and bit-size + * respectively of the key. + * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription + * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription + */ +psa_status_t psa_sign_hash_builtin( + const psa_key_attributes_t *attributes, + const uint8_t *key_buffer, size_t key_buffer_size, + psa_algorithm_t alg, const uint8_t *hash, size_t hash_length, + uint8_t *signature, size_t signature_size, size_t *signature_length); + +/** + * \brief Verify the signature a hash or short message using a public key. + * + * \note The signature of this function is that of a PSA driver + * verify_hash entry point. This function behaves as a verify_hash + * entry point as defined in the PSA driver interface specification for + * transparent drivers. + * + * \param[in] attributes The attributes of the key to use for the + * operation. + * \param[in] key_buffer The buffer containing the key context. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in bytes. + * \param[in] alg A signature algorithm that is compatible with + * the type of the key. + * \param[in] hash The hash or message whose signature is to be + * verified. + * \param[in] hash_length Size of the \p hash buffer in bytes. + * \param[in] signature Buffer containing the signature to verify. + * \param[in] signature_length Size of the \p signature buffer in bytes. + * + * \retval #PSA_SUCCESS + * The signature is valid. + * \retval #PSA_ERROR_INVALID_SIGNATURE + * The calculation was performed successfully, but the passed + * signature is not a valid signature. + * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription + * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + */ +psa_status_t psa_verify_hash_builtin( + const psa_key_attributes_t *attributes, + const uint8_t *key_buffer, size_t key_buffer_size, + psa_algorithm_t alg, const uint8_t *hash, size_t hash_length, + const uint8_t *signature, size_t signature_length); + +/** + * \brief Validate the key bit size for unstructured keys. + * + * \note Check that the bit size is acceptable for a given key type for + * unstructured keys. + * + * \param[in] type The key type + * \param[in] bits The number of bits of the key + * + * \retval #PSA_SUCCESS + * The key type and size are valid. + * \retval #PSA_ERROR_INVALID_ARGUMENT + * The size in bits of the key is not valid. + * \retval #PSA_ERROR_NOT_SUPPORTED + * The type and/or the size in bits of the key or the combination of + * the two is not supported. + */ +psa_status_t psa_validate_unstructured_key_bit_size(psa_key_type_t type, + size_t bits); + +/** Perform a key agreement and return the raw shared secret, using + built-in raw key agreement functions. + * + * \note The signature of this function is that of a PSA driver + * key_agreement entry point. This function behaves as a key_agreement + * entry point as defined in the PSA driver interface specification for + * transparent drivers. + * + * \param[in] attributes The attributes of the key to use for the + * operation. + * \param[in] key_buffer The buffer containing the private key + * context. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in + * bytes. + * \param[in] alg A key agreement algorithm that is + * compatible with the type of the key. + * \param[in] peer_key The buffer containing the key context + * of the peer's public key. + * \param[in] peer_key_length Size of the \p peer_key buffer in + * bytes. + * \param[out] shared_secret The buffer to which the shared secret + * is to be written. + * \param[in] shared_secret_size Size of the \p shared_secret buffer in + * bytes. + * \param[out] shared_secret_length On success, the number of bytes that make + * up the returned shared secret. + * \retval #PSA_SUCCESS + * Success. Shared secret successfully calculated. + * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription + * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription + * \retval #PSA_ERROR_INVALID_ARGUMENT + * \p alg is not a key agreement algorithm, or + * \p private_key is not compatible with \p alg, + * or \p peer_key is not valid for \p alg or not compatible with + * \p private_key. + * \retval #PSA_ERROR_BUFFER_TOO_SMALL + * \p shared_secret_size is too small + * \retval #PSA_ERROR_NOT_SUPPORTED + * \p alg is not a supported key agreement algorithm. + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription + * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription + * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription + * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription + * \retval #PSA_ERROR_BAD_STATE \emptydescription + */ +psa_status_t psa_key_agreement_raw_builtin( + const psa_key_attributes_t *attributes, + const uint8_t *key_buffer, + size_t key_buffer_size, + psa_algorithm_t alg, + const uint8_t *peer_key, + size_t peer_key_length, + uint8_t *shared_secret, + size_t shared_secret_size, + size_t *shared_secret_length); + +/** + * \brief Set the maximum number of ops allowed to be executed by an + * interruptible function in a single call. + * + * \note The signature of this function is that of a PSA driver + * interruptible_set_max_ops entry point. This function behaves as an + * interruptible_set_max_ops entry point as defined in the PSA driver + * interface specification for transparent drivers. + * + * \param[in] max_ops The maximum number of ops to be executed in a + * single call, this can be a number from 0 to + * #PSA_INTERRUPTIBLE_MAX_OPS_UNLIMITED, where 0 + * is obviously the least amount of work done per + * call. + */ +void mbedtls_psa_interruptible_set_max_ops(uint32_t max_ops); + +/** + * \brief Get the maximum number of ops allowed to be executed by an + * interruptible function in a single call. + * + * \note The signature of this function is that of a PSA driver + * interruptible_get_max_ops entry point. This function behaves as an + * interruptible_get_max_ops entry point as defined in the PSA driver + * interface specification for transparent drivers. + * + * \return Maximum number of ops allowed to be executed + * by an interruptible function in a single call. + */ +uint32_t mbedtls_psa_interruptible_get_max_ops(void); + +/** + * \brief Get the number of ops that a hash signing operation has taken for the + * previous call. If no call or work has taken place, this will return + * zero. + * + * \note The signature of this function is that of a PSA driver + * sign_hash_get_num_ops entry point. This function behaves as an + * sign_hash_get_num_ops entry point as defined in the PSA driver + * interface specification for transparent drivers. + * + * \param operation The \c + * mbedtls_psa_sign_hash_interruptible_operation_t + * to use. This must be initialized first. + * + * \return Number of ops that were completed + * in the last call to \c + * mbedtls_psa_sign_hash_complete(). + */ +uint32_t mbedtls_psa_sign_hash_get_num_ops( + const mbedtls_psa_sign_hash_interruptible_operation_t *operation); + +/** + * \brief Get the number of ops that a hash verification operation has taken for + * the previous call. If no call or work has taken place, this will + * return zero. + * + * \note The signature of this function is that of a PSA driver + * verify_hash_get_num_ops entry point. This function behaves as an + * verify_hash_get_num_ops entry point as defined in the PSA driver + * interface specification for transparent drivers. + * + * \param operation The \c + * mbedtls_psa_verify_hash_interruptible_operation_t + * to use. This must be initialized first. + * + * \return Number of ops that were completed + * in the last call to \c + * mbedtls_psa_verify_hash_complete(). + */ +uint32_t mbedtls_psa_verify_hash_get_num_ops( + const mbedtls_psa_verify_hash_interruptible_operation_t *operation); + +/** + * \brief Start signing a hash or short message with a private key, in an + * interruptible manner. + * + * \note The signature of this function is that of a PSA driver + * sign_hash_start entry point. This function behaves as a + * sign_hash_start entry point as defined in the PSA driver interface + * specification for transparent drivers. + * + * \param[in] operation The \c + * mbedtls_psa_sign_hash_interruptible_operation_t + * to use. This must be initialized first. + * \param[in] attributes The attributes of the key to use for the + * operation. + * \param[in] key_buffer The buffer containing the key context. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in bytes. + * \param[in] alg A signature algorithm that is compatible with + * the type of the key. + * \param[in] hash The hash or message to sign. + * \param hash_length Size of the \p hash buffer in bytes. + * + * \retval #PSA_SUCCESS + * The operation started successfully - call \c psa_sign_hash_complete() + * with the same context to complete the operation + * \retval #PSA_ERROR_INVALID_ARGUMENT + * An unsupported, incorrectly formatted or incorrect type of key was + * used. + * \retval #PSA_ERROR_NOT_SUPPORTED Either no internal interruptible operations + * are currently supported, or the key type is currently unsupported. + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY + * There was insufficient memory to load the key representation. + */ +psa_status_t mbedtls_psa_sign_hash_start( + mbedtls_psa_sign_hash_interruptible_operation_t *operation, + const psa_key_attributes_t *attributes, const uint8_t *key_buffer, + size_t key_buffer_size, psa_algorithm_t alg, + const uint8_t *hash, size_t hash_length); + +/** + * \brief Continue and eventually complete the action of signing a hash or + * short message with a private key, in an interruptible manner. + * + * \note The signature of this function is that of a PSA driver + * sign_hash_complete entry point. This function behaves as a + * sign_hash_complete entry point as defined in the PSA driver interface + * specification for transparent drivers. + * + * \param[in] operation The \c + * mbedtls_psa_sign_hash_interruptible_operation_t + * to use. This must be initialized first. + * + * \param[out] signature Buffer where the signature is to be written. + * \param signature_size Size of the \p signature buffer in bytes. This + * must be appropriate for the selected + * algorithm and key. + * \param[out] signature_length On success, the number of bytes that make up + * the returned signature value. + * + * \retval #PSA_SUCCESS + * Operation completed successfully + * + * \retval #PSA_OPERATION_INCOMPLETE + * Operation was interrupted due to the setting of \c + * psa_interruptible_set_max_ops(), there is still work to be done, + * please call this function again with the same operation object. + * + * \retval #PSA_ERROR_BUFFER_TOO_SMALL + * The size of the \p signature buffer is too small. You can + * determine a sufficient buffer size by calling + * #PSA_SIGN_OUTPUT_SIZE(\c key_type, \c key_bits, \p alg) + * where \c key_type and \c key_bits are the type and bit-size + * respectively of \p key. + * + * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription + * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription + */ +psa_status_t mbedtls_psa_sign_hash_complete( + mbedtls_psa_sign_hash_interruptible_operation_t *operation, + uint8_t *signature, size_t signature_size, + size_t *signature_length); + +/** + * \brief Abort a sign hash operation. + * + * \note The signature of this function is that of a PSA driver sign_hash_abort + * entry point. This function behaves as a sign_hash_abort entry point as + * defined in the PSA driver interface specification for transparent + * drivers. + * + * \param[in] operation The \c + * mbedtls_psa_sign_hash_interruptible_operation_t + * to abort. + * + * \retval #PSA_SUCCESS + * The operation was aborted successfully. + */ +psa_status_t mbedtls_psa_sign_hash_abort( + mbedtls_psa_sign_hash_interruptible_operation_t *operation); + +/** + * \brief Start reading and verifying a hash or short message, in an + * interruptible manner. + * + * \note The signature of this function is that of a PSA driver + * verify_hash_start entry point. This function behaves as a + * verify_hash_start entry point as defined in the PSA driver interface + * specification for transparent drivers. + * + * \param[in] operation The \c + * mbedtls_psa_verify_hash_interruptible_operation_t + * to use. This must be initialized first. + * \param[in] attributes The attributes of the key to use for the + * operation. + * \param[in] key_buffer The buffer containing the key context. + * \param[in] key_buffer_size Size of the \p key_buffer buffer in bytes. + * \param[in] alg A signature algorithm that is compatible with + * the type of the key. + * \param[in] hash The hash whose signature is to be verified. + * \param hash_length Size of the \p hash buffer in bytes. + * \param[in] signature Buffer containing the signature to verify. + * \param signature_length Size of the \p signature buffer in bytes. + * + * \retval #PSA_SUCCESS + * The operation started successfully - call \c psa_sign_hash_complete() + * with the same context to complete the operation + * \retval #PSA_ERROR_INVALID_ARGUMENT + * An unsupported or incorrect type of key was used. + * \retval #PSA_ERROR_NOT_SUPPORTED + * Either no internal interruptible operations are currently supported, + * or the key type is currently unsupported. + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY + * There was insufficient memory either to load the key representation, + * or to prepare the operation. + */ +psa_status_t mbedtls_psa_verify_hash_start( + mbedtls_psa_verify_hash_interruptible_operation_t *operation, + const psa_key_attributes_t *attributes, + const uint8_t *key_buffer, size_t key_buffer_size, + psa_algorithm_t alg, + const uint8_t *hash, size_t hash_length, + const uint8_t *signature, size_t signature_length); + +/** + * \brief Continue and eventually complete the action of signing a hash or + * short message with a private key, in an interruptible manner. + * + * \note The signature of this function is that of a PSA driver + * sign_hash_complete entry point. This function behaves as a + * sign_hash_complete entry point as defined in the PSA driver interface + * specification for transparent drivers. + * + * \param[in] operation The \c + * mbedtls_psa_sign_hash_interruptible_operation_t + * to use. This must be initialized first. + * + * \retval #PSA_SUCCESS + * Operation completed successfully, and the passed signature is valid. + * + * \retval #PSA_OPERATION_INCOMPLETE + * Operation was interrupted due to the setting of \c + * psa_interruptible_set_max_ops(), there is still work to be done, + * please call this function again with the same operation object. + * + * \retval #PSA_ERROR_INVALID_SIGNATURE + * The calculation was performed successfully, but the passed + * signature is not a valid signature. + * + * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription + * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription + * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription + */ +psa_status_t mbedtls_psa_verify_hash_complete( + mbedtls_psa_verify_hash_interruptible_operation_t *operation); + +/** + * \brief Abort a verify signed hash operation. + * + * \note The signature of this function is that of a PSA driver + * verify_hash_abort entry point. This function behaves as a + * verify_hash_abort entry point as defined in the PSA driver interface + * specification for transparent drivers. + * + * \param[in] operation The \c + * mbedtls_psa_verify_hash_interruptible_operation_t + * to abort. + * + * \retval #PSA_SUCCESS + * The operation was aborted successfully. + */ +psa_status_t mbedtls_psa_verify_hash_abort( + mbedtls_psa_verify_hash_interruptible_operation_t *operation); + +typedef struct psa_crypto_local_input_s { + uint8_t *buffer; + size_t length; +} psa_crypto_local_input_t; + +#define PSA_CRYPTO_LOCAL_INPUT_INIT ((psa_crypto_local_input_t) { NULL, 0 }) + +/** Allocate a local copy of an input buffer and copy the contents into it. + * + * \param[in] input Pointer to input buffer. + * \param[in] input_len Length of the input buffer. + * \param[out] local_input Pointer to a psa_crypto_local_input_t struct + * containing a local input copy. + * \return #PSA_SUCCESS, if the buffer was successfully + * copied. + * \return #PSA_ERROR_INSUFFICIENT_MEMORY, if a copy of + * the buffer cannot be allocated. + */ +psa_status_t psa_crypto_local_input_alloc(const uint8_t *input, size_t input_len, + psa_crypto_local_input_t *local_input); + +/** Free a local copy of an input buffer. + * + * \param[in] local_input Pointer to a psa_crypto_local_input_t struct + * populated by a previous call to + * psa_crypto_local_input_alloc(). + */ +void psa_crypto_local_input_free(psa_crypto_local_input_t *local_input); + +typedef struct psa_crypto_local_output_s { + uint8_t *original; + uint8_t *buffer; + size_t length; +} psa_crypto_local_output_t; + +#define PSA_CRYPTO_LOCAL_OUTPUT_INIT ((psa_crypto_local_output_t) { NULL, NULL, 0 }) + +/** Allocate a local copy of an output buffer. + * + * \note This does not copy any data from the original + * output buffer but only allocates a buffer + * whose contents will be copied back to the + * original in a future call to + * psa_crypto_local_output_free(). + * + * \param[in] output Pointer to output buffer. + * \param[in] output_len Length of the output buffer. + * \param[out] local_output Pointer to a psa_crypto_local_output_t struct to + * populate with the local output copy. + * \return #PSA_SUCCESS, if the buffer was successfully + * copied. + * \return #PSA_ERROR_INSUFFICIENT_MEMORY, if a copy of + * the buffer cannot be allocated. + */ +psa_status_t psa_crypto_local_output_alloc(uint8_t *output, size_t output_len, + psa_crypto_local_output_t *local_output); + +/** Copy from a local copy of an output buffer back to the original, then + * free the local copy. + * + * \param[in] local_output Pointer to a psa_crypto_local_output_t struct + * populated by a previous call to + * psa_crypto_local_output_alloc(). + * \return #PSA_SUCCESS, if the local output was + * successfully copied back to the original. + * \return #PSA_ERROR_CORRUPTION_DETECTED, if the output + * could not be copied back to the original. + */ +psa_status_t psa_crypto_local_output_free(psa_crypto_local_output_t *local_output); + +#endif /* PSA_CRYPTO_CORE_H */ diff --git a/third_party/mbedtls/library/psa_util_internal.h b/third_party/mbedtls/library/psa_util_internal.h index 456385123cf7..70a08a02cd8e 100644 --- a/third_party/mbedtls/library/psa_util_internal.h +++ b/third_party/mbedtls/library/psa_util_internal.h @@ -1 +1,100 @@ -// dummy \ No newline at end of file +/** + * \file psa_util_internal.h + * + * \brief Internal utility functions for use of PSA Crypto. + */ +/* + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ + +#ifndef MBEDTLS_PSA_UTIL_INTERNAL_H +#define MBEDTLS_PSA_UTIL_INTERNAL_H + +/* Include the public header so that users only need one include. */ +#include "mbedtls/psa_util.h" + +#include "psa/crypto.h" + +#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) + +/************************************************************************* + * FFDH + ************************************************************************/ + +#define MBEDTLS_PSA_MAX_FFDH_PUBKEY_LENGTH \ + PSA_KEY_EXPORT_FFDH_PUBLIC_KEY_MAX_SIZE(PSA_VENDOR_FFDH_MAX_KEY_BITS) + +/************************************************************************* + * ECC + ************************************************************************/ + +#define MBEDTLS_PSA_MAX_EC_PUBKEY_LENGTH \ + PSA_KEY_EXPORT_ECC_PUBLIC_KEY_MAX_SIZE(PSA_VENDOR_ECC_MAX_CURVE_BITS) + +#define MBEDTLS_PSA_MAX_EC_KEY_PAIR_LENGTH \ + PSA_KEY_EXPORT_ECC_KEY_PAIR_MAX_SIZE(PSA_VENDOR_ECC_MAX_CURVE_BITS) + +/************************************************************************* + * Error translation + ************************************************************************/ + +typedef struct { + /* Error codes used by PSA crypto are in -255..-128, fitting in 16 bits. */ + int16_t psa_status; + /* Error codes used by Mbed TLS are in one of the ranges + * -127..-1 (low-level) or -32767..-4096 (high-level with a low-level + * code optionally added), fitting in 16 bits. */ + int16_t mbedtls_error; +} mbedtls_error_pair_t; + +#if defined(MBEDTLS_MD_LIGHT) +extern const mbedtls_error_pair_t psa_to_md_errors[4]; +#endif + +#if defined(MBEDTLS_BLOCK_CIPHER_SOME_PSA) +extern const mbedtls_error_pair_t psa_to_cipher_errors[4]; +#endif + +#if defined(MBEDTLS_LMS_C) +extern const mbedtls_error_pair_t psa_to_lms_errors[3]; +#endif + +#if defined(MBEDTLS_USE_PSA_CRYPTO) || defined(MBEDTLS_SSL_PROTO_TLS1_3) +extern const mbedtls_error_pair_t psa_to_ssl_errors[7]; +#endif + +#if defined(PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY) || \ + defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) +extern const mbedtls_error_pair_t psa_to_pk_rsa_errors[8]; +#endif + +#if defined(MBEDTLS_USE_PSA_CRYPTO) && \ + defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY) +extern const mbedtls_error_pair_t psa_to_pk_ecdsa_errors[7]; +#endif + +/* Generic fallback function for error translation, + * when the received state was not module-specific. */ +int psa_generic_status_to_mbedtls(psa_status_t status); + +/* This function iterates over provided local error translations, + * and if no match was found - calls the fallback error translation function. */ +int psa_status_to_mbedtls(psa_status_t status, + const mbedtls_error_pair_t *local_translations, + size_t local_errors_num, + int (*fallback_f)(psa_status_t)); + +/* The second out of three-stage error handling functions of the pk module, + * acts as a fallback after RSA / ECDSA error translation, and if no match + * is found, it itself calls psa_generic_status_to_mbedtls. */ +int psa_pk_status_to_mbedtls(psa_status_t status); + +/* Utility macro to shorten the defines of error translator in modules. */ +#define PSA_TO_MBEDTLS_ERR_LIST(status, error_list, fallback_f) \ + psa_status_to_mbedtls(status, error_list, \ + sizeof(error_list)/sizeof(error_list[0]), \ + fallback_f) + +#endif /* MBEDTLS_PSA_CRYPTO_CLIENT */ +#endif /* MBEDTLS_PSA_UTIL_INTERNAL_H */ diff --git a/third_party/mbedtls/library/rsa.cpp b/third_party/mbedtls/library/rsa.cpp index 07aeb69cd4ac..63faf1c8cc74 100644 --- a/third_party/mbedtls/library/rsa.cpp +++ b/third_party/mbedtls/library/rsa.cpp @@ -2458,12 +2458,12 @@ int mbedtls_rsa_rsassa_pkcs1_v15_sign(mbedtls_rsa_context *ctx, * temporary buffer and check it before returning it. */ - sig_try = (unsigned char*) mbedtls_calloc(1, ctx->len); + sig_try = (unsigned char *) mbedtls_calloc(1, ctx->len); if (sig_try == NULL) { return MBEDTLS_ERR_MPI_ALLOC_FAILED; } - verif = (unsigned char*) mbedtls_calloc(1, ctx->len); + verif = (unsigned char *) mbedtls_calloc(1, ctx->len); if (verif == NULL) { mbedtls_free(sig_try); return MBEDTLS_ERR_MPI_ALLOC_FAILED; @@ -2692,8 +2692,8 @@ int mbedtls_rsa_rsassa_pkcs1_v15_verify(mbedtls_rsa_context *ctx, * Prepare expected PKCS1 v1.5 encoding of hash. */ - if ((encoded = (unsigned char*) mbedtls_calloc(1, sig_len)) == NULL || - (encoded_expected = (unsigned char*) mbedtls_calloc(1, sig_len)) == NULL) { + if ((encoded = (unsigned char *) mbedtls_calloc(1, sig_len)) == NULL || + (encoded_expected = (unsigned char *) mbedtls_calloc(1, sig_len)) == NULL) { ret = MBEDTLS_ERR_MPI_ALLOC_FAILED; goto cleanup; } diff --git a/third_party/mbedtls/mbedtls_wrapper.cpp b/third_party/mbedtls/mbedtls_wrapper.cpp index 9a6e7184a61d..7dc0af7fd199 100644 --- a/third_party/mbedtls/mbedtls_wrapper.cpp +++ b/third_party/mbedtls/mbedtls_wrapper.cpp @@ -233,7 +233,7 @@ void MbedTlsWrapper::SHA1State::FinishHex(char *out) { const mbedtls_cipher_info_t *MbedTlsWrapper::AESStateMBEDTLS::GetCipher(size_t key_len){ switch(cipher){ - case GCM: + case duckdb::EncryptionTypes::CipherType::GCM: switch (key_len) { case 16: return mbedtls_cipher_info_from_type(MBEDTLS_CIPHER_AES_128_GCM); @@ -242,10 +242,9 @@ const mbedtls_cipher_info_t *MbedTlsWrapper::AESStateMBEDTLS::GetCipher(size_t k case 32: return mbedtls_cipher_info_from_type(MBEDTLS_CIPHER_AES_256_GCM); default: - throw runtime_error("Invalid AES key length"); + throw runtime_error("Invalid AES key length for GCM"); } - - case CTR: + case duckdb::EncryptionTypes::CipherType::CTR: switch (key_len) { case 16: return mbedtls_cipher_info_from_type(MBEDTLS_CIPHER_AES_128_CTR); @@ -254,25 +253,40 @@ const mbedtls_cipher_info_t *MbedTlsWrapper::AESStateMBEDTLS::GetCipher(size_t k case 32: return mbedtls_cipher_info_from_type(MBEDTLS_CIPHER_AES_256_CTR); default: - throw runtime_error("Invalid AES key length"); + throw runtime_error("Invalid AES key length for CTR"); } - + case duckdb::EncryptionTypes::CipherType::CBC: + switch (key_len) { + case 16: + return mbedtls_cipher_info_from_type(MBEDTLS_CIPHER_AES_128_CBC); + case 24: + return mbedtls_cipher_info_from_type(MBEDTLS_CIPHER_AES_192_CBC); + case 32: + return mbedtls_cipher_info_from_type(MBEDTLS_CIPHER_AES_256_CBC); + default: + throw runtime_error("Invalid AES key length for CBC"); + } default: - throw duckdb::InternalException("Invalid Encryption/Decryption Cipher: %d", static_cast(cipher)); + throw duckdb::InternalException("Invalid Encryption/Decryption Cipher: %s", duckdb::EncryptionTypes::CipherToString(cipher)); } } -MbedTlsWrapper::AESStateMBEDTLS::AESStateMBEDTLS(duckdb::const_data_ptr_t key, duckdb::idx_t key_len) : context(duckdb::make_uniq()) { +MbedTlsWrapper::AESStateMBEDTLS::AESStateMBEDTLS(duckdb::EncryptionTypes::CipherType cipher_p, duckdb::idx_t key_len) : EncryptionState(cipher_p, key_len), context(duckdb::make_uniq()) { mbedtls_cipher_init(context.get()); auto cipher_info = GetCipher(key_len); if (!cipher_info) { - runtime_error("Failed to get Cipher"); + throw runtime_error("Failed to get Cipher"); } - if (mbedtls_cipher_setup(context.get(), cipher_info) != 0) { - runtime_error("Failed to initialize cipher context"); + if (mbedtls_cipher_setup(context.get(), cipher_info)) { + throw runtime_error("Failed to initialize cipher context"); + } + + if (cipher == duckdb::EncryptionTypes::CBC && mbedtls_cipher_set_padding_mode(context.get(), MBEDTLS_PADDING_PKCS7)) { + throw runtime_error("Failed to set CBC padding"); + } } @@ -285,7 +299,7 @@ MbedTlsWrapper::AESStateMBEDTLS::~AESStateMBEDTLS() { void MbedTlsWrapper::AESStateMBEDTLS::GenerateRandomDataStatic(duckdb::data_ptr_t data, duckdb::idx_t len) { duckdb::RandomEngine random_engine; - while (len != 0) { + while (len) { const auto random_integer = random_engine.NextRandomInteger(); const auto next = duckdb::MinValue(len, sizeof(random_integer)); memcpy(data, duckdb::const_data_ptr_cast(&random_integer), next); @@ -299,34 +313,38 @@ void MbedTlsWrapper::AESStateMBEDTLS::GenerateRandomData(duckdb::data_ptr_t data } void MbedTlsWrapper::AESStateMBEDTLS::InitializeInternal(duckdb::const_data_ptr_t iv, duckdb::idx_t iv_len, duckdb::const_data_ptr_t aad, duckdb::idx_t aad_len){ - if (mbedtls_cipher_set_iv(context.get(), iv, iv_len) != 0) { - runtime_error("Failed to set IV for encryption"); + if (mbedtls_cipher_set_iv(context.get(), iv, iv_len)) { + throw runtime_error("Failed to set IV for encryption"); } - + if (aad_len > 0) { - auto ret = mbedtls_cipher_update_ad(context.get(), aad, aad_len); - if (ret != 0) { + if (mbedtls_cipher_update_ad(context.get(), aad, aad_len)) { throw std::runtime_error("Failed to set AAD"); } } } +void MbedTlsWrapper::AESStateMBEDTLS::InitializeEncryption(duckdb::const_data_ptr_t iv, duckdb::idx_t iv_len, duckdb::const_data_ptr_t key, duckdb::idx_t key_len_p, duckdb::const_data_ptr_t aad, duckdb::idx_t aad_len) { + mode = duckdb::EncryptionTypes::ENCRYPT; -void MbedTlsWrapper::AESStateMBEDTLS::InitializeEncryption(duckdb::const_data_ptr_t iv, duckdb::idx_t iv_len, duckdb::const_data_ptr_t key, duckdb::idx_t key_len, duckdb::const_data_ptr_t aad, duckdb::idx_t aad_len) { - mode = ENCRYPT; - - if (mbedtls_cipher_setkey(context.get(), key, key_len * 8, MBEDTLS_ENCRYPT) != 0) { - runtime_error("Failed to set AES key for encryption"); + if (key_len_p != key_len) { + throw duckdb::InternalException("Invalid encryption key length, expected %llu, got %llu", key_len, key_len_p); + } + if (mbedtls_cipher_setkey(context.get(), key, key_len * 8, MBEDTLS_ENCRYPT)) { + throw runtime_error("Failed to set AES key for encryption"); } InitializeInternal(iv, iv_len, aad, aad_len); } -void MbedTlsWrapper::AESStateMBEDTLS::InitializeDecryption(duckdb::const_data_ptr_t iv, duckdb::idx_t iv_len, duckdb::const_data_ptr_t key, duckdb::idx_t key_len, duckdb::const_data_ptr_t aad, duckdb::idx_t aad_len) { - mode = DECRYPT; +void MbedTlsWrapper::AESStateMBEDTLS::InitializeDecryption(duckdb::const_data_ptr_t iv, duckdb::idx_t iv_len, duckdb::const_data_ptr_t key, duckdb::idx_t key_len_p, duckdb::const_data_ptr_t aad, duckdb::idx_t aad_len) { + mode = duckdb::EncryptionTypes::DECRYPT; - if (mbedtls_cipher_setkey(context.get(), key, key_len * 8, MBEDTLS_DECRYPT) != 0) { - runtime_error("Failed to set AES key for encryption"); + if (key_len_p != key_len) { + throw duckdb::InternalException("Invalid encryption key length, expected %llu, got %llu", key_len, key_len_p); + } + if (mbedtls_cipher_setkey(context.get(), key, key_len * 8, MBEDTLS_DECRYPT)) { + throw runtime_error("Failed to set AES key for encryption"); } InitializeInternal(iv, iv_len, aad, aad_len); @@ -334,28 +352,42 @@ void MbedTlsWrapper::AESStateMBEDTLS::InitializeDecryption(duckdb::const_data_pt size_t MbedTlsWrapper::AESStateMBEDTLS::Process(duckdb::const_data_ptr_t in, duckdb::idx_t in_len, duckdb::data_ptr_t out, duckdb::idx_t out_len) { - size_t result; - if (mbedtls_cipher_update(context.get(), reinterpret_cast(in), in_len, out, - &result) != 0) { - runtime_error("Encryption or Decryption failed at Process"); + + // GCM works in-place, CTR and CBC don't + auto use_out_copy = in == out && cipher != duckdb::EncryptionTypes::CipherType::GCM; + + auto out_ptr = out; + std::unique_ptr out_copy; + if (use_out_copy) { + out_copy.reset(new duckdb::data_t[out_len]); + out_ptr = out_copy.get(); + } + + size_t out_len_res = duckdb::NumericCast(out_len); + if (mbedtls_cipher_update(context.get(), reinterpret_cast(in), in_len, out_ptr, + &out_len_res)) { + throw runtime_error("Encryption or Decryption failed at Process"); }; - return result; + if (use_out_copy) { + memcpy(out, out_ptr, out_len_res); + } + return out_len_res; } void MbedTlsWrapper::AESStateMBEDTLS::FinalizeGCM(duckdb::data_ptr_t tag, duckdb::idx_t tag_len){ switch (mode) { - case ENCRYPT: { - if (mbedtls_cipher_write_tag(context.get(), tag, tag_len) != 0) { - runtime_error("Writing tag failed"); + case duckdb::EncryptionTypes::ENCRYPT: { + if (mbedtls_cipher_write_tag(context.get(), tag, tag_len)) { + throw runtime_error("Writing tag failed"); } break; } - case DECRYPT: { - if (mbedtls_cipher_check_tag(context.get(), tag, tag_len) != 0) { + case duckdb::EncryptionTypes::DECRYPT: { + if (mbedtls_cipher_check_tag(context.get(), tag, tag_len)) { throw duckdb::InvalidInputException( "Computed AES tag differs from read AES tag, are you using the right key?"); } @@ -368,10 +400,13 @@ void MbedTlsWrapper::AESStateMBEDTLS::FinalizeGCM(duckdb::data_ptr_t tag, duckdb } size_t MbedTlsWrapper::AESStateMBEDTLS::Finalize(duckdb::data_ptr_t out, duckdb::idx_t out_len, duckdb::data_ptr_t tag, - duckdb::idx_t tag_len) { + duckdb::idx_t tag_len) { size_t result = out_len; - mbedtls_cipher_finish(context.get(), out, &result); - FinalizeGCM(tag, tag_len); - + if (mbedtls_cipher_finish(context.get(), out, &result)) { + throw runtime_error("Encryption or Decryption failed at Finalize"); + } + if (cipher == duckdb::EncryptionTypes::GCM) { + FinalizeGCM(tag, tag_len); + } return result; } diff --git a/third_party/yyjson/include/yyjson_utils.hpp b/third_party/yyjson/include/yyjson_utils.hpp new file mode 100644 index 000000000000..a848d44a9a2c --- /dev/null +++ b/third_party/yyjson/include/yyjson_utils.hpp @@ -0,0 +1,33 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// yyjson_utils.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "yyjson.hpp" + +using namespace duckdb_yyjson; // NOLINT + +namespace duckdb { + +struct ConvertedJSONHolder { +public: + ~ConvertedJSONHolder() { + if (doc) { + yyjson_mut_doc_free(doc); + } + if (stringified_json) { + free(stringified_json); + } + } + +public: + yyjson_mut_doc *doc = nullptr; + char *stringified_json = nullptr; +}; + +} // namespace duckdb diff --git a/tools/juliapkg/Project.toml b/tools/juliapkg/Project.toml index b7c0519587b0..4256dc97084f 100644 --- a/tools/juliapkg/Project.toml +++ b/tools/juliapkg/Project.toml @@ -1,7 +1,7 @@ name = "DuckDB" uuid = "d2f5444f-75bc-4fdf-ac35-56f514c445e1" authors = ["Mark Raasveldt "] -version = "1.3.2" +version = "1.4.1" [deps] DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" @@ -14,7 +14,7 @@ WeakRefStrings = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" [compat] DBInterface = "2.5" -DuckDB_jll = "1.3.2" +DuckDB_jll = "1.4.1" FixedPointDecimals = "0.4, 0.5, 0.6" Tables = "1.7" WeakRefStrings = "1.4" diff --git a/tools/shell/linenoise/linenoise.cpp b/tools/shell/linenoise/linenoise.cpp index 4f4f86dfb938..af63fd7c5c94 100644 --- a/tools/shell/linenoise/linenoise.cpp +++ b/tools/shell/linenoise/linenoise.cpp @@ -575,7 +575,7 @@ void Linenoise::EditMoveEnd() { /* Move cursor to the start of the line. */ void Linenoise::EditMoveStartOfLine() { - while (pos > 0 && buf[pos] != '\n') { + while (pos > 0 && buf[pos - 1] != '\n') { pos--; } RefreshLine(); diff --git a/tools/shell/shell.cpp b/tools/shell/shell.cpp index 468d4305acc5..9fbdc8965dc0 100644 --- a/tools/shell/shell.cpp +++ b/tools/shell/shell.cpp @@ -81,6 +81,7 @@ #include #include "duckdb_shell_wrapper.h" #include "duckdb/common/box_renderer.hpp" +#include "duckdb/common/file_system.hpp" #include "duckdb/parser/qualified_name.hpp" #include "sqlite3.h" typedef sqlite3_int64 i64; @@ -1981,6 +1982,71 @@ static void toggleSelectOrder(sqlite3 *db) { sqlite3_exec(db, zStmt, 0, 0, 0); } +/* +** Lookup the schema for a table using the information schema. +*/ +static char *getTableSchema(sqlite3 *db, const char *zTable) { + char *zSchema = NULL; + sqlite3_stmt *pStmt = NULL; + char *zSql = sqlite3_mprintf("SELECT table_schema FROM information_schema.tables " + "WHERE table_name = %Q AND table_type='BASE TABLE' " + "ORDER BY (table_schema='main') DESC LIMIT 1", + zTable); + + if (!zSql) + return NULL; + + int rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + + if (rc == SQLITE_OK && pStmt) { + if (sqlite3_step(pStmt) == SQLITE_ROW) { + const char *schema = (const char *)sqlite3_column_text(pStmt, 0); + if (schema) + zSchema = sqlite3_mprintf("%s", schema); + } + sqlite3_finalize(pStmt); + } + + return zSchema; +} + +/* +** Quote an identifier if needed. +*/ +static char *quoteIdentifier(const char *zIdent) { + if (!zIdent) + return NULL; + + bool needsQuote = !isalpha((unsigned char)zIdent[0]) && zIdent[0] != '_'; + if (!needsQuote) { + for (const char *p = zIdent; *p; p++) { + if (!isalnum((unsigned char)*p) && *p != '_') { + needsQuote = true; + break; + } + } + } + + return needsQuote ? sqlite3_mprintf("\"%w\"", zIdent) : sqlite3_mprintf("%s", zIdent); +} + +/* +** Build a qualified name: schema.table +*/ +static char *buildQualifiedName(const char *zSchema, const char *zTable) { + char *quotedSchema = quoteIdentifier(zSchema); + char *quotedTable = quoteIdentifier(zTable); + char *result = NULL; + + if (quotedSchema && quotedTable) { + result = sqlite3_mprintf("%s.%s", quotedSchema, quotedTable); + } + sqlite3_free(quotedSchema); + sqlite3_free(quotedTable); + return result; +} + /* ** This is a different callback routine used for dumping the database. ** Each row received by this callback consists of a table name, @@ -2026,16 +2092,33 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed) { int i; char *savedDestTable; RenderMode savedMode; + char *zSchema = NULL; + char *zQualifiedName = NULL; + + zSchema = getTableSchema(p->db, zTable); + if (!zSchema) + zSchema = sqlite3_mprintf("main"); - azCol = p->TableColumnList(zTable); + zQualifiedName = buildQualifiedName(zSchema, zTable); + if (!zQualifiedName) { + sqlite3_free(zSchema); + p->nErr++; + return 0; + } + + azCol = p->TableColumnList(zQualifiedName); if (azCol == 0) { + sqlite3_free(zQualifiedName); + sqlite3_free(zSchema); p->nErr++; return 0; } - /* Always quote the table name, even if it appears to be pure ascii, - ** in case it is a keyword. Ex: INSERT INTO "table" ... */ - appendText(sTable, zTable, quoteChar(zTable)); + if (strcmp(zSchema, "main") != 0) { + appendText(sTable, zQualifiedName, 0); + } else { + appendText(sTable, zTable, quoteChar(zTable)); + } /* If preserving the rowid, add a column list after the table name. ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" ** instead of the usual "INSERT INTO tab VALUES(...)". @@ -2064,7 +2147,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed) { } freeColumnList(azCol); appendText(sSelect, " FROM ", 0); - appendText(sSelect, zTable, quoteChar(zTable)); + appendText(sSelect, zQualifiedName, 0); savedDestTable = p->zDestTable; savedMode = p->mode; @@ -2079,7 +2162,9 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed) { } p->zDestTable = savedDestTable; p->mode = savedMode; - if (rc) + sqlite3_free(zQualifiedName); + sqlite3_free(zSchema); + if (rc != SQLITE_OK && rc != SQLITE_DONE) p->nErr++; } return 0; @@ -2647,7 +2732,7 @@ static void output_file_close(FILE *f) { ** filename is "off". */ static FILE *output_file_open(const char *zFile, int bTextMode) { - FILE *f; + FILE *f = nullptr; if (strcmp(zFile, "stdout") == 0) { f = stdout; } else if (strcmp(zFile, "stderr") == 0) { @@ -2655,7 +2740,8 @@ static FILE *output_file_open(const char *zFile, int bTextMode) { } else if (strcmp(zFile, "off") == 0) { f = 0; } else { - f = fopen(zFile, bTextMode ? "w" : "wb"); + const string expanded_path = duckdb::FileSystem::ExpandPath(zFile, /*opener=*/nullptr); + f = fopen(expanded_path.c_str(), bTextMode ? "w" : "wb"); if (f == 0) { utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); } @@ -3117,6 +3203,30 @@ MetadataResult DumpTable(ShellState &state, const char **azArg, idx_t nArg) { state.nErr = 0; if (zLike == 0) zLike = sqlite3_mprintf("true"); + + // Emit CREATE SCHEMA for non-main schemas first + zSql = sqlite3_mprintf("SELECT DISTINCT table_schema FROM information_schema.tables " + "WHERE table_schema != 'main' AND table_schema NOT LIKE 'pg_%%' " + "AND table_schema != 'information_schema' " + "AND table_name IN (SELECT name FROM sqlite_schema WHERE (%s) AND type=='table') " + "ORDER BY table_schema", + zLike); + sqlite3_stmt *pStmt = NULL; + if (sqlite3_prepare_v2(state.db, zSql, -1, &pStmt, 0) == SQLITE_OK) { + while (sqlite3_step(pStmt) == SQLITE_ROW) { + const char *schema = (const char *)sqlite3_column_text(pStmt, 0); + if (schema) { + char *quotedSchema = quoteIdentifier(schema); + if (quotedSchema) { + raw_printf(state.out, "CREATE SCHEMA IF NOT EXISTS %s;\n", quotedSchema); + sqlite3_free(quotedSchema); + } + } + } + sqlite3_finalize(pStmt); + } + sqlite3_free(zSql); + zSql = sqlite3_mprintf("SELECT name, type, sql FROM sqlite_schema " "WHERE (%s) AND type=='table'" " AND sql NOT NULL" @@ -4756,23 +4866,24 @@ static const char zOptions[] = " -s COMMAND run \"COMMAND\" and exit\n" " -safe enable safe-mode\n" " -separator SEP set output column separator. Default: '|'\n" + " -storage-version V database storage compatibility version to use. Default: 'v0.10.0'\n" " -table set output mode to 'table'\n" " -ui launches a web interface using the ui extension (configurable with .ui_command)\n" " -unredacted allow printing unredacted secrets\n" " -unsigned allow loading of unsigned extensions\n" " -version show DuckDB version\n"; static void usage(int showDetail) { - utf8_printf(stderr, + utf8_printf(stdout, "Usage: %s [OPTIONS] FILENAME [SQL]\n" "FILENAME is the name of a DuckDB database. A new database is created\n" "if the file does not previously exist.\n", program_name); if (showDetail) { - utf8_printf(stderr, "OPTIONS include:\n%s", zOptions); + utf8_printf(stdout, "OPTIONS include:\n%s", zOptions); } else { - raw_printf(stderr, "Use the -help option for additional information\n"); + raw_printf(stdout, "Use the -help option for additional information\n"); } - exit(1); + exit(0); } /* @@ -4965,12 +5076,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv) { data.openFlags |= DUCKDB_UNSIGNED_EXTENSIONS; } else if (strcmp(z, "-safe") == 0) { safe_mode = true; - } else if (strcmp(z, "-storage_version") == 0) { + } else if (strcmp(z, "-storage_version") == 0 || strcmp(z, "-storage-version") == 0) { auto storage_version = string(cmdline_option_value(argc, argv, ++i)); if (storage_version != "latest") { utf8_printf( stderr, - "%s: Error: unknown argument (%s) for '-storage_version', only 'latest' is supported currently\n", + "%s: Error: unknown argument (%s) for '-storage-version', only 'latest' is supported currently\n", program_name, storage_version.c_str()); } else { data.openFlags |= DUCKDB_LATEST_STORAGE_VERSION; diff --git a/tools/shell/tests/test_shell_basics.py b/tools/shell/tests/test_shell_basics.py index ef836cb5744c..88ca4202c69a 100644 --- a/tools/shell/tests/test_shell_basics.py +++ b/tools/shell/tests/test_shell_basics.py @@ -913,6 +913,76 @@ def test_dump_blobs(shell): result = test.run() result.check_stdout("'\\x07\\x08\\x09'") +def test_dump_schema_qualified(shell): + test = ( + ShellTest(shell) + .statement("CREATE SCHEMA other;") + .statement("CREATE TABLE other.t_in_other(a INT);") + .statement(".dump") + ) + result = test.run() + result.check_stdout('CREATE SCHEMA IF NOT EXISTS other;') + result.check_stdout('CREATE TABLE other.t_in_other(a INTEGER);') + result.check_stdout('COMMIT') + +def test_dump_schema_with_data(shell): + test = ( + ShellTest(shell) + .statement("CREATE SCHEMA test_schema;") + .statement("CREATE TABLE test_schema.tbl(x INT, y VARCHAR);") + .statement(".changes off") + .statement("INSERT INTO test_schema.tbl VALUES (1, 'hello'), (2, 'world');") + .statement(".dump") + ) + result = test.run() + result.check_stdout('CREATE SCHEMA IF NOT EXISTS test_schema;') + result.check_stdout('CREATE TABLE test_schema.tbl(x INTEGER, y VARCHAR);') + result.check_stdout("INSERT INTO test_schema.tbl VALUES(1,'hello');") + result.check_stdout('COMMIT') + +def test_dump_multiple_schemas(shell): + test = ( + ShellTest(shell) + .statement("CREATE SCHEMA s1;") + .statement("CREATE SCHEMA s2;") + .statement("CREATE TABLE s1.t1(a INT);") + .statement("CREATE TABLE s2.t2(b INT);") + .statement(".changes off") + .statement("INSERT INTO s1.t1 VALUES (10);") + .statement("INSERT INTO s2.t2 VALUES (20);") + .statement(".dump") + ) + result = test.run() + result.check_stdout('CREATE SCHEMA IF NOT EXISTS s1;') + result.check_stdout('CREATE SCHEMA IF NOT EXISTS s2;') + result.check_stdout('INSERT INTO s1.t1 VALUES(10);') + result.check_stdout('INSERT INTO s2.t2 VALUES(20);') + +def test_dump_quoted_schema(shell): + test = ( + ShellTest(shell) + .statement('CREATE SCHEMA "my-schema";') + .statement('CREATE TABLE "my-schema"."my-table"(a INT);') + .statement(".dump") + ) + result = test.run() + result.check_stdout('CREATE SCHEMA IF NOT EXISTS "my-schema";') + result.check_stdout('CREATE TABLE IF NOT EXISTS "my-schema"."my-table"(a INTEGER);') + +def test_dump_if_not_exists(shell): + test = ( + ShellTest(shell) + .statement("CREATE SCHEMA other;") + .statement("CREATE TABLE IF NOT EXISTS other.tbl(x INT);") + .statement(".changes off") + .statement("INSERT INTO other.tbl VALUES (42);") + .statement(".dump") + ) + result = test.run() + result.check_stdout('CREATE SCHEMA IF NOT EXISTS other;') + result.check_stdout('INSERT INTO other.tbl VALUES(42);') + result.check_stdout('COMMIT') + def test_invalid_csv(shell, tmp_path): file = tmp_path / 'nonsencsv.csv' with open(file, 'wb+') as f: @@ -1132,5 +1202,10 @@ def test_tables_invalid_pattern_handling(shell): # Should show usage message for invalid pattern result.check_stderr("Usage: .tables ?TABLE?") +def test_help_prints_to_stdout(shell): + test = ShellTest(shell, ["--help"]) + result = test.run() + result.check_stdout("OPTIONS include:") + # fmt: on