Merge remote-tracking branch 'upstream/main'

Author: Tishj

.clang-tidy

@@ -1,4 +1,4 @@
-Checks:          '-*,clang-diagnostic-*,bugprone-*,performance-*,google-explicit-constructor,google-build-using-namespace,google-runtime-int,misc-definitions-in-headers,modernize-use-nullptr,modernize-use-override,-bugprone-macro-parentheses,readability-braces-around-statements,-bugprone-branch-clone,readability-identifier-naming,hicpp-exception-baseclass,misc-throw-by-value-catch-by-reference,-bugprone-signed-char-misuse,-bugprone-misplaced-widening-cast,-bugprone-sizeof-expression,-bugprone-easily-swappable-parameters,google-global-names-in-headers,llvm-header-guard,misc-definitions-in-headers,modernize-use-emplace,modernize-use-bool-literals,-performance-inefficient-string-concatenation,-performance-no-int-to-ptr,readability-container-size-empty,cppcoreguidelines-pro-type-cstyle-cast,-llvm-header-guard,-performance-enum-size,cppcoreguidelines-pro-type-const-cast,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-interfaces-global-init,cppcoreguidelines-slicing,cppcoreguidelines-rvalue-reference-param-not-moved,cppcoreguidelines-virtual-class-destructor,-readability-identifier-naming,-bugprone-exception-escape,-bugprone-unused-local-non-trivial-variable,-bugprone-empty-catch'
+Checks:          '-*,clang-diagnostic-*,bugprone-*,performance-*,google-explicit-constructor,google-build-using-namespace,google-runtime-int,misc-definitions-in-headers,modernize-use-nullptr,modernize-use-override,-bugprone-macro-parentheses,readability-braces-around-statements,-bugprone-branch-clone,readability-identifier-naming,hicpp-exception-baseclass,misc-throw-by-value-catch-by-reference,-bugprone-signed-char-misuse,-bugprone-misplaced-widening-cast,-bugprone-sizeof-expression,-bugprone-easily-swappable-parameters,google-global-names-in-headers,llvm-header-guard,misc-definitions-in-headers,modernize-use-emplace,modernize-use-bool-literals,-performance-inefficient-string-concatenation,-performance-no-int-to-ptr,readability-container-size-empty,cppcoreguidelines-pro-type-cstyle-cast,-llvm-header-guard,-performance-enum-size,cppcoreguidelines-pro-type-const-cast,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-interfaces-global-init,cppcoreguidelines-slicing,cppcoreguidelines-rvalue-reference-param-not-moved,cppcoreguidelines-virtual-class-destructor,-readability-identifier-naming,-bugprone-exception-escape,-bugprone-unused-local-non-trivial-variable,-bugprone-empty-catch,-misc-use-internal-linkage,-readability-static-definition-in-anonymous-namespace'
 WarningsAsErrors: '*'
 HeaderFilterRegex: 'src/include/duckdb/.*'
 FormatStyle:     none
@@ -49,4 +49,7 @@ CheckOptions:
     value:           '::duckdb::shared_ptr;::duckdb::unique_ptr;::std::auto_ptr;::duckdb::weak_ptr'
   - key:             cppcoreguidelines-rvalue-reference-param-not-moved.IgnoreUnnamedParams
     value:           true
-
+  - key:             misc-use-internal-linkage
+    value:           true
+  - key:             readability-static-definition-in-anonymous-namespace
+    value:           true

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -23,6 +23,7 @@ body:
             * [AWS extension](https://github.com/duckdb/duckdb-aws/issues/new)
             * [Azure extension](https://github.com/duckdb/duckdb-azure/issues/new)
             * [Delta extension](https://github.com/duckdb/duckdb-delta/issues/new)
+            * [Encodings extension](https://github.com/duckdb/duckdb-encodings/issues/new)
             * [Excel extension](https://github.com/duckdb/duckdb-excel/issues/new)
             * [fts (full text search) extension](https://github.com/duckdb/duckdb-fts/issues/new)
             * [httpfs extension](https://github.com/duckdb/duckdb-httpfs/issues/new)

.github/actions/build_extensions/action.yml

@@ -4,10 +4,10 @@ inputs:
   # Test config
   run_tests:
     description: 'Run extension tests after build'
-    default: 1
+    default: '1'
   run_autoload_tests:
     description: 'Runs the autoloading tests'
-    default: 1
+    default: '1'
 
   # Deploy config
   deploy_as:
@@ -32,19 +32,19 @@ inputs:
     default: ''
   static_link_build:
     description: 'Links DuckDB statically to the loadable extensions'
-    default: 1
+    default: '1'
   no_static_linking:
     description: 'Disables linking extensions into DuckDB for testing'
-    default: 0
+    default: '0'
   vcpkg_build:
     description: 'Installs vcpkg and pass its toolchain to CMakes'
-    default: 1
+    default: '1'
   build_dir:
     description: 'DuckDB source directory to run the build in'
     default: '.'
   ninja:
     description: 'Use ninja for building'
-    default: 0
+    default: '0'
   openssl_path:
     description: 'Directory of OpenSSL installation'
     default: ''
@@ -53,25 +53,28 @@ inputs:
     default: ''
   treat_warn_as_error:
     description: 'Treat compilation warnings as errors'
-    default: 1
+    default: '1'
   build_in_tree_extensions:
     description: 'Build in-tree extensions'
-    default: 1
+    default: '1'
   build_out_of_tree_extensions:
     description: 'Build out-of-tree extensions'
-    default: 1
+    default: '1'
+  build_complete_extensions_set:
+    description: 'Whether all extensions needs to be built'
+    default: '1'
   bundle_static_lib_mode:
     description: 'Build the default bundled extensions to publish the static libs'
-    default: 0
+    default: '0'
   osx_universal:
     description: 'Build Universal Binary for OSX'
-    default: 0
+    default: '0'
   osx_arch:
     description: 'Build specific architecture for OSX'
     default: ''
   aarch64_cross_compile:
     description: 'Enable Linux aarch64 cross-compiling'
-    default: 0
+    default: '0'
   vcpkg_target_triplet:
     description: 'Target triplet for installing vcpkg dependencies'
     default: ''
@@ -100,17 +103,18 @@ runs:
         echo "EXTENSION_CONFIGS=$EXTENSION_CONFIGS" >> $GITHUB_ENV
 
     - name: Setup vcpkg
-      if: inputs.vcpkg_build == 1
+      if: ${{ inputs.vcpkg_build == 1 }}
       uses: lukka/[email protected]
       with:
-        vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
+        vcpkgGitCommitId: ce613c41372b23b1f51333815feb3edd87ef8a8b
 
     - name: Set vcpkg env variables
-      if: inputs.vcpkg_build == 1
+      if: ${{ inputs.vcpkg_build == 1 }}
       shell: bash
       run: |
         echo "VCPKG_TOOLCHAIN_PATH=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" >> $GITHUB_ENV
         echo "VCPKG_TARGET_TRIPLET=${{ inputs.vcpkg_target_triplet }}" >> $GITHUB_ENV
+        echo "BUILD_COMPLETE_EXTENSION_SET=${{ inputs.build_complete_extensions_set }}" >> $GITHUB_ENV
 
     - name: workaround for https://github.com/duckdb/duckdb/issues/8360
       if: inputs.vcpkg_target_triplet == 'x64-windows-static-md'
@@ -130,7 +134,7 @@ runs:
         echo "OPENSSL_ROOT_DIR=${{ inputs.openssl_path }}" >> $GITHUB_ENV
 
     - name: Create combined vcpkg manifest
-      if: inputs.vcpkg_build == 1 && inputs.build_out_of_tree_extensions == 1
+      if: ${{ inputs.vcpkg_build == '1' && inputs.build_out_of_tree_extensions == '1' }}
       shell: bash
       run: |
         make extension_configuration
@@ -150,6 +154,7 @@ runs:
         LOCAL_EXTENSION_REPO: ${{ inputs.run_autoload_tests == 1 && github.workspace || ''}}
         GEN: ${{ inputs.ninja == 1 && 'ninja' || '' }}
         USE_MERGED_VCPKG_MANIFEST: 1
+        # TODO we should no longer override this but we should probably check that it is what we expect
         DUCKDB_PLATFORM: ${{ inputs.duckdb_arch }}
         CMAKE_VARS_BUILD: ${{ inputs.cmake_flags }}
 
@@ -169,7 +174,7 @@ runs:
     # - statically linked extensions are disable on startup but a sqlogictest require will call their load function
     # - loadable-only extensions have their loadable extension loaded with the LOAD statement on a sqlogictest require
     - name: Test statically linked extensions
-      if: ${{ inputs.run_tests == 1 && inputs.no_static_linking == 0}}
+      if: ${{ inputs.run_tests == '1' && inputs.no_static_linking == '0' }}
       shell: bash
       run: |
         ${{ inputs.unittest_script }}
@@ -179,19 +184,19 @@ runs:
       shell: bash
       run: |
         ls
-        cd  ${{ inputs.build_dir}}
+        cd  ${{ inputs.build_dir }}
         ${{ inputs.post_install }}
 
     # The reason we need to rebuild is we need to test auto-loading extensions: this is only possible without the other
     # extensions linked
     - name: Rebuild DuckDB without any extensions, but with all extension tests
-      if: ${{ inputs.run_autoload_tests == 1 }}
+      if: ${{ inputs.run_autoload_tests == '1' }}
       shell: bash
       env:
         EXTENSION_TESTS_ONLY: 1
         ENABLE_EXTENSION_AUTOLOADING: 1
         ENABLE_EXTENSION_AUTOINSTALL: 1
-        GEN: ${{ inputs.ninja == 1 && 'ninja' || '' }}
+        GEN: ${{ inputs.ninja == '1' && 'ninja' || '' }}
         USE_MERGED_VCPKG_MANIFEST: 1
       run: |
         cd  ${{ inputs.build_dir}}
@@ -202,10 +207,11 @@ runs:
 
     # Run all unittests (including the out-of-tree tests) without any extensions linked, relying on the autoloader
     - name: Run tests with auto loading
-      if: ${{ inputs.run_autoload_tests == 1 }}
+      if: ${{ inputs.run_autoload_tests == '1' }}
       shell: bash
       env:
-        LOCAL_EXTENSION_REPO: ${{ inputs.run_autoload_tests == 1 && github.workspace || ''}}
+        LOCAL_EXTENSION_REPO: ${{ inputs.run_autoload_tests == '1' && github.workspace || ''}}
+        DUCKDB_TEST_DESCRIPTION: 'Extension autoloading tests. All `require` calls are ignored and auto-loading is tested. Use require no_extension_autoloading in the test to skip tests.'
       run: |
         cd  ${{ inputs.build_dir}}
         python3 scripts/get_test_list.py --file-contains 'require ' --list '"*.test"' > test.list

.github/actions/build_extensions_dockerized/action.yml

@@ -4,7 +4,7 @@ inputs:
   # Build config
   run_tests:
     description: 'Run extension tests after build'
-    default: 1
+    default: 'true'
   duckdb_arch:
     description: 'Provide DUCKDB_PLATFORM to build system for cross compilation'
     default: ''
@@ -14,6 +14,12 @@ inputs:
   override_git_describe:
     description: 'Override git describe'
     default: ''
+  build_complete_extensions_set:
+    description: 'Whether all extensions needs to be built'
+    default: '1'
+  save_cache:
+    description: 'Should cache be saved'
+    default: '1'
 
 runs:
   using: "composite"
@@ -44,7 +50,7 @@ runs:
         run: |
           docker build \
             --build-arg 'vcpkg_url=https://github.com/microsoft/vcpkg.git' \
-            --build-arg 'vcpkg_commit=5e5d0e1cd7785623065e77eff011afdeec1a3574' \
+            --build-arg 'vcpkg_commit=ce613c41372b23b1f51333815feb3edd87ef8a8b' \
             --build-arg 'extra_toolchains=;python3;' \
             -t duckdb/${{ inputs.duckdb_arch }} \
             ./extension-ci-tools/docker/${{ inputs.duckdb_arch }}
@@ -58,27 +64,25 @@ runs:
           echo "OPENSSL_ROOT_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ inputs.vcpkg_target_triplet }}" >> docker_env.txt
           echo "OPENSSL_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ inputs.vcpkg_target_triplet }}" >> docker_env.txt
           echo "OPENSSL_USE_STATIC_LIBS=true" >> docker_env.txt
+          # TODO this should also turn into a check
           echo "DUCKDB_PLATFORM=${{ inputs.duckdb_arch }}" >> docker_env.txt
           echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.override_git_describe }}" >> docker_env.txt
           echo "LINUX_CI_IN_DOCKER=1" >> docker_env.txt
-          echo "TOOLCHAIN_FLAGS=${{ inputs.duckdb_arch == 'linux_arm64' && '-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran' || '' }}" >> docker_env.txt
-          echo "CC=${{ inputs.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }}" >> docker_env.txt
-          echo "CXX=${{ inputs.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }}" >> docker_env.txt 
+          echo "BUILD_COMPLETE_EXTENSION_SET=${{ inputs.build_complete_extensions_set }}" >> docker_env.txt
 
       - name: Generate timestamp for Ccache entry
-        shell: cmake -P {0}
+        shell: bash
         id: ccache_timestamp
         run: |
-          string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-          message("::set-output name=timestamp::${current_date}")
+          date --utc +'timestamp=%Y-%m-%d-%H;%M;%S' >> "$GITHUB_OUTPUT"
 
       - name: Create Ccache directory
         shell: bash
         run: |
           mkdir ccache_dir
 
       - name: Load Ccache
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
           path: ./ccache_dir
           key: ccache-extension-distribution-${{ inputs.duckdb_arch }}-${{ steps.ccache_timestamp.outputs.timestamp }}
@@ -95,8 +99,15 @@ runs:
         run: |
           docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ inputs.duckdb_arch }} make release
 
+      - name: Save Ccache
+        if: ${{ inputs.save_cache == '1' }}
+        uses: actions/cache/save@v4
+        with:
+          path: ./ccache_dir
+          key: ccache-extension-distribution-${{ inputs.duckdb_arch }}-${{ steps.ccache_timestamp.outputs.timestamp }}
+
       - name: Test extension (inside docker)
         shell: bash
-        if: ${{ inputs.run_tests == 1 && inputs.duckdb_arch != 'linux_arm64'}}
+        if: ${{ inputs.run_tests == 'true' && inputs.duckdb_arch != 'linux_arm64'}}
         run: |
           docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ inputs.duckdb_arch }} make test_release

.github/actions/manylinux_2014_setup/action.yml

@@ -3,17 +3,12 @@
     "include": [
       {
         "duckdb_arch": "linux_amd64",
-        "container": "ubuntu:18.04",
-        "vcpkg_triplet": "x64-linux"
-      },
-      {
-        "duckdb_arch": "linux_amd64_gcc4",
-        "container": "quay.io/pypa/manylinux2014_x86_64",
+        "container": "quay.io/pypa/manylinux_2_28_x86_64",
         "vcpkg_triplet": "x64-linux"
       },
       {
         "duckdb_arch": "linux_arm64",
-        "container": "ubuntu:18.04",
+        "container": "quay.io/pypa/manylinux_2_28_aarch64",
         "vcpkg_triplet": "arm64-linux"
       },
       {