Get latest libmagic from MSYS2 for Windows (#10)

Author: ddelange

.github/workflows/ci.yml

@@ -1,5 +1,8 @@
 name: ci
 on: [push, pull_request]
+concurrency:  # https://stackoverflow.com/questions/66335225#comment133398800_72408109
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 jobs:
   ci:
     strategy:

.github/workflows/wheels.yml

@@ -8,8 +8,11 @@ on:
     types: [released, prereleased]
   workflow_dispatch:  # allows running workflow manually from the Actions tab
 
-jobs:
+concurrency:  # https://stackoverflow.com/questions/66335225#comment133398800_72408109
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
+jobs:
   build-sdist:
     runs-on: ubuntu-latest
 
@@ -86,14 +89,50 @@ jobs:
       if: runner.os == 'Linux'
       uses: docker/setup-qemu-action@v3
 
+    # For Windows, grabbing latest file from MSYS2 is easier than building from source
+    # It's generally up to date ref https://packages.msys2.org/base/mingw-w64-file
+    - name: Setup MSYS2 and install file
+      if: runner.os == 'Windows'
+      uses: msys2/[email protected]
+      with:
+        msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}
+        location: D:\
+        install: >-
+          ${{ endsWith(matrix.only, '32') && 'mingw-w64-i686-file' || 'mingw-w64-x86_64-file' }}
+
+    # The DLL dependency tree flattened out ref "Dependencies" https://packages.msys2.org/packages/mingw-w64-x86_64-file
+    - name: Copy Windows ddl and mgc
+      if: runner.os == 'Windows'
+      run: |
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/share/misc/magic.mgc" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libmagic-1.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libsystre-0.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libtre-5.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libasprintf-0.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libintl-8.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libatomic-1.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libgomp-1.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libquadmath-0.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libstdc++-6.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libcharset-1.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libiconv-2.dll" "magic"
+
+    # These are needed additionally in the win32 wheel ref https://packages.msys2.org/packages/mingw-w64-i686-file
+    - name: Copy additional 32-bit runtime DLLs
+      if: runner.os == 'Windows' && endsWith(matrix.only, '32')
+      run: |
+        cp "/msys64/mingw32/bin/libgcc_s_dw2-1.dll" "magic"
+        cp "/msys64/mingw32/bin/libwinpthread-1.dll" "magic"
+
     - uses: pypa/[email protected]  # sync version with pip install cibuildwheel above
       timeout-minutes: 10
       with:
         only: ${{ matrix.only }}
       env:
         CIBW_BUILD_VERBOSITY: 1
         # add compiled libmagic to the build directory (to include in the wheel)
-        CIBW_BEFORE_BUILD: ${{ ( startsWith( matrix.os, 'macos' ) && 'sudo -E bash add_libmagic.sh' ) || 'bash add_libmagic.sh' }}
+        CIBW_BEFORE_BUILD_MACOS: sudo -E bash add_libmagic.sh
+        CIBW_BEFORE_BUILD_LINUX: bash add_libmagic.sh
         # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag)
         MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }}
         # simple smoke test run on each wheel: this is an HLS MP4 video, only recognised in recent versions of libmagic

add_libmagic.sh

@@ -27,7 +27,6 @@ install_precompiled() {
     # Debian https://packages.ubuntu.com/libmagic1
     # Alpine https://pkgs.alpinelinux.org/package/libmagic
     # RHEL https://git.almalinux.org/rpms/file
-    # Windows https://github.com/julian-r/file-windows
     if [ -n "$(which brew)" ]; then
         brew install libmagic
     elif [ -n "$(which apt-get)" ]; then
@@ -37,20 +36,6 @@ install_precompiled() {
         apk add --update libmagic
     elif [ -n "$(which dnf)" ]; then
         dnf --setopt install_weak_deps=false -y install file-libs
-    else
-        # windows (no install, just download into current working directory)
-        # could also consider install using `pacman`: https://packages.msys2.org/base/mingw-w64-file
-        # which would require an update of copy_libmagic below to account for new magic.mgc paths
-        python <<EOF
-import platform, sysconfig, io, zipfile, urllib.request
-assert platform.system() == "Windows"
-machine = "x86" if sysconfig.get_platform() == "win32" else "x64"
-url = f"https://github.com/julian-r/file-windows/releases/download/v5.44/file_5.44-build104-vs2022-{machine}.zip"
-print("Downloading", url)
-zipfile.ZipFile(io.BytesIO(urllib.request.urlopen(url).read())).extractall(".")
-EOF
-        # check what was copied
-        ls -ltra
     fi
 }
 
@@ -60,12 +45,14 @@ copy_libmagic() {
     # this python command relies on current working directory containing `./magic/loader.py`
     libmagic_path="$(python -c 'from magic.loader import load_lib; print(load_lib()._name)')" &&
     cp "${libmagic_path}" "magic" &&
-    # only on linux/macos: additionally copy compiled db into magic dir (prefer the one installed by install_source)
+    # additionally copy compiled db into magic dir (prefer the one installed by install_source)
     ( ( cp "/usr/local/share/misc/magic.mgc" "magic" || cp "/usr/share/misc/magic.mgc" "magic" ) || true ) &&
     # check what was copied
     ls -ltra magic
 }
 
+# skip windows (taken care of separately in wheels.yml)
+python -c 'import platform; assert platform.system() != "Windows"' || ( echo "skipping on windows" && exit 0 )
 # prefer a recent build from source
 install_source || install_precompiled
 # files to be copied into the wheel

setup.py

@@ -6,11 +6,6 @@
 import os
 import sys
 
-# python packages should not install succesfully if libraries are missing
-from magic.loader import load_lib
-
-load_lib()._name
-
 
 def read(file_name):
     """Read a text file and return the content as a string."""