diff --git a/BinaryBuilderAuditor.jl/Manifest.toml b/BinaryBuilderAuditor.jl/Manifest.toml index f1b4dfe..d64f2b5 100644 --- a/BinaryBuilderAuditor.jl/Manifest.toml +++ b/BinaryBuilderAuditor.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "f0465bc14e77aee28eba0944a179c9c65f95a304" @@ -41,7 +41,7 @@ uuid = "21737265-7a69-6e4f-4974-6375646f7270" version = "0.1.0" [[deps.BinaryBuilderSources]] -deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TimerOutputs", "TreeArchival"] +deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TOML", "TimerOutputs", "TreeArchival"] path = "../BinaryBuilderSources.jl" uuid = "316c416d-4527-6863-7465-466137743047" version = "0.1.0" @@ -54,9 +54,9 @@ version = "0.2.0" [[deps.Compat]] deps = ["TOML", "UUIDs"] -git-tree-sha1 = "3a3dfb30697e96a440e4149c8c51bf32f818c0f3" +git-tree-sha1 = "0037835448781bb46feb39866934e243886d756a" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.17.0" +version = "4.18.0" [deps.Compat.extensions] CompatLinearAlgebraExt = "LinearAlgebra" @@ -77,9 +77,9 @@ version = "1.6.0" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" +git-tree-sha1 = "7bb1361afdb33c7f2b085aa49ea8fe1b0fb14e58" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.5+0" +version = "2.7.1+0" [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" @@ -91,24 +91,30 @@ uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" [[deps.Git]] -deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] -git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "cb151153e40ad40a6dbf984fcd767e1d266fcc9c" +git-tree-sha1 = "e2aef26f7d273f1e5b1daba56837c47b49b4388f" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.50.1+0" +version = "2.51.1+0" [[deps.HistoricalStdlibVersions]] deps = ["Pkg", "PrecompileTools"] -git-tree-sha1 = "1c2bfcc3d0ad9628a0fc247a267d8b06153182d4" +git-tree-sha1 = "4e7f6506af8fab111202d306abc5aab5c8d1d8ac" repo-rev = "main" repo-url = "https://github.com/JuliaPackaging/HistoricalStdlibVersions.jl.git" uuid = "6df8b67a-e8a0-4029-b4b7-ac196fe72102" -version = "2.0.3" +version = "2.0.4" [[deps.InteractiveUtils]] deps = ["Markdown"] @@ -127,15 +133,15 @@ weakdeps = ["BinaryBuilderSources"] [[deps.JLLPrefixes]] deps = ["Artifacts", "Git", "HistoricalStdlibVersions", "Pkg", "Preferences", "SHA", "Scratch"] -git-tree-sha1 = "c199272df7cab0230eb6febc3003c87873599107" +git-tree-sha1 = "3b772c89f2fb0f2877e9149d3f529276ca8b33f2" uuid = "afc68a34-7891-4c5a-9da1-1c62935e7b0d" -version = "0.3.13" +version = "0.4.1" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.KeywordArgumentExtraction]] deps = ["ExprTools", "InteractiveUtils"] @@ -226,15 +232,15 @@ version = "0.4.4" [[deps.OpenSSH_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] -git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +git-tree-sha1 = "1f2f0911e1c02f28a390bb720f97f3349c4dcefb" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "10.0.1+0" +version = "10.0.2+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "87510f7292a2b21aeff97912b0898f9553cc5c2c" +git-tree-sha1 = "2ae7d4ddec2e13ad3bddf5c0796f7547cf682391" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.1+0" +version = "3.5.2+0" [[deps.OrderedCollections]] git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" @@ -265,9 +271,9 @@ version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/BinaryBuilderAuditor.jl/src/AuditResult.jl b/BinaryBuilderAuditor.jl/src/AuditResult.jl index 7605b0b..a723790 100644 --- a/BinaryBuilderAuditor.jl/src/AuditResult.jl +++ b/BinaryBuilderAuditor.jl/src/AuditResult.jl @@ -1,5 +1,5 @@ using JLLGenerator, StyledStrings -export PassResult, AuditResult, print_results, success +export PassResult, AuditResult, success struct PassResult # One of `:success`, `:warn` or `:fail` @@ -42,8 +42,8 @@ function status_style(status::Symbol) )[status] end -print_results(ar::AuditResult; kwargs...) = print_results(ar.pass_results; kwargs...) -function print_results(pass_results::Dict{String,Vector{PassResult}}; io::IO = stdout) +Base.show(io::IO, ar::AuditResult) = show(io, ar.pass_results) +function Base.show(io::IO, pass_results::Dict{String,Vector{PassResult}}) println(io, "Audit results:") for (name, results) in pass_results println(io, " -> $(name) ($(length(results)) entries)") diff --git a/BinaryBuilderAuditor.jl/src/AuditorToolchain.jl b/BinaryBuilderAuditor.jl/src/AuditorToolchain.jl index b4b70e1..eba66fe 100644 --- a/BinaryBuilderAuditor.jl/src/AuditorToolchain.jl +++ b/BinaryBuilderAuditor.jl/src/AuditorToolchain.jl @@ -23,7 +23,7 @@ struct AuditorToolchain <: AbstractToolchain platform; uuid=Base.UUID("1e42d1a4-ec21-5f39-ae07-c1fb720fbc4b"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/CCTools_jll.jl", ), # eventually, include a resolved version diff --git a/BinaryBuilderAuditor.jl/src/BinaryBuilderAuditor.jl b/BinaryBuilderAuditor.jl/src/BinaryBuilderAuditor.jl index a6047cb..4154923 100644 --- a/BinaryBuilderAuditor.jl/src/BinaryBuilderAuditor.jl +++ b/BinaryBuilderAuditor.jl/src/BinaryBuilderAuditor.jl @@ -57,7 +57,7 @@ function audit!(prefix::String, licenses_present(scan, pass_results) if verbose - print_results(pass_results) + show(pass_results) end # Cleanup temporary directories containing toolchains diff --git a/BinaryBuilderAuditor.jl/src/SystemLibraries.jl b/BinaryBuilderAuditor.jl/src/SystemLibraries.jl index 59653ce..bda8937 100644 --- a/BinaryBuilderAuditor.jl/src/SystemLibraries.jl +++ b/BinaryBuilderAuditor.jl/src/SystemLibraries.jl @@ -62,8 +62,8 @@ function is_system_library(soname::AbstractString, platform::AbstractPlatform) "libpmsample.dylib", "libsandbox.1.dylib", "libsystem.b.dylib", - # This is not built by clang or GCC, so we leave it as a system library "libc++.1.dylib", + "libstdc++.6.dylib", "libresolv.9.dylib", # Frameworks in the SDK "accelerate", @@ -181,6 +181,17 @@ function is_system_library(soname::AbstractString, platform::AbstractPlatform) "libexecinfo.so.1", "libkvm.so.7", "libutil.so.9", + "libc.so.7", + "libc.so.6", + "libthr.so.3", + "libm.so.5", + "libm.so.6", + + # compiler support libraries + "libc++.so.1", + "libcxxrt.so.1", + "libgcc_s.so.1", + "libstdc++.so.6", ] return lowercase(soname) ∈ sdk_libs else diff --git a/BinaryBuilderAuditor.jl/test/passes/DynamicLinkageTests.jl b/BinaryBuilderAuditor.jl/test/passes/DynamicLinkageTests.jl index ae9d0a4..8f4826d 100644 --- a/BinaryBuilderAuditor.jl/test/passes/DynamicLinkageTests.jl +++ b/BinaryBuilderAuditor.jl/test/passes/DynamicLinkageTests.jl @@ -20,7 +20,7 @@ function soname_flag(platform, soname) end # We're gonna make use of a C toolchain for lots of these tests -for target_platform in (Platform("x86_64", "linux"), Platform("aarch64", "macos")) +for target_platform in (Platform("x86_64", "linux"), Platform("aarch64", "macos"; os_version=v"20")) platform = CrossPlatform(BBHostPlatform() => target_platform) toolchain = CToolchain(platform; use_ccache=false) diff --git a/BinaryBuilderAuditor.jl/test/passes/LibrarySONAMETests.jl b/BinaryBuilderAuditor.jl/test/passes/LibrarySONAMETests.jl index c428513..75d2491 100644 --- a/BinaryBuilderAuditor.jl/test/passes/LibrarySONAMETests.jl +++ b/BinaryBuilderAuditor.jl/test/passes/LibrarySONAMETests.jl @@ -19,7 +19,7 @@ function make_libfoo(prefix, target) end @testset "ensure_sonames" begin - for target in (Platform("x86_64", "linux"), Platform("aarch64", "macos")) + for target in (Platform("x86_64", "linux"), Platform("aarch64", "macos"; os_version=v"20")) mktempdir() do prefix libfoo_path = make_libfoo(prefix, target) @@ -59,7 +59,7 @@ end ensure_sonames!(scan, pass_results) @test !success(pass_results) io = IOBuffer() - print_results(pass_results; io) + show(io, pass_results) @test occursin("Failed to set SONAME:", String(take!(io))) @test any(r.identifier == relpath(libfoo_path, prefix) && r.status == :fail for r in pass_results["ensure_sonames!"]) end diff --git a/BinaryBuilderAuditor.jl/test/runtests.jl b/BinaryBuilderAuditor.jl/test/runtests.jl index b3ba285..9457734 100644 --- a/BinaryBuilderAuditor.jl/test/runtests.jl +++ b/BinaryBuilderAuditor.jl/test/runtests.jl @@ -1,4 +1,4 @@ -using Test, BinaryBuilderAuditor, JLLGenerator, BinaryBuilderToolchains, TreeArchival +using Test, BinaryBuilderProducts, BinaryBuilderAuditor, JLLGenerator, BinaryBuilderToolchains, TreeArchival include("ScanningTests.jl") include("passes/RelativeSymlinkTests.jl") diff --git a/BinaryBuilderGitUtils.jl/Manifest.toml b/BinaryBuilderGitUtils.jl/Manifest.toml index 55876a4..023ac3f 100644 --- a/BinaryBuilderGitUtils.jl/Manifest.toml +++ b/BinaryBuilderGitUtils.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "5065db5cbc116447b19556c4543ba261ca11385f" @@ -15,27 +15,33 @@ version = "1.11.0" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" +git-tree-sha1 = "7bb1361afdb33c7f2b085aa49ea8fe1b0fb14e58" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.5+0" +version = "2.7.1+0" [[deps.Git]] -deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] -git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "cb151153e40ad40a6dbf984fcd767e1d266fcc9c" +git-tree-sha1 = "e2aef26f7d273f1e5b1daba56837c47b49b4388f" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.50.1+0" +version = "2.51.1+0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] @@ -70,15 +76,15 @@ version = "0.1.0" [[deps.OpenSSH_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] -git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +git-tree-sha1 = "1f2f0911e1c02f28a390bb720f97f3349c4dcefb" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "10.0.1+0" +version = "10.0.2+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "87510f7292a2b21aeff97912b0898f9553cc5c2c" +git-tree-sha1 = "2ae7d4ddec2e13ad3bddf5c0796f7547cf682391" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.1+0" +version = "3.5.2+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] @@ -87,9 +93,9 @@ version = "10.42.0+1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/BinaryBuilderPlatformExtensions.jl/Manifest.toml b/BinaryBuilderPlatformExtensions.jl/Manifest.toml index 91bf23f..8c94a24 100644 --- a/BinaryBuilderPlatformExtensions.jl/Manifest.toml +++ b/BinaryBuilderPlatformExtensions.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "02c91ea86434b619a90551de48dcdc5edac5866c" @@ -22,9 +22,9 @@ version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/BinaryBuilderPlatformExtensions.jl/src/Microarchitectures.jl b/BinaryBuilderPlatformExtensions.jl/src/Microarchitectures.jl index 1e0c6d0..5621db8 100644 --- a/BinaryBuilderPlatformExtensions.jl/src/Microarchitectures.jl +++ b/BinaryBuilderPlatformExtensions.jl/src/Microarchitectures.jl @@ -89,6 +89,9 @@ const ARCHITECTURE_FLAGS = Dict( #"power9" => ["-mcpu=power9", "-mtune=power9"], # Eventually, we'll support power10, once we have compilers that support it. #"power10" => ["-mcpu=power10", "-mtune=power10"], + ), + "riscv64" => OrderedDict( + "riscv64" => ["-march=rv64gc"], ) ), "gcc" => Dict( diff --git a/BinaryBuilderPlatformExtensions.jl/src/PlatformProperties.jl b/BinaryBuilderPlatformExtensions.jl/src/PlatformProperties.jl index 81160ae..5a13327 100644 --- a/BinaryBuilderPlatformExtensions.jl/src/PlatformProperties.jl +++ b/BinaryBuilderPlatformExtensions.jl/src/PlatformProperties.jl @@ -1,4 +1,4 @@ -export macos_version, macos_kernel_version, freebsd_kernel_version, nbits, proc_family, exeext, dlext +export macos_version, macos_kernel_version, nbits, proc_family, exeext, dlext function macos_version(kernel_version::Integer) # See https://en.wikipedia.org/wiki/Darwin_(operating_system)#Release_history @@ -30,14 +30,18 @@ understand, return the closest valid number. """ function macos_kernel_version(v::VersionNumber) if v.major < 10 + # kernel version 12 (10.8) as our minimum return 12 elseif v.major == 10 + # 10.8 is our typical minimum if v.minor <= 8 return 12 else + # 19 (10.15) is the last `10.x` version return min(12 + (v.minor - 8), 19) end else + # Somewhat easier counting for 11.x and beyond return 20 + (v.major - 11) end end @@ -74,16 +78,6 @@ function macos_kernel_version(p::AbstractPlatform) return Int(os_version(p).major) end -function freebsd_kernel_version(p::AbstractPlatform) - if os(p) != "freebsd" - return nothing - end - if os_version(p) === nothing - return nothing - end - return Int(os_version(p).major) -end - """ nbits(p::AbstractPlatform) @@ -92,7 +86,7 @@ Return the number of bits in the architecture of the given Platform. Usually either 32 or 64. """ function nbits(p::AbstractPlatform) - if arch(p) ∈ ("x86_64", "aarch64", "powerpc64le") + if arch(p) ∈ ("x86_64", "aarch64", "powerpc64le", "riscv64") return 64 elseif arch(p) ∈ ("i686", "armv7l", "armv6l") return 32 @@ -118,6 +112,8 @@ function proc_family(p::AbstractPlatform) return "arm" elseif arch(p) == "powerpc64le" return "power" + elseif arch(p) == "riscv64" + return "riscv" else error("Unknown processor family for architecture of platform $(triplet(p))") end diff --git a/BinaryBuilderPlatformExtensions.jl/src/Utils.jl b/BinaryBuilderPlatformExtensions.jl/src/Utils.jl index e63848f..edca6fb 100644 --- a/BinaryBuilderPlatformExtensions.jl/src/Utils.jl +++ b/BinaryBuilderPlatformExtensions.jl/src/Utils.jl @@ -6,7 +6,7 @@ export gcc_platform, gcc_target_triplet Strip out any tags that are not the basic annotations like `libc` and `call_abi`. """ function gcc_platform(p::Platform) - keeps = ("libc", "call_abi") + keeps = ("libc", "call_abi", "os_version") filtered_tags = Dict{Symbol,String}(Symbol(k) => v for (k, v) in tags(p) if k ∈ keeps) return Platform(arch(p)::String, os(p)::String; filtered_tags...) end diff --git a/BinaryBuilderPlatformExtensions.jl/test/runtests.jl b/BinaryBuilderPlatformExtensions.jl/test/runtests.jl index aeabe37..805c1da 100644 --- a/BinaryBuilderPlatformExtensions.jl/test/runtests.jl +++ b/BinaryBuilderPlatformExtensions.jl/test/runtests.jl @@ -229,7 +229,7 @@ end end for p in [Platform("x86_64", "linux"), Platform("x86_64", "windows"), Platform("aarch64", "linux"), - Platform("powerpc64le", "linux"), Platform("x86_64", "macos")] + Platform("powerpc64le", "linux"), Platform("x86_64", "macos"), Platform("riscv64", "linux")] @test nbits(p) == 64 end @@ -245,6 +245,7 @@ end @test proc_family(p) == "arm" end @test proc_family(Platform("powerpc64le", "linux")) == "power" + @test proc_family(Platform("riscv64", "linux")) == "riscv" @test proc_family(CrossPlatform(Platform("aarch64", "linux") => Platform("i686", "linux"))) == "intel" @test proc_family(AnyPlatform()) == "any" end @@ -287,7 +288,7 @@ end end # Get all architectures and all microarchitectures for the different architectures - @test sort(get_all_arch_names()) == ["aarch64", "armv6l", "armv7l", "i686", "powerpc64le", "x86_64"] + @test sort(get_all_arch_names()) == ["aarch64", "armv6l", "armv7l", "i686", "powerpc64le", "riscv64", "x86_64"] @test sort(get_all_march_names("x86_64")) == ["avx", "avx2", "avx512", "x86_64"] @test sort(get_all_march_names("armv7l")) == ["armv7l", "neonvfpv4"] end diff --git a/BinaryBuilderProducts.jl/Manifest.toml b/BinaryBuilderProducts.jl/Manifest.toml index a08f080..14b1e76 100644 --- a/BinaryBuilderProducts.jl/Manifest.toml +++ b/BinaryBuilderProducts.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "c2f035581add52d2ffa9427da668fde9005dd078" @@ -35,16 +35,16 @@ uuid = "213f2928-4d72-6f46-7461-4c705f634367" version = "0.1.0" [[deps.BinaryBuilderSources]] -deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TimerOutputs", "TreeArchival"] +deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TOML", "TimerOutputs", "TreeArchival"] path = "../BinaryBuilderSources.jl" uuid = "316c416d-4527-6863-7465-466137743047" version = "0.1.0" [[deps.Compat]] deps = ["TOML", "UUIDs"] -git-tree-sha1 = "3a3dfb30697e96a440e4149c8c51bf32f818c0f3" +git-tree-sha1 = "0037835448781bb46feb39866934e243886d756a" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.17.0" +version = "4.18.0" [deps.Compat.extensions] CompatLinearAlgebraExt = "LinearAlgebra" @@ -65,9 +65,9 @@ version = "1.6.0" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" +git-tree-sha1 = "7bb1361afdb33c7f2b085aa49ea8fe1b0fb14e58" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.5+0" +version = "2.7.1+0" [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" @@ -79,22 +79,28 @@ uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" [[deps.Git]] -deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] -git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "cb151153e40ad40a6dbf984fcd767e1d266fcc9c" +git-tree-sha1 = "e2aef26f7d273f1e5b1daba56837c47b49b4388f" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.50.1+0" +version = "2.51.1+0" [[deps.HistoricalStdlibVersions]] -deps = ["Pkg"] -git-tree-sha1 = "7e72d0ce251105cebeab1e156977af2346cbe558" +deps = ["Pkg", "PrecompileTools"] +git-tree-sha1 = "0eab8a68b8162a4c6a1f10bcdb54ec23d647412f" uuid = "6df8b67a-e8a0-4029-b4b7-ac196fe72102" -version = "2.0.2" +version = "2.0.4" [[deps.InteractiveUtils]] deps = ["Markdown"] @@ -113,15 +119,15 @@ weakdeps = ["BinaryBuilderSources"] [[deps.JLLPrefixes]] deps = ["Artifacts", "Git", "HistoricalStdlibVersions", "Pkg", "Preferences", "SHA", "Scratch"] -git-tree-sha1 = "c199272df7cab0230eb6febc3003c87873599107" +git-tree-sha1 = "3b772c89f2fb0f2877e9149d3f529276ca8b33f2" uuid = "afc68a34-7891-4c5a-9da1-1c62935e7b0d" -version = "0.3.13" +version = "0.4.1" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.KeywordArgumentExtraction]] deps = ["ExprTools", "InteractiveUtils"] @@ -200,15 +206,15 @@ version = "1.2.0" [[deps.OpenSSH_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] -git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +git-tree-sha1 = "1f2f0911e1c02f28a390bb720f97f3349c4dcefb" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "10.0.1+0" +version = "10.0.2+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "87510f7292a2b21aeff97912b0898f9553cc5c2c" +git-tree-sha1 = "2ae7d4ddec2e13ad3bddf5c0796f7547cf682391" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.1+0" +version = "3.5.2+0" [[deps.OrderedCollections]] git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" @@ -239,9 +245,9 @@ version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/BinaryBuilderSources.jl/Manifest.toml b/BinaryBuilderSources.jl/Manifest.toml index dc23d8e..38481de 100644 --- a/BinaryBuilderSources.jl/Manifest.toml +++ b/BinaryBuilderSources.jl/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" -project_hash = "d96164522edd454f48a2bfbfedd7965dc8fc7aa5" +project_hash = "87574461b525f80d9a9aefdc10f9a9a6a8963ebb" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -40,9 +40,9 @@ version = "1.6.0" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" +git-tree-sha1 = "7bb1361afdb33c7f2b085aa49ea8fe1b0fb14e58" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.5+0" +version = "2.7.1+0" [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" @@ -54,34 +54,40 @@ uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" [[deps.Git]] -deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] -git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "cb151153e40ad40a6dbf984fcd767e1d266fcc9c" +git-tree-sha1 = "e2aef26f7d273f1e5b1daba56837c47b49b4388f" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.50.1+0" +version = "2.51.1+0" [[deps.HistoricalStdlibVersions]] -deps = ["Pkg"] -git-tree-sha1 = "7e72d0ce251105cebeab1e156977af2346cbe558" +deps = ["Pkg", "PrecompileTools"] +git-tree-sha1 = "0eab8a68b8162a4c6a1f10bcdb54ec23d647412f" uuid = "6df8b67a-e8a0-4029-b4b7-ac196fe72102" -version = "2.0.2" +version = "2.0.4" [[deps.JLLPrefixes]] deps = ["Artifacts", "Git", "HistoricalStdlibVersions", "Pkg", "Preferences", "SHA", "Scratch"] -git-tree-sha1 = "c199272df7cab0230eb6febc3003c87873599107" +git-tree-sha1 = "3b772c89f2fb0f2877e9149d3f529276ca8b33f2" uuid = "afc68a34-7891-4c5a-9da1-1c62935e7b0d" -version = "0.3.13" +version = "0.4.1" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] @@ -148,15 +154,15 @@ version = "1.2.0" [[deps.OpenSSH_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] -git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +git-tree-sha1 = "1f2f0911e1c02f28a390bb720f97f3349c4dcefb" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "10.0.1+0" +version = "10.0.2+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "87510f7292a2b21aeff97912b0898f9553cc5c2c" +git-tree-sha1 = "2ae7d4ddec2e13ad3bddf5c0796f7547cf682391" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.1+0" +version = "3.5.2+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] @@ -174,11 +180,17 @@ version = "1.11.0" [deps.Pkg.weakdeps] REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/BinaryBuilderSources.jl/Project.toml b/BinaryBuilderSources.jl/Project.toml index e8a8c0f..f407c5e 100644 --- a/BinaryBuilderSources.jl/Project.toml +++ b/BinaryBuilderSources.jl/Project.toml @@ -22,7 +22,7 @@ TreeArchival = "216c6a2e-6c61-7669-6863-726165657274" BinaryBuilderGitUtils = "0.1.0" Dates = "1.11.0" Git = "1.3.1" -JLLPrefixes = "0.3.5" +JLLPrefixes = "0.4.1" MultiHashParsing = "0.1.0" Random = "1.11.0" Scratch = "1" diff --git a/BinaryBuilderSources.jl/src/JLLSource.jl b/BinaryBuilderSources.jl/src/JLLSource.jl index 66bc37a..3d6f15e 100644 --- a/BinaryBuilderSources.jl/src/JLLSource.jl +++ b/BinaryBuilderSources.jl/src/JLLSource.jl @@ -203,6 +203,16 @@ function prepare(jlls::Vector{JLLSource}; push!(jlls_by_prefix[jll.target], jll) end + # Load UUIDs contained within our project, which were previously built by us + built_uuids = Set{Base.UUID}() + project_path = Pkg.Types.projectfile_path(project_dir) + if isfile(project_path) + project_deps = get(TOML.parsefile(project_path), "deps", String[]) + for (_, uuid) in project_deps + push!(built_uuids, Base.UUID(uuid)) + end + end + # For each group of platforms and sharded by prefix, we are able to download as a group: # We can't do it all together because it's totally valid for us to try and download # two different versions of the same JLL to different prefixes. @@ -230,16 +240,44 @@ function prepare(jlls::Vector{JLLSource}; if ispath(cache_path) cache = TOML.parsefile(cache_path) + if !haskey(cache, "dep_uuids") + @debug("JLLSource cache lacks dep_uuids! Cache corrupt or out of date!") + clear_cache!() + continue + end + + cached_dep_uuids = Set{Base.UUID}() + for uuid in cache["dep_uuids"] + push!(cached_dep_uuids, Base.UUID(uuid)) + end + for jll in jlls_slice # This should be impossible since we address the cache by spec_hash, but let's be paranoid if !haskey(cache, string(jll.package.uuid)) - @error("JLLSource cache does not contain all UUIDs! This should be impossible!", name=jll.package.name, uuid=string(jll.package.uuid), cache_path) + @debug("JLLSource cache does not contain all UUIDs! This should be impossible!", name=jll.package.name, uuid=string(jll.package.uuid), cache_path) + clear_cache!() + break + end + + # Drop the package cache if has the wrong structure (this is how we gracefully deal with Elliot adding new fields) + package_cache = cache[string(jll.package.uuid)] + if !haskey(package_cache, "artifact_paths") || !haskey(package_cache, "tree_hash") + @debug("JLLSource cache lacks artifact paths or tree_hash! Cache corrupt?", name=jll.package.name, uuid=string(jll.package.uuid), cache_path) clear_cache!() break end - if isempty(jll.artifact_paths) - cached_paths = cache[string(jll.package.uuid)] + # Don't use the cache (but don't drop it) if any of the dep_uuids + # are in our universe (e.g. they were built by us previously) + overridden_deps = intersect(built_uuids, cached_dep_uuids) + if !isempty(overridden_deps) + @debug("Universe overrides some deps, rejecting cache!", name=jll.package.name, overridden_deps) + break + end + + if isempty(jll.artifact_paths) || jll.package.tree_hash === nothing + cached_paths = package_cache["artifact_paths"] + tree_hash = Base.SHA1(package_cache["tree_hash"]) if !isa(cached_paths, Vector{AbstractString}) @debug("cached_paths is not a String[], clearing", name=jll.package.name, uuid=string(jll.package.uuid), cache_path) @@ -248,19 +286,22 @@ function prepare(jlls::Vector{JLLSource}; end # Only use the cached paths if they actually exist on-disk (e.g. they haven't been Pkg.gc()'ed) - if all(isdir.(cached_paths)) + if all(isdir.(cached_paths)) && isdir(Pkg.Operations.find_installed(jll.package.name, jll.package.uuid, tree_hash)) + @debug("Loaded for $(jll.package.name)", cached_paths=basename.(cached_paths), platform, prefix) append!(jll.artifact_paths, cached_paths) + jll.package.tree_hash = tree_hash end end end end if any(isempty(jll.artifact_paths) for jll in jlls_slice) - @timeit to "collect_artifact_paths" begin - art_paths = collect_artifact_paths([jll.package for jll in jlls_slice]; platform, project_dir, pkg_depot=depot, verbose) + @timeit to "collect_artifact_metas" begin + slice_deps = [jll.package for jll in jlls_slice] + artifact_metas = collect_artifact_metas(slice_deps; platform, project_dir, pkg_depot=depot, verbose) + art_paths = collect_artifact_paths(artifact_metas, slice_deps) end for jll in jlls_slice - @debug("Prepared", jll, cache_key=jll_cache_paths[jll]) pkg = only([pkg for (pkg, _) in art_paths if pkg.uuid == jll.package.uuid]) # Update `jll.package` with things from `pkg` if pkg.version != Pkg.Types.VersionSpec() @@ -276,25 +317,103 @@ function prepare(jlls::Vector{JLLSource}; jll.package.repo = pkg.repo end append!(jll.artifact_paths, art_paths[pkg]) + @debug("Prepared", jll, pkg, cache_path, prefix, platform) end - # Write out the result - mkpath(dirname(cache_path)) - open(cache_path; write=true) do io - TOML.print(io, Dict(string(jll.package.uuid) => jll.artifact_paths for jll in jlls_slice)) + # Write out the result into a cache so we don't have to go through this resolution again. + # Only do that if all jll's had a `tree_hash`: + if all(jll.package.tree_hash !== nothing for jll in jlls_slice) + # Record here all recursive dependencies of the slice_deps, so that if we know one of + # these is changed (e.g. by being built as part of the current universe), then we + # invalidate this entire slice. + slice_uuids = union([m["dep_uuids"] for m in values(artifact_metas)]...) + + mkpath(dirname(cache_path)) + open(cache_path; write=true) do io + cache_entry = Dict( + # We record both the artifact path and the gitsha as those are the primary outputs + # of our resolution and artifact meta collection. + (string(jll.package.uuid) => Dict( + # Useful for debugging + "name" => jll.package.name, + "artifact_paths" => jll.artifact_paths, + "tree_hash" => string(jll.package.tree_hash), + ) + for jll in jlls_slice)..., + + # We also record a full set of UUIDs involved in this resolution, so that if we + # come back in the future but have rebuilt one of these as part of the build + # process, we can reject the cache (because we have overridden that UUID). + "dep_uuids" => string.(slice_uuids), + ) + TOML.print(io, cache_entry) + end end end end end +end - # Serialize all artifact paths into a hashed cache for each jll, so we don't have - # to do this again until our next update. - for jll in jlls - +verify(jll::JLLSource) = !isempty(jll.artifact_paths) + +function find_duplicate_artifact_paths(artifact_paths::Vector{String}) + seen_files = Dict{String,Any}() + for art_path in artifact_paths + for (root, dirs, files) in walkdir(art_path) + for file in files + rel_path = relpath(joinpath(root, file), art_path) + if !haskey(seen_files, rel_path) + seen_files[rel_path] = [] + end + push!(seen_files[rel_path], art_path) + end + end end + + # Only keep files that are listed in more than one artifact path + filter!(seen_files) do (file, art_paths) + length(art_paths) > 1 + end + return seen_files end -verify(jll::JLLSource) = !isempty(jll.artifact_paths) +function find_overlapping_jlls(jlls::Vector{JLLSource}) + paths = unique(vcat((jll.artifact_paths for jll in jlls)...)) + dup_files = find_duplicate_artifact_paths(paths) + + conflicting_sets = Set{Set{String}}() + for (file, art_paths) in dup_files + push!(conflicting_sets, Set{String}(art_paths)) + end + + function find_jlls_for_art_path(art_path) + jll_names = String[] + for jll in jlls + if art_path ∈ jll.artifact_paths + push!(jll_names, jll.package.name) + end + end + return jll_names + end + + for conflicting_set in conflicting_sets + @warn("Conflicting JLLs found:") + for jll in jlls + jll_arts = filter(p -> p ∈ jll.artifact_paths, collect(conflicting_set)) + if !isempty(jll_arts) + println("$(jll.package.name):") + for art in jll_arts + println(" - $(art)") + for (file, art_paths) in dup_files + if art ∈ art_paths + println(" - $(file)") + end + end + end + end + end + end +end """ deploy(jlls::Vector{JLLSource}, prefix::String) @@ -318,6 +437,9 @@ function deploy(jlls::Vector{JLLSource}, prefix::String) # Install each to their relative targets for (target, target_jlls) in jlls_by_prefix + # For debugging only + #find_overlapping_jlls(target_jlls) + install_path = joinpath(prefix, target) mkpath(install_path) paths = unique(vcat((jll.artifact_paths for jll in target_jlls)...)) diff --git a/BinaryBuilderSources.jl/test/runtests.jl b/BinaryBuilderSources.jl/test/runtests.jl index 6301bfd..03b4bb4 100644 --- a/BinaryBuilderSources.jl/test/runtests.jl +++ b/BinaryBuilderSources.jl/test/runtests.jl @@ -403,6 +403,12 @@ const binlib = Sys.iswindows() ? "bin" : "lib" @test_throws ArgumentError deduplicate_jlls([zstd_impossible_dep, zstd_specific_dep]) @test startswith(source(zstd_specific_dep), "Zstd_jll@v") @test startswith(source(zstd_any_dep), "Zstd_jll@v") + + # Test that disjoint versions in separate prefixes is okay: + @test length(deduplicate_jlls([ + retarget(zstd_specific_dep, "foo"), + retarget(zstd_impossible_dep, "bar"), + ])) == 2 end end end diff --git a/BinaryBuilderToolchains.jl/Manifest.toml b/BinaryBuilderToolchains.jl/Manifest.toml index efeb51f..cf10d72 100644 --- a/BinaryBuilderToolchains.jl/Manifest.toml +++ b/BinaryBuilderToolchains.jl/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" -project_hash = "3676fb91021d825e1bc8130dfea38704c137d7bc" +project_hash = "5063bebb0647ae880f9e333f45d5b6cf1617bd63" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -35,7 +35,7 @@ uuid = "213f2928-4d72-6f46-7461-4c705f634367" version = "0.1.0" [[deps.BinaryBuilderSources]] -deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TimerOutputs", "TreeArchival"] +deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TOML", "TimerOutputs", "TreeArchival"] path = "../BinaryBuilderSources.jl" uuid = "316c416d-4527-6863-7465-466137743047" version = "0.1.0" @@ -52,9 +52,9 @@ version = "1.6.0" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" +git-tree-sha1 = "7bb1361afdb33c7f2b085aa49ea8fe1b0fb14e58" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.5+0" +version = "2.7.1+0" [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" @@ -66,36 +66,42 @@ uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" [[deps.Git]] -deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] -git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "cb151153e40ad40a6dbf984fcd767e1d266fcc9c" +git-tree-sha1 = "e2aef26f7d273f1e5b1daba56837c47b49b4388f" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.50.1+0" +version = "2.51.1+0" [[deps.HistoricalStdlibVersions]] deps = ["Pkg", "PrecompileTools"] -git-tree-sha1 = "1c2bfcc3d0ad9628a0fc247a267d8b06153182d4" +git-tree-sha1 = "4e7f6506af8fab111202d306abc5aab5c8d1d8ac" repo-rev = "main" repo-url = "https://github.com/JuliaPackaging/HistoricalStdlibVersions.jl.git" uuid = "6df8b67a-e8a0-4029-b4b7-ac196fe72102" -version = "2.0.3" +version = "2.0.4" [[deps.JLLPrefixes]] deps = ["Artifacts", "Git", "HistoricalStdlibVersions", "Pkg", "Preferences", "SHA", "Scratch"] -git-tree-sha1 = "c199272df7cab0230eb6febc3003c87873599107" +git-tree-sha1 = "3b772c89f2fb0f2877e9149d3f529276ca8b33f2" uuid = "afc68a34-7891-4c5a-9da1-1c62935e7b0d" -version = "0.3.13" +version = "0.4.1" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.LazyJLLWrappers]] deps = ["Artifacts", "Libdl", "Pkg", "Preferences", "TOML"] @@ -168,15 +174,15 @@ version = "1.2.0" [[deps.OpenSSH_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] -git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +git-tree-sha1 = "1f2f0911e1c02f28a390bb720f97f3349c4dcefb" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "10.0.1+0" +version = "10.0.2+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "87510f7292a2b21aeff97912b0898f9553cc5c2c" +git-tree-sha1 = "2ae7d4ddec2e13ad3bddf5c0796f7547cf682391" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.1+0" +version = "3.5.2+0" [[deps.OrderedCollections]] git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" @@ -207,9 +213,9 @@ version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/BinaryBuilderToolchains.jl/Project.toml b/BinaryBuilderToolchains.jl/Project.toml index cb57b96..5de7ef0 100644 --- a/BinaryBuilderToolchains.jl/Project.toml +++ b/BinaryBuilderToolchains.jl/Project.toml @@ -22,7 +22,7 @@ TreeArchival = "216c6a2e-6c61-7669-6863-726165657274" BinaryBuilderPlatformExtensions = "0.1.0" BinaryBuilderSources = "0.1.0" HistoricalStdlibVersions = "2.0.1" -JLLPrefixes = "0.3.5" +JLLPrefixes = "0.3.8, 0.4" PrecompileTools = "1.2.1" Reexport = "1.2.2" Scratch = "1.2" diff --git a/BinaryBuilderToolchains.jl/src/BinaryBuilderToolchains.jl b/BinaryBuilderToolchains.jl/src/BinaryBuilderToolchains.jl index 117ba64..d2ebba3 100644 --- a/BinaryBuilderToolchains.jl/src/BinaryBuilderToolchains.jl +++ b/BinaryBuilderToolchains.jl/src/BinaryBuilderToolchains.jl @@ -54,6 +54,7 @@ end include("WrapperUtils.jl") include("PathUtils.jl") +include("toolchains/BinutilsToolchain.jl") include("toolchains/CToolchain.jl") include("toolchains/HostToolsToolchain.jl") include("toolchains/CMakeToolchain.jl") @@ -65,7 +66,7 @@ include("InteractiveUtils.jl") BBHostPlatform(), Platform("x86_64", "windows"), Platform("armv7l", "linux"; libc=:musl), - Platform("aarch64", "macos"), + Platform("aarch64", "macos"; os_version=v"20"), ] platforms = [CrossPlatform(BBHostPlatform() => target) for target in targets] @compile_workload begin @@ -79,7 +80,7 @@ include("InteractiveUtils.jl") HostToolsToolchain(platform) ]) catch e - @warn("Failed to precompile support for platform", platform, e) + @error("Failed to precompile support for platform", platform, exception=(e, catch_backtrace())) end end end diff --git a/BinaryBuilderToolchains.jl/src/InteractiveUtils.jl b/BinaryBuilderToolchains.jl/src/InteractiveUtils.jl index 7dd1f80..f730aa0 100644 --- a/BinaryBuilderToolchains.jl/src/InteractiveUtils.jl +++ b/BinaryBuilderToolchains.jl/src/InteractiveUtils.jl @@ -63,7 +63,7 @@ function with_cxx_csls(f::Function, env) BBHostPlatform(); uuid=Base.UUID("3ba1ab17-c18f-5d2d-9d5a-db37f286de95"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/libstdcxx_jll.jl", ), ), @@ -72,7 +72,7 @@ function with_cxx_csls(f::Function, env) BBHostPlatform(); uuid=Base.UUID("899a7460-a157-599b-96c7-ccb58ef9beb5"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LLVMLibcxx_jll.jl", ), ), @@ -81,7 +81,7 @@ function with_cxx_csls(f::Function, env) BBHostPlatform(); uuid=Base.UUID("871c935c-5660-55ad-bb68-d1283357316b"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LLVMLibunwind_jll.jl", ), ), diff --git a/BinaryBuilderToolchains.jl/src/toolchains/BinutilsToolchain.jl b/BinaryBuilderToolchains.jl/src/toolchains/BinutilsToolchain.jl new file mode 100644 index 0000000..cbb2d64 --- /dev/null +++ b/BinaryBuilderToolchains.jl/src/toolchains/BinutilsToolchain.jl @@ -0,0 +1,587 @@ +export BinutilsToolchain + +""" + BinutilsToolchain + +This toolchain is typically used within `CToolchain`, the only reason it's +split out like this is so that during bootstrap for macOS and FreeBSD, we +can deploy Binutils while bootstrapping our compilers. This is taken care +of for us by crosstool-ng +""" +struct BinutilsToolchain <: AbstractToolchain + platform::CrossPlatform + + # See CToolchain for explanation of these fields + vendor::Symbol + deps::Vector{JLLSource} + wrapper_prefixes::Vector{String} + env_prefixes::Vector{String} + use_ccache::Bool + + # We must store the GCC version (if we have one) so that we can + # find our LTO plugins. + gcc_version::Union{Nothing,VersionNumber} + cache_key::String + + function BinutilsToolchain(platform::CrossPlatform, + vendor::Symbol; + wrapper_prefixes = ["\${triplet}-", ""], + env_prefixes = [""], + use_ccache = true, + gcc_version = nothing) + cache_key = string( + triplet(platform), + vendor, + env_prefixes..., + wrapper_prefixes..., + use_ccache ? "true" : "false", + gcc_version, + ) + cache_key = string( + "BinutilsToolchain-", + bytes2hex(sha1(cache_key)) + ) + return new( + platform, + vendor, + binutils_jll_source_selection(vendor, platform), + wrapper_prefixes, + env_prefixes, + use_ccache, + gcc_version, + cache_key, + ) + end +end + +function Base.show(io::IO, toolchain::BinutilsToolchain) + println(io, "BinutilsToolchain ($(toolchain.platform))") + for dep in toolchain.deps + println(io, " - $(dep.package.name[1:end-4]) v$(dep.package.version)") + end +end + + +function binutils_jll_source_selection(vendor, platform) + deps = JLLSource[] + # If this is a crosstool-ng based "gcc bootstrap" toolchain, binutils is already included. + if os(platform.target) ∈ ("linux", "windows") && vendor ∈ (:gcc_bootstrap,) + return deps + end + simple_vendor = get_simple_vendor(vendor) + + if os(platform.target) == "macos" + append!(deps, [ + JLLSource( + "CCTools_jll", + platform; + uuid=Base.UUID("1e42d1a4-ec21-5f39-ae07-c1fb720fbc4b"), + repo=Pkg.Types.GitRepo( + rev="bb2/GCCBootstrap-x86_64-linux-gnu", + source="https://github.com/staticfloat/CCTools_jll.jl", + ), + # eventually, include a resolved version + version=v"986.0.0", + target=simple_vendor, + ), + # JLLSource( + # "libtapi_jll", + # platform.host; + # uuid=Base.UUID("defda0c2-6d1f-5f19-8ead-78afca958a10"), + # repo=Pkg.Types.GitRepo( + # rev="bb2/GCCBootstrap-x86_64-linux-gnu", + # source="https://github.com/staticfloat/libtapi_jll.jl", + # ), + # # eventually, include a resolved version + # version=v"1300.6.0", + # target=simple_vendor, + # ), + JLLSource("ldid_jll", platform.host), + ]) + + if simple_vendor != "clang" + append!(deps, [ + JLLSource( + "LLVMBootstrap_Clang_jll", + platform; + uuid=Base.UUID("b81fd3a9-9257-59d0-818a-b16b9f1e1eb9"), + repo=Pkg.Types.GitRepo( + rev="bb2/GCCBootstrap-x86_64-linux-gnu", + source="https://github.com/staticfloat/LLVMBootstrap_Clang_jll.jl" + ), + version=v"17.0.0", + target=get_simple_vendor(vendor), + ), + JLLSource( + "LLVMBootstrap_libLLVM_jll", + platform; + uuid=Base.UUID("de72bca2-3cdf-50cb-9084-6e985cd8d9f3"), + repo=Pkg.Types.GitRepo( + rev="bb2/GCCBootstrap-x86_64-linux-gnu", + source="https://github.com/staticfloat/LLVMBootstrap_libLLVM_jll.jl" + ), + version=v"17.0.0", + target=get_simple_vendor(vendor), + ), + ]) + end + else + push!(deps, JLLSource( + "Binutils_jll", + platform; + repo=Pkg.Types.GitRepo( + rev="bb2/GCCBootstrap-x86_64-linux-gnu", + source="https://github.com/staticfloat/Binutils_jll.jl", + ), + # eventually, include a resolved version + version=v"2.41.0", + target=simple_vendor, + )) + end + return deps +end + +cache_key(toolchain::BinutilsToolchain) = toolchain.cache_key + +function add_ccache_preamble(io, toolchain) + if toolchain.use_ccache + println(io, """ + # If `ccache` is available, use it! + if which ccache >/dev/null; then + PROG=( ccache "compiler_check=string:$(cache_key(toolchain))" "\${PROG[@]}" ) + fi + """) + end +end + +function binutils_wrappers(toolchain::BinutilsToolchain, dir::String) + p = toolchain.platform.target + toolchain_prefix = "\$(dirname \"\${WRAPPER_DIR}\")/$(get_simple_vendor(toolchain.vendor))" + gcc_triplet = toolchain.vendor == :gcc_bootstrap ? gcc_target_triplet(p) : triplet(gcc_platform(p)) + + # These tools don't need anything fancy; just `compiler_wrapper()` + simple_tools = String[] + @warn("TODO: Verify that `as` does not need adjusted MACOSX_DEPLOYMENT_TARGET", maxlog=1) + @warn("TODO: Add in `ld.64` and `ld.target-triplet` again", maxlog=1) + # Apple has some extra simple tools + if Sys.isapple(p) + append!(simple_tools, [ + "install_name_tool", + "lipo", + "otool", + ]) + else + # Everyone except for `macOS` has a `readelf` command. + append!(simple_tools, [ + "readelf", + ]) + end + + # Windows has some extra simple tools + if Sys.iswindows(p) + append!(simple_tools, [ + "windres", + "winmc", + ]) + end + + function _ld_wrapper(io) + if Sys.iswindows(p) + _warn_nondeterministic_definition(io, "uses the '--insert-timestamps' flag which embeds timestamps") + + # Warn if someone has asked for timestamps + flagmatch(io, [flag"--insert-timestamps"]) do io + println(io, "warn_nondeterministic") + end + + # Default to not using timestamps + flagmatch(io, [!flag"--insert-timestamps", !flag"--no-insert-timestamps"]) do io + append_flags(io, :PRE, "--no-insert-timestamp") + end + end + + # If `ccache` is allowed, sneak `ccache` in as the first argument to `PROG` + add_ccache_preamble(io, toolchain) + end + + # Many of our tools have nondeterministic + function _warn_nondeterministic_definition(io, nondeterminism_description="uses flags that cause nondeterministic output!") + println(io, """ + NONDETERMINISTIC=0 + warn_nondeterministic() { + if [[ "\${NONDETERMINISTIC}" != "1" ]]; then + echo "Non-reproducibility alert: This '\$0' invocation $(nondeterminism_description)." >&2 + echo "\$0 flags: \${ARGS[@]}" >&2 + echo "Continuing build, but please repent." >&2 + fi + NONDETERMINISTIC=1 + } + """) + end + + # Some tools can load an LTO plugin. We make sure this happens by passing in + # `--plugin` automatically if the plugin exists. This is not necessary on newer + # binutils builds which properly install a symlink in `lib/bfd_plugins/`, but + # doesn't hurt anything, so we just always do it. + function lto_plugin_args(io::IO) + if !isa(toolchain.gcc_version, VersionNumber) + return + end + + # We have the version glob here because our patch version may not actually + # correspond to the true patch version. It would be nice to inspect the + # JLL.toml for the GCC build and determine the true `src_version here, + # but that's an incredibly low-priority TODO. + majmin = string(toolchain.gcc_version.major, ".", toolchain.gcc_version.minor) + plugin_path = "`compgen -G \"$(toolchain_prefix)/libexec/gcc/$(gcc_triplet)/$(majmin)*/liblto_plugin.so\"`" + bash_if_statement(io, "-f $(plugin_path)") do io + append_flags(io, :POST, "--plugin=$(plugin_path)") + end + end + + # `ar` and `ranlib` have special treatment due to determinism requirements. + # Additionally, we use the `llvm-` prefixed tools on macOS. + function _ar_wrapper(io) + # We need to detect the `-U` flag that is passed to `ar`. Unfortunately, + # `ar` accepts many forms of its arguments, and we want to catch all of them. + _warn_nondeterministic_definition(io, "uses the '-U' flag which embeds timestamps") + + # We'll start with the easy stuff; `-U` by itself, as any argument! + flagmatch(io, [flag"-U"]) do io + println(io, "warn_nondeterministic") + end + + # However, the more traditional way to use `ar` is to mash a bunch of + # single-letter flags together into the first argument. This can be + # preceeded by a dash, but doesn't have to be (sigh). + flagmatch(io, [flag"-?[^-]*U.*"r]; match_target="\${ARGS[0]}") do io + println(io, "warn_nondeterministic") + end + + # Figure out if we've already set `-D` + flagmatch(io, [flag"-D"]) do io + println(io, "DETERMINISTIC=1") + end + flagmatch(io, [flag"-?[^-]*D"r]; match_target="\${ARGS[0]}") do io + println(io, "DETERMINISTIC=1") + end + + # If we haven't already set `-U`, _and_ we haven't already set `-D`, then + # we'll try to set `-D`: + flagmatch(io, [!flag"--.*"r]; match_target="\${ARGS[0]}") do io + # If our first flag is not a double-dashed option, we will just + # slap `D` onto the end of it: + println(io, raw""" + if [[ "${NONDETERMINISTIC}" != "1" ]] && [[ "${DETERMINISTIC}" != "1" ]]; then + ARGS[0]="${ARGS[0]}D" + DETERMINISTIC="1" + fi + """) + end + + # Eliminate the `u` option, as it's incompatible with `D` and is just an optimization + println(io, raw""" + if [[ "${DETERMINISTIC}" == "1" ]]; then + for ((i=0; i<"${#ARGS[@]}"; ++i)); do + if [[ "${ARGS[i]}" == "-u" ]]; then + unset ARGS[$i] + fi + done + + # Also find examples like `ar -ruD` or `ar ruD` + if [[ " ${ARGS[0]} " == *'u'* ]]; then + ARGS[0]=$(echo "${ARGS[0]}" | tr -d u) + fi + fi + """) + + # If we've got a `liblto_plugin`, load it in: + lto_plugin_args(io) + end + + # Multiple tools (`ranlib`, `strip`) have a `-U` or `-D` option that switches them + # from nondeterministic mode to determinstic mode. We, of course, _always_ want + # deterministic mode, and so do some common option parsing here. `ar` is a special + # case due to handling the `-u` flag, which is why it has all that extra logic above. + function _simple_U_D_determinism(io) + _warn_nondeterministic_definition(io, "uses the '-U' flag which embeds UIDs and timestamps") + # Warn the user if they provide `-U` in their build script + flagmatch(io, [flag"-[^-]*U.*"r]) do io + println(io, "warn_nondeterministic") + end + + # If there's no `-U`, and we haven't already provided `-D`, insert it! + flagmatch(io, [!flag"-[^-]*U.*"r, !flag"-[^-]*D.*"r]) do io + append_flags(io, :PRE, "-D") + end + end + + function _ranlib_wrapper(io) + _simple_U_D_determinism(io) + + # ranlib can take in `--plugin` + lto_plugin_args(io) + end + + function _nm_wrapper(io) + # nm can take in `--plugin` + lto_plugin_args(io) + end + + function _strip_wrapper_pre(io) + _simple_U_D_determinism(io) + + # On non-apple platforms, there's nothing else to be done! + if !Sys.isapple(p) + return + end + + # Otherwise, we need to do some RATHER ONEROUS parsing. + # We need to identify every file touched by `strip` and then + # re-sign them all using `ldid`. Because `strip` can take + # multiple output files, we end up doing a bunch of custom + # argument parsing here to identify all files that will be signed. + println(io, raw""" + FILES_TO_SIGN=() + # Parse arguments to figure out what files are being stripped, + # so we know what to re-sign after all is said and done. + get_files_to_sign() + { + for ARG_IDX in "${!ARGS[@]}"; do + # If `-o` is passed, that's the only file to sign, ignore + # everything else and finish off immediately. + if [[ "${ARGS[ARG_IDX]}" == "-o" ]] && (( ARG_IDX + 1 < ${#ARGS[@]} )); then + FILES_TO_SIGN=( "${ARGS[ARG_IDX+1]}" ) + return + elif [[ "${ARGS[ARG_IDX]}" == "-o"* ]]; then + filename="${ARGS[ARG_IDX}]}" + FILES_TO_SIGN=( "${filename%%-o}" ) + return + fi + + # Otherwise, we collect arguments we don't know what to do with, + # assuming they are files we should be signing. + if [[ "${ARGS[ARG_IDX]}" != -* ]]; then + FILES_TO_SIGN+=( "${ARGS[ARG_IDX]}" ) + fi + done + } + """) + + # Tell `strip` not to complain about us invalidating a code signature, since + # we're gonna fix it up with `ldid` immediately afterward. + append_flags(io, :PRE, ["-no_code_signature_warning"]) + end + + function _strip_wrapper_post(io) + # On non-apple platforms, we don't need to do anything + if !Sys.isapple(p) + return + end + + println(io, raw""" + # Re-sign all files listed in `FILES_TO_SIGN` + for file in "${FILES_TO_SIGN[@]}"; do + ldid -S "${file}" + done + """) + end + + function _as_wrapper(io) + if Sys.isapple(p) + # Warn if someone has asked for timestamps + flagmatch(io, [!flag"-arch"]) do io + # macOS likes to use `arm64`, not `aarch64`: + arch_str = arch(p) == "aarch64" ? "arm64" : arch(p) + append_flags(io, :PRE, ["-arch", arch_str]) + end + + # Tell the `as` executable how to find our clang. We use a special name here + # so that our wrapper for this doesn't conflict with an actual clang. + println(io, "export CCTOOLS_CLANG_AS_EXECUTABLE='$(gcc_triplet)-clang-as'") + end + + # If `ccache` is allowed, sneak `ccache` in as the first argument to `PROG` + add_ccache_preamble(io, toolchain) + end + + # Simple `clang` wrapper for being used as `clang-as` + function _clang_as_wrapper(io) + append_flags(io, :PRE, [ + # Set the `target` for `clang` so it generates the right kind of code + "--target=$(gcc_triplet)", + ]) + + # If `ccache` is allowed, sneak `ccache` in as the first argument to `PROG` + add_ccache_preamble(io, toolchain) + end + + # For all simple tools, create the target-specific name, and the basename if we're the default toolchain + for tool in simple_tools + make_tool_wrappers(toolchain, dir, tool, "$(gcc_triplet)-$(tool)"; toolchain_prefix) + end + + # `ld` is a simple tool, except that it can be wrapped with `ccache`: + make_tool_wrappers(toolchain, dir, "ld", "$(gcc_triplet)-ld"; wrapper=_ld_wrapper, toolchain_prefix) + + # `as` is a simple tool, except that on macOS it needs an `-arch` specified: + make_tool_wrappers(toolchain, dir, "as", "$(gcc_triplet)-as"; wrapper=_as_wrapper, toolchain_prefix) + + # Our `as` on macOS subs out to `$(gcc_triplet)-clang-as`, so we generate that here, + # which in turn subs out to `clang`. + if Sys.isapple(p) + make_tool_wrappers(toolchain, dir, "clang-as", "clang"; wrapper=_clang_as_wrapper, toolchain_prefix) + end + + # `nm` is a simple tool, except that it can take in `--plugin` for LTO + make_tool_wrappers(toolchain, dir, "nm", "$(gcc_triplet)-nm"; wrapper=_nm_wrapper, toolchain_prefix) + + # `strip` needs complicated option parsing if we're on macOS + make_tool_wrappers(toolchain, dir, "strip", "$(gcc_triplet)-strip"; wrapper=_strip_wrapper_pre, post_func=_strip_wrapper_post, toolchain_prefix) + + # c++filt uses `llvm-cxxfilt` on macOS, `c++filt` elsewhere + cxxfilt_name = Sys.isapple(p) ? "llvm-cxxfilt" : "$(gcc_triplet)-c++filt" + make_tool_wrappers(toolchain, dir, "c++filt", cxxfilt_name; toolchain_prefix) + + ar_name = Sys.isapple(p) ? "llvm-ar" : "$(gcc_triplet)-ar" + make_tool_wrappers(toolchain, dir, "ar", ar_name; wrapper=_ar_wrapper, toolchain_prefix) + + ranlib_name = Sys.isapple(p) ? "llvm-ranlib" : "$(gcc_triplet)-ranlib" + make_tool_wrappers(toolchain, dir, "ranlib", ranlib_name; wrapper=_ranlib_wrapper, toolchain_prefix) + + objcopy_name = Sys.isapple(p) ? "llvm-objcopy" : "$(gcc_triplet)-objcopy" + make_tool_wrappers(toolchain, dir, "objcopy", objcopy_name; toolchain_prefix) + + objdump_name = Sys.isapple(p) ? "llvm-objdump" : "$(gcc_triplet)-objdump" + make_tool_wrappers(toolchain, dir, "objdump", objdump_name; toolchain_prefix) + + if Sys.isapple(p) + # dsymutil is just called `dsymutil` + make_tool_wrappers(toolchain, dir, "dsymutil", "dsymutil"; toolchain_prefix) + end + + # dlltool needs some determinism fixes as well + if Sys.iswindows(p) + function _dlltool_wrapper(io) + append_flags(io, :PRE, ["--temp-prefix", "/tmp/dlltool-\${ARGS_HASH}"]) + end + make_tool_wrappers(toolchain, dir, "dlltool", "$(gcc_triplet)-dlltool"; wrapper=_dlltool_wrapper, toolchain_prefix) + end +end + +function toolchain_sources(toolchain::BinutilsToolchain) + sources = AbstractSource[] + + installing_jll(name) = get_jll(toolchain, name) !== nothing + push!(sources, CachedGeneratedSource(cache_key(toolchain); target="wrappers") do out_dir + binutils_wrappers(toolchain, out_dir) + end) + append!(sources, toolchain.deps) + + # We only ever use the latest binutils, no version selection + return sources +end + +function toolchain_env(toolchain::BinutilsToolchain, deployed_prefix::String) + env = Dict{String,String}() + + insert_PATH!(env, :PRE, [ + joinpath(deployed_prefix, "wrappers"), + joinpath(deployed_prefix, get_simple_vendor(toolchain), "bin") + ]) + + function set_envvars(envvar_prefix::String, tool_prefix::String) + env["$(envvar_prefix)AR"] = "$(tool_prefix)ar" + env["$(envvar_prefix)AS"] = "$(tool_prefix)as" + env["$(envvar_prefix)LD"] = "$(tool_prefix)ld" + env["$(envvar_prefix)NM"] = "$(tool_prefix)nm" + env["$(envvar_prefix)RANLIB"] = "$(tool_prefix)ranlib" + env["$(envvar_prefix)OBJCOPY"] = "$(tool_prefix)objcopy" + env["$(envvar_prefix)OBJDUMP"] = "$(tool_prefix)objdump" + env["$(envvar_prefix)STRIP"] = "$(tool_prefix)strip" + + if Sys.isapple(toolchain.platform.target) + env["$(envvar_prefix)DSYMUTIL"] = "$(tool_prefix)dsymutil" + env["$(envvar_prefix)LIPO"] = "$(tool_prefix)lipo" + end + + if !Sys.isapple(toolchain.platform.target) + env["$(envvar_prefix)READELF"] = "$(tool_prefix)readelf" + end + + if Sys.iswindows(toolchain.platform.target) + env["$(envvar_prefix)DLLTOOL"] = "$(tool_prefix)dlltool" + env["$(envvar_prefix)WINDRES"] = "$(tool_prefix)windres" + env["$(envvar_prefix)WINMC"] = "$(tool_prefix)winmc" + end + end + + wrapper_prefixes = replace.(toolchain.wrapper_prefixes, ("\${triplet}" => triplet(gcc_platform(toolchain.platform.target)),)) + wrapper_prefix = wrapper_prefixes[argmax(length.(wrapper_prefixes))] + for env_prefix in toolchain.env_prefixes + set_envvars(env_prefix, wrapper_prefix) + end + + if Sys.isapple(toolchain.platform.target) + # If toolchain platform already has an `os_version`, we need to obey that, otherwise we + # use the default deployment targets for the architecture being built: + function default_macos_kernel_version(arch) + if arch == "x86_64" + return 14 + elseif arch == "aarch64" + return 20 + else + throw(ArgumentError("Unknown macOS architecture '$(arch)'!")) + end + end + + kernel_version = something( + os_version(toolchain.platform.target), + default_macos_kernel_version(arch(toolchain.platform.target)) + ) + env["MACOSX_DEPLOYMENT_TARGET"] = macos_version(kernel_version) + end + + if Sys.isfreebsd(toolchain.platform.target) + function default_freebsd_sdk_version() + return v"14.1" + end + freebsd_version = something( + os_version(toolchain.platform.target), + default_freebsd_sdk_version(), + ) + env["FREEBSD_TARGET_SDK"] = "$(freebsd_version.major).$(freebsd_version.minor)" + end + + return env +end + +platform(toolchain::BinutilsToolchain) = toolchain.platform + +function supported_platforms(::Type{BinutilsToolchain}; experimental::Bool = false) + # Maybe make this inspect the supported platforms of GCC_jll or something like that? + return [ + Platform("x86_64", "linux"), + Platform("i686", "linux"), + Platform("aarch64", "linux"), + Platform("armv7l", "linux"), + Platform("ppc64le", "linux"), + + Platform("x86_64", "linux"; libc="musl"), + Platform("i686", "linux"; libc="musl"), + Platform("aarch64", "linux"; libc="musl"), + Platform("armv6l", "linux"; libc="musl"), + Platform("armv7l", "linux"; libc="musl"), + + Platform("x86_64", "windows"), + Platform("i686", "windows"), + + # These os version numbers come from the currently-default macOSSDK_jll and FreeBSDSysroot_jll versions + Platform("x86_64", "macos"; os_version=string(macos_kernel_version("11.1"))), + Platform("aarch64", "macos"; os_version=string(macos_kernel_version("11.1"))), + + Platform("x86_64", "freebsd"; os_version="14.1"), + Platform("aarch64", "freebsd"; os_version="14.1"), + ] +end + diff --git a/BinaryBuilderToolchains.jl/src/toolchains/CToolchain.jl b/BinaryBuilderToolchains.jl/src/toolchains/CToolchain.jl index 3367c95..72165e1 100644 --- a/BinaryBuilderToolchains.jl/src/toolchains/CToolchain.jl +++ b/BinaryBuilderToolchains.jl/src/toolchains/CToolchain.jl @@ -42,6 +42,12 @@ struct CToolchain <: AbstractToolchain extra_cflags::Vector{String} extra_ldflags::Vector{String} + # Cache key that we use to store our generated wrappers + cache_key::String + + # We internally create a BinutilsToolchain + binutils_toolchain::BinutilsToolchain + function CToolchain(platform::CrossPlatform; vendor = :auto, env_prefixes = [""], @@ -50,7 +56,6 @@ struct CToolchain <: AbstractToolchain use_ccache = true, gcc_version = VersionSpec("9"), llvm_version = VersionSpec("17"), - binutils_version = v"2.41.0", glibc_version = :oldest, compiler_runtime = :auto, cxx_runtime = :auto, @@ -111,7 +116,6 @@ struct CToolchain <: AbstractToolchain platform, gcc_version, llvm_version, - binutils_version, glibc_version, use_ccache, compiler_runtime, @@ -121,12 +125,49 @@ struct CToolchain <: AbstractToolchain # Concretize the JLLSource's `PackageSpec`'s version (and UUID) now: resolve_versions!(deps; julia_version=nothing) + gcc_version = nothing + for name in ("GCC", "GCCBootstrap") + jll = get_jll(deps, string(name, "_jll")) + if jll !== nothing + gcc_version = jll.package.version + break + end + end + + wrapper_prefixes = string.(wrapper_prefixes) + env_prefixes = string.(env_prefixes) + binutils_toolchain = BinutilsToolchain( + platform, + get_vendor(vendor, platform); + wrapper_prefixes, + env_prefixes, + use_ccache, + gcc_version, + ) + + cache_key = string( + triplet(platform), + lock_microarchitecture ? "true" : "false", + use_ccache ? "true" : "false", + compiler_runtime, + cxx_runtime, + vendor, + env_prefixes..., + wrapper_prefixes..., + extra_cflags..., + extra_ldflags..., + ) + cache_key = string( + "CToolchain-", + bytes2hex(sha1(cache_key)) + ) + return new( platform, vendor, deps, - string.(wrapper_prefixes), - string.(env_prefixes), + wrapper_prefixes, + env_prefixes, lock_microarchitecture, use_ccache, compiler_runtime, @@ -134,10 +175,13 @@ struct CToolchain <: AbstractToolchain linker, string.(extra_cflags), string.(extra_ldflags), + cache_key, + binutils_toolchain, ) end end +cache_key(toolchain::CToolchain) = toolchain.cache_key function get_vendor(vendor::Symbol, platform::AbstractPlatform) clang_default(p) = os(target_if_crossplatform(p)) ∈ ("macos", "freebsd") if vendor == :auto @@ -156,7 +200,7 @@ function get_vendor(vendor::Symbol, platform::AbstractPlatform) end return vendor end -get_vendor(ct::CToolchain) = get_vendor(ct.vendor, ct.platform) +get_vendor(toolchain) = get_vendor(toolchain.vendor, toolchain.platform) # This one only returns `gcc` or `clang`, no `bootstrap` distinction. function get_simple_vendor(vendor::Symbol) @@ -168,7 +212,7 @@ function get_simple_vendor(vendor::Symbol) return string(vendor) end end -get_simple_vendor(toolchain::CToolchain) = get_simple_vendor(get_vendor(toolchain)) +get_simple_vendor(toolchain) = get_simple_vendor(get_vendor(toolchain)) function auto_chooser(criteria, val, platform, choices) @@ -220,7 +264,6 @@ get_linker(ct::CToolchain) = get_linker(ct.linker, ct.platform) function jll_source_selection(vendor::Symbol, platform::CrossPlatform, gcc_version, llvm_version, - binutils_version, glibc_version, use_ccache, compiler_runtime, @@ -233,12 +276,12 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, # Manual version selection, drop this once these are registered! if v"2.17" == glibc_version glibc_repo = Pkg.Types.GitRepo( - rev="2f33ece6d34f813332ff277ffaea52b075f1af67", + rev="d3f2e355cd60300022f455f61aa608d963868e35", source="https://github.com/staticfloat/Glibc_jll.jl" ) elseif v"2.19" == glibc_version glibc_repo = Pkg.Types.GitRepo( - rev="a3d1c4ed6e676a47c4659aeecc8f396a2233757d", + rev="462af198d91ad3ad73815e77e64dd34e6866b1f3", source="https://github.com/staticfloat/Glibc_jll.jl" ) else @@ -260,7 +303,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, "Musl_jll", platform.target; repo=Pkg.Types.GitRepo( - rev="827bfab690e1cab77b4d48e1a250c8acd3547443", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Musl_jll.jl" ), target=sysroot_path, @@ -271,7 +314,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, platform.target; repo=Pkg.Types.GitRepo( source="https://github.com/staticfloat/macOSSDK_jll.jl", - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", ), version=v"11.1", target=sysroot_path, @@ -287,56 +330,26 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, "Mingw_jll", platform.target; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Mingw_jll.jl", ), target=sysroot_path, )] - else - error("Unknown libc for $(triplet(platform.target))") - end - - if os(platform.target) == "macos" - binutils_jlls = [ - JLLSource( - "CCTools_jll", - platform; - uuid=Base.UUID("1e42d1a4-ec21-5f39-ae07-c1fb720fbc4b"), - repo=Pkg.Types.GitRepo( - rev="main", - source="https://github.com/staticfloat/CCTools_jll.jl", - ), - # eventually, include a resolved version - version=v"986.0.0", - target=get_simple_vendor(vendor), - ), - JLLSource( - "libtapi_jll", - platform.host; - uuid=Base.UUID("defda0c2-6d1f-5f19-8ead-78afca958a10"), - repo=Pkg.Types.GitRepo( - rev="main", - source="https://github.com/staticfloat/libtapi_jll.jl", - ), - # eventually, include a resolved version - version=v"1300.6.0", - target=get_simple_vendor(vendor), - ), - JLLSource("ldid_jll", platform.host), - ] - else - binutils_jlls = [JLLSource( - "Binutils_jll", - platform; + elseif os(platform.target) == "freebsd" + freebsd_sdk_jll = JLLSource( + "FreeBSDSysroot_jll", + platform.target; + uuid=Base.UUID("671a10c0-f9bf-59ae-b52a-dff4adda89ae"), repo=Pkg.Types.GitRepo( - rev="main", - #rev="c5da93839bef6c88d3b7ecf4109eb9fe0c716a34", - source="https://github.com/staticfloat/Binutils_jll.jl", + source="https://github.com/staticfloat/FreeBSDSysroot_jll.jl", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", ), - # eventually, include a resolved version - version=v"2.41.0", - target=get_simple_vendor(vendor), - )] + version=v"14.1", + target=sysroot_path, + ) + libc_jlls = [freebsd_sdk_jll] + else + error("Unknown libc for $(triplet(platform.target))") end # Both GCC and Clang can use the GCC support libraries @@ -346,7 +359,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, platform.target; uuid=Base.UUID("465c4c53-7f13-5720-b733-07d6cbd50c3b"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/GCC_support_libraries_jll.jl", ), version=v"14.2.0", @@ -357,7 +370,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, platform.target; uuid=Base.UUID("7bc14925-bf4e-535d-80f2-90698dc22d13"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/GCC_crt_objects_jll.jl", ), version=v"14.2.0", @@ -371,7 +384,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, platform.target; uuid=Base.UUID("4e17d02c-6bf5-513e-be62-445f41c75a11"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LLVMCompilerRT_jll.jl", ), version=v"17.0.7", @@ -385,7 +398,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, platform.target; uuid=Base.UUID("3ba1ab17-c18f-5d2d-9d5a-db37f286de95"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/libstdcxx_jll.jl", ), version=v"14.2.0", @@ -398,7 +411,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, platform.target; uuid=Base.UUID("899a7460-a157-599b-96c7-ccb58ef9beb5"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LLVMLibcxx_jll.jl", ), version=v"17.0.1", @@ -409,7 +422,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, platform.target; uuid=Base.UUID("871c935c-5660-55ad-bb68-d1283357316b"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LLVMLibunwind_jll.jl", ), version=v"17.0.1", @@ -417,31 +430,28 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, ), ] - # These JLLs get installed not only when we're actually asking for - # `:clang_bootstrap`, but also when we're asking for `:gcc_bootstrap` - # on macOS, because `gcc` -> `as` -> `clang -intergrated-as`. :( clang_bootstrap_jlls = [ JLLSource( "LLVMBootstrap_Clang_jll", platform; uuid=Base.UUID("b81fd3a9-9257-59d0-818a-b16b9f1e1eb9"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LLVMBootstrap_Clang_jll.jl" ), version=v"17.0.0", - target="clang", + target=get_simple_vendor(vendor), ), JLLSource( "LLVMBootstrap_libLLVM_jll", platform; uuid=Base.UUID("de72bca2-3cdf-50cb-9084-6e985cd8d9f3"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LLVMBootstrap_libLLVM_jll.jl" ), version=v"17.0.0", - target="clang", + target=get_simple_vendor(vendor), ), ] @@ -451,44 +461,42 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, "Clang_jll", platform; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Clang_jll.jl", ), version=v"17.0.7", - target="clang", + target=get_simple_vendor(vendor), ), JLLSource( "libLLVM_jll", platform; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/libLLVM_jll.jl", ), version=v"17.0.7", - target="clang", + target=get_simple_vendor(vendor), ), ] # If we're asking for a bootstrap toolchain, give just that and nothing else, # which is why we `return` from within here. if vendor == :gcc_bootstrap - if Sys.isapple(platform.target) + if Sys.isapple(platform.target) || Sys.isbsd(platform.target) + # Darwin and FreeBSD use clang natively and aren't supported by + # crosstool-ng, so we manually cobble together a GCCBootstrap-like build here: append!(deps, [ JLLSource( - "GCCBootstrapMacOS_jll", + "GCCBootstrapManual_jll", platform; - uuid = Base.UUID("117daf6b-c727-5bed-b063-6a70e57c2a0e"), + uuid = Base.UUID("f60369a9-3442-5217-af1b-bf3786f10eba"), repo=Pkg.Types.GitRepo( rev="bb2/GCCBootstrap-$(triplet(platform.host))", - source="https://github.com/staticfloat/GCCBootstrapMacOS_jll.jl" + source="https://github.com/staticfloat/GCCBootstrapManual_jll.jl" ), version=v"14.2.0", target="gcc", ), - - # binutils actually needs `clang` to act as assembler, so we include it here. - clang_bootstrap_jlls..., - binutils_jlls..., libc_jlls..., ]) else @@ -516,7 +524,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, "LinuxKernelHeaders_jll", platform.target; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LinuxKernelHeaders_jll.jl" ), # LinuxKernelHeaders gets installed into `//usr` @@ -528,7 +536,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, append!(deps, libc_jlls) if vendor == :gcc - # Include GCC as well as Binutils + # Include GCC # These are compilers, so they take in the full cross platform. # TODO: Get `GCC_jll.jl` packaged so that I don't # have to pull down a special commit like this! @@ -538,7 +546,7 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, platform; uuid=Base.UUID("ec15993a-68c6-5861-8652-ef539d7ffb0b"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/GCC_jll.jl", ), # eventually, include a resolved version @@ -548,23 +556,15 @@ function jll_source_selection(vendor::Symbol, platform::CrossPlatform, ), gcc_support_libs..., libstdcxx_libs..., - binutils_jlls..., ]) - - # binutils actually needs `clang` to act as assembler, so we include it here. - if Sys.isapple(platform.target) - append!(deps, clang_bootstrap_jlls) - end elseif vendor == :clang || vendor == :clang_bootstrap if vendor == :clang append!(deps, [ clang_jlls..., - binutils_jlls..., ]) else append!(deps, [ clang_bootstrap_jlls..., - binutils_jlls..., ]) end comp_runtime = get_compiler_runtime(compiler_runtime, platform) @@ -603,14 +603,15 @@ function Base.show(io::IO, toolchain::CToolchain) end end -function get_jll(toolchain::CToolchain, name::String) - for jll in toolchain.deps +function get_jll(deps::Vector{JLLSource}, name::String) + for jll in deps if jll.package.name == name return jll end end return nothing end +get_jll(toolchain, name::String) = get_jll(toolchain.deps, name) function toolchain_sources(toolchain::CToolchain) sources = AbstractSource[] @@ -619,34 +620,16 @@ function toolchain_sources(toolchain::CToolchain) # Create a `GeneratedSource` that, at `prepare()` time, will JIT out # our compiler wrappers. We store it with a cache key that is sensitive # to basically all inputs, so that it can be cached. - cache_key = string( - triplet(toolchain.platform), - toolchain.lock_microarchitecture ? "true" : "false", - toolchain.use_ccache ? "true" : "false", - toolchain.compiler_runtime, - toolchain.cxx_runtime, - toolchain.vendor, - toolchain.env_prefixes..., - toolchain.wrapper_prefixes..., - toolchain.extra_cflags..., - toolchain.extra_ldflags..., - ) - cache_key = string( - "CToolchain-", - bytes2hex(sha1(cache_key)) - ) - push!(sources, CachedGeneratedSource(cache_key; target="wrappers") do out_dir - if installing_jll("GCC_jll") || installing_jll("GCCBootstrap_jll") || installing_jll("GCCBootstrapMacOS_jll") + push!(sources, CachedGeneratedSource(cache_key(toolchain); target="wrappers") do out_dir + if installing_jll("GCC_jll") || installing_jll("GCCBootstrap_jll") || installing_jll("GCCBootstrapManual_jll") gcc_wrappers(toolchain, out_dir) end if installing_jll("Clang_jll") || installing_jll("LLVMBootstrap_Clang_jll") clang_wrappers(toolchain, out_dir) end - if installing_jll("Binutils_jll") || installing_jll("CCTools_jll") || installing_jll("GCCBootstrap_jll") - binutils_wrappers(toolchain, out_dir) - end end) + append!(sources, toolchain_sources(toolchain.binutils_toolchain)) append!(sources, toolchain.deps) return sources end @@ -654,49 +637,15 @@ end function toolchain_env(toolchain::CToolchain, deployed_prefix::String) env = Dict{String,String}() - if get_vendor(toolchain) ∈ (:gcc, :gcc_bootstrap) - insert_PATH!(env, :PRE, [ - joinpath(deployed_prefix, "gcc", "bin"), - ]) - end - - if get_vendor(toolchain) ∈ (:clang, :clang_bootstrap) - insert_PATH!(env, :PRE, [ - joinpath(deployed_prefix, "clang", "bin"), - ]) - end - insert_PATH!(env, :PRE, [ joinpath(deployed_prefix, "wrappers"), + joinpath(deployed_prefix, get_simple_vendor(toolchain), "bin") ]) function set_envvars(envvar_prefix::String, tool_prefix::String) - env["$(envvar_prefix)AR"] = "$(tool_prefix)ar" - env["$(envvar_prefix)AS"] = "$(tool_prefix)as" env["$(envvar_prefix)CC"] = "$(tool_prefix)cc" env["$(envvar_prefix)CXX"] = "$(tool_prefix)c++" env["$(envvar_prefix)CPP"] = "$(tool_prefix)cpp" - env["$(envvar_prefix)LD"] = "$(tool_prefix)ld" - env["$(envvar_prefix)NM"] = "$(tool_prefix)nm" - env["$(envvar_prefix)RANLIB"] = "$(tool_prefix)ranlib" - env["$(envvar_prefix)OBJCOPY"] = "$(tool_prefix)objcopy" - env["$(envvar_prefix)OBJDUMP"] = "$(tool_prefix)objdump" - env["$(envvar_prefix)STRIP"] = "$(tool_prefix)strip" - - if Sys.isapple(toolchain.platform.target) - env["$(envvar_prefix)DSYMUTIL"] = "$(tool_prefix)dsymutil" - env["$(envvar_prefix)LIPO"] = "$(tool_prefix)lipo" - end - - if !Sys.isapple(toolchain.platform.target) - env["$(envvar_prefix)READELF"] = "$(tool_prefix)readelf" - end - - if Sys.iswindows(toolchain.platform.target) - env["$(envvar_prefix)DLLTOOL"] = "$(tool_prefix)dlltool" - env["$(envvar_prefix)WINDRES"] = "$(tool_prefix)windres" - env["$(envvar_prefix)WINMC"] = "$(tool_prefix)winmc" - end env["$(envvar_prefix)CC_TARGET"] = triplet(gcc_platform(toolchain.platform.target)) end @@ -708,26 +657,8 @@ function toolchain_env(toolchain::CToolchain, deployed_prefix::String) set_envvars(env_prefix, wrapper_prefix) end - if Sys.isapple(toolchain.platform.target) - # If toolchain platform already has an `os_version`, we need to obey that, otherwise we - # use the default deployment targets for the architecture being built: - function default_kernel_version(arch) - if arch == "x86_64" - return 14 - elseif arch == "aarch64" - return 20 - else - throw(ArgumentError("Unknown macOS architecture '$(arch)'!")) - end - end - - kernel_version = something( - os_version(toolchain.platform.target), - default_kernel_version(arch(toolchain.platform.target)) - ) - env["MACOSX_DEPLOYMENT_TARGET"] = macos_version(kernel_version) - end - + # Merge in Binutils environment variables + merge!(env, toolchain_env(toolchain.binutils_toolchain, deployed_prefix)) return env end @@ -894,36 +825,6 @@ function add_macos_flags(io, toolchain) end end -function add_ccache_preamble(io, toolchain) - if toolchain.use_ccache - # Build hash of compiler JLLs that we will feed to `ccache` to identify our - # specific compiler set, to key our `ccache` cache correctly. - compiler_treehash = bytes2hex(sha256( - join([dep.package.tree_hash for dep in toolchain.deps]) - )) - - println(io, """ - # If `ccache` is available, use it! - if which ccache >/dev/null; then - PROG=( ccache "compiler_check=string:$(compiler_treehash)" "\${PROG[@]}" ) - fi - """) - end -end - -function get_gcc_version(toolchain::CToolchain) - gcc_version = nothing - for name in ("GCC", "GCCBootstrap") - jll = get_jll(toolchain, string(name, "_jll")) - if jll !== nothing - gcc_version = jll.package.version - break - end - end - return gcc_version -end - - """ gcc_wrappers(toolchain::CToolchain, dir::String) @@ -936,7 +837,7 @@ wrapper names `cc`, `gcc`, `c++`, etc... function gcc_wrappers(toolchain::CToolchain, dir::String) p = toolchain.platform.target toolchain_prefix = "\$(dirname \"\${WRAPPER_DIR}\")/gcc" - gcc_version = something(get_gcc_version(toolchain), v"0") + gcc_version = something(toolchain.binutils_toolchain.gcc_version, v"0") function _gcc_wrapper(io) add_microarchitectural_flags(io, toolchain) @@ -1117,333 +1018,5 @@ function clang_wrappers(toolchain::CToolchain, dir::String) end end - -function binutils_wrappers(toolchain::CToolchain, dir::String) - p = toolchain.platform.target - toolchain_prefix = "\$(dirname \"\${WRAPPER_DIR}\")/$(get_simple_vendor(toolchain))" - gcc_triplet = get_vendor(toolchain) == :gcc_bootstrap ? gcc_target_triplet(p) : triplet(gcc_platform(p)) - gcc_version = get_gcc_version(toolchain) - - # These tools don't need anything fancy; just `compiler_wrapper()` - simple_tools = String[] - @warn("TODO: Verify that `as` does not need adjusted MACOSX_DEPLOYMENT_TARGET", maxlog=1) - @warn("TODO: Add in `ld.64` and `ld.target-triplet` again", maxlog=1) - # Apple has some extra simple tools - if Sys.isapple(p) - append!(simple_tools, [ - "install_name_tool", - "lipo", - "otool", - ]) - else - # Everyone except for `macOS` has a `readelf` command. - append!(simple_tools, [ - "readelf", - ]) - end - - # Windows has some extra simple tools - if Sys.iswindows(p) - append!(simple_tools, [ - "windres", - "winmc", - ]) - end - - function _ld_wrapper(io) - if Sys.iswindows(p) - _warn_nondeterministic_definition(io, "uses the '--insert-timestamps' flag which embeds timestamps") - - # Warn if someone has asked for timestamps - flagmatch(io, [flag"--insert-timestamps"]) do io - println(io, "warn_nondeterministic") - end - - # Default to not using timestamps - flagmatch(io, [!flag"--insert-timestamps", !flag"--no-insert-timestamps"]) do io - append_flags(io, :PRE, "--no-insert-timestamp") - end - end - - # If `ccache` is allowed, sneak `ccache` in as the first argument to `PROG` - add_ccache_preamble(io, toolchain) - end - - # Many of our tools have nondeterministic - function _warn_nondeterministic_definition(io, nondeterminism_description="uses flags that cause nondeterministic output!") - println(io, """ - NONDETERMINISTIC=0 - warn_nondeterministic() { - if [[ "\${NONDETERMINISTIC}" != "1" ]]; then - echo "Non-reproducibility alert: This '\$0' invocation $(nondeterminism_description)." >&2 - echo "\$0 flags: \${ARGS[@]}" >&2 - echo "Continuing build, but please repent." >&2 - fi - NONDETERMINISTIC=1 - } - """) - end - - # Some tools can load an LTO plugin. We make sure this happens by passing in - # `--plugin` automatically if the plugin exists. This is not necessary on newer - # binutils builds which properly install a symlink in `lib/bfd_plugins/`, but - # doesn't hurt anything, so we just always do it. - function lto_plugin_args(io::IO) - if !isa(gcc_version, VersionNumber) - return - end - - # We have the version glob here because our patch version may not actually - # correspond to the true patch version. It would be nice to inspect the - # JLL.toml for the GCC build and determine the true `src_version here, - # but that's an incredibly low-priority TODO. - plugin_path = "`compgen -G \"$(toolchain_prefix)/libexec/gcc/$(gcc_triplet)/$(gcc_version.major).$(gcc_version.minor)*/liblto_plugin.so\"`" - bash_if_statement(io, "-f $(plugin_path)") do io - append_flags(io, :POST, "--plugin=$(plugin_path)") - end - end - - # `ar` and `ranlib` have special treatment due to determinism requirements. - # Additionally, we use the `llvm-` prefixed tools on macOS. - function _ar_wrapper(io) - # We need to detect the `-U` flag that is passed to `ar`. Unfortunately, - # `ar` accepts many forms of its arguments, and we want to catch all of them. - _warn_nondeterministic_definition(io, "uses the '-U' flag which embeds timestamps") - - # We'll start with the easy stuff; `-U` by itself, as any argument! - flagmatch(io, [flag"-U"]) do io - println(io, "warn_nondeterministic") - end - - # However, the more traditional way to use `ar` is to mash a bunch of - # single-letter flags together into the first argument. This can be - # preceeded by a dash, but doesn't have to be (sigh). - flagmatch(io, [flag"-?[^-]*U.*"r]; match_target="\${ARGS[0]}") do io - println(io, "warn_nondeterministic") - end - - # Figure out if we've already set `-D` - flagmatch(io, [flag"-D"]) do io - println(io, "DETERMINISTIC=1") - end - flagmatch(io, [flag"-?[^-]*D"r]; match_target="\${ARGS[0]}") do io - println(io, "DETERMINISTIC=1") - end - - # If we haven't already set `-U`, _and_ we haven't already set `-D`, then - # we'll try to set `-D`: - flagmatch(io, [!flag"--.*"r]; match_target="\${ARGS[0]}") do io - # If our first flag is not a double-dashed option, we will just - # slap `D` onto the end of it: - println(io, raw""" - if [[ "${NONDETERMINISTIC}" != "1" ]] && [[ "${DETERMINISTIC}" != "1" ]]; then - ARGS[0]="${ARGS[0]}D" - DETERMINISTIC="1" - fi - """) - end - - # Eliminate the `u` option, as it's incompatible with `D` and is just an optimization - println(io, raw""" - if [[ "${DETERMINISTIC}" == "1" ]]; then - for ((i=0; i<"${#ARGS[@]}"; ++i)); do - if [[ "${ARGS[i]}" == "-u" ]]; then - unset ARGS[$i] - fi - done - - # Also find examles like `ar -ruD` or `ar ruD` - if [[ " ${ARGS[0]} " == *'u'* ]]; then - ARGS[0]=$(echo "${ARGS[0]}" | tr -d u) - fi - fi - """) - - # If we've got a `liblto_plugin`, load it in: - lto_plugin_args(io) - end - - # Multiple tools (`ranlib`, `strip`) have a `-U` or `-D` option that switches them - # from nondeterministic mode to determinstic mode. We, of course, _always_ want - # deterministic mode, and so do some common option parsing here. `ar` is a special - # case due to handling the `-u` flag, which is why it has all that extra logic above. - function _simple_U_D_determinism(io) - _warn_nondeterministic_definition(io, "uses the '-U' flag which embeds UIDs and timestamps") - # Warn the user if they provide `-U` in their build script - flagmatch(io, [flag"-[^-]*U.*"r]) do io - println(io, "warn_nondeterministic") - end - - # If there's no `-U`, and we haven't already provided `-D`, insert it! - flagmatch(io, [!flag"-[^-]*U.*"r, !flag"-[^-]*D.*"r]) do io - append_flags(io, :PRE, "-D") - end - end - - function _ranlib_wrapper(io) - _simple_U_D_determinism(io) - - # ranlib can take in `--plugin` - lto_plugin_args(io) - end - - function _nm_wrapper(io) - # nm can take in `--plugin` - lto_plugin_args(io) - end - - function _strip_wrapper_pre(io) - _simple_U_D_determinism(io) - - # On non-apple platforms, there's nothing else to be done! - if !Sys.isapple(p) - return - end - - # Otherwise, we need to do some RATHER ONEROUS parsing. - # We need to identify every file touched by `strip` and then - # re-sign them all using `ldid`. Because `strip` can take - # multiple output files, we end up doing a bunch of custom - # argument parsing here to identify all files that will be signed. - println(io, raw""" - FILES_TO_SIGN=() - # Parse arguments to figure out what files are being stripped, - # so we know what to re-sign after all is said and done. - get_files_to_sign() - { - for ARG_IDX in "${!ARGS[@]}"; do - # If `-o` is passed, that's the only file to sign, ignore - # everything else and finish off immediately. - if [[ "${ARGS[ARG_IDX]}" == "-o" ]] && (( ARG_IDX + 1 < ${#ARGS[@]} )); then - FILES_TO_SIGN=( "${ARGS[ARG_IDX+1]}" ) - return - elif [[ "${ARGS[ARG_IDX]}" == "-o"* ]]; then - filename="${ARGS[ARG_IDX}]}" - FILES_TO_SIGN=( "${filename%%-o}" ) - return - fi - - # Otherwise, we collect arguments we don't know what to do with, - # assuming they are files we should be signing. - if [[ "${ARGS[ARG_IDX]}" != -* ]]; then - FILES_TO_SIGN+=( "${ARGS[ARG_IDX]}" ) - fi - done - } - """) - - # Tell `strip` not to complain about us invalidating a code signature, since - # we're gonna fix it up with `ldid` immediately afterward. - append_flags(io, :PRE, ["-no_code_signature_warning"]) - end - - function _strip_wrapper_post(io) - # On non-apple platforms, we don't need to do anything - if !Sys.isapple(p) - return - end - - println(io, raw""" - # Re-sign all files listed in `FILES_TO_SIGN` - for file in "${FILES_TO_SIGN[@]}"; do - ldid -S "${file}" - done - """) - end - - function _as_wrapper(io) - if Sys.isapple(p) - # Warn if someone has asked for timestamps - flagmatch(io, [!flag"-arch"]) do io - # macOS likes to use `arm64`, not `aarch64`: - arch_str = arch(p) == "aarch64" ? "arm64" : arch(p) - append_flags(io, :PRE, ["-arch", arch_str]) - end - - # Tell the `as` executable how to find the corresponding clang - println(io, "export CCTOOLS_CLANG_AS_EXECUTABLE='$(gcc_triplet)-clang'") - end - - # If `ccache` is allowed, sneak `ccache` in as the first argument to `PROG` - add_ccache_preamble(io, toolchain) - end - - # For all simple tools, create the target-specific name, and the basename if we're the default toolchain - for tool in simple_tools - make_tool_wrappers(toolchain, dir, tool, "$(gcc_triplet)-$(tool)"; toolchain_prefix) - end - - # `ld` is a simple tool, except that it can be wrapped with `ccache`: - make_tool_wrappers(toolchain, dir, "ld", "$(gcc_triplet)-ld"; wrapper=_ld_wrapper, toolchain_prefix) - - # `as` is a simple tool, except that on macOS it needs an `-arch` specified: - make_tool_wrappers(toolchain, dir, "as", "$(gcc_triplet)-as"; wrapper=_as_wrapper, toolchain_prefix) - - # `nm` is a simple tool, except that it can take in `--plugin` for LTO - make_tool_wrappers(toolchain, dir, "nm", "$(gcc_triplet)-nm"; wrapper=_nm_wrapper, toolchain_prefix) - - # `strip` needs complicated option parsing if we're on macOS - make_tool_wrappers(toolchain, dir, "strip", "$(gcc_triplet)-strip"; wrapper=_strip_wrapper_pre, post_func=_strip_wrapper_post, toolchain_prefix) - - - # Used by llvm tools like `llvm-ar` if we're on macOS - if Sys.isapple(p) - llvm_toolchain_prefix = "\$(dirname \"\${WRAPPER_DIR}\")/clang" - else - llvm_toolchain_prefix = toolchain_prefix - end - - # c++filt uses `llvm-cxxfilt` on macOS, `c++filt` elsewhere - cxxfilt_name = Sys.isapple(p) ? "llvm-cxxfilt" : "$(gcc_triplet)-c++filt" - make_tool_wrappers(toolchain, dir, "c++filt", cxxfilt_name; toolchain_prefix=llvm_toolchain_prefix) - - ar_name = Sys.isapple(p) ? "llvm-ar" : "$(gcc_triplet)-ar" - make_tool_wrappers(toolchain, dir, "ar", ar_name; wrapper=_ar_wrapper, toolchain_prefix=llvm_toolchain_prefix) - - ranlib_name = Sys.isapple(p) ? "llvm-ranlib" : "$(gcc_triplet)-ranlib" - make_tool_wrappers(toolchain, dir, "ranlib", ranlib_name; wrapper=_ranlib_wrapper, toolchain_prefix=llvm_toolchain_prefix) - - objcopy_name = Sys.isapple(p) ? "llvm-objcopy" : "$(gcc_triplet)-objcopy" - make_tool_wrappers(toolchain, dir, "objcopy", objcopy_name; toolchain_prefix=llvm_toolchain_prefix) - - objdump_name = Sys.isapple(p) ? "llvm-objdump" : "$(gcc_triplet)-objdump" - make_tool_wrappers(toolchain, dir, "objdump", objdump_name; toolchain_prefix=llvm_toolchain_prefix) - - if Sys.isapple(p) - # dsymutil is just called `dsymutil` - make_tool_wrappers(toolchain, dir, "dsymutil", "dsymutil"; toolchain_prefix=llvm_toolchain_prefix) - end - - # dlltool needs some determinism fixes as well - if Sys.iswindows(p) - function _dlltool_wrapper(io) - append_flags(io, :PRE, ["--temp-prefix", "/tmp/dlltool-\${ARGS_HASH}"]) - end - make_tool_wrappers(toolchain, dir, "dlltool", "$(gcc_triplet)-dlltool"; wrapper=_dlltool_wrapper, toolchain_prefix) - end -end - - - -function supported_platforms(::Type{CToolchain}; experimental::Bool = false) - # Maybe make this inspect the supported platforms of GCC_jll or something like that? - return [ - Platform("x86_64", "linux"), - Platform("i686", "linux"), - Platform("aarch64", "linux"), - Platform("armv7l", "linux"), - Platform("ppc64le", "linux"), - - Platform("x86_64", "linux"; libc="musl"), - Platform("i686", "linux"; libc="musl"), - Platform("aarch64", "linux"; libc="musl"), - Platform("armv6l", "linux"; libc="musl"), - Platform("armv7l", "linux"; libc="musl"), - - Platform("x86_64", "windows"), - Platform("i686", "windows"), - - Platform("x86_64", "macos"), - Platform("aarch64", "macos"), - ] -end +# Sub off to BinutilsToolchain +supported_platforms(::Type{CToolchain}; experimental::Bool = false) = supported_platforms(BinutilsToolchain; experimental) diff --git a/BinaryBuilderToolchains.jl/src/toolchains/HostToolsToolchain.jl b/BinaryBuilderToolchains.jl/src/toolchains/HostToolsToolchain.jl index ac25605..3f80a66 100644 --- a/BinaryBuilderToolchains.jl/src/toolchains/HostToolsToolchain.jl +++ b/BinaryBuilderToolchains.jl/src/toolchains/HostToolsToolchain.jl @@ -145,15 +145,7 @@ function toolchain_sources(toolchain::HostToolsToolchain) end end) - registries = Pkg.Registry.reachable_registries(; depots=[BinaryBuilderSources.default_jll_source_depot()]) - cache_key = string( - "HostTools-", - bytes2hex(sha1(string( - triplet(toolchain.platform), - BinaryBuilderSources.jll_cache_name(toolchain.deps, registries), - ))), - ) - push!(sources, CachedGeneratedSource(cache_key; target="wrappers") do out_dir + push!(sources, CachedGeneratedSource("HostTools"; target="wrappers") do out_dir toolchain_prefix = "\$(dirname \"\${WRAPPER_DIR}\")" if any(jll.package.name == "Tar_jll" for jll in toolchain.deps) # Forcibly insert --no-same-owner into every tar invocation, diff --git a/BinaryBuilderToolchains.jl/test/CToolchainTests.jl b/BinaryBuilderToolchains.jl/test/CToolchainTests.jl index 184f344..c75ffab 100644 --- a/BinaryBuilderToolchains.jl/test/CToolchainTests.jl +++ b/BinaryBuilderToolchains.jl/test/CToolchainTests.jl @@ -178,11 +178,15 @@ ENV["TESTSUITE_OUTPUT_DIR"] = mktempdir(;cleanup=false) end # Ensure that `strip` works for macOS without needing to resign - macos_cp = CrossPlatform(BBHostPlatform() => Platform("aarch64", "macos")) - toolchains = [CToolchain(macos_cp), htt_toolchain] - with_toolchains(toolchains) do prefix, env - cd(joinpath(@__DIR__, "testsuite", "CToolchain", "08_strip_resigning")) do - @test success(setenv(Cmd(["/bin/bash", "-c", "make clean; make check"]), env)) + @testset "macos strip resigning" begin + for macos_target in filter(Sys.isapple, supported_platforms(CToolchain)) + macos_cp = CrossPlatform(BBHostPlatform() => macos_target) + toolchains = [CToolchain(macos_cp), htt_toolchain] + with_toolchains(toolchains) do prefix, env + cd(joinpath(@__DIR__, "testsuite", "CToolchain", "08_strip_resigning")) do + @test success(setenv(Cmd(["/bin/bash", "-c", "make clean; make check"]), env)) + end + end end end end diff --git a/BinaryBuilderToolchains.jl/test/common.jl b/BinaryBuilderToolchains.jl/test/common.jl index ded49dd..68f051e 100644 --- a/BinaryBuilderToolchains.jl/test/common.jl +++ b/BinaryBuilderToolchains.jl/test/common.jl @@ -22,38 +22,45 @@ end function toolchain_tests(prefix, env, platform, testsuite; do_cxxabi_tests::Bool = false) testsuite_path = joinpath(@__DIR__, "testsuite", testsuite) - cd(testsuite_path) do - # Run our entire test suite first - p = run(ignorestatus(setenv(`make -s cleancheck-all`, env))) - # If this fails, run it again, but with `make` not set to silent - if !success(p) - run(setenv(`make cleancheck-all`, env)) - end - @test success(p) - - # Run the `cxx_string_abi` with `BB_WRAPPERS_VERBOSE` and ensure that we get the right - # `cxxstring_abi` defines showing in the build log: - if do_cxxabi_tests - @test haskey(platform.target, "cxxstring_abi") - cxxstring_abi_define = string( - "-D_GLIBCXX_USE_CXX11_ABI=", - platform.target["cxxstring_abi"] == "cxx11" ? "1" : "0", - ) - - # Turn on verbose wrappers - debug_env = copy(env) - debug_env["BB_WRAPPERS_VERBOSE"] = "true" - p, debug_out = capture_output(setenv(`make cleancheck-02_cxx_string_abi`, debug_env)) + @testset "$(triplet(platform)) $(do_cxxabi_tests ? "cxxabi" : "")" begin + cd(testsuite_path) do + # Run our entire test suite first + p = run(ignorestatus(setenv(`make -s cleancheck-all`, env))) + # If this fails, run it again, but with `make` not set to silent + if !success(p) + run(setenv(`make cleancheck-all`, env)) + end @test success(p) - @test occursin(cxxstring_abi_define, debug_out) + + # Run the `cxx_string_abi` with `BB_WRAPPERS_VERBOSE` and ensure that we get the right + # `cxxstring_abi` defines showing in the build log: + if do_cxxabi_tests + @test haskey(platform.target, "cxxstring_abi") + cxxstring_abi_define = string( + "-D_GLIBCXX_USE_CXX11_ABI=", + platform.target["cxxstring_abi"] == "cxx11" ? "1" : "0", + ) + + # Turn on verbose wrappers + debug_env = copy(env) + debug_env["BB_WRAPPERS_VERBOSE"] = "true" + p, debug_out = capture_output(setenv(`make cleancheck-02_cxx_string_abi`, debug_env)) + @test success(p) + @test occursin(cxxstring_abi_define, debug_out) + end end - end - # Ensure that every wrapper we generate actually runs (e.g. no dangling tool references) - for wrapper in readdir(joinpath(prefix, "wrappers"); join=true) - if Sys.isexecutable(wrapper) - @testset "$(basename(wrapper))" begin - @test success(setenv(`$(wrapper) --version`, env)) + # Ensure that every wrapper we generate actually runs (e.g. no dangling tool references) + for wrapper in readdir(joinpath(prefix, "wrappers"); join=true) + # Skip these, they're special + if endswith(basename(wrapper), "-clang-as") + continue + end + + if Sys.isexecutable(wrapper) + @testset "$(basename(wrapper))" begin + @test success(setenv(`$(wrapper) --version`, env)) + end end end end diff --git a/JLLGenerator.jl/Manifest.toml b/JLLGenerator.jl/Manifest.toml index 7777907..8309a69 100644 --- a/JLLGenerator.jl/Manifest.toml +++ b/JLLGenerator.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "4c4ab75912f35859ba877902052211c9224cfa9d" @@ -24,9 +24,9 @@ version = "0.1.0" [[deps.Compat]] deps = ["TOML", "UUIDs"] -git-tree-sha1 = "3a3dfb30697e96a440e4149c8c51bf32f818c0f3" +git-tree-sha1 = "0037835448781bb46feb39866934e243886d756a" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.17.0" +version = "4.18.0" [deps.Compat.extensions] CompatLinearAlgebraExt = "LinearAlgebra" @@ -51,9 +51,9 @@ version = "1.11.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] @@ -142,9 +142,9 @@ version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/JLLGenerator.jl/contrib/Manifest.toml b/JLLGenerator.jl/contrib/Manifest.toml index f79a44b..221cd34 100644 --- a/JLLGenerator.jl/contrib/Manifest.toml +++ b/JLLGenerator.jl/contrib/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "fcdbf9cb21a021388d521efbf1ddd319c241e596" @@ -31,21 +31,27 @@ version = "1.11.0" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" +git-tree-sha1 = "7bb1361afdb33c7f2b085aa49ea8fe1b0fb14e58" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.5+0" +version = "2.7.1+0" [[deps.Git]] -deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] -git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "cb151153e40ad40a6dbf984fcd767e1d266fcc9c" +git-tree-sha1 = "e2aef26f7d273f1e5b1daba56837c47b49b4388f" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.50.1+0" +version = "2.51.1+0" [[deps.InteractiveUtils]] deps = ["Markdown"] @@ -54,9 +60,9 @@ version = "1.11.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] @@ -100,15 +106,15 @@ version = "0.1.0" [[deps.OpenSSH_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] -git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +git-tree-sha1 = "1f2f0911e1c02f28a390bb720f97f3349c4dcefb" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "10.0.1+0" +version = "10.0.2+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "87510f7292a2b21aeff97912b0898f9553cc5c2c" +git-tree-sha1 = "2ae7d4ddec2e13ad3bddf5c0796f7547cf682391" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.1+0" +version = "3.5.2+0" [[deps.OrderedCollections]] git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" @@ -128,9 +134,9 @@ version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/JLLGenerator.jl/ext/BinaryBuilderSourcesExt.jl b/JLLGenerator.jl/ext/BinaryBuilderSourcesExt.jl index 171fe07..a667d7d 100644 --- a/JLLGenerator.jl/ext/BinaryBuilderSourcesExt.jl +++ b/JLLGenerator.jl/ext/BinaryBuilderSourcesExt.jl @@ -10,10 +10,14 @@ function Base.pkgdir(j::JLLSource) if j.package.path !== nothing return j.package.path end + if j.package.tree_hash === nothing + @warn("BinaryBuilder2 error: this package didn't get resolved! Perhaps lacking a platform mapping?", j.package) + end return Pkg.Operations.find_installed(j.package.name, j.package.uuid, j.package.tree_hash) end -function JLLGenerator.parse_toml_dict(j::JLLSource; depot::Union{Nothing,String} = nothing) +function JLLGenerator.parse_toml_dict(j::JLLSource; project::Union{Nothing,String} = nothing, depot::Union{Nothing,String} = nothing) local pkg_dir + if depot === nothing pkg_dir = pkgdir(j) else diff --git a/KeywordArgumentExtraction.jl/Manifest.toml b/KeywordArgumentExtraction.jl/Manifest.toml index f04918c..d1d618b 100644 --- a/KeywordArgumentExtraction.jl/Manifest.toml +++ b/KeywordArgumentExtraction.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "eae609e7fd30def3659dd38809f62040d962a8a5" diff --git a/LazyJLLWrappers.jl/Manifest.toml b/LazyJLLWrappers.jl/Manifest.toml index cd2290c..a718e65 100644 --- a/LazyJLLWrappers.jl/Manifest.toml +++ b/LazyJLLWrappers.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "671eaf2f5a02947991c1fbbb10b169ad695dbfcf" @@ -94,9 +94,9 @@ version = "1.11.0" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/Manifest-v1.12.toml b/Manifest-v1.12.toml index af69943..a5140f4 100644 --- a/Manifest-v1.12.toml +++ b/Manifest-v1.12.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.12.0-rc1" +julia_version = "1.12.1" manifest_format = "2.0" -project_hash = "536abf74de62fb937e599ab9ced7d9e62fdaf80d" +project_hash = "6a7af71fce7ab846053b1c85790be288c0ae2e0d" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -23,7 +23,7 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" [[deps.BinaryBuilder2]] -deps = ["Artifacts", "BinaryBuilderAuditor", "BinaryBuilderGitUtils", "BinaryBuilderPlatformExtensions", "BinaryBuilderProducts", "BinaryBuilderSources", "BinaryBuilderToolchains", "Ccache_jll", "Dates", "Downloads", "Git", "HistoricalStdlibVersions", "JLLGenerator", "JLLPrefixes", "KeywordArgumentExtraction", "LazyJLLWrappers", "Libdl", "LocalRegistry", "Logging", "LoggingExtras", "MultiHashParsing", "ObjectFile", "OrderedCollections", "OutputCollectors", "Pkg", "PrecompileTools", "Preferences", "Random", "Reexport", "RegistryTools", "SHA", "Sandbox", "Scratch", "ScratchSpaceGarbageCollector", "StyledStrings", "TOML", "Test", "TimerOutputs", "TreeArchival", "UserNSSandbox_jll", "gh_cli_jll"] +deps = ["Artifacts", "BinaryBuilderAuditor", "BinaryBuilderGitUtils", "BinaryBuilderPlatformExtensions", "BinaryBuilderProducts", "BinaryBuilderSources", "BinaryBuilderToolchains", "Ccache_jll", "Dates", "Downloads", "Git", "HistoricalStdlibVersions", "Infiltrator", "JLLGenerator", "JLLPrefixes", "KeywordArgumentExtraction", "LazyJLLWrappers", "Libdl", "LocalRegistry", "Logging", "LoggingExtras", "MultiHashParsing", "ObjectFile", "OrderedCollections", "OutputCollectors", "Pkg", "PrecompileTools", "Preferences", "Random", "Reexport", "RegistryTools", "SHA", "Sandbox", "Scratch", "ScratchSpaceGarbageCollector", "StyledStrings", "TOML", "Test", "TimerOutputs", "TreeArchival", "UserNSSandbox_jll", "gh_cli_jll"] path = "." uuid = "12aac903-9f7c-5d81-afc2-d9565ea332af" version = "2.0.0" @@ -53,7 +53,7 @@ uuid = "21737265-7a69-6e4f-4974-6375646f7270" version = "0.1.0" [[deps.BinaryBuilderSources]] -deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TimerOutputs", "TreeArchival"] +deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TOML", "TimerOutputs", "TreeArchival"] path = "BinaryBuilderSources.jl" uuid = "316c416d-4527-6863-7465-466137743047" version = "0.1.0" @@ -72,9 +72,9 @@ version = "4.9.1+0" [[deps.Compat]] deps = ["TOML", "UUIDs"] -git-tree-sha1 = "3a3dfb30697e96a440e4149c8c51bf32f818c0f3" +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.17.0" +version = "4.18.1" [deps.Compat.extensions] CompatLinearAlgebraExt = "LinearAlgebra" @@ -105,9 +105,9 @@ version = "1.0.5" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" +git-tree-sha1 = "27af30de8b5445644e8ffe3bcb0d72049c089cf1" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.5+0" +version = "2.7.3+0" [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" @@ -119,10 +119,16 @@ uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" [[deps.Git]] -deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] -git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] @@ -132,11 +138,15 @@ version = "2.44.0+2" [[deps.HistoricalStdlibVersions]] deps = ["Pkg", "PrecompileTools"] -git-tree-sha1 = "1c2bfcc3d0ad9628a0fc247a267d8b06153182d4" -repo-rev = "main" -repo-url = "https://github.com/JuliaPackaging/HistoricalStdlibVersions.jl.git" +git-tree-sha1 = "abdd6437ede003ee3c412b7da655a8dde356a573" uuid = "6df8b67a-e8a0-4029-b4b7-ac196fe72102" -version = "2.0.3" +version = "2.0.6" + +[[deps.Infiltrator]] +deps = ["InteractiveUtils", "Markdown", "REPL", "UUIDs"] +git-tree-sha1 = "5c248f54dd39381e320f63363e1fbd12f8a5a822" +uuid = "5903a43b-9cc3-4c30-8d17-598619ec4e9b" +version = "1.9.3" [[deps.InteractiveUtils]] deps = ["Markdown"] @@ -155,15 +165,15 @@ weakdeps = ["BinaryBuilderSources"] [[deps.JLLPrefixes]] deps = ["Artifacts", "Git", "HistoricalStdlibVersions", "Pkg", "Preferences", "SHA", "Scratch"] -git-tree-sha1 = "c199272df7cab0230eb6febc3003c87873599107" +git-tree-sha1 = "3b772c89f2fb0f2877e9149d3f529276ca8b33f2" uuid = "afc68a34-7891-4c5a-9da1-1c62935e7b0d" -version = "0.3.13" +version = "0.4.1" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.JuliaSyntaxHighlighting]] deps = ["StyledStrings"] @@ -245,9 +255,9 @@ version = "1.11.0" [[deps.LoggingExtras]] deps = ["Dates", "Logging"] -git-tree-sha1 = "f02b56007b064fbfddb4c9cd60161b6dd0f40df3" +git-tree-sha1 = "f00544d95982ea270145636c181ceda21c4e2575" uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "1.1.0" +version = "1.2.0" [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] @@ -276,9 +286,9 @@ version = "0.4.4" [[deps.OpenSSH_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] -git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +git-tree-sha1 = "301412a644646fdc0ad67d0a87487466b491e53d" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "10.0.1+0" +version = "10.2.1+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] @@ -306,31 +316,34 @@ version = "10.44.0+1" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.13.0" +version = "1.12.0" +weakdeps = ["REPL"] [deps.Pkg.extensions] REPLExt = "REPL" - [deps.Pkg.weakdeps] - REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "516f18f048a195409d6e072acf879a9f017d3900" +git-tree-sha1 = "07a921781cab75691315adc645096ed5e370cb77" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.3.2" +version = "1.3.3" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" version = "1.11.0" +[[deps.REPL]] +deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + [[deps.Random]] deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -379,6 +392,10 @@ version = "0.1.0" uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" version = "1.11.0" +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + [[deps.StructEquality]] deps = ["Compat"] git-tree-sha1 = "192a9f1de3cfef80ab1a4ba7b150bb0e11ceedcf" diff --git a/Manifest.toml b/Manifest.toml index 30bdf5c..170892d 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" -project_hash = "47465796e0025cc764e0dfceded113c86e325acf" +project_hash = "b60a247efae7baf1f3767724093d7628696e42bc" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -47,7 +47,7 @@ uuid = "21737265-7a69-6e4f-4974-6375646f7270" version = "0.1.0" [[deps.BinaryBuilderSources]] -deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TimerOutputs", "TreeArchival"] +deps = ["BinaryBuilderGitUtils", "Dates", "Downloads", "Git", "JLLPrefixes", "MultiHashParsing", "Pkg", "Random", "SHA", "Scratch", "TOML", "TimerOutputs", "TreeArchival"] path = "BinaryBuilderSources.jl" uuid = "316c416d-4527-6863-7465-466137743047" version = "0.1.0" @@ -66,9 +66,9 @@ version = "4.9.1+0" [[deps.Compat]] deps = ["TOML", "UUIDs"] -git-tree-sha1 = "3a3dfb30697e96a440e4149c8c51bf32f818c0f3" +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.17.0" +version = "4.18.1" [deps.Compat.extensions] CompatLinearAlgebraExt = "LinearAlgebra" @@ -99,9 +99,9 @@ version = "1.0.5" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" +git-tree-sha1 = "27af30de8b5445644e8ffe3bcb0d72049c089cf1" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.5+0" +version = "2.7.3+0" [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" @@ -113,30 +113,34 @@ uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" [[deps.Git]] -deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] -git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" +version = "1.5.0" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.0+0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "cb151153e40ad40a6dbf984fcd767e1d266fcc9c" +git-tree-sha1 = "b6a684587ebe896d9f68ae777f648205940f0f70" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.50.1+0" +version = "2.51.3+0" [[deps.HistoricalStdlibVersions]] deps = ["Pkg", "PrecompileTools"] -git-tree-sha1 = "1c2bfcc3d0ad9628a0fc247a267d8b06153182d4" -repo-rev = "main" -repo-url = "https://github.com/JuliaPackaging/HistoricalStdlibVersions.jl.git" +git-tree-sha1 = "abdd6437ede003ee3c412b7da655a8dde356a573" uuid = "6df8b67a-e8a0-4029-b4b7-ac196fe72102" -version = "2.0.3" +version = "2.0.6" [[deps.Infiltrator]] deps = ["InteractiveUtils", "Markdown", "REPL", "UUIDs"] -git-tree-sha1 = "c287eec86806afafb74fd633ac448b69245602fa" +git-tree-sha1 = "479e360b411c64358004179e9fd057d59cf8dda6" uuid = "5903a43b-9cc3-4c30-8d17-598619ec4e9b" -version = "1.9.2" +version = "1.9.4" [[deps.InteractiveUtils]] deps = ["Markdown"] @@ -155,15 +159,15 @@ weakdeps = ["BinaryBuilderSources"] [[deps.JLLPrefixes]] deps = ["Artifacts", "Git", "HistoricalStdlibVersions", "Pkg", "Preferences", "SHA", "Scratch"] -git-tree-sha1 = "c199272df7cab0230eb6febc3003c87873599107" +git-tree-sha1 = "3b772c89f2fb0f2877e9149d3f529276ca8b33f2" uuid = "afc68a34-7891-4c5a-9da1-1c62935e7b0d" -version = "0.3.13" +version = "0.4.1" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.KeywordArgumentExtraction]] deps = ["ExprTools", "InteractiveUtils"] @@ -240,9 +244,9 @@ version = "1.11.0" [[deps.LoggingExtras]] deps = ["Dates", "Logging"] -git-tree-sha1 = "f02b56007b064fbfddb4c9cd60161b6dd0f40df3" +git-tree-sha1 = "f00544d95982ea270145636c181ceda21c4e2575" uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "1.1.0" +version = "1.2.0" [[deps.Markdown]] deps = ["Base64"] @@ -276,15 +280,15 @@ version = "0.4.4" [[deps.OpenSSH_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] -git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +git-tree-sha1 = "301412a644646fdc0ad67d0a87487466b491e53d" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "10.0.1+0" +version = "10.2.1+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "87510f7292a2b21aeff97912b0898f9553cc5c2c" +git-tree-sha1 = "f19301ae653233bc88b1810ae908194f07f8db9d" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.1+0" +version = "3.5.4+0" [[deps.OrderedCollections]] git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" @@ -321,9 +325,9 @@ version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/MultiHashParsing.jl/Manifest.toml b/MultiHashParsing.jl/Manifest.toml index 6d67367..fd6958e 100644 --- a/MultiHashParsing.jl/Manifest.toml +++ b/MultiHashParsing.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "ac28bdb0880dfb01cce60c7e8b8fffd54ef48828" diff --git a/Project.toml b/Project.toml index 3c69fa8..c1f4a90 100644 --- a/Project.toml +++ b/Project.toml @@ -54,7 +54,6 @@ BinaryBuilderPlatformExtensions = {path = "BinaryBuilderPlatformExtensions.jl"} BinaryBuilderProducts = {path = "BinaryBuilderProducts.jl"} BinaryBuilderSources = {path = "BinaryBuilderSources.jl"} BinaryBuilderToolchains = {path = "BinaryBuilderToolchains.jl"} -HistoricalStdlibVersions = {rev = "main", url = "https://github.com/JuliaPackaging/HistoricalStdlibVersions.jl.git"} JLLGenerator = {path = "JLLGenerator.jl"} KeywordArgumentExtraction = {path = "KeywordArgumentExtraction.jl"} LazyJLLWrappers = {path = "LazyJLLWrappers.jl"} @@ -67,7 +66,7 @@ TreeArchival = {path = "TreeArchival.jl"} Accessors = "0.1.42" HistoricalStdlibVersions = "2" Infiltrator = "1.9.2" -JLLPrefixes = "0.3.8" +JLLPrefixes = "0.3.8, 0.4" Logging = "1.11.0" LoggingExtras = "1.1.0" OutputCollectors = "1" diff --git a/ScratchSpaceGarbageCollector.jl/Manifest.toml b/ScratchSpaceGarbageCollector.jl/Manifest.toml index accd8cf..46276d3 100644 --- a/ScratchSpaceGarbageCollector.jl/Manifest.toml +++ b/ScratchSpaceGarbageCollector.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "44c8c55107df54f019cb556c5f03a42c7df72fb6" diff --git a/TreeArchival.jl/Manifest.toml b/TreeArchival.jl/Manifest.toml index ba42445..dc1bfb1 100644 --- a/TreeArchival.jl/Manifest.toml +++ b/TreeArchival.jl/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.6" +julia_version = "1.11.7" manifest_format = "2.0" project_hash = "2fdfbba0dad651d5e0fd91885d3878c3e32509f1" @@ -25,9 +25,9 @@ version = "1.11.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -41,9 +41,9 @@ version = "1.18.0+0" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.Printf]] deps = ["Unicode"] diff --git a/bootstrap/Binutils/build_tarballs.jl b/bootstrap/Binutils/build_tarballs.jl index 265e86d..d8b6de1 100644 --- a/bootstrap/Binutils/build_tarballs.jl +++ b/bootstrap/Binutils/build_tarballs.jl @@ -1,18 +1,12 @@ -using Revise using BinaryBuilder2, Pkg -meta = BinaryBuilder2.get_default_meta() -host = Platform(arch(HostPlatform()), "linux") -binutils_version_sources = Dict{VersionNumber,Vector}( - v"2.24" => [ - ArchiveSource("https://ftp.gnu.org/gnu/binutils/binutils-2.24.tar.bz2", - "e5e8c5be9664e7f7f96e0d09919110ab5ad597794f5b1809871177a0f0f14137"), - ], - v"2.41" => [ - ArchiveSource("https://ftp.gnu.org/gnu/binutils/binutils-2.41.tar.xz", - "ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450"), - ] -) +# `--bootstrap` causes us to only build `host => target` binutils for everything +# This does not require a target C toolchain. +bootstrap_mode = false +if "--bootstrap" ∈ ARGS + bootstrap_mode = true + filter!(x -> x != "--bootstrap", ARGS) +end script = raw""" cd ${WORKSPACE}/srcdir/binutils-*/ @@ -56,37 +50,31 @@ make -j${nproc} ${MAKEVARS[@]} make install ${MAKEVARS[@]} """ -# Build for these host platforms -host_platforms = [ - Platform("x86_64", "linux"), - Platform("aarch64", "linux"), -] +if bootstrap_mode + host_platforms = [ + Platform(arch(HostPlatform()), "linux") + ] +else + # Build for these host platforms + host_platforms = [ + Platform("x86_64", "linux"), + Platform("aarch64", "linux"), + ] +end # Build for all supported target platforms, except for macOS, which uses cctools, not binutils :( -target_platforms = [ - Platform("x86_64", "linux"), - Platform("i686", "linux"), - Platform("aarch64", "linux"), - Platform("armv6l", "linux"), - Platform("armv7l", "linux"), - Platform("powerpc64le", "linux"), - - Platform("x86_64", "linux"; libc="musl"), - Platform("i686", "linux"; libc="musl"), - Platform("aarch64", "linux"; libc="musl"), - Platform("armv6l", "linux"; libc="musl"), - Platform("armv7l", "linux"; libc="musl"), - - Platform("x86_64", "windows"), - Platform("i686", "windows"), -] +target_platforms = supported_platforms(CToolchain) +target_platforms = filter(!Sys.isapple, target_platforms) -platforms = vcat( +platforms = vec([ # Build cross-binutils from `host => target` - (CrossPlatform(host, target) for host in host_platforms, target in target_platforms if host != target)..., + CrossPlatform(host, target) for host in host_platforms, target in target_platforms +]) + +if !bootstrap_mode # Build native binutils for all targets as well - (CrossPlatform(target, target) for target in target_platforms)..., -) + append!(platforms, [CrossPlatform(target, target) for target in target_platforms if target ∉ host_platforms]) +end tool_names = [ :ar, :as, :ld, :nm, :objcopy, :objdump, :ranlib, :readelf, :strings, :binutils_strip, @@ -103,30 +91,33 @@ for varname in tool_names push!(products, ExecutableProduct("\${bindir}/\${target}-$(tool_name)", varname)) end -for version in (v"2.41",) #keys(binutils_version_sources) - build_tarballs(; - src_name = "Binutils", - src_version = version, - sources = [ - binutils_version_sources[version]..., - # We've got a bevvy of patches for Binutils, include them in. - DirectorySource("./patches-v$(version)"; follow_symlinks=true, target="patches"), - ], - target_dependencies = [ - JLLSource( - "Zlib_jll"; - # TODO: Drop this once `Zlib_jll` on `General` is built by BB2. - repo=Pkg.Types.GitRepo( - rev="main", - source="https://github.com/staticfloat/Zlib_jll.jl" - ), - ), - ], - script, - platforms, - products, - host_toolchains = [CToolchain(;vendor=:bootstrap), HostToolsToolchain()], - target_toolchains = [CToolchain(;vendor=:bootstrap)], - meta, - ) +extra_kwargs = Dict(:target_toolchains => []) +if !bootstrap_mode + extra_kwargs[:target_toolchains] = [CToolchain(;vendor=:gcc_bootstrap)] end + +build_tarballs(; + src_name = "Binutils", + src_version = v"2.41", + sources = [ + ArchiveSource("https://ftp.wayne.edu/gnu/binutils/binutils-2.41.tar.xz", + "ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450"), + # We've got a bevvy of patches for Binutils, include them in. + DirectorySource("./patches-v2.41.0"; follow_symlinks=true, target="patches"), + ], + target_dependencies = [ + JLLSource( + "Zlib_jll"; + # TODO: Drop this once `Zlib_jll` on `General` is built by BB2. + repo=Pkg.Types.GitRepo( + rev="bb2/GCCBootstrap-x86_64-linux-gnu", + source="https://github.com/staticfloat/Zlib_jll.jl" + ), + ), + ], + script, + platforms, + products, + host_toolchains = [CToolchain(;vendor=:gcc_bootstrap), HostToolsToolchain()], + extra_kwargs..., +) diff --git a/bootstrap/CCTools/build_tarballs.jl b/bootstrap/CCTools/build_tarballs.jl index 7c89529..dd8284b 100644 --- a/bootstrap/CCTools/build_tarballs.jl +++ b/bootstrap/CCTools/build_tarballs.jl @@ -1,15 +1,30 @@ using BinaryBuilder2, Pkg +include("../bootstrap_common.jl") + +# `--bootstrap` causes us to only build `host => target` binutils for everything +# This does not require a target C toolchain. +bootstrap_mode = false +if "--bootstrap" ∈ ARGS + bootstrap_mode = true + filter!(x -> x != "--bootstrap", ARGS) +end meta = BinaryBuilder2.get_default_meta() -host_platforms = [ - Platform("x86_64", "linux"), - Platform("aarch64", "linux"), -] -mac_platforms = [ - Platform("x86_64", "macos"), - Platform("aarch64", "macos"), -] + +if bootstrap_mode + host_platforms = bootstrap_host_platforms +else + # Build for these host platforms + host_platforms = [ + Platform("x86_64", "linux"), + Platform("aarch64", "linux"), + ] +end + +mac_platforms = filter(Sys.isapple, supported_platforms()) target_platforms = [CrossPlatform(host => target) for host in host_platforms, target in mac_platforms][:] +host_toolchains = [CToolchain(;vendor=:clang_bootstrap), CMakeToolchain(), HostToolsToolchain()] +target_toolchains = [CToolchain(;vendor=:clang_bootstrap), CMakeToolchain()] build_tarballs(; src_name = "tblgen", @@ -46,16 +61,18 @@ build_tarballs(; JLLSource( "Zlib_jll"; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Zlib_jll.jl" ), ), ], + host_toolchains, + target_toolchains, meta, # Don't package this JLL, we're just using this to get the `tblgen_source` below. package_jll = false, ) -tblgen_source = ExtractResultSource(BinaryBuilder2.get_extract_result(meta, "tblgen")) +tblgen_source = ExtractResultSource(only(BinaryBuilder2.get_extract_results(meta, "tblgen"))) build_tarballs(; src_name = "libtapi", @@ -92,11 +109,13 @@ build_tarballs(; JLLSource( "Zlib_jll"; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Zlib_jll.jl" ), ), ], + host_toolchains, + target_toolchains, meta, ) @@ -107,7 +126,7 @@ function cctools_build_spec_generator(host, platform) "build", CrossPlatform(host => host), [CToolchain(;vendor=:clang_bootstrap), CMakeToolchain(), HostToolsToolchain()], - [], #[JLLSource("Python_jll")], + [], Set([:host]), ), BuildTargetSpec( @@ -120,7 +139,7 @@ function cctools_build_spec_generator(host, platform) "Zlib_jll"; # TODO: Drop this once `Zlib_jll` on `General` is built by BB2. repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Zlib_jll.jl" ), ), diff --git a/bootstrap/CrosstoolNG/build_tarballs.jl b/bootstrap/CrosstoolNG/build_tarballs.jl index f8ff016..28b243e 100644 --- a/bootstrap/CrosstoolNG/build_tarballs.jl +++ b/bootstrap/CrosstoolNG/build_tarballs.jl @@ -16,21 +16,21 @@ build_tarballs(; JLLSource( "Zlib_jll"; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Zlib_jll.jl" ), ), JLLSource( "Ncurses_jll"; repo=Pkg.Types.GitRepo( - rev="bb2/GCCBootstrap", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Ncurses_jll.jl" ), ), JLLSource( "Readline_jll"; repo=Pkg.Types.GitRepo( - rev="bb2/GCCBootstrap", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Readline_jll.jl" ), ), diff --git a/bootstrap/FreeBSDSysroot/build_tarballs.jl b/bootstrap/FreeBSDSysroot/build_tarballs.jl new file mode 100644 index 0000000..c000538 --- /dev/null +++ b/bootstrap/FreeBSDSysroot/build_tarballs.jl @@ -0,0 +1,91 @@ +using BinaryBuilder2 + +meta = BinaryBuilder2.get_default_meta() +platforms = [ + Platform("x86_64", "freebsd"; os_version="14.1"), + Platform("aarch64", "freebsd"; os_version="14.1"), +] + +# Build the tarballs, and possibly a `build.jl` as well. +build_tarballs(; + src_name = "FreeBSDSysroot", + src_version = v"14.1", + sources = [ + # We can't use `ArchiveSource` here, because it trips Tar.jl's attack detection: ("Tarball contains hardlink with non-existent target") + FileSource("http://ftp-archive.freebsd.org/pub/FreeBSD-Archive/old-releases/arm64/14.1-RELEASE/base.txz", + "sha256:b25830252e0dce0161004a5b69a159cbbd92d5e92ae362b06158dbb3f2568d32"; + target="freebsd-aarch64.tar.xz"), + FileSource("http://ftp-archive.freebsd.org/pub/FreeBSD-Archive/old-releases/amd64/14.1-RELEASE/base.txz", + "sha256:bb451694e8435e646b5ff7ddc5e94d5c6c9649f125837a34b2a2dd419732f347"; + target="freebsd-x86_64.tar.xz"), + ], + script = raw""" + TARBALL_NAME="freebsd-$(echo "${target}" | cut -d- -f1).tar.xz" + + FILES_TO_EXTRACT=( + # Extract headers + ./usr/include + + # Extract crt*.o + ./usr/lib/Scrt1.o + ./usr/lib/crti.o + ./usr/lib/crtbeginS.o + ./usr/lib/crtendS.o + ./usr/lib/crtn.o + ./usr/lib/crtbegin.o + ./usr/lib/crtend.o + ./usr/lib/crtbeginT.o + ./usr/lib/gcrt1.o + ./usr/lib/crt1.o + + # Compiler runtimes + ./usr/lib/libcompiler_rt.a + ./usr/lib/libgcc.a + ./usr/lib/libgcc_s.so + ./usr/lib/libgcc_eh.a + ./lib/libgcc_s.so.1 + ./usr/lib/libc++.so + ./usr/lib/libc++.a + ./usr/lib/libc++experimental.a + ./lib/libc++.so.1 + ./usr/lib/libcxxrt.a + ./usr/lib/libcxxrt.so + ./lib/libcxxrt.so.1 + + # We're only going to extract certain libraries that are required to bootstrap GCC/Clang: + # libc + ./lib/libc.so.7 + ./usr/lib/libc.so + ./usr/lib/libc_nonshared.a + + # libdl + ./usr/lib/libdl.so.1 + ./usr/lib/libdl.so + + # libm + ./lib/libm.so.5 + ./usr/lib/libm.so + + # libthr + ./lib/libthr.so.3 + ./usr/lib/libthr.so + + # libpthread + ./usr/lib/libpthread.so + ./usr/lib/libpthread.a + ) + + tar -C ${prefix} -xJf "${TARBALL_NAME}" "${FILES_TO_EXTRACT[@]}" + + # Extract license + tar -xJf "${TARBALL_NAME}" ./COPYRIGHT + install_license COPYRIGHT + """, + platforms, + products = [ + FileProduct("usr/include/unistd.h", :unistd_h), + ], + host_toolchains = [HostToolsToolchain()], + target_toolchains = [], + meta, +) diff --git a/bootstrap/GCC/gcc_common.jl b/bootstrap/GCC/gcc_common.jl index d331409..333ad32 100644 --- a/bootstrap/GCC/gcc_common.jl +++ b/bootstrap/GCC/gcc_common.jl @@ -1,5 +1,6 @@ using BinaryBuilder2, Pkg using BinaryBuilder2: BuildTargetSpec, gcc_platform, get_target_spec_by_name +include("../bootstrap_common.jl") meta = BinaryBuilder2.get_default_meta() @@ -132,7 +133,7 @@ cd ${WORKSPACE}/srcdir # Figure out the GCC version from the directory name gcc_version="$(echo gcc-* | cut -d- -f2)" -if [[ "${target}" != *mingw* ]] && [[ "${target}" != *darwin* ]]; then +if [[ "${target}" != *mingw* ]] && [[ "${target}" != *darwin* ]] && [[ "${target}" != *freebsd* ]]; then lib64="lib${target_nbits%32}" else lib64="lib" @@ -201,6 +202,12 @@ elif [[ "${target}" == *-darwin* ]]; then ICONV_PATHS+=( /opt/host-* ) fi find ${ICONV_PATHS[@]} -name iconv.h -o -name libiconv\* | xargs rm -fv + +elif [[ "${target}" == *-freebsd* ]]; then + # If we don't already have a version number, add one + if [[ "${target}" == *-freebsd ]]; then + target_suffix="${FREEBSD_TARGET_SDK}" + fi fi # Get rid of version numbers at the end of GCC deps @@ -232,14 +239,18 @@ for TOOL in CC CPP CXX AS AR NM LD RANLIB; do BUILD_NAME="BUILD_${TOOL}" export ${TOOL}_FOR_BUILD=${!BUILD_NAME} TARGET_NAME="TARGET_${TOOL}" - export ${TOOL}_FOR_TARGET=${!TARGET_NAME} + if [[ -v "${TARGET_NAME}" ]]; then + export ${TOOL}_FOR_TARGET=${!TARGET_NAME} - # These target tool autodetections do not work - export ac_cv_path_${TOOL}_FOR_TARGET=${!TARGET_NAME} + # These target tool autodetections do not work + export ac_cv_path_${TOOL}_FOR_TARGET=${!TARGET_NAME} + fi done # libcc1 fails with an error about `-rdynamic` unless we define this -export gcc_cv_nm="${NM_FOR_TARGET}" +if [[ -v "NM_FOR_TARGET" ]]; then + export gcc_cv_nm="${NM_FOR_TARGET}" +fi # Make sure the tools that GCC itself wants to use ("ld", "as", "dysmutil") are available # not just as "host-ld" or "host-as", etc... Otherwise, the `collect2` we generate looks @@ -259,7 +270,7 @@ $WORKSPACE/srcdir/gcc-*/configure \ --prefix="${host_prefix}" \ --build="${build}" \ --host="${host}" \ - --target="${target}" \ + --target="${target}${target_suffix:-}" \ --disable-multilib \ --disable-bootstrap \ --disable-werror \ @@ -369,26 +380,7 @@ host_platforms = [ ] # Build for all supported target platforms -target_platforms = [ - Platform("x86_64", "linux"), - Platform("i686", "linux"), - Platform("aarch64", "linux"), - Platform("armv6l", "linux"), - Platform("armv7l", "linux"), - Platform("powerpc64le", "linux"), - - Platform("x86_64", "linux"; libc="musl"), - Platform("i686", "linux"; libc="musl"), - Platform("aarch64", "linux"; libc="musl"), - Platform("armv6l", "linux"; libc="musl"), - Platform("armv7l", "linux"; libc="musl"), - - Platform("x86_64", "windows"), - Platform("i686", "windows"), - - Platform("x86_64", "macos"), - Platform("aarch64", "macos"), -] +target_platforms = supported_platforms() function gcc_platforms(version::VersionNumber) platforms = vcat( @@ -415,7 +407,7 @@ function gcc_build_spec_generator(host, platform) "LinuxKernelHeaders_jll", platform.target; repo=Pkg.Types.GitRepo( - rev="bb2/GCC", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LinuxKernelHeaders_jll.jl" ), target=joinpath(target_str, "usr"), @@ -426,13 +418,13 @@ function gcc_build_spec_generator(host, platform) if arch(platform.target) ∈ ("x86_64", "i686", "powerpc64le") # v2.17 glibc_repo = Pkg.Types.GitRepo( - rev="2f33ece6d34f813332ff277ffaea52b075f1af67", + rev="e5b3b44098fd3ca7a67c7a0e439091a79027a7ee", source="https://github.com/staticfloat/Glibc_jll.jl" ) else # v2.19 glibc_repo = Pkg.Types.GitRepo( - rev="a3d1c4ed6e676a47c4659aeecc8f396a2233757d", + rev="b4779ca103670a2cc631da614c7eecc9e126c98f", source="https://github.com/staticfloat/Glibc_jll.jl" ) end @@ -448,7 +440,7 @@ function gcc_build_spec_generator(host, platform) "Musl_jll", platform.target; repo=Pkg.Types.GitRepo( - rev="827bfab690e1cab77b4d48e1a250c8acd3547443", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Musl_jll.jl" ), target=target_str, @@ -469,11 +461,22 @@ function gcc_build_spec_generator(host, platform) platform.target; uuid=Base.UUID("52f8e75f-aed1-5264-b4c9-b8da5a6d5365"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/macOSSDK_jll.jl" ), target=target_str, )) + elseif os(platform.target) == "freebsd" + push!(target_sources, JLLSource( + "FreeBSDSysroot_jll", + platform.target; + uuid=Base.UUID("671a10c0-f9bf-59ae-b52a-dff4adda89ae"), + repo=Pkg.Types.GitRepo( + source="https://github.com/staticfloat/FreeBSDSysroot_jll.jl", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", + ), + target=target_str, + )) else throw(ArgumentError("Don't know how to install libc sources for $(triplet(platform.target))")) end diff --git a/bootstrap/GCCBootstrap/build_tarballs.jl b/bootstrap/GCCBootstrap/build_tarballs.jl index 9e93630..d4be88c 100644 --- a/bootstrap/GCCBootstrap/build_tarballs.jl +++ b/bootstrap/GCCBootstrap/build_tarballs.jl @@ -93,7 +93,7 @@ build_tarballs(; uuid = Base.UUID("86569e53-7a4c-551c-9ab0-bc1131c15cd4"), repo = Pkg.Types.GitRepo( source="https://github.com/staticfloat/CrosstoolNG_jll.jl", - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", ), ), ], diff --git a/bootstrap/GCCBootstrapMacOS/build_tarballs.jl b/bootstrap/GCCBootstrapMacOS/build_tarballs.jl deleted file mode 100644 index 2a8409e..0000000 --- a/bootstrap/GCCBootstrapMacOS/build_tarballs.jl +++ /dev/null @@ -1,27 +0,0 @@ -using BinaryBuilder2 - -include("../GCC/gcc_common.jl") - -for version in (v"14.2.0",) - build_tarballs(; - src_name = "GCCBootstrapMacOS", - src_version = version, - sources = [ - gcc_version_sources[version]..., - DirectorySource("./patches-v$(version)"; follow_symlinks=true, target="patches"), - ], - script, - platforms = [ - CrossPlatform(Platform(arch(HostPlatform()), "linux") => Platform("aarch64", "macos")), - CrossPlatform(Platform(arch(HostPlatform()), "linux") => Platform("x86_64", "macos")), - ], - products = [ - FileProduct("bin", :bindir), - ExecutableProduct("\${target}-gcc", :gcc), - ExecutableProduct("\${target}-g++", :gxx), - ], - build_spec_generator = gcc_build_spec_generator, - meta, - duplicate_extraction_handling = :ignore_all, - ) -end diff --git a/bootstrap/GCCBootstrapManual/build_tarballs.jl b/bootstrap/GCCBootstrapManual/build_tarballs.jl new file mode 100644 index 0000000..ea63ba1 --- /dev/null +++ b/bootstrap/GCCBootstrapManual/build_tarballs.jl @@ -0,0 +1,85 @@ +using BinaryBuilder2 + +include("../GCC/gcc_common.jl") + + +function notarget_build_spec_generator(host, platform) + target_str = triplet(gcc_platform(platform.target)) + lock_microarchitecture = false + + target_sources = [] + if os(platform.target) == "macos" + push!(target_sources, JLLSource( + "macOSSDK_jll", + platform.target; + uuid=Base.UUID("52f8e75f-aed1-5264-b4c9-b8da5a6d5365"), + repo=Pkg.Types.GitRepo( + rev="bb2/GCCBootstrap-x86_64-linux-gnu", + source="https://github.com/staticfloat/macOSSDK_jll.jl" + ), + target=target_str, + )) + elseif os(platform.target) == "freebsd" + push!(target_sources, JLLSource( + "FreeBSDSysroot_jll", + platform.target; + uuid=Base.UUID("671a10c0-f9bf-59ae-b52a-dff4adda89ae"), + repo=Pkg.Types.GitRepo( + source="https://github.com/staticfloat/FreeBSDSysroot_jll.jl", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", + ), + target=target_str, + )) + else + throw(ArgumentError("Don't know how to install libc sources for $(triplet(platform.target))")) + end + + return [ + BuildTargetSpec( + "build", + CrossPlatform(host => host), + [CToolchain(; vendor=:gcc_bootstrap, lock_microarchitecture), HostToolsToolchain()], + [], + Set([:host]), + ), + BuildTargetSpec( + "host", + CrossPlatform(host => platform.host), + [CToolchain(; vendor=:gcc_bootstrap, lock_microarchitecture)], + [], + Set([:default]), + ), + BuildTargetSpec( + "target", + CrossPlatform(host => platform.target), + [BinutilsToolchain(:gcc_bootstrap)], + #[CToolchain(; vendor=:gcc_bootstrap, lock_microarchitecture)], + target_sources, + Set([]), + ), + ] +end + + +for version in (v"14.2.0",) + platforms = filter(p -> Sys.isapple(p) || Sys.isfreebsd(p), supported_platforms()) + platforms = [CrossPlatform(Platform(arch(HostPlatform()), "linux") => p) for p in platforms] + build_tarballs(; + src_name = "GCCBootstrapManual", + src_version = version, + sources = [ + gcc_version_sources[version]..., + DirectorySource("./patches-v$(version)"; follow_symlinks=true, target="patches"), + ], + script, + platforms, + products = [ + FileProduct("bin", :bindir), + ExecutableProduct("\${target}-gcc", :gcc), + ExecutableProduct("\${target}-g++", :gxx), + ], + build_spec_generator = notarget_build_spec_generator, + meta, + duplicate_extraction_handling = :ignore_all, + ) +end diff --git a/bootstrap/GCCBootstrapMacOS/patches-v14.2.0 b/bootstrap/GCCBootstrapManual/patches-v14.2.0 similarity index 100% rename from bootstrap/GCCBootstrapMacOS/patches-v14.2.0 rename to bootstrap/GCCBootstrapManual/patches-v14.2.0 diff --git a/bootstrap/GCCBootstrapMacOS/patches-v9.4.0 b/bootstrap/GCCBootstrapManual/patches-v9.4.0 similarity index 100% rename from bootstrap/GCCBootstrapMacOS/patches-v9.4.0 rename to bootstrap/GCCBootstrapManual/patches-v9.4.0 diff --git a/bootstrap/LLVM/clang.jl b/bootstrap/LLVM/clang.jl index 0aa78c2..e3ac7b6 100644 --- a/bootstrap/LLVM/clang.jl +++ b/bootstrap/LLVM/clang.jl @@ -6,7 +6,6 @@ build_tarballs(; sources = llvm_sources, script = clang_buildscript(llvm_version), platforms = llvm_platforms(;is_bootstrap=false), - host, build_spec_generator = clang_build_spec_generator(;is_bootstrap=false), extract_spec_generator = (build, plat) -> clang_extract_spec_generator(build, plat; is_bootstrap=false), jll_extraction_map = clang_extraction_map(;is_bootstrap=false) diff --git a/bootstrap/LLVM/clang_bootstrap.jl b/bootstrap/LLVM/clang_bootstrap.jl index dc89056..d87c17c 100644 --- a/bootstrap/LLVM/clang_bootstrap.jl +++ b/bootstrap/LLVM/clang_bootstrap.jl @@ -6,7 +6,6 @@ build_tarballs(; sources = llvm_sources, script = clang_buildscript(llvm_version), platforms = llvm_platforms(;is_bootstrap=true), - host, build_spec_generator = clang_build_spec_generator(;is_bootstrap=true), extract_spec_generator = (build, plat) -> clang_extract_spec_generator(build, plat; is_bootstrap=true), jll_extraction_map = clang_extraction_map(;is_bootstrap=true) diff --git a/bootstrap/LLVM/compiler_rt.jl b/bootstrap/LLVM/compiler_rt.jl index 499aa70..c8c9ffe 100644 --- a/bootstrap/LLVM/compiler_rt.jl +++ b/bootstrap/LLVM/compiler_rt.jl @@ -6,14 +6,17 @@ build_tarballs(; sources = llvm_sources, script = llvm_script_prefix * raw""" # Specify exactly which pieces of compiler-rt we want - CMAKE_FLAGS+=("-DCOMPILER_RT_BUILD_BUILTINS=ON") - CMAKE_FLAGS+=("-DCOMPILER_RT_BUILD_CRT=ON") - CMAKE_FLAGS+=("-DCOMPILER_RT_BUILD_LIBFUZZER=OFF") - CMAKE_FLAGS+=("-DCOMPILER_RT_BUILD_MEMPROF=OFF") - CMAKE_FLAGS+=("-DCOMPILER_RT_BUILD_PROFILE=OFF") - CMAKE_FLAGS+=("-DCOMPILER_RT_BUILD_SANITIZERS=OFF") - CMAKE_FLAGS+=("-DCOMPILER_RT_BUILD_XRAY=OFF") - CMAKE_FLAGS+=("-DCOMPILER_RT_DEFAULT_TARGET_TRIPLE=${target}") + CMAKE_FLAGS+=( + -DCOMPILER_RT_BUILD_BUILTINS=ON + -DCOMPILER_RT_BUILD_CRT=ON + -DCOMPILER_RT_BUILD_LIBFUZZER=OFF + -DCOMPILER_RT_BUILD_MEMPROF=OFF + -DCOMPILER_RT_BUILD_PROFILE=OFF + -DCOMPILER_RT_BUILD_SANITIZERS=OFF + -DCOMPILER_RT_BUILD_XRAY=OFF + -DCOMPILER_RT_ENABLE_IOS=OFF + -DCOMPILER_RT_DEFAULT_TARGET_TRIPLE="${target}" + ) # configure, build, install! ${CMAKE} ${WORKSPACE}/srcdir/llvm-project/compiler-rt ${CMAKE_FLAGS[@]} @@ -21,14 +24,13 @@ build_tarballs(; ninja install """, platforms=supported_platforms(), - host, # We need python, and we need to build with clang host_dependencies = [JLLSource("Python_jll")], target_dependencies = [ JLLSource( "Zlib_jll"; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Zlib_jll.jl" ), ), diff --git a/bootstrap/LLVM/error.txt b/bootstrap/LLVM/error.txt deleted file mode 100644 index 024c99d..0000000 --- a/bootstrap/LLVM/error.txt +++ /dev/null @@ -1,23 +0,0 @@ -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZNSt3__122__libcpp_verbose_abortEPKcz: symbol not defined -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdaPv: symbol not defined -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdaPvRKSt9nothrow_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdaPvSt11align_val_t: symbol not defined -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdaPvSt11align_val_tRKSt9nothrow_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdaPvy: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdaPvySt11align_val_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdlPv: symbol not defined -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdlPvRKSt9nothrow_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdlPvSt11align_val_t: symbol not defined -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdlPvSt11align_val_tRKSt9nothrow_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdlPvy: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZdlPvySt11align_val_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _Znay: symbol not defined -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZnayRKSt9nothrow_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZnaySt11align_val_t: symbol not defined -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZnaySt11align_val_tRKSt9nothrow_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _Znwy: symbol not defined -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZnwyRKSt9nothrow_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZnwySt11align_val_t: symbol not defined -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: cannot export _ZnwySt11align_val_tRKSt9nothrow_t: symbol wrong type (2 vs 3) -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: libcxxabi/src/CMakeFiles/cxxabi_shared_objects.dir/cxa_demangle.cpp.obj:cxa_demangle.cpp:(.text+0xad41): undefined reference to `operator delete(void*)' -/opt/target-x86_64-w64-mingw32/clang/bin/x86_64-w64-mingw32-ld: libcxxabi/src/CMakeFiles/cxxabi_shared_objects.dir/cxa_demangle.cpp.obj:cxa_demangle.cpp:(.text+0xae71): undefined reference to `operator delete(void*)' diff --git a/bootstrap/LLVM/libcxx.jl b/bootstrap/LLVM/libcxx.jl index 2ba7226..10484a3 100644 --- a/bootstrap/LLVM/libcxx.jl +++ b/bootstrap/LLVM/libcxx.jl @@ -2,13 +2,6 @@ include("llvm_common.jl") using BinaryBuilderPlatformExtensions: macos_kernel_version platforms = supported_platforms() -platforms = map(platforms) do p - # x86_64-apple-darwin needs to target at least 10.13 - if Sys.isapple(p) && arch(p) == "x86_64" - p["os_version"] = string(macos_kernel_version("10.13")) - end - return p -end build_tarballs(; src_name = "LLVMLibcxx", @@ -67,14 +60,16 @@ build_tarballs(; ninja install """, platforms, - host, # We need python, and we need to build with clang host_dependencies = [JLLSource("Python_jll")], + + ## I think zlib_jll needs to get overwritten to have `path=...` if it's in-universe + ## Instead of just pulling things out from the environment target_dependencies = [ JLLSource( "Zlib_jll"; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Zlib_jll.jl", ), ), diff --git a/bootstrap/LLVM/llvm_common.jl b/bootstrap/LLVM/llvm_common.jl index 09ffc86..92ed332 100644 --- a/bootstrap/LLVM/llvm_common.jl +++ b/bootstrap/LLVM/llvm_common.jl @@ -1,16 +1,13 @@ using BinaryBuilder2, Pkg using BinaryBuilder2: BuildTargetSpec, get_target_spec_by_name, get_default_target_spec, ExtractSpec, get_package_result - -# The build host -host = Platform(arch(HostPlatform()), "linux") +include("../bootstrap_common.jl") # Check to see if the user has asked for a "bootstrap" build, which # builds clang only for a few targets, using GCC instead of clang. function llvm_platforms(;is_bootstrap::Bool = false) platforms = CrossPlatform[] - # If this is a bootstrap build, only build for these hosts if is_bootstrap - for host in [Platform("x86_64", "linux"), Platform("aarch64", "linux")] + for host in bootstrap_host_platforms push!(platforms, CrossPlatform(host => AnyPlatform())) end else @@ -42,7 +39,7 @@ function clang_build_spec_generator(;is_bootstrap::Bool = false) "LLVMTblgen_jll", uuid=Base.UUID("47b65027-ac0b-59bd-a35b-966a6339d635"), repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/LLVMTblgen_jll.jl", ), ), @@ -67,7 +64,7 @@ function clang_build_spec_generator(;is_bootstrap::Bool = false) JLLSource( "Zlib_jll"; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Zlib_jll.jl" ), ), diff --git a/bootstrap/LLVM/tblgen.jl b/bootstrap/LLVM/tblgen.jl index 1b67e88..34c3020 100644 --- a/bootstrap/LLVM/tblgen.jl +++ b/bootstrap/LLVM/tblgen.jl @@ -32,7 +32,6 @@ build_tarballs(; mv ./bin "${bindir}" """, platforms = [BBHostPlatform()], - host, products = [ ExecutableProduct(["llvm-tblgen"], :llvm_tblgen), ExecutableProduct(["clang-tblgen"], :clang_tblgen), @@ -44,9 +43,11 @@ build_tarballs(; JLLSource( "Zlib_jll"; repo=Pkg.Types.GitRepo( - rev="main", + rev="bb2/GCCBootstrap-x86_64-linux-gnu", source="https://github.com/staticfloat/Zlib_jll.jl", ), ), ], + host_toolchains = [CToolchain(;vendor=:gcc_bootstrap), CMakeToolchain(), HostToolsToolchain()], + target_toolchains = [CToolchain(;vendor=:gcc_bootstrap), CMakeToolchain()], ) diff --git a/bootstrap/Ncurses/build_tarballs.jl b/bootstrap/Ncurses/build_tarballs.jl index 837f6a1..43e22da 100644 --- a/bootstrap/Ncurses/build_tarballs.jl +++ b/bootstrap/Ncurses/build_tarballs.jl @@ -4,7 +4,7 @@ build_tarballs(; src_name = "Ncurses", src_version = v"6.4", sources = [ - ArchiveSource("https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz", + ArchiveSource("https://ftp.wayne.edu/gnu/ncurses/ncurses-6.4.tar.gz", "6931283d9ac87c5073f30b6290c4c75f21632bb4fc3603ac8100812bed248159"), ], script = raw""" diff --git a/bootstrap/Readline/build_tarballs.jl b/bootstrap/Readline/build_tarballs.jl index 7103870..60f6f25 100644 --- a/bootstrap/Readline/build_tarballs.jl +++ b/bootstrap/Readline/build_tarballs.jl @@ -3,12 +3,10 @@ using BinaryBuilder2, Pkg host_linux = Platform(arch(HostPlatform()), "linux") build_tarballs(; src_name = "Readline", - src_version = v"8.2", + src_version = v"8.3", sources = [ - ArchiveSource("https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz", - "3feb7171f16a84ee82ca18a36d7b9be109a52c04f492a053331d7d1095007c35"), - FileSource("https://ftp.gnu.org/gnu/readline/readline-8.2-patches/readline82-001", - "bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"), + ArchiveSource("https://ftp.wayne.edu/gnu/readline/readline-8.3.tar.gz", + "fe5383204467828cd495ee8d1d3c037a7eba1389c22bc6a041f627976f9061cc"), ], target_dependencies = [ JLLSource( @@ -21,8 +19,6 @@ build_tarballs(; ], script = raw""" cd $WORKSPACE/srcdir/readline-*/ - - atomic_patch -p0 ${WORKSPACE}/srcdir/readline82-001 export CPPFLAGS="-I${includedir}" ./configure --prefix=${prefix} --build=${MACHTYPE} --host=${target} --with-curses @@ -35,6 +31,5 @@ build_tarballs(; LibraryProduct(["libhistory", "libhistory8"], :libhistory), LibraryProduct(["libreadline", "libreadline8"], :libreadline), ], - # We're only building for the host here, as this is part of bootstrap platforms = supported_platforms(), ) diff --git a/bootstrap/Zlib/build_tarballs.jl b/bootstrap/Zlib/build_tarballs.jl index b24decd..3b40f8c 100644 --- a/bootstrap/Zlib/build_tarballs.jl +++ b/bootstrap/Zlib/build_tarballs.jl @@ -25,26 +25,7 @@ build_tarballs(; products = [ LibraryProduct("libz", :libz), ], - host_toolchains = [CToolchain(;vendor=:bootstrap), HostToolsToolchain(), CMakeToolchain()], - target_toolchains = [CToolchain(;vendor=:bootstrap), CMakeToolchain()], - platforms = [ - Platform("x86_64", "linux"), - Platform("i686", "linux"), - Platform("aarch64", "linux"), - Platform("armv6l", "linux"), - Platform("armv7l", "linux"), - Platform("powerpc64le", "linux"), - - Platform("x86_64", "linux"; libc="musl"), - Platform("i686", "linux"; libc="musl"), - Platform("aarch64", "linux"; libc="musl"), - Platform("armv6l", "linux"; libc="musl"), - Platform("armv7l", "linux"; libc="musl"), - - Platform("x86_64", "windows"), - Platform("i686", "windows"), - - Platform("x86_64", "macos"), - Platform("aarch64", "macos"), - ], + host_toolchains = [CToolchain(;vendor=:gcc_bootstrap), HostToolsToolchain(), CMakeToolchain()], + target_toolchains = [CToolchain(;vendor=:gcc_bootstrap), CMakeToolchain()], + platforms = supported_platforms(), ) diff --git a/bootstrap/bootstrap_common.jl b/bootstrap/bootstrap_common.jl new file mode 100644 index 0000000..748c222 --- /dev/null +++ b/bootstrap/bootstrap_common.jl @@ -0,0 +1,5 @@ +using BinaryBuilder2 + +bootstrap_host_platforms = [ + Platform(arch(HostPlatform()), "linux") +] diff --git a/bootstrap/macOSSDK/build_tarballs.jl b/bootstrap/macOSSDK/build_tarballs.jl index 54b4769..7f58b3c 100644 --- a/bootstrap/macOSSDK/build_tarballs.jl +++ b/bootstrap/macOSSDK/build_tarballs.jl @@ -1,10 +1,7 @@ using BinaryBuilder2 meta = BinaryBuilder2.get_default_meta() -platforms = [ - Platform("x86_64", "macos"), - Platform("aarch64", "macos"), -] +platforms = filter(Sys.isapple, supported_platforms()) # Build the tarballs, and possibly a `build.jl` as well. build_tarballs(; diff --git a/bootstrap/run_full_bootstrap.jl b/bootstrap/run_full_bootstrap.jl index 1a600b0..89451be 100644 --- a/bootstrap/run_full_bootstrap.jl +++ b/bootstrap/run_full_bootstrap.jl @@ -2,7 +2,7 @@ using BinaryBuilder2 host = Platform(arch(HostPlatform()), "linux") parsed_args = BinaryBuilder2.parse_build_tarballs_args(ARGS) -if !haskey(parsed_args, :universe_name) +if get(parsed_args, :universe_name, nothing) === nothing parsed_args[:universe_name] = "GCCBootstrap-$(triplet(host))" end meta = BuildMeta(;parsed_args...) @@ -28,16 +28,27 @@ run_build_tarballs(ctng_meta, "CrosstoolNG/build_tarballs.jl") @info("Building GCCBootstrap...") run_build_tarballs(meta, "GCCBootstrap/build_tarballs.jl") -# Build GCCBootstrapMacOS -@info("Building GCCBootstrapMacOS...") +# Build Binutils and Zlib (but bootstrap mode, which is restricted to only host => targets, rather than also building targets => targets) +run_build_tarballs(meta, "Binutils/build_tarballs.jl", ["--bootstrap"]) + +# Build tblgen and ClangBootstrap for the current host +run_build_tarballs(ctng_meta, "LLVM/tblgen.jl") +clangbootstrap_target = CrossPlatform(BBHostPlatform() => AnyPlatform()) +run_build_tarballs(meta, "LLVM/clang_bootstrap.jl") + +# Build GCCBootstrapManual +@info("Building GCCBootstrapManual...") run_build_tarballs(meta, "macOSSDK/build_tarballs.jl") -run_build_tarballs(meta, "CCTools/build_tarballs.jl") -run_build_tarballs(meta, "GCCBootstrapMacOS/build_tarballs.jl") +run_build_tarballs(meta, "CCTools/build_tarballs.jl", ["--bootstrap"]) +run_build_tarballs(meta, "FreeBSDSysroot/build_tarballs.jl") +run_build_tarballs(meta, "GCCBootstrapManual/build_tarballs.jl") -GCC_TOOLS=[ - # Build Zlib again, this time targeting everything - "Zlib", +# Build Zlib again, this time targeting everything, so that we can build compiler_rt and libcxx +run_build_tarballs(meta, "Zlib/build_tarballs.jl") +run_build_tarballs(meta, "LLVM/compiler_rt.jl") +run_build_tarballs(meta, "LLVM/libcxx.jl") +GCC_TOOLS=[ # Platform header/library bundles "LinuxKernelHeaders", "Mingw", @@ -55,19 +66,5 @@ for tool in GCC_TOOLS run_build_tarballs(meta, "$(tool)/build_tarballs.jl") end -# Build tblgen and ClangBootstrap for the current host -run_build_tarballs(ctng_meta, "LLVM/tblgen.jl") -clangbootstrap_target = CrossPlatform(BBHostPlatform() => AnyPlatform()) -run_build_tarballs(meta, "LLVM/clang_bootstrap.jl") - -# Next, use ClangBootstrap to build actual `clang` for all platforms, then use it to compile `compiler_rt`, -# and then use clang+compiler_rt to build `libcxx`! -LLVM_TOOLS=[ - "clang", - "compiler_rt", - "libcxx", -] -for tool in LLVM_TOOLS - @info("Building $(tool)") - run_build_tarballs(meta, "LLVM/$(tool).jl") -end +# Then finally, Clang +run_build_tarballs(meta, "LLVM/clang.jl") diff --git a/src/BuildCache.jl b/src/BuildCache.jl index d4b8a93..3dae554 100644 --- a/src/BuildCache.jl +++ b/src/BuildCache.jl @@ -13,6 +13,13 @@ struct BuildCache envs::Dict{SHA1Hash,Dict{String,String}} end +function Base.show(io::IO, bc::BuildCache) + println(io, "BuildCache in $(bc.cache_dir):") + println(io, " - $(length(bc.extractions)) extractions") + println(io, " - $(length(bc.build_logs)) build logs, $(length(bc.extract_logs)) extraction logs") + println(io, " - $(length(bc.envs)) environment maps") +end + default_buildcache_dir() = @get_scratch!("buildcache_database") function BuildCache(;cache_dir = default_buildcache_dir()) diff --git a/src/Maintenance.jl b/src/Maintenance.jl index 3ec506f..470d362 100644 --- a/src/Maintenance.jl +++ b/src/Maintenance.jl @@ -3,6 +3,7 @@ module Maintenance # This file will collect entrypoints for maintenance of a build system. using Ccache_jll import ..BinaryBuilder2: ccache_cache +import ..BinaryBuilderSources: generated_source_cache, jll_resolve_cache function ccache_env(cmd::Cmd) return addenv(cmd, "CCACHE_DIR" => ccache_cache()) @@ -33,6 +34,11 @@ function ccache_show_stats() run(ccache_env(`$(ccache()) --show-stats --show-compression --verbose`)) end +function clear_sources_cache() + rm(generated_source_cache("."); recursive=true, force=true) + rm(jll_resolve_cache("."); recursive=true, force=true) +end + # TODO: # - artifact LRU trimming? # - Old universe deletion? diff --git a/src/PlatformlessWrappers.jl b/src/PlatformlessWrappers.jl index 4b236a2..118788f 100644 --- a/src/PlatformlessWrappers.jl +++ b/src/PlatformlessWrappers.jl @@ -69,6 +69,22 @@ function PlatformlessWrapper(ct::CToolchain) return CToolchain(;vendor=BinaryBuilderToolchains.get_vendor(ct)) end +# BinutilsToolchain support +function BinaryBuilderToolchains.BinutilsToolchain(vendor; kwargs...) + return PlatformlessWrapper{BinutilsToolchain}(; args=[vendor], kwargs=Dict(kwargs...)) +end +function apply_platform(pw::PlatformlessWrapper{BinutilsToolchain}, platform::CrossPlatform) + return BinutilsToolchain(platform, pw.args...; pw.kwargs...) +end +function apply_platform(bt::BinutilsToolchain, p::AbstractPlatform) + if !platforms_match(bt.platform, p) + throw(ArgumentError("Attempted to `apply_platform` a BinutilsToolchain with platform $(triplet(bt.platform)) but for $(triplet(p))")) + end + return bt +end +function PlatformlessWrapper(bt::BinutilsToolchain) + return BinutilsToolchain(bt.vendor) +end # CMakeToolchain support function BinaryBuilderToolchains.CMakeToolchain(; kwargs...) diff --git a/src/Universes.jl b/src/Universes.jl index c27d3f6..9001c2c 100644 --- a/src/Universes.jl +++ b/src/Universes.jl @@ -146,7 +146,11 @@ struct Universe string(registry_url), ) prune!(uni) - dev_bb2_packages(uni) + try + dev_bb2_packages(uni) + catch e + @warn("Unable to dev BB2 packages in universe, perhaps you need to clean it out", uni.depot_path, e) + end # If we are not persistent, clean this universe up at the end of our run if !persistent @@ -559,7 +563,7 @@ function deploy_jll(jll_path::String, if branch_name !== nothing repo_url = "$(repo_url)/tree/$(branch_name)" end - @info("Pushing JLL code", repo_url) + @info("Pushing JLL code", repo_url, branch_name) end github_remote = "https://github.com/$(deploy_org)/$(jll_name)_jll.jl" remote_url!(jll_path, deploy_org, github_remote) @@ -634,7 +638,7 @@ function init_jll_repo(u::Universe, jll_name::String) end const fetched_registries = Set{Base.UUID}() -function get_registry_clone(uni::Universe, reg::RegistrySpec, branch_name::String; +function get_registry_clone(uni::Universe, reg::RegistrySpec, branch_name::Union{Nothing,String}; cache_dir::String = source_download_cache("registry_clones"), force::Bool = false) reg_checkout_path = joinpath(uni.depot_path, "deploy_registries", reg.name) @@ -652,7 +656,7 @@ function get_registry_clone(uni::Universe, reg::RegistrySpec, branch_name::Strin if !isdir(reg_checkout_path) # Check out the head commit to that path head_commit = only(log(reg_clone_path; limit=1)) - reg_branch_name = branch_name !== nothing ? branch_name : head_branch(reg_clone_path) + reg_branch_name = @something(branch_name, head_branch(reg_clone_path)) checkout!(reg_clone_path, reg_checkout_path, head_commit) # Make sure we're on the right branch name @@ -682,19 +686,20 @@ function register_jll!(u::Universe, jll::JLLInfo; skip_artifact_export::Bool = f rm(export_dir; force=true, recursive=true) mkpath(export_dir) - uni_branch_name = "bb2/$(u.name)" if u.name !== nothing + uni_branch_name = "bb2/$(u.name)" if isbranch(jll_bare_repo, uni_branch_name) src_branch = uni_branch_name else src_branch = head_branch(jll_bare_repo) end else + uni_branch_name = nothing src_branch = head_branch(jll_bare_repo) end checkout!(jll_bare_repo, jll_path, src_branch; verbose) - if u.name !== nothing + if uni_branch_name !== nothing branch!(jll_path, uni_branch_name) end @@ -734,6 +739,8 @@ function register_jll!(u::Universe, jll::JLLInfo; skip_artifact_export::Bool = f push=false, # We add `.git` here to better match what is already in the General repository repo="$(jll_repo_url).git", + # When iterating, this is pretty useful. + ignore_reregistration=true, ) end @@ -754,6 +761,8 @@ function register_jll!(u::Universe, jll::JLLInfo; skip_artifact_export::Bool = f commit=true, push=false, repo=jll_repo_url, + # When iterating, this is pretty useful. + ignore_reregistration=true, ) end diff --git a/src/build_api/BuildConfig.jl b/src/build_api/BuildConfig.jl index 7a6ebf7..ade65ea 100644 --- a/src/build_api/BuildConfig.jl +++ b/src/build_api/BuildConfig.jl @@ -36,7 +36,7 @@ struct BuildConfig # Contains sources, host dependencies, target dependencies, and toolchains. # Organized by installation prefix (e.g. `/opt/$(triplet)` for toolchains, # `/workspace/srcdir` for sources, `/workspace/destdir/$(triplet)` for dependencies, etc...) - source_trees::Dict{String,Vector{<:AbstractSource}} + source_trees::Dict{String,Vector{AbstractSource}} env::Dict{String,String} # Bash script that will perform the actual build itself @@ -435,21 +435,33 @@ function build!(config::BuildConfig; # Hit our build cache and see if we've already done this exact build. if build_cache_enabled(meta) && !disable_cache && !isempty(extract_arg_hints) prepare(config; verbose) - build_hash = content_hash(config) - if all(haskey(meta.build_cache, build_hash, extract_content_hash(args...)) for args in extract_arg_hints) - if verbose - @info("Build cached", config, build_hash=content_hash(config)) - end - try - result = BuildResult_cached(config) - meta.builds[config] = result - return result - catch exception - @error("Error while reading from build cache", exception=(exception, catch_backtrace())) + try + build_hash = content_hash(config) + if all(haskey(meta.build_cache, build_hash, extract_content_hash(args...)) for args in extract_arg_hints) + if verbose + @info("Build cached", config, build_hash) + end + try + result = BuildResult_cached(config) + meta.builds[config] = result + return result + catch exception + @error("Error while reading from build cache", exception=(exception, catch_backtrace())) + end + else + @debug("Build not cached", config) + for extract_args in extract_arg_hints + @debug(" -> FAIL: ", build_hash, extract_hash=extract_content_hash(extract_args...)) + end end + catch e + @error("Unable to hit build cache", exception=(e, catch_backtrace())) end end + # Declare these all as `local` so that we can inspect them in the `@infiltrate` below + local run_status, run_exception, build_log + # Write build script out into a logfile build_log_io = IOBuffer() mounts = deploy(config; verbose) @@ -457,20 +469,30 @@ function build!(config::BuildConfig; build_log_io, config, mounts; verbose, ) - local run_status, run_exception exe = Sandbox.preferred_executor()() - if "build-start" ∈ debug_modes - @warn("Launching debug shell") - runshell(config; verbose) - end - @timeit config.to "build" begin - run_status, run_exception = run_trycatch(exe, sandbox_config, `$(metadir_prefix())/build_script.sh`) + try + if "build-start" ∈ debug_modes + @warn("Launching debug shell") + runshell(config; verbose) + end + + @timeit config.to "build" begin + run_status, run_exception = run_trycatch(exe, sandbox_config, `$(metadir_prefix())/build_script.sh`) + end + if run_status != :success && verbose @error("Build failed", run_status, run_exception) end + wait(collector) + catch e + run_status = :errored + run_exception = e + if verbose + @error("Build failed (internal error)", run_status, run_exception) + end end - wait(collector) + build_log = String(take!(build_log_io)) # Generate "log" artifact that will later be packaged up. @@ -503,8 +525,10 @@ function build!(config::BuildConfig; meta.builds[config] = result if "build-stop" ∈ debug_modes || ("build-error" ∈ debug_modes && run_status != :success) - @warn("Dropping into Julia REPL for debugging, variables of interest include: `meta::BuildMeta`, `config::BuildConfig` and `result::BuildResult`") - @warn("Use `runshell(result; verbose)` to enter debug shell within extraction environment.") + @warn(""" + Dropping into Julia REPL, variables of interest include: `meta::BuildMeta`, `config::BuildConfig` and `result::BuildResult` + Use `runshell(result; verbose)` to enter debug shell within extraction environment. + """) if !verbose for line in split(build_log, "\n")[end-50:end] printstyled(line; color=:red) @@ -512,9 +536,8 @@ function build!(config::BuildConfig; end end - if run_status != :success - @infiltrate - end + # Drop into REPL + @infiltrate end return result end diff --git a/src/build_api/BuildMeta.jl b/src/build_api/BuildMeta.jl index 92ff174..ab314ef 100644 --- a/src/build_api/BuildMeta.jl +++ b/src/build_api/BuildMeta.jl @@ -373,14 +373,14 @@ function get_package_result(meta::BuildMeta, name::String) return meta.packagings[only(filter(criteria, keys(meta.packagings)))] end -function get_extract_result(meta::BuildMeta, src_name::String, platform::AbstractPlatform = AnyPlatform()) +function get_extract_results(meta::BuildMeta, src_name::String, platform::AbstractPlatform = AnyPlatform()) criteria(config) = config.build.config.src_name == src_name && platforms_match(config.platform, platform) - return meta.extractions[only(filter(criteria, keys(meta.extractions)))] + return [meta.extractions[k] for k in filter(criteria, keys(meta.extractions))] end -function get_build_result(meta::BuildMeta, src_name::String) +function get_build_results(meta::BuildMeta, src_name::String) criteria(config) = config.src_name == src_name - return meta.builds[only(filter(criteria, keys(meta.builds)))] + return [meta.builds[k] for k in filter(criteria, keys(meta.builds))] end import BinaryBuilderToolchains: indent diff --git a/src/build_api/BuildTargetSpec.jl b/src/build_api/BuildTargetSpec.jl index 55c5d05..93cc897 100644 --- a/src/build_api/BuildTargetSpec.jl +++ b/src/build_api/BuildTargetSpec.jl @@ -31,8 +31,6 @@ struct BuildTargetSpec # Alter wrapper and environment prefix mappings based on our `name`: toolchain = rename_wrapper_prefixes(name, flags, toolchain) - - # Re-name `wrapper_prefixes` return apply_platform(toolchain, platform) end diff --git a/src/build_api/Convenience.jl b/src/build_api/Convenience.jl index e617157..dd0ba3d 100644 --- a/src/build_api/Convenience.jl +++ b/src/build_api/Convenience.jl @@ -310,7 +310,7 @@ so try to avoid doing that in any script you want to use this function with. The `dry_run` keyword argument exists as a convenient way to override the `dry_run` parameter inside of `meta`. """ -function run_build_tarballs(meta::AbstractBuildMeta, build_tarballs_path::AbstractString; dry_run::Bool = false) +function run_build_tarballs(meta::AbstractBuildMeta, build_tarballs_path::AbstractString, args::Vector{String} = String[]; dry_run::Bool = false) build_tarballs_path = abspath(build_tarballs_path) # If `dry_run` is set, we need to toggle `dry_run` in `meta`, but only # for the duration of this call: @@ -319,6 +319,9 @@ function run_build_tarballs(meta::AbstractBuildMeta, build_tarballs_path::Abstra empty!(meta.dry_run) push!.((meta.dry_run,), (:build, :extract, :package)) end + old_ARGS = copy(ARGS) + empty!(ARGS) + append!(ARGS, args) try cd(dirname(build_tarballs_path)) do with_default_meta(meta) do @@ -332,7 +335,10 @@ function run_build_tarballs(meta::AbstractBuildMeta, build_tarballs_path::Abstra end finally empty!(meta.dry_run) + # Can't use `append!()` because this is a `Set` push!.((meta.dry_run,), old_dry_run) + empty!(ARGS) + append!(ARGS, old_ARGS) end end diff --git a/src/build_api/ExtractConfig.jl b/src/build_api/ExtractConfig.jl index d13a249..7e2d027 100644 --- a/src/build_api/ExtractConfig.jl +++ b/src/build_api/ExtractConfig.jl @@ -96,7 +96,7 @@ function BinaryBuilderSources.content_hash(config::ExtractConfig) return extract_content_hash(config.script, config.products) end -function runshell(config::ExtractConfig; output_dir::String=mktempdir(builds_dir()), shell::Cmd = `/bin/bash`) +function runshell(config::ExtractConfig; output_dir::String=mktempdir(builds_dir(".")), shell::Cmd = `/bin/bash`) sandbox_config = SandboxConfig(config, output_dir; save_env=false) run(config.build.exe, sandbox_config, ignorestatus(shell)) end @@ -147,6 +147,25 @@ function SandboxConfig(config::ExtractConfig, output_dir::String, mounts = copy( return SandboxConfig(config.build.config, mounts; env, kwargs...) end +function load_dep_jllinfos(config::ExtractConfig) + build_config = config.build.config + meta = AbstractBuildMeta(config) + jll_infos = JLLInfo[] + prefix_alias = target_prefix(config.target_spec) + for d in build_config.source_trees[prefix_alias] + if isa(d, JLLSource) && platforms_match(d.platform, host_if_crossplatform(config.platform)) + jll_info = try + parse_toml_dict(d; depot=meta.universe.depot_path) + catch + @error("Unable to parse JLLInfo TOML dict for dependency", dep=d) + rethrow() + end + push!(jll_infos, jll_info) + end + end + return jll_infos +end + function BinaryBuilderAuditor.audit!(config::ExtractConfig, artifact_dir::String; verbose::Bool = AbstractBuildMeta(config).verbose, kwargs...) build_config = config.build.config @@ -154,7 +173,7 @@ function BinaryBuilderAuditor.audit!(config::ExtractConfig, artifact_dir::String @timeit config.to "audit" begin prefix_alias = target_prefix(config.target_spec) # Load JLLInfo structures for each dependency - dep_jll_infos = JLLInfo[parse_toml_dict(d; depot=meta.universe.depot_path) for d in build_config.source_trees[prefix_alias] if isa(d, JLLSource) && platforms_match(d.platform, host_if_crossplatform(config.platform))] + dep_jll_infos = load_dep_jllinfos(config) platform = host_if_crossplatform(config.platform) # Get libraries for all JLL dependencies @@ -229,6 +248,10 @@ function extract!(config::ExtractConfig; catch exception @error("Error while reading from build cache", exception=(exception, catch_backtrace())) end + else + extract_hash = content_hash(config) + build_hash = content_hash(config.build.config) + @debug("Extraction not cached", config, extract_hash, build_hash) end end @@ -270,7 +293,7 @@ function extract!(config::ExtractConfig; audit_result = audit!(config, artifact_dir) if !success(audit_result) @error("Audit failed, Running again with debugging enabled, then erroring out!") - print_results(audit_result) + show(audit_result) withenv("JULIA_DEBUG" => "all") do audit!(config, artifact_dir) end @@ -299,7 +322,7 @@ function extract!(config::ExtractConfig; end if audit_result !== nothing open(joinpath(artifact_dir, "$(build_config.src_name)-audit.log"); write=true) do io - print_results(audit_result; io) + show(io, audit_result) end end end @@ -319,8 +342,10 @@ function extract!(config::ExtractConfig; end meta.extractions[config] = result if "extract-stop" ∈ debug_modes || ("extract-error" ∈ debug_modes && run_status != :success) - @warn("Dropping into Julia REPL, variables of interest include `meta::BuildMeta`, `config::ExtractConfig` and `result::ExtractResult`") - @warn("Use `runshell(result; verbose)` to enter debug shell within extraction environment.") + @warn(""" + Dropping into Julia REPL, variables of interest include `meta::BuildMeta`, `config::ExtractConfig` and `result::ExtractResult` + Use `runshell(result; verbose)` to enter debug shell within extraction environment. + """) @infiltrate end return result diff --git a/src/build_api/PackageConfig.jl b/src/build_api/PackageConfig.jl index 5eba2b6..ad39e30 100644 --- a/src/build_api/PackageConfig.jl +++ b/src/build_api/PackageConfig.jl @@ -250,15 +250,14 @@ function add_os_version(platform::Platform, target_spec::BuildTargetSpec) libc_jll_name = "macOSSDK_jll" version_map = macos_kernel_version elseif Sys.isfreebsd(platform) - libc_jll_name = "freebsd_something_jll" - version_map = freebsd_kernel_version + libc_jll_name = "FreeBSDSysroot_jll" else # Other platforms don't do versioning yet return platform end for toolchain in target_spec.toolchains - if !isa(toolchain, CToolchain) + if !isa(toolchain, CToolchain) && !isa(toolchain, BinutilsToolchain) continue end @@ -268,6 +267,11 @@ function add_os_version(platform::Platform, target_spec::BuildTargetSpec) platform["os_version"] = string(version_map(tenv["MACOSX_DEPLOYMENT_TARGET"])) return platform end + + if haskey(tenv, "FREEBSD_TARGET_SDK") + platform["os_version"] = tenv["FREEBSD_TARGET_SDK"] + return platform + end end # Unable to find an OS version, that's fine! diff --git a/test/BuildAPITests/ConvenienceTests.jl b/test/BuildAPITests/ConvenienceTests.jl index 6f33aee..0447890 100644 --- a/test/BuildAPITests/ConvenienceTests.jl +++ b/test/BuildAPITests/ConvenienceTests.jl @@ -51,7 +51,7 @@ using BinaryBuilder2: get_package_result # All the tests above target a native linux; let's run a test on Windows and macOS, just so that # we're exercising those troublesome platforms in at least some way: @testset "Zlib on Windows and macOS" begin - meta = BuildMeta(;universe_name,target_list=[Platform("x86_64", "windows"), Platform("aarch64", "macos")]) + meta = BuildMeta(;universe_name,target_list=[Platform("x86_64", "windows"), Platform("aarch64", "macos"; os_version=v"20")]) run_build_tarballs(meta, joinpath(bootstrap_dir, "Zlib", "build_tarballs.jl")) package_result = get_package_result(meta, "Zlib") @test package_result.status == :success