New Subpackage for LBFGS

Project.toml

@@ -35,7 +35,6 @@ Flux = "0.13, 0.14, 0.15, 0.16"
 ForwardDiff = "0.10, 1"
 Ipopt = "1"
 IterTools = "1.3"
-LBFGSB = "0.4.1"
 LinearAlgebra = "1.10"
 Logging = "1.10"
 LoggingExtras = "0.4, 1"
@@ -44,6 +43,7 @@ MLUtils = "0.4"
 ModelingToolkit = "10"
 Mooncake = "0.4.138"
 Optim = ">= 1.4.1"
+Optimisers = ">= 0.2.5"
 OptimizationBase = "2"
 OptimizationMOI = "0.5"
 OptimizationOptimJL = "0.4"
@@ -52,7 +52,7 @@ OrdinaryDiffEqTsit5 = "1"
 Pkg = "1"
 Printf = "1.10"
 ProgressLogging = "0.1"
-Random = "1.10" 
+Random = "1.10"
 Reexport = "1.2"
 ReverseDiff = "1"
 SafeTestsets = "0.1"
@@ -64,7 +64,6 @@ Symbolics = "6"
 TerminalLoggers = "0.1"
 Test = "1.10"
 Tracker = "0.2"
-Optimisers = ">= 0.2.5"
 Zygote = "0.6, 0.7"
 julia = "1.10"
 

lib/OptimizationLBFGS/Project.toml

@@ -0,0 +1,23 @@
+name = "OptimizationLBFGS"
+uuid = "22f7324a-a79d-40f2-bebe-3af60c77bd15"
+authors = ["paramthakkar123 <[email protected]>"]
+version = "0.1.0"
+
+[deps]
+DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LBFGSB = "5be7bae1-8223-5378-bac3-9e7378a2f6e6"
+MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
+Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba"
+OptimizationBase = "bca83a33-5cc9-4baa-983d-23429ab6bcbb"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
+
+[compat]
+DocStringExtensions = "0.9.5"
+ForwardDiff = "1.0.1"
+LBFGSB = "0.4.1"
+MLUtils = "0.4.8"
+Optimization = "4.4.0"
+OptimizationBase = "2.10.0"
+Zygote = "0.7.10"

src/lbfgsb.jl → lib/OptimizationLBFGS/src/OptimizationLBFGS.jl

@@ -1,4 +1,12 @@
-using Optimization.SciMLBase, LBFGSB
+module OptimizationLBFGS
+
+using Optimization
+using DocStringExtensions
+using LBFGSB
+using OptimizationBase.SciMLBase: OptimizationStats, OptimizationFunction
+using OptimizationBase: ReturnCode
+using OptimizationBase.LinearAlgebra: norm
+using Optimization: deduce_retcode
 
 """
 $(TYPEDEF)
@@ -33,7 +41,7 @@ function task_message_to_string(task::Vector{UInt8})
     return String(task)
 end
 
-function __map_optimizer_args(cache::Optimization.OptimizationCache, opt::LBFGS;
+function __map_optimizer_args(cache::OptimizationCache, opt::LBFGS;
         callback = nothing,
         maxiters::Union{Number, Nothing} = nothing,
         maxtime::Union{Number, Nothing} = nothing,
@@ -125,7 +133,7 @@ function SciMLBase.__solve(cache::OptimizationCache{
             cons_tmp[eq_inds] .= cons_tmp[eq_inds] - cache.lcons[eq_inds]
             cons_tmp[ineq_inds] .= cons_tmp[ineq_inds] .- cache.ucons[ineq_inds]
             opt_state = Optimization.OptimizationState(
-                u = θ, objective = x[1], p = cache.p, iter = iter_count[])
+                u = θ, objective = x[1])
             if cache.callback(opt_state, x...)
                 error("Optimization halted by callback.")
             end
@@ -204,7 +212,7 @@ function SciMLBase.__solve(cache::OptimizationCache{
             end
         end
 
-        stats = Optimization.OptimizationStats(; iterations = maxiters,
+        stats = OptimizationStats(; iterations = maxiters,
             time = 0.0, fevals = maxiters, gevals = maxiters)
         return SciMLBase.build_solution(
             cache, cache.opt, res[2], cache.f(res[2], cache.p)[1],
@@ -215,7 +223,7 @@ function SciMLBase.__solve(cache::OptimizationCache{
             x = cache.f(θ, cache.p)
             iter_count[] += 1
             opt_state = Optimization.OptimizationState(
-                u = θ, objective = x[1], p = cache.p, iter = iter_count[])
+                u = θ, objective = x[1])
             if cache.callback(opt_state, x...)
                 error("Optimization halted by callback.")
             end
@@ -256,3 +264,5 @@ function SciMLBase.__solve(cache::OptimizationCache{
             retcode = opt_ret, original = optimizer)
     end
 end
+
+end

lib/OptimizationLBFGS/test/runtests.jl

@@ -0,0 +1,57 @@
+using OptimizationBase
+using OptimizationBase: ReturnCode
+using OptimizationBase.SciMLBase: OptimizationFunction, OptimizationProblem
+using ForwardDiff, Zygote
+using OptimizationLBFGS
+using MLUtils
+using LBFGSB
+using Test
+
+@testset "OptimizationLBFGS.jl" begin
+    x0 = zeros(2)
+    rosenbrock(x, p = nothing) = (1 - x[1])^2 + 100 * (x[2] - x[1]^2)^2
+    l1 = rosenbrock(x0)
+
+    optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoForwardDiff())
+    prob = OptimizationProblem(optf, x0)
+    @time res = solve(prob, OptimizationLBFGS.LBFGS(), maxiters = 100)
+    @test res.retcode == ReturnCode.Success
+
+    prob = OptimizationProblem(optf, x0, lb = [-1.0, -1.0], ub = [1.0, 1.0])
+    @time res = solve(prob, OptimizationLBFGS.LBFGS(), maxiters = 100)
+    @test res.retcode == ReturnCode.Success
+
+    function con2_c(res, x, p)
+        res .= [x[1]^2 + x[2]^2, (x[2] * sin(x[1]) + x[1]) - 5]
+    end
+
+    optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoZygote(), cons = con2_c)
+    prob = OptimizationProblem(optf, x0, lcons = [1.0, -Inf],
+        ucons = [1.0, 0.0], lb = [-1.0, -1.0],
+        ub = [1.0, 1.0])
+    @time res = solve(prob, OptimizationLBFGS.LBFGS(), maxiters = 100)
+    @test res.retcode == SciMLBase.ReturnCode.Success
+
+    x0 = (-pi):0.001:pi
+    y0 = sin.(x0)
+    data = MLUtils.DataLoader((x0, y0), batchsize = 126)
+    function loss(coeffs, data)
+        ypred = [evalpoly(data[1][i], coeffs) for i in eachindex(data[1])]
+        return sum(abs2, ypred .- data[2])
+    end
+
+    function cons1(res, coeffs, p = nothing)
+        res[1] = coeffs[1] * coeffs[5] - 1
+        return nothing
+    end
+
+    optf = OptimizationFunction(loss, AutoSparseForwardDiff(), cons = cons1)
+    callback = (st, l) -> (@show l; return false)
+
+    initpars = rand(5)
+    l0 = optf(initpars, (x0, y0))
+    prob = OptimizationProblem(optf, initpars, (x0, y0), lcons = [-Inf], ucons = [0.5],
+        lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0])
+    opt1 = solve(prob, OptimizationLBFGS.LBFGS(), maxiters = 1000, callback = callback)
+    @test opt1.objective < l0
+end

src/Optimization.jl

@@ -22,7 +22,6 @@ export ObjSense, MaxSense, MinSense
 
 include("utils.jl")
 include("state.jl")
-include("lbfgsb.jl")
 include("sophia.jl")
 include("auglag.jl")
 

test/native.jl

@@ -1,31 +1,6 @@
 using Optimization
 using ForwardDiff, Zygote, ReverseDiff, FiniteDiff
 using Test
-
-x0 = zeros(2)
-rosenbrock(x, p = nothing) = (1 - x[1])^2 + 100 * (x[2] - x[1]^2)^2
-l1 = rosenbrock(x0)
-
-optf = OptimizationFunction(rosenbrock, AutoForwardDiff())
-prob = OptimizationProblem(optf, x0)
-@time res = solve(prob, Optimization.LBFGS(), maxiters = 100)
-@test res.retcode == Optimization.SciMLBase.ReturnCode.Success
-
-prob = OptimizationProblem(optf, x0, lb = [-1.0, -1.0], ub = [1.0, 1.0])
-@time res = solve(prob, Optimization.LBFGS(), maxiters = 100)
-@test res.retcode == Optimization.SciMLBase.ReturnCode.Success
-
-function con2_c(res, x, p)
-    res .= [x[1]^2 + x[2]^2, (x[2] * sin(x[1]) + x[1]) - 5]
-end
-
-optf = OptimizationFunction(rosenbrock, AutoZygote(), cons = con2_c)
-prob = OptimizationProblem(optf, x0, lcons = [1.0, -Inf],
-    ucons = [1.0, 0.0], lb = [-1.0, -1.0],
-    ub = [1.0, 1.0])
-@time res = solve(prob, Optimization.LBFGS(), maxiters = 100)
-@test res.retcode == SciMLBase.ReturnCode.Success
-
 using MLUtils, OptimizationOptimisers
 
 x0 = (-pi):0.001:pi
@@ -46,10 +21,6 @@ callback = (st, l) -> (@show l; return false)
 
 initpars = rand(5)
 l0 = optf(initpars, (x0, y0))
-prob = OptimizationProblem(optf, initpars, (x0, y0), lcons = [-Inf], ucons = [0.5],
-    lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0])
-opt1 = solve(prob, Optimization.LBFGS(), maxiters = 1000, callback = callback)
-@test opt1.objective < l0
 
 prob = OptimizationProblem(optf, initpars, data, lcons = [-Inf], ucons = [1],
     lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0])