From 1429b2ed3d7aa2542c3b55b5ea67ccf9419dd25a Mon Sep 17 00:00:00 2001 From: paramthakkar123 Date: Thu, 31 Jul 2025 22:18:24 +0530 Subject: [PATCH 1/7] Added a new Subpackage for Augmented Lagrangian --- lib/OptimizationAuglag/Project.toml | 4 ++++ .../OptimizationAuglag/src/OptimizationAuglag.jl | 4 ++++ lib/OptimizationAuglag/test/runtests.jl | 0 src/Optimization.jl | 1 - 4 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 lib/OptimizationAuglag/Project.toml rename src/auglag.jl => lib/OptimizationAuglag/src/OptimizationAuglag.jl (99%) create mode 100644 lib/OptimizationAuglag/test/runtests.jl diff --git a/lib/OptimizationAuglag/Project.toml b/lib/OptimizationAuglag/Project.toml new file mode 100644 index 000000000..3a0812299 --- /dev/null +++ b/lib/OptimizationAuglag/Project.toml @@ -0,0 +1,4 @@ +name = "OptimizationAuglag" +uuid = "2ea93f80-9333-43a1-a68d-1f53b957a421" +authors = ["paramthakkar123 "] +version = "0.1.0" diff --git a/src/auglag.jl b/lib/OptimizationAuglag/src/OptimizationAuglag.jl similarity index 99% rename from src/auglag.jl rename to lib/OptimizationAuglag/src/OptimizationAuglag.jl index c3b4af753..ff0cea2e8 100644 --- a/src/auglag.jl +++ b/lib/OptimizationAuglag/src/OptimizationAuglag.jl @@ -1,3 +1,5 @@ +module OptimizationAuglag + @kwdef struct AugLag inner::Any τ = 0.5 @@ -179,3 +181,5 @@ function SciMLBase.__solve(cache::OptimizationCache{ stats = stats, retcode = opt_ret) end end + +end diff --git a/lib/OptimizationAuglag/test/runtests.jl b/lib/OptimizationAuglag/test/runtests.jl new file mode 100644 index 000000000..e69de29bb diff --git a/src/Optimization.jl b/src/Optimization.jl index 8d0257dd1..4cfeead6e 100644 --- a/src/Optimization.jl +++ b/src/Optimization.jl @@ -24,7 +24,6 @@ include("utils.jl") include("state.jl") include("lbfgsb.jl") include("sophia.jl") -include("auglag.jl") export solve From 36d765456e6e5e0860707a3efcef34252dd33137 Mon Sep 17 00:00:00 2001 From: paramthakkar123 Date: Thu, 31 Jul 2025 22:44:15 +0530 Subject: [PATCH 2/7] Added OptimizationAuglag tests --- lib/OptimizationAuglag/Project.toml | 14 +++++++++ .../src/OptimizationAuglag.jl | 2 ++ lib/OptimizationAuglag/test/runtests.jl | 31 +++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/lib/OptimizationAuglag/Project.toml b/lib/OptimizationAuglag/Project.toml index 3a0812299..20c74eb36 100644 --- a/lib/OptimizationAuglag/Project.toml +++ b/lib/OptimizationAuglag/Project.toml @@ -2,3 +2,17 @@ name = "OptimizationAuglag" uuid = "2ea93f80-9333-43a1-a68d-1f53b957a421" authors = ["paramthakkar123 "] version = "0.1.0" + +[deps] +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" +Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" +OptimizationOptimisers = "42dfb2eb-d2b4-4451-abcd-913932933ac1" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +ForwardDiff = "1.0.1" +MLUtils = "0.4.8" +Optimization = "4.4.0" +OptimizationOptimisers = "0.3.8" +Test = "1.11.0" diff --git a/lib/OptimizationAuglag/src/OptimizationAuglag.jl b/lib/OptimizationAuglag/src/OptimizationAuglag.jl index ff0cea2e8..2ca6b9f33 100644 --- a/lib/OptimizationAuglag/src/OptimizationAuglag.jl +++ b/lib/OptimizationAuglag/src/OptimizationAuglag.jl @@ -1,5 +1,7 @@ module OptimizationAuglag +using Optimization.SciMLBase, Optimization + @kwdef struct AugLag inner::Any τ = 0.5 diff --git a/lib/OptimizationAuglag/test/runtests.jl b/lib/OptimizationAuglag/test/runtests.jl index e69de29bb..eef9d1c75 100644 --- a/lib/OptimizationAuglag/test/runtests.jl +++ b/lib/OptimizationAuglag/test/runtests.jl @@ -0,0 +1,31 @@ +using MLUtils, OptimizationOptimisers +using ForwardDiff +using Test + +@testset "OptimizationAuglag.jl" begin + x0 = (-pi):0.001:pi + y0 = sin.(x0) + data = MLUtils.DataLoader((x0, y0), batchsize = 126) + + function loss(coeffs, data) + ypred = [evalpoly(data[1][i], coeffs) for i in eachindex(data[1])] + return sum(abs2, ypred .- data[2]) + end + + function cons1(res, coeffs, p = nothing) + res[1] = coeffs[1] * coeffs[5] - 1 + return nothing + end + + optf = OptimizationFunction(loss, AutoSparseForwardDiff(), cons = cons1) + callback = (st, l) -> (@show l; return false) + + initpars = rand(5) + l0 = optf(initpars, (x0, y0)) + + prob = OptimizationProblem(optf, initpars, data, lcons = [-Inf], ucons = [1], + lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0]) + opt = solve( + prob, Optimization.AugLag(; inner = Adam()), maxiters = 10000, callback = callback) + @test opt.objective < l0 +end \ No newline at end of file From b38ae9ce2ab6f957f6b852553214c66e95f805bf Mon Sep 17 00:00:00 2001 From: paramthakkar123 Date: Thu, 31 Jul 2025 22:45:08 +0530 Subject: [PATCH 3/7] Added tests to OptimizationAuglag --- test/native.jl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/native.jl b/test/native.jl index 0c6c0f6e5..f7385fd0d 100644 --- a/test/native.jl +++ b/test/native.jl @@ -51,12 +51,6 @@ prob = OptimizationProblem(optf, initpars, (x0, y0), lcons = [-Inf], ucons = [0. opt1 = solve(prob, Optimization.LBFGS(), maxiters = 1000, callback = callback) @test opt1.objective < l0 -prob = OptimizationProblem(optf, initpars, data, lcons = [-Inf], ucons = [1], - lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0]) -opt = solve( - prob, Optimization.AugLag(; inner = Adam()), maxiters = 10000, callback = callback) -@test opt.objective < l0 - optf1 = OptimizationFunction(loss, AutoSparseForwardDiff()) prob1 = OptimizationProblem(optf1, rand(5), data) sol1 = solve(prob1, OptimizationOptimisers.Adam(), maxiters = 1000, callback = callback) From 8909e756e7b6eb7de5b9ea4552ee89b6b7a474f5 Mon Sep 17 00:00:00 2001 From: paramthakkar123 Date: Fri, 1 Aug 2025 00:19:28 +0530 Subject: [PATCH 4/7] Updates --- lib/OptimizationAuglag/Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/OptimizationAuglag/Project.toml b/lib/OptimizationAuglag/Project.toml index 20c74eb36..3f01a0e7d 100644 --- a/lib/OptimizationAuglag/Project.toml +++ b/lib/OptimizationAuglag/Project.toml @@ -7,6 +7,7 @@ version = "0.1.0" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" +OptimizationBase = "bca83a33-5cc9-4baa-983d-23429ab6bcbb" OptimizationOptimisers = "42dfb2eb-d2b4-4451-abcd-913932933ac1" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" @@ -14,5 +15,6 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" ForwardDiff = "1.0.1" MLUtils = "0.4.8" Optimization = "4.4.0" +OptimizationBase = "2.10.0" OptimizationOptimisers = "0.3.8" Test = "1.11.0" From 93ef8ed0f077f240bc24644f81434c9710160c60 Mon Sep 17 00:00:00 2001 From: paramthakkar123 Date: Fri, 1 Aug 2025 09:55:18 +0530 Subject: [PATCH 5/7] OptimizationBase switch --- lib/OptimizationAuglag/src/OptimizationAuglag.jl | 3 ++- lib/OptimizationAuglag/test/runtests.jl | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/OptimizationAuglag/src/OptimizationAuglag.jl b/lib/OptimizationAuglag/src/OptimizationAuglag.jl index 2ca6b9f33..5efdfb776 100644 --- a/lib/OptimizationAuglag/src/OptimizationAuglag.jl +++ b/lib/OptimizationAuglag/src/OptimizationAuglag.jl @@ -1,6 +1,7 @@ module OptimizationAuglag -using Optimization.SciMLBase, Optimization +using OptimizationBase.SciMLBase, Optimization +using OptimizationBase.LinearAlgebra: norm @kwdef struct AugLag inner::Any diff --git a/lib/OptimizationAuglag/test/runtests.jl b/lib/OptimizationAuglag/test/runtests.jl index eef9d1c75..60f994265 100644 --- a/lib/OptimizationAuglag/test/runtests.jl +++ b/lib/OptimizationAuglag/test/runtests.jl @@ -1,5 +1,10 @@ -using MLUtils, OptimizationOptimisers +using OptimizationBase +using MLUtils +using OptimizationOptimisers +using OptimizationAuglag using ForwardDiff +using OptimizationBase: OptimizationCache +using OptimizationBase.SciMLBase: OptimizationFunction using Test @testset "OptimizationAuglag.jl" begin @@ -17,7 +22,7 @@ using Test return nothing end - optf = OptimizationFunction(loss, AutoSparseForwardDiff(), cons = cons1) + optf = OptimizationFunction(loss, OptimizationBase.AutoSparseForwardDiff(), cons = cons1) callback = (st, l) -> (@show l; return false) initpars = rand(5) @@ -26,6 +31,6 @@ using Test prob = OptimizationProblem(optf, initpars, data, lcons = [-Inf], ucons = [1], lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0]) opt = solve( - prob, Optimization.AugLag(; inner = Adam()), maxiters = 10000, callback = callback) + prob, OptimizationAuglag.AugLag(; inner = Adam()), maxiters = 10000, callback = callback) @test opt.objective < l0 end \ No newline at end of file From 3c311219aae346f4b819a2b3ca7b05c256ec4a6a Mon Sep 17 00:00:00 2001 From: paramthakkar123 Date: Fri, 1 Aug 2025 10:10:14 +0530 Subject: [PATCH 6/7] Updates --- lib/OptimizationAuglag/src/OptimizationAuglag.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/OptimizationAuglag/src/OptimizationAuglag.jl b/lib/OptimizationAuglag/src/OptimizationAuglag.jl index 5efdfb776..ae74fed87 100644 --- a/lib/OptimizationAuglag/src/OptimizationAuglag.jl +++ b/lib/OptimizationAuglag/src/OptimizationAuglag.jl @@ -1,6 +1,7 @@ module OptimizationAuglag using OptimizationBase.SciMLBase, Optimization +using OptimizationBase.SciMLBase: OptimizationProblem, OptimizationFunction, OptimizationStats using OptimizationBase.LinearAlgebra: norm @kwdef struct AugLag @@ -20,7 +21,7 @@ SciMLBase.requiresgradient(::AugLag) = true SciMLBase.allowsconstraints(::AugLag) = true SciMLBase.requiresconsjac(::AugLag) = true -function __map_optimizer_args(cache::Optimization.OptimizationCache, opt::AugLag; +function __map_optimizer_args(cache::OptimizationBase.OptimizationCache, opt::AugLag; callback = nothing, maxiters::Union{Number, Nothing} = nothing, maxtime::Union{Number, Nothing} = nothing, @@ -110,7 +111,7 @@ function SciMLBase.__solve(cache::OptimizationCache{ cache.f.cons(cons_tmp, θ) cons_tmp[eq_inds] .= cons_tmp[eq_inds] - cache.lcons[eq_inds] cons_tmp[ineq_inds] .= cons_tmp[ineq_inds] .- cache.ucons[ineq_inds] - opt_state = Optimization.OptimizationState(u = θ, objective = x[1], p = p) + opt_state = Optimization.OptimizationState(u = θ, objective = x[1]) if cache.callback(opt_state, x...) error("Optimization halted by callback.") end @@ -177,7 +178,7 @@ function SciMLBase.__solve(cache::OptimizationCache{ break end end - stats = Optimization.OptimizationStats(; iterations = maxiters, + stats = OptimizationStats(; iterations = maxiters, time = 0.0, fevals = maxiters, gevals = maxiters) return SciMLBase.build_solution( cache, cache.opt, θ, x, From ce51b14ae0268eaa2da92aca0c836d023f10759a Mon Sep 17 00:00:00 2001 From: paramthakkar123 Date: Fri, 1 Aug 2025 10:26:23 +0530 Subject: [PATCH 7/7] Updates --- lib/OptimizationAuglag/src/OptimizationAuglag.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/OptimizationAuglag/src/OptimizationAuglag.jl b/lib/OptimizationAuglag/src/OptimizationAuglag.jl index ae74fed87..ad89ccc95 100644 --- a/lib/OptimizationAuglag/src/OptimizationAuglag.jl +++ b/lib/OptimizationAuglag/src/OptimizationAuglag.jl @@ -1,6 +1,6 @@ module OptimizationAuglag -using OptimizationBase.SciMLBase, Optimization +using Optimization using OptimizationBase.SciMLBase: OptimizationProblem, OptimizationFunction, OptimizationStats using OptimizationBase.LinearAlgebra: norm