Adapt the Trust RegionsState, start with MADS.

Author: kellertuer

src/plans/mesh_adaptive_plan.jl

@@ -99,15 +99,8 @@ $(_fields([:retraction_method, :vector_transport_method]))
 $(_kwargs([:retraction_method, :vector_transport_method, :X]))
 """
 mutable struct LowerTriangularAdaptivePoll{
-        P,
-        T,
-        F <: Real,
-        V <: AbstractVector{F},
-        M <: AbstractMatrix{F},
-        I <: Int,
-        B,
-        VTM <: AbstractVectorTransportMethod,
-        RM <: AbstractRetractionMethod,
+        P, T, F <: Real, V <: AbstractVector{F}, M <: AbstractMatrix{F},
+        I <: Int, B, VTM <: AbstractVectorTransportMethod, RM <: AbstractRetractionMethod,
     } <: AbstractMeshPollFunction
     base_point::P
     candidate::P
@@ -353,12 +346,8 @@ function show(io::IO, dmads::DefaultMeshAdaptiveDirectSearch)
     return print(io, s)
 end
 function (dmads::DefaultMeshAdaptiveDirectSearch)(
-        amp::AbstractManoptProblem,
-        mesh_size::Real,
-        p,
-        X;
-        scale_mesh::Real = 1.0,
-        max_stepsize::Real = Inf,
+        amp::AbstractManoptProblem, mesh_size::Real, p, X;
+        scale_mesh::Real = 1.0, max_stepsize::Real = Inf,
     )
     M = get_manifold(amp)
     dmads.X .= (4 * mesh_size * scale_mesh) .* X
@@ -391,11 +380,7 @@ $(_fields(:stopping_criterion; name = "stop"))
 
 """
 mutable struct MeshAdaptiveDirectSearchState{
-        P,
-        F <: Real,
-        PT <: AbstractMeshPollFunction,
-        ST <: AbstractMeshSearchFunction,
-        SC <: StoppingCriterion,
+        P, F <: Real, PT <: AbstractMeshPollFunction, ST <: AbstractMeshSearchFunction, SC <: StoppingCriterion,
     } <: AbstractManoptSolverState
     p::P
     mesh_size::F
@@ -407,8 +392,7 @@ mutable struct MeshAdaptiveDirectSearchState{
     search::ST
 end
 function MeshAdaptiveDirectSearchState(
-        M::AbstractManifold,
-        p::P = rand(M);
+        M::AbstractManifold, p::P = rand(M);
         mesh_basis::B = default_basis(M, typeof(p)),
         scale_mesh::F = injectivity_radius(M) / 2,
         max_stepsize::F = injectivity_radius(M),
@@ -428,12 +412,8 @@ function MeshAdaptiveDirectSearchState(
             M, copy(M, p); retraction_method = retraction_method
         ),
     ) where {
-        P,
-        F,
-        PT <: AbstractMeshPollFunction,
-        ST <: AbstractMeshSearchFunction,
-        SC <: StoppingCriterion,
-        B <: AbstractBasis,
+        P, F, PT <: AbstractMeshPollFunction, ST <: AbstractMeshSearchFunction,
+        SC <: StoppingCriterion, B <: AbstractBasis,
     }
     poll_s = manifold_dimension(M) * 1.0
     return MeshAdaptiveDirectSearchState{P, F, PT, ST, SC}(
@@ -443,9 +423,13 @@ end
 get_iterate(mads::MeshAdaptiveDirectSearchState) = mads.p
 
 function status_summary(mads::MeshAdaptiveDirectSearchState; context::Symbol = :default)
-    i = get_count(mads, :Iterations)
+    (context === :short) && repr(mads)
+    i = get_count(trs, :Iterations)
+    conv_inl = (i > 0) ? (indicates_convergence(trs.stop) ? " (converged" : " (stopped") * " after $i iterations)" : ""
+    (context === :inline) && return "A solver state for the trust region solver$(conv_inl)"
     Iter = (i > 0) ? "After $i iterations\n" : ""
-    _is_inline(context) && (return "$(repr(mads)) – $(Iter) $(has_converged(mads) ? "(converged)" : "")")
+    Conv = indicates_convergence(trs.stop) ? "Yes" : "No"
+    (context === :inline) && (return "A trust regions method state – $(Iter) $(has_converged(trs) ? "(converged)" : "")")
     s = """
     # Solver state for `Manopt.jl`s mesh adaptive direct search
     $Iter
@@ -454,11 +438,12 @@ function status_summary(mads::MeshAdaptiveDirectSearchState; context::Symbol = :
     * scale_mesh: $(mads.scale_mesh)
     * max_stepsize: $(mads.max_stepsize)
     * poll_size: $(mads.poll_size)
-    * poll:\n  $(replace(repr(mads.poll), "\n" => "\n  ")[1:(end - 3)])
-    * search:\n  $(replace(repr(mads.search), "\n" => "\n  ")[1:(end - 3)])
+    * poll:\n  $(_in_str(repr(mads.poll); indent = 1))
+    * search:\n  $(_in_str(repr(mads.search); indent = 1))
 
     ## Stopping criterion
     $(status_summary(mads.stop; context = context))
+    This indicates convergence: $Conv
     """
     return s
 end

src/solvers/exact_penalty_method.jl

@@ -126,7 +126,7 @@ function status_summary(epms::ExactPenaltyMethodState; context::Symbol = :defaul
     (context === :inline) && return "A solver state for the exact panelty method$(conv_inl)"
     Iter = (i > 0) ? "After $i iterations\n" : ""
     Conv = indicates_convergence(epms.stop) ? "Yes" : "No"
-    _is_inline(context) && (return "$(repr(epms)) – $(Iter) $(has_converged(epms) ? "(converged)" : "")")
+    (context === :inline) && (return "An exact penalty method state – $(Iter) $(has_converged(epms) ? "(converged)" : "")")
     s = """
     # Solver state for `Manopt.jl`s Exact Penalty Method
     $Iter

src/solvers/trust_regions.jl

@@ -75,36 +75,33 @@ mutable struct TrustRegionsState{
     sub_state::St
     p_proposal::P
     f_proposal::R
+    σ::R
+    reduction_threshold::R
+    reduction_factor::R
+    augmentation_threshold::R
+    augmentation_factor::R
     # Only required for Random mode Random
     HX::T
     Y::T
     HY::T
     Z::T
     HZ::T
     τ::R
-    σ::R
-    reduction_threshold::R
-    reduction_factor::R
-    augmentation_threshold::R
-    augmentation_factor::R
-    function TrustRegionsState{P, T, Pr, St, SC, RTR, R, Proj}(
-            p::P,
-            X::T,
-            trust_region_radius::R,
-            max_trust_region_radius::R,
-            acceptance_rate::R,
-            ρ_regularization::R,
-            randomize::Bool,
-            stopping_criterion::SC,
-            retraction_method::RTR,
-            reduction_threshold::R,
-            augmentation_threshold::R,
-            sub_problem::Pr,
-            sub_state::St,
-            project!::Proj = (copyto!),
-            reduction_factor = 0.25,
-            augmentation_factor = 2.0,
-            σ::R = random ? 1.0e-6 : 0.0,
+    function TrustRegionsState(
+            sub_problem::Pr, sub_state::St;
+            p::P, X::T,
+            trust_region_radius::R, max_trust_region_radius::R, acceptance_rate::R,
+            ρ_regularization::R, randomize::Bool,
+            stopping_criterion::SC, retraction_method::RTR, reduction_threshold::R,
+            augmentation_threshold::R, project!::Proj = (copyto!),
+            reduction_factor::R, augmentation_factor::R, σ::R,
+            #random mode ones can stay uninitielized if not provided
+            HX::Union{T, Nothing} = nothing,
+            Y::Union{T, Nothing} = nothing,
+            HY::Union{T, Nothing} = nothing,
+            Z::Union{T, Nothing} = nothing,
+            HZ::Union{T, Nothing} = nothing,
+            τ::Union{R, Nothing} = nothing,
         ) where {
             P, T, Pr, St <: AbstractManoptSolverState,
             SC <: StoppingCriterion, RTR <: AbstractRetractionMethod, R <: Real, Proj,
@@ -127,51 +124,52 @@ mutable struct TrustRegionsState{
         trs.augmentation_factor = augmentation_factor
         trs.project! = project!
         trs.σ = σ
+        !isnothing(HX) && (trs.HX = HX)
+        !isnothing(Y) && (trs.Y = Y)
+        !isnothing(HY) && (trs.HY = HY)
+        !isnothing(Z) && (trs.HZ = Z)
+        !isnothing(HZ) && (trs.HZ = HZ)
+        !isnothing(τ) && (trs.τ = τ)
         return trs
     end
 end
-
+TrustRegionsState(M::AbstractManifold, st::AbstractManoptSolverState; kwargs...) = error("Trust region method state can not be constructed based on $M and the sub state $st, a sub_problem is missing")
 function TrustRegionsState(
         M::AbstractManifold, sub_problem::Pr, sub_state::St;
         p::P = rand(M), X::T = zero_vector(M, p),
-        acceptance_rate = 0.1,
-        ρ_regularization::R = 1000.0,
+        acceptance_rate::Real = 0.1, ρ_regularization::Real = 1000.0,
         randomize::Bool = false,
         stopping_criterion::SC = StopAfterIteration(1000) | StopWhenGradientNormLess(1.0e-6),
-        max_trust_region_radius::R = sqrt(manifold_dimension(M)),
-        trust_region_radius::R = max_trust_region_radius / 8,
+        max_trust_region_radius::Real = sqrt(manifold_dimension(M)),
+        trust_region_radius::Real = max_trust_region_radius / 8,
         retraction_method::RTR = default_retraction_method(M, typeof(p)),
-        reduction_threshold::R = 0.1,
-        reduction_factor = 0.25,
-        augmentation_threshold::R = 0.75,
-        augmentation_factor = 2.0,
-        project!::Proj = (copyto!),
-        σ = randomize ? 1.0e-4 : 0.0,
+        reduction_threshold::Real = 0.1, reduction_factor = 0.25,
+        augmentation_threshold::Real = 0.75, augmentation_factor::Real = 2.0,
+        project!::Proj = (copyto!), σ::Real = randomize ? 1.0e-4 : 0.0,
     ) where {
         P, T, Pr <: Union{AbstractManoptProblem, F} where {F}, St <: AbstractManoptSolverState,
-        R <: Real,
-        SC <: StoppingCriterion,
-        RTR <: AbstractRetractionMethod,
-        Proj,
+        SC <: StoppingCriterion, RTR <: AbstractRetractionMethod, Proj,
     }
-    return TrustRegionsState{P, T, Pr, St, SC, RTR, R, Proj}(
-        p,
-        X,
-        trust_region_radius,
-        max_trust_region_radius,
-        acceptance_rate,
-        ρ_regularization,
-        randomize,
-        stopping_criterion,
-        retraction_method,
-        reduction_threshold,
-        augmentation_threshold,
-        sub_problem,
-        sub_state,
-        project!,
-        reduction_factor,
-        augmentation_factor,
-        σ,
+    R = promote_type(
+        typeof(acceptance_rate), typeof(ρ_regularization), typeof(max_trust_region_radius),
+        typeof(trust_region_radius), typeof(reduction_threshold), typeof(reduction_factor),
+        typeof(augmentation_factor), typeof(augmentation_threshold), typeof(σ)
+    )
+    acceptance_rate = convert(R, acceptance_rate); ρ_regularization = convert(R, ρ_regularization)
+    max_trust_region_radius = convert(R, max_trust_region_radius); trust_region_radius = convert(R, trust_region_radius)
+    reduction_threshold = convert(R, reduction_threshold); reduction_factor = convert(R, reduction_factor)
+    augmentation_factor = convert(R, augmentation_factor); augmentation_threshold = convert(R, augmentation_threshold)
+    σ = convert(R, σ)
+
+    return TrustRegionsState(
+        sub_problem, sub_state;
+        p = p, X = X,
+        trust_region_radius = trust_region_radius, max_trust_region_radius = max_trust_region_radius,
+        acceptance_rate = acceptance_rate, ρ_regularization = ρ_regularization,
+        (project!) = project!, randomize = randomize, σ = σ,
+        stopping_criterion = stopping_criterion, retraction_method = retraction_method,
+        reduction_threshold = reduction_threshold, augmentation_threshold = augmentation_threshold,
+        reduction_factor = reduction_factor, augmentation_factor = augmentation_factor,
     )
 end
 function TrustRegionsState(
@@ -203,14 +201,31 @@ function get_message(dcs::TrustRegionsState)
     # for now only the sub solver might have messages
     return get_message(dcs.sub_state)
 end
+function Base.show(io::IO, trs::TrustRegionsState)
+    print(io, "TrustRegionsState("); print(io, trs.sub_problem); print(io, ", "); print(io, trs.sub_state)
+    print(io, "; ")
+    print(io, "p = $(trs.p), X = $(trs.X), ")
+    print(io, "trust_region_radius = $(trs.trust_region_radius), max_trust_region_radius = $(trs.max_trust_region_radius), ")
+    print(io, "acceptance_rate = $(trs.acceptance_rate), ρ_regularization = $(trs.ρ_regularization), randomize = $(trs.randomize), ")
+    print(io, "reduction_threshold = $(trs.reduction_threshold), augmentation_threshold = $(trs.augmentation_threshold), ")
+    print(io, "(project!) = $(trs.project!), reduction_factor = $(trs.reduction_factor), augmentation_factor = $(trs.augmentation_factor), σ = $(trs.σ), ")
+    isdefined(trs, :HX) && print(io, "HX = $(trs.HX), ")
+    isdefined(trs, :Y) && print(io, "Y = $(trs.Y), ")
+    isdefined(trs, :HY) && print(io, "HY = $(trs.HY), ")
+    isdefined(trs, :Z) && print(io, "Z = $(trs.Z), ")
+    isdefined(trs, :HZ) && print(io, "HZ = $(trs.HZ), ")
+    isdefined(trs, :τ) && print(io, "τ = $(trs.τ), ")
+    print(io, "stopping_criterion = $(trs.stop), retraction_method = $(trs.retraction_method)")
+    return print(io, ")")
+end
 function status_summary(trs::TrustRegionsState; context::Symbol = :default)
     (context === :short) && return repr(trs)
     i = get_count(trs, :Iterations)
     conv_inl = (i > 0) ? (indicates_convergence(trs.stop) ? " (converged" : " (stopped") * " after $i iterations)" : ""
     (context === :inline) && return "A solver state for the trust region solver$(conv_inl)"
     Iter = (i > 0) ? "After $i iterations\n" : ""
     Conv = indicates_convergence(trs.stop) ? "Yes" : "No"
-    _is_inline(context) && (return "$(repr(trs)) – $(Iter) $(has_converged(trs) ? "(converged)" : "")")
+    (context === :inline) && (return "A trust regions method state – $(Iter) $(has_converged(trs) ? "(converged)" : "")")
     sub = _in_str(status_summary(trs.sub_state; context = context); indent = 1, headers = 1, indent_end = "| ")
     s = """
     # Solver state for `Manopt.jl`s Trust Region Method

test/solvers/test_trust_regions.jl

@@ -30,6 +30,7 @@ include("trust_region_model.jl")
         sub_state = TruncatedConjugateGradientState(TpM; X = get_gradient(M, mho, p))
         trs1 = TrustRegionsState(M, sub_problem)
         trs2 = TrustRegionsState(M, sub_problem, sub_state)
+        @test_throws ErrorException TrustRegionsState(M, sub_state)
         trs3 = TrustRegionsState(M, sub_problem; p = p)
         @test Manopt.get_gradient_function(sub_objective)(M, p) == X
     end
@@ -58,17 +59,24 @@ include("trust_region_model.jl")
             Manopt.status_summary(s; context = :default),
             "# Solver state for `Manopt.jl`s Trust Region Method\n"
         )
+        @test startswith(repr(s), "TrustRegionsState(")
+        # not a random one -> does not contain HZ
+        @test !contains(repr(s), "HZ = ")
         p1 = get_solver_result(s)
         q = copy(M, p)
         set_gradient!(s, M, p, zero_vector(M, p))
         @test norm(M, p, get_gradient(s)) ≈ 0.0
         trust_regions!(M, f, rgrad, rhess, q; max_trust_region_radius = 8.0)
         @test isapprox(M, p1, q)
         Random.seed!(42)
-        p2 = trust_regions(
-            M, f, rgrad, rhess, p; max_trust_region_radius = 8.0, randomize = true
+        s2 = trust_regions(
+            M, f, rgrad, rhess, p; max_trust_region_radius = 8.0, randomize = true, return_state = true
         )
+        @test startswith(repr(s2), "TrustRegionsState(")
+        # a random one -> does contain HZ
+        @test contains(repr(s2), "HZ = ")
 
+        p2 = get_solver_result(s2)
         @test f(M, p2) ≈ f(M, p1)
 
         p3 = trust_regions(