Use @test_broken instead of skipping (#1105)

A global flag, test_broken (default false), determines whether
this runs during CI. Locally it's useful to be able to tally the
failures, e.g., #1104

Also:
- Fix one failure in L-BFGS that causes CI to fail
- Remove several tests from the exclusions lists that now pass

Author: timholy

test/multivariate/solvers/first_order/cg.jl

@@ -6,12 +6,10 @@
     run_optim_tests(ConjugateGradient(),
                     skip=skip,
                     convergence_exceptions = (("Powell", 1), ("Powell", 2), ("Polynomial", 1),
-                                              ("Extended Rosenbrock", 1),
                                               ("Extended Powell", 1),
                                               ("Extended Powell", 2)),
                     minimum_exceptions = (("Paraboloid Diagonal", 1)),
-                    minimizer_exceptions = (("Paraboloid Diagonal", 1),
-                                            ("Extended Powell", 1),
+                    minimizer_exceptions = (("Extended Powell", 1),
                                             ("Extended Powell", 2)),
                     f_increase_exceptions = (("Hosaki"),),
                     iteration_exceptions = (("Paraboloid Diagonal", 10000),),

test/multivariate/solvers/first_order/l_bfgs.jl

@@ -2,11 +2,7 @@
     # Trigonometric gets stuck in a local minimum?
     skip = ("Trigonometric",)
 
-    if Sys.WORD_SIZE == 32
-        iteration_exceptions = (("Extended Powell", 2000),)
-    else
-        iteration_exceptions = ()
-    end
+    iteration_exceptions = (("Extended Powell", 2000),)
 
     run_optim_tests(LBFGS(),
                     f_increase_exceptions = ("Extended Rosenbrock",),

test/multivariate/solvers/first_order/momentum_gradient_descent.jl

@@ -4,10 +4,7 @@
             "Trigonometric", "Penalty Function I", "Beale","Paraboloid Random Matrix")
     run_optim_tests(MomentumGradientDescent(),
                     skip = skip,
-                    convergence_exceptions = (("Large Polynomial",1),  ("Himmelblau",1),
-                                              ("Fletcher-Powell", 1),("Fletcher-Powell", 2),
-                                              ("Powell", 1)),
-                    minimum_exceptions = (("Large Polynomial", 1), ("Large Polynomial", 2)),
+                    convergence_exceptions = (("Fletcher-Powell", 1), ("Fletcher-Powell", 2),),
                     iteration_exceptions = (("Paraboloid Diagonal", 10000),
                                             ("Powell", 10000)),
                     f_increase_exceptions = ("Exponential", "Polynomial",

test/runtests.jl

@@ -14,6 +14,7 @@ import LinearAlgebra: norm, diag, I, Diagonal, dot, eigen, issymmetric, mul!
 import SparseArrays: normalize!, spdiagm
 
 debug_printing = false
+test_broken = false
 
 special_tests = [
     "bigfloat/initial_convergence",
@@ -155,15 +156,22 @@ function run_optim_tests(method; convergence_exceptions = (),
                         printstyled(name, " did not converge with i = ", i, "\n", color=:red)
                         printstyled(results, "\n", color=:red)
                     end
+                elseif test_broken
+                    @test_broken Optim.converged(results)
                 end
                 if !((name, i) in minimum_exceptions)
                     @test Optim.minimum(results) < prob.minimum + sqrt(eps(typeof(prob.minimum)))
+                elseif test_broken
+                    @test_broken Optim.minimum(results) < prob.minimum + sqrt(eps(typeof(prob.minimum)))
                 end
                 if !((name, i) in minimizer_exceptions)
                     @test norm(Optim.minimizer(results) - prob.solutions) < 1e-2
+                elseif test_broken
+                    @test_broken norm(Optim.minimizer(results) - prob.solutions) < 1e-2
                 end
             end
         else
+            @test_broken false    # marked skipped tests as broken
             debug_printing && printstyled("Skipping $name\n", color=:blue)
         end
     end
@@ -216,14 +224,21 @@ function run_optim_tests_constrained(method; convergence_exceptions = (),
                     printstyled(name, "did not converge\n", color=:red)
                     printstyled(results, "\n", color=:red)
                 end
+            elseif test_broken
+                @test_broken Optim.converged(results)
             end
             if !(name in minimum_exceptions)
                 @test Optim.minimum(results) < prob.minimum + sqrt(eps(typeof(prob.minimum)))
+            elseif test_broken
+                @test_broken Optim.minimum(results) < prob.minimum + sqrt(eps(typeof(prob.minimum)))
             end
             if !(name in minimizer_exceptions)
                 @test norm(Optim.minimizer(results) - prob.solutions) < 1e-2
+            elseif test_broken
+                @test_broken norm(Optim.minimizer(results) - prob.solutions) < 1e-2
             end
         else
+            @test_broken false
             debug_printing && printstyled("Skipping $name\n", color=:blue)
         end
     end