support for generators instead of curly syntax in Julia 0.5

src/nlpmacros.jl

@@ -11,6 +11,62 @@
 # values is the name of the list of constants which appear in the expression
 function parseNLExpr(m, x, tapevar, parent, values)
 
+    if isexpr(x,:call) && length(x.args) >= 2 && isexpr(x.args[2],:generator)
+        header = x.args[1]
+        if issum(header)
+            operatorid = operator_to_id[:+]
+        elseif isprod(header)
+            operatorid = operator_to_id[:*]
+        else
+            error("Unrecognized expression $header(...)")
+        end
+        codeblock = :(let; end)
+        block = codeblock.args[1]
+        push!(block.args, :(push!($tapevar, NodeData(CALL, $operatorid, $parent))))
+        parentvar = gensym()
+        push!(block.args, :($parentvar = length($tapevar)))
+
+        # we have a filter condition
+        if isexpr(x.args[2].args[2],:filter)
+            cond = x.args[2].args[2].args[1]
+            # generate inner loop code first and then wrap in for loops
+            innercode = parseNLExpr(m, x.args[2].args[1], tapevar, parentvar, values)
+            code = quote
+                if $(esc(cond))
+                    $innercode
+                end
+            end
+            for level in length(x.args[2].args[2]):-1:2
+                _idxvar, idxset = parseIdxSet(x.args[2].args[2].args[level]::Expr)
+                idxvar = esc(_idxvar)
+                code = :(let
+                    $(localvar(idxvar))
+                    for $idxvar in $(esc(idxset))
+                        $code
+                    end
+                end)
+            end
+            push!(block.args, code)
+        else # no condition
+            innercode = parseNLExpr(m, x.args[2].args[1], tapevar, parentvar, values)
+            code = quote
+                $innercode
+            end
+            for level in length(x.args[2].args):-1:2
+                _idxvar, idxset = parseIdxSet(x.args[2].args[level]::Expr)
+                idxvar = esc(_idxvar)
+                code = :(let
+                    $(localvar(idxvar))
+                    for $idxvar in $(esc(idxset))
+                        $code
+                    end
+                end)
+            end
+            push!(block.args, code)
+        end
+        return codeblock
+    end
+
     if isexpr(x, :call)
         if length(x.args) == 2 # univariate
             code = :(let; end)

src/parseExpr_staged.jl

@@ -443,6 +443,152 @@ function parseNorm(normp::Symbol, x::Expr, aff::Symbol, lcoeffs, rcoeffs, newaff
     end
 end
 
+function parseGenerator(x::Expr, aff::Symbol, lcoeffs, rcoeffs, newaff=gensym())
+    @assert isexpr(x,:call)
+    @assert length(x.args) > 1
+    @assert isexpr(x.args[2],:generator)
+    header = x.args[1]
+    if issum(header)
+        parseGeneratorSum(x.args[2], aff, lcoeffs, rcoeffs, newaff)
+    elseif header ∈ [:norm1, :norm2, :norminf, :norm∞]
+        parseGeneratorNorm(header, x.args[2], aff, lcoeffs, rcoeffs, newaff)
+    else
+        error("Expected sum or norm2 outside generator expression; got $header")
+    end
+end
+
+function parseGeneratorSum(x::Expr, aff::Symbol, lcoeffs, rcoeffs, newaff)
+    # we have a filter condition
+    if isexpr(x.args[2],:filter)
+        cond = x.args[2].args[1]
+        # generate inner loop code first and then wrap in for loops
+        inneraff, innercode = parseExpr(x.args[1], aff, lcoeffs, rcoeffs, aff)
+        code = quote
+            if $(esc(cond))
+                $innercode
+            end
+        end
+        for level in length(x.args[2].args):-1:2
+            _idxvar, idxset = parseIdxSet(x.args[2].args[level]::Expr)
+            idxvar = esc(_idxvar)
+            code = :(let
+                $(localvar(idxvar))
+                for $idxvar in $(esc(idxset))
+                    $code
+                end
+            end)
+        end
+    else # no condition
+        inneraff, code = parseExpr(x.args[1], aff, lcoeffs, rcoeffs, aff)
+        for level in length(x.args):-1:2
+            _idxvar, idxset = parseIdxSet(x.args[level]::Expr)
+            idxvar = esc(_idxvar)
+            code = :(let
+                $(localvar(idxvar))
+                for $idxvar in $(esc(idxset))
+                    $code
+                end
+            end)
+        end
+        len = :len
+        # precompute the number of elements to add
+        # this is unncessary if we're just summing constants
+        _, lastidxset = parseIdxSet(x.args[length(x.args)]::Expr)
+        preblock = :($len += length($(esc(lastidxset))))
+        for level in (length(x.args)-1):-1:2
+            _idxvar, idxset = parseIdxSet(x.args[level]::Expr)
+            idxvar = esc(_idxvar)
+            preblock = :(let
+                $(localvar(idxvar))
+                for $idxvar in $(esc(idxset))
+                    $preblock
+                end
+            end)
+        end
+        preblock = quote
+            $len = 0
+            $preblock
+            _sizehint_expr!($aff,$len)
+        end
+        code = :($preblock;$code)
+    end
+    :($code; $newaff=$aff)
+end
+
+function parseGeneratorNorm(normp::Symbol, x::Expr, aff::Symbol, lcoeffs, rcoeffs, newaff)
+    @assert string(normp)[1:4] == "norm"
+    finalaff = gensym()
+    gennorm  = gensym()
+    len      = gensym()
+    normexpr = gensym()
+    # we have a filter condition
+    if isexpr(x.args[2],:filter)
+        cond = x.args[2].args[1]
+        # generate inner loop code first and then wrap in for loops
+        inneraff, innercode = parseExprToplevel(x.args[1], :normaff)
+        code = quote
+            if $(esc(cond))
+                normaff = 0.0
+                $innercode
+                push!($normexpr, $inneraff)
+            end
+        end
+        for level in length(x.args[2].args):-1:2
+            _idxvar, idxset = parseIdxSet(x.args[2].args[level]::Expr)
+            idxvar = esc(_idxvar)
+            code = :(let
+                $(localvar(idxvar))
+                for $idxvar in $(esc(idxset))
+                    $code
+                end
+            end)
+        end
+        preblock = :($normexpr = GenericAffExpr[])
+    else # no condition
+        inneraff, code = parseExprToplevel(x.args[1], :normaff)
+        code = :(normaff = 0.0; $code; push!($normexpr, $inneraff))
+        _, lastidxset = parseIdxSet(x.args[length(x.args)]::Expr)
+        preblock = :($len += length($(esc(lastidxset))))
+        for level in length(x.args):-1:2
+            _idxvar, idxset = parseIdxSet(x.args[level]::Expr)
+            idxvar = esc(_idxvar)
+            code = :(let
+                $(localvar(idxvar))
+                for $idxvar in $(esc(idxset))
+                    $code
+                end
+            end)
+            preblock = :(let
+                $(localvar(idxvar))
+                for $idxvar in $(esc(idxset))
+                    $preblock
+                end
+            end)
+        end
+        preblock = quote
+            $len = 0
+            $preblock
+            $normexpr = GenericAffExpr[]
+            _sizehint_expr!($normexpr, $len)
+        end
+    end
+    if normp == :norm2
+        param = 2
+    elseif normp == :norm1
+        param = 1
+    elseif normp ∈ [:norminf, :norm∞]
+        param = Inf
+    else
+        error("Unrecognized norm: $normp")
+    end
+    quote
+        $preblock
+        $code
+        $gennorm = _build_norm($param,$normexpr)
+        $newaff = $(Expr(:call, :addtoexpr_reorder, aff,lcoeffs...,gennorm,rcoeffs...))
+    end
+end
+
 is_complex_expr(ex) = isa(ex,Expr) && !isexpr(ex,:ref)
 
 parseExprToplevel(x, aff::Symbol) = parseExpr(x, aff, [], [])
@@ -522,6 +668,8 @@ function parseExpr(x, aff::Symbol, lcoeffs::Vector, rcoeffs::Vector, newaff::Sym
             numerator = x.args[2]
             denom = x.args[3]
             return parseExpr(numerator, aff, lcoeffs,vcat(esc(:(1/$denom)),rcoeffs),newaff)
+        elseif isexpr(x,:call) && length(x.args) >= 2 && isexpr(x.args[2],:generator)
+            return newaff, parseGenerator(x,aff,lcoeffs,rcoeffs,newaff)
         elseif x.head == :curly
             return newaff, parseCurly(x,aff,lcoeffs,rcoeffs,newaff)
         else # at lowest level?

test/macros.jl

@@ -42,6 +42,30 @@ facts("[macros] Check Julia expression parsing") do
     @fact sumexpr.args[4].head --> :(=)
 end
 
+# generator syntax only parses on 0.5
+if VERSION >= v"0.5-dev+5475"
+    eval("""
+facts("[macros] Check Julia expression parsing (0.5)") do
+    sumexpr = :(sum(x[i,j] * y[i,j] for i = 1:N, j in 1:M if i != j))
+    @fact sumexpr.head --> :call
+    @fact sumexpr.args[1] --> :sum
+    @fact sumexpr.args[2].head --> :generator
+    @fact sumexpr.args[2].args[1] --> :(x[i,j] * y[i,j])
+    @fact sumexpr.args[2].args[2].head --> :filter
+    @fact sumexpr.args[2].args[2].args[1] --> :(i != j)
+    @fact sumexpr.args[2].args[2].args[2] --> :(i = 1:N)
+    @fact sumexpr.args[2].args[2].args[3] --> :(j = 1:M)
+
+    sumexpr = :(sum(x[i,j] * y[i,j] for i = 1:N, j in 1:M))
+    @fact sumexpr.head --> :call
+    @fact sumexpr.args[1] --> :sum
+    @fact sumexpr.args[2].head --> :generator
+    @fact sumexpr.args[2].args[1] --> :(x[i,j] * y[i,j])
+    @fact sumexpr.args[2].args[2] --> :(i = 1:N)
+    @fact sumexpr.args[2].args[3] --> :(j = 1:M)
+end
+"""); end
+
 facts("[macros] Check @constraint basics") do
     m = Model()
     @variable(m, w)
@@ -106,6 +130,29 @@ facts("[macros] sum{}") do
 
 end
 
+if VERSION >= v"0.5-dev+5475"
+eval("""
+facts("[macros] sum(generator)") do
+    m = Model()
+    @variable(m, x[1:3,1:3])
+    @variable(m, y)
+    C = [1 2 3; 4 5 6; 7 8 9]
+    @constraint(m, sum( C[i,j]*x[i,j] for i in 1:2, j = 2:3 ) <= 1)
+    @fact string(m.linconstr[end]) --> "2 x[1,2] + 3 x[1,3] + 5 x[2,2] + 6 x[2,3] $leq 1"
+    @constraint(m, sum( C[i,j]*x[i,j] for i = 1:3, j in 1:3 if i != j) == y)
+    @fact string(m.linconstr[end]) --> "2 x[1,2] + 3 x[1,3] + 4 x[2,1] + 6 x[2,3] + 7 x[3,1] + 8 x[3,2] - y $eq 0"
+
+    @constraint(m, sum( C[i,j]*x[i,j] for i = 1:3, j = 1:i) == 0);
+    @fact string(m.linconstr[end]) --> "x[1,1] + 4 x[2,1] + 5 x[2,2] + 7 x[3,1] + 8 x[3,2] + 9 x[3,3] $eq 0"
+
+    @constraint(m, sum( 0*x[i,1] for i=1:3) == 0)
+    @fact string(m.linconstr[end]) --> "0 $eq 0"
+
+    @constraint(m, sum( 0*x[i,1] + y for i=1:3) == 0)
+    @fact string(m.linconstr[end]) --> "3 y $eq 0"
+
+end"""); end
+
 facts("[macros] Problem modification") do
     m = Model()
     @variable(m, x[1:3,1:3])
@@ -387,6 +434,21 @@ facts("[macros] Norm parsing") do
     @fact_throws @constraint(model, norm2{x[i,j], i=1:2, j=1:2} + x[1,2] >= -1)
 end
 
+if VERSION >= v"0.5-dev+5475"
+eval("""
+facts("[macros] Generator norm parsing") do
+    model = Model()
+    @variable(model, x[1:2,1:2])
+    @constraint(model, -2norm2(x[i,j] for i in 1:2, j=1:2) + x[1,2] >= -1)
+    @constraint(model, -2norm2(x[i,j] for i=1:2, j in 1:2 if iseven(i+j)) + x[1,2] >= -1)
+    @constraint(model, 1 >= 2*norm2(x[i,1] for i in 1:2))
+    @fact string(model.socconstr[1]) --> "2.0 $Vert[x[1,1],x[1,2],x[2,1],x[2,2]]$Vert$sub2 $leq x[1,2] + 1"
+    @fact string(model.socconstr[2]) --> "2.0 $Vert[x[1,1],x[2,2]]$Vert$sub2 $leq x[1,2] + 1"
+    @fact string(model.socconstr[3]) --> "2.0 $Vert[x[1,1],x[2,1]]$Vert$sub2 $leq 1"
+    @fact_throws @constraint(model, (x[1,1]+1)*norm2(x[i,j] for i=1:2, j=1:2) + x[1,2] >= -1)
+    @fact_throws @constraint(model, norm2(x[i,j] for i=1:2, j=1:2) + x[1,2] >= -1)
+end""");end
+
 facts("[macros] Extraneous terms in QuadExpr (#535)") do
     model = Model()
     @variable(model, x)

test/nonlinear.jl

@@ -60,6 +60,31 @@ context("With solver $(typeof(nlp_solver))") do
         [1.000000, 4.742999, 3.821150, 1.379408], 1e-5)
 end; end; end
 
+if VERSION >= v"0.5-dev+5475"
+eval("""
+facts("[nonlinear] Test HS071 solves correctly (generators)") do
+    # hs071
+    # Polynomial objective and constraints
+    # min x1 * x4 * (x1 + x2 + x3) + x3
+    # st  x1 * x2 * x3 * x4 >= 25
+    #     x1^2 + x2^2 + x3^2 + x4^2 = 40
+    #     1 <= x1, x2, x3, x4 <= 5
+    # Start at (1,5,5,1)
+    # End at (1.000..., 4.743..., 3.821..., 1.379...)
+    m = Model()
+    initval = [1,5,5,1]
+    @variable(m, 1 <= x[i=1:4] <= 5, start=initval[i])
+    @NLobjective(m, Min, x[1]*x[4]*(x[1]+x[2]+x[3]) + x[3])
+    @NLconstraint(m, x[1]*x[2]*x[3]*x[4] >= 25)
+    @NLconstraint(m, sum(x[i]^2 for i=1:4) == 40)
+    @fact MathProgBase.numconstr(m) --> 2
+    status = solve(m)
+
+    @fact status --> :Optimal
+    @fact getvalue(x)[:] --> roughly(
+        [1.000000, 4.742999, 3.821150, 1.379408], 1e-5)
+end""");end
+
 
 facts("[nonlinear] Test HS071 solves correctly, epigraph") do
 for nlp_solver in nlp_solvers
@@ -497,6 +522,15 @@ context("With solver $(typeof(nlp_solver))") do
     @fact getvalue(x) --> roughly(ones(18),1e-4)
 end; end; end
 
+if VERSION >= v"0.5-dev+5475"
+eval("""
+facts("[nonlinear] Test Hessian chunking code (generators)") do
+    m = Model()
+    @variable(m, x[1:18] >= 1, start = 1.2)
+    @NLobjective(m, Min, prod(x[i] for i=1:18))
+    @fact solve(m) --> :Optimal
+    @fact getvalue(x) --> roughly(ones(18),1e-4)
+end"""); end
 
 #############################################################################
 # Test that output is produced in correct MPB form