From e8f66b7eee07aa573a5fc87227410f507731219e Mon Sep 17 00:00:00 2001 From: odow Date: Thu, 21 Jul 2022 11:32:01 +1200 Subject: [PATCH 1/4] [FileFormats.MPS] add writers for quadratic models --- src/FileFormats/MPS/MPS.jl | 242 ++++++++++++++++++++++++++++-------- test/FileFormats/MPS/MPS.jl | 137 +++++++++++++++++--- 2 files changed, 312 insertions(+), 67 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 6ddd4c1ac7..e8f64b4fb3 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -32,25 +32,32 @@ MOI.Utilities.@model( (), (MOI.SOS1, MOI.SOS2), (), - (MOI.ScalarAffineFunction,), + (MOI.ScalarAffineFunction, MOI.ScalarQuadraticFunction), (MOI.VectorOfVariables,), () ) -function MOI.supports( - ::Model{T}, - ::MOI.ObjectiveFunction{MOI.ScalarQuadraticFunction{T}}, -) where {T} - return false -end +@enum( + QuadraticFormat, + kQuadraticFormatCPLEX, + kQuadraticFormatGurobi, + # TODO(odow): kQuadraticFormatMosek +) struct Options warn::Bool objsense::Bool generic_names::Bool + quadratic_format::QuadraticFormat end -get_options(m::Model) = get(m.ext, :MPS_OPTIONS, Options(false, false, false)) +function get_options(m::Model) + return get( + m.ext, + :MPS_OPTIONS, + Options(false, false, false, kQuadraticFormatGurobi), + ) +end """ Model(; kwargs...) @@ -64,14 +71,19 @@ Keyword arguments are: - `generic_names::Bool=false`: strip all names in the model and replace them with the generic names `C\$i` and `R\$i` for the i'th column and row respectively. + - `quadratic_format::QuadraticFormat = kQuadraticFormatGurobi`: specify the + solver-specific extension used when writing the quadratic components of the + model. Options are `kQuadraticFormatGurobi` and `kQuadraticFormatCPLEX`. """ function Model(; warn::Bool = false, print_objsense::Bool = false, generic_names::Bool = false, + quadratic_format::QuadraticFormat = kQuadraticFormatGurobi, ) model = Model{Float64}() - model.ext[:MPS_OPTIONS] = Options(warn, print_objsense, generic_names) + model.ext[:MPS_OPTIONS] = + Options(warn, print_objsense, generic_names, quadratic_format) return model end @@ -184,10 +196,12 @@ function Base.write(io::IO, model::Model) end ordered_names = String[] names = Dict{MOI.VariableIndex,String}() - for x in MOI.get(model, MOI.ListOfVariableIndices()) + var_to_column = Dict{MOI.VariableIndex,Int}() + for (i, x) in enumerate(MOI.get(model, MOI.ListOfVariableIndices())) n = MOI.get(model, MOI.VariableName(), x) push!(ordered_names, n) names[x] = n + var_to_column[x] = i end write_model_name(io, model) flip_obj = false @@ -205,7 +219,16 @@ function Base.write(io::IO, model::Model) write_rhs(io, model, obj_const) write_ranges(io, model) write_bounds(io, model, ordered_names, names) + write_quadobj(io, model, ordered_names, var_to_column) + if options.quadratic_format == kQuadraticFormatGurobi + # Gurobi needs qcons _after_ quadobj and _before_ SOS. + write_quadcons(io, model, ordered_names, var_to_column) + end write_sos(io, model, names) + if options.quadratic_format == kQuadraticFormatCPLEX + # CPLEX needs qcons _after_ SOS. + write_quadcons(io, model, ordered_names, var_to_column) + end println(io, "ENDATA") return end @@ -231,11 +254,11 @@ const SET_TYPES = ( (MOI.Interval{Float64}, "L"), # See the note in the RANGES section. ) -function _write_rows(io, model, S, sense_char) - for index in MOI.get( - model, - MOI.ListOfConstraintIndices{MOI.ScalarAffineFunction{Float64},S}(), - ) +const FUNC_TYPES = + (MOI.ScalarAffineFunction{Float64}, MOI.ScalarQuadraticFunction{Float64}) + +function _write_rows(io, model, F, S, sense_char) + for index in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) row_name = MOI.get(model, MOI.ConstraintName(), index) if row_name == "" error("Row name is empty: $(index).") @@ -245,11 +268,8 @@ function _write_rows(io, model, S, sense_char) return end -function _write_rows(io, model, S::Type{MOI.Interval{Float64}}, ::Any) - for index in MOI.get( - model, - MOI.ListOfConstraintIndices{MOI.ScalarAffineFunction{Float64},S}(), - ) +function _write_rows(io, model, F, S::Type{MOI.Interval{Float64}}, ::Any) + for index in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) row_name = MOI.get(model, MOI.ConstraintName(), index) set = MOI.get(model, MOI.ConstraintSet(), index) if set.lower == -Inf && set.upper == Inf @@ -269,7 +289,9 @@ function write_rows(io::IO, model::Model) println(io, "ROWS") println(io, Card(f1 = "N", f2 = "OBJ")) for (set_type, sense_char) in SET_TYPES - _write_rows(io, model, set_type, sense_char) + for F in FUNC_TYPES + _write_rows(io, model, F, set_type, sense_char) + end end return end @@ -310,16 +332,29 @@ function _extract_terms( return end +function _extract_terms( + v_names::Dict{MOI.VariableIndex,String}, + coefficients::Dict{String,Vector{Tuple{String,Float64}}}, + row_name::String, + func::MOI.ScalarQuadraticFunction, + flip_sign::Bool = false, +) + for term in func.affine_terms + variable_name = v_names[term.variable] + coef = flip_sign ? -term.coefficient : term.coefficient + push!(coefficients[variable_name], (row_name, coef)) + end + return +end + function _collect_coefficients( model, + F, S, v_names::Dict{MOI.VariableIndex,String}, coefficients::Dict{String,Vector{Tuple{String,Float64}}}, ) - for index in MOI.get( - model, - MOI.ListOfConstraintIndices{MOI.ScalarAffineFunction{Float64},S}(), - ) + for index in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) row_name = MOI.get(model, MOI.ConstraintName(), index) func = MOI.get(model, MOI.ConstraintFunction(), index) _extract_terms(v_names, coefficients, row_name, func) @@ -327,19 +362,27 @@ function _collect_coefficients( return end +function _get_objective(model) + F = MOI.get(model, MOI.ObjectiveFunctionType()) + f = MOI.get(model, MOI.ObjectiveFunction{F}()) + if f isa MOI.VariableIndex + return convert(MOI.ScalarAffineFunction{Float64}, f) + end + return f +end + function write_columns(io::IO, model::Model, flip_obj, ordered_names, names) coefficients = Dict{String,Vector{Tuple{String,Float64}}}( n => Tuple{String,Float64}[] for n in ordered_names ) # Build constraint coefficients for (S, _) in SET_TYPES - _collect_coefficients(model, S, names, coefficients) + for F in FUNC_TYPES + _collect_coefficients(model, F, S, names, coefficients) + end end # Build objective - obj_func = MOI.get( - model, - MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}(), - ) + obj_func = _get_objective(model) _extract_terms(names, coefficients, "OBJ", obj_func, flip_obj) integer_variables = list_of_integer_variables(model, names) println(io, "COLUMNS") @@ -380,11 +423,8 @@ _value(set::MOI.LessThan) = set.upper _value(set::MOI.GreaterThan) = set.lower _value(set::MOI.EqualTo) = set.value -function _write_rhs(io, model, S) - for index in MOI.get( - model, - MOI.ListOfConstraintIndices{MOI.ScalarAffineFunction{Float64},S}(), - ) +function _write_rhs(io, model, F, S) + for index in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) row_name = MOI.get(model, MOI.ConstraintName(), index) set = MOI.get(model, MOI.ConstraintSet(), index) println( @@ -399,11 +439,8 @@ function _write_rhs(io, model, S) return end -function _write_rhs(io, model, S::Type{MOI.Interval{Float64}}) - for index in MOI.get( - model, - MOI.ListOfConstraintIndices{MOI.ScalarAffineFunction{Float64},S}(), - ) +function _write_rhs(io, model, F, S::Type{MOI.Interval{Float64}}) + for index in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) row_name = MOI.get(model, MOI.ConstraintName(), index) set = MOI.get(model, MOI.ConstraintSet(), index) if set.lower == -Inf && set.upper == Inf @@ -425,7 +462,9 @@ end function write_rhs(io::IO, model::Model, obj_const) println(io, "RHS") for (set_type, _) in SET_TYPES - _write_rhs(io, model, set_type) + for F in FUNC_TYPES + _write_rhs(io, model, F, set_type) + end end # Objective constants are added to the RHS as a negative offset. # https://www.ibm.com/docs/en/icos/20.1.0?topic=standard-records-in-mps-format @@ -454,20 +493,14 @@ end # E | + | rhs | rhs + range # E | - | rhs + range | rhs # -# We elect to write out ScalarAffineFunction-in-Interval constraints in terms of -# LessThan (L) constraints with a range shift. The RHS term is set to the upper -# bound, and the RANGE term to upper - lower. +# We elect to write out F-in-Interval constraints in terms of LessThan (L) +# constraints with a range shift. The RHS term is set to the upper bound, and +# the RANGE term to upper - lower. # ============================================================================== -function write_ranges(io::IO, model::Model) - println(io, "RANGES") - for index in MOI.get( - model, - MOI.ListOfConstraintIndices{ - MOI.ScalarAffineFunction{Float64}, - MOI.Interval{Float64}, - }(), - ) +function _write_ranges(io::IO, model::Model, ::Type{F}) where {F} + cis = MOI.get(model, MOI.ListOfConstraintIndices{F,MOI.Interval{Float64}}()) + for index in cis set = MOI.get(model, MOI.ConstraintSet(), index)::MOI.Interval{Float64} if isfinite(set.upper - set.lower) # We only need to write the range if the bounds are both finite @@ -479,6 +512,14 @@ function write_ranges(io::IO, model::Model) return end +function write_ranges(io::IO, model::Model) + println(io, "RANGES") + for F in FUNC_TYPES + _write_ranges(io, model, F) + end + return +end + # ============================================================================== # BOUNDS # Variables default to [0, ∞). @@ -602,6 +643,101 @@ function write_bounds(io::IO, model::Model, ordered_names, names) return end +# ============================================================================== +# QUADRATIC OBJECTIVE +# ============================================================================== + +function write_quadobj(io::IO, model::Model, ordered_names, var_to_column) + f = _get_objective(model) + if !(f isa MOI.ScalarQuadraticFunction{Float64}) + return + end + options = get_options(model) + if options.quadratic_format == kQuadraticFormatGurobi + println(io, "QUADOBJ") + else + println(io, "QMATRIX") + end + _write_q_matrix( + io, + f, + ordered_names, + var_to_column; + duplicate_off_diagonal = options.quadratic_format == + kQuadraticFormatCPLEX, + ) + return +end + +function _write_q_matrix( + io::IO, + f, + ordered_names, + var_to_column; + duplicate_off_diagonal::Bool, +) + # Convert the quadratic terms into matrix form. We don't need to scale + # because MOI uses the same Q/2 format as Gurobi, but we do need to ensure + # we collate off-diagonal terms in the lower-triangular. + terms = Dict{Tuple{Int,Int},Float64}() + for term in f.quadratic_terms + x, y = var_to_column[term.variable_1], var_to_column[term.variable_2] + if x > y + x, y = y, x + end + if haskey(terms, (x, y)) + terms[(x, y)] += term.coefficient + else + terms[(x, y)] = term.coefficient + end + end + # Use sort for reproducibility, and so the Q matrix is given in order. + for (x, y) in sort!(collect(keys(terms))) + println( + io, + Card( + f2 = ordered_names[x], + f3 = ordered_names[y], + f4 = sprint(print_shortest, terms[(x, y)]), + ), + ) + if x != y && duplicate_off_diagonal + println( + io, + Card( + f2 = ordered_names[y], + f3 = ordered_names[x], + f4 = sprint(print_shortest, terms[(x, y)]), + ), + ) + end + end + return +end + +# ============================================================================== +# QUADRATIC CONSTRAINTS +# ============================================================================== + +function write_quadcons(io::IO, model::Model, ordered_names, var_to_column) + F = MOI.ScalarQuadraticFunction{Float64} + for (S, _) in SET_TYPES + for ci in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) + name = MOI.get(model, MOI.ConstraintName(), ci) + println(io, "QCMATRIX $name") + f = MOI.get(model, MOI.ConstraintFunction(), ci) + _write_q_matrix( + io, + f, + ordered_names, + var_to_column; + duplicate_off_diagonal = true, + ) + end + end + return +end + # ============================================================================== # SOS # ============================================================================== diff --git a/test/FileFormats/MPS/MPS.jl b/test/FileFormats/MPS/MPS.jl index 02f11e7f1b..5dfd7415d7 100644 --- a/test/FileFormats/MPS/MPS.jl +++ b/test/FileFormats/MPS/MPS.jl @@ -33,20 +33,6 @@ function test_show() "A Mathematical Programming System (MPS) model" end -function test_quadratic() - model = MPS.Model() - @test_throws( - MOI.UnsupportedAttribute, - MOIU.loadfromstring!( - model, - """ -variables: x -minobjective: 1.0*x*x -""", - ) - ) -end - function test_nonempty() model = MPS.Model() @test MOI.is_empty(model) @@ -691,6 +677,129 @@ function test_infinite_interval() return end +function test_quadobj_gurobi() + model = MPS.Model() + MOIU.loadfromstring!( + model, + """ +variables: x, y +minobjective: x + y + 5.0 * x * x + 1.0 * x * y + 1.0 * y * x + 1.2 * y * y +""", + ) + MOI.write_to_file(model, MPS_TEST_FILE) + @test read(MPS_TEST_FILE, String) == + "NAME \n" * + "ROWS\n" * + " N OBJ\n" * + "COLUMNS\n" * + " x OBJ 1\n" * + " y OBJ 1\n" * + "RHS\n" * + "RANGES\n" * + "BOUNDS\n" * + " FR bounds x\n" * + " FR bounds y\n" * + "QUADOBJ\n" * + " x x 10\n" * + " x y 2\n" * + " y y 2.4\n" * + "ENDATA\n" +end + +function test_quadobj_cplex() + model = MPS.Model(; quadratic_format = MPS.kQuadraticFormatCPLEX) + MOIU.loadfromstring!( + model, + """ +variables: x, y +minobjective: x + y + 5.0 * x * x + 1.0 * x * y + 1.0 * y * x + 1.2 * y * y +""", + ) + MOI.write_to_file(model, MPS_TEST_FILE) + @test read(MPS_TEST_FILE, String) == + "NAME \n" * + "ROWS\n" * + " N OBJ\n" * + "COLUMNS\n" * + " x OBJ 1\n" * + " y OBJ 1\n" * + "RHS\n" * + "RANGES\n" * + "BOUNDS\n" * + " FR bounds x\n" * + " FR bounds y\n" * + "QMATRIX\n" * + " x x 10\n" * + " x y 2\n" * + " y x 2\n" * + " y y 2.4\n" * + "ENDATA\n" +end + +function test_quadcon_gurobi() + model = MPS.Model() + MOIU.loadfromstring!( + model, + """ +variables: x, y +c1: x + y + 5.0 * x * x + 1.0 * x * y + 1.0 * y * x + 1.2 * y * y <= 1.0 +""", + ) + MOI.write_to_file(model, MPS_TEST_FILE) + @test read(MPS_TEST_FILE, String) == + "NAME \n" * + "ROWS\n" * + " N OBJ\n" * + " L c1\n" * + "COLUMNS\n" * + " x c1 1\n" * + " y c1 1\n" * + "RHS\n" * + " rhs c1 1\n" * + "RANGES\n" * + "BOUNDS\n" * + " FR bounds x\n" * + " FR bounds y\n" * + "QCMATRIX c1\n" * + " x x 10\n" * + " x y 2\n" * + " y x 2\n" * + " y y 2.4\n" * + "ENDATA\n" +end + +function test_quadcon_cplex() + model = MPS.Model(; quadratic_format = MPS.kQuadraticFormatCPLEX) + MOIU.loadfromstring!( + model, + """ +variables: x, y +c1: x + y + 5.0 * x * x + 1.0 * x * y + 1.0 * y * x + 1.2 * y * y <= 1.0 +""", + ) + MOI.write_to_file(model, MPS_TEST_FILE) + @test read(MPS_TEST_FILE, String) == + "NAME \n" * + "ROWS\n" * + " N OBJ\n" * + " L c1\n" * + "COLUMNS\n" * + " x c1 1\n" * + " y c1 1\n" * + "RHS\n" * + " rhs c1 1\n" * + "RANGES\n" * + "BOUNDS\n" * + " FR bounds x\n" * + " FR bounds y\n" * + "QCMATRIX c1\n" * + " x x 10\n" * + " x y 2\n" * + " y x 2\n" * + " y y 2.4\n" * + "ENDATA\n" +end + function runtests() for name in names(@__MODULE__, all = true) if startswith("$(name)", "test_") From 770182626c747535074b9806d9367e28d095f68d Mon Sep 17 00:00:00 2001 From: odow Date: Thu, 21 Jul 2022 12:20:35 +1200 Subject: [PATCH 2/4] Add readers for quadratic models --- src/FileFormats/MPS/MPS.jl | 142 +++++++++++++++++++++++++++++++++--- test/FileFormats/MPS/MPS.jl | 56 ++++++++++++++ 2 files changed, 186 insertions(+), 12 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index e8f64b4fb3..4ade52c9b4 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -848,6 +848,9 @@ mutable struct TempMPSModel row_to_name::Vector{String} intorg_flag::Bool # A flag used to parse COLUMNS section. sos_constraints::Vector{_SOSConstraint} + quad_obj::Vector{Tuple{String,String,Float64}} + qc_matrix::Dict{String,Vector{Tuple{String,String,Float64}}} + current_qc_matrix::String end function TempMPSModel() @@ -870,6 +873,9 @@ function TempMPSModel() String[], false, _SOSConstraint[], + Tuple{String,String,Float64}[], + Dict{String,Vector{Tuple{String,String,Float64}}}(), + "", ) end @@ -885,6 +891,9 @@ end HEADER_SOS, HEADER_ENDATA, HEADER_UNKNOWN, + HEADER_QUADOBJ, + HEADER_QMATRIX, + HEADER_QCMATRIX, ) # Headers(s) gets called _alot_ (on every line), so we try very hard to be @@ -913,11 +922,22 @@ function Headers(s::AbstractString) elseif N == 7 if (x == 'C' || x == 'c') && (uppercase(s) == "COLUMNS") return HEADER_COLUMNS + elseif (x == 'Q' || x == 'q') + header = uppercase(s) + if header == "QUADOBJ" + return HEADER_QUADOBJ + elseif header == "QMATRIX" + return HEADER_QMATRIX + end end elseif N == 12 if (x == 'O' || x == 'o') && startswith(uppercase(s), "OBJSENSE") return HEADER_OBJSENSE end + elseif N > 12 + if (x == 'Q' || x == 'q') && startswith(uppercase(s), "QCMATRIX") + return HEADER_QCMATRIX + end end return HEADER_UNKNOWN end @@ -952,6 +972,14 @@ function Base.read!(io::IO, model::Model) @assert sense == "MAX" || sense == "MIN" data.is_minimization = sense == "MIN" continue + elseif h == HEADER_QCMATRIX + items = line_to_items(line) + @assert length(items) == 2 + data.current_qc_matrix = String(items[2]) + header = HEADER_QCMATRIX + data.qc_matrix[data.current_qc_matrix] = + Tuple{String,String,Float64}[] + continue elseif h != HEADER_UNKNOWN header = h continue @@ -976,6 +1004,12 @@ function Base.read!(io::IO, model::Model) parse_bounds_line(data, items) elseif header == HEADER_SOS parse_sos_line(data, items) + elseif header == HEADER_QUADOBJ + parse_quadobj_line(data, items) + elseif header == HEADER_QMATRIX + parse_qmatrix_line(data, items) + elseif header == HEADER_QCMATRIX + parse_qcmatrix_line(data, items) else @assert header == HEADER_ENDATA break @@ -1008,7 +1042,7 @@ function copy_to(model::Model, data::TempMPSModel) for (j, c_name) in enumerate(data.row_to_name) set = bounds_to_set(data.row_lower[j], data.row_upper[j]) if set !== nothing - _add_linear_constraint(model, data, variable_map, j, c_name, set) + _add_constraint(model, data, variable_map, j, c_name, set) end # `else` is a free constraint. Don't add it. end @@ -1044,17 +1078,32 @@ function _add_objective(model, data, variable_map) else MOI.set(model, MOI.ObjectiveSense(), MOI.MAX_SENSE) end - MOI.set( - model, - MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}(), - MOI.ScalarAffineFunction( - [ - MOI.ScalarAffineTerm(data.c[i], variable_map[v]) for - (i, v) in enumerate(data.col_to_name) if !iszero(data.c[i]) - ], - -data.obj_constant, - ), - ) + affine_terms = MOI.ScalarAffineTerm{Float64}[ + MOI.ScalarAffineTerm(data.c[i], variable_map[v]) for + (i, v) in enumerate(data.col_to_name) if !iszero(data.c[i]) + ] + q_terms = MOI.ScalarQuadraticTerm{Float64}[] + for (i, j, q) in data.quad_obj + x = variable_map[i] + y = variable_map[j] + push!(q_terms, MOI.ScalarQuadraticTerm(q, x, y)) + end + obj = if length(q_terms) == 0 + MOI.ScalarAffineFunction(affine_terms, -data.obj_constant) + else + MOI.ScalarQuadraticFunction(q_terms, affine_terms, -data.obj_constant) + end + + MOI.set(model, MOI.ObjectiveFunction{typeof(obj)}(), obj) + return +end + +function _add_constraint(model, data, variable_map, j, c_name, set) + if haskey(data.qc_matrix, c_name) + _add_quad_constraint(model, data, variable_map, j, c_name, set) + else + _add_linear_constraint(model, data, variable_map, j, c_name, set) + end return end @@ -1068,6 +1117,24 @@ function _add_linear_constraint(model, data, variable_map, j, c_name, set) return end +function _add_quad_constraint(model, data, variable_map, j, c_name, set) + aff_terms = MOI.ScalarAffineTerm{Float64}[ + MOI.ScalarAffineTerm(coef, variable_map[data.col_to_name[i]]) for + (i, coef) in data.A[j] + ] + quad_terms = MOI.ScalarQuadraticTerm{Float64}[] + for (x, y, q) in data.qc_matrix[c_name] + push!( + quad_terms, + MOI.ScalarQuadraticTerm(q, variable_map[x], variable_map[y]), + ) + end + f = MOI.ScalarQuadraticFunction(quad_terms, aff_terms, 0.0) + c = MOI.add_constraint(model, f, set) + MOI.set(model, MOI.ConstraintName(), c, c_name) + return +end + # ============================================================================== # NAME # ============================================================================== @@ -1409,6 +1476,10 @@ function parse_bounds_line(data::TempMPSModel, items::Vector{String}) return end +# ============================================================================== +# SOS +# ============================================================================== + function parse_sos_line(data, items) if length(items) != 2 error("Malformed SOS line: $(join(items, " "))") @@ -1424,4 +1495,51 @@ function parse_sos_line(data, items) return end +# ============================================================================== +# QUADOBJ +# ============================================================================== + +function parse_quadobj_line(data, items) + if length(items) != 3 + error("Malformed QUADOBJ line: $(join(items, " "))") + end + push!(data.quad_obj, (items[1], items[2], parse(Float64, items[3]))) + return +end + +# ============================================================================== +# QMATRIX +# ============================================================================== + +function parse_qmatrix_line(data, items) + if length(items) != 3 + error("Malformed QMATRIX line: $(join(items, " "))") + end + if data.name_to_col[items[1]] <= data.name_to_col[items[2]] + # Off-diagonals have duplicate entries! We don't need to store both + # triangles. + push!(data.quad_obj, (items[1], items[2], parse(Float64, items[3]))) + end + return +end + +# ============================================================================== +# QMATRIX +# ============================================================================== + +function parse_qcmatrix_line(data, items) + if length(items) != 3 + error("Malformed QCMATRIX line: $(join(items, " "))") + end + if data.name_to_col[items[1]] <= data.name_to_col[items[2]] + # Off-diagonals have duplicate entries! We don't need to store both + # triangles. + push!( + data.qc_matrix[data.current_qc_matrix], + (items[1], items[2], parse(Float64, items[3])), + ) + end + return +end + end diff --git a/test/FileFormats/MPS/MPS.jl b/test/FileFormats/MPS/MPS.jl index 5dfd7415d7..3b2a830111 100644 --- a/test/FileFormats/MPS/MPS.jl +++ b/test/FileFormats/MPS/MPS.jl @@ -704,6 +704,7 @@ minobjective: x + y + 5.0 * x * x + 1.0 * x * y + 1.0 * y * x + 1.2 * y * y " x y 2\n" * " y y 2.4\n" * "ENDATA\n" + return end function test_quadobj_cplex() @@ -734,6 +735,7 @@ minobjective: x + y + 5.0 * x * x + 1.0 * x * y + 1.0 * y * x + 1.2 * y * y " y x 2\n" * " y y 2.4\n" * "ENDATA\n" + return end function test_quadcon_gurobi() @@ -766,6 +768,7 @@ c1: x + y + 5.0 * x * x + 1.0 * x * y + 1.0 * y * x + 1.2 * y * y <= 1.0 " y x 2\n" * " y y 2.4\n" * "ENDATA\n" + return end function test_quadcon_cplex() @@ -798,6 +801,59 @@ c1: x + y + 5.0 * x * x + 1.0 * x * y + 1.0 * y * x + 1.2 * y * y <= 1.0 " y x 2\n" * " y y 2.4\n" * "ENDATA\n" + return +end + +function test_round_trip_quadobj_gurobi() + _test_model_equality( + """ +variables: x, y +minobjective: 1.2x + 2.1 * x * x + 1.2 * x * y + 0.2 * y * x + 0.5 * y * y +""", + ["x", "y"], + String[], + ) + return +end + +function test_round_trip_qmatrix_cplex() + _test_model_equality( + """ +variables: x, y +minobjective: 1.2x + 2.1 * x * x + 1.2 * x * y + 0.2 * y * x + 0.5 * y * y +""", + ["x", "y"], + String[]; + quadratic_format = MPS.kQuadraticFormatCPLEX, + ) + return +end + +function test_round_trip_qcmatrix_gurobi() + _test_model_equality( + """ +variables: x, y +minobjective: 1.3 * x * x + 0.5 * x * y +c1: 1.2x + 2.1 * x * x + 1.2 * x * y + 0.2 * y * x + 0.5 * y * y <= 1.0 +""", + ["x", "y"], + ["c1"], + ) + return +end + +function test_round_trip_qcmatrix_cplex() + _test_model_equality( + """ +variables: x, y +minobjective: 1.3 * x * x + 0.5 * x * y +c1: 1.2x + 2.1 * x * x + 1.2 * x * y + 0.2 * y * x + 0.5 * y * y <= 1.0 +""", + ["x", "y"], + ["c1"]; + quadratic_format = MPS.kQuadraticFormatCPLEX, + ) + return end function runtests() From aa1ed71ff63d39d95a7cc1336189dc5b98b19681 Mon Sep 17 00:00:00 2001 From: odow Date: Thu, 21 Jul 2022 13:13:19 +1200 Subject: [PATCH 3/4] Add QSECTION support --- src/FileFormats/MPS/MPS.jl | 57 ++++++++++++++++++++++++++++++------- test/FileFormats/MPS/MPS.jl | 14 +++++++++ 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 4ade52c9b4..5056d632c5 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -41,7 +41,7 @@ MOI.Utilities.@model( QuadraticFormat, kQuadraticFormatCPLEX, kQuadraticFormatGurobi, - # TODO(odow): kQuadraticFormatMosek + kQuadraticFormatMosek, ) struct Options @@ -73,7 +73,8 @@ Keyword arguments are: respectively. - `quadratic_format::QuadraticFormat = kQuadraticFormatGurobi`: specify the solver-specific extension used when writing the quadratic components of the - model. Options are `kQuadraticFormatGurobi` and `kQuadraticFormatCPLEX`. + model. Options are `kQuadraticFormatGurobi`, `kQuadraticFormatCPLEX`, and + `kQuadraticFormatMosek`. """ function Model(; warn::Bool = false, @@ -220,7 +221,7 @@ function Base.write(io::IO, model::Model) write_ranges(io, model) write_bounds(io, model, ordered_names, names) write_quadobj(io, model, ordered_names, var_to_column) - if options.quadratic_format == kQuadraticFormatGurobi + if options.quadratic_format != kQuadraticFormatCPLEX # Gurobi needs qcons _after_ quadobj and _before_ SOS. write_quadcons(io, model, ordered_names, var_to_column) end @@ -655,8 +656,11 @@ function write_quadobj(io::IO, model::Model, ordered_names, var_to_column) options = get_options(model) if options.quadratic_format == kQuadraticFormatGurobi println(io, "QUADOBJ") - else + elseif options.quadratic_format == kQuadraticFormatCPLEX println(io, "QMATRIX") + else + @assert options.quadratic_format == kQuadraticFormatMosek + println(io, "QSECTION OBJ") end _write_q_matrix( io, @@ -720,18 +724,24 @@ end # ============================================================================== function write_quadcons(io::IO, model::Model, ordered_names, var_to_column) + options = get_options(model) F = MOI.ScalarQuadraticFunction{Float64} for (S, _) in SET_TYPES for ci in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) name = MOI.get(model, MOI.ConstraintName(), ci) - println(io, "QCMATRIX $name") + if options.quadratic_format == kQuadraticFormatMosek + println(io, "QSECTION $name") + else + println(io, "QCMATRIX $name") + end f = MOI.get(model, MOI.ConstraintFunction(), ci) _write_q_matrix( io, f, ordered_names, var_to_column; - duplicate_off_diagonal = true, + duplicate_off_diagonal = options.quadratic_format != + kQuadraticFormatMosek, ) end end @@ -894,6 +904,7 @@ end HEADER_QUADOBJ, HEADER_QMATRIX, HEADER_QCMATRIX, + HEADER_QSECTION, ) # Headers(s) gets called _alot_ (on every line), so we try very hard to be @@ -935,8 +946,13 @@ function Headers(s::AbstractString) return HEADER_OBJSENSE end elseif N > 12 - if (x == 'Q' || x == 'q') && startswith(uppercase(s), "QCMATRIX") - return HEADER_QCMATRIX + if (x == 'Q' || x == 'q') + header = uppercase(s) + if startswith(header, "QCMATRIX") + return HEADER_QCMATRIX + elseif startswith(header, "QSECTION") + return HEADER_QSECTION + end end end return HEADER_UNKNOWN @@ -972,11 +988,11 @@ function Base.read!(io::IO, model::Model) @assert sense == "MAX" || sense == "MIN" data.is_minimization = sense == "MIN" continue - elseif h == HEADER_QCMATRIX + elseif h == HEADER_QCMATRIX || h == HEADER_QSECTION items = line_to_items(line) @assert length(items) == 2 data.current_qc_matrix = String(items[2]) - header = HEADER_QCMATRIX + header = h data.qc_matrix[data.current_qc_matrix] = Tuple{String,String,Float64}[] continue @@ -1010,6 +1026,8 @@ function Base.read!(io::IO, model::Model) parse_qmatrix_line(data, items) elseif header == HEADER_QCMATRIX parse_qcmatrix_line(data, items) + elseif header == HEADER_QSECTION + parse_qsection_line(data, items) else @assert header == HEADER_ENDATA break @@ -1542,4 +1560,23 @@ function parse_qcmatrix_line(data, items) return end +# ============================================================================== +# QSECTION +# ============================================================================== + +function parse_qsection_line(data, items) + if length(items) != 3 + error("Malformed QSECTION line: $(join(items, " "))") + end + if data.current_qc_matrix == "OBJ" + push!(data.quad_obj, (items[1], items[2], parse(Float64, items[3]))) + else + push!( + data.qc_matrix[data.current_qc_matrix], + (items[1], items[2], parse(Float64, items[3])), + ) + end + return +end + end diff --git a/test/FileFormats/MPS/MPS.jl b/test/FileFormats/MPS/MPS.jl index 3b2a830111..6f2ae1d427 100644 --- a/test/FileFormats/MPS/MPS.jl +++ b/test/FileFormats/MPS/MPS.jl @@ -856,6 +856,20 @@ c1: 1.2x + 2.1 * x * x + 1.2 * x * y + 0.2 * y * x + 0.5 * y * y <= 1.0 return end +function test_round_trip_qcmatrix_mosek() + _test_model_equality( + """ +variables: x, y +minobjective: 1.3 * x * x + 0.5 * x * y +c1: 1.2x + 2.1 * x * x + 1.2 * x * y + 0.2 * y * x + 0.5 * y * y <= 1.0 +""", + ["x", "y"], + ["c1"]; + quadratic_format = MPS.kQuadraticFormatMosek, + ) + return +end + function runtests() for name in names(@__MODULE__, all = true) if startswith("$(name)", "test_") From 6912f5297e01a82da7396512b42ea2b06fa30475 Mon Sep 17 00:00:00 2001 From: odow Date: Thu, 21 Jul 2022 14:50:32 +1200 Subject: [PATCH 4/4] Add tests for invalid models --- .../MPS/failing_models/malformed_qcmatrix.mps | 13 +++++++++++++ .../MPS/failing_models/malformed_qmatrix.mps | 12 ++++++++++++ .../MPS/failing_models/malformed_qsection.mps | 12 ++++++++++++ .../MPS/failing_models/malformed_quadobj.mps | 12 ++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 test/FileFormats/MPS/failing_models/malformed_qcmatrix.mps create mode 100644 test/FileFormats/MPS/failing_models/malformed_qmatrix.mps create mode 100644 test/FileFormats/MPS/failing_models/malformed_qsection.mps create mode 100644 test/FileFormats/MPS/failing_models/malformed_quadobj.mps diff --git a/test/FileFormats/MPS/failing_models/malformed_qcmatrix.mps b/test/FileFormats/MPS/failing_models/malformed_qcmatrix.mps new file mode 100644 index 0000000000..487c61e18c --- /dev/null +++ b/test/FileFormats/MPS/failing_models/malformed_qcmatrix.mps @@ -0,0 +1,13 @@ +NAME +OBJSENSE MIN +ROWS + N obj + L c +COLUMNS + x obj 1 +RHS +BOUNDS + FR bounds x +QCMATRIX c + x 1 +ENDATA diff --git a/test/FileFormats/MPS/failing_models/malformed_qmatrix.mps b/test/FileFormats/MPS/failing_models/malformed_qmatrix.mps new file mode 100644 index 0000000000..ec6e145463 --- /dev/null +++ b/test/FileFormats/MPS/failing_models/malformed_qmatrix.mps @@ -0,0 +1,12 @@ +NAME +OBJSENSE MIN +ROWS + N obj +COLUMNS + x obj 1 +RHS +BOUNDS + FR bounds x +QMATRIX + x 1 +ENDATA diff --git a/test/FileFormats/MPS/failing_models/malformed_qsection.mps b/test/FileFormats/MPS/failing_models/malformed_qsection.mps new file mode 100644 index 0000000000..193b7c567a --- /dev/null +++ b/test/FileFormats/MPS/failing_models/malformed_qsection.mps @@ -0,0 +1,12 @@ +NAME +OBJSENSE MIN +ROWS + N obj +COLUMNS + x obj 1 +RHS +BOUNDS + FR bounds x +QSECTION obj + x 1 +ENDATA diff --git a/test/FileFormats/MPS/failing_models/malformed_quadobj.mps b/test/FileFormats/MPS/failing_models/malformed_quadobj.mps new file mode 100644 index 0000000000..9859147b92 --- /dev/null +++ b/test/FileFormats/MPS/failing_models/malformed_quadobj.mps @@ -0,0 +1,12 @@ +NAME +OBJSENSE MIN +ROWS + N obj +COLUMNS + x obj 1 +RHS +BOUNDS + FR bounds x +QUADOBJ + x 1 +ENDATA