Skip to content

Commit bc29975

Browse files
AnHeuermannAnHeuermann
committed
Moving regression tests in separate directory
1 parent fb15d1e commit bc29975

File tree

7 files changed

+316
-10
lines changed

7 files changed

+316
-10
lines changed

‎.github/workflows/regressionTests.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ name: Regression Tests
33
on:
44
push:
55
branches: ['master', 'maintenance/*']
6+
pull_request:
67
schedule:
78
- cron: "25 4 * * 3" # Every Wednesday at 04:25
89
workflow_dispatch:
@@ -31,7 +32,6 @@ jobs:
3132
omc
3233
libraries: |
3334
'Modelica 4.0.0'
34-
'Buildings 10.0.0'
3535
3636
- run: "omc --version"
3737

@@ -48,13 +48,13 @@ jobs:
4848
uses: julia-actions/julia-buildpkg@v1
4949

5050
- name: Install dependencies
51-
run: julia --project=test/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
51+
run: julia --project=regression-tests/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
5252

5353
- name: "Run regression test"
54-
run: julia --project=test/. -e "include(\"test/regressionTests.jl\"); runTests(libraries, models)"
54+
run: julia --project=regression-tests/. -e "include(\"regression-tests/regressionTests.jl\"); runTests(libraries, models)"
5555

5656
- name: Archive FMUs
5757
uses: actions/upload-artifact@v3
5858
with:
5959
name: fmu-export
60-
path: test/test-regressionTests/**/*.fmu
60+
path: regression-tests/temp/**/*.fmu

‎regression-tests/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
temp/

‎regression-tests/Project.toml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[deps]
2+
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
3+
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
4+
FMI = "14a09403-18e3-468f-ad8a-74f8dda2d9ac"
5+
OMJulia = "0f4fe800-344e-11e9-2949-fb537ad918e1"
6+
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

‎regression-tests/regressionTests.jl

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
#=
2+
This file is part of OpenModelica.
3+
Copyright (c) 1998-2023, Open Source Modelica Consortium (OSMC),
4+
c/o Linköpings universitet, Department of Computer and Information Science,
5+
SE-58183 Linköping, Sweden.
6+
7+
All rights reserved.
8+
9+
THIS PROGRAM IS PROVIDED UNDER THE TERMS OF THE BSD NEW LICENSE OR THE
10+
GPL VERSION 3 LICENSE OR THE OSMC PUBLIC LICENSE (OSMC-PL) VERSION 1.2.
11+
ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS PROGRAM CONSTITUTES
12+
RECIPIENT'S ACCEPTANCE OF THE OSMC PUBLIC LICENSE OR THE GPL VERSION 3,
13+
ACCORDING TO RECIPIENTS CHOICE.
14+
15+
The OpenModelica software and the OSMC (Open Source Modelica Consortium)
16+
Public License (OSMC-PL) are obtained from OSMC, either from the above
17+
address, from the URLs: http://www.openmodelica.org or
18+
http://www.ida.liu.se/projects/OpenModelica, and in the OpenModelica
19+
distribution. GNU version 3 is obtained from:
20+
http://www.gnu.org/copyleft/gpl.html. The New BSD License is obtained from:
21+
http://www.opensource.org/licenses/BSD-3-Clause.
22+
23+
This program is distributed WITHOUT ANY WARRANTY; without even the implied
24+
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, EXCEPT AS
25+
EXPRESSLY SET FORTH IN THE BY RECIPIENT SELECTED SUBSIDIARY LICENSE
26+
CONDITIONS OF OSMC-PL.
27+
=#
28+
29+
using Test
30+
31+
"""
32+
Timeout error.
33+
"""
34+
struct TimeOutError <: Exception
35+
cmd::Cmd
36+
end
37+
function Base.showerror(io::IO, e::TimeOutError)
38+
println(io, "Timeout reached running command")
39+
println(io, e.cmd)
40+
end
41+
42+
"""
43+
Run single test process.
44+
45+
Start a new Julia process.
46+
Kill process and throw TimeOutError when timeout is reached.
47+
Catch InterruptException, kill process and rethorw InterruptException.
48+
49+
# Arguments
50+
- `library`: Modelica library name.
51+
- `version`: Library version.
52+
- `model`: Modelica model from library to test.
53+
- `testdir`: Test working directory.
54+
55+
# Keywords
56+
- `timeout=10*60::Integer`: Timeout in seconds. Defaults to 10 minutes.
57+
"""
58+
function singleTest(library, version, model, testdir;
59+
timeout=10*60::Integer)
60+
61+
mkpath(testdir)
62+
logFile = joinpath(testdir, "runSingleTest.log")
63+
rm(logFile, force=true)
64+
65+
@info "Testing $model"
66+
67+
cmd = Cmd(`$(joinpath(Sys.BINDIR, "julia")) runSingleTest.jl $(library) $(version) $(model) $(testdir)`, dir=@__DIR__)
68+
@info cmd
69+
plp = pipeline(cmd, stdout=logFile, stderr=logFile)
70+
process = run(plp, wait=false)
71+
72+
try
73+
timer = Timer(0; interval=1)
74+
for _ in 1:timeout
75+
wait(timer)
76+
if !process_running(process)
77+
close(timer)
78+
break
79+
end
80+
end
81+
if process_running(process)
82+
@error "Killing $(process)"
83+
kill(process)
84+
end
85+
catch e
86+
if isa(e, InterruptException) && process_running(p)
87+
@error "Killing process $(cmd)."
88+
kill(p)
89+
end
90+
rethrow(e)
91+
end
92+
93+
println(read(logFile, String))
94+
95+
status = (process.exitcode == 0) &&
96+
isfile(joinpath(testdir, "$(model).fmu")) &&
97+
isfile(joinpath(testdir, "FMI_results.csv"))
98+
99+
return status
100+
end
101+
102+
"""
103+
Run all tests.
104+
105+
Start a new Julia process for each test.
106+
Kill process and throw TimeOutError when timeout is reached.
107+
Catch InterruptException, kill process and rethorw InterruptException.
108+
109+
# Arguments
110+
- `libraries::Vector{Tuple{S,S}}`: Vector of tuples with library and version to test.
111+
- `models::Vector{Vector{S}}`: Vector of vectors with models to test for each library.
112+
113+
# Keywords
114+
- `workdir`: Root working directory.
115+
"""
116+
function runTests(libraries::Vector{Tuple{S,S}},
117+
models::Vector{Vector{S}};
118+
workdir=abspath(joinpath(@__DIR__, "temp"))) where S<:AbstractString
119+
120+
rm(workdir, recursive=true, force=true) # This can break on Windows when some program or file is still open
121+
mkpath(workdir)
122+
123+
@testset "OpenModelica" begin
124+
for (i, (library, version)) in enumerate(libraries)
125+
@testset verbose=true "$library" begin
126+
libdir = joinpath(workdir, library)
127+
mkpath(libdir)
128+
129+
for model in models[i]
130+
modeldir = joinpath(libdir, model)
131+
@testset "$model" begin
132+
@test singleTest(library, version, model, modeldir)
133+
end
134+
end
135+
end
136+
end
137+
end
138+
139+
return
140+
end
141+
142+
libraries = [
143+
("Modelica", "4.0.0")
144+
]
145+
146+
models = [
147+
[
148+
"Modelica.Blocks.Examples.Filter",
149+
"Modelica.Electrical.Analog.Examples.CauerLowPassAnalog"
150+
"Modelica.Blocks.Examples.RealNetwork1",
151+
"Modelica.Electrical.Digital.Examples.FlipFlop",
152+
"Modelica.Mechanics.Rotational.Examples.FirstGrounded",
153+
"Modelica.Mechanics.Rotational.Examples.CoupledClutches",
154+
"Modelica.Mechanics.MultiBody.Examples.Elementary.DoublePendulum",
155+
"Modelica.Mechanics.MultiBody.Examples.Elementary.FreeBody",
156+
"Modelica.Fluid.Examples.TraceSubstances.RoomCO2WithControls",
157+
"Modelica.Clocked.Examples.SimpleControlledDrive.ClockedWithDiscreteTextbookController",
158+
"Modelica.Fluid.Examples.PumpingSystem"
159+
]
160+
]

‎regression-tests/runSingleTest.jl

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#=
2+
This file is part of OpenModelica.
3+
Copyright (c) 1998-2023, Open Source Modelica Consortium (OSMC),
4+
c/o Linköpings universitet, Department of Computer and Information Science,
5+
SE-58183 Linköping, Sweden.
6+
7+
All rights reserved.
8+
9+
THIS PROGRAM IS PROVIDED UNDER THE TERMS OF THE BSD NEW LICENSE OR THE
10+
GPL VERSION 3 LICENSE OR THE OSMC PUBLIC LICENSE (OSMC-PL) VERSION 1.2.
11+
ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS PROGRAM CONSTITUTES
12+
RECIPIENT'S ACCEPTANCE OF THE OSMC PUBLIC LICENSE OR THE GPL VERSION 3,
13+
ACCORDING TO RECIPIENTS CHOICE.
14+
15+
The OpenModelica software and the OSMC (Open Source Modelica Consortium)
16+
Public License (OSMC-PL) are obtained from OSMC, either from the above
17+
address, from the URLs: http://www.openmodelica.org or
18+
http://www.ida.liu.se/projects/OpenModelica, and in the OpenModelica
19+
distribution. GNU version 3 is obtained from:
20+
http://www.gnu.org/copyleft/gpl.html. The New BSD License is obtained from:
21+
http://www.opensource.org/licenses/BSD-3-Clause.
22+
23+
This program is distributed WITHOUT ANY WARRANTY; without even the implied
24+
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, EXCEPT AS
25+
EXPRESSLY SET FORTH IN THE BY RECIPIENT SELECTED SUBSIDIARY LICENSE
26+
CONDITIONS OF OSMC-PL.
27+
=#
28+
29+
import Pkg; Pkg.activate(@__DIR__)
30+
import OMJulia
31+
import FMI
32+
33+
using Test
34+
using DataFrames
35+
using CSV
36+
37+
"""
38+
Simulate single model to generate a result file.
39+
"""
40+
function testSimulation(omc::OMJulia.OMCSession, className::String)
41+
@info "\tSimulation"
42+
@testset "Simulation" begin
43+
res = OMJulia.API.simulate(omc, className; outputFormat="csv")
44+
resultFile = res["resultFile"]
45+
46+
@test isfile(resultFile)
47+
return resultFile
48+
end
49+
end
50+
51+
"""
52+
Build a FMU for a single model, import the generated FMU, simulate it and compare to given reference results.
53+
"""
54+
function testFmuExport(omc::OMJulia.OMCSession, className::String, referenceResult, recordValues; workdir::String)
55+
fmuPath = ""
56+
fmuImportSuccess = false
57+
@info "\tFMU Export"
58+
@testset "Export" begin
59+
fmuPath = OMJulia.API.buildModelFMU(omc, className)
60+
@test isfile(fmuPath)
61+
@test splitext(splitpath(fmuPath)[end]) == (className, ".fmu")
62+
end
63+
64+
@info "\tFMU Import"
65+
@testset "Import" begin
66+
if isfile(fmuPath)
67+
fmu = FMI.fmiLoad(fmuPath)
68+
solution = FMI.fmiSimulate(fmu; recordValues = recordValues, showProgress=false)
69+
70+
# Own implementation of CSV export, workaround for https://github.com/ThummeTo/FMI.jl/issues/198
71+
df = DataFrames.DataFrame(time = solution.values.t)
72+
for i in 1:length(solution.values.saveval[1])
73+
for var in FMI.fmi2ValueReferenceToString(fmu, solution.valueReferences[i])
74+
if in(var, recordValues)
75+
df[!, Symbol(var)] = [val[i] for val in solution.values.saveval]
76+
end
77+
end
78+
end
79+
fmiResult = joinpath(workdir, "FMI_results.csv")
80+
CSV.write(fmiResult, df)
81+
82+
#FMI.fmiSaveSolution(solution, "FMI_results.csv")
83+
fmuImportSuccess = true
84+
end
85+
@test fmuImportSuccess
86+
end
87+
88+
@info "\tCheck Results"
89+
@testset "Verification" begin
90+
if fmuImportSuccess
91+
@test (true, String[]) == OMJulia.API.diffSimulationResults(omc, "FMI_results.csv", referenceResult, "diff")
92+
else
93+
@test false
94+
end
95+
end
96+
end
97+
98+
"""
99+
Run Simulation and FMU export/import test for all models.
100+
"""
101+
function runSingleTest(library, version, model, modeldir)
102+
local resultFile
103+
104+
@info "Testing library: $library, model $model"
105+
mkpath(modeldir)
106+
omc = OMJulia.OMCSession()
107+
108+
try
109+
@testset "$model" verbose=true begin
110+
@testset "Simulation" begin
111+
OMJulia.API.cd(omc, modeldir)
112+
113+
@test OMJulia.API.loadModel(omc, library; priorityVersion = [version], requireExactVersion = true)
114+
resultFile = testSimulation(omc, model)
115+
end
116+
117+
@testset "FMI" begin
118+
if isfile(resultFile)
119+
recordValues = names(CSV.read(resultFile, DataFrame))[2:end]
120+
filter!(val -> !startswith(val, "\$"), recordValues) # Filter internal variables
121+
testFmuExport(omc, model, resultFile, recordValues; workdir=modeldir)
122+
else
123+
@test false
124+
end
125+
end
126+
end
127+
finally
128+
OMJulia.quit(omc)
129+
end
130+
end
131+
132+
# Comand-line interface
133+
if !isempty(PROGRAM_FILE)
134+
if length(ARGS) == 4
135+
library = ARGS[1]
136+
version = ARGS[2]
137+
model = ARGS[3]
138+
modeldir = ARGS[4]
139+
runSingleTest(library, version, model, modeldir)
140+
else
141+
@error "Wrong number of arguments"
142+
for a in ARGS; println(a); end
143+
return -1
144+
end
145+
end

‎test/Project.toml

-5
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,6 @@
11
[deps]
22
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
33
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
4-
FMI = "14a09403-18e3-468f-ad8a-74f8dda2d9ac"
54
OMJulia = "0f4fe800-344e-11e9-2949-fb537ad918e1"
6-
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
75
SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
86
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
9-
10-
[compat]
11-
FMI = "0.13"

‎test/runtests.jl

-1
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,4 @@ using Test
3434
@safetestset "OMCSession" begin include("omcTest.jl") end
3535
@safetestset "ModelicaSystem" begin include("modelicaSystemTest.jl") end
3636
@safetestset "API" begin include("apiTest.jl") end
37-
@safetestset "Fast regressions test" begin include("regressionTests.jl"); runTests(libraries, [models[1][1:2]]) end
3837
end

0 commit comments

Comments
 (0)