Skip to content

Commit d8c81e6

Browse files
AnHeuermannAnHeuermann
authored
Regression tests (OpenModelica#111)
* Regression tests to perform - Simulation - FMU export - FMU import with FMI.jl ^v0.13 * Test each model in a new process to survive segmentation faults
1 parent 3b97b91 commit d8c81e6

File tree

6 files changed

+362
-1
lines changed

6 files changed

+362
-1
lines changed

‎.github/workflows/regressionTests.yml

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Regression Tests
2+
3+
on:
4+
push:
5+
branches: ['master', 'maintenance/*']
6+
pull_request:
7+
schedule:
8+
- cron: "25 4 * * 3" # Every Wednesday at 04:25
9+
workflow_dispatch:
10+
11+
jobs:
12+
regression-test:
13+
if: github.repository == 'OpenModelica/OMJulia.jl' # Run only on OpenModelica/OMJulia.jl to prevent spamming forks
14+
runs-on: ${{ matrix.os }}
15+
timeout-minutes: 60
16+
strategy:
17+
fail-fast: false
18+
matrix:
19+
julia-version: ['1.8', '1.9']
20+
julia-arch: ['x64']
21+
os: ['ubuntu-latest', 'windows-latest']
22+
omc-version: ['stable', 'nightly', '1.21']
23+
24+
steps:
25+
- uses: actions/checkout@v4
26+
27+
- name: "Set up OpenModelica Compiler"
28+
uses: AnHeuermann/[email protected]
29+
with:
30+
version: ${{ matrix.omc-version }}
31+
packages: |
32+
omc
33+
libraries: |
34+
'Modelica 4.0.0'
35+
36+
- run: "omc --version"
37+
38+
- name: "Set up Julia"
39+
uses: julia-actions/setup-julia@v1
40+
with:
41+
version: ${{ matrix.julia-version }}
42+
arch: ${{ matrix.julia-arch }}
43+
44+
- name: Cache Julia
45+
uses: julia-actions/cache@v1
46+
47+
- name: "Build OMJulia"
48+
uses: julia-actions/julia-buildpkg@v1
49+
50+
- name: Install dependencies
51+
run: julia --project=regression-tests/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
52+
53+
- name: "Run regression test"
54+
shell: bash
55+
run: julia --project=regression-tests/. -e 'include("regression-tests/regressionTests.jl"); runTests(libraries, models)'
56+
57+
- name: Archive FMUs
58+
uses: actions/upload-artifact@v3
59+
with:
60+
name: fmu-export
61+
path: regression-tests/temp/**/*.fmu

‎README.md

+4-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
*Julia scripting [OpenModelica](https://openmodelica.org/) interface.*
44

5-
[![][docs-dev-img]][docs-dev-url] [![][GHA-test-img]][GHA-test-url]
5+
[![][docs-dev-img]][docs-dev-url] [![][GHA-test-img]][GHA-test-url] [![][GHA-regressions-img]][GHA-regressions-url]
66

77
## Requirements
88

@@ -82,3 +82,6 @@ CONDITIONS OF OSMC-PL.
8282

8383
[GHA-test-img]: https://github.com/OpenModelica/OMJulia.jl/actions/workflows/Test.yml/badge.svg?branch=master
8484
[GHA-test-url]: https://github.com/OpenModelica/OMJulia.jl/actions/workflows/Test.yml
85+
86+
[GHA-regressions-img]: https://github.com/OpenModelica/OMJulia.jl/actions/workflows/regressionTests.yml/badge.svg?branch=master
87+
[GHA-regressions-url]: https://github.com/OpenModelica/OMJulia.jl/actions/workflows/regressionTests.yml

‎regression-tests/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
temp/

‎regression-tests/Project.toml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[deps]
2+
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
3+
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
4+
FMI = "14a09403-18e3-468f-ad8a-74f8dda2d9ac"
5+
OMJulia = "0f4fe800-344e-11e9-2949-fb537ad918e1"
6+
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

‎regression-tests/regressionTests.jl

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#=
2+
This file is part of OpenModelica.
3+
Copyright (c) 1998-2023, Open Source Modelica Consortium (OSMC),
4+
c/o Linköpings universitet, Department of Computer and Information Science,
5+
SE-58183 Linköping, Sweden.
6+
7+
All rights reserved.
8+
9+
THIS PROGRAM IS PROVIDED UNDER THE TERMS OF THE BSD NEW LICENSE OR THE
10+
GPL VERSION 3 LICENSE OR THE OSMC PUBLIC LICENSE (OSMC-PL) VERSION 1.2.
11+
ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS PROGRAM CONSTITUTES
12+
RECIPIENT'S ACCEPTANCE OF THE OSMC PUBLIC LICENSE OR THE GPL VERSION 3,
13+
ACCORDING TO RECIPIENTS CHOICE.
14+
15+
The OpenModelica software and the OSMC (Open Source Modelica Consortium)
16+
Public License (OSMC-PL) are obtained from OSMC, either from the above
17+
address, from the URLs: http://www.openmodelica.org or
18+
http://www.ida.liu.se/projects/OpenModelica, and in the OpenModelica
19+
distribution. GNU version 3 is obtained from:
20+
http://www.gnu.org/copyleft/gpl.html. The New BSD License is obtained from:
21+
http://www.opensource.org/licenses/BSD-3-Clause.
22+
23+
This program is distributed WITHOUT ANY WARRANTY; without even the implied
24+
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, EXCEPT AS
25+
EXPRESSLY SET FORTH IN THE BY RECIPIENT SELECTED SUBSIDIARY LICENSE
26+
CONDITIONS OF OSMC-PL.
27+
=#
28+
29+
using Test
30+
31+
"""
32+
Run single test process.
33+
34+
Start a new Julia process.
35+
Kill process and throw an error when timeout is reached.
36+
Catch InterruptException, kill process and rethorw InterruptException.
37+
38+
# Arguments
39+
- `library`: Modelica library name.
40+
- `version`: Library version.
41+
- `model`: Modelica model from library to test.
42+
- `testdir`: Test working directory.
43+
44+
# Keywords
45+
- `timeout=10*60::Integer`: Timeout in seconds. Defaults to 10 minutes.
46+
"""
47+
function singleTest(library, version, model, testdir;
48+
timeout=10*60::Integer)
49+
50+
mkpath(testdir)
51+
logFile = joinpath(testdir, "runSingleTest.log")
52+
rm(logFile, force=true)
53+
54+
@info "Testing $model"
55+
56+
cmd = Cmd(`$(joinpath(Sys.BINDIR, "julia")) runSingleTest.jl $(library) $(version) $(model) $(testdir)`, dir=@__DIR__)
57+
@info cmd
58+
plp = pipeline(cmd, stdout=logFile, stderr=logFile)
59+
process = run(plp, wait=false)
60+
61+
try
62+
timer = Timer(0; interval=1)
63+
for _ in 1:timeout
64+
wait(timer)
65+
if !process_running(process)
66+
close(timer)
67+
break
68+
end
69+
end
70+
if process_running(process)
71+
@error "Killing $(process)"
72+
kill(process)
73+
end
74+
catch e
75+
if isa(e, InterruptException) && process_running(p)
76+
@error "Killing process $(cmd)."
77+
kill(p)
78+
end
79+
rethrow(e)
80+
end
81+
82+
println(read(logFile, String))
83+
84+
status = (process.exitcode == 0) &&
85+
isfile(joinpath(testdir, "$(model).fmu")) &&
86+
isfile(joinpath(testdir, "FMI_results.csv"))
87+
88+
return status
89+
end
90+
91+
"""
92+
Run all tests.
93+
94+
# Arguments
95+
- `libraries::Vector{Tuple{S,S}}`: Vector of tuples with library and version to test.
96+
- `models::Vector{Vector{S}}`: Vector of vectors with models to test for each library.
97+
98+
# Keywords
99+
- `workdir`: Root working directory.
100+
"""
101+
function runTests(libraries::Vector{Tuple{S,S}},
102+
models::Vector{Vector{S}};
103+
workdir=abspath(joinpath(@__DIR__, "temp"))) where S<:AbstractString
104+
105+
rm(workdir, recursive=true, force=true) # This can break on Windows when some program or file is still open
106+
mkpath(workdir)
107+
108+
@testset "OpenModelica" begin
109+
for (i, (library, version)) in enumerate(libraries)
110+
@testset verbose=true "$library" begin
111+
libdir = joinpath(workdir, library)
112+
mkpath(libdir)
113+
114+
for model in models[i]
115+
modeldir = joinpath(libdir, model)
116+
@testset "$model" begin
117+
@test singleTest(library, version, model, modeldir)
118+
end
119+
end
120+
end
121+
end
122+
end
123+
124+
return
125+
end
126+
127+
libraries = [
128+
("Modelica", "4.0.0")
129+
]
130+
131+
models = [
132+
[
133+
"Modelica.Blocks.Examples.Filter",
134+
"Modelica.Electrical.Analog.Examples.CauerLowPassAnalog",
135+
"Modelica.Blocks.Examples.RealNetwork1",
136+
"Modelica.Electrical.Digital.Examples.FlipFlop",
137+
"Modelica.Mechanics.Rotational.Examples.FirstGrounded",
138+
"Modelica.Mechanics.Rotational.Examples.CoupledClutches",
139+
"Modelica.Mechanics.MultiBody.Examples.Elementary.DoublePendulum",
140+
"Modelica.Mechanics.MultiBody.Examples.Elementary.FreeBody",
141+
"Modelica.Fluid.Examples.TraceSubstances.RoomCO2WithControls",
142+
"Modelica.Clocked.Examples.SimpleControlledDrive.ClockedWithDiscreteTextbookController",
143+
"Modelica.Fluid.Examples.PumpingSystem"
144+
]
145+
]

‎regression-tests/runSingleTest.jl

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#=
2+
This file is part of OpenModelica.
3+
Copyright (c) 1998-2023, Open Source Modelica Consortium (OSMC),
4+
c/o Linköpings universitet, Department of Computer and Information Science,
5+
SE-58183 Linköping, Sweden.
6+
7+
All rights reserved.
8+
9+
THIS PROGRAM IS PROVIDED UNDER THE TERMS OF THE BSD NEW LICENSE OR THE
10+
GPL VERSION 3 LICENSE OR THE OSMC PUBLIC LICENSE (OSMC-PL) VERSION 1.2.
11+
ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS PROGRAM CONSTITUTES
12+
RECIPIENT'S ACCEPTANCE OF THE OSMC PUBLIC LICENSE OR THE GPL VERSION 3,
13+
ACCORDING TO RECIPIENTS CHOICE.
14+
15+
The OpenModelica software and the OSMC (Open Source Modelica Consortium)
16+
Public License (OSMC-PL) are obtained from OSMC, either from the above
17+
address, from the URLs: http://www.openmodelica.org or
18+
http://www.ida.liu.se/projects/OpenModelica, and in the OpenModelica
19+
distribution. GNU version 3 is obtained from:
20+
http://www.gnu.org/copyleft/gpl.html. The New BSD License is obtained from:
21+
http://www.opensource.org/licenses/BSD-3-Clause.
22+
23+
This program is distributed WITHOUT ANY WARRANTY; without even the implied
24+
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, EXCEPT AS
25+
EXPRESSLY SET FORTH IN THE BY RECIPIENT SELECTED SUBSIDIARY LICENSE
26+
CONDITIONS OF OSMC-PL.
27+
=#
28+
29+
import Pkg; Pkg.activate(@__DIR__)
30+
import OMJulia
31+
import FMI
32+
33+
using Test
34+
using DataFrames
35+
using CSV
36+
37+
"""
38+
Simulate single model to generate a result file.
39+
"""
40+
function testSimulation(omc::OMJulia.OMCSession, className::String)
41+
@info "\tSimulation"
42+
@testset "Simulation" begin
43+
res = OMJulia.API.simulate(omc, className; outputFormat="csv")
44+
resultFile = res["resultFile"]
45+
46+
@test isfile(resultFile)
47+
return resultFile
48+
end
49+
end
50+
51+
"""
52+
Build a FMU for a single model, import the generated FMU, simulate it and compare to given reference results.
53+
"""
54+
function testFmuExport(omc::OMJulia.OMCSession, className::String, referenceResult, recordValues; workdir::String)
55+
fmuPath = ""
56+
fmuImportSuccess = false
57+
@info "\tFMU Export"
58+
@testset "Export" begin
59+
fmuPath = OMJulia.API.buildModelFMU(omc, className)
60+
@test isfile(fmuPath)
61+
@test splitext(splitpath(fmuPath)[end]) == (className, ".fmu")
62+
end
63+
64+
@info "\tFMU Import"
65+
@testset "Import" begin
66+
if isfile(fmuPath)
67+
fmu = FMI.fmiLoad(fmuPath)
68+
solution = FMI.fmiSimulate(fmu; recordValues = recordValues, showProgress=false)
69+
70+
# Own implementation of CSV export, workaround for https://github.com/ThummeTo/FMI.jl/issues/198
71+
df = DataFrames.DataFrame(time = solution.values.t)
72+
for i in 1:length(solution.values.saveval[1])
73+
for var in FMI.fmi2ValueReferenceToString(fmu, solution.valueReferences[i])
74+
if in(var, recordValues)
75+
df[!, Symbol(var)] = [val[i] for val in solution.values.saveval]
76+
end
77+
end
78+
end
79+
fmiResult = joinpath(workdir, "FMI_results.csv")
80+
CSV.write(fmiResult, df)
81+
82+
#FMI.fmiSaveSolution(solution, "FMI_results.csv")
83+
fmuImportSuccess = true
84+
end
85+
@test fmuImportSuccess
86+
end
87+
88+
@info "\tCheck Results"
89+
@testset "Verification" begin
90+
if fmuImportSuccess
91+
@test (true, String[]) == OMJulia.API.diffSimulationResults(omc, "FMI_results.csv", referenceResult, "diff")
92+
else
93+
@test false
94+
end
95+
end
96+
end
97+
98+
"""
99+
Run Simulation and FMU export/import test for all models.
100+
"""
101+
function runSingleTest(library, version, model, modeldir)
102+
local resultFile
103+
104+
@info "Testing library: $library, model $model"
105+
mkpath(modeldir)
106+
omc = OMJulia.OMCSession()
107+
108+
try
109+
@testset "$model" verbose=true begin
110+
@testset "Simulation" begin
111+
OMJulia.API.cd(omc, modeldir)
112+
113+
@test OMJulia.API.loadModel(omc, library; priorityVersion = [version], requireExactVersion = true)
114+
resultFile = testSimulation(omc, model)
115+
end
116+
117+
@testset "FMI" begin
118+
if isfile(resultFile)
119+
recordValues = names(CSV.read(resultFile, DataFrame))[2:end]
120+
filter!(val -> !startswith(val, "\$"), recordValues) # Filter internal variables
121+
testFmuExport(omc, model, resultFile, recordValues; workdir=modeldir)
122+
else
123+
@test false
124+
end
125+
end
126+
end
127+
finally
128+
OMJulia.quit(omc)
129+
end
130+
end
131+
132+
# Comand-line interface
133+
if !isempty(PROGRAM_FILE)
134+
if length(ARGS) == 4
135+
library = ARGS[1]
136+
version = ARGS[2]
137+
model = ARGS[3]
138+
modeldir = ARGS[4]
139+
runSingleTest(library, version, model, modeldir)
140+
else
141+
@error "Wrong number of arguments"
142+
for a in ARGS; println(a); end
143+
return -1
144+
end
145+
end

0 commit comments

Comments
 (0)