Skip to content

Commit 0232044

Browse files
sjoelundclaude
sjoelund
and
claude
committed
Separate wasm-jit compile time from sim time
The wasm-jit target JIT-compiles the model inside omc, and the test tool ran `translateModel` followed by `simulate(resimulateExecutable=)`. That path never calls `buildModel`, so `timeCompile` is not measured and the compile lands in the simulation time instead — reported as a build time of 0 and a correspondingly inflated `sim`. `--nobuildmodel` translates, builds and runs in a single `simulate()` call, which reports the phase times itself, and takes the build and simulation times from that record rather than the wall clock around the command. For VehicleInterfaces the same run splits as: | Flag | build | sim | |-------------------|---------|---------| | (none) | 0.00000 | 0.44668 | | `--nobuildmodel` | 0.21299 | 0.21309 | `--coldhot` simulates each model twice in the same omc. The second run reuses the module compiled for the first, so it is the hot number; both are printed but only the hot one is stored, since the database columns are enumerated in the insert. Both flags only apply to `simCodeTarget=wasm-jit`, and only set their config key when passed, so the confighash — and therefore the stored history — of a normal run is unchanged. `runRegressiontest` gains a `testFlags` argument, passed to `test.py` verbatim; the wasm-jit job uses it for `--nobuildmodel`. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent 90e8bc6 commit 0232044

3 files changed

Lines changed: 95 additions & 37 deletions

File tree

‎.CI/Jenkinsfile‎

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ pipeline {
5757
expression { params.v1_26 }
5858
}
5959
steps {
60-
runRegressiontest('maintenance/v1.26', 'v1.26', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false)
60+
runRegressiontest('maintenance/v1.26', 'v1.26', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false)
6161
}
6262
}
6363

@@ -74,7 +74,7 @@ pipeline {
7474
expression { params.v1_27 }
7575
}
7676
steps {
77-
runRegressiontest('maintenance/v1.27', 'v1.27', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false)
77+
runRegressiontest('maintenance/v1.27', 'v1.27', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false)
7878
}
7979
}
8080

@@ -91,7 +91,7 @@ pipeline {
9191
expression { params.master }
9292
}
9393
steps {
94-
runRegressiontest('master', 'master', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false)
94+
runRegressiontest('master', 'master', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false)
9595
}
9696
}
9797

@@ -108,7 +108,7 @@ pipeline {
108108
expression { params.conversion_script }
109109
}
110110
steps {
111-
runRegressiontest('master', 'conversion', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, true)
111+
runRegressiontest('master', 'conversion', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, true)
112112
}
113113
}
114114

@@ -125,7 +125,7 @@ pipeline {
125125
expression { params.newInst_newBackend }
126126
}
127127
steps {
128-
runRegressiontest('master', 'newInst-newBackend', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend")', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false)
128+
runRegressiontest('master', 'newInst-newBackend', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend")', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false)
129129
}
130130
}
131131

@@ -142,7 +142,7 @@ pipeline {
142142
expression { params.fmi_v1_26 }
143143
}
144144
steps {
145-
runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
145+
runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
146146
}
147147
}
148148
stage('v1.27 FMI with OMSimulator') {
@@ -158,7 +158,7 @@ pipeline {
158158
expression { params.fmi_v1_27 }
159159
}
160160
steps {
161-
runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
161+
runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
162162
}
163163
}
164164
stage('master FMI with OMSimulator') {
@@ -174,7 +174,7 @@ pipeline {
174174
expression { params.fmi_master }
175175
}
176176
steps {
177-
runRegressiontest('master', 'master-fmi', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
177+
runRegressiontest('master', 'master-fmi', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
178178
}
179179
}
180180

@@ -191,7 +191,7 @@ pipeline {
191191
expression { params.cs_fmu_cvode_v1_26 }
192192
}
193193
steps {
194-
runRegressiontest('maintenance/v1.26', 'v1.26-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
194+
runRegressiontest('maintenance/v1.26', 'v1.26-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
195195
}
196196
}
197197
stage('v1.27 CVODE CS-FMUs with OMSimulator') {
@@ -207,7 +207,7 @@ pipeline {
207207
expression { params.cs_fmu_cvode_v1_27 }
208208
}
209209
steps {
210-
runRegressiontest('maintenance/v1.27', 'v1.27-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
210+
runRegressiontest('maintenance/v1.27', 'v1.27-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
211211
}
212212
}
213213
stage('master CVODE CS-FMUs with OMSimulator') {
@@ -223,7 +223,7 @@ pipeline {
223223
expression { params.cs_fmu_cvode_master }
224224
}
225225
steps {
226-
runRegressiontest('master', 'master-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
226+
runRegressiontest('master', 'master-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
227227
}
228228
}
229229

@@ -240,7 +240,7 @@ pipeline {
240240
expression { params.fmpy_fmi_v1_26 }
241241
}
242242
steps {
243-
runRegressiontest('maintenance/v1.26', 'v1.26-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
243+
runRegressiontest('maintenance/v1.26', 'v1.26-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
244244
}
245245
}
246246

@@ -257,7 +257,7 @@ pipeline {
257257
expression { params.fmpy_fmi_v1_27 }
258258
}
259259
steps {
260-
runRegressiontest('maintenance/v1.27', 'v1.27-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
260+
runRegressiontest('maintenance/v1.27', 'v1.27-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
261261
}
262262
}
263263

@@ -274,7 +274,7 @@ pipeline {
274274
expression { params.fmpy_fmi_master }
275275
}
276276
steps {
277-
runRegressiontest('master', 'master-fmi-fmpy', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
277+
runRegressiontest('master', 'master-fmi-fmpy', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
278278
}
279279
}
280280

@@ -291,7 +291,7 @@ pipeline {
291291
expression { params.newInst_daeMode }
292292
}
293293
steps {
294-
runRegressiontest('master', 'newInst-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --daeMode=true")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
294+
runRegressiontest('master', 'newInst-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --daeMode=true")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
295295
}
296296
}
297297
stage('newBackend-daeMode') {
@@ -307,7 +307,7 @@ pipeline {
307307
expression { params.newBackend_daeMode }
308308
}
309309
steps {
310-
runRegressiontest('master', 'newBackend-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend --daeMode=true")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
310+
runRegressiontest('master', 'newBackend-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend --daeMode=true")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
311311
}
312312
}
313313
stage('oldInst') {
@@ -323,7 +323,7 @@ pipeline {
323323
expression { params.oldInst }
324324
}
325325
steps {
326-
runRegressiontest('master', 'oldInst', 'setCommandLineOptions("-d=nonewInst")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
326+
runRegressiontest('master', 'oldInst', 'setCommandLineOptions("-d=nonewInst")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
327327
}
328328
}
329329
stage('cvode') {
@@ -339,7 +339,7 @@ pipeline {
339339
expression { params.cvode }
340340
}
341341
steps {
342-
runRegressiontest('master', 'cvode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s cvode', false, false)
342+
runRegressiontest('master', 'cvode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s cvode', '', false, false)
343343
}
344344
}
345345
stage('gbode') {
@@ -355,7 +355,7 @@ pipeline {
355355
expression { params.gbode }
356356
}
357357
steps {
358-
runRegressiontest('master', 'gbode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s gbode -gbm=radauIIA3', false, false)
358+
runRegressiontest('master', 'gbode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s gbode -gbm=radauIIA3', '', false, false)
359359
}
360360
}
361361
stage('ida') {
@@ -371,7 +371,7 @@ pipeline {
371371
expression { params.ida }
372372
}
373373
steps {
374-
runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s ida', false, false)
374+
runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '-s ida', '', false, false)
375375
}
376376
}
377377
stage('wasm-jit') {
@@ -387,7 +387,7 @@ pipeline {
387387
expression { params.wasm_jit }
388388
}
389389
steps {
390-
runRegressiontest('master', 'wasm-jit', 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
390+
runRegressiontest('master', 'wasm-jit', 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '--nobuildmodel', false, false)
391391
}
392392
}
393393
stage('generateSymbolicJacobian') {
@@ -403,7 +403,7 @@ pipeline {
403403
expression { params.generateSymbolicJacobian }
404404
}
405405
steps {
406-
runRegressiontest('master', 'generateSymbolicJacobian', 'setCommandLineOptions("--generateSymbolicJacobian")', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false)
406+
runRegressiontest('master', 'generateSymbolicJacobian', 'setCommandLineOptions("--generateSymbolicJacobian")', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false)
407407
}
408408
}
409409
stage('heavy_tests') {
@@ -419,7 +419,7 @@ pipeline {
419419
expression { params.heavy_tests }
420420
}
421421
steps {
422-
runRegressiontest('master', 'heavy_tests', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', false, false, 1, 'configs/heavy_tests.json')
422+
runRegressiontest('master', 'heavy_tests', '', '', 'ripper1', 'LibraryTestingRipper1DB', false, '', '', false, false, 1, 'configs/heavy_tests.json')
423423
}
424424
}
425425

@@ -436,7 +436,7 @@ pipeline {
436436
expression { params.cpp_v1_26 }
437437
}
438438
steps {
439-
runRegressiontest('maintenance/v1.26', 'v1.26-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
439+
runRegressiontest('maintenance/v1.26', 'v1.26-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
440440
}
441441
}
442442
stage('C++ v1.27') {
@@ -452,7 +452,7 @@ pipeline {
452452
expression { params.cpp_v1_27 }
453453
}
454454
steps {
455-
runRegressiontest('maintenance/v1.27', 'v1.27-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
455+
runRegressiontest('maintenance/v1.27', 'v1.27-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
456456
}
457457
}
458458

@@ -469,7 +469,7 @@ pipeline {
469469
expression { params.cpp }
470470
}
471471
steps {
472-
runRegressiontest('master', 'cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', false, false)
472+
runRegressiontest('master', 'cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false)
473473
}
474474
}
475475
} }
@@ -694,6 +694,7 @@ done
694694
* @parm sshConfig: SSH configuration saved on test node.
695695
* @param omcompiler: Checkout old OMCompiler submodule. Should be `false` nowadays.
696696
* @param extrasimflags: Additional simulation flags passed to test.py via flag `--extrasimflags`.
697+
* @param testFlags: Additional flags passed to test.py verbatim, e.g. `--nobuildmodel`.
697698
* @param removePackageOrder: Passed to `installLibraries`.
698699
* @param conversionScript: Passed to `installLibraries`.
699700
* @param jobs: The number of tests/jobs to launch in parallel.
@@ -702,7 +703,7 @@ done
702703
* @param libs_config_file: The config file to be used for testing.
703704
* This file specifies which libraries to test and what options to use for them.
704705
*/
705-
def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, omcompiler, extrasimflags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json') {
706+
def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json') {
706707
sh '''
707708
find /tmp -name "*openmodelica.hudson*" -exec rm {} ";" || true
708709
mkdir -p ~/TEST_LIBS_BACKUP
@@ -911,7 +912,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om
911912
912913
cd OpenModelicaLibraryTesting
913914
# Force /usr/bin/omc as being used for generating the mos-files. Ensures consistent behavior among all tested OMC versions
914-
stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1
915+
stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1
915916
"""
916917
sh 'date'
917918
sh "rm -f OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp"

‎test.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
parser.add_argument('--extrasimflags', default='')
3535
parser.add_argument('--ompython_omhome', default='')
3636
parser.add_argument('--noclean', action="store_true", default=False)
37+
parser.add_argument('--nobuildmodel', action="store_true", help="Translate, build and simulate in a single simulate() call instead of translateModel() followed by simulate(resimulateExecutable=...), so the JIT compile is reported as build time rather than simulation time. Only used by simCodeTarget=wasm-jit.", default=False)
38+
parser.add_argument('--coldhot', action="store_true", help="Simulate each model twice in the same omc; the second run reuses the compiled module. Both times are printed, but only the hot one is stored. Only used by simCodeTarget=wasm-jit.", default=False)
3739
parser.add_argument('--fmisimulator', default='')
3840
parser.add_argument('--ulimitvmem', help="Virtual memory limit (in kB) (linux only)", type=int, default=8*1024*1024)
3941
parser.add_argument('--default', action='append', help="Add a default value for some configuration key, such as --default=ulimitExe=60. The equals sign is mandatory.", default=[])
@@ -572,6 +574,11 @@ def hashReferenceFiles(s):
572574
skipped_libs = {}
573575
tests=[]
574576
for (library,conf) in configs:
577+
# Only when asked, so a normal run's confighash is unchanged
578+
if args.nobuildmodel:
579+
conf["noBuildModel"] = True
580+
if args.coldhot:
581+
conf["coldHot"] = True
575582
c=conf.copy()
576583
del(c["configFromFile"])
577584
if "referenceFiles" in c:
@@ -901,6 +908,15 @@ def loadJsonOrEmptySet(f):
901908
#for k in sorted(stats.keys(), key=lambda c: stats[c][3]["exectime"], reverse=True):
902909
# print("%s: exectime %.2f" % (k, stats[k][3]["exectime"]))
903910

911+
if args.coldhot:
912+
# Only "sim" (the hot run) is stored; the cold one is reported here
913+
print("Cold vs hot simulation time:")
914+
for key in sorted(stats.keys(), key=lambda k: stats[k][1]):
915+
(name,model,libname,data)=stats[key]
916+
if data.get("simcold") is not None:
917+
print(" %-70s cold %8.4f hot %8.4f" % (model, data["simcold"], data.get("sim") or 0.0))
918+
sys.stdout.flush()
919+
904920
for key in stats.keys():
905921
(name,model,libname,data)=stats[key]
906922
stats_by_libname[libname]["stats"].append(stats[key])

0 commit comments

Comments
 (0)