diff --git a/.github/workflows/job-examples.yml b/.github/workflows/job-examples.yml index 623fd3d..bf47f00 100644 --- a/.github/workflows/job-examples.yml +++ b/.github/workflows/job-examples.yml @@ -21,13 +21,13 @@ jobs: run: | cd examples python3 -m unittest - - name: Run test generation examples with command line tool + - name: Run test generation examples with mopyregtest command line tool run: | cd examples/generate_tests - /github/home/.local/bin/mopyregtest ./gen_tests BlocksLpDist_from_cli \ + /github/home/.local/bin/mopyregtest generate --metric=Lp_dist \ + ./gen_tests BlocksLpDist_from_cli \ ~/".openmodelica/libraries/Modelica 4.0.0+maint.om/" \ - Modelica.Blocks.Sources.Sine,Modelica.Blocks.Sources.ExpSine,Modelica.Blocks.Sources.Step \ - --metric=Lp_dist + Modelica.Blocks.Sources.Sine,Modelica.Blocks.Sources.ExpSine,Modelica.Blocks.Sources.Step cd gen_tests python3 test_blockslpdist_from_cli.py - name: Run test generation examples with script @@ -36,4 +36,13 @@ jobs: python3 gentests_modelica_blocks_sources.py cd gen_tests python3 test_blockslpdist_from_script.py - python3 test_blocksuserdef_from_script.py \ No newline at end of file + python3 test_blocksuserdef_from_script.py + - name: Run CSV file comparison with mopyregtest command line tool + run: | + /github/home/.local/bin/mopyregtest compare \ + --metric=Lp_dist \ + --tol=0.015 \ + --vars=y \ + --fill-in-method=interpolate \ + ./examples/test_user_defined_metrics/references/SineNoisy_res.csv \ + ./examples/test_user_defined_metrics/references/Sine_res.csv \ No newline at end of file diff --git a/.github/workflows/job-unit-tests.yml b/.github/workflows/job-unit-tests.yml index 4eb183d..f6cfbbf 100644 --- a/.github/workflows/job-unit-tests.yml +++ b/.github/workflows/job-unit-tests.yml @@ -3,6 +3,7 @@ on: [workflow_dispatch] jobs: unit-test: runs-on: ubuntu-latest + container: openmodelica/openmodelica:v1.21.0-minimal strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] @@ -18,6 +19,9 @@ jobs: run: | python -m pip install --upgrade pip pip install numpy pandas + - name: Install Modelica STL 4.0.0 + run: | + echo "installPackage(Modelica, \"4.0.0+maint.om\", exactMatch=true);" > installModelicaStl.mos && omc installModelicaStl.mos - name: Run unit tests run: | pip3 install --user . diff --git a/examples/generate_tests/README.md b/examples/generate_tests/README.md index 2d4a19b..ee457c0 100644 --- a/examples/generate_tests/README.md +++ b/examples/generate_tests/README.md @@ -25,8 +25,30 @@ mopyregtest --help which will print ```bash -usage: mopyregtest [-h] [--metric {norm_p_dist,norm_infty_dist,Lp_dist,Linfty_dist}] [--references REFERENCES] - test_folder test_name package_folder models_in_package +usage: mopyregtest [-h] {generate} ... + +options: + -h, --help show this help message and exit + +subcommands: + {generate} mopyregtest CLI command overview + generate Generate test case definitions + +Command line interface for MoPyRegtest, the CI friendly regression testing tool for Modelica models. This command line interface is a simplified version to +interact with MoPyRegtest. If you want to use all options, please consider creating a dedicated Python script. +``` + +and furthermore + +```bash +mopyregtest generate --help +``` + +tells you + +```bash +usage: mopyregtest generate [-h] [--metric {norm_p_dist,norm_infty_dist,Lp_dist,Linfty_dist}] [--references REFERENCES] + test_folder test_name package_folder models_in_package positional arguments: test_folder Path where test shall be generated. Advice: Should not exist yet @@ -37,14 +59,11 @@ positional arguments: options: -h, --help show this help message and exit --metric {norm_p_dist,norm_infty_dist,Lp_dist,Linfty_dist} - Metric to be used. Choose here from predefined values. For user-defined metrics please consider creating the tests with a - dedicated script. + Metric to be used. Choose here from predefined values. For user-defined metrics please consider creating the tests with a dedicated + script. --references REFERENCES - Comma separated list like :,:. Missing references for models - here will be generated. - -Command line interface for the experimental MoPyRegtest test case generator. This interface is a simplified version. If you want to use all options, -please consider creating a dedicated Python script. + Comma separated list like :,:. Missing references for models here will + be generated. ``` Just like in the @@ -56,7 +75,7 @@ change to the folder [examples/generate_tests](/examples/generate_tests), and si ```bash cd examples/generate_tests -mopyregtest ./gen_tests BlocksLpDist_from_cli ~/".openmodelica/libraries/Modelica 4.0.0+maint.om/" Modelica.Blocks.Sources.Sine,Modelica.Blocks.Sources.ExpSine,Modelica.Blocks.Sources.Step --metric=Lp_dist +mopyregtest generate --metric=Lp_dist ./gen_tests BlocksLpDist_from_cli ~/".openmodelica/libraries/Modelica 4.0.0+maint.om/" Modelica.Blocks.Sources.Sine,Modelica.Blocks.Sources.ExpSine,Modelica.Blocks.Sources.Step ``` and for Windows just adapt the path to the Modelica Standard library, e.g. to diff --git a/examples/test_user_defined_metrics/references/Sine_res.csv b/examples/test_user_defined_metrics/references/Sine_res.csv new file mode 100644 index 0000000..1e31602 --- /dev/null +++ b/examples/test_user_defined_metrics/references/Sine_res.csv @@ -0,0 +1,503 @@ +"time","y" +0,0 +0.002,0.01256603988335261 +0.004,0.02513009544333748 +0.006,0.03769018266993454 +0.008,0.05024431817976956 +0.01,0.06279051952931337 +0.012,0.07532680552793272 +0.014,0.08785119655074317 +0.016,0.1003617148512149 +0.018,0.1128563848734817 +0.02,0.1253332335643043 +0.022,0.1377902906846381 +0.024,0.1502255891207571 +0.026,0.1626371651948836 +0.028,0.175023058975276 +0.03,0.1873813145857246 +0.032,0.199709980514407 +0.034,0.2120071099220546 +0.036,0.2242707609493811 +0.038,0.2364989970237247 +0.04,0.2486898871648548 +0.042,0.2608415062898969 +0.044,0.2729519355173252 +0.046,0.2850192624699761 +0.048,0.2970415815770349 +0.05,0.3090169943749474 +0.052,0.3209436098072095 +0.054,0.3328195445229866 +0.056,0.3446429231745171 +0.058,0.3564118787132508 +0.06,0.3681245526846779 +0.062,0.3797790955218011 +0.064,0.3913736668372024 +0.066,0.4029064357136626 +0.068,0.4143755809932841 +0.07000000000000001,0.4257792915650727 +0.07199999999999999,0.4371157666509328 +0.074,0.4483832160900322 +0.076,0.4595798606214878 +0.078,0.4707039321653325 +0.08,0.4817536741017153 +0.082,0.4927273415482916 +0.08400000000000001,0.5036232016357608 +0.08599999999999999,0.5144395337815064 +0.08799999999999999,0.5251746299612956 +0.09,0.5358267949789967 +0.092,0.5463943467342691 +0.094,0.556875616488188 +0.096,0.5672689491267565 +0.098,0.5775727034222676 +0.1,0.5877852522924731 +0.102,0.5979049830575188 +0.104,0.6079302976946054 +0.106,0.6178596130903343 +0.108,0.6276913612907005 +0.11,0.6374239897486896 +0.112,0.6470559615694442 +0.114,0.6565857557529565 +0.116,0.6660118674342517 +0.118,0.6753328081210244 +0.12,0.6845471059286886 +0.122,0.6936533058128049 +0.124,0.7026499697988492 +0.126,0.7115356772092853 +0.128,0.7203090248879069 +0.13,0.7289686274214116 +0.132,0.7375131173581739 +0.134,0.7459411454241821 +0.136,0.7542513807361038 +0.138,0.7624425110114479 +0.14,0.7705132427757893 +0.142,0.7784623015670233 +0.144,0.7862884321366188 +0.146,0.7939903986478353 +0.148,0.8015669848708765 +0.15,0.8090169943749475 +0.152,0.8163392507171839 +0.154,0.8235325976284275 +0.156,0.8305958991958126 +0.158,0.8375280400421417 +0.16,0.8443279255020151 +0.162,0.8509944817946918 +0.164,0.8575266561936523 +0.166,0.8639234171928353 +0.168,0.8701837546695257 +0.17,0.8763066800438637 +0.172,0.8822912264349532 +0.174,0.8881364488135445 +0.176,0.8938414241512637 +0.178,0.899405251566371 +0.18,0.9048270524660196 +0.182,0.9101059706849957 +0.184,0.9152411726209175 +0.186,0.9202318473658703 +0.188,0.925077206834458 +0.19,0.9297764858882513 +0.192,0.934328942456612 +0.194,0.9387338576538741 +0.196,0.9429905358928644 +0.198,0.9470983049947443 +0.2,0.9510565162951535 +0.202,0.954864544746643 +0.204,0.9585217890173758 +0.206,0.9620276715860858 +0.208,0.9653816388332739 +0.21,0.9685831611286311 +0.212,0.9716317329146739 +0.214,0.9745268727865771 +0.216,0.9772681235681935 +0.218,0.9798550523842469 +0.22,0.9822872507286886 +0.222,0.9845643345292053 +0.224,0.986685944207868 +0.226,0.9886517447379141 +0.228,0.9904614256966512 +0.23,0.9921147013144779 +0.232,0.9936113105200084 +0.234,0.9949510169813002 +0.236,0.9961336091431725 +0.238,0.9971589002606139 +0.24,0.9980267284282716 +0.242,0.9987369566060175 +0.244,0.9992894726405892 +0.246,0.9996841892832999 +0.248,0.9999210442038161 +0.25,1 +0.252,0.9999210442038161 +0.254,0.9996841892832999 +0.256,0.9992894726405892 +0.258,0.9987369566060175 +0.26,0.9980267284282716 +0.262,0.9971589002606139 +0.264,0.9961336091431725 +0.266,0.9949510169813002 +0.268,0.9936113105200084 +0.27,0.9921147013144778 +0.272,0.9904614256966512 +0.274,0.988651744737914 +0.276,0.986685944207868 +0.278,0.9845643345292053 +0.28,0.9822872507286886 +0.282,0.9798550523842469 +0.284,0.9772681235681935 +0.286,0.9745268727865772 +0.288,0.971631732914674 +0.29,0.9685831611286312 +0.292,0.965381638833274 +0.294,0.9620276715860859 +0.296,0.9585217890173759 +0.298,0.9548645447466431 +0.3,0.9510565162951536 +0.302,0.9470983049947443 +0.304,0.9429905358928645 +0.306,0.9387338576538742 +0.308,0.9343289424566121 +0.31,0.9297764858882513 +0.312,0.925077206834458 +0.314,0.9202318473658704 +0.316,0.9152411726209176 +0.318,0.9101059706849957 +0.32,0.9048270524660195 +0.322,0.8994052515663712 +0.324,0.8938414241512639 +0.326,0.8881364488135446 +0.328,0.8822912264349533 +0.33,0.8763066800438635 +0.332,0.8701837546695257 +0.334,0.8639234171928354 +0.336,0.8575266561936522 +0.338,0.8509944817946917 +0.34,0.844327925502015 +0.342,0.8375280400421418 +0.344,0.8305958991958129 +0.346,0.8235325976284276 +0.348,0.8163392507171841 +0.35,0.8090169943749475 +0.352,0.8015669848708769 +0.354,0.7939903986478355 +0.356,0.786288432136619 +0.358,0.7784623015670235 +0.36,0.7705132427757893 +0.362,0.7624425110114481 +0.364,0.754251380736104 +0.366,0.7459411454241822 +0.368,0.7375131173581739 +0.37,0.7289686274214114 +0.372,0.720309024887907 +0.374,0.7115356772092855 +0.376,0.7026499697988492 +0.378,0.6936533058128049 +0.38,0.6845471059286888 +0.382,0.6753328081210246 +0.384,0.6660118674342517 +0.386,0.6565857557529564 +0.388,0.6470559615694442 +0.39,0.6374239897486899 +0.392,0.6276913612907006 +0.394,0.6178596130903343 +0.396,0.6079302976946053 +0.398,0.5979049830575187 +0.4,0.5877852522924732 +0.402,0.5775727034222676 +0.404,0.5672689491267564 +0.406,0.5568756164881878 +0.408,0.5463943467342692 +0.41,0.535826794978997 +0.412,0.525174629961296 +0.414,0.5144395337815066 +0.416,0.5036232016357609 +0.418,0.4927273415482916 +0.42,0.4817536741017156 +0.422,0.4707039321653328 +0.424,0.459579860621488 +0.426,0.4483832160900323 +0.428,0.4371157666509329 +0.43,0.4257792915650729 +0.432,0.4143755809932843 +0.434,0.4029064357136627 +0.436,0.3913736668372024 +0.438,0.3797790955218014 +0.44,0.3681245526846781 +0.442,0.3564118787132508 +0.444,0.3446429231745171 +0.446,0.3328195445229865 +0.448,0.3209436098072097 +0.45,0.3090169943749475 +0.452,0.2970415815770349 +0.454,0.2850192624699761 +0.456,0.2729519355173251 +0.458,0.2608415062898971 +0.46,0.2486898871648548 +0.462,0.2364989970237246 +0.464,0.224270760949381 +0.466,0.2120071099220548 +0.468,0.1997099805144071 +0.47,0.187381314585725 +0.472,0.1750230589752763 +0.474,0.1626371651948838 +0.476,0.1502255891207571 +0.478,0.1377902906846385 +0.48,0.1253332335643045 +0.482,0.1128563848734819 +0.484,0.100361714851215 +0.486,0.08785119655074315 +0.488,0.07532680552793304 +0.49,0.06279051952931358 +0.492,0.05024431817976966 +0.494,0.03769018266993453 +0.496,0.02513009544333781 +0.498,0.01256603988335284 +0.5,1.224646799147353e-16 +0.502,-0.01256603988335259 +0.504,-0.02513009544333757 +0.506,-0.03769018266993429 +0.508,-0.05024431817976942 +0.51,-0.06279051952931335 +0.512,-0.07532680552793279 +0.514,-0.0878511965507429 +0.516,-0.1003617148512147 +0.518,-0.1128563848734816 +0.52,-0.1253332335643043 +0.522,-0.1377902906846382 +0.524,-0.1502255891207569 +0.526,-0.1626371651948835 +0.528,-0.1750230589752761 +0.53,-0.1873813145857248 +0.532,-0.1997099805144072 +0.534,-0.2120071099220545 +0.536,-0.2242707609493812 +0.538,-0.2364989970237248 +0.54,-0.248689887164855 +0.542,-0.2608415062898968 +0.544,-0.2729519355173252 +0.546,-0.2850192624699762 +0.548,-0.2970415815770351 +0.55,-0.3090169943749477 +0.552,-0.3209436098072095 +0.554,-0.3328195445229867 +0.556,-0.3446429231745172 +0.5580000000000001,-0.356411878713251 +0.5600000000000001,-0.3681245526846783 +0.5620000000000001,-0.3797790955218012 +0.5639999999999999,-0.3913736668372021 +0.5659999999999999,-0.4029064357136621 +0.5679999999999999,-0.4143755809932836 +0.57,-0.4257792915650723 +0.572,-0.4371157666509327 +0.574,-0.4483832160900317 +0.576,-0.4595798606214874 +0.578,-0.4707039321653322 +0.58,-0.481753674101715 +0.582,-0.4927273415482914 +0.584,-0.5036232016357604 +0.586,-0.5144395337815061 +0.588,-0.5251746299612954 +0.59,-0.5358267949789964 +0.592,-0.546394346734269 +0.594,-0.5568756164881876 +0.596,-0.5672689491267562 +0.598,-0.5775727034222674 +0.6,-0.587785252292473 +0.602,-0.5979049830575185 +0.604,-0.607930297694605 +0.606,-0.6178596130903341 +0.608,-0.6276913612907004 +0.61,-0.6374239897486896 +0.612,-0.647055961569444 +0.614,-0.6565857557529562 +0.616,-0.6660118674342514 +0.618,-0.6753328081210244 +0.62,-0.6845471059286887 +0.622,-0.6936533058128047 +0.624,-0.7026499697988491 +0.626,-0.7115356772092852 +0.628,-0.7203090248879069 +0.63,-0.7289686274214113 +0.632,-0.7375131173581737 +0.634,-0.7459411454241821 +0.636,-0.7542513807361038 +0.638,-0.7624425110114479 +0.64,-0.7705132427757894 +0.642,-0.7784623015670236 +0.644,-0.7862884321366186 +0.646,-0.793990398647835 +0.648,-0.8015669848708764 +0.65,-0.8090169943749473 +0.652,-0.8163392507171839 +0.654,-0.8235325976284275 +0.656,-0.8305958991958128 +0.658,-0.8375280400421419 +0.66,-0.8443279255020153 +0.662,-0.8509944817946921 +0.664,-0.8575266561936521 +0.666,-0.8639234171928352 +0.668,-0.8701837546695256 +0.67,-0.8763066800438636 +0.672,-0.8822912264349534 +0.674,-0.8881364488135446 +0.676,-0.8938414241512639 +0.678,-0.8994052515663712 +0.68,-0.9048270524660198 +0.6820000000000001,-0.9101059706849955 +0.6840000000000001,-0.9152411726209175 +0.6860000000000001,-0.9202318473658704 +0.6879999999999999,-0.9250772068344577 +0.6899999999999999,-0.9297764858882511 +0.6919999999999999,-0.9343289424566118 +0.694,-0.938733857653874 +0.696,-0.9429905358928643 +0.698,-0.9470983049947442 +0.7,-0.9510565162951535 +0.702,-0.954864544746643 +0.704,-0.9585217890173756 +0.706,-0.9620276715860857 +0.708,-0.9653816388332738 +0.71,-0.968583161128631 +0.712,-0.9716317329146739 +0.714,-0.9745268727865771 +0.716,-0.9772681235681934 +0.718,-0.9798550523842469 +0.72,-0.9822872507286887 +0.722,-0.9845643345292054 +0.724,-0.9866859442078679 +0.726,-0.988651744737914 +0.728,-0.9904614256966512 +0.73,-0.9921147013144778 +0.732,-0.9936113105200084 +0.734,-0.9949510169813002 +0.736,-0.9961336091431725 +0.738,-0.9971589002606139 +0.74,-0.9980267284282716 +0.742,-0.9987369566060175 +0.744,-0.9992894726405892 +0.746,-0.9996841892832999 +0.748,-0.9999210442038161 +0.75,-1 +0.752,-0.9999210442038161 +0.754,-0.9996841892832999 +0.756,-0.9992894726405892 +0.758,-0.9987369566060175 +0.76,-0.9980267284282716 +0.762,-0.9971589002606139 +0.764,-0.9961336091431725 +0.766,-0.9949510169813002 +0.768,-0.9936113105200084 +0.77,-0.9921147013144779 +0.772,-0.9904614256966512 +0.774,-0.988651744737914 +0.776,-0.986685944207868 +0.778,-0.9845643345292054 +0.78,-0.9822872507286887 +0.782,-0.979855052384247 +0.784,-0.9772681235681935 +0.786,-0.9745268727865771 +0.788,-0.971631732914674 +0.79,-0.9685831611286311 +0.792,-0.9653816388332738 +0.794,-0.9620276715860858 +0.796,-0.9585217890173757 +0.798,-0.9548645447466431 +0.8,-0.9510565162951536 +0.802,-0.9470983049947443 +0.804,-0.9429905358928644 +0.806,-0.9387338576538741 +0.8080000000000001,-0.934328942456612 +0.8100000000000001,-0.9297764858882512 +0.8120000000000001,-0.9250772068344579 +0.8139999999999999,-0.9202318473658705 +0.8159999999999999,-0.9152411726209176 +0.8179999999999999,-0.9101059706849958 +0.82,-0.9048270524660199 +0.822,-0.8994052515663714 +0.824,-0.8938414241512641 +0.826,-0.8881364488135448 +0.828,-0.8822912264349535 +0.83,-0.8763066800438638 +0.832,-0.8701837546695258 +0.834,-0.8639234171928354 +0.836,-0.8575266561936523 +0.838,-0.8509944817946923 +0.84,-0.8443279255020155 +0.842,-0.8375280400421421 +0.844,-0.8305958991958129 +0.846,-0.8235325976284277 +0.848,-0.8163392507171842 +0.85,-0.8090169943749476 +0.852,-0.8015669848708766 +0.854,-0.7939903986478353 +0.856,-0.7862884321366188 +0.858,-0.7784623015670239 +0.86,-0.7705132427757896 +0.862,-0.7624425110114481 +0.864,-0.7542513807361041 +0.866,-0.7459411454241823 +0.868,-0.737513117358174 +0.87,-0.7289686274214116 +0.872,-0.7203090248879068 +0.874,-0.7115356772092852 +0.876,-0.7026499697988496 +0.878,-0.6936533058128054 +0.88,-0.6845471059286889 +0.882,-0.6753328081210247 +0.884,-0.6660118674342518 +0.886,-0.6565857557529565 +0.888,-0.6470559615694443 +0.89,-0.6374239897486896 +0.892,-0.6276913612907002 +0.894,-0.6178596130903348 +0.896,-0.6079302976946057 +0.898,-0.5979049830575192 +0.9,-0.5877852522924734 +0.902,-0.5775727034222677 +0.904,-0.5672689491267565 +0.906,-0.556875616488188 +0.908,-0.5463943467342689 +0.91,-0.5358267949789963 +0.912,-0.5251746299612954 +0.914,-0.5144395337815068 +0.916,-0.503623201635761 +0.918,-0.4927273415482917 +0.92,-0.4817536741017153 +0.922,-0.4707039321653325 +0.924,-0.4595798606214877 +0.926,-0.448383216090032 +0.928,-0.4371157666509325 +0.93,-0.4257792915650722 +0.9320000000000001,-0.4143755809932844 +0.9340000000000001,-0.4029064357136629 +0.9360000000000001,-0.3913736668372025 +0.9379999999999999,-0.3797790955218019 +0.9399999999999999,-0.3681245526846787 +0.9419999999999999,-0.3564118787132513 +0.944,-0.3446429231745176 +0.946,-0.3328195445229871 +0.948,-0.3209436098072098 +0.95,-0.3090169943749476 +0.952,-0.297041581577035 +0.954,-0.285019262469977 +0.956,-0.272951935517326 +0.958,-0.2608415062898976 +0.96,-0.2486898871648553 +0.962,-0.2364989970237251 +0.964,-0.2242707609493816 +0.966,-0.2120071099220549 +0.968,-0.1997099805144072 +0.97,-0.1873813145857247 +0.972,-0.175023058975276 +0.974,-0.1626371651948843 +0.976,-0.1502255891207577 +0.978,-0.1377902906846386 +0.98,-0.1253332335643046 +0.982,-0.112856384873482 +0.984,-0.1003617148512151 +0.986,-0.08785119655074328 +0.988,-0.07532680552793272 +0.99,-0.06279051952931326 +0.992,-0.05024431817977022 +0.994,-0.0376901826699351 +0.996,-0.02513009544333794 +0.998,-0.01256603988335296 +1,-2.449293598294706e-16 +1,-2.449293598294706e-16 diff --git a/mopyregtest/cli.py b/mopyregtest/cli.py index 75f7a58..f0ee508 100644 --- a/mopyregtest/cli.py +++ b/mopyregtest/cli.py @@ -8,31 +8,25 @@ import argparse import pathlib +import sys from mopyregtest import metrics -from mopyregtest import Generator +from mopyregtest import Generator, RegressionTest -def main(): - parser = argparse.ArgumentParser( - epilog="Command line interface for the experimental MoPyRegtest test case generator. " - "This interface is a simplified version. If you want to use all options, please consider " - "creating a dedicated Python script.") - parser.add_argument("test_folder", type=str, help="Path where test shall be generated. Advice: Should not exist yet") - parser.add_argument("test_name", type=str, help="Name of the test. Do not use special characters") - parser.add_argument("package_folder", type=str, - help="Path to Modelica package from which models shall be tested. Relative paths are possible") - parser.add_argument("models_in_package", type=str, - help="Comma separated list of model names like , " - "to be turned into regression tests") - parser.add_argument("--metric", type=str, help="Metric to be used. Choose here from predefined values. " - "For user-defined metrics please consider creating the tests with a dedicated script.", - choices=["norm_p_dist", "norm_infty_dist", "Lp_dist", "Linfty_dist"], default="norm_infty_dist") - parser.add_argument("--references", type=str, - help="Comma separated list like :,:. " - "Missing references for models here will be generated.") - - args = parser.parse_args() +def metric_str_to_func(m: str): + if m == "norm_p_dist": + return metrics.norm_p_dist + elif m == "norm_infty_dist": + return metrics.norm_infty_dist + elif m == "Lp_dist": + return metrics.Lp_dist + elif m == "Linfty_dist": + return metrics.Linfty_dist + else: + raise ValueError("Invalid value for metric") + +def generate(args): test_name = args.test_name test_folder = pathlib.Path(args.test_folder) result_folder = "results" @@ -46,16 +40,88 @@ def main(): metric = metrics.norm_infty_dist if args.metric is not None: - if args.metric == "norm_p_dist": - metric = metrics.norm_p_dist - elif args.metric == "norm_infty_dist": - metric = metrics.norm_infty_dist - elif args.metric == "Lp_dist": - metric = metrics.Lp_dist - elif args.metric == "Linfty_dist": - metric = metrics.Linfty_dist - else: - raise ValueError("Invalid value for metric") + metric = metric_str_to_func(args.metric) gen = Generator(package_folder=package_folder, models_in_package=models_in_package, metric=metric) gen.generate_tests(test_folder, test_name, result_folder, references) + + return + + +def compare(args): + reference_result = pathlib.Path(args.ref_csv_file).absolute() + simulation_result = pathlib.Path(args.sim_csv_file).absolute() + + if args.vars is not None: + validated_cols = args.vars.split(",") + else: + validated_cols = [] + + metric = metrics.norm_infty_dist + if args.metric is not None: + metric = metric_str_to_func(args.metric) + + RegressionTest.compare_csv_files(reference_result, simulation_result, + args.tol, validated_cols, metric, True, args.fill_in_method) + + return + + +def parse_args(cmd_args): + # Main parser + main_parser = argparse.ArgumentParser( + prog="mopyregtest", + epilog="Command line interface for MoPyRegtest, the CI friendly regression testing tool for Modelica models. " + "This command line interface is a simplified version to interact with MoPyRegtest. " + "If you want to use all options, please consider creating a dedicated Python script.") + subparsers = main_parser.add_subparsers(title="subcommands", help="mopyregtest CLI command overview") + + # mopyregtest generate + generate_parser = subparsers.add_parser("generate", help="Generate test case definitions") + + # mopyregtest generate [--metric {norm_p_dist,norm_infty_dist,Lp_dist,Linfty_dist}] [--references REFERENCES] test_folder test_name package_folder models_in_package + generate_parser.add_argument("test_folder", type=str, help="Path where test shall be generated. Advice: Should not exist yet") + generate_parser.add_argument("test_name", type=str, help="Name of the test. Do not use special characters") + generate_parser.add_argument("package_folder", type=str, + help="Path to Modelica package from which models shall be tested. Relative paths are possible") + generate_parser.add_argument("models_in_package", type=str, + help="Comma separated list of model names like , " + "to be turned into regression tests") + generate_parser.add_argument("--metric", type=str, + help="Metric to be used. Choose here from predefined values. " + "For user-defined metrics please consider creating the tests with a dedicated script. " + "If omitted, the default is norm_infty_dist", + choices=["norm_p_dist", "norm_infty_dist", "Lp_dist", "Linfty_dist"], default="norm_infty_dist") + generate_parser.add_argument("--references", type=str, + help="Comma separated list like :,:. " + "Missing references for models here will be generated.") + generate_parser.set_defaults(func=generate) + + # mopyregtest compare + compare_parser = subparsers.add_parser("compare", help="Compare CSV result files") + + # mopyregtest compare [--metric {norm_p_dist,norm_infty_dist,Lp_dist,Linfty_dist}] [--vars VARS] csv_file_1 csv_fil_2 + compare_parser.add_argument("ref_csv_file", type=str, help="Path of the reference CSV result file to compare") + compare_parser.add_argument("sim_csv_file", type=str, help="Path of the simulation CSV result file to compare") + compare_parser.add_argument("--metric", type=str, + help="Metric to be used. Choose here from predefined values. " + "For user-defined metrics please consider creating the tests with a dedicated script.", + choices=["norm_p_dist", "norm_infty_dist", "Lp_dist", "Linfty_dist"], + default="norm_infty_dist") + compare_parser.add_argument("--vars", type=str, + help="Comma separated list like ,. " + "If omitted, then all common column names from both CSV files will be used.") + compare_parser.add_argument("--tol", type=float, + help="Absolute tolerance up to which deviation in the comparison metric is accepted", + default=1e-7) + compare_parser.add_argument("--fill-in-method", type=str, + help="Defines the method used to fill in data when calling RegressionTest._unify_timestamps", + choices=["ffill", "bfill", "interpolate"], default="ffill") + compare_parser.set_defaults(func=compare) + + args = main_parser.parse_args(cmd_args) + args.func(args) + + +def main(): + parse_args(sys.argv[1:]) diff --git a/mopyregtest/modelicaregressiontest.py b/mopyregtest/modelicaregressiontest.py index 7797976..f64d404 100644 --- a/mopyregtest/modelicaregressiontest.py +++ b/mopyregtest/modelicaregressiontest.py @@ -182,11 +182,11 @@ def _unify_timestamps(results: List[pd.DataFrame], fill_in_method="ffill"): # Fill in values at missing timestamps if fill_in_method == "ffill": - results_ext[i] = results_ext[i].fillna(method="ffill", axis=0) + results_ext[i] = results_ext[i].ffill(axis=0) elif fill_in_method == "bfill": - results_ext[i] = results_ext[i].fillna(method="bfill", axis=0) + results_ext[i] = results_ext[i].bfill(axis=0) elif fill_in_method == "interpolate": - results_ext[i] = results_ext[i].interpolate() + results_ext[i] = results_ext[i].interpolate(axis=0) else: raise ValueError("Unknown filling method for NaN values") @@ -216,6 +216,61 @@ def _import_and_simulate(self): return + @staticmethod + def compare_csv_files(reference_result, simulation_result, tol=1e-7, validated_cols=[], + metric=metrics.norm_infty_dist, + unify_timestamps=True, fill_in_method="ffill"): + """ + Compares two CSV files from Modelica simulation runs, one as a reference result, the other one as the actual + simulation result. + + The default values of this function are identical to the ones in RegressionTest.compare_result + + Parameters + ---------- + reference_result : str + Path to a reference .csv file + simulation_result : str + Path to a simulation result .csv file + tol : float + See doc string of RegressionTest.compare_result + validated_cols : list + See doc string of RegressionTest.compare_result + metric : Callable + See doc string of RegressionTest.compare_result + unify_timestamps : bool + See doc string of RegressionTest.compare_result + fill_in_method : str + See doc string of RegressionTest.compare_result + + Returns + ------- + out : None + """ + ref_data = pd.read_csv(filepath_or_buffer=reference_result, delimiter=',') + result_data = pd.read_csv(filepath_or_buffer=simulation_result, delimiter=',') + + if unify_timestamps: + data_ext = RegressionTest._unify_timestamps([ref_data, result_data], fill_in_method) + ref_data = data_ext[0] + result_data = data_ext[1] + + # Determine common columns by comparing column headers + common_cols = set(ref_data.columns).intersection(set(result_data.columns)) + + if not validated_cols: + validated_cols = common_cols + + for c in validated_cols: + print("Comparing column \"{}\"".format(c)) + delta = metric(ref_data[["time", c]].values, result_data[["time", c]].values) + if np.abs(delta) >= tol: + raise AssertionError( + f"Values in Colum {c} of results {simulation_result} and {reference_result} differ by " \ + f"{np.abs(delta)} which is larger than {tol}.") + + return + def compare_result(self, reference_result, tol=1e-7, validated_cols=[], metric=metrics.norm_infty_dist, unify_timestamps=True, fill_in_method="ffill"): @@ -228,7 +283,7 @@ def compare_result(self, reference_result, tol=1e-7, validated_cols=[], reference_result : str Path to a reference .csv file containing the expected results of the model tol : float - Absolute tolerance up to which equality is tested + Absolute tolerance up to which deviation in the comparison metric is accepted validated_cols : list List of variable names (from the file header) in the reference .csv file that are used in the regression test metric : Callable @@ -277,26 +332,9 @@ def compare_result(self, reference_result, tol=1e-7, validated_cols=[], print("Comparing simulation result {} and reference {}".format(simulation_result, reference_result)) - ref_data = pd.read_csv(filepath_or_buffer=reference_result, delimiter=',') - result_data = pd.read_csv(filepath_or_buffer=simulation_result, delimiter=',') - - if unify_timestamps: - data_ext = self._unify_timestamps([ref_data, result_data], fill_in_method) - ref_data = data_ext[0] - result_data = data_ext[1] - - # Determine common columns by comparing column headers - common_cols = set(ref_data.columns).intersection(set(result_data.columns)) - - if not validated_cols: - validated_cols = common_cols - - for c in validated_cols: - print("Comparing column \"{}\"".format(c)) - delta = metric(ref_data[["time", c]].values, result_data[["time", c]].values) - if np.abs(delta) >= tol: - raise AssertionError(f"Values in Colum {c} of results {simulation_result} and {reference_result} differ by " \ - f"{np.abs(delta)} which is larger than {tol}.") + RegressionTest.compare_csv_files(reference_result, simulation_result, tol, validated_cols, + metric, + unify_timestamps, fill_in_method) return diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..c7e57b3 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,42 @@ +import unittest +import pathlib +from mopyregtest import cli + +this_folder = pathlib.Path(__file__).absolute().parent + +class TestCli(unittest.TestCase): + def test_generate(self): + cmd_args = ["generate", + "--metric=Lp_dist", + str(this_folder / "../examples/generate_tests/gen_tests"), + "BlocksLpDist_from_cli", + "~/.openmodelica/libraries/Modelica 4.0.0+maint.om/", + "Modelica.Blocks.Sources.Sine,Modelica.Blocks.Sources.ExpSine,Modelica.Blocks.Sources.Step"] + + cli.parse_args(cmd_args) + + return + + def test_compare1(self): + cmd_args = ["compare", + "--metric=Lp_dist", + "--tol=2.5e-4", + "--vars=sine.y,y", + "--fill-in-method=interpolate", + str(this_folder / "../examples/test_user_defined_metrics/references/SineNoisy_res.csv"), + str(this_folder / "../examples/test_user_defined_metrics/references/SineNoisy_res.csv")] + + cli.parse_args(cmd_args) + + return + + def test_compare2(self): + cmd_args = ["compare", + "--vars=y", + "--fill-in-method=interpolate", + str(this_folder / "../examples/test_user_defined_metrics/references/SineNoisy_res.csv"), + str(this_folder / "../examples/test_user_defined_metrics/references/Sine_res.csv")] + + self.assertRaises(AssertionError, cli.parse_args, cmd_args=cmd_args) + + return diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 856d364..63e7345 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -10,13 +10,13 @@ def test_Lp_norm(self): [10, -5]]) p = 2 - self.assertAlmostEquals(mopyregtest.metrics.Lp_norm(f), ((1-0) * 1**p + (3-1) * 2**p + (10-3) * 3**p)**(1/p)) + self.assertAlmostEqual(mopyregtest.metrics.Lp_norm(f), ((1-0) * 1**p + (3-1) * 2**p + (10-3) * 3**p)**(1/p)) p = 3 - self.assertAlmostEquals(mopyregtest.metrics.Lp_norm(f, p), ((1-0) * 1**p + (3-1) * 2**p + (10-3) * 3**p)**(1/p)) + self.assertAlmostEqual(mopyregtest.metrics.Lp_norm(f, p), ((1-0) * 1**p + (3-1) * 2**p + (10-3) * 3**p)**(1/p)) p = 1 - self.assertAlmostEquals(mopyregtest.metrics.Lp_norm(f, p), ((1-0) * 1**p + (3-1) * 2**p + (10-3) * 3**p)**(1/p)) + self.assertAlmostEqual(mopyregtest.metrics.Lp_norm(f, p), ((1-0) * 1**p + (3-1) * 2**p + (10-3) * 3**p)**(1/p)) def test_Lp_dist(self): f1 = np.array([[0, 1], @@ -41,10 +41,10 @@ def test_Lp_dist(self): [10, -5]]) p = 2 - self.assertAlmostEquals(mopyregtest.metrics.Lp_dist(f1, f2), ((1-0) * 6**p + (3-1) * 4**p + (10-3) * 8**p)**(1/p)) + self.assertAlmostEqual(mopyregtest.metrics.Lp_dist(f1, f2), ((1-0) * 6**p + (3-1) * 4**p + (10-3) * 8**p)**(1/p)) p = 3 - self.assertAlmostEquals(mopyregtest.metrics.Lp_dist(f1, f2, p), ((1-0) * 6**p + (3-1) * 4**p + (10-3) * 8**p)**(1/p)) + self.assertAlmostEqual(mopyregtest.metrics.Lp_dist(f1, f2, p), ((1-0) * 6**p + (3-1) * 4**p + (10-3) * 8**p)**(1/p)) p = 2 self.assertRaises(ValueError, mopyregtest.metrics.Lp_dist, f1=f1, f2=f3, p=p) @@ -58,7 +58,7 @@ def test_Linfty_norm(self): [3, -3], [10, -5]]) - self.assertAlmostEquals(mopyregtest.metrics.Linfty_norm(f), 5) + self.assertAlmostEqual(mopyregtest.metrics.Linfty_norm(f), 5) def test_Linfty_dist(self): f1 = np.array([[0, 1], @@ -82,7 +82,7 @@ def test_Linfty_dist(self): [3, -3], [10, -5]]) - self.assertAlmostEquals(mopyregtest.metrics.Linfty_dist(f1, f2), 8) + self.assertAlmostEqual(mopyregtest.metrics.Linfty_dist(f1, f2), 8) self.assertRaises(ValueError, mopyregtest.metrics.Linfty_dist, f1=f1, f2=f3) @@ -111,12 +111,12 @@ def test_norm_p_dist(self): [10, -5]]) p = 2 - self.assertAlmostEquals(mopyregtest.metrics.norm_p_dist(f1, f2), - (6 ** p + 4 ** p + 8 ** p + 7 ** p)**(1/p)) + self.assertAlmostEqual(mopyregtest.metrics.norm_p_dist(f1, f2), + (6 ** p + 4 ** p + 8 ** p + 7 ** p)**(1/p)) p = 3 - self.assertAlmostEquals(mopyregtest.metrics.norm_p_dist(f1, f2, p), - (6 ** p + 4 ** p + 8 ** p + 7 ** p)**(1/p)) + self.assertAlmostEqual(mopyregtest.metrics.norm_p_dist(f1, f2, p), + (6 ** p + 4 ** p + 8 ** p + 7 ** p)**(1/p)) p = 2 self.assertRaises(ValueError, mopyregtest.metrics.norm_p_dist, f1=f1, f2=f3, p=p) @@ -146,7 +146,7 @@ def test_norm_infty_dist(self): [3, -3], [10, -5]]) - self.assertAlmostEquals(mopyregtest.metrics.norm_infty_dist(f1, f2), 8) + self.assertAlmostEqual(mopyregtest.metrics.norm_infty_dist(f1, f2), 8) self.assertRaises(ValueError, mopyregtest.metrics.norm_infty_dist, f1=f1, f2=f3)