From 77e74a1f9aa244b5922ec22782f1f6f019502963 Mon Sep 17 00:00:00 2001 From: Diego Fernandez <110383200+DiFerMa@users.noreply.github.com> Date: Mon, 23 Sep 2024 08:40:36 +0200 Subject: [PATCH] Feature/implement coverage (#78) * implement coverage for lobster * Include coverage report and test outputs to lobster * Add ci-tests to workflow --- .github/workflows/ci.yml | 7 +++++++ Makefile | 14 +++++++++++++- coverage.cfg | 16 ++++++++++++++++ requirements_dev.txt | 1 + test-system/lobster-json/Makefile | 4 +++- test-system/lobster-python/Makefile | 4 +++- test-system/lobster-python/basic.output | 8 ++++---- .../multiple_identical_function_names.output | Bin 6174 -> 2904 bytes .../multiple_identical_function_names.py | 2 ++ test-system/lobster-python/pytest_mark.output | 8 ++++---- util/check_local_modifications.sh | 11 +++++++++++ 11 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 coverage.cfg create mode 100755 util/check_local_modifications.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9b202e7..9777ef0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,3 +65,10 @@ jobs: - name: Executing system tests run: | make system-tests + - name: Coverage analysis + run: | + make coverage + - name: Check output files + if: always() + run: | + util/check_local_modifications.sh diff --git a/Makefile b/Makefile index fa4eccc7..26fdd4bd 100644 --- a/Makefile +++ b/Makefile @@ -45,11 +45,15 @@ integration-tests: packages (cd integration-tests/projects/filter; make) system-tests: + mkdir -p docs make -B -C test-system/lobster-json make -B -C test-system/lobster-python unit-tests: - python3 -m unittest discover -s test-unit -v + coverage run -p \ + --branch --rcfile=coverage.cfg \ + --data-file .coverage \ + -m unittest discover -s test-unit -v test: integration-tests system-tests unit-tests @@ -74,3 +78,11 @@ full-release: make github-release make bump git push + +coverage: + coverage combine -q + coverage html --rcfile=coverage.cfg + coverage report --rcfile=coverage.cfg --fail-under=57 + +test-ci: system-tests unit-tests coverage + util/check_local_modifications.sh diff --git a/coverage.cfg b/coverage.cfg new file mode 100644 index 00000000..85c4728c --- /dev/null +++ b/coverage.cfg @@ -0,0 +1,16 @@ +[report] +exclude_lines = + pragma: no cover + ice_loc + @abstractmethod + if DEBUG_ + assert False + def __repr__(self) + def sanity_test() + if __name__ == "__main__" + +[run] +omit = + /usr/* + */site-packages/* + test-unit/* diff --git a/requirements_dev.txt b/requirements_dev.txt index 546c61b7..700430d4 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,3 +1,4 @@ -r requirements.txt pycodestyle==2.12.0 pylint==3.2.4 +coverage>=7.2 diff --git a/test-system/lobster-json/Makefile b/test-system/lobster-json/Makefile index 83aff3fa..3f3f880a 100644 --- a/test-system/lobster-json/Makefile +++ b/test-system/lobster-json/Makefile @@ -7,7 +7,9 @@ all: $(TARGETS) %.output: %.input @tail +2 $< > $*.json @touch $*.lobster - -$(TOOL) $(shell head -1 $< | tail --bytes=+3) --out=$*.lobster --single > $@ 2>&1 + -@coverage run -p --rcfile=../../coverage.cfg --branch \ + --data-file ../../.coverage \ + $(TOOL) $(shell head -1 $< | tail --bytes=+3) --out=$*.lobster --single > $@ 2>&1 @echo "==========" >> $@ @cat $*.lobster >> $@ @rm $*.json $*.lobster diff --git a/test-system/lobster-python/Makefile b/test-system/lobster-python/Makefile index 43f9e485..1188b10d 100644 --- a/test-system/lobster-python/Makefile +++ b/test-system/lobster-python/Makefile @@ -6,7 +6,9 @@ all: $(TARGETS) %.output: %.py @touch $*.lobster - -$(TOOL) $(shell head -1 $< | tail --bytes=+2) $< --out=$*.lobster --single > $@ 2>&1 + -@coverage run -p --rcfile=../../coverage.cfg --branch \ + --data-file ../../.coverage \ + $(TOOL) $(shell head -1 $< | tail --bytes=+2) $< --out=$*.lobster --single > $@ 2>&1 @echo "==========" >> $@ @cat $*.lobster >> $@ @rm $*.lobster diff --git a/test-system/lobster-python/basic.output b/test-system/lobster-python/basic.output index e622e4c5..e7751c8a 100644 --- a/test-system/lobster-python/basic.output +++ b/test-system/lobster-python/basic.output @@ -21,14 +21,14 @@ Written output for 3 items to basic.lobster "kind": "Function" }, { - "tag": "python basic.Example.helper_function", + "tag": "python basic.Example", "location": { "kind": "file", "file": "basic.py", "line": 13, "column": null }, - "name": "basic.Example.helper_function", + "name": "basic.Example", "messages": [], "just_up": [], "just_down": [], @@ -40,14 +40,14 @@ Written output for 3 items to basic.lobster "kind": "Method" }, { - "tag": "python basic.Example.nor", + "tag": "python basic.Example-1", "location": { "kind": "file", "file": "basic.py", "line": 17, "column": null }, - "name": "basic.Example.nor", + "name": "basic.Example", "messages": [], "just_up": [], "just_down": [], diff --git a/test-system/lobster-python/multiple_identical_function_names.output b/test-system/lobster-python/multiple_identical_function_names.output index a09f9df2307cbcb8f86c4e4c3e10b692475e0877..31c4f3979826f2977b5f4165bc9ac97eabc4c189 100644 GIT binary patch literal 2904 zcmds3O;5ux488YPL^)GLH}NTP<;;!4gsRYbT_8 z^GmFEXC)$NW;|9N+1yxm#w0@N7#dc3g|bqR${aMxOsI73b%xUDRErYau`&-1!A95Z z6TL-@@m!$bS8PV_jD<*eIEsRTyjtSI=pUdLfT@LNFD;Z*fa6CEk}{bHWe{A0PWVq! z=OMvdDromoy#A*6Za&rW<9|wOAiVP?EkR{Q`7)%{Ua3trt7e^tri21^C)!Sp+Ui6B zj&Nr8E%KvxnEL9AU~XPRJ^j4}Nh8$f1toOh37ew)wicmf+xG5eBQUSYZKyRp{Gp01Z~4vB4Q_JZ-wR isdeq#WeX^Sqit4WSynMxk*$p;e*x>(esmhGqUarjpH+eY literal 6174 zcmeHL+fKqj5S?ce|H16Dg#hsqKfq_-iD|H{2rdO%&=A9~t7mpw=w)%OmspZb)5~^e zXV2U^(|Lb;rDHmx6C8p<8qpG`peu}r+ZmqS(eX&rGwnE|7PO!>RzJ|3V$9EIMH1r) z-b;$e$5V)VtlkA!De>+eR51nW9k26oj<7bOlse!YVaBHf_a{(J)DzD#mlQjnmN05m z5(t<-@69~5GZ(GPRnk5ekiqD-=gEKkzKS=`ODWNvM&wZ-QLLlp5s-_R$hPHdk9znH zy1Z@DfsWd8-tN)Bpsn`Cx6S@0xkXm-xy;@E_zqE3rmChNp)PT?`M!UbCKP^8CNO{khy$t9jISwR(dd zTc6GkD!y&@ck#{NW_>j8vgLjCVuw(u&SZ@4SAq`P61~JE>k5iWzt}JJi;I>JsJ{dB p*}2P{_v)*wJyhv4aPY7P?-YQ#%;Q;MNI|#jkq=a#aHFn_{sTX=e!&0$ diff --git a/test-system/lobster-python/multiple_identical_function_names.py b/test-system/lobster-python/multiple_identical_function_names.py index 7e6e9e7b..4eb04247 100644 --- a/test-system/lobster-python/multiple_identical_function_names.py +++ b/test-system/lobster-python/multiple_identical_function_names.py @@ -1,3 +1,5 @@ +# + flag = False requirements_global = None diff --git a/test-system/lobster-python/pytest_mark.output b/test-system/lobster-python/pytest_mark.output index f0fd12cf..df3ff3df 100644 --- a/test-system/lobster-python/pytest_mark.output +++ b/test-system/lobster-python/pytest_mark.output @@ -3,14 +3,14 @@ Written output for 2 items to pytest_mark.lobster { "data": [ { - "tag": "pyunit pytest_mark.TestFrameServerProcessesStart.test_1", + "tag": "pyunit pytest_mark.TestFrameServerProcessesStart.test_1:21", "location": { "kind": "file", "file": "pytest_mark.py", "line": 21, "column": null }, - "name": "pytest_mark.TestFrameServerProcessesStart.test_1", + "name": "pytest_mark.TestFrameServerProcessesStart.test_1:21", "messages": [], "just_up": [], "just_down": [], @@ -23,14 +23,14 @@ Written output for 2 items to pytest_mark.lobster "status": null }, { - "tag": "pyunit pytest_mark.TestFrameServerProcessesStart.test_2", + "tag": "pyunit pytest_mark.TestFrameServerProcessesStart.test_2:24", "location": { "kind": "file", "file": "pytest_mark.py", "line": 24, "column": null }, - "name": "pytest_mark.TestFrameServerProcessesStart.test_2", + "name": "pytest_mark.TestFrameServerProcessesStart.test_2:24", "messages": [], "just_up": [], "just_down": [], diff --git a/util/check_local_modifications.sh b/util/check_local_modifications.sh new file mode 100755 index 00000000..e227600a --- /dev/null +++ b/util/check_local_modifications.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [[ $(git status -s) ]]; then + echo "Local modifications found:" + git -P diff + echo "Summary:" + git status -s + exit 1 +else + exit 0 +fi