Feature/implement coverage (bmw-software-engineering#78)

* implement coverage for lobster

* Include coverage report and test outputs to lobster

* Add ci-tests to workflow

.github/workflows/ci.yml

@@ -65,3 +65,10 @@ jobs:
       - name: Executing system tests
         run: |
           make system-tests
+      - name: Coverage analysis
+        run: |
+          make coverage
+      - name: Check output files
+        if: always()
+        run: |
+          util/check_local_modifications.sh

Makefile

@@ -45,11 +45,15 @@ integration-tests: packages
 	(cd integration-tests/projects/filter; make)
 
 system-tests:
+	mkdir -p docs
 	make -B -C test-system/lobster-json
 	make -B -C test-system/lobster-python
 
 unit-tests:
-	python3 -m unittest discover -s test-unit -v
+	coverage run -p \
+			--branch --rcfile=coverage.cfg \
+			--data-file .coverage \
+			-m unittest discover -s test-unit -v
 
 test: integration-tests system-tests unit-tests
 
@@ -74,3 +78,11 @@ full-release:
 	make github-release
 	make bump
 	git push
+
+coverage:
+	coverage combine -q
+	coverage html --rcfile=coverage.cfg
+	coverage report --rcfile=coverage.cfg --fail-under=57
+
+test-ci: system-tests unit-tests coverage
+	util/check_local_modifications.sh

coverage.cfg

@@ -0,0 +1,16 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    ice_loc
+    @abstractmethod
+    if DEBUG_
+    assert False
+    def __repr__(self)
+    def sanity_test()
+    if __name__ == "__main__"
+
+[run]
+omit =
+    /usr/*
+    */site-packages/*
+    test-unit/*

requirements_dev.txt

@@ -1,3 +1,4 @@
 -r requirements.txt
 pycodestyle==2.12.0
 pylint==3.2.4
+coverage>=7.2

test-system/lobster-json/Makefile

@@ -7,7 +7,9 @@ all: $(TARGETS)
 %.output: %.input
 	@tail +2 $< > $*.json
 	@touch $*.lobster
-	-$(TOOL) $(shell head -1 $< | tail --bytes=+3) --out=$*.lobster --single > $@ 2>&1
+	-@coverage run -p --rcfile=../../coverage.cfg --branch \
+		--data-file ../../.coverage \
+		$(TOOL) $(shell head -1 $< | tail --bytes=+3) --out=$*.lobster --single > $@ 2>&1
 	@echo "==========" >> $@
 	@cat $*.lobster >> $@
 	@rm $*.json $*.lobster

test-system/lobster-python/Makefile

@@ -6,7 +6,9 @@ all: $(TARGETS)
 
 %.output: %.py
 	@touch $*.lobster
-	-$(TOOL) $(shell head -1 $< | tail --bytes=+2) $< --out=$*.lobster --single > $@ 2>&1
+	-@coverage run -p --rcfile=../../coverage.cfg --branch \
+		--data-file ../../.coverage \
+		$(TOOL) $(shell head -1 $< | tail --bytes=+2) $< --out=$*.lobster --single > $@ 2>&1
 	@echo "==========" >> $@
 	@cat $*.lobster >> $@
 	@rm $*.lobster

test-system/lobster-python/basic.output

@@ -21,14 +21,14 @@ Written output for 3 items to basic.lobster
       "kind": "Function"
     },
     {
-      "tag": "python basic.Example.helper_function",
+      "tag": "python basic.Example",
       "location": {
         "kind": "file",
         "file": "basic.py",
         "line": 13,
         "column": null
       },
-      "name": "basic.Example.helper_function",
+      "name": "basic.Example",
       "messages": [],
       "just_up": [],
       "just_down": [],
@@ -40,14 +40,14 @@ Written output for 3 items to basic.lobster
       "kind": "Method"
     },
     {
-      "tag": "python basic.Example.nor",
+      "tag": "python basic.Example-1",
       "location": {
         "kind": "file",
         "file": "basic.py",
         "line": 17,
         "column": null
       },
-      "name": "basic.Example.nor",
+      "name": "basic.Example",
       "messages": [],
       "just_up": [],
       "just_down": [],

test-system/lobster-python/multiple_identical_function_names.py

@@ -1,3 +1,5 @@
+#
+
 flag = False
 requirements_global = None
 

test-system/lobster-python/pytest_mark.output

@@ -3,14 +3,14 @@ Written output for 2 items to pytest_mark.lobster
 {
   "data": [
     {
-      "tag": "pyunit pytest_mark.TestFrameServerProcessesStart.test_1",
+      "tag": "pyunit pytest_mark.TestFrameServerProcessesStart.test_1:21",
       "location": {
         "kind": "file",
         "file": "pytest_mark.py",
         "line": 21,
         "column": null
       },
-      "name": "pytest_mark.TestFrameServerProcessesStart.test_1",
+      "name": "pytest_mark.TestFrameServerProcessesStart.test_1:21",
       "messages": [],
       "just_up": [],
       "just_down": [],
@@ -23,14 +23,14 @@ Written output for 2 items to pytest_mark.lobster
       "status": null
     },
     {
-      "tag": "pyunit pytest_mark.TestFrameServerProcessesStart.test_2",
+      "tag": "pyunit pytest_mark.TestFrameServerProcessesStart.test_2:24",
       "location": {
         "kind": "file",
         "file": "pytest_mark.py",
         "line": 24,
         "column": null
       },
-      "name": "pytest_mark.TestFrameServerProcessesStart.test_2",
+      "name": "pytest_mark.TestFrameServerProcessesStart.test_2:24",
       "messages": [],
       "just_up": [],
       "just_down": [],

util/check_local_modifications.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [[ $(git status -s) ]]; then
+    echo "Local modifications found:"
+    git -P diff
+    echo "Summary:"
+    git status -s
+    exit 1
+else
+    exit 0
+fi