feat(workflows): upload coverage / JUnit results in workflows (#170)

metaopt · Nov 6, 2024 · 518a71d · 518a71d
1 parent b75db44
commit 518a71d
Show file tree

Hide file tree

Showing 7 changed files with 177 additions and 11 deletions.
diff --git a/.github/workflows/tests-with-pydebug.yml b/.github/workflows/tests-with-pydebug.yml
@@ -27,15 +27,17 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
+  CMAKE_BUILD_TYPE: "Debug"
   OPTREE_CXX_WERROR: "ON"
   PYTHON: "python"  # to be updated
+  PYTHON_TAG: "py3"  # to be updated
   PYTHON_VERSION: "3"  # to be updated
   PYENV_ROOT: "~/.pyenv"  # to be updated
   COLUMNS: "128"
 
 jobs:
   test:
-    name: Test for Python ${{ matrix.python-version }}${{ matrix.python-abiflags }} on ${{ matrix.runner }}
+    name: Test for CPython ${{ matrix.python-version }}${{ matrix.python-abiflags }} on ${{ matrix.runner }}
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
@@ -77,6 +79,7 @@ jobs:
         run: |
           export PYENV_ROOT="${HOME}/.pyenv"
           export PATH="${PYENV_ROOT}/bin:${PYENV_ROOT}/shims:${PATH}"
+          export PATH="${PWD}/venv/bin:${PATH}"
           git clone https://github.com/pyenv/pyenv.git "${PYENV_ROOT}"
           echo "PYENV_ROOT=${PYENV_ROOT}" >> "${GITHUB_ENV}"
           echo "PATH=${PATH}" >> "${GITHUB_ENV}"
@@ -88,6 +91,7 @@ jobs:
         run: |
           $Env:PYENV_ROOT = "$Env:USERPROFILE\.pyenv"
           $Env:PATH = "$Env:PYENV_ROOT\pyenv-win\bin;$Env:PYENV_ROOT\pyenv-win\shims;$Env:PATH"
+          $Env:PATH = "$(Get-Location)\venv\Scripts;$Env:PATH"
           git clone https://github.com/pyenv-win/pyenv-win.git "$Env:PYENV_ROOT"
           Write-Output "PYENV_ROOT=$Env:PYENV_ROOT" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
           Write-Output "PATH=$Env:PATH" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
@@ -107,6 +111,7 @@ jobs:
               pyenv install --list | tr -d ' ' | grep -E "^${{ matrix.python-version }}" |
                 grep -vF '-' | grep -E '[0-9]t$' | sort -rV | head -n 1
             )"
+            echo "PYTHON_GIL=0" >> "${GITHUB_ENV}"
           elif ! PYTHON_VERSION="$(pyenv latest --known "${{ matrix.python-version }}")"; then
             PYTHON_VERSION="$(
               pyenv install --list | tr -d ' ' | grep -E "^${{ matrix.python-version }}" |
@@ -137,19 +142,70 @@ jobs:
             fi
           fi
           pyenv install ${{ env.PYTHON_VERSION }} "${PYENV_INSTALL_ARGS[@]}"
+
+          set -x
+          pyenv versions
           pyenv global "$(pyenv versions | grep -F '${{ env.PYTHON_VERSION }}' | tr -d ' ')"
           pyenv rehash
 
       - name: Set up Environment
+        shell: bash
         run: |
+          set -x
+
           ${{ env.PYTHON }} --version
-          ${{ env.PYTHON }} -c 'from pprint import pprint; import sysconfig; pprint(sysconfig.get_config_vars())'
-          ${{ env.PYTHON }} -m pip install --upgrade pip setuptools wheel
+          echo "::group::Upgrade pip"
+          ${{ env.PYTHON }} -m pip install --upgrade pip setuptools wheel rich
+          echo "::endgroup::"
+          echo "::group::Python sysconfig"
+          ${{ env.PYTHON }} -c 'import rich, sysconfig; rich.print(sysconfig.get_config_vars())'
+          echo "::endgroup::"
+
+          export PYTHON_TAG="$(
+            echo 'import sys; print(
+              "{0.name[0]}p{1.major}{1.minor}".format(
+                sys.implementation,
+                sys.version_info,
+              ).lower(),
+            )' | ${{ env.PYTHON }} -
+          )${{ matrix.python-abiflags }}"
+          echo "PYTHON_TAG=${PYTHON_TAG}" | tee -a "${GITHUB_ENV}"
+
+          if [[ "${{ runner.os }}" == 'Windows' ]]; then
+            echo "CMAKE_BUILD_TYPE=Release" | tee -a "${GITHUB_ENV}"
+          fi
 
       - name: Install OpTree
         run: |
           ${{ env.PYTHON }} -m pip install -vv --editable '.[test]'
 
       - name: Test with pytest
+        shell: bash
+        run: |
+          PYTESTOPTS=(
+            "--exitfirst"
+            "--cov-report=xml:coverage-${{ env.PYTHON_TAG }}-${{ runner.os }}.xml"
+            "--junit-xml=junit-${{ env.PYTHON_TAG }}-${{ runner.os }}.xml"
+          )
+          make test PYTESTOPTS="${PYTESTOPTS[*]}"
+
+      - name: List generated files
+        if: ${{ !cancelled() }}
+        shell: bash
         run: |
-          make test PYTESTOPTS="--exitfirst"
+          find . -type f -name '*.py[co]' -delete
+          find . -depth -type d -name "__pycache__" -exec rm -r "{}" +
+          if [[ -n "$(git status --ignored --porcelain | grep -vE '/$')" ]]; then
+            ls -alh $(git status --ignored --porcelain | grep -vE '/$' | cut -d ' ' -f2)
+          fi
+
+      - name: Upload JUnit results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: tests/junit-${{ env.PYTHON_TAG }}.xml
+          flags: junit-${{ env.PYTHON_TAG }}
+          name: junit-umbrella
+          verbose: true
+          fail_ci_if_error: false
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -27,9 +27,11 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
+  CMAKE_BUILD_TYPE: "Debug"
   OPTREE_CXX_WERROR: "ON"
   FULL_TEST_PYTHON_VERSIONS: "3.10;3.11"
   PYTHON: "python"  # to be updated
+  PYTHON_TAG: "py3"  # to be updated
   COLUMNS: "128"
 
 jobs:
@@ -69,8 +71,32 @@ jobs:
           allow-prereleases: true
 
       - name: Set up Environment
+        shell: bash
         run: |
-          ${{ env.PYTHON }} -m pip install --upgrade pip setuptools wheel
+          set -x
+
+          ${{ env.PYTHON }} --version
+          echo "::group::Upgrade pip"
+          ${{ env.PYTHON }} -m pip install --upgrade pip setuptools wheel rich
+          echo "::endgroup::"
+          echo "::group::Python sysconfig"
+          ${{ env.PYTHON }} -c 'import rich, sysconfig; rich.print(sysconfig.get_config_vars())'
+          echo "::endgroup::"
+
+          export PYTHON_TAG="$(
+            echo 'import sys; print(
+              "{0.name[0]}p{1.major}{1.minor}{2}".format(
+                sys.implementation,
+                sys.version_info,
+                getattr(sys, "abiflags", ""),
+              ).lower(),
+            )' | ${{ env.PYTHON }} -
+          )"
+          echo "PYTHON_TAG=${PYTHON_TAG}" | tee -a "${GITHUB_ENV}"
+
+          if [[ "${{ runner.os }}" == 'Windows' ]]; then
+            echo "CMAKE_BUILD_TYPE=Release" | tee -a "${GITHUB_ENV}"
+          fi
 
       - name: Install test dependencies
         shell: bash
@@ -91,15 +117,86 @@ jobs:
           ${{ env.PYTHON }} -m pip install -vv --editable '.[test]'
 
       - name: Test with pytest
+        shell: bash
         run: |
-          make test PYTESTOPTS="--exitfirst"
+          PYTESTOPTS=(
+            "--exitfirst"
+            "--cov-report=xml:coverage-${{ env.PYTHON_TAG }}-${{ runner.os }}.xml"
+            "--junit-xml=junit-${{ env.PYTHON_TAG }}-${{ runner.os }}.xml"
+          )
+          make test PYTESTOPTS="${PYTESTOPTS[*]}"
+
+      - name: List generated files
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          find . -type f -name '*.py[co]' -delete
+          find . -depth -type d -name "__pycache__" -exec rm -r "{}" +
+          if [[ -n "$(git status --ignored --porcelain | grep -vE '/$')" ]]; then
+            ls -alh $(git status --ignored --porcelain | grep -vE '/$' | cut -d ' ' -f2)
+          fi
+
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.python-version }}-${{ runner.os }}
+          path: tests/coverage-*.xml
+          if-no-files-found: error
+
+      - name: Upload JUnit artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: junit-${{ matrix.python-version }}-${{ runner.os }}
+          path: tests/junit-*.xml
+          if-no-files-found: error
+
+  upload-coverage:
+    name: Upload coverage to Codecov
+    runs-on: ubuntu-latest
+    needs: test
+    timeout-minutes: 15
+    steps:
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-*
+          path: coverage
+          merge-multiple: true
+
+      - name: List coverage artifacts
+        run: ls -lh coverage/*
 
       - name: Upload coverage to Codecov
-        if: runner.os == 'Linux'
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          file: ./tests/coverage.xml
+          directory: coverage
           flags: unittests
           name: codecov-umbrella
+          verbose: true
+          fail_ci_if_error: false
+
+  upload-junit:
+    name: Upload JUnit results to Codecov
+    runs-on: ubuntu-latest
+    needs: test
+    timeout-minutes: 15
+    steps:
+      - name: Download JUnit artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: junit-*
+          path: junit
+          merge-multiple: true
+
+      - name: List JUnit artifacts
+        run: ls -lh junit/*
+
+      - name: Upload JUnit results to Codecov
+        uses: codecov/test-results-action@v1
+        with:
+          directory: junit
+          flags: unittests
+          name: junit-umbrella
+          verbose: true
           fail_ci_if_error: false
diff --git a/.gitignore b/.gitignore
@@ -52,11 +52,18 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+coverage-*.xml
+coverage.*.xml
 *.cover
 *.py,cover
 .hypothesis/
 .pytest_cache/
 cover/
+.junit
+.junit.*
+junit.xml
+junit-*.xml
+junit.*.xml
 
 # Translations
 *.mo

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Upload coverage / JUnit results in CI workflows by [@XuehaiPan](https://github.com/XuehaiPan) in [#170](https://github.com/metaopt/optree/pull/170).
 - Add more info to `tree_flatten_one_level` by [@XuehaiPan](https://github.com/XuehaiPan) in [#168](https://github.com/metaopt/optree/pull/168).
 - Improve typing support for generic `PyTree[T]` and registry lookup / register functions by [@XuehaiPan](https://github.com/XuehaiPan) in [#160](https://github.com/metaopt/optree/pull/160) and [#166](https://github.com/metaopt/optree/pull/166).
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -31,6 +31,7 @@ endif()
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
 endif()
+message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
 
 if(NOT DEFINED CMAKE_CXX_STANDARD AND NOT "$ENV{CMAKE_CXX_STANDARD}" STREQUAL "")
     set(CMAKE_CXX_STANDARD "$ENV{CMAKE_CXX_STANDARD}")
@@ -76,6 +77,10 @@ if(MSVC)
         " /wd5262"  # use [[fallthrough]] when a break statement is intentionally omitted between cases
         " /wd5264"  # 'const' variable is not used
     )
+    string(
+        APPEND CMAKE_CXX_FLAGS_DEBUG
+        " /wd4702"  # unreachable code
+    )
     string(APPEND CMAKE_CXX_FLAGS_DEBUG " /Zi")
     string(APPEND CMAKE_CXX_FLAGS_RELEASE " /O2 /Ob2")
 else()

diff --git a/Makefile b/Makefile
@@ -270,8 +270,8 @@ clean-py:
 	find . -depth -type d -name ".ruff_cache" -exec rm -r "{}" +
 	find . -depth -type d -name ".mypy_cache" -exec rm -r "{}" +
 	find . -depth -type d -name ".pytest_cache" -exec rm -r "{}" +
-	rm -f tests/.coverage tests/.coverage.*
-	rm -f tests/coverage.xml
+	rm -f tests/.coverage tests/.coverage.* tests/coverage.xml tests/coverage-*.xml tests/coverage.*.xml
+	rm -f tests/.junit tests/.junit.* tests/junit.xml tests/junit-*.xml tests/junit.*.xml
 
 .PHONY: clean-build
 clean-build:

diff --git a/setup.py b/setup.py
@@ -36,7 +36,7 @@ def build_extension(self, ext):
         build_temp = pathlib.Path(self.build_temp).absolute()
         build_temp.mkdir(parents=True, exist_ok=True)
 
-        config = 'Debug' if self.debug else 'Release'
+        config = os.getenv('CMAKE_BUILD_TYPE', '') or ('Debug' if self.debug else 'Release')
         cmake_args = [
             f'-DCMAKE_BUILD_TYPE={config}',
             f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{config.upper()}={ext_path.parent}',