From 641a121fcde83c49666eed359eeacc0cb5537594 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 9 Nov 2024 23:50:21 +0800 Subject: [PATCH] feat: show backtraces on Windows --- .github/workflows/tests-with-pydebug.yml | 79 +++++++++++++++++------- .github/workflows/tests.yml | 76 ++++++++++++++++++----- Makefile | 5 +- 3 files changed, 120 insertions(+), 40 deletions(-) diff --git a/.github/workflows/tests-with-pydebug.yml b/.github/workflows/tests-with-pydebug.yml index 391eb0b6..7f695ddf 100644 --- a/.github/workflows/tests-with-pydebug.yml +++ b/.github/workflows/tests-with-pydebug.yml @@ -41,7 +41,7 @@ jobs: runs-on: ${{ matrix.runner }} strategy: matrix: - runner: [ubuntu-latest, windows-latest, macos-latest] + runner: [ubuntu-latest, macos-latest, windows-latest] python-version: - "3.7" - "3.8" @@ -203,18 +203,29 @@ jobs: pyenv shims echo "::endgroup::" + PYTHON_EXE="${{ env.PYTHON }}" if [[ "${{ runner.os }}" == 'Windows' ]]; then if [[ "${{ matrix.python-abiflags }}" == *t* ]]; then - PYTHON="${PYTHON}${{ matrix.python-version }}t" + PYTHON_EXE="${PYTHON_EXE}${{ matrix.python-version }}t" fi if [[ "${{ matrix.python-abiflags }}" == *d* ]]; then - PYTHON="${PYTHON}_d" + PYTHON_EXE="${PYTHON_EXE}_d" + export PYTHON="${PYTHON}_d" + echo "PYTHON=${PYTHON}" | tee -a "${GITHUB_ENV}" + else + echo "CMAKE_BUILD_TYPE=Release" | tee -a "${GITHUB_ENV}" fi - export PYTHON="${PYTHON}" - echo "PYTHON=${PYTHON}" | tee -a "${GITHUB_ENV}" fi - "${PYTHON}" -m venv venv # PATH is already updated in step setup-pyenv + "${PYTHON_EXE}" -m venv venv # PATH is already updated in step setup-pyenv + echo "::group::Python executables" + if [[ "${{ runner.os }}" != 'Windows' ]]; then + ls -alh venv/bin + else + ls -alh venv/Scripts + fi + echo "::endgroup::" + "${PYTHON}" --version echo "::group::Upgrade pip" "${PYTHON}" -m pip install --upgrade pip setuptools wheel rich @@ -233,21 +244,47 @@ jobs: )${{ matrix.python-abiflags }}" echo "PYTHON_TAG=${PYTHON_TAG}" | tee -a "${GITHUB_ENV}" - if [[ "${{ runner.os }}" == 'Linux' ]]; then - sudo sysctl -w kernel.core_pattern="core.${PYTHON_TAG}.%P" - sudo sysctl -w kernel.core_uses_pid=0 - sudo sysctl -w fs.suid_dumpable=1 - sysctl kernel.core_pattern kernel.core_uses_pid fs.suid_dumpable - elif [[ "${{ runner.os }}" == 'macOS' ]]; then - sudo sysctl -w kern.corefile="core.${PYTHON_TAG}.%P" - sudo sysctl -w kern.coredump=1 - sudo sysctl -w kern.sugid_coredump=1 - sysctl kern.corefile kern.coredump kern.sugid_coredump - elif [[ "${{ runner.os }}" == 'Windows' ]]; then - if [[ "${{ matrix.python-abiflags }}" != *d* ]]; then - echo "CMAKE_BUILD_TYPE=Release" | tee -a "${GITHUB_ENV}" - fi - fi + - name: Enable core dump generation (Linux) + if: runner.os == 'Linux' + run: | + sudo sysctl -w kernel.core_pattern="core.${PYTHON_TAG}.%P" + sudo sysctl -w kernel.core_uses_pid=0 + sudo sysctl -w fs.suid_dumpable=1 + sysctl kernel.core_pattern kernel.core_uses_pid fs.suid_dumpable + + - name: Enable core dump generation (macOS) + if: runner.os == 'macOS' + run: | + sudo sysctl -w kern.corefile="core.${PYTHON_TAG}.%P" + sudo sysctl -w kern.coredump=1 + sudo sysctl -w kern.sugid_coredump=1 + sysctl kern.corefile kern.coredump kern.sugid_coredump + + - name: Enable core dump generation (Windows) + if: runner.os == 'Windows' + run: | + $pwd = Get-Location + $Env:_NT_SOURCE_PATH = "$pwd;${{ env.PYENV_ROOT }}\pyenv-win\versions\${{ env.PYTHON_VERSION }}" + $Env:_NT_SYMBOL_PATH = "cache*$pwd\.symcache;$Env:_NT_SOURCE_PATH;srv*https://msdl.microsoft.com/download/symbols" + Get-ChildItem -Path "C:\Program Files (x86)\Windows Kits" -Directory | Sort-Object -Property Name + $WindowsKitsDir = "C:\Program Files (x86)\Windows Kits\10" + $DebuggersDir = "$WindowsKitsDir\Debuggers\x64" + Get-ChildItem -Path $DebuggersDir + $Env:PATH = "$DebuggersDir;$Env:PATH" + $PYTEST = 'cdb -hd -gG -o -c ".sympath" -c ".srcpath" ${{ env.PYTHON }} -X dev -m pytest' + Write-Output "_NT_SOURCE_PATH=$Env:_NT_SOURCE_PATH" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + Write-Output "_NT_SYMBOL_PATH=$Env:_NT_SYMBOL_PATH" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + Write-Output "PATH=$Env:PATH" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + Write-Output "PYTEST=$PYTEST" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + + $RegPath="HKLM:\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" + New-Item -Path $RegPath -Force | Out-Null + Set-ItemProperty -Path $RegPath -Name 'DumpFolder' -Value $pwd + Set-ItemProperty -Path $RegPath -Name 'DumpCount' -Value 10 + Set-ItemProperty -Path $RegPath -Name 'DumpType' -Value 2 + Get-ItemProperty -Path $RegPath + + cdb -version - name: Install OpTree run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e5bf0fd6..8495e730 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,7 +40,7 @@ jobs: runs-on: ${{ matrix.runner }} strategy: matrix: - runner: [ubuntu-latest, windows-latest, macos-latest] + runner: [ubuntu-latest, macos-latest, windows-latest] python-version: - "3.7" - "3.8" @@ -75,6 +75,9 @@ jobs: run: | set -x + if [[ "${{ runner.os }}" == 'Windows' ]]; then + echo "CMAKE_BUILD_TYPE=Release" | tee -a "${GITHUB_ENV}" + fi ${{ env.PYTHON }} --version echo "::group::Upgrade pip" ${{ env.PYTHON }} -m pip install --upgrade pip setuptools wheel rich @@ -94,19 +97,47 @@ jobs: )" echo "PYTHON_TAG=${PYTHON_TAG}" | tee -a "${GITHUB_ENV}" - if [[ "${{ runner.os }}" == 'Linux' ]]; then - sudo sysctl -w kernel.core_pattern="core.${PYTHON_TAG}.%P" - sudo sysctl -w kernel.core_uses_pid=0 - sudo sysctl -w fs.suid_dumpable=1 - sysctl kernel.core_pattern kernel.core_uses_pid fs.suid_dumpable - elif [[ "${{ runner.os }}" == 'macOS' ]]; then - sudo sysctl -w kern.corefile="core.${PYTHON_TAG}.%P" - sudo sysctl -w kern.coredump=1 - sudo sysctl -w kern.sugid_coredump=1 - sysctl kern.corefile kern.coredump kern.sugid_coredump - elif [[ "${{ runner.os }}" == 'Windows' ]]; then - echo "CMAKE_BUILD_TYPE=Release" | tee -a "${GITHUB_ENV}" - fi + - name: Enable core dump generation (Linux) + if: runner.os == 'Linux' + run: | + sudo sysctl -w kernel.core_pattern="core.${PYTHON_TAG}.%P" + sudo sysctl -w kernel.core_uses_pid=0 + sudo sysctl -w fs.suid_dumpable=1 + sysctl kernel.core_pattern kernel.core_uses_pid fs.suid_dumpable + + - name: Enable core dump generation (macOS) + if: runner.os == 'macOS' + run: | + sudo sysctl -w kern.corefile="core.${PYTHON_TAG}.%P" + sudo sysctl -w kern.coredump=1 + sudo sysctl -w kern.sugid_coredump=1 + sysctl kern.corefile kern.coredump kern.sugid_coredump + + - name: Enable core dump generation (Windows) + if: runner.os == 'Windows' + run: | + $pwd = Get-Location + $Env:_NT_SOURCE_PATH = "$pwd;${{ env.PYENV_ROOT }}\pyenv-win\versions\${{ env.PYTHON_VERSION }}" + $Env:_NT_SYMBOL_PATH = "cache*$pwd\.symcache;$Env:_NT_SOURCE_PATH;srv*https://msdl.microsoft.com/download/symbols" + Get-ChildItem -Path "C:\Program Files (x86)\Windows Kits" -Directory | Sort-Object -Property Name + $WindowsKitsDir = "C:\Program Files (x86)\Windows Kits\10" + $DebuggersDir = "$WindowsKitsDir\Debuggers\x64" + Get-ChildItem -Path $DebuggersDir + $Env:PATH = "$DebuggersDir;$Env:PATH" + $PYTEST = 'cdb -hd -gG -o -c ".sympath" -c ".srcpath" ${{ env.PYTHON }} -X dev -m pytest' + Write-Output "_NT_SOURCE_PATH=$Env:_NT_SOURCE_PATH" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + Write-Output "_NT_SYMBOL_PATH=$Env:_NT_SYMBOL_PATH" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + Write-Output "PATH=$Env:PATH" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + Write-Output "PYTEST=$PYTEST" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + + $RegPath="HKLM:\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" + New-Item -Path $RegPath -Force | Out-Null + Set-ItemProperty -Path $RegPath -Name 'DumpFolder' -Value $pwd + Set-ItemProperty -Path $RegPath -Name 'DumpCount' -Value 10 + Set-ItemProperty -Path $RegPath -Name 'DumpType' -Value 2 + Get-ItemProperty -Path $RegPath + + cdb -version - name: Install test dependencies shell: bash @@ -154,17 +185,28 @@ jobs: shell: bash run: | if [[ -n "$(find . -iname "core.*.[1-9]*")" ]]; then + echo "Found core dumps:" + ls -alh $(find . -iname "core.*.[1-9]*") + BACKTRACE_COMMAND="" if [[ "${{ runner.os }}" == 'Linux' ]]; then + echo "::group::Install GDB" ( export DEBIAN_FRONTEND=noninteractive sudo apt-get update -qq && sudo apt-get install -yqq gdb ) - echo "Found core dumps:" - ls -alh $(find . -iname "core.*.[1-9]*") + echo "::endgroup::" + BACKTRACE_COMMAND="gdb --exec ${{ env.PYTHON }} --core '{}' -ex 'bt' -ex 'q'" + elif [[ "${{ runner.os }}" == 'macOS' ]]; then + echo "::group::Install LLDB" + brew install llvm --quiet + echo "::endgroup::" + BACKTRACE_COMMAND="lldb --file ${{ env.PYTHON }} --core '{}' -o 'bt' -o 'q'" + fi + if [[ -n "${BACKTRACE_COMMAND}" ]]; then echo "Collecting backtraces:" find . -iname "core.*.[1-9]*" -exec bash -xc " echo '::group::backtrace from: {}'; - gdb ${{ env.PYTHON }} '{}' -ex 'backtrace' -ex 'quit'; + ${BACKTRACE_COMMAND}; echo '::endgroup::'; " ';' fi diff --git a/Makefile b/Makefile index 19f93f8c..cfeae541 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ GOPATH ?= $(HOME)/go GOBIN ?= $(GOPATH)/bin PATH := $(PATH):$(GOBIN) PYTHON ?= $(shell command -v python3 || command -v python) +PYTEST ?= $(PYTHON) -X dev -m pytest PYTESTOPTS ?= CMAKE_CXX_STANDARD ?= 20 OPTREE_CXX_WERROR ?= ON @@ -139,10 +140,10 @@ addlicense-install: go-install .PHONY: pytest test pytest test: pytest-install - $(PYTHON) -m pytest --version + $(PYTEST) --version cd tests && $(PYTHON) -X dev -W 'always' -W 'error' -c 'import $(PROJECT_PATH)' && \ $(PYTHON) -X dev -W 'always' -W 'error' -c 'import $(PROJECT_PATH)._C; print(f"GLIBCXX_USE_CXX11_ABI={$(PROJECT_PATH)._C.GLIBCXX_USE_CXX11_ABI}")' && \ - $(PYTHON) -X dev -m pytest --verbose --color=yes --durations=10 --showlocals \ + $(PYTEST) --verbose --color=yes --durations=10 --showlocals \ --cov="$(PROJECT_PATH)" --cov-config=.coveragerc --cov-report=xml --cov-report=term-missing \ $(PYTESTOPTS) .