refactor: remove Py_Get_ID and cached string objects (#263)
#1602
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests with PyDebug | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: | |
| - labeled | |
| - unlabeled | |
| - opened | |
| - synchronize | |
| - reopened | |
| paths: | |
| - setup.py | |
| - setup.cfg | |
| - pyproject.toml | |
| - MANIFEST.in | |
| - CMakeLists.txt | |
| - include/** | |
| - src/** | |
| - tests/** | |
| - optree/** | |
| - .github/workflows/tests-with-pydebug.yml | |
| # Allow to trigger the workflow manually | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: "${{ github.workflow }}-${{ github.ref }}" | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| CMAKE_BUILD_TYPE: "Debug" | |
| OPTREE_CXX_WERROR: "ON" | |
| _GLIBCXX_USE_CXX11_ABI: "1" | |
| PYTHONUNBUFFERED: "1" | |
| PYTHON: "python" # to be updated | |
| PYTHON_TAG: "py3" # to be updated | |
| PYTHON_VERSION: "3" # to be updated | |
| pybind11_VERSION: "stable" # to be updated | |
| PYENV_ROOT: "~/.pyenv" # to be updated | |
| VENV_BIN_NAME: "bin" # to be updated | |
| COLUMNS: "100" | |
| FORCE_COLOR: "1" | |
| CLICOLOR_FORCE: "1" | |
| XDG_CACHE_HOME: "${{ github.workspace }}/.cache" | |
| PIP_CACHE_DIR: "${{ github.workspace }}/.cache/pip" | |
| PIP_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu" | |
| jobs: | |
| test: | |
| name: Test for CPython ${{ matrix.python-version }}${{ matrix.python-abiflags }} on ${{ matrix.runner }} | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: [ubuntu-latest, macos-latest, windows-latest] | |
| python-version: | |
| - "3.9" | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| - "3.13" | |
| - "3.14" | |
| python-abiflags: ["d", "td"] | |
| exclude: | |
| - python-version: "3.9" | |
| python-abiflags: "td" | |
| - python-version: "3.10" | |
| python-abiflags: "td" | |
| - python-version: "3.11" | |
| python-abiflags: "td" | |
| - python-version: "3.12" | |
| python-abiflags: "td" | |
| fail-fast: false | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up pyenv (Unix) | |
| id: setup-pyenv-unix | |
| if: runner.os != 'Windows' | |
| run: | | |
| export PYENV_ROOT="${HOME}/.pyenv" | |
| export PATH="${PYENV_ROOT}/bin:${PYENV_ROOT}/shims:${PATH}" | |
| export PATH="${PWD}/venv/bin:${PATH}" | |
| git clone https://github.com/pyenv/pyenv.git "${PYENV_ROOT}" | |
| echo "PYENV_ROOT=${PYENV_ROOT}" | tee -a "${GITHUB_ENV}" | |
| echo "PATH=${PATH}" | tee -a "${GITHUB_ENV}" | |
| if [[ "${{ runner.os }}" == 'Linux' ]]; then | |
| sudo apt-get update -qq && sudo apt-get install -yqq --no-install-recommends \ | |
| make \ | |
| build-essential \ | |
| libssl-dev \ | |
| zlib1g-dev \ | |
| libbz2-dev \ | |
| libsqlite3-dev \ | |
| libncurses-dev \ | |
| libreadline-dev \ | |
| libgdbm-dev \ | |
| liblzma-dev | |
| elif [[ "${{ runner.os }}" == 'macOS' ]]; then | |
| brew update --quiet --force | |
| brew install --only-dependencies pyenv python@3 | |
| fi | |
| - name: Set up pyenv (Windows) | |
| id: setup-pyenv-windows | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: | | |
| $Env:PYENV_ROOT = "${Env:USERPROFILE}\.pyenv" | |
| $Env:PATH = "${Env:PYENV_ROOT}\pyenv-win\bin;${Env:PYENV_ROOT}\pyenv-win\shims;${Env:PATH}" | |
| $Env:PATH = "$(Get-Location)\venv\Scripts;${Env:PATH}" | |
| git clone https://github.com/pyenv-win/pyenv-win.git "${Env:PYENV_ROOT}" | |
| Write-Output "PYENV_ROOT=${Env:PYENV_ROOT}" | Out-File -FilePath "${Env:GITHUB_ENV}" -Encoding utf8 -Append | |
| Write-Output "VENV_BIN_NAME=Scripts" | Out-File -FilePath "${Env:GITHUB_ENV}" -Encoding utf8 -Append | |
| Write-Output "PATH=${Env:PATH}" | Out-File -FilePath "${Env:GITHUB_ENV}" -Encoding utf8 -Append | |
| - name: Determine Python version | |
| shell: bash | |
| run: | | |
| if [[ "${{ runner.os }}" == 'Windows' ]]; then | |
| for ((i = 0; i < 3; ++i)); do | |
| pyenv update && break | |
| done | |
| fi | |
| echo "::group::pyenv install --list" | |
| pyenv install --list | |
| echo "::endgroup::" | |
| if [[ "${{ matrix.python-abiflags }}" == *t* ]]; then | |
| echo "PYTHON_GIL=0" | tee -a "${GITHUB_ENV}" | |
| fi | |
| if [[ "${{ runner.os }}" != 'Windows' && "${{ matrix.python-abiflags }}" == *t* ]]; then | |
| PYTHON_VERSION="$( | |
| pyenv install --list | tr -d ' ' | grep -E "^${{ matrix.python-version }}" | | |
| grep -vF '-' | grep -E '[0-9]t$' | sort -rV | head -n 1 | |
| )" | |
| elif ! PYTHON_VERSION="$(pyenv latest --known "${{ matrix.python-version }}")"; then | |
| PYTHON_VERSION="$( | |
| pyenv install --list | tr -d ' ' | grep -E "^${{ matrix.python-version }}" | | |
| grep -vF '-' | grep -E '[0-9]$' | sort -rV | head -n 1 | |
| )" | |
| fi | |
| echo "PYTHON_VERSION=${PYTHON_VERSION}" | tee -a "${GITHUB_ENV}" | |
| - name: Set up pyenv cache | |
| id: pyenv-cache | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{ env.PYENV_ROOT }} | |
| key: pyenv-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}${{ matrix.python-abiflags }}-${{ env.PYTHON_VERSION }}$ | |
| - name: Set up Python | |
| if: steps.pyenv-cache.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| set -x | |
| PYENV_INSTALL_ARGS=() | |
| if [[ "${{ runner.os }}" != 'Windows' ]]; then | |
| if [[ "${{ matrix.python-abiflags }}" == *d* ]]; then | |
| PYENV_INSTALL_ARGS+=("--debug") | |
| fi | |
| else | |
| for ((i = 0; i < 3; ++i)); do | |
| pyenv update && break | |
| done | |
| fi | |
| pyenv install ${{ env.PYTHON_VERSION }} "${PYENV_INSTALL_ARGS[@]}" | |
| if [[ "${{ runner.os }}" == 'Windows' ]]; then | |
| INSTALL_ARGS=( | |
| "SimpleInstall=1" | |
| "InstallAllUsers=0" | |
| "Include_dev=1" | |
| "Include_lib=1" | |
| "Include_exe=1" | |
| "Include_pip=1" | |
| "Include_tools=1" | |
| "Include_launcher=0" | |
| "Include_test=0" | |
| ) | |
| if [[ "${{ matrix.python-abiflags }}" == *d* ]]; then | |
| INSTALL_ARGS+=( | |
| "Include_debug=1" | |
| "Include_symbols=1" | |
| ) | |
| fi | |
| if [[ "${{ matrix.python-abiflags }}" == *t* ]]; then | |
| INSTALL_ARGS+=( | |
| "Include_freethreaded=1" | |
| ) | |
| fi | |
| INSTALL_ARGS+=( | |
| 'TargetDir="${{ env.PYENV_ROOT }}\pyenv-win\versions\${{ env.PYTHON_VERSION }}"' | |
| ) | |
| INSTALLER='${{ env.PYENV_ROOT }}\pyenv-win\install_cache\'"$( | |
| cd '${{ env.PYENV_ROOT }}\pyenv-win\install_cache' && | |
| find . -name "*-${{ env.PYTHON_VERSION }}-*.exe" | | |
| head -n 1 | cut -c3- | |
| )" | |
| pyenv uninstall ${{ env.PYTHON_VERSION }} | |
| pwsh -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command \ | |
| "${INSTALLER} /quiet /uninstall 2>&1 | Out-Default" | |
| pwsh -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command \ | |
| "${INSTALLER} /quiet ${INSTALL_ARGS[*]} 2>&1 | Out-Default" | |
| fi | |
| pyenv versions | |
| pyenv global "$(pyenv versions | grep -F '${{ env.PYTHON_VERSION }}' | tr -d ' ')" | |
| pyenv rehash | |
| - name: Set up pip cache | |
| id: pip-cache | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{ env.PIP_CACHE_DIR }} | |
| key: pip-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}${{ matrix.python-abiflags }}-${{ hashFiles('pyproject.toml', 'requirements*.txt', '*/requirements*.txt') }}$ | |
| - name: Set up Environment | |
| shell: bash | |
| run: | | |
| set -x | |
| echo "::group::pyenv shims" | |
| pyenv shims | |
| echo "::endgroup::" | |
| PYTHON_EXE="${{ env.PYTHON }}" | |
| if [[ "${{ runner.os }}" == 'Windows' ]]; then | |
| if [[ "${{ matrix.python-abiflags }}" == *t* ]]; then | |
| PYTHON_EXE="${PYTHON_EXE}${{ matrix.python-version }}t" | |
| fi | |
| if [[ "${{ matrix.python-abiflags }}" == *d* ]]; then | |
| PYTHON_EXE="${PYTHON_EXE}_d" | |
| export PYTHON="${PYTHON}_d" | |
| echo "PYTHON=${PYTHON}" | tee -a "${GITHUB_ENV}" | |
| else | |
| echo "CMAKE_BUILD_TYPE=Release" | tee -a "${GITHUB_ENV}" | |
| fi | |
| fi | |
| "${PYTHON_EXE}" -m venv venv # PATH is already updated in step setup-pyenv | |
| echo "::group::Python executables" | |
| ls -alh "venv/${VENV_BIN_NAME}" | |
| source "venv/${VENV_BIN_NAME}/activate" | |
| echo "::endgroup::" | |
| "$(command -v "${PYTHON}")" -VV | |
| echo "::group::Upgrade pip" | |
| "${PYTHON}" -m pip install --upgrade pip setuptools wheel rich | |
| echo "::endgroup::" | |
| echo "::group::Python sysconfig" | |
| "${PYTHON}" -c 'import rich, sysconfig; rich.print(sysconfig.get_config_vars())' | |
| echo "::endgroup::" | |
| export PYTHON_TAG="$( | |
| echo 'import sys; print( | |
| "{0.name[0]}p{1.major}{1.minor}".format( | |
| sys.implementation, | |
| sys.version_info, | |
| ).lower(), | |
| )' | "${PYTHON}" - | |
| )${{ matrix.python-abiflags }}" | |
| echo "PYTHON_TAG=${PYTHON_TAG}" | tee -a "${GITHUB_ENV}" | |
| - name: Enable core dump generation (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo sysctl -w kernel.core_pattern="core.${PYTHON_TAG}.%P" | |
| sudo sysctl -w kernel.core_uses_pid=0 | |
| sudo sysctl -w fs.suid_dumpable=1 | |
| sysctl kernel.core_pattern kernel.core_uses_pid fs.suid_dumpable | |
| - name: Enable core dump generation (macOS) | |
| if: runner.os == 'macOS' | |
| run: | | |
| sudo sysctl -w kern.corefile="core.${PYTHON_TAG}.%P" | |
| sudo sysctl -w kern.coredump=1 | |
| sudo sysctl -w kern.sugid_coredump=1 | |
| sysctl kern.corefile kern.coredump kern.sugid_coredump | |
| - name: Enable core dump generation (Windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| choco install windowsdriverkit11 --no-progress --force --yes | |
| $pwd = Get-Location | |
| $Env:_NT_SOURCE_PATH = "${pwd};${{ env.PYENV_ROOT }}\pyenv-win\versions\${{ env.PYTHON_VERSION }}" | |
| $Env:_NT_SYMBOL_PATH = "cache*${pwd}\.symcache;${Env:_NT_SOURCE_PATH};srv*https://msdl.microsoft.com/download/symbols" | |
| Get-ChildItem -Path "${Env:ProgramFiles(x86)}\Windows Kits" -Directory | Sort-Object -Property Name | |
| $WindowsKitsDir = "${Env:ProgramFiles(x86)}\Windows Kits\10" | |
| $DebuggersDir = "${WindowsKitsDir}\Debuggers\x64" | |
| Get-ChildItem -Path "${DebuggersDir}" | |
| $Env:PATH = "${DebuggersDir};${Env:PATH}" | |
| $PYTEST = 'cdb -gG -o -c ".dump /ma /u core.dmp; !py; g; q" ${{ env.PYTHON }} -X dev -m pytest -Walways' | |
| Write-Output "_NT_SOURCE_PATH=${Env:_NT_SOURCE_PATH}" | Out-File -FilePath "${Env:GITHUB_ENV}" -Encoding utf8 -Append | |
| Write-Output "_NT_SYMBOL_PATH=${Env:_NT_SYMBOL_PATH}" | Out-File -FilePath "${Env:GITHUB_ENV}" -Encoding utf8 -Append | |
| Write-Output "PATH=${Env:PATH}" | Out-File -FilePath "${Env:GITHUB_ENV}" -Encoding utf8 -Append | |
| Write-Output "PYTEST=${PYTEST}" | Out-File -FilePath "${Env:GITHUB_ENV}" -Encoding utf8 -Append | |
| cdb -version | |
| - name: Use nightly pybind11 | |
| shell: bash | |
| if: | | |
| github.event_name == 'pull_request' && | |
| contains(github.event.pull_request.labels.*.name, 'test-with-nightly-pybind11') | |
| run: | | |
| source "venv/${VENV_BIN_NAME}/activate" | |
| ${{ env.PYTHON }} .github/workflows/set_setup_requires.py | |
| echo "::group::pyproject.toml" | |
| cat pyproject.toml | |
| echo "::endgroup::" | |
| echo "pybind11_VERSION=HEAD" | tee -a "${GITHUB_ENV}" | |
| - name: Test buildable without Python frontend | |
| if: runner.os != 'Windows' | |
| shell: bash | |
| run: | | |
| source "venv/${VENV_BIN_NAME}/activate" | |
| make cmake-build PYTHON="${{ env.PYTHON }}" && make clean | |
| - name: Install OpTree | |
| shell: bash | |
| run: | | |
| source "venv/${VENV_BIN_NAME}/activate" | |
| ${{ env.PYTHON }} -m pip install -v --editable '.[test]' | |
| - name: Test with pytest | |
| shell: bash | |
| run: | | |
| source "venv/${VENV_BIN_NAME}/activate" | |
| set -x | |
| ulimit -c unlimited | |
| ulimit -a | |
| PYTESTOPTS=( | |
| "--exitfirst" | |
| "--cov-report=xml:coverage-${{ env.PYTHON_TAG }}-${{ runner.os }}.xml" | |
| "--junit-xml=junit-${{ env.PYTHON_TAG }}-${{ runner.os }}.xml" | |
| ) | |
| make test PYTESTOPTS="${PYTESTOPTS[*]}" | |
| CORE_DUMP_FILES="$( | |
| find . -type d -path "./venv" -prune \ | |
| -o '(' -iname "core.*.[1-9]*" -o -iname "core_*.dmp" ')' -print | |
| )" | |
| if [[ -n "${CORE_DUMP_FILES}" ]]; then | |
| echo "::error::Coredump files found, indicating a crash during tests." >&2 | |
| echo "Coredump files:" >&2 | |
| ls -alh ${CORE_DUMP_FILES} >&2 | |
| exit 1 | |
| fi | |
| - name: List generated files | |
| if: ${{ !cancelled() }} | |
| shell: bash | |
| run: | | |
| find . -type f -name '*.py[co]' -delete | |
| find . -depth -type d -name "__pycache__" -exec rm -r "{}" + | |
| if git status --ignored --porcelain | grep -qvE '/$'; then | |
| ls -alh $(git status --ignored --porcelain | grep -vE '/$' | grep -oE '\S+$') | |
| fi | |
| - name: Collect backtraces from coredumps (if any) | |
| if: ${{ !cancelled() }} | |
| shell: bash | |
| run: | | |
| CORE_DUMP_FILES="$( | |
| find . -type d -path "./venv" -prune \ | |
| -o '(' -iname "core.*.[1-9]*" -o -iname "core_*.dmp" ')' -print | |
| )" | |
| if [[ -n "${CORE_DUMP_FILES}" ]]; then | |
| echo "Found core dumps:" | |
| ls -alh ${CORE_DUMP_FILES} | |
| BACKTRACE_COMMAND="" | |
| if [[ "${{ runner.os }}" == 'Linux' ]]; then | |
| echo "::group::Install GDB" | |
| ( | |
| export DEBIAN_FRONTEND=noninteractive | |
| sudo apt-get update -qq && sudo apt-get install -yqq gdb | |
| ) | |
| echo "::endgroup::" | |
| BACKTRACE_COMMAND="gdb --exec ${{ env.PYTHON }} --core '{}' -ex 'bt -full' -ex 'q'" | |
| elif [[ "${{ runner.os }}" == 'macOS' ]]; then | |
| echo "::group::Install LLDB" | |
| brew update --quiet --force | |
| brew list --formula llvm >/dev/null || brew install --formula llvm | |
| echo "::endgroup::" | |
| BACKTRACE_COMMAND="$(brew --prefix llvm)/bin/lldb --file ${{ env.PYTHON }} --core '{}' -o 'bt all' -o 'q'" | |
| elif [[ "${{ runner.os }}" == 'Windows' ]]; then | |
| BACKTRACE_COMMAND="cdb -z '{}' -c \"!py; !analyze -vv; .ecxr; kP; q\"" | |
| fi | |
| if [[ -n "${BACKTRACE_COMMAND}" ]]; then | |
| echo "Collecting backtraces:" | |
| find . -type d -path "./venv" -prune \ | |
| -o -iname "core.*.[1-9]*" \ | |
| -exec bash -xc " | |
| echo '::group::backtrace from: {}'; | |
| ${BACKTRACE_COMMAND}; | |
| echo '::endgroup::'; | |
| " ';' | |
| find . -type d -path "./venv" -prune \ | |
| -o -iname "core_*.dmp" \ | |
| -exec bash -xc " | |
| echo '::group::backtrace from: {}'; | |
| ${BACKTRACE_COMMAND}; | |
| echo '::endgroup::'; | |
| " ';' | |
| fi | |
| echo "::warning::Coredump files found, see backtraces above for details." >&2 | |
| exit 1 | |
| fi | |
| - name: Upload coverage to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: tests/coverage-${{ env.PYTHON_TAG }}-${{ runner.os }}.xml | |
| flags: unittests-pydebug,unittests-pydebug-${{ env.PYTHON_TAG }}-${{ runner.os }} | |
| name: codecov-pydebug | |
| verbose: true | |
| fail_ci_if_error: false | |
| - name: Upload JUnit results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: tests/junit-${{ env.PYTHON_TAG }}-${{ runner.os }}.xml | |
| flags: junit-pydebug,junit-pydebug-${{ env.PYTHON_TAG }}-${{ runner.os }} | |
| name: junit-pydebug | |
| verbose: true | |
| fail_ci_if_error: false | |
| - name: Upload core dump file | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: coredump-${{ env.PYTHON_TAG }}-${{ runner.os }} | |
| path: | | |
| tests/core.* | |
| tests/core_*.dmp | |
| if-no-files-found: ignore |