From f50f3ba0497fb621423bc95446b7225f671f91bf Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 15 Nov 2025 17:43:08 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=93=A6=20Build=20release=20artifacts?= =?UTF-8?q?=20in=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a narrowly scoped initial integration of `reusable-tox.yml` into the project's CI/CD infra. It makes use of the following external action and reusable workflow repositories, some are optional: * `codecov/codecov-action` (transitively, via `tox-dev/workflow`) * `irongut/CodeCoverageSummary` (transitively, via `tox-dev/workflow`) * `re-actors/cache-python-deps` « (directly; and also transitively, via `tox-dev/workflow`) * `re-actors/checkout-python-sdist` « (transitively, via `tox-dev/workflow`) * `test-summary/action` (transitively, via `tox-dev/workflow`) * `tox-dev/workflow` « (directly) The marked projects are under my control. This patch plugs the main CI workflow as reusable into the release one. The CI workflow gets two new jobs — pre-setup that computes a number of variables, and build that makes the artifacts and stores them for later consumption by the PyPI upload job. The build job also double-checks that the artifacts are created with expected names. The invocation of `pypa/build` moved to tox so that it's possible to run the same process locally. In follow-up PRs, these dists will also be used in testing. --- .github/actions/cache-keys/action.yml | 48 ++++ .../hooks/post-tox-run/action.yml | 82 ++++++ .github/workflows/ci.yml | 246 +++++++++++++++++- .github/workflows/release.yml | 88 ++----- tox.ini | 71 +++++ 5 files changed, 471 insertions(+), 64 deletions(-) create mode 100644 .github/actions/cache-keys/action.yml create mode 100644 .github/reusables/tox-dev/workflow/reusable-tox/hooks/post-tox-run/action.yml diff --git a/.github/actions/cache-keys/action.yml b/.github/actions/cache-keys/action.yml new file mode 100644 index 000000000..68e762193 --- /dev/null +++ b/.github/actions/cache-keys/action.yml @@ -0,0 +1,48 @@ +--- + +name: placeholder +description: placeholder + +outputs: + cache-key-for-dep-files: + description: >- + A cache key string derived from the dependency declaration files. + value: ${{ steps.calc-cache-key-files.outputs.files-hash-key }} + +runs: + using: composite + steps: + - name: >- + Calculate dependency files' combined hash value + for use in the cache key + id: calc-cache-key-files + run: | + from os import environ + from pathlib import Path + + FILE_APPEND_MODE = 'a' + + files_derived_hash = '${{ + hashFiles( + 'tox.ini', + 'pyproject.toml', + '.pre-commit-config.yaml', + 'pytest.ini', + 'dependencies/**', + 'dependencies/*/**', + 'setup.cfg' + ) + }}' + + print(f'Computed file-derived hash is {files_derived_hash}.') + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print( + f'files-hash-key={files_derived_hash}', + file=outputs_file, + ) + shell: python + +... diff --git a/.github/reusables/tox-dev/workflow/reusable-tox/hooks/post-tox-run/action.yml b/.github/reusables/tox-dev/workflow/reusable-tox/hooks/post-tox-run/action.yml new file mode 100644 index 000000000..4072d6b2b --- /dev/null +++ b/.github/reusables/tox-dev/workflow/reusable-tox/hooks/post-tox-run/action.yml @@ -0,0 +1,82 @@ +--- + +inputs: + calling-job-context: + description: A JSON with the calling job inputs + type: string + current-job-steps: + description: >- + The `$ {{ steps }}` context passed from the reusable workflow's + tox job encoded as a JSON string. The caller passes it this input + as follows: + `current-job-steps: $ {{ toJSON(steps) }}`. + type: string + job-dependencies-context: + default: >- + {} + description: >- + The `$ {{ needs }}` context passed from the calling workflow + encoded as a JSON string. The caller is expected to form this + input as follows: + `job-dependencies-context: $ {{ toJSON(needs) }}`. + required: false + type: string + +runs: + using: composite + steps: + - name: Verify that the artifacts with expected names got created + if: fromJSON(inputs.calling-job-context).toxenv == 'build-dists' + run: > + # Verify that the artifacts with expected names got created + + + ls -1 + 'dist/${{ + fromJSON( + inputs.job-dependencies-context + ).pre-setup.outputs.sdist-artifact-name + }}' + 'dist/${{ + fromJSON( + inputs.job-dependencies-context + ).pre-setup.outputs.wheel-artifact-name + }}' + shell: bash + - name: Store the distribution packages + if: fromJSON(inputs.calling-job-context).toxenv == 'build-dists' + uses: actions/upload-artifact@v4 + with: + name: >- + ${{ + fromJSON( + inputs.job-dependencies-context + ).pre-setup.outputs.dists-artifact-name + }} + # NOTE: Exact expected file names are specified here + # NOTE: as a safety measure — if anything weird ends + # NOTE: up being in this dir or not all dists will be + # NOTE: produced, this will fail the workflow. + path: | + dist/${{ + fromJSON( + inputs.job-dependencies-context + ).pre-setup.outputs.sdist-artifact-name + }} + dist/${{ + fromJSON( + inputs.job-dependencies-context + ).pre-setup.outputs.wheel-artifact-name + }} + retention-days: >- + ${{ + fromJSON( + fromJSON( + inputs.job-dependencies-context + ).pre-setup.outputs.release-requested + ) + && 90 + || 30 + }} + +... diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27aa62ddb..c8f795be5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,14 +14,43 @@ on: description: >- A JSON string with pip versions to test against under CPython. - required: true + required: false type: string cpython-versions: description: >- A JSON string with CPython versions to test against. - required: true + required: false type: string + release-version: + description: >- + Target PEP440-compliant version to release. + Please, don't prepend `v`. + required: false + type: string + release-committish: + default: '' + description: >- + The commit to be released to PyPI and tagged + in Git as `release-version`. Normally, you + should keep this empty. + type: string + + outputs: + dists-artifact-name: + description: Workflow artifact name containing dists. + value: ${{ jobs.pre-setup.outputs.dists-artifact-name }} + is-upstream-repository: + description: >- + A flag representing whether the workflow runs in the upstream + repository or a fork. + value: ${{ jobs.pre-setup.outputs.is-upstream-repository }} + project-name: + description: PyPI project name. + value: ${{ jobs.pre-setup.outputs.project-name }} + project-version: + description: PyPI project version string. + value: ${{ jobs.pre-setup.outputs.dist-version }} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -44,8 +73,221 @@ env: PYTEST_THEME PYTEST_THEME_MODE PRE_COMMIT_COLOR + UPSTREAM_REPOSITORY_ID: >- + 5746963 jobs: + pre-setup: + name: ⚙️ Pre-set global build settings + + runs-on: ubuntu-latest + + timeout-minutes: 2 # network is slow sometimes when fetching from Git + + defaults: + run: + shell: python + + outputs: + # NOTE: These aren't env vars because the `${{ env }}` context is + # NOTE: inaccessible when passing inputs to reusable workflows. + dists-artifact-name: python-package-distributions + dist-version: >- + ${{ + steps.request-check.outputs.release-requested == 'true' + && inputs.release-version + || steps.scm-version.outputs.dist-version + }} + project-name: ${{ steps.metadata.outputs.project-name }} + release-requested: >- + ${{ + steps.request-check.outputs.release-requested || false + }} + cache-key-for-dep-files: >- + ${{ steps.calc-cache-key-files.outputs.cache-key-for-dep-files }} + sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }} + wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }} + is-upstream-repository: >- + ${{ toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id) }} + + steps: + - name: Switch to using Python 3.14 by default + uses: actions/setup-python@v6 + with: + python-version: 3.14 + - name: >- + Mark the build as untagged '${{ + github.event.repository.default_branch + }}' branch build + id: untagged-check + if: >- + github.event_name == 'push' && + github.ref_type == 'branch' && + github.ref_name == github.event.repository.default_branch + run: | + from os import environ + from pathlib import Path + + FILE_APPEND_MODE = 'a' + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print('is-untagged-devel=true', file=outputs_file) + - name: Mark the build as "release request" + id: request-check + if: inputs.release-version != '' + run: | + from os import environ + from pathlib import Path + + FILE_APPEND_MODE = 'a' + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print('release-requested=true', file=outputs_file) + - name: Check out src from Git + uses: actions/checkout@v4 + with: + fetch-depth: >- + ${{ + steps.request-check.outputs.release-requested == 'true' + && 1 || 0 + }} + ref: ${{ inputs.release-committish }} + - name: Scan static PEP 621 core packaging metadata + id: metadata + run: | + from os import environ + from pathlib import Path + from tomllib import loads as parse_toml_from_string + + FILE_APPEND_MODE = 'a' + + pyproject_toml_txt = Path('pyproject.toml').read_text() + metadata = parse_toml_from_string(pyproject_toml_txt)['project'] + project_name = metadata["name"] + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print(f'project-name={project_name}', file=outputs_file) + - name: >- + Calculate dependency files' combined hash value + for use in the cache key + if: >- + steps.request-check.outputs.release-requested != 'true' + id: calc-cache-key-files + uses: ./.github/actions/cache-keys + - name: Set up pip cache + if: >- + steps.request-check.outputs.release-requested != 'true' + uses: re-actors/cache-python-deps@release/v1 + with: + cache-key-for-dependency-files: >- + ${{ steps.calc-cache-key-files.outputs.cache-key-for-dep-files }} + - name: Drop Git tags from HEAD for non-release requests + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + git tag --points-at HEAD + | + xargs git tag --delete + shell: bash -eEuxo pipefail {0} + - name: Set up versioning prerequisites + if: >- + steps.request-check.outputs.release-requested != 'true' + run: >- + python -m + pip install + --user + setuptools-scm + shell: bash -eEuxo pipefail {0} + - name: Set the current dist version from Git + if: steps.request-check.outputs.release-requested != 'true' + id: scm-version + run: | + from os import environ + from pathlib import Path + + import setuptools_scm + + FILE_APPEND_MODE = 'a' + + ver = setuptools_scm.get_version(local_scheme='dirty-tag') + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print(f'dist-version={ver}', file=outputs_file) + print( + f'dist-version-for-filenames={ver.replace("+", "-")}', + file=outputs_file, + ) + - name: Set the expected dist artifact names + id: artifact-name + env: + PROJECT_NAME: ${{ steps.metadata.outputs.project-name }} + run: | + from os import environ + from pathlib import Path + + FILE_APPEND_MODE = 'a' + + whl_file_prj_base_name = environ['PROJECT_NAME'].replace('-', '_') + sdist_file_prj_base_name = ( + whl_file_prj_base_name. + replace('.', '_'). + lower() + ) + + with Path(environ['GITHUB_OUTPUT']).open( + mode=FILE_APPEND_MODE, + ) as outputs_file: + print( + f"sdist={sdist_file_prj_base_name !s}-${{ + steps.request-check.outputs.release-requested == 'true' + && inputs.release-version + || steps.scm-version.outputs.dist-version + }}.tar.gz", + file=outputs_file, + ) + print( + f"wheel={whl_file_prj_base_name !s}-${{ + steps.request-check.outputs.release-requested == 'true' + && inputs.release-version + || steps.scm-version.outputs.dist-version + }}-py3-none-any.whl", + file=outputs_file, + ) + + build: + name: >- + 📦 Build dists + needs: + - pre-setup # transitive, for accessing settings + + uses: tox-dev/workflow/.github/workflows/reusable-tox.yml@617ca35caa695c572377861016677905e58a328c # yamllint disable-line rule:line-length + with: + cache-key-for-dependency-files: >- + ${{ needs.pre-setup.outputs.cache-key-for-dep-files }} + check-name: Build dists under 🐍3.12 + checkout-src-git-committish: >- + ${{ inputs.release-committish }} + checkout-src-git-fetch-depth: >- + ${{ + fromJSON(needs.pre-setup.outputs.release-requested) + && 1 + || 0 + }} + job-dependencies-context: >- # context for hooks + ${{ toJSON(needs) }} + python-version: 3.12 + runner-vm-os: ubuntu-latest + timeout-minutes: 2 + toxenv: build-dists + xfail: false + linters: name: Linters uses: ./.github/workflows/reusable-qa.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 414006915..bebe3b045 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,10 +2,6 @@ name: 📦 Packaging on: - pull_request: - push: - branches: - - main release: types: - published @@ -17,7 +13,6 @@ env: PIP_NO_PYTHON_VERSION_WARNING: 1 # Hide "this Python is deprecated" message PIP_NO_WARN_SCRIPT_LOCATION: 1 # Hide "script dir is not in $PATH" message PRE_COMMIT_COLOR: always - PROJECT_NAME: pip-tools PY_COLORS: 1 # Recognized by the `py` package, dependency of `pytest` PYTHONIOENCODING: utf-8 PYTHONUTF8: 1 @@ -36,13 +31,11 @@ env: PYTHONIOENCODING PYTHONLEGACYWINDOWSSTDIO PYTHONUTF8 - UPSTREAM_REPOSITORY_ID: >- - 5746963 run-name: >- ${{ github.event.action == 'published' - && format('📦 Releasing v{0}...', github.ref_name) + && format('📦 Releasing {0}...', github.ref_name) || format('🌱 Smoke-testing packaging for commit {0}', github.sha) }} triggered by: ${{ github.event_name }} of ${{ @@ -59,68 +52,29 @@ run-name: >- }}) jobs: - build: + build-and-test: name: >- - 📦 v${{ github.ref_name }} + 📦 ${{ github.ref_name }} [mode: ${{ github.event.action == 'published' && 'release' || 'nightly' }}] - - runs-on: ubuntu-latest - - timeout-minutes: 2 - - outputs: - # NOTE: These aren't env vars because the `${{ env }}` context is - # NOTE: inaccessible when passing inputs to reusable workflows. - upstream-repository-id: ${{ env.UPSTREAM_REPOSITORY_ID }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - - name: Install dependencies - run: | - python -Im pip install -U twine build - - - name: Build package - run: | - python -Im build - twine check --strict dist/* - - - name: Store the distribution packages - uses: actions/upload-artifact@v4 - with: - name: python-package-distributions - # NOTE: Exact expected file names are specified here - # NOTE: as a safety measure — if anything weird ends - # NOTE: up being in this dir or not all dists will be - # NOTE: produced, this will fail the workflow. - path: | - dist/*.tar.gz - dist/*.whl - retention-days: >- - ${{ - github.event.action == 'published' - && 90 || 30 - }} + uses: ./.github/workflows/ci.yml + with: + release-version: ${{ github.ref_name }} + release-committish: '' publish-pypi: name: >- 📦 - Publish v${{ github.ref_name }} to PyPI + Publish v${{ + needs.build-and-test.outputs.project-version + }} to PyPI needs: - - build + - build-and-test if: >- github.event.action == 'published' - && needs.build.outputs.upstream-repository-id == github.repository_id + && fromJSON(needs.build-and-test.outputs.is-upstream-repository) runs-on: ubuntu-latest @@ -129,7 +83,11 @@ jobs: environment: name: pypi url: >- - https://pypi.org/project/${{ env.PROJECT_NAME }}/${{ github.ref_name }} + https://pypi.org/project/${{ + needs.build-and-test.outputs.project-name + }}/${{ + needs.build-and-test.outputs.project-version + }} permissions: id-token: write # PyPI Trusted Publishing (OIDC) @@ -138,11 +96,14 @@ jobs: - name: Download all the dists uses: actions/download-artifact@v4 with: - name: python-package-distributions + name: >- + ${{ needs.build-and-test.outputs.dists-artifact-name }} path: dist/ - name: >- 📦 - Publish v${{ github.ref_name }} to PyPI + Publish v${{ + needs.build-and-test.outputs.project-version + }} to PyPI 🔏 uses: pypa/gh-action-pypi-publish@release/v1 - name: Clean up the publish attestation leftovers @@ -152,4 +113,7 @@ jobs: with: user: jazzband password: ${{ secrets.JAZZBAND_RELEASE_KEY }} - repository-url: https://jazzband.co/projects/${{ env.PROJECT_NAME }}/upload + repository-url: >- + https://jazzband.co/projects/${{ + needs.build-and-test.outputs.project-name + }}/upload diff --git a/tox.ini b/tox.ini index 561ca7b2f..57201d420 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,16 @@ envlist = readme skip_missing_interpreters = True +[python-cli-options] +byte-warnings = -b +byte-errors = -bb +max-isolation = -E -s -I +# some-isolation = -I +# FIXME: Python 2 shim. Is this equivalent to the above? +some-isolation = -E -s +warnings-to-errors = -Werror + + [testenv] description = run the tests with pytest extras = @@ -191,3 +201,64 @@ commands_pre = commands = {envpython} -bb -I -Werror \ -c 'import sys, subprocess; subprocess.run([sys.executable, "-bb", "-I", "-Werror", "-m", "towncrier", "build", "--version", "DRAFT_VERSION", "--draft", *sys.argv[1:]], stderr=subprocess.DEVNULL)' {posargs} + + +[testenv:cleanup-dists] +description = + Wipe the dist{/} folder +deps = +commands_pre = +commands = + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -c \ + 'import os, shutil, sys; \ + dists_dir = "{toxinidir}{/}dist{/}"; \ + shutil.rmtree(dists_dir, ignore_errors=True); \ + sys.exit(os.path.exists(dists_dir))' +commands_post = +package = skip + + +[testenv:build-dists] +allowlist_externals = + env +description = + Build dists with {basepython} and put them into the dist{/} folder +depends = + cleanup-dists +deps = + build +commands_pre = +commands = + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -m build \ + {posargs:} +commands_post = +package = skip + + +[testenv:metadata-validation] +description = + Verify that dists under the `dist{/}` dir + have valid metadata +depends = + build-dists +deps = + twine +commands = + {envpython} \ + {[python-cli-options]byte-errors} \ + {[python-cli-options]max-isolation} \ + {[python-cli-options]warnings-to-errors} \ + -m twine \ + check \ + --strict \ + dist{/}* +commands_post = +package = skip From 0d7c0845e5e135f06ac896e45e98a4a6d8105726 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 28 Nov 2025 22:48:51 +0100 Subject: [PATCH 2/2] Add a change note for PR #2274 --- changelog.d/2274.contrib.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog.d/2274.contrib.md diff --git a/changelog.d/2274.contrib.md b/changelog.d/2274.contrib.md new file mode 100644 index 000000000..7a608461e --- /dev/null +++ b/changelog.d/2274.contrib.md @@ -0,0 +1,6 @@ +The CI/CD is now set up so that the distribution build job +is a part of the test pipeline. That pipeline is included in +the release workflow which sources the artifact in produces. +The tests must now pass for the release to be published to PyPI. + +-- by {user}`webknjaz`