Expose PEP 740 attestations functionality

PR #236

This patch adds PEP 740 attestation generation to the workflow: when the Trusted Publishing flow is used, this will generate a publish attestation for each distribution being uploaded. These generated attestations are then fed into `twine`, which newly supports them via `--attestations`.

Ref: https://github.com/pypi/warehouse/issues/15871
This commit is contained in:
William Woodruff 2024-08-31 20:50:29 -04:00 committed by GitHub
parent fb9fc6a4e6
commit 8a08d61689
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 274 additions and 8 deletions

View File

@ -28,6 +28,7 @@ COPY LICENSE.md .
COPY twine-upload.sh . COPY twine-upload.sh .
COPY print-hash.py . COPY print-hash.py .
COPY oidc-exchange.py . COPY oidc-exchange.py .
COPY attestations.py .
RUN chmod +x twine-upload.sh RUN chmod +x twine-upload.sh
ENTRYPOINT ["/app/twine-upload.sh"] ENTRYPOINT ["/app/twine-upload.sh"]

View File

@ -99,6 +99,31 @@ filter to the job:
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
``` ```
### Generating and uploading attestations
> [!IMPORTANT]
> Support for generating and uploading [digital attestations] is currently
> experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI.
> Support for this feature is not yet stable; the settings and behavior described
> below may change without prior notice.
> [!NOTE]
> Generating and uploading digital attestations currently requires
> authentication with a [trusted publisher].
You can generate signed [digital attestations] for all the distribution files and
upload them all together by enabling the `attestations` setting:
```yml
with:
attestations: true
```
This will use [Sigstore] to create attestation
objects for each distribution package, signing them with the identity provided
by the GitHub's OIDC token associated with the current workflow. This means
both the trusted publishing authentication and the attestations are tied to the
same identity.
## Non-goals ## Non-goals
@ -287,3 +312,7 @@ https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md
[configured on PyPI]: https://docs.pypi.org/trusted-publishers/adding-a-publisher/ [configured on PyPI]: https://docs.pypi.org/trusted-publishers/adding-a-publisher/
[how to specify username and password]: #specifying-a-different-username [how to specify username and password]: #specifying-a-different-username
[digital attestations]: https://peps.python.org/pep-0740/
[Sigstore]: https://www.sigstore.dev/
[trusted publisher]: #trusted-publishing

View File

@ -80,6 +80,13 @@ inputs:
Use `print-hash` instead. Use `print-hash` instead.
required: false required: false
default: 'false' default: 'false'
attestations:
description: >-
[EXPERIMENTAL]
Enable experimental support for PEP 740 attestations.
Only works with PyPI and TestPyPI via Trusted Publishing.
required: false
default: 'false'
branding: branding:
color: yellow color: yellow
icon: upload-cloud icon: upload-cloud
@ -95,3 +102,4 @@ runs:
- ${{ inputs.skip-existing }} - ${{ inputs.skip-existing }}
- ${{ inputs.verbose }} - ${{ inputs.verbose }}
- ${{ inputs.print-hash }} - ${{ inputs.print-hash }}
- ${{ inputs.attestations }}

114
attestations.py Normal file
View File

@ -0,0 +1,114 @@
import logging
import os
import sys
from pathlib import Path
from typing import NoReturn
from pypi_attestations import Attestation, Distribution
from sigstore.oidc import IdentityError, IdentityToken, detect_credential
from sigstore.sign import Signer, SigningContext
# Be very verbose.
sigstore_logger = logging.getLogger('sigstore')
sigstore_logger.setLevel(logging.DEBUG)
sigstore_logger.addHandler(logging.StreamHandler())
_GITHUB_STEP_SUMMARY = Path(os.getenv('GITHUB_STEP_SUMMARY'))
# The top-level error message that gets rendered.
# This message wraps one of the other templates/messages defined below.
_ERROR_SUMMARY_MESSAGE = """
Attestation generation failure:
{message}
You're seeing this because the action attempted to generated PEP 740
attestations for its inputs, but failed to do so.
"""
# Rendered if OIDC identity token retrieval fails for any reason.
_TOKEN_RETRIEVAL_FAILED_MESSAGE = """
OpenID Connect token retrieval failed: {identity_error}
This failure occurred after a successful Trusted Publishing Flow,
suggesting a transient error.
""" # noqa: S105; not a password
def die(msg: str) -> NoReturn:
with _GITHUB_STEP_SUMMARY.open('a', encoding='utf-8') as io:
print(_ERROR_SUMMARY_MESSAGE.format(message=msg), file=io)
# HACK: GitHub Actions' annotations don't work across multiple lines naively;
# translating `\n` into `%0A` (i.e., HTML percent-encoding) is known to work.
# See: https://github.com/actions/toolkit/issues/193
msg = msg.replace('\n', '%0A')
print(f'::error::Attestation generation failure: {msg}', file=sys.stderr)
sys.exit(1)
def debug(msg: str):
print(f'::debug::{msg}', file=sys.stderr)
def collect_dists(packages_dir: Path) -> list[Path]:
# Collect all sdists and wheels.
dist_paths = [sdist.resolve() for sdist in packages_dir.glob('*.tar.gz')]
dist_paths.extend(whl.resolve() for whl in packages_dir.glob('*.whl'))
# Make sure everything that looks like a dist actually is one.
# We do this up-front to prevent partial signing.
if (invalid_dists := [path for path in dist_paths if path.is_file()]):
invalid_dist_list = ', '.join(map(str, invalid_dists))
die(
'The following paths look like distributions but '
f'are not actually files: {invalid_dist_list}',
)
return dist_paths
def attest_dist(dist_path: Path, signer: Signer) -> None:
# We are the publishing step, so there should be no pre-existing publish
# attestation. The presence of one indicates user confusion.
attestation_path = Path(f'{dist_path}.publish.attestation')
if attestation_path.exists():
die(f'{dist_path} already has a publish attestation: {attestation_path}')
dist = Distribution.from_file(dist_path)
attestation = Attestation.sign(signer, dist)
attestation_path.write_text(attestation.model_dump_json(), encoding='utf-8')
debug(f'saved publish attestation: {dist_path=} {attestation_path=}')
def get_identity_token() -> IdentityToken:
# Will raise `sigstore.oidc.IdentityError` if it fails to get the token
# from the environment or if the token is malformed.
# NOTE: audience is always sigstore.
oidc_token = detect_credential()
return IdentityToken(oidc_token)
def main() -> None:
packages_dir = Path(sys.argv[1])
try:
identity = get_identity_token()
except IdentityError as identity_error:
# NOTE: We only perform attestations in trusted publishing flows, so we
# don't need to re-check for the "PR from fork" error mode, only
# generic token retrieval errors. We also render a simpler error,
# since permissions can't be to blame at this stage.
die(_TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error))
dist_paths = collect_dists(packages_dir)
with SigningContext.production().signer(identity, cache=True) as s:
debug(f'attesting to dists: {dist_paths}')
for dist_path in dist_paths:
attest_dist(dist_path, s)
if __name__ == '__main__':
main()

View File

@ -1,9 +1,14 @@
twine twine
# NOTE: Used to detect an ambient OIDC credential for OIDC publishing. # NOTE: Used to detect an ambient OIDC credential for OIDC publishing,
# NOTE: as well as PEP 740 attestations.
id ~= 1.0 id ~= 1.0
# NOTE: This is pulled in transitively through `twine`, but we also declare # NOTE: This is pulled in transitively through `twine`, but we also declare
# NOTE: it explicitly here because `oidc-exchange.py` uses it. # NOTE: it explicitly here because `oidc-exchange.py` uses it.
# Ref: https://github.com/di/id # Ref: https://github.com/di/id
requests requests
# NOTE: Used to generate attestations.
pypi-attestations ~= 0.0.11
sigstore ~= 3.2.0

View File

@ -6,16 +6,41 @@
# #
annotated-types==0.6.0 annotated-types==0.6.0
# via pydantic # via pydantic
betterproto==2.0.0b6
# via sigstore-protobuf-specs
certifi==2024.2.2 certifi==2024.2.2
# via requests # via requests
cffi==1.16.0
# via cryptography
charset-normalizer==3.3.2 charset-normalizer==3.3.2
# via requests # via requests
cryptography==42.0.7
# via
# pyopenssl
# pypi-attestations
# sigstore
dnspython==2.6.1
# via email-validator
docutils==0.21.2 docutils==0.21.2
# via readme-renderer # via readme-renderer
email-validator==2.1.1
# via pydantic
grpclib==0.4.7
# via betterproto
h2==4.1.0
# via grpclib
hpack==4.0.0
# via h2
hyperframe==6.0.1
# via h2
id==1.4.0 id==1.4.0
# via -r runtime.in # via
# -r runtime.in
# sigstore
idna==3.7 idna==3.7
# via requests # via
# email-validator
# requests
importlib-metadata==7.1.0 importlib-metadata==7.1.0
# via twine # via twine
jaraco-classes==3.4.0 jaraco-classes==3.4.0
@ -34,33 +59,77 @@ more-itertools==10.2.0
# via # via
# jaraco-classes # jaraco-classes
# jaraco-functools # jaraco-functools
multidict==6.0.5
# via grpclib
nh3==0.2.17 nh3==0.2.17
# via readme-renderer # via readme-renderer
packaging==24.1
# via pypi-attestations
pkginfo==1.10.0 pkginfo==1.10.0
# via twine # via twine
platformdirs==4.2.2
# via sigstore
pyasn1==0.6.0
# via sigstore
pycparser==2.22
# via cffi
pydantic==2.7.1 pydantic==2.7.1
# via id # via
# id
# pypi-attestations
# sigstore
# sigstore-rekor-types
pydantic-core==2.18.2 pydantic-core==2.18.2
# via pydantic # via pydantic
pygments==2.18.0 pygments==2.18.0
# via # via
# readme-renderer # readme-renderer
# rich # rich
pyjwt==2.8.0
# via sigstore
pyopenssl==24.1.0
# via sigstore
pypi-attestations==0.0.11
# via -r runtime.in
python-dateutil==2.9.0.post0
# via betterproto
readme-renderer==43.0 readme-renderer==43.0
# via twine # via twine
requests==2.32.0 requests==2.32.3
# via # via
# -r runtime.in # -r runtime.in
# id # id
# requests-toolbelt # requests-toolbelt
# sigstore
# tuf
# twine # twine
requests-toolbelt==1.0.0 requests-toolbelt==1.0.0
# via twine # via twine
rfc3986==2.0.0 rfc3986==2.0.0
# via twine # via twine
rfc8785==0.1.2
# via sigstore
rich==13.7.1 rich==13.7.1
# via twine # via
twine==5.1.0 # sigstore
# twine
securesystemslib==1.0.0
# via tuf
sigstore==3.2.0
# via
# -r runtime.in
# pypi-attestations
sigstore-protobuf-specs==0.3.2
# via
# pypi-attestations
# sigstore
sigstore-rekor-types==0.0.13
# via sigstore
six==1.16.0
# via python-dateutil
tuf==5.0.0
# via sigstore
twine==5.1.1
# via -r runtime.in # via -r runtime.in
typing-extensions==4.11.0 typing-extensions==4.11.0
# via # via

View File

@ -39,6 +39,7 @@ INPUT_PACKAGES_DIR="$(get-normalized-input 'packages-dir')"
INPUT_VERIFY_METADATA="$(get-normalized-input 'verify-metadata')" INPUT_VERIFY_METADATA="$(get-normalized-input 'verify-metadata')"
INPUT_SKIP_EXISTING="$(get-normalized-input 'skip-existing')" INPUT_SKIP_EXISTING="$(get-normalized-input 'skip-existing')"
INPUT_PRINT_HASH="$(get-normalized-input 'print-hash')" INPUT_PRINT_HASH="$(get-normalized-input 'print-hash')"
INPUT_ATTESTATIONS="$(get-normalized-input 'attestations')"
PASSWORD_DEPRECATION_NUDGE="::error title=Password-based uploads disabled::\ PASSWORD_DEPRECATION_NUDGE="::error title=Password-based uploads disabled::\
As of 2024, PyPI requires all users to enable Two-Factor \ As of 2024, PyPI requires all users to enable Two-Factor \
@ -53,7 +54,37 @@ environments like GitHub Actions without needing to use username/password \
combinations or API tokens to authenticate with PyPI. Read more: \ combinations or API tokens to authenticate with PyPI. Read more: \
https://docs.pypi.org/trusted-publishers" https://docs.pypi.org/trusted-publishers"
if [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] ; then ATTESTATIONS_WITHOUT_TP_WARNING="::warning title=attestations input ignored::\
The workflow was run with the 'attestations: true' input, but an explicit \
password was also set, disabling Trusted Publishing. As a result, the \
attestations input is ignored."
ATTESTATIONS_WRONG_INDEX_WARNING="::warning title=attestations input ignored::\
The workflow was run with 'attestations: true' input, but the specified \
repository URL does not support PEP 740 attestations. As a result, the \
attestations input is ignored."
[[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] \
&& TRUSTED_PUBLISHING=true || TRUSTED_PUBLISHING=false
if [[ "${INPUT_ATTESTATIONS}" != "false" ]] ; then
# Setting `attestations: true` without Trusted Publishing indicates
# user confusion, since attestations (currently) require it.
if ! "${TRUSTED_PUBLISHING}" ; then
echo "${ATTESTATIONS_WITHOUT_TP_WARNING}"
INPUT_ATTESTATIONS="false"
fi
# Setting `attestations: true` with an index other than PyPI or TestPyPI
# indicates user confusion, since attestations are not supported on other
# indices presently.
if [[ ! "${INPUT_REPOSITORY_URL}" =~ pypi\.org ]] ; then
echo "${ATTESTATIONS_WRONG_INDEX_WARNING}"
INPUT_ATTESTATIONS="false"
fi
fi
if "${TRUSTED_PUBLISHING}" ; then
# No password supplied by the user implies that we're in the OIDC flow; # No password supplied by the user implies that we're in the OIDC flow;
# retrieve the OIDC credential and exchange it for a PyPI API token. # retrieve the OIDC credential and exchange it for a PyPI API token.
echo "::debug::Authenticating to ${INPUT_REPOSITORY_URL} via Trusted Publishing" echo "::debug::Authenticating to ${INPUT_REPOSITORY_URL} via Trusted Publishing"
@ -130,6 +161,15 @@ if [[ ${INPUT_VERBOSE,,} != "false" ]] ; then
TWINE_EXTRA_ARGS="--verbose $TWINE_EXTRA_ARGS" TWINE_EXTRA_ARGS="--verbose $TWINE_EXTRA_ARGS"
fi fi
if [[ ${INPUT_ATTESTATIONS,,} != "false" ]] ; then
# NOTE: Intentionally placed after `twine check`, to prevent attestation
# NOTE: generation on distributions with invalid metadata.
echo "::notice::Generating and uploading digital attestations"
python /app/attestations.py "${INPUT_PACKAGES_DIR%%/}"
TWINE_EXTRA_ARGS="--attestations $TWINE_EXTRA_ARGS"
fi
if [[ ${INPUT_PRINT_HASH,,} != "false" || ${INPUT_VERBOSE,,} != "false" ]] ; then if [[ ${INPUT_PRINT_HASH,,} != "false" || ${INPUT_VERBOSE,,} != "false" ]] ; then
python /app/print-hash.py ${INPUT_PACKAGES_DIR%%/} python /app/print-hash.py ${INPUT_PACKAGES_DIR%%/}
fi fi