From 8a08d616893759ef8e1aa1f2785787c0b97e20d6 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Sat, 31 Aug 2024 20:50:29 -0400 Subject: [PATCH] Expose PEP 740 attestations functionality PR #236 This patch adds PEP 740 attestation generation to the workflow: when the Trusted Publishing flow is used, this will generate a publish attestation for each distribution being uploaded. These generated attestations are then fed into `twine`, which newly supports them via `--attestations`. Ref: https://github.com/pypi/warehouse/issues/15871 --- Dockerfile | 1 + README.md | 29 ++++++++++ action.yml | 8 +++ attestations.py | 114 +++++++++++++++++++++++++++++++++++++++ requirements/runtime.in | 7 ++- requirements/runtime.txt | 81 +++++++++++++++++++++++++--- twine-upload.sh | 42 ++++++++++++++- 7 files changed, 274 insertions(+), 8 deletions(-) create mode 100644 attestations.py diff --git a/Dockerfile b/Dockerfile index bff054f..a308f4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,7 @@ COPY LICENSE.md . COPY twine-upload.sh . COPY print-hash.py . COPY oidc-exchange.py . +COPY attestations.py . RUN chmod +x twine-upload.sh ENTRYPOINT ["/app/twine-upload.sh"] diff --git a/README.md b/README.md index 32a95f3..c0998c5 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,31 @@ filter to the job: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') ``` +### Generating and uploading attestations + +> [!IMPORTANT] +> Support for generating and uploading [digital attestations] is currently +> experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI. +> Support for this feature is not yet stable; the settings and behavior described +> below may change without prior notice. + +> [!NOTE] +> Generating and uploading digital attestations currently requires +> authentication with a [trusted publisher]. + +You can generate signed [digital attestations] for all the distribution files and +upload them all together by enabling the `attestations` setting: + +```yml + with: + attestations: true +``` + +This will use [Sigstore] to create attestation +objects for each distribution package, signing them with the identity provided +by the GitHub's OIDC token associated with the current workflow. This means +both the trusted publishing authentication and the attestations are tied to the +same identity. ## Non-goals @@ -287,3 +312,7 @@ https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md [configured on PyPI]: https://docs.pypi.org/trusted-publishers/adding-a-publisher/ [how to specify username and password]: #specifying-a-different-username + +[digital attestations]: https://peps.python.org/pep-0740/ +[Sigstore]: https://www.sigstore.dev/ +[trusted publisher]: #trusted-publishing diff --git a/action.yml b/action.yml index 0c79949..40fed97 100644 --- a/action.yml +++ b/action.yml @@ -80,6 +80,13 @@ inputs: Use `print-hash` instead. required: false default: 'false' + attestations: + description: >- + [EXPERIMENTAL] + Enable experimental support for PEP 740 attestations. + Only works with PyPI and TestPyPI via Trusted Publishing. + required: false + default: 'false' branding: color: yellow icon: upload-cloud @@ -95,3 +102,4 @@ runs: - ${{ inputs.skip-existing }} - ${{ inputs.verbose }} - ${{ inputs.print-hash }} + - ${{ inputs.attestations }} diff --git a/attestations.py b/attestations.py new file mode 100644 index 0000000..6863c83 --- /dev/null +++ b/attestations.py @@ -0,0 +1,114 @@ +import logging +import os +import sys +from pathlib import Path +from typing import NoReturn + +from pypi_attestations import Attestation, Distribution +from sigstore.oidc import IdentityError, IdentityToken, detect_credential +from sigstore.sign import Signer, SigningContext + +# Be very verbose. +sigstore_logger = logging.getLogger('sigstore') +sigstore_logger.setLevel(logging.DEBUG) +sigstore_logger.addHandler(logging.StreamHandler()) + +_GITHUB_STEP_SUMMARY = Path(os.getenv('GITHUB_STEP_SUMMARY')) + +# The top-level error message that gets rendered. +# This message wraps one of the other templates/messages defined below. +_ERROR_SUMMARY_MESSAGE = """ +Attestation generation failure: + +{message} + +You're seeing this because the action attempted to generated PEP 740 +attestations for its inputs, but failed to do so. +""" + +# Rendered if OIDC identity token retrieval fails for any reason. +_TOKEN_RETRIEVAL_FAILED_MESSAGE = """ +OpenID Connect token retrieval failed: {identity_error} + +This failure occurred after a successful Trusted Publishing Flow, +suggesting a transient error. +""" # noqa: S105; not a password + + +def die(msg: str) -> NoReturn: + with _GITHUB_STEP_SUMMARY.open('a', encoding='utf-8') as io: + print(_ERROR_SUMMARY_MESSAGE.format(message=msg), file=io) + + # HACK: GitHub Actions' annotations don't work across multiple lines naively; + # translating `\n` into `%0A` (i.e., HTML percent-encoding) is known to work. + # See: https://github.com/actions/toolkit/issues/193 + msg = msg.replace('\n', '%0A') + print(f'::error::Attestation generation failure: {msg}', file=sys.stderr) + sys.exit(1) + + +def debug(msg: str): + print(f'::debug::{msg}', file=sys.stderr) + + +def collect_dists(packages_dir: Path) -> list[Path]: + # Collect all sdists and wheels. + dist_paths = [sdist.resolve() for sdist in packages_dir.glob('*.tar.gz')] + dist_paths.extend(whl.resolve() for whl in packages_dir.glob('*.whl')) + + # Make sure everything that looks like a dist actually is one. + # We do this up-front to prevent partial signing. + if (invalid_dists := [path for path in dist_paths if path.is_file()]): + invalid_dist_list = ', '.join(map(str, invalid_dists)) + die( + 'The following paths look like distributions but ' + f'are not actually files: {invalid_dist_list}', + ) + + return dist_paths + + +def attest_dist(dist_path: Path, signer: Signer) -> None: + # We are the publishing step, so there should be no pre-existing publish + # attestation. The presence of one indicates user confusion. + attestation_path = Path(f'{dist_path}.publish.attestation') + if attestation_path.exists(): + die(f'{dist_path} already has a publish attestation: {attestation_path}') + + dist = Distribution.from_file(dist_path) + attestation = Attestation.sign(signer, dist) + + attestation_path.write_text(attestation.model_dump_json(), encoding='utf-8') + debug(f'saved publish attestation: {dist_path=} {attestation_path=}') + + +def get_identity_token() -> IdentityToken: + # Will raise `sigstore.oidc.IdentityError` if it fails to get the token + # from the environment or if the token is malformed. + # NOTE: audience is always sigstore. + oidc_token = detect_credential() + return IdentityToken(oidc_token) + + +def main() -> None: + packages_dir = Path(sys.argv[1]) + + try: + identity = get_identity_token() + except IdentityError as identity_error: + # NOTE: We only perform attestations in trusted publishing flows, so we + # don't need to re-check for the "PR from fork" error mode, only + # generic token retrieval errors. We also render a simpler error, + # since permissions can't be to blame at this stage. + die(_TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error)) + + dist_paths = collect_dists(packages_dir) + + with SigningContext.production().signer(identity, cache=True) as s: + debug(f'attesting to dists: {dist_paths}') + for dist_path in dist_paths: + attest_dist(dist_path, s) + + +if __name__ == '__main__': + main() diff --git a/requirements/runtime.in b/requirements/runtime.in index 57be3b9..50f52b6 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,9 +1,14 @@ twine -# NOTE: Used to detect an ambient OIDC credential for OIDC publishing. +# NOTE: Used to detect an ambient OIDC credential for OIDC publishing, +# NOTE: as well as PEP 740 attestations. id ~= 1.0 # NOTE: This is pulled in transitively through `twine`, but we also declare # NOTE: it explicitly here because `oidc-exchange.py` uses it. # Ref: https://github.com/di/id requests + +# NOTE: Used to generate attestations. +pypi-attestations ~= 0.0.11 +sigstore ~= 3.2.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 06e1b7d..d50cd5b 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -6,16 +6,41 @@ # annotated-types==0.6.0 # via pydantic +betterproto==2.0.0b6 + # via sigstore-protobuf-specs certifi==2024.2.2 # via requests +cffi==1.16.0 + # via cryptography charset-normalizer==3.3.2 # via requests +cryptography==42.0.7 + # via + # pyopenssl + # pypi-attestations + # sigstore +dnspython==2.6.1 + # via email-validator docutils==0.21.2 # via readme-renderer +email-validator==2.1.1 + # via pydantic +grpclib==0.4.7 + # via betterproto +h2==4.1.0 + # via grpclib +hpack==4.0.0 + # via h2 +hyperframe==6.0.1 + # via h2 id==1.4.0 - # via -r runtime.in + # via + # -r runtime.in + # sigstore idna==3.7 - # via requests + # via + # email-validator + # requests importlib-metadata==7.1.0 # via twine jaraco-classes==3.4.0 @@ -34,33 +59,77 @@ more-itertools==10.2.0 # via # jaraco-classes # jaraco-functools +multidict==6.0.5 + # via grpclib nh3==0.2.17 # via readme-renderer +packaging==24.1 + # via pypi-attestations pkginfo==1.10.0 # via twine +platformdirs==4.2.2 + # via sigstore +pyasn1==0.6.0 + # via sigstore +pycparser==2.22 + # via cffi pydantic==2.7.1 - # via id + # via + # id + # pypi-attestations + # sigstore + # sigstore-rekor-types pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via # readme-renderer # rich +pyjwt==2.8.0 + # via sigstore +pyopenssl==24.1.0 + # via sigstore +pypi-attestations==0.0.11 + # via -r runtime.in +python-dateutil==2.9.0.post0 + # via betterproto readme-renderer==43.0 # via twine -requests==2.32.0 +requests==2.32.3 # via # -r runtime.in # id # requests-toolbelt + # sigstore + # tuf # twine requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine +rfc8785==0.1.2 + # via sigstore rich==13.7.1 - # via twine -twine==5.1.0 + # via + # sigstore + # twine +securesystemslib==1.0.0 + # via tuf +sigstore==3.2.0 + # via + # -r runtime.in + # pypi-attestations +sigstore-protobuf-specs==0.3.2 + # via + # pypi-attestations + # sigstore +sigstore-rekor-types==0.0.13 + # via sigstore +six==1.16.0 + # via python-dateutil +tuf==5.0.0 + # via sigstore +twine==5.1.1 # via -r runtime.in typing-extensions==4.11.0 # via diff --git a/twine-upload.sh b/twine-upload.sh index 9c608ad..12b57b2 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -39,6 +39,7 @@ INPUT_PACKAGES_DIR="$(get-normalized-input 'packages-dir')" INPUT_VERIFY_METADATA="$(get-normalized-input 'verify-metadata')" INPUT_SKIP_EXISTING="$(get-normalized-input 'skip-existing')" INPUT_PRINT_HASH="$(get-normalized-input 'print-hash')" +INPUT_ATTESTATIONS="$(get-normalized-input 'attestations')" PASSWORD_DEPRECATION_NUDGE="::error title=Password-based uploads disabled::\ As of 2024, PyPI requires all users to enable Two-Factor \ @@ -53,7 +54,37 @@ environments like GitHub Actions without needing to use username/password \ combinations or API tokens to authenticate with PyPI. Read more: \ https://docs.pypi.org/trusted-publishers" -if [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] ; then +ATTESTATIONS_WITHOUT_TP_WARNING="::warning title=attestations input ignored::\ +The workflow was run with the 'attestations: true' input, but an explicit \ +password was also set, disabling Trusted Publishing. As a result, the \ +attestations input is ignored." + +ATTESTATIONS_WRONG_INDEX_WARNING="::warning title=attestations input ignored::\ +The workflow was run with 'attestations: true' input, but the specified \ +repository URL does not support PEP 740 attestations. As a result, the \ +attestations input is ignored." + +[[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] \ + && TRUSTED_PUBLISHING=true || TRUSTED_PUBLISHING=false + +if [[ "${INPUT_ATTESTATIONS}" != "false" ]] ; then + # Setting `attestations: true` without Trusted Publishing indicates + # user confusion, since attestations (currently) require it. + if ! "${TRUSTED_PUBLISHING}" ; then + echo "${ATTESTATIONS_WITHOUT_TP_WARNING}" + INPUT_ATTESTATIONS="false" + fi + + # Setting `attestations: true` with an index other than PyPI or TestPyPI + # indicates user confusion, since attestations are not supported on other + # indices presently. + if [[ ! "${INPUT_REPOSITORY_URL}" =~ pypi\.org ]] ; then + echo "${ATTESTATIONS_WRONG_INDEX_WARNING}" + INPUT_ATTESTATIONS="false" + fi +fi + +if "${TRUSTED_PUBLISHING}" ; then # No password supplied by the user implies that we're in the OIDC flow; # retrieve the OIDC credential and exchange it for a PyPI API token. echo "::debug::Authenticating to ${INPUT_REPOSITORY_URL} via Trusted Publishing" @@ -130,6 +161,15 @@ if [[ ${INPUT_VERBOSE,,} != "false" ]] ; then TWINE_EXTRA_ARGS="--verbose $TWINE_EXTRA_ARGS" fi +if [[ ${INPUT_ATTESTATIONS,,} != "false" ]] ; then + # NOTE: Intentionally placed after `twine check`, to prevent attestation + # NOTE: generation on distributions with invalid metadata. + echo "::notice::Generating and uploading digital attestations" + python /app/attestations.py "${INPUT_PACKAGES_DIR%%/}" + + TWINE_EXTRA_ARGS="--attestations $TWINE_EXTRA_ARGS" +fi + if [[ ${INPUT_PRINT_HASH,,} != "false" || ${INPUT_VERBOSE,,} != "false" ]] ; then python /app/print-hash.py ${INPUT_PACKAGES_DIR%%/} fi