From b526ff8902c42cc0c89f77f551de12b1e850c61b Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 2 May 2024 12:58:33 -0400 Subject: [PATCH] requirements: Add initial support for uploading PEP 740 attestations Signed-off-by: William Woodruff --- Dockerfile | 1 + README.md | 21 ++++++++ action.yml | 8 +++ attestations.py | 102 +++++++++++++++++++++++++++++++++++++++ requirements/runtime.in | 7 ++- requirements/runtime.txt | 75 +++++++++++++++++++++++----- twine-upload.sh | 37 ++++++++++++++ 7 files changed, 239 insertions(+), 12 deletions(-) create mode 100644 attestations.py diff --git a/Dockerfile b/Dockerfile index 72e1d22..96c54f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ COPY LICENSE.md . COPY twine-upload.sh . COPY print-hash.py . COPY oidc-exchange.py . +COPY attestations.py . RUN chmod +x twine-upload.sh ENTRYPOINT ["/app/twine-upload.sh"] diff --git a/README.md b/README.md index 32a95f3..f036a34 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,25 @@ for example. See [Creating & using secrets]. While still secure, [trusted publishing] is now encouraged over API tokens as a best practice on supported platforms (like GitHub). +### Generating and uploading attestations (EXPERIMENTAL) + +> [!NOTE] +> Support for generating and uploading [PEP 740 attestations] is currently +> experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI. + +You can generate signed [PEP 740 attestations] for all the distribution files and +upload them all together by enabling the `attestations` setting: + +```yml + with: + attestations: true +``` + +This will use `sigstore` to create attestation objects for each distribution package, +signing them with the identity provided by the GitHub's OIDC token associated with the +current workflow. This means both the trusted publishing authentication and the +attestations are tied to the same identity. + ## License The Dockerfile and associated scripts and documentation in this project @@ -287,3 +306,5 @@ https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md [configured on PyPI]: https://docs.pypi.org/trusted-publishers/adding-a-publisher/ [how to specify username and password]: #specifying-a-different-username + +[PEP 740 attestations]: https://peps.python.org/pep-0740/ diff --git a/action.yml b/action.yml index 0c79949..40fed97 100644 --- a/action.yml +++ b/action.yml @@ -80,6 +80,13 @@ inputs: Use `print-hash` instead. required: false default: 'false' + attestations: + description: >- + [EXPERIMENTAL] + Enable experimental support for PEP 740 attestations. + Only works with PyPI and TestPyPI via Trusted Publishing. + required: false + default: 'false' branding: color: yellow icon: upload-cloud @@ -95,3 +102,4 @@ runs: - ${{ inputs.skip-existing }} - ${{ inputs.verbose }} - ${{ inputs.print-hash }} + - ${{ inputs.attestations }} diff --git a/attestations.py b/attestations.py new file mode 100644 index 0000000..faab9f9 --- /dev/null +++ b/attestations.py @@ -0,0 +1,102 @@ +import logging +import os +import sys +from pathlib import Path +from typing import NoReturn + +from pypi_attestation_models import AttestationPayload +from sigstore.oidc import IdentityError, IdentityToken, detect_credential +from sigstore.sign import Signer, SigningContext + +# Be very verbose. +sigstore_logger = logging.getLogger("sigstore") +sigstore_logger.setLevel(logging.DEBUG) +sigstore_logger.addHandler(logging.StreamHandler()) + +_GITHUB_STEP_SUMMARY = Path(os.getenv("GITHUB_STEP_SUMMARY")) + +# The top-level error message that gets rendered. +# This message wraps one of the other templates/messages defined below. +_ERROR_SUMMARY_MESSAGE = """ +Attestation generation failure: + +{message} + +You're seeing this because the action attempted to generated PEP 740 +attestations for its inputs, but failed to do so. +""" + +# Rendered if OIDC identity token retrieval fails for any reason. +_TOKEN_RETRIEVAL_FAILED_MESSAGE = """ +OpenID Connect token retrieval failed: {identity_error} + +This generally indicates a workflow configuration error, such as insufficient +permissions. Make sure that your workflow has `id-token: write` configured +at the job level, e.g.: + +```yaml +permissions: + id-token: write +``` + +Learn more at https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings. +""" + + +def die(msg: str) -> NoReturn: + with _GITHUB_STEP_SUMMARY.open("a", encoding="utf-8") as io: + print(_ERROR_SUMMARY_MESSAGE.format(message=msg), file=io) + + # HACK: GitHub Actions' annotations don't work across multiple lines naively; + # translating `\n` into `%0A` (i.e., HTML percent-encoding) is known to work. + # See: https://github.com/actions/toolkit/issues/193 + msg = msg.replace("\n", "%0A") + print(f"::error::Attestation generation failure: {msg}", file=sys.stderr) + sys.exit(1) + + +def debug(msg: str): + print(f"::debug::{msg}", file=sys.stderr) + + +# pylint: disable=redefined-outer-name +def attest_dist(dist: Path, signer: Signer) -> None: + # We are the publishing step, so there should be no pre-existing publish + # attestation. The presence of one indicates user confusion. + attestation_path = Path(f"{dist}.publish.attestation") + if attestation_path.is_file(): + die(f"{dist} already has a publish attestation: {attestation_path}") + + payload = AttestationPayload.from_dist(dist) + attestation = payload.sign(signer) + + attestation_path.write_text(attestation.model_dump_json(), encoding="utf-8") + debug(f"saved publish attestation: {dist=} {attestation_path=}") + + +packages_dir = Path(sys.argv[1]) + +try: + # NOTE: audience is always sigstore. + oidc_token = detect_credential() + identity = IdentityToken(oidc_token) +except IdentityError as identity_error: + # NOTE: We only perform attestations in trusted publishing flows, so we + # don't need to re-check for the "PR from fork" error mode, only + # generic token retrieval errors. + cause = _TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error) + die(cause) + +# Collect all sdists and wheels. +dists = [sdist.absolute() for sdist in packages_dir.glob("*.tar.gz")] +dists.extend(whl.absolute() for whl in packages_dir.glob("*.whl")) + +with SigningContext.production().signer(identity, cache=True) as signer: + for dist in dists: + # This should never really happen, but some versions of GitHub's + # download-artifact will create a subdirectory with the same name + # as the artifact being downloaded, e.g. `dist/foo.whl/foo.whl`. + if not dist.is_file(): + die(f"Path looks like a distribution but is not a file: {dist}") + + attest_dist(dist, signer) diff --git a/requirements/runtime.in b/requirements/runtime.in index 57be3b9..4148e6c 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,9 +1,14 @@ twine -# NOTE: Used to detect an ambient OIDC credential for OIDC publishing. +# NOTE: Used to detect an ambient OIDC credential for OIDC publishing, +# as well as PEP 740 attestations. id ~= 1.0 # NOTE: This is pulled in transitively through `twine`, but we also declare # NOTE: it explicitly here because `oidc-exchange.py` uses it. # Ref: https://github.com/di/id requests + +# NOTE: Used to generate attestations. +pypi-attestation-models == 0.0.2 +sigstore ~= 3.0.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 57fde0c..4a57cb0 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -6,6 +6,8 @@ # annotated-types==0.6.0 # via pydantic +betterproto==2.0.0b6 + # via sigstore-protobuf-specs certifi==2024.2.2 # via requests cffi==1.16.0 @@ -13,13 +15,32 @@ cffi==1.16.0 charset-normalizer==3.3.2 # via requests cryptography==42.0.7 - # via secretstorage + # via + # pyopenssl + # pypi-attestation-models + # sigstore +dnspython==2.6.1 + # via email-validator docutils==0.21.2 # via readme-renderer +email-validator==2.1.1 + # via pydantic +grpclib==0.4.7 + # via betterproto +h2==4.1.0 + # via grpclib +hpack==4.0.0 + # via h2 +hyperframe==6.0.1 + # via h2 id==1.4.0 - # via -r runtime.in + # via + # -r runtime.in + # sigstore idna==3.7 - # via requests + # via + # email-validator + # requests importlib-metadata==7.1.0 # via twine jaraco-classes==3.4.0 @@ -28,10 +49,6 @@ jaraco-context==5.3.0 # via keyring jaraco-functools==4.0.1 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage keyring==25.2.1 # via twine markdown-it-py==3.0.0 @@ -42,20 +59,38 @@ more-itertools==10.2.0 # via # jaraco-classes # jaraco-functools +multidict==6.0.5 + # via grpclib nh3==0.2.17 # via readme-renderer pkginfo==1.10.0 # via twine +platformdirs==4.2.2 + # via sigstore +pyasn1==0.6.0 + # via sigstore pycparser==2.22 # via cffi pydantic==2.7.1 - # via id + # via + # id + # pypi-attestation-models + # sigstore + # sigstore-rekor-types pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via # readme-renderer # rich +pyjwt==2.8.0 + # via sigstore +pyopenssl==24.1.0 + # via sigstore +pypi-attestation-models==0.0.2 + # via -r runtime.in +python-dateutil==2.9.0.post0 + # via betterproto readme-renderer==43.0 # via twine requests==2.31.0 @@ -63,15 +98,33 @@ requests==2.31.0 # -r runtime.in # id # requests-toolbelt + # sigstore + # tuf # twine requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine +rfc8785==0.1.2 + # via sigstore rich==13.7.1 - # via twine -secretstorage==3.3.3 - # via keyring + # via + # sigstore + # twine +securesystemslib==1.0.0 + # via tuf +sigstore==3.0.0 + # via + # -r runtime.in + # pypi-attestation-models +sigstore-protobuf-specs==0.3.2 + # via sigstore +sigstore-rekor-types==0.0.13 + # via sigstore +six==1.16.0 + # via python-dateutil +tuf==5.0.0 + # via sigstore twine==5.1.0 # via -r runtime.in typing-extensions==4.11.0 diff --git a/twine-upload.sh b/twine-upload.sh index 9c608ad..8493825 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -39,6 +39,7 @@ INPUT_PACKAGES_DIR="$(get-normalized-input 'packages-dir')" INPUT_VERIFY_METADATA="$(get-normalized-input 'verify-metadata')" INPUT_SKIP_EXISTING="$(get-normalized-input 'skip-existing')" INPUT_PRINT_HASH="$(get-normalized-input 'print-hash')" +INPUT_ATTESTATIONS="$(get-normalized-input 'attestations')" PASSWORD_DEPRECATION_NUDGE="::error title=Password-based uploads disabled::\ As of 2024, PyPI requires all users to enable Two-Factor \ @@ -53,6 +54,33 @@ environments like GitHub Actions without needing to use username/password \ combinations or API tokens to authenticate with PyPI. Read more: \ https://docs.pypi.org/trusted-publishers" +ATTESTATIONS_WITHOUT_TP_WARNING="::warning title=attestations setting ignored::\ +The workflow was run with 'attestations: true', but an explicit password was \ +also supplied, disabling Trusted Publishing. As a result, the attestations \ +setting is ignored." + +ATTESTATIONS_WRONG_INDEX_WARNING="::warning title=attestations setting ignored::\ +The workflow was run with 'attestations: true', but the specified repository URL \ +does not support PEP 740 attestations. As a result, the attestations setting \ +is ignored." + +if [[ "${INPUT_ATTESTATIONS}" != "false" ]] ; then + # Setting `attestations: true` and explicitly passing a password indicates + # user confusion, since attestations (currently) require Trusted Publishing. + if [[ -n "${INPUT_PASSWORD}" ]] ; then + echo "${ATTESTATIONS_WITHOUT_TP_WARNING}" + INPUT_ATTESTATIONS="false" + fi + + # Setting `attestations: true` with an index other than PyPI or TestPyPI + # indicates user confusion, since attestations are not supported on other + # indices presently. + if [[ ! "${INPUT_REPOSITORY_URL}" =~ pypi\.org ]] ; then + echo "${ATTESTATIONS_WRONG_INDEX_WARNING}" + INPUT_ATTESTATIONS="false" + fi +fi + if [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] ; then # No password supplied by the user implies that we're in the OIDC flow; # retrieve the OIDC credential and exchange it for a PyPI API token. @@ -130,6 +158,15 @@ if [[ ${INPUT_VERBOSE,,} != "false" ]] ; then TWINE_EXTRA_ARGS="--verbose $TWINE_EXTRA_ARGS" fi +if [[ ${INPUT_ATTESTATIONS,,} != "false" ]] ; then + # NOTE: Intentionally placed after `twine check`, to prevent attestation + # generation on distributions with invalid metadata. + echo "::debug::Generating and uploading PEP 740 attestations" + python /app/attestations.py "${INPUT_PACKAGES_DIR%%/}" + + TWINE_EXTRA_ARGS="--attestations $TWINE_EXTRA_ARGS" +fi + if [[ ${INPUT_PRINT_HASH,,} != "false" || ${INPUT_VERBOSE,,} != "false" ]] ; then python /app/print-hash.py ${INPUT_PACKAGES_DIR%%/} fi