From 363a467b3df34900fffeb42b3fe6b8c33012217d Mon Sep 17 00:00:00 2001 From: Blake Gentry Date: Fri, 2 Jan 2026 15:40:06 -0600 Subject: [PATCH 1/5] fix(ci) harden offline sigstore bundle verification The offline bundle verification step would fail when oras discover returned a referrers index with manifests=null. To resolve this and further harden the publish flow, I implemented a separate script which: - Extracts referrers prefetch + offline verification into scripts/ci - Force-fetches subject + referrers via curl (custom header) before oras - Retries discovery briefly and handles empty referrer sets safely --- .github/workflows/docker-riverproui.yaml | 77 +++--------- scripts/ci/prefetch-referrers-content.sh | 71 +++++++++++ scripts/ci/verify-sigstore-bundles-offline.sh | 119 ++++++++++++++++++ 3 files changed, 207 insertions(+), 60 deletions(-) create mode 100644 scripts/ci/prefetch-referrers-content.sh create mode 100644 scripts/ci/verify-sigstore-bundles-offline.sh diff --git a/.github/workflows/docker-riverproui.yaml b/.github/workflows/docker-riverproui.yaml index 778cb34..20f9f6b 100644 --- a/.github/workflows/docker-riverproui.yaml +++ b/.github/workflows/docker-riverproui.yaml @@ -506,6 +506,12 @@ jobs: # If neither MANIFEST_DIGEST (from merge job) nor ecr_manifest_digest (manual input) # is available, we skip index referrer prefetch rather than fail. - name: Prefetch all referrers content + env: + AUTH_USER: river + AUTH_PASSWORD: ${{ secrets.RIVERPRO_GO_MOD_CREDENTIAL }} + FORCE_FETCH_SECRET: ${{ secrets.FORCE_FETCH_SECRET }} + REGISTRY_MANIFEST_URL: https://riverqueue.com/v2/riverproui/manifests + REGISTRY_REFERRERS_URL: https://riverqueue.com/v2/riverproui/referrers run: | # Include both the ECR-derived index digest (attestations live here) # and the live registry's index digest (may differ but could gain referrers). @@ -518,24 +524,7 @@ jobs: echo "Note: no index subject digest provided; skipping index referrers prefetch." fi subjects+=("${{ steps.resolve-digest.outputs.digest }}" "${{ steps.platform-digests.outputs.amd64_digest }}" "${{ steps.platform-digests.outputs.arm64_digest }}") - for subject in "${subjects[@]}"; do - oras discover --format json "$IMAGE_NAME@$subject" | tee /tmp/referrers.json - digests=$(jq -r '.manifests[]?.digest // empty' /tmp/referrers.json) - for d in $digests; do - tmpfile=$(mktemp) - # Fetch the full referrer manifest to a file so we can parse layers + config - oras manifest fetch --output "$tmpfile" "$IMAGE_NAME@$d" - # Prefetch config blob (often empty but warms cache) - cfg=$(jq -r '.config.digest // empty' "$tmpfile") - if [ -n "$cfg" ]; then - oras blob fetch --output /dev/null "$IMAGE_NAME@$cfg" - fi - # Prefetch any layer blobs (DSSE envelope, bundle, etc.) - jq -r '.layers[]?.digest // empty' "$tmpfile" | while read -r ld; do - [ -n "$ld" ] && oras blob fetch --output /dev/null "$IMAGE_NAME@$ld" - done - done - done + bash scripts/ci/prefetch-referrers-content.sh "${subjects[@]}" # Verify index (if digest is available) and per-arch attestations against live registry. # Index-level attestation is bound to the ECR index digest; provide it via MANIFEST_DIGEST @@ -576,55 +565,23 @@ jobs: # Offline verification of Sigstore bundle: only attempt for index if digest provided. - name: Verify Sigstore bundle against subject bytes (offline crypto) + env: + AUTH_USER: river + AUTH_PASSWORD: ${{ secrets.RIVERPRO_GO_MOD_CREDENTIAL }} + FORCE_FETCH_SECRET: ${{ secrets.FORCE_FETCH_SECRET }} + REGISTRY_MANIFEST_URL: https://riverqueue.com/v2/riverproui/manifests + REGISTRY_REFERRERS_URL: https://riverqueue.com/v2/riverproui/referrers run: | FALLBACK="${{ inputs.ecr_manifest_digest }}" INDEX_SUBJECT="${MANIFEST_DIGEST:-$FALLBACK}" - verify_bundle () { - subject="$1" - # Exact subject bytes - oras manifest fetch --output /tmp/subject.json "$IMAGE_NAME@$subject" - - # Find *all* bundle candidates for this subject (both media-type spellings) - refjson=$(mktemp) - oras discover --format json "$IMAGE_NAME@$subject" > "$refjson" - mapfile -t bundle_mfs < <(jq -r '.manifests[] - | select(.artifactType=="application/vnd.dev.sigstore.bundle.v0.3+json" - or .artifactType=="application/vnd.dev.sigstore.bundle+json;version=0.3") - | .digest' "$refjson") - - ok=0 - for bmf in "${bundle_mfs[@]}"; do - [ -z "$bmf" ] && continue - mf=$(mktemp) - oras manifest fetch --output "$mf" "$IMAGE_NAME@$bmf" - bundle_blob=$(jq -r '.layers[0].digest' "$mf") - oras blob fetch --output /tmp/bundle.json "$IMAGE_NAME@$bundle_blob" - - if cosign verify-blob-attestation \ - --bundle /tmp/bundle.json \ - --new-bundle-format \ - --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ - --certificate-identity-regexp '^https://github.com/riverqueue/riverui/.*' \ - /tmp/subject.json > /tmp/verify.out 2>&1; then - echo "Verified OK with bundle $bmf" - ok=1 - break - fi - done - - if [ "$ok" -ne 1 ]; then - echo "No matching bundle verified for subject $subject" - cat /tmp/verify.out || true - exit 1 - fi - } if [ -n "$INDEX_SUBJECT" ]; then - verify_bundle "$INDEX_SUBJECT" + bash scripts/ci/verify-sigstore-bundles-offline.sh "$INDEX_SUBJECT" else echo "Skipping offline index bundle verification: no MANIFEST_DIGEST or ecr_manifest_digest provided." fi - verify_bundle "${{ steps.platform-digests.outputs.amd64_digest }}" - verify_bundle "${{ steps.platform-digests.outputs.arm64_digest }}" + bash scripts/ci/verify-sigstore-bundles-offline.sh \ + "${{ steps.platform-digests.outputs.amd64_digest }}" \ + "${{ steps.platform-digests.outputs.arm64_digest }}" - name: Upload debug artifacts if: failure() || cancelled() diff --git a/scripts/ci/prefetch-referrers-content.sh b/scripts/ci/prefetch-referrers-content.sh new file mode 100644 index 0000000..219d69c --- /dev/null +++ b/scripts/ci/prefetch-referrers-content.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Prefetch referrers for one or more OCI subjects (digests), and warm their blobs. +# +# Required env: +# - IMAGE_NAME: e.g. "riverqueue.com/riverproui" +# - AUTH_USER: basic auth username +# - AUTH_PASSWORD: basic auth password +# - FORCE_FETCH_SECRET: value for X-Force-Fetch-From-Upstream (best-effort) +# - REGISTRY_MANIFEST_URL: e.g. "https://riverqueue.com/v2/riverproui/manifests" +# - REGISTRY_REFERRERS_URL: e.g. "https://riverqueue.com/v2/riverproui/referrers" +# +# Usage: +# bash scripts/ci/prefetch-referrers-content.sh sha256:... sha256:... + +: "${IMAGE_NAME:?IMAGE_NAME is required}" +: "${AUTH_USER:?AUTH_USER is required}" +: "${AUTH_PASSWORD:?AUTH_PASSWORD is required}" +: "${FORCE_FETCH_SECRET:?FORCE_FETCH_SECRET is required}" +: "${REGISTRY_MANIFEST_URL:?REGISTRY_MANIFEST_URL is required}" +: "${REGISTRY_REFERRERS_URL:?REGISTRY_REFERRERS_URL is required}" + +ACCEPT_MANIFESTS="application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json" + +force_fetch_manifest_and_referrers() { + subject="$1" + + # Best-effort: warm the pull-through cache for the subject bytes and its referrers index. + curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \ + -H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \ + -H "Accept: ${ACCEPT_MANIFESTS}" \ + "${REGISTRY_MANIFEST_URL}/${subject}" -o /dev/null || true + + curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \ + -H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \ + -H "Accept: application/vnd.oci.image.index.v1+json" \ + "${REGISTRY_REFERRERS_URL}/${subject}" -o /dev/null || true +} + +if [ "$#" -lt 1 ]; then + echo "usage: $0 [ ...]" >&2 + exit 2 +fi + +for subject in "$@"; do + [ -z "$subject" ] && continue + + force_fetch_manifest_and_referrers "$subject" + + # Discover and warm each referrer manifest and its blobs. + oras discover --format json "${IMAGE_NAME}@${subject}" | tee /tmp/referrers.json + digests="$(jq -r '.manifests[]?.digest // empty' /tmp/referrers.json)" + for d in $digests; do + [ -z "$d" ] && continue + + tmpfile="$(mktemp)" + oras manifest fetch --output "$tmpfile" "${IMAGE_NAME}@${d}" + + cfg="$(jq -r '.config.digest // empty' "$tmpfile")" + if [ -n "$cfg" ]; then + oras blob fetch --output /dev/null "${IMAGE_NAME}@${cfg}" + fi + + jq -r '.layers[]?.digest // empty' "$tmpfile" | while read -r ld; do + [ -n "$ld" ] && oras blob fetch --output /dev/null "${IMAGE_NAME}@${ld}" + done + done +done + + diff --git a/scripts/ci/verify-sigstore-bundles-offline.sh b/scripts/ci/verify-sigstore-bundles-offline.sh new file mode 100644 index 0000000..da88174 --- /dev/null +++ b/scripts/ci/verify-sigstore-bundles-offline.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Offline verification of Sigstore bundles against exact subject bytes. +# +# Required env: +# - IMAGE_NAME: e.g. "riverqueue.com/riverproui" +# - AUTH_USER: basic auth username +# - AUTH_PASSWORD: basic auth password +# - FORCE_FETCH_SECRET: value for X-Force-Fetch-From-Upstream (best-effort) +# - REGISTRY_MANIFEST_URL: e.g. "https://riverqueue.com/v2/riverproui/manifests" +# - REGISTRY_REFERRERS_URL: e.g. "https://riverqueue.com/v2/riverproui/referrers" +# +# Usage: +# bash scripts/ci/verify-sigstore-bundles-offline.sh sha256:... sha256:... sha256:... +# +# Notes: +# - We "force fetch" the subject + referrers index first to avoid ORAS limitations around custom headers. +# - We retry discovery briefly to avoid eventual-consistency/race flakes in pull-through caches. + +: "${IMAGE_NAME:?IMAGE_NAME is required}" +: "${AUTH_USER:?AUTH_USER is required}" +: "${AUTH_PASSWORD:?AUTH_PASSWORD is required}" +: "${FORCE_FETCH_SECRET:?FORCE_FETCH_SECRET is required}" +: "${REGISTRY_MANIFEST_URL:?REGISTRY_MANIFEST_URL is required}" +: "${REGISTRY_REFERRERS_URL:?REGISTRY_REFERRERS_URL is required}" + +ACCEPT_MANIFESTS="application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json" + +echo "oras: $(oras version 2>/dev/null || echo unknown)" +echo "cosign: $(cosign version 2>/dev/null || echo unknown)" +echo "jq: $(jq --version 2>/dev/null || echo unknown)" + +force_fetch_manifest_and_referrers() { + subject="$1" + + curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \ + -H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \ + -H "Accept: ${ACCEPT_MANIFESTS}" \ + "${REGISTRY_MANIFEST_URL}/${subject}" -o /dev/null || true + + curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \ + -H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \ + -H "Accept: application/vnd.oci.image.index.v1+json" \ + "${REGISTRY_REFERRERS_URL}/${subject}" -o /dev/null || true +} + +verify_bundle_for_subject() { + subject="$1" + : > /tmp/verify.out + + # Exact subject bytes (do not go through tag indirections). + oras manifest fetch --output /tmp/subject.json "${IMAGE_NAME}@${subject}" + + bundle_mfs=() + refjson="" + for attempt in 1 2 3 4 5; do + refjson="$(mktemp)" + force_fetch_manifest_and_referrers "$subject" + oras discover --format json "${IMAGE_NAME}@${subject}" > "$refjson" + + mapfile -t bundle_mfs < <(jq -r '(.manifests // [])[] + | select(.artifactType=="application/vnd.dev.sigstore.bundle.v0.3+json" + or .artifactType=="application/vnd.dev.sigstore.bundle+json;version=0.3") + | .digest' "$refjson") + + if [ ${#bundle_mfs[@]} -gt 0 ]; then + break + fi + + if [ "$attempt" -lt 5 ]; then + echo "No bundle referrers discovered yet for ${subject} (attempt ${attempt}/5); retrying soon..." + sleep $((attempt * 2)) + fi + done + + ok=0 + for bmf in "${bundle_mfs[@]}"; do + [ -z "$bmf" ] && continue + + mf="$(mktemp)" + oras manifest fetch --output "$mf" "${IMAGE_NAME}@${bmf}" + bundle_blob="$(jq -r '.layers[0].digest' "$mf")" + oras blob fetch --output /tmp/bundle.json "${IMAGE_NAME}@${bundle_blob}" + + if cosign verify-blob-attestation \ + --bundle /tmp/bundle.json \ + --new-bundle-format \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + --certificate-identity-regexp '^https://github.com/riverqueue/riverui/.*' \ + /tmp/subject.json > /tmp/verify.out 2>&1; then + echo "Verified OK with bundle ${bmf} for subject ${subject}" + ok=1 + break + fi + done + + if [ "$ok" -ne 1 ]; then + echo "No matching bundle verified for subject ${subject}" + if [ ${#bundle_mfs[@]} -eq 0 ]; then + echo "Note: no bundle referrers were discovered for this subject. Referrers index JSON:" + cat "$refjson" || true + fi + cat /tmp/verify.out || true + exit 1 + fi +} + +if [ "$#" -lt 1 ]; then + echo "usage: $0 [ ...]" >&2 + exit 2 +fi + +for subject in "$@"; do + [ -z "$subject" ] && continue + verify_bundle_for_subject "$subject" +done + + From b0d312283527d197e86bcefada6a2032e2661933 Mon Sep 17 00:00:00 2001 From: Blake Gentry Date: Fri, 2 Jan 2026 15:42:40 -0600 Subject: [PATCH 2/5] move prefetch script to scripts/ci --- .github/workflows/docker-riverproui.yaml | 2 +- scripts/{ => ci}/prefetch-buildx-attestation-manifests.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{ => ci}/prefetch-buildx-attestation-manifests.sh (100%) diff --git a/.github/workflows/docker-riverproui.yaml b/.github/workflows/docker-riverproui.yaml index 20f9f6b..699a0f7 100644 --- a/.github/workflows/docker-riverproui.yaml +++ b/.github/workflows/docker-riverproui.yaml @@ -475,7 +475,7 @@ jobs: FORCE_FETCH_SECRET: ${{ secrets.FORCE_FETCH_SECRET }} REGISTRY_MANIFEST_URL: https://riverqueue.com/v2/riverproui/manifests run: | - bash scripts/prefetch-buildx-attestation-manifests.sh /tmp/index-manifest-docker.json + bash scripts/ci/prefetch-buildx-attestation-manifests.sh /tmp/index-manifest-docker.json - name: Fetch amd64 manifest with crane (Docker media type) run: crane manifest "$IMAGE_NAME@${{ steps.platform-digests.outputs.amd64_digest }}" > /tmp/amd64-manifest-docker.json diff --git a/scripts/prefetch-buildx-attestation-manifests.sh b/scripts/ci/prefetch-buildx-attestation-manifests.sh similarity index 100% rename from scripts/prefetch-buildx-attestation-manifests.sh rename to scripts/ci/prefetch-buildx-attestation-manifests.sh From 18dbe0c7f4648b3c407e737868a9db392a8c5e8d Mon Sep 17 00:00:00 2001 From: Blake Gentry Date: Sat, 3 Jan 2026 14:01:05 -0600 Subject: [PATCH 3/5] development.md updates --- docs/development.md | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/development.md b/docs/development.md index 1a7495d..6ef2d29 100644 --- a/docs/development.md +++ b/docs/development.md @@ -89,13 +89,47 @@ $ npm run build 2. Prepare a PR with the changes, updating `CHANGELOG.md` with any necessary additions at the same time. Have it reviewed and merged. -3. Upon merge, pull down the changes, tag each module with the new version, and push the new tags: +3. Upon merge, pull down the changes, tag the main riverui module with the new version, and push the new tag: ```shell git pull origin master git tag $VERSION - git tag riverproui/$VERSION -m "release riverproui/$VERSION" git push --tags ``` 4. The build will cut a new release and create binaries automatically, but it won't have a good release message. Go the [release list](https://github.com/riverqueue/riverui/releases), find `$VERSION` and change the description to the release content in `CHANGELOG.md` (again, the build will have to finish first). + +### Releasing riverproui + +The `riverproui` submodule depends on the top level `riverui` module and in development it is customary to leave a `replace` directive in its `go.mod` so that it can be developed against the live local version. However, this `replace` directive makes it incompatible with `go install ...@latest`. + +As such, we must use a two-phase release for these modules: + +1. Release `riverui` with an initial version (i.e. all the steps above). + +2. Comment out `replace` directives to riverui `./riverproui/go.mod`. These were probably needed for developing the new feature, but need to be removed because they prevent the module from being `go install`-able. + +3. From `./riverproui`, `go get` to upgrade to the main package versions were just released (make sure you're getting `$VERSION` and not thwarted by shenanigans in Go's module proxy): + + ```shell + cd ./riverproui + go get -u riverqueue.com/riverui@$VERSION + ``` + +4. Run `go mod tidy`: + + ```shell + go mod tidy + ``` + +5. Prepare a PR with the changes. Have it reviewed and merged. + +6. Pull the changes back down, add a tag for `riverproui/$VERSION`, and push it to GitHub: + + ```shell + git pull origin master + git tag riverproui/$VERSION -m "release riverproui/$VERSION" + git push --tags + ``` + + The main `$VERSION` tag and `riverproui/$VERSION` will point to different commits, and although a little odd, is tolerable. From 6c6afe32444e492300a1824a9479f9486549fe61 Mon Sep 17 00:00:00 2001 From: Blake Gentry Date: Sat, 3 Jan 2026 14:01:26 -0600 Subject: [PATCH 4/5] fix(ci) parse ORAS referrers index correctly ORAS v1.3.0 emits discovery results under .referrers, not .manifests. Update CI helper scripts to accept both shapes so bundle discovery works and offline verification stops failing. --- scripts/ci/prefetch-referrers-content.sh | 5 ++++- scripts/ci/verify-sigstore-bundles-offline.sh | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/ci/prefetch-referrers-content.sh b/scripts/ci/prefetch-referrers-content.sh index 219d69c..99f56b4 100644 --- a/scripts/ci/prefetch-referrers-content.sh +++ b/scripts/ci/prefetch-referrers-content.sh @@ -50,7 +50,10 @@ for subject in "$@"; do # Discover and warm each referrer manifest and its blobs. oras discover --format json "${IMAGE_NAME}@${subject}" | tee /tmp/referrers.json - digests="$(jq -r '.manifests[]?.digest // empty' /tmp/referrers.json)" + # ORAS output shape differs by version/flags: + # - some versions return { "manifests": [...] } + # - some return { "referrers": [...] } + digests="$(jq -r '((.manifests // .referrers // []) | .[]? | .digest // empty)' /tmp/referrers.json)" for d in $digests; do [ -z "$d" ] && continue diff --git a/scripts/ci/verify-sigstore-bundles-offline.sh b/scripts/ci/verify-sigstore-bundles-offline.sh index da88174..eac481a 100644 --- a/scripts/ci/verify-sigstore-bundles-offline.sh +++ b/scripts/ci/verify-sigstore-bundles-offline.sh @@ -59,7 +59,7 @@ verify_bundle_for_subject() { force_fetch_manifest_and_referrers "$subject" oras discover --format json "${IMAGE_NAME}@${subject}" > "$refjson" - mapfile -t bundle_mfs < <(jq -r '(.manifests // [])[] + mapfile -t bundle_mfs < <(jq -r '((.manifests // .referrers // []) | .[]?) | select(.artifactType=="application/vnd.dev.sigstore.bundle.v0.3+json" or .artifactType=="application/vnd.dev.sigstore.bundle+json;version=0.3") | .digest' "$refjson") From 9cfebbcb6194236cd50912b5e986a92e871686f9 Mon Sep 17 00:00:00 2001 From: Blake Gentry Date: Sat, 3 Jan 2026 14:11:09 -0600 Subject: [PATCH 5/5] fix(ci) workflow boolean checks --- .github/workflows/docker-riverproui.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-riverproui.yaml b/.github/workflows/docker-riverproui.yaml index 699a0f7..e2ac4e3 100644 --- a/.github/workflows/docker-riverproui.yaml +++ b/.github/workflows/docker-riverproui.yaml @@ -37,7 +37,7 @@ jobs: name: "Build image: riverproui" runs-on: ubuntu-latest # Skip this job when manually dispatched with verify_only=true - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.verify_only == 'true') }} + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.verify_only) }} env: ECR_ACCOUNT_ID: ${{ secrets.ECR_CACHE_AWS_ACCOUNT_ID }} ECR_ROLE_ARN: ${{ secrets.ECR_CACHE_ROLE_ARN }} @@ -124,7 +124,7 @@ jobs: needs: - build-riverproui # Skip this job when manually dispatched with verify_only=true - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.verify_only == 'true') }} + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.verify_only) }} permissions: contents: read id-token: write @@ -363,7 +363,7 @@ jobs: # Only run on release tag events (refs/tags/riverproui/vX.Y.Z), # or when manually forced via workflow_dispatch: force_prefetch=true. # For workflow_dispatch, inputs.ref should be in the form 'riverproui/vX.Y.Z'. - if: startsWith(github.ref, 'refs/tags/riverproui/v') || (github.event_name == 'workflow_dispatch' && (startsWith(inputs.ref, 'riverproui/v') || inputs.force_prefetch == 'true')) + if: startsWith(github.ref, 'refs/tags/riverproui/v') || (github.event_name == 'workflow_dispatch' && (startsWith(inputs.ref, 'riverproui/v') || inputs.force_prefetch)) permissions: contents: read steps: