Release test report: reuse suites that already passed on the same code

.github/scripts/junit-totals.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Prints "<tests> <failed> <skipped>" over all JUnit XML in an extracted report bundle - from both
+# Gradle (test-results) and AGP (androidTest-results): one <testsuite> per file with
+# tests/failures/errors/skipped attributes. Failed counts failures and errors.
+#
+# Usage: junit-totals.sh <extracted bundle dir>
+set -euo pipefail
+
+# the AGP file names contain spaces ("...(AVD) - 11-_app-.xml"), so the files are read through
+# find -exec instead of being passed around as a word list
+find "$1" -path '*-results/*' -name '*.xml' -exec cat {} + | awk '
+  function attr(name) { return match($0, name "=\"[0-9]+\"") ? substr($0, RSTART + length(name) + 2, RLENGTH - length(name) - 3) + 0 : 0 }
+  /<testsuite[ >]/ { t += attr("tests"); f += attr("failures") + attr("errors"); s += attr("skipped") }
+  END { print t + 0, f + 0, s + 0 }'

.github/scripts/publish-reports.sh

@@ -8,9 +8,15 @@
 # Needs the gh CLI on the runner and GH_TOKEN with contents: write (to create the release/tag and
 # upload/delete its assets).
 #
-# With REPORTS_TAG set (release-test-report.yml), the bundle goes to that version's pre-release
-# instead, under its plain name and without the retention cleanup - that release is created by the
-# calling workflow and kept until deleted by hand.
+# With REPORTS_TAG set (release-test-report.yml), the bundle also goes to that version's
+# pre-release, under its plain name - that release is created by the calling workflow and kept
+# until deleted by hand.
+#
+# The name on ci-reports holds the git tree of the tested code, so a later release report run on
+# the same code can reuse a passing bundle instead of running the tests again (see
+# reuse-verified-reports.sh). The tree, not the commit: a PR run tests GitHub's merge preview
+# commit, and the merge commit made when merging the PR has the same tree as long as the base
+# branch hasn't changed in between.
 #
 # Usage: publish-reports.sh <bundle file>
 set -euo pipefail
@@ -28,7 +34,6 @@ if [ -n "${REPORTS_TAG:-}" ]; then
     echo "## Reports: $name"
     echo "[Download $name]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download/$REPORTS_TAG/$name) - attached to the [$REPORTS_TAG]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/$REPORTS_TAG) pre-release"
   } >> "$GITHUB_STEP_SUMMARY"
-  exit 0
 fi
 
 # unit-test.yml and android-test.yml run at the same time, so both may try to create the release on
@@ -41,7 +46,8 @@ fi
 
 # run id + attempt in the name, since all runs share this one release (a re-run gets its own file
 # instead of replacing the reports of the attempt before it)
-name="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(basename "$file")"
+tree=$(git rev-parse 'HEAD^{tree}')
+name="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-tree-${tree}-$(basename "$file")"
 staged="$RUNNER_TEMP/$name"
 cp "$file" "$staged"
 gh release upload "$tag" "$staged" --clobber

.github/scripts/release-report-summary.sh

@@ -3,6 +3,9 @@
 # results (and unit test line coverage) per suite, and writes them as the release notes and the
 # job summary. Used by release-test-report.yml after the test jobs have uploaded their bundles.
 #
+# REUSED_UNIT, REUSED_API30 and REUSED_API34 hold the id of the earlier run whose bundle was reused
+# for that suite (see reuse-verified-reports.sh), empty if this run tested it.
+#
 # Needs the gh CLI on the runner and GH_TOKEN with contents: write.
 #
 # Usage: release-report-summary.sh <release tag> <version>
@@ -26,53 +29,52 @@ run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
   echo "- Run: [$GITHUB_RUN_ID]($run_url)"
   echo "- Date: $(date -u +%Y-%m-%d) (UTC)"
   echo
-  echo "| Suite | Result | Tests | Passed | Failed | Skipped | Report |"
-  echo "|---|---|---:|---:|---:|---:|---|"
+  echo "| Suite | Result | Tests | Passed | Failed | Skipped | Tested in | Report |"
+  echo "|---|---|---:|---:|---:|---:|---|---|"
 } > "$notes"
 
 failures="$work/failures.md"
 : > "$failures"
 coverage=""
 
-for suite in "Unit tests:unit-test-reports.tar.gz" \
-             "Instrumentation tests (API 30, small screen):android-test-reports-api30.tar.gz" \
-             "Instrumentation tests (API 34, Pixel 6):android-test-reports-api34.tar.gz"; do
-  label="${suite%%:*}"
-  bundle="${suite#*:}"
+for suite in "${REUSED_UNIT:-}:Unit tests:unit-test-reports.tar.gz" \
+             "${REUSED_API30:-}:Instrumentation tests (API 30, small screen):android-test-reports-api30.tar.gz" \
+             "${REUSED_API34:-}:Instrumentation tests (API 34, Pixel 6):android-test-reports-api34.tar.gz"; do
+  reused_from="${suite%%:*}"
+  rest="${suite#*:}"
+  label="${rest%%:*}"
+  bundle="${rest#*:}"
+  tested_in="this run"
+  if [ -n "$reused_from" ]; then
+    tested_in="[run $reused_from]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$reused_from), same code"
+  fi
   if [ ! -f "$work/bundles/$bundle" ]; then
-    echo "| $label | ⚠️ no report | | | | | |" >> "$notes"
+    echo "| $label | ⚠️ no report | | | | | $tested_in | |" >> "$notes"
     continue
   fi
   dir="$work/extract/${bundle%.tar.gz}"
   mkdir -p "$dir"
   tar -xzf "$work/bundles/$bundle" -C "$dir"
 
-  # JUnit XML from both Gradle (test-results) and AGP (androidTest-results): one <testsuite> per
-  # file with tests/failures/errors/skipped attributes, failing <testcase>s followed by <failure>
-  # or <error>
-  xmls=$(find "$dir" -path '*-results/*' -name '*.xml' | sort)
-  if [ -z "$xmls" ]; then
-    echo "| $label | ⚠️ no results | | | | | |" >> "$notes"
+  if [ -z "$(find "$dir" -path '*-results/*' -name '*.xml')" ]; then
+    echo "| $label | ⚠️ no results | | | | | $tested_in | |" >> "$notes"
     continue
   fi
-  # shellcheck disable=SC2086
-  read -r tests failed skipped < <(awk '
-    function attr(name,   m) { return match($0, name "=\"[0-9]+\"") ? substr($0, RSTART + length(name) + 2, RLENGTH - length(name) - 3) + 0 : 0 }
-    /<testsuite[ >]/ { t += attr("tests"); f += attr("failures") + attr("errors"); s += attr("skipped") }
-    END { print t + 0, f + 0, s + 0 }' $xmls)
+  read -r tests failed skipped < <(bash "$(dirname "$0")/junit-totals.sh" "$dir")
   passed=$((tests - failed - skipped))
   result="✅ passed"
   [ "$failed" -gt 0 ] && result="❌ failed"
   url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download/$tag/$bundle"
-  echo "| $label | $result | $tests | $passed | $failed | $skipped | [$bundle]($url) |" >> "$notes"
+  echo "| $label | $result | $tests | $passed | $failed | $skipped | $tested_in | [$bundle]($url) |" >> "$notes"
 
   if [ "$failed" -gt 0 ]; then
     echo "**$label**" >> "$failures"
-    # shellcheck disable=SC2086
-    awk '
+    # failing <testcase>s are followed by <failure> or <error>; the AGP file names contain spaces,
+    # see junit-totals.sh
+    find "$dir" -path '*-results/*' -name '*.xml' -exec cat {} + | awk '
       function attr(name,   m) { return match($0, name "=\"[^\"]*\"") ? substr($0, RSTART + length(name) + 2, RLENGTH - length(name) - 3) : "" }
       /<testcase[ >]/ { tc = attr("classname") "." attr("name") }
-      /<failure[ >]|<error[ >]/ { if (tc != "") print "- `" tc "`"; tc = "" }' $xmls | sort -u >> "$failures"
+      /<failure[ >]|<error[ >]/ { if (tc != "") print "- `" tc "`"; tc = "" }' | sort -u >> "$failures"
     echo >> "$failures"
   fi
 

.github/scripts/reuse-verified-reports.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Looks on the ci-reports pre-release for the newest bundle of the given suite that was made from
+# exactly the checked-out code (same git tree, see publish-reports.sh) - e.g. by the PR run before
+# merging, or by an earlier release report run whose summary failed. If it passed (tests ran, none
+# failed), attaches it to the version's pre-release under its plain name, as if this run had
+# tested it, and prints the id of the run that made it. Prints nothing if there's none to reuse,
+# then the suite has to be tested again.
+#
+# Only the newest bundle counts: if the newest run on that code failed, an older passing one
+# doesn't make it verified.
+#
+# Needs the gh CLI on the runner and GH_TOKEN with contents: write.
+#
+# Usage: reuse-verified-reports.sh <release tag> <bundle name, e.g. unit-test-reports.tar.gz>
+set -euo pipefail
+
+tag="$1"
+bundle="$2"
+tree=$(git rev-parse 'HEAD^{tree}')
+suffix="-tree-$tree-$bundle"
+
+release_id=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/ci-reports" --jq .id 2>/dev/null) || {
+  echo "No ci-reports pre-release, testing $bundle" >&2
+  exit 0
+}
+name=$(gh api --paginate "repos/$GITHUB_REPOSITORY/releases/$release_id/assets" \
+  --jq ".[] | select(.name | endswith(\"$suffix\")) | [.created_at, .name] | @tsv" |
+  sort | tail -1 | cut -f2)
+if [ -z "$name" ]; then
+  echo "No $bundle tested on tree $tree, testing it" >&2
+  exit 0
+fi
+
+work="$RUNNER_TEMP/reuse-$bundle"
+rm -rf "$work"
+mkdir -p "$work/extract"
+gh release download ci-reports --pattern "$name" --dir "$work"
+tar -xzf "$work/$name" -C "$work/extract"
+read -r tests failed skipped < <(bash "$(dirname "$0")/junit-totals.sh" "$work/extract")
+if [ "$tests" -eq 0 ] || [ "$failed" -gt 0 ]; then
+  echo "$name has $failed of $tests tests failed, testing $bundle again" >&2
+  exit 0
+fi
+
+cp "$work/$name" "$work/$bundle"
+gh release upload "$tag" "$work/$bundle" --clobber
+rm -rf "$work"
+echo "Reusing $name: $tests tests, none failed, $skipped skipped" >&2
+# <run id>-<attempt>-tree-...
+echo "${name%%-*}"

.github/workflows/android-test.yml

@@ -12,6 +12,10 @@ on:
       release-tag:
         type: string
         default: ""
+      # JSON list of the API levels to test, so it can leave out the ones that already passed
+      api-levels:
+        type: string
+        default: "[30, 34]"
 
 permissions:
   contents: write # attaching the report bundle to the ci-reports pre-release (publish-reports.sh)
@@ -23,28 +27,27 @@ jobs:
     # not for PRs from forks - see unit-test.yml
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on: [self-hosted, macOS]
+    # screen per API level: API 30 keeps the emulator's default small screen (320x640 @ 160dpi
+    # = 320x640dp, smaller than nearly any real phone) as the small-screen check; API 34 runs
+    # on a normal Full HD phone (pixel_6: 1080x2400 @ 420dpi, ~411x914dp).
+    # AVD_SUFFIX goes into the AVD name and cache key - the runner's disk keeps AVDs between
+    # runs and reuses one by name (force-avd-creation: false), so a new screen needs a new name.
+    # Set here rather than with matrix include entries, since an include entry for an API level
+    # left out by the api-levels input would be added as a job of its own
+    env:
+      AVD_PROFILE: ${{ matrix.api-level == 34 && 'pixel_6' || '' }}
+      AVD_SUFFIX: ${{ matrix.api-level == 34 && '-pixel6' || '' }}
     strategy:
       max-parallel: 1 # multiple runners are registered on the same physical Mac - without this,
       # matrix legs can run concurrently and contend for the same real GPU, which is what was
       # causing "Failed to find ColorBuffer" emulator errors regardless of the -gpu backend used
       matrix:
-        api-level:
-          - 30 # min API with arm64-v8a system images
-          - 34 # close to targetSdk, well-supported arm64-v8a image
+        # 30: min API with arm64-v8a system images; 34: close to targetSdk, well-supported
+        # arm64-v8a image. When not called by release-test-report.yml (pull_request,
+        # workflow_dispatch), there's no api-levels input, so both are tested
+        api-level: ${{ fromJSON(inputs.api-levels || '[30, 34]') }}
         arch:
           - arm64-v8a
-        # screen per API level: API 30 keeps the emulator's default small screen (320x640 @ 160dpi
-        # = 320x640dp, smaller than nearly any real phone) as the small-screen check; API 34 runs
-        # on a normal Full HD phone (pixel_6: 1080x2400 @ 420dpi, ~411x914dp).
-        # avd-suffix goes into the AVD name and cache key - the runner's disk keeps AVDs between
-        # runs and reuses one by name (force-avd-creation: false), so a new screen needs a new name
-        include:
-          - api-level: 30
-            profile: ""
-            avd-suffix: ""
-          - api-level: 34
-            profile: pixel_6
-            avd-suffix: "-pixel6"
     steps:
       - uses: actions/checkout@v4
 
@@ -57,7 +60,7 @@ jobs:
           path: |
             ~/.android/avd/*
             ~/.android/adb*
-          key: avd-${{ matrix.arch }}-${{ matrix.api-level }}-google-apis-swiftshader${{ matrix.avd-suffix }}
+          key: avd-${{ matrix.arch }}-${{ matrix.api-level }}-google-apis-swiftshader${{ env.AVD_SUFFIX }}
 
       - name: Create AVD and generate snapshot for caching
         if: steps.avd-cache.outputs.cache-hit != 'true'
@@ -66,13 +69,13 @@ jobs:
           api-level: ${{ matrix.api-level }}
           arch: ${{ matrix.arch }}
           target: google_apis
-          profile: ${{ matrix.profile }}
+          profile: ${{ env.AVD_PROFILE }}
           # this is a self-hosted runner with a persistent disk (unlike GitHub-hosted runners,
           # which start every job with an empty disk) - without a name derived from the actual
           # config, the action's default fixed "test" AVD name means any future change to
           # api-level/arch/target/profile here would silently keep reusing a stale AVD built under
           # the OLD config instead of picking up the new one, since force-avd-creation stays false
-          avd-name: ci-${{ matrix.arch }}-api${{ matrix.api-level }}-google-apis${{ matrix.avd-suffix }}
+          avd-name: ci-${{ matrix.arch }}-api${{ matrix.api-level }}-google-apis${{ env.AVD_SUFFIX }}
           force-avd-creation: false
           emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           disable-animations: false
@@ -84,8 +87,8 @@ jobs:
           api-level: ${{ matrix.api-level }}
           arch: ${{ matrix.arch }}
           target: google_apis
-          profile: ${{ matrix.profile }}
-          avd-name: ci-${{ matrix.arch }}-api${{ matrix.api-level }}-google-apis${{ matrix.avd-suffix }}
+          profile: ${{ env.AVD_PROFILE }}
+          avd-name: ci-${{ matrix.arch }}-api${{ matrix.api-level }}-google-apis${{ env.AVD_SUFFIX }}
           force-avd-creation: false
           emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           disable-animations: true

.github/workflows/release-test-report.yml

@@ -4,6 +4,12 @@ name: Release test report
 # the unit + instrumentation test reports for a release build. They're attached to a
 # "v<version>-test-reports" pre-release titled "<version> release test reports summary", whose
 # notes hold the pass/fail summary. Running again for the same version replaces that pre-release.
+#
+# A suite that already passed on exactly this code (same git tree) isn't tested again: its bundle
+# from that run (usually the PR's, or an earlier run of this workflow) is reused, see
+# reuse-verified-reports.sh. So after a merged PR whose tests passed, or a run where only some
+# suites failed, only what's missing runs. The "tested in" column of the summary says which run
+# each suite's reports come from.
 
 on:
   workflow_dispatch:
@@ -12,6 +18,10 @@ on:
         description: "Version name (leave empty to use appVersionName from app/build.gradle.kts)"
         type: string
         default: ""
+      retest:
+        description: "Run all tests again, even if they already passed on this code"
+        type: boolean
+        default: false
 
 permissions:
   contents: write # creating the pre-release and attaching the report bundles
@@ -27,6 +37,10 @@ jobs:
     outputs:
       version: ${{ steps.release.outputs.version }}
       tag: ${{ steps.release.outputs.tag }}
+      reused-unit: ${{ steps.reuse.outputs.unit }}
+      reused-api30: ${{ steps.reuse.outputs.api30 }}
+      reused-api34: ${{ steps.reuse.outputs.api34 }}
+      api-levels: ${{ steps.reuse.outputs.api-levels }}
     steps:
       - uses: actions/checkout@v4
 
@@ -53,17 +67,44 @@ jobs:
           echo "version=$version" >> "$GITHUB_OUTPUT"
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
 
+      - name: Reuse reports of suites that already passed on this code
+        id: reuse
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RETEST: ${{ inputs.retest }}
+          TAG: ${{ steps.release.outputs.tag }}
+        run: |
+          unit="" api30="" api34=""
+          # a failing lookup only means testing again
+          if [ "$RETEST" != "true" ]; then
+            unit=$(bash .github/scripts/reuse-verified-reports.sh "$TAG" unit-test-reports.tar.gz || true)
+            api30=$(bash .github/scripts/reuse-verified-reports.sh "$TAG" android-test-reports-api30.tar.gz || true)
+            api34=$(bash .github/scripts/reuse-verified-reports.sh "$TAG" android-test-reports-api34.tar.gz || true)
+          fi
+          levels=""
+          [ -z "$api30" ] && levels="$levels,30"
+          [ -z "$api34" ] && levels="$levels,34"
+          {
+            echo "unit=$unit"
+            echo "api30=$api30"
+            echo "api34=$api34"
+            echo "api-levels=[${levels#,}]"
+          } >> "$GITHUB_OUTPUT"
+
   unit-test:
     needs: prepare
+    if: needs.prepare.outputs.reused-unit == ''
     uses: ./.github/workflows/unit-test.yml
     with:
       release-tag: ${{ needs.prepare.outputs.tag }}
 
   android-test:
     needs: prepare
+    if: needs.prepare.outputs.api-levels != '[]'
     uses: ./.github/workflows/android-test.yml
     with:
       release-tag: ${{ needs.prepare.outputs.tag }}
+      api-levels: ${{ needs.prepare.outputs.api-levels }}
 
   summary:
     needs: [prepare, unit-test, android-test]
@@ -76,4 +117,7 @@ jobs:
       - name: Write release notes
         env:
           GH_TOKEN: ${{ github.token }}
+          REUSED_UNIT: ${{ needs.prepare.outputs.reused-unit }}
+          REUSED_API30: ${{ needs.prepare.outputs.reused-api30 }}
+          REUSED_API34: ${{ needs.prepare.outputs.reused-api34 }}
         run: bash .github/scripts/release-report-summary.sh "${{ needs.prepare.outputs.tag }}" "${{ needs.prepare.outputs.version }}"