From aea6d4125f8c625266041c1fd5bccae7091e9e3a Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 15:23:55 -0700 Subject: [PATCH 1/4] refactor(taskfile): adopt lib v2.6.0 lib v2.6.0 supplies three things this Taskfile still carried: - C1: the rsync image ships aws.config and sets AWS_CONFIG_FILE, so the repo's aws.config and the root env: block that named it go. - C2: the rsync engine names its image, so the toolbox include drops IMAGE. - C6, C35: the toolbox no longer defines report-engine, so the include drops its exclude, and the "Two includes, one namespace" comment goes with it. The render does not move for any of these. The workflow pins and the render-update follow once v2.6.0 is tagged. --- Taskfile.yml | 9 --------- aws.config | 10 ---------- 2 files changed, 19 deletions(-) delete mode 100644 aws.config diff --git a/Taskfile.yml b/Taskfile.yml index 8ce2e1f..a58a3bf 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -21,22 +21,13 @@ vars: CANARY: aspell/dict/0index.html # 8 plain http:// links, which the zone's HTML rewriters would alter FRESH_KEY: mirror-updated-timestamp.txt # ftp.gnu.org's clock, Unix time; a day old fails the run -env: - AWS_CONFIG_FILE: '{{.ROOT_DIR}}/aws.config' - -# Two includes, one namespace. report-engine is both the toolbox's no-op and the engine's, -# so the toolbox include excludes it. MAX_BATCHES and BATCH_GB are the engine's inline -# defaults and RECONCILE the toolbox's, so a run sets them: task sync -- MAX_BATCHES=8 -# RECONCILE=true. includes: toolbox: taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/toolbox.yml flatten: true - excludes: [report-engine] vars: NAME: gnu DESC: a twice-daily mirror of ftp.gnu.org/gnu, from {{.SOURCE}}, at https://{{.HOST}}/ - IMAGE: ghcr.io/katoptra/toolbox:rsync-v2 rsync: taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/engines/rsync.yml flatten: true diff --git a/aws.config b/aws.config deleted file mode 100644 index 7a7a797..0000000 --- a/aws.config +++ /dev/null @@ -1,10 +0,0 @@ -# Read via AWS_CONFIG_FILE. Anything under 4 GiB is one PutObject; the largest file in the -# tree is 1.51 GB. A file past 4 GiB (R2's single-part limit is 4.995 GiB) goes up in -# 512 MiB parts. -[default] -retry_mode = standard -max_attempts = 10 -s3 = - multipart_threshold = 4GB - multipart_chunksize = 512MB - max_concurrent_requests = 32 From 417f92ea6da892cfb68245f00839aa9ae0fb4f07 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 15:36:15 -0700 Subject: [PATCH 2/4] docs(comments): rewrite the Taskfile and workflow comments in STE - C35: the Taskfile header follows the shared template. It names what the mirror copies, where, how often and how, points to lib's README for what the toolbox and the engine supply, lists the values that are this mirror's own, and says how the secrets reach the container. The previous header called these values "identity and nothing else", which was false: PAGE_FOOT, CANARY and FRESH_KEY are not identity. - C3: INDEX's comment no longer claims that no upstream path can carry the name. One can, and the directory page wins. - C78: every other Taskfile comment, the sync workflow's comment and op.env's header are in ASD-STE100. op.env's header is the one that lib's examples and every mirror share, and no op:// line changes. A comment longer than one short clause now sits on its own lines above its var or key, wrapped at 92 columns. No comment sits inside a command, so the render does not move. --- .github/workflows/sync.yml | 4 ++- Taskfile.yml | 60 ++++++++++++++++++++++++++------------ op.env | 5 ++-- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index ab242c5..846d18f 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -8,7 +8,9 @@ on: default: '' permissions: contents: read - actions: write # the called workflow chains the next run; a called workflow cannot raise this + # The called workflow uses this permission to start the next run of a chain. A called + # workflow cannot increase this permission. + actions: write concurrency: {group: sync, cancel-in-progress: false} jobs: sync: diff --git a/Taskfile.yml b/Taskfile.yml index a58a3bf..685d299 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,25 +1,49 @@ -# Mirror GNU's release tree, ftp.gnu.org/gnu, into a Cloudflare R2 bucket twice a day, from -# a list diff: a secondary's listing against the state the last run left in the bucket. The -# toolbox, which starts a run, contains it in an image, resolves its secrets, renders and -# checks it, and reports it, and the rsync engine, which moves the bytes, draws the -# directory pages, reads them back and orders the pipeline, are katoptra/lib's, included -# below at v2 and flattened into this file. What is here is the mirror's identity and -# nothing else. R2 is reached through the AWS CLI, configured by the three AWS_* variables -# op.env resolves from 1Password, which cross into the image by name. +# This mirror keeps a copy of GNU's release tree, ftp.gnu.org/gnu, in a Cloudflare R2 +# bucket. Twice a day, a run compares the listing of a secondary mirror with the state that +# the last run put in the bucket (a list diff). +# +# The toolbox and the rsync engine are from katoptra/lib. Its README shows the items that +# each one supplies. This file includes them at v2, with `flatten: true`. +# +# The values in this file are for this mirror only: +# - SOURCE, HOST and BUCKET identify the mirror. +# - CEILING_GB and LIST_FLOOR are the limits for a run. +# - INDEX and PAGE_FOOT are for the directory pages. +# - CANARY and FRESH_KEY are for the checks at the end of each run. +# - NAME and DESC are for the menu. +# +# The secrets are the names in op.env. The toolbox gets their values from the vault in +# 1Password. It gives each value to the container through the environment. Only the name +# is on the command line. version: '3' vars: - SOURCE: rsync://mirror.csclub.uwaterloo.ca/gnu/ # a secondary, as GNU asks; about 2 h behind ftp.gnu.org - HOST: gnu.katoptra.org # a fork changes this line and the address in PAGE_FOOT - BUCKET: gnu # bucket name; objects sit at its root under GNU's own paths - CEILING_GB: 350 # refuse to run past this many decimal GB upstream; 274 GB in September 2026 - LIST_FLOOR: 40000 # a listing under this many lines is a truncated one; the tree is ~43,850 files - INDEX: '{{.HOST}}.directory.index.html' # the page drawn in every directory; no upstream path can carry the name - # The line every directory page closes on. GNU's mirror guidelines allow short, - # explanatory text and no link but a bug-reporting one: an address that takes mail. + # SOURCE is a secondary mirror, which GNU recommends. It gets changes approximately 2 h + # after ftp.gnu.org. + SOURCE: rsync://mirror.csclub.uwaterloo.ca/gnu/ + HOST: gnu.katoptra.org # a fork changes this line and the address in PAGE_FOOT. + # BUCKET is the name of the bucket. The objects are at its root, and their keys are GNU's + # paths. + BUCKET: gnu + # If upstream has more than this number of decimal GB, the run stops with an error. + # Upstream had 274 GB in September 2026. + CEILING_GB: 350 + # A listing must have more lines than this value, or it is a truncated listing. The tree + # has approximately 43,850 files. + LIST_FLOOR: 40000 + # INDEX is the key suffix of the directory pages. The Transform Rule of the zone uses only + # this name. If an upstream file has this name, the bucket keeps the page, not the file. + INDEX: '{{.HOST}}.directory.index.html' + # PAGE_FOOT is the last line of each directory page. In GNU's mirror guidelines, text + # must be short and "strictly explanatory". The only link that is permitted is a link for + # bug reports. That link is an email address, which must receive mail. PAGE_FOOT: 'A mirror of https://ftp.gnu.org/gnu/, updated twice a day. Sizes in bytes, times as upstream lists them. Report a problem to gnu@katoptra.org.' - CANARY: aspell/dict/0index.html # 8 plain http:// links, which the zone's HTML rewriters would alter - FRESH_KEY: mirror-updated-timestamp.txt # ftp.gnu.org's clock, Unix time; a day old fails the run + # CANARY is a file with 8 http:// links. If the zone uses an HTML rewriter, the rewriter + # can change these links. + CANARY: aspell/dict/0index.html + # FRESH_KEY is the file with ftp.gnu.org's clock, in Unix time. If that time is more than + # 24 h before the check, the run stops with an error. + FRESH_KEY: mirror-updated-timestamp.txt includes: toolbox: diff --git a/op.env b/op.env index 170facd..f30a2f6 100644 --- a/op.env +++ b/op.env @@ -1,5 +1,6 @@ -# op:// references only; `op run --env-file=op.env` resolves them at run time. -# One vault, addressed by UUID so a rename cannot break it; item gnu. See katoptra/lib, Secrets. +# This file contains only op:// references. `op run --env-file=op.env` gets their values +# at run time. The secrets are in one vault, addressed by UUID. Thus, the references stay +# correct when the vault gets a new name. The item is gnu. Refer to katoptra/lib, Secrets. AWS_ACCESS_KEY_ID=op://y6y6b6l2zjbpv7szc5ym5sprne/gnu/r2/access_key_id AWS_SECRET_ACCESS_KEY=op://y6y6b6l2zjbpv7szc5ym5sprne/gnu/r2/secret_access_key AWS_ENDPOINT_URL=op://y6y6b6l2zjbpv7szc5ym5sprne/gnu/r2/endpoint From 3f44ea3285df114ffbab775844b0f3228caf5990 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 16:08:05 -0700 Subject: [PATCH 3/4] docs(comments): polish the Taskfile comments after review - C33's one number: CEILING_GB's comment says approximately 275 GB, the measured 274.56 GB that the $4.12 a month rests on. - The header gives each secret to the image with its name: STE approves BY only as a mathematical function word. - The header keeps C35's "below": the includes sit below the vars. - The header's check line and FRESH_KEY's comment say that the engine also sends the FRESH_KEY file to the bucket last. - HOST's trailing comment is a sentence, with a capital and a period. No comment sits inside a command, so the render does not move. --- Taskfile.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 685d299..024c294 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,30 +3,30 @@ # the last run put in the bucket (a list diff). # # The toolbox and the rsync engine are from katoptra/lib. Its README shows the items that -# each one supplies. This file includes them at v2, with `flatten: true`. +# each one supplies. This file includes them below the vars, at v2, with `flatten: true`. # # The values in this file are for this mirror only: # - SOURCE, HOST and BUCKET identify the mirror. # - CEILING_GB and LIST_FLOOR are the limits for a run. # - INDEX and PAGE_FOOT are for the directory pages. -# - CANARY and FRESH_KEY are for the checks at the end of each run. +# - CANARY and FRESH_KEY are for the checks at the end of each run. The engine also sends +# the FRESH_KEY file to the bucket last. # - NAME and DESC are for the menu. # # The secrets are the names in op.env. The toolbox gets their values from the vault in -# 1Password. It gives each value to the container through the environment. Only the name -# is on the command line. +# 1Password. It gives each value to the image with its name, not on a command line. version: '3' vars: # SOURCE is a secondary mirror, which GNU recommends. It gets changes approximately 2 h # after ftp.gnu.org. SOURCE: rsync://mirror.csclub.uwaterloo.ca/gnu/ - HOST: gnu.katoptra.org # a fork changes this line and the address in PAGE_FOOT. + HOST: gnu.katoptra.org # A fork changes this line and the address in PAGE_FOOT. # BUCKET is the name of the bucket. The objects are at its root, and their keys are GNU's # paths. BUCKET: gnu # If upstream has more than this number of decimal GB, the run stops with an error. - # Upstream had 274 GB in September 2026. + # Upstream had approximately 275 GB in September 2026. CEILING_GB: 350 # A listing must have more lines than this value, or it is a truncated listing. The tree # has approximately 43,850 files. @@ -42,7 +42,8 @@ vars: # can change these links. CANARY: aspell/dict/0index.html # FRESH_KEY is the file with ftp.gnu.org's clock, in Unix time. If that time is more than - # 24 h before the check, the run stops with an error. + # 24 h before the check, the run stops with an error. The engine also sends this file to + # the bucket last. FRESH_KEY: mirror-updated-timestamp.txt includes: From a14727fa38d5498bd779e4d8f6a55548537d7986 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 16:49:51 -0700 Subject: [PATCH 4/4] chore(render): accept lib v2.6.0 and pin its workflows --- .github/workflows/check.yml | 2 +- .github/workflows/sync.yml | 2 +- render.txt | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 29ad511..2721983 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,4 +3,4 @@ on: {pull_request: {}} permissions: {contents: read} jobs: check: - uses: katoptra/lib/.github/workflows/check.yml@3854982b72617790be2b8ed0fb73c22e87b8a42e # v2.3.0 + uses: katoptra/lib/.github/workflows/check.yml@36d75e6b21473400b5b15003025d9a2e81d4eeea # v2.6.0 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 846d18f..8da51ad 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -14,6 +14,6 @@ permissions: concurrency: {group: sync, cancel-in-progress: false} jobs: sync: - uses: katoptra/lib/.github/workflows/sync.yml@3854982b72617790be2b8ed0fb73c22e87b8a42e # v2.3.0 + uses: katoptra/lib/.github/workflows/sync.yml@36d75e6b21473400b5b15003025d9a2e81d4eeea # v2.6.0 with: {vars: '${{ inputs.vars }}'} secrets: inherit diff --git a/render.txt b/render.txt index daef47d..766b003 100644 --- a/render.txt +++ b/render.txt @@ -23,7 +23,7 @@ echo "due: RECONCILE=$r RECONCILE_HOURS=$h, last reconcile $ago: $(test -f /work task: [list] rm -f /work/.run/publish.txt /work/.run/labeled.txt /work/.run/orphans.txt /work/.run/unlanded.txt /work/.run/fresh.txt task: [list] mkdir -p /work/.run /work/staging task: [retry] for i in 1 2 3 4 5; do - rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -rL --list-only rsync://mirror.csclub.uwaterloo.ca/gnu/ > /work/.run/listing.txt ) || rc=$? # subshell so CMD's exit cannot end the loop; || so Task's errexit cannot + rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -rL --list-only rsync://mirror.csclub.uwaterloo.ca/gnu/ > /work/.run/listing.txt ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s @@ -76,7 +76,7 @@ task: [split] awk -F'\t' -v OFS='\t' -v CAP=$((4 * 1000000000)) -v DIR=/work/.ru task: [prepare] mkdir -p /work/.run/tl task: [prepare] printf '%s\n' tlpkg/texlive.tlpdb.xz tlpkg/texlive.tlpdb.sha512 tlpkg/texlive.tlpdb.sha512.asc tlpkg/gpg/pubring.gpg > /work/.run/tl/files.txt task: [retry] for i in 1 2 3 4 5; do - rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -Lt --files-from=/work/.run/tl/files.txt rsync://mirror.csclub.uwaterloo.ca/gnu// /work/.run/tl/ ) || rc=$? # subshell so CMD's exit cannot end the loop; || so Task's errexit cannot + rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -Lt --files-from=/work/.run/tl/files.txt rsync://mirror.csclub.uwaterloo.ca/gnu// /work/.run/tl/ ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s @@ -87,7 +87,7 @@ task: [prepare] cd /work/.run/tl/tlpkg && shasum -a 512 -c texlive.tlpdb.sha512 task: [fetch] find /work/staging -mindepth 1 -delete task: [fetch] cut -f1 > /work/.run/files.txt task: [retry] for i in 1 2 3 4 5; do - rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -Lt --files-from=/work/.run/files.txt --ignore-missing-args rsync://mirror.csclub.uwaterloo.ca/gnu/ /work/staging/ ) || rc=$? # subshell so CMD's exit cannot end the loop; || so Task's errexit cannot + rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -Lt --files-from=/work/.run/files.txt --ignore-missing-args rsync://mirror.csclub.uwaterloo.ca/gnu/ /work/staging/ ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s @@ -168,7 +168,7 @@ if to == b"up": with open(labels, "rb") as f: for line in f: path, t = line.rstrip(b"\n").split(b"\t") - if path != b"timestamp": + if path != b"mirror-updated-timestamp.txt": n = trees.setdefault(t, str(len(trees) + 1).encode()) move(os.path.join(staging, path), os.path.join(up, n, path)) for t, n in trees.items(): @@ -200,7 +200,7 @@ if to == b"up": with open(labels, "rb") as f: for line in f: path, t = line.rstrip(b"\n").split(b"\t") - if path != b"timestamp": + if path != b"mirror-updated-timestamp.txt": n = trees.setdefault(t, str(len(trees) + 1).encode()) move(os.path.join(staging, path), os.path.join(up, n, path)) for t, n in trees.items(): @@ -216,7 +216,7 @@ else: shutil.rmtree(up, ignore_errors=True) PY -task: [publish] test ! -f /work/staging/timestamp || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --content-type "$(awk -F'\t' '$1 == "timestamp" { print $2 }' /work/.run/labels.txt)" /work/staging/timestamp s3://gnu/timestamp | tee -a /work/.run/publish.txt +task: [publish] test ! -f /work/staging/mirror-updated-timestamp.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --content-type "$(awk -F'\t' '$1 == "mirror-updated-timestamp.txt" { print $2 }' /work/.run/labels.txt)" /work/staging/mirror-updated-timestamp.txt s3://gnu/mirror-updated-timestamp.txt | tee -a /work/.run/publish.txt task: [merge] (cd /work/staging && find . -type f | sed 's|^\./||' | LC_ALL=C sort) > /work/.run/landed.txt task: [merge] awk -F'\t' 'NR == FNR { drop[$1] = 1; next } !($1 in drop)' /work/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/.run/landed.txt ) | LC_ALL=C sort > /work/.run/applied.new task: [push] xz -T0 -c /work/.run/applied.new > /work/.run/applied.new.xz @@ -321,7 +321,7 @@ task: [smoke] h=aspell/dict/0index.html; n=$(awk -F'\t' -v k="$h" '$1 == k { pri task: [smoke] g=$(awk -F'\t' '$1 ~ /\.tar\.gz$/ && $2 > 0 { print $1; exit }' /work/.run/applied.txt 2>/dev/null); case "https://gnu.katoptra.org" in http*) ;; *) g= ;; esac; if test -z "$g"; then echo "content-encoding check skipped (no non-empty .tar.gz in the state, or the URL is not http)"; else u=$(printf '%s\n' "$g" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2) && h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://gnu.katoptra.org/$u") && if printf '%s' "$h" | grep -qi '^content-encoding:'; then echo "$g: served with a content-encoding" >&2; exit 1; fi && echo "$g: served with no content-encoding"; fi -task: [smoke] if test -f /work/.run/changed.txt && test "$(ls /work/.run/batch-*.txt 2>/dev/null | wc -l)" -le 4; then LC_ALL=C comm -23 /work/.run/changed.txt /work/.run/applied.txt > /work/.run/unlanded.txt; else : > /work/.run/unlanded.txt; fi; cut -f1 /work/.run/unlanded.txt | head -20 | while IFS= read -r p; do echo "::warning::listed upstream, never fetched: $p"; done +task: [smoke] if test -f /work/.run/changed.txt && test ! -f /work/.run/chain; then LC_ALL=C comm -23 /work/.run/changed.txt /work/.run/applied.txt > /work/.run/unlanded.txt; else : > /work/.run/unlanded.txt; fi; cut -f1 /work/.run/unlanded.txt | head -20 | while IFS= read -r p; do echo "::warning::listed upstream, never fetched: $p"; done task: [fresh] k=mirror-updated-timestamp.txt; if ! awk -F'\t' -v k="$k" '$1 == k { f = 1 } END { exit !f }' /work/.run/applied.txt 2>/dev/null; then echo "freshness skipped ($k is not in the state)"; else case "24" in ''|*[!0-9]*) echo "FRESH_HOURS is a whole number" >&2; exit 1 ;; esac; u=$(printf '%s\n' "$k" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2); t=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 "https://gnu.katoptra.org/$u" | awk '!/^#/ { for (i = 1; i <= NF; i++) {