From 94aa1f102805ffa8a5a3f7be5f3651c36b5052db Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 12 Nov 2026 02:46:03 +0900 Subject: [PATCH 1/9] ci(template): pin actions by SHA and declare permissions like the satellites The template workflows used tag pins, validate-data had no permissions block and carried a comment about game-catalog's record count. They now take the form game-catalog runs: the same pinned SHAs, read-only permissions, concurrency and timeouts. --- .../.github/workflows/deploy-pages.yml | 9 +++++---- .../.github/workflows/validate-data.yml | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/machine/satellite_template/.github/workflows/deploy-pages.yml b/machine/satellite_template/.github/workflows/deploy-pages.yml index 06421b4..94bde29 100644 --- a/machine/satellite_template/.github/workflows/deploy-pages.yml +++ b/machine/satellite_template/.github/workflows/deploy-pages.yml @@ -19,24 +19,25 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 # history.json replays every data commit - - uses: actions/setup-python@v5 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.12" - name: Build summary + history run: python site/build.py - - uses: actions/upload-pages-artifact@v3 + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: site deploy: needs: build runs-on: ubuntu-latest + timeout-minutes: 10 environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: - id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/machine/satellite_template/.github/workflows/validate-data.yml b/machine/satellite_template/.github/workflows/validate-data.yml index 02763cc..17442b4 100644 --- a/machine/satellite_template/.github/workflows/validate-data.yml +++ b/machine/satellite_template/.github/workflows/validate-data.yml @@ -1,17 +1,26 @@ name: validate-data +# GITHUB_TOKEN only: no personal access tokens, no bot account, no writes. on: pull_request: push: branches: [develop, main] +permissions: + contents: read + +concurrency: + group: validate-data-${{ github.ref }} + cancel-in-progress: true + jobs: validate: runs-on: ubuntu-latest - timeout-minutes: 90 # 962k files; measure a real run before trimming + # Generous for a new catalog; measure a real run before trimming. + timeout-minutes: 30 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.12" - name: Validate {{title}} From 79ac4be77230e17d74cc27c99c4e0910a79d1618 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 12 Nov 2026 02:46:32 +0900 Subject: [PATCH 2/9] feat(template): add the label-triggered final check to new satellites Both satellites run a final-check workflow and the template had none. The new one is game-catalog's with the repository and category filled in. The checklist now registers the repo in repos.json with its allowed secret and names the label and the token to set up, and the tests allow TECHENGINEBOT_TOKEN in final-check.yml only. --- machine/new_satellite.py | 7 +- .../.github/workflows/final-check.yml | 183 ++++++++++++++++++ tests/test_new_satellite.py | 24 ++- 3 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 machine/satellite_template/.github/workflows/final-check.yml diff --git a/machine/new_satellite.py b/machine/new_satellite.py index 725b534..db552c6 100644 --- a/machine/new_satellite.py +++ b/machine/new_satellite.py @@ -122,10 +122,15 @@ def checklist(v: dict[str, str]) -> str: 3. gh api -X POST repos/{repo}/pages -f build_type=workflow 4. gh api -X POST repos/{repo}/environments/github-pages/deployment-branch-policies -f name=main (skip this and every deploy fails with no log — game-catalog, 2026-09-17) - 5. add {{"name": "{repo}", "branches": ["develop", "main"]}} to TechMachine machine/repos.json + 5. add {{"name": "{repo}", "branches": ["develop", "main"], + "allowed_secrets": ["TECHENGINEBOT_TOKEN"]}} to TechMachine machine/repos.json and the summary.json URL to its endpoints 6. add {{ key: "{v['plural']}", label: "{v['plural']}", base: "https://gettechapi.github.io/{v['repo']}/" }} to SATELLITES in TechAPI site/src/scripts/techapi.js + 7. for the final check: create the `final-check` label and set the + TECHENGINEBOT_TOKEN secret to a fine-grained token of TechEngineBot with + Pull requests: read and write on {repo} only. Without the secret the + verdict stays in the run summary and nothing is posted. """ diff --git a/machine/satellite_template/.github/workflows/final-check.yml b/machine/satellite_template/.github/workflows/final-check.yml new file mode 100644 index 0000000..44b424d --- /dev/null +++ b/machine/satellite_template/.github/workflows/final-check.yml @@ -0,0 +1,183 @@ +name: final-check + +# A maintainer adds the `final-check` label to an important PR. The check job +# runs, on the PR merged into its base, everything a push to develop/main +# would: the full-catalog validator, the tests and the site build. TechEngineBot +# leaves ONE comment with the verdict; adding the label again edits that +# comment. This is the only place the TechEngineBot account is used here. +# +# The check job runs PR code, so it gets no secrets and a read-only token. +# Only the comment job sees TECHENGINEBOT_TOKEN, which must be a fine-grained +# token of TechEngineBot with Pull requests: read and write; the job refuses +# any other kind of token or account. Fork PRs never get a comment. +on: + pull_request: + types: [labeled] + branches: [develop, main] + +permissions: {} + +jobs: + check: + if: github.event.label.name == 'final-check' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + concurrency: + group: final-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + outputs: + body: ${{ steps.report.outputs.body }} + steps: + # The merge commit, as a push would build it after merging. Full history: + # site/build.py replays every data commit for history.json. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Validate the whole catalog + id: validate + continue-on-error: true + run: | + status=0 + python -m app.validate > "$RUNNER_TEMP/validate.log" 2>&1 || status=$? + tail -n 40 "$RUNNER_TEMP/validate.log" + exit "$status" + + - name: Tests + id: tests + continue-on-error: true + run: | + status=0 + { python -m pip install --disable-pip-version-check pytest && python -m pytest -q; } \ + > "$RUNNER_TEMP/tests.log" 2>&1 || status=$? + tail -n 40 "$RUNNER_TEMP/tests.log" + exit "$status" + + - name: Build the site + id: site + continue-on-error: true + run: | + status=0 + python site/build.py > "$RUNNER_TEMP/site.log" 2>&1 || status=$? + cat "$RUNNER_TEMP/site.log" + exit "$status" + + # The comment quotes tool output, which can carry text from the PR's + # records: logs stay inside code fences with backticks and @ removed, so + # they can neither break out into markdown nor mention anyone. + - name: Report + id: report + if: always() + env: + VALIDATE: ${{ steps.validate.outcome }} + TESTS: ${{ steps.tests.outcome }} + SITE: ${{ steps.site.outcome }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + clean() { tr -d '`@' | cut -c1-300; } + fence() { # fence + echo '```text' + tail -n "$2" "$1" 2>/dev/null | clean + echo '```' + } + verdict='✅ Final check passed' + [ "$VALIDATE/$TESTS/$SITE" = success/success/success ] || verdict='❌ Final check failed' + changed=$(git diff --name-only HEAD^1 HEAD -- data/{{category}} | grep -c '\.json$' || true) + { + echo '' + echo "### $verdict" + echo + echo "Head \`${HEAD_SHA::7}\`, checked as merged into the base · ${changed:-0} changed record(s) · [run]($RUN_URL)" + echo + echo '| Step | Result |' + echo '| --- | --- |' + echo "| Validator, whole catalog | ${VALIDATE:-skipped} |" + echo "| Tests | ${TESTS:-skipped} |" + echo "| Site build (summary.json, history.json) | ${SITE:-skipped} |" + [ "$VALIDATE" = failure ] && { echo; echo '
Validator log'; echo; fence "$RUNNER_TEMP/validate.log" 30; echo '
'; } + [ "$TESTS" = failure ] && { echo; echo '
Test log'; echo; fence "$RUNNER_TEMP/tests.log" 30; echo '
'; } + [ "$SITE" = failure ] && { echo; echo '
Site build log'; echo; fence "$RUNNER_TEMP/site.log" 20; echo '
'; } + true + } > "$RUNNER_TEMP/body.md" + cat "$RUNNER_TEMP/body.md" >> "$GITHUB_STEP_SUMMARY" + delim="BODY_$(openssl rand -hex 16)" + { echo "body<<$delim"; cat "$RUNNER_TEMP/body.md"; echo "$delim"; } >> "$GITHUB_OUTPUT" + [ "$verdict" = '✅ Final check passed' ] + + comment: + needs: check + if: >- + always() && github.event.label.name == 'final-check' && + (needs.check.result == 'success' || needs.check.result == 'failure') && + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - name: Check the token + id: token + env: + TOKEN: ${{ secrets.TECHENGINEBOT_TOKEN }} + run: | + if [ -z "$TOKEN" ]; then + echo "::warning::TECHENGINEBOT_TOKEN is not set; the verdict is in the check job's summary only." + echo "ok=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + case "$TOKEN" in + github_pat_*) echo "ok=true" >> "$GITHUB_OUTPUT" ;; + *) echo "::error::TECHENGINEBOT_TOKEN must be a fine-grained token (github_pat_...) of TechEngineBot with Pull requests: read and write." + exit 1 ;; + esac + + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + if: steps.token.outputs.ok == 'true' + env: + BODY: ${{ needs.check.outputs.body }} + PR: ${{ github.event.pull_request.number }} + with: + github-token: ${{ secrets.TECHENGINEBOT_TOKEN }} + script: | + const marker = ''; + // The check job ran the PR's code, so its report is untrusted here: + // drop every @ and break issue references and issue/PR links, so the + // comment can neither mention anyone nor cross-reference anything. + const zw = String.fromCharCode(0x200b); // zero-width space + const body = (process.env.BODY || '') + .replace(/@/g, '') + .replace(/(? c.user?.login === me.login && c.body?.startsWith(marker)); + if (!mine) { + const { data } = await github.rest.issues.createComment({ owner, repo, issue_number, body }); + core.info(`Posted ${data.html_url}`); + } else if (mine.body !== body) { + await github.rest.issues.updateComment({ owner, repo, comment_id: mine.id, body }); + core.info(`Updated ${mine.html_url}`); + } else { + core.info(`Unchanged: ${mine.html_url}`); + } diff --git a/tests/test_new_satellite.py b/tests/test_new_satellite.py index a705484..161aabe 100644 --- a/tests/test_new_satellite.py +++ b/tests/test_new_satellite.py @@ -60,9 +60,29 @@ def test_unknown_placeholder_is_an_error(): render("{{nope}}", {}) -def test_generated_workflows_use_only_the_builtin_token(generated: Path): +# TechEngineBot's token is allowed in one place: the label-triggered final check. +BOT_SECRET = {"final-check.yml": {"TECHENGINEBOT_TOKEN"}} + + +def test_generated_workflows_use_the_bot_token_only_in_final_check(generated: Path): from machine.health import foreign_secrets workflows = sorted((generated / ".github" / "workflows").glob("*.yml")) assert workflows for path in workflows: - assert foreign_secrets(path.read_text(encoding="utf-8")) == [], path + text = path.read_text(encoding="utf-8") + assert foreign_secrets(text, BOT_SECRET.get(path.name, set())) == [], path + final_check = (generated / ".github" / "workflows" / "final-check.yml").read_text(encoding="utf-8") + assert foreign_secrets(final_check) == ["TECHENGINEBOT_TOKEN"] + + +def test_generated_final_check_is_named_after_the_repo(generated: Path): + text = (generated / ".github" / "workflows" / "final-check.yml").read_text(encoding="utf-8") + assert text.count("") == 2 + assert "data/demo " in text + + +def test_checklist_registers_the_repo_with_the_bot_secret(): + from machine.new_satellite import checklist + text = checklist({"repo": "demo-catalog", "plural": "demos"}) + assert '"allowed_secrets": ["TECHENGINEBOT_TOKEN"]' in text + assert "final-check" in text From c5fbcf8148fb2382c7e32bf17ae565f7fe069ca0 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 12 Nov 2026 02:46:36 +0900 Subject: [PATCH 3/9] feat(template): add CODEOWNERS and a pull request template New satellites start with the maintainers team as code owners and a short PR template: title convention, develop as the target, validator and tests, cited sources, and the sidebar filled in by hand. Issue templates are left to the organisation's shared ones. The template validator's docstring no longer quotes game-catalog's record count. --- machine/satellite_template/.github/CODEOWNERS | 2 ++ .../.github/pull_request_template.md | 12 ++++++++++++ machine/satellite_template/app/validate.py | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 machine/satellite_template/.github/CODEOWNERS create mode 100644 machine/satellite_template/.github/pull_request_template.md diff --git a/machine/satellite_template/.github/CODEOWNERS b/machine/satellite_template/.github/CODEOWNERS new file mode 100644 index 0000000..22c2c89 --- /dev/null +++ b/machine/satellite_template/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Default owners for everything in this repository. +* @GetTechAPI/maintainers diff --git a/machine/satellite_template/.github/pull_request_template.md b/machine/satellite_template/.github/pull_request_template.md new file mode 100644 index 0000000..2424c07 --- /dev/null +++ b/machine/satellite_template/.github/pull_request_template.md @@ -0,0 +1,12 @@ + + +## What & why + + + +## Checklist + +- [ ] Targets `develop` +- [ ] `python -m app.validate` and `python -m pytest -q` pass locally +- [ ] Every new or changed record cites at least one public reference in `source_urls` +- [ ] Sidebar filled in by hand: assignees, labels, milestone, projects diff --git a/machine/satellite_template/app/validate.py b/machine/satellite_template/app/validate.py index a8bb295..f580949 100644 --- a/machine/satellite_template/app/validate.py +++ b/machine/satellite_template/app/validate.py @@ -2,7 +2,7 @@ Generated by TechMachine's satellite template. Records are checked one at a time rather than loaded into a list first, so the check scales to catalogs of -any size (game-catalog validates ~1M records in about 90 seconds on CI). +any size. Run with: python -m app.validate """ From 365d273e5337dfef2a61ab041e7e604c0161e444 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 12 Nov 2026 02:47:05 +0900 Subject: [PATCH 4/9] test: check pinned actions in the template's workflows too The pinned-action rule was a grep inside final-check.yml that only saw this repository's own workflows, so the template could generate tag-pinned ones unnoticed. It is now a test over both directories, and the final check runs that test. CI installs a fixed pytest instead of the newest. --- .github/workflows/final-check.yml | 14 ++++---- .github/workflows/test.yml | 2 +- tests/test_pinned_actions.py | 54 +++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 tests/test_pinned_actions.py diff --git a/.github/workflows/final-check.yml b/.github/workflows/final-check.yml index 3b972ca..2b9fb03 100644 --- a/.github/workflows/final-check.yml +++ b/.github/workflows/final-check.yml @@ -45,7 +45,7 @@ jobs: continue-on-error: true run: | status=0 - python -m pip install --disable-pip-version-check pytest > "$RUNNER_TEMP/install.log" 2>&1 || status=$? + python -m pip install --disable-pip-version-check 'pytest==8.4.2' > "$RUNNER_TEMP/install.log" 2>&1 || status=$? tail -n 20 "$RUNNER_TEMP/install.log" exit "$status" @@ -91,20 +91,18 @@ jobs: cat "$RUNNER_TEMP/config.log" exit "$status" - # The repository only allows GitHub-owned actions pinned by full SHA. + # The organisation only runs GitHub-owned actions pinned by full SHA. The + # rule lives in tests/test_pinned_actions.py, which also covers the + # workflows the satellite template generates. - name: Syntax and pinned actions id: lint + if: steps.install.outcome == 'success' continue-on-error: true run: | status=0 { python -m compileall -q machine tests || status=1 - unpinned=$(grep -nE '^\s*-?\s*uses:' .github/workflows/*.yml | grep -vE 'uses: actions/[a-z-]+@[0-9a-f]{40}( |$)' || true) - if [ -n "$unpinned" ]; then - echo "Not a GitHub-owned action pinned by SHA:" - echo "$unpinned" - status=1 - fi + python -m pytest -q -p no:cacheprovider tests/test_pinned_actions.py || status=1 } > "$RUNNER_TEMP/lint.log" 2>&1 cat "$RUNNER_TEMP/lint.log" exit "$status" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d37a388..96dfb3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,5 +18,5 @@ jobs: with: python-version: "3.12" - run: | - pip install pytest + pip install 'pytest==8.4.2' python -m pytest -q diff --git a/tests/test_pinned_actions.py b/tests/test_pinned_actions.py new file mode 100644 index 0000000..4c00048 --- /dev/null +++ b/tests/test_pinned_actions.py @@ -0,0 +1,54 @@ +"""Every action is GitHub-owned and pinned by full commit SHA. + +The organisation only runs such actions, in this repository's workflows and +in the ones the satellite template generates. A tag (`@v7`) can be moved to +other code; a SHA cannot. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +WORKFLOW_DIRS = ( + ROOT / ".github" / "workflows", + ROOT / "machine" / "satellite_template" / ".github" / "workflows", +) +USES = re.compile(r"^\s*(?:-\s*)?uses:\s*(\S+)", re.MULTILINE) +PINNED = re.compile(r"actions/[a-z0-9-]+@[0-9a-f]{40}\Z") + + +def unpinned(workflow_text: str) -> list[str]: + """The `uses:` references that are not `actions/@<40-hex-sha>`.""" + return [ref.strip("'\"") for ref in USES.findall(workflow_text) + if not PINNED.match(ref.strip("'\""))] + + +def workflow_files() -> list[Path]: + return sorted(path for folder in WORKFLOW_DIRS + for pattern in ("*.yml", "*.yaml") for path in folder.glob(pattern)) + + +def test_both_workflow_directories_are_scanned(): + folders = {path.parent for path in workflow_files()} + assert folders == set(WORKFLOW_DIRS) + + +@pytest.mark.parametrize("path", workflow_files(), ids=lambda p: p.relative_to(ROOT).as_posix()) +def test_actions_are_github_owned_and_pinned_by_sha(path: Path): + text = path.read_text(encoding="utf-8") + assert USES.search(text), f"{path} uses no action; is the pattern still right?" + assert unpinned(text) == [] + + +def test_the_rule_rejects_tags_branches_and_other_owners(): + sha = "3d3c42e5aac5ba805825da76410c181273ba90b1" + assert unpinned(f" - uses: actions/checkout@{sha} # v7.0.1\n") == [] + assert unpinned(f" uses: actions/deploy-pages@{sha}\n") == [] + assert unpinned(" - uses: actions/checkout@v7\n") == ["actions/checkout@v7"] + assert unpinned(" - uses: actions/checkout@main\n") == ["actions/checkout@main"] + assert unpinned(f" - uses: someone/checkout@{sha}\n") == [f"someone/checkout@{sha}"] + assert unpinned(f" - uses: actions/checkout@{sha[:12]}\n") == [f"actions/checkout@{sha[:12]}"] From 2b26db0e19193ea57c5249bf605b8f0223cce669 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 12 Nov 2026 02:47:16 +0900 Subject: [PATCH 5/9] feat(health): check the org profile repo and TechMachine itself The .github repo has no workflows and no endpoint; the check lists its workflows, finds none and reports nothing, which a test now pins down. TechMachine was the one repository with workflows that nothing watched. --- machine/repos.json | 16 ++++++++++++++++ tests/test_health.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/machine/repos.json b/machine/repos.json index 0852fa7..cce135d 100644 --- a/machine/repos.json +++ b/machine/repos.json @@ -38,6 +38,22 @@ "allowed_secrets": [ "TECHENGINEBOT_TOKEN" ] + }, + { + "name": "GetTechAPI/.github", + "branches": [ + "main" + ] + }, + { + "name": "GetTechAPI/TechMachine", + "branches": [ + "develop", + "main" + ], + "allowed_secrets": [ + "TECHENGINEBOT_TOKEN" + ] } ], "endpoints": [ diff --git a/tests/test_health.py b/tests/test_health.py index d162ffd..42e3de7 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -83,6 +83,42 @@ def test_other_workflows_are_never_leftovers(): assert not leftover_workflow(ci, has_dependabot_config=False) +def test_a_repo_without_workflows_or_endpoints_is_clean(monkeypatch): + # The org profile repo (.github) holds community files only: no workflows, + # no Pages. One listing call, nothing else, and no finding. + calls = [] + + def get(url: str, token: str | None) -> dict: + calls.append(url) + return {"total_count": 0, "workflows": []} + monkeypatch.setattr(health, "_get", get) + config = {"repos": [{"name": "org/.github", "branches": ["main"]}], "endpoints": []} + assert health.check(config, None) == [] + assert calls == [f"{health.API}/repos/org/.github/actions/workflows?per_page=100"] + assert "Checked .github and 0 public endpoints" in render([], config) + + +def test_a_repo_that_cannot_be_listed_is_reported(monkeypatch): + def missing(url: str, token: str | None) -> dict: + raise urllib.error.HTTPError(url, 404, "Not Found", None, None) + monkeypatch.setattr(health, "_get", missing) + config = {"repos": [{"name": "org/gone", "branches": ["main"]}], "endpoints": []} + findings = health.check(config, None) + assert [f.where for f in findings] == ["org/gone"] + assert "could not list workflows" in findings[0].what + + +def test_repos_json_is_well_formed(): + import json + config = json.loads(health.CONFIG.read_text(encoding="utf-8")) + names = [repo["name"] for repo in config["repos"]] + assert len(names) == len(set(names)) + for repo in config["repos"]: + assert repo["name"].startswith("GetTechAPI/") and repo["branches"], repo + for endpoint in config["endpoints"]: + assert endpoint["url"].startswith("https://") and endpoint["expect"], endpoint + + def test_endpoint_without_expected_field_is_a_problem(): assert endpoint_finding("summary", "https://x", "count", {"other": 1}) assert endpoint_finding("summary", "https://x", "count", "404") From 6be95686318a81ef1c3821f094b4f843e77fdba1 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 12 Nov 2026 02:47:16 +0900 Subject: [PATCH 6/9] test(template): pin the list of files a new satellite starts with Generating a repository must produce exactly the expected tree, so adding or dropping a template file is a deliberate change to the test as well. --- tests/test_new_satellite.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_new_satellite.py b/tests/test_new_satellite.py index 161aabe..a45d972 100644 --- a/tests/test_new_satellite.py +++ b/tests/test_new_satellite.py @@ -22,6 +22,40 @@ def generated(tmp_path: Path) -> Path: return out +# Everything a new satellite starts with. A template change shows up here, so +# adding, renaming or dropping a generated file is a deliberate edit. +EXPECTED_FILES = [ + ".github/CODEOWNERS", + ".github/pull_request_template.md", + ".github/workflows/deploy-pages.yml", + ".github/workflows/final-check.yml", + ".github/workflows/validate-data.yml", + ".gitignore", + "DATA_LICENSE.md", + "LICENSE", + "README.md", + "app/__init__.py", + "app/validate.py", + "pyproject.toml", + "site/.nojekyll", + "site/build.py", + "site/index.html", + "tests/test_validate.py", +] + + +def test_generated_tree_is_the_expected_file_list(generated: Path): + files = sorted(p.relative_to(generated).as_posix() for p in generated.rglob("*") if p.is_file()) + assert files == sorted(EXPECTED_FILES) + assert (generated / "data" / "demo").is_dir() + + +def test_generated_metadata_is_filled_in(generated: Path): + assert "* @GetTechAPI/maintainers" in (generated / ".github" / "CODEOWNERS").read_text(encoding="utf-8") + template = (generated / ".github" / "pull_request_template.md").read_text(encoding="utf-8") + assert "data(demo):" in template and "{{" not in template + + def test_no_placeholder_survives(generated: Path): for path in generated.rglob("*"): if path.is_file() and path.suffix in {".py", ".md", ".toml", ".yml", ".html"}: From ca165c8e0247dbb3a4befc7b07a14a663a64f640 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 12 Nov 2026 02:47:54 +0900 Subject: [PATCH 7/9] docs(readme): describe the health rules, the scaffolded files and the org's repositories The README now lists what the health check actually inspects, the files the scaffolder writes and the steps for creating a satellite, with a short table of the organisation's repositories. --- README.md | 158 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 114 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 42fc9a9..729e157 100644 --- a/README.md +++ b/README.md @@ -2,52 +2,75 @@ Repository management for the GetTechAPI organisation. Where [TechEngine](https://github.com/GetTechAPI/TechEngine) processes the data, -TechMachine looks after the repositories that hold it. +TechMachine looks after the repositories that hold it. It does two things: +a health check over every repository, and a scaffolder for new satellite +data repositories. -## Health check - -`machine/health.py` runs when a person starts it — the `health` workflow's -*Run workflow* button, or locally. For every repository in [`machine/repos.json`](machine/repos.json) -it reads the newest completed run of each active workflow on each tracked -branch, and it fetches the public endpoints the TechAPI homepage depends on. +## The organisation's repositories -Anything other than `success`, `skipped` or `neutral` is reported — -`cancelled` and `timed_out` included. That is deliberate: a job killed at -GitHub's six-hour limit reports itself as *cancelled*, and several automations -here failed that way for days without anyone noticing. +| Repository | What it is | +| --- | --- | +| [TechAPI](https://github.com/GetTechAPI/TechAPI) | The dataset and the site | +| [TechEngine](https://github.com/GetTechAPI/TechEngine) | The engine that processes the data; TechAPI pins it as a submodule | +| [game-catalog](https://github.com/GetTechAPI/game-catalog) | Satellite data repository: games | +| [cpu-engineering-samples](https://github.com/GetTechAPI/cpu-engineering-samples) | Satellite data repository: CPU engineering samples | +| [.github](https://github.com/GetTechAPI/.github) | The organisation profile and shared community files | +| TechMachine | This repository: health check and satellite scaffolder | -It queries each workflow separately rather than scanning "the last N runs", -because weekly jobs scroll out of any fixed window behind the daily ones — and -the weekly jobs are the ones that go quiet. +## Health check -It also reads each active workflow file and reports any that uses a secret -other than `GITHUB_TOKEN`. Automation here must act as `github-actions[bot]`, -never as a person: workflows running under a personal token are how a -maintainer's account got suspended. Disabling the workflow in the Actions -settings clears the finding; a non-token secret a repository genuinely needs -goes in that repository's `allowed_secrets` in `repos.json` — TechAPI's -label-triggered final check is the one place TechEngineBot's token is allowed. +`machine/health.py` runs when a person starts it: the `health` workflow's +*Run workflow* button, or locally. It reads [`machine/repos.json`](machine/repos.json) +and reports three kinds of problem. + +1. **The latest completed run of each active workflow, per tracked branch.** + Anything other than `success`, `skipped` or `neutral` is reported, + `cancelled` and `timed_out` included. That is deliberate: a job killed at + GitHub's six-hour limit reports itself as *cancelled*, and several + automations here failed that way for days without anyone noticing. Each + workflow is queried separately rather than scanning "the last N runs", + because weekly jobs scroll out of any fixed window behind the daily ones, + and the weekly jobs are the ones that go quiet. Disabled workflows are + skipped, and so is GitHub's own *Dependabot Updates* workflow in a + repository that no longer has a `.github/dependabot.yml` (or `.yaml`). +2. **Secrets.** It reads each active workflow file on each tracked branch and + reports any that uses a secret other than `GITHUB_TOKEN` or the + repository's `allowed_secrets`. Automation here must act as + `github-actions[bot]`, never as a person: workflows running under a + personal token are how a maintainer's account got suspended. The one + allowed secret is `TECHENGINEBOT_TOKEN`, which the label-triggered final + check uses to post its comment. Disabling a workflow in the Actions + settings clears its finding. +3. **Endpoints.** It fetches each public URL listed under `endpoints` and + reports one that is unreachable, is not JSON, or lacks its `expect` field. + These are the files the TechAPI homepage depends on. + +A repository that cannot be read is reported too, rather than passed over. It only reads. The report goes to stdout and the run's step summary; it -writes nothing to GitHub, mentions nobody, and runs on no schedule — an +writes nothing to GitHub, mentions nobody, and runs on no schedule. An account here was suspended over automation, and a report nobody asked for is -how that starts. +how that starts. The run itself always ends green: the report is the output. ```bash python -m machine.health # print the report (GITHUB_TOKEN raises the rate limit) python -m pytest -q ``` -Adding a repository or endpoint is one entry in `machine/repos.json`. +Adding a repository or endpoint is one entry in `machine/repos.json`: a +`name`, the `branches` to watch and, where the final check runs, +`"allowed_secrets": ["TECHENGINEBOT_TOKEN"]`. A repository with no workflows +(the `.github` profile repository) is listed with its branch only. ## Final check Adding the `final-check` label to a pull request asks -[TechEngineBot](https://github.com/TechEngineBot) for a last look, in place of -Copilot. The check job runs the tests (including a generated satellite -repository), validates `repos.json`, and checks syntax and that every action -is GitHub-owned and pinned by SHA. TechEngineBot then leaves **one** comment -with the verdict; adding the label again edits it. +[TechEngineBot](https://github.com/TechEngineBot) for a last look. The check +job runs the tests (including a generated satellite repository), validates +`repos.json`, and checks syntax and that every action is GitHub-owned and +pinned by full commit SHA, both in this repository's workflows and in the +ones the template generates (`tests/test_pinned_actions.py`). TechEngineBot +then leaves **one** comment with the verdict; adding the label again edits it. The check job runs the PR's code with no secrets and a read-only token. Only the comment job sees `TECHENGINEBOT_TOKEN`, and it posts nothing unless that is @@ -57,27 +80,74 @@ as TechAPI's final check. ## Satellite repositories -Categories that do not belong in TechAPI live in their own repository (games: -[game-catalog](https://github.com/GetTechAPI/game-catalog)). The split -criterion is identity, not size — software and websites are tech data and -stay in TechAPI. +Categories that do not belong in TechAPI live in their own repository +([game-catalog](https://github.com/GetTechAPI/game-catalog), +[cpu-engineering-samples](https://github.com/GetTechAPI/cpu-engineering-samples)). +The split criterion is identity, not size: software and websites are tech +data and stay in TechAPI. -`machine/new_satellite.py` writes a new one from the layout game-catalog -proved out: a streaming validator, a site build that publishes -`summary.json` + `history.json` (never a listing of every record), CI, and -licences. +`machine/new_satellite.py` writes a new one from +[`machine/satellite_template/`](machine/satellite_template): ```bash -python -m machine.new_satellite --repo game-catalog --category game --title "Game catalog" --plural games --date-field release_date --range rating:0:5 --range metacritic:0:100 --out ../game-catalog +python -m machine.new_satellite --repo game-catalog --category game \ + --title "Game catalog" --plural games --date-field release_date \ + --range rating:0:5 --range metacritic:0:100 --out ../game-catalog ``` -It only writes files. Creating the repository changes the organisation, so -that is left to a person; the remaining steps are printed at the end, -including adding `main` to the Pages environment's deployment branches — -without it every deploy fails and leaves no log. +### What it generates + +``` +.github/CODEOWNERS # * @GetTechAPI/maintainers +.github/pull_request_template.md +.github/workflows/validate-data.yml # validator + tests on PRs and pushes to develop/main +.github/workflows/deploy-pages.yml # builds and publishes the site from main +.github/workflows/final-check.yml # the label-triggered check and TechEngineBot's comment +.gitignore +DATA_LICENSE.md # CC BY-SA 4.0 for the records +LICENSE # MIT for the code +README.md +app/__init__.py +app/validate.py # required fields, slug, dates, ranges, duplicates +pyproject.toml +site/.nojekyll +site/build.py # summary.json + history.json, never a listing of every record +site/index.html +tests/test_validate.py +data// # empty; records go in /.json +``` -The tests generate a repository and run *its* test suite and validator, so a -template change that breaks generated repos fails here. +The workflows follow the organisation's rules: GitHub-owned actions pinned +by full commit SHA, least-privilege `permissions:`, no schedules, nothing +that opens pull requests or issues, and no secret except +`TECHENGINEBOT_TOKEN` in the final check's comment job. There are no issue +templates; those are shared from the `.github` repository. + +The tests generate a repository, compare its tree with the list above, and +run *its* test suite and validator, so a template change that breaks or +reshapes generated repositories fails here. + +### Creating a satellite + +The scaffolder only writes files. Everything else changes the organisation, +so it is left to a person; the same list is printed after generating. + +1. Generate the files with the command above. +2. Create the repository: `gh repo create GetTechAPI/ --public`. +3. Push the scaffold to `develop`, then `develop` to `main`. +4. Turn Pages on with the workflow as its source: + `gh api -X POST repos/GetTechAPI//pages -f build_type=workflow`. +5. Add `main` to the Pages environment's deployment branches: + `gh api -X POST repos/GetTechAPI//environments/github-pages/deployment-branch-policies -f name=main`. + Without it every deploy fails and leaves no log. +6. Add the repository to `machine/repos.json` here, with + `"allowed_secrets": ["TECHENGINEBOT_TOKEN"]`, and its `summary.json` URL + to `endpoints`. +7. Add it to `SATELLITES` in TechAPI's `site/src/scripts/techapi.js` so the + homepage counts it. +8. For the final check, create the `final-check` label and set the + `TECHENGINEBOT_TOKEN` secret to a fine-grained token of TechEngineBot with + *Pull requests: read and write* on that repository only. ## Branching From b2d6548352cfd7edf47d99ee889d7175c8d6c779 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 12 Nov 2026 02:48:04 +0900 Subject: [PATCH 8/9] chore: add .editorconfig and .gitattributes Every tracked text file is already LF; this keeps it that way on Windows. --- .editorconfig | 15 +++++++++++++++ .gitattributes | 1 + 2 files changed, 16 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitattributes diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ab0905a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.{yml,yaml,json,toml,html}] +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf From f2231fa737e17c0039ca39834b2ce247cc392945 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 12 Nov 2026 02:48:52 +0900 Subject: [PATCH 9/9] fix(template): name an owner that exists in CODEOWNERS The organisation has no teams, so a team handle would own nothing. --- README.md | 2 +- machine/satellite_template/.github/CODEOWNERS | 2 +- tests/test_new_satellite.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 729e157..701d541 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ python -m machine.new_satellite --repo game-catalog --category game \ ### What it generates ``` -.github/CODEOWNERS # * @GetTechAPI/maintainers +.github/CODEOWNERS # * @Seungpyo1007 .github/pull_request_template.md .github/workflows/validate-data.yml # validator + tests on PRs and pushes to develop/main .github/workflows/deploy-pages.yml # builds and publishes the site from main diff --git a/machine/satellite_template/.github/CODEOWNERS b/machine/satellite_template/.github/CODEOWNERS index 22c2c89..e36cfc2 100644 --- a/machine/satellite_template/.github/CODEOWNERS +++ b/machine/satellite_template/.github/CODEOWNERS @@ -1,2 +1,2 @@ # Default owners for everything in this repository. -* @GetTechAPI/maintainers +* @Seungpyo1007 diff --git a/tests/test_new_satellite.py b/tests/test_new_satellite.py index a45d972..e41cb16 100644 --- a/tests/test_new_satellite.py +++ b/tests/test_new_satellite.py @@ -51,7 +51,7 @@ def test_generated_tree_is_the_expected_file_list(generated: Path): def test_generated_metadata_is_filled_in(generated: Path): - assert "* @GetTechAPI/maintainers" in (generated / ".github" / "CODEOWNERS").read_text(encoding="utf-8") + assert "* @Seungpyo1007" in (generated / ".github" / "CODEOWNERS").read_text(encoding="utf-8") template = (generated / ".github" / "pull_request_template.md").read_text(encoding="utf-8") assert "data(demo):" in template and "{{" not in template