From 5b5c6c70d246aac542e4c73793d1b420307e0b64 Mon Sep 17 00:00:00 2001 From: 0XSIKIPON Date: Thu, 17 Sep 2026 02:23:22 +0100 Subject: [PATCH 1/2] feat(audit): add repo scaffolding detection to skip non-plugin files in audits --- e2e/Makefile.devkit | 8 ++++- src/a0_plugin_testkit/real/a0_api.py | 4 +++ src/a0_plugin_testkit/real/deps.py | 4 +++ src/a0_plugin_testkit/real/validator.py | 12 +++++-- src/a0_plugin_testkit/scan_scope.py | 42 +++++++++++++++++++++++++ 5 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/a0_plugin_testkit/scan_scope.py diff --git a/e2e/Makefile.devkit b/e2e/Makefile.devkit index 36d3d91..6873e4d 100644 --- a/e2e/Makefile.devkit +++ b/e2e/Makefile.devkit @@ -81,10 +81,16 @@ link-gemini: ## copy the .gemini AI-reviewer styleguide into this repo (DEC-066) build: ## assemble the plugin dir (no-op for source-only plugins) @echo "build: nothing to assemble for '$(PLUGIN_NAME)'" +# Mirrors the reusable plugin-e2e workflow's zip excludes so root-layout +# plugins (.devkit.yml plugin_dir: .) don't pack .git, the _testkit submodule, +# caller workflows, etc. into the zip. No-ops for subdir layouts. The last +# three are local-only build output that never exists on a CI runner. package: ## produce the versioned plugin zip from $(PLUGIN_DIR) @mkdir -p $(DIST_DIR) @rm -f $(PLUGIN_ZIP) - cd $(PLUGIN_DIR) && zip -qr $(abspath $(PLUGIN_ZIP)) . -x '*.DS_Store' 'tests/*' '.github/*' '*/node_modules/*' + cd $(PLUGIN_DIR) && zip -qr $(abspath $(PLUGIN_ZIP)) . \ + -x '*.DS_Store' '.git/*' 'tests/*' '.github/*' '.skills/*' 'dist/*' '*/node_modules/*' \ + '.devkit.yml' '.gitmodules' '.gitignore' '.claude/*' '.gemini/*' 'artifacts/*' @echo "packaged → $(PLUGIN_ZIP)" e2e: package ## behaviour BDD if tests/e2e/features/ exists, else classic lifecycle — in the devcontainer diff --git a/src/a0_plugin_testkit/real/a0_api.py b/src/a0_plugin_testkit/real/a0_api.py index 8a2a4b8..b83f8c2 100644 --- a/src/a0_plugin_testkit/real/a0_api.py +++ b/src/a0_plugin_testkit/real/a0_api.py @@ -21,6 +21,8 @@ from functools import lru_cache from pathlib import Path +from ..scan_scope import is_repo_scaffolding + # A0-internal top-level modules whose attribute access we'll track. Anything # else (stdlib, pip packages) is out of scope for this check. _A0_INTERNAL_TOPLEVEL = frozenset({ @@ -212,6 +214,8 @@ def audit_a0_api_usage( audit = A0ApiAudit(plugin_dir=plugin_dir, a0_root=a0_root) for py in plugin_dir.rglob("*.py"): + if is_repo_scaffolding(py.relative_to(plugin_dir).as_posix()): + continue try: tree = ast.parse(py.read_text(encoding="utf-8", errors="ignore")) except SyntaxError: diff --git a/src/a0_plugin_testkit/real/deps.py b/src/a0_plugin_testkit/real/deps.py index be14f68..e37fa19 100644 --- a/src/a0_plugin_testkit/real/deps.py +++ b/src/a0_plugin_testkit/real/deps.py @@ -29,6 +29,8 @@ from dataclasses import dataclass from pathlib import Path +from ..scan_scope import is_repo_scaffolding + # --------------------------------------------------------------------------- # # Knowledge base # --------------------------------------------------------------------------- # @@ -249,6 +251,8 @@ def audit_dependencies( undeclared: list[_ImportSite] = [] for py in plugin_dir.rglob("*.py"): rel = py.relative_to(plugin_dir).as_posix() + if is_repo_scaffolding(rel): + continue for site in _extract_top_level_imports(py): mod = _norm(site.module) if mod in _STDLIB or mod in _A0_INTERNAL: diff --git a/src/a0_plugin_testkit/real/validator.py b/src/a0_plugin_testkit/real/validator.py index 1997f13..f121e65 100644 --- a/src/a0_plugin_testkit/real/validator.py +++ b/src/a0_plugin_testkit/real/validator.py @@ -23,6 +23,7 @@ discover_html_surfaces, discover_js_hooks, ) +from ..scan_scope import is_repo_scaffolding _NAME_RE = re.compile(r"^[a-z0-9_]+$") @@ -147,7 +148,11 @@ def _check_manifest(plugin_dir: Path) -> list[ValidationFinding]: message=f"plugin name {name!r} must match {_NAME_RE.pattern}", path="plugin.yaml", )) - if name != plugin_dir.name: + # Root-layout plugins (.devkit.yml plugin_dir: .) ARE the repo, so the + # audited dir carries the repo name, not the plugin name. At runtime + # A0 always installs into usr/plugins/, so the invariant still + # holds where it matters — only enforce it for subdir layouts. + if name != plugin_dir.name and not (plugin_dir / ".devkit.yml").is_file(): findings.append(ValidationFinding( check="manifest", severity="error", @@ -218,12 +223,15 @@ def _check_structure(plugin_dir: Path) -> list[ValidationFinding]: # Catch __pycache__ accidentally zipped in. pycache_dirs = list(plugin_dir.rglob("__pycache__")) for pd in pycache_dirs: + rel = pd.relative_to(plugin_dir).as_posix() + if is_repo_scaffolding(rel): + continue findings.append(ValidationFinding( check="structure", severity="warning", code="structure.pycache", message="__pycache__ directory present — should be gitignored / zip-excluded", - path=str(pd.relative_to(plugin_dir)), + path=rel, )) # config.html must coexist with settings_sections (warn, don't error). diff --git a/src/a0_plugin_testkit/scan_scope.py b/src/a0_plugin_testkit/scan_scope.py new file mode 100644 index 0000000..81937b1 --- /dev/null +++ b/src/a0_plugin_testkit/scan_scope.py @@ -0,0 +1,42 @@ +"""Scan scope for whole-tree audits — plugin code vs repo scaffolding. + +Root-layout plugins (``.devkit.yml`` ``plugin_dir: .``) keep their dev +scaffolding INSIDE the audited tree: the test suite + the vendored ``_testkit`` +submodule, caller workflows, local agent dirs, build output. None of that is +shipped plugin code — the reusable plugin-e2e workflow's zip step already +excludes the same set when packaging a root-layout plugin — so the whole-tree +audits (deps, A0-API usage, validator structure checks) skip it too. + +No-ops for subdir layouts (``plugin_dir: usr/plugins/``): none of these +folders sit inside a nested plugin dir, so those audits behave exactly as +before. +""" +from __future__ import annotations + +# First path segment (relative to the plugin dir) that marks repo scaffolding. +# Mirrors the zip excludes of the reusable plugin-e2e workflow, plus +# local-only build output that never exists on a CI runner. +_REPO_SCAFFOLD_DIRS = frozenset({ + "tests", # plugin's own suite + the vendored _testkit submodule + ".github", # caller workflows + ".skills", # devkit-linked skills + ".agent-zero", # vendored A0 source (inside _testkit) + ".git", + ".devkit.yml", # standardization wrapper (root layout only) + ".gitmodules", + ".gitignore", + "dist", # packaged zips + "artifacts", # e2e artifacts + ".claude", # local agent workspace + ".gemini", # local agent styleguide +}) + + +def is_repo_scaffolding(rel_posix: str) -> bool: + """True if a plugin-dir-relative POSIX path belongs to repo scaffolding. + + Only the FIRST segment is matched, so e.g. ``tests/_testkit/src/x.py`` + and ``dist/foo.zip`` are scaffolding, while ``extensions/python/x.py`` + and a root-level ``hooks.py`` are not. + """ + return rel_posix.split("/", 1)[0] in _REPO_SCAFFOLD_DIRS From 6b196003a94c0b32b07470f65342e2745a5cab0d Mon Sep 17 00:00:00 2001 From: 0XSIKIPON Date: Thu, 17 Sep 2026 02:48:15 +0100 Subject: [PATCH 2/2] feat(audit): add self-tests for root-layout plugin repo audits to skip scaffolding --- CHANGELOG.md | 22 +++++-- SPEC.md | 19 +++++++ tests/test_scan_scope.py | 120 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 tests/test_scan_scope.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 41f2c9a..6e4e8a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,18 @@ the frozen Make target contract (SPEC Appendix E.1), the reusable workflow input `Makefile.devkit` / `.devkit.yml` interface, or a tightening of the enforcement gates. **MINOR** = new backward-compatible targets/checks/assets. **PATCH** = fixes that don't change the contract. +## v2.2.0 — unreleased + +- **New: root-layout plugin repos are fully supported** (DEC-083). A repo with `.devkit.yml +`plugin_dir: .`— the layout hub installs require, since A0 git-clones the repo into`usr/plugins/` as-is — now passes the whole-tree audits: the dependency sweep, the A0-API +sweep, and the validator skip repo scaffolding (`tests/`+ the vendored`_testkit`, caller +workflows, local agent dirs, build output), mirroring the reusable workflow's zip excludes so +"audited" and "shipped" cover the same files. `manifest.name.dir_mismatch`is enforced only for +subdir layouts (runtime installs always land in`usr/plugins/`). No-op for subdir layouts. +- `make package` now uses the same exclude list as the reusable workflow's zip step, so a local + root-layout zip doesn't pack `.git`, the `_testkit` submodule, or caller workflows. No-op for + subdir layouts. + ## v2.1.6 — 2026-08-05 - **New: `make docs`** (DEC-077). Builds `docs/BEHAVIOUR.md` + `docs/screenshots/` from the plugin's @@ -52,7 +64,7 @@ the frozen Make target contract (SPEC Appendix E.1), the reusable workflow input harnesses, `e2e/ci/prune-artifacts.sh` invoked by both workflows. - **Fix: e2e runs uploaded a near-empty artifact.** Two independent defects, both of which made a - *green* run undiagnosable. (1) `trace: retain-on-failure` meant a passing run produced no trace at + _green_ run undiagnosable. (1) `trace: retain-on-failure` meant a passing run produced no trace at all, so there was no ground truth to diff a later regression against — traces are now **always captured** (DEC-073), still overridable via `BDD_TRACE`. (2) Both harnesses (`run-bdd.sh` **and** `run-lifecycle.sh`) copied only `trace.zip`, silently dropping the `.png`/`.webm` Playwright writes @@ -81,7 +93,7 @@ the frozen Make target contract (SPEC Appendix E.1), the reusable workflow input ## v2.1.2 — 2026-07-04 - **Skills: durable debugging/lifecycle learnings (from the chat-radar v2 adoption).** - - `a0-plugin-e2e-bdd` → "Debugging & harness truths": isolate what a symptom *proves* before fixing + - `a0-plugin-e2e-bdd` → "Debugging & harness truths": isolate what a symptom _proves_ before fixing (renders-blank-live = environment/global-collision; renders-in-pod-but-empty = data/seed, e.g. an `/import_chat` 404); an asserting test can encode the wrong spec intent; harness plumbing (features/ flips the harness + retires the seed hook, filesystem seed seam via `A0_CONTAINER`, fresh page + @@ -97,7 +109,7 @@ the frozen Make target contract (SPEC Appendix E.1), the reusable workflow input - **Skill (`a0-plugin-e2e-bdd`): fixture patterns for seamless UI plugins.** Captures the three patterns proven across share-chat / fullscreen-toggle / mermaid-diagrams / diff-visualizer / chat-comments — - pure-UI control, render-a-code-block (inject the node A0 emits; wait for the CDN renderer *before* + pure-UI control, render-a-code-block (inject the node A0 emits; wait for the CDN renderer _before_ injecting; assert the rendered output, not the transient `data-…-processed` marker; feed diff2html a full git diff), and store-driven (drive public store methods, assert observable effects) — plus the local-A0 boot gotcha (neutralize `run_sshd`; reinstall chromium after cache eviction). @@ -109,7 +121,7 @@ the frozen Make target contract (SPEC Appendix E.1), the reusable workflow input `npx playwright show-trace ` or trace.playwright.dev. Both harnesses (`run-bdd`, `run-lifecycle`) collect `trace.zip`; the flaky `ffmpeg`→GIF step is gone. Artifact renamed `e2e-recording-*` → `e2e-traces-*`. - **Capture scope:** failing scenarios by default (`retain-on-failure`). Run the `plugin-e2e` workflow via - **workflow_dispatch with `capture_all_traces: true`** to capture a trace for *every* scenario + **workflow_dispatch with `capture_all_traces: true`** to capture a trace for _every_ scenario (threads through to `BDD_TRACE=on`). Locally: `BDD_TRACE=on make e2e`. - Consumers re-run `make link-workflows` to pick up the new dispatch input (the default already flows via `@main`). @@ -120,7 +132,7 @@ the frozen Make target contract (SPEC Appendix E.1), the reusable workflow input (`plugin_dir: usr/plugins/`) where the lint previously looked in the wrong place. No effect on root-layout plugins. -## v2.0.0 — 2026-07-04 (BREAKING) +## v2.0.0 — 2026-07-04 (BREAKING) - **BDD behaviour tests are now mandatory** (DEC-069). `bdd_lint` **hard-fails a plugin with no `tests/e2e/features/`** — the previous "self-skip for non-BDD plugins" was a silent loophole (no tests diff --git a/SPEC.md b/SPEC.md index 25450b3..4411d0f 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1277,6 +1277,25 @@ report job asserts the matrix ran if the probe asked for it. The runs pin the resolved **digest**, not `:latest`, so a failure is reproducible against the exact image that broke it rather than whatever `:latest` points at by the time someone investigates. + +**DEC-083 — Root-layout plugin repos: the whole-tree audits scan shipped plugin code, not repo scaffolding.** +A root-layout repo (`.devkit.yml` `plugin_dir: .`) is its own plugin dir — required for hub installs, +which `git clone` the repo and drop it into `usr/plugins/` as-is — so its dev scaffolding sits +*inside* the tree the audits sweep: the test suite with the vendored `_testkit` submodule (and its +nested `.agent-zero`), caller workflows, local agent dirs, build output. Against such a tree the +dependency audit flags `pytest` and friends as undeclared, the A0-API audit cross-references A0's own +source and fails on it, and the validator demands dir-name == plugin-name against what is actually +the repo name. All three are false alarms about code that never ships. +The fix follows the packaging contract rather than inventing a second rule: the reusable workflow's +zip step already excludes exactly these trees for root-layout plugins, so "audited" and "shipped" +now cover the same files. One predicate (`a0_plugin_testkit.scan_scope.is_repo_scaffolding`, first +path segment match against the zip-exclude set) gates the dependency sweep, the A0-API sweep, and +the validator's `__pycache__` scan. `manifest.name.dir_mismatch` is enforced only when the plugin +dir has no `.devkit.yml` — at runtime A0 always installs into `usr/plugins/`, so the invariant +still holds where it matters. +No-op for subdir layouts: none of the scaffolding folders sit inside a nested plugin dir, so +previously-green consumers behave identically (MINOR, not MAJOR). + > **Numbering note:** DEC-070–072 were originally introduced as code comments only. They are now > written up above, so this section documents an unbroken 070–082. The lesson is worth keeping: a > decision that exists only as a code comment is invisible to anyone reading the SPEC, and all three diff --git a/tests/test_scan_scope.py b/tests/test_scan_scope.py new file mode 100644 index 0000000..4c02786 --- /dev/null +++ b/tests/test_scan_scope.py @@ -0,0 +1,120 @@ +"""Self-tests for DEC-083 — root-layout audits scan shipped plugin code only. + +Root-layout repos (``.devkit.yml`` ``plugin_dir: .``) keep their dev scaffolding +inside the audited tree; the whole-tree audits must skip it (mirroring the +reusable workflow's zip excludes) while still auditing everything that ships. +""" + +from __future__ import annotations + +from pathlib import Path + +from a0_plugin_testkit.real.deps import audit_dependencies +from a0_plugin_testkit.real.validator import static_validate +from a0_plugin_testkit.scan_scope import is_repo_scaffolding + + +# ----- predicate ------------------------------------------------------------ # + + +def test_scaffolding_trees_are_detected() -> None: + for rel in ( + "tests/conftest.py", + "tests/_testkit/src/a0_plugin_testkit/x.py", + ".github/workflows/plugin-e2e.yml", + ".claude/temp/notes.py", + ".gemini/styleguide.md", + "dist/plugin.zip", + "artifacts/trace.zip", + ".git/config", + ): + assert is_repo_scaffolding(rel), rel + + +def test_shipped_plugin_code_is_not_detected() -> None: + for rel in ( + "hooks.py", + "__init__.py", + "extensions/python/system_prompt/_10_nudge.py", + "api/handlers.py", + "skills/mermaid/SKILL.md", + "webui/thumbnail.png", + ): + assert not is_repo_scaffolding(rel), rel + + +def test_boundary_root_test_file_is_not_scaffolding() -> None: + # The predicate matches directories, not file names — a root-level + # ``test_hook.py`` is still audited. (The security check's separate + # ``test_`` heuristic is deliberately not duplicated here.) + assert not is_repo_scaffolding("test_hook.py") + + +# ----- dependency audit ----------------------------------------------------- # + + +def _fake_a0_root(tmp_path: Path) -> Path: + root = tmp_path / "fake-a0" + root.mkdir() + (root / "requirements.txt").write_text("", encoding="utf-8") + return root + + +def test_deps_audit_skips_scaffolding_but_not_plugin_code(tmp_path: Path) -> None: + plugin = tmp_path / "my_plugin" + (plugin / "tests").mkdir(parents=True) + (plugin / "tests" / "conftest.py").write_text("import pytest\n", encoding="utf-8") + (plugin / "hooks.py").write_text( + "import pandas\n", encoding="utf-8" # undeclared on purpose + ) + audit = audit_dependencies(plugin, a0_root=_fake_a0_root(tmp_path)) + + audited = {site.path for site in audit.third_party} + assert "tests/conftest.py" not in audited, "scaffolding must be skipped" + assert "hooks.py" in audited, "shipped code must still be audited" + assert [site.module for site in audit.undeclared] == ["pandas"] + + +# ----- validator ------------------------------------------------------------- # + + +def _root_layout_repo(tmp_path: Path) -> Path: + repo = tmp_path / "my-plugin-repo" # repo name ≠ plugin name (root layout) + repo.mkdir() + (repo / ".devkit.yml").write_text("plugin_dir: .\n", encoding="utf-8") + (repo / "plugin.yaml").write_text( + "name: my_plugin\ntitle: X\ndescription: Y\nversion: 0.1.0\n", + encoding="utf-8", + ) + return repo + + +def test_dir_mismatch_not_enforced_for_root_layout(tmp_path: Path) -> None: + report = static_validate(_root_layout_repo(tmp_path)) + codes = {f.code for f in report.findings} + assert "manifest.name.dir_mismatch" not in codes + + +def test_dir_mismatch_still_enforced_for_subdir_layout(tmp_path: Path) -> None: + # Same mismatched name, but no .devkit.yml inside the plugin dir → the + # invariant still holds (subdir layout, and runtime usr/plugins/). + plugin = _root_layout_repo(tmp_path).rename(tmp_path / "my_plugin_wrong_dir") + (plugin / ".devkit.yml").unlink() + report = static_validate(plugin) + codes = {f.code for f in report.findings} + assert "manifest.name.dir_mismatch" in codes + + +def test_pycache_in_scaffolding_is_not_flagged(tmp_path: Path) -> None: + repo = _root_layout_repo(tmp_path) + (repo / "tests" / "__pycache__").mkdir(parents=True) + (repo / "tests" / "__pycache__" / "x.pyc").write_text("", encoding="utf-8") + report = static_validate(repo) + pycache = [f for f in report.findings if f.code == "structure.pycache"] + assert pycache == [], "scaffolding __pycache__ must not be flagged" + + (repo / "__pycache__").mkdir() + (repo / "__pycache__" / "x.pyc").write_text("", encoding="utf-8") + report = static_validate(repo) + pycache = [f for f in report.findings if f.code == "structure.pycache"] + assert [f.path for f in pycache] == ["__pycache__"]