From 8fe1402bc58a6c2fef3a8fd51795635897685261 Mon Sep 17 00:00:00 2001 From: Jonathan Burdge Date: Fri, 9 Oct 2026 18:19:24 +0300 Subject: [PATCH] Document YAML schemas and check reference examples against validators --- configs/README.md | 2 +- docs/configuration.md | 680 +++++++++++++++++++++++++------------- docs/development.md | 16 + docs/python-api.md | 8 +- tests/check.py | 3 +- tests/config_docs.py | 81 +++++ tests/test_config_docs.py | 107 ++++++ tests/test_engines.py | 19 +- 8 files changed, 674 insertions(+), 242 deletions(-) create mode 100644 tests/config_docs.py create mode 100644 tests/test_config_docs.py diff --git a/configs/README.md b/configs/README.md index 0ef6cb3..503e424 100644 --- a/configs/README.md +++ b/configs/README.md @@ -17,7 +17,7 @@ Put repeated language `evidence` and `note` under `language_defaults` in the eva Each profile or set needs a distinct `name` within its directory. To change a selector's startup choice, edit that directory's `default.txt` to name one YAML file. The catalogue is selected at build time with `--catalogue`, or temporarily loaded in the browser. -The [configuration reference](../docs/configuration.md) describes all three formats with small examples. [examples/](examples/) contains the fictional catalogue and weights used by [examples/scores.csv](../examples/scores.csv). +The [configuration reference](../docs/configuration.md) lists every field's type, required/optional status, allowed values, and default behavior. Start with [eval sets](../docs/configuration.md#eval-sets), [weighting profiles](../docs/configuration.md#weighting-profiles), or [per-eval definitions](../docs/configuration.md#per-eval-definitions). It also covers catalogue manifests, portable catalogues, language assignments, and startup selections. [examples/](examples/) contains the fictional catalogue and weights used by [examples/scores.csv](../examples/scores.csv). Submit changes as a PR, then check the generated dashboard: diff --git a/docs/configuration.md b/docs/configuration.md index 53b3178..38f6607 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,184 +9,245 @@ Quickdash separates four inputs: model results, a weighting profile, an optional | Eval set | Optional expected evals and variants | `configs/sets/flagship-1.yaml` on dashboard startup | | Global catalogue | Match tasks to evals, categories, metrics, normalization and languages | `configs/catalogue.yaml` → `configs/evals/` | -## Choose weights and expected coverage +Use the field tables below when authoring YAML. The dashboard and Python API use the same schemas. These files select and interpret existing results; they do not launch evaluations or change how a model was evaluated. -Two profiles are supplied. **Original** is selected on startup; **Code & math emphasis** shifts weight toward those two categories. +- [Eval sets](#eval-sets): required evals, few-shot and metric overrides, exclusions, pinned tasks. +- [Weighting profiles](#weighting-profiles): category weights, English shares, calculation mode. +- [Catalogue manifest](#catalogue-manifest): load a directory of per-eval definitions. +- [Per-eval definitions](#per-eval-definitions): task matching, metric selection, normalization, components. +- [Language assignments and defaults](#shared-language-metadata): ordinary, pooled, and translation results. +- [Portable catalogue](#portable-catalogue): a complete single-file catalogue for browser import or Python. +- [Startup selection](#startup-selection): initial A/B models and default profile/set filenames. -| Category | Original | Code & math emphasis | -| --- | ---: | ---: | -| Code | 0.15 | 0.20 | -| Math | 0.15 | 0.20 | -| Reasoning | 0.15 | 0.10 | -| Knowledge | 0.15 | 0.125 | -| Commonsense | 0.15 | 0.125 | -| Reading | 0.15 | 0.15 | -| Translation | 0.1/3 | 0.1/3 | -| Language | 0.1/3 | 0.1/3 | -| Instruction following | 0.1/3 | 0.1/3 | +For the shipped choices, see [current defaults](#choose-weights-and-expected-coverage). For loading files, see [build defaults and browser imports](#build-defaults-and-browser-imports) and the [Python API](python-api.md#load-the-repositorys-per-eval-files). -Both profiles default to the standard calculation and store an English share of 0.5 per category, which applies only when an English-balance calculation is selected. +## YAML conventions -**flagship-1** is the startup eval set. The **Weighting profile** and **Eval set** selectors operate independently. Switching a profile resets category weights, English shares, and the calculation to that profile's values, leaving the eval set unchanged. Switching eval sets preserves your current weights and calculation. Export edits before switching profiles if you want to keep them. +- Use YAML 1.2 Core. Unknown fields and duplicate mapping keys are rejected. Names, category labels, metrics, filters, and task strings are case-sensitive. +- **Required** means the field must be present. **Optional** means omit the field to get the behavior described in its table; `null` does not mean “use the default.” Empty strings and empty lists are accepted only where stated. +- Write numeric values as numbers, not quoted strings. Numbers must be finite. Shot counts are integers from `0` through `9007199254740991` (`2^53 - 1`); `0` means zero-shot, not unrestricted. +- Write booleans as `true` or `false`. Quote strings such as `"none"`, `""`, and regexes when that makes their meaning clearer. `metric_filter: "none"` matches literal `none`; `metric_filter: ""` matches a blank CSV filter. Neither is a wildcard. +- Top-level `notes`, where allowed, is a list of strings and defaults to no notes. It records context without generating warnings. Eval-level `warning` is a separate field for an active scoring caveat. +- Document intentional exclusions with inline YAML comments explaining why. Parsers and browser exports do not preserve comments; authored YAML in version control retains those explanations. -**Any available** uses recognized selected measurements shared by A and B. Measurements present on only one side generate comparison warnings and are excluded from both scores. Catalogue entries absent from both models do not generate warnings. The supplied freeform set explicitly excludes prompted Global PIQA pending validation; present data for it appears in the **Intentional exclusions** informational summary. +## Eval sets -**flagship-1** requires the catalogue's known tasks for each listed eval. Georgian is excluded throughout the set because it was omitted from tokenizer training, and this was discovered too late to correct for this model generation. Georgian is therefore neither targeted nor evaluated for this generation. English is excluded specifically for X-CSQA to avoid overlap with CommonsenseQA. Original and translated tasks stay under their existing eval group (for example `arc_challenge`). A missing required result generates a warning even if other languages for that eval are present. The score is labelled **INCOMPLETE** and uses the shared subset with redistributed weights. Excluded tasks are not requirements; present explicitly excluded data appears in one informational summary and remains inspectable. +Files: [`configs/sets/*.yaml`](../configs/sets/). Python argument: `load_config(..., eval_set=...)`. Builder flag: `--eval-set`. -The set uses the corrected CoT reasoning and code-continuation protocols from the `flag-evals-471` exports. Each has its own catalogue entry, so the original and corrected runs retain distinct task identities: +A set controls membership and expected coverage. `fixed` declares required evals; `available` uses recognized selected data without requiring absent evals. In a comparison, both modes score shared measurements only. Omitting `eval_set` in Python creates an unrestricted available-mode set; it does not select `flagship-1` or load the repository's `any-available.yaml` exclusions. -| Original eval | Selected corrected eval | -| --- | --- | -| AIME24, AIME25, AMC23 | `aime24_cot`, `aime25_cot`, `amc23_cot` | -| GPQADiamond, JEEBench, MATH500 | `gpqa_diamond_cot`, `jeebench_cot`, `math500_cot` | -| HumanEval, LiveCodeBench, mbpp | `humaneval_cont`, `livecodebench_cont`, `mbpp_cont` | -| polymath | `polymath_cot` (all four difficulty levels in each of six languages) | +### Set fields -These entries select `pass@1` with filter `all`, at 0 shots except `mbpp_cont` at 3 shots. Alternate metrics such as `pass@4` and `think_closed` remain raw inspection fields. Category assignments, normalization floors, and PolyMath difficulty weights are retained. The inherited JEEBench floor of 0.1055 is a shared scoring convention, not a newly measured baseline for CoT prompting. + -`flagship-1` excludes the original entries, avoiding duplicate contributions from the same benchmark. Missing corrected results leave coverage incomplete, even when an original run is present; relaxed matching does not substitute a different task or metric. Original entries remain available for custom sets and inspection. **Any available** can include both original and corrected entries as separate evals, so its score answers a different comparison question. +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `version` | Required | Integer `1` | Schema version. | +| `name` | Required | Nonempty string | Display name; must be unique among sets offered by one build. | +| `mode` | Required | `fixed` or `available` | There is no default in a set file. | +| `evals` | Required for `fixed`; forbidden for `available` | Nonempty list of [eval entries](#set-eval-entries) | Declares required evals. Even `evals: []` is invalid in available mode. | +| `exclude` | Optional | List of unique, exact catalogue eval names; `[]` allowed | No explicit eval exclusions. Supported in both modes; every name must exist in the catalogue. | +| `exclude_languages` | Optional | List of unique canonical language codes; `[]` allowed | No set-wide language exclusions. Removes matching tasks across the set. | +| `notes` | Optional | List of strings; `[]` allowed | No notes. | + +### Set eval entries -A named set can be concise: +Each item under `evals` has these fields. **Add overrides to the existing entry** rather than adding the same eval twice. + + + +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `name` | Required | Exact catalogue eval name | Must exist, be unique in this set, and not also appear in `exclude`. It is not a CSV task name unless the catalogue uses the same name. | +| `shots` | Optional | Nonnegative integer | Inherit catalogue `shots`. If neither specifies it, no shot count is selected by configuration. | +| `metric` | Optional | Nonempty string | Inherit catalogue `metric`. Overrides the exact CSV `metric` to select. | +| `metric_filter` | Optional | String, including `""` | Inherit catalogue `metric_filter`. Overrides the exact CSV `filter` to select. | +| `exclude_languages` | Optional | List of unique canonical language codes; `[]` allowed | No additional exclusions for this eval. Combines with set-wide exclusions. | +| `variants` | Optional | Nonempty list of [task requirements](#pinned-task-requirements) | Require all eligible known tasks for this eval after language exclusions. | + +For example, a complete custom set using entries from the shipped catalogue: + + ```yaml version: 1 -name: Example required set +name: Custom reasoning checks mode: fixed -exclude_languages: [kat_Geor] evals: - name: arc_challenge shots: 10 - name: sib200 metric: acc - metric_filter: none - - name: xcsqa - exclude_languages: [eng_Latn] + metric_filter: "none" + - name: gpqa_diamond_cot + shots: 0 + metric: "pass@1" + metric_filter: "all" +exclude: + - GPQADiamond # Use gpqa_diamond_cot with the corrected CoT protocol. ``` -This example requires a catalogue containing those evals and language assignments. `mode: fixed` means a declared set of required evals; `mode: available` means whatever recognized results are shared by the models. This choice is independent of strict/relaxed matching. +You can put any combination of `shots`, `metric`, and `metric_filter` on an eval. The override applies to the **whole eval group**, including translated tasks and components. Scale, normalization, task matching, and component rules stay in the catalogue; they cannot be overridden by a set. Choose a metric compatible with that scale and normalization. + +There is no top-level `shots` or per-language setting override. A set cannot unset a catalogue shot expectation with `null`; to make it unrestricted, omit `shots` from the catalogue rule as well. Strict/relaxed matching is a runtime option, not a YAML field; see [matching behavior](#strict-and-relaxed-matching). + +### Pinned task requirements -For each named eval, omitted `metric`, `metric_filter`, and `shots` inherit from the catalogue. Set overrides apply to the **whole eval group**, including its translated tasks. They do not alter the global catalogue. Score scale and normalization remain the catalogue's interpretation of that eval; an alternative metric must be compatible with that interpretation. There are no variant-specific setting overrides. **Eval configuration** displays the effective settings and identifies fields overridden by the active set. Changing sets reinterprets loaded results and regenerates synthetic comparisons while preserving weight edits. +Use `variants` only when you want an explicit task inventory instead of the catalogue's eligible known tasks. Each item has: -Known tasks come from the catalogue's explicit task-language assignments and literal `match.name`, limited by the eval's `match` and optional `select` rules. Required coverage is independent of the CSVs: absence from both models still warns. Updating the catalogue can therefore expand a named set's requirements. An eval using only a regex and no known concrete tasks cannot be required until its task inventory is supplied. A recognized task outside the known inventory is excluded from a fixed set, with a warning. Freeform mode can still use it and report missing language metadata. + -`exclude_languages` applies across the entire set or just one eval. Set-wide and local exclusions combine. They use explicit canonical language assignments, not substrings of task names. For translation, either endpoint excludes the pair. Pooled results use their declared language label; exclusions cannot split pooled scores. Excluding a whole component language group is valid; partial component selections are configuration errors. If every task is excluded, the score is unavailable rather than zero. +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `task` | Required | Exact nonempty CSV task name | Must be known to this eval in the catalogue and eligible under `match` and `select`. | +| `n_shot` | Optional | Nonnegative integer | No additional shot constraint beyond the effective eval setting. If supplied, pins this task's membership to that shot count. | -Unknown eval names and language exclusions with no matching catalogue assignment are configuration errors. A local language exclusion must exist within that eval's eligible catalogue tasks; a set-wide exclusion must exist somewhere in the catalogue's eligible tasks. Duplicate/invalid language codes and misspelled fields also fail. A known selected task missing from the CSV is instead a recoverable warning. Failed imports preserve the active dashboard. +For the [portable example catalogue](#portable-catalogue): -Available-mode sets can exclude entire evals or languages without creating required coverage: + ```yaml version: 1 -name: Any available +name: English example only +mode: fixed +evals: + - name: Example reasoning + shots: 0 + variants: + - task: example_en + n_shot: 0 +``` + +`shots` changes the eval's expected setting; `variants[].n_shot` is a hard membership requirement. A pin must agree with effective `shots` when set, and relaxed matching does not loosen it. Duplicate task/shot entries and overlapping unrestricted/pinned entries are invalid. Tasks forming a component group must be selected completely with compatible shot settings. + +Known tasks come from explicit catalogue language assignments and literal `match.name`, restricted by `match` and `select`. A regex alone does not enumerate tasks. A fixed eval with no eligible known tasks is invalid. Catalogue additions can expand an unpinned set's requirements; absent required results warn and make its coverage incomplete. + +### Intentional exclusions + +A complete available-mode set can exclude data without declaring required evals: + + + +```yaml +version: 1 +name: Available validated evals mode: available exclude: - global_piqa_prompted # Scoring and normalization await validation. ``` -`exclude` works in both modes and contains unique, exact catalogue eval names. All names must exist even if no corresponding model results are loaded. An eval cannot be both required under `evals` and listed in `exclude`. Fixed sets exclude unlisted evals automatically; listing an intentional exclusion explicitly explains why its presence should not warn. +Language exclusions may be global or local. This complete set uses the shipped catalogue: -Document each exclusion with an inline YAML comment explaining the replacement or known issue. For example: + ```yaml version: 1 -name: Corrected AIME +name: Flagship language scope mode: fixed +exclude_languages: + - kat_Geor # Georgian was omitted from tokenizer training; not targeted or evaluated for this model generation. evals: - - name: aime25_cot -exclude: - - AIME25 # Replaced by aime25_cot with corrected CoT evaluation settings. + - name: commonsense_qa + - name: xcsqa + exclude_languages: + - eng_Latn # Avoid overlap with CommonsenseQA. ``` -An excluded eval is outside this set even if its results arrive later. To require it again, remove it from `exclude` and add it to `evals`. An intentionally missing eval therefore does not become an unsatisfied requirement; genuinely missing required evals still warn and make coverage incomplete. This mechanism does not waive missing components or protocols of an included eval. +Language codes use the catalogue's explicit assignments, not substrings in task names. A global exclusion must occur in the catalogue's eligible task assignments; a local exclusion must occur in that eval's eligible assignments. Unknown or duplicate codes are errors. Translation is excluded when **either endpoint** matches. Pooled results use their declared language label and cannot be split. Entire component language groups may be excluded; partial component selections are invalid. -Explicit eval and language exclusions produce one **Intentional exclusions** entry under **Information** in the Warnings view when eligible data is present, including rows with only the wrong metric. It combines the affected models and evals, with expandable task lists for each exclusion. It does not increase the warning badge, emit Python warnings, or fail strict diagnostic checks. If no excluded data is present, no informational entry is needed. Unlisted evals and unselected variants without an explicit exclusion still produce **Not used** warnings. Alternate metric rows or summary children of a selected eval do not warn merely because another field or summary was selected. +A fixed set already excludes unlisted evals, but explicit `exclude` entries distinguish intentional exclusions from unexpected extra results. Explicit eval/language exclusions produce one `intentional_exclusion` informational diagnostic when affected data is present, even if it has only the wrong metric. They do not increase the warning count, emit Python warnings, or fail `diagnostics="error"` / CLI `--strict`. With no affected data, no information record is needed. Unlisted evals and unselected variants without an explicit exclusion can still warn. -YAML comments document the source file; the parsers and browser exports do not preserve comments. Exclusion names survive import/export. Keep the authored YAML in version control as the record of why each exclusion exists. +An excluded eval stays outside the set if results arrive later. Remove it from `exclude` and add it to `evals` to require it again. Exclusions do not waive missing metrics, tasks, or components of an included eval. Missing included requirements still warn. -For an explicitly pinned task inventory, a fixed entry may still use `variants: [{task: example_en}]`, optionally with a hard `n_shot` requirement. Each task must be known and selected by its catalogue rule. These are membership constraints, not setting overrides; a pinned `n_shot` must agree with the effective eval settings and is not loosened by relaxed matching. Ordinary named sets need no `variants` list. +## Weighting profiles -## Strict and relaxed matching +Files: [`configs/weights/*.yaml`](../configs/weights/). Python argument: `load_config(..., weights=...)`. Builder flag: `--weights`. -Strict shot-mismatch warnings group tasks by eval, model, and expected/actual shot-count pair, with a count and expandable task list. They replace duplicate missing-setting/coverage warnings for those same tasks; missing requirements still count toward incomplete coverage. + -The dashboard starts with **Strict matching**. Expected settings are resolved in this order: catalogue defaults, then optional whole-eval set overrides. Strict matching requires the configured metric, metric filter, and (when specified) shot count. Shared comparisons also require the same harness and backend. +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `version` | Required | Integer `1` | Schema version. | +| `name` | Required | Nonempty string | Display name; must be unique among profiles offered by one build. | +| `weights` | Required | Nonempty mapping of category labels to nonnegative numbers | Category weights must sum to `1` (absolute tolerance `1e-8`). Labels match catalogue `category` exactly. | +| `aggregate` | Optional | `standard`, `english_eval`, or `english_category` | `standard`; see the modes below. | +| `english_weights` | Optional | Mapping of categories in `weights` to numbers in `[0, 1]`; `{}` allowed | Each omitted category defaults to `0`, which **disables English balancing** for that category. Shares do not sum to 1 across categories. | +| `notes` | Optional | List of strings; `[]` allowed | No notes. | -**Relaxed — allow few-shot differences** may select a different shot count for each concrete task. It prefers the expected count; otherwise it uses the uniquely closest available count, independently for each model and task with the same metric/filter/harness/backend. Equally close alternatives are ambiguous and excluded with a warning. Scores never influence that choice. Without a configured shot expectation, shot counts still have to match across models. - -Relaxed matching never substitutes a different metric or metric filter, or pairs different harnesses/backends. Component groups must still contain every component at one consistent actual shot count. If per-task selection leaves a mixed-shot or incomplete component group, the group is excluded; the engine does not search for a different combination to rescue it. - -When a differing shot count actually contributes, the page prominently reports **INCONSISTENT EVALUATION SETTINGS**. Warnings give the eval, model, tasks, expected count, and actual count. Real measurement identities retain their actual settings. Enabling relaxed mode alone does not label a comparison inconsistent if no mismatched measurements contribute. The shipped working expectations are 25 shots for ARC Challenge, 10 for PIQA, and 5 for MGSM; their 0-shot translated/global variants need relaxed matching unless the set overrides the expectation. - -A weighting profile works with either mode: + ```yaml version: 1 -name: Reasoning weights -weights: {Reasoning: 1} -aggregate: standard -english_weights: {Reasoning: 0.5} +name: Reasoning and math +weights: + Reasoning: 0.6 + Math: 0.4 +aggregate: english_eval +english_weights: + Reasoning: 0.5 + Math: 0.5 ``` -Weights must be nonnegative and sum to 1. Empty categories have their weight redistributed. A catalogue category omitted from the profile receives zero weight and produces a warning when shared measurements use it. The editor exposes that zero so it can be assigned weight. Weight profiles never list evals. All three YAML types accept optional `notes` (a list of strings), require `version: 1` and a nonempty `name`, and reject unknown fields. +| `aggregate` value | Calculation within each category | +| --- | --- | +| `standard` | Average selected variants within each eval, then evals equally. Ignores `english_weights`. | +| `english_eval` | Combine English and other-language means within each eval, then average evals equally. | +| `english_category` | Average represented evals separately on each language side, then combine the two category means. | -## Build defaults and browser imports +Component groups are combined first. Category weights apply last. With a positive English share, a group containing only one language side keeps its full weight. A share of `1` gives English all weight where both sides exist; `0` uses the ordinary unsplit calculation, rather than selecting only non-English results. See the [worked aggregate example](#choosing-an-aggregate). -The builder embeds YAML profiles directly in `configs/weights/` and sets directly in `configs/sets/`. Each directory's `default.txt` contains one YAML filename selected on startup. Missing or invalid defaults stop the build before replacing output. Names must be unique within each selector. +The shipped profiles explicitly use English shares of `0.5`; that is a configuration choice, **not the parser default**. A catalogue category omitted from `weights` receives zero weight and warns when shared measurements use it. A category with no usable data has its weight redistributed proportionally among categories with data. Category labels are configurable, not a fixed list of allowed names. Profiles do not list evals or change their protocols. -- `--catalogue PATH` chooses a complete catalogue YAML or a manifest pointing to per-eval files. Relative `evals_dir` paths resolve against the manifest’s directory, independent of the working directory. -- `--weights PATH` chooses a profile; used alone, it embeds only that profile. `--weights-dir DIR` offers the profiles in another directory and uses its `default.txt` unless an explicit profile is supplied. -- `--eval-set PATH` and `--sets-dir DIR` work the same way for eval sets. -- `--results-dir DIR` embeds CSVs directly in that directory. A checkpoint label may occur in only one file; one file can contain multiple models. An optional `default.yaml` in that directory selects the initial comparison using `a` and `b` checkpoint labels. Both must exist in the embedded data; invalid defaults stop the build before replacing output. Other models remain selectable. See [choosing the startup models](../results/README.md). -- `--sample-csv FILE`, used with `--results-dir`, supplies a fallback only if that directory has no CSVs. Invalid shared files stop the build; they never trigger the fallback. Pages uses `examples/sample-evals.csv` for this option. +## Catalogue manifest -Every offered set is validated against the catalogue and every profile before writing output. Raw results are interpreted using the catalogue with the active set’s overrides; changing sets can change both membership and scoring-field selection. An empty results directory starts without models unless `--sample-csv` supplies a fallback. Omitting both input options starts without models. +File: [`configs/catalogue.yaml`](../configs/catalogue.yaml). Python argument: `load_config(..., catalogue=PATH)`. Builder flag: `--catalogue`. -Under **Eval configuration**, load or export the catalogue, weights, and eval set separately. Uploaded choices are temporary; reload restores published defaults. Catalogue and eval-set imports reinterpret all loaded real models and regenerate synthetic comparisons. Invalid imports preserve the previous models and settings. When replacing a catalogue, first load a compatible set. A neutral set with `mode: available` and no exclusions works with any catalogue; the project's supplied Any available set references prompted Global PIQA and requires that catalogue entry. `configs/examples/eval-set.yaml` is a neutral set for the fictional example. **Clear models** retains settings. +A manifest loads per-eval YAML files from disk: -`analysis.json` records the catalogue, selected profile and set, available `profiles` and `suites`, and source filenames/hashes. It also contains the resolved internal `scheme` used for arithmetic; that combined object is not a YAML input format. Generated `catalogue.yaml` contains the assembled, portable catalogue, with no `evals_dir` reference. `weights.yaml` and `eval-set.yaml` record the other configuration inputs. Browser export also saves the complete catalogue in one file; browser import accepts complete catalogues, not filesystem manifests. + -## Share a view +```yaml +version: 1 +name: Example catalogue +evals_dir: evals +notes: + - Each eval file owns its language assignments. +``` -The browser address updates as you use the dashboard. Copy that address to reopen -or share the current view. A short link such as `#view=languages` opens a tab with -the page's defaults. Generated links also record A/B checkpoint labels, the eval -set and weighting profile filenames, strict/relaxed matching, calculation mode, -edited category weights and English shares, filters, search, sorting, the category -being explored, and expanded eval/language details. Switching tabs adds a browser -history entry; edits within a tab update its entry. Back and Forward restore them. + -Links use the CSVs and configs embedded in the page being opened; they do not pin -a historical dataset or config version. They contain settings and labels, not CSV -contents or uploaded YAML. A view using temporary uploads is marked as requiring -those files; its link cannot reproduce the comparison by itself. Share the files -separately. A local `file:` address also needs the same HTML file at that path; -use the hosted page for links to other people, or share a standalone HTML build. +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `version` | Required | Integer `1` | Format version shared by the per-eval files. | +| `name` | Required | Nonempty string | Catalogue display name. | +| `evals_dir` | Required | Nonempty directory path string | Relative paths resolve against the manifest's directory. Absolute paths are also accepted. | +| `notes` | Optional | List of strings; `[]` allowed | No notes. | -Malformed links and unavailable model/config references show a notice without -changing the current comparison. On first load, the page's defaults remain visible -with that notice. The invalid anchor is retained so it can be inspected; choosing -a new view or changing a control resumes address updates. Display settings in a -link do not alter the scoring rules or suppress data warnings. +Every `.yaml` or `.yml` file directly in `evals_dir` is loaded in filename order; subdirectories and other extensions are ignored. The directory must exist and contain at least one eval definition. Adding a file needs no registration list. The manifest cannot also contain `evals` or `languages`. + +Manifests require filesystem access: pass a **filename**, not a manifest mapping, to Python. Browser import accepts the [portable catalogue](#portable-catalogue), not a manifest or an individual eval file. The builder's generated `catalogue.yaml` is portable. + +## Per-eval definitions -## Edit one eval +Files: [`configs/evals/*.yaml`](../configs/evals/). Each file contains one eval and its language assignments; the manifest loads them together. Per-eval files do **not** take `version` or `notes` fields. -The repository keeps each eval’s interpretation and language mappings in one file. -For example, this `evals/example.yaml` selects `acc_norm` with the `none` metric filter for two language variants -of a four-choice eval: +This complete fictional definition can be saved as `evals/example.yaml` beside the example manifest: + + ```yaml -name: Example eval +name: Example reasoning category: Reasoning match: {regex: 'example_(en|fr)'} metric: acc_norm -metric_filter: none +metric_filter: "none" shots: 0 score: {scale: 1} normalize: - min: 0.25 # Four choices: uniform guessing gets 1/4 correct. + min: 0.25 max: 1 basis: uniform_choice - note: Four-choice chance correction is enabled for this example. + note: Four-choice chance correction for this fictional example. languages: - language: eng_Latn tasks: [example_en] @@ -194,113 +255,317 @@ languages: tasks: [example_fr] ``` -A `catalogue.yaml` beside the `evals/` directory gathers those files: +### Eval fields -```yaml -version: 1 -name: Example scoring -evals_dir: evals + + +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `name` | Required | Nonempty string, unique across catalogue evals | Display/group name and the name referenced by sets. | +| `category` | Required | Nonempty string | Category label used by weighting profiles. | +| `match` | Required | [Match rule](#task-matching-rules) | Identifies the CSV tasks belonging to this eval. Overlapping eval matches are errors. | +| `metric` | Required | Nonempty string | Exact CSV `metric`, such as `acc_norm`, `exact_match`, or `pass@1`. No automatic metric fallback. | +| `metric_filter` | Required | String, including `""` | Exact CSV `filter`. Blank and literal `"none"` are different. | +| `score` | Required | Mapping with required `scale` | Source score interpretation; see below. | +| `shots` | Optional | Nonnegative integer | No shot restriction; multiple settings may be selected. Comparisons still enforce compatible protocols. | +| `select` | Optional | [Match rule](#task-matching-rules) | All matched tasks are eligible. When supplied, only tasks also matching `select` contribute; others remain auditable. | +| `normalize` | Optional | [Normalization mapping](#normalization-fields) | Identity normalization: `min: 0`, `max: 1`, clipping enabled. | +| `warning` | Optional | Nonempty string | No config caveat. Supplied text generates a caveat when the eval participates in the comparison and is always visible in its configuration details. Does not change scores. | +| `aggregation` | Optional | [Component mapping](#component-fields) | Ordinary variant averaging; no required component groups. | +| `languages` | Required in a per-eval file | List of [language assignments](#shared-language-metadata); `[]` allowed | Explicit task-language metadata owned by this eval. Empty means none is known. In a portable catalogue this list moves to the catalogue root. | +| `language_defaults` | Optional; per-eval files only | Mapping with optional `evidence` and `note` strings | No shared language metadata. Expanded during catalogue assembly. | + +`score` accepts only the following field: + + + +| Field | Required? | Type / allowed values | Meaning | +| --- | --- | --- | --- | +| `scale` | Required | Positive finite number | Divide the CSV value by this upper scale: `1` for fractions, `100` for percentages or native chrF points. Selected values must be within `0..scale`. | + +An eval's `metric`, `metric_filter`, and `shots` are defaults that a set can override. `score`, `normalize`, `match`, `select`, languages, and component rules belong to the catalogue. All metrics are treated as higher-is-better. + +### Task matching rules + +`match`, `select`, and component `match` use the same mapping. Supply **exactly one** field: + + + +| Field | Required? | Type / allowed values | Meaning | +| --- | --- | --- | --- | +| `name` | One of `name` or `regex` | Nonempty string | Exact, case-sensitive CSV task name. | +| `regex` | One of `name` or `regex` | Nonempty portable regex string | Must match the entire task name. | + +For example, `match: {name: example_en}` is exact; `match: {regex: 'example_(en|fr)'}` recognizes both names. A `select` rule further restricts `match`; it cannot bring unmatched tasks into the eval. + +Use the shared Python/JavaScript regex subset: literal text, character classes, alternatives, capturing/noncapturing groups, and ordinary quantifiers. Flags, lookarounds, named groups, backreferences, and possessive quantifiers are rejected. `\d` and `\w` use ASCII classes; `\s` uses ECMAScript whitespace and cannot appear inside a character class. Dot excludes line terminators and matches one Unicode code point. Regex captures do not assign languages. + +### Normalization fields + +If `normalize` is present, it accepts: + + + +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `min` | Required | Number | Lower bound after division by `score.scale`. Must satisfy `0 <= min < max <= 1`. | +| `max` | Required | Number | Upper bound after division by `score.scale`; same constraint. | +| `clip` | Optional | Boolean | `true`: clamp the normalized score to `0..100`. `false` permits normalized scores outside that range; source values must still be within `0..scale`. | +| `basis` | Optional | `uniform_choice`, `uniform_integer`, `not_applicable`, or `unresolved` | No rationale label. Metadata only; it does not compute or change the bounds. | +| `note` | Optional | String; `""` allowed | No explanation. | +| `sources` | Optional | List of HTTP(S) URL strings; `[]` allowed | No source links. | + +```text +raw_fraction = value / score.scale +raw_score = 100 × raw_fraction +normalized_score = 100 × (raw_fraction − min) / (max − min) +# With clip: true, clamp normalized_score to [0, 100]. ``` -`evals_dir` is a directory path. The loader reads its direct `.yaml` and `.yml` -files in filename order; subdirectories and other extensions are ignored. Each -file is one eval definition with its own `languages` list. There is no per-file -`version` field: the manifest specifies the format version. Add a new eval by -adding a file; no separate list needs updating. Metadata `notes` may be supplied -on the manifest as a list of strings. +With `scale: 1`, `min: 0.25`, `max: 1`, a source value of `0.625` gives raw score `62.5` and normalized score `50`. Use `min: 0`, `max: 1` to disable chance correction. `acc_norm` is an evaluator metric name; it does not itself apply this chance correction. See [baseline choices](#initial-chance-baselines) for the shipped catalogue's rationale. + +### Component fields + +Use `aggregation` when multiple task results form one eval score. The mapping accepts: + + + +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `components` | Required | Nonempty list of component entries | Every component is required in each represented language/protocol group. | +| `note` | Optional | String; `""` allowed | No explanation. | +| `sources` | Optional | List of HTTP(S) URL strings; `[]` allowed | No source links. | -Every explicit language task must match the eval in its file. Duplicate eval -names, duplicate task-language assignments, known tasks matching multiple evals, -and incompatible component configurations are errors. Missing or empty eval -directories and malformed files stop loading. Errors identify the source file -when a single file is invalid. All validation finishes before the builder replaces -output or a browser import takes effect. +Each item in `components` accepts: + + + +| Field | Required? | Type / allowed values | Meaning | +| --- | --- | --- | --- | +| `name` | Required | Nonempty string, unique within this eval | Component label. | +| `match` | Required | [Match rule](#task-matching-rules) | Identifies this component's tasks within the parent eval. | +| `relative_weight` | Required | Positive finite number | Relative share; weights are divided by their sum, which must also be finite. | + +Components inherit the parent metric, filter, shots, scale, and normalization. They cannot override them. Each selected task must match exactly one component and have explicit language metadata. A valid set must select complete groups; missing result data excludes the whole affected group with a warning. See [the PolyMath example and completeness rules](#weighted-components-within-an-eval). ## Shared language metadata -Put repeated language evidence and notes in `language_defaults` within the eval -file. A language entry inherits each omitted field and can override either field -independently: +In per-eval files, `languages` is a required list, and each item accepts: + + + +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `tasks` | Required | Nonempty list of exact nonempty task names | Every task must match the eval owning this file. A task may have only one language assignment in the catalogue. | +| `scope` | Optional in per-eval files; required in portable catalogues | `single`, `pooled`, or `translation` | Inferred during assembly from the fields below. | +| `language` | Required for `single` or `pooled`; forbidden for `translation` | Canonical code such as `eng_Latn`; `mul` allowed only for `pooled` | Language/group label. | +| `source_language` | Required for `translation`; forbidden otherwise | Canonical code, excluding `mul` | Translation source. | +| `target_language` | Required for `translation`; forbidden otherwise | Canonical code, excluding `mul` | Translation target. | +| `evidence` | Optional | HTTP(S) URL string or `""` | Inherit `language_defaults.evidence` in a per-eval file, otherwise no evidence link. | +| `note` | Optional | String; `""` allowed | Inherit `language_defaults.note` in a per-eval file, otherwise no authored note. | + +Canonical codes have the form `xxx_Ssss` (three lowercase letters, underscore, capitalized four-letter script), such as `fra_Latn` and `srp_Cyrl`, plus the pooled label `mul`. Short aliases and spelled-out names are not resolved. An unlisted task stays Unknown even if its name looks like a language code. + +Scope inference applies **only to per-eval files**: + +| Declared fields | Inferred scope | +| --- | --- | +| `language: eng_Latn` (or another canonical code) | `single` | +| `language: mul` | `pooled` | +| `source_language` or `target_language` | `translation`; both endpoints are still required | + +Use explicit `scope: pooled` when an inseparable pooled score is grouped under one specific code rather than `mul`. Explicit scope must agree with the supplied fields. + +The optional `language_defaults` mapping accepts only these fields: + + + +| Field | Required? | Type / allowed values | Behavior when omitted | +| --- | --- | --- | --- | +| `evidence` | Optional | HTTP(S) URL string or `""` | No inherited evidence link. | +| `note` | Optional | String; `""` allowed | No inherited note. | + +Each language entry inherits omitted metadata and can override either field independently. `""` clears an inherited value; `null` is invalid. Invalid defaults are rejected even if every entry overrides them. This excerpt belongs in a per-eval definition whose `match` covers the listed tasks: + + ```yaml language_defaults: - evidence: https://huggingface.co/datasets/Qwen/PolyMath - note: Language assignments follow the benchmark configuration. + evidence: https://example.org/benchmark-definition + note: Language labels come from the benchmark definition. languages: - - language: deu_Latn - tasks: [polymath_de_low, polymath_de_medium, polymath_de_high, polymath_de_top] - language: eng_Latn - tasks: [polymath_en_low, polymath_en_medium, polymath_en_high, polymath_en_top] - note: A language-specific explanation can replace the shared note. + tasks: [example_en] + - language: fra_Latn + tasks: [example_fr] + note: A more specific explanation for this task. ``` -`language_defaults` accepts only `evidence` and `note`. Both must be strings; -nonempty evidence must be an HTTP(S) URL. An explicit empty string clears an -inherited field; `null` is invalid. Invalid defaults are rejected even if every -language overrides them. The assembled catalogue and single-file exports contain -the resolved metadata, so they remain self-contained. - -In per-eval files, `scope` is optional: `language` implies `single`, `language: mul` -implies `pooled`, and source/target fields imply `translation`. Both translation -languages are required. Use an explicit `scope: pooled` when a score pools languages -but is grouped under one specific language code. Explicit overrides must agree -with the fields: translation cannot also have `language`, and single/pooled cannot -have source/target fields. Assembly records an explicit scope for every entry. -This uses the declared fields; it does not infer language identity from task names. - -## Complete small example - -For a portable single file, the format remains `version`, `name`, `evals`, and -`languages`, with optional `notes`. The loader collects each per-eval definition -under `evals` and its language groups under `languages`. This assembled format is -what the browser and scoring engines use. It contains no directory references. -See the complete [fictional catalogue](../configs/examples/catalogue.yaml), or -build a dashboard and use its generated `catalogue.yaml`. Both formats work with -`--catalogue` and Python’s `load_config`; only the complete format can be imported -into a standalone browser page. A manifest cannot also contain `evals` or -`languages`. - -For an input row with `value=0.625`, the raw score is 62.5 and the normalized score is 50. Raw comparisons show the former; the composite uses the latter. A row using `acc` is retained in the configuration audit but excluded because this config selects `acc_norm`. The UI shows the original source value for every metric, including excluded metrics; the scaled and normalized columns apply to selected scores. Eval summaries label the selected metric and shot counts actually used, then list other available metrics and settings. Expanded selection rules explain blank extraction-filter fields and unrestricted shot counts separately from the data currently selected. A shot is one example in the prompt; 0-shot means no examples. Tasks eligible under `select` that lack the configured metric/filter/shot combination are highlighted and listed in Warnings for each affected model, even when other tasks in the eval have selected scores. - -## Eval fields - -| Field | Meaning | -|---|---| -| `language_defaults` | Optional per-eval defaults for language `evidence` and `note`; individual language entries override them. Expanded during assembly. | -| `languages` | In a per-eval file, the explicit language groups for that eval. Required; use `[]` when unknown. In a complete catalogue, these groups live in the top-level `languages` list. | -| `name` | Unique display name of the eval, grouping its variants. | -| `category` | Category label used by weighting profiles and breakdowns. | -| `match` | Exactly one of `{name: exact task name}` or `{regex: 'full-match pattern'}`. Unmatched tasks are excluded and listed in Warnings; overlapping matches are an error. | -| `metric` | Exact CSV scoring field, such as `acc`, `acc_norm`, or `python_pass@1`. | -| `metric_filter` | Exact value of the CSV `filter` column, including `""` if empty. | -| `shots` | Optional nonnegative integer selecting the CSV `n_shot`. Omit to include all shot settings. | -| `select` | Optional name/regex rule restricting which matched tasks contribute. Useful for selecting summaries while retaining child-task audits. | -| `score.scale` | Raw metric's upper scale: 1 for fractional accuracy; 100 for percentage or chrF scores. Selected values must be finite and within 0..scale. | -| `warning` | Optional nonempty text describing an unresolved scoring assumption. Appears once in Warnings when present in the selected comparison and in this eval’s configuration details; it does not change scores. | -| `aggregation` | Optional component rules with positive relative weights; see [weighted components](#weighted-components-within-an-eval). | -| `normalize` | Optional object with `min`, `max`, and optional `clip`, `basis`, `note`, and `sources`. Thresholds are fractions after division by `score.scale`. | +The URL is illustrative. Translation uses two endpoints, for example this language-entry excerpt: -`metric_filter` selects the evaluator’s response-processing label recorded beside a metric; Quickdash does not execute that processing. `metric_filter: ''` matches a blank CSV field, while `metric_filter: none` matches the literal text `none`. These are distinct values, and neither means “any filter.” + -Use the shared Python/JavaScript regex subset: literal text, character classes, alternatives, capturing/noncapturing groups, and ordinary quantifiers. Patterns match the entire task name. Flags, lookarounds, named groups, backreferences and possessive quantifiers are rejected. `\d` and `\w` use ASCII character classes; `\s` uses ECMAScript whitespace and must not appear inside a character class. Dot excludes line terminators and matches one Unicode code point. Language extraction does not use these patterns. Exact-name rules are available when regexes are unnecessary. +```yaml +tasks: ['flores200:eng_Latn-spa_Latn'] +scope: translation +source_language: eng_Latn +target_language: spa_Latn +``` -The [Python API](python-api.md) consumes these same configurations and returns score trees, audits, coverage, and structured diagnostics. All configuration counts are derived from the supplied inputs. +Assembly writes explicit scopes and resolved metadata into the portable catalogue. The [language views](#explicit-language-assignments) keep source/target pairs distinct without double-counting scores. -The Original weighting profile gives Code, Math, Reasoning, Knowledge, Commonsense, and Reading a weight of 0.15 each; Translation, Language, and Instruction following each receive 0.1/3. Category weights must be nonnegative and sum to 1. Names, metrics, and task strings are case-sensitive. Unknown config fields are rejected to catch typos. `version` must be 1; `name` labels the active config. Optional top-level `notes` is a list of strings. +## Portable catalogue -## Normalization and contributions +A complete single-file catalogue can be loaded by the browser, builder, or Python. It contains the same eval rules as the per-eval layout, with all language assignments collected at the root. -For metric value `v`, scale `s`, and normalization bounds `lo`, `hi`: + -```text -raw_fraction = v / s -raw_score = 100 × raw_fraction -normalized_score = 100 × clamp((raw_fraction − lo) / (hi − lo), 0, 1) +| Field | Required? | Type / allowed values | Behavior when omitted / meaning | +| --- | --- | --- | --- | +| `version` | Required | Integer `1` | Schema version. | +| `name` | Required | Nonempty string | Catalogue display name. | +| `evals` | Required | Nonempty list of [eval rules](#eval-fields) | Each entry uses the eval fields above, **without** `languages` or `language_defaults`. | +| `languages` | Required | List of [language assignments](#shared-language-metadata); `[]` allowed | Each assignment requires explicit `scope`. No scope/default expansion occurs in this format. | +| `notes` | Optional | List of strings; `[]` allowed | No notes. | + + + +```yaml +version: 1 +name: Portable example catalogue +evals: + - name: Example reasoning + category: Reasoning + match: {regex: 'example_(en|fr)'} + metric: acc_norm + metric_filter: "none" + shots: 0 + score: {scale: 1} + normalize: {min: 0.25, max: 1} +languages: + - tasks: [example_en] + scope: single + language: eng_Latn + - tasks: [example_fr] + scope: single + language: fra_Latn ``` -`0 ≤ lo < hi ≤ 1` is required. Omitting normalization gives `lo=0`, `hi=1`. `clip` defaults to true; setting it false allows normalized scores below 0 or above 100. All metrics are treated as higher-is-better. +There is no `evals_dir`, `weights`, `aggregate`, or eval-set membership in this format. Generated `catalogue.yaml` and browser catalogue exports use this layout. The resolved internal `scheme` in `analysis.json` combines inputs for arithmetic; it is not another YAML input format. + +## Startup selection + +### Initial models: `results/default.yaml` + +This optional file is read when building with `--results-dir`. It accepts **exactly** these two fields, with no `version` or `name`: + + + +| Field | Required? | Type / allowed values | Meaning | +| --- | --- | --- | --- | +| `a` | Required when the file exists | Exact checkpoint label from the embedded CSV data | Initial model A. | +| `b` | Required when the file exists | Exact checkpoint label from the embedded CSV data | Initial model B. May equal `a`. | + + + +```yaml +a: v1annealC_120k_l0fix +b: v2anneal_120k +``` + +Use checkpoint labels, not filenames. Missing labels or unknown fields fail the build. Without the file, the dashboard uses the first two checkpoint labels alphabetically, or a synthetic comparison if there is only one model. Direct single-CSV builds do not read it. This affects only the initial selection, not scoring or which other models are available. See [result-file setup](../results/README.md). + +### Initial profile and set: `default.txt` + +[`configs/weights/default.txt`](../configs/weights/default.txt) and [`configs/sets/default.txt`](../configs/sets/default.txt) are **plain text, not YAML**. Each contains one existing `.yaml` or `.yml` filename in its own directory, such as `oellm.yaml` or `flagship-1.yaml`. Paths and multiple lines are invalid. These files are required when the builder chooses a directory's default; an explicit `--weights` or `--eval-set` bypasses that default selection. Python's `load_config` does not read these files. + +## Choose weights and expected coverage + +Two profiles are supplied. **Original** is selected on startup; **Code & math emphasis** shifts weight toward those two categories. + +| Category | Original | Code & math emphasis | +| --- | ---: | ---: | +| Code | 0.15 | 0.20 | +| Math | 0.15 | 0.20 | +| Reasoning | 0.15 | 0.10 | +| Knowledge | 0.15 | 0.125 | +| Commonsense | 0.15 | 0.125 | +| Reading | 0.15 | 0.15 | +| Translation | 0.1/3 | 0.1/3 | +| Language | 0.1/3 | 0.1/3 | +| Instruction following | 0.1/3 | 0.1/3 | + +Both profiles default to the standard calculation and store an English share of 0.5 per category, which applies only when an English-balance calculation is selected. -For evals without component rules, under the original aggregate, one variant's weighted contribution is its normalized score multiplied by the effective category weight, divided by the number of available evals in that category and by the number of shared selected variants for that eval. Filtering does not change those denominators. Weighted A−B contributions sum to the composite difference over shared coverage. The English-balance modes redistribute contributions as described in [Choosing an aggregate](#choosing-an-aggregate). +**flagship-1** is the startup eval set. The **Weighting profile** and **Eval set** selectors operate independently. Switching a profile resets category weights, English shares, and the calculation to that profile's values, leaving the eval set unchanged. Switching eval sets preserves your current weights and calculation. Export edits before switching profiles if you want to keep them. + +**Any available** uses recognized selected measurements shared by A and B. Measurements present on only one side generate comparison warnings and are excluded from both scores. Catalogue entries absent from both models do not generate warnings. The supplied freeform set explicitly excludes prompted Global PIQA pending validation; present data for it appears in the **Intentional exclusions** informational summary. + +**flagship-1** requires the catalogue's known tasks for each listed eval. Georgian is excluded throughout the set because it was omitted from tokenizer training, and this was discovered too late to correct for this model generation. Georgian is therefore neither targeted nor evaluated for this generation. English is excluded specifically for X-CSQA to avoid overlap with CommonsenseQA. Original and translated tasks stay under their existing eval group (for example `arc_challenge`). A missing required result generates a warning even if other languages for that eval are present. The score is labelled **INCOMPLETE** and uses the shared subset with redistributed weights. Excluded tasks are not requirements; present explicitly excluded data appears in one informational summary and remains inspectable. + +The set uses the corrected CoT reasoning and code-continuation protocols from the `flag-evals-471` exports. Each has its own catalogue entry, so the original and corrected runs retain distinct task identities: + +| Original eval | Selected corrected eval | +| --- | --- | +| AIME24, AIME25, AMC23 | `aime24_cot`, `aime25_cot`, `amc23_cot` | +| GPQADiamond, JEEBench, MATH500 | `gpqa_diamond_cot`, `jeebench_cot`, `math500_cot` | +| HumanEval, LiveCodeBench, mbpp | `humaneval_cont`, `livecodebench_cont`, `mbpp_cont` | +| polymath | `polymath_cot` (all four difficulty levels in each of six languages) | + +These entries select `pass@1` with filter `all`, at 0 shots except `mbpp_cont` at 3 shots. Alternate metrics such as `pass@4` and `think_closed` remain raw inspection fields. Category assignments, normalization floors, and PolyMath difficulty weights are retained. The inherited JEEBench floor of 0.1055 is a shared scoring convention, not a newly measured baseline for CoT prompting. + +`flagship-1` excludes the original entries, avoiding duplicate contributions from the same benchmark. Missing corrected results leave coverage incomplete, even when an original run is present; relaxed matching does not substitute a different task or metric. Original entries remain available for custom sets and inspection. **Any available** can include both original and corrected entries as separate evals, so its score answers a different comparison question. + +## Strict and relaxed matching + +Strict shot-mismatch warnings group tasks by eval, model, and expected/actual shot-count pair, with a count and expandable task list. They replace duplicate missing-setting/coverage warnings for those same tasks; missing requirements still count toward incomplete coverage. + +The dashboard starts with **Strict matching**. Expected settings are resolved in this order: catalogue defaults, then optional whole-eval set overrides. Strict matching requires the configured metric, metric filter, and (when specified) shot count. Shared comparisons also require the same harness and backend. + +**Relaxed — allow few-shot differences** may select a different shot count for each concrete task. It prefers the expected count; otherwise it uses the uniquely closest available count, independently for each model and task with the same metric/filter/harness/backend. Equally close alternatives are ambiguous and excluded with a warning. Scores never influence that choice. Without a configured shot expectation, shot counts still have to match across models. + +Relaxed matching never substitutes a different metric or metric filter, or pairs different harnesses/backends. Component groups must still contain every component at one consistent actual shot count. If per-task selection leaves a mixed-shot or incomplete component group, the group is excluded; the engine does not search for a different combination to rescue it. + +When a differing shot count actually contributes, the page prominently reports **INCONSISTENT EVALUATION SETTINGS**. Warnings give the eval, model, tasks, expected count, and actual count. Real measurement identities retain their actual settings. Enabling relaxed mode alone does not label a comparison inconsistent if no mismatched measurements contribute. The shipped working expectations are 25 shots for ARC Challenge, 10 for PIQA, and 5 for MGSM; their 0-shot translated/global variants need relaxed matching unless the set overrides the expectation. + +## Build defaults and browser imports + +The builder embeds YAML profiles directly in `configs/weights/` and sets directly in `configs/sets/`. Each directory's `default.txt` contains one YAML filename selected on startup. Missing or invalid defaults stop the build before replacing output. Names must be unique within each selector. + +- `--catalogue PATH` chooses a complete catalogue YAML or a manifest pointing to per-eval files. Relative `evals_dir` paths resolve against the manifest’s directory, independent of the working directory. +- `--weights PATH` chooses a profile; used alone, it embeds only that profile. `--weights-dir DIR` offers the profiles in another directory and uses its `default.txt` unless an explicit profile is supplied. +- `--eval-set PATH` and `--sets-dir DIR` work the same way for eval sets. +- `--results-dir DIR` embeds CSVs directly in that directory. A checkpoint label may occur in only one file; one file can contain multiple models. An optional `default.yaml` in that directory selects the initial comparison using `a` and `b` checkpoint labels. Both must exist in the embedded data; invalid defaults stop the build before replacing output. Other models remain selectable. See [choosing the startup models](../results/README.md). +- `--sample-csv FILE`, used with `--results-dir`, supplies a fallback only if that directory has no CSVs. Invalid shared files stop the build; they never trigger the fallback. Pages uses `examples/sample-evals.csv` for this option. + +Every offered set is validated against the catalogue and every profile before writing output. Raw results are interpreted using the catalogue with the active set’s overrides; changing sets can change both membership and scoring-field selection. An empty results directory starts without models unless `--sample-csv` supplies a fallback. Omitting both input options starts without models. + +Under **Eval configuration**, load or export the catalogue, weights, and eval set separately. Uploaded choices are temporary; reload restores published defaults. Catalogue and eval-set imports reinterpret all loaded real models and regenerate synthetic comparisons. Invalid imports preserve the previous models and settings. When replacing a catalogue, first load a compatible set. A neutral set with `mode: available` and no exclusions works with any catalogue; the project's supplied Any available set references prompted Global PIQA and requires that catalogue entry. `configs/examples/eval-set.yaml` is a neutral set for the fictional example. **Clear models** retains settings. + +`analysis.json` records the catalogue, selected profile and set, available `profiles` and `suites`, and source filenames/hashes. It also contains the resolved internal `scheme` used for arithmetic; that combined object is not a YAML input format. Generated `catalogue.yaml` contains the assembled, portable catalogue, with no `evals_dir` reference. `weights.yaml` and `eval-set.yaml` record the other configuration inputs. Browser export also saves the complete catalogue in one file; browser import accepts complete catalogues, not filesystem manifests. + +## Share a view + +The browser address updates as you use the dashboard. Copy that address to reopen +or share the current view. A short link such as `#view=languages` opens a tab with +the page's defaults. Generated links also record A/B checkpoint labels, the eval +set and weighting profile filenames, strict/relaxed matching, calculation mode, +edited category weights and English shares, filters, search, sorting, the category +being explored, and expanded eval/language details. Switching tabs adds a browser +history entry; edits within a tab update its entry. Back and Forward restore them. + +Links use the CSVs and configs embedded in the page being opened; they do not pin +a historical dataset or config version. They contain settings and labels, not CSV +contents or uploaded YAML. A view using temporary uploads is marked as requiring +those files; its link cannot reproduce the comparison by itself. Share the files +separately. A local `file:` address also needs the same HTML file at that path; +use the hosted page for links to other people, or share a standalone HTML build. + +Malformed links and unavailable model/config references show a notice without +changing the current comparison. On first load, the page's defaults remain visible +with that notice. The invalid anchor is retained so it can be inspected; choosing +a new view or changing a control resumes address updates. Display settings in a +link do not alter the scoring rules or suppress data warnings. ## Weighted components within an eval @@ -308,6 +573,8 @@ Use `aggregation` when several task results form one eval score and the exporter For PolyMath, add this block to its eval entry (an excerpt, not a complete catalogue): + + ```yaml aggregation: components: @@ -384,32 +651,9 @@ AMC23 is open-ended in the selected evaluator: the original contest's answer opt ## Explicit language assignments -Each group lists exact CSV task names. Several names may share one assignment: - -```yaml -tasks: [example_english, example_en, example_eng_Latn] -scope: single -language: eng_Latn -evidence: https://example.org/benchmark-definition -note: The benchmark definition identifies all three subsets as English. - -``` - -The URL above is illustrative; the supplied config contains actual benchmark source links. `evidence` and `note` are optional. Evidence links must use HTTP or HTTPS. Duplicate task assignments are rejected. Unlisted task names remain Unknown, even if they look like language codes. - -Use `scope: pooled` for scores combining languages that cannot be separated, for example `language: mul`. The supplied MultiBLiMP `multiblimp_hbs` assignment instead uses `scope: single` and `language: srp_Latn` as an explicit grouping approximation: Serbian has more speakers than Croatian. Its eval-level `warning` and language-assignment `note` record that the score pools both languages; no data separation or Serbian-only measurement is implied. Identifiers otherwise follow `xxx_Ssss` form, such as `fra_Latn` or `srp_Cyrl`. Normalized identifiers are written directly; short aliases and spelled-out names are not interpreted at runtime. Language identifiers follow the [upstream OELLM catalogue](https://github.com/OpenEuroLLM/training-data-catalogue/blob/b4823c623c8de4c98de2de5beb24c1f1781b8123/languages). That inventory lists language names and codes, not eval-to-language assignments. Builds and browser imports use the explicit assignments in the YAML; they do not fetch the catalogue or infer languages from task names. +[Language assignments](#shared-language-metadata) determine the language views; task names are not parsed to guess a language. The supplied MultiBLiMP `multiblimp_hbs` assignment uses `scope: single` and `language: srp_Latn` as a grouping approximation for pooled Croatian/Serbian results. Its warning and note explain the approximation; it is not a Serbian-only measurement. -Translation requires both endpoints and no single `language` field: - -```yaml -tasks: ['flores200:eng_Latn-spa_Latn'] -scope: translation -source_language: eng_Latn -target_language: spa_Latn - -``` - -This result appears in: +A translation task assigned `source_language: eng_Latn` and `target_language: spa_Latn` appears in: - Category first: `Translation → FLORES200 → eng_Latn → From eng_Latn → eng_Latn → spa_Latn`. - Language first: `eng_Latn → Translation → FLORES200 → From eng_Latn → eng_Latn → spa_Latn`. @@ -449,23 +693,7 @@ The prominent **Score calculation** panel offers three modes: For evals with component rules, combine complete components first and use complete language/protocol groups in place of variants in the table above. All modes apply the configured category weights last. The selector affects both model score cards, category/eval contributions, effective weights, and weighted delta bars. Raw comparison columns and descriptive language/category breakdowns retain their meanings. -Optional top-level fields in the **weighting profile** persist the choice and shares: - -```yaml -aggregate: english_eval -english_weights: - Code: 0.5 - Math: 0.5 - Reasoning: 0.5 - Knowledge: 0.5 - Commonsense: 0.5 - Reading: 0.5 - Translation: 0.5 - Language: 0.5 - Instruction following: 0.5 -``` - -The weight editor has one category per row. **English share applies only when either English-balance mode is selected at the top.** Switching modes preserves the stored shares; they are inactive under Original. Keys must be configured categories and values must be numbers from 0 to 1. Omitted categories default to 0. Zero disables the split for that category and retains its original calculation. Positive shares apply inside every eval in that category under `english_eval`, or to the category mean under `english_category`. Shares do not sum to 1 across categories; outer category weights still do. +English shares are configured in the [weighting profile](#weighting-profiles). They apply only in an English-balance mode; zero disables the split for that category. A share of 0.5 gives English half and other languages half collectively wherever both language groups exist. When an eval (per-eval mode) or category (per-category mode) contains only one language side, that side automatically retains the full weight regardless of its configured positive share. Code and Instruction following can therefore use 0.5 without requiring the user to know their language coverage. No scores means exclusion, not an invented score. diff --git a/docs/development.md b/docs/development.md index feb6471..97fd624 100644 --- a/docs/development.md +++ b/docs/development.md @@ -16,6 +16,22 @@ Run commands from the repository root. Install the Python package with `python - | `docs/` | Configuration and contributor documentation. | | `data/`, `output/` | Ignored local inputs and generated files. | +## Keep configuration docs in sync + +The public checks validate the schema tables and YAML examples in +[the configuration reference](configuration.md). Invisible `config-schema` labels +identify field tables; `config-example` labels identify executable examples. +Keep those labels when moving sections or rewriting prose. Every YAML block +needs an example label and a validation context in `tests/config_docs.py`. + +`tests/test_config_docs.py` observes the Python validators' field checks to +compare allowed fields and unconditional requirements with the tables. It also +loads the manifest and startup examples from disk. `tests/test_engines.py` runs +the scoring examples through both engines. These checks do not snapshot prose, +line numbers, or heading order. Conditional requirements, defaults, and numeric +semantics still need behavioral regressions when the corresponding contract +changes; the tests cannot prove that every explanatory sentence is correct. + ## Build and inspect a change ```sh diff --git a/docs/python-api.md b/docs/python-api.md index 6cdd931..bdd91b8 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -95,7 +95,7 @@ scopes are resolved during loading; local metadata overrides are preserved. The complete in-memory catalogue, so later analysis does not access those files. Existing single-file catalogues remain supported. When passing a dictionary instead of a filename, supply the complete catalogue; a filesystem manifest needs -a filename to resolve its directory. See [editing an eval](configuration.md#edit-one-eval). +a filename to resolve its directory. See the [per-eval YAML format](configuration.md#per-eval-definitions). ## Expected settings and relaxed comparisons @@ -103,8 +103,10 @@ Both `analyze()` and `compare()` accept `matching="strict"` (the default) or `matching="relaxed"`. They first apply the eval set's optional whole-eval `metric`, `metric_filter`, and `shots` overrides to the catalogue defaults. Set-wide and per-eval `exclude_languages` remove results from membership and required coverage. -See the [configuration contract](configuration.md#strict-and-relaxed-matching) -for selection, ambiguity, and component-group rules. +See the [set field reference](configuration.md#set-eval-entries) for `shots`, +`metric`, and `metric_filter` override syntax, and the +[matching rules](configuration.md#strict-and-relaxed-matching) for selection, +ambiguity, and component-group behavior. ```python comparison = compare( diff --git a/tests/check.py b/tests/check.py index 63bc6fb..141aca5 100644 --- a/tests/check.py +++ b/tests/check.py @@ -103,7 +103,8 @@ def main(): # Browser tests invoke python3; use this command's Python environment for builds. os.environ['PATH'] = str(Path(sys.executable).parent) + os.pathsep + os.environ.get('PATH', '') subprocess.run([sys.executable, '-m', 'unittest', 'tests.test_check', - 'tests.test_data', 'tests.test_engines', 'tests.test_pages_preview'], cwd=ROOT, check=True) + 'tests.test_data', 'tests.test_engines', 'tests.test_config_docs', + 'tests.test_pages_preview'], cwd=ROOT, check=True) subprocess.run(['node', '--test'] + ['tests/' + name for name in NODE_TESTS], cwd=ROOT, check=True) run_browser(chrome) print('All public checks passed, including the browser.') diff --git a/tests/config_docs.py b/tests/config_docs.py new file mode 100644 index 0000000..f4d2440 --- /dev/null +++ b/tests/config_docs.py @@ -0,0 +1,81 @@ +"""Read stable schema/example labels without depending on prose or heading order.""" + +from pathlib import Path +import re + +from quickdash.io import parse_yaml +from quickdash.config import load_catalogue + +ROOT = Path(__file__).resolve().parent.parent +REFERENCE = ROOT / "docs/configuration.md" + + +def examples(source=None): + source = REFERENCE.read_text() if source is None else source + entries = re.findall( + r"\s*```yaml\s*\n(.*?)\n```", + source, re.S, + ) + if len(entries) != len(re.findall(r"(?m)^```yaml\s*$", source)): + raise AssertionError("Every YAML example needs a config-example label") + result = {name: parse_yaml(body) for name, body in entries} + if len(result) != len(entries): + raise AssertionError("Duplicate config-example label") + return result + + +def schemas(source=None): + source = REFERENCE.read_text() if source is None else source + result = {} + for name, table in re.findall( + r"\s*((?:\|[^\n]*\n)+)", source + ): + if name in result: + raise AssertionError("Duplicate config-schema label: " + name) + fields = {} + for row in table.splitlines(): + match = re.match(r"\|\s*`(\w+)`\s*\|\s*([^|]+)\|", row) + if match: + key, requirement = match.groups() + if key in fields: + raise AssertionError("Duplicate documented field: " + name + "." + key) + fields[key] = requirement.strip() + if not fields: + raise AssertionError("Missing field table for " + name) + result[name] = fields + return result + + +def engine_cases(): + """Exercise every scoring example in its documented catalogue context.""" + docs = examples() + production = load_catalogue(ROOT / "configs/catalogue.yaml") + available = dict(version=1, name="Documentation checks", mode="available") + base = dict(catalogue=docs["catalogue"], profile=docs["profile"], suite=available) + metadata = {k: v for k, v in docs["manifest"].items() if k != "evals_dir"} + cases = {} + for name, example in docs.items(): + config = dict(base) + case = dict(config=config, rows=[]) + if name.startswith("set-"): + config.update(catalogue=docs["catalogue"] if name == "set-pinned" else production, + suite=example) + elif name in {"catalogue", "profile"}: + config[name] = example + elif name in {"manifest", "eval", "language-defaults", "translation", "components"}: + definition = dict(docs["eval"]) + if name == "language-defaults": + definition.update(example) + elif name == "translation": + definition.update(match=dict(name=example["tasks"][0]), languages=[example]) + elif name == "components": + definition = parse_yaml((ROOT / "configs/evals/polymath.yaml").read_text()) + definition.update(example) + config["catalogue"] = metadata + case["eval_definitions"] = [definition] + elif name == "startup": + continue # Builder-only format; covered in test_config_docs. + else: + raise AssertionError("Add a validation context for config-example: " + name) + cases[name] = case + return cases diff --git a/tests/test_config_docs.py b/tests/test_config_docs.py new file mode 100644 index 0000000..56874f7 --- /dev/null +++ b/tests/test_config_docs.py @@ -0,0 +1,107 @@ +"""Keep schema field tables and executable examples aligned with the validators.""" + +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import yaml + +from app.build import default_comparison +from quickdash import config +from tests.config_docs import examples, schemas + + +class ConfigurationDocs(unittest.TestCase): + def test_schema_fields_follow_live_validators(self): + docs = schemas() + # Fictional inputs exercise every nested mapping. Field inventories come + # from object_keys calls in the validators, not a second schema in tests. + match = dict(regex="example_(en|fr)") + language = dict(tasks=["example_en"], scope="single", language="eng_Latn") + component = dict(name="only", match=match, relative_weight=1) + rule = dict(name="Example", category="C", match=match, metric="acc", + metric_filter="none", score=dict(scale=1), + normalize=dict(min=0, max=1), + aggregation=dict(components=[component])) + catalogue = dict(version=1, name="Example catalogue", evals=[rule], languages=[language]) + variant = dict(task="example_en") + entry = dict(name="Example", variants=[variant]) + suite = dict(version=1, name="Example set", mode="fixed", evals=[entry]) + profile = dict(version=1, name="Example weights", weights=dict(C=1)) + defaults = dict(note="Language rationale") + definition = dict(rule, languages=[language], language_defaults=defaults) + metadata = dict(version=1, name="Example manifest") + manifest = dict(metadata, evals_dir="evals") + targets = dict(catalogue=catalogue, eval=rule, match=match, score=rule["score"], + normalize=rule["normalize"], aggregation=rule["aggregation"], + component=component, language=language, set=suite, + profile=profile, manifest=manifest) + targets.update({"set-eval": entry, "variant": variant, "language-defaults": defaults}) + observed = {} + original = config.object_keys + + def record(value, allowed, required=()): + original(value, allowed, required) + for name, target in targets.items(): + if value == target: + signature = (set(allowed), set(required)) + if name in observed: + self.assertEqual(observed[name], signature, name) + observed[name] = signature + + with patch.object(config, "object_keys", side_effect=record): + config.validate_catalogue(catalogue) + config.validate_suite(suite) + config.validate_profile(profile) + config.assemble_catalogue(metadata, [definition]) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "evals").mkdir() + (path / "evals/example.yaml").write_text(yaml.safe_dump(definition)) + (path / "catalogue.yaml").write_text(yaml.safe_dump(manifest)) + config.load_catalogue(path / "catalogue.yaml") + + self.assertEqual(set(observed), set(targets), "Every documented mapping must exercise its validator") + # Per-eval assembly moves these two file-level fields outside the rule. + observed["eval"][0].update(definition.keys() - rule.keys()) + observed["eval"][1].add("languages") + self.assertEqual(set(docs), set(observed) | {"startup"}) + for name, (allowed, required) in observed.items(): + with self.subTest(schema=name): + self.assertEqual(set(docs[name]), allowed, "Document every allowed field, and no obsolete fields") + for field in required: + self.assertIn("required", docs[name][field].lower(), field) + for field, status in docs[name].items(): + if status.lower() == "required": + self.assertIn(field, required, field) + + def test_examples_load_from_disk_with_the_documented_layout(self): + docs = examples() + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "evals").mkdir() + (path / "catalogue.yaml").write_text(yaml.safe_dump(docs["manifest"])) + (path / "evals/example.yaml").write_text(yaml.safe_dump(docs["eval"])) + (path / "weights.yaml").write_text(yaml.safe_dump(docs["profile"])) + (path / "set.yaml").write_text(yaml.safe_dump(docs["set-pinned"])) + loaded = config.load_config(catalogue=path / "catalogue.yaml", + weights=path / "weights.yaml", eval_set=path / "set.yaml") + self.assertEqual(loaded["catalogue"]["evals"][0]["name"], docs["eval"]["name"]) + (path / "default.yaml").write_text(yaml.safe_dump(docs["startup"])) + labels = set(docs["startup"].values()) + self.assertEqual(default_comparison(path, labels), docs["startup"]) + self.assertEqual(set(schemas()["startup"]), set(docs["startup"])) + for field in docs["startup"]: + invalid = {k: v for k, v in docs["startup"].items() if k != field} + (path / "default.yaml").write_text(yaml.safe_dump(invalid)) + with self.assertRaises(ValueError): + default_comparison(path, labels) + + def test_markers_allow_prose_and_table_formatting_changes(self): + self.assertEqual(schemas("Intro\n\n\n" + "| Field | Required? | Details |\n|:---|:---|---|\n" + "| `name` | Required | Wording can change. |\n"), + {"demo": {"name": "Required"}}) + self.assertEqual(examples("New heading\n\n\n" + "```yaml\nname: Example # A comment\n```\n"), + {"demo": {"name": "Example"}}) diff --git a/tests/test_engines.py b/tests/test_engines.py index e48806b..4acb78e 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -250,17 +250,14 @@ def both(self, cases): results.append((py, other)) return results - def test_documented_eval_sets_resolve_in_both_engines(self): - section = (ROOT / "docs/configuration.md").read_text().split("## Strict and relaxed matching")[0] - examples = [parse_yaml(block) for block in re.findall(r"```yaml\n(.*?)```", section, re.S)] - self.assertTrue(examples, "The eval-set guide must contain executable examples") - base = load_config(catalogue=ROOT / "configs/catalogue.yaml", - weights=ROOT / "configs/weights/oellm.yaml") - rows = parse_csv((ROOT / "examples/sample-evals.csv").read_text()) - cases = [dict(config={**base, "suite": suite}, rows=rows) for suite in examples] - for results in self.both(cases): - for result in results: - self.assertNotIn("error", result) + def test_documented_configurations_resolve_in_both_engines(self): + from tests.config_docs import engine_cases + cases = engine_cases() + self.assertTrue(cases, "The configuration guide must contain executable examples") + for name, results in zip(cases, self.both(list(cases.values()))): + with self.subTest(example=name): + for result in results: + self.assertNotIn("error", result) def test_per_eval_catalogue_assembly(self): original = fixture()