From e67f4f4812d0d8e707356c76740e368c655c08b2 Mon Sep 17 00:00:00 2001
From: Pigbibi <20649888+Pigbibi@users.noreply.github.com>
Date: Wed, 7 Oct 2026 19:38:07 +0800
Subject: [PATCH] feat(research): add offline Treasury ten-year rates parser

---
 .../providers/treasury_rates.py               | 179 +++++++++++++
 tests/test_treasury_rates.py                  | 250 ++++++++++++++++++
 2 files changed, 429 insertions(+)
 create mode 100644 src/market_signal_sources/providers/treasury_rates.py
 create mode 100644 tests/test_treasury_rates.py

diff --git a/src/market_signal_sources/providers/treasury_rates.py b/src/market_signal_sources/providers/treasury_rates.py
new file mode 100644
index 0000000..0412595
--- /dev/null
+++ b/src/market_signal_sources/providers/treasury_rates.py
@@ -0,0 +1,179 @@
+"""Offline Treasury nominal/real ten-year XML to the existing QSP research v1.
+
+The caller supplies bytes and capture evidence; this module has no downloader,
+clock, filesystem, registration, or execution authority. Its input evidence is
+a declaration, not a license, authenticated capture, or historical PIT proof.
+
+Official formats and missing-element semantics:
+https://home.treasury.gov/treasury-daily-interest-rate-xml-feed
+https://home.treasury.gov/developer-notice-xml-changes
+
+Each evidence mapping has exactly dataset, raw_sha256 and rows. Rows is a sequence aligned to XML entry order, with observation_date,
+available_at, received_at and revision_id. Missing row evidence or missing
+time/revision fields remain unknown. The raw hash binds this row order; an
+explicit observation_date must also match its XML entry. available_at MUST describe evidence of that row revision's
+publication, never a date-derived clock, first-seen time or receipt fallback.
+The caller must retain original bytes and this evidence outside the QSP v1
+snapshot: v1 has no field for the raw capture hash. A raw capture hash is not a
+row revision identifier; appending a future row must not revise earlier rows.
+"""
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from datetime import date, datetime, timezone
+import hashlib
+import math
+import re
+from xml.etree import ElementTree as ET
+
+
+_ATOM = "{http://www.w3.org/2005/Atom}"
+_META = "{http://schemas.microsoft.com/ado/2007/08/dataservices/metadata}"
+_DATA = "{http://schemas.microsoft.com/ado/2007/08/dataservices}"
+_DATE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T00:00:00")
+_NUMBER = re.compile(r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[Ee][+-]?[0-9]+)?")
+_ROW_FIELDS = {"observation_date", "available_at", "received_at", "revision_id"}
+
+
+class TreasuryRatesError(ValueError):
+    """Malformed or ambiguous source/capture contract, without input echo."""
+
+
+def _time(value):
+    if not isinstance(value, str):
+        return None
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None
+    except (ValueError, OverflowError):
+        return None
+
+
+def _stamp(value):
+    parsed = _time(value)
+    return None if parsed is None else parsed.isoformat().replace("+00:00", "Z")
+
+
+def _single(parent, tag):
+    matches = parent.findall(tag)
+    if len(matches) > 1:
+        raise TreasuryRatesError("duplicate_field")
+    return matches[0] if matches else None
+
+
+def _observation_date(properties):
+    field = _single(properties, _DATA + "NEW_DATE")
+    if field is not None and (len(field) or field.attrib.get(_META + "null") not in (None, "false", "0") or field.attrib.get(_META + "type") not in (None, "Edm.DateTime")):
+        raise TreasuryRatesError("observation_date_invalid")
+    value = "" if field is None or field.text is None else field.text.strip()
+    if not _DATE.fullmatch(value):
+        raise TreasuryRatesError("observation_date_invalid")
+    try:
+        return date.fromisoformat(value[:10])
+    except ValueError as exc:
+        raise TreasuryRatesError("observation_date_invalid") from exc
+
+
+def _value(properties, field):
+    element = _single(properties, _DATA + field)
+    if element is None or len(element) or element.attrib.get(_META + "null") not in (None, "false", "0"):
+        return None
+    if element.attrib.get(_META + "type") not in (None, "Edm.Double", "Edm.Decimal"):
+        return None
+    text = (element.text or "").strip()
+    if not _NUMBER.fullmatch(text):
+        return None
+    try:
+        value = float(text)
+        return value if math.isfinite(value) else None
+    except (ValueError, OverflowError):
+        return None
+
+
+def _series(raw, evidence, dataset, field, decision):
+    if not isinstance(raw, bytes):
+        raise TreasuryRatesError("xml_bytes_required")
+    if not isinstance(evidence, Mapping) or set(evidence) != {"dataset", "raw_sha256", "rows"}:
+        raise TreasuryRatesError("capture_evidence_invalid")
+    if evidence["dataset"] != dataset:
+        raise TreasuryRatesError("dataset_mismatch")
+    if evidence["raw_sha256"] != hashlib.sha256(raw).hexdigest():
+        raise TreasuryRatesError("raw_hash_mismatch")
+    row_evidence = evidence["rows"]
+    if not isinstance(row_evidence, Sequence) or isinstance(row_evidence, (str, bytes, bytearray)):
+        raise TreasuryRatesError("row_evidence_invalid")
+    try:
+        text = raw.decode("utf-8")
+        if "\x00" in text or "<!DOCTYPE" in text.upper() or "<!ENTITY" in text.upper():
+            raise TreasuryRatesError("xml_declaration_forbidden")
+        root = ET.fromstring(text)
+    except (UnicodeError, ET.ParseError) as exc:
+        raise TreasuryRatesError("xml_invalid") from exc
+    if root.tag != _ATOM + "feed":
+        raise TreasuryRatesError("xml_root_invalid")
+    rows = []
+    previous = None
+    seen = set()
+    for ordinal, entry in enumerate(root.findall(_ATOM + "entry")):
+        content = _single(entry, _ATOM + "content")
+        properties = None if content is None else _single(content, _META + "properties")
+        if properties is None:
+            raise TreasuryRatesError("xml_properties_missing")
+        observed = _observation_date(properties)
+        # Project before reading maturity values, revision or non-selector fields.
+        if observed > decision.date():
+            continue
+        evidence_row = row_evidence[ordinal] if ordinal < len(row_evidence) else {}
+        if not isinstance(evidence_row, Mapping):
+            raise TreasuryRatesError("row_evidence_invalid")
+        available = _time(evidence_row.get("available_at"))
+        received = _time(evidence_row.get("received_at"))
+        if (available is not None and available > decision) or (received is not None and received > decision):
+            continue
+        if evidence_row and evidence_row.get("observation_date") != observed.isoformat():
+            raise TreasuryRatesError("row_evidence_date_mismatch")
+        if set(evidence_row) - _ROW_FIELDS:
+            raise TreasuryRatesError("row_evidence_fields_invalid")
+        if observed in seen:
+            raise TreasuryRatesError("duplicate_observation")
+        if previous is not None and observed < previous:
+            raise TreasuryRatesError("observation_order")
+        seen.add(observed)
+        previous = observed
+        revision = evidence_row.get("revision_id")
+        if not isinstance(revision, str) or not revision or revision != revision.strip() or len(revision) > 256 or revision == "latest":
+            revision = None
+        rows.append({
+            "observation_date": observed.isoformat(),
+            "value": _value(properties, field),
+            "available_at": _stamp(evidence_row.get("available_at")),
+            "received_at": _stamp(evidence_row.get("received_at")),
+            "revision_id": revision,
+        })
+    return {"source_id": "us_treasury.daily_par_yield_curves",
+            "series_id": dataset + "." + field,
+            "basis": "treasury_par_yield", "unit": "percent", "rows": rows}
+
+
+def build_treasury_rates_research_input(
+    *, nominal_xml: bytes, real_xml: bytes,
+    nominal_evidence: Mapping, real_evidence: Mapping, decision_at: str,
+) -> dict:
+    """Build QSP rates-input.v1; data/evidence gaps remain explicit unknowns.
+
+    This does not generate reported breakeven or run the observer. The owning
+    research caller must pass the result to QSP's existing v1 observer with its
+    frozen explicit window/age configuration. All historical/economic/runtime
+    qualification remains external; no source registry or runtime is changed.
+    """
+    decision = _time(decision_at)
+    if decision is None:
+        raise TreasuryRatesError("decision_at_invalid")
+    return {
+        "schema_version": "qsl.rates-context-input.research.v1",
+        "decision_at": decision.isoformat().replace("+00:00", "Z"),
+        "series": {
+            "nominal_10y": _series(nominal_xml, nominal_evidence, "daily_treasury_yield_curve", "BC_10YEAR", decision),
+            "real_10y": _series(real_xml, real_evidence, "daily_treasury_real_yield_curve", "TC_10YEAR", decision),
+        },
+    }
diff --git a/tests/test_treasury_rates.py b/tests/test_treasury_rates.py
new file mode 100644
index 0000000..aca94d8
--- /dev/null
+++ b/tests/test_treasury_rates.py
@@ -0,0 +1,250 @@
+"""SYNTHETIC_FIXTURE_ONLY: no downloaded observations."""
+import copy
+import hashlib
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+from market_signal_sources.providers.treasury_rates import (
+    TreasuryRatesError,
+    build_treasury_rates_research_input,
+)
+
+DATES = ("2024-01-02", "2024-01-03")
+DECISION = "2024-01-05T12:00:00Z"
+
+
+def xml(rows, field="BC_10YEAR"):
+    entries = "".join(
+        '<entry><content type="application/xml"><m:properties>'
+        f'<d:NEW_DATE m:type="Edm.DateTime">{day}T00:00:00</d:NEW_DATE>'
+        + ("" if value is None else f'<d:{field} m:type="Edm.Double">{value}</d:{field}>')
+        + '</m:properties></content></entry>'
+        for day, value in rows
+    )
+    return ('<feed xmlns="http://www.w3.org/2005/Atom" '
+            'xmlns:m="http://schemas.microsoft.com/ado/2007/08/dataservices/metadata" '
+            'xmlns:d="http://schemas.microsoft.com/ado/2007/08/dataservices">'
+            + entries + '</feed>').encode()
+
+
+def evidence(raw, dataset):
+    return {"dataset": dataset, "raw_sha256": hashlib.sha256(raw).hexdigest(),
+            "rows": [{"observation_date": day, "available_at": f"{day}T21:00:00Z",
+                           "received_at": f"{day}T21:01:00Z",
+                           "revision_id": f"SYNTHETIC_FIXTURE_ONLY:{day}:v1"}
+                     for day in DATES]}
+
+
+def arguments():
+    nominal = xml(zip(DATES, (4.0, 4.2)))
+    real = xml(zip(DATES, (1.0, 1.1)), "TC_10YEAR")
+    return {"nominal_xml": nominal, "real_xml": real,
+            "nominal_evidence": evidence(nominal, "daily_treasury_yield_curve"),
+            "real_evidence": evidence(real, "daily_treasury_real_yield_curve"),
+            "decision_at": DECISION}
+
+
+def replace_nominal(args, raw):
+    args["nominal_xml"] = raw
+    args["nominal_evidence"]["raw_sha256"] = hashlib.sha256(raw).hexdigest()
+
+
+def rows(args):
+    return build_treasury_rates_research_input(**args)["series"]["nominal_10y"]["rows"]
+
+
+def test_existing_qsp_shape_identities_and_no_breakeven():
+    args = arguments()
+    before = copy.deepcopy(args)
+    result = build_treasury_rates_research_input(**args)
+    assert args == before
+    assert result["schema_version"] == "qsl.rates-context-input.research.v1"
+    assert result["decision_at"] == DECISION
+    assert set(result) == {"schema_version", "decision_at", "series"}
+    assert set(result["series"]) == {"nominal_10y", "real_10y"}
+    nominal, real = result["series"].values()
+    assert nominal["source_id"] == real["source_id"] == "us_treasury.daily_par_yield_curves"
+    assert nominal["series_id"] == "daily_treasury_yield_curve.BC_10YEAR"
+    assert real["series_id"] == "daily_treasury_real_yield_curve.TC_10YEAR"
+    assert nominal["basis"] == real["basis"] == "treasury_par_yield"
+    assert nominal["unit"] == real["unit"] == "percent"
+    assert nominal["rows"][1] == {"observation_date": DATES[1], "value": 4.2,
+                                 **args["nominal_evidence"]["rows"][1]}
+
+
+@pytest.mark.parametrize("value", ["-0.25", "0", "4.2"])
+def test_finite_percent_values(value):
+    args = arguments()
+    replace_nominal(args, xml(zip(DATES, (value, value))))
+    assert rows(args)[0]["value"] == float(value)
+
+
+@pytest.mark.parametrize("value", [None, "NaN", "INF", "-Infinity", "true", "bad", "1e9999", "1_000", ""])
+def test_invalid_or_missing_value_is_not_zero(value):
+    args = arguments()
+    replace_nominal(args, xml(zip(DATES, (value, 4.2))))
+    assert rows(args)[0]["value"] is None
+
+
+def test_xml_null_and_wrong_maturity_are_missing():
+    for raw in [
+        xml(zip(DATES, (4.0, 4.2))).replace(
+            b'<d:BC_10YEAR m:type="Edm.Double">4.0</d:BC_10YEAR>',
+            b'<d:BC_10YEAR m:null="true"/>'),
+        xml(zip(DATES, (4.0, 4.2)), "BC_5YEAR"),
+    ]:
+        args = arguments()
+        replace_nominal(args, raw)
+        assert rows(args)[0]["value"] is None
+
+
+@pytest.mark.parametrize("missing", ["available_at", "received_at", "revision_id"])
+def test_evidence_is_never_inferred(missing):
+    args = arguments()
+    del args["nominal_evidence"]["rows"][0][missing]
+    assert rows(args)[0][missing] is None
+    if missing == "available_at":
+        assert rows(args)[0]["received_at"] == f"{DATES[0]}T21:01:00Z"
+
+
+def test_missing_all_row_evidence_remains_unknown():
+    args = arguments()
+    args["nominal_evidence"]["rows"] = []
+    assert all(r["available_at"] is r["received_at"] is r["revision_id"] is None for r in rows(args))
+
+
+@pytest.mark.parametrize("side", ["nominal", "real"])
+def test_raw_hash_mismatch(side):
+    args = arguments()
+    args[f"{side}_evidence"]["raw_sha256"] = "0" * 64
+    with pytest.raises(TreasuryRatesError, match="raw_hash_mismatch"):
+        rows(args)
+
+
+def test_dataset_role_mismatch():
+    args = arguments()
+    args["nominal_evidence"]["dataset"] = "daily_treasury_real_yield_curve"
+    with pytest.raises(TreasuryRatesError, match="dataset_mismatch"):
+        rows(args)
+
+
+@pytest.mark.parametrize("raw,reason", [
+    (xml([(DATES[0], 4), (DATES[0], 7), (DATES[1], 4.2)]), "duplicate_observation"),
+    (xml([(DATES[1], 4.2), (DATES[0], 4)]), "observation_order"),
+])
+def test_visible_conflicting_revisions_and_order_are_not_repaired(raw, reason):
+    args = arguments()
+    replace_nominal(args, raw)
+    args["nominal_evidence"]["rows"] = []
+    with pytest.raises(TreasuryRatesError, match=reason):
+        rows(args)
+
+
+@pytest.mark.parametrize("selector", ["future_date", "available_at", "received_at"])
+def test_future_or_late_append_cannot_change_past_output(selector):
+    args = arguments()
+    original = build_treasury_rates_research_input(**args)
+    day = "2024-01-06" if selector == "future_date" else "2024-01-04"
+    replace_nominal(args, xml([(DATES[0], 4), (DATES[1], 4.2), (day, "bad")]))
+    ev = {"unexpected": object()}
+    if selector != "future_date":
+        ev[selector] = "2024-01-06T00:00:00Z"
+    args["nominal_evidence"]["rows"].append(ev)
+    assert build_treasury_rates_research_input(**args) == original
+
+
+def test_late_required_row_is_omitted():
+    args = arguments()
+    args["nominal_evidence"]["rows"][1]["received_at"] = "2024-01-06T00:00:00Z"
+    assert len(rows(args)) == 1
+
+
+@pytest.mark.parametrize("bad", ["", "2024-01-05", "2024-01-05T12:00:00", "garbage"])
+def test_decision_requires_timezone(bad):
+    args = arguments()
+    args["decision_at"] = bad
+    with pytest.raises(TreasuryRatesError, match="decision_at_invalid"):
+        rows(args)
+
+
+@pytest.mark.parametrize("mutation", ["dtd", "broken", "wrong_root", "duplicate_field", "bad_date"])
+def test_unsafe_or_ambiguous_xml(mutation):
+    args = arguments()
+    raw = args["nominal_xml"]
+    if mutation == "dtd":
+        raw = b'<!DOCTYPE feed [<!ENTITY value "4.0">]>' + raw
+    elif mutation == "broken":
+        raw = raw[:-2]
+    elif mutation == "wrong_root":
+        raw = raw.replace(b'<feed ', b'<other ').replace(b'</feed>', b'</other>')
+    elif mutation == "duplicate_field":
+        raw = raw.replace(b'</m:properties>', b'<d:BC_10YEAR>7</d:BC_10YEAR></m:properties>', 1)
+    else:
+        raw = raw.replace(b'2024-01-02T00:00:00', b'not-a-date')
+    replace_nominal(args, raw)
+    with pytest.raises(TreasuryRatesError):
+        rows(args)
+
+
+def test_namespace_prefix_is_not_identity():
+    args = arguments()
+    original = build_treasury_rates_research_input(**args)
+    raw = args["nominal_xml"].replace(b'xmlns:d=', b'xmlns:value=')
+    raw = raw.replace(b'<d:', b'<value:').replace(b'</d:', b'</value:')
+    replace_nominal(args, raw)
+    assert build_treasury_rates_research_input(**args) == original
+
+
+def test_module_can_be_loaded_without_optional_package_imports():
+    path = Path(__file__).parents[1] / "src/market_signal_sources/providers/treasury_rates.py"
+    spec = importlib.util.spec_from_file_location("isolated_treasury_rates", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    assert module.build_treasury_rates_research_input(**arguments()) == build_treasury_rates_research_input(**arguments())
+
+
+def test_late_same_day_revision_does_not_hide_original_revision():
+    args = arguments()
+    original = build_treasury_rates_research_input(**args)
+    replace_nominal(args, xml([(DATES[0], 4), (DATES[1], 4.2), (DATES[0], "bad")]))
+    args["nominal_evidence"]["rows"].append({"observation_date": DATES[0],
+        "available_at": "2024-01-06T00:00:00Z", "revision_id": "later-revision"})
+    assert build_treasury_rates_research_input(**args) == original
+
+
+def test_row_evidence_cannot_be_reassigned_to_another_observation():
+    args = arguments()
+    args["nominal_evidence"]["rows"][0]["observation_date"] = DATES[1]
+    with pytest.raises(TreasuryRatesError, match="row_evidence_date_mismatch"):
+        rows(args)
+
+
+@pytest.mark.parametrize('attribute', ['m:null="true"', 'm:null="1"', 'm:null="invalid"', 'm:type="Edm.Double"'])
+def test_observation_date_metadata_cannot_claim_a_null_or_nondatetime(attribute):
+    args = arguments()
+    raw = args['nominal_xml'].replace(b'<d:NEW_DATE m:type="Edm.DateTime">', f'<d:NEW_DATE {attribute}>'.encode(), 1)
+    replace_nominal(args, raw)
+    with pytest.raises(TreasuryRatesError, match='observation_date_invalid'):
+        rows(args)
+
+
+def test_observation_date_must_be_scalar():
+    args = arguments()
+    raw = args['nominal_xml'].replace(b'2024-01-02T00:00:00</d:NEW_DATE>', b'2024-01-02T00:00:00<d:extra>bad</d:extra></d:NEW_DATE>')
+    replace_nominal(args, raw)
+    with pytest.raises(TreasuryRatesError, match='observation_date_invalid'):
+        rows(args)
+
+
+@pytest.mark.parametrize('payload', [
+    b'<d:BC_10YEAR>4.0<d:extra>999</d:extra></d:BC_10YEAR>',
+    b'<d:BC_10YEAR m:null="1">4.0</d:BC_10YEAR>',
+    b'<d:BC_10YEAR m:null="invalid">4.0</d:BC_10YEAR>',
+])
+def test_malformed_or_null_numeric_content_cannot_use_leading_text(payload):
+    args = arguments()
+    raw = args['nominal_xml'].replace(b'<d:BC_10YEAR m:type="Edm.Double">4.0</d:BC_10YEAR>', payload)
+    replace_nominal(args, raw)
+    assert rows(args)[0]['value'] is None