diff --git a/src/market_signal_sources/providers/treasury_rates.py b/src/market_signal_sources/providers/treasury_rates.py new file mode 100644 index 0000000..0412595 --- /dev/null +++ b/src/market_signal_sources/providers/treasury_rates.py @@ -0,0 +1,179 @@ +"""Offline Treasury nominal/real ten-year XML to the existing QSP research v1. + +The caller supplies bytes and capture evidence; this module has no downloader, +clock, filesystem, registration, or execution authority. Its input evidence is +a declaration, not a license, authenticated capture, or historical PIT proof. + +Official formats and missing-element semantics: +https://home.treasury.gov/treasury-daily-interest-rate-xml-feed +https://home.treasury.gov/developer-notice-xml-changes + +Each evidence mapping has exactly dataset, raw_sha256 and rows. Rows is a sequence aligned to XML entry order, with observation_date, +available_at, received_at and revision_id. Missing row evidence or missing +time/revision fields remain unknown. The raw hash binds this row order; an +explicit observation_date must also match its XML entry. available_at MUST describe evidence of that row revision's +publication, never a date-derived clock, first-seen time or receipt fallback. +The caller must retain original bytes and this evidence outside the QSP v1 +snapshot: v1 has no field for the raw capture hash. A raw capture hash is not a +row revision identifier; appending a future row must not revise earlier rows. +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import date, datetime, timezone +import hashlib +import math +import re +from xml.etree import ElementTree as ET + + +_ATOM = "{http://www.w3.org/2005/Atom}" +_META = "{http://schemas.microsoft.com/ado/2007/08/dataservices/metadata}" +_DATA = "{http://schemas.microsoft.com/ado/2007/08/dataservices}" +_DATE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T00:00:00") +_NUMBER = re.compile(r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[Ee][+-]?[0-9]+)?") +_ROW_FIELDS = {"observation_date", "available_at", "received_at", "revision_id"} + + +class TreasuryRatesError(ValueError): + """Malformed or ambiguous source/capture contract, without input echo.""" + + +def _time(value): + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None + except (ValueError, OverflowError): + return None + + +def _stamp(value): + parsed = _time(value) + return None if parsed is None else parsed.isoformat().replace("+00:00", "Z") + + +def _single(parent, tag): + matches = parent.findall(tag) + if len(matches) > 1: + raise TreasuryRatesError("duplicate_field") + return matches[0] if matches else None + + +def _observation_date(properties): + field = _single(properties, _DATA + "NEW_DATE") + if field is not None and (len(field) or field.attrib.get(_META + "null") not in (None, "false", "0") or field.attrib.get(_META + "type") not in (None, "Edm.DateTime")): + raise TreasuryRatesError("observation_date_invalid") + value = "" if field is None or field.text is None else field.text.strip() + if not _DATE.fullmatch(value): + raise TreasuryRatesError("observation_date_invalid") + try: + return date.fromisoformat(value[:10]) + except ValueError as exc: + raise TreasuryRatesError("observation_date_invalid") from exc + + +def _value(properties, field): + element = _single(properties, _DATA + field) + if element is None or len(element) or element.attrib.get(_META + "null") not in (None, "false", "0"): + return None + if element.attrib.get(_META + "type") not in (None, "Edm.Double", "Edm.Decimal"): + return None + text = (element.text or "").strip() + if not _NUMBER.fullmatch(text): + return None + try: + value = float(text) + return value if math.isfinite(value) else None + except (ValueError, OverflowError): + return None + + +def _series(raw, evidence, dataset, field, decision): + if not isinstance(raw, bytes): + raise TreasuryRatesError("xml_bytes_required") + if not isinstance(evidence, Mapping) or set(evidence) != {"dataset", "raw_sha256", "rows"}: + raise TreasuryRatesError("capture_evidence_invalid") + if evidence["dataset"] != dataset: + raise TreasuryRatesError("dataset_mismatch") + if evidence["raw_sha256"] != hashlib.sha256(raw).hexdigest(): + raise TreasuryRatesError("raw_hash_mismatch") + row_evidence = evidence["rows"] + if not isinstance(row_evidence, Sequence) or isinstance(row_evidence, (str, bytes, bytearray)): + raise TreasuryRatesError("row_evidence_invalid") + try: + text = raw.decode("utf-8") + if "\x00" in text or " decision.date(): + continue + evidence_row = row_evidence[ordinal] if ordinal < len(row_evidence) else {} + if not isinstance(evidence_row, Mapping): + raise TreasuryRatesError("row_evidence_invalid") + available = _time(evidence_row.get("available_at")) + received = _time(evidence_row.get("received_at")) + if (available is not None and available > decision) or (received is not None and received > decision): + continue + if evidence_row and evidence_row.get("observation_date") != observed.isoformat(): + raise TreasuryRatesError("row_evidence_date_mismatch") + if set(evidence_row) - _ROW_FIELDS: + raise TreasuryRatesError("row_evidence_fields_invalid") + if observed in seen: + raise TreasuryRatesError("duplicate_observation") + if previous is not None and observed < previous: + raise TreasuryRatesError("observation_order") + seen.add(observed) + previous = observed + revision = evidence_row.get("revision_id") + if not isinstance(revision, str) or not revision or revision != revision.strip() or len(revision) > 256 or revision == "latest": + revision = None + rows.append({ + "observation_date": observed.isoformat(), + "value": _value(properties, field), + "available_at": _stamp(evidence_row.get("available_at")), + "received_at": _stamp(evidence_row.get("received_at")), + "revision_id": revision, + }) + return {"source_id": "us_treasury.daily_par_yield_curves", + "series_id": dataset + "." + field, + "basis": "treasury_par_yield", "unit": "percent", "rows": rows} + + +def build_treasury_rates_research_input( + *, nominal_xml: bytes, real_xml: bytes, + nominal_evidence: Mapping, real_evidence: Mapping, decision_at: str, +) -> dict: + """Build QSP rates-input.v1; data/evidence gaps remain explicit unknowns. + + This does not generate reported breakeven or run the observer. The owning + research caller must pass the result to QSP's existing v1 observer with its + frozen explicit window/age configuration. All historical/economic/runtime + qualification remains external; no source registry or runtime is changed. + """ + decision = _time(decision_at) + if decision is None: + raise TreasuryRatesError("decision_at_invalid") + return { + "schema_version": "qsl.rates-context-input.research.v1", + "decision_at": decision.isoformat().replace("+00:00", "Z"), + "series": { + "nominal_10y": _series(nominal_xml, nominal_evidence, "daily_treasury_yield_curve", "BC_10YEAR", decision), + "real_10y": _series(real_xml, real_evidence, "daily_treasury_real_yield_curve", "TC_10YEAR", decision), + }, + } diff --git a/tests/test_treasury_rates.py b/tests/test_treasury_rates.py new file mode 100644 index 0000000..aca94d8 --- /dev/null +++ b/tests/test_treasury_rates.py @@ -0,0 +1,250 @@ +"""SYNTHETIC_FIXTURE_ONLY: no downloaded observations.""" +import copy +import hashlib +import importlib.util +from pathlib import Path + +import pytest + +from market_signal_sources.providers.treasury_rates import ( + TreasuryRatesError, + build_treasury_rates_research_input, +) + +DATES = ("2024-01-02", "2024-01-03") +DECISION = "2024-01-05T12:00:00Z" + + +def xml(rows, field="BC_10YEAR"): + entries = "".join( + '' + f'{day}T00:00:00' + + ("" if value is None else f'{value}') + + '' + for day, value in rows + ) + return ('' + + entries + '').encode() + + +def evidence(raw, dataset): + return {"dataset": dataset, "raw_sha256": hashlib.sha256(raw).hexdigest(), + "rows": [{"observation_date": day, "available_at": f"{day}T21:00:00Z", + "received_at": f"{day}T21:01:00Z", + "revision_id": f"SYNTHETIC_FIXTURE_ONLY:{day}:v1"} + for day in DATES]} + + +def arguments(): + nominal = xml(zip(DATES, (4.0, 4.2))) + real = xml(zip(DATES, (1.0, 1.1)), "TC_10YEAR") + return {"nominal_xml": nominal, "real_xml": real, + "nominal_evidence": evidence(nominal, "daily_treasury_yield_curve"), + "real_evidence": evidence(real, "daily_treasury_real_yield_curve"), + "decision_at": DECISION} + + +def replace_nominal(args, raw): + args["nominal_xml"] = raw + args["nominal_evidence"]["raw_sha256"] = hashlib.sha256(raw).hexdigest() + + +def rows(args): + return build_treasury_rates_research_input(**args)["series"]["nominal_10y"]["rows"] + + +def test_existing_qsp_shape_identities_and_no_breakeven(): + args = arguments() + before = copy.deepcopy(args) + result = build_treasury_rates_research_input(**args) + assert args == before + assert result["schema_version"] == "qsl.rates-context-input.research.v1" + assert result["decision_at"] == DECISION + assert set(result) == {"schema_version", "decision_at", "series"} + assert set(result["series"]) == {"nominal_10y", "real_10y"} + nominal, real = result["series"].values() + assert nominal["source_id"] == real["source_id"] == "us_treasury.daily_par_yield_curves" + assert nominal["series_id"] == "daily_treasury_yield_curve.BC_10YEAR" + assert real["series_id"] == "daily_treasury_real_yield_curve.TC_10YEAR" + assert nominal["basis"] == real["basis"] == "treasury_par_yield" + assert nominal["unit"] == real["unit"] == "percent" + assert nominal["rows"][1] == {"observation_date": DATES[1], "value": 4.2, + **args["nominal_evidence"]["rows"][1]} + + +@pytest.mark.parametrize("value", ["-0.25", "0", "4.2"]) +def test_finite_percent_values(value): + args = arguments() + replace_nominal(args, xml(zip(DATES, (value, value)))) + assert rows(args)[0]["value"] == float(value) + + +@pytest.mark.parametrize("value", [None, "NaN", "INF", "-Infinity", "true", "bad", "1e9999", "1_000", ""]) +def test_invalid_or_missing_value_is_not_zero(value): + args = arguments() + replace_nominal(args, xml(zip(DATES, (value, 4.2)))) + assert rows(args)[0]["value"] is None + + +def test_xml_null_and_wrong_maturity_are_missing(): + for raw in [ + xml(zip(DATES, (4.0, 4.2))).replace( + b'4.0', + b''), + xml(zip(DATES, (4.0, 4.2)), "BC_5YEAR"), + ]: + args = arguments() + replace_nominal(args, raw) + assert rows(args)[0]["value"] is None + + +@pytest.mark.parametrize("missing", ["available_at", "received_at", "revision_id"]) +def test_evidence_is_never_inferred(missing): + args = arguments() + del args["nominal_evidence"]["rows"][0][missing] + assert rows(args)[0][missing] is None + if missing == "available_at": + assert rows(args)[0]["received_at"] == f"{DATES[0]}T21:01:00Z" + + +def test_missing_all_row_evidence_remains_unknown(): + args = arguments() + args["nominal_evidence"]["rows"] = [] + assert all(r["available_at"] is r["received_at"] is r["revision_id"] is None for r in rows(args)) + + +@pytest.mark.parametrize("side", ["nominal", "real"]) +def test_raw_hash_mismatch(side): + args = arguments() + args[f"{side}_evidence"]["raw_sha256"] = "0" * 64 + with pytest.raises(TreasuryRatesError, match="raw_hash_mismatch"): + rows(args) + + +def test_dataset_role_mismatch(): + args = arguments() + args["nominal_evidence"]["dataset"] = "daily_treasury_real_yield_curve" + with pytest.raises(TreasuryRatesError, match="dataset_mismatch"): + rows(args) + + +@pytest.mark.parametrize("raw,reason", [ + (xml([(DATES[0], 4), (DATES[0], 7), (DATES[1], 4.2)]), "duplicate_observation"), + (xml([(DATES[1], 4.2), (DATES[0], 4)]), "observation_order"), +]) +def test_visible_conflicting_revisions_and_order_are_not_repaired(raw, reason): + args = arguments() + replace_nominal(args, raw) + args["nominal_evidence"]["rows"] = [] + with pytest.raises(TreasuryRatesError, match=reason): + rows(args) + + +@pytest.mark.parametrize("selector", ["future_date", "available_at", "received_at"]) +def test_future_or_late_append_cannot_change_past_output(selector): + args = arguments() + original = build_treasury_rates_research_input(**args) + day = "2024-01-06" if selector == "future_date" else "2024-01-04" + replace_nominal(args, xml([(DATES[0], 4), (DATES[1], 4.2), (day, "bad")])) + ev = {"unexpected": object()} + if selector != "future_date": + ev[selector] = "2024-01-06T00:00:00Z" + args["nominal_evidence"]["rows"].append(ev) + assert build_treasury_rates_research_input(**args) == original + + +def test_late_required_row_is_omitted(): + args = arguments() + args["nominal_evidence"]["rows"][1]["received_at"] = "2024-01-06T00:00:00Z" + assert len(rows(args)) == 1 + + +@pytest.mark.parametrize("bad", ["", "2024-01-05", "2024-01-05T12:00:00", "garbage"]) +def test_decision_requires_timezone(bad): + args = arguments() + args["decision_at"] = bad + with pytest.raises(TreasuryRatesError, match="decision_at_invalid"): + rows(args) + + +@pytest.mark.parametrize("mutation", ["dtd", "broken", "wrong_root", "duplicate_field", "bad_date"]) +def test_unsafe_or_ambiguous_xml(mutation): + args = arguments() + raw = args["nominal_xml"] + if mutation == "dtd": + raw = b']>' + raw + elif mutation == "broken": + raw = raw[:-2] + elif mutation == "wrong_root": + raw = raw.replace(b'', b'') + elif mutation == "duplicate_field": + raw = raw.replace(b'', b'7', 1) + else: + raw = raw.replace(b'2024-01-02T00:00:00', b'not-a-date') + replace_nominal(args, raw) + with pytest.raises(TreasuryRatesError): + rows(args) + + +def test_namespace_prefix_is_not_identity(): + args = arguments() + original = build_treasury_rates_research_input(**args) + raw = args["nominal_xml"].replace(b'xmlns:d=', b'xmlns:value=') + raw = raw.replace(b'', f''.encode(), 1) + replace_nominal(args, raw) + with pytest.raises(TreasuryRatesError, match='observation_date_invalid'): + rows(args) + + +def test_observation_date_must_be_scalar(): + args = arguments() + raw = args['nominal_xml'].replace(b'2024-01-02T00:00:00', b'2024-01-02T00:00:00bad') + replace_nominal(args, raw) + with pytest.raises(TreasuryRatesError, match='observation_date_invalid'): + rows(args) + + +@pytest.mark.parametrize('payload', [ + b'4.0999', + b'4.0', + b'4.0', +]) +def test_malformed_or_null_numeric_content_cannot_use_leading_text(payload): + args = arguments() + raw = args['nominal_xml'].replace(b'4.0', payload) + replace_nominal(args, raw) + assert rows(args)[0]['value'] is None