From e16083f1aba5656c0a81d92b511edee860200d3f Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Sat, 29 Aug 2026 05:49:31 +0530 Subject: [PATCH 1/7] fix(model): wire openai_compatible configuration across trainer, eval, and config pipeline --- scripts/eval_only.py | 131 ++++++++++- scripts/train.py | 42 +++- skillopt/config.py | 18 ++ skillopt/engine/trainer.py | 21 ++ skillopt/model/__init__.py | 12 + skillopt/model/openai_compatible_backend.py | 18 +- tests/test_openai_compatible_backend.py | 16 ++ tests/test_openai_compatible_config.py | 248 ++++++++++++++++++++ 8 files changed, 494 insertions(+), 12 deletions(-) create mode 100644 tests/test_openai_compatible_config.py diff --git a/scripts/eval_only.py b/scripts/eval_only.py index abe44905..8b6fbb63 100644 --- a/scripts/eval_only.py +++ b/scripts/eval_only.py @@ -38,6 +38,7 @@ configure_copilot_exec, configure_cursor_exec, configure_minimax_chat, + configure_openai_compatible, configure_qwen_chat, set_optimizer_backend, set_optimizer_deployment, @@ -161,7 +162,24 @@ def parse_args() -> argparse.Namespace: # Legacy flat overrides p.add_argument("--env", type=str) p.add_argument("--backend", type=str, - choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "copilot", "copilot_chat", "copilot_exec", "minimax", "minimax_chat"]) + choices=[ + "azure_openai", + "codex", + "codex_exec", + "claude", + "claude_chat", + "claude_code_exec", + "cursor", + "cursor_exec", + "copilot", + "copilot_chat", + "copilot_exec", + "qwen", + "qwen_chat", + "minimax", + "minimax_chat", + "openai_compatible", + ]) p.add_argument("--optimizer_model", type=str) p.add_argument("--target_model", type=str) p.add_argument("--optimizer_backend", type=str) @@ -189,6 +207,45 @@ def parse_args() -> argparse.Namespace: p.add_argument("--target_azure_openai_auth_mode", type=str) p.add_argument("--target_azure_openai_ad_scope", type=str) p.add_argument("--target_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--qwen_chat_base_url", type=str) + p.add_argument("--qwen_chat_api_key", type=str) + p.add_argument("--qwen_chat_temperature", type=float) + p.add_argument("--qwen_chat_timeout_seconds", type=float) + p.add_argument("--qwen_chat_max_tokens", type=int) + p.add_argument("--qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--qwen_chat_thinking_mode", type=str) + p.add_argument("--optimizer_qwen_chat_base_url", type=str) + p.add_argument("--optimizer_qwen_chat_api_key", type=str) + p.add_argument("--optimizer_qwen_chat_temperature", type=float) + p.add_argument("--optimizer_qwen_chat_timeout_seconds", type=float) + p.add_argument("--optimizer_qwen_chat_max_tokens", type=int) + p.add_argument("--optimizer_qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--optimizer_qwen_chat_thinking_mode", type=str) + p.add_argument("--target_qwen_chat_base_url", type=str) + p.add_argument("--target_qwen_chat_api_key", type=str) + p.add_argument("--target_qwen_chat_temperature", type=float) + p.add_argument("--target_qwen_chat_timeout_seconds", type=float) + p.add_argument("--target_qwen_chat_max_tokens", type=int) + p.add_argument("--target_qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--target_qwen_chat_thinking_mode", type=str) + p.add_argument("--openai_compatible_base_url", type=str) + p.add_argument("--openai_compatible_api_key", type=str) + p.add_argument("--openai_compatible_model", type=str) + p.add_argument("--openai_compatible_temperature", type=float) + p.add_argument("--openai_compatible_timeout_seconds", type=float) + p.add_argument("--openai_compatible_max_tokens", type=int) + p.add_argument("--optimizer_openai_compatible_base_url", type=str) + p.add_argument("--optimizer_openai_compatible_api_key", type=str) + p.add_argument("--optimizer_openai_compatible_model", type=str) + p.add_argument("--optimizer_openai_compatible_temperature", type=float) + p.add_argument("--optimizer_openai_compatible_timeout_seconds", type=float) + p.add_argument("--optimizer_openai_compatible_max_tokens", type=int) + p.add_argument("--target_openai_compatible_base_url", type=str) + p.add_argument("--target_openai_compatible_api_key", type=str) + p.add_argument("--target_openai_compatible_model", type=str) + p.add_argument("--target_openai_compatible_temperature", type=float) + p.add_argument("--target_openai_compatible_timeout_seconds", type=float) + p.add_argument("--target_openai_compatible_max_tokens", type=int) p.add_argument("--codex_exec_path", type=str) p.add_argument("--codex_exec_sandbox", type=str) p.add_argument("--codex_exec_profile", type=str) @@ -244,10 +301,8 @@ def parse_args() -> argparse.Namespace: return p.parse_args() -def main() -> None: - args = parse_args() - - from skillopt.config import load_config as _load, flatten_config, is_structured +def load_config(args: argparse.Namespace) -> dict: + from skillopt.config import flatten_config, is_structured, load_config as _load cfg = _load(args.config, overrides=args.cfg_options) structured = is_structured(cfg) @@ -286,6 +341,45 @@ def main() -> None: "target_azure_openai_auth_mode": "model.target_azure_openai_auth_mode", "target_azure_openai_ad_scope": "model.target_azure_openai_ad_scope", "target_azure_openai_managed_identity_client_id": "model.target_azure_openai_managed_identity_client_id", + "qwen_chat_base_url": "model.qwen_chat_base_url", + "qwen_chat_api_key": "model.qwen_chat_api_key", + "qwen_chat_temperature": "model.qwen_chat_temperature", + "qwen_chat_timeout_seconds": "model.qwen_chat_timeout_seconds", + "qwen_chat_max_tokens": "model.qwen_chat_max_tokens", + "qwen_chat_enable_thinking": "model.qwen_chat_enable_thinking", + "qwen_chat_thinking_mode": "model.qwen_chat_thinking_mode", + "optimizer_qwen_chat_base_url": "model.optimizer_qwen_chat_base_url", + "optimizer_qwen_chat_api_key": "model.optimizer_qwen_chat_api_key", + "optimizer_qwen_chat_temperature": "model.optimizer_qwen_chat_temperature", + "optimizer_qwen_chat_timeout_seconds": "model.optimizer_qwen_chat_timeout_seconds", + "optimizer_qwen_chat_max_tokens": "model.optimizer_qwen_chat_max_tokens", + "optimizer_qwen_chat_enable_thinking": "model.optimizer_qwen_chat_enable_thinking", + "optimizer_qwen_chat_thinking_mode": "model.optimizer_qwen_chat_thinking_mode", + "target_qwen_chat_base_url": "model.target_qwen_chat_base_url", + "target_qwen_chat_api_key": "model.target_qwen_chat_api_key", + "target_qwen_chat_temperature": "model.target_qwen_chat_temperature", + "target_qwen_chat_timeout_seconds": "model.target_qwen_chat_timeout_seconds", + "target_qwen_chat_max_tokens": "model.target_qwen_chat_max_tokens", + "target_qwen_chat_enable_thinking": "model.target_qwen_chat_enable_thinking", + "target_qwen_chat_thinking_mode": "model.target_qwen_chat_thinking_mode", + "openai_compatible_base_url": "model.openai_compatible_base_url", + "openai_compatible_api_key": "model.openai_compatible_api_key", + "openai_compatible_model": "model.openai_compatible_model", + "openai_compatible_temperature": "model.openai_compatible_temperature", + "openai_compatible_timeout_seconds": "model.openai_compatible_timeout_seconds", + "openai_compatible_max_tokens": "model.openai_compatible_max_tokens", + "optimizer_openai_compatible_base_url": "model.optimizer_openai_compatible_base_url", + "optimizer_openai_compatible_api_key": "model.optimizer_openai_compatible_api_key", + "optimizer_openai_compatible_model": "model.optimizer_openai_compatible_model", + "optimizer_openai_compatible_temperature": "model.optimizer_openai_compatible_temperature", + "optimizer_openai_compatible_timeout_seconds": "model.optimizer_openai_compatible_timeout_seconds", + "optimizer_openai_compatible_max_tokens": "model.optimizer_openai_compatible_max_tokens", + "target_openai_compatible_base_url": "model.target_openai_compatible_base_url", + "target_openai_compatible_api_key": "model.target_openai_compatible_api_key", + "target_openai_compatible_model": "model.target_openai_compatible_model", + "target_openai_compatible_temperature": "model.target_openai_compatible_temperature", + "target_openai_compatible_timeout_seconds": "model.target_openai_compatible_timeout_seconds", + "target_openai_compatible_max_tokens": "model.target_openai_compatible_max_tokens", "codex_exec_path": "model.codex_exec_path", "codex_exec_sandbox": "model.codex_exec_sandbox", "codex_exec_profile": "model.codex_exec_profile", @@ -466,6 +560,12 @@ def _set_role(key: str, value: str) -> None: cfg["out_root"] = os.path.join("outputs", f"eval_{env}_{model}_{ts}") cfg["out_root"] = os.path.abspath(cfg["out_root"]) + return cfg + + +def main() -> None: + args = parse_args() + cfg = load_config(args) out_root = cfg["out_root"] os.makedirs(out_root, exist_ok=True) @@ -501,6 +601,7 @@ def _set_role(key: str, value: str) -> None: cfg.get("target_azure_openai_managed_identity_client_id") or None ), ) + backend = cfg.get("model_backend") or cfg.get("target_backend") or "azure_openai" set_optimizer_backend(cfg.get("optimizer_backend", "openai_chat")) set_target_backend(cfg.get("target_backend", "openai_chat")) set_optimizer_deployment(cfg.get("optimizer_model", default_model_for_backend(backend))) @@ -554,6 +655,26 @@ def _set_role(key: str, value: str) -> None: minimax_model_cfg = cfg.get("minimax_model") if minimax_model_cfg and cfg.get("target_backend") == "minimax_chat": set_target_deployment(str(minimax_model_cfg)) + configure_openai_compatible( + base_url=cfg.get("openai_compatible_base_url") or None, + api_key=cfg.get("openai_compatible_api_key") or None, + model=cfg.get("openai_compatible_model") or None, + temperature=cfg.get("openai_compatible_temperature"), + timeout_seconds=cfg.get("openai_compatible_timeout_seconds"), + max_tokens=cfg.get("openai_compatible_max_tokens"), + optimizer_base_url=cfg.get("optimizer_openai_compatible_base_url") or None, + optimizer_api_key=cfg.get("optimizer_openai_compatible_api_key") or None, + optimizer_model=cfg.get("optimizer_openai_compatible_model") or None, + optimizer_temperature=cfg.get("optimizer_openai_compatible_temperature"), + optimizer_timeout_seconds=cfg.get("optimizer_openai_compatible_timeout_seconds"), + optimizer_max_tokens=cfg.get("optimizer_openai_compatible_max_tokens"), + target_base_url=cfg.get("target_openai_compatible_base_url") or None, + target_api_key=cfg.get("target_openai_compatible_api_key") or None, + target_model=cfg.get("target_openai_compatible_model") or None, + target_temperature=cfg.get("target_openai_compatible_temperature"), + target_timeout_seconds=cfg.get("target_openai_compatible_timeout_seconds"), + target_max_tokens=cfg.get("target_openai_compatible_max_tokens"), + ) set_reasoning_effort(cfg.get("reasoning_effort", "") or None) # Build adapter diff --git a/scripts/train.py b/scripts/train.py index f8f6cbf5..53db7532 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -202,6 +202,24 @@ def parse_args() -> argparse.Namespace: p.add_argument("--minimax_temperature", type=float) p.add_argument("--minimax_max_tokens", type=int) p.add_argument("--minimax_enable_thinking", type=_BOOL) + p.add_argument("--openai_compatible_base_url", type=str) + p.add_argument("--openai_compatible_api_key", type=str) + p.add_argument("--openai_compatible_model", type=str) + p.add_argument("--openai_compatible_temperature", type=float) + p.add_argument("--openai_compatible_timeout_seconds", type=float) + p.add_argument("--openai_compatible_max_tokens", type=int) + p.add_argument("--optimizer_openai_compatible_base_url", type=str) + p.add_argument("--optimizer_openai_compatible_api_key", type=str) + p.add_argument("--optimizer_openai_compatible_model", type=str) + p.add_argument("--optimizer_openai_compatible_temperature", type=float) + p.add_argument("--optimizer_openai_compatible_timeout_seconds", type=float) + p.add_argument("--optimizer_openai_compatible_max_tokens", type=int) + p.add_argument("--target_openai_compatible_base_url", type=str) + p.add_argument("--target_openai_compatible_api_key", type=str) + p.add_argument("--target_openai_compatible_model", type=str) + p.add_argument("--target_openai_compatible_temperature", type=float) + p.add_argument("--target_openai_compatible_timeout_seconds", type=float) + p.add_argument("--target_openai_compatible_max_tokens", type=int) p.add_argument("--codex_exec_path", type=str) p.add_argument("--codex_exec_sandbox", type=str) p.add_argument("--codex_exec_profile", type=str) @@ -426,6 +444,24 @@ def _retired_option_sources( "minimax_temperature": "model.minimax_temperature", "minimax_max_tokens": "model.minimax_max_tokens", "minimax_enable_thinking": "model.minimax_enable_thinking", + "openai_compatible_base_url": "model.openai_compatible_base_url", + "openai_compatible_api_key": "model.openai_compatible_api_key", + "openai_compatible_model": "model.openai_compatible_model", + "openai_compatible_temperature": "model.openai_compatible_temperature", + "openai_compatible_timeout_seconds": "model.openai_compatible_timeout_seconds", + "openai_compatible_max_tokens": "model.openai_compatible_max_tokens", + "optimizer_openai_compatible_base_url": "model.optimizer_openai_compatible_base_url", + "optimizer_openai_compatible_api_key": "model.optimizer_openai_compatible_api_key", + "optimizer_openai_compatible_model": "model.optimizer_openai_compatible_model", + "optimizer_openai_compatible_temperature": "model.optimizer_openai_compatible_temperature", + "optimizer_openai_compatible_timeout_seconds": "model.optimizer_openai_compatible_timeout_seconds", + "optimizer_openai_compatible_max_tokens": "model.optimizer_openai_compatible_max_tokens", + "target_openai_compatible_base_url": "model.target_openai_compatible_base_url", + "target_openai_compatible_api_key": "model.target_openai_compatible_api_key", + "target_openai_compatible_model": "model.target_openai_compatible_model", + "target_openai_compatible_temperature": "model.target_openai_compatible_temperature", + "target_openai_compatible_timeout_seconds": "model.target_openai_compatible_timeout_seconds", + "target_openai_compatible_max_tokens": "model.target_openai_compatible_max_tokens", "codex_exec_path": "model.codex_exec_path", "codex_exec_sandbox": "model.codex_exec_sandbox", "codex_exec_profile": "model.codex_exec_profile", @@ -484,7 +520,8 @@ def _retired_option_sources( def load_config(args: argparse.Namespace) -> dict: """Load config with _base_ inheritance, then apply CLI overrides.""" import warnings - from skillopt.config import load_config as _load, flatten_config, is_structured + + from skillopt.config import flatten_config, is_structured, load_config as _load # F08: Warn when API keys are supplied on the CLI. Keep the replacement # guidance specific to each backend and, where applicable, each role. @@ -509,6 +546,9 @@ def load_config(args: argparse.Namespace) -> dict: "optimizer_qwen_chat_api_key": "OPTIMIZER_QWEN_CHAT_API_KEY", "target_qwen_chat_api_key": "TARGET_QWEN_CHAT_API_KEY", "minimax_api_key": "MINIMAX_API_KEY", + "openai_compatible_api_key": "OPENAI_COMPATIBLE_API_KEY", + "optimizer_openai_compatible_api_key": "OPTIMIZER_OPENAI_COMPATIBLE_API_KEY", + "target_openai_compatible_api_key": "TARGET_OPENAI_COMPATIBLE_API_KEY", } for _cli_key, _guidance in _credential_guidance.items(): if getattr(args, _cli_key, None): diff --git a/skillopt/config.py b/skillopt/config.py index e1fdec04..12230d31 100644 --- a/skillopt/config.py +++ b/skillopt/config.py @@ -123,6 +123,24 @@ "model.minimax_temperature": "minimax_temperature", "model.minimax_max_tokens": "minimax_max_tokens", "model.minimax_enable_thinking": "minimax_enable_thinking", + "model.openai_compatible_base_url": "openai_compatible_base_url", + "model.openai_compatible_api_key": "openai_compatible_api_key", + "model.openai_compatible_model": "openai_compatible_model", + "model.openai_compatible_temperature": "openai_compatible_temperature", + "model.openai_compatible_timeout_seconds": "openai_compatible_timeout_seconds", + "model.openai_compatible_max_tokens": "openai_compatible_max_tokens", + "model.optimizer_openai_compatible_base_url": "optimizer_openai_compatible_base_url", + "model.optimizer_openai_compatible_api_key": "optimizer_openai_compatible_api_key", + "model.optimizer_openai_compatible_model": "optimizer_openai_compatible_model", + "model.optimizer_openai_compatible_temperature": "optimizer_openai_compatible_temperature", + "model.optimizer_openai_compatible_timeout_seconds": "optimizer_openai_compatible_timeout_seconds", + "model.optimizer_openai_compatible_max_tokens": "optimizer_openai_compatible_max_tokens", + "model.target_openai_compatible_base_url": "target_openai_compatible_base_url", + "model.target_openai_compatible_api_key": "target_openai_compatible_api_key", + "model.target_openai_compatible_model": "target_openai_compatible_model", + "model.target_openai_compatible_temperature": "target_openai_compatible_temperature", + "model.target_openai_compatible_timeout_seconds": "target_openai_compatible_timeout_seconds", + "model.target_openai_compatible_max_tokens": "target_openai_compatible_max_tokens", "train.num_epochs": "num_epochs", "train.train_size": "train_size", "train.steps_per_epoch": "steps_per_epoch", diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py index 520a90eb..d461345a 100644 --- a/skillopt/engine/trainer.py +++ b/skillopt/engine/trainer.py @@ -67,6 +67,7 @@ configure_copilot_exec, configure_cursor_exec, configure_minimax_chat, + configure_openai_compatible, configure_qwen_chat, get_qwen_thinking_modes, get_token_summary, @@ -832,6 +833,26 @@ def _build_eval_env(split: str, env_num: int, seed: int): minimax_model_cfg = cfg.get("minimax_model") if minimax_model_cfg and cfg.get("target_backend") == "minimax_chat": set_target_deployment(str(minimax_model_cfg)) + configure_openai_compatible( + base_url=cfg.get("openai_compatible_base_url") or None, + api_key=cfg.get("openai_compatible_api_key") or None, + model=cfg.get("openai_compatible_model") or None, + temperature=cfg.get("openai_compatible_temperature"), + timeout_seconds=cfg.get("openai_compatible_timeout_seconds"), + max_tokens=cfg.get("openai_compatible_max_tokens"), + optimizer_base_url=cfg.get("optimizer_openai_compatible_base_url") or None, + optimizer_api_key=cfg.get("optimizer_openai_compatible_api_key") or None, + optimizer_model=cfg.get("optimizer_openai_compatible_model") or None, + optimizer_temperature=cfg.get("optimizer_openai_compatible_temperature"), + optimizer_timeout_seconds=cfg.get("optimizer_openai_compatible_timeout_seconds"), + optimizer_max_tokens=cfg.get("optimizer_openai_compatible_max_tokens"), + target_base_url=cfg.get("target_openai_compatible_base_url") or None, + target_api_key=cfg.get("target_openai_compatible_api_key") or None, + target_model=cfg.get("target_openai_compatible_model") or None, + target_temperature=cfg.get("target_openai_compatible_temperature"), + target_timeout_seconds=cfg.get("target_openai_compatible_timeout_seconds"), + target_max_tokens=cfg.get("target_openai_compatible_max_tokens"), + ) _configure_trace_to_optimizer_gates(target_backend, cfg) reasoning = cfg.get("reasoning_effort", "") or None set_reasoning_effort(reasoning) diff --git a/skillopt/model/__init__.py b/skillopt/model/__init__.py index 9033bc98..1d0edd02 100644 --- a/skillopt/model/__init__.py +++ b/skillopt/model/__init__.py @@ -764,9 +764,15 @@ def configure_openai_compatible( optimizer_base_url: str | None = None, optimizer_api_key: str | None = None, optimizer_model: str | None = None, + optimizer_temperature: float | str | None = None, + optimizer_timeout_seconds: float | str | None = None, + optimizer_max_tokens: int | str | None = None, target_base_url: str | None = None, target_api_key: str | None = None, target_model: str | None = None, + target_temperature: float | str | None = None, + target_timeout_seconds: float | str | None = None, + target_max_tokens: int | str | None = None, ) -> None: _openai_compat.configure_openai_compatible( base_url=base_url, @@ -778,9 +784,15 @@ def configure_openai_compatible( optimizer_base_url=optimizer_base_url, optimizer_api_key=optimizer_api_key, optimizer_model=optimizer_model, + optimizer_temperature=optimizer_temperature, + optimizer_timeout_seconds=optimizer_timeout_seconds, + optimizer_max_tokens=optimizer_max_tokens, target_base_url=target_base_url, target_api_key=target_api_key, target_model=target_model, + target_temperature=target_temperature, + target_timeout_seconds=target_timeout_seconds, + target_max_tokens=target_max_tokens, ) diff --git a/skillopt/model/openai_compatible_backend.py b/skillopt/model/openai_compatible_backend.py index 7462be41..4fef4ce1 100644 --- a/skillopt/model/openai_compatible_backend.py +++ b/skillopt/model/openai_compatible_backend.py @@ -372,9 +372,15 @@ def configure_openai_compatible( optimizer_base_url: str | None = None, optimizer_api_key: str | None = None, optimizer_model: str | None = None, + optimizer_temperature: float | str | None = None, + optimizer_timeout_seconds: float | str | None = None, + optimizer_max_tokens: int | str | None = None, target_base_url: str | None = None, target_api_key: str | None = None, target_model: str | None = None, + target_temperature: float | str | None = None, + target_timeout_seconds: float | str | None = None, + target_max_tokens: int | str | None = None, ) -> None: """Configure the generic OpenAI-compatible backend at runtime. @@ -400,9 +406,9 @@ def configure_openai_compatible( base_url=optimizer_base_url if optimizer_base_url is not None else base_url, api_key=optimizer_api_key if optimizer_api_key is not None else api_key, deployment=optimizer_model if optimizer_model is not None else model, - temperature=temperature, - timeout_seconds=timeout_seconds, - max_tokens=max_tokens, + temperature=optimizer_temperature if optimizer_temperature is not None else temperature, + timeout_seconds=optimizer_timeout_seconds if optimizer_timeout_seconds is not None else timeout_seconds, + max_tokens=optimizer_max_tokens if optimizer_max_tokens is not None else max_tokens, ) _update_config( TARGET_CONFIG, @@ -410,9 +416,9 @@ def configure_openai_compatible( base_url=target_base_url if target_base_url is not None else base_url, api_key=target_api_key if target_api_key is not None else api_key, deployment=target_model if target_model is not None else model, - temperature=temperature, - timeout_seconds=timeout_seconds, - max_tokens=max_tokens, + temperature=target_temperature if target_temperature is not None else temperature, + timeout_seconds=target_timeout_seconds if target_timeout_seconds is not None else timeout_seconds, + max_tokens=target_max_tokens if target_max_tokens is not None else max_tokens, ) _reset_clients() diff --git a/tests/test_openai_compatible_backend.py b/tests/test_openai_compatible_backend.py index 4482e206..f5abe41b 100644 --- a/tests/test_openai_compatible_backend.py +++ b/tests/test_openai_compatible_backend.py @@ -49,20 +49,36 @@ def test_configure_preserves_role_specific_values() -> None: base_url="https://shared.example/v1", api_key="shared-key", model="shared-model", + temperature=0.5, + timeout_seconds=60, + max_tokens=2048, optimizer_base_url="https://optimizer.example/v1", optimizer_api_key="optimizer-key", optimizer_model="optimizer-model", + optimizer_temperature=0.2, + optimizer_timeout_seconds=120, + optimizer_max_tokens=4096, target_base_url="https://target.example/v1", target_api_key="target-key", target_model="target-model", + target_temperature=0.7, + target_timeout_seconds=90, + target_max_tokens=8192, ) assert backend.OPTIMIZER_CONFIG.base_url == "https://optimizer.example/v1" assert backend.OPTIMIZER_CONFIG.api_key == "optimizer-key" assert backend.OPTIMIZER_CONFIG.deployment == "optimizer-model" + assert backend.OPTIMIZER_CONFIG.temperature == 0.2 + assert backend.OPTIMIZER_CONFIG.timeout_seconds == 120.0 + assert backend.OPTIMIZER_CONFIG.max_tokens == 4096 + assert backend.TARGET_CONFIG.base_url == "https://target.example/v1" assert backend.TARGET_CONFIG.api_key == "target-key" assert backend.TARGET_CONFIG.deployment == "target-model" + assert backend.TARGET_CONFIG.temperature == 0.7 + assert backend.TARGET_CONFIG.timeout_seconds == 90.0 + assert backend.TARGET_CONFIG.max_tokens == 8192 def test_optimizer_and_target_route_to_their_own_clients(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_openai_compatible_config.py b/tests/test_openai_compatible_config.py new file mode 100644 index 00000000..da464c13 --- /dev/null +++ b/tests/test_openai_compatible_config.py @@ -0,0 +1,248 @@ +"""Tests for OpenAI-compatible backend configuration pipeline.""" + +from __future__ import annotations + +import tempfile +from unittest import mock + +import pytest +import yaml + +import scripts.eval_only as eval_only_script +import scripts.train as train_script +from skillopt.config import flatten_config, load_config + + +def test_flatten_config_maps_openai_compatible_keys() -> None: + structured_cfg = { + "model": { + "backend": "openai_compatible", + "openai_compatible_base_url": "https://api.deepseek.com/v1", + "openai_compatible_api_key": "sk-deepseek-test", + "openai_compatible_model": "deepseek-chat", + "openai_compatible_temperature": 0.3, + "openai_compatible_timeout_seconds": 180.0, + "openai_compatible_max_tokens": 4096, + "optimizer_openai_compatible_base_url": "https://api.together.xyz/v1", + "optimizer_openai_compatible_api_key": "sk-together-test", + "optimizer_openai_compatible_model": "together-model", + "optimizer_openai_compatible_temperature": 0.1, + "optimizer_openai_compatible_timeout_seconds": 240.0, + "optimizer_openai_compatible_max_tokens": 8192, + "target_openai_compatible_base_url": "https://api.groq.com/openai/v1", + "target_openai_compatible_api_key": "sk-groq-test", + "target_openai_compatible_model": "groq-model", + "target_openai_compatible_temperature": 0.7, + "target_openai_compatible_timeout_seconds": 90.0, + "target_openai_compatible_max_tokens": 2048, + }, + "train": {"num_epochs": 1}, + "env": {"name": "searchqa"}, + } + + flat = flatten_config(structured_cfg) + + assert flat["model_backend"] == "openai_compatible" + assert flat["openai_compatible_base_url"] == "https://api.deepseek.com/v1" + assert flat["openai_compatible_api_key"] == "sk-deepseek-test" + assert flat["openai_compatible_model"] == "deepseek-chat" + assert flat["openai_compatible_temperature"] == 0.3 + assert flat["openai_compatible_timeout_seconds"] == 180.0 + assert flat["openai_compatible_max_tokens"] == 4096 + + assert flat["optimizer_openai_compatible_base_url"] == "https://api.together.xyz/v1" + assert flat["optimizer_openai_compatible_api_key"] == "sk-together-test" + assert flat["optimizer_openai_compatible_model"] == "together-model" + assert flat["optimizer_openai_compatible_temperature"] == 0.1 + assert flat["optimizer_openai_compatible_timeout_seconds"] == 240.0 + assert flat["optimizer_openai_compatible_max_tokens"] == 8192 + + assert flat["target_openai_compatible_base_url"] == "https://api.groq.com/openai/v1" + assert flat["target_openai_compatible_api_key"] == "sk-groq-test" + assert flat["target_openai_compatible_model"] == "groq-model" + assert flat["target_openai_compatible_temperature"] == 0.7 + assert flat["target_openai_compatible_timeout_seconds"] == 90.0 + assert flat["target_openai_compatible_max_tokens"] == 2048 + + +def test_load_config_with_cfg_options_overrides() -> None: + raw = { + "model": { + "backend": "openai_compatible", + "openai_compatible_base_url": "http://localhost:11434/v1", + "openai_compatible_model": "llama3", + }, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + path = f.name + + overrides = [ + "model.openai_compatible_base_url=http://custom-host:8000/v1", + "model.openai_compatible_temperature=0.8", + ] + cfg = load_config(path, overrides=overrides) + flat = flatten_config(cfg) + + assert flat["openai_compatible_base_url"] == "http://custom-host:8000/v1" + assert flat["openai_compatible_temperature"] == 0.8 + assert flat["openai_compatible_model"] == "llama3" + + +def test_train_script_cli_to_structured_mapping() -> None: + cli_args = [ + "--config", "configs/base.yaml", + "--openai_compatible_base_url", "https://api.openai-compat.com/v1", + "--openai_compatible_model", "custom-model", + "--openai_compatible_temperature", "0.4", + "--openai_compatible_timeout_seconds", "150", + "--openai_compatible_max_tokens", "3000", + "--optimizer_openai_compatible_base_url", "https://api.opt.com/v1", + "--target_openai_compatible_base_url", "https://api.target.com/v1", + ] + + with mock.patch("sys.argv", ["train.py"] + cli_args): + args = train_script.parse_args() + + assert args.openai_compatible_base_url == "https://api.openai-compat.com/v1" + assert args.openai_compatible_model == "custom-model" + assert args.openai_compatible_temperature == 0.4 + assert args.openai_compatible_timeout_seconds == 150.0 + assert args.openai_compatible_max_tokens == 3000 + assert args.optimizer_openai_compatible_base_url == "https://api.opt.com/v1" + assert args.target_openai_compatible_base_url == "https://api.target.com/v1" + + for flag in [ + "openai_compatible_base_url", + "openai_compatible_api_key", + "openai_compatible_model", + "openai_compatible_temperature", + "openai_compatible_timeout_seconds", + "openai_compatible_max_tokens", + "optimizer_openai_compatible_base_url", + "optimizer_openai_compatible_api_key", + "optimizer_openai_compatible_model", + "optimizer_openai_compatible_temperature", + "optimizer_openai_compatible_timeout_seconds", + "optimizer_openai_compatible_max_tokens", + "target_openai_compatible_base_url", + "target_openai_compatible_api_key", + "target_openai_compatible_model", + "target_openai_compatible_temperature", + "target_openai_compatible_timeout_seconds", + "target_openai_compatible_max_tokens", + ]: + assert flag in train_script._LEGACY_TO_STRUCTURED + assert train_script._LEGACY_TO_STRUCTURED[flag] == f"model.{flag}" + + +def test_eval_only_parse_args_and_map() -> None: + raw = { + "model": { + "backend": "openai_compatible", + }, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + cli_args = [ + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + "--openai_compatible_base_url", "https://eval.example/v1", + "--openai_compatible_model", "eval-model", + "--target_openai_compatible_temperature", "0.6", + ] + + with mock.patch("sys.argv", ["eval_only.py"] + cli_args): + args = eval_only_script.parse_args() + cfg = eval_only_script.load_config(args) + + assert args.backend == "openai_compatible" + assert args.openai_compatible_base_url == "https://eval.example/v1" + assert args.openai_compatible_model == "eval-model" + assert args.target_openai_compatible_temperature == 0.6 + assert cfg["openai_compatible_base_url"] == "https://eval.example/v1" + assert cfg["openai_compatible_model"] == "eval-model" + assert cfg["target_openai_compatible_temperature"] == 0.6 + + +def test_train_script_credential_warnings_for_openai_compatible() -> None: + raw = { + "model": {"backend": "openai_compatible"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + cli_args = [ + "--config", config_path, + "--openai_compatible_api_key", "sk-secret-test", + ] + + with mock.patch("sys.argv", ["train.py"] + cli_args): + with pytest.deprecated_call(match="OPENAI_COMPATIBLE_API_KEY"): + train_script.load_config(train_script.parse_args()) + + +def test_trainer_initialization_configures_openai_compatible(monkeypatch: pytest.MonkeyPatch) -> None: + import skillopt.engine.trainer as trainer_mod + from skillopt.envs.base import EnvAdapter + + configured_kwargs: dict = {} + + def fake_configure(**kwargs): + configured_kwargs.update(kwargs) + + monkeypatch.setattr(trainer_mod, "configure_openai_compatible", fake_configure) + + class EarlyExit(Exception): + pass + + def stop_after_model_config(*args, **kwargs): + raise EarlyExit() + + monkeypatch.setattr(trainer_mod, "_configure_trace_to_optimizer_gates", stop_after_model_config) + + cfg = { + "model_backend": "openai_compatible", + "optimizer_backend": "openai_compatible", + "target_backend": "openai_compatible", + "optimizer_model": "opt-model", + "target_model": "target-model", + "openai_compatible_base_url": "https://api.test.com/v1", + "openai_compatible_api_key": "test-key", + "openai_compatible_model": "test-model", + "openai_compatible_temperature": 0.5, + "openai_compatible_timeout_seconds": 120.0, + "openai_compatible_max_tokens": 4096, + "skill_init": "skills/empty.md", + "num_epochs": 1, + "train_size": 1, + "batch_size": 1, + "accumulation": 1, + "merge_batch_size": 2, + "edit_budget": 2, + "seed": 42, + "out_root": "/tmp/out", + } + + mock_adapter = mock.create_autospec(EnvAdapter, instance=True) + mock_adapter.requires_ray.return_value = False + mock_adapter.get_dataloader.return_value = None + + trainer = trainer_mod.ReflACTTrainer(cfg, mock_adapter) + with pytest.raises(EarlyExit): + trainer.train() + + assert configured_kwargs["base_url"] == "https://api.test.com/v1" + assert configured_kwargs["api_key"] == "test-key" + assert configured_kwargs["model"] == "test-model" + assert configured_kwargs["temperature"] == 0.5 + assert configured_kwargs["timeout_seconds"] == 120.0 + assert configured_kwargs["max_tokens"] == 4096 + From 451f2e8f7b7ffe7fd34d0466b3d345199b2dcfbf Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Sun, 30 Aug 2026 23:33:49 +0530 Subject: [PATCH 2/7] fix(model): resolve entry-point gaps in train and eval-only pipelines --- scripts/eval_only.py | 42 +++++ scripts/train.py | 22 ++- tests/test_openai_compatible_config.py | 212 +++++++++++++++++++++++++ 3 files changed, 275 insertions(+), 1 deletion(-) diff --git a/scripts/eval_only.py b/scripts/eval_only.py index 8b6fbb63..46508cc5 100644 --- a/scripts/eval_only.py +++ b/scripts/eval_only.py @@ -493,6 +493,9 @@ def _set_role(key: str, value: str) -> None: elif backend == "copilot_exec": _set_role("optimizer_backend", "openai_chat") _set_role("target_backend", "copilot_exec") + elif backend == "qwen_chat": + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "qwen_chat") elif backend == "minimax_chat": _set_role("optimizer_backend", "openai_chat") _set_role("target_backend", "minimax_chat") @@ -518,6 +521,22 @@ def _set_role(key: str, value: str) -> None: and not _has_model_override("model.optimizer", "optimizer_model") ): cfg["optimizer_model"] = default_model_for_backend("claude_code_exec") + if cfg.get("optimizer_backend") == "qwen_chat": + if ( + str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + cfg["optimizer_model"] = default_model_for_backend("qwen_chat") + if cfg.get("optimizer_backend") == "openai_compatible": + if ( + str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + cfg["optimizer_model"] = ( + cfg.get("optimizer_openai_compatible_model") + or cfg.get("openai_compatible_model") + or default_model_for_backend("openai_compatible") + ) if cfg.get("target_backend") == "claude_chat": if ( str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS @@ -543,6 +562,12 @@ def _set_role(key: str, value: str) -> None: ): # Copilot CLI model IDs are independent of Azure deployment names. cfg["target_model"] = "" + if cfg.get("target_backend") == "qwen_chat": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = default_model_for_backend("qwen_chat") if cfg.get("target_backend") == "minimax_chat": if ( str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS @@ -552,6 +577,16 @@ def _set_role(key: str, value: str) -> None: cfg.get("minimax_model") or default_model_for_backend("minimax_chat") ) + if cfg.get("target_backend") == "openai_compatible": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = ( + cfg.get("target_openai_compatible_model") + or cfg.get("openai_compatible_model") + or default_model_for_backend("openai_compatible") + ) if not cfg.get("out_root"): env = cfg.get("env", "unknown") @@ -636,6 +671,13 @@ def main() -> None: max_tokens=cfg.get("qwen_chat_max_tokens"), enable_thinking=cfg.get("qwen_chat_enable_thinking"), thinking_mode=cfg.get("qwen_chat_thinking_mode"), + optimizer_base_url=cfg.get("optimizer_qwen_chat_base_url") or None, + optimizer_api_key=cfg.get("optimizer_qwen_chat_api_key") or None, + optimizer_temperature=cfg.get("optimizer_qwen_chat_temperature"), + optimizer_timeout_seconds=cfg.get("optimizer_qwen_chat_timeout_seconds"), + optimizer_max_tokens=cfg.get("optimizer_qwen_chat_max_tokens"), + optimizer_enable_thinking=cfg.get("optimizer_qwen_chat_enable_thinking"), + optimizer_thinking_mode=cfg.get("optimizer_qwen_chat_thinking_mode"), target_base_url=cfg.get("target_qwen_chat_base_url") or None, target_api_key=cfg.get("target_qwen_chat_api_key") or None, target_temperature=cfg.get("target_qwen_chat_temperature"), diff --git a/scripts/train.py b/scripts/train.py index 53db7532..af162659 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -144,7 +144,7 @@ def parse_args() -> argparse.Namespace: # Legacy flat CLI overrides (still work, prefer --cfg-options for new usage) p.add_argument("--env", type=str) p.add_argument("--backend", type=str, - choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "copilot", "copilot_chat", "copilot_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"]) + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "copilot", "copilot_chat", "copilot_exec", "qwen", "qwen_chat", "minimax", "minimax_chat", "openai_compatible"]) p.add_argument("--optimizer_model", type=str) p.add_argument("--target_model", type=str) p.add_argument("--optimizer_backend", type=str) @@ -744,6 +744,16 @@ def _set_role(key: str, value: str) -> None: and not _has_model_override("model.optimizer", "optimizer_model") ): flat["optimizer_model"] = default_model_for_backend("qwen_chat") + if flat.get("optimizer_backend") == "openai_compatible": + if ( + str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + flat["optimizer_model"] = ( + flat.get("optimizer_openai_compatible_model") + or flat.get("openai_compatible_model") + or default_model_for_backend("openai_compatible") + ) if flat.get("target_backend") == "claude_chat": if ( str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS @@ -784,6 +794,16 @@ def _set_role(key: str, value: str) -> None: flat.get("minimax_model") or default_model_for_backend("minimax_chat") ) + if flat.get("target_backend") == "openai_compatible": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = ( + flat.get("target_openai_compatible_model") + or flat.get("openai_compatible_model") + or default_model_for_backend("openai_compatible") + ) # Auto-generate output root if not flat.get("out_root"): diff --git a/tests/test_openai_compatible_config.py b/tests/test_openai_compatible_config.py index da464c13..bec6e709 100644 --- a/tests/test_openai_compatible_config.py +++ b/tests/test_openai_compatible_config.py @@ -246,3 +246,215 @@ def stop_after_model_config(*args, **kwargs): assert configured_kwargs["timeout_seconds"] == 120.0 assert configured_kwargs["max_tokens"] == 4096 + +def test_train_script_backend_choices_accepts_openai_compatible() -> None: + raw = { + "model": {"backend": "azure_openai", "optimizer": "gpt-5.5", "target": "gpt-5.5"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + cli_args = [ + "--config", config_path, + "--backend", "openai_compatible", + ] + with mock.patch("sys.argv", ["train.py"] + cli_args): + args = train_script.parse_args() + flat = train_script.load_config(args) + + assert args.backend == "openai_compatible" + assert flat["optimizer_backend"] == "openai_compatible" + assert flat["target_backend"] == "openai_compatible" + assert flat["optimizer_model"] == "gpt-4o-mini" + assert flat["target_model"] == "gpt-4o-mini" + + +def test_train_script_openai_compatible_model_precedence() -> None: + raw = { + "model": {"backend": "azure_openai", "optimizer": "gpt-5.5", "target": "gpt-5.5"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + # 1. Shared model override + with mock.patch("sys.argv", [ + "train.py", + "--config", config_path, + "--backend", "openai_compatible", + "--openai_compatible_model", "deepseek-chat", + ]): + flat = train_script.load_config(train_script.parse_args()) + assert flat["optimizer_model"] == "deepseek-chat" + assert flat["target_model"] == "deepseek-chat" + + # 2. Per-role compatible model overrides + with mock.patch("sys.argv", [ + "train.py", + "--config", config_path, + "--backend", "openai_compatible", + "--openai_compatible_model", "fallback-shared", + "--optimizer_openai_compatible_model", "deepseek-coder", + "--target_openai_compatible_model", "deepseek-v3", + ]): + flat = train_script.load_config(train_script.parse_args()) + assert flat["optimizer_model"] == "deepseek-coder" + assert flat["target_model"] == "deepseek-v3" + + # 3. Explicit per-role model overrides take highest precedence + with mock.patch("sys.argv", [ + "train.py", + "--config", config_path, + "--backend", "openai_compatible", + "--openai_compatible_model", "fallback-shared", + "--optimizer_model", "explicit-opt-model", + "--target_model", "explicit-tgt-model", + ]): + flat = train_script.load_config(train_script.parse_args()) + assert flat["optimizer_model"] == "explicit-opt-model" + assert flat["target_model"] == "explicit-tgt-model" + + +@pytest.mark.parametrize("backend_flag", ["qwen", "qwen_chat"]) +def test_eval_only_qwen_role_and_model_resolution(backend_flag: str) -> None: + raw = { + "model": {"backend": "azure_openai", "optimizer": "gpt-5.5", "target": "gpt-5.5"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + cli_args = [ + "--config", config_path, + "--skill", "skills/test.md", + "--backend", backend_flag, + ] + with mock.patch("sys.argv", ["eval_only.py"] + cli_args): + args = eval_only_script.parse_args() + cfg = eval_only_script.load_config(args) + + assert cfg["optimizer_backend"] == "openai_chat" + assert cfg["target_backend"] == "qwen_chat" + assert cfg["optimizer_model"] == "gpt-5.5" # openai_chat keeps base sentinel or default + assert cfg["target_model"] == "Qwen/Qwen3.5-4B" # normalized from gpt-5.5 sentinel + + +def test_eval_only_openai_compatible_model_precedence() -> None: + raw = { + "model": {"backend": "azure_openai", "optimizer": "gpt-5.5", "target": "gpt-5.5"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + # 1. Default fallback + with mock.patch("sys.argv", [ + "eval_only.py", + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + ]): + cfg = eval_only_script.load_config(eval_only_script.parse_args()) + assert cfg["optimizer_model"] == "gpt-4o-mini" + assert cfg["target_model"] == "gpt-4o-mini" + + # 2. Shared model override + with mock.patch("sys.argv", [ + "eval_only.py", + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + "--openai_compatible_model", "deepseek-chat", + ]): + cfg = eval_only_script.load_config(eval_only_script.parse_args()) + assert cfg["optimizer_model"] == "deepseek-chat" + assert cfg["target_model"] == "deepseek-chat" + + # 3. Per-role compatible model overrides + with mock.patch("sys.argv", [ + "eval_only.py", + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + "--optimizer_openai_compatible_model", "opt-compat-model", + "--target_openai_compatible_model", "tgt-compat-model", + ]): + cfg = eval_only_script.load_config(eval_only_script.parse_args()) + assert cfg["optimizer_model"] == "opt-compat-model" + assert cfg["target_model"] == "tgt-compat-model" + + # 4. Explicit per-role overrides take precedence + with mock.patch("sys.argv", [ + "eval_only.py", + "--config", config_path, + "--skill", "skills/test.md", + "--backend", "openai_compatible", + "--openai_compatible_model", "fallback-model", + "--optimizer_model", "explicit-opt", + "--target_model", "explicit-tgt", + ]): + cfg = eval_only_script.load_config(eval_only_script.parse_args()) + assert cfg["optimizer_model"] == "explicit-opt" + assert cfg["target_model"] == "explicit-tgt" + + +def test_eval_only_forwards_optimizer_qwen_chat_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: + raw = { + "model": {"backend": "qwen_chat"}, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as f: + f.write("# Dummy Skill\n") + skill_path = f.name + + cli_args = [ + "--config", config_path, + "--skill", skill_path, + "--optimizer_qwen_chat_base_url", "http://opt-qwen:8000/v1", + "--optimizer_qwen_chat_api_key", "opt-qwen-key", + "--optimizer_qwen_chat_temperature", "0.2", + "--optimizer_qwen_chat_timeout_seconds", "90", + "--optimizer_qwen_chat_max_tokens", "2048", + "--optimizer_qwen_chat_enable_thinking", "true", + "--optimizer_qwen_chat_thinking_mode", "deep", + "--target_qwen_chat_base_url", "http://tgt-qwen:8000/v1", + ] + + qwen_kwargs: dict = {} + + def fake_configure_qwen(**kwargs): + qwen_kwargs.update(kwargs) + + monkeypatch.setattr(eval_only_script, "configure_qwen_chat", fake_configure_qwen) + + class StopExecution(Exception): + pass + + def stop_at_adapter(*args, **kwargs): + raise StopExecution() + + monkeypatch.setattr(eval_only_script, "get_adapter", stop_at_adapter) + + with mock.patch("sys.argv", ["eval_only.py"] + cli_args): + with pytest.raises(StopExecution): + eval_only_script.main() + + assert qwen_kwargs["optimizer_base_url"] == "http://opt-qwen:8000/v1" + assert qwen_kwargs["optimizer_api_key"] == "opt-qwen-key" + assert qwen_kwargs["optimizer_temperature"] == 0.2 + assert qwen_kwargs["optimizer_timeout_seconds"] == 90.0 + assert qwen_kwargs["optimizer_max_tokens"] == 2048 + assert qwen_kwargs["optimizer_enable_thinking"] is True + assert qwen_kwargs["optimizer_thinking_mode"] == "deep" + assert qwen_kwargs["target_base_url"] == "http://tgt-qwen:8000/v1" + + From d1ed2f2da738ea9b3800181c962454f521f3ec8d Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Mon, 31 Aug 2026 13:55:39 +0530 Subject: [PATCH 3/7] fix(model): initialize MiniMax OPTIMIZER_DEPLOYMENT at import and add fresh-import regression --- skillopt/model/__init__.py | 3 +- skillopt/model/minimax_backend.py | 46 ++++++++++- tests/test_minimax_backend.py | 130 ++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 4 deletions(-) diff --git a/skillopt/model/__init__.py b/skillopt/model/__init__.py index 1d0edd02..28ade6fb 100644 --- a/skillopt/model/__init__.py +++ b/skillopt/model/__init__.py @@ -326,7 +326,7 @@ def chat_optimizer_messages( timeout=timeout, ) if get_optimizer_backend() == "minimax_chat": - return _minimax.chat_target_messages( + return _minimax.chat_optimizer_messages( messages=messages, max_completion_tokens=max_completion_tokens, retries=retries, @@ -821,5 +821,6 @@ def set_optimizer_deployment(deployment: str) -> None: _claude.set_optimizer_deployment(deployment) _claude_code.set_optimizer_deployment(deployment) _qwen.set_optimizer_deployment(deployment) + _minimax.set_optimizer_deployment(deployment) _openai_compat.set_optimizer_deployment(deployment) _codex.set_optimizer_deployment(deployment) diff --git a/skillopt/model/minimax_backend.py b/skillopt/model/minimax_backend.py index 9736a533..3737d292 100644 --- a/skillopt/model/minimax_backend.py +++ b/skillopt/model/minimax_backend.py @@ -69,6 +69,10 @@ def base_url_for_region(region: str | None) -> str: "TARGET_DEPLOYMENT", default_model_for_backend("minimax_chat"), ) +OPTIMIZER_DEPLOYMENT = os.environ.get( + "OPTIMIZER_DEPLOYMENT", + default_model_for_backend("minimax_chat"), +) # Models whose thinking cannot actually be turned off. Per MiniMax's # OpenAI-compatible docs the M2.x family accepts ``{"type": "disabled"}`` but @@ -304,7 +308,7 @@ def chat_target( stage: str = "target", reasoning_effort: str | None = None, timeout: float | None = None, -) -> tuple[str, dict[int]]: +) -> tuple[str, dict[str, int]]: del reasoning_effort messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] return _chat_messages_impl( @@ -312,6 +316,7 @@ def chat_target( max_completion_tokens, retries, stage, + deployment=TARGET_DEPLOYMENT, timeout=timeout, ) @@ -324,7 +329,7 @@ def chat_optimizer( stage: str = "optimizer", reasoning_effort: str | None = None, timeout: float | None = None, -) -> tuple[str, dict[int]]: +) -> tuple[str, dict[str, int]]: """Optimizer chat call. Backend stores the trained skill; uses the same MiniMax-proxied OpenAI-compat endpoint as `chat_target`. Added in the parallel-training fix; previously missing in skillopt 0.2.0's @@ -332,12 +337,14 @@ def chat_optimizer( (Azure) and produced "[skip] no usable patches" for any user running optimizer+target on `minimax_chat`. """ + del reasoning_effort messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] return _chat_messages_impl( messages, max_completion_tokens, retries, stage, + deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout, ) @@ -363,6 +370,33 @@ def chat_target_messages( tools=tools, tool_choice=tool_choice, return_message=return_message, + deployment=TARGET_DEPLOYMENT, + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + del reasoning_effort + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout, ) @@ -382,4 +416,10 @@ def set_reasoning_effort(effort: str | None) -> None: def set_target_deployment(deployment: str) -> None: global TARGET_DEPLOYMENT TARGET_DEPLOYMENT = deployment or default_model_for_backend("minimax_chat") - os.environ["TARGET_DEPLOYMENT"] = TARGET_DEPLOYMENT \ No newline at end of file + os.environ["TARGET_DEPLOYMENT"] = TARGET_DEPLOYMENT + + +def set_optimizer_deployment(deployment: str) -> None: + global OPTIMIZER_DEPLOYMENT + OPTIMIZER_DEPLOYMENT = deployment or default_model_for_backend("minimax_chat") + os.environ["OPTIMIZER_DEPLOYMENT"] = OPTIMIZER_DEPLOYMENT \ No newline at end of file diff --git a/tests/test_minimax_backend.py b/tests/test_minimax_backend.py index ec049a21..8c914c7b 100644 --- a/tests/test_minimax_backend.py +++ b/tests/test_minimax_backend.py @@ -2,8 +2,10 @@ from __future__ import annotations +import importlib import importlib.util import json +import os import sys import types from collections.abc import Iterator @@ -71,6 +73,7 @@ def minimax_backend() -> Iterator[Any]: snapshot = { "ENABLE_THINKING": backend.ENABLE_THINKING, "TARGET_DEPLOYMENT": backend.TARGET_DEPLOYMENT, + "OPTIMIZER_DEPLOYMENT": backend.OPTIMIZER_DEPLOYMENT, "API_KEY": backend.API_KEY, "BASE_URL": backend.BASE_URL, } @@ -91,6 +94,8 @@ def test_default_deployment_is_current_model(minimax_backend: Any) -> None: from skillopt.model.common import default_model_for_backend assert default_model_for_backend("minimax_chat") == "MiniMax-M3" + assert minimax_backend.TARGET_DEPLOYMENT == "MiniMax-M3" + assert minimax_backend.OPTIMIZER_DEPLOYMENT == "MiniMax-M3" def test_always_on_model_sends_adaptive_not_disabled( @@ -169,3 +174,128 @@ def test_unknown_deployment_defaults_to_adaptive( minimax_backend.chat_target("system", "user", retries=1) assert recorder.calls[0]["payload"]["thinking"] == {"type": "adaptive"} + + +def test_chat_optimizer_and_target_use_respective_deployments( + monkeypatch: pytest.MonkeyPatch, minimax_backend: Any +) -> None: + minimax_backend.TARGET_DEPLOYMENT = "MiniMax-Target-Model" + minimax_backend.OPTIMIZER_DEPLOYMENT = "MiniMax-Optimizer-Model" + recorder = _record_urlopen(monkeypatch, minimax_backend) + + minimax_backend.chat_target("sys_target", "user_target", retries=1) + minimax_backend.chat_optimizer("sys_opt", "user_opt", retries=1) + minimax_backend.chat_target_messages([{"role": "user", "content": "msg_target"}], retries=1) + minimax_backend.chat_optimizer_messages([{"role": "user", "content": "msg_opt"}], retries=1) + + assert recorder.calls[0]["payload"]["model"] == "MiniMax-Target-Model" + assert recorder.calls[1]["payload"]["model"] == "MiniMax-Optimizer-Model" + assert recorder.calls[2]["payload"]["model"] == "MiniMax-Target-Model" + assert recorder.calls[3]["payload"]["model"] == "MiniMax-Optimizer-Model" + + +def test_set_optimizer_and_target_deployment(minimax_backend: Any) -> None: + minimax_backend.set_target_deployment("MiniMax-New-Target") + assert minimax_backend.TARGET_DEPLOYMENT == "MiniMax-New-Target" + assert os.environ.get("TARGET_DEPLOYMENT") == "MiniMax-New-Target" + + minimax_backend.set_optimizer_deployment("MiniMax-New-Optimizer") + assert minimax_backend.OPTIMIZER_DEPLOYMENT == "MiniMax-New-Optimizer" + assert os.environ.get("OPTIMIZER_DEPLOYMENT") == "MiniMax-New-Optimizer" + + +def test_timeout_forwarded_to_urlopen( + monkeypatch: pytest.MonkeyPatch, minimax_backend: Any +) -> None: + recorder = _record_urlopen(monkeypatch, minimax_backend) + + minimax_backend.chat_target("system", "user", retries=1, timeout=42.5) + minimax_backend.chat_optimizer("system", "user", retries=1, timeout=55.0) + minimax_backend.chat_target_messages([{"role": "user", "content": "hi"}], retries=1, timeout=60.0) + minimax_backend.chat_optimizer_messages([{"role": "user", "content": "hi"}], retries=1, timeout=75.0) + + assert recorder.calls[0]["timeout"] == 42.5 + assert recorder.calls[1]["timeout"] == 55.0 + assert recorder.calls[2]["timeout"] == 60.0 + assert recorder.calls[3]["timeout"] == 75.0 + + +def test_fresh_import_optimizer_calls_without_setter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression: calling chat_optimizer or chat_optimizer_messages without calling + + set_optimizer_deployment() on a fresh import must not raise NameError for + OPTIMIZER_DEPLOYMENT. + """ + _install_openai_stub() + monkeypatch.delenv("OPTIMIZER_DEPLOYMENT", raising=False) + monkeypatch.delenv("TARGET_DEPLOYMENT", raising=False) + + from skillopt.model import minimax_backend as backend + + module = importlib.reload(backend) + recorder = _record_urlopen(monkeypatch, module) + + text, usage = module.chat_optimizer("system prompt", "user query", retries=1) + assert text == "answer" + assert recorder.calls[0]["payload"]["model"] == "MiniMax-M3" + + msg, usage_msg = module.chat_optimizer_messages( + [{"role": "user", "content": "user query"}], retries=1 + ) + assert msg == "answer" + assert recorder.calls[1]["payload"]["model"] == "MiniMax-M3" + + +def test_fresh_import_respects_optimizer_deployment_env( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_openai_stub() + monkeypatch.setenv("OPTIMIZER_DEPLOYMENT", "MiniMax-Env-Optimizer") + monkeypatch.setenv("TARGET_DEPLOYMENT", "MiniMax-Env-Target") + + from skillopt.model import minimax_backend as backend + + module = importlib.reload(backend) + recorder = _record_urlopen(monkeypatch, module) + + module.chat_optimizer("system prompt", "user query", retries=1) + module.chat_optimizer_messages( + [{"role": "user", "content": "user query"}], retries=1 + ) + module.chat_target("system prompt", "user query", retries=1) + module.chat_target_messages( + [{"role": "user", "content": "user query"}], retries=1 + ) + + assert recorder.calls[0]["payload"]["model"] == "MiniMax-Env-Optimizer" + assert recorder.calls[1]["payload"]["model"] == "MiniMax-Env-Optimizer" + assert recorder.calls[2]["payload"]["model"] == "MiniMax-Env-Target" + assert recorder.calls[3]["payload"]["model"] == "MiniMax-Env-Target" + + +def test_model_dispatcher_chat_optimizer_messages_minimax( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_openai_stub() + import skillopt.model as model + from skillopt.model import backend_config, minimax_backend as backend + + module = importlib.reload(backend) + recorder = _record_urlopen(monkeypatch, module) + + backend_config.set_optimizer_backend("minimax_chat") + model.set_optimizer_deployment("MiniMax-Custom-Opt") + + res, _ = model.chat_optimizer("system", "user", retries=1, timeout=99) + assert res == "answer" + assert recorder.calls[0]["payload"]["model"] == "MiniMax-Custom-Opt" + assert recorder.calls[0]["timeout"] == 99 + + res_msg, _ = model.chat_optimizer_messages( + [{"role": "user", "content": "test"}], retries=1, timeout=88 + ) + assert res_msg == "answer" + assert recorder.calls[1]["payload"]["model"] == "MiniMax-Custom-Opt" + assert recorder.calls[1]["timeout"] == 88 From 0e88e3852b109dfe14cff3772fc05112b10e9977 Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Sun, 6 Sep 2026 05:08:24 +0530 Subject: [PATCH 4/7] Fix model precedence and fixture isolation issues --- scripts/eval_only.py | 12 ++++- skillopt/engine/trainer.py | 13 ++++- tests/test_minimax_backend.py | 8 ++- tests/test_openai_compatible_config.py | 73 ++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/scripts/eval_only.py b/scripts/eval_only.py index 46508cc5..f898cc4f 100644 --- a/scripts/eval_only.py +++ b/scripts/eval_only.py @@ -706,13 +706,21 @@ def main() -> None: max_tokens=cfg.get("openai_compatible_max_tokens"), optimizer_base_url=cfg.get("optimizer_openai_compatible_base_url") or None, optimizer_api_key=cfg.get("optimizer_openai_compatible_api_key") or None, - optimizer_model=cfg.get("optimizer_openai_compatible_model") or None, + optimizer_model=( + cfg["optimizer_model"] + if cfg.get("optimizer_backend") == "openai_compatible" + else (cfg.get("optimizer_openai_compatible_model") or None) + ), optimizer_temperature=cfg.get("optimizer_openai_compatible_temperature"), optimizer_timeout_seconds=cfg.get("optimizer_openai_compatible_timeout_seconds"), optimizer_max_tokens=cfg.get("optimizer_openai_compatible_max_tokens"), target_base_url=cfg.get("target_openai_compatible_base_url") or None, target_api_key=cfg.get("target_openai_compatible_api_key") or None, - target_model=cfg.get("target_openai_compatible_model") or None, + target_model=( + cfg["target_model"] + if cfg.get("target_backend") == "openai_compatible" + else (cfg.get("target_openai_compatible_model") or None) + ), target_temperature=cfg.get("target_openai_compatible_temperature"), target_timeout_seconds=cfg.get("target_openai_compatible_timeout_seconds"), target_max_tokens=cfg.get("target_openai_compatible_max_tokens"), diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py index d461345a..8910ea38 100644 --- a/skillopt/engine/trainer.py +++ b/skillopt/engine/trainer.py @@ -842,13 +842,22 @@ def _build_eval_env(split: str, env_num: int, seed: int): max_tokens=cfg.get("openai_compatible_max_tokens"), optimizer_base_url=cfg.get("optimizer_openai_compatible_base_url") or None, optimizer_api_key=cfg.get("optimizer_openai_compatible_api_key") or None, - optimizer_model=cfg.get("optimizer_openai_compatible_model") or None, + optimizer_model=( + cfg["optimizer_model"] + if cfg.get("optimizer_backend") == "openai_compatible" + else (cfg.get("optimizer_openai_compatible_model") or None) + ), optimizer_temperature=cfg.get("optimizer_openai_compatible_temperature"), optimizer_timeout_seconds=cfg.get("optimizer_openai_compatible_timeout_seconds"), optimizer_max_tokens=cfg.get("optimizer_openai_compatible_max_tokens"), target_base_url=cfg.get("target_openai_compatible_base_url") or None, target_api_key=cfg.get("target_openai_compatible_api_key") or None, - target_model=cfg.get("target_openai_compatible_model") or None, + target_model=( + cfg["target_model"] + if cfg.get("target_backend") == "openai_compatible" + else (cfg.get("target_openai_compatible_model") or None) + ), + target_temperature=cfg.get("target_openai_compatible_temperature"), target_timeout_seconds=cfg.get("target_openai_compatible_timeout_seconds"), target_max_tokens=cfg.get("target_openai_compatible_max_tokens"), diff --git a/tests/test_minimax_backend.py b/tests/test_minimax_backend.py index 8c914c7b..52145584 100644 --- a/tests/test_minimax_backend.py +++ b/tests/test_minimax_backend.py @@ -65,9 +65,13 @@ def _install_openai_stub() -> None: sys.modules["openai"] = openai_stub -@pytest.fixture() -def minimax_backend() -> Iterator[Any]: +@pytest.fixture +def minimax_backend(monkeypatch: pytest.MonkeyPatch) -> Iterator[Any]: _install_openai_stub() + + # Isolate environment variables to avoid cross-test pollution + monkeypatch.setattr(os, "environ", os.environ.copy()) + from skillopt.model import minimax_backend as backend snapshot = { diff --git a/tests/test_openai_compatible_config.py b/tests/test_openai_compatible_config.py index bec6e709..fef17ce1 100644 --- a/tests/test_openai_compatible_config.py +++ b/tests/test_openai_compatible_config.py @@ -458,3 +458,76 @@ def stop_at_adapter(*args, **kwargs): assert qwen_kwargs["target_base_url"] == "http://tgt-qwen:8000/v1" + +def test_trainer_preserves_explicit_role_models(monkeypatch: pytest.MonkeyPatch) -> None: + import skillopt.engine.trainer as trainer_mod + from skillopt.envs.base import EnvAdapter + import skillopt.model.openai_compatible_backend as openai_compat + + class EarlyExit(Exception): + pass + + def stop_after_model_config(*args, **kwargs): + raise EarlyExit() + + monkeypatch.setattr(trainer_mod, "_configure_trace_to_optimizer_gates", stop_after_model_config) + + cfg = { + "model_backend": "openai_compatible", + "optimizer_backend": "openai_compatible", + "target_backend": "openai_compatible", + "optimizer_model": "explicit-optimizer-model", + "target_model": "explicit-target-model", + "openai_compatible_base_url": "https://api.test.com/v1", + "openai_compatible_model": "fallback-shared", + "skill_init": "skills/empty.md", + "num_epochs": 1, + "train_size": 1, + "batch_size": 1, + "accumulation": 1, + "merge_batch_size": 2, + "edit_budget": 2, + "seed": 42, + "out_root": "/tmp/out", + } + + mock_adapter = mock.create_autospec(EnvAdapter, instance=True) + mock_adapter.requires_ray.return_value = False + mock_adapter.get_dataloader.return_value = None + + trainer = trainer_mod.ReflACTTrainer(cfg, mock_adapter) + with pytest.raises(EarlyExit): + trainer.train() + + assert openai_compat.OPTIMIZER_CONFIG.deployment == "explicit-optimizer-model" + assert openai_compat.TARGET_CONFIG.deployment == "explicit-target-model" + + +def test_eval_only_preserves_explicit_role_models(monkeypatch: pytest.MonkeyPatch) -> None: + import skillopt.model.openai_compatible_backend as openai_compat + + cfg = { + "model_backend": "openai_compatible", + "optimizer_backend": "openai_compatible", + "target_backend": "openai_compatible", + "optimizer_model": "eval-optimizer-model", + "target_model": "eval-target-model", + "openai_compatible_base_url": "https://api.test.com/v1", + "openai_compatible_model": "fallback-shared", + } + + class EarlyExit(Exception): + pass + + def stop_after_config(*args, **kwargs): + raise EarlyExit() + + monkeypatch.setattr(eval_only_script, "set_reasoning_effort", stop_after_config) + + with mock.patch("scripts.eval_only.load_config", return_value=cfg): + with mock.patch("scripts.eval_only.parse_args"): + with pytest.raises(EarlyExit): + eval_only_script.main() + + assert openai_compat.OPTIMIZER_CONFIG.deployment == "eval-optimizer-model" + assert openai_compat.TARGET_CONFIG.deployment == "eval-target-model" From 84ec2ee1b56824564881529647a18319e6be49ef Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Sun, 13 Sep 2026 05:11:23 +0530 Subject: [PATCH 5/7] Fix test isolation and mock defects --- tests/test_minimax_backend.py | 3 +++ tests/test_openai_compatible_config.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_minimax_backend.py b/tests/test_minimax_backend.py index 52145584..5786d574 100644 --- a/tests/test_minimax_backend.py +++ b/tests/test_minimax_backend.py @@ -71,8 +71,11 @@ def minimax_backend(monkeypatch: pytest.MonkeyPatch) -> Iterator[Any]: # Isolate environment variables to avoid cross-test pollution monkeypatch.setattr(os, "environ", os.environ.copy()) + monkeypatch.delenv("TARGET_DEPLOYMENT", raising=False) + monkeypatch.delenv("OPTIMIZER_DEPLOYMENT", raising=False) from skillopt.model import minimax_backend as backend + importlib.reload(backend) snapshot = { "ENABLE_THINKING": backend.ENABLE_THINKING, diff --git a/tests/test_openai_compatible_config.py b/tests/test_openai_compatible_config.py index fef17ce1..e3bd2d90 100644 --- a/tests/test_openai_compatible_config.py +++ b/tests/test_openai_compatible_config.py @@ -514,8 +514,13 @@ def test_eval_only_preserves_explicit_role_models(monkeypatch: pytest.MonkeyPatc "target_model": "eval-target-model", "openai_compatible_base_url": "https://api.test.com/v1", "openai_compatible_model": "fallback-shared", + "out_root": "/tmp/out", } + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as f: + f.write("# Dummy Skill\n") + skill_path = f.name + class EarlyExit(Exception): pass @@ -524,10 +529,14 @@ def stop_after_config(*args, **kwargs): monkeypatch.setattr(eval_only_script, "set_reasoning_effort", stop_after_config) + fake_args = mock.Mock() + fake_args.skill = skill_path + with mock.patch("scripts.eval_only.load_config", return_value=cfg): - with mock.patch("scripts.eval_only.parse_args"): + with mock.patch("scripts.eval_only.parse_args", return_value=fake_args): with pytest.raises(EarlyExit): eval_only_script.main() assert openai_compat.OPTIMIZER_CONFIG.deployment == "eval-optimizer-model" assert openai_compat.TARGET_CONFIG.deployment == "eval-target-model" + From f3b053afcae7aac2b2d1755d9cb99249c9b36044 Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Wed, 16 Sep 2026 06:09:12 +0530 Subject: [PATCH 6/7] chore: remove trailing whitespaces --- tests/test_minimax_backend.py | 4 ++-- tests/test_openai_compatible_config.py | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_minimax_backend.py b/tests/test_minimax_backend.py index 5786d574..1ac8f7dc 100644 --- a/tests/test_minimax_backend.py +++ b/tests/test_minimax_backend.py @@ -68,12 +68,12 @@ def _install_openai_stub() -> None: @pytest.fixture def minimax_backend(monkeypatch: pytest.MonkeyPatch) -> Iterator[Any]: _install_openai_stub() - + # Isolate environment variables to avoid cross-test pollution monkeypatch.setattr(os, "environ", os.environ.copy()) monkeypatch.delenv("TARGET_DEPLOYMENT", raising=False) monkeypatch.delenv("OPTIMIZER_DEPLOYMENT", raising=False) - + from skillopt.model import minimax_backend as backend importlib.reload(backend) diff --git a/tests/test_openai_compatible_config.py b/tests/test_openai_compatible_config.py index e3bd2d90..bf9c7b80 100644 --- a/tests/test_openai_compatible_config.py +++ b/tests/test_openai_compatible_config.py @@ -516,17 +516,17 @@ def test_eval_only_preserves_explicit_role_models(monkeypatch: pytest.MonkeyPatc "openai_compatible_model": "fallback-shared", "out_root": "/tmp/out", } - + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as f: f.write("# Dummy Skill\n") skill_path = f.name class EarlyExit(Exception): pass - + def stop_after_config(*args, **kwargs): raise EarlyExit() - + monkeypatch.setattr(eval_only_script, "set_reasoning_effort", stop_after_config) fake_args = mock.Mock() @@ -539,4 +539,3 @@ def stop_after_config(*args, **kwargs): assert openai_compat.OPTIMIZER_CONFIG.deployment == "eval-optimizer-model" assert openai_compat.TARGET_CONFIG.deployment == "eval-target-model" - From dc27040038c247d2d7ba806cebf6cd43d474e75a Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Fri, 2 Oct 2026 00:33:39 +0530 Subject: [PATCH 7/7] fix(model): resolve missing role models in minimal YAML and harden runtime configuration - Populate missing role models in load_config() for minimal YAML configurations across scripts/eval_only.py and scripts/train.py, falling back through role-specific -> shared -> default_model_for_backend. - Guard role-model lookups with .get() in eval_only.py and trainer.py to prevent KeyError during deployment and runtime configuration. - Preserve #255 timeout forwarding and environment isolation in Minimax backend. - Add real-entry regression tests verifying minimal YAML resolution, CLI overrides precedence, and runtime wire configs. --- scripts/eval_only.py | 29 +++--- scripts/train.py | 29 +++--- skillopt/engine/trainer.py | 78 +++++++------- tests/test_minimax_backend.py | 4 +- tests/test_openai_compatible_config.py | 137 ++++++++++++++++++++++++- 5 files changed, 210 insertions(+), 67 deletions(-) diff --git a/scripts/eval_only.py b/scripts/eval_only.py index f898cc4f..e92c05c4 100644 --- a/scripts/eval_only.py +++ b/scripts/eval_only.py @@ -302,7 +302,8 @@ def parse_args() -> argparse.Namespace: def load_config(args: argparse.Namespace) -> dict: - from skillopt.config import flatten_config, is_structured, load_config as _load + from skillopt.config import flatten_config, is_structured + from skillopt.config import load_config as _load cfg = _load(args.config, overrides=args.cfg_options) structured = is_structured(cfg) @@ -511,25 +512,25 @@ def _set_role(key: str, value: str) -> None: if cfg.get("optimizer_backend") == "claude_chat": if ( - str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("optimizer_model") or str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.optimizer", "optimizer_model") ): cfg["optimizer_model"] = default_model_for_backend("claude_chat") if cfg.get("optimizer_backend") == "claude_code_exec": if ( - str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("optimizer_model") or str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.optimizer", "optimizer_model") ): cfg["optimizer_model"] = default_model_for_backend("claude_code_exec") if cfg.get("optimizer_backend") == "qwen_chat": if ( - str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("optimizer_model") or str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.optimizer", "optimizer_model") ): cfg["optimizer_model"] = default_model_for_backend("qwen_chat") if cfg.get("optimizer_backend") == "openai_compatible": if ( - str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("optimizer_model") or str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.optimizer", "optimizer_model") ): cfg["optimizer_model"] = ( @@ -539,38 +540,38 @@ def _set_role(key: str, value: str) -> None: ) if cfg.get("target_backend") == "claude_chat": if ( - str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("target_model") or str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): cfg["target_model"] = default_model_for_backend("claude_chat") if cfg.get("target_backend") == "claude_code_exec": if ( - str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("target_model") or str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): cfg["target_model"] = default_model_for_backend("claude_chat") if cfg.get("target_backend") == "cursor_exec": if ( - str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("target_model") or str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): cfg["target_model"] = default_model_for_backend("cursor_exec") if cfg.get("target_backend") == "copilot_exec": if ( - str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("target_model") or str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): # Copilot CLI model IDs are independent of Azure deployment names. cfg["target_model"] = "" if cfg.get("target_backend") == "qwen_chat": if ( - str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("target_model") or str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): cfg["target_model"] = default_model_for_backend("qwen_chat") if cfg.get("target_backend") == "minimax_chat": if ( - str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("target_model") or str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): cfg["target_model"] = ( @@ -579,7 +580,7 @@ def _set_role(key: str, value: str) -> None: ) if cfg.get("target_backend") == "openai_compatible": if ( - str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not cfg.get("target_model") or str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): cfg["target_model"] = ( @@ -707,7 +708,7 @@ def main() -> None: optimizer_base_url=cfg.get("optimizer_openai_compatible_base_url") or None, optimizer_api_key=cfg.get("optimizer_openai_compatible_api_key") or None, optimizer_model=( - cfg["optimizer_model"] + cfg.get("optimizer_model") if cfg.get("optimizer_backend") == "openai_compatible" else (cfg.get("optimizer_openai_compatible_model") or None) ), @@ -717,7 +718,7 @@ def main() -> None: target_base_url=cfg.get("target_openai_compatible_base_url") or None, target_api_key=cfg.get("target_openai_compatible_api_key") or None, target_model=( - cfg["target_model"] + cfg.get("target_model") if cfg.get("target_backend") == "openai_compatible" else (cfg.get("target_openai_compatible_model") or None) ), diff --git a/scripts/train.py b/scripts/train.py index af162659..630f0562 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -521,7 +521,8 @@ def load_config(args: argparse.Namespace) -> dict: """Load config with _base_ inheritance, then apply CLI overrides.""" import warnings - from skillopt.config import flatten_config, is_structured, load_config as _load + from skillopt.config import flatten_config, is_structured + from skillopt.config import load_config as _load # F08: Warn when API keys are supplied on the CLI. Keep the replacement # guidance specific to each backend and, where applicable, each role. @@ -728,25 +729,25 @@ def _set_role(key: str, value: str) -> None: if flat.get("optimizer_backend") == "claude_chat": if ( - str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("optimizer_model") or str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.optimizer", "optimizer_model") ): flat["optimizer_model"] = default_model_for_backend("claude_chat") if flat.get("optimizer_backend") == "claude_code_exec": if ( - str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("optimizer_model") or str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.optimizer", "optimizer_model") ): flat["optimizer_model"] = default_model_for_backend("claude_code_exec") if flat.get("optimizer_backend") == "qwen_chat": if ( - str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("optimizer_model") or str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.optimizer", "optimizer_model") ): flat["optimizer_model"] = default_model_for_backend("qwen_chat") if flat.get("optimizer_backend") == "openai_compatible": if ( - str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("optimizer_model") or str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.optimizer", "optimizer_model") ): flat["optimizer_model"] = ( @@ -756,38 +757,38 @@ def _set_role(key: str, value: str) -> None: ) if flat.get("target_backend") == "claude_chat": if ( - str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("target_model") or str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): flat["target_model"] = default_model_for_backend("claude_chat") if flat.get("target_backend") == "claude_code_exec": if ( - str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("target_model") or str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): flat["target_model"] = default_model_for_backend("claude_chat") if flat.get("target_backend") == "cursor_exec": if ( - str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("target_model") or str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): flat["target_model"] = default_model_for_backend("cursor_exec") if flat.get("target_backend") == "copilot_exec": if ( - str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("target_model") or str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): # Copilot CLI model IDs are independent of Azure deployment names. flat["target_model"] = "" if flat.get("target_backend") == "qwen_chat": if ( - str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("target_model") or str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): flat["target_model"] = default_model_for_backend("qwen_chat") if flat.get("target_backend") == "minimax_chat": if ( - str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("target_model") or str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): flat["target_model"] = ( @@ -796,7 +797,7 @@ def _set_role(key: str, value: str) -> None: ) if flat.get("target_backend") == "openai_compatible": if ( - str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + (not flat.get("target_model") or str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS) and not _has_model_override("model.target", "target_model") ): flat["target_model"] = ( @@ -823,7 +824,7 @@ def main() -> None: cfg = load_config(args) print(f"\n{'='*60}") - print(f" SkillOpt — Executive Strategy for Self-Evolving Agent Skills") + print(" SkillOpt — Executive Strategy for Self-Evolving Agent Skills") print(f"{'='*60}") print(f" env: {cfg.get('env')}") print(f" optimizer_model: {cfg.get('optimizer_model')}") @@ -834,7 +835,7 @@ def main() -> None: print(f" rewrite_effort: {cfg.get('rewrite_reasoning_effort') or 'off'}") print(f" epochs: {cfg.get('num_epochs')}") print(f" train_size: {cfg.get('train_size') or 'from dataset'}") - print(f" steps/epoch: auto") + print(" steps/epoch: auto") print(f" batch_size: {cfg.get('batch_size')}") print(f" edit_budget: {cfg.get('edit_budget')}") print(f" lr_scheduler: {cfg.get('lr_scheduler', 'constant')}") diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py index 8910ea38..144e37dc 100644 --- a/skillopt/engine/trainer.py +++ b/skillopt/engine/trainer.py @@ -26,21 +26,45 @@ from skillopt.envs.base import EnvAdapter from skillopt.evaluation.gate import GateResult, evaluate_gate, select_gate_score from skillopt.gradient.aggregate import merge_patches -from skillopt.optimizer.meta_skill import run_meta_skill +from skillopt.model import ( + chat_optimizer, + configure_azure_openai, + configure_claude_code_exec, + configure_codex_exec_from_config, + configure_copilot_chat, + configure_copilot_exec, + configure_cursor_exec, + configure_minimax_chat, + configure_openai_compatible, + configure_qwen_chat, + get_qwen_thinking_modes, + get_token_summary, + set_optimizer_backend, + set_optimizer_deployment, + set_reasoning_effort, + set_target_backend, + set_target_deployment, +) +from skillopt.model.common import default_model_for_backend, normalize_backend_name +from skillopt.optimizer.appendix import ( + _strip_all_appendix_fields, + append_to_appendix_field, + inject_empty_appendix_field, +) +from skillopt.optimizer.appendix import ( + extract_appendix_notes as extract_appendix_notes_from_skill, +) from skillopt.optimizer.clip import rank_and_select from skillopt.optimizer.lr_autonomous import decide_autonomous_learning_rate +from skillopt.optimizer.meta_skill import run_meta_skill from skillopt.optimizer.rewrite import rewrite_skill_from_suggestions from skillopt.optimizer.scheduler import build_scheduler from skillopt.optimizer.skill import apply_patch_with_report -from skillopt.optimizer.appendix import ( - append_to_appendix_field, - extract_appendix_notes as extract_appendix_notes_from_skill, - inject_empty_appendix_field, - _strip_all_appendix_fields, -) from skillopt.optimizer.skill_aware import ( configure_skill_aware_reflection, consolidate_appendix_notes, +) +from skillopt.optimizer.skill_aware import ( extract_appendix_notes as extract_appendix_notes_from_result, ) from skillopt.optimizer.slow_update import ( @@ -58,30 +82,8 @@ payload_label, short_item_summary, ) -from skillopt.model import ( - chat_optimizer, - configure_azure_openai, - configure_claude_code_exec, - configure_codex_exec_from_config, - configure_copilot_chat, - configure_copilot_exec, - configure_cursor_exec, - configure_minimax_chat, - configure_openai_compatible, - configure_qwen_chat, - get_qwen_thinking_modes, - get_token_summary, - reset_token_tracker, - set_reasoning_effort, - set_target_backend, - set_target_deployment, - set_optimizer_backend, - set_optimizer_deployment, -) -from skillopt.model.common import normalize_backend_name from skillopt.utils import compute_score, skill_hash - # ── Skill-aware reflection: appendix flush ─────────────────────────────────── def _flush_skill_aware_appendix( @@ -776,8 +778,12 @@ def _build_eval_env(split: str, env_num: int, seed: int): cfg.get("target_azure_openai_managed_identity_client_id") or None ), ) - set_optimizer_deployment(cfg["optimizer_model"]) - set_target_deployment(cfg["target_model"]) + set_optimizer_deployment( + cfg.get("optimizer_model") or default_model_for_backend(optimizer_backend) + ) + set_target_deployment( + cfg.get("target_model") or default_model_for_backend(target_backend) + ) configure_claude_code_exec( path=cfg.get("claude_code_exec_path", "claude"), profile=cfg.get("claude_code_exec_profile", ""), @@ -843,7 +849,7 @@ def _build_eval_env(split: str, env_num: int, seed: int): optimizer_base_url=cfg.get("optimizer_openai_compatible_base_url") or None, optimizer_api_key=cfg.get("optimizer_openai_compatible_api_key") or None, optimizer_model=( - cfg["optimizer_model"] + cfg.get("optimizer_model") if cfg.get("optimizer_backend") == "openai_compatible" else (cfg.get("optimizer_openai_compatible_model") or None) ), @@ -853,7 +859,7 @@ def _build_eval_env(split: str, env_num: int, seed: int): target_base_url=cfg.get("target_openai_compatible_base_url") or None, target_api_key=cfg.get("target_openai_compatible_api_key") or None, target_model=( - cfg["target_model"] + cfg.get("target_model") if cfg.get("target_backend") == "openai_compatible" else (cfg.get("target_openai_compatible_model") or None) ), @@ -867,8 +873,8 @@ def _build_eval_env(split: str, env_num: int, seed: int): set_reasoning_effort(reasoning) print( f" [model config] backend={backend} " - f"optimizer={cfg['optimizer_model']} ({optimizer_backend}) " - f"target={cfg['target_model']} ({target_backend}) " + f"optimizer={cfg.get('optimizer_model')} ({optimizer_backend}) " + f"target={cfg.get('target_model')} ({target_backend}) " f"reasoning={reasoning or 'off'}" ) @@ -2413,7 +2419,7 @@ def _persist_runtime_state(last_completed_step: int) -> None: # Comparison delta_hard = (test_hard or 0) - (baseline_test_hard or 0) - print(f"\n === Improvement vs baseline (init S_0) ===") + print("\n === Improvement vs baseline (init S_0) ===") print( f" [2] best-on-val hard: {baseline_test_hard:.4f} -> {test_hard:.4f} " f"(delta={delta_hard:+.4f})" diff --git a/tests/test_minimax_backend.py b/tests/test_minimax_backend.py index bc2f95e2..8d3a787b 100644 --- a/tests/test_minimax_backend.py +++ b/tests/test_minimax_backend.py @@ -90,11 +90,11 @@ def _record_urlopen(monkeypatch: pytest.MonkeyPatch, backend: Any) -> _UrlopenRe def test_default_deployment_is_current_model(monkeypatch: pytest.MonkeyPatch) -> None: from skillopt.model.common import default_model_for_backend - + _install_openai_stub() monkeypatch.delenv("TARGET_DEPLOYMENT", raising=False) monkeypatch.delenv("OPTIMIZER_DEPLOYMENT", raising=False) - + from skillopt.model import minimax_backend as backend module = importlib.reload(backend) diff --git a/tests/test_openai_compatible_config.py b/tests/test_openai_compatible_config.py index bf9c7b80..93c63cc2 100644 --- a/tests/test_openai_compatible_config.py +++ b/tests/test_openai_compatible_config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import tempfile from unittest import mock @@ -461,8 +462,8 @@ def stop_at_adapter(*args, **kwargs): def test_trainer_preserves_explicit_role_models(monkeypatch: pytest.MonkeyPatch) -> None: import skillopt.engine.trainer as trainer_mod - from skillopt.envs.base import EnvAdapter import skillopt.model.openai_compatible_backend as openai_compat + from skillopt.envs.base import EnvAdapter class EarlyExit(Exception): pass @@ -539,3 +540,137 @@ def stop_after_config(*args, **kwargs): assert openai_compat.OPTIMIZER_CONFIG.deployment == "eval-optimizer-model" assert openai_compat.TARGET_CONFIG.deployment == "eval-target-model" + + +def test_eval_only_minimal_yaml_resolves_runtime_models(monkeypatch: pytest.MonkeyPatch) -> None: + import skillopt.model.openai_compatible_backend as openai_compat + + raw = { + "model": { + "backend": "openai_compatible", + "openai_compatible_model": "synthetic-model", + "openai_compatible_base_url": "http://audit.invalid/v1", + }, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f_yaml: + yaml.dump(raw, f_yaml) + config_path = f_yaml.name + + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as f_skill: + f_skill.write("# Test Skill\n") + skill_path = f_skill.name + + class EarlyExit(Exception): + pass + + def stop_after_config(*args, **kwargs): + raise EarlyExit() + + monkeypatch.setattr(eval_only_script, "set_reasoning_effort", stop_after_config) + + cli_args = ["eval_only.py", "--config", config_path, "--skill", skill_path] + with mock.patch("sys.argv", cli_args): + with pytest.raises(EarlyExit): + eval_only_script.main() + + assert openai_compat.OPTIMIZER_CONFIG.deployment == "synthetic-model" + assert openai_compat.TARGET_CONFIG.deployment == "synthetic-model" + assert os.environ.get("OPTIMIZER_DEPLOYMENT") == "synthetic-model" + assert os.environ.get("TARGET_DEPLOYMENT") == "synthetic-model" + assert openai_compat.OPTIMIZER_CONFIG.base_url == "http://audit.invalid/v1" + assert openai_compat.TARGET_CONFIG.base_url == "http://audit.invalid/v1" + + +def test_eval_only_minimal_yaml_with_role_override_and_backend_default(monkeypatch: pytest.MonkeyPatch) -> None: + import skillopt.model.openai_compatible_backend as openai_compat + + # Case 1: Role-specific compatible override + raw_role = { + "model": { + "backend": "openai_compatible", + "openai_compatible_model": "synthetic-shared", + "optimizer_openai_compatible_model": "synthetic-opt", + "openai_compatible_base_url": "http://audit.invalid/v1", + }, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f_yaml: + yaml.dump(raw_role, f_yaml) + config_path = f_yaml.name + + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as f_skill: + f_skill.write("# Test Skill\n") + skill_path = f_skill.name + + class EarlyExit(Exception): + pass + + def stop_after_config(*args, **kwargs): + raise EarlyExit() + + monkeypatch.setattr(eval_only_script, "set_reasoning_effort", stop_after_config) + + with mock.patch("sys.argv", ["eval_only.py", "--config", config_path, "--skill", skill_path]): + with pytest.raises(EarlyExit): + eval_only_script.main() + + assert openai_compat.OPTIMIZER_CONFIG.deployment == "synthetic-opt" + assert openai_compat.TARGET_CONFIG.deployment == "synthetic-shared" + + # Case 2: Minimal YAML with no model specified falls back to backend default (gpt-4o-mini) + raw_default = { + "model": { + "backend": "openai_compatible", + "openai_compatible_base_url": "http://audit.invalid/v1", + }, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f_yaml_def: + yaml.dump(raw_default, f_yaml_def) + config_path_def = f_yaml_def.name + + with mock.patch("sys.argv", ["eval_only.py", "--config", config_path_def, "--skill", skill_path]): + with pytest.raises(EarlyExit): + eval_only_script.main() + + assert openai_compat.OPTIMIZER_CONFIG.deployment == "gpt-4o-mini" + assert openai_compat.TARGET_CONFIG.deployment == "gpt-4o-mini" + + # Case 3: Explicit per-role CLI overrides take precedence over minimal YAML + with mock.patch( + "sys.argv", + [ + "eval_only.py", + "--config", config_path, + "--skill", skill_path, + "--optimizer_model", "cli-optimizer", + "--target_model", "cli-target", + ], + ): + with pytest.raises(EarlyExit): + eval_only_script.main() + + assert openai_compat.OPTIMIZER_CONFIG.deployment == "cli-optimizer" + assert openai_compat.TARGET_CONFIG.deployment == "cli-target" + + +def test_train_script_minimal_yaml_resolves_role_models() -> None: + raw = { + "model": { + "backend": "openai_compatible", + "openai_compatible_model": "synthetic-train-model", + "openai_compatible_base_url": "http://audit.invalid/v1", + }, + "env": {"name": "searchqa"}, + } + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + yaml.dump(raw, f) + config_path = f.name + + with mock.patch("sys.argv", ["train.py", "--config", config_path]): + args = train_script.parse_args() + cfg = train_script.load_config(args) + + assert cfg["optimizer_model"] == "synthetic-train-model" + assert cfg["target_model"] == "synthetic-train-model"