330 lines
11 KiB
Python
330 lines
11 KiB
Python
"""Tests for LLMRunner."""
|
|
import logging
|
|
import sys
|
|
from types import ModuleType
|
|
|
|
import pytest
|
|
|
|
from orchestrator.config import OrchestratorConfig
|
|
from orchestrator.contracts.error_taxonomy import ReasonCode
|
|
from orchestrator.llm_runner import LLMRunner
|
|
|
|
|
|
def _clear_runtime_llm_env(monkeypatch):
|
|
for env_name in (
|
|
"TRADINGAGENTS_LLM_PROVIDER",
|
|
"TRADINGAGENTS_BACKEND_URL",
|
|
"TRADINGAGENTS_MODEL",
|
|
"TRADINGAGENTS_DEEP_MODEL",
|
|
"TRADINGAGENTS_QUICK_MODEL",
|
|
"ANTHROPIC_BASE_URL",
|
|
"OPENAI_BASE_URL",
|
|
"ANTHROPIC_API_KEY",
|
|
"MINIMAX_API_KEY",
|
|
"OPENAI_API_KEY",
|
|
):
|
|
monkeypatch.delenv(env_name, raising=False)
|
|
|
|
|
|
@pytest.fixture
|
|
def runner(tmp_path, monkeypatch):
|
|
_clear_runtime_llm_env(monkeypatch)
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "anthropic",
|
|
"backend_url": "https://api.minimaxi.com/anthropic",
|
|
"deep_think_llm": "MiniMax-M2.7-highspeed",
|
|
"quick_think_llm": "MiniMax-M2.7-highspeed",
|
|
},
|
|
)
|
|
return LLMRunner(cfg)
|
|
|
|
|
|
# All 5 known ratings
|
|
@pytest.mark.parametrize("rating,expected", [
|
|
("BUY", (1, 0.9)),
|
|
("OVERWEIGHT", (1, 0.6)),
|
|
("HOLD", (0, 0.5)),
|
|
("UNDERWEIGHT", (-1, 0.6)),
|
|
("SELL", (-1, 0.9)),
|
|
])
|
|
def test_map_rating_known(runner, rating, expected):
|
|
assert runner._map_rating(rating) == expected
|
|
|
|
|
|
# Unknown rating → (0, 0.5)
|
|
def test_map_rating_unknown(runner):
|
|
assert runner._map_rating("STRONG_BUY") == (0, 0.5)
|
|
|
|
|
|
# Case-insensitive
|
|
def test_map_rating_lowercase(runner):
|
|
assert runner._map_rating("buy") == (1, 0.9)
|
|
assert runner._map_rating("sell") == (-1, 0.9)
|
|
assert runner._map_rating("hold") == (0, 0.5)
|
|
|
|
|
|
# Empty string → (0, 0.5)
|
|
def test_map_rating_empty_string(runner):
|
|
assert runner._map_rating("") == (0, 0.5)
|
|
|
|
|
|
def test_get_graph_preserves_explicit_empty_selected_analysts(monkeypatch, tmp_path):
|
|
captured_kwargs = {}
|
|
|
|
class FakeTradingAgentsGraph:
|
|
def __init__(self, **kwargs):
|
|
captured_kwargs.update(kwargs)
|
|
|
|
fake_module = ModuleType("tradingagents.graph.trading_graph")
|
|
fake_module.TradingAgentsGraph = FakeTradingAgentsGraph
|
|
monkeypatch.setitem(sys.modules, "tradingagents.graph.trading_graph", fake_module)
|
|
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={"selected_analysts": [], "llm_provider": "anthropic"},
|
|
)
|
|
|
|
runner = LLMRunner(cfg)
|
|
graph = runner._get_graph()
|
|
|
|
assert isinstance(graph, FakeTradingAgentsGraph)
|
|
assert captured_kwargs["config"] == cfg.trading_agents_config
|
|
assert captured_kwargs["selected_analysts"] == []
|
|
|
|
|
|
def test_get_signal_returns_reason_code_on_propagate_failure(monkeypatch, tmp_path):
|
|
_clear_runtime_llm_env(monkeypatch)
|
|
class BrokenGraph:
|
|
def propagate(self, ticker, date):
|
|
raise RuntimeError("graph unavailable")
|
|
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "anthropic",
|
|
"backend_url": "https://api.minimaxi.com/anthropic",
|
|
"deep_think_llm": "MiniMax-M2.7-highspeed",
|
|
"quick_think_llm": "MiniMax-M2.7-highspeed",
|
|
},
|
|
)
|
|
runner = LLMRunner(cfg)
|
|
monkeypatch.setattr(runner, "_get_graph", lambda: BrokenGraph())
|
|
|
|
signal = runner.get_signal("AAPL", "2024-01-02")
|
|
|
|
assert signal.degraded is True
|
|
assert signal.reason_code == ReasonCode.LLM_SIGNAL_FAILED.value
|
|
assert signal.metadata["error"] == "graph unavailable"
|
|
|
|
|
|
def test_get_signal_classifies_provider_auth_failure(monkeypatch, tmp_path):
|
|
_clear_runtime_llm_env(monkeypatch)
|
|
|
|
class BrokenGraph:
|
|
def propagate(self, ticker, date):
|
|
raise RuntimeError(
|
|
"Error code: 401 - {'type': 'error', 'error': {'type': 'authentication_error', 'message': \"login fail: Please carry the API secret key in the Authorization field\"}}"
|
|
)
|
|
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "anthropic",
|
|
"backend_url": "https://api.minimaxi.com/anthropic",
|
|
"deep_think_llm": "MiniMax-M2.7-highspeed",
|
|
"quick_think_llm": "MiniMax-M2.7-highspeed",
|
|
},
|
|
)
|
|
runner = LLMRunner(cfg)
|
|
monkeypatch.setattr(runner, "_get_graph", lambda: BrokenGraph())
|
|
|
|
signal = runner.get_signal("AAPL", "2024-01-02")
|
|
|
|
assert signal.degraded is True
|
|
assert signal.reason_code == ReasonCode.PROVIDER_AUTH_FAILED.value
|
|
assert signal.metadata["data_quality"]["state"] == "provider_auth_failed"
|
|
|
|
|
|
def test_get_signal_returns_provider_mismatch_before_graph_init(tmp_path):
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "anthropic",
|
|
"backend_url": "https://api.openai.com/v1",
|
|
},
|
|
)
|
|
runner = LLMRunner(cfg)
|
|
|
|
signal = runner.get_signal("AAPL", "2024-01-02")
|
|
|
|
assert signal.degraded is True
|
|
assert signal.reason_code == ReasonCode.PROVIDER_MISMATCH.value
|
|
assert signal.metadata["data_quality"]["state"] == "provider_mismatch"
|
|
|
|
|
|
def test_get_signal_persists_research_provenance_on_success(monkeypatch, tmp_path):
|
|
_clear_runtime_llm_env(monkeypatch)
|
|
class SuccessfulGraph:
|
|
def propagate(self, ticker, date):
|
|
return {
|
|
"investment_debate_state": {
|
|
"research_status": "degraded",
|
|
"research_mode": "degraded_synthesis",
|
|
"timed_out_nodes": ["Bull Researcher"],
|
|
"degraded_reason": "bull_researcher_timeout",
|
|
"covered_dimensions": ["market"],
|
|
"manager_confidence": None,
|
|
}
|
|
,
|
|
"final_trade_decision_structured": {
|
|
"rating": "BUY",
|
|
"hold_subtype": "N/A",
|
|
},
|
|
}, "BUY"
|
|
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "anthropic",
|
|
"backend_url": "https://api.minimaxi.com/anthropic",
|
|
"deep_think_llm": "MiniMax-M2.7-highspeed",
|
|
"quick_think_llm": "MiniMax-M2.7-highspeed",
|
|
},
|
|
)
|
|
runner = LLMRunner(cfg)
|
|
monkeypatch.setattr(runner, "_get_graph", lambda: SuccessfulGraph())
|
|
|
|
signal = runner.get_signal("AAPL", "2024-01-02")
|
|
|
|
assert signal.degraded is False
|
|
assert signal.metadata["research"]["research_status"] == "degraded"
|
|
assert signal.metadata["sample_quality"] == "degraded_research"
|
|
assert signal.metadata["data_quality"]["state"] == "research_degraded"
|
|
assert signal.metadata["decision_structured"]["rating"] == "BUY"
|
|
|
|
|
|
# Phase 2: Provider matrix validation tests
|
|
def test_detect_provider_mismatch_google_with_openai_url(tmp_path):
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "google",
|
|
"backend_url": "https://api.openai.com/v1",
|
|
},
|
|
)
|
|
runner = LLMRunner(cfg)
|
|
signal = runner.get_signal("AAPL", "2024-01-02")
|
|
|
|
assert signal.degraded is True
|
|
assert signal.reason_code == ReasonCode.PROVIDER_MISMATCH.value
|
|
|
|
|
|
def test_detect_provider_mismatch_xai_with_anthropic_url(tmp_path):
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "xai",
|
|
"backend_url": "https://api.minimaxi.com/anthropic",
|
|
},
|
|
)
|
|
runner = LLMRunner(cfg)
|
|
signal = runner.get_signal("AAPL", "2024-01-02")
|
|
|
|
assert signal.degraded is True
|
|
assert signal.reason_code == ReasonCode.PROVIDER_MISMATCH.value
|
|
|
|
|
|
def test_detect_provider_mismatch_ollama_with_openai_url(tmp_path):
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "ollama",
|
|
"backend_url": "https://api.openai.com/v1",
|
|
},
|
|
)
|
|
runner = LLMRunner(cfg)
|
|
signal = runner.get_signal("AAPL", "2024-01-02")
|
|
|
|
assert signal.degraded is True
|
|
assert signal.reason_code == ReasonCode.PROVIDER_MISMATCH.value
|
|
|
|
|
|
def test_detect_provider_mismatch_valid_anthropic_minimax(tmp_path):
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "anthropic",
|
|
"backend_url": "https://api.minimaxi.com/anthropic",
|
|
},
|
|
)
|
|
runner = LLMRunner(cfg)
|
|
mismatch = runner._detect_provider_mismatch()
|
|
|
|
assert mismatch is None
|
|
|
|
|
|
def test_detect_provider_mismatch_valid_openai(tmp_path):
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "openai",
|
|
"backend_url": "https://api.openai.com/v1",
|
|
},
|
|
)
|
|
runner = LLMRunner(cfg)
|
|
mismatch = runner._detect_provider_mismatch()
|
|
|
|
assert mismatch is None
|
|
|
|
|
|
# Phase 3: Timeout configuration validation tests
|
|
def test_timeout_validation_warns_for_multiple_analysts_low_timeout(tmp_path, caplog):
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "anthropic",
|
|
"backend_url": "https://api.minimaxi.com/anthropic",
|
|
"selected_analysts": ["market", "social", "news", "fundamentals"],
|
|
"analyst_node_timeout_secs": 75.0,
|
|
},
|
|
)
|
|
with caplog.at_level(logging.WARNING):
|
|
runner = LLMRunner(cfg)
|
|
|
|
assert any("analyst_node_timeout_secs=75.0s may be insufficient" in record.message for record in caplog.records)
|
|
|
|
|
|
def test_timeout_validation_no_warn_for_single_analyst(tmp_path, caplog):
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "anthropic",
|
|
"backend_url": "https://api.minimaxi.com/anthropic",
|
|
"selected_analysts": ["market"],
|
|
"analyst_node_timeout_secs": 75.0,
|
|
},
|
|
)
|
|
with caplog.at_level(logging.WARNING):
|
|
runner = LLMRunner(cfg)
|
|
|
|
assert not any("may be insufficient" in record.message for record in caplog.records)
|
|
|
|
|
|
def test_timeout_validation_no_warn_for_sufficient_timeout(tmp_path, caplog):
|
|
cfg = OrchestratorConfig(
|
|
cache_dir=str(tmp_path),
|
|
trading_agents_config={
|
|
"llm_provider": "anthropic",
|
|
"backend_url": "https://api.minimaxi.com/anthropic",
|
|
"selected_analysts": ["market", "social", "news", "fundamentals"],
|
|
"analyst_node_timeout_secs": 120.0,
|
|
"research_node_timeout_secs": 75.0,
|
|
},
|
|
)
|
|
with caplog.at_level(logging.WARNING):
|
|
runner = LLMRunner(cfg)
|
|
|
|
assert not any("may be insufficient" in record.message for record in caplog.records)
|