diff --git a/.env.example b/.env.example index 1328b838..47ac745d 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,23 @@ -# LLM Providers (set the one you use) +# LLM Provider API Keys (set the ones you use) OPENAI_API_KEY= GOOGLE_API_KEY= ANTHROPIC_API_KEY= XAI_API_KEY= OPENROUTER_API_KEY= + +# Data Provider API Keys +ALPHA_VANTAGE_API_KEY= + +# ── Configuration overrides ────────────────────────────────────────── +# Any setting in DEFAULT_CONFIG can be overridden with a +# TRADINGAGENTS_ environment variable. Unset or empty values +# are ignored (the hardcoded default is kept). +# +# Examples: +# TRADINGAGENTS_LLM_PROVIDER=openrouter +# TRADINGAGENTS_QUICK_THINK_LLM=deepseek/deepseek-chat-v3-0324 +# TRADINGAGENTS_DEEP_THINK_LLM=deepseek/deepseek-r1-0528 +# TRADINGAGENTS_BACKEND_URL=https://openrouter.ai/api/v1 +# TRADINGAGENTS_RESULTS_DIR=./my_results +# TRADINGAGENTS_MAX_DEBATE_ROUNDS=2 +# TRADINGAGENTS_VENDOR_SCANNER_DATA=alpha_vantage diff --git a/CLAUDE.md b/CLAUDE.md index 24e293a6..89b44310 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,6 +75,7 @@ OpenAI, Anthropic, Google, xAI, OpenRouter, Ollama - LLM tiers configuration - Vendor routing - Debate rounds settings +- All values overridable via `TRADINGAGENTS_` env vars (see `.env.example`) ## Patterns to Follow @@ -91,27 +92,38 @@ OpenAI, Anthropic, Google, xAI, OpenRouter, Ollama - **Tool execution**: Trading graph uses `ToolNode` in graph. Scanner agents use `run_tool_loop()` inline. If `bind_tools()` is used, there MUST be a tool execution path. - **yfinance DataFrames**: `top_companies` has ticker as INDEX, not column. Always check `.index` and `.columns`. - **yfinance Sector/Industry**: `Sector.overview` has NO performance data. Use ETF proxies for performance. -- **Vendor fallback**: Functions inside `route_to_vendor` must RAISE on failure, not embed errors in return values. Catch `AlphaVantageError` (base class), not just `RateLimitError`. +- **Vendor fallback**: Functions inside `route_to_vendor` must RAISE on failure, not embed errors in return values. Catch `(AlphaVantageError, ConnectionError, TimeoutError)`, not just `RateLimitError`. - **LangGraph parallel writes**: Any state field written by parallel nodes MUST have a reducer (`Annotated[str, reducer_fn]`). - **Ollama remote host**: Never hardcode `localhost:11434`. Use configured `base_url`. -- **.env loading**: Check actual env var values when debugging auth. Worktree and main repo may have different `.env` files. +- **.env loading**: `load_dotenv()` runs at module level in `default_config.py` — import-order-independent. Check actual env var values when debugging auth. +- **Rate limiter locks**: Never hold a lock during `sleep()` or IO. Release, sleep, re-acquire. +- **Config fallback keys**: `llm_provider` and `backend_url` must always exist at top level — `scanner_graph.py` and `trading_graph.py` use them as fallbacks. ## Project Tracking Files -- `DECISIONS.md` — Architecture decision records (vendor strategy, LLM setup, tool execution) +- `DECISIONS.md` — Architecture decision records (vendor strategy, LLM setup, tool execution, env overrides) - `PROGRESS.md` — Feature progress, what works, TODOs -- `MISTAKES.md` — Past bugs and lessons learned (9 documented mistakes) +- `MISTAKES.md` — Past bugs and lessons learned (10 documented mistakes) -## Current LLM Configuration (Hybrid) +## LLM Configuration -``` -quick_think: qwen3.5:27b via Ollama (http://192.168.50.76:11434) -mid_think: qwen3.5:27b via Ollama (http://192.168.50.76:11434) -deep_think: deepseek/deepseek-r1-0528 via OpenRouter +Per-tier provider overrides in `tradingagents/default_config.py`: +- Each tier (`quick_think`, `mid_think`, `deep_think`) can have its own `_llm_provider` and `_backend_url` +- Falls back to top-level `llm_provider` and `backend_url` when per-tier values are None +- All config values overridable via `TRADINGAGENTS_` env vars +- Keys for LLM providers: `.env` file (e.g., `OPENROUTER_API_KEY`, `ALPHA_VANTAGE_API_KEY`) + +### Env Var Override Convention + +```env +# Pattern: TRADINGAGENTS_=value +TRADINGAGENTS_LLM_PROVIDER=openrouter +TRADINGAGENTS_DEEP_THINK_LLM=deepseek/deepseek-r1-0528 +TRADINGAGENTS_MAX_DEBATE_ROUNDS=3 +TRADINGAGENTS_VENDOR_SCANNER_DATA=alpha_vantage ``` -Config: `tradingagents/default_config.py` (per-tier `_llm_provider` keys) -Keys: `.env` file (`OPENROUTER_API_KEY`, `ALPHA_VANTAGE_API_KEY`) +Empty or unset vars preserve the hardcoded default. `None`-default fields (like `mid_think_llm`) stay `None` when unset, preserving fallback semantics. ## Running the Scanner diff --git a/DECISIONS.md b/DECISIONS.md index 55a436c1..13fcbbb7 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -105,10 +105,75 @@ Download 6 months of history via `yf.download()` and compute 1-day, 1-week, 1-mo ## Decision 007: .env Loading Strategy **Date**: 2026-03-17 -**Status**: Implemented ✅ +**Status**: Superseded by Decision 008 ⚠️ **Context**: `load_dotenv()` loads from CWD. When running from a git worktree, the worktree `.env` may have placeholder values while the main repo `.env` has real keys. **Decision**: `cli/main.py` calls `load_dotenv()` (CWD) then `load_dotenv(Path(__file__).parent.parent / ".env")` as fallback. The worktree `.env` was also updated with real API keys. **Note for future**: If `.env` issues recur, check which `.env` file is being picked up. The worktree and main repo each have their own `.env`. + +**Update**: Decision 008 moves `load_dotenv()` into `default_config.py` itself, making it import-order-independent. The CLI-level `load_dotenv()` in `main.py` is now defense-in-depth only. + +--- + +## Decision 008: Environment Variable Config Overrides + +**Date**: 2026-03-17 +**Status**: Implemented ✅ + +**Context**: `DEFAULT_CONFIG` hardcoded all values (LLM providers, models, vendor routing, debate rounds). Users had to edit `default_config.py` to change any setting. The `load_dotenv()` call in `cli/main.py` ran *after* `DEFAULT_CONFIG` was already evaluated at import time, so env vars like `TRADINGAGENTS_LLM_PROVIDER` had no effect. This also created a latent bug (Mistake #9): `llm_provider` and `backend_url` were removed from the config but `scanner_graph.py` still referenced them as fallbacks. + +**Decision**: +1. **Module-level `.env` loading**: `default_config.py` calls `load_dotenv()` at the top of the module, before `DEFAULT_CONFIG` is evaluated. Loads from CWD first, then falls back to project root (`Path(__file__).resolve().parent.parent / ".env"`). +2. **`_env()` / `_env_int()` helpers**: Read `TRADINGAGENTS_` from environment. Return the hardcoded default when the env var is unset or empty (preserving `None` semantics for per-tier fallbacks). +3. **Restored top-level keys**: `llm_provider` (default: `"openai"`) and `backend_url` (default: `"https://api.openai.com/v1"`) restored as env-overridable keys. Resolves Mistake #9. +4. **All config keys overridable**: LLM models, providers, backend URLs, debate rounds, data vendor categories — all follow the `TRADINGAGENTS_` pattern. +5. **Explicit dependency**: Added `python-dotenv>=1.0.0` to `pyproject.toml` (was used but undeclared). + +**Naming convention**: `TRADINGAGENTS_` prefix + uppercase config key. Examples: +``` +TRADINGAGENTS_LLM_PROVIDER=openrouter +TRADINGAGENTS_DEEP_THINK_LLM=deepseek/deepseek-r1-0528 +TRADINGAGENTS_MAX_DEBATE_ROUNDS=3 +TRADINGAGENTS_VENDOR_SCANNER_DATA=alpha_vantage +``` + +**Files changed**: +- `tradingagents/default_config.py` — core implementation +- `main.py` — moved `load_dotenv()` before imports (defense-in-depth) +- `pyproject.toml` — added `python-dotenv>=1.0.0` +- `.env.example` — documented all overrides +- `tests/test_env_override.py` — 15 tests + +**Alternative considered**: YAML/TOML config file. Rejected — env vars are simpler, work with Docker/CI, and don't require a new config file format. + +--- + +## Decision 009: Thread-Safe Rate Limiter for Alpha Vantage + +**Date**: 2026-03-17 +**Status**: Implemented ✅ + +**Context**: The Alpha Vantage rate limiter in `alpha_vantage_common.py` initially slept *inside* the lock when re-checking the rate window. This blocked all other threads from making API requests during the sleep period, effectively serializing all AV calls. + +**Decision**: Two-phase rate limiting: +1. **First check**: Acquire lock, check timestamps, release lock, sleep if needed. +2. **Re-check loop**: Acquire lock, re-check timestamps. If still over limit, release lock *before* sleeping, then retry. Only append timestamp and break when under the limit. + +This ensures the lock is never held during `sleep()` calls. + +**File**: `tradingagents/dataflows/alpha_vantage_common.py` + +--- + +## Decision 010: Broader Vendor Fallback Exception Handling + +**Date**: 2026-03-17 +**Status**: Implemented ✅ + +**Context**: `route_to_vendor()` only caught `AlphaVantageError` for fallback. But network issues (`ConnectionError`, `TimeoutError`) from the `requests` library wouldn't trigger fallback — they'd crash the pipeline instead. + +**Decision**: Broadened the catch in `route_to_vendor()` to `(AlphaVantageError, ConnectionError, TimeoutError)`. Similarly, `_make_api_request()` now catches `requests.exceptions.RequestException` as a general fallback and wraps `raise_for_status()` in a try/except to convert HTTP errors to `ThirdPartyError`. + +**Files**: `tradingagents/dataflows/interface.py`, `tradingagents/dataflows/alpha_vantage_common.py` diff --git a/MISTAKES.md b/MISTAKES.md index b3629182..b296dc0e 100644 --- a/MISTAKES.md +++ b/MISTAKES.md @@ -96,6 +96,27 @@ Documenting bugs and wrong assumptions to avoid repeating them. **What happened**: Removed `llm_provider` from `default_config.py` (since we have per-tier providers). But `scanner_graph.py` line 78 does `self.config.get(f"{tier}_llm_provider") or self.config["llm_provider"]` — would crash if per-tier provider is ever None. -**Status**: Works currently because per-tier providers are always set. But it's a latent bug. +**Status**: ✅ RESOLVED in PR #9. Top-level `llm_provider` (default: `"openai"`) and `backend_url` (default: `"https://api.openai.com/v1"`) restored as env-overridable config keys. Per-tier providers safely fall back to these when `None`. -**TODO**: Add a safe fallback or remove the dead code path. +**Lesson**: Always preserve fallback keys that downstream code depends on. When refactoring config, grep for all references before removing keys. + +--- + +## Mistake 10: Rate limiter held lock during sleep + +**What happened**: The Alpha Vantage rate limiter's re-check path in `_rate_limited_request()` called `_time.sleep(extra_sleep)` while holding `_rate_lock`. This blocked all other threads from making API requests during the sleep period, effectively serializing all AV calls even though the pipeline runs parallel scanner agents. + +**Root cause**: Initial implementation only had one lock section. When the re-check-after-sleep pattern was added to prevent race conditions, the sleep was left inside the `with _rate_lock:` block. + +**Fix**: Restructured the re-check as a `while True` loop that releases the lock before sleeping: +```python +while True: + with _rate_lock: + if len(_call_timestamps) < _RATE_LIMIT: + _call_timestamps.append(_time.time()) + break + extra_sleep = 60 - (now - _call_timestamps[0]) + 0.1 + _time.sleep(extra_sleep) # ← outside lock +``` + +**Lesson**: Never hold a lock during a sleep/IO operation. Always release the lock, perform the blocking operation, then re-acquire. diff --git a/PROGRESS.md b/PROGRESS.md index 8beafb32..229aaab0 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -15,10 +15,11 @@ The 3-phase scanner pipeline runs successfully from `python -m cli.main scan --d | Phase 3: Macro Synthesis | ✅ | OpenRouter/DeepSeek R1, pure LLM synthesis (no tools) | | Parallel fan-out (Phase 1) | ✅ | LangGraph with `_last_value` reducers | | Tool execution loop | ✅ | `run_tool_loop()` in `tool_runner.py` | -| Data vendor fallback | ✅ | AV → yfinance fallback on `AlphaVantageError` | +| Data vendor fallback | ✅ | AV → yfinance fallback on `AlphaVantageError`, `ConnectionError`, `TimeoutError` | | CLI `--date` flag | ✅ | `python -m cli.main scan --date YYYY-MM-DD` | -| .env loading | ✅ | Keys loaded from project root `.env` | -| Tests (23 total) | ✅ | 14 original + 9 scanner fallback tests | +| .env loading | ✅ | `load_dotenv()` at module level in `default_config.py` — import-order-independent | +| Env var config overrides | ✅ | All `DEFAULT_CONFIG` keys overridable via `TRADINGAGENTS_` env vars | +| Tests (38 total) | ✅ | 14 original + 9 scanner fallback + 15 env override tests | ### Output Quality (Sample Run 2026-03-17) @@ -41,14 +42,40 @@ The 3-phase scanner pipeline runs successfully from `python -m cli.main scan --d - `tradingagents/graph/scanner_setup.py` — LangGraph workflow setup - `tradingagents/dataflows/yfinance_scanner.py` — yfinance data for scanner - `tradingagents/dataflows/alpha_vantage_scanner.py` — Alpha Vantage data for scanner +- `tradingagents/pipeline/macro_bridge.py` — scan → filter → per-ticker analysis bridge - `tests/test_scanner_fallback.py` — 9 fallback tests +- `tests/test_env_override.py` — 15 env override tests **Modified files:** -- `tradingagents/default_config.py` — per-tier LLM provider config (hybrid setup) +- `tradingagents/default_config.py` — env var overrides via `_env()`/`_env_int()` helpers, `load_dotenv()` at module level, restored top-level `llm_provider` and `backend_url` keys - `tradingagents/llm_clients/openai_client.py` — Ollama remote host support -- `tradingagents/dataflows/interface.py` — broadened fallback catch to `AlphaVantageError` -- `cli/main.py` — `scan` command with `--date` flag, `.env` loading fix -- `.env` — real API keys +- `tradingagents/dataflows/interface.py` — broadened fallback catch to `(AlphaVantageError, ConnectionError, TimeoutError)` +- `tradingagents/dataflows/alpha_vantage_common.py` — thread-safe rate limiter (sleep outside lock), broader `RequestException` catch, wrapped `raise_for_status` +- `tradingagents/graph/scanner_graph.py` — debug mode fix (stream for debug, invoke for result) +- `tradingagents/pipeline/macro_bridge.py` — `get_running_loop()` over deprecated `get_event_loop()` +- `cli/main.py` — `scan` command with `--date` flag, `try/except` in `run_pipeline`, `.env` loading fix +- `main.py` — `load_dotenv()` before tradingagents imports +- `pyproject.toml` — `python-dotenv>=1.0.0` dependency declared +- `.env.example` — documented all `TRADINGAGENTS_*` overrides and `ALPHA_VANTAGE_API_KEY` + +--- + +## Milestone: Env Var Config Overrides ✅ COMPLETE (PR #9) + +All `DEFAULT_CONFIG` values are now overridable via `TRADINGAGENTS_` environment variables without code changes. This resolves the latent bug from Mistake #9 (missing top-level `llm_provider`). + +### What Changed + +| Component | Detail | +|-----------|--------| +| `default_config.py` | `load_dotenv()` at module level + `_env()`/`_env_int()` helpers | +| Top-level fallback keys | Restored `llm_provider` and `backend_url` (defaults: `"openai"`, `"https://api.openai.com/v1"`) | +| Per-tier overrides | All `None` by default — fall back to top-level when not set via env | +| Integer config keys | `max_debate_rounds`, `max_risk_discuss_rounds`, `max_recur_limit` use `_env_int()` | +| Data vendor keys | `data_vendors.*` overridable via `TRADINGAGENTS_VENDOR_` | +| `.env.example` | Complete reference of all overridable settings | +| `python-dotenv` | Added to `pyproject.toml` as explicit dependency | +| Tests | 15 new tests in `tests/test_env_override.py` | --- @@ -78,4 +105,4 @@ The 3-phase scanner pipeline runs successfully from `python -m cli.main scan --d - [ ] **Streaming output**: Scanner currently runs with `Live(Spinner(...))` — no intermediate output. Could stream phase completions to the console. -- [ ] **Remove top-level `llm_provider` references**: `scanner_graph.py` lines 69, 78 still fall back to `self.config["llm_provider"]` which doesn't exist in current config. Works because per-tier providers are always set, but will crash if they're ever `None`. +- [x] ~~**Remove top-level `llm_provider` references**~~: Resolved in PR #9 — `llm_provider` and `backend_url` restored as top-level keys with `"openai"` / `"https://api.openai.com/v1"` defaults. Per-tier providers fall back to these when `None`. diff --git a/cli/main.py b/cli/main.py index ab094d31..d9a9d023 100644 --- a/cli/main.py +++ b/cli/main.py @@ -1,5 +1,6 @@ from typing import Optional import datetime +import json import typer from pathlib import Path from functools import wraps @@ -1201,8 +1202,6 @@ def run_scan(date: Optional[str] = None): raise typer.Exit(1) # Save reports - import json as _json - for key in ["geopolitical_report", "market_movers_report", "sector_performance_report", "industry_deep_dive_report", "macro_scan_summary"]: content = result.get(key, "") @@ -1217,7 +1216,7 @@ def run_scan(date: Optional[str] = None): # Try to parse and show watchlist table try: - summary_data = _json.loads(summary) + summary_data = json.loads(summary) stocks = summary_data.get("stocks_to_investigate", []) if stocks: table = Table(title="Stocks to Investigate", box=box.ROUNDED) @@ -1235,16 +1234,16 @@ def run_scan(date: Optional[str] = None): s.get("thesis_angle", ""), ) console.print(table) - except (_json.JSONDecodeError, KeyError): + except (json.JSONDecodeError, KeyError): pass # Summary wasn't valid JSON — already printed as markdown + console.print(f"\n[green]Results saved to {save_dir}[/green]") def run_pipeline(): """Full pipeline: scan -> filter -> per-ticker deep dive.""" import asyncio - import json as _json from tradingagents.pipeline.macro_bridge import ( parse_macro_output, filter_candidates, @@ -1293,10 +1292,14 @@ def run_pipeline(): output_dir = Path("results/macro_pipeline") console.print(f"\n[cyan]Running TradingAgents for {len(candidates)} tickers...[/cyan]") - with Live(Spinner("dots", text="Analyzing..."), console=console, transient=True): - results = asyncio.run( - run_all_tickers(candidates, macro_context, config, analysis_date) - ) + try: + with Live(Spinner("dots", text="Analyzing..."), console=console, transient=True): + results = asyncio.run( + run_all_tickers(candidates, macro_context, config, analysis_date) + ) + except Exception as e: + console.print(f"[red]Pipeline failed: {e}[/red]") + raise typer.Exit(1) save_results(results, macro_context, output_dir) diff --git a/main.py b/main.py index 7e8b20e8..be020d0c 100644 --- a/main.py +++ b/main.py @@ -1,11 +1,13 @@ -from tradingagents.graph.trading_graph import TradingAgentsGraph -from tradingagents.default_config import DEFAULT_CONFIG - from dotenv import load_dotenv -# Load environment variables from .env file +# Load environment variables from .env file BEFORE importing any +# tradingagents modules so TRADINGAGENTS_* vars are visible to +# DEFAULT_CONFIG at import time. load_dotenv() +from tradingagents.graph.trading_graph import TradingAgentsGraph +from tradingagents.default_config import DEFAULT_CONFIG + # Create a custom config config = DEFAULT_CONFIG.copy() config["deep_think_llm"] = "gpt-5-mini" # Use a different model diff --git a/pyproject.toml b/pyproject.toml index 9213d7f6..d361508b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "langgraph>=0.4.8", "pandas>=2.3.0", "parsel>=1.10.0", + "python-dotenv>=1.0.0", "pytz>=2025.2", "questionary>=2.1.0", "rank-bm25>=0.2.2", diff --git a/tests/test_env_override.py b/tests/test_env_override.py new file mode 100644 index 00000000..1bf4e54b --- /dev/null +++ b/tests/test_env_override.py @@ -0,0 +1,108 @@ +"""Tests that TRADINGAGENTS_* environment variables override DEFAULT_CONFIG.""" + +import importlib +import os +from unittest.mock import patch + +import pytest + + +class TestEnvOverridesDefaults: + """Verify that setting TRADINGAGENTS_ env vars changes DEFAULT_CONFIG.""" + + def _reload_config(self): + """Force-reimport default_config so the module-level dict is rebuilt.""" + import tradingagents.default_config as mod + + importlib.reload(mod) + return mod.DEFAULT_CONFIG + + def test_llm_provider_override(self): + with patch.dict(os.environ, {"TRADINGAGENTS_LLM_PROVIDER": "openrouter"}): + cfg = self._reload_config() + assert cfg["llm_provider"] == "openrouter" + + def test_backend_url_override(self): + with patch.dict(os.environ, {"TRADINGAGENTS_BACKEND_URL": "http://localhost:1234"}): + cfg = self._reload_config() + assert cfg["backend_url"] == "http://localhost:1234" + + def test_deep_think_llm_override(self): + with patch.dict(os.environ, {"TRADINGAGENTS_DEEP_THINK_LLM": "deepseek/deepseek-r1"}): + cfg = self._reload_config() + assert cfg["deep_think_llm"] == "deepseek/deepseek-r1" + + def test_quick_think_llm_override(self): + with patch.dict(os.environ, {"TRADINGAGENTS_QUICK_THINK_LLM": "gpt-4o-mini"}): + cfg = self._reload_config() + assert cfg["quick_think_llm"] == "gpt-4o-mini" + + def test_mid_think_llm_none_by_default(self): + """mid_think_llm defaults to None (falls back to quick_think_llm).""" + with patch.dict(os.environ, {}, clear=False): + # Remove the env var if it happens to be set + os.environ.pop("TRADINGAGENTS_MID_THINK_LLM", None) + cfg = self._reload_config() + assert cfg["mid_think_llm"] is None + + def test_mid_think_llm_override(self): + with patch.dict(os.environ, {"TRADINGAGENTS_MID_THINK_LLM": "gpt-4o"}): + cfg = self._reload_config() + assert cfg["mid_think_llm"] == "gpt-4o" + + def test_empty_env_var_keeps_default(self): + """An empty string is treated the same as unset (keeps the default).""" + with patch.dict(os.environ, {"TRADINGAGENTS_LLM_PROVIDER": ""}): + cfg = self._reload_config() + assert cfg["llm_provider"] == "openai" + + def test_empty_env_var_keeps_none_default(self): + """An empty string for a None-default field stays None.""" + with patch.dict(os.environ, {"TRADINGAGENTS_DEEP_THINK_LLM_PROVIDER": ""}): + cfg = self._reload_config() + assert cfg["deep_think_llm_provider"] is None + + def test_per_tier_provider_override(self): + with patch.dict(os.environ, {"TRADINGAGENTS_DEEP_THINK_LLM_PROVIDER": "anthropic"}): + cfg = self._reload_config() + assert cfg["deep_think_llm_provider"] == "anthropic" + + def test_per_tier_backend_url_override(self): + with patch.dict(os.environ, {"TRADINGAGENTS_MID_THINK_BACKEND_URL": "http://my-ollama:11434"}): + cfg = self._reload_config() + assert cfg["mid_think_backend_url"] == "http://my-ollama:11434" + + def test_max_debate_rounds_int(self): + with patch.dict(os.environ, {"TRADINGAGENTS_MAX_DEBATE_ROUNDS": "3"}): + cfg = self._reload_config() + assert cfg["max_debate_rounds"] == 3 + + def test_max_debate_rounds_bad_value(self): + """Non-numeric string falls back to hardcoded default.""" + with patch.dict(os.environ, {"TRADINGAGENTS_MAX_DEBATE_ROUNDS": "abc"}): + cfg = self._reload_config() + assert cfg["max_debate_rounds"] == 1 + + def test_results_dir_override(self): + with patch.dict(os.environ, {"TRADINGAGENTS_RESULTS_DIR": "/tmp/my_results"}): + cfg = self._reload_config() + assert cfg["results_dir"] == "/tmp/my_results" + + def test_vendor_scanner_data_override(self): + with patch.dict(os.environ, {"TRADINGAGENTS_VENDOR_SCANNER_DATA": "alpha_vantage"}): + cfg = self._reload_config() + assert cfg["data_vendors"]["scanner_data"] == "alpha_vantage" + + def test_defaults_unchanged_when_no_env_set(self): + """Without any TRADINGAGENTS_* vars, defaults are the original hardcoded values.""" + # Clear all TRADINGAGENTS_ vars + env_clean = {k: v for k, v in os.environ.items() if not k.startswith("TRADINGAGENTS_")} + with patch.dict(os.environ, env_clean, clear=True): + cfg = self._reload_config() + assert cfg["llm_provider"] == "openai" + assert cfg["deep_think_llm"] == "gpt-5.2" + assert cfg["mid_think_llm"] is None + assert cfg["quick_think_llm"] == "gpt-5-mini" + assert cfg["backend_url"] == "https://api.openai.com/v1" + assert cfg["max_debate_rounds"] == 1 + assert cfg["data_vendors"]["scanner_data"] == "yfinance" diff --git a/tradingagents/agents/utils/tool_runner.py b/tradingagents/agents/utils/tool_runner.py index e1b8d4c3..3c07d5a4 100644 --- a/tradingagents/agents/utils/tool_runner.py +++ b/tradingagents/agents/utils/tool_runner.py @@ -12,7 +12,9 @@ from typing import Any, List from langchain_core.messages import AIMessage, ToolMessage -MAX_TOOL_ROUNDS = 5 # safety limit to avoid infinite loops +# Most LLM tool-calling patterns resolve within 2-3 rounds; +# 5 provides headroom for complex scenarios while preventing runaway loops. +MAX_TOOL_ROUNDS = 5 def run_tool_loop( diff --git a/tradingagents/dataflows/alpha_vantage_common.py b/tradingagents/dataflows/alpha_vantage_common.py index 2314a68b..5aaa27a5 100644 --- a/tradingagents/dataflows/alpha_vantage_common.py +++ b/tradingagents/dataflows/alpha_vantage_common.py @@ -2,6 +2,8 @@ import os import requests import pandas as pd import json +import threading +import time as _time from datetime import datetime from io import StringIO @@ -73,8 +75,6 @@ class ThirdPartyParseError(AlphaVantageError): # ─── Rate-limited request helper ───────────────────────────────────────────── -import threading -import time as _time _rate_lock = threading.Lock() _call_timestamps: list[float] = [] @@ -83,14 +83,34 @@ _RATE_LIMIT = 75 # calls per minute (Alpha Vantage premium) def _rate_limited_request(function_name: str, params: dict, timeout: int = 30) -> dict | str: """Make an API request with rate limiting (75 calls/min for premium key).""" + sleep_time = 0.0 with _rate_lock: now = _time.time() # Remove timestamps older than 60 seconds _call_timestamps[:] = [t for t in _call_timestamps if now - t < 60] if len(_call_timestamps) >= _RATE_LIMIT: sleep_time = 60 - (now - _call_timestamps[0]) + 0.1 - _time.sleep(sleep_time) - _call_timestamps.append(_time.time()) + + # Sleep outside the lock to avoid blocking other threads + if sleep_time > 0: + _time.sleep(sleep_time) + + # Re-check and register under lock to avoid races where multiple + # threads calculate similar sleep times and then all fire at once. + while True: + with _rate_lock: + now = _time.time() + _call_timestamps[:] = [t for t in _call_timestamps if now - t < 60] + if len(_call_timestamps) >= _RATE_LIMIT: + # Another thread filled the window while we slept — wait again + extra_sleep = 60 - (now - _call_timestamps[0]) + 0.1 + else: + _call_timestamps.append(_time.time()) + break + # Sleep outside the lock to avoid blocking other threads + _time.sleep(extra_sleep) + + return _make_api_request(function_name, params, timeout=timeout) @@ -131,6 +151,8 @@ def _make_api_request(function_name: str, params: dict, timeout: int = 30) -> di ) except requests.exceptions.ConnectionError as exc: raise ThirdPartyError(f"Connection error: function={function_name}, error={exc}") + except requests.exceptions.RequestException as exc: + raise ThirdPartyError(f"Request failed: function={function_name}, error={exc}") # HTTP-level errors if response.status_code == 401: @@ -146,7 +168,13 @@ def _make_api_request(function_name: str, params: dict, timeout: int = 30) -> di f"Server error: status={response.status_code}, function={function_name}, " f"body={response.text[:200]}" ) - response.raise_for_status() + try: + response.raise_for_status() + except requests.exceptions.HTTPError as exc: + raise ThirdPartyError( + f"HTTP error: status={response.status_code}, function={function_name}, " + f"body={response.text[:200]}" + ) from exc response_text = response.text diff --git a/tradingagents/dataflows/interface.py b/tradingagents/dataflows/interface.py index 03789fd2..adddb290 100644 --- a/tradingagents/dataflows/interface.py +++ b/tradingagents/dataflows/interface.py @@ -201,7 +201,7 @@ def route_to_vendor(method: str, *args, **kwargs): try: return impl_func(*args, **kwargs) - except AlphaVantageError: - continue # Any AV error triggers fallback to next vendor + except (AlphaVantageError, ConnectionError, TimeoutError): + continue # Any AV error or connection/timeout triggers fallback to next vendor raise RuntimeError(f"No available vendor for '{method}'") \ No newline at end of file diff --git a/tradingagents/default_config.py b/tradingagents/default_config.py index 4611ebf4..e42787b1 100644 --- a/tradingagents/default_config.py +++ b/tradingagents/default_config.py @@ -1,45 +1,83 @@ import os +from pathlib import Path + +from dotenv import load_dotenv + +# Load .env so that TRADINGAGENTS_* variables are available before +# DEFAULT_CONFIG is evaluated. CWD is checked first, then the project +# root (two levels up from this file). load_dotenv never overwrites +# variables that are already present in the environment. +load_dotenv() +load_dotenv(Path(__file__).resolve().parent.parent / ".env") + + +def _env(key: str, default=None): + """Read ``TRADINGAGENTS_`` from the environment. + + Returns *default* when the variable is unset **or** empty, so that + ``TRADINGAGENTS_MID_THINK_LLM=`` in a ``.env`` file is treated the + same as not setting it at all (preserving the ``None`` semantics for + "fall back to the parent setting"). + """ + val = os.getenv(f"TRADINGAGENTS_{key.upper()}") + if not val: # None or "" + return default + return val + + +def _env_int(key: str, default=None): + """Like :func:`_env` but coerces the value to ``int``.""" + val = _env(key) + if val is None: + return default + try: + return int(val) + except (ValueError, TypeError): + return default + DEFAULT_CONFIG = { "project_dir": os.path.abspath(os.path.join(os.path.dirname(__file__), ".")), - "results_dir": os.getenv("TRADINGAGENTS_RESULTS_DIR", "./results"), + "results_dir": _env("RESULTS_DIR", "./results"), "data_cache_dir": os.path.join( os.path.abspath(os.path.join(os.path.dirname(__file__), ".")), "dataflows/data_cache", ), - # LLM settings - "mid_think_llm": "qwen3.5:27b", # falls back to quick_think_llm when None - "quick_think_llm": "qwen3.5:27b", + # LLM settings — all overridable via TRADINGAGENTS_ env vars + "llm_provider": _env("LLM_PROVIDER", "openai"), + "deep_think_llm": _env("DEEP_THINK_LLM", "gpt-5.2"), + "mid_think_llm": _env("MID_THINK_LLM"), # falls back to quick_think_llm when None + "quick_think_llm": _env("QUICK_THINK_LLM", "gpt-5-mini"), + "backend_url": _env("BACKEND_URL", "https://api.openai.com/v1"), # Per-role provider overrides (fall back to llm_provider / backend_url when None) - "deep_think_llm_provider": "openrouter", - "deep_think_llm": "deepseek/deepseek-r1-0528", - "deep_think_backend_url": None, # uses OpenRouter's default URL - "mid_think_llm_provider": "ollama", # falls back to ollama - "mid_think_backend_url": "http://192.168.50.76:11434", # falls back to backend_url (ollama host) - "quick_think_llm_provider": "ollama", # falls back to ollama - "quick_think_backend_url": "http://192.168.50.76:11434", # falls back to backend_url (ollama host) + "deep_think_llm_provider": _env("DEEP_THINK_LLM_PROVIDER"), # e.g. "google", "anthropic", "openrouter" + "deep_think_backend_url": _env("DEEP_THINK_BACKEND_URL"), # override backend URL for deep-think model + "mid_think_llm_provider": _env("MID_THINK_LLM_PROVIDER"), # e.g. "ollama" + "mid_think_backend_url": _env("MID_THINK_BACKEND_URL"), # override backend URL for mid-think model + "quick_think_llm_provider": _env("QUICK_THINK_LLM_PROVIDER"), # e.g. "openai", "ollama" + "quick_think_backend_url": _env("QUICK_THINK_BACKEND_URL"), # override backend URL for quick-think model # Provider-specific thinking configuration (applies to all roles unless overridden) - "google_thinking_level": None, # "high", "minimal", etc. - "openai_reasoning_effort": None, # "medium", "high", "low" + "google_thinking_level": _env("GOOGLE_THINKING_LEVEL"), # "high", "minimal", etc. + "openai_reasoning_effort": _env("OPENAI_REASONING_EFFORT"), # "medium", "high", "low" # Per-role provider-specific thinking configuration - "deep_think_google_thinking_level": None, - "deep_think_openai_reasoning_effort": None, - "mid_think_google_thinking_level": None, - "mid_think_openai_reasoning_effort": None, - "quick_think_google_thinking_level": None, - "quick_think_openai_reasoning_effort": None, + "deep_think_google_thinking_level": _env("DEEP_THINK_GOOGLE_THINKING_LEVEL"), + "deep_think_openai_reasoning_effort": _env("DEEP_THINK_OPENAI_REASONING_EFFORT"), + "mid_think_google_thinking_level": _env("MID_THINK_GOOGLE_THINKING_LEVEL"), + "mid_think_openai_reasoning_effort": _env("MID_THINK_OPENAI_REASONING_EFFORT"), + "quick_think_google_thinking_level": _env("QUICK_THINK_GOOGLE_THINKING_LEVEL"), + "quick_think_openai_reasoning_effort": _env("QUICK_THINK_OPENAI_REASONING_EFFORT"), # Debate and discussion settings - "max_debate_rounds": 1, - "max_risk_discuss_rounds": 1, - "max_recur_limit": 100, + "max_debate_rounds": _env_int("MAX_DEBATE_ROUNDS", 1), + "max_risk_discuss_rounds": _env_int("MAX_RISK_DISCUSS_ROUNDS", 1), + "max_recur_limit": _env_int("MAX_RECUR_LIMIT", 100), # Data vendor configuration # Category-level configuration (default for all tools in category) "data_vendors": { - "core_stock_apis": "yfinance", # Options: alpha_vantage, yfinance - "technical_indicators": "yfinance", # Options: alpha_vantage, yfinance - "fundamental_data": "yfinance", # Options: alpha_vantage, yfinance - "news_data": "yfinance", # Options: alpha_vantage, yfinance - "scanner_data": "alpha_vantage", # Options: alpha_vantage (primary), yfinance (fallback) + "core_stock_apis": _env("VENDOR_CORE_STOCK_APIS", "yfinance"), + "technical_indicators": _env("VENDOR_TECHNICAL_INDICATORS", "yfinance"), + "fundamental_data": _env("VENDOR_FUNDAMENTAL_DATA", "yfinance"), + "news_data": _env("VENDOR_NEWS_DATA", "yfinance"), + "scanner_data": _env("VENDOR_SCANNER_DATA", "yfinance"), }, # Tool-level configuration (takes precedence over category-level) "tool_vendors": { diff --git a/tradingagents/graph/scanner_graph.py b/tradingagents/graph/scanner_graph.py index a6abab52..9bccd0ff 100644 --- a/tradingagents/graph/scanner_graph.py +++ b/tradingagents/graph/scanner_graph.py @@ -139,9 +139,10 @@ class ScannerGraph: } if self.debug: - trace = [] + # stream() yields partial state updates; use invoke() for the + # full accumulated state and print chunks for debugging only. for chunk in self.graph.stream(initial_state): - trace.append(chunk) - return trace[-1] if trace else initial_state + print(f"[scanner debug] chunk keys: {list(chunk.keys())}") + # Fall through to invoke() for the correct accumulated result return self.graph.invoke(initial_state) diff --git a/tradingagents/pipeline/macro_bridge.py b/tradingagents/pipeline/macro_bridge.py index 4f06ee9d..53d7a1fa 100644 --- a/tradingagents/pipeline/macro_bridge.py +++ b/tradingagents/pipeline/macro_bridge.py @@ -238,10 +238,10 @@ async def run_all_tickers( List of TickerResult in completion order. """ semaphore = asyncio.Semaphore(max_concurrent) - loop = asyncio.get_event_loop() async def _run_one(candidate: StockCandidate) -> TickerResult: async with semaphore: + loop = asyncio.get_running_loop() # TradingAgentsGraph is synchronous — run it in a thread pool return await loop.run_in_executor( None,