fix: add exponential backoff retry for yfinance rate limits (#426)

tradingagents/dataflows/stockstats_utils.py

@@ -1,10 +1,35 @@
+import time
+import logging
+
 import pandas as pd
 import yfinance as yf
+from yfinance.exceptions import YFRateLimitError
 from stockstats import wrap
 from typing import Annotated
 import os
 from .config import get_config
 
+logger = logging.getLogger(__name__)
+
+
+def yf_retry(func, max_retries=3, base_delay=2.0):
+    """Execute a yfinance call with exponential backoff on rate limits.
+
+    yfinance raises YFRateLimitError on HTTP 429 responses but does not
+    retry them internally. This wrapper adds retry logic specifically
+    for rate limits. Other exceptions propagate immediately.
+    """
+    for attempt in range(max_retries + 1):
+        try:
+            return func()
+        except YFRateLimitError:
+            if attempt < max_retries:
+                delay = base_delay * (2 ** attempt)
+                logger.warning(f"Yahoo Finance rate limited, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries})")
+                time.sleep(delay)
+            else:
+                raise
+
 
 def _clean_dataframe(data: pd.DataFrame) -> pd.DataFrame:
     """Normalize a stock DataFrame for stockstats: parse dates, drop invalid rows, fill price gaps."""
@@ -51,14 +76,14 @@ class StockstatsUtils:
         if os.path.exists(data_file):
             data = pd.read_csv(data_file, on_bad_lines="skip")
         else:
-            data = yf.download(
+            data = yf_retry(lambda: yf.download(
                 symbol,
                 start=start_date_str,
                 end=end_date_str,
                 multi_level_index=False,
                 progress=False,
                 auto_adjust=True,
-            )
+            ))
             data = data.reset_index()
             data.to_csv(data_file, index=False)
 

tradingagents/dataflows/y_finance.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from dateutil.relativedelta import relativedelta
 import yfinance as yf
 import os
-from .stockstats_utils import StockstatsUtils, _clean_dataframe
+from .stockstats_utils import StockstatsUtils, _clean_dataframe, yf_retry
 
 def get_YFin_data_online(
     symbol: Annotated[str, "ticker symbol of the company"],
@@ -18,7 +18,7 @@ def get_YFin_data_online(
     ticker = yf.Ticker(symbol.upper())
 
     # Fetch historical data for the specified date range
-    data = ticker.history(start=start_date, end=end_date)
+    data = yf_retry(lambda: ticker.history(start=start_date, end=end_date))
 
     # Check if data is empty
     if data.empty:
@@ -234,14 +234,14 @@ def _get_stock_stats_bulk(
         if os.path.exists(data_file):
             data = pd.read_csv(data_file, on_bad_lines="skip")
         else:
-            data = yf.download(
+            data = yf_retry(lambda: yf.download(
                 symbol,
                 start=start_date_str,
                 end=end_date_str,
                 multi_level_index=False,
                 progress=False,
                 auto_adjust=True,
-            )
+            ))
             data = data.reset_index()
             data.to_csv(data_file, index=False)
 
@@ -300,7 +300,7 @@ def get_fundamentals(
     """Get company fundamentals overview from yfinance."""
     try:
         ticker_obj = yf.Ticker(ticker.upper())
-        info = ticker_obj.info
+        info = yf_retry(lambda: ticker_obj.info)
 
         if not info:
             return f"No fundamentals data found for symbol '{ticker}'"
@@ -360,9 +360,9 @@ def get_balance_sheet(
         ticker_obj = yf.Ticker(ticker.upper())
 
         if freq.lower() == "quarterly":
-            data = ticker_obj.quarterly_balance_sheet
+            data = yf_retry(lambda: ticker_obj.quarterly_balance_sheet)
         else:
-            data = ticker_obj.balance_sheet
+            data = yf_retry(lambda: ticker_obj.balance_sheet)
             
         if data.empty:
             return f"No balance sheet data found for symbol '{ticker}'"
@@ -390,9 +390,9 @@ def get_cashflow(
         ticker_obj = yf.Ticker(ticker.upper())
 
         if freq.lower() == "quarterly":
-            data = ticker_obj.quarterly_cashflow
+            data = yf_retry(lambda: ticker_obj.quarterly_cashflow)
         else:
-            data = ticker_obj.cashflow
+            data = yf_retry(lambda: ticker_obj.cashflow)
             
         if data.empty:
             return f"No cash flow data found for symbol '{ticker}'"
@@ -420,9 +420,9 @@ def get_income_statement(
         ticker_obj = yf.Ticker(ticker.upper())
 
         if freq.lower() == "quarterly":
-            data = ticker_obj.quarterly_income_stmt
+            data = yf_retry(lambda: ticker_obj.quarterly_income_stmt)
         else:
-            data = ticker_obj.income_stmt
+            data = yf_retry(lambda: ticker_obj.income_stmt)
             
         if data.empty:
             return f"No income statement data found for symbol '{ticker}'"
@@ -446,7 +446,7 @@ def get_insider_transactions(
     """Get insider transactions data from yfinance."""
     try:
         ticker_obj = yf.Ticker(ticker.upper())
-        data = ticker_obj.insider_transactions
+        data = yf_retry(lambda: ticker_obj.insider_transactions)
         
         if data is None or data.empty:
             return f"No insider transactions data found for symbol '{ticker}'"