diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e7991eef --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +# Tests package + diff --git a/tests/test_openai_news_warnings.py b/tests/test_openai_news_warnings.py new file mode 100644 index 00000000..f2219f73 --- /dev/null +++ b/tests/test_openai_news_warnings.py @@ -0,0 +1,124 @@ +""" +Tests for OpenAI news dataflow functions to ensure proper warnings about hallucination risks. + +Issue #274: OpenAI is hallucinating and provides outdated news. +The OpenAI vendor for news retrieval doesn't have reliable real-time web search access, +so it may generate fake or outdated news based on its training data. +""" + +import pytest +import warnings +from unittest.mock import Mock, patch +from tradingagents.dataflows.openai import ( + get_global_news_openai, + get_stock_news_openai, + get_fundamentals_openai, +) + + +class TestOpenAINewsWarnings: + """Test that OpenAI news functions emit appropriate warnings about hallucination risks.""" + + @patch("tradingagents.dataflows.openai.OpenAI") + @patch("tradingagents.dataflows.openai.get_config") + def test_get_global_news_emits_warning(self, mock_get_config, mock_openai_class): + """Test that get_global_news_openai emits a warning about potential hallucination.""" + # Setup mocks + mock_config = { + "backend_url": "https://api.openai.com/v1", + "quick_think_llm": "gpt-4o-mini", + } + mock_get_config.return_value = mock_config + + mock_client = Mock() + mock_openai_class.return_value = mock_client + + # Mock the response + mock_response = Mock() + mock_response.output = [None, Mock(content=[Mock(text="Fake news content")])] + mock_client.responses.create.return_value = mock_response + + # Test that a warning is emitted + with pytest.warns(UserWarning, match="may hallucinate|outdated|unreliable"): + result = get_global_news_openai("2024-11-14", look_back_days=7, limit=5) + + assert result is not None + + @patch("tradingagents.dataflows.openai.OpenAI") + @patch("tradingagents.dataflows.openai.get_config") + def test_get_stock_news_emits_warning(self, mock_get_config, mock_openai_class): + """Test that get_stock_news_openai emits a warning about potential hallucination.""" + # Setup mocks + mock_config = { + "backend_url": "https://api.openai.com/v1", + "quick_think_llm": "gpt-4o-mini", + } + mock_get_config.return_value = mock_config + + mock_client = Mock() + mock_openai_class.return_value = mock_client + + # Mock the response + mock_response = Mock() + mock_response.output = [None, Mock(content=[Mock(text="Fake stock news")])] + mock_client.responses.create.return_value = mock_response + + # Test that a warning is emitted + with pytest.warns(UserWarning, match="may hallucinate|outdated|unreliable"): + result = get_stock_news_openai("NVDA", "2024-11-01", "2024-11-14") + + assert result is not None + + @patch("tradingagents.dataflows.openai.OpenAI") + @patch("tradingagents.dataflows.openai.get_config") + def test_get_fundamentals_emits_warning(self, mock_get_config, mock_openai_class): + """Test that get_fundamentals_openai emits a warning about potential hallucination.""" + # Setup mocks + mock_config = { + "backend_url": "https://api.openai.com/v1", + "quick_think_llm": "gpt-4o-mini", + } + mock_get_config.return_value = mock_config + + mock_client = Mock() + mock_openai_class.return_value = mock_client + + # Mock the response + mock_response = Mock() + mock_response.output = [None, Mock(content=[Mock(text="Fake fundamentals")])] + mock_client.responses.create.return_value = mock_response + + # Test that a warning is emitted + with pytest.warns(UserWarning, match="may hallucinate|outdated|unreliable"): + result = get_fundamentals_openai("NVDA", "2024-11-14") + + assert result is not None + + def test_warning_message_content(self): + """Test that warning messages contain helpful information about alternatives.""" + # This test verifies the warning message suggests using alternative vendors + with patch("tradingagents.dataflows.openai.OpenAI"), \ + patch("tradingagents.dataflows.openai.get_config") as mock_get_config: + + mock_get_config.return_value = { + "backend_url": "https://api.openai.com/v1", + "quick_think_llm": "gpt-4o-mini", + } + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + try: + get_global_news_openai("2024-11-14") + except Exception: + pass # We're only testing the warning, not the full execution + + # Check that at least one warning was issued + assert len(w) > 0 + + # Check that the warning mentions alternatives + warning_text = str(w[0].message).lower() + assert any(keyword in warning_text for keyword in [ + "alpha_vantage", "google", "local", "alternative", "vendor" + ]) + diff --git a/tradingagents/dataflows/openai.py b/tradingagents/dataflows/openai.py index 91a2258b..d9c3b78b 100644 --- a/tradingagents/dataflows/openai.py +++ b/tradingagents/dataflows/openai.py @@ -1,8 +1,50 @@ +import warnings from openai import OpenAI from .config import get_config +def _warn_hallucination_risk(data_type="news", category="news_data", alternatives=None): + """ + Emit a warning about potential hallucination when using OpenAI for data retrieval. + + Args: + data_type: Type of data being retrieved (e.g., "news", "fundamental data") + category: Config category for vendor selection (e.g., "news_data", "fundamental_data") + alternatives: List of alternative vendor names (default: ["alpha_vantage", "google", "local"]) + """ + if alternatives is None: + alternatives = ["alpha_vantage", "google", "local"] + + alternatives_str = "', '".join(alternatives) + warnings.warn( + f"OpenAI {data_type} vendor may hallucinate or provide outdated {data_type}. " + f"For reliable {data_type}, use alternative vendors: '{alternatives_str}'. " + f"Configure in config['data_vendors']['{category}'].", + UserWarning, + stacklevel=3 + ) + + def get_stock_news_openai(query, start_date, end_date): + """ + Retrieve stock news using OpenAI's LLM. + + WARNING: This function may hallucinate or provide outdated news because it relies on + the LLM's training data rather than real-time web search. For reliable, up-to-date news, + consider using alternative vendors such as 'alpha_vantage', 'google', or 'local'. + + Configure alternative vendors in your config: + config["data_vendors"]["news_data"] = "alpha_vantage" # or "google" or "local" + + Args: + query: Stock ticker or search query + start_date: Start date in yyyy-mm-dd format + end_date: End date in yyyy-mm-dd format + + Returns: + str: News content (may be hallucinated or outdated) + """ + _warn_hallucination_risk(data_type="news", category="news_data") config = get_config() client = OpenAI(base_url=config["backend_url"]) @@ -38,6 +80,26 @@ def get_stock_news_openai(query, start_date, end_date): def get_global_news_openai(curr_date, look_back_days=7, limit=5): + """ + Retrieve global news using OpenAI's LLM. + + WARNING: This function may hallucinate or provide outdated news because it relies on + the LLM's training data rather than real-time web search. For reliable, up-to-date news, + consider using alternative vendors such as 'alpha_vantage', 'google', or 'local'. + + Configure alternative vendors in your config: + config["data_vendors"]["news_data"] = "alpha_vantage" # or "google" or "local" + + Args: + curr_date: Current date in yyyy-mm-dd format + look_back_days: Number of days to look back (default: 7) + limit: Maximum number of articles to return (default: 5) + + Returns: + str: News content (may be hallucinated or outdated) + """ + _warn_hallucination_risk(data_type="news", category="news_data") + config = get_config() client = OpenAI(base_url=config["backend_url"]) @@ -73,6 +135,29 @@ def get_global_news_openai(curr_date, look_back_days=7, limit=5): def get_fundamentals_openai(ticker, curr_date): + """ + Retrieve fundamental data using OpenAI's LLM. + + WARNING: This function may hallucinate or provide outdated data because it relies on + the LLM's training data rather than real-time data sources. For reliable, up-to-date + fundamental data, consider using alternative vendors such as 'alpha_vantage', 'yfinance', or 'local'. + + Configure alternative vendors in your config: + config["data_vendors"]["fundamental_data"] = "alpha_vantage" # or "yfinance" or "local" + + Args: + ticker: Stock ticker symbol + curr_date: Current date in yyyy-mm-dd format + + Returns: + str: Fundamental data (may be hallucinated or outdated) + """ + _warn_hallucination_risk( + data_type="fundamental data", + category="fundamental_data", + alternatives=["alpha_vantage", "yfinance", "local"] + ) + config = get_config() client = OpenAI(base_url=config["backend_url"])