fix: suppress memory injection when memory is empty (hallucination guard, closes #572)

When past_memories is empty, all five agents previously injected an
empty string into their prompts while still instructing the LLM to
"address reflections and learn from past mistakes" — causing the LLM
to hallucinate fabricated lessons on first run.

Each agent now conditionally builds its memory section only when
past_memories is non-empty, so the injection and its instruction
are both absent when there is nothing to recall.

Also fixes import ordering in memory.py (logger after imports).

Tests: tests/test_hallucination_guard.py covers empty and populated
memory for all five agents (bull, bear, trader, research manager,
portfolio manager).

Companion to #563 (memory persistence).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Zhigong Liu 2026-04-18 22:30:31 -04:00
parent bf7d27e0a9
commit 2cdada6300
7 changed files with 247 additions and 30 deletions

View File

@ -0,0 +1,204 @@
"""Tests for memory hallucination guard across all five agents.
Verifies that:
- Memory section headers are ABSENT from prompts when memory is empty.
- Memory content IS injected when memory is populated.
No LLM API calls are made llm.invoke() is mocked.
"""
from unittest.mock import MagicMock
from tradingagents.agents.researchers.bull_researcher import create_bull_researcher
from tradingagents.agents.researchers.bear_researcher import create_bear_researcher
from tradingagents.agents.trader.trader import create_trader
from tradingagents.agents.managers.research_manager import create_research_manager
from tradingagents.agents.managers.portfolio_manager import create_portfolio_manager
# ---------------------------------------------------------------------------
# Shared helpers
# ---------------------------------------------------------------------------
def _make_state():
return {
"investment_debate_state": {
"history": "",
"bull_history": "",
"bear_history": "",
"current_response": "",
"count": 0,
},
"market_report": "market data",
"sentiment_report": "sentiment data",
"news_report": "news data",
"fundamentals_report": "fundamentals data",
"company_of_interest": "AAPL",
"trade_date": "2024-01-15",
"investment_plan": "buy AAPL",
"trader_investment_plan": "",
"risk_debate_state": {
"history": "",
"aggressive_history": "",
"conservative_history": "",
"neutral_history": "",
"judge_decision": "",
"count": 0,
"latest_speaker": "",
"current_aggressive_response": "",
"current_conservative_response": "",
"current_neutral_response": "",
},
}
def _mock_llm():
llm = MagicMock()
llm.invoke.return_value = MagicMock(content="mocked response")
return llm
def _empty_memory():
m = MagicMock()
m.get_memories.return_value = []
return m
def _populated_memory(lesson="Past lesson: watch macro risk"):
m = MagicMock()
m.get_memories.return_value = [{"recommendation": lesson, "similarity_score": 0.9}]
return m
# ---------------------------------------------------------------------------
# Bull Researcher
# ---------------------------------------------------------------------------
def test_bull_omits_memory_section_when_empty():
"""No reflections header or instruction when memory is empty."""
llm = _mock_llm()
node = create_bull_researcher(llm, _empty_memory())
node(_make_state())
prompt = llm.invoke.call_args[0][0]
assert "Reflections from similar situations" not in prompt
assert "address reflections" not in prompt.lower()
assert "learn from lessons and mistakes" not in prompt.lower()
def test_bull_includes_memory_section_when_populated():
"""Lesson text appears in prompt when memory is populated."""
llm = _mock_llm()
node = create_bull_researcher(llm, _populated_memory("Reduce tech exposure on rate hikes"))
node(_make_state())
prompt = llm.invoke.call_args[0][0]
assert "Reduce tech exposure on rate hikes" in prompt
assert "Reflections from similar situations" in prompt
# ---------------------------------------------------------------------------
# Bear Researcher
# ---------------------------------------------------------------------------
def test_bear_omits_memory_section_when_empty():
"""No reflections header or instruction when memory is empty."""
llm = _mock_llm()
node = create_bear_researcher(llm, _empty_memory())
node(_make_state())
prompt = llm.invoke.call_args[0][0]
assert "Reflections from similar situations" not in prompt
assert "address reflections" not in prompt.lower()
assert "learn from lessons and mistakes" not in prompt.lower()
def test_bear_includes_memory_section_when_populated():
"""Lesson text appears in prompt when memory is populated."""
llm = _mock_llm()
node = create_bear_researcher(llm, _populated_memory("Overestimated resilience in 2022"))
node(_make_state())
prompt = llm.invoke.call_args[0][0]
assert "Overestimated resilience in 2022" in prompt
assert "Reflections from similar situations" in prompt
# ---------------------------------------------------------------------------
# Trader
# ---------------------------------------------------------------------------
def test_trader_omits_reflection_clause_when_empty():
"""No reflection text or 'No past memories found.' in system message when empty."""
llm = _mock_llm()
node = create_trader(llm, _empty_memory())
node(_make_state())
messages = llm.invoke.call_args[0][0]
system_content = messages[0]["content"]
assert "No past memories found" not in system_content
assert "Here are reflections" not in system_content
assert "Apply lessons from past decisions" not in system_content
def test_trader_includes_reflection_clause_when_populated():
"""Lesson text appears in system message when memory is populated."""
llm = _mock_llm()
node = create_trader(llm, _populated_memory("Avoid chasing momentum tops"))
node(_make_state())
messages = llm.invoke.call_args[0][0]
system_content = messages[0]["content"]
assert "Avoid chasing momentum tops" in system_content
assert "Apply lessons from past decisions" in system_content
# ---------------------------------------------------------------------------
# Research Manager
# ---------------------------------------------------------------------------
def test_research_manager_omits_memory_section_when_empty():
"""No past reflections header when memory is empty."""
llm = _mock_llm()
node = create_research_manager(llm, _empty_memory())
node(_make_state())
prompt = llm.invoke.call_args[0][0]
assert "Here are your past reflections on mistakes" not in prompt
assert "Take into account your past mistakes" not in prompt
def test_research_manager_includes_memory_section_when_populated():
"""Lesson text and header appear in prompt when memory is populated."""
llm = _mock_llm()
node = create_research_manager(llm, _populated_memory("Missed earnings surprise signal"))
node(_make_state())
prompt = llm.invoke.call_args[0][0]
assert "Missed earnings surprise signal" in prompt
assert "Here are your past reflections on mistakes" in prompt
# ---------------------------------------------------------------------------
# Portfolio Manager
# ---------------------------------------------------------------------------
def test_portfolio_manager_omits_lessons_line_when_empty():
"""No 'Lessons from past decisions' line when memory is empty."""
llm = _mock_llm()
node = create_portfolio_manager(llm, _empty_memory())
node(_make_state())
prompt = llm.invoke.call_args[0][0]
assert "Lessons from past decisions" not in prompt
def test_portfolio_manager_includes_lessons_line_when_populated():
"""Lesson text and label appear in prompt when memory is populated."""
llm = _mock_llm()
node = create_portfolio_manager(llm, _populated_memory("Size down in low-liquidity names"))
node(_make_state())
prompt = llm.invoke.call_args[0][0]
assert "Size down in low-liquidity names" in prompt
assert "Lessons from past decisions" in prompt

View File

@ -18,9 +18,11 @@ def create_portfolio_manager(llm, memory):
curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}" curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}"
past_memories = memory.get_memories(curr_situation, n_matches=2) past_memories = memory.get_memories(curr_situation, n_matches=2)
past_memory_str = "" if past_memories:
for i, rec in enumerate(past_memories, 1): past_memory_str = "".join(rec["recommendation"] + "\n\n" for rec in past_memories)
past_memory_str += rec["recommendation"] + "\n\n" lessons_line = f"- Lessons from past decisions: **{past_memory_str}**\n"
else:
lessons_line = ""
prompt = f"""As the Portfolio Manager, synthesize the risk analysts' debate and deliver the final trading decision. prompt = f"""As the Portfolio Manager, synthesize the risk analysts' debate and deliver the final trading decision.
@ -38,7 +40,7 @@ def create_portfolio_manager(llm, memory):
**Context:** **Context:**
- Research Manager's investment plan: **{research_plan}** - Research Manager's investment plan: **{research_plan}**
- Trader's transaction proposal: **{trader_plan}** - Trader's transaction proposal: **{trader_plan}**
- Lessons from past decisions: **{past_memory_str}** {lessons_line}
**Required Output Structure:** **Required Output Structure:**
1. **Rating**: State one of Buy / Overweight / Hold / Underweight / Sell. 1. **Rating**: State one of Buy / Overweight / Hold / Underweight / Sell.

View File

@ -16,9 +16,15 @@ def create_research_manager(llm, memory):
curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}" curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}"
past_memories = memory.get_memories(curr_situation, n_matches=2) past_memories = memory.get_memories(curr_situation, n_matches=2)
past_memory_str = "" if past_memories:
for i, rec in enumerate(past_memories, 1): past_memory_str = "".join(rec["recommendation"] + "\n\n" for rec in past_memories)
past_memory_str += rec["recommendation"] + "\n\n" memory_section = (
"Take into account your past mistakes on similar situations."
" Use these insights to refine your decision-making and ensure you are learning and improving.\n\n"
f"Here are your past reflections on mistakes:\n\"{past_memory_str}\"\n"
)
else:
memory_section = ""
prompt = f"""As the portfolio manager and debate facilitator, your role is to critically evaluate this round of debate and make a definitive decision: align with the bear analyst, the bull analyst, or choose Hold only if it is strongly justified based on the arguments presented. prompt = f"""As the portfolio manager and debate facilitator, your role is to critically evaluate this round of debate and make a definitive decision: align with the bear analyst, the bull analyst, or choose Hold only if it is strongly justified based on the arguments presented.
@ -29,11 +35,9 @@ Additionally, develop a detailed investment plan for the trader. This should inc
Your Recommendation: A decisive stance supported by the most convincing arguments. Your Recommendation: A decisive stance supported by the most convincing arguments.
Rationale: An explanation of why these arguments lead to your conclusion. Rationale: An explanation of why these arguments lead to your conclusion.
Strategic Actions: Concrete steps for implementing the recommendation. Strategic Actions: Concrete steps for implementing the recommendation.
Take into account your past mistakes on similar situations. Use these insights to refine your decision-making and ensure you are learning and improving. Present your analysis conversationally, as if speaking naturally, without special formatting. Present your analysis conversationally, as if speaking naturally, without special formatting.
Here are your past reflections on mistakes:
\"{past_memory_str}\"
{memory_section}
{instrument_context} {instrument_context}
Here is the debate: Here is the debate:

View File

@ -15,9 +15,12 @@ def create_bear_researcher(llm, memory):
curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}" curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}"
past_memories = memory.get_memories(curr_situation, n_matches=2) past_memories = memory.get_memories(curr_situation, n_matches=2)
past_memory_str = "" memory_section = (
for i, rec in enumerate(past_memories, 1): "Reflections from similar situations and lessons learned:\n"
past_memory_str += rec["recommendation"] + "\n\n" + "".join(rec["recommendation"] + "\n\n" for rec in past_memories)
+ "\nUse these past reflections to strengthen your argument."
if past_memories else ""
)
prompt = f"""You are a Bear Analyst making the case against investing in the stock. Your goal is to present a well-reasoned argument emphasizing risks, challenges, and negative indicators. Leverage the provided research and data to highlight potential downsides and counter bullish arguments effectively. prompt = f"""You are a Bear Analyst making the case against investing in the stock. Your goal is to present a well-reasoned argument emphasizing risks, challenges, and negative indicators. Leverage the provided research and data to highlight potential downsides and counter bullish arguments effectively.
@ -37,8 +40,8 @@ Latest world affairs news: {news_report}
Company fundamentals report: {fundamentals_report} Company fundamentals report: {fundamentals_report}
Conversation history of the debate: {history} Conversation history of the debate: {history}
Last bull argument: {current_response} Last bull argument: {current_response}
Reflections from similar situations and lessons learned: {past_memory_str} {memory_section}
Use this information to deliver a compelling bear argument, refute the bull's claims, and engage in a dynamic debate that demonstrates the risks and weaknesses of investing in the stock. You must also address reflections and learn from lessons and mistakes you made in the past. Use this information to deliver a compelling bear argument, refute the bull's claims, and engage in a dynamic debate that demonstrates the risks and weaknesses of investing in the stock.
""" """
response = llm.invoke(prompt) response = llm.invoke(prompt)

View File

@ -15,9 +15,12 @@ def create_bull_researcher(llm, memory):
curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}" curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}"
past_memories = memory.get_memories(curr_situation, n_matches=2) past_memories = memory.get_memories(curr_situation, n_matches=2)
past_memory_str = "" memory_section = (
for i, rec in enumerate(past_memories, 1): "Reflections from similar situations and lessons learned:\n"
past_memory_str += rec["recommendation"] + "\n\n" + "".join(rec["recommendation"] + "\n\n" for rec in past_memories)
+ "\nUse these past reflections to strengthen your argument."
if past_memories else ""
)
prompt = f"""You are a Bull Analyst advocating for investing in the stock. Your task is to build a strong, evidence-based case emphasizing growth potential, competitive advantages, and positive market indicators. Leverage the provided research and data to address concerns and counter bearish arguments effectively. prompt = f"""You are a Bull Analyst advocating for investing in the stock. Your task is to build a strong, evidence-based case emphasizing growth potential, competitive advantages, and positive market indicators. Leverage the provided research and data to address concerns and counter bearish arguments effectively.
@ -35,8 +38,8 @@ Latest world affairs news: {news_report}
Company fundamentals report: {fundamentals_report} Company fundamentals report: {fundamentals_report}
Conversation history of the debate: {history} Conversation history of the debate: {history}
Last bear argument: {current_response} Last bear argument: {current_response}
Reflections from similar situations and lessons learned: {past_memory_str} {memory_section}
Use this information to deliver a compelling bull argument, refute the bear's concerns, and engage in a dynamic debate that demonstrates the strengths of the bull position. You must also address reflections and learn from lessons and mistakes you made in the past. Use this information to deliver a compelling bull argument, refute the bear's concerns, and engage in a dynamic debate that demonstrates the strengths of the bull position.
""" """
response = llm.invoke(prompt) response = llm.invoke(prompt)

View File

@ -16,12 +16,12 @@ def create_trader(llm, memory):
curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}" curr_situation = f"{market_research_report}\n\n{sentiment_report}\n\n{news_report}\n\n{fundamentals_report}"
past_memories = memory.get_memories(curr_situation, n_matches=2) past_memories = memory.get_memories(curr_situation, n_matches=2)
past_memory_str = "" reflection_clause = (
if past_memories: " Apply lessons from past decisions to strengthen your analysis."
for i, rec in enumerate(past_memories, 1): " Here are reflections from similar situations you traded in and the lessons learned:\n"
past_memory_str += rec["recommendation"] + "\n\n" + "".join(rec["recommendation"] + "\n\n" for rec in past_memories)
else: if past_memories else ""
past_memory_str = "No past memories found." )
context = { context = {
"role": "user", "role": "user",
@ -31,7 +31,7 @@ def create_trader(llm, memory):
messages = [ messages = [
{ {
"role": "system", "role": "system",
"content": f"""You are a trading agent analyzing market data to make investment decisions. Based on your analysis, provide a specific recommendation to buy, sell, or hold. End with a firm decision and always conclude your response with 'FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL**' to confirm your recommendation. Apply lessons from past decisions to strengthen your analysis. Here are reflections from similar situations you traded in and the lessons learned: {past_memory_str}""", "content": f"""You are a trading agent analyzing market data to make investment decisions. Based on your analysis, provide a specific recommendation to buy, sell, or hold. End with a firm decision and always conclude your response with 'FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL**' to confirm your recommendation.{reflection_clause}""",
}, },
context, context,
] ]

View File

@ -6,13 +6,14 @@ no token limits, works offline with any LLM provider.
import json import json
import logging import logging
import re
import tempfile import tempfile
from pathlib import Path from pathlib import Path
logger = logging.getLogger(__name__)
from rank_bm25 import BM25Okapi from rank_bm25 import BM25Okapi
from typing import List, Tuple from typing import List, Tuple
import re
logger = logging.getLogger(__name__)
class FinancialSituationMemory: class FinancialSituationMemory: