diff --git a/MERGE_DOCUMENTATION.md b/MERGE_DOCUMENTATION.md index 1a62cdaf..196c5417 100644 --- a/MERGE_DOCUMENTATION.md +++ b/MERGE_DOCUMENTATION.md @@ -5,6 +5,7 @@ This document provides a comprehensive overview of the successful merge of Chinese market features from the TradingAgentsCN project into the main TradingAgents repository. The integration includes DashScope (Alibaba Cloud) LLM support, Tushare API for A-share data, advanced database caching, and enhanced CLI market selection. **Merge Details**: + - **Date**: January 2025 - **Branch**: `full-merge-chinese-features` - **Source**: TradingAgentsCN directory @@ -20,6 +21,7 @@ This document provides a comprehensive overview of the successful merge of Chine **Status**: ✅ **COMPLETE AND TESTED** **What was added**: + - Complete DashScope LLM provider integration in CLI - Support for Qwen model family: qwen-turbo, qwen-plus, qwen-max, qwen-max-longcontext - DashScope embedding service for memory system @@ -27,6 +29,7 @@ This document provides a comprehensive overview of the successful merge of Chine - Comprehensive error handling and API key validation **Key Files**: + ``` cli/utils.py - DashScope LLM provider options tradingagents/graph/trading_graph.py - DashScope LLM initialization @@ -35,6 +38,7 @@ tradingagents/default_config.py - Configuration examples ``` **Configuration Required**: + ```env DASHSCOPE_API_KEY=your_dashscope_api_key_here ``` @@ -48,10 +52,11 @@ DASHSCOPE_API_KEY=your_dashscope_api_key_here **Status**: ✅ **COMPLETE AND TESTED** **What was added**: + - Tushare API integration for real-time A-share data - Support for all major Chinese stock exchanges: - Shanghai Stock Exchange: 60xxxx (e.g., 600036) - - Shenzhen Stock Exchange: 00xxxx (e.g., 000001) + - Shenzhen Stock Exchange: 00xxxx (e.g., 000001) - ChiNext Board: 30xxxx (e.g., 300001) - STAR Market: 68xxxx (e.g., 688001) - Optimized China data provider with intelligent caching @@ -59,6 +64,7 @@ DASHSCOPE_API_KEY=your_dashscope_api_key_here - Unified stock data service with automatic fallback **Key Files**: + ``` tradingagents/dataflows/tushare_utils.py - Tushare data provider tradingagents/dataflows/optimized_china_data.py - Optimized A-share data @@ -67,12 +73,14 @@ tradingagents/dataflows/stock_data_service.py - Unified data service ``` **Dependencies Added**: + ``` tushare>=1.2.0 beautifulsoup4>=4.9.0 ``` **Data Flow Architecture**: + ``` MongoDB Database → Tushare API → File Cache → Error Handling ``` @@ -84,6 +92,7 @@ MongoDB Database → Tushare API → File Cache → Error Handling **Status**: ✅ **COMPLETE AND TESTED** **What was added**: + - MongoDB integration for persistent data storage and analytics - Redis integration for high-performance caching - Database cache manager with intelligent routing @@ -92,6 +101,7 @@ MongoDB Database → Tushare API → File Cache → Error Handling - Integrated cache manager with adaptive performance optimization **Key Files**: + ``` tradingagents/config/database_config.py - Database configuration tradingagents/config/database_manager.py - Connection management @@ -103,12 +113,14 @@ tradingagents/dataflows/adaptive_cache.py - Adaptive cache system ``` **Dependencies Added**: + ``` pymongo>=4.0.0 redis>=4.0.0 ``` **Configuration (Optional)**: + ```env # MongoDB MONGODB_ENABLED=false @@ -135,6 +147,7 @@ REDIS_DB=0 **Status**: ✅ **COMPLETE AND TESTED** **What was added**: + - Interactive market selection interface - Market-specific ticker format validation with examples - Automatic data source routing based on market selection @@ -142,6 +155,7 @@ REDIS_DB=0 - Comprehensive format validation and error messages **Key Files**: + ``` cli/utils.py - Added select_market() and enhanced get_ticker() cli/main.py - Updated workflow with market selection step @@ -150,11 +164,12 @@ cli/main.py - Updated workflow with market selection step **Supported Markets**: 1. **US Stock Market** + - Format: 1-5 letter symbols (e.g., AAPL, SPY, TSLA) - Data Source: Yahoo Finance - Validation Pattern: `^[A-Z]{1,5}$` +2. **China A-Share Market** -2. **China A-Share Market** - Format: 6-digit numeric codes (e.g., 000001, 600036) - Data Source: Tushare API - Validation Pattern: `^\d{6}$` @@ -168,17 +183,20 @@ cli/main.py - Updated workflow with market selection step **Status**: ✅ **COMPLETE AND TESTED** **What was added**: + - IntegratedCacheManager as default cache system in CLI - Automatic selection between database and file caching - Intelligent fallback mechanisms for high availability - Performance optimization with adaptive caching strategies **Key Changes**: + ``` tradingagents/dataflows/cache_manager.py - Enhanced get_cache() function ``` **Cache Priority Logic**: + ``` 1. Database cache (MongoDB/Redis) - if enabled and available 2. File cache - reliable fallback @@ -190,18 +208,21 @@ tradingagents/dataflows/cache_manager.py - Enhanced get_cache() function ## 🔧 Technical Improvements ### 📦 Dependency Management + - ✅ All new dependencies properly added to `requirements.txt` - ✅ Optional dependencies with graceful fallbacks - ✅ No breaking changes to existing functionality - ✅ Backward compatibility maintained ### 🛡️ Error Handling & Reliability + - ✅ Comprehensive error handling for all new features - ✅ Graceful degradation when external services unavailable - ✅ Detailed error messages with user guidance - ✅ Automatic retry logic for API calls ### 🔄 Backward Compatibility + - ✅ All existing functionality preserved and working - ✅ New features are optional and configurable - ✅ Default behavior unchanged for existing users @@ -212,17 +233,21 @@ tradingagents/dataflows/cache_manager.py - Enhanced get_cache() function ## ⚠️ Known Issues & Limitations ### 1. Optional Dependencies + **Issue**: Some dependencies not installed by default **Impact**: Limited functionality until manually installed -**Solution**: +**Solution**: + ```bash -pip install pytdx beautifulsoup4 +pip install tushare beautifulsoup4 ``` ### 2. Database Services + **Issue**: MongoDB and Redis disabled by default in `.env` **Impact**: Database caching features not active by default **Solution**: Enable in `.env` and start database services: + ```bash # Start services docker run -d -p 27017:27017 --name mongodb mongo @@ -234,6 +259,7 @@ REDIS_ENABLED=true ``` ### 3. API Rate Limits + **Issue**: Tushare API may have rate limiting for free accounts **Impact**: Potential delays in A-share data retrieval **Mitigation**: Intelligent caching and retry logic implemented @@ -243,15 +269,18 @@ REDIS_ENABLED=true ## 🚧 Incomplete Features / Future Work ### 1. Data Source Selection UI + **Status**: ❌ **NOT IMPLEMENTED** **Description**: User interface to manually choose between cache and Tushare API **Current State**: Automatic fallback logic only **Future Enhancement**: Add CLI option for data source preference ### 2. Advanced A-Share Analytics + **Status**: ⚠️ **PARTIALLY IMPLEMENTED** **Completed**: Basic data retrieval and caching **Missing**: + - Real-time market sentiment analysis - A-share specific technical indicators - Chinese financial news sentiment integration @@ -259,8 +288,10 @@ REDIS_ENABLED=true - A-share market hours and trading calendar ### 3. Performance Monitoring & Analytics + **Status**: ❌ **NOT IMPLEMENTED** **Missing Features**: + - Database performance metrics dashboard - Cache hit/miss statistics - API response time monitoring @@ -268,17 +299,21 @@ REDIS_ENABLED=true - Cost tracking for API calls ### 4. Configuration Management + **Status**: ⚠️ **BASIC IMPLEMENTATION** **Current**: Basic configuration validation **Missing**: + - Comprehensive configuration validation wizard - Interactive setup guide for new users - Configuration health checks and diagnostics - Automatic configuration migration tools ### 5. Advanced Tushare Features + **Status**: ❌ **NOT IMPLEMENTED** **Missing**: + - Real-time tick data streaming - Level-2 market data integration - Options and futures data support @@ -290,6 +325,7 @@ REDIS_ENABLED=true ## 📊 Testing & Validation Status ### ✅ Completed & Verified + - **DashScope LLM Integration**: ✅ All models working - **Tushare API Functionality**: ✅ Data retrieval working - **Database Connectivity**: ✅ MongoDB and Redis connections @@ -300,6 +336,7 @@ REDIS_ENABLED=true - **Error Handling**: ✅ Graceful degradation verified ### ⚠️ Needs Further Testing + - End-to-end A-share analysis workflow under load - Database performance with large datasets - Tushare API behavior under rate limiting @@ -311,17 +348,19 @@ REDIS_ENABLED=true ## 🎯 Deployment Recommendations ### For Immediate Use (Minimal Setup) + 1. **Install optional dependencies**: + ```bash pip install pytdx beautifulsoup4 ``` - 2. **Configure DashScope API**: + ```env DASHSCOPE_API_KEY=your_actual_api_key ``` - 3. **Test A-share functionality**: + ```bash python -m cli.main # Select: China A-Share @@ -329,24 +368,27 @@ REDIS_ENABLED=true ``` ### For Production Deployment (Full Features) + 1. **Setup database services**: + ```bash docker run -d -p 27017:27017 --name mongodb mongo docker run -d -p 6379:6379 --name redis redis ``` - 2. **Enable database caching**: + ```env MONGODB_ENABLED=true REDIS_ENABLED=true ``` - 3. **Configure monitoring and logging**: + - Implement application logging - Setup database monitoring - Configure API usage tracking ### For Development Environment + 1. **Setup development databases** with persistent volumes 2. **Configure development API keys** with rate limiting 3. **Enable debug logging** for troubleshooting @@ -357,6 +399,7 @@ REDIS_ENABLED=true ## 📈 Impact Assessment ### ✅ Positive Impacts + - **🇨🇳 Chinese Market Access**: Complete A-share market analysis capability - **🚀 Performance**: Database caching significantly improves data access speed - **🔄 Reliability**: Multiple LLM providers increase system reliability @@ -365,6 +408,7 @@ REDIS_ENABLED=true - **💾 Scalability**: Database integration enables enterprise deployment ### ⚠️ Considerations + - **🔧 Complexity**: Increased system complexity with multiple data sources - **📦 Dependencies**: Additional external dependencies and services - **🛠️ Maintenance**: More components require monitoring and maintenance @@ -378,6 +422,7 @@ REDIS_ENABLED=true This merge represents a significant enhancement to TradingAgents, successfully integrating comprehensive Chinese market support while maintaining system stability and backward compatibility. The integration creates a robust, scalable foundation for global financial market analysis. ### 🏆 Key Achievements + - **🇨🇳 Complete Chinese A-share market support** with Tushare API - **🤖 DashScope LLM integration** with Qwen model family - **🗄️ Enterprise-grade database caching** with MongoDB and Redis @@ -386,7 +431,9 @@ This merge represents a significant enhancement to TradingAgents, successfully i - **📈 Scalable architecture** ready for production deployment ### 🚀 System Status + The system is now **production-ready** for both US and Chinese market analysis, with clear documentation and upgrade paths for future enhancements. Users can immediately benefit from Chinese market support while having the option to enable advanced database features for improved performance and analytics. ### 🔮 Future Roadmap + The foundation is now in place for advanced features like real-time sentiment analysis, advanced Chinese market indicators, and comprehensive performance monitoring. The modular architecture ensures these enhancements can be added incrementally without disrupting existing functionality. diff --git a/MERGE_SUMMARY.md b/MERGE_SUMMARY.md index 5338f622..e3867f1a 100644 --- a/MERGE_SUMMARY.md +++ b/MERGE_SUMMARY.md @@ -5,7 +5,8 @@ 本文档详细记录了从TradingAgentsCN项目向主TradingAgents仓库的全面功能合并。此次合并包括百炼(DashScope)大模型集成、Tushare API的A股数据支持、高级数据库缓存系统,以及增强的CLI市场选择功能。 **合并详情**: -- **合并时间**: 2025年1月 + +- **合并时间**: 2025年7月 - **合并分支**: `full-merge-chinese-features` - **源项目**: TradingAgentsCN目录 - **目标项目**: 主TradingAgents仓库 @@ -28,6 +29,7 @@ **状态**: ✅ **完成并测试通过** **集成内容**: + - CLI中完整的百炼LLM提供商支持 - 支持通义千问模型系列: qwen-turbo, qwen-plus, qwen-max, qwen-max-longcontext - 百炼embedding服务用于记忆系统 @@ -35,6 +37,7 @@ - 全面的错误处理和API密钥验证 **核心文件**: + ``` cli/utils.py - 百炼LLM提供商选项 tradingagents/graph/trading_graph.py - 百炼LLM初始化 @@ -43,6 +46,7 @@ tradingagents/default_config.py - 配置示例 ``` **所需配置**: + ```env DASHSCOPE_API_KEY=your_dashscope_api_key_here ``` @@ -56,6 +60,7 @@ DASHSCOPE_API_KEY=your_dashscope_api_key_here **状态**: ✅ **完成并测试通过** **集成内容**: + - Tushare API集成,获取专业A股数据 - 支持所有主要中国证券交易所: - 上海证券交易所: 60xxxx (如 600036) @@ -67,6 +72,7 @@ DASHSCOPE_API_KEY=your_dashscope_api_key_here - 统一股票数据服务,带自动回退机制 **核心文件**: + ``` tradingagents/dataflows/tushare_utils.py - Tushare数据提供器 tradingagents/dataflows/optimized_china_data.py - 优化A股数据 @@ -75,12 +81,14 @@ tradingagents/dataflows/stock_data_service.py - 统一数据服务 ``` **新增依赖**: + ``` tushare>=1.2.0 beautifulsoup4>=4.9.0 ``` **数据流架构**: + ``` MongoDB数据库 → Tushare API → 文件缓存 → 错误处理 ``` @@ -92,6 +100,7 @@ MongoDB数据库 → Tushare API → 文件缓存 → 错误处理 **状态**: ✅ **完成并测试通过** **集成内容**: + - MongoDB集成,用于持久化数据存储和分析 - Redis集成,用于高性能缓存 - 数据库缓存管理器,带智能路由 @@ -100,6 +109,7 @@ MongoDB数据库 → Tushare API → 文件缓存 → 错误处理 - 集成缓存管理器,带自适应性能优化 **核心文件**: + ``` tradingagents/config/database_config.py - 数据库配置 tradingagents/config/database_manager.py - 连接管理 @@ -111,12 +121,14 @@ tradingagents/dataflows/adaptive_cache.py - 自适应缓存系统 ``` **新增依赖**: + ``` pymongo>=4.0.0 redis>=4.0.0 ``` **配置(可选)**: + ```env # MongoDB MONGODB_ENABLED=false @@ -143,6 +155,7 @@ REDIS_DB=0 **状态**: ✅ **完成并测试通过** **集成内容**: + - 交互式市场选择界面 - 市场特定的股票代码格式验证和示例 - 基于市场选择的自动数据源路由 @@ -150,6 +163,7 @@ REDIS_DB=0 - 全面的格式验证和错误消息 **核心文件**: + ``` cli/utils.py - 添加select_market()和增强get_ticker() cli/main.py - 更新工作流程,包含市场选择步骤 @@ -158,11 +172,12 @@ cli/main.py - 更新工作流程,包含市场选择步骤 **支持的市场**: 1. **美股市场** + - 格式: 1-5位字母代码 (如 AAPL, SPY, TSLA) - 数据源: Yahoo Finance - 验证模式: `^[A-Z]{1,5}$` - 2. **中国A股市场** + - 格式: 6位数字代码 (如 000001, 600036) - 数据源: Tushare API - 验证模式: `^\d{6}$` @@ -176,17 +191,20 @@ cli/main.py - 更新工作流程,包含市场选择步骤 **状态**: ✅ **完成并测试通过** **集成内容**: + - IntegratedCacheManager作为CLI中的默认缓存系统 - 数据库和文件缓存之间的自动选择 - 高可用性的智能回退机制 - 自适应缓存策略的性能优化 **核心变更**: + ``` tradingagents/dataflows/cache_manager.py - 增强get_cache()函数 ``` **缓存优先级逻辑**: + ``` 1. 数据库缓存 (MongoDB/Redis) - 如果启用且可用 2. 文件缓存 - 可靠的回退方案 @@ -198,18 +216,21 @@ tradingagents/dataflows/cache_manager.py - 增强get_cache()函数 ## 🔧 技术改进 ### 📦 依赖管理 + - ✅ 所有新依赖正确添加到`requirements.txt` - ✅ 可选依赖带有优雅回退机制 - ✅ 不破坏现有功能 - ✅ 保持向后兼容性 ### 🛡️ 错误处理和可靠性 + - ✅ 所有新功能的全面错误处理 - ✅ 外部服务不可用时的优雅降级 - ✅ 详细的错误消息和用户指导 - ✅ API调用的自动重试逻辑 ### 🔄 向后兼容性 + - ✅ 所有现有功能保持正常工作 - ✅ 新功能可选且可配置 - ✅ 现有用户的默认行为不变 @@ -220,17 +241,21 @@ tradingagents/dataflows/cache_manager.py - 增强get_cache()函数 ## ⚠️ 已知问题和限制 ### 1. 可选依赖 + **问题**: 部分依赖默认未安装 **影响**: 手动安装前功能受限 **解决方案**: + ```bash -pip install pytdx beautifulsoup4 +pip install tushare beautifulsoup4 ``` ### 2. 数据库服务 + **问题**: MongoDB和Redis在`.env`中默认禁用 **影响**: 数据库缓存功能默认不激活 **解决方案**: 启用数据库并启动服务: + ```bash # 启动服务 docker run -d -p 27017:27017 --name mongodb mongo @@ -242,6 +267,7 @@ REDIS_ENABLED=true ``` ### 3. API频率限制 + **问题**: Tushare API可能有频率限制(免费账户) **影响**: A股数据获取可能出现延迟 **缓解措施**: 已实现智能缓存和重试逻辑 @@ -251,15 +277,18 @@ REDIS_ENABLED=true ## 🚧 未完成功能/未来工作 ### 1. 数据源选择UI + **状态**: ❌ **未实现** **描述**: 用户手动选择缓存和Tushare API的界面 **当前状态**: 仅有自动回退逻辑 **未来增强**: 添加CLI数据源偏好选项 ### 2. 高级A股分析功能 + **状态**: ⚠️ **部分实现** **已完成**: 基础数据获取和缓存 **缺失功能**: + - 实时市场情绪分析 - A股特定技术指标 - 中国财经新闻情绪集成 @@ -267,8 +296,10 @@ REDIS_ENABLED=true - A股交易时间和交易日历 ### 3. 性能监控和分析 + **状态**: ❌ **未实现** **缺失功能**: + - 数据库性能指标仪表板 - 缓存命中/未命中统计 - API响应时间监控 @@ -276,17 +307,21 @@ REDIS_ENABLED=true - API调用成本跟踪 ### 4. 配置管理 + **状态**: ⚠️ **基础实现** **当前**: 基础配置验证 **缺失功能**: + - 全面配置验证向导 - 新用户交互式设置指南 - 配置健康检查和诊断 - 自动配置迁移工具 ### 5. 高级Tushare功能 + **状态**: ❌ **未实现** **缺失功能**: + - 实时tick数据流 - Level-2市场数据集成 - 期权和期货数据支持 @@ -298,6 +333,7 @@ REDIS_ENABLED=true ## 📊 测试和验证状态 ### ✅ 已完成并验证 + - **百炼LLM集成**: ✅ 所有模型正常工作 - **Tushare API功能**: ✅ 数据获取正常工作 - **数据库连接**: ✅ MongoDB和Redis连接 @@ -308,6 +344,7 @@ REDIS_ENABLED=true - **错误处理**: ✅ 优雅降级已验证 ### ⚠️ 需要进一步测试 + - 负载下的端到端A股分析工作流 - 大数据集的数据库性能 - 频率限制下的Tushare API行为 @@ -319,17 +356,19 @@ REDIS_ENABLED=true ## 🎯 部署建议 ### 立即使用(最小设置) + 1. **安装可选依赖**: + ```bash pip install pytdx beautifulsoup4 ``` - 2. **配置百炼API**: + ```env DASHSCOPE_API_KEY=your_actual_api_key ``` - 3. **测试A股功能**: + ```bash python -m cli.main # 选择: China A-Share @@ -337,24 +376,27 @@ REDIS_ENABLED=true ``` ### 生产部署(完整功能) + 1. **设置数据库服务**: + ```bash docker run -d -p 27017:27017 --name mongodb mongo docker run -d -p 6379:6379 --name redis redis ``` - 2. **启用数据库缓存**: + ```env MONGODB_ENABLED=true REDIS_ENABLED=true ``` - 3. **配置监控和日志**: + - 实现应用程序日志 - 设置数据库监控 - 配置API使用跟踪 ### 开发环境 + 1. **设置持久化卷的开发数据库** 2. **配置开发API密钥**,带频率限制 3. **启用调试日志**用于故障排除 @@ -365,6 +407,7 @@ REDIS_ENABLED=true ## 📈 影响评估 ### ✅ 积极影响 + - **🇨🇳 中国市场接入**: 完整的A股市场分析能力 - **🚀 性能提升**: 数据库缓存显著提高数据访问速度 - **🔄 可靠性**: 多LLM提供商增加系统可靠性 @@ -373,6 +416,7 @@ REDIS_ENABLED=true - **💾 可扩展性**: 数据库集成支持企业部署 ### ⚠️ 考虑因素 + - **🔧 复杂性**: 多数据源增加系统复杂性 - **📦 依赖**: 额外的外部依赖和服务 - **🛠️ 维护**: 更多组件需要监控和维护 @@ -386,6 +430,7 @@ REDIS_ENABLED=true 此次合并代表了TradingAgents的重大增强,成功集成了全面的中国市场支持,同时保持系统稳定性和向后兼容性。集成创建了一个强大、可扩展的全球金融市场分析基础。 ### 🏆 关键成就 + - **🇨🇳 完整的中国A股市场支持**,集成Tushare API - **🤖 百炼LLM集成**,支持通义千问模型系列 - **🗄️ 企业级数据库缓存**,支持MongoDB和Redis @@ -394,7 +439,9 @@ REDIS_ENABLED=true - **📈 可扩展架构**,为生产部署做好准备 ### 🚀 系统状态 + 系统现在**生产就绪**,支持美股和中国市场分析,具有清晰的文档和未来增强的升级路径。用户可以立即受益于中国市场支持,同时可以选择启用高级数据库功能以获得更好的性能和分析能力。 ### 🔮 未来路线图 + 现在已为高级功能奠定基础,如实时情绪分析、高级中国市场指标和全面性能监控。模块化架构确保这些增强可以逐步添加,而不会破坏现有功能。 diff --git a/requirements.txt b/requirements.txt index b3aff507..e3a22c3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,5 +29,4 @@ langchain-google-genai dashscope streamlit plotly -tushare # Tushare API for Chinese stock data (replaces pytdx) pymongo # MongoDB database support for token usage storage diff --git a/tradingagents/dataflows/tdx_utils.py b/tradingagents/dataflows/tdx_utils.py deleted file mode 100644 index 3cd317aa..00000000 --- a/tradingagents/dataflows/tdx_utils.py +++ /dev/null @@ -1,856 +0,0 @@ -#!/usr/bin/env python3 -""" -通达信API数据获取工具 -支持A股、港股实时数据和历史数据 -""" - -import pandas as pd -import numpy as np -from datetime import datetime, timedelta -from typing import List, Dict, Optional, Tuple -import warnings -warnings.filterwarnings('ignore') - -# 导入数据库管理器 -try: - from tradingagents.config.database_manager import get_database_manager - DB_MANAGER_AVAILABLE = True -except ImportError: - DB_MANAGER_AVAILABLE = False - print("⚠️ 数据库缓存管理器不可用,尝试文件缓存") - -# 导入MongoDB股票信息查询 -try: - import os - from pymongo import MongoClient - MONGODB_AVAILABLE = True -except ImportError: - MONGODB_AVAILABLE = False - print("⚠️ pymongo未安装,无法从MongoDB获取股票名称") - -try: - from .cache_manager import get_cache - FILE_CACHE_AVAILABLE = True -except ImportError: - FILE_CACHE_AVAILABLE = False - print("⚠️ 文件缓存管理器不可用,将直接从API获取数据") - -try: - # 通达信Python接口 - import pytdx - from pytdx.hq import TdxHq_API - from pytdx.exhq import TdxExHq_API - TDX_AVAILABLE = True -except ImportError: - TDX_AVAILABLE = False - print("⚠️ pytdx库未安装,无法使用通达信API") - print("💡 安装命令: pip install pytdx") - - -class TongDaXinDataProvider: - """通达信数据提供器""" - - def __init__(self): - print(f"🔍 [DEBUG] 初始化通达信数据提供器...") - self.api = None - self.exapi = None # 扩展行情API - self.connected = False - - print(f"🔍 [DEBUG] 检查pytdx库可用性: {TDX_AVAILABLE}") - if not TDX_AVAILABLE: - error_msg = "pytdx库未安装,请运行: pip install pytdx" - print(f"❌ [DEBUG] {error_msg}") - raise ImportError(error_msg) - print(f"✅ [DEBUG] pytdx库检查通过") - - def connect(self): - """连接通达信服务器""" - print(f"🔍 [DEBUG] 开始连接通达信服务器...") - try: - # 尝试从配置文件加载可用服务器 - print(f"🔍 [DEBUG] 加载服务器配置...") - working_servers = self._load_working_servers() - - # 如果没有配置文件,使用默认服务器列表 - if not working_servers: - print(f"🔍 [DEBUG] 未找到配置文件,使用默认服务器列表") - working_servers = [ - {'ip': '115.238.56.198', 'port': 7709}, - {'ip': '115.238.90.165', 'port': 7709}, - {'ip': '180.153.18.170', 'port': 7709}, - {'ip': '119.147.212.81', 'port': 7709}, # 备用 - ] - else: - print(f"🔍 [DEBUG] 从配置文件加载了 {len(working_servers)} 个服务器") - - # 尝试连接可用服务器 - print(f"🔍 [DEBUG] 创建通达信API实例...") - self.api = TdxHq_API() - print(f"🔍 [DEBUG] 开始尝试连接服务器...") - - for i, server in enumerate(working_servers): - try: - print(f"🔍 [DEBUG] 尝试连接服务器 {i+1}/{len(working_servers)}: {server['ip']}:{server['port']}") - result = self.api.connect(server['ip'], server['port']) - print(f"🔍 [DEBUG] 连接结果: {result}") - if result: - print(f"✅ 通达信API连接成功: {server['ip']}:{server['port']}") - self.connected = True - return True - except Exception as e: - print(f"⚠️ 服务器 {server['ip']}:{server['port']} 连接失败: {e}") - continue - - print("❌ 所有通达信服务器连接失败") - self.connected = False - return False - - except Exception as e: - print(f"❌ 通达信API连接失败: {e}") - self.connected = False - return False - - def _load_working_servers(self): - """加载可用服务器配置""" - try: - import json - import os - - config_file = 'tdx_servers_config.json' - if os.path.exists(config_file): - with open(config_file, 'r', encoding='utf-8') as f: - config = json.load(f) - return config.get('working_servers', []) - except Exception: - pass - return [] - - def disconnect(self): - """断开连接""" - try: - if self.api: - self.api.disconnect() - if self.exapi: - self.exapi.disconnect() - self.connected = False - print("✅ 通达信API连接已断开") - except: - pass - - def is_connected(self): - """检查连接状态""" - if not self.connected or not self.api: - return False - - # 尝试简单的API调用来验证连接是否有效 - try: - # 获取市场信息作为连接测试 - result = self.api.get_security_count(0) # 获取深圳市场股票数量 - return result is not None and result > 0 - except Exception as e: - print(f"🔍 [DEBUG] 连接测试失败: {e}") - self.connected = False - return False - - def _get_stock_name(self, stock_code: str) -> str: - """ - 获取股票名称 - 优先级:缓存 -> MongoDB -> 常用股票映射 -> API获取(仅深圳市场) -> 默认格式 - Args: - stock_code: 股票代码 - Returns: - str: 股票名称 - """ - global _stock_name_cache - - # 首先检查缓存 - if stock_code in _stock_name_cache: - return _stock_name_cache[stock_code] - - # 优先从MongoDB获取 - mongodb_name = _get_stock_name_from_mongodb(stock_code) - if mongodb_name: - _stock_name_cache[stock_code] = mongodb_name - return mongodb_name - - # 检查常用股票映射表 - if stock_code in _common_stock_names: - name = _common_stock_names[stock_code] - _stock_name_cache[stock_code] = name - return name - - # 如果API不可用,直接返回默认格式 - if not self.connected: - if not self.connect(): - default_name = f'股票{stock_code}' - _stock_name_cache[stock_code] = default_name - return default_name - - try: - # 仅对深圳市场尝试从API获取(上海市场的get_security_list不可用) - market = self._get_market_code(stock_code) - if market == 0: # 深圳市场 - try: - for start_pos in range(0, 2000, 1000): # 分批获取 - stock_list = self.api.get_security_list(market, start_pos) - if stock_list: - for stock_info in stock_list: - if stock_info.get('code') == stock_code: - stock_name = stock_info.get('name', '').strip() - if stock_name: - _stock_name_cache[stock_code] = stock_name - return stock_name - except Exception as e: - print(f"⚠️ 获取深圳股票列表失败: {e}") - - # 如果都失败了,返回默认格式并缓存 - default_name = f'股票{stock_code}' - _stock_name_cache[stock_code] = default_name - return default_name - - except Exception as e: - print(f"⚠️ 获取股票名称失败: {e}") - default_name = f'股票{stock_code}' - _stock_name_cache[stock_code] = default_name - return default_name - - def get_real_time_data(self, stock_code: str) -> Dict: - """ - 获取股票实时数据 - Args: - stock_code: 股票代码 - Returns: - Dict: 实时数据 - """ - if not self.connected: - if not self.connect(): - return {} - - try: - market = self._get_market_code(stock_code) - - # 获取实时数据 - data = self.api.get_security_quotes([(market, stock_code)]) - - if not data: - return {} - - quote = data[0] - - # 安全获取字段,避免KeyError - def safe_get(key, default=0): - return quote.get(key, default) - - return { - 'code': stock_code, - 'name': self._get_stock_name(stock_code), # 使用独立的股票名称获取方法 - 'price': safe_get('price'), - 'last_close': safe_get('last_close'), - 'open': safe_get('open'), - 'high': safe_get('high'), - 'low': safe_get('low'), - 'volume': safe_get('vol'), - 'amount': safe_get('amount'), - 'change': safe_get('price') - safe_get('last_close'), - 'change_percent': ((safe_get('price') - safe_get('last_close')) / safe_get('last_close') * 100) if safe_get('last_close') > 0 else 0, - 'bid_prices': [safe_get(f'bid{i}') for i in range(1, 6)], - 'bid_volumes': [safe_get(f'bid_vol{i}') for i in range(1, 6)], - 'ask_prices': [safe_get(f'ask{i}') for i in range(1, 6)], - 'ask_volumes': [safe_get(f'ask_vol{i}') for i in range(1, 6)], - 'update_time': datetime.now().strftime('%Y-%m-%d %H:%M:%S') - } - - except Exception as e: - print(f"获取实时数据失败: {e}") - return {} - - def get_stock_history_data(self, stock_code: str, start_date: str, end_date: str, period: str = 'D') -> pd.DataFrame: - """ - 获取股票历史数据 - Args: - stock_code: 股票代码 - start_date: 开始日期 'YYYY-MM-DD' - end_date: 结束日期 'YYYY-MM-DD' - period: 周期 'D'=日线, 'W'=周线, 'M'=月线 - Returns: - DataFrame: 历史数据 - """ - if not self.connected: - if not self.connect(): - return pd.DataFrame() - - try: - market = self._get_market_code(stock_code) - - # 计算需要获取的数据量 - start_dt = datetime.strptime(start_date, '%Y-%m-%d') - end_dt = datetime.strptime(end_date, '%Y-%m-%d') - days_diff = (end_dt - start_dt).days - - # 根据周期调整数据量 - if period == 'D': - count = min(days_diff + 10, 800) # 日线最多800条 - elif period == 'W': - count = min(days_diff // 7 + 10, 800) - elif period == 'M': - count = min(days_diff // 30 + 10, 800) - else: - count = 800 - - # 获取K线数据 - category_map = {'D': 9, 'W': 5, 'M': 6} - category = category_map.get(period, 9) - - data = self.api.get_security_bars(category, market, stock_code, 0, count) - - if not data: - return pd.DataFrame() - - # 转换为DataFrame - df = pd.DataFrame(data) - - # 处理数据格式 - df['datetime'] = pd.to_datetime(df['datetime']) - df = df.set_index('datetime') - df = df.sort_index() - - # 筛选日期范围 - df = df[start_date:end_date] - - # 重命名列以匹配Yahoo Finance格式 - df = df.rename(columns={ - 'open': 'Open', - 'high': 'High', - 'low': 'Low', - 'close': 'Close', - 'vol': 'Volume', - 'amount': 'Amount' - }) - - # 添加股票代码信息 - df['Symbol'] = stock_code - - return df - - except Exception as e: - print(f"获取历史数据失败: {e}") - return pd.DataFrame() - - def get_stock_technical_indicators(self, stock_code: str, period: int = 20) -> Dict: - """ - 计算技术指标 - Args: - stock_code: 股票代码 - period: 计算周期 - Returns: - Dict: 技术指标数据 - """ - try: - # 获取最近的历史数据 - end_date = datetime.now().strftime('%Y-%m-%d') - start_date = (datetime.now() - timedelta(days=period*2)).strftime('%Y-%m-%d') - - df = self.get_stock_history_data(stock_code, start_date, end_date) - - if df.empty: - return {} - - # 计算技术指标 - indicators = {} - - # 移动平均线 - indicators['MA5'] = df['Close'].rolling(5).mean().iloc[-1] if len(df) >= 5 else None - indicators['MA10'] = df['Close'].rolling(10).mean().iloc[-1] if len(df) >= 10 else None - indicators['MA20'] = df['Close'].rolling(20).mean().iloc[-1] if len(df) >= 20 else None - - # RSI - if len(df) >= 14: - delta = df['Close'].diff() - gain = (delta.where(delta > 0, 0)).rolling(14).mean() - loss = (-delta.where(delta < 0, 0)).rolling(14).mean() - rs = gain / loss - indicators['RSI'] = (100 - (100 / (1 + rs))).iloc[-1] - - # MACD - if len(df) >= 26: - exp1 = df['Close'].ewm(span=12).mean() - exp2 = df['Close'].ewm(span=26).mean() - macd = exp1 - exp2 - signal = macd.ewm(span=9).mean() - indicators['MACD'] = macd.iloc[-1] - indicators['MACD_Signal'] = signal.iloc[-1] - indicators['MACD_Histogram'] = (macd - signal).iloc[-1] - - # 布林带 - if len(df) >= 20: - sma = df['Close'].rolling(20).mean() - std = df['Close'].rolling(20).std() - indicators['BB_Upper'] = (sma + 2 * std).iloc[-1] - indicators['BB_Middle'] = sma.iloc[-1] - indicators['BB_Lower'] = (sma - 2 * std).iloc[-1] - - return indicators - - except Exception as e: - print(f"计算技术指标失败: {e}") - return {} - - def search_stocks(self, keyword: str) -> List[Dict]: - """ - 搜索股票 - Args: - keyword: 搜索关键词(股票代码或名称) - Returns: - List[Dict]: 搜索结果 - """ - if not self.connected: - if not self.connect(): - return [] - - try: - # 通达信没有直接的搜索API,这里提供一个简化的实现 - # 实际使用中可以维护一个股票代码表 - - # 常见股票代码映射 - stock_mapping = { - '平安银行': '000001', - '万科A': '000002', - '中国平安': '601318', - '贵州茅台': '600519', - '招商银行': '600036', - '五粮液': '000858', - '格力电器': '000651', - '美的集团': '000333', - '中国石化': '600028', - '工商银行': '601398' - } - - results = [] - - # 按关键词搜索 - for name, code in stock_mapping.items(): - if keyword.lower() in name.lower() or keyword in code: - # 获取实时数据 - realtime_data = self.get_real_time_data(code) - if realtime_data: - results.append({ - 'code': code, - 'name': name, - 'price': realtime_data.get('price', 0), - 'change_percent': realtime_data.get('change_percent', 0) - }) - - return results - - except Exception as e: - print(f"搜索股票失败: {e}") - return [] - - def _get_market_code(self, stock_code: str) -> int: - """ - 根据股票代码判断市场 - Args: - stock_code: 股票代码 - Returns: - int: 市场代码 (0=深圳, 1=上海) - """ - if stock_code.startswith(('000', '002', '003', '300')): - return 0 # 深圳 - elif stock_code.startswith(('600', '601', '603', '605', '688')): - return 1 # 上海 - else: - return 0 # 默认深圳 - - def get_market_overview(self) -> Dict: - """获取市场概览""" - if not self.connected: - if not self.connect(): - return {} - - try: - # 获取主要指数数据 - indices = { - '上证指数': ('1', '000001'), - '深证成指': ('0', '399001'), - '创业板指': ('0', '399006'), - '科创50': ('1', '000688') - } - - market_data = {} - - for name, (market, code) in indices.items(): - try: - data = self.api.get_security_quotes([(int(market), code)]) - if data: - quote = data[0] - market_data[name] = { - 'price': quote['price'], - 'change': quote['price'] - quote['last_close'], - 'change_percent': ((quote['price'] - quote['last_close']) / quote['last_close'] * 100) if quote['last_close'] > 0 else 0, - 'volume': quote['vol'] - } - except: - continue - - return market_data - - except Exception as e: - print(f"获取市场概览失败: {e}") - return {} - - -# 全局实例和缓存 -_tdx_provider = None -_stock_name_cache = {} # 股票名称缓存,避免重复API调用 -_mongodb_client = None -_mongodb_db = None - -def _get_mongodb_connection(): - """获取MongoDB连接""" - global _mongodb_client, _mongodb_db - - if not MONGODB_AVAILABLE: - return None, None - - if _mongodb_client is None or _mongodb_db is None: - try: - # 从环境变量获取MongoDB配置 - config = { - 'host': os.getenv('MONGODB_HOST', 'localhost'), - 'port': int(os.getenv('MONGODB_PORT', 27018)), - 'username': os.getenv('MONGODB_USERNAME'), - 'password': os.getenv('MONGODB_PASSWORD'), - 'database': os.getenv('MONGODB_DATABASE', 'tradingagents'), - 'auth_source': os.getenv('MONGODB_AUTH_SOURCE', 'admin') - } - - # 构建连接字符串 - if config.get('username') and config.get('password'): - connection_string = f"mongodb://{config['username']}:{config['password']}@{config['host']}:{config['port']}/{config['auth_source']}" - else: - connection_string = f"mongodb://{config['host']}:{config['port']}/" - - # 创建客户端 - _mongodb_client = MongoClient( - connection_string, - serverSelectionTimeoutMS=3000 # 3秒超时 - ) - - # 测试连接 - _mongodb_client.admin.command('ping') - - # 选择数据库 - _mongodb_db = _mongodb_client[config['database']] - - except Exception as e: - print(f"⚠️ MongoDB连接失败: {e}") - _mongodb_client = None - _mongodb_db = None - - return _mongodb_client, _mongodb_db - -def _get_stock_name_from_mongodb(stock_code: str) -> Optional[str]: - """从MongoDB获取股票名称""" - try: - client, db = _get_mongodb_connection() - if db is None: - return None - - collection = db['stock_basic_info'] - stock_info = collection.find_one({'code': stock_code}) - - if stock_info and 'name' in stock_info: - return stock_info['name'].strip() - - return None - - except Exception as e: - print(f"⚠️ 从MongoDB获取股票名称失败: {e}") - return None - -# 精简的常用股票名称映射(仅包含最常见的股票) -_common_stock_names = { - # 深圳主板 - '000001': '平安银行', - '000002': '万科A', - '000858': '五粮液', - '000895': '双汇发展', - - # 深圳中小板 - '002594': '比亚迪', - '002415': '海康威视', - '002304': '洋河股份', - - # 深圳创业板 - '300059': '东方财富', - '300750': '宁德时代', - '300015': '爱尔眼科', - - # 上海主板 - '600519': '贵州茅台', - '600036': '招商银行', - '601398': '工商银行', - '601127': '小康股份', - '600000': '浦发银行', - '601318': '中国平安', - '600276': '恒瑞医药', - '600887': '伊利股份', - - # 科创板 - '688981': '中芯国际', - '688599': '天合光能', -} - -def get_tdx_provider() -> TongDaXinDataProvider: - """获取通达信数据提供器实例""" - global _tdx_provider - if _tdx_provider is None: - print(f"🔍 [DEBUG] 创建新的通达信数据提供器实例...") - _tdx_provider = TongDaXinDataProvider() - print(f"🔍 [DEBUG] 通达信数据提供器实例创建完成") - else: - print(f"🔍 [DEBUG] 使用现有的通达信数据提供器实例") - # 检查连接状态,如果连接断开则重新创建 - if not _tdx_provider.is_connected(): - print(f"🔍 [DEBUG] 检测到连接断开,重新创建通达信数据提供器...") - _tdx_provider = TongDaXinDataProvider() - print(f"🔍 [DEBUG] 通达信数据提供器重新创建完成") - return _tdx_provider - - -def get_china_stock_data(stock_code: str, start_date: str, end_date: str) -> str: - """ - 获取中国股票数据的主要接口函数(支持缓存) - Args: - stock_code: 股票代码 (如 '000001') - start_date: 开始日期 'YYYY-MM-DD' - end_date: 结束日期 'YYYY-MM-DD' - Returns: - str: 格式化的股票数据 - """ - print(f"📊 正在获取中国股票数据: {stock_code} ({start_date} 到 {end_date})") - - # 优先尝试从数据库缓存加载数据(使用统一的database_manager) - try: - from tradingagents.config.database_manager import get_database_manager - db_manager = get_database_manager() - if db_manager.is_mongodb_available(): - # 直接使用MongoDB客户端查询缓存数据 - mongodb_client = db_manager.get_mongodb_client() - if mongodb_client: - db = mongodb_client[db_manager.mongodb_config["database"]] - collection = db.stock_data - - # 查询最近的缓存数据 - from datetime import datetime, timedelta - cutoff_time = datetime.utcnow() - timedelta(hours=6) - - cached_doc = collection.find_one({ - "symbol": stock_code, - "market_type": "china", - "created_at": {"$gte": cutoff_time} - }, sort=[("created_at", -1)]) - - if cached_doc and 'data' in cached_doc: - print(f"🗄️ 从MongoDB缓存加载数据: {stock_code}") - return cached_doc['data'] - except Exception as e: - print(f"⚠️ 从MongoDB加载缓存失败: {e}") - - # 如果数据库缓存不可用,尝试文件缓存 - if FILE_CACHE_AVAILABLE: - cache = get_cache() - cache_key = cache.find_cached_stock_data( - symbol=stock_code, - start_date=start_date, - end_date=end_date, - data_source="tdx", - max_age_hours=6 # 6小时内的缓存有效 - ) - - if cache_key: - cached_data = cache.load_stock_data(cache_key) - if cached_data: - print(f"💾 从文件缓存加载数据: {stock_code} -> {cache_key}") - return cached_data - - print(f"🌐 从通达信API获取数据: {stock_code}") - - try: - provider = get_tdx_provider() - - # 获取历史数据 - df = provider.get_stock_history_data(stock_code, start_date, end_date) - - if df.empty: - error_msg = f"❌ 未能获取股票 {stock_code} 的历史数据" - print(error_msg) - return error_msg - - # 获取实时数据 - realtime_data = provider.get_real_time_data(stock_code) - - # 获取技术指标 - indicators = provider.get_stock_technical_indicators(stock_code) - - # 格式化输出 - result = f""" -# {stock_code} 股票数据分析 - -## 📊 实时行情 -- 股票名称: {realtime_data.get('name', 'N/A')} -- 当前价格: ¥{realtime_data.get('price', 0):.2f} -- 涨跌幅: {realtime_data.get('change_percent', 0):.2f}% -- 成交量: {realtime_data.get('volume', 0):,}手 -- 更新时间: {realtime_data.get('update_time', 'N/A')} - -## 📈 历史数据概览 -- 数据期间: {start_date} 至 {end_date} -- 数据条数: {len(df)}条 -- 期间最高: ¥{df['High'].max():.2f} -- 期间最低: ¥{df['Low'].min():.2f} -- 期间涨幅: {((df['Close'].iloc[-1] - df['Close'].iloc[0]) / df['Close'].iloc[0] * 100):.2f}% - -## 🔍 技术指标 -- MA5: ¥{indicators.get('MA5', 0):.2f} -- MA10: ¥{indicators.get('MA10', 0):.2f} -- MA20: ¥{indicators.get('MA20', 0):.2f} -- RSI: {indicators.get('RSI', 0):.2f} -- MACD: {indicators.get('MACD', 0):.4f} - -## 📋 最近5日数据 -{df.tail().to_string()} - -数据来源: 通达信API (实时数据) -""" - - # 优先保存到数据库缓存(使用统一的database_manager) - try: - from tradingagents.config.database_manager import get_database_manager - db_manager = get_database_manager() - if db_manager.is_mongodb_available(): - # 直接使用MongoDB客户端保存数据 - mongodb_client = db_manager.get_mongodb_client() - if mongodb_client: - db = mongodb_client[db_manager.mongodb_config["database"]] - collection = db.stock_data - - doc = { - "symbol": stock_code, - "market_type": "china", - "data": result, - "metadata": { - 'start_date': start_date, - 'end_date': end_date, - 'data_source': 'tdx', - 'realtime_data': realtime_data, - 'indicators': indicators, - 'history_count': len(df) - }, - "created_at": datetime.utcnow(), - "updated_at": datetime.utcnow() - } - - collection.replace_one( - {"symbol": stock_code, "market_type": "china"}, - doc, - upsert=True - ) - print(f"💾 数据已保存到MongoDB: {stock_code}") - except Exception as e: - print(f"⚠️ 保存到MongoDB失败: {e}") - - # 同时保存到文件缓存作为备份 - if FILE_CACHE_AVAILABLE: - cache = get_cache() - cache.save_stock_data( - symbol=stock_code, - data=result, - start_date=start_date, - end_date=end_date, - data_source="tdx" - ) - - return result - - except Exception as e: - import traceback - error_details = traceback.format_exc() - print(f"❌ [DEBUG] 通达信API调用失败:") - print(f"❌ [DEBUG] 错误类型: {type(e).__name__}") - print(f"❌ [DEBUG] 错误信息: {str(e)}") - print(f"❌ [DEBUG] 详细堆栈:") - print(error_details) - - return f""" -❌ 中国股票数据获取失败 - {stock_code} -错误类型: {type(e).__name__} -错误信息: {str(e)} - -🔍 调试信息: -{error_details} - -💡 解决建议: -1. 检查pytdx库是否已安装: pip install pytdx -2. 确认股票代码格式正确 (如: 000001, 600519) -3. 检查网络连接是否正常 -4. 尝试重新连接通达信服务器 - -注: 通达信API需要网络连接到通达信服务器 -""" - - -def get_china_market_overview() -> str: - """获取中国股市概览""" - try: - provider = get_tdx_provider() - market_data = provider.get_market_overview() - - if not market_data: - return "无法获取市场概览数据" - - result = "# 中国股市概览\n\n" - - for name, data in market_data.items(): - change_symbol = "📈" if data['change'] >= 0 else "📉" - result += f"## {change_symbol} {name}\n" - result += f"- 当前点位: {data['price']:.2f}\n" - result += f"- 涨跌点数: {data['change']:+.2f}\n" - result += f"- 涨跌幅: {data['change_percent']:+.2f}%\n" - result += f"- 成交量: {data['volume']:,}\n\n" - - result += f"更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n" - result += "数据来源: 通达信API\n" - - return result - - except Exception as e: - return f"获取市场概览失败: {str(e)}" - -# 在文件末尾添加以下函数 - -def get_china_stock_data_enhanced(stock_code: str, start_date: str, end_date: str) -> str: - """ - 增强版中国股票数据获取函数(完整降级机制) - 这是get_china_stock_data的增强版本 - - Args: - stock_code: 股票代码 (如 '000001') - start_date: 开始日期 'YYYY-MM-DD' - end_date: 结束日期 'YYYY-MM-DD' - Returns: - str: 格式化的股票数据 - """ - try: - from .stock_data_service import get_stock_data_service - service = get_stock_data_service() - return service.get_stock_data_with_fallback(stock_code, start_date, end_date) - except ImportError: - # 如果新服务不可用,降级到原有函数 - print("⚠️ 增强服务不可用,使用原有函数") - return get_china_stock_data(stock_code, start_date, end_date) - except Exception as e: - print(f"⚠️ 增强服务出错,降级到原有函数: {e}") - return get_china_stock_data(stock_code, start_date, end_date) - -# ... existing code ...