From f35feafe9a03c605c729edc11c5f0033019177db Mon Sep 17 00:00:00 2001 From: treasuraid Date: Sun, 8 Feb 2026 18:06:32 +0900 Subject: [PATCH] feat: Add top 10 OpenRouter models to CLI model selection - Added top-ranked models from OpenRouter.ai rankings: - Kimi K2.5 0127 (#1 ranked) - Claude Opus 4.5 / Sonnet 4.5 - Gemini 3 Flash Preview / 2.5 Flash / 2.5 Flash Lite - Deepseek V3.2 - Grok 4.1 Fast / Grok Code Fast 1 - Minimax M2.1 - Refactored SHALLOW_AGENT_OPTIONS and DEEP_AGENT_OPTIONS to module-level constants - Added comprehensive tests for OpenRouter model configuration in tests/test_cli_utils.py - Preserved existing free models (Nemotron, GLM) The model options dictionaries were moved from function scope to module level to enable better testing and reusability. --- cli/utils.py | 232 ++++++++++++++++++++++++---------------- tests/__init__.py | 0 tests/test_cli_utils.py | 143 +++++++++++++++++++++++++ 3 files changed, 285 insertions(+), 90 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_cli_utils.py diff --git a/cli/utils.py b/cli/utils.py index aa097fb5..32d85f5a 100644 --- a/cli/utils.py +++ b/cli/utils.py @@ -10,6 +10,130 @@ ANALYST_ORDER = [ ("Fundamentals Analyst", AnalystType.FUNDAMENTALS), ] +# Define shallow thinking llm engine options with their corresponding model names +SHALLOW_AGENT_OPTIONS = { + "openai": [ + ("GPT-5 Mini - Cost-optimized reasoning", "gpt-5-mini"), + ("GPT-5 Nano - Ultra-fast, high-throughput", "gpt-5-nano"), + ("GPT-5.2 - Latest flagship", "gpt-5.2"), + ("GPT-5.1 - Flexible reasoning", "gpt-5.1"), + ("GPT-4.1 - Smartest non-reasoning, 1M context", "gpt-4.1"), + ], + "anthropic": [ + ("Claude Haiku 4.5 - Fast + extended thinking", "claude-haiku-4-5"), + ("Claude Sonnet 4.5 - Best for agents/coding", "claude-sonnet-4-5"), + ("Claude Sonnet 4 - High-performance", "claude-sonnet-4-20250514"), + ], + "google": [ + ("Gemini 3 Flash - Next-gen fast", "gemini-3-flash-preview"), + ("Gemini 2.5 Flash - Balanced, recommended", "gemini-2.5-flash"), + ("Gemini 3 Pro - Reasoning-first", "gemini-3-pro-preview"), + ("Gemini 2.5 Flash Lite - Fast, low-cost", "gemini-2.5-flash-lite"), + ], + "xai": [ + ( + "Grok 4.1 Fast (Non-Reasoning) - Speed optimized, 2M ctx", + "grok-4-1-fast-non-reasoning", + ), + ( + "Grok 4 Fast (Non-Reasoning) - Speed optimized", + "grok-4-fast-non-reasoning", + ), + ( + "Grok 4.1 Fast (Reasoning) - High-performance, 2M ctx", + "grok-4-1-fast-reasoning", + ), + ("Grok 4 Fast (Reasoning) - High-performance", "grok-4-fast-reasoning"), + ], + "openrouter": [ + ("Kimi K2.5 0127 - Top ranked", "moonshotai/kimi-k2.5-0127"), + ( + "Claude Sonnet 4.5 - Best for agents", + "anthropic/claude-4.5-sonnet-20250929", + ), + ("Claude Opus 4.5 - Premium", "anthropic/claude-4.5-opus-20251124"), + ("Gemini 3 Flash Preview - Fast", "google/gemini-3-flash-preview-20251217"), + ("Gemini 2.5 Flash - Balanced", "google/gemini-2.5-flash"), + ("Gemini 2.5 Flash Lite - Efficient", "google/gemini-2.5-flash-lite"), + ("Deepseek V3.2 - Cost effective", "deepseek/deepseek-v3.2-20251201"), + ("Grok 4.1 Fast - High performance", "x-ai/grok-4.1-fast"), + ("Grok Code Fast 1 - Coding optimized", "x-ai/grok-code-fast-1"), + ("Minimax M2.1 - Rising star", "minimax/minimax-m2.1"), + ( + "NVIDIA Nemotron 3 Nano 30B (free)", + "nvidia/nemotron-3-nano-30b-a3b:free", + ), + ("Z.AI GLM 4.5 Air (free)", "z-ai/glm-4.5-air:free"), + ], + "ollama": [ + ("Qwen3:latest (8B, local)", "qwen3:latest"), + ("GPT-OSS:latest (20B, local)", "gpt-oss:latest"), + ("GLM-4.7-Flash:latest (30B, local)", "glm-4.7-flash:latest"), + ], +} + +# Define deep thinking llm engine options with their corresponding model names +DEEP_AGENT_OPTIONS = { + "openai": [ + ("GPT-5.2 - Latest flagship", "gpt-5.2"), + ("GPT-5.1 - Flexible reasoning", "gpt-5.1"), + ("GPT-5 - Advanced reasoning", "gpt-5"), + ("GPT-4.1 - Smartest non-reasoning, 1M context", "gpt-4.1"), + ("GPT-5 Mini - Cost-optimized reasoning", "gpt-5-mini"), + ("GPT-5 Nano - Ultra-fast, high-throughput", "gpt-5-nano"), + ], + "anthropic": [ + ("Claude Sonnet 4.5 - Best for agents/coding", "claude-sonnet-4-5"), + ("Claude Opus 4.5 - Premium, max intelligence", "claude-opus-4-5"), + ("Claude Opus 4.1 - Most capable model", "claude-opus-4-1-20250805"), + ("Claude Haiku 4.5 - Fast + extended thinking", "claude-haiku-4-5"), + ("Claude Sonnet 4 - High-performance", "claude-sonnet-4-20250514"), + ], + "google": [ + ("Gemini 3 Pro - Reasoning-first", "gemini-3-pro-preview"), + ("Gemini 3 Flash - Next-gen fast", "gemini-3-flash-preview"), + ("Gemini 2.5 Flash - Balanced, recommended", "gemini-2.5-flash"), + ], + "xai": [ + ( + "Grok 4.1 Fast (Reasoning) - High-performance, 2M ctx", + "grok-4-1-fast-reasoning", + ), + ("Grok 4 Fast (Reasoning) - High-performance", "grok-4-fast-reasoning"), + ("Grok 4 - Flagship model", "grok-4-0709"), + ( + "Grok 4.1 Fast (Non-Reasoning) - Speed optimized, 2M ctx", + "grok-4-1-fast-non-reasoning", + ), + ("Grok 4 Fast (Non-Reasoning) - Speed optimized", "grok-4-fast-non-reasoning"), + ], + "openrouter": [ + ("Kimi K2.5 0127 - Top ranked", "moonshotai/kimi-k2.5-0127"), + ( + "Claude Opus 4.5 - Premium intelligence", + "anthropic/claude-4.5-opus-20251124", + ), + ("Claude Sonnet 4.5 - Best for agents", "anthropic/claude-4.5-sonnet-20250929"), + ( + "Gemini 3 Flash Preview - Fast reasoning", + "google/gemini-3-flash-preview-20251217", + ), + ("Gemini 2.5 Flash - Balanced", "google/gemini-2.5-flash"), + ("Gemini 2.5 Flash Lite - Efficient", "google/gemini-2.5-flash-lite"), + ("Grok 4.1 Fast - High performance", "x-ai/grok-4.1-fast"), + ("Deepseek V3.2 - Cost effective", "deepseek/deepseek-v3.2-20251201"), + ("Grok Code Fast 1 - Coding optimized", "x-ai/grok-code-fast-1"), + ("Minimax M2.1 - Rising star", "minimax/minimax-m2.1"), + ("Z.AI GLM 4.5 Air (free)", "z-ai/glm-4.5-air:free"), + ("NVIDIA Nemotron 3 Nano 30B (free)", "nvidia/nemotron-3-nano-30b-a3b:free"), + ], + "ollama": [ + ("GLM-4.7-Flash:latest (30B, local)", "glm-4.7-flash:latest"), + ("GPT-OSS:latest (20B, local)", "gpt-oss:latest"), + ("Qwen3:latest (8B, local)", "qwen3:latest"), + ], +} + def get_ticker() -> str: """Prompt the user to enter a ticker symbol.""" @@ -125,43 +249,6 @@ def select_research_depth() -> int: def select_shallow_thinking_agent(provider) -> str: """Select shallow thinking llm engine using an interactive selection.""" - # Define shallow thinking llm engine options with their corresponding model names - SHALLOW_AGENT_OPTIONS = { - "openai": [ - ("GPT-5 Mini - Cost-optimized reasoning", "gpt-5-mini"), - ("GPT-5 Nano - Ultra-fast, high-throughput", "gpt-5-nano"), - ("GPT-5.2 - Latest flagship", "gpt-5.2"), - ("GPT-5.1 - Flexible reasoning", "gpt-5.1"), - ("GPT-4.1 - Smartest non-reasoning, 1M context", "gpt-4.1"), - ], - "anthropic": [ - ("Claude Haiku 4.5 - Fast + extended thinking", "claude-haiku-4-5"), - ("Claude Sonnet 4.5 - Best for agents/coding", "claude-sonnet-4-5"), - ("Claude Sonnet 4 - High-performance", "claude-sonnet-4-20250514"), - ], - "google": [ - ("Gemini 3 Flash - Next-gen fast", "gemini-3-flash-preview"), - ("Gemini 2.5 Flash - Balanced, recommended", "gemini-2.5-flash"), - ("Gemini 3 Pro - Reasoning-first", "gemini-3-pro-preview"), - ("Gemini 2.5 Flash Lite - Fast, low-cost", "gemini-2.5-flash-lite"), - ], - "xai": [ - ("Grok 4.1 Fast (Non-Reasoning) - Speed optimized, 2M ctx", "grok-4-1-fast-non-reasoning"), - ("Grok 4 Fast (Non-Reasoning) - Speed optimized", "grok-4-fast-non-reasoning"), - ("Grok 4.1 Fast (Reasoning) - High-performance, 2M ctx", "grok-4-1-fast-reasoning"), - ("Grok 4 Fast (Reasoning) - High-performance", "grok-4-fast-reasoning"), - ], - "openrouter": [ - ("NVIDIA Nemotron 3 Nano 30B (free)", "nvidia/nemotron-3-nano-30b-a3b:free"), - ("Z.AI GLM 4.5 Air (free)", "z-ai/glm-4.5-air:free"), - ], - "ollama": [ - ("Qwen3:latest (8B, local)", "qwen3:latest"), - ("GPT-OSS:latest (20B, local)", "gpt-oss:latest"), - ("GLM-4.7-Flash:latest (30B, local)", "glm-4.7-flash:latest"), - ], - } - choice = questionary.select( "Select Your [Quick-Thinking LLM Engine]:", choices=[ @@ -190,46 +277,6 @@ def select_shallow_thinking_agent(provider) -> str: def select_deep_thinking_agent(provider) -> str: """Select deep thinking llm engine using an interactive selection.""" - # Define deep thinking llm engine options with their corresponding model names - DEEP_AGENT_OPTIONS = { - "openai": [ - ("GPT-5.2 - Latest flagship", "gpt-5.2"), - ("GPT-5.1 - Flexible reasoning", "gpt-5.1"), - ("GPT-5 - Advanced reasoning", "gpt-5"), - ("GPT-4.1 - Smartest non-reasoning, 1M context", "gpt-4.1"), - ("GPT-5 Mini - Cost-optimized reasoning", "gpt-5-mini"), - ("GPT-5 Nano - Ultra-fast, high-throughput", "gpt-5-nano"), - ], - "anthropic": [ - ("Claude Sonnet 4.5 - Best for agents/coding", "claude-sonnet-4-5"), - ("Claude Opus 4.5 - Premium, max intelligence", "claude-opus-4-5"), - ("Claude Opus 4.1 - Most capable model", "claude-opus-4-1-20250805"), - ("Claude Haiku 4.5 - Fast + extended thinking", "claude-haiku-4-5"), - ("Claude Sonnet 4 - High-performance", "claude-sonnet-4-20250514"), - ], - "google": [ - ("Gemini 3 Pro - Reasoning-first", "gemini-3-pro-preview"), - ("Gemini 3 Flash - Next-gen fast", "gemini-3-flash-preview"), - ("Gemini 2.5 Flash - Balanced, recommended", "gemini-2.5-flash"), - ], - "xai": [ - ("Grok 4.1 Fast (Reasoning) - High-performance, 2M ctx", "grok-4-1-fast-reasoning"), - ("Grok 4 Fast (Reasoning) - High-performance", "grok-4-fast-reasoning"), - ("Grok 4 - Flagship model", "grok-4-0709"), - ("Grok 4.1 Fast (Non-Reasoning) - Speed optimized, 2M ctx", "grok-4-1-fast-non-reasoning"), - ("Grok 4 Fast (Non-Reasoning) - Speed optimized", "grok-4-fast-non-reasoning"), - ], - "openrouter": [ - ("Z.AI GLM 4.5 Air (free)", "z-ai/glm-4.5-air:free"), - ("NVIDIA Nemotron 3 Nano 30B (free)", "nvidia/nemotron-3-nano-30b-a3b:free"), - ], - "ollama": [ - ("GLM-4.7-Flash:latest (30B, local)", "glm-4.7-flash:latest"), - ("GPT-OSS:latest (20B, local)", "gpt-oss:latest"), - ("Qwen3:latest (8B, local)", "qwen3:latest"), - ], - } - choice = questionary.select( "Select Your [Deep-Thinking LLM Engine]:", choices=[ @@ -252,6 +299,7 @@ def select_deep_thinking_agent(provider) -> str: return choice + def select_llm_provider() -> tuple[str, str]: """Select the OpenAI api url using interactive selection.""" # Define OpenAI api options with their corresponding endpoints @@ -263,7 +311,7 @@ def select_llm_provider() -> tuple[str, str]: ("Openrouter", "https://openrouter.ai/api/v1"), ("Ollama", "http://localhost:11434/v1"), ] - + choice = questionary.select( "Select your LLM Provider:", choices=[ @@ -279,11 +327,11 @@ def select_llm_provider() -> tuple[str, str]: ] ), ).ask() - + if choice is None: console.print("\n[red]no OpenAI backend selected. Exiting...[/red]") exit(1) - + display_name, url = choice print(f"You selected: {display_name}\tURL: {url}") @@ -300,11 +348,13 @@ def ask_openai_reasoning_effort() -> str: return questionary.select( "Select Reasoning Effort:", choices=choices, - style=questionary.Style([ - ("selected", "fg:cyan noinherit"), - ("highlighted", "fg:cyan noinherit"), - ("pointer", "fg:cyan noinherit"), - ]), + style=questionary.Style( + [ + ("selected", "fg:cyan noinherit"), + ("highlighted", "fg:cyan noinherit"), + ("pointer", "fg:cyan noinherit"), + ] + ), ).ask() @@ -320,9 +370,11 @@ def ask_gemini_thinking_config() -> str | None: questionary.Choice("Enable Thinking (recommended)", "high"), questionary.Choice("Minimal/Disable Thinking", "minimal"), ], - style=questionary.Style([ - ("selected", "fg:green noinherit"), - ("highlighted", "fg:green noinherit"), - ("pointer", "fg:green noinherit"), - ]), + style=questionary.Style( + [ + ("selected", "fg:green noinherit"), + ("highlighted", "fg:green noinherit"), + ("pointer", "fg:green noinherit"), + ] + ), ).ask() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_cli_utils.py b/tests/test_cli_utils.py new file mode 100644 index 00000000..4a0de04a --- /dev/null +++ b/tests/test_cli_utils.py @@ -0,0 +1,143 @@ +"""Tests for CLI utilities, specifically OpenRouter model configuration.""" + +import pytest +from cli.utils import SHALLOW_AGENT_OPTIONS, DEEP_AGENT_OPTIONS + + +class TestOpenRouterModels: + """Test suite for OpenRouter model configuration.""" + + def test_shallow_thinking_openrouter_models_count(self): + """Verify OpenRouter shallow thinking models list has expected entries.""" + openrouter_models = SHALLOW_AGENT_OPTIONS.get("openrouter", []) + + # Should have at least 12 models (top 10 + 2 free models) + assert len(openrouter_models) >= 12, ( + f"Expected at least 12 models, got {len(openrouter_models)}" + ) + + def test_deep_thinking_openrouter_models_count(self): + """Verify OpenRouter deep thinking models list has expected entries.""" + openrouter_models = DEEP_AGENT_OPTIONS.get("openrouter", []) + + # Should have at least 12 models (top 10 + 2 free models) + assert len(openrouter_models) >= 12, ( + f"Expected at least 12 models, got {len(openrouter_models)}" + ) + + def test_openrouter_models_have_required_format(self): + """Verify all OpenRouter models follow the expected format.""" + for model_list in [ + SHALLOW_AGENT_OPTIONS.get("openrouter", []), + DEEP_AGENT_OPTIONS.get("openrouter", []), + ]: + for display_name, model_id in model_list: + # Display name should be a string + assert isinstance(display_name, str), ( + f"Display name must be string, got {type(display_name)}" + ) + assert len(display_name) > 0, "Display name cannot be empty" + + # Model ID should be a string with provider/model format + assert isinstance(model_id, str), ( + f"Model ID must be string, got {type(model_id)}" + ) + assert "/" in model_id, ( + f"Model ID {model_id} should contain provider prefix" + ) + + def test_top_models_included(self): + """Verify top ranked models from OpenRouter are included.""" + # Key models that should be present (based on OpenRouter rankings) + expected_models = [ + "moonshotai/kimi-k2.5-0127", # #1 ranked + "anthropic/claude-4.5-opus-20251124", # Top Claude + "anthropic/claude-4.5-sonnet-20250929", # Popular Claude + "google/gemini-3-flash-preview-20251217", # Top Gemini + "deepseek/deepseek-v3.2-20251201", # Popular open source + "x-ai/grok-4.1-fast", # xAI model + ] + + all_models = [] + for options in [SHALLOW_AGENT_OPTIONS, DEEP_AGENT_OPTIONS]: + all_models.extend( + [model_id for _, model_id in options.get("openrouter", [])] + ) + + for expected in expected_models: + assert expected in all_models, ( + f"Expected model {expected} not found in configuration" + ) + + def test_free_models_still_available(self): + """Verify free models are still included.""" + free_models = [ + "nvidia/nemotron-3-nano-30b-a3b:free", + "z-ai/glm-4.5-air:free", + ] + + all_models = [] + for options in [SHALLOW_AGENT_OPTIONS, DEEP_AGENT_OPTIONS]: + all_models.extend( + [model_id for _, model_id in options.get("openrouter", [])] + ) + + for free_model in free_models: + assert free_model in all_models, ( + f"Free model {free_model} should be preserved" + ) + + def test_no_duplicate_model_ids(self): + """Verify no duplicate model IDs exist in OpenRouter lists.""" + for options in [SHALLOW_AGENT_OPTIONS, DEEP_AGENT_OPTIONS]: + model_ids = [model_id for _, model_id in options.get("openrouter", [])] + assert len(model_ids) == len(set(model_ids)), "Duplicate model IDs found" + + def test_all_providers_have_models(self): + """Verify all supported providers have model entries.""" + expected_providers = [ + "openai", + "anthropic", + "google", + "xai", + "openrouter", + "ollama", + ] + + for provider in expected_providers: + assert provider in SHALLOW_AGENT_OPTIONS, ( + f"Provider {provider} missing from shallow options" + ) + assert len(SHALLOW_AGENT_OPTIONS[provider]) > 0, ( + f"Provider {provider} has no shallow models" + ) + + assert provider in DEEP_AGENT_OPTIONS, ( + f"Provider {provider} missing from deep options" + ) + assert len(DEEP_AGENT_OPTIONS[provider]) > 0, ( + f"Provider {provider} has no deep models" + ) + + def test_model_consistency_between_lists(self): + """Verify common models appear in both shallow and deep lists where applicable.""" + shallow_models = set( + model_id for _, model_id in SHALLOW_AGENT_OPTIONS.get("openrouter", []) + ) + deep_models = set( + model_id for _, model_id in DEEP_AGENT_OPTIONS.get("openrouter", []) + ) + + # Free models should be in both lists + free_models = {"nvidia/nemotron-3-nano-30b-a3b:free", "z-ai/glm-4.5-air:free"} + for free_model in free_models: + assert free_model in shallow_models, ( + f"Free model {free_model} should be in shallow list" + ) + assert free_model in deep_models, ( + f"Free model {free_model} should be in deep list" + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"])