TradingAgents/.claude/hooks/validate_docs_consistency.py

#!/usr/bin/env python3
"""
Documentation Consistency Validation Hook - Layer 3 Defense with GenAI Semantic Validation

This pre-commit hook validates that documentation stays in sync with code.
It's OPTIONAL - can be annoying to block commits, but catches drift early.

Features:
- Count validation (exact matches)
- GenAI semantic validation of descriptions (accuracy checking)
- Catches misleading or inaccurate documentation
- Graceful degradation with fallback heuristics

Enable via:
    .claude/settings.local.json:
    {
      "hooks": {
        "PreCommit": {
          "*": ["python .claude/hooks/validate_docs_consistency.py"]
        }
      }
    }

Or via git pre-commit hook:
    ln -s ../../.claude/hooks/validate_docs_consistency.py .git/hooks/pre-commit

What it checks:
- README.md skill/agent/command counts match reality
- GenAI validates descriptions match actual functionality
- Cross-document consistency (SYNC-STATUS, UPDATES, marketplace.json)
- No references to non-existent skills
- marketplace.json metrics match actual counts

Exit codes:
- 0: All checks passed
- 1: Documentation inconsistency detected (blocks commit)
"""

import sys
import json
import re
import os
from pathlib import Path
from typing import Tuple

from genai_utils import GenAIAnalyzer, parse_binary_response
from genai_prompts import DESCRIPTION_VALIDATION_PROMPT

# Initialize GenAI analyzer (with feature flag support)
analyzer = GenAIAnalyzer(
    use_genai=os.environ.get("GENAI_DOCS_VALIDATE", "true").lower() == "true"
)


def get_plugin_root() -> Path:
    """Find plugin root directory."""
    # First, check if we're running from .claude/hooks (dogfooding)
    hook_dir = Path(__file__).parent
    repo_root = hook_dir.parent.parent  # .claude/hooks -> .claude -> repo_root

    plugin_path = repo_root / "plugins" / "autonomous-dev"
    if plugin_path.exists():
        return plugin_path

    # Fallback: check if we're already in the plugin directory
    current = hook_dir.parent
    if (current / "agents").exists() and (current / "skills").exists():
        return current

    # Give up
    raise FileNotFoundError("Could not find plugin root directory")


def count_skills(plugin_root: Path) -> int:
    """Count actual skills in skills/ directory."""
    skills_dir = plugin_root / "skills"
    return len([
        d for d in skills_dir.iterdir()
        if d.is_dir() and not d.name.startswith(".")
    ])


def count_agents(plugin_root: Path) -> int:
    """Count actual agents in agents/ directory."""
    agents_dir = plugin_root / "agents"
    return len([
        f for f in agents_dir.iterdir()
        if f.is_file() and f.suffix == ".md" and not f.name.startswith(".")
    ])


def count_commands(plugin_root: Path) -> int:
    """Count actual commands in commands/ directory."""
    commands_dir = plugin_root / "commands"
    return len([
        f for f in commands_dir.iterdir()
        if f.is_file() and f.suffix == ".md" and not f.name.startswith(".")
    ])


def check_readme_skill_count(plugin_root: Path, actual_count: int) -> Tuple[bool, str]:
    """Check README.md skill count matches actual."""
    readme_path = plugin_root / "README.md"
    if not readme_path.exists():
        return False, "README.md not found"

    content = readme_path.read_text()
    pattern = rf"\b{actual_count}\s+[Ss]kills"

    if not re.search(pattern, content):
        return False, (
            f"README.md shows incorrect skill count (expected {actual_count})\n"
            f"Fix: Update README.md to show '{actual_count} Skills (Comprehensive SDLC Coverage)'"
        )

    return True, "✅ README.md skill count correct"


def check_readme_agent_count(plugin_root: Path, actual_count: int) -> Tuple[bool, str]:
    """Check README.md agent count matches actual."""
    readme_path = plugin_root / "README.md"
    content = readme_path.read_text()
    pattern = rf"\b{actual_count}\s+[Ss]pecialized\s+[Aa]gents|\b{actual_count}\s+[Aa]gents"

    if not re.search(pattern, content):
        return False, (
            f"README.md shows incorrect agent count (expected {actual_count})\n"
            f"Fix: Update README.md to show '{actual_count} Specialized Agents'"
        )

    return True, "✅ README.md agent count correct"


def check_readme_command_count(plugin_root: Path, actual_count: int) -> Tuple[bool, str]:
    """Check README.md command count matches actual."""
    readme_path = plugin_root / "README.md"
    content = readme_path.read_text()
    pattern = rf"\b{actual_count}\s+[Ss]lash\s+[Cc]ommands|\b{actual_count}\s+[Cc]ommands"

    if not re.search(pattern, content):
        return False, (
            f"README.md shows incorrect command count (expected {actual_count})\n"
            f"Fix: Update README.md to show '{actual_count} Slash Commands'"
        )

    return True, "✅ README.md command count correct"


def check_marketplace_json(plugin_root: Path, skill_count: int, agent_count: int, command_count: int) -> Tuple[bool, str]:
    """Check marketplace.json metrics match actual counts."""
    marketplace_path = plugin_root / ".claude-plugin" / "marketplace.json"
    if not marketplace_path.exists():
        return True, "⚠️  marketplace.json not found (skipping)"

    try:
        data = json.loads(marketplace_path.read_text())
        metrics = data.get("metrics", {})

        errors = []
        if metrics.get("skills") != skill_count:
            errors.append(f"skills: {metrics.get('skills')} (should be {skill_count})")
        if metrics.get("agents") != agent_count:
            errors.append(f"agents: {metrics.get('agents')} (should be {agent_count})")
        if metrics.get("commands") != command_count:
            errors.append(f"commands: {metrics.get('commands')} (should be {command_count})")

        if errors:
            return False, (
                f"marketplace.json metrics incorrect:\n"
                + "\n".join(f"  - {e}" for e in errors) +
                f"\nFix: Update .claude-plugin/marketplace.json metrics section"
            )

        return True, "✅ marketplace.json metrics correct"

    except json.JSONDecodeError:
        return False, "marketplace.json is invalid JSON"


def check_no_broken_skill_references(plugin_root: Path) -> Tuple[bool, str]:
    """Check for references to non-existent skills."""
    # Get actual skills
    skills_dir = plugin_root / "skills"
    actual_skills = set(
        d.name for d in skills_dir.iterdir()
        if d.is_dir() and not d.name.startswith(".")
    )

    # Known problematic skills that have been removed
    problematic_skills = ['engineering-standards']

    readme_path = plugin_root / "README.md"
    readme_content = readme_path.read_text()

    broken_references = []
    for skill in problematic_skills:
        if skill not in actual_skills and skill in readme_content:
            broken_references.append(skill)

    if broken_references:
        return False, (
            f"README.md references non-existent skills: {broken_references}\n"
            f"Fix: Remove or replace these skill references"
        )

    return True, "✅ No broken skill references"


def check_cross_document_consistency(plugin_root: Path, skill_count: int) -> Tuple[bool, str]:
    """Check all documentation files show same skill count."""
    files_to_check = [
        "README.md",
        "docs/SYNC-STATUS.md",
        "docs/UPDATES.md",
        "INSTALL_TEMPLATE.md",
    ]

    inconsistencies = []

    for file_path in files_to_check:
        full_path = plugin_root / file_path
        if not full_path.exists():
            continue

        content = full_path.read_text()
        # Look for skill count mentions
        if str(skill_count) not in content or "skills" not in content.lower():
            # Check if it mentions a different count
            skill_mentions = re.findall(r'(\d+)\s+[Ss]kills', content)
            if skill_mentions and int(skill_mentions[0]) != skill_count:
                inconsistencies.append(f"{file_path}: shows {skill_mentions[0]} skills (should be {skill_count})")

    if inconsistencies:
        return False, (
            f"Cross-document skill count inconsistency:\n"
            + "\n".join(f"  - {i}" for i in inconsistencies) +
            f"\nFix: Update all files to show {skill_count} skills"
        )

    return True, "✅ Cross-document consistency verified"


def validate_description_accuracy_with_genai(plugin_root: Path, entity_type: str) -> Tuple[bool, str]:
    """Use GenAI to validate if descriptions match actual implementation.

    Delegates to shared GenAI utility with graceful fallback.

    Args:
        plugin_root: Root directory of plugin
        entity_type: 'agents', 'skills', or 'commands'

    Returns:
        (passed, message) tuple
    """
    # Get README.md section for the entity type
    readme_path = plugin_root / "README.md"
    if not readme_path.exists():
        return True, f"⏭️  No README.md found"

    readme_content = readme_path.read_text()

    # Extract the relevant section (simplified - looks for entity type mentions)
    section_start = readme_content.lower().find(entity_type.lower())
    if section_start == -1:
        return True, f"⏭️  No {entity_type} section found in README.md"

    # Get a reasonable chunk of the section
    section_end = min(section_start + 2000, len(readme_content))
    section = readme_content[section_start:section_end]

    # Call shared GenAI analyzer
    response = analyzer.analyze(
        DESCRIPTION_VALIDATION_PROMPT,
        entity_type=entity_type,
        section=section[:1000]
    )

    # Parse response using shared utility
    if response:
        is_accurate = parse_binary_response(
            response,
            true_keywords=["ACCURATE"],
            false_keywords=["MISLEADING"]
        )
        if is_accurate is not None:
            if is_accurate:
                return True, f"✅ GenAI validated {entity_type} descriptions are accurate"
            else:
                return False, (
                    f"⚠️  GenAI found potential inaccuracies in {entity_type} descriptions\n"
                    f"Review README.md {entity_type} section for misleading or vague descriptions"
                )

    # Fallback: if GenAI unavailable or ambiguous, skip validation
    return True, "⏭️  GenAI validation skipped (call failed or ambiguous)"


def main() -> int:
    """Run all documentation consistency checks.

    Returns:
        0 if all checks pass
        1 if any check fails
    """
    use_genai = os.environ.get("GENAI_DOCS_VALIDATE", "true").lower() == "true"
    genai_status = "🤖 (with GenAI semantic validation)" if use_genai else ""
    print(f"🔍 Validating documentation consistency... {genai_status}")
    print()

    try:
        plugin_root = get_plugin_root()
    except FileNotFoundError as e:
        print(f"❌ Error: {e}")
        return 1

    # Count actual resources
    skill_count = count_skills(plugin_root)
    agent_count = count_agents(plugin_root)
    command_count = count_commands(plugin_root)

    print(f"📊 Actual counts:")
    print(f"   - Skills: {skill_count}")
    print(f"   - Agents: {agent_count}")
    print(f"   - Commands: {command_count}")
    print()

    # Run all checks
    checks = [
        ("README.md skill count", check_readme_skill_count(plugin_root, skill_count)),
        ("README.md agent count", check_readme_agent_count(plugin_root, agent_count)),
        ("README.md command count", check_readme_command_count(plugin_root, command_count)),
        ("marketplace.json metrics", check_marketplace_json(plugin_root, skill_count, agent_count, command_count)),
        ("Broken skill references", check_no_broken_skill_references(plugin_root)),
        ("Cross-document consistency", check_cross_document_consistency(plugin_root, skill_count)),
    ]

    # Add GenAI semantic validation if enabled
    if use_genai:
        checks.extend([
            ("Agent descriptions accuracy", validate_description_accuracy_with_genai(plugin_root, "agents")),
            ("Command descriptions accuracy", validate_description_accuracy_with_genai(plugin_root, "commands")),
        ])

    all_passed = True

    for check_name, (passed, message) in checks:
        if passed:
            print(f"{message}")
        else:
            print(f"❌ {check_name} FAILED:")
            print(f"   {message}")
            print()
            all_passed = False

    print()

    if all_passed:
        print("✅ All documentation consistency checks passed!")
        return 0
    else:
        print("❌ Documentation consistency checks FAILED!")
        print()
        print("Fix the issues above before committing.")
        print("Or run: pytest tests/test_documentation_consistency.py -v")
        print()
        print("To skip this hook (NOT RECOMMENDED):")
        print("  git commit --no-verify")
        return 1


if __name__ == "__main__":
    sys.exit(main())