694 lines
22 KiB
Python
694 lines
22 KiB
Python
"""Retrofit verification and readiness assessment.
|
|
|
|
This module verifies retrofit execution results and assesses project readiness
|
|
for autonomous development. Runs compliance checks, test suites, and compatibility
|
|
verification.
|
|
|
|
Classes:
|
|
ComplianceCheck: Single compliance check result
|
|
TestResult: Test suite execution results
|
|
CompatibilityReport: Tool and dependency compatibility
|
|
VerificationResult: Complete verification results
|
|
RetrofitVerifier: Main verification coordinator
|
|
|
|
Security:
|
|
- CWE-22: Path validation via security_utils
|
|
- CWE-78: Command injection prevention
|
|
- CWE-117: Audit logging with sanitization
|
|
|
|
Related:
|
|
- GitHub Issue #59: Brownfield retrofit command implementation
|
|
|
|
See error-handling-patterns skill for exception hierarchy and error handling best practices.
|
|
|
|
|
|
Design Patterns:
|
|
See library-design-patterns skill for standardized design patterns.
|
|
"""
|
|
|
|
import subprocess
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
from .security_utils import audit_log, validate_path
|
|
from .retrofit_executor import ExecutionResult
|
|
|
|
|
|
@dataclass
|
|
class ComplianceCheck:
|
|
"""Single compliance check result.
|
|
|
|
Attributes:
|
|
check_name: Name of the check
|
|
passed: Whether check passed
|
|
message: Result message
|
|
remediation: Remediation steps if failed
|
|
"""
|
|
check_name: str
|
|
passed: bool
|
|
message: str
|
|
remediation: Optional[str] = None
|
|
|
|
def to_dict(self) -> dict:
|
|
"""Convert to dictionary representation.
|
|
|
|
Returns:
|
|
Dictionary with check data
|
|
"""
|
|
return {
|
|
"check_name": self.check_name,
|
|
"passed": self.passed,
|
|
"message": self.message,
|
|
"remediation": self.remediation
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class TestResult:
|
|
"""Test suite execution results.
|
|
|
|
Attributes:
|
|
framework: Test framework used
|
|
passed: Number of passing tests
|
|
failed: Number of failing tests
|
|
skipped: Number of skipped tests
|
|
coverage: Test coverage percentage (0-100)
|
|
"""
|
|
framework: str = "unknown"
|
|
passed: int = 0
|
|
failed: int = 0
|
|
skipped: int = 0
|
|
coverage: float = 0.0
|
|
|
|
def to_dict(self) -> dict:
|
|
"""Convert to dictionary representation.
|
|
|
|
Returns:
|
|
Dictionary with test results
|
|
"""
|
|
return {
|
|
"framework": self.framework,
|
|
"passed": self.passed,
|
|
"failed": self.failed,
|
|
"skipped": self.skipped,
|
|
"coverage": self.coverage,
|
|
"total": self.passed + self.failed + self.skipped
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class CompatibilityReport:
|
|
"""Tool and dependency compatibility.
|
|
|
|
Attributes:
|
|
version_checks: Dict mapping tool to version string
|
|
dependency_checks: Dict mapping dependency to status
|
|
issues: List of compatibility issues found
|
|
"""
|
|
version_checks: Dict[str, str] = field(default_factory=dict)
|
|
dependency_checks: Dict[str, str] = field(default_factory=dict)
|
|
issues: List[str] = field(default_factory=list)
|
|
|
|
def to_dict(self) -> dict:
|
|
"""Convert to dictionary representation.
|
|
|
|
Returns:
|
|
Dictionary with compatibility data
|
|
"""
|
|
return {
|
|
"version_checks": self.version_checks,
|
|
"dependency_checks": self.dependency_checks,
|
|
"issues": self.issues,
|
|
"compatible": len(self.issues) == 0
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class VerificationResult:
|
|
"""Complete verification results.
|
|
|
|
Attributes:
|
|
compliance_checks: List of compliance check results
|
|
test_result: Test suite results
|
|
compatibility_report: Compatibility check results
|
|
readiness_score: Overall readiness score (0-100)
|
|
blockers: List of critical blockers
|
|
ready_for_auto_implement: Whether ready for /auto-implement
|
|
"""
|
|
compliance_checks: List[ComplianceCheck] = field(default_factory=list)
|
|
test_result: Optional[TestResult] = None
|
|
compatibility_report: Optional[CompatibilityReport] = None
|
|
readiness_score: float = 0.0
|
|
blockers: List[str] = field(default_factory=list)
|
|
ready_for_auto_implement: bool = False
|
|
|
|
def to_dict(self) -> dict:
|
|
"""Convert to dictionary representation.
|
|
|
|
Returns:
|
|
Dictionary with verification results
|
|
"""
|
|
return {
|
|
"compliance_checks": [check.to_dict() for check in self.compliance_checks],
|
|
"test_result": self.test_result.to_dict() if self.test_result else None,
|
|
"compatibility_report": self.compatibility_report.to_dict() if self.compatibility_report else None,
|
|
"readiness_score": self.readiness_score,
|
|
"blockers": self.blockers,
|
|
"ready_for_auto_implement": self.ready_for_auto_implement,
|
|
"checks_passed": sum(1 for check in self.compliance_checks if check.passed),
|
|
"checks_failed": sum(1 for check in self.compliance_checks if not check.passed)
|
|
}
|
|
|
|
|
|
class RetrofitVerifier:
|
|
"""Main retrofit verification coordinator.
|
|
|
|
Verifies retrofit execution results and assesses project readiness for
|
|
autonomous development via comprehensive compliance and compatibility checks.
|
|
"""
|
|
|
|
def __init__(self, project_root: Path):
|
|
"""Initialize retrofit verifier.
|
|
|
|
Args:
|
|
project_root: Path to project root directory
|
|
|
|
Raises:
|
|
ValueError: If project_root invalid
|
|
"""
|
|
# Security: Validate project root path (CWE-22)
|
|
validated_root = validate_path(
|
|
project_root,
|
|
"project_root",
|
|
allow_missing=False,
|
|
)
|
|
self.project_root = Path(validated_root)
|
|
|
|
# Audit log initialization
|
|
audit_log(
|
|
"retrofit_verifier_init",
|
|
project_root=str(self.project_root),
|
|
success=True
|
|
)
|
|
|
|
def verify(self, execution_result: ExecutionResult) -> VerificationResult:
|
|
"""Verify retrofit execution and assess readiness.
|
|
|
|
Args:
|
|
execution_result: Retrofit execution results
|
|
|
|
Returns:
|
|
Verification results with readiness assessment
|
|
|
|
Raises:
|
|
ValueError: If execution_result invalid
|
|
"""
|
|
if not execution_result:
|
|
raise ValueError("Execution result required")
|
|
|
|
audit_log(
|
|
"retrofit_verification_start",
|
|
project_root=str(self.project_root),
|
|
completed_steps=len(execution_result.completed_steps),
|
|
failed_steps=len(execution_result.failed_steps)
|
|
)
|
|
|
|
try:
|
|
result = VerificationResult()
|
|
|
|
# Run compliance checks
|
|
result.compliance_checks = self.run_compliance_checks()
|
|
|
|
# Run test suite
|
|
result.test_result = self.run_test_suite()
|
|
|
|
# Check compatibility
|
|
result.compatibility_report = self.check_compatibility()
|
|
|
|
# Assess readiness
|
|
result.readiness_score = self.assess_readiness()
|
|
|
|
# Identify blockers
|
|
result.blockers = self._identify_blockers(result)
|
|
|
|
# Determine if ready for /auto-implement
|
|
result.ready_for_auto_implement = (
|
|
len(result.blockers) == 0 and
|
|
result.readiness_score >= 70.0
|
|
)
|
|
|
|
audit_log(
|
|
"retrofit_verification_complete",
|
|
project_root=str(self.project_root),
|
|
readiness_score=result.readiness_score,
|
|
blockers=len(result.blockers),
|
|
ready=result.ready_for_auto_implement,
|
|
success=True
|
|
)
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
audit_log(
|
|
"retrofit_verification_failed",
|
|
project_root=str(self.project_root),
|
|
error=str(e),
|
|
success=False
|
|
)
|
|
raise
|
|
|
|
def run_compliance_checks(self) -> List[ComplianceCheck]:
|
|
"""Run compliance checks for autonomous-dev standards.
|
|
|
|
Returns:
|
|
List of compliance check results
|
|
"""
|
|
checks = []
|
|
|
|
# Check: PROJECT.md exists
|
|
checks.append(self.verify_project_md())
|
|
|
|
# Check: File organization
|
|
checks.append(self.verify_file_organization())
|
|
|
|
# Check: Test structure
|
|
checks.append(self._verify_test_structure())
|
|
|
|
# Check: Documentation
|
|
checks.append(self._verify_documentation())
|
|
|
|
# Check: Git configuration
|
|
checks.append(self._verify_git_config())
|
|
|
|
return checks
|
|
|
|
def run_test_suite(self) -> TestResult:
|
|
"""Run test suite if available.
|
|
|
|
Returns:
|
|
Test execution results
|
|
"""
|
|
result = TestResult()
|
|
|
|
try:
|
|
# Check if pytest available
|
|
pytest_path = self.project_root / "pytest.ini"
|
|
has_pytest_config = pytest_path.exists() or (self.project_root / "pyproject.toml").exists()
|
|
|
|
if not has_pytest_config:
|
|
result.framework = "none"
|
|
return result
|
|
|
|
result.framework = "pytest"
|
|
|
|
# Run pytest (simplified - would use subprocess in real implementation)
|
|
# Security: Command injection prevention (CWE-78)
|
|
tests_dir = self.project_root / "tests"
|
|
if tests_dir.exists():
|
|
# Would run: pytest --tb=short --quiet
|
|
# For now, return placeholder results
|
|
result.passed = 0 # Would parse from pytest output
|
|
result.failed = 0
|
|
result.skipped = 0
|
|
result.coverage = 0.0
|
|
|
|
audit_log(
|
|
"test_suite_executed",
|
|
framework=result.framework,
|
|
passed=result.passed,
|
|
failed=result.failed
|
|
)
|
|
|
|
except Exception as e:
|
|
audit_log(
|
|
"test_suite_failed",
|
|
error=str(e),
|
|
success=False
|
|
)
|
|
|
|
return result
|
|
|
|
def verify_project_md(self) -> ComplianceCheck:
|
|
"""Verify PROJECT.md exists and has required sections.
|
|
|
|
Returns:
|
|
Compliance check result
|
|
"""
|
|
project_md = self.project_root / ".claude" / "PROJECT.md"
|
|
|
|
if not project_md.exists():
|
|
return ComplianceCheck(
|
|
check_name="project_md_exists",
|
|
passed=False,
|
|
message="PROJECT.md not found",
|
|
remediation="Create .claude/PROJECT.md with GOALS, SCOPE, CONSTRAINTS sections"
|
|
)
|
|
|
|
try:
|
|
content = project_md.read_text(encoding='utf-8')
|
|
|
|
# Check for required sections
|
|
required_sections = ["GOALS", "SCOPE", "CONSTRAINTS"]
|
|
missing_sections = [s for s in required_sections if f"## {s}" not in content]
|
|
|
|
if missing_sections:
|
|
return ComplianceCheck(
|
|
check_name="project_md_sections",
|
|
passed=False,
|
|
message=f"PROJECT.md missing sections: {', '.join(missing_sections)}",
|
|
remediation=f"Add missing sections to PROJECT.md: {', '.join(missing_sections)}"
|
|
)
|
|
|
|
return ComplianceCheck(
|
|
check_name="project_md_complete",
|
|
passed=True,
|
|
message="PROJECT.md exists with all required sections"
|
|
)
|
|
|
|
except Exception as e:
|
|
return ComplianceCheck(
|
|
check_name="project_md_read",
|
|
passed=False,
|
|
message=f"Failed to read PROJECT.md: {e}",
|
|
remediation="Verify PROJECT.md is readable and properly formatted"
|
|
)
|
|
|
|
def verify_file_organization(self) -> ComplianceCheck:
|
|
"""Verify file organization follows standards.
|
|
|
|
Returns:
|
|
Compliance check result
|
|
"""
|
|
# Check for standard directories
|
|
has_src = (self.project_root / "src").is_dir()
|
|
has_tests = (self.project_root / "tests").is_dir()
|
|
has_docs = (self.project_root / "docs").is_dir()
|
|
|
|
# Check for scattered source files in root
|
|
root_py_files = list(self.project_root.glob("*.py"))
|
|
# Exclude common root files
|
|
excluded = {"setup.py", "conftest.py", "__init__.py"}
|
|
scattered_files = [f for f in root_py_files if f.name not in excluded]
|
|
|
|
if len(scattered_files) > 3:
|
|
return ComplianceCheck(
|
|
check_name="file_organization",
|
|
passed=False,
|
|
message=f"{len(scattered_files)} Python files in root directory",
|
|
remediation="Move source files to src/ directory for better organization"
|
|
)
|
|
|
|
if not has_src and len(root_py_files) > 5:
|
|
return ComplianceCheck(
|
|
check_name="file_organization",
|
|
passed=False,
|
|
message="No src/ directory structure",
|
|
remediation="Create src/ directory and organize source files"
|
|
)
|
|
|
|
score = sum([has_src, has_tests, has_docs])
|
|
if score >= 2:
|
|
return ComplianceCheck(
|
|
check_name="file_organization",
|
|
passed=True,
|
|
message=f"Good file organization (score: {score}/3)"
|
|
)
|
|
else:
|
|
return ComplianceCheck(
|
|
check_name="file_organization",
|
|
passed=False,
|
|
message=f"Poor file organization (score: {score}/3)",
|
|
remediation="Create standard directories: src/, tests/, docs/"
|
|
)
|
|
|
|
def check_compatibility(self) -> CompatibilityReport:
|
|
"""Check tool and dependency compatibility.
|
|
|
|
Returns:
|
|
Compatibility report
|
|
"""
|
|
report = CompatibilityReport()
|
|
|
|
# Check Python version
|
|
try:
|
|
result = subprocess.run(
|
|
["python", "--version"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5
|
|
)
|
|
if result.returncode == 0:
|
|
version = result.stdout.strip()
|
|
report.version_checks["python"] = version
|
|
|
|
# Check if Python 3.8+
|
|
if "Python 3." in version:
|
|
major, minor = version.split()[1].split(".")[:2]
|
|
if int(minor) < 8:
|
|
report.issues.append(f"Python version {version} < 3.8 (recommended: 3.8+)")
|
|
else:
|
|
report.issues.append("Python not found")
|
|
|
|
except Exception as e:
|
|
report.issues.append(f"Failed to check Python version: {e}")
|
|
|
|
# Check git
|
|
try:
|
|
result = subprocess.run(
|
|
["git", "--version"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5
|
|
)
|
|
if result.returncode == 0:
|
|
report.version_checks["git"] = result.stdout.strip()
|
|
else:
|
|
report.issues.append("Git not found")
|
|
|
|
except Exception as e:
|
|
report.issues.append(f"Failed to check Git version: {e}")
|
|
|
|
# Check if git repository
|
|
if not (self.project_root / ".git").is_dir():
|
|
report.issues.append("Not a Git repository")
|
|
report.dependency_checks["git_repo"] = "missing"
|
|
else:
|
|
report.dependency_checks["git_repo"] = "present"
|
|
|
|
# Check for package manager
|
|
has_requirements = (self.project_root / "requirements.txt").exists()
|
|
has_pyproject = (self.project_root / "pyproject.toml").exists()
|
|
has_setup = (self.project_root / "setup.py").exists()
|
|
|
|
if has_requirements or has_pyproject or has_setup:
|
|
report.dependency_checks["package_manager"] = "present"
|
|
else:
|
|
report.issues.append("No package manager configuration found (requirements.txt, pyproject.toml, or setup.py)")
|
|
report.dependency_checks["package_manager"] = "missing"
|
|
|
|
audit_log(
|
|
"compatibility_check_complete",
|
|
issues=len(report.issues),
|
|
compatible=len(report.issues) == 0
|
|
)
|
|
|
|
return report
|
|
|
|
def assess_readiness(self) -> float:
|
|
"""Assess overall readiness score.
|
|
|
|
Returns:
|
|
Readiness score (0-100)
|
|
"""
|
|
score = 0.0
|
|
|
|
# Component weights (total 100)
|
|
weights = {
|
|
"project_md": 20.0,
|
|
"file_organization": 20.0,
|
|
"test_structure": 20.0,
|
|
"documentation": 15.0,
|
|
"git_config": 10.0,
|
|
"compatibility": 15.0
|
|
}
|
|
|
|
# Project.md check
|
|
project_md_exists = (self.project_root / ".claude" / "PROJECT.md").exists()
|
|
if project_md_exists:
|
|
score += weights["project_md"]
|
|
|
|
# File organization
|
|
has_src = (self.project_root / "src").is_dir()
|
|
has_tests = (self.project_root / "tests").is_dir()
|
|
org_score = sum([has_src, has_tests]) / 2
|
|
score += weights["file_organization"] * org_score
|
|
|
|
# Test structure
|
|
if has_tests:
|
|
test_files = list((self.project_root / "tests").glob("test_*.py"))
|
|
if len(test_files) > 0:
|
|
score += weights["test_structure"]
|
|
|
|
# Documentation
|
|
readme_exists = (self.project_root / "README.md").exists()
|
|
if readme_exists:
|
|
score += weights["documentation"]
|
|
|
|
# Git config
|
|
is_git_repo = (self.project_root / ".git").is_dir()
|
|
if is_git_repo:
|
|
score += weights["git_config"]
|
|
|
|
# Compatibility
|
|
has_package_manager = (
|
|
(self.project_root / "requirements.txt").exists() or
|
|
(self.project_root / "pyproject.toml").exists()
|
|
)
|
|
if has_package_manager:
|
|
score += weights["compatibility"]
|
|
|
|
audit_log(
|
|
"readiness_assessed",
|
|
score=score,
|
|
ready=(score >= 70.0)
|
|
)
|
|
|
|
return score
|
|
|
|
# Private helper methods
|
|
|
|
def _verify_test_structure(self) -> ComplianceCheck:
|
|
"""Verify test directory structure.
|
|
|
|
Returns:
|
|
Compliance check result
|
|
"""
|
|
tests_dir = self.project_root / "tests"
|
|
|
|
if not tests_dir.exists():
|
|
return ComplianceCheck(
|
|
check_name="test_structure",
|
|
passed=False,
|
|
message="No tests/ directory found",
|
|
remediation="Create tests/ directory and add test files"
|
|
)
|
|
|
|
# Check for test files
|
|
test_files = list(tests_dir.glob("test_*.py"))
|
|
if len(test_files) == 0:
|
|
return ComplianceCheck(
|
|
check_name="test_structure",
|
|
passed=False,
|
|
message="No test files found in tests/",
|
|
remediation="Add test files following test_*.py naming convention"
|
|
)
|
|
|
|
return ComplianceCheck(
|
|
check_name="test_structure",
|
|
passed=True,
|
|
message=f"Test structure valid ({len(test_files)} test files)"
|
|
)
|
|
|
|
def _verify_documentation(self) -> ComplianceCheck:
|
|
"""Verify documentation exists.
|
|
|
|
Returns:
|
|
Compliance check result
|
|
"""
|
|
readme = self.project_root / "README.md"
|
|
|
|
if not readme.exists():
|
|
return ComplianceCheck(
|
|
check_name="documentation",
|
|
passed=False,
|
|
message="README.md not found",
|
|
remediation="Create README.md with project overview"
|
|
)
|
|
|
|
try:
|
|
content = readme.read_text(encoding='utf-8')
|
|
if len(content.strip()) < 100:
|
|
return ComplianceCheck(
|
|
check_name="documentation",
|
|
passed=False,
|
|
message="README.md is too sparse",
|
|
remediation="Add detailed project documentation to README.md"
|
|
)
|
|
|
|
return ComplianceCheck(
|
|
check_name="documentation",
|
|
passed=True,
|
|
message="Documentation complete"
|
|
)
|
|
|
|
except Exception as e:
|
|
return ComplianceCheck(
|
|
check_name="documentation",
|
|
passed=False,
|
|
message=f"Failed to read README.md: {e}",
|
|
remediation="Verify README.md is readable"
|
|
)
|
|
|
|
def _verify_git_config(self) -> ComplianceCheck:
|
|
"""Verify git configuration.
|
|
|
|
Returns:
|
|
Compliance check result
|
|
"""
|
|
git_dir = self.project_root / ".git"
|
|
|
|
if not git_dir.is_dir():
|
|
return ComplianceCheck(
|
|
check_name="git_config",
|
|
passed=False,
|
|
message="Not a Git repository",
|
|
remediation="Initialize Git repository with: git init"
|
|
)
|
|
|
|
# Check for .gitignore
|
|
gitignore = self.project_root / ".gitignore"
|
|
if not gitignore.exists():
|
|
return ComplianceCheck(
|
|
check_name="git_config",
|
|
passed=False,
|
|
message="No .gitignore file found",
|
|
remediation="Create .gitignore to exclude build artifacts and sensitive files"
|
|
)
|
|
|
|
return ComplianceCheck(
|
|
check_name="git_config",
|
|
passed=True,
|
|
message="Git properly configured"
|
|
)
|
|
|
|
def _identify_blockers(self, result: VerificationResult) -> List[str]:
|
|
"""Identify critical blockers preventing /auto-implement.
|
|
|
|
Args:
|
|
result: Verification result
|
|
|
|
Returns:
|
|
List of blocker descriptions
|
|
"""
|
|
blockers = []
|
|
|
|
# Check critical compliance failures
|
|
for check in result.compliance_checks:
|
|
if not check.passed:
|
|
# Critical checks
|
|
if check.check_name in ["project_md_exists", "git_config"]:
|
|
blockers.append(f"CRITICAL: {check.message}")
|
|
|
|
# Check compatibility issues
|
|
if result.compatibility_report:
|
|
for issue in result.compatibility_report.issues:
|
|
if "not found" in issue.lower() or "missing" in issue.lower():
|
|
blockers.append(f"COMPATIBILITY: {issue}")
|
|
|
|
# Check test failures
|
|
if result.test_result and result.test_result.failed > 0:
|
|
blockers.append(f"TESTS: {result.test_result.failed} failing tests")
|
|
|
|
return blockers
|