Add evaluation framework and fix call graph extraction

- Add evaluation test suite with TypeScript and Python fixtures
- Fix MCP server to defer CodeGraph init until rootUri received
- Fix call edge extraction by calling resolveReferences() after indexAll/sync
- Fix glob matching for root-level files (e.g., **/*.py now matches auth.py)
- Fix duplicate node extraction for methods inside classes
- Update context tests to use buildContext for semantic search + graph traversal
- Export unused formatter functions to fix build

Evaluation results:
- TypeScript: 96% precision, 79% recall, 85% F1
- Python: 99% precision, 80% recall, 85% F1

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenryandClaude Opus 4.5 committed 2026-01-18 18:48:22 -06:00
1 parent e306114607
commit 6b672f9152
30 files changed
+2600 -129

No files matched your search

@@ -0,0 +1,79 @@
"""Authentication service."""
import hashlib
import secrets
from datetime import datetime
from typing import Optional, Tuple
from models import User
from database import db
from validation import validate_email, validate_password
def hash_password(password: str) -> str:
"""Hash a password for storage."""
salt = secrets.token_hex(16)
hash_obj = hashlib.sha256((password + salt).encode())
return f"{salt}:{hash_obj.hexdigest()}"
def verify_password(password: str, password_hash: str) -> bool:
"""Verify a password against its hash."""
salt, stored_hash = password_hash.split(":")
hash_obj = hashlib.sha256((password + salt).encode())
return hash_obj.hexdigest() == stored_hash
def generate_token() -> str:
"""Generate a secure random token."""
return secrets.token_urlsafe(32)
class AuthService:
def __init__(self):
self.tokens: dict = {}
def register(self, email: str, password: str, name: str) -> Tuple[bool, str]:
"""Register a new user."""
if not validate_email(email):
return False, "Invalid email format"
if not validate_password(password):
return False, "Password too weak"
if db.get_user_by_email(email):
return False, "Email already registered"
user = User(
id=generate_token(),
email=email,
name=name,
password_hash=hash_password(password),
created_at=datetime.now(),
)
db.create_user(user)
return True, user.id
def login(self, email: str, password: str) -> Optional[str]:
"""Authenticate user and return token."""
user = db.get_user_by_email(email)
if not user:
return None
if not verify_password(password, user.password_hash):
return None
token = generate_token()
self.tokens[token] = user.id
return token
def logout(self, token: str) -> None:
"""Invalidate a token."""
self.tokens.pop(token, None)
def get_user_id(self, token: str) -> Optional[str]:
"""Get user ID from token."""
return self.tokens.get(token)
auth_service = AuthService()
@@ -0,0 +1,54 @@
"""Database operations."""
from typing import Optional, List, Dict
from models import User, Task, Project
class Database:
def __init__(self):
self.users: Dict[str, User] = {}
self.tasks: Dict[str, Task] = {}
self.projects: Dict[str, Project] = {}
def get_user(self, user_id: str) -> Optional[User]:
return self.users.get(user_id)
def get_user_by_email(self, email: str) -> Optional[User]:
for user in self.users.values():
if user.email == email:
return user
return None
def create_user(self, user: User) -> None:
self.users[user.id] = user
def get_task(self, task_id: str) -> Optional[Task]:
return self.tasks.get(task_id)
def get_user_tasks(self, user_id: str) -> List[Task]:
return [t for t in self.tasks.values() if t.user_id == user_id]
def create_task(self, task: Task) -> None:
self.tasks[task.id] = task
def update_task(self, task_id: str, **updates) -> Optional[Task]:
task = self.tasks.get(task_id)
if task:
for key, value in updates.items():
setattr(task, key, value)
return task
def delete_task(self, task_id: str) -> bool:
if task_id in self.tasks:
del self.tasks[task_id]
return True
return False
def get_project(self, project_id: str) -> Optional[Project]:
return self.projects.get(project_id)
def create_project(self, project: Project) -> None:
self.projects[project.id] = project
db = Database()
@@ -0,0 +1,305 @@
/**
* Ground truth definitions for the Python task management fixture
*/
import { FixtureGroundTruth } from '../../types';
export const pythonFixture: FixtureGroundTruth = {
name: 'python-taskmanager',
path: '__tests__/evaluation/fixtures/python-project',
language: 'python',
totalFiles: 5,
approximateTokens: 1200, // Rough estimate
testCases: [
// =========================================================================
// Search Tests
// =========================================================================
{
id: 'py-search-auth',
description: 'Search for authentication functionality',
query: 'authentication login',
type: 'search',
expectedSymbols: ['AuthService', 'AuthService.login', 'AuthService.register', 'verify_password'],
irrelevantSymbols: ['TaskService', 'validate_task_title', 'Project'],
minRecall: 0.7,
minPrecision: 0.5,
},
{
id: 'py-search-task',
description: 'Search for task management',
query: 'task create complete',
type: 'search',
expectedSymbols: ['TaskService', 'TaskService.create_task', 'TaskService.complete_task', 'Task'],
irrelevantSymbols: ['AuthService', 'validate_email', 'hash_password'],
minRecall: 0.7,
minPrecision: 0.5,
},
{
id: 'py-search-validation',
description: 'Search for validation',
query: 'validate',
type: 'search',
expectedSymbols: ['validate_email', 'validate_password', 'validate_task_title'],
irrelevantSymbols: ['hash_password', 'generate_token', 'TaskService'],
minRecall: 0.8,
minPrecision: 0.6,
},
// =========================================================================
// Context Tests
// =========================================================================
{
id: 'py-context-login-bug',
description: 'Build context for fixing login issues',
query: 'debug why users cannot log in',
type: 'context',
expectedSymbols: [
'AuthService.login',
'verify_password',
'db.get_user_by_email',
'User',
'hash_password',
],
irrelevantSymbols: [
'TaskService',
'validate_task_title',
'Project',
'Task',
],
minRecall: 0.8,
minPrecision: 0.6,
},
{
id: 'py-context-task-creation',
description: 'Build context for task creation flow',
query: 'understand how tasks are created',
type: 'context',
expectedSymbols: [
'TaskService.create_task',
'validate_task_title',
'auth_service.get_user_id',
'db.create_task',
'Task',
'generate_token',
],
irrelevantSymbols: [
'validate_email',
'hash_password',
'AuthService.register',
'Project',
],
minRecall: 0.7,
minPrecision: 0.5,
},
{
id: 'py-context-user-registration',
description: 'Build context for user registration',
query: 'add email confirmation to registration',
type: 'context',
expectedSymbols: [
'AuthService.register',
'validate_email',
'validate_password',
'hash_password',
'db.create_user',
'User',
],
irrelevantSymbols: [
'TaskService',
'validate_task_title',
'Task',
'Project',
],
minRecall: 0.7,
minPrecision: 0.6,
},
// =========================================================================
// Callers Tests
// =========================================================================
{
id: 'py-callers-get_user_id',
description: 'Find all callers of auth_service.get_user_id',
query: 'get_user_id',
type: 'callers',
targetSymbol: 'get_user_id',
expectedSymbols: [
'TaskService.create_task',
'TaskService.get_task',
'TaskService.get_user_tasks',
],
irrelevantSymbols: [
'AuthService.login',
'validate_email',
'hash_password',
],
minRecall: 1.0,
minPrecision: 1.0,
},
{
id: 'py-callers-validate_email',
description: 'Find all callers of validate_email',
query: 'validate_email',
type: 'callers',
targetSymbol: 'validate_email',
expectedSymbols: [
'AuthService.register',
],
irrelevantSymbols: [
'TaskService',
'validate_password',
'hash_password',
],
minRecall: 1.0,
minPrecision: 1.0,
},
{
id: 'py-callers-generate_token',
description: 'Find all callers of generate_token',
query: 'generate_token',
type: 'callers',
targetSymbol: 'generate_token',
expectedSymbols: [
'AuthService.register',
'AuthService.login',
'TaskService.create_task',
],
irrelevantSymbols: [
'validate_email',
'validate_password',
'db.get_user',
],
minRecall: 1.0,
minPrecision: 1.0,
},
// =========================================================================
// Callees Tests
// =========================================================================
{
id: 'py-callees-login',
description: 'Find what AuthService.login calls',
query: 'login',
type: 'callees',
targetSymbol: 'login',
expectedSymbols: [
'db.get_user_by_email',
'verify_password',
'generate_token',
],
irrelevantSymbols: [
'validate_email',
'hash_password',
'validate_task_title',
],
minRecall: 1.0,
minPrecision: 1.0,
},
{
id: 'py-callees-create_task',
description: 'Find what TaskService.create_task calls',
query: 'create_task',
type: 'callees',
targetSymbol: 'TaskService.create_task',
expectedSymbols: [
'auth_service.get_user_id',
'validate_task_title',
'generate_token',
'db.create_task',
],
irrelevantSymbols: [
'validate_email',
'hash_password',
'db.get_user',
],
minRecall: 0.8,
minPrecision: 0.8,
},
// =========================================================================
// Impact Tests
// =========================================================================
{
id: 'py-impact-generate_token',
description: 'Impact of changing generate_token',
query: 'generate_token',
type: 'impact',
targetSymbol: 'generate_token',
expectedSymbols: [
// Direct callers
'AuthService.register',
'AuthService.login',
'TaskService.create_task',
],
irrelevantSymbols: [
'validate_email',
'validate_task_title',
'db.get_project',
],
minRecall: 0.8,
minPrecision: 0.7,
},
{
id: 'py-impact-get_user_id',
description: 'Impact of changing get_user_id',
query: 'get_user_id',
type: 'impact',
targetSymbol: 'get_user_id',
expectedSymbols: [
'TaskService.create_task',
'TaskService.get_task',
'TaskService.get_user_tasks',
'TaskService.complete_task',
'TaskService.delete_task',
],
irrelevantSymbols: [
'AuthService.register',
'validate_email',
'hash_password',
],
minRecall: 0.8,
minPrecision: 0.7,
},
],
// Known call graph edges for validation
callGraph: [
// Auth -> Database
{ caller: 'AuthService.register', callee: 'db.get_user_by_email' },
{ caller: 'AuthService.register', callee: 'db.create_user' },
{ caller: 'AuthService.login', callee: 'db.get_user_by_email' },
// Auth -> Crypto
{ caller: 'AuthService.register', callee: 'hash_password' },
{ caller: 'AuthService.register', callee: 'generate_token' },
{ caller: 'AuthService.login', callee: 'verify_password' },
{ caller: 'AuthService.login', callee: 'generate_token' },
// Auth -> Validation
{ caller: 'AuthService.register', callee: 'validate_email' },
{ caller: 'AuthService.register', callee: 'validate_password' },
// Task -> Auth
{ caller: 'TaskService.create_task', callee: 'auth_service.get_user_id' },
{ caller: 'TaskService.get_task', callee: 'auth_service.get_user_id' },
{ caller: 'TaskService.get_user_tasks', callee: 'auth_service.get_user_id' },
// Task -> Database
{ caller: 'TaskService.create_task', callee: 'db.create_task' },
{ caller: 'TaskService.get_task', callee: 'db.get_task' },
{ caller: 'TaskService.get_user_tasks', callee: 'db.get_user_tasks' },
{ caller: 'TaskService.complete_task', callee: 'db.update_task' },
{ caller: 'TaskService.delete_task', callee: 'db.delete_task' },
// Task -> Crypto
{ caller: 'TaskService.create_task', callee: 'generate_token' },
// Task -> Validation
{ caller: 'TaskService.create_task', callee: 'validate_task_title' },
// Task -> Task (internal)
{ caller: 'TaskService.complete_task', callee: 'TaskService.get_task' },
{ caller: 'TaskService.delete_task', callee: 'TaskService.get_task' },
],
};
@@ -0,0 +1,34 @@
"""Data models for the application."""
from dataclasses import dataclass
from datetime import datetime
from typing import Optional, List
@dataclass
class User:
id: str
email: str
name: str
password_hash: str
created_at: datetime
@dataclass
class Task:
id: str
user_id: str
title: str
description: Optional[str]
completed: bool
created_at: datetime
completed_at: Optional[datetime] = None
@dataclass
class Project:
id: str
user_id: str
name: str
tasks: List[str] # Task IDs
created_at: datetime
@@ -0,0 +1,72 @@
"""Task management service."""
from datetime import datetime
from typing import Optional, List
from models import Task
from database import db
from auth import auth_service, generate_token
from validation import validate_task_title
class TaskService:
def create_task(
self, token: str, title: str, description: Optional[str] = None
) -> Optional[Task]:
"""Create a new task."""
user_id = auth_service.get_user_id(token)
if not user_id:
return None
if not validate_task_title(title):
return None
task = Task(
id=generate_token(),
user_id=user_id,
title=title,
description=description,
completed=False,
created_at=datetime.now(),
)
db.create_task(task)
return task
def get_task(self, token: str, task_id: str) -> Optional[Task]:
"""Get a task by ID."""
user_id = auth_service.get_user_id(token)
if not user_id:
return None
task = db.get_task(task_id)
if task and task.user_id == user_id:
return task
return None
def get_user_tasks(self, token: str) -> List[Task]:
"""Get all tasks for the authenticated user."""
user_id = auth_service.get_user_id(token)
if not user_id:
return []
return db.get_user_tasks(user_id)
def complete_task(self, token: str, task_id: str) -> bool:
"""Mark a task as completed."""
task = self.get_task(token, task_id)
if not task:
return False
db.update_task(task_id, completed=True, completed_at=datetime.now())
return True
def delete_task(self, token: str, task_id: str) -> bool:
"""Delete a task."""
task = self.get_task(token, task_id)
if not task:
return False
return db.delete_task(task_id)
task_service = TaskService()
@@ -0,0 +1,27 @@
"""Validation utilities."""
import re
def validate_email(email: str) -> bool:
"""Validate email format."""
pattern = r'^[^\s@]+@[^\s@]+\.[^\s@]+$'
return bool(re.match(pattern, email))
def validate_password(password: str) -> bool:
"""Validate password strength."""
if len(password) < 8:
return False
if not re.search(r'[A-Z]', password):
return False
if not re.search(r'[a-z]', password):
return False
if not re.search(r'[0-9]', password):
return False
return True
def validate_task_title(title: str) -> bool:
"""Validate task title."""
return bool(title and len(title.strip()) >= 1 and len(title) <= 200)