Add evaluation framework and fix call graph extraction
- Add evaluation test suite with TypeScript and Python fixtures - Fix MCP server to defer CodeGraph init until rootUri received - Fix call edge extraction by calling resolveReferences() after indexAll/sync - Fix glob matching for root-level files (e.g., **/*.py now matches auth.py) - Fix duplicate node extraction for methods inside classes - Update context tests to use buildContext for semantic search + graph traversal - Export unused formatter functions to fix build Evaluation results: - TypeScript: 96% precision, 79% recall, 85% F1 - Python: 99% precision, 80% recall, 85% F1 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e306114607
commit
6b672f9152
30 files changed
+2600
-129
No files matched your search
@@ -0,0 +1,79 @@
|
||||
"""Authentication service."""
|
||||
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from models import User
|
||||
from database import db
|
||||
from validation import validate_email, validate_password
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Hash a password for storage."""
|
||||
salt = secrets.token_hex(16)
|
||||
hash_obj = hashlib.sha256((password + salt).encode())
|
||||
return f"{salt}:{hash_obj.hexdigest()}"
|
||||
|
||||
|
||||
def verify_password(password: str, password_hash: str) -> bool:
|
||||
"""Verify a password against its hash."""
|
||||
salt, stored_hash = password_hash.split(":")
|
||||
hash_obj = hashlib.sha256((password + salt).encode())
|
||||
return hash_obj.hexdigest() == stored_hash
|
||||
|
||||
|
||||
def generate_token() -> str:
|
||||
"""Generate a secure random token."""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
class AuthService:
|
||||
def __init__(self):
|
||||
self.tokens: dict = {}
|
||||
|
||||
def register(self, email: str, password: str, name: str) -> Tuple[bool, str]:
|
||||
"""Register a new user."""
|
||||
if not validate_email(email):
|
||||
return False, "Invalid email format"
|
||||
|
||||
if not validate_password(password):
|
||||
return False, "Password too weak"
|
||||
|
||||
if db.get_user_by_email(email):
|
||||
return False, "Email already registered"
|
||||
|
||||
user = User(
|
||||
id=generate_token(),
|
||||
email=email,
|
||||
name=name,
|
||||
password_hash=hash_password(password),
|
||||
created_at=datetime.now(),
|
||||
)
|
||||
db.create_user(user)
|
||||
return True, user.id
|
||||
|
||||
def login(self, email: str, password: str) -> Optional[str]:
|
||||
"""Authenticate user and return token."""
|
||||
user = db.get_user_by_email(email)
|
||||
if not user:
|
||||
return None
|
||||
|
||||
if not verify_password(password, user.password_hash):
|
||||
return None
|
||||
|
||||
token = generate_token()
|
||||
self.tokens[token] = user.id
|
||||
return token
|
||||
|
||||
def logout(self, token: str) -> None:
|
||||
"""Invalidate a token."""
|
||||
self.tokens.pop(token, None)
|
||||
|
||||
def get_user_id(self, token: str) -> Optional[str]:
|
||||
"""Get user ID from token."""
|
||||
return self.tokens.get(token)
|
||||
|
||||
|
||||
auth_service = AuthService()
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Database operations."""
|
||||
|
||||
from typing import Optional, List, Dict
|
||||
from models import User, Task, Project
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self):
|
||||
self.users: Dict[str, User] = {}
|
||||
self.tasks: Dict[str, Task] = {}
|
||||
self.projects: Dict[str, Project] = {}
|
||||
|
||||
def get_user(self, user_id: str) -> Optional[User]:
|
||||
return self.users.get(user_id)
|
||||
|
||||
def get_user_by_email(self, email: str) -> Optional[User]:
|
||||
for user in self.users.values():
|
||||
if user.email == email:
|
||||
return user
|
||||
return None
|
||||
|
||||
def create_user(self, user: User) -> None:
|
||||
self.users[user.id] = user
|
||||
|
||||
def get_task(self, task_id: str) -> Optional[Task]:
|
||||
return self.tasks.get(task_id)
|
||||
|
||||
def get_user_tasks(self, user_id: str) -> List[Task]:
|
||||
return [t for t in self.tasks.values() if t.user_id == user_id]
|
||||
|
||||
def create_task(self, task: Task) -> None:
|
||||
self.tasks[task.id] = task
|
||||
|
||||
def update_task(self, task_id: str, **updates) -> Optional[Task]:
|
||||
task = self.tasks.get(task_id)
|
||||
if task:
|
||||
for key, value in updates.items():
|
||||
setattr(task, key, value)
|
||||
return task
|
||||
|
||||
def delete_task(self, task_id: str) -> bool:
|
||||
if task_id in self.tasks:
|
||||
del self.tasks[task_id]
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_project(self, project_id: str) -> Optional[Project]:
|
||||
return self.projects.get(project_id)
|
||||
|
||||
def create_project(self, project: Project) -> None:
|
||||
self.projects[project.id] = project
|
||||
|
||||
|
||||
db = Database()
|
||||
@@ -0,0 +1,305 @@
|
||||
/**
|
||||
* Ground truth definitions for the Python task management fixture
|
||||
*/
|
||||
|
||||
import { FixtureGroundTruth } from '../../types';
|
||||
|
||||
export const pythonFixture: FixtureGroundTruth = {
|
||||
name: 'python-taskmanager',
|
||||
path: '__tests__/evaluation/fixtures/python-project',
|
||||
language: 'python',
|
||||
totalFiles: 5,
|
||||
approximateTokens: 1200, // Rough estimate
|
||||
|
||||
testCases: [
|
||||
// =========================================================================
|
||||
// Search Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-search-auth',
|
||||
description: 'Search for authentication functionality',
|
||||
query: 'authentication login',
|
||||
type: 'search',
|
||||
expectedSymbols: ['AuthService', 'AuthService.login', 'AuthService.register', 'verify_password'],
|
||||
irrelevantSymbols: ['TaskService', 'validate_task_title', 'Project'],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
{
|
||||
id: 'py-search-task',
|
||||
description: 'Search for task management',
|
||||
query: 'task create complete',
|
||||
type: 'search',
|
||||
expectedSymbols: ['TaskService', 'TaskService.create_task', 'TaskService.complete_task', 'Task'],
|
||||
irrelevantSymbols: ['AuthService', 'validate_email', 'hash_password'],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
{
|
||||
id: 'py-search-validation',
|
||||
description: 'Search for validation',
|
||||
query: 'validate',
|
||||
type: 'search',
|
||||
expectedSymbols: ['validate_email', 'validate_password', 'validate_task_title'],
|
||||
irrelevantSymbols: ['hash_password', 'generate_token', 'TaskService'],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Context Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-context-login-bug',
|
||||
description: 'Build context for fixing login issues',
|
||||
query: 'debug why users cannot log in',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'AuthService.login',
|
||||
'verify_password',
|
||||
'db.get_user_by_email',
|
||||
'User',
|
||||
'hash_password',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'TaskService',
|
||||
'validate_task_title',
|
||||
'Project',
|
||||
'Task',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
{
|
||||
id: 'py-context-task-creation',
|
||||
description: 'Build context for task creation flow',
|
||||
query: 'understand how tasks are created',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'TaskService.create_task',
|
||||
'validate_task_title',
|
||||
'auth_service.get_user_id',
|
||||
'db.create_task',
|
||||
'Task',
|
||||
'generate_token',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
'AuthService.register',
|
||||
'Project',
|
||||
],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
{
|
||||
id: 'py-context-user-registration',
|
||||
description: 'Build context for user registration',
|
||||
query: 'add email confirmation to registration',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
'validate_email',
|
||||
'validate_password',
|
||||
'hash_password',
|
||||
'db.create_user',
|
||||
'User',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'TaskService',
|
||||
'validate_task_title',
|
||||
'Task',
|
||||
'Project',
|
||||
],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Callers Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-callers-get_user_id',
|
||||
description: 'Find all callers of auth_service.get_user_id',
|
||||
query: 'get_user_id',
|
||||
type: 'callers',
|
||||
targetSymbol: 'get_user_id',
|
||||
expectedSymbols: [
|
||||
'TaskService.create_task',
|
||||
'TaskService.get_task',
|
||||
'TaskService.get_user_tasks',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'AuthService.login',
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'py-callers-validate_email',
|
||||
description: 'Find all callers of validate_email',
|
||||
query: 'validate_email',
|
||||
type: 'callers',
|
||||
targetSymbol: 'validate_email',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'TaskService',
|
||||
'validate_password',
|
||||
'hash_password',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'py-callers-generate_token',
|
||||
description: 'Find all callers of generate_token',
|
||||
query: 'generate_token',
|
||||
type: 'callers',
|
||||
targetSymbol: 'generate_token',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
'AuthService.login',
|
||||
'TaskService.create_task',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'validate_password',
|
||||
'db.get_user',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Callees Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-callees-login',
|
||||
description: 'Find what AuthService.login calls',
|
||||
query: 'login',
|
||||
type: 'callees',
|
||||
targetSymbol: 'login',
|
||||
expectedSymbols: [
|
||||
'db.get_user_by_email',
|
||||
'verify_password',
|
||||
'generate_token',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
'validate_task_title',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'py-callees-create_task',
|
||||
description: 'Find what TaskService.create_task calls',
|
||||
query: 'create_task',
|
||||
type: 'callees',
|
||||
targetSymbol: 'TaskService.create_task',
|
||||
expectedSymbols: [
|
||||
'auth_service.get_user_id',
|
||||
'validate_task_title',
|
||||
'generate_token',
|
||||
'db.create_task',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
'db.get_user',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.8,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Impact Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-impact-generate_token',
|
||||
description: 'Impact of changing generate_token',
|
||||
query: 'generate_token',
|
||||
type: 'impact',
|
||||
targetSymbol: 'generate_token',
|
||||
expectedSymbols: [
|
||||
// Direct callers
|
||||
'AuthService.register',
|
||||
'AuthService.login',
|
||||
'TaskService.create_task',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'validate_task_title',
|
||||
'db.get_project',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.7,
|
||||
},
|
||||
{
|
||||
id: 'py-impact-get_user_id',
|
||||
description: 'Impact of changing get_user_id',
|
||||
query: 'get_user_id',
|
||||
type: 'impact',
|
||||
targetSymbol: 'get_user_id',
|
||||
expectedSymbols: [
|
||||
'TaskService.create_task',
|
||||
'TaskService.get_task',
|
||||
'TaskService.get_user_tasks',
|
||||
'TaskService.complete_task',
|
||||
'TaskService.delete_task',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'AuthService.register',
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.7,
|
||||
},
|
||||
],
|
||||
|
||||
// Known call graph edges for validation
|
||||
callGraph: [
|
||||
// Auth -> Database
|
||||
{ caller: 'AuthService.register', callee: 'db.get_user_by_email' },
|
||||
{ caller: 'AuthService.register', callee: 'db.create_user' },
|
||||
{ caller: 'AuthService.login', callee: 'db.get_user_by_email' },
|
||||
|
||||
// Auth -> Crypto
|
||||
{ caller: 'AuthService.register', callee: 'hash_password' },
|
||||
{ caller: 'AuthService.register', callee: 'generate_token' },
|
||||
{ caller: 'AuthService.login', callee: 'verify_password' },
|
||||
{ caller: 'AuthService.login', callee: 'generate_token' },
|
||||
|
||||
// Auth -> Validation
|
||||
{ caller: 'AuthService.register', callee: 'validate_email' },
|
||||
{ caller: 'AuthService.register', callee: 'validate_password' },
|
||||
|
||||
// Task -> Auth
|
||||
{ caller: 'TaskService.create_task', callee: 'auth_service.get_user_id' },
|
||||
{ caller: 'TaskService.get_task', callee: 'auth_service.get_user_id' },
|
||||
{ caller: 'TaskService.get_user_tasks', callee: 'auth_service.get_user_id' },
|
||||
|
||||
// Task -> Database
|
||||
{ caller: 'TaskService.create_task', callee: 'db.create_task' },
|
||||
{ caller: 'TaskService.get_task', callee: 'db.get_task' },
|
||||
{ caller: 'TaskService.get_user_tasks', callee: 'db.get_user_tasks' },
|
||||
{ caller: 'TaskService.complete_task', callee: 'db.update_task' },
|
||||
{ caller: 'TaskService.delete_task', callee: 'db.delete_task' },
|
||||
|
||||
// Task -> Crypto
|
||||
{ caller: 'TaskService.create_task', callee: 'generate_token' },
|
||||
|
||||
// Task -> Validation
|
||||
{ caller: 'TaskService.create_task', callee: 'validate_task_title' },
|
||||
|
||||
// Task -> Task (internal)
|
||||
{ caller: 'TaskService.complete_task', callee: 'TaskService.get_task' },
|
||||
{ caller: 'TaskService.delete_task', callee: 'TaskService.get_task' },
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Data models for the application."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
|
||||
|
||||
@dataclass
|
||||
class User:
|
||||
id: str
|
||||
email: str
|
||||
name: str
|
||||
password_hash: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class Task:
|
||||
id: str
|
||||
user_id: str
|
||||
title: str
|
||||
description: Optional[str]
|
||||
completed: bool
|
||||
created_at: datetime
|
||||
completed_at: Optional[datetime] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Project:
|
||||
id: str
|
||||
user_id: str
|
||||
name: str
|
||||
tasks: List[str] # Task IDs
|
||||
created_at: datetime
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Task management service."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
|
||||
from models import Task
|
||||
from database import db
|
||||
from auth import auth_service, generate_token
|
||||
from validation import validate_task_title
|
||||
|
||||
|
||||
class TaskService:
|
||||
def create_task(
|
||||
self, token: str, title: str, description: Optional[str] = None
|
||||
) -> Optional[Task]:
|
||||
"""Create a new task."""
|
||||
user_id = auth_service.get_user_id(token)
|
||||
if not user_id:
|
||||
return None
|
||||
|
||||
if not validate_task_title(title):
|
||||
return None
|
||||
|
||||
task = Task(
|
||||
id=generate_token(),
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
description=description,
|
||||
completed=False,
|
||||
created_at=datetime.now(),
|
||||
)
|
||||
db.create_task(task)
|
||||
return task
|
||||
|
||||
def get_task(self, token: str, task_id: str) -> Optional[Task]:
|
||||
"""Get a task by ID."""
|
||||
user_id = auth_service.get_user_id(token)
|
||||
if not user_id:
|
||||
return None
|
||||
|
||||
task = db.get_task(task_id)
|
||||
if task and task.user_id == user_id:
|
||||
return task
|
||||
return None
|
||||
|
||||
def get_user_tasks(self, token: str) -> List[Task]:
|
||||
"""Get all tasks for the authenticated user."""
|
||||
user_id = auth_service.get_user_id(token)
|
||||
if not user_id:
|
||||
return []
|
||||
|
||||
return db.get_user_tasks(user_id)
|
||||
|
||||
def complete_task(self, token: str, task_id: str) -> bool:
|
||||
"""Mark a task as completed."""
|
||||
task = self.get_task(token, task_id)
|
||||
if not task:
|
||||
return False
|
||||
|
||||
db.update_task(task_id, completed=True, completed_at=datetime.now())
|
||||
return True
|
||||
|
||||
def delete_task(self, token: str, task_id: str) -> bool:
|
||||
"""Delete a task."""
|
||||
task = self.get_task(token, task_id)
|
||||
if not task:
|
||||
return False
|
||||
|
||||
return db.delete_task(task_id)
|
||||
|
||||
|
||||
task_service = TaskService()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Validation utilities."""
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def validate_email(email: str) -> bool:
|
||||
"""Validate email format."""
|
||||
pattern = r'^[^\s@]+@[^\s@]+\.[^\s@]+$'
|
||||
return bool(re.match(pattern, email))
|
||||
|
||||
|
||||
def validate_password(password: str) -> bool:
|
||||
"""Validate password strength."""
|
||||
if len(password) < 8:
|
||||
return False
|
||||
if not re.search(r'[A-Z]', password):
|
||||
return False
|
||||
if not re.search(r'[a-z]', password):
|
||||
return False
|
||||
if not re.search(r'[0-9]', password):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def validate_task_title(title: str) -> bool:
|
||||
"""Validate task title."""
|
||||
return bool(title and len(title.strip()) >= 1 and len(title) <= 200)
|
||||
Reference in new issue
Block a user