fix(explore): accurately resolve query file paths and find camelCase symbols

Previously, `codegraph_explore` queries explicitly naming files by path (e.g., `src/routes/m/projects/[id]/runs/[runId]/+page.svelte`) were shredded. Bracketed path segments exploded into "named symbol" seeds, and FTS on fragments like `page` or `runs` admitted every sibling file, starving the user's intended target.

This change introduces:
- **Query path pinning:** File paths named in a query are now resolved against the index, "pinned," and stripped from the query. Pinned files are guaranteed inclusion, top ranking, and fair allocation. Unresolvable path-like spans are reported.
- **Segment vocabulary supplement:** Natural language query terms (e.g., "auto-scroll to bottom") can now reach camelCase identifiers (e.g., `pinFeedIfNearBottom`, `feedAtBottom`) by matching against their constituent segments.
- **Variable seeding:** `variable` and `constant` node kinds are now included in identifier seeding, improving recall for `$state`-style variables common in frameworks like Svelte.
This commit is contained in:
Colby McHenry committed 2026-08-20 09:10:07 -07:00
1 parent c6aaa20358
commit 238dbc5cec
15 files changed
+905 -31

No files matched your search

+105
View File
@@ -0,0 +1,105 @@
/**
* End-to-end gate for query-path pinning + the segment-vocab supplement +
* variable seeding, on the bug that motivated all three: an agent named a
* SvelteKit route file by exact path plus behavior words ("scrollToBottom,
* onscroll, atBottom tracking") and got back neither the file's scroll code
* nor the file itself at full weight — the bracketed path was tokenizer
* shrapnel (`runId` seeded as a named symbol, every sibling `+page` admitted)
* and the camelCase scroll symbols were FTS-opaque.
*
* The fixture mirrors that shape in plain TS (bracket/paren directories are
* the crux, not the language): a target file under
* `src/routes/m/projects/[id]/runs/[runId]/` holding `feedAtBottom` /
* `handleFeedScroll` / `pinFeedIfNearBottom`, a decoy chat-window page under
* a `(protected)` route group, and a runs-store decoy defining `runId` and
* `Scope` — the two symbols that headlined the original junk blast radius.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import CodeGraph from '../src/index';
import { ToolHandler } from '../src/mcp/tools';
const FIXTURE = 'explore-path-pinning';
const TARGET = 'src/routes/m/projects/[id]/runs/[runId]/+page.ts';
const DECOY_CHAT = 'src/routes/(protected)/chat-window/+page.ts';
let dir: string;
let cg: CodeGraph;
async function explore(query: string): Promise<string> {
const res = await new ToolHandler(cg).execute('codegraph_explore', { query });
return res.content?.[0]?.text ?? '';
}
/** The response renders a source section for `file`. */
const hasSection = (response: string, file: string): boolean =>
response.includes('**`' + file + '`');
beforeAll(async () => {
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-path-pin-'));
fs.cpSync(path.join(__dirname, 'fixtures', FIXTURE), dir, { recursive: true });
fs.rmSync(path.join(dir, '.codegraph'), { recursive: true, force: true });
cg = CodeGraph.initSync(dir);
await cg.indexAll();
}, 180_000);
afterAll(() => {
cg?.destroy();
if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true });
});
describe('fixture shape — if this rots, the gates below mean nothing', () => {
it('indexes the bracketed-path target with its scroll symbols', () => {
const names = cg.getNodesInFile(TARGET).map((n) => n.name);
expect(names).toContain('feedAtBottom');
expect(names).toContain('handleFeedScroll');
expect(names).toContain('pinFeedIfNearBottom');
});
});
describe('path pinning (fix 1)', () => {
it('a pure-path query renders the named file and says it was pinned', async () => {
const out = await explore(TARGET);
expect(hasSection(out, TARGET)).toBe(true);
expect(out).toContain('pinned from the query');
});
it('the original bug-shaped query renders the pinned file, not path shrapnel', async () => {
const out = await explore(
`run page auto-scroll to bottom logic in ${TARGET} — scrollToBottom, onscroll, atBottom tracking`,
);
expect(hasSection(out, TARGET)).toBe(true);
// The path fragments must not seed: `runId` (runs-store decoy) and the
// bracketed segment's namesakes headlined the original junk blast radius.
const blast = out.split('**Relationships**')[0]!;
expect(blast).not.toMatch(/`runId` \(src\/lib\/runs-store\.ts/);
// The chat decoy MAY render — it genuinely holds scroll-pinning code the
// segment supplement now finds — but the pinned file must rank first.
// (Pre-fix, `+page`/`runs` shrapnel admitted the siblings ABOVE the named
// file and the envelope truncated it.)
const decoyAt = out.indexOf('**`' + DECOY_CHAT + '`');
const targetAt = out.indexOf('**`' + TARGET + '`');
expect(targetAt).toBeGreaterThan(-1);
if (decoyAt !== -1) expect(targetAt).toBeLessThan(decoyAt);
});
it('an unresolvable path is reported, not silently dropped', async () => {
const out = await explore('crash in src/routes/gone/missing-page.ts on load');
expect(out).toContain('No indexed file uniquely matches');
expect(out).toContain('src/routes/gone/missing-page.ts');
});
});
describe('segment supplement + variable seeding (fixes 2–3)', () => {
it('word-level scroll terms reach the camelCase scroll code without a path', async () => {
const out = await explore('feed auto-scroll to bottom pinning behavior');
expect(hasSection(out, TARGET)).toBe(true);
});
it('a camel infix naming only $state-style variables still finds their file', async () => {
const out = await explore('where does the atBottom flag get reset');
expect(hasSection(out, TARGET)).toBe(true);
});
});
@@ -0,0 +1,83 @@
/**
* Pinned files in `allocateExploreBudget` (see query-paths.ts): a file the
* query named by PATH must survive every allocation guard. Its score is
* whatever the path-stripped query happened to match — for a pure-path query,
* nearly nothing — so without the pinned floor the proportional split would
* fund the one file the agent explicitly asked for worst of all, and the
* cliff would zero it outright.
*/
import { describe, it, expect } from 'vitest';
import { allocateExploreBudget, getExploreOutputBudget, EXPLORE_ALLOCATION } from '../src/mcp/tools';
import type { ExploreAllocationCandidate } from '../src/mcp/tools';
const cand = (
path: string,
score: number,
extra: Partial<ExploreAllocationCandidate> = {},
): ExploreAllocationCandidate => ({ path, score, worth: 1, spine: false, ...extra });
const budget = getExploreOutputBudget(1000);
describe('allocateExploreBudget — pinned files', () => {
it('never cliffs a pinned file, however low it scores', () => {
const { allowances, cliffed } = allocateExploreBudget(
[
cand('pinned.svelte', 0.1, { pinned: true }),
cand('hub.ts', 200),
cand('noise.ts', 0.1),
],
budget,
8,
);
expect(cliffed).toContain('noise.ts');
expect(cliffed).not.toContain('pinned.svelte');
expect(allowances.has('pinned.svelte')).toBe(true);
});
it('funds a pinned file at least as well as the strongest candidate', () => {
const { allowances } = allocateExploreBudget(
[
cand('pinned.svelte', 0.5, { pinned: true }),
cand('hub.ts', 300),
cand('helper.ts', 40),
],
budget,
8,
);
expect(allowances.get('pinned.svelte')!).toBeGreaterThanOrEqual(allowances.get('hub.ts')!);
expect(allowances.get('pinned.svelte')!).toBeGreaterThan(allowances.get('helper.ts')!);
});
it('keeps pinned files through the affordability trim', () => {
// Smallest tier: affordable = floor(13000 / (MIN_CHARS + FILE_OVERHEAD)) = 14
// slots. 18 equal-weight candidates admitted → the trim must cut 4. The
// pinned file sits last with a TIED weight (the pinned floor lifts it to
// the top weight), so the stable by-weight sort would slice it off — only
// the explicit spine/pinned keep saves it.
const tiny = getExploreOutputBudget(10);
const fleet = Array.from({ length: 17 }, (_, i) => cand(`f${i}.ts`, 100));
fleet.push(cand('pinned.svelte', 0.1, { pinned: true }));
const { allowances, cliffed } = allocateExploreBudget(fleet, tiny, 18);
expect(allowances.has('pinned.svelte')).toBe(true);
expect(cliffed).not.toContain('pinned.svelte');
expect(allowances.size).toBeLessThan(18);
});
it('an all-pinned zero-score call still allocates (pure-path query)', () => {
const { allowances, pool } = allocateExploreBudget(
[cand('a.svelte', 0, { pinned: true }), cand('b.svelte', 0, { pinned: true })],
budget,
8,
);
expect(pool).toBeGreaterThan(0);
expect(allowances.get('a.svelte')!).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS);
expect(allowances.get('b.svelte')!).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS);
});
it('unpinned behavior is unchanged when no candidate is pinned', () => {
const before = allocateExploreBudget(
[cand('a.ts', 40), cand('b.ts', 10)], budget, 8,
);
expect(before.allowances.get('a.ts')!).toBeGreaterThan(before.allowances.get('b.ts')!);
});
});
@@ -0,0 +1,5 @@
{
"name": "explore-path-pinning-fixture",
"version": "1.0.0",
"private": true
}
@@ -0,0 +1,31 @@
/** In-memory registry of task runs, keyed by run id. */
export interface Scope {
projectId: string;
label: string;
}
export const runId = 'run-000';
const runs = new Map<string, { id: string; scope: Scope; status: string }>();
export function registerRun(id: string, scope: Scope): void {
runs.set(id, { id, scope, status: 'queued' });
}
export function getRun(id: string): { id: string; scope: Scope; status: string } | null {
return runs.get(id) ?? null;
}
export function listRuns(scope: Scope): string[] {
return [...runs.values()]
.filter((r) => r.scope.projectId === scope.projectId)
.map((r) => r.id);
}
export function stopRun(id: string): boolean {
const run = runs.get(id);
if (!run) return false;
run.status = 'cancelled';
return true;
}
@@ -0,0 +1,28 @@
/** Detached chat window page — session presence + streaming state. */
let chatAtBottom = true;
let isStreaming = false;
let messages: string[] = [];
export function handleMessagesScroll(distance: number): void {
chatAtBottom = distance < 50;
}
export function sendMessage(text: string): void {
messages = [...messages, text];
isStreaming = true;
}
export function stopResponse(): void {
isStreaming = false;
}
export function redock(): void {
messages = [];
isStreaming = false;
chatAtBottom = true;
}
export function chatSnapshot(): { messages: string[]; streaming: boolean } {
return { messages: [...messages], streaming: isStreaming };
}
@@ -0,0 +1,67 @@
/** Mobile run feed — event stream + scroll pinning for the run page. */
export interface FeedEvent {
id: string;
kind: 'output' | 'tool' | 'error';
content: string;
}
const EVENT_CAP = 300;
let events: FeedEvent[] = [];
let workingLine: string | null = null;
/** Whether the reader is at the tail of the feed (within 50px). */
let feedAtBottom = true;
interface FeedElement {
scrollTop: number;
scrollHeight: number;
clientHeight: number;
}
let feedEl: FeedElement | null = null;
export function bindFeedElement(el: FeedElement | null): void {
feedEl = el;
}
/** Track the reader's position; called from the feed's scroll listener. */
export function handleFeedScroll(): void {
const el = feedEl;
if (!el) return;
feedAtBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 50;
}
/** Re-pin after async content growth (image loads), only when at the tail. */
export function pinFeedIfNearBottom(): void {
const el = feedEl;
if (!el) return;
if (feedAtBottom) {
el.scrollTop = el.scrollHeight;
}
}
export function appendEvent(event: FeedEvent): void {
events = events.length >= EVENT_CAP
? [...events.slice(-(EVENT_CAP - 1)), event]
: [...events, event];
if (feedAtBottom) {
pinFeedIfNearBottom();
}
}
export function setWorkingLine(line: string | null): void {
workingLine = line;
pinFeedIfNearBottom();
}
export function resetFeed(): void {
events = [];
workingLine = null;
feedAtBottom = true;
}
export function feedSnapshot(): { events: FeedEvent[]; workingLine: string | null } {
return { events: [...events], workingLine };
}
+30
View File
@@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest';
import {
splitIdentifierSegments,
extractProseCandidates,
extractSegmentSearchWords,
normalizeProseWord,
segmentLookupVariants,
} from '../src/search/identifier-segments';
@@ -101,3 +102,32 @@ describe('segmentLookupVariants — light plural folding', () => {
expect(segmentLookupVariants('boxes')).toEqual(['boxes']); // -es strip would go sub-minimum
});
});
describe('extractSegmentSearchWords — query words for the search-side vocab supplement', () => {
it('keeps prose words and adds camel-token segments', () => {
const words = extractSegmentSearchWords('auto-scroll to bottom — atBottom tracking');
// Prose candidates survive as before…
expect(words).toContain('scroll');
expect(words).toContain('bottom');
expect(words).toContain('tracking');
// …and the camel token contributed its ≥4-char segments ("at" is under
// the prose minimum; "bottom" arrives from the split even when the prose
// pass missed it).
expect(extractSegmentSearchWords('where is atBottom set')).toContain('bottom');
});
it('splits multi-hump tokens into every usable segment', () => {
const words = extractSegmentSearchWords('trace pinFeedIfNearBottom please');
expect(words).toEqual(expect.arrayContaining(['feed', 'near', 'bottom']));
});
it('does not invent segments for plain prose', () => {
const words = extractSegmentSearchWords('how does checkout work');
expect(words).toContain('checkout');
expect(words).not.toContain('check');
});
it('returns nothing for an empty query', () => {
expect(extractSegmentSearchWords('')).toEqual([]);
});
});
+129
View File
@@ -0,0 +1,129 @@
/**
* File-path recognition in explore queries (src/search/query-paths.ts).
*
* The originating bug: an agent named two SvelteKit route files by exact path
* (`src/routes/m/projects/[id]/runs/[runId]/+page.svelte`) and the explore
* pipeline shredded them — the seeding tokenizer splits on brackets, so the
* fragments `runId`/`scope` seeded as "named symbols" and headlined the blast
* radius, while FTS admitted every sibling `+page.svelte` off the `page`/`runs`
* fragments. These tests pin the module that stops that: path spans resolve
* against the indexed file list, matching files pin, and the spans leave the
* query. Resolution IS the detector — slash-bearing non-paths stay untouched.
*/
import { describe, it, expect } from 'vitest';
import { extractQueryPaths, queryMightContainPaths } from '../src/search/query-paths';
const INDEX = [
'src/routes/m/projects/[id]/runs/[runId]/+page.svelte',
'src/routes/m/projects/[id]/chat/[scope]/+page.svelte',
'src/routes/m/projects/[id]/+page.svelte',
'src/routes/(protected)/chat-window/+page.svelte',
'src/lib/chat-manager.ts',
'src/lib/task-runner-manager.ts',
'src/lib/stores/sqlite-store.ts',
'src/lib/stores/postgresql-store.ts',
];
describe('queryMightContainPaths — the cheap pre-gate', () => {
it('fires on slashes and dotted basenames', () => {
expect(queryMightContainPaths('look at src/lib/chat-manager.ts')).toBe(true);
expect(queryMightContainPaths('look at chat-manager.ts please')).toBe(true);
});
it('stays quiet on plain prose and Class.method spans', () => {
expect(queryMightContainPaths('how does the scroll pinning work')).toBe(false);
// `.isPackaged` is 10 chars — past the 8-char extension cap.
expect(queryMightContainPaths('what reads app.isPackaged here')).toBe(false);
});
});
describe('extractQueryPaths — resolution and stripping', () => {
it('resolves a bracketed SvelteKit path and strips it from the query', () => {
const q = 'auto-scroll logic in src/routes/m/projects/[id]/runs/[runId]/+page.svelte — atBottom tracking';
const out = extractQueryPaths(q, INDEX);
expect(out.pinnedFiles).toEqual(['src/routes/m/projects/[id]/runs/[runId]/+page.svelte']);
expect(out.strippedQuery).not.toContain('+page.svelte');
expect(out.strippedQuery).not.toContain('runId');
expect(out.strippedQuery).toContain('atBottom tracking');
expect(out.unresolvedPathSpans).toEqual([]);
});
it('pins multiple named files in appearance order', () => {
const q = 'compare src/routes/m/projects/[id]/chat/[scope]/+page.svelte and src/routes/m/projects/[id]/runs/[runId]/+page.svelte';
const out = extractQueryPaths(q, INDEX);
expect(out.pinnedFiles).toEqual([
'src/routes/m/projects/[id]/chat/[scope]/+page.svelte',
'src/routes/m/projects/[id]/runs/[runId]/+page.svelte',
]);
});
it('resolves a (protected) route-group path — parens are path characters', () => {
const out = extractQueryPaths('read src/routes/(protected)/chat-window/+page.svelte', INDEX);
expect(out.pinnedFiles).toEqual(['src/routes/(protected)/chat-window/+page.svelte']);
});
it('resolves an absolute path by walking suffixes to the indexed relative path', () => {
const q = 'fix /Users/colby/dev/beads-live-dashboard/src/lib/chat-manager.ts';
const out = extractQueryPaths(q, INDEX);
expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']);
});
it('resolves a unique basename and a partial path', () => {
expect(extractQueryPaths('see chat-manager.ts', INDEX).pinnedFiles)
.toEqual(['src/lib/chat-manager.ts']);
expect(extractQueryPaths('see stores/sqlite-store.ts', INDEX).pinnedFiles)
.toEqual(['src/lib/stores/sqlite-store.ts']);
});
it('strips wrapping punctuation and line references', () => {
const out = extractQueryPaths('the bug (see `src/lib/chat-manager.ts:243`).', INDEX);
expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']);
const hash = extractQueryPaths('regression at src/lib/task-runner-manager.ts#L88-L120', INDEX);
expect(hash.pinnedFiles).toEqual(['src/lib/task-runner-manager.ts']);
});
it('treats an over-ambiguous basename as unresolved — stripped and reported', () => {
const out = extractQueryPaths('why do all +page.svelte files flash', INDEX);
expect(out.pinnedFiles).toEqual([]);
expect(out.unresolvedPathSpans).toEqual(['+page.svelte']);
expect(out.strippedQuery).toBe('why do all files flash');
});
it('strips and reports a clearly-path-shaped span that matches nothing', () => {
const out = extractQueryPaths('crash in src/routes/gone/missing-page.svelte on load', INDEX);
expect(out.pinnedFiles).toEqual([]);
expect(out.unresolvedPathSpans).toEqual(['src/routes/gone/missing-page.svelte']);
expect(out.strippedQuery).toBe('crash in on load');
});
it('leaves slash-bearing non-paths alone', () => {
const q = 'does gen_server:call/2 block and/or timeout';
const out = extractQueryPaths(q, INDEX);
expect(out.pinnedFiles).toEqual([]);
expect(out.unresolvedPathSpans).toEqual([]);
expect(out.strippedQuery).toBe(q);
});
it('dedupes a path named twice and honors maxPins', () => {
const twice = extractQueryPaths(
'src/lib/chat-manager.ts wraps src/lib/chat-manager.ts', INDEX,
);
expect(twice.pinnedFiles).toEqual(['src/lib/chat-manager.ts']);
const capped = extractQueryPaths(
'src/lib/chat-manager.ts src/lib/task-runner-manager.ts', INDEX, { maxPins: 1 },
);
expect(capped.pinnedFiles).toEqual(['src/lib/chat-manager.ts']);
});
it('matches case-insensitively but returns the indexed spelling', () => {
const out = extractQueryPaths('SRC/LIB/CHAT-MANAGER.TS', INDEX);
expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']);
});
it('passes through untouched when nothing resolves', () => {
const q = 'plain prose question about scrolling';
const out = extractQueryPaths(q, INDEX);
expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [] });
});
});