feat(ui): classify code from the engine's own tree-sitter parse, retiring Shiki (CG-57)
The viewer ran a second highlighter over source the engine had already parsed
with a real grammar: Shiki, plus 56 pruned TextMate grammars shipped in
dist/textmate/. The classification now comes off that tree instead, so a file is
read by exactly the grammar that decided what its symbols are.
The swap is complete rather than flagged: @shikijs/core, @shikijs/engine-javascript
and @shikijs/langs are off the dependency list, scripts/prune-grammars.mjs and
`npm run build:textmate` are deleted, and check-ui-build.mjs asserts the
tree-sitter grammars in dist/extraction/wasm instead of dist/textmate.
The wire contract is unchanged — `[classId, text]` pairs with the class names
alongside — so the viewer's decoder and code blocks did not have to be rewritten.
Two classes are added to the six: `type` (a named type reference, painted at
plain ink) and `def` (the name a definition declares, weight 600), the latter
taken from the extractors' own definition tables so it cannot drift from what
indexing calls a definition.
Three differences are not cosmetic:
* Interpolations (`${…}`, `#{…}`, `$"{…}"`, f-strings) are classified as code,
not as string. The call-site overlay refuses to claim a token classed string,
so calls written inside interpolated strings now link.
* Built-in type words are emitted whole and classed `type` in every language.
The grammars disagree about whether `string` is a type_identifier or an
anonymous token inside a predefined_type, and TextMate scoped them
inconsistently too.
* 3 000 lines of TypeScript cost 24-41 ms instead of ~700 ms.
Given up deliberately: Liquid, Razor, YAML, Twig, XML and .properties render
plain. .svelte/.vue/.astro are classified through their <script> blocks, the same
delegation the SFC extractors do. Pulling html/css/vue out of tree-sitter-wasms
would cover them, but those ABI-13 builds are the known cause of shared-WASM-heap
corruption for every other language in the same process.
Measured parity, per-language before/after screenshots and the reproduction
recipe: docs/design/cg57-highlighting-parity.md.
This commit is contained in:
Colby McHenry
@@ -561,6 +561,19 @@ function looksLikeObjc(source: string): boolean {
|
||||
return /@(?:interface|implementation|protocol|synthesize)\b/.test(sample);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a language has a tree-sitter grammar of its own.
|
||||
*
|
||||
* Narrower than {@link isLanguageSupported}, which also answers true for the
|
||||
* formats handled by custom extractors (SFCs, Liquid, Razor, YAML, XML,
|
||||
* properties) — those have extraction but no grammar, so anything that needs to
|
||||
* PARSE the file (the viewer's syntax classification, for one) has to ask this
|
||||
* instead.
|
||||
*/
|
||||
export function hasTreeSitterGrammar(language: string | undefined | null): boolean {
|
||||
return !!language && language in WASM_GRAMMAR_FILES;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a language is supported (has a grammar defined).
|
||||
* Returns true if the grammar exists, even if not yet loaded.
|
||||
|
||||
@@ -0,0 +1,465 @@
|
||||
/**
|
||||
* Syntax classification from the engine's own tree-sitter parse (CG-57).
|
||||
*
|
||||
* The viewer used to run a second highlighter (Shiki + 56 pruned TextMate
|
||||
* grammars) over source the engine had already parsed with a real grammar. This
|
||||
* takes the classification off the tree instead, which removes the second
|
||||
* dependency, the second grammar set, and — the part that actually mattered —
|
||||
* the second opinion: a `.ts` file is now read by exactly the grammar that
|
||||
* decided what its symbols are.
|
||||
*
|
||||
* ## What comes out
|
||||
*
|
||||
* A flat, ordered, non-overlapping list of {@link SyntaxSpan}s over the source
|
||||
* string. Gaps between spans are whitespace and are the caller's to fill. The
|
||||
* classes are deliberately few, because the design's code colouring is
|
||||
* near-monochrome: comments recede, strings and numbers recede one step less,
|
||||
* keywords carry weight rather than hue, and the only colour in the body is a
|
||||
* call site the graph resolved.
|
||||
*
|
||||
* ## How a node becomes a class
|
||||
*
|
||||
* The rules are language-agnostic on purpose — the engine indexes 40-odd
|
||||
* languages and a per-grammar scope table would be 40 tables to keep true:
|
||||
*
|
||||
* * a node whose type mentions `comment` is a comment, whole, undescended;
|
||||
* * inside a string node every leaf is string, *except* below an interpolation,
|
||||
* where the code starts again (so `${user.name()}` still links);
|
||||
* * a numeric literal node is a number;
|
||||
* * an **anonymous** leaf is a keyword when its text is a bare word and
|
||||
* punctuation otherwise — this is what makes `func`, `fn`, `def`, `END-IF`
|
||||
* and `Sub` all land as keywords without naming any of them;
|
||||
* * a **named** leaf whose text is identifier-shaped is an identifier, unless
|
||||
* the grammar called it a type name, or the extractor's own definition tables
|
||||
* say it is the name of a definition.
|
||||
*
|
||||
* The last of those is the one place per-language knowledge is used, and it is
|
||||
* reused rather than restated: {@link EXTRACTORS} already names every node type
|
||||
* that declares something in each language, plus the field its name hangs on.
|
||||
*/
|
||||
|
||||
import type { Node as SyntaxNode } from 'web-tree-sitter';
|
||||
import { Language } from '../types';
|
||||
import { EXTRACTORS } from './languages';
|
||||
import { getParser, loadGrammarsForLanguages } from './grammars';
|
||||
import type { LanguageExtractor } from './tree-sitter-types';
|
||||
|
||||
/* ------------------------------------------------------------- the classes -- */
|
||||
|
||||
/**
|
||||
* Every class a token can carry, in wire order.
|
||||
*
|
||||
* `other` is punctuation and whitespace both. The design spec lists them apart
|
||||
* (`punct` vs the gaps) but they paint identically — plain ink — and splitting
|
||||
* them would roughly double the token count on a dense line to express a
|
||||
* difference nothing draws.
|
||||
*/
|
||||
export const SYNTAX_TOKEN_CLASSES = [
|
||||
'other',
|
||||
'ident',
|
||||
'comment',
|
||||
'string',
|
||||
'keyword',
|
||||
'number',
|
||||
'type',
|
||||
'def',
|
||||
] as const;
|
||||
|
||||
export type SyntaxTokenClass = (typeof SYNTAX_TOKEN_CLASSES)[number];
|
||||
|
||||
/** A classified run of the source, by JS string index. Half-open. */
|
||||
export interface SyntaxSpan {
|
||||
start: number;
|
||||
end: number;
|
||||
cls: SyntaxTokenClass;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------- node-type tests -- */
|
||||
|
||||
/**
|
||||
* Anything a grammar calls a comment.
|
||||
*
|
||||
* Substring rather than equality because the spelling is per-grammar:
|
||||
* `comment`, `line_comment`, `block_comment`, `doc_comment`, `html_comment`,
|
||||
* `comment_directive`, `preproc_comment`.
|
||||
*/
|
||||
function isCommentType(type: string): boolean {
|
||||
return type.includes('comment');
|
||||
}
|
||||
|
||||
/**
|
||||
* A node whose leaves are string content unless an interpolation interrupts.
|
||||
*
|
||||
* `string` covers the bulk (`string_literal`, `interpreted_string_literal`,
|
||||
* `raw_string_literal`, `encapsed_string`, `string_content`); the rest are the
|
||||
* spellings that avoid the word — Rust/Go/C character literals, shell and PHP
|
||||
* heredocs, and regular expressions, which recede for the same reason a string
|
||||
* does.
|
||||
*/
|
||||
function isStringType(type: string): boolean {
|
||||
return (
|
||||
type.includes('string') ||
|
||||
type.includes('heredoc') ||
|
||||
type.includes('regex') ||
|
||||
type === 'char_literal' ||
|
||||
type === 'character' ||
|
||||
type === 'character_literal' ||
|
||||
type === 'rune_literal' ||
|
||||
type === 'quoted_attribute_value'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Where code resumes inside a string.
|
||||
*
|
||||
* A template literal's `${…}` and an f-string's `{…}` hold real expressions,
|
||||
* and the graph records call sites inside them. Swallowing the whole literal as
|
||||
* one string token would drop those links — the overlay refuses to claim a
|
||||
* token classed `string`, deliberately, so that a word inside a message never
|
||||
* gets underlined.
|
||||
*/
|
||||
function isInterpolationType(type: string): boolean {
|
||||
return (
|
||||
type.includes('interpolation') ||
|
||||
type.includes('substitution') ||
|
||||
type === 'template_substitution' ||
|
||||
type === 'string_interpolation' ||
|
||||
type === 'format_expression'
|
||||
);
|
||||
}
|
||||
|
||||
/** A numeric literal, plus the language constants a theme groups with them. */
|
||||
function isNumberType(type: string): boolean {
|
||||
return (
|
||||
type === 'number' ||
|
||||
type === 'integer' ||
|
||||
type === 'float' ||
|
||||
type === 'number_literal' ||
|
||||
type === 'integer_literal' ||
|
||||
type === 'float_literal' ||
|
||||
type === 'decimal_integer_literal' ||
|
||||
type === 'decimal_floating_point_literal' ||
|
||||
type === 'hex_integer_literal' ||
|
||||
type === 'real_literal' ||
|
||||
type === 'numeric_literal' ||
|
||||
type === 'int_literal' ||
|
||||
type === 'imaginary_literal'
|
||||
);
|
||||
}
|
||||
|
||||
/** A named type reference — `type_identifier` and the equivalents. */
|
||||
function isTypeNameType(type: string): boolean {
|
||||
return type.includes('type_identifier') || type === 'type_name' || type === 'class_type';
|
||||
}
|
||||
|
||||
/**
|
||||
* Built-in type words — `string`, `int`, `u32`, `void`.
|
||||
*
|
||||
* These are emitted WHOLE and undescended, and they carry the same `type` class
|
||||
* a user-defined type name gets. Both halves of that matter, because the
|
||||
* grammars disagree with each other about what a built-in type even is:
|
||||
* tree-sitter-go calls `string` a `type_identifier` (so it would be a type),
|
||||
* tree-sitter-typescript wraps it in a `predefined_type` whose child is an
|
||||
* anonymous token spelled `string` (so it would be a keyword). Reading the
|
||||
* wrapper rather than its children is what stops the same word from painting
|
||||
* two different ways in two languages on the same screen.
|
||||
*/
|
||||
const BUILTIN_TYPE_TYPES: ReadonlySet<string> = new Set([
|
||||
'primitive_type',
|
||||
'predefined_type',
|
||||
'builtin_type',
|
||||
'sized_type_specifier',
|
||||
]);
|
||||
|
||||
/** Literal constants a theme groups with numbers (`constant.language`). */
|
||||
const CONSTANT_TYPES: ReadonlySet<string> = new Set([
|
||||
'true',
|
||||
'false',
|
||||
'null',
|
||||
'nil',
|
||||
'none',
|
||||
'undefined',
|
||||
'null_literal',
|
||||
'nil_literal',
|
||||
'boolean_literal',
|
||||
'true_literal',
|
||||
'false_literal',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Identifier-shaped text, in the loosest sense every indexed language agrees on.
|
||||
*
|
||||
* The high range is there because `\w` is ASCII-only in JavaScript and a symbol
|
||||
* name can be Chinese, Japanese or Cyrillic; a call site in those repositories
|
||||
* has to be linkable too. Hyphens are in because COBOL and Erlang spell words
|
||||
* with them (`END-IF`, `is_record`).
|
||||
*/
|
||||
const IDENT_SHAPE = /^[A-Za-z_$À-][\w$À--]*$/;
|
||||
|
||||
/** A bare word — what separates a keyword from punctuation among anonymous nodes. */
|
||||
const WORD_SHAPE = /^[A-Za-z_][A-Za-z_0-9-]*$/;
|
||||
|
||||
/* ------------------------------------------------------- definition names -- */
|
||||
|
||||
/**
|
||||
* Every node type that declares something, per language, from the extractors.
|
||||
*
|
||||
* This is the single piece of per-language knowledge the classifier uses, and
|
||||
* it is borrowed rather than restated: the same lists drive extraction, so a
|
||||
* language that learns a new declaration form gets its name bolded here for
|
||||
* free — and cannot drift, because there is only one list.
|
||||
*/
|
||||
function definitionTypesFor(extractor: LanguageExtractor): ReadonlySet<string> {
|
||||
return new Set([
|
||||
...extractor.functionTypes,
|
||||
...extractor.classTypes,
|
||||
...extractor.methodTypes,
|
||||
...extractor.interfaceTypes,
|
||||
...extractor.structTypes,
|
||||
...extractor.enumTypes,
|
||||
...extractor.typeAliasTypes,
|
||||
...(extractor.unionTypes ?? []),
|
||||
...(extractor.extraClassNodeTypes ?? []),
|
||||
]);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------- the walker -- */
|
||||
|
||||
interface WalkContext {
|
||||
source: string;
|
||||
out: SyntaxSpan[];
|
||||
defTypes: ReadonlySet<string>;
|
||||
nameField: string;
|
||||
/** Start indices of nodes that are a definition's own name. */
|
||||
defStarts: Set<number>;
|
||||
offset: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify one parsed tree into spans.
|
||||
*
|
||||
* Exported for tests and for anything that already holds a tree; the usual
|
||||
* entry point is {@link tokenizeSource}, which parses first.
|
||||
*/
|
||||
export function classifyTree(
|
||||
root: SyntaxNode,
|
||||
source: string,
|
||||
language: Language,
|
||||
offset = 0
|
||||
): SyntaxSpan[] {
|
||||
const extractor = EXTRACTORS[language];
|
||||
const ctx: WalkContext = {
|
||||
source,
|
||||
out: [],
|
||||
defTypes: extractor ? definitionTypesFor(extractor) : new Set<string>(),
|
||||
nameField: extractor?.nameField ?? 'name',
|
||||
defStarts: new Set<number>(),
|
||||
offset,
|
||||
};
|
||||
visit(root, ctx, false);
|
||||
return ctx.out;
|
||||
}
|
||||
|
||||
function visit(node: SyntaxNode, ctx: WalkContext, inString: boolean): void {
|
||||
const type = node.type;
|
||||
|
||||
if (node.isNamed && isCommentType(type)) {
|
||||
emit(ctx, node.startIndex, node.endIndex, 'comment');
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.isNamed && BUILTIN_TYPE_TYPES.has(type)) {
|
||||
emit(ctx, node.startIndex, node.endIndex, 'type');
|
||||
return;
|
||||
}
|
||||
|
||||
// Record the definition's own name BEFORE descending — the name node is a
|
||||
// descendant, so the mark has to be in place by the time the walk reaches it.
|
||||
if (ctx.defTypes.has(type)) {
|
||||
const name = node.childForFieldName(ctx.nameField);
|
||||
if (name) ctx.defStarts.add(name.startIndex);
|
||||
}
|
||||
|
||||
const childCount = node.childCount;
|
||||
if (childCount === 0) {
|
||||
emit(ctx, node.startIndex, node.endIndex, leafClass(node, ctx, inString));
|
||||
return;
|
||||
}
|
||||
|
||||
const nested = isInterpolationType(type) ? false : inString || isStringType(type);
|
||||
|
||||
for (let i = 0; i < childCount; i++) {
|
||||
const child = node.child(i);
|
||||
if (child) visit(child, ctx, nested);
|
||||
}
|
||||
}
|
||||
|
||||
function leafClass(node: SyntaxNode, ctx: WalkContext, inString: boolean): SyntaxTokenClass {
|
||||
const type = node.type;
|
||||
|
||||
// An ANONYMOUS node's `type` is its own literal text, so none of the
|
||||
// type-name tests below may be applied to one: `key: string` in TypeScript or
|
||||
// PHP is a token whose type is the word `string`, and reading that as a
|
||||
// string literal greys out half of every signature. Anonymous means keyword
|
||||
// or punctuation, decided on shape alone — which is also what makes `func`,
|
||||
// `fn`, `def`, `Sub` and `END-IF` all land right without naming any of them.
|
||||
if (!node.isNamed) {
|
||||
if (inString) return 'string';
|
||||
if (CONSTANT_TYPES.has(type)) return 'number';
|
||||
return WORD_SHAPE.test(type) ? 'keyword' : 'other';
|
||||
}
|
||||
|
||||
if (inString || isStringType(type)) return 'string';
|
||||
if (isNumberType(type) || CONSTANT_TYPES.has(type)) return 'number';
|
||||
|
||||
const text = ctx.source.slice(node.startIndex, node.endIndex);
|
||||
// Ahead of the type tests: a class name is a `type_identifier` in half these
|
||||
// grammars and a plain `identifier` in the other half, and the design bolds
|
||||
// the thing being DECLARED either way.
|
||||
if (ctx.defStarts.has(node.startIndex) && IDENT_SHAPE.test(text)) return 'def';
|
||||
if (isTypeNameType(type)) return 'type';
|
||||
return IDENT_SHAPE.test(text) ? 'ident' : 'other';
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a span, skipping empties and merging a run of the same class.
|
||||
*
|
||||
* Zero-width nodes are real: every grammar with a layout-sensitive scanner
|
||||
* (Python's `_newline`, Erlang's, Swift's) emits them, and a zero-width span
|
||||
* would put an empty token on the wire for nothing.
|
||||
*/
|
||||
function emit(ctx: WalkContext, start: number, end: number, cls: SyntaxTokenClass): void {
|
||||
if (end <= start) return;
|
||||
const last = ctx.out[ctx.out.length - 1];
|
||||
const from = start + ctx.offset;
|
||||
if (last && last.cls === cls && last.end === from) {
|
||||
last.end = end + ctx.offset;
|
||||
return;
|
||||
}
|
||||
ctx.out.push({ start: from, end: end + ctx.offset, cls });
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------- regions -- */
|
||||
|
||||
/**
|
||||
* A stretch of a file written in a different language from the file itself.
|
||||
*
|
||||
* Single-file components are the only case: a `.svelte`, `.vue` or `.astro`
|
||||
* file has no tree-sitter grammar of its own here, but its `<script>` block —
|
||||
* where every symbol the engine indexed in that file lives — is ordinary
|
||||
* TypeScript or JavaScript. The extractors already delegate exactly this way,
|
||||
* so the viewer reads a component's code with the same grammar the graph was
|
||||
* built from. The surrounding markup stays unclassified, which under a
|
||||
* near-monochrome theme costs the recession on tag names and attribute strings
|
||||
* and nothing else.
|
||||
*/
|
||||
export interface SyntaxRegion {
|
||||
start: number;
|
||||
end: number;
|
||||
language: Language;
|
||||
}
|
||||
|
||||
const SCRIPT_BLOCK = /<script(\s[^>]*)?>([\s\S]*?)<\/script>/gi;
|
||||
const TS_LANG_ATTR = /lang\s*=\s*["'](ts|typescript)["']/i;
|
||||
/** Astro's frontmatter: a `---` fence at the very top of the file. */
|
||||
const ASTRO_FRONTMATTER = /^(---\r?\n)([\s\S]*?)\r?\n---/;
|
||||
|
||||
/**
|
||||
* The sub-language regions of a file, or null when the file is one language.
|
||||
*
|
||||
* Null and an empty array mean different things: null is "parse the whole file
|
||||
* as `language`", empty is "this file has a grammar for none of it".
|
||||
*/
|
||||
export function syntaxRegionsFor(source: string, language: Language): SyntaxRegion[] | null {
|
||||
if (language !== 'svelte' && language !== 'vue' && language !== 'astro') return null;
|
||||
|
||||
const regions: SyntaxRegion[] = [];
|
||||
if (language === 'astro') {
|
||||
const front = ASTRO_FRONTMATTER.exec(source);
|
||||
if (front && front[2]) {
|
||||
const start = (front[1] as string).length;
|
||||
regions.push({ start, end: start + (front[2] as string).length, language: 'typescript' });
|
||||
}
|
||||
}
|
||||
|
||||
SCRIPT_BLOCK.lastIndex = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = SCRIPT_BLOCK.exec(source)) !== null) {
|
||||
const body = match[2] ?? '';
|
||||
if (body.trim() === '') continue;
|
||||
const start = match.index + match[0].length - body.length - '</script>'.length;
|
||||
regions.push({
|
||||
start,
|
||||
end: start + body.length,
|
||||
language: TS_LANG_ATTR.test(match[1] ?? '') ? 'typescript' : 'javascript',
|
||||
});
|
||||
}
|
||||
return regions;
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------- the API -- */
|
||||
|
||||
export interface TokenizeResult {
|
||||
spans: SyntaxSpan[];
|
||||
/** The grammar(s) that produced them, for the payload's `grammar` field. */
|
||||
grammars: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `source` and classify it.
|
||||
*
|
||||
* Returns null when nothing in the file has a grammar — a plain answer, which
|
||||
* every caller here already knows how to serve. Never throws: a grammar that
|
||||
* fails to load or a parse that comes back empty is the same outcome as not
|
||||
* having one.
|
||||
*/
|
||||
export async function tokenizeSource(
|
||||
source: string,
|
||||
language: Language
|
||||
): Promise<TokenizeResult | null> {
|
||||
const regions = syntaxRegionsFor(source, language);
|
||||
if (regions === null) {
|
||||
const spans = await tokenizeRegion(source, language, 0);
|
||||
return spans ? { spans, grammars: [language] } : null;
|
||||
}
|
||||
if (regions.length === 0) return null;
|
||||
|
||||
const spans: SyntaxSpan[] = [];
|
||||
const grammars = new Set<string>();
|
||||
for (const region of regions) {
|
||||
const part = await tokenizeRegion(
|
||||
source.slice(region.start, region.end),
|
||||
region.language,
|
||||
region.start
|
||||
);
|
||||
if (!part) continue;
|
||||
grammars.add(region.language);
|
||||
spans.push(...part);
|
||||
}
|
||||
if (spans.length === 0) return null;
|
||||
spans.sort((a, b) => a.start - b.start);
|
||||
return { spans, grammars: [...grammars] };
|
||||
}
|
||||
|
||||
async function tokenizeRegion(
|
||||
source: string,
|
||||
language: Language,
|
||||
offset: number
|
||||
): Promise<SyntaxSpan[] | null> {
|
||||
try {
|
||||
await loadGrammarsForLanguages([language]);
|
||||
const parser = getParser(language);
|
||||
if (!parser) return null;
|
||||
const tree = parser.parse(source);
|
||||
if (!tree?.rootNode) return null;
|
||||
try {
|
||||
return classifyTree(tree.rootNode, source, language, offset);
|
||||
} finally {
|
||||
tree.delete();
|
||||
}
|
||||
} catch {
|
||||
// A grammar that will not load, or a parse that threw: the caller serves
|
||||
// the source unclassified, which is the whole point of the plain path.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,7 @@
|
||||
* before a single page of source has arrived.
|
||||
*
|
||||
* The source itself does NOT ride along. A 6 800-line TypeScript file is ~1.5 s
|
||||
* of TextMate tokenising and megabytes of JSON; the viewer pages it through
|
||||
* of parsing and megabytes of JSON; the viewer pages it through
|
||||
* `/api/source` as the reader scrolls, which is also what lets the graph
|
||||
* facts — ports, arcs, rail rows — be complete from the first frame while the
|
||||
* text fills in behind them.
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
/**
|
||||
* Finding and reading the pruned TextMate grammars on disk.
|
||||
*
|
||||
* Shiki ships 700-odd grammars; the engine indexes 40-odd languages. The build
|
||||
* writes only the closure those 40 need — including the grammars they embed, so
|
||||
* a `.vue` file still gets its `<script lang="ts">` — into `dist/textmate/`,
|
||||
* and `@shikijs/langs` stays a devDependency that never reaches a user's disk.
|
||||
* See `scripts/prune-grammars.mjs`.
|
||||
*
|
||||
* They are located the way `db/index.ts` finds `schema.sql` and `assets.ts`
|
||||
* finds the viewer: relative to `__dirname`, never to `process.cwd()`, which is
|
||||
* whatever directory the user happened to be standing in.
|
||||
*
|
||||
* `dist/textmate`, not `dist/highlight` — `src/ui-server/highlight/` is this
|
||||
* module and tsc already owns `dist/ui-server/highlight/`. The same collision
|
||||
* that put the viewer in `dist/viewer` rather than `dist/ui`.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
/** Overrides where the grammars are read from. For tests and for packagers. */
|
||||
export const TEXTMATE_PATH_ENV = 'CODEGRAPH_TEXTMATE_PATH';
|
||||
|
||||
/** What `scripts/prune-grammars.mjs` writes beside the grammar files. */
|
||||
export interface GrammarManifest {
|
||||
/** Shiki version the grammars were pruned from — surfaced when one fails. */
|
||||
shikiVersion: string;
|
||||
/** Grammar id → the files to load, dependencies first, the grammar itself last. */
|
||||
languages: Record<string, string[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Candidate locations, most-specific first.
|
||||
*
|
||||
* 1. The `CODEGRAPH_TEXTMATE_PATH` override.
|
||||
* 2. `<__dirname>/../../textmate` — the shipped layout
|
||||
* (`dist/ui-server/highlight/` → `dist/textmate/`).
|
||||
* 3. `<__dirname>/../../../dist/textmate` — running the TypeScript straight out
|
||||
* of `src/` (vitest, tsx), where `__dirname` is `src/ui-server/highlight/`.
|
||||
*/
|
||||
export function grammarDirCandidates(): string[] {
|
||||
const override = process.env[TEXTMATE_PATH_ENV]?.trim();
|
||||
const candidates = [
|
||||
path.join(__dirname, '..', '..', 'textmate'),
|
||||
path.join(__dirname, '..', '..', '..', 'dist', 'textmate'),
|
||||
];
|
||||
return override ? [path.resolve(override), ...candidates] : candidates;
|
||||
}
|
||||
|
||||
/**
|
||||
* The grammar directory and its manifest, or null when the build did not run.
|
||||
*
|
||||
* Null is a normal outcome, not an error: a source checkout that has only had
|
||||
* `tsc` run against it has no `dist/textmate`, and the right answer there is
|
||||
* plain text, not a 500 on a request for source.
|
||||
*/
|
||||
export function loadManifest(): { dir: string; manifest: GrammarManifest } | null {
|
||||
for (const dir of grammarDirCandidates()) {
|
||||
try {
|
||||
const raw = fs.readFileSync(path.join(dir, 'manifest.json'), 'utf-8');
|
||||
const manifest = JSON.parse(raw) as GrammarManifest;
|
||||
if (manifest && typeof manifest === 'object' && manifest.languages) return { dir, manifest };
|
||||
} catch {
|
||||
// Not here — try the next candidate.
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the grammar registrations one language needs, dependencies first.
|
||||
*
|
||||
* Shiki resolves a grammar's `embeddedLangs` against what is already in its
|
||||
* registry, so the order the manifest records matters: `vue` must arrive after
|
||||
* the `html`, `css` and `typescript` it embeds, or the embedded blocks come
|
||||
* back unhighlighted.
|
||||
*/
|
||||
export function readGrammarChain(dir: string, manifest: GrammarManifest, id: string): unknown[] {
|
||||
const files = manifest.languages[id];
|
||||
if (!files) return [];
|
||||
return files.map((file) => {
|
||||
const full = path.join(dir, `${file}.json`);
|
||||
return JSON.parse(fs.readFileSync(full, 'utf-8')) as unknown;
|
||||
});
|
||||
}
|
||||
+148
-198
@@ -1,57 +1,61 @@
|
||||
/**
|
||||
* Server-side syntax classification for the viewer's code block (CG-43).
|
||||
* Server-side syntax classification for the viewer's code block.
|
||||
*
|
||||
* The viewer used to lex on the client with a hand-rolled dialect table. This
|
||||
* replaces it with real TextMate grammars, run once here, so a Go file reads as
|
||||
* Go rather than as "something with braces". Three properties keep that from
|
||||
* becoming a liability:
|
||||
* The classes come off the engine's OWN tree-sitter parse (CG-57). Until then
|
||||
* the viewer ran a second highlighter — Shiki, plus 56 pruned TextMate grammars
|
||||
* shipped beside the binary — over source the engine had already parsed with a
|
||||
* real grammar. That is gone: one grammar set, one opinion about what a `.ts`
|
||||
* file is, nothing extra in the bundle, and roughly an order of magnitude off
|
||||
* the cost on the language that used to be worst (see below).
|
||||
*
|
||||
* * **It never fails a request.** A missing grammar, a corrupt grammar file, an
|
||||
* ESM import that did not resolve, a slice too big to be worth tokenising —
|
||||
* every one of them answers `engine: 'plain'` with a reason and the source
|
||||
* still goes out. Highlighting is the part that degrades; nothing else does.
|
||||
* Three properties are unchanged, because they are what make this safe to
|
||||
* depend on:
|
||||
*
|
||||
* * **It never fails a request.** A grammar that will not load, a parse that
|
||||
* throws, a language nobody wrote a grammar for, a slice too big to be worth
|
||||
* parsing — every one of them answers `engine: 'plain'` with a reason and the
|
||||
* source still goes out. Highlighting is the part that degrades; nothing else
|
||||
* does.
|
||||
* * **Identifiers survive whatever token boundaries the grammar chose.** Every
|
||||
* code token is split into identifier runs before it goes on the wire, which
|
||||
* is what lets the viewer wrap a call site as a link by *claiming a token*
|
||||
* rather than re-tokenising the line on top of the highlighter's answer.
|
||||
* * **The classification is a class name, not a colour.** See `theme.ts` — the
|
||||
* viewer paints from CSS custom properties, so one token stream serves light
|
||||
* and dark and the design tokens live in exactly one place.
|
||||
* rather than re-tokenising the line on top of the classifier's answer.
|
||||
* * **The classification is a class name, not a colour.** The viewer paints
|
||||
* from CSS custom properties, so one token stream serves light and dark and
|
||||
* the design tokens live in exactly one place.
|
||||
*
|
||||
* ## Cost, measured
|
||||
*
|
||||
* Shiki's JavaScript regex engine (no oniguruma wasm, no native module) runs at
|
||||
* roughly 17 us/line on Go, 34 us/line on Python and 230 us/line on TypeScript,
|
||||
* whose TextMate grammar is by a wide margin the most expensive one here. A
|
||||
* 3 000-line TypeScript file is therefore ~700 ms cold, which is why
|
||||
* {@link SLICE_CACHE_LIMIT} exists: a slice is keyed by the file's content hash
|
||||
* and its line range, so every re-render — a theme flip, a resize, stepping
|
||||
* back to a symbol — is a map lookup. Phase 1 only ever asks for one symbol's
|
||||
* range (tens of lines); the whole-file view is CG-52 and tree-sitter tokens
|
||||
* from the engine's own parse replace this module entirely in CG-57.
|
||||
* On the dev Mac, 3 000 lines, cold (parse + classify + wire):
|
||||
*
|
||||
* | | TypeScript | Go | Python |
|
||||
* |---|---|---|---|
|
||||
* | Shiki (was) | ~700 ms | 43–57 ms | 35–47 ms |
|
||||
* | tree-sitter (now) | 24–41 ms | ~30 ms | 25–29 ms |
|
||||
*
|
||||
* Rust, Ruby, PHP, C# and Swift all land between 14 and 27 ms on the same
|
||||
* measurement. TypeScript's TextMate grammar was 5–7× every other one and the
|
||||
* cost was regex *execution*, not compilation — nothing about the old module
|
||||
* could have fixed it, and it is now the same order as everything else. The
|
||||
* slice cache still exists, because a re-render (a theme flip, a resize,
|
||||
* stepping back through the trail) should cost nothing at all, and because a
|
||||
* whole-file view pages the same file repeatedly.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ShikiCoreModule,
|
||||
ShikiHighlighter,
|
||||
ShikiJavaScriptEngineModule,
|
||||
ShikiThemedToken,
|
||||
} from './shiki-types';
|
||||
import { CLASS_ID, MONO_THEME, TOKEN_CLASSES, classOf, type TokenClassName } from './theme';
|
||||
import { SYNTAX_TOKEN_CLASSES, tokenizeSource, type SyntaxTokenClass } from '../../extraction/syntax-tokens';
|
||||
import type { Language } from '../../types';
|
||||
import { grammarFor } from './languages';
|
||||
import { loadManifest, readGrammarChain, type GrammarManifest } from './grammars';
|
||||
|
||||
export { TOKEN_CLASSES } from './theme';
|
||||
export { LANGUAGE_GRAMMAR, REQUIRED_GRAMMARS, grammarFor } from './languages';
|
||||
export { TEXTMATE_PATH_ENV } from './grammars';
|
||||
export { COMPONENT_LANGUAGES, grammarFor, isHighlightable } from './languages';
|
||||
export { SYNTAX_TOKEN_CLASSES as TOKEN_CLASSES } from '../../extraction/syntax-tokens';
|
||||
|
||||
/** One token on the wire: its class id, then its text. */
|
||||
export type WireToken = [number, string];
|
||||
|
||||
export interface HighlightResult {
|
||||
/** `shiki` when a grammar produced the classes; `plain` when nothing did. */
|
||||
engine: 'shiki' | 'plain';
|
||||
/** The TextMate grammar used, or null. */
|
||||
/** `tree-sitter` when a grammar produced the classes; `plain` when nothing did. */
|
||||
engine: 'tree-sitter' | 'plain';
|
||||
/** The grammar the source was read with, or null. */
|
||||
grammar: string | null;
|
||||
/** Class names, indexed by the first element of every {@link WireToken}. */
|
||||
classes: readonly string[];
|
||||
@@ -62,25 +66,25 @@ export interface HighlightResult {
|
||||
}
|
||||
|
||||
/**
|
||||
* Lines above this are not tokenised.
|
||||
* Lines above this are not classified.
|
||||
*
|
||||
* Matches `MAX_SOURCE_LINES`, so anything the source endpoint will serve, this
|
||||
* will try to highlight.
|
||||
* will try to classify.
|
||||
*/
|
||||
export const MAX_HIGHLIGHT_LINES = 4000;
|
||||
|
||||
/**
|
||||
* Characters above this are not tokenised.
|
||||
* Characters above this are not classified.
|
||||
*
|
||||
* The line cap alone does not bound the work: one minified bundle line can be
|
||||
* two megabytes, and a TextMate scanner walks it character by character. This
|
||||
* is the guard that keeps a single request from wedging a single-threaded
|
||||
* loopback server, and it is generous — 600 kB is far more source than any
|
||||
* screen renders.
|
||||
* two megabytes, and a parser walks it character by character. This is the
|
||||
* guard that keeps a single request from wedging a single-threaded loopback
|
||||
* server, and it is generous — 600 kB is far more source than any screen
|
||||
* renders.
|
||||
*/
|
||||
export const MAX_HIGHLIGHT_CHARS = 600_000;
|
||||
|
||||
/** Highlighted slices kept in memory. Most are one symbol's body. */
|
||||
/** Classified slices kept in memory. Most are one symbol's body. */
|
||||
export const SLICE_CACHE_LIMIT = 96;
|
||||
|
||||
/**
|
||||
@@ -94,105 +98,23 @@ export const SLICE_CACHE_LIMIT = 96;
|
||||
*/
|
||||
export const SLICE_CACHE_LINES = 20_000;
|
||||
|
||||
/* ----------------------------------------------------------- the runtime -- */
|
||||
/** Class name → its index in {@link SYNTAX_TOKEN_CLASSES}, which is what the wire carries. */
|
||||
const CLASS_ID = Object.fromEntries(
|
||||
SYNTAX_TOKEN_CLASSES.map((name, index) => [name, index])
|
||||
) as Record<SyntaxTokenClass, number>;
|
||||
|
||||
/**
|
||||
* tsc compiles `import()` to `require()` under `module: commonjs`, which fails
|
||||
* for an ESM-only package. Same escape hatch `src/bin/codegraph.ts` uses.
|
||||
*/
|
||||
const importESM = new Function('specifier', 'return import(specifier)') as (
|
||||
specifier: string
|
||||
) => Promise<unknown>;
|
||||
|
||||
/**
|
||||
* Import an ESM-only package from this CommonJS build.
|
||||
* Classes that are never merged with their neighbour.
|
||||
*
|
||||
* The `new Function` route is the one that runs in production. It does NOT run
|
||||
* under Vitest, whose module runner evaluates this file without a dynamic-import
|
||||
* callback ("A dynamic import callback was not specified") — there, the
|
||||
* transformed `import()` below is the working one, and in the shipped CommonJS
|
||||
* build it is the one that cannot work. Each covers exactly the other's gap;
|
||||
* neither alone is enough, which is why both are here.
|
||||
* Every identifier-shaped token has to stay claimable on its own — the overlay
|
||||
* wraps exactly one of them as a call-site link, and two merged into one token
|
||||
* would underline both or neither.
|
||||
*/
|
||||
async function loadEsm<T>(specifier: string): Promise<T> {
|
||||
try {
|
||||
return (await importESM(specifier)) as T;
|
||||
} catch (err) {
|
||||
if (!(err instanceof Error) || !/dynamic import callback/i.test(err.message)) throw err;
|
||||
return (await import(/* @vite-ignore */ specifier)) as T;
|
||||
}
|
||||
}
|
||||
|
||||
interface Runtime {
|
||||
highlighter: ShikiHighlighter;
|
||||
dir: string;
|
||||
manifest: GrammarManifest;
|
||||
}
|
||||
|
||||
let runtimePromise: Promise<Runtime | null> | null = null;
|
||||
/** Why the runtime is unavailable, for the `reason` on a plain answer. */
|
||||
let runtimeFailure: string | null = null;
|
||||
|
||||
async function getRuntime(): Promise<Runtime | null> {
|
||||
if (!runtimePromise) runtimePromise = createRuntime();
|
||||
return runtimePromise;
|
||||
}
|
||||
|
||||
async function createRuntime(): Promise<Runtime | null> {
|
||||
const found = loadManifest();
|
||||
if (!found) {
|
||||
runtimeFailure =
|
||||
'No syntax grammars are installed with this build, so source is shown unhighlighted.';
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const core = await loadEsm<ShikiCoreModule>('@shikijs/core');
|
||||
const engineModule = await loadEsm<ShikiJavaScriptEngineModule>('@shikijs/engine-javascript');
|
||||
const highlighter = core.createHighlighterCoreSync({
|
||||
themes: [MONO_THEME],
|
||||
langs: [],
|
||||
// The JavaScript regex engine, deliberately: no oniguruma wasm and no
|
||||
// native module, so the viewer adds nothing to the install that has to
|
||||
// be compiled or fetched per platform. `forgiving` skips the handful of
|
||||
// Oniguruma-only patterns it cannot translate rather than refusing the
|
||||
// whole grammar over them.
|
||||
engine: engineModule.createJavaScriptRegexEngine({ forgiving: true, cache: new Map() }),
|
||||
});
|
||||
return { highlighter, dir: found.dir, manifest: found.manifest };
|
||||
} catch (err) {
|
||||
runtimeFailure = `Syntax highlighting is unavailable (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}).`;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Grammar ids already handed to the highlighter, and the ones that failed. */
|
||||
const loadedGrammars = new Set<string>();
|
||||
const brokenGrammars = new Map<string, string>();
|
||||
|
||||
function ensureGrammar(runtime: Runtime, id: string): string | null {
|
||||
if (loadedGrammars.has(id)) return null;
|
||||
const broken = brokenGrammars.get(id);
|
||||
if (broken !== undefined) return broken;
|
||||
try {
|
||||
const chain = readGrammarChain(runtime.dir, runtime.manifest, id);
|
||||
if (chain.length === 0) {
|
||||
const reason = `No ${id} grammar shipped with this build, so it is shown unhighlighted.`;
|
||||
brokenGrammars.set(id, reason);
|
||||
return reason;
|
||||
}
|
||||
runtime.highlighter.loadLanguageSync(chain);
|
||||
loadedGrammars.add(id);
|
||||
return null;
|
||||
} catch (err) {
|
||||
const reason = `The ${id} grammar could not be loaded (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}).`;
|
||||
brokenGrammars.set(id, reason);
|
||||
return reason;
|
||||
}
|
||||
}
|
||||
const UNMERGEABLE: ReadonlySet<SyntaxTokenClass> = new Set<SyntaxTokenClass>([
|
||||
'ident',
|
||||
'type',
|
||||
'def',
|
||||
]);
|
||||
|
||||
/* -------------------------------------------------------------- the cache -- */
|
||||
|
||||
@@ -242,7 +164,7 @@ export interface HighlightOptions {
|
||||
language?: string | null;
|
||||
/**
|
||||
* A key that changes whenever the text does — the file's content hash plus
|
||||
* the requested range. Omit it and the slice is tokenised every time.
|
||||
* the requested range. Omit it and the slice is classified every time.
|
||||
*/
|
||||
cacheKey?: string;
|
||||
}
|
||||
@@ -265,13 +187,14 @@ export async function highlightLines(
|
||||
if (hit) return hit;
|
||||
}
|
||||
|
||||
const result = await highlightUncached(lines, grammar);
|
||||
const result = await highlightUncached(lines, options.language ?? null, grammar);
|
||||
if (key) cachePut(key, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function highlightUncached(
|
||||
lines: readonly string[],
|
||||
language: string | null,
|
||||
grammar: string | null
|
||||
): Promise<HighlightResult> {
|
||||
if (!grammar) {
|
||||
@@ -280,55 +203,84 @@ async function highlightUncached(
|
||||
if (lines.length > MAX_HIGHLIGHT_LINES) {
|
||||
return plain(lines, grammar, `Too many lines to highlight (over ${MAX_HIGHLIGHT_LINES}).`);
|
||||
}
|
||||
let chars = 0;
|
||||
for (const line of lines) chars += line.length + 1;
|
||||
if (chars > MAX_HIGHLIGHT_CHARS) {
|
||||
const text = lines.join('\n');
|
||||
if (text.length > MAX_HIGHLIGHT_CHARS) {
|
||||
return plain(lines, grammar, 'Too much text on too few lines to highlight (minified?).');
|
||||
}
|
||||
|
||||
const runtime = await getRuntime();
|
||||
if (!runtime) return plain(lines, grammar, runtimeFailure ?? undefined);
|
||||
|
||||
const failure = ensureGrammar(runtime, grammar);
|
||||
if (failure) return plain(lines, grammar, failure);
|
||||
|
||||
let tokenized: ShikiThemedToken[][];
|
||||
try {
|
||||
tokenized = runtime.highlighter.codeToTokensBase(lines.join('\n'), {
|
||||
lang: grammar,
|
||||
theme: MONO_THEME.name,
|
||||
});
|
||||
} catch (err) {
|
||||
// A grammar that throws once will throw again on the next request for the
|
||||
// same file type, so it is retired rather than retried.
|
||||
const reason = `The ${grammar} grammar failed on this file (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}).`;
|
||||
brokenGrammars.set(grammar, reason);
|
||||
loadedGrammars.delete(grammar);
|
||||
return plain(lines, grammar, reason);
|
||||
const tokenized = await tokenizeSource(text, language as Language);
|
||||
if (!tokenized || tokenized.spans.length === 0) {
|
||||
return plain(lines, grammar, `The ${grammar} grammar is not available in this build.`);
|
||||
}
|
||||
|
||||
// A trailing empty line, or a grammar that answered short, must not shift the
|
||||
// viewer's line numbering — the rows are indexed positionally.
|
||||
const out: WireToken[][] = lines.map((line, i) => {
|
||||
const row = tokenized[i];
|
||||
return row ? atomize(row) : atomizePlain(line);
|
||||
});
|
||||
|
||||
return { engine: 'shiki', grammar, classes: TOKEN_CLASSES, lines: out };
|
||||
return {
|
||||
engine: 'tree-sitter',
|
||||
grammar: tokenized.grammars.join('+') || grammar,
|
||||
classes: SYNTAX_TOKEN_CLASSES,
|
||||
lines: toWireLines(lines, text, tokenized.spans),
|
||||
};
|
||||
}
|
||||
|
||||
function plain(lines: readonly string[], grammar: string | null, reason?: string): HighlightResult {
|
||||
return {
|
||||
engine: 'plain',
|
||||
grammar,
|
||||
classes: TOKEN_CLASSES,
|
||||
classes: SYNTAX_TOKEN_CLASSES,
|
||||
lines: lines.map(atomizePlain),
|
||||
...(reason ? { reason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------- spans to lines -- */
|
||||
|
||||
/**
|
||||
* Cut the classifier's spans into one token list per source line.
|
||||
*
|
||||
* The classifier answers over the whole slice, in string offsets, and leaves
|
||||
* the gaps between spans unclassified — those are whitespace and the layout
|
||||
* separators no grammar names. Here they become plain tokens, multi-line spans
|
||||
* (a block comment, a heredoc) are split at the newlines, and every line ends
|
||||
* up with a token list whose texts concatenate back to exactly that line.
|
||||
*
|
||||
* One entry per line, always: the code block indexes rows positionally, so a
|
||||
* short answer would render every line below it against the wrong source.
|
||||
*/
|
||||
function toWireLines(
|
||||
lines: readonly string[],
|
||||
text: string,
|
||||
spans: readonly { start: number; end: number; cls: SyntaxTokenClass }[]
|
||||
): WireToken[][] {
|
||||
const pieces: { start: number; end: number; cls: SyntaxTokenClass }[] = [];
|
||||
let cursor = 0;
|
||||
for (const span of spans) {
|
||||
if (span.end <= cursor) continue;
|
||||
const start = Math.max(span.start, cursor);
|
||||
if (start > cursor) pieces.push({ start: cursor, end: start, cls: 'other' });
|
||||
pieces.push({ start, end: span.end, cls: span.cls });
|
||||
cursor = span.end;
|
||||
}
|
||||
if (cursor < text.length) pieces.push({ start: cursor, end: text.length, cls: 'other' });
|
||||
|
||||
const out: WireToken[][] = [];
|
||||
let lineStart = 0;
|
||||
let first = 0;
|
||||
for (const line of lines) {
|
||||
const lineEnd = lineStart + line.length;
|
||||
const row: WireToken[] = [];
|
||||
while (first < pieces.length && (pieces[first] as { end: number }).end <= lineStart) first += 1;
|
||||
for (let i = first; i < pieces.length; i++) {
|
||||
const piece = pieces[i] as { start: number; end: number; cls: SyntaxTokenClass };
|
||||
if (piece.start >= lineEnd) break;
|
||||
const from = Math.max(piece.start, lineStart);
|
||||
const to = Math.min(piece.end, lineEnd);
|
||||
if (to > from) pushPiece(row, text.slice(from, to), piece.cls);
|
||||
}
|
||||
out.push(row);
|
||||
lineStart = lineEnd + 1; // the '\n' the join put back
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------- atomisation -- */
|
||||
|
||||
/**
|
||||
@@ -341,29 +293,23 @@ function plain(lines: readonly string[], grammar: string | null, reason?: string
|
||||
const IDENT = /[A-Za-z_$À-][\w$À-]*/g;
|
||||
|
||||
/**
|
||||
* Split a grammar's tokens into identifier runs, merging everything else.
|
||||
* Add one classified run to a line, splitting it into identifier atoms.
|
||||
*
|
||||
* This is the step that makes the graph's call-site links independent of how a
|
||||
* grammar chose to chunk a line. TextMate is free to emit `this.mutex.withLock`
|
||||
* as one token, three, or five, and the viewer has to be able to wrap exactly
|
||||
* `withLock`; giving it identifier-sized atoms up front means the overlay only
|
||||
* ever *claims* a token, never re-cuts one.
|
||||
* grammar chose to chunk a line: the viewer has to be able to wrap exactly
|
||||
* `withLock` in `this.mutex.withLock`, and giving it identifier-sized atoms up
|
||||
* front means the overlay only ever *claims* a token, never re-cuts one.
|
||||
*
|
||||
* Comments and strings are left whole on purpose: no edge points inside one,
|
||||
* and a doc comment split into forty atoms is forty times the wire bytes for
|
||||
* nothing.
|
||||
*/
|
||||
function atomize(tokens: readonly ShikiThemedToken[]): WireToken[] {
|
||||
const out: WireToken[] = [];
|
||||
for (const token of tokens) {
|
||||
const cls = classOf(token.color);
|
||||
if (cls === 'comment' || cls === 'string') {
|
||||
push(out, cls, token.content);
|
||||
continue;
|
||||
}
|
||||
splitIdentifiers(out, token.content, cls);
|
||||
function pushPiece(row: WireToken[], text: string, cls: SyntaxTokenClass): void {
|
||||
if (cls === 'comment' || cls === 'string') {
|
||||
push(row, cls, text);
|
||||
return;
|
||||
}
|
||||
return out;
|
||||
splitIdentifiers(row, text, cls);
|
||||
}
|
||||
|
||||
function atomizePlain(line: string): WireToken[] {
|
||||
@@ -375,31 +321,35 @@ function atomizePlain(line: string): WireToken[] {
|
||||
/**
|
||||
* Emit `text` as alternating non-identifier and identifier runs.
|
||||
*
|
||||
* An identifier inside a token the grammar called a keyword keeps the keyword
|
||||
* An identifier inside a run the grammar called a keyword keeps the keyword
|
||||
* class — `func` should still carry its weight — while the overlay's matcher
|
||||
* looks at a token's *text*, not its class, so a language whose grammar scopes
|
||||
* type names as `storage.type` still links.
|
||||
* looks at a token's *text*, not its class, so a language whose grammar calls a
|
||||
* declared type name something unexpected still links.
|
||||
*/
|
||||
function splitIdentifiers(out: WireToken[], text: string, cls: TokenClassName): void {
|
||||
function splitIdentifiers(out: WireToken[], text: string, cls: SyntaxTokenClass): void {
|
||||
if (text === '') return;
|
||||
IDENT.lastIndex = 0;
|
||||
let at = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = IDENT.exec(text)) !== null) {
|
||||
if (match.index > at) push(out, cls === 'ident' ? 'other' : cls, text.slice(at, match.index));
|
||||
if (match.index > at) push(out, gapClass(cls), text.slice(at, match.index));
|
||||
push(out, cls === 'other' ? 'ident' : cls, match[0]);
|
||||
at = match.index + match[0].length;
|
||||
}
|
||||
if (at < text.length) push(out, cls === 'ident' ? 'other' : cls, text.slice(at));
|
||||
if (at < text.length) push(out, gapClass(cls), text.slice(at));
|
||||
}
|
||||
|
||||
/** The class for the non-identifier remainder of a run. */
|
||||
function gapClass(cls: SyntaxTokenClass): SyntaxTokenClass {
|
||||
return UNMERGEABLE.has(cls) ? 'other' : cls;
|
||||
}
|
||||
|
||||
/** Append, merging into the previous token when it carries the same class. */
|
||||
function push(out: WireToken[], cls: TokenClassName, text: string): void {
|
||||
function push(out: WireToken[], cls: SyntaxTokenClass, text: string): void {
|
||||
if (text === '') return;
|
||||
const id = CLASS_ID[cls];
|
||||
const last = out[out.length - 1];
|
||||
// Identifiers are never merged: each one has to stay claimable on its own.
|
||||
if (last && last[0] === id && id !== CLASS_ID.ident) {
|
||||
if (last && last[0] === id && !UNMERGEABLE.has(cls)) {
|
||||
last[1] += text;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,87 +1,34 @@
|
||||
/**
|
||||
* Engine `Language` → TextMate grammar, and the closure of grammars that has
|
||||
* to ship for those to load.
|
||||
* Which engine language a file's source is classified with (CG-57).
|
||||
*
|
||||
* The engine indexes 40-odd languages; Shiki carries 700-odd grammars. Shipping
|
||||
* all of them would put 11 MB of JSON in the bundle to serve 40, so the build
|
||||
* prunes them (`scripts/prune-grammars.mjs`) to exactly the closure this table
|
||||
* names — which is why the table lives in its own module: the build script
|
||||
* reads the compiled `dist/ui-server/highlight/languages.js` rather than keeping
|
||||
* a second copy of the mapping that could drift from the runtime's.
|
||||
* There is no second grammar table any more. The viewer reads a file with the
|
||||
* grammar the *engine* parsed it with, so this is a question about coverage
|
||||
* rather than about mapping: a language the extractor has a tree-sitter grammar
|
||||
* for classifies; one it does not renders plain, with its identifiers still
|
||||
* split out so the graph's call-site links land exactly as they do everywhere
|
||||
* else. Highlighting is the part that degrades, never the linking.
|
||||
*
|
||||
* A language with no entry (or with `null`) is not an error. It renders as
|
||||
* plain text with its identifiers still split out, so the graph's call-site
|
||||
* links land exactly as they do everywhere else — highlighting is the part that
|
||||
* degrades, never the linking.
|
||||
* The three single-file-component formats are the exception worth naming. A
|
||||
* `.svelte`, `.vue` or `.astro` file has no grammar of its own here — the
|
||||
* extractors pull the `<script>` block out and hand it to TypeScript or
|
||||
* JavaScript — and the classifier does exactly the same thing, so a component's
|
||||
* code is read by the grammar its symbols came from while the surrounding
|
||||
* markup stays plain.
|
||||
*/
|
||||
|
||||
import type { Language } from '../../types';
|
||||
import { hasTreeSitterGrammar } from '../../extraction/grammars';
|
||||
|
||||
/**
|
||||
* The grammar each indexed language is read with.
|
||||
* Formats whose source is classified through their embedded script blocks.
|
||||
*
|
||||
* Three of these are deliberate approximations, marked below: Shiki has no
|
||||
* ColdFusion grammar, and the three CFML dialects the engine distinguishes are
|
||||
* each a close relative of something it does have. An approximate keyword set
|
||||
* is a better answer than no colouring at all, and nothing downstream depends
|
||||
* on the grammar being exact — the links come from the graph.
|
||||
* Kept beside `syntaxRegionsFor`, which decides where those blocks are — this
|
||||
* list only has to agree about *which* formats have them.
|
||||
*/
|
||||
export const LANGUAGE_GRAMMAR: Record<Language, string | null> = {
|
||||
typescript: 'typescript',
|
||||
javascript: 'javascript',
|
||||
tsx: 'tsx',
|
||||
jsx: 'jsx',
|
||||
// ArkTS is TypeScript plus HarmonyOS decorators — the TS grammar reads it.
|
||||
arkts: 'typescript',
|
||||
python: 'python',
|
||||
go: 'go',
|
||||
rust: 'rust',
|
||||
java: 'java',
|
||||
c: 'c',
|
||||
cpp: 'cpp',
|
||||
csharp: 'csharp',
|
||||
razor: 'razor',
|
||||
php: 'php',
|
||||
ruby: 'ruby',
|
||||
swift: 'swift',
|
||||
kotlin: 'kotlin',
|
||||
dart: 'dart',
|
||||
svelte: 'svelte',
|
||||
vue: 'vue',
|
||||
astro: 'astro',
|
||||
liquid: 'liquid',
|
||||
pascal: 'pascal',
|
||||
scala: 'scala',
|
||||
lua: 'lua',
|
||||
luau: 'luau',
|
||||
objc: 'objective-c',
|
||||
r: 'r',
|
||||
solidity: 'solidity',
|
||||
nix: 'nix',
|
||||
yaml: 'yaml',
|
||||
twig: 'twig',
|
||||
xml: 'xml',
|
||||
properties: 'properties',
|
||||
// Approximations — no CFML grammar exists. Tag soup reads as HTML, cfscript
|
||||
// is a JavaScript-shaped dialect, and a <cfquery> body is SQL.
|
||||
cfml: 'html',
|
||||
cfscript: 'javascript',
|
||||
cfquery: 'sql',
|
||||
cobol: 'cobol',
|
||||
vbnet: 'vb',
|
||||
erlang: 'erlang',
|
||||
terraform: 'terraform',
|
||||
// Not a language, the absence of one: a file no extractor claimed.
|
||||
unknown: null,
|
||||
};
|
||||
|
||||
/** Every grammar the build must prune to, de-duplicated, in a stable order. */
|
||||
export const REQUIRED_GRAMMARS: readonly string[] = [
|
||||
...new Set(Object.values(LANGUAGE_GRAMMAR).filter((id): id is string => id !== null)),
|
||||
].sort();
|
||||
export const COMPONENT_LANGUAGES: readonly Language[] = ['svelte', 'vue', 'astro'];
|
||||
|
||||
/**
|
||||
* The grammar for an indexed language, or null when it has none.
|
||||
* The grammar a file of this language is read with, or null when it has none.
|
||||
*
|
||||
* Accepts the raw string off a `FileRecord` rather than a `Language`, because
|
||||
* an index written by an older engine can hold a language this build has since
|
||||
@@ -89,5 +36,12 @@ export const REQUIRED_GRAMMARS: readonly string[] = [
|
||||
*/
|
||||
export function grammarFor(language: string | undefined | null): string | null {
|
||||
if (!language) return null;
|
||||
return LANGUAGE_GRAMMAR[language as Language] ?? null;
|
||||
const lang = language as Language;
|
||||
if (COMPONENT_LANGUAGES.includes(lang)) return 'typescript';
|
||||
return hasTreeSitterGrammar(lang) ? lang : null;
|
||||
}
|
||||
|
||||
/** Whether a file of this language classifies at all. For tests and diagnostics. */
|
||||
export function isHighlightable(language: string | undefined | null): boolean {
|
||||
return grammarFor(language) !== null;
|
||||
}
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
/**
|
||||
* The slice of Shiki's surface this server uses, declared locally.
|
||||
*
|
||||
* `@shikijs/core` is ESM-only and the engine compiles to CommonJS, so it is
|
||||
* loaded through the same `new Function('return import(...)')` escape hatch the
|
||||
* CLI uses for `@clack/prompts` — which means tsc never sees the import and
|
||||
* cannot type it. Rather than fight `.d.mts` resolution under
|
||||
* `module: commonjs`, the four shapes actually touched are written out here.
|
||||
* They are checked against the real package by the highlighter's tests: a
|
||||
* signature change shows up as a failing highlight, not as a silent `any`.
|
||||
*/
|
||||
|
||||
export interface ShikiThemedToken {
|
||||
content: string;
|
||||
color?: string;
|
||||
fontStyle?: number;
|
||||
}
|
||||
|
||||
export interface ShikiHighlighter {
|
||||
loadLanguageSync(lang: unknown): void;
|
||||
getLoadedLanguages(): string[];
|
||||
codeToTokensBase(code: string, options: { lang: string; theme: string }): ShikiThemedToken[][];
|
||||
dispose?(): void;
|
||||
}
|
||||
|
||||
export interface ShikiCoreModule {
|
||||
createHighlighterCoreSync(options: {
|
||||
themes: unknown[];
|
||||
langs: unknown[];
|
||||
engine: unknown;
|
||||
}): ShikiHighlighter;
|
||||
}
|
||||
|
||||
export interface ShikiJavaScriptEngineModule {
|
||||
createJavaScriptRegexEngine(options?: {
|
||||
forgiving?: boolean;
|
||||
target?: 'auto' | 'ES2025' | 'ES2024' | 'ES2018';
|
||||
cache?: Map<string, RegExp | Error> | null;
|
||||
}): unknown;
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
/**
|
||||
* The near-monochrome code theme (design spec §2.2).
|
||||
*
|
||||
* The colouring is deliberately almost absent: comments and strings recede,
|
||||
* keywords carry weight rather than hue, and the ONLY colour in the body is a
|
||||
* resolved call site. A six-colour syntax theme buries exactly the thing the
|
||||
* screen exists to show.
|
||||
*
|
||||
* ## Why the theme's colours are sentinels, not colours
|
||||
*
|
||||
* A TextMate theme classifies by mapping scopes to colours, so that is how the
|
||||
* classification is *expressed* — but the values here are placeholders that
|
||||
* mean "comment", "string", "keyword", "number", nothing. The server turns each
|
||||
* one back into a class name; the viewer paints it from a CSS custom property.
|
||||
*
|
||||
* That indirection is load-bearing, not decoration:
|
||||
*
|
||||
* * **One token stream serves both modes.** The viewer flips light/dark from
|
||||
* `prefers-color-scheme` with no reload and no refetch. Baking `#6a675d` into
|
||||
* the payload would make dark mode a second request for the same source, and
|
||||
* would put the design tokens in two places at once.
|
||||
* * **Contrast is fixed where the tokens live.** `ui/src/app.css` owns the
|
||||
* ramp; a colour change there cannot leave the server's copy behind.
|
||||
*
|
||||
* The sentinels are arbitrary but must be distinct and must never be a colour a
|
||||
* grammar could plausibly emit through some other path, hence the `#00000n`
|
||||
* block: TextMate themes only ever return values *this* theme defines.
|
||||
*/
|
||||
|
||||
/** The classes a token can carry — the viewer's `TokenClass`, server side. */
|
||||
export const TOKEN_CLASSES = ['other', 'ident', 'comment', 'string', 'keyword', 'number'] as const;
|
||||
|
||||
export type TokenClassName = (typeof TOKEN_CLASSES)[number];
|
||||
|
||||
/** Class name → its index in {@link TOKEN_CLASSES}, which is what the wire carries. */
|
||||
export const CLASS_ID: Record<TokenClassName, number> = {
|
||||
other: 0,
|
||||
ident: 1,
|
||||
comment: 2,
|
||||
string: 3,
|
||||
keyword: 4,
|
||||
number: 5,
|
||||
};
|
||||
|
||||
const FG_DEFAULT = '#000001';
|
||||
const FG_COMMENT = '#000002';
|
||||
const FG_STRING = '#000003';
|
||||
const FG_KEYWORD = '#000004';
|
||||
const FG_NUMBER = '#000005';
|
||||
|
||||
/** Sentinel foreground → the class it stands for. */
|
||||
export const SENTINEL_CLASS: Record<string, TokenClassName> = {
|
||||
[FG_DEFAULT]: 'other',
|
||||
[FG_COMMENT]: 'comment',
|
||||
[FG_STRING]: 'string',
|
||||
[FG_KEYWORD]: 'keyword',
|
||||
[FG_NUMBER]: 'number',
|
||||
};
|
||||
|
||||
/**
|
||||
* The theme itself.
|
||||
*
|
||||
* Scope selection follows the spec exactly: `comment` recedes furthest,
|
||||
* `string`/`constant.numeric` sit one step in, `keyword`/`storage` stay ink and
|
||||
* gain weight, everything else is ink. Nothing sets a background — a token that
|
||||
* painted its own would fight the hovered-line and hot-line tints the rails use
|
||||
* to point at it.
|
||||
*/
|
||||
export const MONO_THEME = {
|
||||
name: 'codegraph-mono',
|
||||
type: 'light' as const,
|
||||
fg: FG_DEFAULT,
|
||||
// TextMate wants a background; the viewer never reads it (the code block
|
||||
// paints `--paper`), and it must not equal a foreground sentinel.
|
||||
bg: '#ffffff',
|
||||
settings: [
|
||||
{ settings: { foreground: FG_DEFAULT } },
|
||||
{ scope: ['comment', 'punctuation.definition.comment'], settings: { foreground: FG_COMMENT } },
|
||||
{
|
||||
scope: [
|
||||
'string',
|
||||
'string.template',
|
||||
'punctuation.definition.string',
|
||||
'constant.character.escape',
|
||||
],
|
||||
settings: { foreground: FG_STRING },
|
||||
},
|
||||
{
|
||||
scope: ['constant.numeric', 'constant.language', 'keyword.other.unit'],
|
||||
settings: { foreground: FG_NUMBER },
|
||||
},
|
||||
{
|
||||
scope: ['keyword', 'keyword.control', 'storage', 'storage.type', 'storage.modifier'],
|
||||
settings: { foreground: FG_KEYWORD },
|
||||
},
|
||||
// `keyword.operator` is a keyword scope by name only: it covers `=`, `+`,
|
||||
// `=>` and `?.`. Weighting punctuation buys nothing and costs the calm the
|
||||
// rest of the block is built on, so it drops back to plain ink — while the
|
||||
// operators that are actually WORDS (`new`, `typeof`, `instanceof`, `in`)
|
||||
// keep their weight through the more specific rule below. Shiki resolves
|
||||
// the longest matching scope, so the order here is the order of rescue,
|
||||
// not of priority.
|
||||
{ scope: ['keyword.operator'], settings: { foreground: FG_DEFAULT } },
|
||||
{
|
||||
scope: ['keyword.operator.expression', 'keyword.operator.word', 'keyword.operator.new'],
|
||||
settings: { foreground: FG_KEYWORD },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
/** The class a Shiki token's resolved colour stands for. */
|
||||
export function classOf(color: string | undefined): TokenClassName {
|
||||
if (!color) return 'other';
|
||||
return SENTINEL_CLASS[color.toLowerCase()] ?? 'other';
|
||||
}
|
||||
Reference in New Issue
Block a user