feat(kernel): R7b Ruby walker — ruby module, tree-sitter-ruby 0.23.1 bump, ref-flag wire slot, ruby default-routed (#1379)

Third R7b port, checklist-first recipe (docs/design/ruby-kernel-port-checklist.md).

Grammar bump first, validated standalone (the rust pattern): tree-sitter-ruby
^0.20.1 (tree-sitter-wasms, 2024-02) → v0.23.1 — crate pinned =0.23.1, wasm
built from tag 71bd32f's checked-in parser.c/scanner.c (both sha-matched
against the crates.io tarball; content bump, ABI stays 14). Old-vs-new
full-init dumps: sinatra/jekyll byte-identical; rails = exactly the one
classified hunk (the `recv&.!=` safe-nav operator misparse fix,
`table_name.!` → `table_name.!=`, precision-positive).

Walker (python.rs chassis + the six ruby divergences) preserves bug-for-bug:
the importTypes:['call'] funnel (class-body DSL — attr_accessor, has_many,
define_method incl. its block, sinatra route blocks — emits NOTHING at
non-body scope), hook-handled module multiply-capture (nested modules re-scan
their subtree per level after popping — `this.hooked` fn-refs from class AND
module AND file), the sibling-scan visibility trio (bare `private` invisible;
`private :sym`/`private def` poison all later defs; the inner def stays
public), bare-call statements (do…end body_statement emits, brace-block
block_body doesn't), `.new` instantiates with last-`::`-segment names,
constant-receiver references refs, require/require_relative path refs
(posix-normalized, `.rb`-suffixed, `Kernel.require` and interpolated-path
quirks included), `=begin` docstring marker survival, and the reverse-order
value-ref DFS.

Wire v2: the hook's mixin `implements` refs carry `filePath: ctx.filePath` —
the ONE extraction-ref denormalized field (php's trait-use refs share the
shape). RefRow's first pad byte becomes a flags slot (REF_FLAG_FILE_PATH);
decode re-attaches its own filePath parameter; KERNEL_ABI_VERSION 1→2 on both
sides (mismatched dist/.node pairs degrade to wasm, as designed).

Gates: sweeps 0-diff sinatra 147/147, jekyll 164/164, rails 3452/3452 (3,763
files, 0 deferrals — ruby error incidence 0.00%, any deferral = walker bug);
full-init dumps byte-identical ×3 (7.2k/9.4k/375.6k lines); kernel-ruby-parity
suite (torture + CRLF + wire-flag pin + defer) + ruby grammar-parity row;
full suite 2,613 green ×2 under CODEGRAPH_KERNEL_EXPECT=1 (one unrelated
mcp-initialize timing flake under parallel load, passes solo 3/3 ×3).
DEFAULT_ROUTED += ruby (12 langs).

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

__tests__/fixtures/kernel-parity/torture.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+require "json"
+require "sidekiq/fetch"
+require_relative "../shared/helper"
+require %q(pct/lib)
+require :sym_form
+Kernel.require "some/lib"
+require "interp/#{RUBY_VERSION}"
+
+TOP_LIMIT = 100
+RETRY_MAX = 3
+lower_conf = "x"
+top_var = compute_default
+m1, m2 = 3, 4
+counter ||= 0
+@ivar_top = 1
+$global_top = 2
+X_WITH_NEW = Widget.new
+WORDS = %w[alpha beta]
+
+module Outer
+  module Inner
+    class Deep < Base
+      before_action :hooked
+      def greet; end
+    end
+  end
+end
+
+module A::B
+  A_B_CONST = 5
+end
+
+class TopDoc; end
+
+class Bare; end
+
+class A::B::C
+  def compact_method; end
+end
+
+class ExprSuper < Struct.new(:a)
+end
+
+class ScopedSuper < A::B::C
+end
+
+# Class docs line one.
+# Class docs line two.
+class Documented < BaseThing
+  include Comparable
+  include Helpers, Sortable
+  extend self
+  prepend Memoizer
+  Foo.include Bar
+  include dynamic_mod()
+
+  CLASS_LIMIT = 9
+  not_extracted = 1
+
+  attr_accessor :name, :email
+  attr_reader :created_at
+  alias_method :handle, :name
+  alias fast_name name
+
+  define_method :dyn do
+    invisible_call
+  end
+
+  before_action :hooked_here
+  skip_before_action :check_thing, only: [:index]
+  around_create :wrap_it
+  rescue_from ArgumentError, with: :render_404
+  validates :name
+  validate :custom_check
+  helper_method :help_me
+  set_callback :save, :before, :cb_sym
+
+  def self.build(arg)
+    helper_build(arg)
+  end
+
+  class << self
+    def singleton_style
+      reset_cache
+    end
+  end
+
+  def done?; end
+  def save!; end
+  def value=(v); end
+
+  def visible_before; end
+
+  private
+
+  def still_public_after_bare_private; end
+
+  private :some_sym
+
+  def now_private; end
+
+  private def inner_public; end
+
+  def also_private; end
+
+  public def public_again; end
+
+  def after_public_def; end
+
+  protected
+
+  def bare_protected_is_invisible_so_still_public; end
+
+  def target_cb; end
+
+  def wire(list, button)
+    register(method(:target_cb))
+    register(&method(:target_cb))
+    register(method(:missing_cb))
+    store = method(:target_cb)
+    list.each(&:map_sym)
+  end
+
+  def calls_zoo(cb, lower, neg, other, deep, obj, h)
+    reset
+    compute(1)
+    puts "hi"
+    done? ? cleanup_now : 0
+    helper unless finished?
+    compute rescue nil
+    self.own_method
+    @name.upcase
+    @@class_var.inspect
+    $global.freeze
+    "literal".upcase
+    5.times { beep }
+    [1, 2].sum
+    Klass.static_call
+    NS::Klass.other_call
+    RETRY_MAX.times
+    Widget.new
+    NS::Widget.new
+    lower.new
+    chained.first_call.second_call
+    Widget.create(cb).save
+    deep&.safe_call
+    neg&.!= other
+    obj.attr = 1
+    obj.attr2 &&= refresh
+    h[:k] = 2
+    puts "#{@name.upcase} interp"
+    body = <<~SQL
+      select #{col_name.downcase}
+    SQL
+    yield
+    lam = lambda { inner_lambda_call }
+    pr = proc { inner_proc_call }
+    arrow = ->(a) { arrow_call(a) }
+  end
+
+  def flow_control(list)
+    begin
+      begin_stmt_call
+    rescue ArgumentError => e
+      rescue_stmt_call
+    else
+      else_stmt_call
+    ensure
+      ensure_stmt_call
+    end
+    case list
+    when Array then when_then_call
+    end
+    while list.any? do
+      while_body_call
+      break
+    end
+    if list
+      then_call
+    end
+  end
+
+  def body_requires
+    require "in/body"
+    Kernel.require "other/lib"
+  end
+
+  def container_method
+    def nested_def; end
+    class InnerClass; end
+    module InnerMod
+      def mod_method; end
+    end
+    nested_def
+  end
+
+  def reader_one
+    TOP_LIMIT + CLASS_LIMIT
+  end
+
+  def reader_two
+    a = RETRY_MAX
+    b = TOP_LIMIT
+    lower_conf
+  end
+
+  def shadower
+    lower_conf = "y"
+    lower_conf.upcase
+  end
+end
+
+# Doc for documented_fn.
+# Second line.
+def documented_fn; end
+
+=begin
+Block comment body.
+* star line
+-- dash line
+=end
+def block_doc_fn; end
+
+# lost doc
+private def priv_doc_fn; end
+
+def self.top_singleton
+  singleton_body_call
+end
+
+def obj.weird_singleton; end
+
+__END__
+raw data trailer here

__tests__/kernel-grammar-parity.test.ts

@@ -36,7 +36,7 @@ const kernelBuilt = fs.existsSync(KERNEL_PATH);
 
 // Every kernel-capable language. `jsx` shares the javascript grammar on BOTH
 // paths (langs.rs mirrors WASM_GRAMMAR_FILES), so the distinct grammars are:
-const GRAMMAR_LANGUAGES: Language[] = ['typescript', 'tsx', 'javascript', 'java', 'python', 'go', 'c', 'cpp', 'rust', 'csharp'];
+const GRAMMAR_LANGUAGES: Language[] = ['typescript', 'tsx', 'javascript', 'java', 'python', 'go', 'c', 'cpp', 'rust', 'csharp', 'ruby'];
 
 describe.skipIf(!kernelBuilt)('kernel↔wasm grammar parity', () => {
   beforeAll(async () => {

__tests__/kernel-ruby-parity.test.ts

@@ -0,0 +1,134 @@
+/**
+ * Kernel↔wasm Ruby extraction parity (R7b of the kernel migration).
+ *
+ * Asserts the native walker (codegraph-kernel/src/ruby.rs) produces the SAME
+ * ExtractionResult as the wasm TreeSitterExtractor — nodes, edges, and
+ * unresolved refs compared as canonicalized multisets — over the checked-in
+ * torture fixture (torture.rb: the importTypes:['call'] funnel and its
+ * class-body DSL blindness, mixin implements refs WITH filePath (the v2
+ * ref-flag wire path), module nesting + the hook multiply-capture, the
+ * sibling-scan visibility trio, require/require_relative path refs incl. the
+ * Kernel.require and interpolated-path quirks, the ruby call branch
+ * (`.new` instantiates, constant-receiver references, `&.` joins, raw chain
+ * text), bare-call statements (do…end vs brace-block blindness), heredocs,
+ * `=begin` docstring marker survival, value-ref targets + shadow prune,
+ * `__END__` trailer) and its CRLF variant (derived in-memory — #1329).
+ *
+ * The full-repo sweep lives in scripts/kernel-parity.mjs (sinatra/jekyll/
+ * rails for the §5 gate); this suite keeps the invariant alive in `npm test`.
+ * Skips when no kernel binary is staged; CODEGRAPH_KERNEL_EXPECT=1 turns that
+ * into a failure (kernel-scaffold.test.ts).
+ */
+
+import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import { extractFromSource } from '../src/extraction';
+import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
+import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel';
+import type { ExtractionResult } from '../src/types';
+
+const KERNEL_PATH = path.join(
+  __dirname,
+  '..',
+  'codegraph-kernel',
+  'prebuilds',
+  `${process.platform}-${process.arch}`,
+  'codegraph-kernel.node'
+);
+const kernelBuilt = fs.existsSync(KERNEL_PATH);
+
+const FIXTURE_DIR = path.join(__dirname, 'fixtures', 'kernel-parity');
+
+function canon(result: ExtractionResult): { nodes: string[]; edges: string[]; refs: string[] } {
+  return {
+    nodes: result.nodes
+      .map(({ updatedAt: _u, ...n }) => JSON.stringify(n, Object.keys(n).sort()))
+      .sort(),
+    edges: result.edges.map((e) => JSON.stringify(e, Object.keys(e).sort())).sort(),
+    refs: result.unresolvedReferences
+      .map((r) => JSON.stringify(r, Object.keys(r).sort()))
+      .sort(),
+  };
+}
+
+const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const;
+let savedEnv: Record<string, string | undefined>;
+
+describe.skipIf(!kernelBuilt)('kernel Ruby extraction parity', () => {
+  beforeAll(async () => {
+    await initGrammars();
+    await loadGrammarsForLanguages(['ruby']);
+  });
+
+  beforeEach(() => {
+    savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
+    resetKernelForTests();
+  });
+
+  afterEach(() => {
+    for (const k of ENV_KEYS) {
+      if (savedEnv[k] === undefined) delete process.env[k];
+      else process.env[k] = savedEnv[k];
+    }
+    resetKernelForTests();
+  });
+
+  function assertParity(filePath: string, source: string, minNodes = 3): void {
+    process.env.CODEGRAPH_KERNEL_LANGS = 'all';
+    delete process.env.CODEGRAPH_KERNEL;
+    const viaKernel = tryKernelExtract(filePath, source, 'ruby');
+    expect(viaKernel, `kernel extraction failed for ${filePath}`).not.toBeNull();
+
+    process.env.CODEGRAPH_KERNEL = '0';
+    const viaWasm = extractFromSource(filePath, source, 'ruby');
+    delete process.env.CODEGRAPH_KERNEL;
+
+    const k = canon(viaKernel!);
+    const w = canon(viaWasm);
+    expect(k.nodes, `${filePath}: nodes`).toEqual(w.nodes);
+    expect(k.edges, `${filePath}: edges`).toEqual(w.edges);
+    expect(k.refs, `${filePath}: refs`).toEqual(w.refs);
+    expect(viaWasm.nodes.length).toBeGreaterThanOrEqual(minNodes);
+  }
+
+  it('torture fixture: modules/mixins, visibility scan, requires, call zoo, value refs', () => {
+    const file = path.join(FIXTURE_DIR, 'torture.rb');
+    assertParity('fixtures/torture.rb', fs.readFileSync(file, 'utf8'), 30);
+  });
+
+  // CRLF variant — the shape every Windows autocrlf checkout has. Derived in
+  // memory so no platform or editor can silently normalize it away; pins the
+  // JS-multiline-^ docstring semantics for `#` runs and `=begin` bodies
+  // (#1329), plus heredoc/`%`-literal CRLF parsing.
+  it('torture fixture CRLF parity', () => {
+    const file = path.join(FIXTURE_DIR, 'torture.rb');
+    const crlf = fs.readFileSync(file, 'utf8').replace(/(?<!\r)\n/g, '\r\n');
+    assertParity('fixtures/torture.rb (crlf)', crlf, 30);
+  });
+
+  it('mixin implements refs carry filePath through the v2 ref-flag wire path', () => {
+    const src = 'class Widget\n  include Comparable\nend\n';
+    process.env.CODEGRAPH_KERNEL_LANGS = 'all';
+    delete process.env.CODEGRAPH_KERNEL;
+    const viaKernel = tryKernelExtract('src/widget.rb', src, 'ruby');
+    expect(viaKernel).not.toBeNull();
+    const impl = viaKernel!.unresolvedReferences.find((r) => r.referenceKind === 'implements');
+    expect(impl?.referenceName).toBe('Comparable');
+    expect(impl?.filePath).toBe('src/widget.rb');
+    // Ordinary refs stay un-denormalized.
+    const other = viaKernel!.unresolvedReferences.find((r) => r.referenceKind !== 'implements');
+    if (other) expect(other.filePath).toBeUndefined();
+  });
+
+  it('files with parse errors defer to the wasm extractor (recovery is encoding-dependent)', () => {
+    const broken = 'def broken(\n  x = [1,\nend\n';
+    process.env.CODEGRAPH_KERNEL_LANGS = 'all';
+    delete process.env.CODEGRAPH_KERNEL;
+    expect(tryKernelExtract('src/broken.rb', broken, 'ruby')).toBeNull();
+    process.env.CODEGRAPH_KERNEL = '0';
+    const viaWasm = extractFromSource('src/broken.rb', broken, 'ruby');
+    delete process.env.CODEGRAPH_KERNEL;
+    expect(viaWasm.nodes.some((n) => n.kind === 'file')).toBe(true);
+  });
+});

__tests__/kernel-scaffold.test.ts

@@ -73,11 +73,11 @@ describe.skipIf(!kernelBuilt)('kernel scaffold', () => {
   });
 
   it('TS/JS family + Java + Python + Go route to the kernel by default; others stay wasm', () => {
-    for (const lang of ['typescript', 'tsx', 'javascript', 'jsx', 'java', 'python', 'go'] as const) {
+    for (const lang of ['typescript', 'tsx', 'javascript', 'jsx', 'java', 'python', 'go', 'ruby'] as const) {
       expect(kernelRoutes(lang), lang).toBe(true);
     }
-    expect(kernelRoutes('ruby')).toBe(false);
-    expect(tryKernelExtract('src/a.rb', 'def f\nend\n', 'ruby')).toBeNull();
+    expect(kernelRoutes('php')).toBe(false);
+    expect(tryKernelExtract('src/a.php', '<?php function f() {}\n', 'php')).toBeNull();
     // CODEGRAPH_KERNEL_LANGS REPLACES the default set when present.
     process.env.CODEGRAPH_KERNEL_LANGS = 'tsx';
     expect(kernelRoutes('typescript')).toBe(false);