From b20f443438ccc987319506fae1c86001cbc1c682 Mon Sep 17 00:00:00 2001
From: Theo Ephraim <theo@dmno.dev>
Date: Fri, 3 Jul 2026 22:57:21 -0700
Subject: [PATCH 1/6] feat(gen-tm): line-comment introducer regions for
 structured-comment dialects
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A grammar whose comment token matches only the INTRODUCER (a bare `#`) while
the comment content is parser-tokenized (env-spec decorator comments) renders
badly from flat token rules: the prose after `#` keeps the ordinary text-token
scopes (string.unquoted, constant.numeric, ...), so themes color comments like
code.

New highlight-only token metadata — mirroring `interpolation` — declares the
shape instead:

    token('#', { scope: 'comment.line', lineComment: { richStarters: [DEC_NAME] } })

gen-tm then emits to-end-of-line regions in place of the flat rule:
- a RICH region gated on a lookahead that a declared rich-starter opens the
  body (`# @decorator(...)`) — full token highlighting applies inside via
  $self, on top of the comment base scope, so structured comments keep their
  colors while stray text dims;
- a PLAIN region for every other comment — no inner patterns, so the body
  falls to the comment scope and dims like any comment.

The introducer captures as punctuation.definition.comment. Lexer/parser and
all other generators are untouched; grammars without the metadata are
byte-identical (npm run gen produces no diffs for the six built-in grammars).

Also registers test/env-spec-regressions.ts as a check gate (it was previously
manual-only) and extends it with the new contracts.
---
 src/api.ts                   | 16 +++++++++
 src/gen-tm.ts                | 40 ++++++++++++++++++++++
 src/types.ts                 |  8 +++++
 test/check.ts                |  1 +
 test/env-spec-regressions.ts | 64 +++++++++++++++++++++++++++++++++++-
 5 files changed, 128 insertions(+), 1 deletion(-)

diff --git a/src/api.ts b/src/api.ts
index bb9856f..16acecf 100644
--- a/src/api.ts
+++ b/src/api.ts
@@ -19,6 +19,12 @@ interface TokenOptions {
   // Highlight-only interpolation regions for ordinary string tokens (e.g. env-spec `${…}` / `$(…)`).
   // The parser/lexer stay token-based; generators re-express these as nested regions.
   interpolation?: StringInterpolation | StringInterpolation[];
+  // Highlight-only: this comment token matches only the INTRODUCER (a bare `#`) while the
+  // comment runs to end-of-line with parser-tokenized content (a structured-comment dialect).
+  // Generators emit a to-EOL region in the token's comment scope so prose dims; `richStarters`
+  // lists tokens that keep full token highlighting when one opens the comment body
+  // (e.g. env-spec `# @decorator(...)`). See TokenDecl.lineComment.
+  lineComment?: { richStarters?: TokenRef[] };
   // A regex matching exactly one well-formed escape sequence. Engine-scanned tokens
   // (templates) validate each `\`-escape against it and reject any that don't match —
   // unlike `escape` (highlight-only), this drives tokenization. Skipped in tag
@@ -547,6 +553,16 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
         ? (Array.isArray(tok.opts.interpolation) ? tok.opts.interpolation : [tok.opts.interpolation]).map((i) => ({ ...i }))
         : undefined,
       escapeValidPattern: tok.opts.escapeValid,
+      // richStarters are TokenRefs; resolve to declared names (unknown refs are a config error)
+      lineComment: tok.opts.lineComment
+        ? {
+          richStarters: (tok.opts.lineComment.richStarters ?? []).map((ref) => {
+            const refName = names.get(ref);
+            if (!refName) throw new Error(`lineComment.richStarters on token '${name}' references an undeclared token`);
+            return refName;
+          }),
+        }
+        : undefined,
       embed: tok.opts.embed,
       identifier: tok.opts.identifier,
       identifierPrefix: tok.opts.identifierPrefix,
diff --git a/src/gen-tm.ts b/src/gen-tm.ts
index 92d9c8f..68e4c41 100644
--- a/src/gen-tm.ts
+++ b/src/gen-tm.ts
@@ -5012,6 +5012,46 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
       repository[key] = blockEntry;
       topPatterns.push({ include: `#${key}` });
 
+    } else if (tok.lineComment && scope.startsWith('comment.')) {
+      // ── Line-comment INTRODUCER (declared, highlight-only) ──
+      // The token matches only the marker (a bare `#`) while the comment runs to
+      // end-of-line with content the PARSER still tokenizes (a structured-comment
+      // dialect — env-spec decorator comments). A flat rule would leave the comment
+      // PROSE scoped by the ordinary text tokens (string.unquoted…), so themes render
+      // comments like code. Emit begin/end regions to `$` instead:
+      //   1. a RICH region, gated on a lookahead that one of the declared
+      //      `richStarters` opens the body (`# @decorator(...)`) — full token
+      //      highlighting applies INSIDE (via $self) on top of the comment base
+      //      scope, so structured comments keep their colors while stray text dims;
+      //   2. a PLAIN region for every other comment — no inner patterns, so the
+      //      whole body falls to the comment scope and dims like any comment.
+      // Ordering: rich is registered first so the gated form wins; both sit at this
+      // token's position in the top-level pattern list (declaration order preserved).
+      const introducer = tokenPatternSource(tok);
+      const commentPunct = `punctuation.definition.comment.${langName}`;
+      const richStarters = (tok.lineComment.richStarters ?? [])
+        .map((n) => grammar.tokens.find((t) => t.name === n))
+        .filter((t): t is TokenDecl => !!t);
+      if (richStarters.length) {
+        const startLookahead = richStarters.map((t) => `(?:${tokenPatternSource(t)})`).join('|');
+        repository[`${key}-rich`] = {
+          name: `${scope}.${langName}`,
+          begin: `(${introducer})(?=[ \\t]*(?:${startLookahead}))`,
+          beginCaptures: { '1': { name: commentPunct } },
+          end: '$',
+          patterns: [{ include: '$self' }],
+        };
+        topPatterns.push({ include: `#${key}-rich` });
+      }
+      repository[key] = {
+        name: `${scope}.${langName}`,
+        begin: `(${introducer})`,
+        beginCaptures: { '1': { name: commentPunct } },
+        end: '$',
+        patterns: [],
+      };
+      topPatterns.push({ include: `#${key}` });
+
     } else {
       // The bare-identifier catch-all is scoped `variable.other.readwrite` — the
       // TextMate convention for a mutable variable *reference* (what hand-written
diff --git a/src/types.ts b/src/types.ts
index 21186b0..6bb8754 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -23,6 +23,14 @@ export interface TokenDecl {
   scope?: string;         // @scope(...) override
   escapePattern?: TokenPattern; // @escape pattern — escape sequence pattern (highlight only)
   interpolation?: StringInterpolation[]; // highlight-only interpolation regions inside a string token (e.g. `${…}` / `$(…)`)
+  // Highlight-only: this comment-scoped token matches only the INTRODUCER (e.g. a bare `#`)
+  // while the comment runs to end-of-line with content the PARSER still tokenizes (a
+  // structured-comment dialect — env-spec decorator comments). Highlighter generators emit a
+  // to-end-of-line region carrying this token's comment scope so prose dims like any comment;
+  // `richStarters` names tokens that keep FULL token highlighting when one of them (after
+  // optional blanks) opens the comment body (`# @decorator(...)`). The lexer/parser are
+  // unaffected — exactly like `interpolation`, this is generator metadata.
+  lineComment?: { richStarters?: string[] };
   escapeValidPattern?: TokenPattern; // one well-formed escape; engine-scanned tokens reject non-matching `\`-escapes (skipped in tag position)
   embed?: string;         // @embed(lang) — embedded language scope name
   // ── Lexer hints (keep the engine language-agnostic; all optional) ──
diff --git a/test/check.ts b/test/check.ts
index 1658343..c11264f 100644
--- a/test/check.ts
+++ b/test/check.ts
@@ -31,6 +31,7 @@ const GATES: Gate[] = [
   { group: 'core', name: 'incremental-grammars', args: ['test/incremental-grammars.ts'] },
   { group: 'core', name: 'exhaustive-edits', args: ['test/exhaustive-edits.ts'] },
   { group: 'core', name: 'issue-cases', args: ['test/test-issues.ts'] },
+  { group: 'core', name: 'env-spec-regressions', args: ['test/env-spec-regressions.ts'] },
   { group: 'conformance', name: 'js', args: ['test/js-conformance.ts'] },
   { group: 'conformance', name: 'tsx', args: ['test/tsx-conformance.ts'] },
   { group: 'conformance', name: 'jsx', args: ['test/jsx-conformance.ts'] },
diff --git a/test/env-spec-regressions.ts b/test/env-spec-regressions.ts
index db0f44c..1f9ed1b 100644
--- a/test/env-spec-regressions.ts
+++ b/test/env-spec-regressions.ts
@@ -6,7 +6,7 @@
 //
 // Run with: node test/env-spec-regressions.ts
 import { createParser } from '../src/gen-parser.ts';
-import { defineGrammar, many, opt, rule, token, seq, star, altPattern, oneOf, noneOf, anyChar, never, range, plus, followedBy } from '../src/api.ts';
+import { defineGrammar, many, opt, rule, token, seq, star, altPattern, oneOf, noneOf, anyChar, never, range, plus, followedBy, notPrecededBy } from '../src/api.ts';
 import { generateTmLanguage } from '../src/gen-tm.ts';
 
 let ok = 0;
@@ -83,6 +83,68 @@ const check = (label: string, cond: boolean) => {
   check('parser: multiline inline quoted value is accepted without blockScalar', !threw);
 }
 
+// ---------------------------------------------------------------------------
+// Regression 3: a line-comment INTRODUCER token (`lineComment` metadata) emits
+// to-end-of-line REGIONS in TextMate, not a flat 1-char rule — so comment prose
+// dims to the comment scope while `richStarters`-led comments (env-spec decorator
+// comments `# @dec(...)`) keep full token highlighting inside.
+// ---------------------------------------------------------------------------
+{
+  const hspace = oneOf(' ', '\t');
+  const alpha = oneOf(range('a', 'z'), range('A', 'Z'));
+  const WS = token(plus(hspace), { skip: true, scope: 'meta.whitespace' });
+  const DEC_NAME = token(seq('@', plus(alpha)), { scope: 'variable.annotation' });
+  const HASH = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), {
+    scope: 'comment.line',
+    lineComment: { richStarters: [DEC_NAME] },
+  });
+  const KEY = token(seq(plus(alpha), followedBy('=')), { scope: 'entity.name.tag' });
+  const TEXT = token(plus(noneOf(' ', '\t', '\n', '#', '=', '@')), { scope: 'string.unquoted' });
+  const Part = rule(() => [DEC_NAME, TEXT]);
+  const Comment = rule(() => [[HASH, many(Part)]]);
+  const Item = rule(() => [[KEY, '=', opt(TEXT), opt(Comment)]]);
+  const Line = rule(() => [Item, Comment]);
+  const File = rule(() => [[many(Line)]]);
+  const grammar = defineGrammar({
+    name: 'env-spec-comments',
+    tokens: { WS, HASH, DEC_NAME, KEY, TEXT },
+    rules: { Part, Comment, Item, Line, File },
+    entry: File,
+  });
+
+  const tm = generateTmLanguage(grammar);
+  const plain = tm.repository.hash;
+  const rich = tm.repository['hash-rich'];
+  check('tm: plain comment entry is a to-EOL region', !!plain && plain.end === '$');
+  check('tm: plain comment region carries the comment scope', plain?.name === 'comment.line.env-spec-comments');
+  check('tm: plain comment region has NO inner patterns (prose dims)', Array.isArray(plain?.patterns) && plain.patterns.length === 0);
+  check('tm: rich comment entry exists and is gated on the rich starter', !!rich && typeof rich.begin === 'string' && rich.begin.includes('(?=[ \\t]*'));
+  check('tm: rich comment region keeps full token highlighting via $self', JSON.stringify(rich?.patterns) === JSON.stringify([{ include: '$self' }]));
+  check('tm: rich entry is tried before the plain entry', (() => {
+    const order = tm.patterns.map((p) => (p as { include?: string }).include);
+    return order.indexOf('#hash-rich') !== -1 && order.indexOf('#hash-rich') < order.indexOf('#hash');
+  })());
+  check('tm: introducer captures as comment punctuation', JSON.stringify(plain?.beginCaptures?.['1']) === JSON.stringify({ name: 'punctuation.definition.comment.env-spec-comments' }));
+
+  // parser behavior is UNaffected by the highlight-only metadata
+  const parser = createParser(grammar);
+  let threw = false;
+  try {
+    parser.parse('KEY=val # @dec note\n# plain prose');
+  } catch {
+    threw = true;
+  }
+  check('parser: lineComment metadata does not change parsing', !threw);
+
+  // a comment token WITHOUT the metadata still emits the flat rule (no behavior change)
+  const HASH2 = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), { scope: 'comment.line' });
+  const Comment2 = rule(() => [[HASH2, many(TEXT)]]);
+  const File2 = rule(() => [[many(Comment2)]]);
+  const flatGrammar = defineGrammar({ name: 'no-metadata', tokens: { HASH2, TEXT }, rules: { Comment2, File2 }, entry: File2 });
+  const tm2 = generateTmLanguage(flatGrammar);
+  check('tm: without lineComment metadata the comment token stays a flat match', typeof tm2.repository.hash2?.match === 'string' && tm2.repository.hash2?.begin === undefined);
+}
+
 console.log(
   fail === 0
     ? `\n${ok}/${ok} env-spec regression checks pass`

From 0195ef0df42be65f2ce6e88ee510d352c223a2d0 Mon Sep 17 00:00:00 2001
From: Theo Ephraim <theo@dmno.dev>
Date: Fri, 3 Jul 2026 23:19:06 -0700
Subject: [PATCH 2/6] =?UTF-8?q?feat(gen-tm):=20continuation=20brackets=20?=
 =?UTF-8?q?=E2=80=94=20multi-line=20constructs=20inside=20rich=20comments?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A bracket left OPEN in a rich comment continues the construct across
consecutive introducer-prefixed lines (env-spec multi-line decorator calls and
literals):

    # @import(
    #   ./.env.shared,
    #   pick=[
    #     KEY1, # note
    #   ],
    # )

Declared via `lineComment.continuationBrackets` (bracket pairs). Each pair
emits a begin/end region nested inside the line-scoped rich region — a
TextMate child region suspends its parent's $ end, so the construct spans
lines while single-line comments still close at EOL. Inside a construct:
a line-start introducer is a CONTINUATION MARKER (comment punctuation, not a
new comment); any other introducer opens an embedded comment that dims to
end-of-line; brackets nest recursively; everything else highlights via $self.

An unclosed bracket runs the region to the next closer — the standard
hand-written-grammar hazard; the parser stays the authority on validity.

Opt-in as before: `npm run gen` produces no diffs for the built-in grammars;
41/41 gates pass (env-spec regression contracts extended to 18).
---
 src/api.ts                   |  3 ++-
 src/gen-tm.ts                | 44 +++++++++++++++++++++++++++++++++++-
 src/types.ts                 |  7 +++++-
 test/env-spec-regressions.ts | 34 ++++++++++++++++++++++++++++
 4 files changed, 85 insertions(+), 3 deletions(-)

diff --git a/src/api.ts b/src/api.ts
index 16acecf..bde5b84 100644
--- a/src/api.ts
+++ b/src/api.ts
@@ -24,7 +24,7 @@ interface TokenOptions {
   // Generators emit a to-EOL region in the token's comment scope so prose dims; `richStarters`
   // lists tokens that keep full token highlighting when one opens the comment body
   // (e.g. env-spec `# @decorator(...)`). See TokenDecl.lineComment.
-  lineComment?: { richStarters?: TokenRef[] };
+  lineComment?: { richStarters?: TokenRef[]; continuationBrackets?: [string, string][] };
   // A regex matching exactly one well-formed escape sequence. Engine-scanned tokens
   // (templates) validate each `\`-escape against it and reject any that don't match —
   // unlike `escape` (highlight-only), this drives tokenization. Skipped in tag
@@ -561,6 +561,7 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
             if (!refName) throw new Error(`lineComment.richStarters on token '${name}' references an undeclared token`);
             return refName;
           }),
+          continuationBrackets: tok.opts.lineComment.continuationBrackets?.map((pair) => [...pair] as [string, string]),
         }
         : undefined,
       embed: tok.opts.embed,
diff --git a/src/gen-tm.ts b/src/gen-tm.ts
index 68e4c41..4c79984 100644
--- a/src/gen-tm.ts
+++ b/src/gen-tm.ts
@@ -5034,12 +5034,54 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
         .filter((t): t is TokenDecl => !!t);
       if (richStarters.length) {
         const startLookahead = richStarters.map((t) => `(?:${tokenPatternSource(t)})`).join('|');
+        // ── Multi-line constructs (`continuationBrackets`) ──
+        // A bracket left open in a rich comment continues the construct across consecutive
+        // introducer-prefixed lines (`# @import(` … `#   KEY1,` … `# )`). Each pair becomes a
+        // begin/end region nested INSIDE the line-scoped rich region — a TextMate child region
+        // suspends its parent's `$` end, so the construct spans lines while the parent still
+        // closes single-line comments. Inside a construct:
+        //   - a line-start introducer is a CONTINUATION MARKER (comment punctuation, not a new
+        //     comment) — matched first (anchored `^`, earliest-match wins);
+        //   - any other introducer opens an embedded comment to end-of-line that dims
+        //     (`#   KEY1, # note`), popping at `$` so the construct resumes next line;
+        //   - brackets nest recursively; everything else highlights via $self.
+        // An unclosed bracket runs the region to the next closer (the same hazard every
+        // hand-written TextMate grammar with multi-line constructs has) — the parser is the
+        // authority on validity, the highlighter degrades soft.
+        const contBrackets = tok.lineComment.continuationBrackets ?? [];
+        const bracketKeyOf = (open: string) => `${key}-rich-cont-${[...open].map((c) => c.charCodeAt(0).toString(16)).join('')}`;
+        const bracketIncludes = contBrackets.map(([open]) => ({ include: `#${bracketKeyOf(open)}` }));
+        if (contBrackets.length) {
+          repository[`${key}-rich-cont-marker`] = {
+            match: `^[ \\t]*(${introducer})`,
+            captures: { '1': { name: `${scope}.${langName} ${commentPunct}` } },
+          };
+          repository[`${key}-rich-cont-comment`] = {
+            name: `${scope}.${langName}`,
+            begin: `(${introducer})`,
+            beginCaptures: { '1': { name: commentPunct } },
+            end: '$',
+            patterns: [],
+          };
+          for (const [open, close] of contBrackets) {
+            repository[bracketKeyOf(open)] = {
+              begin: escapeRegex(open),
+              end: escapeRegex(close),
+              patterns: [
+                { include: `#${key}-rich-cont-marker` },
+                { include: `#${key}-rich-cont-comment` },
+                ...bracketIncludes,
+                { include: '$self' },
+              ],
+            };
+          }
+        }
         repository[`${key}-rich`] = {
           name: `${scope}.${langName}`,
           begin: `(${introducer})(?=[ \\t]*(?:${startLookahead}))`,
           beginCaptures: { '1': { name: commentPunct } },
           end: '$',
-          patterns: [{ include: '$self' }],
+          patterns: [...bracketIncludes, { include: '$self' }],
         };
         topPatterns.push({ include: `#${key}-rich` });
       }
diff --git a/src/types.ts b/src/types.ts
index 6bb8754..f94d370 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -30,7 +30,12 @@ export interface TokenDecl {
   // `richStarters` names tokens that keep FULL token highlighting when one of them (after
   // optional blanks) opens the comment body (`# @decorator(...)`). The lexer/parser are
   // unaffected — exactly like `interpolation`, this is generator metadata.
-  lineComment?: { richStarters?: string[] };
+  // `continuationBrackets`: bracket pairs that, when left OPEN inside a rich comment, continue
+  // the construct across consecutive introducer-prefixed lines (env-spec multi-line decorator
+  // calls/literals — `# @import(` … `#   KEY1,` … `# )`). Each opens a begin/end region that
+  // outlives the line-scoped comment region (a TextMate child region suspends its parent's end),
+  // with the line-start introducer scoped as a continuation marker rather than a new comment.
+  lineComment?: { richStarters?: string[]; continuationBrackets?: [string, string][] };
   escapeValidPattern?: TokenPattern; // one well-formed escape; engine-scanned tokens reject non-matching `\`-escapes (skipped in tag position)
   embed?: string;         // @embed(lang) — embedded language scope name
   // ── Lexer hints (keep the engine language-agnostic; all optional) ──
diff --git a/test/env-spec-regressions.ts b/test/env-spec-regressions.ts
index 1f9ed1b..0718382 100644
--- a/test/env-spec-regressions.ts
+++ b/test/env-spec-regressions.ts
@@ -136,6 +136,40 @@ const check = (label: string, cond: boolean) => {
   }
   check('parser: lineComment metadata does not change parsing', !threw);
 
+  // continuationBrackets: a bracket left open in a rich comment continues the construct
+  // across introducer-prefixed lines via nested begin/end regions
+  const HASH_ML = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), {
+    scope: 'comment.line',
+    lineComment: { richStarters: [DEC_NAME], continuationBrackets: [['(', ')'], ['[', ']']] },
+  });
+  const CommentMl = rule(() => [[HASH_ML, many(Part)]]);
+  const FileMl = rule(() => [[many(CommentMl)]]);
+  const mlGrammar = defineGrammar({
+    name: 'env-spec-ml',
+    tokens: { WS, HASH_ML, DEC_NAME, KEY, TEXT },
+    rules: { Part, CommentMl, FileMl },
+    entry: FileMl,
+  });
+  const tmMl = generateTmLanguage(mlGrammar);
+  const parenKey = Object.keys(tmMl.repository).find((k) => k.startsWith('hash_ml-rich-cont-') && tmMl.repository[k].begin === '\\(');
+  check('tm: continuation bracket pair emits a begin/end region', !!parenKey && tmMl.repository[parenKey!].end === '\\)');
+  const parenRegion = parenKey ? tmMl.repository[parenKey] : undefined;
+  const parenIncludes = (parenRegion?.patterns ?? []).map((pp) => (pp as { include?: string }).include);
+  check('tm: construct interior tries marker, embedded comment, nested brackets, then $self',
+    parenIncludes[0] === '#hash_ml-rich-cont-marker'
+    && parenIncludes[1] === '#hash_ml-rich-cont-comment'
+    && parenIncludes.includes('$self')
+    && parenIncludes.filter((n) => n?.startsWith('#hash_ml-rich-cont-') && n !== '#hash_ml-rich-cont-marker' && n !== '#hash_ml-rich-cont-comment').length === 2);
+  const marker = tmMl.repository['hash_ml-rich-cont-marker'];
+  check('tm: continuation marker is line-anchored and scoped as comment punctuation',
+    typeof marker?.match === 'string' && marker.match.startsWith('^[ \\t]*')
+    && JSON.stringify(marker?.captures?.['1']?.name ?? '').includes('punctuation.definition.comment'));
+  const embedded = tmMl.repository['hash_ml-rich-cont-comment'];
+  check('tm: embedded comment inside a construct dims to end-of-line', embedded?.end === '$' && Array.isArray(embedded?.patterns) && embedded.patterns.length === 0);
+  const richMl = tmMl.repository['hash_ml-rich'];
+  const richIncludes = (richMl?.patterns ?? []).map((pp) => (pp as { include?: string }).include);
+  check('tm: rich region tries construct brackets before $self', richIncludes[richIncludes.length - 1] === '$self' && richIncludes.length === 3);
+
   // a comment token WITHOUT the metadata still emits the flat rule (no behavior change)
   const HASH2 = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), { scope: 'comment.line' });
   const Comment2 = rule(() => [[HASH2, many(TEXT)]]);

From 09f1a165390301fb75129fdd125080bf6c07a09a Mon Sep 17 00:00:00 2001
From: Theo Ephraim <theo@dmno.dev>
Date: Fri, 3 Jul 2026 23:43:03 -0700
Subject: [PATCH 3/6] feat: contextualScopes (rule-context token scopes) +
 lineComment markup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two more generic highlight-fidelity features hand-written grammars have that
generated ones lacked:

1. `contextualScopes` — grammar-level, highlight-only: token T carries scope S
   when it appears within rule R (T's immediate enclosing rule):

       contextualScopes: [
         { token: ASSIGN_KEY, within: [FunctionArgKeyValue], scope: 'entity.other.attribute-name' },
       ]

   Each generator consumes the SAME declaration at its own fidelity:
   - tree-sitter: exact `(rule (token) @capture)` queries, emitted last
     (highlight resolution is last-wins) so they override the token's flat
     capture inside the declared rules only;
   - gen-tm: the flat grammar approximates rule context with derived CONSTRUCT
     regions — a call-argument region (callee + `(` … `)`, nested bracket
     regions inside, only emitted when contextual scopes are declared) and the
     lineComment continuation-bracket interiors — where the override rules are
     tried before the token's flat rule; top-level occurrences keep the
     declared scope. The callee capture reuses the grammar's own callee-token
     scope when one is declared.

   Motivating case: env-spec option keys (`fn(retry=3)`, `@import(pick=…)`)
   styling as attribute names, distinct from top-level env var keys.

2. `lineComment.markup` — declared doc-markup patterns (token-pattern IR, e.g.
   `**bold**` / `__italic__`) highlighted inside PLAIN comment bodies.

Opt-in as before: npm run gen produces no diffs for the six built-in grammars;
41/41 gates pass (env-spec regression contracts extended to 25).
---
 src/api.ts                   | 23 ++++++++++-
 src/gen-tm.ts                | 74 +++++++++++++++++++++++++++++++++++-
 src/gen-treesitter.ts        | 24 ++++++++++++
 src/types.ts                 | 16 +++++++-
 test/env-spec-regressions.ts | 72 +++++++++++++++++++++++++++++++++++
 5 files changed, 203 insertions(+), 6 deletions(-)

diff --git a/src/api.ts b/src/api.ts
index bde5b84..99d693f 100644
--- a/src/api.ts
+++ b/src/api.ts
@@ -24,7 +24,11 @@ interface TokenOptions {
   // Generators emit a to-EOL region in the token's comment scope so prose dims; `richStarters`
   // lists tokens that keep full token highlighting when one opens the comment body
   // (e.g. env-spec `# @decorator(...)`). See TokenDecl.lineComment.
-  lineComment?: { richStarters?: TokenRef[]; continuationBrackets?: [string, string][] };
+  lineComment?: {
+    richStarters?: TokenRef[];
+    continuationBrackets?: [string, string][];
+    markup?: { pattern: TokenPattern; scope: string }[];
+  };
   // A regex matching exactly one well-formed escape sequence. Engine-scanned tokens
   // (templates) validate each `\`-escape against it and reject any that don't match —
   // unlike `escape` (highlight-only), this drives tokenization. Skipped in tag
@@ -513,6 +517,9 @@ interface GrammarConfig {
   ledPrec?: LedPrec[];
   rules: Record<string, RuleRef>;
   scopes?: Record<string, string[]>;
+  // Highlight-only contextual token scopes: token T carries scope S within rule R (T's
+  // immediate enclosing rule) — see CstGrammar.contextualScopes for generator fidelity.
+  contextualScopes?: { token: TokenRef; within: RuleRef | RuleRef[]; scope: string }[];
   entry: RuleRef;
   markup?: MarkupConfig;  // opt-in markup-mode tokenization (HTML/Vue)
   indent?: IndentConfig;  // opt-in indentation-sensitive tokenization (YAML)
@@ -562,6 +569,7 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
             return refName;
           }),
           continuationBrackets: tok.opts.lineComment.continuationBrackets?.map((pair) => [...pair] as [string, string]),
+          markup: tok.opts.lineComment.markup?.map((m) => ({ ...m })),
         }
         : undefined,
       embed: tok.opts.embed,
@@ -608,5 +616,16 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
     }
   }
 
-  return { name: config.name, scopeName: config.scopeName, tokens, precs, ledPrecs: config.ledPrec, rules, scopeOverrides, markup: config.markup, indent: config.indent, newline: config.newline, expressionRule: config.expression ? names.get(config.expression) : undefined, aliasScopes: config.aliasScopes, canonicalRepoNames: config.canonicalRepoNames, manifest: config.manifest };
+  const contextualScopes = (config.contextualScopes ?? []).map((entry) => {
+    const tokenName = names.get(entry.token);
+    if (!tokenName) throw new Error('contextualScopes entry references an undeclared token');
+    const withinRefs = Array.isArray(entry.within) ? entry.within : [entry.within];
+    const within = withinRefs.map((ref) => {
+      const ruleName = names.get(ref);
+      if (!ruleName) throw new Error(`contextualScopes entry for token '${tokenName}' references an undeclared rule`);
+      return ruleName;
+    });
+    return { token: tokenName, within, scope: entry.scope };
+  });
+  return { name: config.name, scopeName: config.scopeName, tokens, precs, ledPrecs: config.ledPrec, rules, scopeOverrides, contextualScopes, markup: config.markup, indent: config.indent, newline: config.newline, expressionRule: config.expression ? names.get(config.expression) : undefined, aliasScopes: config.aliasScopes, canonicalRepoNames: config.canonicalRepoNames, manifest: config.manifest };
 }
diff --git a/src/gen-tm.ts b/src/gen-tm.ts
index 4c79984..5895169 100644
--- a/src/gen-tm.ts
+++ b/src/gen-tm.ts
@@ -4819,6 +4819,23 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
   // to the region's own `meta.type.*` name. The classification is the same
   // scope-family test the token loop already uses, so the set is fully derived
   // from the grammar's own token scopes — no hardcoded `"`/digit literals.
+  // ── Contextual token-scope overrides (grammar.contextualScopes, highlight-only) ──
+  // Token T carries scope S within rule R. The FLAT grammar approximates rule context with its
+  // derived CONSTRUCT regions — call-argument lists and lineComment continuation brackets —
+  // where these override rules are tried before the token's flat rule; top-level occurrences
+  // keep the token's declared scope. (tree-sitter consumes the same declaration exactly, as
+  // `(rule (token) @capture)` queries.)
+  const ctxOverrideIncludes = (grammar.contextualScopes ?? []).map((entry, i) => {
+    const target = grammar.tokens.find((t) => t.name === entry.token);
+    if (!target) return null;
+    const ctxKey = `ctx-scope-${i}-${entry.token.toLowerCase()}`;
+    repository[ctxKey] ??= {
+      match: tokenPatternSource(target),
+      name: `${entry.scope}.${langName}`,
+    };
+    return { include: `#${ctxKey}` };
+  }).filter((x): x is { include: string } => !!x);
+
   const literalLiteralKeys: string[] = [];
   const literalTokenNames = new Set<string>();
   const rememberLiteralKey = (scope: string, repoKey: string, tokName?: string) => {
@@ -5029,6 +5046,12 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
       // token's position in the top-level pattern list (declaration order preserved).
       const introducer = tokenPatternSource(tok);
       const commentPunct = `punctuation.definition.comment.${langName}`;
+      // Declared doc-markup patterns (`**bold**`, `__italic__`, …) highlighted inside
+      // PLAIN comment bodies — token-pattern IR, nothing language-specific.
+      const markupPatterns: TmPattern[] = (tok.lineComment.markup ?? []).map((m) => ({
+        match: tokenPatternToRegex(m.pattern),
+        name: `${m.scope}.${langName}`,
+      }));
       const richStarters = (tok.lineComment.richStarters ?? [])
         .map((n) => grammar.tokens.find((t) => t.name === n))
         .filter((t): t is TokenDecl => !!t);
@@ -5061,7 +5084,7 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
             begin: `(${introducer})`,
             beginCaptures: { '1': { name: commentPunct } },
             end: '$',
-            patterns: [],
+            patterns: markupPatterns,
           };
           for (const [open, close] of contBrackets) {
             repository[bracketKeyOf(open)] = {
@@ -5070,6 +5093,7 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
               patterns: [
                 { include: `#${key}-rich-cont-marker` },
                 { include: `#${key}-rich-cont-comment` },
+                ...ctxOverrideIncludes,
                 ...bracketIncludes,
                 { include: '$self' },
               ],
@@ -5090,7 +5114,7 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
         begin: `(${introducer})`,
         beginCaptures: { '1': { name: commentPunct } },
         end: '$',
-        patterns: [],
+        patterns: markupPatterns,
       };
       topPatterns.push({ include: `#${key}` });
 
@@ -7518,6 +7542,49 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
     topPatterns.push({ include: '#function-call' });
   }
 
+  // ── 4b2. Call-argument construct regions (contextualScopes) ──
+  // Only when the grammar declares contextual scopes: a call's argument list becomes a real
+  // begin/end region (callee + `(` … `)`), so the overrides apply inside it — and inside any
+  // nested bracket constructs — while everything else still highlights via $self. The nested
+  // pair regions are NOT top-level includes (a bare top-level `{…}` value keeps flat token
+  // scopes); they exist only inside a call or inside each other.
+  if (hasCallExpr && ctxOverrideIncludes.length) {
+    const allRuleLits = new Set<string>();
+    for (const r of grammar.rules) for (const lit of collectLiterals(r.body)) allRuleLits.add(lit);
+    const ctxPairs = ([['(', ')'], ['[', ']'], ['{', '}']] as [string, string][])
+      .filter(([o, c]) => allRuleLits.has(o) && allRuleLits.has(c) && (o !== '(' || true));
+    const pairKeyOf = (open: string) => `ctx-pair-${[...open].map((ch) => ch.charCodeAt(0).toString(16)).join('')}`;
+    const pairIncludes = ctxPairs.filter(([o]) => o !== '(').map(([o]) => ({ include: `#${pairKeyOf(o)}` }));
+    for (const [open, close] of ctxPairs) {
+      if (open === '(') continue;   // `(` is owned by the call region itself
+      repository[pairKeyOf(open)] = {
+        begin: escapeRegex(open),
+        beginCaptures: { '0': { name: `${getScope(scopeOverrides, open) ?? 'punctuation.section.brackets.begin'}.${langName}` } },
+        end: escapeRegex(close),
+        endCaptures: { '0': { name: `${getScope(scopeOverrides, close) ?? 'punctuation.section.brackets.end'}.${langName}` } },
+        patterns: [...ctxOverrideIncludes, ...pairIncludes, { include: '$self' }],
+      };
+    }
+    // callee scope: reuse the grammar's own callee token scope when one is declared (a token
+    // whose pattern is gated on a following `(`), else the TextMate-conventional call scope
+    const calleeTok = grammar.tokens.find((t) => {
+      const sc = t.scope ?? classifyToken(t).scope;
+      return (sc.startsWith('variable.function') || sc.startsWith('entity.name.function')) && tokenPatternSource(t).includes('\\(');
+    });
+    const calleeScope = calleeTok?.scope ?? 'entity.name.function';
+    repository['ctx-call-args'] = {
+      begin: `(${identPattern})\\s*(\\()`,
+      beginCaptures: {
+        '1': { name: `${calleeScope}.${langName}` },
+        '2': { name: `${getScope(scopeOverrides, '(') ?? 'punctuation.section.parens.begin'}.${langName}` },
+      },
+      end: '\\)',
+      endCaptures: { '0': { name: `${getScope(scopeOverrides, ')') ?? 'punctuation.section.parens.end'}.${langName}` } },
+      patterns: [...ctxOverrideIncludes, ...pairIncludes, { include: '$self' }],
+    };
+    topPatterns.push({ include: '#ctx-call-args' });
+  }
+
   // ── 4b1a. Object method key: `key: (params) => ...` or `key: function(...)` ──
   // In object literals, a property key followed by `:` and a function value
   // should be scoped as entity.name.function (not plain variable.other).
@@ -8460,6 +8527,9 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
     // tried before #punctuation (which would otherwise claim the `{`/`[` as a bare bracket) and
     // before the scalar tokens. Its `{`/`[` can never lead a plain scalar, so this ranking is safe.
     if (key === 'flow-mapping' || key === 'flow-sequence') return 0.85;
+    // A call-argument construct region (contextualScopes) opens on `ident(` — it must be tried
+    // before the flat callee/ident token rules that would otherwise consume the name alone.
+    if (key === 'ctx-call-args') return 0.87;
     // A top-level token-match scoped `entity.name.tag` (e.g. an indentation grammar's mapping
     // KEY — a scalar that is the LHS of `:`) is a NAME, more specific than any string/typed-value
     // scalar it overlaps, so it must be tried first. (Markup tag names live inside begin/end
diff --git a/src/gen-treesitter.ts b/src/gen-treesitter.ts
index ff67d11..ac98eac 100644
--- a/src/gen-treesitter.ts
+++ b/src/gen-treesitter.ts
@@ -1313,6 +1313,7 @@ function scopeToCapture(scope: string): string | null {
     ['entity.name.function', '@function'],
     ['entity.name.type', '@type'],
     ['entity.name', '@constructor'],
+    ['entity.other.attribute-name', '@attribute'],
     ['entity.other.document', '@punctuation.delimiter'],  // YAML --- / ... markers
     ['entity.other.property', '@property'],
     ['support.type.primitive', '@type.builtin'],
@@ -1690,6 +1691,29 @@ function buildHighlightsScm(
   }
   out.push('');
 
+  // ── Contextual token scopes (grammar.contextualScopes, highlight-only) ──
+  // Token T carries scope S within rule R (T's immediate enclosing rule) — expressed
+  // exactly as `(rule (token) @capture)` queries. Emitted LAST: highlight resolution is
+  // last-wins (nvim-treesitter / tree-sitter-cli), so these override the token's flat
+  // capture above inside their declared rules only.
+  const ctxQueries: string[] = [];
+  for (const entry of grammar.contextualScopes ?? []) {
+    const cap = scopeToCapture(entry.scope);
+    if (!cap) continue;
+    const tokSnake = ctx.tokenSnake.get(entry.token);
+    if (!tokSnake) continue;
+    for (const ruleName of entry.within) {
+      const ruleSnake = ctx.ruleSnake.get(ruleName);
+      if (!ruleSnake) continue;
+      ctxQueries.push(`(${ruleSnake} (${tokSnake}) ${cap})`);
+    }
+  }
+  if (ctxQueries.length) {
+    out.push(';; Contextual token scopes (grammar.contextualScopes).');
+    for (const q of ctxQueries) out.push(q);
+    out.push('');
+  }
+
   return out.join('\n');
 }
 
diff --git a/src/types.ts b/src/types.ts
index f94d370..bf2817b 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -35,7 +35,13 @@ export interface TokenDecl {
   // calls/literals — `# @import(` … `#   KEY1,` … `# )`). Each opens a begin/end region that
   // outlives the line-scoped comment region (a TextMate child region suspends its parent's end),
   // with the line-start introducer scoped as a continuation marker rather than a new comment.
-  lineComment?: { richStarters?: string[]; continuationBrackets?: [string, string][] };
+  // `markup`: doc-markup patterns highlighted inside PLAIN comment bodies (declared as
+  // token-pattern IR — e.g. `**bold**` / `__italic__` — nothing language-specific here).
+  lineComment?: {
+    richStarters?: string[];
+    continuationBrackets?: [string, string][];
+    markup?: { pattern: TokenPattern; scope: string }[];
+  };
   escapeValidPattern?: TokenPattern; // one well-formed escape; engine-scanned tokens reject non-matching `\`-escapes (skipped in tag position)
   embed?: string;         // @embed(lang) — embedded language scope name
   // ── Lexer hints (keep the engine language-agnostic; all optional) ──
@@ -554,7 +560,13 @@ export interface CstGrammar {
   precs: PrecLevel[];
   ledPrecs?: LedPrec[];
   rules: RuleDecl[];
-  scopeOverrides: Map<string, string[]>;  // literal → scope overrides from `scopes` section (multiple if keyword appears in multiple groups)
+  scopeOverrides: Map<string, string[]>;
+  // Highlight-only CONTEXTUAL token scopes: token T carries scope S when it appears within
+  // rule R (T's immediate enclosing rule). Generators consume this at their own fidelity:
+  // tree-sitter emits exact `(rule (token) @capture)` queries; the TextMate generator applies
+  // the override inside its derived bracket-construct regions (call args, continuation
+  // brackets) — the flat top-level rules keep the token's declared scope.
+  contextualScopes?: { token: string; within: string[]; scope: string }[];  // literal → scope overrides from `scopes` section (multiple if keyword appears in multiple groups)
   name?: string;
   scopeName?: string;  // declared TextMate scope name (e.g. source.ts); its suffix drives every scope's language tag
   markup?: MarkupConfig;  // opt-in markup-mode tokenization (HTML/Vue); absent for token-stream languages
diff --git a/test/env-spec-regressions.ts b/test/env-spec-regressions.ts
index 0718382..16dc065 100644
--- a/test/env-spec-regressions.ts
+++ b/test/env-spec-regressions.ts
@@ -8,6 +8,7 @@
 import { createParser } from '../src/gen-parser.ts';
 import { defineGrammar, many, opt, rule, token, seq, star, altPattern, oneOf, noneOf, anyChar, never, range, plus, followedBy, notPrecededBy } from '../src/api.ts';
 import { generateTmLanguage } from '../src/gen-tm.ts';
+import { generateTreeSitter } from '../src/gen-treesitter.ts';
 
 let ok = 0;
 let fail = 0;
@@ -83,6 +84,11 @@ const check = (label: string, cond: boolean) => {
   check('parser: multiline inline quoted value is accepted without blockScalar', !threw);
 }
 
+// ---------------------------------------------------------------------------
+// Regression 4 (declared below regression 3's grammar pieces): contextualScopes +
+// lineComment.markup — context-dependent token scopes and declared comment markup.
+// ---------------------------------------------------------------------------
+
 // ---------------------------------------------------------------------------
 // Regression 3: a line-comment INTRODUCER token (`lineComment` metadata) emits
 // to-end-of-line REGIONS in TextMate, not a flat 1-char rule — so comment prose
@@ -179,6 +185,72 @@ const check = (label: string, cond: boolean) => {
   check('tm: without lineComment metadata the comment token stays a flat match', typeof tm2.repository.hash2?.match === 'string' && tm2.repository.hash2?.begin === undefined);
 }
 
+// ---------------------------------------------------------------------------
+// Regression 4: contextualScopes — token T carries scope S within rule R.
+//   tm: overrides apply inside derived construct regions (call args + continuation
+//       brackets); flat top-level rules keep the declared scope.
+//   tree-sitter: exact `(rule (token) @capture)` queries, emitted last (last-wins).
+// Plus lineComment.markup: declared doc-markup patterns inside plain comment bodies.
+// ---------------------------------------------------------------------------
+{
+  const hspace = oneOf(' ', '\t');
+  const alpha = oneOf(range('a', 'z'), range('A', 'Z'));
+  const WS = token(plus(hspace), { skip: true, scope: 'meta.whitespace' });
+  const DEC_NAME = token(seq('@', plus(alpha)), { scope: 'variable.annotation' });
+  const HASH = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), {
+    scope: 'comment.line',
+    lineComment: {
+      richStarters: [DEC_NAME],
+      continuationBrackets: [['(', ')']],
+      markup: [{ pattern: seq('**', star(noneOf('*', '\n')), '**'), scope: 'markup.bold' }],
+    },
+  });
+  const KEY = token(seq(plus(alpha), followedBy('=')), { scope: 'entity.name.tag' });
+  const FN_NAME = token(seq(plus(alpha), followedBy(seq(star(hspace), '('))), { scope: 'variable.function' });
+  const TEXT = token(plus(noneOf(' ', '\t', '\n', '#', '=', '@', '(', ')', ',')), { scope: 'string.unquoted' });
+  const ArgKV = rule(() => [[KEY, '=', TEXT]]);
+  const Arg = rule(() => [ArgKV, TEXT]);
+  const Args = rule(() => [['(', opt(Arg), ')']]);
+  const Call = rule(() => [[FN_NAME, Args]]);
+  const Part = rule(() => [DEC_NAME, Call, TEXT, KEY, '=', ',', '(', ')']);
+  const Comment = rule(() => [[HASH, many(Part)]]);
+  const Item = rule(() => [[KEY, '=', opt(Call), opt(Comment)]]);
+  const Line = rule(() => [Item, Comment]);
+  const File = rule(() => [[many(Line)]]);
+  const grammar = defineGrammar({
+    name: 'env-spec-ctx',
+    tokens: { WS, HASH, DEC_NAME, KEY, FN_NAME, TEXT },
+    rules: { ArgKV, Arg, Args, Call, Part, Comment, Item, Line, File },
+    contextualScopes: [{ token: KEY, within: [ArgKV], scope: 'entity.other.attribute-name' }],
+    entry: File,
+  });
+
+  const tm = generateTmLanguage(grammar);
+  const callArgs = tm.repository['ctx-call-args'];
+  check('tm: contextualScopes derives a call-args construct region', !!callArgs && callArgs.end === '\\)');
+  const callIncludes = (callArgs?.patterns ?? []).map((pp) => (pp as { include?: string }).include);
+  check('tm: construct region tries contextual overrides before $self',
+    callIncludes[0]?.startsWith('#ctx-scope-') === true && callIncludes[callIncludes.length - 1] === '$self');
+  const override = tm.repository[callIncludes[0]!.slice(1)];
+  check('tm: the override rule carries the contextual scope',
+    override?.name === 'entity.other.attribute-name.env-spec-ctx');
+  const contParen = Object.keys(tm.repository).find((k) => k.startsWith('hash-rich-cont-') && tm.repository[k].begin === '\\(');
+  const contIncludes = contParen ? (tm.repository[contParen].patterns ?? []).map((pp) => (pp as { include?: string }).include) : [];
+  check('tm: continuation-bracket interiors include the contextual overrides',
+    contIncludes.some((n) => n?.startsWith('#ctx-scope-')));
+  check('tm: flat top-level token rule keeps the declared scope',
+    tm.repository.key?.name === 'entity.name.tag.env-spec-ctx');
+  const plain = tm.repository.hash;
+  check('tm: plain comment region carries the declared markup patterns',
+    Array.isArray(plain?.patterns) && plain.patterns.length === 1
+    && (plain.patterns[0] as { name?: string }).name === 'markup.bold.env-spec-ctx');
+
+  const ts = generateTreeSitter(grammar, 'env-spec-ctx');
+  check('tree-sitter: contextual scope emits an exact rule-scoped query, last-wins',
+    ts.highlightsScm.includes('(arg_kv (key) @attribute)')
+    && ts.highlightsScm.lastIndexOf('(arg_kv (key) @attribute)') > ts.highlightsScm.lastIndexOf('Keyword, operator, and punctuation literals'));
+}
+
 console.log(
   fail === 0
     ? `\n${ok}/${ok} env-spec regression checks pass`

From 6bfa8aff116dc1315c13f9e58d6ff2e578746c24 Mon Sep 17 00:00:00 2001
From: Theo Ephraim <theo@dmno.dev>
Date: Fri, 3 Jul 2026 23:48:11 -0700
Subject: [PATCH 4/6] fix(gen-tm): derive the call-region callee from the
 grammar's callee token

The generic identifier fallback can resolve to a placeholder (never-matching)
token in indentation grammars, leaving ctx-call-args dead. Prefer the token
whose pattern is gated on a following '(' (the grammar's own callee), and skip
the region entirely when no usable callee pattern exists.
---
 src/gen-tm.ts | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/gen-tm.ts b/src/gen-tm.ts
index 5895169..5dbf2be 100644
--- a/src/gen-tm.ts
+++ b/src/gen-tm.ts
@@ -7565,15 +7565,19 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
         patterns: [...ctxOverrideIncludes, ...pairIncludes, { include: '$self' }],
       };
     }
-    // callee scope: reuse the grammar's own callee token scope when one is declared (a token
-    // whose pattern is gated on a following `(`), else the TextMate-conventional call scope
+    // callee pattern + scope: prefer the grammar's own CALLEE token (one whose pattern is
+    // gated on a following `(` — e.g. env-spec FUNCTION_NAME), else the identifier pattern.
+    // The generic ident fallback can resolve to a placeholder token in indentation grammars,
+    // so a never-matching callee skips the region entirely rather than emitting a dead rule.
     const calleeTok = grammar.tokens.find((t) => {
       const sc = t.scope ?? classifyToken(t).scope;
       return (sc.startsWith('variable.function') || sc.startsWith('entity.name.function')) && tokenPatternSource(t).includes('\\(');
     });
     const calleeScope = calleeTok?.scope ?? 'entity.name.function';
-    repository['ctx-call-args'] = {
-      begin: `(${identPattern})\\s*(\\()`,
+    const calleePattern = calleeTok ? tokenPatternSource(calleeTok) : identPattern;
+    if (!calleePattern.includes('(?!)')) {
+      repository['ctx-call-args'] = {
+      begin: `(${calleePattern})\\s*(\\()`,
       beginCaptures: {
         '1': { name: `${calleeScope}.${langName}` },
         '2': { name: `${getScope(scopeOverrides, '(') ?? 'punctuation.section.parens.begin'}.${langName}` },
@@ -7581,8 +7585,9 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
       end: '\\)',
       endCaptures: { '0': { name: `${getScope(scopeOverrides, ')') ?? 'punctuation.section.parens.end'}.${langName}` } },
       patterns: [...ctxOverrideIncludes, ...pairIncludes, { include: '$self' }],
-    };
-    topPatterns.push({ include: '#ctx-call-args' });
+      };
+      topPatterns.push({ include: '#ctx-call-args' });
+    }
   }
 
   // ── 4b1a. Object method key: `key: (params) => ...` or `key: function(...)` ──

From 6a2ce9dca9b31d0604d77dff85ce088308d9eac2 Mon Sep 17 00:00:00 2001
From: Theo Ephraim <theo@dmno.dev>
Date: Fri, 3 Jul 2026 23:57:33 -0700
Subject: [PATCH 5/6] test: rewrite structured-comment regressions as a
 behavioral spec
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous contracts asserted the generated grammar's internal shape
(repository keys, include ordering) — they break on refactor without saying
what behavior to preserve. Rewritten against RENDERED OUTPUT instead: a real
document is tokenized with vscode-textmate and the assertions read as the
spec, line by line —

- the same key token paints three ways: env key at top level, attribute name
  inside call args (contextualScopes), attribute name inside decorator
  comment constructs
- plain comment prose dims as comment (never as a value string); declared
  markup (**bold**) highlights inside it
- decorator comments keep rich token scopes
- an open bracket continues the construct across #-prefixed lines: content
  keeps token scopes, the line-start # is a continuation marker, an embedded
  '# aside' dims to end-of-line, and after the closer a plain comment dims
  again
- the parser CST is byte-identical with and without the highlight metadata
  (highlight-only, proven not asserted)
- without the metadata, generation is unchanged (opt-in)
- tree-sitter emits the same declaration as exact last-wins queries

The implementation can be thrown away; these tests say what any replacement
must do.
---
 test/env-spec-regressions.ts | 291 ++++++++++++++++++-----------------
 1 file changed, 149 insertions(+), 142 deletions(-)

diff --git a/test/env-spec-regressions.ts b/test/env-spec-regressions.ts
index 16dc065..09da902 100644
--- a/test/env-spec-regressions.ts
+++ b/test/env-spec-regressions.ts
@@ -84,171 +84,178 @@ const check = (label: string, cond: boolean) => {
   check('parser: multiline inline quoted value is accepted without blockScalar', !threw);
 }
 
-// ---------------------------------------------------------------------------
-// Regression 4 (declared below regression 3's grammar pieces): contextualScopes +
-// lineComment.markup — context-dependent token scopes and declared comment markup.
-// ---------------------------------------------------------------------------
 
 // ---------------------------------------------------------------------------
-// Regression 3: a line-comment INTRODUCER token (`lineComment` metadata) emits
-// to-end-of-line REGIONS in TextMate, not a flat 1-char rule — so comment prose
-// dims to the comment scope while `richStarters`-led comments (env-spec decorator
-// comments `# @dec(...)`) keep full token highlighting inside.
+// Regressions 3–6: structured-comment highlighting — BEHAVIORAL SPEC.
+//
+// These are written against the *rendered output* (vscode-textmate tokenization
+// of a real document), not the generated grammar's internal shape, so they pin
+// the desired outcome independently of how the generators achieve it.
+//
+// The dialect under test is an env-spec-style DSL: comments carry a decorator
+// DSL, so the PARSER must tokenize comment bodies — but a comment is still a
+// comment to a THEME. The contract, line by line:
+//
+//   KEY=fn(retry=3, plain)         KEY = env key; retry = option key (attribute,
+//                                  contextualScopes); plain = positional value
+//   # a note with **bold** mark    prose dims as comment; **bold** = markup
+//   # @dec(opt=1)                  decorator comments keep rich token scopes
+//   # @import(                     an OPEN bracket continues the construct
+//   #   first,                     across `#`-prefixed lines: content keeps its
+//   #   pick=[                     token scopes; the line-start `#` is a
+//   #     ITEM, # aside            continuation marker; `# aside` dims to EOL
+//   #   ],
+//   # )
+//   # after                        construct closed → plain dim comment again
+//
+// And all of it is HIGHLIGHT-ONLY: the parser must produce a byte-identical
+// CST whether or not the highlight metadata is declared.
 // ---------------------------------------------------------------------------
-{
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import vsctm from 'vscode-textmate';
+import onigLib from 'vscode-oniguruma';
+
+const { INITIAL, Registry, parseRawGrammar } = vsctm;
+const { loadWASM, OnigScanner, OnigString } = onigLib;
+const require = createRequire(import.meta.url);
+const wasmBin = readFileSync(require.resolve('vscode-oniguruma/release/onig.wasm'));
+await loadWASM(wasmBin.buffer.slice(wasmBin.byteOffset, wasmBin.byteOffset + wasmBin.byteLength));
+
+function makeDialect(withHighlightMetadata: boolean) {
   const hspace = oneOf(' ', '\t');
-  const alpha = oneOf(range('a', 'z'), range('A', 'Z'));
+  const alnum = oneOf(range('a', 'z'), range('A', 'Z'), range('0', '9'));
   const WS = token(plus(hspace), { skip: true, scope: 'meta.whitespace' });
-  const DEC_NAME = token(seq('@', plus(alpha)), { scope: 'variable.annotation' });
+  const DEC_NAME = token(seq('@', plus(alnum)), { scope: 'variable.annotation' });
   const HASH = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), {
     scope: 'comment.line',
-    lineComment: { richStarters: [DEC_NAME] },
+    ...(withHighlightMetadata ? {
+      lineComment: {
+        richStarters: [DEC_NAME],
+        continuationBrackets: [['(', ')'], ['[', ']']] as [string, string][],
+        markup: [{ pattern: seq('**', star(noneOf('*', '\n')), '**'), scope: 'markup.bold' }],
+      },
+    } : {}),
   });
-  const KEY = token(seq(plus(alpha), followedBy('=')), { scope: 'entity.name.tag' });
-  const TEXT = token(plus(noneOf(' ', '\t', '\n', '#', '=', '@')), { scope: 'string.unquoted' });
-  const Part = rule(() => [DEC_NAME, TEXT]);
+  const KEY = token(seq(plus(alnum), followedBy('=')), { scope: 'entity.name.tag' });
+  const FN_NAME = token(seq(plus(alnum), followedBy(seq(star(hspace), '('))), { scope: 'variable.function' });
+  const TEXT = token(plus(noneOf(' ', '\t', '\n', '#', '=', '@', '(', ')', '[', ']', ',', '*')), { scope: 'string.unquoted' });
+  const ArgKV = rule(() => [[KEY, '=', TEXT]]);
+  const Arg = rule(() => [ArgKV, TEXT]);
+  const Args = rule(() => [['(', opt(Arg), opt(','), opt(Arg), ')']]);
+  const Call = rule(() => [[FN_NAME, Args]]);
+  const Part = rule(() => [DEC_NAME, Call, ArgKV, TEXT, KEY, '=', ',', '(', ')', '[', ']', '**']);
   const Comment = rule(() => [[HASH, many(Part)]]);
-  const Item = rule(() => [[KEY, '=', opt(TEXT), opt(Comment)]]);
+  const Item = rule(() => [[KEY, '=', Call, opt(Comment)]]);
   const Line = rule(() => [Item, Comment]);
   const File = rule(() => [[many(Line)]]);
-  const grammar = defineGrammar({
-    name: 'env-spec-comments',
-    tokens: { WS, HASH, DEC_NAME, KEY, TEXT },
-    rules: { Part, Comment, Item, Line, File },
+  return defineGrammar({
+    name: 'env-spec-dialect',
+    tokens: { WS, HASH, DEC_NAME, KEY, FN_NAME, TEXT },
+    rules: { ArgKV, Arg, Args, Call, Part, Comment, Item, Line, File },
+    ...(withHighlightMetadata ? {
+      contextualScopes: [{ token: KEY, within: [ArgKV], scope: 'entity.other.attribute-name' }],
+    } : {}),
     entry: File,
   });
+}
 
-  const tm = generateTmLanguage(grammar);
-  const plain = tm.repository.hash;
-  const rich = tm.repository['hash-rich'];
-  check('tm: plain comment entry is a to-EOL region', !!plain && plain.end === '$');
-  check('tm: plain comment region carries the comment scope', plain?.name === 'comment.line.env-spec-comments');
-  check('tm: plain comment region has NO inner patterns (prose dims)', Array.isArray(plain?.patterns) && plain.patterns.length === 0);
-  check('tm: rich comment entry exists and is gated on the rich starter', !!rich && typeof rich.begin === 'string' && rich.begin.includes('(?=[ \\t]*'));
-  check('tm: rich comment region keeps full token highlighting via $self', JSON.stringify(rich?.patterns) === JSON.stringify([{ include: '$self' }]));
-  check('tm: rich entry is tried before the plain entry', (() => {
-    const order = tm.patterns.map((p) => (p as { include?: string }).include);
-    return order.indexOf('#hash-rich') !== -1 && order.indexOf('#hash-rich') < order.indexOf('#hash');
-  })());
-  check('tm: introducer captures as comment punctuation', JSON.stringify(plain?.beginCaptures?.['1']) === JSON.stringify({ name: 'punctuation.definition.comment.env-spec-comments' }));
-
-  // parser behavior is UNaffected by the highlight-only metadata
-  const parser = createParser(grammar);
-  let threw = false;
-  try {
-    parser.parse('KEY=val # @dec note\n# plain prose');
-  } catch {
-    threw = true;
-  }
-  check('parser: lineComment metadata does not change parsing', !threw);
+const DOC = [
+  'KEY=fn(retry=3, plain)',
+  '# a note with **bold** mark',
+  '# @dec(opt=1)',
+  '# @import(',
+  '#   first,',
+  '#   pick=[',
+  '#     ITEM, # aside',
+  '#   ],',
+  '# )',
+  '# after',
+];
 
-  // continuationBrackets: a bracket left open in a rich comment continues the construct
-  // across introducer-prefixed lines via nested begin/end regions
-  const HASH_ML = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), {
-    scope: 'comment.line',
-    lineComment: { richStarters: [DEC_NAME], continuationBrackets: [['(', ')'], ['[', ']']] },
+async function tokenizeDoc(grammarDef: ReturnType<typeof makeDialect>) {
+  const tm = generateTmLanguage(grammarDef);
+  const registry = new Registry({
+    onigLib: Promise.resolve({
+      createOnigScanner: (p: string[]) => new OnigScanner(p),
+      createOnigString: (str: string) => new OnigString(str),
+    }),
+    loadGrammar: async (scopeName: string) => scopeName === 'source.env-spec-dialect'
+      ? parseRawGrammar(JSON.stringify(tm), 'g.json') : null,
   });
-  const CommentMl = rule(() => [[HASH_ML, many(Part)]]);
-  const FileMl = rule(() => [[many(CommentMl)]]);
-  const mlGrammar = defineGrammar({
-    name: 'env-spec-ml',
-    tokens: { WS, HASH_ML, DEC_NAME, KEY, TEXT },
-    rules: { Part, CommentMl, FileMl },
-    entry: FileMl,
+  const grammar = (await registry.loadGrammar('source.env-spec-dialect'))!;
+  let stack = INITIAL;
+  return DOC.map((line) => {
+    const r = grammar.tokenizeLine(line, stack);
+    stack = r.ruleStack;
+    return r.tokens.map((t) => ({ text: line.slice(t.startIndex, t.endIndex), scopes: t.scopes }));
   });
-  const tmMl = generateTmLanguage(mlGrammar);
-  const parenKey = Object.keys(tmMl.repository).find((k) => k.startsWith('hash_ml-rich-cont-') && tmMl.repository[k].begin === '\\(');
-  check('tm: continuation bracket pair emits a begin/end region', !!parenKey && tmMl.repository[parenKey!].end === '\\)');
-  const parenRegion = parenKey ? tmMl.repository[parenKey] : undefined;
-  const parenIncludes = (parenRegion?.patterns ?? []).map((pp) => (pp as { include?: string }).include);
-  check('tm: construct interior tries marker, embedded comment, nested brackets, then $self',
-    parenIncludes[0] === '#hash_ml-rich-cont-marker'
-    && parenIncludes[1] === '#hash_ml-rich-cont-comment'
-    && parenIncludes.includes('$self')
-    && parenIncludes.filter((n) => n?.startsWith('#hash_ml-rich-cont-') && n !== '#hash_ml-rich-cont-marker' && n !== '#hash_ml-rich-cont-comment').length === 2);
-  const marker = tmMl.repository['hash_ml-rich-cont-marker'];
-  check('tm: continuation marker is line-anchored and scoped as comment punctuation',
-    typeof marker?.match === 'string' && marker.match.startsWith('^[ \\t]*')
-    && JSON.stringify(marker?.captures?.['1']?.name ?? '').includes('punctuation.definition.comment'));
-  const embedded = tmMl.repository['hash_ml-rich-cont-comment'];
-  check('tm: embedded comment inside a construct dims to end-of-line', embedded?.end === '$' && Array.isArray(embedded?.patterns) && embedded.patterns.length === 0);
-  const richMl = tmMl.repository['hash_ml-rich'];
-  const richIncludes = (richMl?.patterns ?? []).map((pp) => (pp as { include?: string }).include);
-  check('tm: rich region tries construct brackets before $self', richIncludes[richIncludes.length - 1] === '$self' && richIncludes.length === 3);
+}
 
-  // a comment token WITHOUT the metadata still emits the flat rule (no behavior change)
-  const HASH2 = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), { scope: 'comment.line' });
-  const Comment2 = rule(() => [[HASH2, many(TEXT)]]);
-  const File2 = rule(() => [[many(Comment2)]]);
-  const flatGrammar = defineGrammar({ name: 'no-metadata', tokens: { HASH2, TEXT }, rules: { Comment2, File2 }, entry: File2 });
-  const tm2 = generateTmLanguage(flatGrammar);
-  check('tm: without lineComment metadata the comment token stays a flat match', typeof tm2.repository.hash2?.match === 'string' && tm2.repository.hash2?.begin === undefined);
+type Span = { text: string; scopes: string[] };
+// the scopes painted on `text` in DOC line `lineNo` (1-based); nth occurrence via `skip`
+function paint(lines: Span[][], lineNo: number, text: string, skip = 0): string[] {
+  let seen = 0;
+  for (const span of lines[lineNo - 1]) {
+    if (span.text.includes(text) || (text.length > span.text.length && span.text.trim() !== '' && text.includes(span.text.trim()) && span.text.trim().length > 2)) {
+      if (seen === skip) return span.scopes;
+      seen += 1;
+    }
+  }
+  return [];
 }
+const has = (scopes: string[], frag: string) => scopes.some((sc) => sc.includes(frag));
 
-// ---------------------------------------------------------------------------
-// Regression 4: contextualScopes — token T carries scope S within rule R.
-//   tm: overrides apply inside derived construct regions (call args + continuation
-//       brackets); flat top-level rules keep the declared scope.
-//   tree-sitter: exact `(rule (token) @capture)` queries, emitted last (last-wins).
-// Plus lineComment.markup: declared doc-markup patterns inside plain comment bodies.
-// ---------------------------------------------------------------------------
 {
-  const hspace = oneOf(' ', '\t');
-  const alpha = oneOf(range('a', 'z'), range('A', 'Z'));
-  const WS = token(plus(hspace), { skip: true, scope: 'meta.whitespace' });
-  const DEC_NAME = token(seq('@', plus(alpha)), { scope: 'variable.annotation' });
-  const HASH = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), {
-    scope: 'comment.line',
-    lineComment: {
-      richStarters: [DEC_NAME],
-      continuationBrackets: [['(', ')']],
-      markup: [{ pattern: seq('**', star(noneOf('*', '\n')), '**'), scope: 'markup.bold' }],
-    },
-  });
-  const KEY = token(seq(plus(alpha), followedBy('=')), { scope: 'entity.name.tag' });
-  const FN_NAME = token(seq(plus(alpha), followedBy(seq(star(hspace), '('))), { scope: 'variable.function' });
-  const TEXT = token(plus(noneOf(' ', '\t', '\n', '#', '=', '@', '(', ')', ',')), { scope: 'string.unquoted' });
-  const ArgKV = rule(() => [[KEY, '=', TEXT]]);
-  const Arg = rule(() => [ArgKV, TEXT]);
-  const Args = rule(() => [['(', opt(Arg), ')']]);
-  const Call = rule(() => [[FN_NAME, Args]]);
-  const Part = rule(() => [DEC_NAME, Call, TEXT, KEY, '=', ',', '(', ')']);
-  const Comment = rule(() => [[HASH, many(Part)]]);
-  const Item = rule(() => [[KEY, '=', opt(Call), opt(Comment)]]);
-  const Line = rule(() => [Item, Comment]);
-  const File = rule(() => [[many(Line)]]);
-  const grammar = defineGrammar({
-    name: 'env-spec-ctx',
-    tokens: { WS, HASH, DEC_NAME, KEY, FN_NAME, TEXT },
-    rules: { ArgKV, Arg, Args, Call, Part, Comment, Item, Line, File },
-    contextualScopes: [{ token: KEY, within: [ArgKV], scope: 'entity.other.attribute-name' }],
-    entry: File,
-  });
+  const lines = await tokenizeDoc(makeDialect(true));
 
-  const tm = generateTmLanguage(grammar);
-  const callArgs = tm.repository['ctx-call-args'];
-  check('tm: contextualScopes derives a call-args construct region', !!callArgs && callArgs.end === '\\)');
-  const callIncludes = (callArgs?.patterns ?? []).map((pp) => (pp as { include?: string }).include);
-  check('tm: construct region tries contextual overrides before $self',
-    callIncludes[0]?.startsWith('#ctx-scope-') === true && callIncludes[callIncludes.length - 1] === '$self');
-  const override = tm.repository[callIncludes[0]!.slice(1)];
-  check('tm: the override rule carries the contextual scope',
-    override?.name === 'entity.other.attribute-name.env-spec-ctx');
-  const contParen = Object.keys(tm.repository).find((k) => k.startsWith('hash-rich-cont-') && tm.repository[k].begin === '\\(');
-  const contIncludes = contParen ? (tm.repository[contParen].patterns ?? []).map((pp) => (pp as { include?: string }).include) : [];
-  check('tm: continuation-bracket interiors include the contextual overrides',
-    contIncludes.some((n) => n?.startsWith('#ctx-scope-')));
-  check('tm: flat top-level token rule keeps the declared scope',
-    tm.repository.key?.name === 'entity.name.tag.env-spec-ctx');
-  const plain = tm.repository.hash;
-  check('tm: plain comment region carries the declared markup patterns',
-    Array.isArray(plain?.patterns) && plain.patterns.length === 1
-    && (plain.patterns[0] as { name?: string }).name === 'markup.bold.env-spec-ctx');
+  // ── Regression 3: contextual scopes — the SAME token, three different paints ──
+  check('spec: a top-level env key keeps its declared scope', has(paint(lines, 1, 'KEY'), 'entity.name.tag'));
+  check('spec: an option key inside call args paints as an attribute name', has(paint(lines, 1, 'retry'), 'entity.other.attribute-name'));
+  check('spec: a positional arg value is NOT an attribute name', !has(paint(lines, 1, 'plain'), 'attribute-name') && has(paint(lines, 1, 'plain'), 'string.unquoted'));
+  check('spec: the callee keeps its function scope', has(paint(lines, 1, 'fn'), 'function'));
+
+  // ── Regression 4: plain comments dim; markup highlights ──
+  check('spec: plain comment prose paints as comment, not as a value string', has(paint(lines, 2, 'a note with'), 'comment.line') && !has(paint(lines, 2, 'a note with'), 'string.unquoted'));
+  check('spec: the comment introducer is comment punctuation', has(paint(lines, 2, '#'), 'punctuation.definition.comment'));
+  check('spec: declared markup highlights inside plain comments', has(paint(lines, 2, '**bold**'), 'markup.bold'));
+
+  // ── Regression 5: decorator comments stay rich ──
+  check('spec: a decorator in a rich comment keeps its annotation scope', has(paint(lines, 3, '@dec'), 'variable.annotation'));
+  check('spec: an option key inside a decorator call paints as an attribute name', has(paint(lines, 3, 'opt'), 'entity.other.attribute-name'));
+
+  // ── Regression 6: multi-line constructs — an open bracket continues the construct ──
+  check('spec: construct content on a continuation line keeps its token scope (not dimmed)', has(paint(lines, 5, 'first'), 'string.unquoted'));
+  check('spec: the line-start `#` inside a construct is a continuation marker (comment punctuation)', has(paint(lines, 5, '#'), 'punctuation.definition.comment'));
+  check('spec: a nested option key on a continuation line paints as an attribute name', has(paint(lines, 6, 'pick'), 'entity.other.attribute-name'));
+  check('spec: a nested array element keeps its token scope', has(paint(lines, 7, 'ITEM'), 'string.unquoted'));
+  check('spec: an embedded `# aside` after content dims to end-of-line', has(paint(lines, 7, 'aside'), 'comment.line') && !has(paint(lines, 7, 'aside'), 'string.unquoted'));
+  check('spec: after the construct closes, a plain comment dims again', has(paint(lines, 10, 'after'), 'comment.line') && !has(paint(lines, 10, 'after'), 'string.unquoted'));
+}
 
-  const ts = generateTreeSitter(grammar, 'env-spec-ctx');
-  check('tree-sitter: contextual scope emits an exact rule-scoped query, last-wins',
-    ts.highlightsScm.includes('(arg_kv (key) @attribute)')
-    && ts.highlightsScm.lastIndexOf('(arg_kv (key) @attribute)') > ts.highlightsScm.lastIndexOf('Keyword, operator, and punctuation literals'));
+// ── Highlight metadata is HIGHLIGHT-ONLY: identical CSTs with and without it ──
+{
+  const withMeta = createParser(makeDialect(true));
+  const withoutMeta = createParser(makeDialect(false));
+  const text = DOC.join('\n');
+  const a = JSON.stringify(withMeta.parse(text));
+  const b = JSON.stringify(withoutMeta.parse(text));
+  check('spec: the parser CST is byte-identical with and without highlight metadata', a === b);
+}
+
+// ── Without the metadata, generation is unchanged (the features are opt-in) ──
+{
+  const plainGrammar = makeDialect(false);
+  const tm = generateTmLanguage(plainGrammar);
+  check('spec: without lineComment metadata the comment token stays a flat match', typeof tm.repository.hash?.match === 'string' && tm.repository.hash?.begin === undefined);
+  check('spec: without contextualScopes no construct regions are derived', !Object.keys(tm.repository).some((k) => k.startsWith('ctx-')));
+}
+
+// ── tree-sitter: the same contextualScopes declaration becomes exact queries ──
+{
+  const ts = generateTreeSitter(makeDialect(true), 'env-spec-dialect');
+  check('spec: tree-sitter emits an exact rule-scoped capture for the contextual scope', ts.highlightsScm.includes('(arg_kv (key) @attribute)'));
+  check('spec: contextual captures come last (highlight resolution is last-wins)', ts.highlightsScm.lastIndexOf('(arg_kv (key) @attribute)') > ts.highlightsScm.lastIndexOf('] @'));
 }
 
 console.log(

From 1f1f4c2c6b18b5a07a57bba1bf08e30d3a0bfe49 Mon Sep 17 00:00:00 2001
From: Theo Ephraim <theo@dmno.dev>
Date: Sat, 4 Jul 2026 11:51:40 -0700
Subject: [PATCH 6/6] fix(gen-tm): call constructs inside comment blocks must
 not leak past their closer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A callee-anchored call-args region (reached via $self inside a rich comment)
starts matching at the CALLEE — an earlier position than the continuation
bracket's bare '(' — so it hijacked '# @dec=someFn(' constructs with an
interior that treats a line-start '#' as an embedded comment: the '# )' closer
was eaten, the region ran away past the comment block, and the FOLLOWING
config item painted as comment/attribute content.

Emit a continuation-aware call variant inside rich comments and construct
interiors (same marker-aware interior as the paren construct, begin consumes
the callee), tried before $self so it wins the position tie.

Spec test added: a call construct in a decorator comment keeps continuation
content rich, ends at its '# )' closer, and the following config item carries
no comment scope.
---
 src/gen-tm.ts                | 34 +++++++++++++++++++++++++++++++++-
 test/env-spec-regressions.ts | 20 +++++++++++++++++---
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/src/gen-tm.ts b/src/gen-tm.ts
index 5dbf2be..d7c7a94 100644
--- a/src/gen-tm.ts
+++ b/src/gen-tm.ts
@@ -5074,6 +5074,19 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
         const contBrackets = tok.lineComment.continuationBrackets ?? [];
         const bracketKeyOf = (open: string) => `${key}-rich-cont-${[...open].map((c) => c.charCodeAt(0).toString(16)).join('')}`;
         const bracketIncludes = contBrackets.map(([open]) => ({ include: `#${bracketKeyOf(open)}` }));
+        // A CALL opening a construct (`# @dec=someFn(`) must get the marker-aware interior
+        // too: a callee-anchored region (e.g. the contextualScopes call-args region reached
+        // via $self) starts matching at the CALLEE — an earlier position than the bare `(`
+        // — so without a call-aware variant here it would hijack the construct with an
+        // interior that treats line-start `#` as a comment, eat the `# )` closer, and run
+        // away past the comment block. Same interior as the paren construct, begin consumes
+        // the callee (scoped like the grammar's callee token). Tried before $self.
+        const calleeForCont = grammar.tokens.find((t) => {
+          const sc = t.scope ?? classifyToken(t).scope;
+          return (sc.startsWith('variable.function') || sc.startsWith('entity.name.function')) && tokenPatternSource(t).includes('\\(');
+        });
+        const hasParenPair = contBrackets.some(([open]) => open === '(');
+        const contCallInclude = (calleeForCont && hasParenPair) ? [{ include: `#${key}-rich-cont-call` }] : [];
         if (contBrackets.length) {
           repository[`${key}-rich-cont-marker`] = {
             match: `^[ \\t]*(${introducer})`,
@@ -5090,6 +5103,25 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
             repository[bracketKeyOf(open)] = {
               begin: escapeRegex(open),
               end: escapeRegex(close),
+              patterns: [
+                { include: `#${key}-rich-cont-marker` },
+                { include: `#${key}-rich-cont-comment` },
+                ...ctxOverrideIncludes,
+                ...contCallInclude,
+                ...bracketIncludes,
+                { include: '$self' },
+              ],
+            };
+          }
+          if (calleeForCont && hasParenPair) {
+            repository[`${key}-rich-cont-call`] = {
+              begin: `(${tokenPatternSource(calleeForCont)})\\s*(\\()`,
+              beginCaptures: {
+                '1': { name: `${(calleeForCont.scope ?? 'entity.name.function')}.${langName}` },
+                '2': { name: `${getScope(grammar.scopeOverrides, '(') ?? 'punctuation.section.parens.begin'}.${langName}` },
+              },
+              end: '\\)',
+              endCaptures: { '0': { name: `${getScope(grammar.scopeOverrides, ')') ?? 'punctuation.section.parens.end'}.${langName}` } },
               patterns: [
                 { include: `#${key}-rich-cont-marker` },
                 { include: `#${key}-rich-cont-comment` },
@@ -5105,7 +5137,7 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
           begin: `(${introducer})(?=[ \\t]*(?:${startLookahead}))`,
           beginCaptures: { '1': { name: commentPunct } },
           end: '$',
-          patterns: [...bracketIncludes, { include: '$self' }],
+          patterns: [...contCallInclude, ...bracketIncludes, { include: '$self' }],
         };
         topPatterns.push({ include: `#${key}-rich` });
       }
diff --git a/test/env-spec-regressions.ts b/test/env-spec-regressions.ts
index 09da902..4b2134c 100644
--- a/test/env-spec-regressions.ts
+++ b/test/env-spec-regressions.ts
@@ -6,7 +6,7 @@
 //
 // Run with: node test/env-spec-regressions.ts
 import { createParser } from '../src/gen-parser.ts';
-import { defineGrammar, many, opt, rule, token, seq, star, altPattern, oneOf, noneOf, anyChar, never, range, plus, followedBy, notPrecededBy } from '../src/api.ts';
+import { alt, defineGrammar, many, opt, rule, token, seq, star, altPattern, oneOf, noneOf, anyChar, never, range, plus, followedBy, notPrecededBy } from '../src/api.ts';
 import { generateTmLanguage } from '../src/gen-tm.ts';
 import { generateTreeSitter } from '../src/gen-treesitter.ts';
 
@@ -142,11 +142,11 @@ function makeDialect(withHighlightMetadata: boolean) {
   const TEXT = token(plus(noneOf(' ', '\t', '\n', '#', '=', '@', '(', ')', '[', ']', ',', '*')), { scope: 'string.unquoted' });
   const ArgKV = rule(() => [[KEY, '=', TEXT]]);
   const Arg = rule(() => [ArgKV, TEXT]);
-  const Args = rule(() => [['(', opt(Arg), opt(','), opt(Arg), ')']]);
+  const Args = rule(() => [['(', many(alt(Arg, ',', HASH)), ')']]);
   const Call = rule(() => [[FN_NAME, Args]]);
   const Part = rule(() => [DEC_NAME, Call, ArgKV, TEXT, KEY, '=', ',', '(', ')', '[', ']', '**']);
   const Comment = rule(() => [[HASH, many(Part)]]);
-  const Item = rule(() => [[KEY, '=', Call, opt(Comment)]]);
+  const Item = rule(() => [[KEY, '=', alt(Call, TEXT), opt(Comment)]]);
   const Line = rule(() => [Item, Comment]);
   const File = rule(() => [[many(Line)]]);
   return defineGrammar({
@@ -171,6 +171,11 @@ const DOC = [
   '#   ],',
   '# )',
   '# after',
+  '# @dec2=someFn(',
+  '#   argone,',
+  '#   keytwo=v,',
+  '# )',
+  'NEXT=ok',
 ];
 
 async function tokenizeDoc(grammarDef: ReturnType<typeof makeDialect>) {
@@ -231,6 +236,15 @@ const has = (scopes: string[], frag: string) => scopes.some((sc) => sc.includes(
   check('spec: a nested array element keeps its token scope', has(paint(lines, 7, 'ITEM'), 'string.unquoted'));
   check('spec: an embedded `# aside` after content dims to end-of-line', has(paint(lines, 7, 'aside'), 'comment.line') && !has(paint(lines, 7, 'aside'), 'string.unquoted'));
   check('spec: after the construct closes, a plain comment dims again', has(paint(lines, 10, 'after'), 'comment.line') && !has(paint(lines, 10, 'after'), 'string.unquoted'));
+
+  // ── Regression 7: a CALL opening a construct must not leak past its closer ──
+  // `# @dec2=someFn(` … `# )` — the call-args region must be continuation-aware; a
+  // naive callee-anchored region would treat `# )` as an embedded comment, never see
+  // the closer, and swallow the FOLLOWING config item into the comment.
+  check('spec: a call construct keeps continuation content rich', has(paint(lines, 12, 'argone'), 'string.unquoted'));
+  check('spec: a nested option key in a call construct paints as an attribute name', has(paint(lines, 13, 'keytwo'), 'entity.other.attribute-name'));
+  check('spec: the call construct ends at its `# )` closer', has(paint(lines, 15, 'NEXT'), 'entity.name.tag') && !has(paint(lines, 15, 'NEXT'), 'comment'));
+  check('spec: the config item after the construct is not comment-scoped at all', !has(paint(lines, 15, 'ok'), 'comment'));
 }
 
 // ── Highlight metadata is HIGHLIGHT-ONLY: identical CSTs with and without it ──