Highlight fidelity for structured-comment dialects: lineComment regions, continuation brackets, contextualScopes, comment markup

src/api.ts

@@ -19,6 +19,16 @@ interface TokenOptions {
   // Highlight-only interpolation regions for ordinary string tokens (e.g. env-spec `${…}` / `$(…)`).
   // The parser/lexer stay token-based; generators re-express these as nested regions.
   interpolation?: StringInterpolation | StringInterpolation[];
+  // Highlight-only: this comment token matches only the INTRODUCER (a bare `#`) while the
+  // comment runs to end-of-line with parser-tokenized content (a structured-comment dialect).
+  // Generators emit a to-EOL region in the token's comment scope so prose dims; `richStarters`
+  // lists tokens that keep full token highlighting when one opens the comment body
+  // (e.g. env-spec `# @decorator(...)`). See TokenDecl.lineComment.
+  lineComment?: {
+    richStarters?: TokenRef[];
+    continuationBrackets?: [string, string][];
+    markup?: { pattern: TokenPattern; scope: string }[];
+  };
   // A regex matching exactly one well-formed escape sequence. Engine-scanned tokens
   // (templates) validate each `\`-escape against it and reject any that don't match —
   // unlike `escape` (highlight-only), this drives tokenization. Skipped in tag
@@ -507,6 +517,9 @@ interface GrammarConfig {
   ledPrec?: LedPrec[];
   rules: Record<string, RuleRef>;
   scopes?: Record<string, string[]>;
+  // Highlight-only contextual token scopes: token T carries scope S within rule R (T's
+  // immediate enclosing rule) — see CstGrammar.contextualScopes for generator fidelity.
+  contextualScopes?: { token: TokenRef; within: RuleRef | RuleRef[]; scope: string }[];
   entry: RuleRef;
   markup?: MarkupConfig;  // opt-in markup-mode tokenization (HTML/Vue)
   indent?: IndentConfig;  // opt-in indentation-sensitive tokenization (YAML)
@@ -547,6 +560,18 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
         ? (Array.isArray(tok.opts.interpolation) ? tok.opts.interpolation : [tok.opts.interpolation]).map((i) => ({ ...i }))
         : undefined,
       escapeValidPattern: tok.opts.escapeValid,
+      // richStarters are TokenRefs; resolve to declared names (unknown refs are a config error)
+      lineComment: tok.opts.lineComment
+        ? {
+          richStarters: (tok.opts.lineComment.richStarters ?? []).map((ref) => {
+            const refName = names.get(ref);
+            if (!refName) throw new Error(`lineComment.richStarters on token '${name}' references an undeclared token`);
+            return refName;
+          }),
+          continuationBrackets: tok.opts.lineComment.continuationBrackets?.map((pair) => [...pair] as [string, string]),
+          markup: tok.opts.lineComment.markup?.map((m) => ({ ...m })),
+        }
+        : undefined,
       embed: tok.opts.embed,
       identifier: tok.opts.identifier,
       identifierPrefix: tok.opts.identifierPrefix,
@@ -591,5 +616,16 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
     }
   }
 
-  return { name: config.name, scopeName: config.scopeName, tokens, precs, ledPrecs: config.ledPrec, rules, scopeOverrides, markup: config.markup, indent: config.indent, newline: config.newline, expressionRule: config.expression ? names.get(config.expression) : undefined, aliasScopes: config.aliasScopes, canonicalRepoNames: config.canonicalRepoNames, manifest: config.manifest };
+  const contextualScopes = (config.contextualScopes ?? []).map((entry) => {
+    const tokenName = names.get(entry.token);
+    if (!tokenName) throw new Error('contextualScopes entry references an undeclared token');
+    const withinRefs = Array.isArray(entry.within) ? entry.within : [entry.within];
+    const within = withinRefs.map((ref) => {
+      const ruleName = names.get(ref);
+      if (!ruleName) throw new Error(`contextualScopes entry for token '${tokenName}' references an undeclared rule`);
+      return ruleName;
+    });
+    return { token: tokenName, within, scope: entry.scope };
+  });
+  return { name: config.name, scopeName: config.scopeName, tokens, precs, ledPrecs: config.ledPrec, rules, scopeOverrides, contextualScopes, markup: config.markup, indent: config.indent, newline: config.newline, expressionRule: config.expression ? names.get(config.expression) : undefined, aliasScopes: config.aliasScopes, canonicalRepoNames: config.canonicalRepoNames, manifest: config.manifest };
 }

src/gen-tm.ts

@@ -4819,6 +4819,23 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
   // to the region's own `meta.type.*` name. The classification is the same
   // scope-family test the token loop already uses, so the set is fully derived
   // from the grammar's own token scopes — no hardcoded `"`/digit literals.
+  // ── Contextual token-scope overrides (grammar.contextualScopes, highlight-only) ──
+  // Token T carries scope S within rule R. The FLAT grammar approximates rule context with its
+  // derived CONSTRUCT regions — call-argument lists and lineComment continuation brackets —
+  // where these override rules are tried before the token's flat rule; top-level occurrences
+  // keep the token's declared scope. (tree-sitter consumes the same declaration exactly, as
+  // `(rule (token) @capture)` queries.)
+  const ctxOverrideIncludes = (grammar.contextualScopes ?? []).map((entry, i) => {
+    const target = grammar.tokens.find((t) => t.name === entry.token);
+    if (!target) return null;
+    const ctxKey = `ctx-scope-${i}-${entry.token.toLowerCase()}`;
+    repository[ctxKey] ??= {
+      match: tokenPatternSource(target),
+      name: `${entry.scope}.${langName}`,
+    };
+    return { include: `#${ctxKey}` };
+  }).filter((x): x is { include: string } => !!x);
+
   const literalLiteralKeys: string[] = [];
   const literalTokenNames = new Set<string>();
   const rememberLiteralKey = (scope: string, repoKey: string, tokName?: string) => {
@@ -5012,6 +5029,127 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
       repository[key] = blockEntry;
       topPatterns.push({ include: `#${key}` });
 
+    } else if (tok.lineComment && scope.startsWith('comment.')) {
+      // ── Line-comment INTRODUCER (declared, highlight-only) ──
+      // The token matches only the marker (a bare `#`) while the comment runs to
+      // end-of-line with content the PARSER still tokenizes (a structured-comment
+      // dialect — env-spec decorator comments). A flat rule would leave the comment
+      // PROSE scoped by the ordinary text tokens (string.unquoted…), so themes render
+      // comments like code. Emit begin/end regions to `$` instead:
+      //   1. a RICH region, gated on a lookahead that one of the declared
+      //      `richStarters` opens the body (`# @decorator(...)`) — full token
+      //      highlighting applies INSIDE (via $self) on top of the comment base
+      //      scope, so structured comments keep their colors while stray text dims;
+      //   2. a PLAIN region for every other comment — no inner patterns, so the
+      //      whole body falls to the comment scope and dims like any comment.
+      // Ordering: rich is registered first so the gated form wins; both sit at this
+      // token's position in the top-level pattern list (declaration order preserved).
+      const introducer = tokenPatternSource(tok);
+      const commentPunct = `punctuation.definition.comment.${langName}`;
+      // Declared doc-markup patterns (`**bold**`, `__italic__`, …) highlighted inside
+      // PLAIN comment bodies — token-pattern IR, nothing language-specific.
+      const markupPatterns: TmPattern[] = (tok.lineComment.markup ?? []).map((m) => ({
+        match: tokenPatternToRegex(m.pattern),
+        name: `${m.scope}.${langName}`,
+      }));
+      const richStarters = (tok.lineComment.richStarters ?? [])
+        .map((n) => grammar.tokens.find((t) => t.name === n))
+        .filter((t): t is TokenDecl => !!t);
+      if (richStarters.length) {
+        const startLookahead = richStarters.map((t) => `(?:${tokenPatternSource(t)})`).join('|');
+        // ── Multi-line constructs (`continuationBrackets`) ──
+        // A bracket left open in a rich comment continues the construct across consecutive
+        // introducer-prefixed lines (`# @import(` … `#   KEY1,` … `# )`). Each pair becomes a
+        // begin/end region nested INSIDE the line-scoped rich region — a TextMate child region
+        // suspends its parent's `$` end, so the construct spans lines while the parent still
+        // closes single-line comments. Inside a construct:
+        //   - a line-start introducer is a CONTINUATION MARKER (comment punctuation, not a new
+        //     comment) — matched first (anchored `^`, earliest-match wins);
+        //   - any other introducer opens an embedded comment to end-of-line that dims
+        //     (`#   KEY1, # note`), popping at `$` so the construct resumes next line;
+        //   - brackets nest recursively; everything else highlights via $self.
+        // An unclosed bracket runs the region to the next closer (the same hazard every
+        // hand-written TextMate grammar with multi-line constructs has) — the parser is the
+        // authority on validity, the highlighter degrades soft.
+        const contBrackets = tok.lineComment.continuationBrackets ?? [];
+        const bracketKeyOf = (open: string) => `${key}-rich-cont-${[...open].map((c) => c.charCodeAt(0).toString(16)).join('')}`;
+        const bracketIncludes = contBrackets.map(([open]) => ({ include: `#${bracketKeyOf(open)}` }));
+        // A CALL opening a construct (`# @dec=someFn(`) must get the marker-aware interior
+        // too: a callee-anchored region (e.g. the contextualScopes call-args region reached
+        // via $self) starts matching at the CALLEE — an earlier position than the bare `(`
+        // — so without a call-aware variant here it would hijack the construct with an
+        // interior that treats line-start `#` as a comment, eat the `# )` closer, and run
+        // away past the comment block. Same interior as the paren construct, begin consumes
+        // the callee (scoped like the grammar's callee token). Tried before $self.
+        const calleeForCont = grammar.tokens.find((t) => {
+          const sc = t.scope ?? classifyToken(t).scope;
+          return (sc.startsWith('variable.function') || sc.startsWith('entity.name.function')) && tokenPatternSource(t).includes('\\(');
+        });
+        const hasParenPair = contBrackets.some(([open]) => open === '(');
+        const contCallInclude = (calleeForCont && hasParenPair) ? [{ include: `#${key}-rich-cont-call` }] : [];
+        if (contBrackets.length) {
+          repository[`${key}-rich-cont-marker`] = {
+            match: `^[ \\t]*(${introducer})`,
+            captures: { '1': { name: `${scope}.${langName} ${commentPunct}` } },
+          };
+          repository[`${key}-rich-cont-comment`] = {
+            name: `${scope}.${langName}`,
+            begin: `(${introducer})`,
+            beginCaptures: { '1': { name: commentPunct } },
+            end: '$',
+            patterns: markupPatterns,
+          };
+          for (const [open, close] of contBrackets) {
+            repository[bracketKeyOf(open)] = {
+              begin: escapeRegex(open),
+              end: escapeRegex(close),
+              patterns: [
+                { include: `#${key}-rich-cont-marker` },
+                { include: `#${key}-rich-cont-comment` },
+                ...ctxOverrideIncludes,
+                ...contCallInclude,
+                ...bracketIncludes,
+                { include: '$self' },
+              ],
+            };
+          }
+          if (calleeForCont && hasParenPair) {
+            repository[`${key}-rich-cont-call`] = {
+              begin: `(${tokenPatternSource(calleeForCont)})\\s*(\\()`,
+              beginCaptures: {
+                '1': { name: `${(calleeForCont.scope ?? 'entity.name.function')}.${langName}` },
+                '2': { name: `${getScope(grammar.scopeOverrides, '(') ?? 'punctuation.section.parens.begin'}.${langName}` },
+              },
+              end: '\\)',
+              endCaptures: { '0': { name: `${getScope(grammar.scopeOverrides, ')') ?? 'punctuation.section.parens.end'}.${langName}` } },
+              patterns: [
+                { include: `#${key}-rich-cont-marker` },
+                { include: `#${key}-rich-cont-comment` },
+                ...ctxOverrideIncludes,
+                ...bracketIncludes,
+                { include: '$self' },
+              ],
+            };
+          }
+        }
+        repository[`${key}-rich`] = {
+          name: `${scope}.${langName}`,
+          begin: `(${introducer})(?=[ \\t]*(?:${startLookahead}))`,
+          beginCaptures: { '1': { name: commentPunct } },
+          end: '$',
+          patterns: [...contCallInclude, ...bracketIncludes, { include: '$self' }],
+        };
+        topPatterns.push({ include: `#${key}-rich` });
+      }
+      repository[key] = {
+        name: `${scope}.${langName}`,
+        begin: `(${introducer})`,
+        beginCaptures: { '1': { name: commentPunct } },
+        end: '$',
+        patterns: markupPatterns,
+      };
+      topPatterns.push({ include: `#${key}` });
+
     } else {
       // The bare-identifier catch-all is scoped `variable.other.readwrite` — the
       // TextMate convention for a mutable variable *reference* (what hand-written
@@ -7436,6 +7574,54 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
     topPatterns.push({ include: '#function-call' });
   }
 
+  // ── 4b2. Call-argument construct regions (contextualScopes) ──
+  // Only when the grammar declares contextual scopes: a call's argument list becomes a real
+  // begin/end region (callee + `(` … `)`), so the overrides apply inside it — and inside any
+  // nested bracket constructs — while everything else still highlights via $self. The nested
+  // pair regions are NOT top-level includes (a bare top-level `{…}` value keeps flat token
+  // scopes); they exist only inside a call or inside each other.
+  if (hasCallExpr && ctxOverrideIncludes.length) {
+    const allRuleLits = new Set<string>();
+    for (const r of grammar.rules) for (const lit of collectLiterals(r.body)) allRuleLits.add(lit);
+    const ctxPairs = ([['(', ')'], ['[', ']'], ['{', '}']] as [string, string][])
+      .filter(([o, c]) => allRuleLits.has(o) && allRuleLits.has(c) && (o !== '(' || true));
+    const pairKeyOf = (open: string) => `ctx-pair-${[...open].map((ch) => ch.charCodeAt(0).toString(16)).join('')}`;
+    const pairIncludes = ctxPairs.filter(([o]) => o !== '(').map(([o]) => ({ include: `#${pairKeyOf(o)}` }));
+    for (const [open, close] of ctxPairs) {
+      if (open === '(') continue;   // `(` is owned by the call region itself
+      repository[pairKeyOf(open)] = {
+        begin: escapeRegex(open),
+        beginCaptures: { '0': { name: `${getScope(scopeOverrides, open) ?? 'punctuation.section.brackets.begin'}.${langName}` } },
+        end: escapeRegex(close),
+        endCaptures: { '0': { name: `${getScope(scopeOverrides, close) ?? 'punctuation.section.brackets.end'}.${langName}` } },
+        patterns: [...ctxOverrideIncludes, ...pairIncludes, { include: '$self' }],
+      };
+    }
+    // callee pattern + scope: prefer the grammar's own CALLEE token (one whose pattern is
+    // gated on a following `(` — e.g. env-spec FUNCTION_NAME), else the identifier pattern.
+    // The generic ident fallback can resolve to a placeholder token in indentation grammars,
+    // so a never-matching callee skips the region entirely rather than emitting a dead rule.
+    const calleeTok = grammar.tokens.find((t) => {
+      const sc = t.scope ?? classifyToken(t).scope;
+      return (sc.startsWith('variable.function') || sc.startsWith('entity.name.function')) && tokenPatternSource(t).includes('\\(');
+    });
+    const calleeScope = calleeTok?.scope ?? 'entity.name.function';
+    const calleePattern = calleeTok ? tokenPatternSource(calleeTok) : identPattern;
+    if (!calleePattern.includes('(?!)')) {
+      repository['ctx-call-args'] = {
+      begin: `(${calleePattern})\\s*(\\()`,
+      beginCaptures: {
+        '1': { name: `${calleeScope}.${langName}` },
+        '2': { name: `${getScope(scopeOverrides, '(') ?? 'punctuation.section.parens.begin'}.${langName}` },
+      },
+      end: '\\)',
+      endCaptures: { '0': { name: `${getScope(scopeOverrides, ')') ?? 'punctuation.section.parens.end'}.${langName}` } },
+      patterns: [...ctxOverrideIncludes, ...pairIncludes, { include: '$self' }],
+      };
+      topPatterns.push({ include: '#ctx-call-args' });
+    }
+  }
+
   // ── 4b1a. Object method key: `key: (params) => ...` or `key: function(...)` ──
   // In object literals, a property key followed by `:` and a function value
   // should be scoped as entity.name.function (not plain variable.other).
@@ -8378,6 +8564,9 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
     // tried before #punctuation (which would otherwise claim the `{`/`[` as a bare bracket) and
     // before the scalar tokens. Its `{`/`[` can never lead a plain scalar, so this ranking is safe.
     if (key === 'flow-mapping' || key === 'flow-sequence') return 0.85;
+    // A call-argument construct region (contextualScopes) opens on `ident(` — it must be tried
+    // before the flat callee/ident token rules that would otherwise consume the name alone.
+    if (key === 'ctx-call-args') return 0.87;
     // A top-level token-match scoped `entity.name.tag` (e.g. an indentation grammar's mapping
     // KEY — a scalar that is the LHS of `:`) is a NAME, more specific than any string/typed-value
     // scalar it overlaps, so it must be tried first. (Markup tag names live inside begin/end

src/gen-treesitter.ts

@@ -1313,6 +1313,7 @@ function scopeToCapture(scope: string): string | null {
     ['entity.name.function', '@function'],
     ['entity.name.type', '@type'],
     ['entity.name', '@constructor'],
+    ['entity.other.attribute-name', '@attribute'],
     ['entity.other.document', '@punctuation.delimiter'],  // YAML --- / ... markers
     ['entity.other.property', '@property'],
     ['support.type.primitive', '@type.builtin'],
@@ -1690,6 +1691,29 @@ function buildHighlightsScm(
   }
   out.push('');
 
+  // ── Contextual token scopes (grammar.contextualScopes, highlight-only) ──
+  // Token T carries scope S within rule R (T's immediate enclosing rule) — expressed
+  // exactly as `(rule (token) @capture)` queries. Emitted LAST: highlight resolution is
+  // last-wins (nvim-treesitter / tree-sitter-cli), so these override the token's flat
+  // capture above inside their declared rules only.
+  const ctxQueries: string[] = [];
+  for (const entry of grammar.contextualScopes ?? []) {
+    const cap = scopeToCapture(entry.scope);
+    if (!cap) continue;
+    const tokSnake = ctx.tokenSnake.get(entry.token);
+    if (!tokSnake) continue;
+    for (const ruleName of entry.within) {
+      const ruleSnake = ctx.ruleSnake.get(ruleName);
+      if (!ruleSnake) continue;
+      ctxQueries.push(`(${ruleSnake} (${tokSnake}) ${cap})`);
+    }
+  }
+  if (ctxQueries.length) {
+    out.push(';; Contextual token scopes (grammar.contextualScopes).');
+    for (const q of ctxQueries) out.push(q);
+    out.push('');
+  }
+
   return out.join('\n');
 }