Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion crates/codegraph-core/src/extractors/javascript.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1896,7 +1896,14 @@ fn walk_ast_nodes_depth(node: &Node, source: &[u8], ast_nodes: &mut Vec<AstNode>
}
return;
}
"string" | "template_string" => {
// Guard on `is_named()`: tree-sitter-typescript's `predefined_type`
// production (the `string`/`number`/`boolean`/... primitive type
// keywords) lexes its keyword as an anonymous token whose `kind()`
// string is identical to the *named* `string` literal node type.
// Without this guard, `name: string` type annotations are
// misclassified as string-literal ast_nodes (#1729). Mirrors the
// WASM-side guard in `ast-store-visitor.ts::resolveAstKind`.
"string" | "template_string" if node.is_named() => {
let raw = node_text(node, source);
// Strip quotes to get content
let content = raw
Expand Down
2 changes: 2 additions & 0 deletions src/ast-analysis/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import { computeLOCMetrics, computeMaintainabilityIndex } from './metrics.js';
import {
AST_STRING_CONFIGS,
AST_TYPE_MAPS,
astRequiresNamedNode,
astStopRecurseKinds,
CFG_RULES,
COMPLEXITY_RULES,
Expand Down Expand Up @@ -475,6 +476,7 @@ function setupAstVisitor(
nodeIdMap,
stringConfig,
astStopRecurseKinds(langId),
astRequiresNamedNode(langId),
);
}

Expand Down
20 changes: 20 additions & 0 deletions src/ast-analysis/rules/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -378,3 +378,23 @@ export function astStopRecurseKinds(langId: string): ReadonlySet<string> {
}
return EMPTY_STOP_RECURSE;
}

// ─── Per-language "named-node-required" guard ────────────────────────────
//
// tree-sitter-typescript's `predefined_type` production (the `string`,
// `number`, `boolean`, `void`, ... primitive type keywords) lexes its
// keyword as an anonymous grammar token whose `type` string is identical to
// the *named* `string` node type used for real string literals — the
// grammar's own node-types.json lists both a `"string", named: true` entry
// (the literal) and a `"string", named: false` entry (the keyword token).
// Matching by `node.type` alone therefore misclassifies `name: string`
// type annotations as string-literal ast_nodes (#1729). Mirrors the native
// `is_named()` guard in `extractors/javascript.rs::walk_ast_nodes_depth`.
//
// Scoped to the JS family: no other bundled grammar currently maps an
// AST_TYPE_MAPS key that collides with an anonymous token of the same name.
const JS_REQUIRES_NAMED_NODE: ReadonlySet<string> = new Set(['javascript', 'typescript', 'tsx']);

export function astRequiresNamedNode(langId: string): boolean {
return JS_REQUIRES_NAMED_NODE.has(langId);
}
24 changes: 22 additions & 2 deletions src/ast-analysis/visitors/ast-store-visitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -273,9 +273,28 @@ function collectNode(ctx: CollectCtx, node: TreeSitterNode, kind: string): void
* Object() function), which then crashes the worker boundary with
* "function Object() { [native code] } could not be cloned" when the
* resulting astNodes row is structured-cloned back to the main thread.
*
* When `requireNamedNode` is set, anonymous `string`/`template_string`
* grammar tokens are rejected even though their `type` string matches a
* mapped key — e.g. TypeScript's `predefined_type` keyword (`string`,
* `number`, ...) lexes as an unnamed token whose type collides with the
* named `string` literal node (#1729). Scoped to just those two node types
* (not every mapped kind) to mirror the native `is_named()` guard, which
* only appears on the `"string" | "template_string"` match arm in
* `extractors/javascript.rs::walk_ast_nodes_depth` — every other mapped kind
* (call, new, throw, await, regex) is always a named production, so gating
* them too would silently diverge from the native engine's behavior the
* moment that stops being true.
*/
function resolveAstKind(node: TreeSitterNode, astTypeMap: Record<string, string>): string | null {
const NAMED_NODE_REQUIRED_TYPES: ReadonlySet<string> = new Set(['string', 'template_string']);

function resolveAstKind(
node: TreeSitterNode,
astTypeMap: Record<string, string>,
requireNamedNode: boolean,
): string | null {
if (!Object.hasOwn(astTypeMap, node.type)) return null;
if (requireNamedNode && !node.isNamed && NAMED_NODE_REQUIRED_TYPES.has(node.type)) return null;
return astTypeMap[node.type] || null;
Comment thread
carlos-alm marked this conversation as resolved.
}

Expand All @@ -286,6 +305,7 @@ export function createAstStoreVisitor(
nodeIdMap: Map<string, number>,
stringConfig: AstStringConfig = DEFAULT_STRING_CONFIG,
stopRecurseKinds: ReadonlySet<string> = new Set(),
requireNamedNode = false,
): Visitor {
const newTypes = newTypesFor(astTypeMap);
// When nodeIdMap is empty, parentNodeId resolution is wasted work — the
Expand All @@ -311,7 +331,7 @@ export function createAstStoreVisitor(
// unrelated subtree. The parent call's skipChildren handles the intended case.
if (ctx.matched.has(node.id)) return;

const kind = resolveAstKind(node, astTypeMap);
const kind = resolveAstKind(node, astTypeMap, requireNamedNode);
if (!kind) return;

collectNode(ctx, node, kind);
Expand Down
2 changes: 2 additions & 0 deletions src/domain/wasm-worker-entry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import { computeLOCMetrics, computeMaintainabilityIndex } from '../ast-analysis/
import {
AST_STRING_CONFIGS,
AST_TYPE_MAPS,
astRequiresNamedNode,
astStopRecurseKinds,
CFG_RULES,
COMPLEXITY_RULES,
Expand Down Expand Up @@ -606,6 +607,7 @@ function buildAstVisitor(
new Map<string, number>(),
stringConfig,
astStopRecurseKinds(langId),
astRequiresNamedNode(langId),
);
}

Expand Down
2 changes: 2 additions & 0 deletions src/features/ast.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import path from 'node:path';
import {
AST_STRING_CONFIGS,
AST_TYPE_MAPS,
astRequiresNamedNode,
astStopRecurseKinds,
} from '../ast-analysis/rules/index.js';
import { buildExtensionSet } from '../ast-analysis/shared.js';
Expand Down Expand Up @@ -244,6 +245,7 @@ function walkAst(
nodeIdMap,
stringConfig,
astStopRecurseKinds(langId),
astRequiresNamedNode(langId),
);
const results = walkWithVisitors(rootNode, [visitor], langId);
const collected = (results['ast-store'] || []) as AstRow[];
Expand Down
10 changes: 10 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -777,6 +777,16 @@ export interface TreeSitterNode {
id: number;
type: string;
text: string;
/**
* Whether this node is a named grammar production vs. an anonymous token
* (keyword/punctuation). Tree-sitter grammars can define an anonymous
* token whose text matches a *named* node's type string — e.g.
* tree-sitter-typescript's `predefined_type` wraps an anonymous `string`
* keyword token that collides with the named `string` literal node type
* (#1729). Consumers matching by `type` alone must also check `isNamed`
* to avoid misclassifying keyword tokens as literal/expression nodes.
*/
isNamed: boolean;
startPosition: { row: number; column: number };
endPosition: { row: number; column: number };
childCount: number;
Expand Down
2 changes: 2 additions & 0 deletions tests/engines/ast-parity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { beforeAll, describe, expect, it } from 'vitest';
import {
AST_STRING_CONFIGS,
AST_TYPE_MAPS,
astRequiresNamedNode,
astStopRecurseKinds,
} from '../../src/ast-analysis/rules/index.js';
import { walkWithVisitors } from '../../src/ast-analysis/visitor.js';
Expand Down Expand Up @@ -420,6 +421,7 @@ let run () =
new Map(),
stringConfig,
astStopRecurseKinds(langId),
astRequiresNamedNode(langId),
);
const results = walkWithVisitors(tree.rootNode as any, [visitor], langId);
const rows = (results['ast-store'] || []) as unknown[];
Expand Down
Loading
Loading