Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 4 additions & 28 deletions src/domain/graph/builder/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import fs from 'node:fs';
import path from 'node:path';
import { purgeFilesData } from '../../../db/index.js';
import { debug, warn } from '../../../infrastructure/logger.js';
import { getOrCreatePerDbChunkStmt } from '../../../shared/chunked-stmt-cache.js';
import { buildIgnoreSet, EXTENSIONS, normalizePath } from '../../../shared/constants.js';
import { toErrorMessage } from '../../../shared/errors.js';
import { compileGlobs, globToRegex, matchesAny } from '../../../shared/globs.js';
Expand Down Expand Up @@ -357,33 +358,8 @@ const BATCH_CHUNK = 500;
const nodeStmtCache = new WeakMap<BetterSqlite3Database, Map<number, SqliteStatement>>();
const edgeStmtCache = new WeakMap<BetterSqlite3Database, Map<number, SqliteStatement>>();

/**
* Get (or lazily prepare + cache) a multi-value INSERT statement for a given
* chunk size, keyed per-database. Shared by getNodeStmt/getEdgeStmt, which
* previously duplicated this exact WeakMap<db, Map<chunkSize, stmt>>
* cache-getter shape — only the SQL text differed.
*/
function getOrCreateBatchStmt(
cache: WeakMap<BetterSqlite3Database, Map<number, SqliteStatement>>,
db: BetterSqlite3Database,
chunkSize: number,
buildSql: (chunkSize: number) => string,
): SqliteStatement {
let perDb = cache.get(db);
if (!perDb) {
perDb = new Map();
cache.set(db, perDb);
}
let stmt = perDb.get(chunkSize);
if (!stmt) {
stmt = db.prepare(buildSql(chunkSize));
perDb.set(chunkSize, stmt);
}
return stmt;
}

function getNodeStmt(db: BetterSqlite3Database, chunkSize: number): SqliteStatement {
return getOrCreateBatchStmt(nodeStmtCache, db, chunkSize, (n) => {
return getOrCreatePerDbChunkStmt(nodeStmtCache, db, chunkSize, (n) => {
const ph = '(?,?,?,?,?,?,?,?,?)';
return (
'INSERT OR IGNORE INTO nodes (name,kind,file,line,end_line,parent_id,qualified_name,scope,visibility) VALUES ' +
Expand All @@ -393,7 +369,7 @@ function getNodeStmt(db: BetterSqlite3Database, chunkSize: number): SqliteStatem
}

function getEdgeStmt(db: BetterSqlite3Database, chunkSize: number): SqliteStatement {
return getOrCreateBatchStmt(edgeStmtCache, db, chunkSize, (n) => {
return getOrCreatePerDbChunkStmt(edgeStmtCache, db, chunkSize, (n) => {
const ph = '(?,?,?,?,?,?,?)';
return (
'INSERT INTO edges (source_id,target_id,kind,confidence,dynamic,technique,dynamic_kind) VALUES ' +
Expand Down Expand Up @@ -451,7 +427,7 @@ export function batchInsertEdges(db: BetterSqlite3Database, rows: unknown[][]):
const exportStmtCache = new WeakMap<BetterSqlite3Database, Map<number, SqliteStatement>>();

function getExportStmt(db: BetterSqlite3Database, chunkSize: number): SqliteStatement {
return getOrCreateBatchStmt(exportStmtCache, db, chunkSize, (n) => {
return getOrCreatePerDbChunkStmt(exportStmtCache, db, chunkSize, (n) => {
const conditions = Array.from(
{ length: n },
() => '(name = ? AND kind = ? AND file = ? AND line = ?)',
Expand Down
15 changes: 7 additions & 8 deletions src/features/structure.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import path from 'node:path';
import { getBuildMeta, getNodeId, setBuildMeta, testFilterSQL } from '../db/index.js';
import { debug } from '../infrastructure/logger.js';
import { getOrCreateChunkStmt } from '../shared/chunked-stmt-cache.js';
import { getAncestorDirs, normalizePath } from '../shared/constants.js';
import type { BetterSqlite3Database } from '../types.js';
import type { BetterSqlite3Database, SqliteStatement as DbSqliteStatement } from '../types.js';

// ─── Build-time helpers ───────────────────────────────────────────────

Expand Down Expand Up @@ -538,19 +539,17 @@ function batchUpdateRoles(
resetFn: () => void,
): void {
const ROLE_CHUNK = 500;
const roleStmtCache = new Map<number, SqliteStatement>();
const roleStmtCache = new Map<number, DbSqliteStatement>();
db.transaction(() => {
resetFn();
for (const [role, ids] of idsByRole) {
for (let i = 0; i < ids.length; i += ROLE_CHUNK) {
const end = Math.min(i + ROLE_CHUNK, ids.length);
const chunkSize = end - i;
let stmt = roleStmtCache.get(chunkSize);
if (!stmt) {
const placeholders = Array.from({ length: chunkSize }, () => '?').join(',');
stmt = db.prepare(`UPDATE nodes SET role = ? WHERE id IN (${placeholders})`);
roleStmtCache.set(chunkSize, stmt);
}
const stmt = getOrCreateChunkStmt(roleStmtCache, db, chunkSize, (n) => {
const placeholders = Array.from({ length: n }, () => '?').join(',');
return `UPDATE nodes SET role = ? WHERE id IN (${placeholders})`;
});
const vals: unknown[] = [role];
for (let j = i; j < end; j++) vals.push(ids[j]);
stmt.run(...vals);
Expand Down
58 changes: 58 additions & 0 deletions src/shared/chunked-stmt-cache.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/**
* Chunk-size-keyed prepared-statement cache.
*
* Many call sites build a multi-value SQL statement (`INSERT ... VALUES
* (?,?,?),(?,?,?),...` or `UPDATE ... WHERE id IN (?,?,?)`) sized to a batch
* of rows, then re-run that exact statement for every batch of the same size.
* Recompiling the statement per batch is wasteful — this module memoizes the
* prepared statement by chunk size so it is built once per distinct size.
*
* Extracted from a duplicate between the node/edge/export batch-insert
* helpers in `domain/graph/builder/helpers.ts` and the node-role batch-update
* in `features/structure.ts`, which each re-implemented this exact
* check-cache/prepare/cache-set shape independently.
*/
import type { BetterSqlite3Database, SqliteStatement } from '../types.js';

/**
* Get (or lazily prepare + cache) a SQL statement for a given chunk size,
* memoized in `cache`. `buildSql` is only invoked on a cache miss.
*/
export function getOrCreateChunkStmt<TRow = unknown>(
cache: Map<number, SqliteStatement<TRow>>,
db: BetterSqlite3Database,
chunkSize: number,
buildSql: (chunkSize: number) => string,
): SqliteStatement<TRow> {
let stmt = cache.get(chunkSize);
if (!stmt) {
stmt = db.prepare<TRow>(buildSql(chunkSize));
cache.set(chunkSize, stmt);
}
return stmt;
}

/**
* Per-database variant of {@link getOrCreateChunkStmt}: resolves (or lazily
* creates) the chunk-size cache scoped to `db` inside `dbCache`, then
* delegates to it.
*
* Use this when the cache must persist across many calls (e.g. repeated
* batch inserts over the lifetime of a build) and needs to be keyed per
* database instance — so independent connections (such as isolated test
* databases) never collide, and cached statements are released once `db` is
* garbage collected.
*/
export function getOrCreatePerDbChunkStmt<TRow = unknown>(
dbCache: WeakMap<BetterSqlite3Database, Map<number, SqliteStatement<TRow>>>,
db: BetterSqlite3Database,
chunkSize: number,
buildSql: (chunkSize: number) => string,
): SqliteStatement<TRow> {
let perDb = dbCache.get(db);
if (!perDb) {
perDb = new Map();
dbCache.set(db, perDb);
}
return getOrCreateChunkStmt(perDb, db, chunkSize, buildSql);
}
Loading