Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions crates/codegraph-core/src/db/connection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -857,6 +857,14 @@ impl NativeDatabase {
/// Batches are received as `serde_json::Value` and deserialized via serde so
/// that `null` visibility values map to `None` instead of crashing napi's
/// `Option<String>` object conversion (#709).
///
/// `file_hashes` is committed in its own transaction, separate from node
/// insertion (#1731) — callers that need edge-consistent hashes (i.e. the
/// standard incremental build pipeline) should pass an empty array here
/// and commit hashes themselves once resolveImports/buildEdges have
/// finished rebuilding the affected files' edges (see
/// `insertNodes.commitFileHashes` on the JS side, or
/// `insert_nodes::commit_file_hashes` for the all-Rust orchestrator).
#[napi(ts_args_type = "batches: Array<{ file: string; definitions: Array<{ name: string; kind: string; line: number; endLine?: number; visibility?: string; children: Array<{ name: string; kind: string; line: number; endLine?: number; visibility?: string }> }>; exports: Array<{ name: string; kind: string; line: number }> }>, fileHashes: FileHashEntry[], removedFiles: string[]")]
pub fn bulk_insert_nodes(
&self,
Expand All @@ -869,9 +877,16 @@ impl NativeDatabase {
napi::Error::from_reason(format!("bulk_insert_nodes: invalid batches: {e}"))
})?;
let conn = self.conn()?;
Ok(insert_nodes::do_insert_nodes(conn, &batches, &file_hashes, &removed_files)
let insert_ok = insert_nodes::do_insert_nodes(conn, &batches, &removed_files)
.inspect_err(|e| eprintln!("[NativeDatabase] bulk_insert_nodes failed: {e}"))
.is_ok())
.is_ok();
if !insert_ok {
return Ok(false);
}
let hashes_ok = insert_nodes::commit_file_hashes(conn, &file_hashes)
.inspect_err(|e| eprintln!("[NativeDatabase] bulk_insert_nodes hash commit failed: {e}"))
.is_ok();
Ok(hashes_ok)
}

/// Bulk-insert edge rows using chunked multi-value INSERT statements.
Expand Down
25 changes: 22 additions & 3 deletions crates/codegraph-core/src/domain/graph/builder/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,13 @@
//! 2. Collect files (with gitignore + extension filter)
//! 3. Detect changes (tiered: journal/mtime/hash)
//! 4. Parse files in parallel (existing `parallel::parse_files_parallel`)
//! 5. Insert nodes (existing `insert_nodes::do_insert_nodes`)
//! 5. Insert nodes (existing `insert_nodes::do_insert_nodes`) — file_hashes
//! for changed files is NOT written here; see step 7
//! 6. Resolve imports (existing `resolve::resolve_imports_batch`)
//! 6b. Re-parse barrel candidates (incremental only)
//! 7. Build import edges + call edges + barrel resolution
//! 7. Build import edges + call edges + barrel resolution, then commit
//! file_hashes for changed files (`insert_nodes::commit_file_hashes`) now
//! that their edges match this revision (#1731)
//! 8. Structure metrics + role classification
//! 9. Finalize (metadata, journal)

Expand Down Expand Up @@ -505,13 +508,20 @@ pub fn run_pipeline(
timing.parse_ms = t0.elapsed().as_secs_f64() * 1000.0;

// ── Stage 5: Insert nodes ──────────────────────────────────────────
// file_hashes for these files is deliberately NOT written here — only
// node/edge-adjacent (contains, parameter_of) data plus removed-file hash
// cleanup. Committing a changed file's hash this early (before Stage 7
// rebuilds its import/call edges) would let the hash claim "up to date"
// even if edge-building later fails or is interrupted, permanently
// desyncing file_hashes from the edges it's supposed to gate re-parsing
// on (#1731). The hash is committed at the end of Stage 7 instead, once
// edges genuinely match this revision — see `commit_file_hashes` below.
let t0 = Instant::now();
let insert_batches = build_insert_batches(&file_symbols);
let file_hashes = build_file_hash_entries(&parse_changes);
let _ = crate::domain::graph::builder::stages::insert_nodes::do_insert_nodes(
conn,
&insert_batches,
&file_hashes,
&change_result.removed,
);
detect_changes::heal_metadata(conn, &change_result.metadata_updates);
Expand Down Expand Up @@ -564,6 +574,15 @@ pub fn run_pipeline(
build_and_insert_call_edges(conn, &file_symbols, &import_ctx, !change_result.is_full_build);

reconnect_saved_reverse_dep_edges(conn, &saved_reverse_dep_edges);

// Now that edges reflect this revision, commit file_hashes for the
// changed files (#1731). Deferred from Stage 5 — see the comment there.
if let Err(e) = crate::domain::graph::builder::stages::insert_nodes::commit_file_hashes(
conn,
&file_hashes,
) {
eprintln!("[codegraph] commit_file_hashes failed: {e}");
}
timing.edges_ms = t0.elapsed().as_secs_f64() * 1000.0;

// ── Stage 8: Structure + roles ─────────────────────────────────────
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,17 +94,41 @@ fn query_node_ids(
Ok(map)
}

/// Insert nodes/children/containment edges, plus hash cleanup for removed
/// files. `file_hashes` for CHANGED files is intentionally NOT written here —
/// see [`commit_file_hashes`] below (#1731): committing a file's hash this
/// early (before import/call edges are built) would let the hash claim
/// "up to date" even if edge-building later fails or is interrupted. Deleting
/// a removed file's hash has no such coupling risk (the file has no edges to
/// keep in sync with), so `removed_files` is still handled immediately.
pub(crate) fn do_insert_nodes(
conn: &Connection,
batches: &[InsertNodesBatch],
file_hashes: &[FileHashEntry],
removed_files: &[String],
) -> rusqlite::Result<()> {
let tx = conn.unchecked_transaction()?;
insert_file_nodes(&tx, batches)?;
let (contains_edges, param_of_edges) = insert_symbol_nodes(&tx, batches)?;
upsert_node_batch(&tx, &contains_edges, &param_of_edges)?;
upsert_file_hashes(&tx, file_hashes, removed_files)?;
delete_removed_file_hashes(&tx, removed_files)?;
tx.commit()
}

/// Commit `file_hashes` for changed files. Called separately from
/// [`do_insert_nodes`], strictly AFTER import/call edges have been rebuilt
/// for these files (#1731) — see that function's doc comment and the call
/// site in `pipeline.rs` Stage 7. Own transaction: node insertion and hash
/// commit are two distinct steps in the coupling this fix establishes, not
/// one atomic unit.
pub(crate) fn commit_file_hashes(
conn: &Connection,
file_hashes: &[FileHashEntry],
) -> rusqlite::Result<()> {
if file_hashes.is_empty() {
return Ok(());
}
let tx = conn.unchecked_transaction()?;
upsert_file_hashes(&tx, file_hashes)?;
tx.commit()
}

Expand Down Expand Up @@ -297,43 +321,57 @@ fn upsert_node_batch(
Ok(())
}

/// Phase 4: upsert file hashes and remove hashes for deleted files. No-ops
/// gracefully when the `file_hashes` table has not been created yet (e.g.
/// during the initial schema migration).
/// Returns true iff the `file_hashes` table exists. Both halves of the old
/// combined upsert/delete function (now [`upsert_file_hashes`] and
/// [`delete_removed_file_hashes`]) no-op gracefully when it's absent — e.g.
/// during the initial schema migration.
fn has_file_hashes_table(tx: &rusqlite::Transaction) -> bool {
tx.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='file_hashes'")
.and_then(|mut s| s.query_row([], |_| Ok(true)))
.unwrap_or(false)
}

/// Phase 4a: upsert file hashes for changed files. Split out from the
/// removed-files delete (below) so the two can be committed at different
/// points in the pipeline — see [`commit_file_hashes`] (#1731).
fn upsert_file_hashes(
tx: &rusqlite::Transaction,
file_hashes: &[FileHashEntry],
removed_files: &[String],
) -> rusqlite::Result<()> {
let has_file_hashes = tx
.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='file_hashes'")
.and_then(|mut s| s.query_row([], |_| Ok(true)))
.unwrap_or(false);

if !has_file_hashes {
if file_hashes.is_empty() || !has_file_hashes_table(tx) {
return Ok(());
}

{
let mut upsert = tx.prepare_cached(
"INSERT OR REPLACE INTO file_hashes (file, hash, mtime, size) \
VALUES (?1, ?2, ?3, ?4)",
)?;
for entry in file_hashes {
upsert.execute(params![
&entry.file,
&entry.hash,
entry.mtime as i64,
entry.size as i64
])?;
}
let mut upsert = tx.prepare_cached(
"INSERT OR REPLACE INTO file_hashes (file, hash, mtime, size) \
VALUES (?1, ?2, ?3, ?4)",
)?;
for entry in file_hashes {
upsert.execute(params![
&entry.file,
&entry.hash,
entry.mtime as i64,
entry.size as i64
])?;
}

if !removed_files.is_empty() {
let mut delete = tx.prepare_cached("DELETE FROM file_hashes WHERE file = ?1")?;
for file in removed_files {
delete.execute(params![file])?;
}
Ok(())
}

/// Phase 4b: remove file_hashes rows for deleted files. Safe to run
/// immediately alongside node insertion (unlike the upsert above) — a
/// removed file has no edges that need to stay in sync with its hash.
fn delete_removed_file_hashes(
tx: &rusqlite::Transaction,
removed_files: &[String],
) -> rusqlite::Result<()> {
if removed_files.is_empty() || !has_file_hashes_table(tx) {
return Ok(());
}

let mut delete = tx.prepare_cached("DELETE FROM file_hashes WHERE file = ?1")?;
for file in removed_files {
delete.execute(params![file])?;
}

Ok(())
Expand Down
26 changes: 25 additions & 1 deletion src/domain/graph/builder/incremental.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import {
resolveReceiverEdge,
resolveSameClassQualifiedMethod,
} from './call-resolver.js';
import { BUILTIN_RECEIVERS, readFileSafe } from './helpers.js';
import { BUILTIN_RECEIVERS, fileHash, fileStat, readFileSafe } from './helpers.js';
import { importNamePairs } from './import-utils.js';

// ── Local types ─────────────────────────────────────────────────────────
Expand All @@ -42,6 +42,14 @@ export interface IncrementalStmts {
listSymbols: { all: (...params: unknown[]) => unknown[] };
findNodeInFile: { all: (...params: unknown[]) => unknown[] };
findNodeByName: { all: (...params: unknown[]) => unknown[] };
/**
* Upsert a `file_hashes` row: `(relPath, hash, mtime, size)`. Called only
* after a file's edges have been fully rebuilt (#1731) — see the call site
* in `rebuildFile` for why this can't happen any earlier.
*/
upsertFileHash: { run: (...params: unknown[]) => unknown };
/** Delete a `file_hashes` row for a file removed from disk. */
deleteFileHash: { run: (...params: unknown[]) => unknown };
}

interface RebuildResult {
Expand Down Expand Up @@ -938,6 +946,10 @@ export async function rebuildFile(

if (!fs.existsSync(filePath)) {
if (cache) (cache as { remove(p: string): void }).remove(filePath);
// The file no longer exists, so it has no edges to keep in sync with a
// hash — delete it immediately (mirrors the full-build removed-file path
// in insertNodes.ts, which is likewise unconditional).
stmts.deleteFileHash.run(relPath);
return buildDeletionResult(relPath, oldNodes, edgesBefore, oldSymbols, diffSymbols);
}

Expand Down Expand Up @@ -982,6 +994,18 @@ export async function rebuildFile(
// their edges unchanged.
const totalEdgesBefore = edgesBefore + reverseDepsEdgesBefore;

// Commit file_hashes now that relPath's edges have been fully rebuilt to
// match `code` (#1731). Writing this any earlier — or not at all, as
// before this fix — would leave file_hashes stale relative to the edges
// rebuildEdgesForTargetFile just wrote, so the next full/incremental
// `codegraph build` would either redundantly reprocess an already-correct
// file (stale-hash direction) or, combined with other divergent writers,
// risk trusting a hash that doesn't actually reflect these edges.
const stat = fileStat(filePath);
if (stat) {
stmts.upsertFileHash.run(relPath, fileHash(code), stat.mtime, stat.size);
}

const symbolDiff = diffSymbols ? diffSymbols(oldSymbols, newSymbols) : null;
const event = oldNodes === 0 ? 'added' : 'modified';

Expand Down
10 changes: 9 additions & 1 deletion src/domain/graph/builder/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import { buildStructure } from './stages/build-structure.js';
import { collectFiles } from './stages/collect-files.js';
import { detectChanges, detectNoChanges } from './stages/detect-changes.js';
import { finalize } from './stages/finalize.js';
import { insertNodes } from './stages/insert-nodes.js';
import { commitFileHashes, insertNodes } from './stages/insert-nodes.js';
import {
closeNativeDb,
refreshJsDb,
Expand Down Expand Up @@ -319,6 +319,14 @@ async function runPipelineStages(ctx: PipelineContext): Promise<void> {

await resolveImports(ctx);
await buildEdges(ctx);

// Commit file_hashes for changed files now that their edges have been
// rebuilt to match (#1731) — see commitFileHashes() for the rationale.
// Placed before buildStructure/runAnalyses so a failure there still
// benefits from a hash that accurately reflects the (now-consistent)
// node+edge state committed so far.
commitFileHashes(ctx);

await buildStructure(ctx);

// Reopen nativeDb for feature modules (ast, cfg, complexity, dataflow).
Expand Down
Loading
Loading