From 067674fcc8284db0722afa5cfaf6fed0ef14c92a Mon Sep 17 00:00:00 2001 From: carlos-alm Date: Mon, 6 Jul 2026 05:12:49 -0600 Subject: [PATCH 1/2] refactor: dedupe computeSavings via pct helper and persist partial token-benchmark results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit computeSavings reimplemented the percentage-reduction formula inline instead of using the pct helper already defined for computeAggregate. Both call sites now share pct — verified byte-identical output across normal, zero-denominator, negative-savings, and null-guard cases. main()'s per-issue loop only serialized results to stdout after the full loop completed, so a crash partway through (another issue throwing, or the optional --perf benchmarks failing) discarded every already-computed result. The loop now overwrites token-benchmark.partial.json after each issue; the file is removed once the full run succeeds and the final JSON has been printed. Fixes #1757 Impact: 5 functions changed, 17 affected --- .gitignore | 1 + scripts/token-benchmark.ts | 56 +++++++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 74b27941c..d8e9f3c1a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ crates/codegraph-core/*.node .claude/worktrees/ generated/DEPENDENCIES.md generated/DEPENDENCIES.json +token-benchmark.partial.json artifacts/ pkg/ target/ diff --git a/scripts/token-benchmark.ts b/scripts/token-benchmark.ts index 7e66cb6b8..87d4c9698 100644 --- a/scripts/token-benchmark.ts +++ b/scripts/token-benchmark.ts @@ -81,6 +81,11 @@ function round2(n) { return Math.round(n * 100) / 100; } +/** Percentage reduction from `a` to `b` (e.g. token/cost savings). Returns 0 when `a <= 0`. */ +function pct(a, b) { + return a > 0 ? Math.round(((a - b) / a) * 100) : 0; +} + function git(args, cwd) { return execFileSync('git', args, { cwd, stdio: 'pipe', encoding: 'utf8' }).trim(); } @@ -423,19 +428,9 @@ function medianForRuns(runs) { /** Token + cost savings (% reduction) between two median objects. */ function computeSavings(baselineMedian, codegraphMedian) { if (!baselineMedian || !codegraphMedian || baselineMedian.inputTokens <= 0) return null; - const tokenSavings = - ((baselineMedian.inputTokens - codegraphMedian.inputTokens) / - baselineMedian.inputTokens) * - 100; - const costSavings = - baselineMedian.totalCostUsd > 0 - ? ((baselineMedian.totalCostUsd - codegraphMedian.totalCostUsd) / - baselineMedian.totalCostUsd) * - 100 - : 0; return { - inputTokensPct: Math.round(tokenSavings), - costPct: Math.round(costSavings), + inputTokensPct: pct(baselineMedian.inputTokens, codegraphMedian.inputTokens), + costPct: pct(baselineMedian.totalCostUsd, codegraphMedian.totalCostUsd), }; } @@ -484,7 +479,6 @@ function computeAggregate(results) { const totalCodegraphTokens = sum((r) => r.codegraph.median.inputTokens); const totalBaselineCost = sum((r) => r.baseline.median.totalCostUsd); const totalCodegraphCost = sum((r) => r.codegraph.median.totalCostUsd); - const pct = (a, b) => (a > 0 ? Math.round(((a - b) / a) * 100) : 0); return { savings: { @@ -502,6 +496,36 @@ function computeAggregate(results) { // ── Main ────────────────────────────────────────────────────────────────── +/** Where in-progress results are (re)written after each issue completes. */ +const PARTIAL_RESULTS_PATH = path.join(root, 'token-benchmark.partial.json'); + +/** + * Overwrite the partial-results snapshot with the results collected so far. + * Called after each issue completes so a later crash (another issue + * throwing, or `runPerfBenchmarks` failing) doesn't discard the + * already-computed results for the issues that already finished (#1757). + */ +function writePartialResults(results, totalIssues) { + const partialOutput = { + version: benchVersion, + date: new Date().toISOString().slice(0, 10), + model: MODEL, + runsPerIssue: RUNS, + maxTurns: MAX_TURNS, + maxBudgetUsd: MAX_BUDGET, + partial: true, + completedIssues: results.length, + totalIssues, + issues: results, + aggregate: computeAggregate(results), + perfBenchmarks: null, + }; + fs.writeFileSync(PARTIAL_RESULTS_PATH, JSON.stringify(partialOutput, null, 2)); + console.error( + ` Partial results written to ${PARTIAL_RESULTS_PATH} (${results.length}/${totalIssues} issues)`, + ); +} + async function main() { // Resolve Next.js directory const nextjsDir = flags['nextjs-dir'] @@ -522,6 +546,7 @@ async function main() { const results = []; for (const issue of selectedIssues) { results.push(await runIssueExperiment(issue, nextjsDir)); + writePartialResults(results, selectedIssues.length); } const aggregate = computeAggregate(results); @@ -549,6 +574,11 @@ async function main() { }; console.log(JSON.stringify(output, null, 2)); + + // Full run (including perf benchmarks, if requested) succeeded — the + // complete results are now in the stdout output above, so the partial + // snapshot is no longer needed. + fs.rmSync(PARTIAL_RESULTS_PATH, { force: true }); } main().catch((err) => { From 1a4e1a9ea4c0939a97f0ba21f6c1b3f97d6fac8b Mon Sep 17 00:00:00 2001 From: carlos-alm Date: Mon, 6 Jul 2026 15:35:32 -0600 Subject: [PATCH 2/2] fix: wrap partial-results write in try/catch to avoid losing progress on disk errors (Greptile) --- scripts/token-benchmark.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/token-benchmark.ts b/scripts/token-benchmark.ts index 87d4c9698..65834e8a0 100644 --- a/scripts/token-benchmark.ts +++ b/scripts/token-benchmark.ts @@ -520,10 +520,14 @@ function writePartialResults(results, totalIssues) { aggregate: computeAggregate(results), perfBenchmarks: null, }; - fs.writeFileSync(PARTIAL_RESULTS_PATH, JSON.stringify(partialOutput, null, 2)); - console.error( - ` Partial results written to ${PARTIAL_RESULTS_PATH} (${results.length}/${totalIssues} issues)`, - ); + try { + fs.writeFileSync(PARTIAL_RESULTS_PATH, JSON.stringify(partialOutput, null, 2)); + console.error( + ` Partial results written to ${PARTIAL_RESULTS_PATH} (${results.length}/${totalIssues} issues)`, + ); + } catch (err) { + console.error(` Warning: could not write partial results — ${err.message}`); + } } async function main() {