diff --git a/src/__tests__/atlas-adapter-episodic.test.ts b/src/__tests__/atlas-adapter-episodic.test.ts index 4135813..b15cdd4 100644 --- a/src/__tests__/atlas-adapter-episodic.test.ts +++ b/src/__tests__/atlas-adapter-episodic.test.ts @@ -170,10 +170,16 @@ describe("episodic leaf adapter (aimock)", () => { expect(fragment.provenance.classification.freshness.as_of).toBe( "2026-06-07", ); - // The conv path is the provenance url + source label so the fragment is - // traceable to its transcript. + // The conv path is the provenance url so the fragment stays traceable to + // its transcript — `provenance.url` is a bounded attribution slot retained + // as harvest attribution (spec §3.1-E3 rationale). expect(fragment.provenance.url).toBe(CONV_PATH); - expect(fragment.provenance.source).toBe(CONV_PATH); + // `provenance.source`, by contrast, is externally persisted via + // toSeedEntryRow (spec §3.1) and so is run through the env-reference + // sanitizer (§3.3): the machine-local `~/.claude/…` conv path has no + // recognized repo top-level segment, so E1 rewrites it to `` — + // the private home-dir path never enters the external corpus. + expect(fragment.provenance.source).toBe(""); // The top-level provenance.date carries the SAME window date as // freshness.as_of — canonicalize.ts reads provenance.date (not // freshness.as_of) for recency() and supersedes(), so a fragment without diff --git a/src/__tests__/atlas-adapter-github.test.ts b/src/__tests__/atlas-adapter-github.test.ts index 8d4a75d..81f4e66 100644 --- a/src/__tests__/atlas-adapter-github.test.ts +++ b/src/__tests__/atlas-adapter-github.test.ts @@ -567,6 +567,158 @@ describe("distillBodyToContent — the NARROW shared helper (B2)", () => { expect(out).not.toContain("boilerplate line"); }); + it("keeps fence parity when a real fence has an INTERNAL blank line and closes, then a boilerplate section follows (P3)", () => { + // Over-KEEP bucket (a) / parity-inversion: the outside-section + // unterminated-fence recovery fires on a BLANK line inside a LEGITIMATE + // fenced block that DOES later close. The recovery flips `inFence` false + // mid-fence; the block's REAL closing ``` then re-toggles `inFence` to + // true while the parser is actually OUTSIDE any fence — inverting parity + // for the rest of the body. With parity inverted and NO trailing blank + // line after the closer, a subsequent boilerplate heading (`## Test plan`) + // takes the `if (inFence)` branch, so heading parsing is short-circuited + // and the whole boilerplate section is silently RETAINED as literal + // content; symmetrically, real trailing prose can be swallowed. Fence + // parity must be correct regardless of internal blank lines in a real + // fenced block. + const body = + "Real rationale prose.\n" + + "```sh\n" + // opens outside any dropped section + "echo one\n" + + "\n" + // INTERNAL blank line inside the real fence + "echo two\n" + + "```\n" + // the REAL closer — NO trailing blank line before the heading + "## Test plan\n" + // a REAL boilerplate heading that MUST still drop + "- [x] ran the suite\n" + + "boilerplate detail line\n" + + "\n" + + "## Rationale\n" + + "This trailing prose is REAL why/how content that must survive."; + const out = distillBodyToContent(body); + // The real prose and the whole fenced block survive. + expect(out).toContain("Real rationale prose."); + expect(out).toContain("echo one"); + expect(out).toContain("echo two"); + // The boilerplate section after the closed fence STILL drops. + expect(out).not.toContain("## Test plan"); + expect(out).not.toContain("ran the suite"); + expect(out).not.toContain("boilerplate detail line"); + // Real trailing prose after the boilerplate is retained (not swallowed). + expect(out).toContain("## Rationale"); + expect(out).toContain( + "This trailing prose is REAL why/how content that must survive.", + ); + }); + + it("keeps fence parity when an UNTERMINATED fence's blank-line recovery is followed by an INDEPENDENT fenced block, then boilerplate (P3-fix-2)", () => { + // Over-KEEP bucket (a) / parity-inversion re-introduced by p3-fix-1: the + // outside-section blank-line recovery arms the closer-absorb for a fence + // that NEVER really closes. A later INDEPENDENT ```code``` block's OPENER + // then satisfies the same `absorb && !inFence` condition and gets absorbed + // WITHOUT setting `inFence = true`. That block's real CLOSER then toggles + // `inFence` to true while the parser is actually OUTSIDE any fence — + // inverting parity for the rest of the body, so a subsequent boilerplate + // heading (`## Test plan`) takes the `if (inFence)` branch and is silently + // RETAINED. The recovery must not let a NEW block's opener be swallowed by + // the pending absorb. + const body = + "Real rationale prose.\n" + + "```sh\n" + // opens outside any dropped section, NEVER closed (unterminated) + "echo build\n" + + "\n" + // paragraph break → recovery boundary for the open fence + "Prose between the fences that must survive.\n" + + "```js\n" + // OPENER of an INDEPENDENT, well-formed fenced block + "const x = 1;\n" + + "```\n" + // CLOSER of that independent block + "## Test plan\n" + // a REAL boilerplate heading that MUST still drop + "- [x] ran the suite\n" + + "boilerplate detail line\n" + + "\n" + + "## Rationale\n" + + "This trailing prose is REAL why/how content that must survive."; + const out = distillBodyToContent(body); + // The real prose survives, and so does the independent block's content. + expect(out).toContain("Real rationale prose."); + expect(out).toContain("echo build"); + expect(out).toContain("Prose between the fences that must survive."); + expect(out).toContain("const x = 1;"); + // The boilerplate section after the independent block STILL drops. + expect(out).not.toContain("## Test plan"); + expect(out).not.toContain("ran the suite"); + expect(out).not.toContain("boilerplate detail line"); + // Real trailing prose after the boilerplate is retained (not swallowed). + expect(out).toContain("## Rationale"); + expect(out).toContain( + "This trailing prose is REAL why/how content that must survive.", + ); + }); + + it("keeps stripping to EOF after a truly-unterminated fence with an internal blank line and NO later fence marker (EOF protection)", () => { + // EOF-protection invariant: a fence opens outside any dropped section, + // contains an internal blank line, and NEVER closes — no later ``` marker + // at all. The blank-line recovery must resume heading-parsing so a trailing + // boilerplate heading still strips, and parity must NOT latch `inFence` + // true to EOF. + const body = + "Why prose.\n" + + "```sh\n" + // unterminated fence, no closer anywhere + "echo one\n" + + "\n" + // internal blank line → recovery boundary + "echo two\n" + + "## Test plan\n" + // boilerplate heading AFTER the recovery — must drop + "- [x] ran it\n" + + "boilerplate tail"; + const out = distillBodyToContent(body); + expect(out).toContain("Why prose."); + expect(out).toContain("echo one"); + expect(out).toContain("echo two"); + expect(out).not.toContain("## Test plan"); + expect(out).not.toContain("ran it"); + expect(out).not.toContain("boilerplate tail"); + }); + + it("keeps fence parity after a DROPPED section's unterminated fence recovers and a later fence marker follows, then boilerplate + real prose (P3-fix-3)", () => { + // Over-KEEP + content-LOSS bucket (a) / parity-inversion: symmetric sibling + // of P3-fix-2, but the unterminated fence opens INSIDE a dropped boilerplate + // section. The dropped-section blank-line recovery (`inDroppedFence` branch, + // L310) exits the drop but — before p3-fix-3 — failed to arm the sticky + // `recoveredOutsideFence` flag that its OUTSIDE-section sibling (L282) arms. + // So a later fence marker toggled `inFence` TRUE while the parser is + // actually OUTSIDE any fence, inverting parity for the rest of the body: + // a subsequent boilerplate heading (`## Test plan`) took the `if (inFence)` + // branch and was silently OVER-KEPT, and — symmetrically, with a second + // marker — real trailing why/how prose is swallowed to EOF. The + // dropped-section recovery must arm the sticky flag exactly like the + // outside-section recovery, so every post-recovery ``` is kept as literal + // content and NEVER re-toggles parity. + const body = + "Why prose.\n" + + "## Screenshots\n" + // boilerplate heading → drop starts + "```sh\n" + // opens INSIDE the dropped section, unterminated + "echo build\n" + + "\n" + // paragraph break → dropped-section recovery boundary (L310) + "Middle prose keep me.\n" + + "```\n" + // a later fence marker — must NOT re-toggle parity post-recovery + "## Test plan\n" + // a REAL boilerplate heading that MUST still drop + "- [x] ran it\n" + + "boilerplate tail\n" + + "\n" + + "## Rationale\n" + + "Closing prose. This trailing why/how content must survive."; + const out = distillBodyToContent(body); + // Real prose before and between fences survives. + expect(out).toContain("Why prose."); + expect(out).toContain("Middle prose keep me."); + // The boilerplate section after the recovery STILL drops (no over-keep). + expect(out).not.toContain("## Test plan"); + expect(out).not.toContain("ran it"); + expect(out).not.toContain("boilerplate tail"); + // Real trailing prose after the boilerplate is retained (not dropped to EOF). + expect(out).toContain("## Rationale"); + expect(out).toContain( + "Closing prose. This trailing why/how content must survive.", + ); + }); + it("skips the CONTRIBUTING ack drop inside code fences (U2)", () => { const body = "Prose.\n" + "```\n" + "- [x] I have read the CONTRIBUTING doc\n" + "```"; @@ -897,6 +1049,30 @@ describe("distillBodyToContent title-prefix interplay is unaffected; distillTitl }); }); +// Env-reference sanitization (spec §3.3): every adapter runs its emitted +// `content` (and `provenance.source` where set) through `sanitizeEnvRefs` +// before returning from extract(). A machine-local `/Users//…` path in +// the PR body must be rewritten to its repo-relative tail, and a bare session +// UUID must be stripped, so neither leaks into the external corpus. +describe("githubAdapter — env-reference sanitization (§3.3)", () => { + it("sanitizes a /Users/ path + session UUID out of the emitted content", async () => { + const unit = loadFixture("pr.json"); + (unit as GitHubPullRequestUnit).pullRequest.body = + "We patched the bug in " + + "/Users/jpr5/proj/cpk/pathfinder/src/atlas/distillation-gate.ts " + + "during session e654541f-dcb7-4152-8ee8-f669848555ee — see the fix."; + const [fragment] = await githubAdapter.extract(unit, ctx); + + // E1: the machine-local prefix is stripped; the repo-relative tail survives. + expect(fragment.content).toContain("src/atlas/distillation-gate.ts"); + expect(fragment.content).not.toContain("/Users/jpr5"); + // E2: the bare session UUID is gone. + expect(fragment.content).not.toContain( + "e654541f-dcb7-4152-8ee8-f669848555ee", + ); + }); +}); + // Type-level guard: the adapter conforms to the LeafAdapter contract. const _typecheck: CandidateFragment["sourcetype"] = githubAdapter.sourcetype; void _typecheck; diff --git a/src/__tests__/atlas-adapter-memory.test.ts b/src/__tests__/atlas-adapter-memory.test.ts index cbf7fc7..55ee3c2 100644 --- a/src/__tests__/atlas-adapter-memory.test.ts +++ b/src/__tests__/atlas-adapter-memory.test.ts @@ -91,10 +91,16 @@ describe("memory leaf adapter", () => { // source_name carries the memory filename (the unit identity) expect(fragment.source_name).toBe("reference_1password_cli.md"); - // originSessionId → provenance (session is the primary source of the fact) - expect(fragment.provenance.source).toContain( + // provenance.source is `memory:` with NO session suffix (spec + // §3.1-E2 / §3.3): the authoring session UUID is machine-local glue that + // must never enter the external corpus, so it is absent and the source is + // the basename form. + expect(fragment.provenance.source).not.toContain( "e654541f-dcb7-4152-8ee8-f669848555ee", ); + expect(fragment.provenance.source).toBe( + "memory:reference_1password_cli.md", + ); // description → summary lives on provenance (validated_against is the // single free-text provenance slot for the distilled summary) expect(fragment.provenance.validated_against).toBe( diff --git a/src/__tests__/atlas-audience-gate.test.ts b/src/__tests__/atlas-audience-gate.test.ts new file mode 100644 index 0000000..d3cae70 --- /dev/null +++ b/src/__tests__/atlas-audience-gate.test.ts @@ -0,0 +1,402 @@ +// Audience/relevance gate tests (Atlas corpus-scoping, spec §4/§8). +// +// The gate has two deterministic pre-screen paths (no LLM) and one injected-judge +// path. Per the org rule the JUDGE is the ONLY LLM seam — but these tests exercise +// (a) the deterministic pre-screens (which never call the judge) and (b) the +// verdict-routing behavior of `enforceAudienceRelevance` (borderline → needsReview, +// internal-ops → INTERNAL_OPS_MARKER stamp → approvable floor). For (b) we drive +// the judge path with a DETERMINISTIC fake AudienceJudge stub, NOT a real LLM: the +// aimock-backed real-judge coverage lives in atlas-llm.test.ts (§8). The stub also +// lets us assert the pre-screen paths NEVER call the judge (call-count === 0). +// +// The CRITICAL COUPLING under test is S2 → S0/S4: an internal-ops verdict must +// stamp INTERNAL_OPS_MARKER onto the SAME `provenance.validated_against` carrier +// the shared floor predicate reads, so `isApprovableFloored` (the S0 predicate +// validate + canonicalize both consume) returns true. We assert the FLOOR firing, +// not just that a token string appears. + +import { describe, expect, it } from "vitest"; + +import { + enforceAudienceRelevance, + type AudienceJudge, + type AudienceVerdict, +} from "../atlas/audience-gate.js"; +import { INTERNAL_OPS_MARKER, isApprovableFloored } from "../atlas/types.js"; +import type { Candidate, KnowledgeType } from "../atlas/types.js"; + +// A deterministic fake judge: returns a fixed verdict and counts its calls so a +// test can assert the deterministic pre-screen never reached the LLM seam. +function fakeJudge( + verdict: AudienceVerdict, +): AudienceJudge & { calls: number } { + const j = { + calls: 0, + async judge(): Promise { + j.calls += 1; + return verdict; + }, + }; + return j; +} + +// A judge that MUST NOT be called — throws if the gate ever reaches it. Used to +// prove a deterministic pre-screen path bypasses the LLM seam entirely. +const throwingJudge: AudienceJudge = { + async judge(): Promise { + throw new Error( + "judge must not be called on a deterministic pre-screen path", + ); + }, +}; + +function makeCandidate(over: { + title: string; + content: string; + knowledge_type?: KnowledgeType; + subsystem?: string; +}): Candidate { + const subsystem = over.subsystem ?? "runtime"; + return { + sourcetype: "github-pr", + subsystem, + claimSlugHint: "claim", + source_name: "atlas", + repo_url: "https://github.com/CopilotKit/pathfinder", + ref: "main", + title: over.title, + content: over.content, + provenance: { + source: "github-pr", + url: "https://github.com/CopilotKit/pathfinder/pull/1", + date: "2026-06-01", + classification: { + sensitivity: "public", + knowledge_type: over.knowledge_type ?? "architecture", + audience: "all-staff", + validation_status: "unverified", + confidence: "high", + provenance_class: "primary", + freshness: { as_of: "2026-06-01" }, + }, + }, + canonical_key: `github-pr:${subsystem}:claim`, + rankScore: 1, + approvable: true, + evidence: [], + needsReview: false, + validationTargets: [], + }; +} + +describe("enforceAudienceRelevance — deterministic pre-screen (no LLM)", () => { + it("scopes a railway-* subsystem + operational + deploy vocab candidate to internal-ops WITHOUT calling the judge", async () => { + const judge = fakeJudge({ kind: "relevant" }); + const cand = makeCandidate({ + title: "Redeployed the web service", + content: + "We redeployed the web service on Railway after the config change landed.", + knowledge_type: "operational", + subsystem: "railway-web", + }); + const [out] = await enforceAudienceRelevance([cand], { judge }); + expect(judge.calls).toBe(0); + // Internal-ops → INTERNAL_OPS_MARKER stamped → S0 floor predicate fires. + expect(isApprovableFloored(out)).toBe(true); + expect(out.provenance.validated_against).toContain(INTERNAL_OPS_MARKER); + }); + + it("passes an architecture candidate carrying a product-portable specific as relevant WITHOUT calling the judge", async () => { + const cand = makeCandidate({ + title: "Admin reindex requires an authenticated POST", + content: + "The reindex job is triggered via POST /admin/reindex; an unauthenticated call returns 401.", + knowledge_type: "architecture", + subsystem: "runtime", + }); + // throwingJudge proves the pre-screen bypassed the LLM seam entirely. + const [out] = await enforceAudienceRelevance([cand], { + judge: throwingJudge, + }); + expect(isApprovableFloored(out)).toBe(false); + expect(out.needsReview).toBe(false); + expect(out.provenance.validated_against ?? "").not.toContain( + INTERNAL_OPS_MARKER, + ); + }); + + // ── Finding 1: PRODUCT_PORTABLE_SPECIFIC_RE must match ONLY genuine + // product-portable specifics, not ordinary prose. ────────────────────── + it("does NOT clear-relevant an internal root-cause candidate whose only 'specifics' are an incidental 4xx/5xx count and a prose parenthetical — it reaches the judge", async () => { + // knowledge_type is clear-relevant-eligible (root-cause) so ONLY the + // over-broad regex could force the bypass. The body has an incidental + // "503 candidates" integer and an ordinary "orchestrator (see …)" + // parenthetical — neither is a product-portable specific. This candidate + // must reach the judge (which rules it internal-ops), NOT be force-passed. + const judge = fakeJudge({ + kind: "internal-ops", + reason: "internal root-cause narrative, no external-builder utility", + }); + const cand = makeCandidate({ + title: "Root cause of the July starvation incident", + content: + "The orchestrator (see the incident timeline) starved because 503 candidates " + + "piled up behind a single replica; we promoted more capacity to drain them.", + knowledge_type: "root-cause", + subsystem: "harvest", + }); + const [out] = await enforceAudienceRelevance([cand], { judge }); + expect(judge.calls).toBe(1); // reached the judge, was NOT clear-relevant + expect(isApprovableFloored(out)).toBe(true); + expect(out.provenance.validated_against).toContain(INTERNAL_OPS_MARKER); + }); + + it("does NOT clear-relevant an internal-ops candidate whose bare 4xx/5xx code sits next to incidental prose words ('code'/'error'/'status'), not real HTTP context — it reaches the judge", async () => { + // Finding 4 (floor escape): STATUS_VERB included bare "code"/"error"/ + // "status", so ordinary internal-ops prose ("during code review we saw 503 + // candidates") matched HTTP_STATUS_IN_CONTEXT_RE and took the clear-relevant + // bypass — escaping the fail-restrictive judge. A bare 4xx/5xx must only + // qualify as an HTTP status in GENUINE HTTP context (returns/responds/HTTP/ + // status code), never because an incidental word like "code" happens to sit + // within a few characters of the number. + const judge = fakeJudge({ + kind: "internal-ops", + reason: "internal code-review narrative, no external-builder utility", + }); + const cand = makeCandidate({ + title: "Code-review triage note", + content: + "During code review we saw 503 candidates queued behind the reindex; " + + "the error status was noted and we drained them by hand.", + knowledge_type: "root-cause", + subsystem: "harvest", + }); + const [out] = await enforceAudienceRelevance([cand], { judge }); + expect(judge.calls).toBe(1); // reached the judge, was NOT clear-relevant + expect(isApprovableFloored(out)).toBe(true); + expect(out.provenance.validated_against).toContain(INTERNAL_OPS_MARKER); + }); + + it("still clear-relevants a genuine HTTP status in real context ('returns 401' / '401 error code') WITHOUT calling the judge", async () => { + // Finding 4 guardrail: tightening STATUS_VERB must PRESERVE real HTTP-status + // true-positives. A status verb ("returns 401") and the "error code"/"status + // code" noun phrase (in either order) are genuine HTTP context and must + // still take the clear-relevant bypass on a clear-relevant knowledge_type. + for (const content of [ + "The admin endpoint returns 401 when the token is missing.", + "A stale token yields a 401 error code the client must refetch on.", + ]) { + const cand = makeCandidate({ + title: "HTTP status contract", + content, + knowledge_type: "protocol", + subsystem: "runtime", + }); + const [out] = await enforceAudienceRelevance([cand], { + judge: throwingJudge, + }); + expect(isApprovableFloored(out)).toBe(false); + expect(out.needsReview).toBe(false); + } + }); + + it("still clear-relevants a genuine API-call product-portable specific (useCoAgent(...) / runHarvest(...)) WITHOUT calling the judge", async () => { + for (const content of [ + "Register the agent with `useCoAgent({ name })` on the client.", + "The harvest driver entrypoint is runHarvest(opts) — call it per run.", + ]) { + const cand = makeCandidate({ + title: "API usage", + content, + knowledge_type: "protocol", + subsystem: "runtime", + }); + const [out] = await enforceAudienceRelevance([cand], { + judge: throwingJudge, + }); + expect(isApprovableFloored(out)).toBe(false); + expect(out.needsReview).toBe(false); + } + }); +}); + +describe("enforceAudienceRelevance — injected judge path", () => { + it("sets needsReview for a borderline judge verdict", async () => { + // A candidate that clears NEITHER pre-screen (process knowledge_type, no + // deploy vocab, no infra subsystem, not a clear-relevant knowledge_type) so + // it falls through to the injected judge. + const judge = fakeJudge({ + kind: "borderline", + reason: "advanced internals", + }); + const cand = makeCandidate({ + title: "How the harvest run bookkeeping works", + content: + "The harvest driver tracks per-run bookkeeping for the pipeline.", + knowledge_type: "process", + subsystem: "harvest", + }); + const [out] = await enforceAudienceRelevance([cand], { judge }); + expect(judge.calls).toBe(1); + expect(out.needsReview).toBe(true); + // Borderline does NOT floor approvable. + expect(isApprovableFloored(out)).toBe(false); + }); + + it("stamps INTERNAL_OPS_MARKER and floors approvable for an internal-ops judge verdict", async () => { + const judge = fakeJudge({ + kind: "internal-ops", + reason: "pure deploy log", + }); + const cand = makeCandidate({ + title: "PR closeout bookkeeping", + content: "PR #142 shipped X and the closeout note was filed.", + knowledge_type: "process", + subsystem: "harvest", + }); + const [out] = await enforceAudienceRelevance([cand], { judge }); + expect(judge.calls).toBe(1); + expect(out.provenance.validated_against).toContain(INTERNAL_OPS_MARKER); + // The marker must be READ by the shared S0 floor predicate, not just written. + expect(isApprovableFloored(out)).toBe(true); + }); + + // ── Finding 2: verdict-flip hygiene — a candidate a PRIOR run scoped to + // internal-ops carries INTERNAL_OPS_MARKER; a re-run that flips it to + // relevant/borderline must STRIP the stale marker so it is not + // stale-floored forever (mirrors distillation-gate's stripRestatementMarker). + it("strips a stale INTERNAL_OPS_MARKER when a re-run judges the candidate relevant", async () => { + const judge = fakeJudge({ kind: "relevant" }); + const cand = makeCandidate({ + title: "Now-relevant claim", + content: "Some prose the pre-screen cannot classify, judged relevant.", + knowledge_type: "process", + subsystem: "harvest", + }); + // Simulate the prior run's internal-ops stamp already on the carrier. + cand.provenance.validated_against = INTERNAL_OPS_MARKER; + const [out] = await enforceAudienceRelevance([cand], { judge }); + expect(judge.calls).toBe(1); + expect(isApprovableFloored(out)).toBe(false); + expect(out.provenance.validated_against ?? "").not.toContain( + INTERNAL_OPS_MARKER, + ); + }); + + it("strips a stale INTERNAL_OPS_MARKER when a re-run judges the candidate borderline (preserving co-resident tokens)", async () => { + const judge = fakeJudge({ kind: "borderline", reason: "some utility" }); + const cand = makeCandidate({ + title: "Now-borderline claim", + content: "Some prose the pre-screen cannot classify, judged borderline.", + knowledge_type: "process", + subsystem: "harvest", + }); + // A co-resident, unrelated token must survive; only the stale floor goes. + cand.provenance.validated_against = `${INTERNAL_OPS_MARKER}; some-other-token`; + const [out] = await enforceAudienceRelevance([cand], { judge }); + expect(judge.calls).toBe(1); + expect(out.needsReview).toBe(true); + expect(isApprovableFloored(out)).toBe(false); + expect(out.provenance.validated_against ?? "").not.toContain( + INTERNAL_OPS_MARKER, + ); + expect(out.provenance.validated_against).toContain("some-other-token"); + }); + + // ── Finding (conf 80): the stale-marker strip must be DUAL-CARRIER. The S0 + // floor predicate (hasFloorMarker) reads INTERNAL_OPS_MARKER from EITHER + // `provenance.validated_against` OR a `fused_from` evidence ref. A prior run + // can stamp the marker on the fused_from carrier (an idiom the S0 marker + // supports); a re-run that flips the candidate to relevant/borderline must + // un-floor it on BOTH carriers, or the "applied UNIFORMLY … no verdict-flip + // path can leave a stale floor" invariant is not delivered. + it("strips a stale INTERNAL_OPS_MARKER carried as a fused_from evidence ref when a re-run judges the candidate relevant", async () => { + const judge = fakeJudge({ kind: "relevant" }); + const cand = makeCandidate({ + title: "Now-relevant claim (fused_from carrier)", + content: "Some prose the pre-screen cannot classify, judged relevant.", + knowledge_type: "process", + subsystem: "harvest", + }); + // Prior run stamped the floor marker on the fused_from evidence carrier. + cand.evidence = [{ kind: "fused_from", ref: INTERNAL_OPS_MARKER }]; + const [out] = await enforceAudienceRelevance([cand], { judge }); + expect(judge.calls).toBe(1); + // Must un-floor on BOTH carriers. + expect(isApprovableFloored(out)).toBe(false); + expect( + out.evidence.some( + (e) => e.kind === "fused_from" && e.ref === INTERNAL_OPS_MARKER, + ), + ).toBe(false); + }); + + it("strips a stale INTERNAL_OPS_MARKER fused_from ref when a re-run judges the candidate borderline (preserving co-resident evidence)", async () => { + const judge = fakeJudge({ kind: "borderline", reason: "some utility" }); + const cand = makeCandidate({ + title: "Now-borderline claim (fused_from carrier)", + content: "Some prose the pre-screen cannot classify, judged borderline.", + knowledge_type: "process", + subsystem: "harvest", + }); + // A co-resident, unrelated fused_from ref must survive; only the floor goes. + cand.evidence = [ + { kind: "fused_from", ref: INTERNAL_OPS_MARKER }, + { kind: "fused_from", ref: "some-other-ref" }, + { kind: "changed_file", path: "src/atlas/audience-gate.ts" }, + ]; + const [out] = await enforceAudienceRelevance([cand], { judge }); + expect(judge.calls).toBe(1); + expect(out.needsReview).toBe(true); + expect(isApprovableFloored(out)).toBe(false); + expect( + out.evidence.some( + (e) => e.kind === "fused_from" && e.ref === INTERNAL_OPS_MARKER, + ), + ).toBe(false); + // Co-resident evidence survives (whole-ref match, no over-strip). + expect( + out.evidence.some( + (e) => e.kind === "fused_from" && e.ref === "some-other-ref", + ), + ).toBe(true); + expect( + out.evidence.some( + (e) => + e.kind === "changed_file" && e.path === "src/atlas/audience-gate.ts", + ), + ).toBe(true); + }); +}); + +describe("enforceAudienceRelevance — NEVER-DROP contract", () => { + it("returns a same-length, same-order output and never mutates the input", async () => { + const judge = fakeJudge({ kind: "relevant" }); + const inputs = [ + makeCandidate({ + title: "Redeployed web", + content: "We redeployed the web service on Railway.", + knowledge_type: "operational", + subsystem: "railway-web", + }), + makeCandidate({ + title: "POST /admin/reindex returns 401 unauthenticated", + content: "POST /admin/reindex returns 401 when unauthenticated.", + knowledge_type: "architecture", + subsystem: "runtime", + }), + makeCandidate({ + title: "Fall-through to the judge", + content: "Some prose the pre-screen cannot classify.", + knowledge_type: "process", + subsystem: "harvest", + }), + ]; + const before = inputs.map((c) => c.canonical_key); + const out = await enforceAudienceRelevance(inputs, { judge }); + expect(out).toHaveLength(inputs.length); + expect(out.map((c) => c.canonical_key)).toEqual(before); + // Input candidates are not mutated in place (fresh objects returned). + expect(inputs[0]?.provenance.validated_against).toBeUndefined(); + }); +}); diff --git a/src/__tests__/atlas-canonicalize.test.ts b/src/__tests__/atlas-canonicalize.test.ts index b41e142..9d94a3d 100644 --- a/src/__tests__/atlas-canonicalize.test.ts +++ b/src/__tests__/atlas-canonicalize.test.ts @@ -10,6 +10,8 @@ import { CandidateSchema, parseCanonicalKey, RAG_NO_DELTA_MARKER, + RESTATEMENT_MARKER, + INTERNAL_OPS_MARKER, } from "../atlas/types.js"; import type { CandidateFragment, @@ -469,14 +471,23 @@ describe("canonicalize — rank ordering", () => { // duplicate's rankScore so it OUT-RANKS its un-duplicated twin (the §6.2 rank // inversion). It must be excluded from the evidence-depth count exactly like // the rag-corpus-overlap: prefix already is. + // + // This test isolates the evidence-DEPTH exclusion, so both fragments are + // `unverified` — validation weight 1 for both, whether or not the shared + // rank floor fires — leaving evidence depth as the ONLY variable. (The + // RAG_NO_DELTA_MARKER now ALSO floors the validation weight via the shared + // isApprovableFloored predicate; that floor is covered by the dedicated + // RESTATEMENT/INTERNAL_OPS floor tests above.) const floored = makeFragment({ subsystem: "s", claimSlugHint: "floored", + validation_status: "unverified", evidence: [{ kind: "fused_from", ref: RAG_NO_DELTA_MARKER }], }); const bare = makeFragment({ subsystem: "s", claimSlugHint: "bare-nd", + validation_status: "unverified", evidence: [], }); const out = canonicalize([floored, bare]); @@ -486,6 +497,149 @@ describe("canonicalize — rank ordering", () => { expect(row("floored").rankScore).toBe(row("bare-nd").rankScore); }); + it("an INTERNAL_OPS floor marker is rank-NEUTRAL for evidence depth (does not out-rank a genuine twin)", () => { + // §6.2: an INTERNAL_OPS_MARKER floor is carried on the SAME `fused_from` ref + // idiom as RESTATEMENT/RAG_NO_DELTA. It is a provenance floor trace, NOT + // corroboration for the claim — counting it toward evidence depth would + // inflate the floored candidate's rankScore so it OUT-RANKS a genuine twin + // with no extra corroboration (inverting §6.2). This isolates the + // evidence-DEPTH exclusion: both fragments are `unverified` (validation + // weight 1 for both, so the validation-weight floor is not the variable), + // leaving the floor `fused_from` ref as the ONLY difference. The genuine + // twin has NO evidence, so if the floor ref counted, the floored candidate + // would win. + const floored = makeFragment({ + subsystem: "s", + claimSlugHint: "io-depth-floored", + validation_status: "unverified", + evidence: [{ kind: "fused_from", ref: INTERNAL_OPS_MARKER }], + }); + const genuineTwin = makeFragment({ + subsystem: "s", + claimSlugHint: "io-depth-twin", + validation_status: "unverified", + evidence: [], + }); + const out = canonicalize([floored, genuineTwin]); + const row = (slug: string) => + out.find((c) => c.canonical_key.endsWith(`:${slug}`))!; + // The floor marker must NOT count as evidence: the floored candidate must + // NOT out-rank its genuine twin — they tie. + expect(row("io-depth-floored").rankScore).toBe( + row("io-depth-twin").rankScore, + ); + expect(row("io-depth-floored").rankScore).not.toBeGreaterThan( + row("io-depth-twin").rankScore, + ); + }); + + it("a RESTATEMENT floor marker is rank-NEUTRAL for evidence depth (does not out-rank a genuine twin)", () => { + // Same evidence-depth exclusion for the RESTATEMENT_MARKER floor: it rides + // the identical `fused_from` ref carrier, so it must ALSO be excluded from + // the corroboration count. Both twins `unverified` to isolate depth. + const floored = makeFragment({ + subsystem: "s", + claimSlugHint: "rs-depth-floored", + validation_status: "unverified", + evidence: [{ kind: "fused_from", ref: RESTATEMENT_MARKER }], + }); + const genuineTwin = makeFragment({ + subsystem: "s", + claimSlugHint: "rs-depth-twin", + validation_status: "unverified", + evidence: [], + }); + const out = canonicalize([floored, genuineTwin]); + const row = (slug: string) => + out.find((c) => c.canonical_key.endsWith(`:${slug}`))!; + expect(row("rs-depth-floored").rankScore).toBe( + row("rs-depth-twin").rankScore, + ); + expect(row("rs-depth-floored").rankScore).not.toBeGreaterThan( + row("rs-depth-twin").rankScore, + ); + }); + + it("floors the rank validation-weight for an INTERNAL_OPS-marked candidate (§6.2 shared floor predicate)", () => { + // §6.2 rank-score decision: internal-ops MUST also floor rank, via the S0 + // shared `isApprovableFloored` predicate — not just RESTATEMENT. An + // internal-ops candidate is approvable=false (audience floor), and its rank + // must not be lifted above a genuine claim by the dominant validation weight. + // The floor is carried on either idiom; use the fused_from ref here. + // + // A `showcase-verified` internal-ops candidate carries the DOMINANT + // validation weight (3). Its twin is IDENTICAL — same INTERNAL_OPS_MARKER + // (so evidence-depth matches) — but already `unverified` (weight 1), so the + // comparison isolates the validation-weight floor. Before the generalization, + // computeRankScore reads hasRestatementMarker — which ignores INTERNAL_OPS — + // so the showcase-verified row keeps its full weight and strictly OUT-RANKS + // the unverified twin (RED). After: floored to `unverified`, they tie (GREEN). + const internalOps = makeFragment({ + subsystem: "s", + claimSlugHint: "internal-ops", + validation_status: "showcase-verified", + evidence: [{ kind: "fused_from", ref: INTERNAL_OPS_MARKER }], + }); + const twin = makeFragment({ + subsystem: "s", + claimSlugHint: "bare-io", + validation_status: "unverified", + evidence: [{ kind: "fused_from", ref: INTERNAL_OPS_MARKER }], + }); + const out = canonicalize([internalOps, twin]); + const row = (slug: string) => + out.find((c) => c.canonical_key.endsWith(`:${slug}`))!; + expect(row("internal-ops").rankScore).toBe(row("bare-io").rankScore); + }); + + it("still floors the rank validation-weight for a RESTATEMENT-marked candidate (no regression)", () => { + // The two pre-existing floor markers must keep flooring rank exactly as + // before the generalization. A `showcase-verified` restatement (weight 3) + // ties its `unverified` twin (weight 1) once floored — the twin carries the + // SAME RESTATEMENT_MARKER so evidence-depth matches and only validation + // weight differs. + const restatement = makeFragment({ + subsystem: "s", + claimSlugHint: "restatement", + validation_status: "showcase-verified", + evidence: [{ kind: "fused_from", ref: RESTATEMENT_MARKER }], + }); + const twin = makeFragment({ + subsystem: "s", + claimSlugHint: "bare-rs", + validation_status: "unverified", + evidence: [{ kind: "fused_from", ref: RESTATEMENT_MARKER }], + }); + const out = canonicalize([restatement, twin]); + const row = (slug: string) => + out.find((c) => c.canonical_key.endsWith(`:${slug}`))!; + expect(row("restatement").rankScore).toBe(row("bare-rs").rankScore); + }); + + it("does NOT floor the rank validation-weight for a clean (unmarked) candidate", () => { + // No floor marker → the full promoted validation weight applies. A clean + // `showcase-verified` candidate (weight 3) strictly OUT-RANKS a bare + // `unverified` twin (weight 1). This guards against the floor over-firing. + const clean = makeFragment({ + subsystem: "s", + claimSlugHint: "clean-sv", + validation_status: "showcase-verified", + evidence: [], + }); + const bare = makeFragment({ + subsystem: "s", + claimSlugHint: "bare-clean", + validation_status: "unverified", + evidence: [], + }); + const out = canonicalize([clean, bare]); + const row = (slug: string) => + out.find((c) => c.canonical_key.endsWith(`:${slug}`))!; + expect(row("clean-sv").rankScore).toBeGreaterThan( + row("bare-clean").rankScore, + ); + }); + it("ranks a more recent fact higher, all else equal", () => { const recent = makeFragment({ subsystem: "s", diff --git a/src/__tests__/atlas-exclude.test.ts b/src/__tests__/atlas-exclude.test.ts index 500ad76..a507d6d 100644 --- a/src/__tests__/atlas-exclude.test.ts +++ b/src/__tests__/atlas-exclude.test.ts @@ -236,10 +236,14 @@ describe("applyExclusions — flag rules (pure, no LLM)", () => { (r): r is Extract => r.kind === "english", ); - expect(englishRules.length).toBeGreaterThanOrEqual(2); + expect(englishRules.length).toBeGreaterThanOrEqual(3); const joined = englishRules.map((r) => r.text.toLowerCase()).join(" | "); expect(joined).toMatch(/credential|secret|api key|token|password/); expect(joined).toMatch(/customer|client|account/); + // Internal-ops trivia: Railway/CI deploy logs, PR-closeout records, + // internal-infra topology with zero external-builder value. + expect(joined).toMatch(/internal operational trivia/); + expect(joined).toMatch(/pr-closeout/); }); }); diff --git a/src/__tests__/atlas-harvest-cli.test.ts b/src/__tests__/atlas-harvest-cli.test.ts index 133123e..2491770 100644 --- a/src/__tests__/atlas-harvest-cli.test.ts +++ b/src/__tests__/atlas-harvest-cli.test.ts @@ -61,6 +61,7 @@ import type { CorpusHit, ValidationStatus, } from "../atlas/types.js"; +import { INTERNAL_OPS_MARKER } from "../atlas/types.js"; import type { FeatureRegistry } from "../atlas/adapters/showcase.js"; import type { ValidationContext } from "../atlas/validate.js"; import type { ExclusionRule } from "../atlas/exclude.js"; @@ -73,14 +74,18 @@ import { buildLeafAdapterRegistry, runHarvest, buildArtifactCandidates, + processCandidatePipeline, parseMinOverlap, resolveBaseUrl, resolveToken, formatCliError, runAtlasHarvestCli, + resolveSharedPipelineSeams, + __internalsForTest, type RunHarvestDeps, } from "../atlas/harvest-cli.js"; import type { DistillationJudge } from "../atlas/distillation-gate.js"; +import type { AudienceJudge } from "../atlas/audience-gate.js"; // The sync MODULE is mocked file-wide: the sync-summary CLI test below asserts // only the driver's output plumbing — sync's own enactment semantics live in @@ -208,6 +213,17 @@ const passThroughJudge: DistillationJudge = { judge: async () => ({ kind: "distilled" }), }; +// A pass-through audience judge: rules every candidate `relevant` (a no-op). +// Injected alongside `passThroughJudge` into the tests that exercise +// NON-audience-gate concerns (upsert/manifest/dedup-ordering/parity/re-rank) so +// `runHarvest`/`buildArtifactCandidates` do not construct a real OpenAIDistiller +// for the audience-gate default (which would need an API key). The audience +// gate's OWN wiring is covered by the dedicated describe block at the end of +// this file, and its transform behavior in atlas-audience-gate.test.ts. +const passThroughAudienceJudge: AudienceJudge = { + judge: async () => ({ kind: "relevant" }), +}; + // No-op semantic-dedup seams (Theme B). Injected alongside `passThroughJudge` // into the tests that exercise NON-dedup concerns (upsert/manifest/ordering) so // `runHarvest` does not construct a real OpenAIDistiller for the embed/distill @@ -306,6 +322,7 @@ describe("atlas-harvest driver — run pipeline (real PGlite)", () => { upsert: true, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -350,6 +367,7 @@ describe("atlas-harvest driver — run pipeline (real PGlite)", () => { upsert: true, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -384,6 +402,7 @@ describe("atlas-harvest driver — run pipeline (real PGlite)", () => { runsDir, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -409,6 +428,7 @@ describe("atlas-harvest driver — run pipeline (real PGlite)", () => { dryRun: true, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -431,6 +451,7 @@ describe("atlas-harvest driver — run pipeline (real PGlite)", () => { runsDir, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -469,6 +490,7 @@ describe("atlas-harvest driver — run pipeline (real PGlite)", () => { runsDir, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), deps, @@ -501,6 +523,7 @@ describe("atlas-harvest driver — run pipeline (real PGlite)", () => { upsert: true, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -579,6 +602,7 @@ describe("atlas-harvest driver — artifact/run pipeline parity (FIX 1)", () => runsDir, validationContext: ctx, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ragClient: parityClient, ...passThroughSemanticDedup, }); @@ -662,6 +686,7 @@ describe("atlas-harvest driver — artifact/run pipeline parity (FIX 1)", () => runsDir, validationContext: ctx, judge: restatementJudge, + audienceJudge: passThroughAudienceJudge, ragClient: restatementClient, ...passThroughSemanticDedup, }); @@ -722,6 +747,7 @@ describe("atlas-harvest driver — artifact/run pipeline parity (FIX 1)", () => runsDir, validationContext: ctx, judge: rewriteJudge, + audienceJudge: passThroughAudienceJudge, ragClient: rewriteClient, ...passThroughSemanticDedup, }); @@ -940,6 +966,7 @@ describe("atlas-harvest driver — artifact/upsert SEMANTIC rag-dedup parity (st runsDir, validationContext: ctx, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...seams, vectorSearch: vectorSearchAbove(), }); @@ -953,6 +980,7 @@ describe("atlas-harvest driver — artifact/upsert SEMANTIC rag-dedup parity (st upsert: true, validationContext: ctx, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...seams, vectorSearch: vectorSearchAbove(), }); @@ -1024,6 +1052,7 @@ describe("atlas-harvest driver — artifact/upsert SEMANTIC rag-dedup parity (st runsDir, validationContext: ctx, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...seams, vectorSearch: vectorSearchAbove(), }); @@ -1036,6 +1065,7 @@ describe("atlas-harvest driver — artifact/upsert SEMANTIC rag-dedup parity (st upsert: true, validationContext: ctx, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...seams, vectorSearch: vectorSearchAbove(), }); @@ -1245,6 +1275,7 @@ describe("atlas-harvest driver — run manifest (V80)", () => { runsDir, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -1270,6 +1301,7 @@ describe("atlas-harvest driver — run manifest (V80)", () => { runsDir, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -1296,6 +1328,7 @@ describe("atlas-harvest driver — run manifest (V80)", () => { runsDir, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -1320,6 +1353,7 @@ describe("atlas-harvest driver — run manifest (V80)", () => { dryRun: true, ragClient: client, judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, ...passThroughSemanticDedup, validationContext: emptyValidationContext(checkoutDir), }); @@ -1387,6 +1421,7 @@ describe("atlas-harvest driver — post-validate re-rank (V57)", () => { // Pass-through distillation judge so this re-rank test does not construct a // real OpenAIDistiller (the gate's behavior is covered by the parity tests). judge: passThroughJudge, + audienceJudge: passThroughAudienceJudge, // No-op rag-dedup seams keep this re-rank test isolated to the rank stage. ragClient: rerankClient, ...passThroughSemanticDedup, @@ -1628,6 +1663,21 @@ describe("atlas-harvest driver — artifact without --prior-run-id warns (X12)", match: { systemMessage: "WHY-vs-WHAT judge" }, response: { content: JSON.stringify({ verdict: "distilled" }) }, }); + // The artifact path now ALSO runs the audience-relevance gate (§4.5), which — + // with no judge injection point on the CLI — constructs a real + // OpenAIDistiller-backed audience judge. The fragment is a product-portable + // why/how claim, so respond with the `relevant` verdict for any user message + // under the audience-judge system prompt. Keeps the LLM seam honest (org rule: + // aimock, never a vi.fn stub) without a real key. + mock.addFixture({ + match: { systemMessage: "audience-relevance judge" }, + response: { + content: JSON.stringify({ + verdict: "relevant", + reason: "product-portable why/how claim", + }), + }, + }); await mock.start(); process.env.OPENAI_BASE_URL = `${mock.url}/v1`; process.env.OPENAI_API_KEY = "mock"; @@ -1719,3 +1769,202 @@ describe("atlas-harvest driver — artifact without --prior-run-id warns (X12)", ); }); }); + +// ── S7: audience-relevance gate wiring into the shared pipeline (§4.5) ─────────── +// +// The gate is wired into processCandidatePipeline AFTER enforceDistillation and +// BEFORE dedupAgainstRagCorpus. These tests prove the wiring END-TO-END with a +// FAKE AudienceJudge stub (never a real LLM): a candidate the stub rules +// `internal-ops` flows through the SHARED pipeline and comes out approvable=false +// (the gate stamps INTERNAL_OPS_MARKER → validate reads it as an approvability +// floor). RED before the wiring existed (no audience gate → the candidate stays +// approvable=true); GREEN after. The second test pins the pipeline ORDER: +// enforceAudienceRelevance runs BETWEEN the distillation gate and rag-dedup. +describe("atlas-harvest driver — audience-relevance gate wiring (S7 §4.5)", () => { + let runsDir: string; + let checkoutDir: string; + + beforeAll(() => { + runsDir = fs.mkdtempSync(path.join(os.tmpdir(), "atlas-harvest-aud-")); + checkoutDir = fs.mkdtempSync( + path.join(os.tmpdir(), "atlas-harvest-audco-"), + ); + }); + + afterAll(() => { + fs.rmSync(runsDir, { recursive: true, force: true }); + fs.rmSync(checkoutDir, { recursive: true, force: true }); + }); + + // An architecture fact WITH a resolvable validationTarget: without the audience + // gate it source-verifies and stays approvable=true; the internal-ops floor is + // the ONLY thing that drives it to approvable=false, so it isolates the gate's + // wiring. Content/subsystem/knowledge_type are chosen so the gate's + // DETERMINISTIC pre-screen does NOT fire (subsystem "runtime" is not an infra + // slug; content carries no deploy/PR-closeout vocab; knowledge_type + // "architecture" is a clear-relevant type but the content lacks a + // product-portable-specific token, so it falls through to the injected judge) — + // this way the FAKE judge stub actually decides the verdict. + function audienceFragment(): CandidateFragment { + const symbolFile = path.join(checkoutDir, "src", "runtime", "stream.ts"); + fs.mkdirSync(path.dirname(symbolFile), { recursive: true }); + fs.writeFileSync(symbolFile, "export const drainToolQueue = () => {};\n"); + return fragment({ + claimSlugHint: "audience-internal-ops", + title: "An operational note with no external-builder value", + content: + "The team keeps an internal note about coordinating the weekly review " + + "cadence across sessions.", + provenance: { + source: "github-pr", + url: "https://github.com/CopilotKit/pathfinder/pull/77", + date: "2026-06-01", + classification: { + sensitivity: "public", + knowledge_type: "architecture", + audience: "all-staff", + validation_status: "unverified", + confidence: "high", + provenance_class: "primary", + freshness: { as_of: "2026-06-01" }, + }, + }, + validationTargets: ["drainToolQueue"], + }); + } + + it("a candidate the audience judge rules internal-ops flows through the pipeline and comes out approvable=false", async () => { + const runId = "run-audience-internal-ops"; + seedRunDir(runsDir, runId, [audienceFragment()]); + + // FAKE audience judge: rules the candidate internal-ops (NOT a real LLM). + const internalOpsJudge: AudienceJudge = { + judge: async () => ({ + kind: "internal-ops", + reason: "purely internal operational trivia", + }), + }; + + const { client } = makeSearchClient(); + const store = new RunStore(runsDir); + const cands = await processCandidatePipeline({ + runId, + store, + validationContext: emptyValidationContext(checkoutDir), + judge: passThroughJudge, + audienceJudge: internalOpsJudge, + ragClient: client, + ...passThroughSemanticDedup, + }); + + expect(cands).toHaveLength(1); + const cand = cands[0]!; + // The gate stamped the floor marker onto the shared carrier... + expect(cand.provenance.validated_against ?? "").toContain( + INTERNAL_OPS_MARKER, + ); + // ...and validate READ it as an approvability floor, even though the + // candidate source-verifies (the drainToolQueue symbol resolves) — WITHOUT + // the audience gate this same candidate comes out approvable=true. + expect(cand.approvable).toBe(false); + }); + + it("runs enforceAudienceRelevance AFTER the distillation gate and BEFORE rag-dedup (pipeline order)", async () => { + const runId = "run-audience-order"; + seedRunDir(runsDir, runId, [audienceFragment()]); + + const order: string[] = []; + + // Order-recording distillation judge (passes through as distilled). + const orderingDistillJudge: DistillationJudge = { + judge: async () => { + order.push("distill"); + return { kind: "distilled" }; + }, + }; + // Order-recording audience judge (relevant → no floor, keeps the order test + // focused purely on WHEN the gate runs). + const orderingAudienceJudge: AudienceJudge = { + judge: async () => { + order.push("audience"); + return { kind: "relevant" }; + }, + }; + // Order-recording dedup stub (passes candidates through unchanged). + const orderingDedup = async (c: Candidate[]): Promise => { + order.push("dedup"); + return c; + }; + + const { client } = makeSearchClient(); + const store = new RunStore(runsDir); + await processCandidatePipeline({ + runId, + store, + validationContext: emptyValidationContext(checkoutDir), + judge: orderingDistillJudge, + audienceJudge: orderingAudienceJudge, + ragClient: client, + ...passThroughSemanticDedup, + dedup: orderingDedup, + }); + + // distillation gate → audience gate → rag-dedup. + expect(order).toEqual(["distill", "audience", "dedup"]); + expect(order.indexOf("audience")).toBeGreaterThan(order.indexOf("distill")); + expect(order.indexOf("audience")).toBeLessThan(order.indexOf("dedup")); + }); +}); + +// ── Shared-pipeline seam defaults route through the ONE buildLlm() factory ─────── +// +// resolveSharedPipelineSeams defaults every LLM-backed seam (judge, audienceJudge, +// embed, distillDelta) to the SAME canonical `buildLlm()` factory. The audience +// judge MUST NOT construct its own OpenAIDistiller directly — doing so silently +// bypasses whatever config buildLlm applies (or later gains: apiKey sentinel, +// model selection, baseURL). This test pins the default audienceJudge to the +// buildLlm seam by swapping `__internalsForTest.buildLlm` for a spy and asserting +// the default judge routes through it (never a real LLM — the fake is returned in +// place of a distiller). RED before the fix: the default audienceJudge built its +// own `new OpenAIDistiller()`, so the spy is never reached (and, with no +// OPENAI_API_KEY, the real distiller throws at construction). GREEN after: the +// default audienceJudge calls buildLlm().judge (the AudienceJudge seam method, +// which OpenAIDistiller delegates to judgeAudience) like its sibling seams. +describe("atlas-harvest driver — shared-pipeline seam defaults route through buildLlm", () => { + const realBuildLlm = __internalsForTest.buildLlm; + + afterEach(() => { + __internalsForTest.buildLlm = realBuildLlm; + vi.restoreAllMocks(); + }); + + it("defaults the audienceJudge seam through buildLlm() (not a direct new OpenAIDistiller)", async () => { + const judge = vi.fn(async () => ({ kind: "relevant" as const })); + // Fake LLM the buildLlm seam returns — exposes just the methods the seam + // defaults invoke; if the audienceJudge default bypasses buildLlm, this + // judge is never called (the AudienceJudge seam method OpenAIDistiller.judge + // delegates to judgeAudience). + const fakeLlm = { + judge, + judgeAudience: vi.fn(), + judgeDistillation: vi.fn(), + embed: vi.fn(), + distillDelta: vi.fn(), + }; + const buildLlmSpy = vi.fn(() => fakeLlm); + __internalsForTest.buildLlm = buildLlmSpy as unknown as typeof realBuildLlm; + + const seams = resolveSharedPipelineSeams({}); + await seams.audienceJudge.judge({ + title: "t", + content: "c", + knowledge_type: "architecture", + subsystem: "runtime", + }); + + // The default audienceJudge routed through the buildLlm seam... + expect(buildLlmSpy).toHaveBeenCalled(); + // ...and invoked the judge (AudienceJudge) method on what buildLlm returned. + expect(judge).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/__tests__/atlas-llm.test.ts b/src/__tests__/atlas-llm.test.ts index 19fd584..1c6ee7f 100644 --- a/src/__tests__/atlas-llm.test.ts +++ b/src/__tests__/atlas-llm.test.ts @@ -33,6 +33,8 @@ import { CandidateFragmentSchema } from "../atlas/types.js"; // gate each fixture to exactly one operation. const EPISODIC_SYSTEM_MARKER = "knowledge-distillation engine"; const EXCLUSION_SYSTEM_MARKER = "exclusion-rule judge"; +const AUDIENCE_SYSTEM_MARKER = "audience-relevance judge"; +const DISTILLATION_SYSTEM_MARKER = "WHY-vs-WHAT judge"; // The distilled-fragment JSON the "model" returns for the episodic call. const EPISODIC_MODEL_OUTPUT = { @@ -93,6 +95,37 @@ const EPISODIC_UNRECOGNIZED_MODEL_OUTPUT = { validationTargets: [], }; +// An UNRECOGNIZED knowledge_type value: not a valid enum member even after +// trim/lowercase. The distiller must NOT silently coerce it to a +// CLEAR-RELEVANT knowledge_type (architecture/design-rationale/root-cause/ +// protocol/security) — doing so hands a garbled internal-ops candidate the +// clear-relevant pre-screen bypass in the audience gate, escaping the +// fail-restrictive judge and the internal-ops floor. It must WARN (mirroring +// the sensitivity coercion) and default to a NON-clear-relevant type so +// unknowns fall through to the judge. +const UNRECOGNIZED_KNOWLEDGE_TYPE_MARKER = "UNRECOGNIZED-KNOWLEDGE-TYPE-WINDOW"; +const EPISODIC_UNRECOGNIZED_KNOWLEDGE_TYPE_OUTPUT = { + title: "Internal deploy-cadence bookkeeping for the harvest cron", + content: + "The harvest cron's deploy cadence bookkeeping is tracked internally; this is why the on-call rotation owns the redeploy window.", + subsystem: "harvest", + knowledge_type: "totally-made-up-knowledge-type", + validationTargets: [], +}; + +// The set of clear-relevant knowledge types (mirrored from audience-gate.ts's +// CLEAR_RELEVANT_KNOWLEDGE_TYPES, which is module-private). A coerced-default +// knowledge_type MUST NOT land in this set, or an unrecognized internal-ops +// candidate takes the clear-relevant bypass and skips the fail-restrictive +// judge. +const CLEAR_RELEVANT_KNOWLEDGE_TYPES = new Set([ + "architecture", + "design-rationale", + "root-cause", + "protocol", + "security", +]); + // A model subsystem containing ':' (a canonical-key structural delimiter) plus // padded/blank validationTargets entries. CandidateFragmentSchema rejects ':' // in subsystem, so without sanitization the distiller's "always parses against @@ -145,6 +178,105 @@ const EXCLUSION_KEEP_OUTPUT = { reason: "Candidate is a generic architecture fact with no customer data.", }; +// Audience-judge verdicts (§4.4). Each fixture is gated on the audience system +// prompt plus a distinct user-payload marker so it fires only for its case. +const AUDIENCE_INTERNAL_OPS_MARKER = "AUDIENCE-INTERNAL-OPS"; +const AUDIENCE_RELEVANT_MARKER = "AUDIENCE-RELEVANT"; +const AUDIENCE_BORDERLINE_MARKER = "AUDIENCE-BORDERLINE"; +const AUDIENCE_INTERNAL_OPS_OUTPUT = { + verdict: "internal-ops", + reason: "PR-closeout/deploy log with zero external-builder utility.", +}; +const AUDIENCE_RELEVANT_OUTPUT = { + verdict: "relevant", + reason: "Explains why timingSafeEqual guards the admin endpoint.", +}; +const AUDIENCE_BORDERLINE_OUTPUT = { + verdict: "borderline", + reason: "Internal-leaning but an advanced builder might want it.", +}; + +// Audience-judge delimiter VARIANTS of the internal-ops verdict. The canonical +// verdict is "internal-ops" but a nondeterministic model may emit the same +// multi-token verdict with a space or underscore ("internal ops" / +// "internal_ops"). judgeAudience must recognize these as internal-ops, NOT +// throw and kill the whole harvest run. Each gated on a distinct user marker. +const AUDIENCE_INTERNAL_OPS_UNDERSCORE_MARKER = "AUDIENCE-IOPS-UNDERSCORE"; +const AUDIENCE_INTERNAL_OPS_SPACE_MARKER = "AUDIENCE-IOPS-SPACE"; +const AUDIENCE_INTERNAL_OPS_CASED_MARKER = "AUDIENCE-IOPS-CASED"; +const AUDIENCE_INTERNAL_OPS_UNDERSCORE_OUTPUT = { + verdict: "internal_ops", + reason: "Underscore-delimited internal-ops verdict variant.", +}; +const AUDIENCE_INTERNAL_OPS_SPACE_OUTPUT = { + verdict: "internal ops", + reason: "Space-delimited internal-ops verdict variant.", +}; +const AUDIENCE_INTERNAL_OPS_CASED_OUTPUT = { + verdict: " Internal_Ops ", + reason: "Cased/padded internal-ops verdict variant.", +}; + +// Audience-judge INTERIOR-RUN delimiter variants of the internal-ops verdict. +// The round-1 fix only canonicalized SINGLE-delimiter variants (one space, one +// underscore, one hyphen). A model can just as easily emit a DOUBLE space or a +// tab between the two tokens ("internal ops" / "internal\tops"). Those fall +// through the single-delimiter list to the borderline fallback, silently +// UN-FLOORING what is clearly an internal-ops verdict (internal-ops floors +// approvable=false; borderline does not). judgeAudience must collapse ANY run +// of internal whitespace/underscore/hyphen delimiters and still map to +// internal-ops. Each gated on a distinct user marker. +const AUDIENCE_INTERNAL_OPS_DBLSPACE_MARKER = "AUDIENCE-IOPS-DBLSPACE"; +const AUDIENCE_INTERNAL_OPS_TAB_MARKER = "AUDIENCE-IOPS-TAB"; +const AUDIENCE_INTERNAL_OPS_DBLSPACE_OUTPUT = { + verdict: "internal ops", + reason: "Double-space-delimited internal-ops verdict variant.", +}; +const AUDIENCE_INTERNAL_OPS_TAB_OUTPUT = { + verdict: "internal\tops", + reason: "Tab-delimited internal-ops verdict variant.", +}; + +// Audience-judge GARBAGE verdict: an unrecognized value that is neither a known +// verdict nor an internal-ops delimiter variant. The gate is NEVER-DROP and +// fail-restrictive: an unrecognized verdict must fall back to "borderline" +// (keep for human review), NEVER throw and kill the run. +const AUDIENCE_GARBAGE_MARKER = "AUDIENCE-GARBAGE-VERDICT"; +const AUDIENCE_GARBAGE_OUTPUT = { + verdict: "totally-made-up-verdict", + reason: "Model emitted a verdict outside the known set.", +}; + +// Audience-judge internal-ops verdicts with a TRAILING TOKEN beyond the two +// canonical tokens: "internal-ops (trivia)" and "internal-ops." The round-3 fix +// used EXACT-equality after delimiter-collapse, so a verdict with an extra +// trailing token fell through to the non-flooring borderline fallback — a +// floor escape. judgeAudience must match internal-ops when the collapsed +// verdict's LEADING token/prefix is "internal-ops", anchored at the start (so a +// verdict that merely MENTIONS the phrase mid-sentence does not over-match). +const AUDIENCE_INTERNAL_OPS_TRAILING_MARKER = "AUDIENCE-IOPS-TRAILING"; +const AUDIENCE_INTERNAL_OPS_TRAILING_DOT_MARKER = "AUDIENCE-IOPS-DOTSUFFIX"; +const AUDIENCE_INTERNAL_OPS_TRAILING_OUTPUT = { + verdict: "internal-ops (trivia)", + reason: "Internal-ops verdict with a trailing parenthetical token.", +}; +const AUDIENCE_INTERNAL_OPS_TRAILING_DOT_OUTPUT = { + verdict: "internal-ops.", + reason: "Internal-ops verdict with a trailing period.", +}; + +// A candidate whose ONLY "specific" is a machine-local absolute path (env glue, +// per §6.1). The patched DISTILLATION_SYSTEM_PROMPT must NOT let the presence of +// that opaque path make the claim "distilled" — the model rates it a restatement +// (or a rewrite that drops the env glue). Gated on the distillation system prompt +// plus this marker in the user payload. +const MACHINE_PATH_DISTILL_MARKER = "MACHINE-PATH-DISTILL"; +const MACHINE_PATH_DISTILL_OUTPUT = { + verdict: "restatement", + reason: + "The only 'specific' is a machine-local /Users path (env glue), not a product-portable specific, so there is no distilled why/how claim.", +}; + // Markers for windows whose "model" response is VALID JSON but NOT an object — // the distiller must fail loud (same path as a parse failure), never treat a // bare string / null / array as the expected shape. @@ -212,6 +344,17 @@ const fixtures: Fixture[] = [ }, response: { content: JSON.stringify(EPISODIC_UNRECOGNIZED_MODEL_OUTPUT) }, }, + // Episodic distillation — UNRECOGNIZED knowledge_type. Listed before the + // catch-all episodic fixture so the more specific match wins. + { + match: { + systemMessage: EPISODIC_SYSTEM_MARKER, + userMessage: UNRECOGNIZED_KNOWLEDGE_TYPE_MARKER, + }, + response: { + content: JSON.stringify(EPISODIC_UNRECOGNIZED_KNOWLEDGE_TYPE_OUTPUT), + }, + }, // Episodic distillation — ':'-bearing subsystem + padded validationTargets. // Listed before the catch-all episodic fixture so the more specific match wins. { @@ -277,6 +420,115 @@ const fixtures: Fixture[] = [ }), }, }, + // Audience judge — INTERNAL-OPS: gate on the audience system prompt AND the + // internal-ops marker in the user payload. + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_INTERNAL_OPS_MARKER, + }, + response: { content: JSON.stringify(AUDIENCE_INTERNAL_OPS_OUTPUT) }, + }, + // Audience judge — RELEVANT. + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_RELEVANT_MARKER, + }, + response: { content: JSON.stringify(AUDIENCE_RELEVANT_OUTPUT) }, + }, + // Audience judge — BORDERLINE. + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_BORDERLINE_MARKER, + }, + response: { content: JSON.stringify(AUDIENCE_BORDERLINE_OUTPUT) }, + }, + // Audience judge — internal-ops delimiter VARIANTS: underscore / space / cased. + // The model emits the multi-token verdict with a non-canonical delimiter; the + // gate must map each to the internal-ops verdict, not throw. + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_INTERNAL_OPS_UNDERSCORE_MARKER, + }, + response: { + content: JSON.stringify(AUDIENCE_INTERNAL_OPS_UNDERSCORE_OUTPUT), + }, + }, + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_INTERNAL_OPS_SPACE_MARKER, + }, + response: { content: JSON.stringify(AUDIENCE_INTERNAL_OPS_SPACE_OUTPUT) }, + }, + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_INTERNAL_OPS_CASED_MARKER, + }, + response: { content: JSON.stringify(AUDIENCE_INTERNAL_OPS_CASED_OUTPUT) }, + }, + // Audience judge — internal-ops INTERIOR-RUN variants: double-space / tab. + // A run of >1 whitespace (or mixed) delimiters between the two tokens must + // still map to internal-ops, not slip through to the borderline fallback. + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_INTERNAL_OPS_DBLSPACE_MARKER, + }, + response: { + content: JSON.stringify(AUDIENCE_INTERNAL_OPS_DBLSPACE_OUTPUT), + }, + }, + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_INTERNAL_OPS_TAB_MARKER, + }, + response: { content: JSON.stringify(AUDIENCE_INTERNAL_OPS_TAB_OUTPUT) }, + }, + // Audience judge — GARBAGE verdict: unrecognized value falls back to + // borderline (never-drop, fail-restrictive), NOT a thrown error. + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_GARBAGE_MARKER, + }, + response: { content: JSON.stringify(AUDIENCE_GARBAGE_OUTPUT) }, + }, + // Audience judge — internal-ops with a TRAILING TOKEN: "internal-ops + // (trivia)" / "internal-ops." A prefix-anchored match must still floor these; + // exact-equality after delimiter-collapse let them escape to borderline. + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_INTERNAL_OPS_TRAILING_MARKER, + }, + response: { + content: JSON.stringify(AUDIENCE_INTERNAL_OPS_TRAILING_OUTPUT), + }, + }, + { + match: { + systemMessage: AUDIENCE_SYSTEM_MARKER, + userMessage: AUDIENCE_INTERNAL_OPS_TRAILING_DOT_MARKER, + }, + response: { + content: JSON.stringify(AUDIENCE_INTERNAL_OPS_TRAILING_DOT_OUTPUT), + }, + }, + // Distillation judge — a candidate whose only "specific" is a machine-local + // /Users path: the patched prompt must NOT rate it "distilled" (§6.1). + { + match: { + systemMessage: DISTILLATION_SYSTEM_MARKER, + userMessage: MACHINE_PATH_DISTILL_MARKER, + }, + response: { content: JSON.stringify(MACHINE_PATH_DISTILL_OUTPUT) }, + }, // Embedding — WRONG dimension: the "model" returns a 7-element vector even // though embed() requested this.embeddingDimensions (1536). embed() must // surface the mismatch (fail loud), not pass a wrong-length vector downstream. @@ -430,6 +682,37 @@ describe("OpenAIDistiller (aimock)", () => { } }); + it("WARNS and coerces an unrecognized knowledge_type to a NON-clear-relevant default (no clear-relevant bypass escape)", async () => { + // Finding 1 (floor escape): an unrecognized model knowledge_type was + // silently coerced to "design-rationale" — which is a CLEAR-RELEVANT type. + // That hands a garbled internal-ops candidate the clear-relevant pre-screen + // bypass in the audience gate, skipping the fail-restrictive judge and the + // internal-ops floor. Fail-restrictive: an unrecognized type must WARN + // (mirroring coerceEpisodicSensitivity, whose sibling was silent) AND + // default to a type that is NOT clear-relevant, so unknowns fall through to + // the judge instead of being force-passed. + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const fragment = await distiller.distillEpisodicWindow( + `Transcript window ${UNRECOGNIZED_KNOWLEDGE_TYPE_MARKER}: deploy cadence.`, + { sourceName: "session-unknown-kt", subsystem: "harvest" }, + ); + + expect(() => CandidateFragmentSchema.parse(fragment)).not.toThrow(); + const kt = fragment.provenance.classification.knowledge_type; + // The coerced default must NOT be one of the clear-relevant types, or a + // garbled internal-ops candidate escapes the floor via the bypass. + expect(CLEAR_RELEVANT_KNOWLEDGE_TYPES.has(kt)).toBe(false); + // The warning names the discarded value so the operator can see what the + // model actually said (mirrors the sensitivity coercion warn). + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("totally-made-up-knowledge-type"), + ); + } finally { + warnSpy.mockRestore(); + } + }); + it("sanitizes a ':'-bearing model subsystem and trims validationTargets so the fragment stays schema-valid", async () => { // ':' is a canonical-key structural delimiter; CandidateFragmentSchema // rejects it in subsystem. The distiller promises the returned fragment @@ -578,6 +861,210 @@ describe("OpenAIDistiller (aimock)", () => { expect(verdict.reason).toBe("Candidate is a generic fact."); }); }); + + describe("judgeAudience (§4.4)", () => { + it("rates a PR-closeout/deploy candidate internal-ops", async () => { + const verdict = await distiller.judgeAudience({ + title: `PR-closeout log ${AUDIENCE_INTERNAL_OPS_MARKER}`, + content: + "PR #742 shipped the reindex tweak and redeployed the Railway service.", + knowledge_type: "operational", + subsystem: "pr-closeout", + }); + expect(verdict.kind).toBe("internal-ops"); + if (verdict.kind === "internal-ops") { + expect(verdict.reason).toBe(AUDIENCE_INTERNAL_OPS_OUTPUT.reason); + } + }); + + it("rates a why/how claim about timingSafeEqual relevant", async () => { + const verdict = await distiller.judgeAudience({ + title: `Why timingSafeEqual ${AUDIENCE_RELEVANT_MARKER}`, + content: + "POST /admin/:op compares the token via timingSafeEqual so a wrong key returns 401 without leaking timing.", + knowledge_type: "security", + subsystem: "admin-api", + }); + expect(verdict.kind).toBe("relevant"); + }); + + it("rates an internal-leaning-but-maybe-useful claim borderline", async () => { + const verdict = await distiller.judgeAudience({ + title: `Advanced internals ${AUDIENCE_BORDERLINE_MARKER}`, + content: + "The reindex job batches by subsystem; a sophisticated builder might reuse the batching idea.", + knowledge_type: "operational", + subsystem: "reindex", + }); + expect(verdict.kind).toBe("borderline"); + if (verdict.kind === "borderline") { + expect(verdict.reason).toBe(AUDIENCE_BORDERLINE_OUTPUT.reason); + } + }); + + it("maps an underscore-delimited 'internal_ops' verdict to internal-ops (no throw)", async () => { + const verdict = await distiller.judgeAudience({ + title: `Deploy log ${AUDIENCE_INTERNAL_OPS_UNDERSCORE_MARKER}`, + content: "PR #99 redeployed the Railway service.", + knowledge_type: "operational", + subsystem: "pr-closeout", + }); + expect(verdict.kind).toBe("internal-ops"); + if (verdict.kind === "internal-ops") { + expect(verdict.reason).toBe( + AUDIENCE_INTERNAL_OPS_UNDERSCORE_OUTPUT.reason, + ); + } + }); + + it("maps a space-delimited 'internal ops' verdict to internal-ops (no throw)", async () => { + const verdict = await distiller.judgeAudience({ + title: `Deploy log ${AUDIENCE_INTERNAL_OPS_SPACE_MARKER}`, + content: "PR #100 redeployed the Railway service.", + knowledge_type: "operational", + subsystem: "pr-closeout", + }); + expect(verdict.kind).toBe("internal-ops"); + if (verdict.kind === "internal-ops") { + expect(verdict.reason).toBe(AUDIENCE_INTERNAL_OPS_SPACE_OUTPUT.reason); + } + }); + + it("maps a cased/padded ' Internal_Ops ' verdict to internal-ops (no throw)", async () => { + const verdict = await distiller.judgeAudience({ + title: `Deploy log ${AUDIENCE_INTERNAL_OPS_CASED_MARKER}`, + content: "PR #101 redeployed the Railway service.", + knowledge_type: "operational", + subsystem: "pr-closeout", + }); + expect(verdict.kind).toBe("internal-ops"); + if (verdict.kind === "internal-ops") { + expect(verdict.reason).toBe(AUDIENCE_INTERNAL_OPS_CASED_OUTPUT.reason); + } + }); + + it("maps a double-space 'internal ops' verdict to internal-ops (not borderline)", async () => { + // Round-1 canonicalized only single-delimiter variants; an interior run + // of >1 space slipped through to the borderline fallback, UN-FLOORING an + // internal-ops verdict (borderline does not floor approvable=false). + const verdict = await distiller.judgeAudience({ + title: `Deploy log ${AUDIENCE_INTERNAL_OPS_DBLSPACE_MARKER}`, + content: "PR #102 redeployed the Railway service.", + knowledge_type: "operational", + subsystem: "pr-closeout", + }); + expect(verdict.kind).toBe("internal-ops"); + if (verdict.kind === "internal-ops") { + expect(verdict.reason).toBe( + AUDIENCE_INTERNAL_OPS_DBLSPACE_OUTPUT.reason, + ); + } + }); + + it("maps a tab-delimited 'internal\\tops' verdict to internal-ops (not borderline)", async () => { + // A tab between the two tokens is the same interior-run gap as a double + // space — it must collapse to the canonical internal-ops verdict. + const verdict = await distiller.judgeAudience({ + title: `Deploy log ${AUDIENCE_INTERNAL_OPS_TAB_MARKER}`, + content: "PR #103 redeployed the Railway service.", + knowledge_type: "operational", + subsystem: "pr-closeout", + }); + expect(verdict.kind).toBe("internal-ops"); + if (verdict.kind === "internal-ops") { + expect(verdict.reason).toBe(AUDIENCE_INTERNAL_OPS_TAB_OUTPUT.reason); + } + }); + + it("falls back to borderline on an unrecognized verdict (never-drop, no throw)", async () => { + const verdict = await distiller.judgeAudience({ + title: `Ambiguous claim ${AUDIENCE_GARBAGE_MARKER}`, + content: "Some claim the model classified with a bogus verdict.", + knowledge_type: "operational", + subsystem: "misc", + }); + expect(verdict.kind).toBe("borderline"); + if (verdict.kind === "borderline") { + expect(verdict.reason).toBe(AUDIENCE_GARBAGE_OUTPUT.reason); + } + }); + + it("WARNS at the seam when it falls back to borderline on an unrecognized verdict (observability, still never-drop)", async () => { + // Finding 2: the unrecognized-verdict → borderline fallback logged + // NOTHING, unlike every sibling branch (which throws or warns). A drifting + // model silently inflates the human-review queue with no operator signal. + // The fallback must WARN at the seam (mirroring the sibling warn style) + // while KEEPING the never-throw, borderline behavior. + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const verdict = await distiller.judgeAudience({ + title: `Ambiguous claim ${AUDIENCE_GARBAGE_MARKER}`, + content: "Some claim the model classified with a bogus verdict.", + knowledge_type: "operational", + subsystem: "misc", + }); + expect(verdict.kind).toBe("borderline"); + // The warn names the discarded verdict so a drifting model is visible. + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("totally-made-up-verdict"), + ); + } finally { + warnSpy.mockRestore(); + } + }); + + it("maps an 'internal-ops (trivia)' verdict (trailing token) to internal-ops, not borderline", async () => { + // Finding 3: the floor used EXACT-equality after delimiter-collapse, so a + // verdict with an extra trailing token fell through to the non-flooring + // borderline fallback — a floor escape. A prefix-anchored match must floor + // "internal-ops (trivia)". + const verdict = await distiller.judgeAudience({ + title: `Deploy log ${AUDIENCE_INTERNAL_OPS_TRAILING_MARKER}`, + content: "PR #200 redeployed the Railway service.", + knowledge_type: "operational", + subsystem: "pr-closeout", + }); + expect(verdict.kind).toBe("internal-ops"); + if (verdict.kind === "internal-ops") { + expect(verdict.reason).toBe( + AUDIENCE_INTERNAL_OPS_TRAILING_OUTPUT.reason, + ); + } + }); + + it("maps an 'internal-ops.' verdict (trailing period) to internal-ops, not borderline", async () => { + const verdict = await distiller.judgeAudience({ + title: `Deploy log ${AUDIENCE_INTERNAL_OPS_TRAILING_DOT_MARKER}`, + content: "PR #201 redeployed the Railway service.", + knowledge_type: "operational", + subsystem: "pr-closeout", + }); + expect(verdict.kind).toBe("internal-ops"); + if (verdict.kind === "internal-ops") { + expect(verdict.reason).toBe( + AUDIENCE_INTERNAL_OPS_TRAILING_DOT_OUTPUT.reason, + ); + } + }); + }); + + describe("judgeDistillation env-glue reconciliation (§6.1)", () => { + it("does NOT rate a candidate 'distilled' when its only specific is a machine-local path", async () => { + // §6.1: machine-local absolute paths (/Users/…) are env glue, NOT + // product-portable specifics. The patched prompt must stop the gate from + // rating an env-glue-only candidate "distilled" just because it carries a + // long opaque path string. The aimock fixture returns "restatement" for + // this input; the assertion is that the verdict is NOT "distilled". + const verdict = await distiller.judgeDistillation({ + title: `Added the atlas gate ${MACHINE_PATH_DISTILL_MARKER}`, + content: + "Added the audience gate at /Users/dev/proj/cpk/pathfinder/src/atlas/audience-gate.ts.", + knowledge_type: "operational", + }); + expect(verdict.kind).not.toBe("distilled"); + expect(["restatement", "rewritten"]).toContain(verdict.kind); + }); + }); }); describe("OpenAIDistiller constructor (API-key guard, no LLM calls)", () => { diff --git a/src/__tests__/atlas-sanitize-env-refs.test.ts b/src/__tests__/atlas-sanitize-env-refs.test.ts new file mode 100644 index 0000000..c0c320d --- /dev/null +++ b/src/__tests__/atlas-sanitize-env-refs.test.ts @@ -0,0 +1,709 @@ +import { describe, expect, it } from "vitest"; + +import { sanitizeEnvRefs } from "../atlas/adapters/sanitize-env-refs.js"; + +// The sanitizer is a pure deterministic pass (§3.1 E1-E6). Each describe below +// is one detection category; the final blocks cover the criterion-7 no-op +// harness and the whole-corpus red-green anchor from §8. +// +// Convenience: most category tests only care about `content`, so this helper +// runs the sanitizer with an empty provenanceSource and returns the sanitized +// content. +function clean(content: string): string { + return sanitizeEnvRefs(content, "").content; +} + +describe("sanitizeEnvRefs — E1 machine-local absolute paths", () => { + it("keeps the repo-relative tail from the first recognized top-level segment", () => { + expect( + clean("see /Users/x/proj/cpk/pathfinder/src/atlas/distillation-gate.ts"), + ).toBe("see src/atlas/distillation-gate.ts"); + }); + + it("infers repo-relative from /proj/ paths (bin/ root)", () => { + expect(clean("run /proj/cpk/pathfinder/bin/showcase now")).toBe( + "run bin/showcase now", + ); + }); + + it("infers repo-relative from ~/ paths (tests/ root)", () => { + expect(clean("edit ~/proj/cpk/pathfinder/tests/foo.ts")).toBe( + "edit tests/foo.ts", + ); + }); + + it("replaces the whole path with when no recognized root is present", () => { + expect(clean("open /Users/x/random-notes.md")).toBe("open "); + }); + + it("keeps from the FIRST recognized root on nested cases", () => { + expect(clean("path /a/src/b/src/c.ts here")).toBe( + "path src/b/src/c.ts here", + ); + }); + + it("F7 truncation guard: matches the WHOLE path with no leftover tail", () => { + // The path token runs to end-of-path (whitespace/quote/paren/backtick/EOS), + // NOT the next slash — so there is no leftover `/pathfinder/...` fragment. + const out = clean("at /proj/cpk/pathfinder/src/atlas/x.ts:12"); + expect(out).toBe("at src/atlas/x.ts:12"); + expect(out).not.toContain("/proj"); + expect(out).not.toContain("pathfinder/"); + }); + + it("terminates the path token at a closing paren", () => { + expect(clean("(see /Users/x/random-notes.md) done")).toBe( + "(see ) done", + ); + }); + + it("terminates the path token at a backtick", () => { + expect(clean("`/proj/cpk/pathfinder/src/atlas/x.ts`")).toBe( + "`src/atlas/x.ts`", + ); + }); + + it("sanitizes /home/ paths with no recognized root to ", () => { + expect(clean("cd /home/alice/random/notes.md")).toBe("cd "); + }); + + it("does NOT anchor on a user/home segment named like a repo dir (CR finding 1)", () => { + // The home/user segment happens to be literally named an anchor dir. It is + // machine-local structure, NOT a repo-relative tail — must collapse fully. + expect(clean("open /Users/deploy/proj/app.ts")).toBe("open "); + expect(clean("cd /home/src/x/notes.md")).toBe("cd "); + }); + + it("sanitizes a home-root file with no interior slash (round-3 finding 3)", () => { + // A `~/`-rooted file with no interior slash (`~/secret.env`) is still a + // machine-local reference and must collapse to — the comment + // advertised `~/…` handling but the interior-slash requirement missed it. + expect(clean("cat ~/secret.env")).toBe("cat "); + expect(clean("edit ~/.npmrc now")).toBe("edit now"); + }); + + it("keeps the portable tail for a path rooted at a top-level anchor dir (round-3 finding 4)", () => { + // A path rooted directly at a repo top-level dir (`/src/x.ts`, `/bin/y`) is + // portable — keep the tail from that anchor, do NOT collapse to . + expect(clean("see /src/x.ts here")).toBe("see src/x.ts here"); + expect(clean("run /bin/y now")).toBe("run bin/y now"); + // Guard the round-1 fix: a home/user root must STILL collapse fully and must + // NOT emit the username segment. + expect(clean("open /Users/deploy/proj/app.ts")).toBe("open "); + // Guard the §8 nested anchor still resolves from the first genuine src/. + expect(clean("path /a/src/b/src/c.ts here")).toBe( + "path src/b/src/c.ts here", + ); + }); + + it("stops the path at trailing prose punctuation (CR finding 2)", () => { + // A trailing comma / period is prose, not part of the path tail. It must + // not be absorbed into the sanitized path. + expect(clean("see /Users/x/random-notes.md, then continue")).toBe( + "see , then continue", + ); + expect(clean("look at /Users/x/notes.md.")).toBe("look at ."); + expect(clean("path /proj/cpk/pathfinder/src/atlas/x.ts; next")).toBe( + "path src/atlas/x.ts; next", + ); + }); + + it("sanitizes machine-local paths in provenanceSource too", () => { + const { source } = sanitizeEnvRefs( + "", + "/Users/x/proj/cpk/pathfinder/src/atlas/memory.ts", + ); + expect(source).toBe("src/atlas/memory.ts"); + }); + + it("is idempotent", () => { + const x = + "see /Users/x/proj/cpk/pathfinder/src/atlas/x.ts and /Users/x/z.md"; + expect(clean(clean(x))).toBe(clean(x)); + }); +}); + +describe("sanitizeEnvRefs — E2 session UUIDs", () => { + const UUID = "e654541f-dcb7-4152-8ee8-f669848555ee"; + + it("strips a session UUID clause from content", () => { + const out = clean(`built as memory:foo.md (session ${UUID})`); + expect(out).toBe("built as memory:foo.md"); + expect(out).not.toContain(UUID); + }); + + it("strips a session UUID from provenanceSource", () => { + const { source } = sanitizeEnvRefs("", `memory:foo.md (session ${UUID})`); + expect(source).not.toContain(UUID); + }); + + it("is idempotent", () => { + const x = `note (session ${UUID}) tail`; + expect(clean(clean(x))).toBe(clean(x)); + }); +}); + +describe("sanitizeEnvRefs — E3 Notion page ids/URLs", () => { + it("replaces a notion.so URL in content with ", () => { + const out = clean( + "see https://www.notion.so/copilotkit/Some-Page-3963aa38185281db80b8e4bf73de0ea5", + ); + expect(out).toBe("see "); + expect(out).not.toContain("3963aa38185281db80b8e4bf73de0ea5"); + }); + + it("replaces a bare 32-hex in notion.so context", () => { + const out = clean("notion.so/3963aa38185281db80b8e4bf73de0ea5 here"); + expect(out).toBe(" here"); + }); + + it("replaces a 32-hex in /p/ context", () => { + const out = clean("app.notion.com/p/3963aa38185281db80b8e4bf73de0ea5"); + expect(out).toBe(""); + }); + + it("replaces a dashed-UUID notion.so link with (CR finding 3)", () => { + const out = clean( + "see https://www.notion.so/copilotkit/Some-Page-3963aa38-1852-81db-80b8-e4bf73de0ea5 tail", + ); + expect(out).toBe("see tail"); + expect(out).not.toContain("notion.so"); + expect(out).not.toContain("3963aa38-1852-81db-80b8-e4bf73de0ea5"); + }); + + it("replaces a single-segment notion.so/ URL cleanly (CR finding 4)", () => { + const out = clean( + "open https://notion.so/3963aa38185281db80b8e4bf73de0ea5 now", + ); + expect(out).toBe("open now"); + expect(out).not.toContain("https://"); + expect(out).not.toContain("notion.so"); + }); + + it("collapses a multi-segment scheme-less notion.so link whole (round-3 finding 2)", () => { + // A scheme-less notion.so link with 2+ intermediate path segments must + // collapse WHOLE — no leaked internal page id, no dangling prefix. + const out = clean( + "see notion.so/team/space/Page-3963aa38185281db80b8e4bf73de0ea5 tail", + ); + expect(out).toBe("see tail"); + expect(out).not.toContain("3963aa38185281db80b8e4bf73de0ea5"); + expect(out).not.toContain("notion.so"); + expect(out).not.toContain("team/space"); + }); + + it("does NOT strip a bare 32-hex commit SHA in code context", () => { + const sha = "6a10cc2f8d3e4b1a9c7f2e5d8b0a3c6f9e1d4b7a"; + // A commit-SHA-shaped 40-hex must survive; and a bare 32-hex outside notion + // context must survive too. + const hex32 = "3963aa38185281db80b8e4bf73de0ea5"; + expect(clean(`git checkout ${sha}`)).toBe(`git checkout ${sha}`); + expect(clean(`blob hash ${hex32} in tree`)).toBe( + `blob hash ${hex32} in tree`, + ); + }); + + it("collapses a genuine /p/ Notion link (with same-string notion context) to , not (crfix3 finding: ordering + context anchor)", () => { + // A bare leading-slash `/p/` looks like a machine-local absolute path to + // E1, so today it wrongly collapses to . When Notion context is + // present in the same string, it is a genuine Notion page link and must + // collapse to . The reconciliation anchors the /p/ arm on + // Notion context AND orders the context-anchored Notion arms before E1. + const hex = "3963aa38185281db80b8e4bf73de0ea5"; + const out = clean(`the notion page /p/${hex} (see notion.so)`); + expect(out).toContain(""); + expect(out).not.toContain(""); + expect(out).not.toContain(hex); + }); + + it("does NOT turn a context-free /p/ (no notion anywhere) into , and does not leak the raw id (spec OQ5: no context-free strip)", () => { + // No Notion host/marker in the string => no justification for the Notion + // label. It must NOT become . It falls through to E1, + // which collapses the whole machine-local path to — so the raw + // id does not leak either. Best of both: no false Notion label, no id leak. + const hex = "3963aa38185281db80b8e4bf73de0ea5"; + const out = clean(`see the route /p/${hex} for details`); + expect(out).not.toContain(""); + expect(out).not.toContain(hex); + expect(out).toBe("see the route for details"); + }); + + it("fires the /p/ arm when Notion context existed anywhere pre-replacement, so a notion.so URL AND a /p/ in the SAME string both collapse (round-4 finding: marker must be tested pre-replacement)", () => { + // A string carrying BOTH a notion.so URL AND a bare /p/ link: both are + // genuine Notion page links and must become . The /p/ arm + // is gated on the presence of a Notion marker; that gate must be evaluated + // against the PRE-replacement (original) content, not the already-partially- + // replaced output — otherwise the gate's correctness accidentally depends on + // the replacement placeholder happening to still contain the word "notion". + const hex1 = "3963aa38185281db80b8e4bf73de0ea5"; + const hex2 = "1111aa38185281db80b8e4bf73de0ea5"; + const out = clean( + `see https://www.notion.so/copilotkit/Page-${hex1} and /p/${hex2}`, + ); + expect(out).toBe("see and "); + expect(out).not.toContain(""); + expect(out).not.toContain(hex1); + expect(out).not.toContain(hex2); + }); + + it("collapses a Notion host carrying leading subdomain labels WHOLE, leaking no leading label (round-4 host hardening)", () => { + // `notion.so` / `notion.com` matched as a bare host leaves any leading + // subdomain label dangling (`foo.notion.so/` => `foo.`). + // The host arms must consume ANY number of leading labels (and the scheme, if + // present) so the whole reference collapses. + const hex = "3963aa38185281db80b8e4bf73de0ea5"; + + const ctx = clean(`see foo.notion.so/${hex} tail`); + expect(ctx).toBe("see tail"); + expect(ctx).not.toContain("foo."); + + const ctxCom = clean(`see app.notion.com/${hex} tail`); + expect(ctxCom).toBe("see tail"); + expect(ctxCom).not.toContain("app."); + + const url = clean(`open https://foo.notion.so/Page-${hex} now`); + expect(url).toBe("open now"); + expect(url).not.toContain("foo."); + expect(url).not.toContain("https://"); + }); + + it("leaves provenanceSource / url untouched by E3", () => { + const url = + "https://www.notion.so/copilotkit/Some-Page-3963aa38185281db80b8e4bf73de0ea5"; + const { source } = sanitizeEnvRefs("", url); + expect(source).toBe(url); + }); + + it("is idempotent", () => { + const x = + "see https://www.notion.so/copilotkit/Some-Page-3963aa38185281db80b8e4bf73de0ea5 tail"; + expect(clean(clean(x))).toBe(clean(x)); + }); +}); + +describe("sanitizeEnvRefs — E4 internal Slack channels (closed allowlist)", () => { + it("replaces #engr with ", () => { + expect(clean("ping #engr about it")).toBe( + "ping about it", + ); + }); + + it("replaces #oss-alerts with ", () => { + expect(clean("see #oss-alerts")).toBe("see "); + }); + + it("replaces #deploys with ", () => { + expect(clean("watch #deploys")).toBe("watch "); + }); + + it("does NOT touch #teamwork (F5 negative)", () => { + expect(clean("the #teamwork tag")).toBe("the #teamwork tag"); + }); + + it("does NOT touch #teams (F5 negative)", () => { + expect(clean("join #teams")).toBe("join #teams"); + }); + + it("does NOT touch a Markdown heading '# Infra' (F5 negative)", () => { + expect(clean("# Infra\nbody")).toBe("# Infra\nbody"); + }); + + it("does NOT touch 'see the #infra section' (F5 negative)", () => { + expect(clean("see the #infra section")).toBe("see the #infra section"); + }); + + it("does NOT match #deploys inside #deployserver (word boundary)", () => { + expect(clean("the #deployserver host")).toBe("the #deployserver host"); + }); + + it("is idempotent", () => { + const x = "ping #engr and #oss-alerts"; + expect(clean(clean(x))).toBe(clean(x)); + }); +}); + +describe("sanitizeEnvRefs — E5 railway hosts", () => { + it("replaces railway.app/project/ with ", () => { + const out = clean( + "deployed to railway.app/project/6a10cc2f-8d3e-4b1a-9c7f-2e5d8b0a3c6f", + ); + expect(out).toBe("deployed to "); + expect(out).not.toContain("railway.app/project"); + }); + + it("replaces .up.railway.app with ", () => { + const out = clean("host pathfinder-web.up.railway.app is up"); + expect(out).toBe("host is up"); + expect(out).not.toContain("up.railway.app"); + }); + + it("replaces an uppercase-labelled railway host case-insensitively (round-3 finding 1)", () => { + // The host label may contain uppercase (`MyApp.up.railway.app`); the arm + // must still strip it or the internal host leaks unsanitized. + const out = clean("host MyApp.up.railway.app is up"); + expect(out).toBe("host is up"); + expect(out).not.toMatch(/railway\.app/i); + }); + + it("replaces an uppercase-domain railway project link case-insensitively (round-3 finding 1)", () => { + // The literal `railway.app` host label can appear with uppercase (e.g. a + // copy-pasted `Railway.app/project/…`); the arm must strip it too. + const out = clean("deployed to Railway.app/project/MyProject-123"); + expect(out).toBe("deployed to "); + expect(out).not.toMatch(/railway\.app/i); + }); + + it("collapses a MULTI-LABEL railway host whole, leaking no leading label (round-4 finding: multi-label leak)", () => { + // `\b[a-z0-9-]+\.up\.railway\.app` captures only the single label before + // `.up`, so a multi-label host leaks its leading subdomains + // (`pr-42.myapp.up.railway.app` => `pr-42.`). The arm must + // consume ANY number of leading labels and collapse the whole host. + const out1 = clean("host pr-42.myapp.up.railway.app is up"); + expect(out1).toBe("host is up"); + expect(out1).not.toContain("pr-42"); + expect(out1).not.toMatch(/railway\.app/i); + + const out2 = clean("host svc.myapp.up.railway.app is up"); + expect(out2).toBe("host is up"); + expect(out2).not.toContain("svc."); + expect(out2).not.toMatch(/railway\.app/i); + }); + + it("collapses an uppercase MULTI-LABEL railway host whole (case-insensitive, no leading-label leak)", () => { + const out = clean("host PR-42.MyApp.up.railway.app is up"); + expect(out).toBe("host is up"); + expect(out).not.toContain("PR-42"); + expect(out).not.toMatch(/railway\.app/i); + }); + + it("is idempotent", () => { + const x = + "host pathfinder-web.up.railway.app and railway.app/project/abc-123"; + expect(clean(clean(x))).toBe(clean(x)); + }); +}); + +describe("sanitizeEnvRefs — E6 internal emails", () => { + it("replaces an @copilotkit.ai email with ", () => { + expect(clean("ask alice@copilotkit.ai for help")).toBe( + "ask for help", + ); + }); + + it("does NOT touch @copilotkit/react-core (F6 negative)", () => { + expect(clean("import @copilotkit/react-core")).toBe( + "import @copilotkit/react-core", + ); + }); + + it("does NOT touch @types/node (F6 negative)", () => { + expect(clean("add @types/node dep")).toBe("add @types/node dep"); + }); + + it("does NOT touch @Injectable (F6 negative)", () => { + expect(clean("use @Injectable() here")).toBe("use @Injectable() here"); + }); + + it("does NOT touch @copilotkit handle (F6 negative)", () => { + expect(clean("follow @copilotkit on X")).toBe("follow @copilotkit on X"); + }); + + it("is idempotent", () => { + const x = "ask alice@copilotkit.ai and bob@copilotkit.ai"; + expect(clean(clean(x))).toBe(clean(x)); + }); +}); + +describe("sanitizeEnvRefs — round-5 uniform host-boundary findings", () => { + const HEX = "3963aa38185281db80b8e4bf73de0ea5"; + + // Finding 1: RAILWAY_PROJECT was never hardened for leading subdomain labels + // (round-5 hardened RAILWAY_HOST but skipped the project arm). A subdomained + // host in front of `railway.app/project/` leaks the leading label. + it("collapses a subdomained railway.app/project/ WHOLE, leaking no leading label (finding 1)", () => { + const out1 = clean("deployed to pr-1.railway.app/project/xyz here"); + expect(out1).toBe("deployed to here"); + expect(out1).not.toContain("pr-1"); + expect(out1).not.toMatch(/railway\.app/i); + + const out2 = clean("deployed to foo.railway.app/project/my-id here"); + expect(out2).toBe("deployed to here"); + expect(out2).not.toContain("foo."); + expect(out2).not.toMatch(/railway\.app/i); + }); + + // Finding 2: NOTION_CONTEXT_ID had no left boundary, so it matched + // `notion.so`/`notion.com` MID-LABEL: `evilnotion.so/` corrupted to + // `evil`. + it("does NOT match notion.so mid-label — evilnotion.so must not be corrupted (finding 2)", () => { + const out = clean(`see evilnotion.so/${HEX} here`); + expect(out).not.toBe(`see evil here`); + expect(out).not.toContain(""); + // The genuine (non-Notion) host is left intact; only its bare id would ever + // be a concern, but `evilnotion.so` is not a Notion host at all. + expect(out).toContain("evilnotion.so"); + }); + + // Finding 3 (round-5) → REVISED by crfix7 F1: the round-5 fix forbade `>` in + // E1's LEADING_BOUNDARY so a placeholder's trailing `>` could not be an E1 + // anchor. But because the boundary is a ZERO-WIDTH lookbehind (it never + // consumes the `>`), that `>`-exclusion was the WRONG remedy: it caused a + // GENUINE machine-local path abutting an emitted placeholder's `>` (e.g. + // `notion.so//Users/jpr5/x.ts` → E3 → `/Users/jpr5/x.ts`) + // to be SKIPPED by E1 — leaking the username verbatim (crfix7 F1). Removing + // `>` from the forbidden set re-anchors E1 after the placeholder's `>` WITHOUT + // deleting the `>` (zero-width), so the placeholder stays intact AND the + // following genuine path is sanitized. Consequently a `/`-rooted interior-slash + // path immediately after a placeholder is now correctly collapsed: `/foo/bar` + // has no recognized repo top-level segment (segment 1 `foo` is not a HOME_ROOT + // and firstAnchorable=1 finds no REPO_TOP_LEVEL match), so it becomes + // . The placeholder's `>` is preserved (never consumed); no `/` is + // deleted; and the result is idempotent (the emitted / + // contain no re-anchorable `/`-rooted path). + it("sanitizes a genuine path abutting a placeholder's '>' without consuming the '>' (crfix7 F1; revises round-5 finding 3)", () => { + const input = "/foo/bar and more"; + const out = clean(input); + // The genuine absolute path after the placeholder is sanitized (no leak, no + // corruption). `/foo/bar` has no repo top-level anchor → . + expect(out).toBe(" and more"); + // The placeholder's own `>` is preserved (zero-width lookbehind never + // consumed it) — the placeholder is not fused into a malformed `<...><`. + expect(out).not.toContain(" { + const out = clean("host up.railway.app is up"); + expect(out).toBe("host is up"); + expect(out).not.toMatch(/railway\.app/i); + }); + + // Boundary: a bare-word `notion` marker authorizes the /p/ arm. + it("bare-word notion marker authorizes the /p/ arm (boundary)", () => { + const out = clean(`the notion page /p/${HEX} here`); + expect(out).toContain(""); + expect(out).not.toContain(""); + expect(out).not.toContain(HEX); + }); + + // Boundary: a bare dashed-UUID with no Notion/session clause context is a + // free-standing UUID and is stripped by the E2 bare-UUID arm. + it("strips a bare dashed-UUID via the non-clause E2 branch (boundary)", () => { + const uuid = "e654541f-dcb7-4152-8ee8-f669848555ee"; + const out = clean(`ref ${uuid} in prose`); + expect(out).not.toContain(uuid); + }); +}); + +describe("sanitizeEnvRefs — crfix6 leading-boundary leak (E1 anchor allowlist)", () => { + // THE LEAK: E1's leading anchor was a consuming ALLOWLIST char class that + // omitted common path-preceding separators `=`, `:`, `,` (and `;`). So a + // machine-local absolute path preceded by any of them was left ENTIRELY + // UNSANITIZED and leaked the full username-bearing path. The systemic fix + // replaces the allowlist with a principled zero-width negative lookbehind that + // forbids only path-continuation chars (so we don't match mid-token) and the + // placeholder terminator `>` (so E1 stays placeholder-safe / idempotent). + + it("sanitizes a path preceded by `=` (KEY=/Users/... shell/env assignment)", () => { + // `/Users/jpr5/proj/...` has a username at seg 2; per firstAnchorable the + // scan lands on the first genuine repo top-level (`src/`). + const out = clean("CONFIG=/Users/jpr5/proj/src/x.ts"); + expect(out).toBe("CONFIG=src/x.ts"); + expect(out).not.toContain("/Users/"); + expect(out).not.toContain("jpr5"); + }); + + it("sanitizes a path preceded by `=` in a --flag=value form", () => { + // No recognized repo top-level under the username => collapses to . + const out = clean("--config=/Users/alice/secrets/src/prod.ts"); + expect(out).toBe("--config=src/prod.ts"); + expect(out).not.toContain("/Users/"); + expect(out).not.toContain("alice"); + }); + + it("sanitizes a path preceded by `=` where no repo root follows (cwd=)", () => { + const out = clean("cwd=/Users/alice/x"); + expect(out).toBe("cwd="); + expect(out).not.toContain("/Users/"); + expect(out).not.toContain("alice"); + }); + + it("sanitizes a path preceded by `: ` (YAML/prose `path: /Users/...`)", () => { + // Colon-space: the space already anchored, but a colon with no space must + // ALSO anchor. Guard the colon-space prose form explicitly. + const out = clean("path: /Users/alice/x"); + expect(out).toBe("path: "); + expect(out).not.toContain("/Users/"); + expect(out).not.toContain("alice"); + }); + + it("sanitizes a path preceded by a bare `:` (no space)", () => { + const out = clean("path:/Users/alice/x"); + expect(out).toBe("path:"); + expect(out).not.toContain("/Users/"); + expect(out).not.toContain("alice"); + }); + + it("sanitizes a path preceded by a bare `,` (comma anchor, tail to EOS)", () => { + // The bare `,` immediately before the path must anchor E1 (it was omitted + // from the old allowlist and leaked). Tail runs to end-of-string. + const out = clean("list,/Users/alice/x"); + expect(out).toBe("list,"); + expect(out).not.toContain("/Users/"); + expect(out).not.toContain("alice"); + }); + + it("sanitizes a path preceded by a bare `;` (semicolon anchor, tail to EOS)", () => { + const out = clean("first;/Users/alice/x"); + expect(out).toBe("first;"); + expect(out).not.toContain("/Users/"); + expect(out).not.toContain("alice"); + }); + + it("sanitizes a machine-local path preceded by `=` in provenanceSource too", () => { + const { source } = sanitizeEnvRefs( + "", + "CONFIG=/Users/x/proj/cpk/pathfinder/src/atlas/memory.ts", + ); + expect(source).toBe("CONFIG=src/atlas/memory.ts"); + expect(source).not.toContain("/Users/"); + }); + + it("does NOT over-match a `=`-preceded time value (no real path shape)", () => { + // `time=12:30` has no `/`-rooted or `~/`-rooted absolute-path shape after + // the boundary, so E1 must leave it entirely alone. + expect(clean("time=12:30")).toBe("time=12:30"); + }); + + it("does NOT over-match a `=`-preceded ratio (single interior slash, no path root)", () => { + // `ratio=3/4` is a bare `/`-separated token, not a `/`-rooted absolute path + // (the leading `/` is missing), so it must NOT be mangled into a path. + expect(clean("ratio=3/4")).toBe("ratio=3/4"); + }); + + it("still does NOT anchor mid-token / inside a word (word char before `/`)", () => { + // A slash immediately preceded by a word char (`abc/def`) is not a + // machine-local absolute path anchor and must be left alone. + expect(clean("pkg/subpath/file")).toBe("pkg/subpath/file"); + }); + + it("stays idempotent for a `=`-preceded path", () => { + const x = "CONFIG=/Users/jpr5/proj/src/x.ts and cwd=/Users/alice/z.md"; + expect(clean(clean(x))).toBe(clean(x)); + }); +}); + +describe("sanitizeEnvRefs — criterion 7 no-op harness", () => { + it("leaves pathfinder.yaml untouched", () => { + expect(clean("configure pathfinder.yaml keys")).toBe( + "configure pathfinder.yaml keys", + ); + }); + + it("leaves public API symbols untouched", () => { + expect(clean("call dedupAgainstRagCorpus() on the corpus")).toBe( + "call dedupAgainstRagCorpus() on the corpus", + ); + }); + + it("leaves HTTP codes untouched", () => { + expect(clean("returns 401 on bad token")).toBe("returns 401 on bad token"); + }); + + it("leaves @scope/pkg npm specifiers untouched", () => { + expect(clean("depends on @scope/pkg")).toBe("depends on @scope/pkg"); + }); +}); + +describe("sanitizeEnvRefs — whole-corpus red-green anchor (§8)", () => { + const UUID = "e654541f-dcb7-4152-8ee8-f669848555ee"; + + it("a real memory candidate body has no E1-E6 leak after sanitizing", () => { + const body = `Built as memory:reference_1password_cli.md (session ${UUID}) at /Users/x/proj/cpk/pathfinder/src/atlas/adapters/memory.ts:341. Ping #engr and see https://www.notion.so/copilotkit/Notes-3963aa38185281db80b8e4bf73de0ea5 or ask alice@copilotkit.ai on pathfinder-web.up.railway.app.`; + const out = clean(body); + + // E1: no machine-local path prefixes. + expect(out).not.toMatch(/\/Users\//); + expect(out).not.toMatch(/\/home\//); + expect(out).not.toMatch(/\/proj\//); + // E2: no session UUID. + expect(out).not.toContain(UUID); + // E3: no notion page id. + expect(out).not.toContain("3963aa38185281db80b8e4bf73de0ea5"); + // E4: no internal channel. + expect(out).not.toMatch(/#(?:engr|oss-alerts|deploys)\b/); + // E5: no railway host. + expect(out).not.toMatch(/up\.railway\.app/); + // E6: no internal email. + expect(out).not.toMatch(/@copilotkit\.ai\b/); + + // But the product-portable repo-relative path survives. + expect(out).toContain("src/atlas/adapters/memory.ts"); + }); + + it("is idempotent on the whole-corpus body", () => { + const body = `memory:reference_1password_cli.md (session ${UUID}) at /Users/x/proj/cpk/pathfinder/src/atlas/adapters/memory.ts ping #engr https://www.notion.so/copilotkit/N-3963aa38185281db80b8e4bf73de0ea5 alice@copilotkit.ai pathfinder-web.up.railway.app`; + expect(clean(clean(body))).toBe(clean(body)); + }); +}); + +describe("sanitizeEnvRefs — crfix7 F1: genuine path abutting an emitted placeholder", () => { + // THE LEAK (crfix7 F1, regression from the round-5 `>`-exclusion): E3/E5 run + // BEFORE E1 and emit placeholders (``, ``) + // that end in `>`. When a GENUINE machine-local absolute path immediately + // follows such a placeholder's `>` (because the original URL had the path + // appended to it), E1's `>`-forbidding LEADING_BOUNDARY SKIPPED that path — so + // the username-bearing prefix leaked verbatim. The fix removes `>` from the + // forbidden set; the boundary stays a ZERO-WIDTH lookbehind so the `>` is never + // consumed (placeholder intact, no `/` deleted, still idempotent). + + it("sanitizes a real /Users path appended to a collapsed notion link — no username leak (F1)", () => { + // notion.so/<32hex>/Users/jpr5/x.ts → E3 → `/Users/jpr5/x.ts` + // → E1 must sanitize the appended machine-local path (previously skipped). + const hex = "3963aa38185281db80b8e4bf73de0ea5"; + const out = clean(`notion.so/${hex}/Users/jpr5/x.ts`); + expect(out).not.toContain("jpr5"); + expect(out).not.toContain("/Users/"); + expect(out).toContain(""); + // /Users/jpr5/x.ts has a username at seg 2, no repo top-level → . + expect(out).toBe(""); + }); + + it("is idempotent for a path appended to a collapsed notion link (F1)", () => { + const hex = "3963aa38185281db80b8e4bf73de0ea5"; + const x = `notion.so/${hex}/Users/jpr5/x.ts`; + expect(clean(clean(x))).toBe(clean(x)); + }); + + it("sanitizes a real /Users path appended to a collapsed railway host — no username leak (F1)", () => { + // myapp.up.railway.app/Users/jpr5/x.ts → E5 → `/Users/jpr5/x.ts` + // → E1 must sanitize the appended machine-local path. + const out = clean("myapp.up.railway.app/Users/jpr5/x.ts"); + expect(out).not.toContain("jpr5"); + expect(out).not.toContain("/Users/"); + expect(out).toContain(""); + expect(out).toBe(""); + }); + + it("is idempotent for a path appended to a collapsed railway host (F1)", () => { + const x = "myapp.up.railway.app/Users/jpr5/x.ts"; + expect(clean(clean(x))).toBe(clean(x)); + }); +}); + +describe("sanitizeEnvRefs — crfix7 .com fold (NOTION_URL scheme arm)", () => { + // FOLD-IN: NOTION_URL hardcoded `notion\.so/` while NOTION_CONTEXT_ID accepts + // both `.so` and `.com`. So a schemed `https://notion.com/` bypassed the + // scheme arm; NOTION_CONTEXT_ID then matched only `notion.com/` and left a + // dangling `https://` in front of the placeholder. NOTION_URL now accepts + // `notion.com` too so the whole schemed URL (scheme included) collapses. + it("collapses a schemed https://notion.com/ whole with no dangling https://", () => { + const hex = "3963aa38185281db80b8e4bf73de0ea5"; + const out = clean(`open https://notion.com/${hex} now`); + expect(out).toBe("open now"); + expect(out).not.toContain("https://"); + expect(out).not.toContain("notion.com"); + }); +}); diff --git a/src/__tests__/atlas-search-endpoint.test.ts b/src/__tests__/atlas-search-endpoint.test.ts index bb8d555..5e732d9 100644 --- a/src/__tests__/atlas-search-endpoint.test.ts +++ b/src/__tests__/atlas-search-endpoint.test.ts @@ -790,6 +790,11 @@ describe("atlas harvest E2E — full pipeline against the live /api/search route // path, not the why-vs-what gate (covered in atlas-distillation-gate.test.ts), // so avoid constructing a real OpenAIDistiller (which needs an API key). judge: { judge: async () => ({ kind: "distilled" as const }) }, + // Pass-through audience judge (same rationale as `judge` above): this E2E + // asserts the rag-dedup/upsert path, not the audience gate, so avoid a real + // OpenAIDistiller and make no LLM call — always "relevant" so the gate is a + // no-op and existing expectations are unchanged. + audienceJudge: { judge: async () => ({ kind: "relevant" as const }) }, validationContext, }).finally(() => { // Capture-then-restore even if the run throws — mockRestore CLEARS diff --git a/src/__tests__/atlas-types.test.ts b/src/__tests__/atlas-types.test.ts index ea4ee74..3dc54d8 100644 --- a/src/__tests__/atlas-types.test.ts +++ b/src/__tests__/atlas-types.test.ts @@ -7,6 +7,10 @@ import { KnowledgeType, BEHAVIOR_KNOWLEDGE_TYPES, RESTATEMENT_MARKER, + RAG_NO_DELTA_MARKER, + INTERNAL_OPS_MARKER, + APPROVABLE_FLOOR_MARKERS, + isApprovableFloored, buildCanonicalKey, parseCanonicalKey, mostRestrictiveSensitivity, @@ -814,6 +818,126 @@ describe("RESTATEMENT_MARKER (O2 shared literal)", () => { }); }); +// ── S0 (FOUNDATION): shared approvable-floor marker set + predicate ──────────── +// The audience gate's new INTERNAL_OPS_MARKER and the two existing floor markers +// (RESTATEMENT_MARKER, RAG_NO_DELTA_MARKER) must be read from ONE place so +// validate's approvable-floor and canonicalize's rank-floor cannot drift. This +// slot owns the shared set + predicate both consumers will import. +describe("isApprovableFloored / APPROVABLE_FLOOR_MARKERS (S0 shared floor primitive)", () => { + // Build a minimal floor-marker carrier: a candidate whose validated_against + // carries `marker` as a whole "; "-delimited token (the idiom the upstream + // gates use), everything else clean. + function candidateWithMarker( + marker: string, + ): Pick { + return { + provenance: { + source: "x", + validated_against: `some prior context; ${marker}; and more`, + classification: { + sensitivity: "internal", + knowledge_type: "architecture", + audience: "all-staff", + validation_status: "source-verified", + confidence: "high", + provenance_class: "derived", + freshness: { as_of: "2026-06-08", re_verify_by: "2026-09-08" }, + }, + } as Candidate["provenance"], + evidence: [], + }; + } + + it("exports the audience internal-ops marker literal", () => { + expect(INTERNAL_OPS_MARKER).toBe("audience:internal-ops"); + }); + + it("enumerates exactly the three approvable-floor markers", () => { + expect([...APPROVABLE_FLOOR_MARKERS].sort()).toEqual( + [RESTATEMENT_MARKER, RAG_NO_DELTA_MARKER, INTERNAL_OPS_MARKER].sort(), + ); + }); + + // RED anchor: the NEW audience marker must floor. + it("floors a candidate carrying INTERNAL_OPS_MARKER in validated_against", () => { + expect(isApprovableFloored(candidateWithMarker(INTERNAL_OPS_MARKER))).toBe( + true, + ); + }); + + // No regression on the two pre-existing markers. + it("still floors on RESTATEMENT_MARKER and RAG_NO_DELTA_MARKER", () => { + expect(isApprovableFloored(candidateWithMarker(RESTATEMENT_MARKER))).toBe( + true, + ); + expect(isApprovableFloored(candidateWithMarker(RAG_NO_DELTA_MARKER))).toBe( + true, + ); + }); + + // Also reads the fused_from evidence-ref carrier idiom. + it("floors on a fused_from evidence ref equal to a floor marker", () => { + const c: Pick = { + provenance: { + source: "x", + classification: { + sensitivity: "internal", + knowledge_type: "architecture", + audience: "all-staff", + validation_status: "source-verified", + confidence: "high", + provenance_class: "derived", + freshness: { as_of: "2026-06-08", re_verify_by: "2026-09-08" }, + }, + } as Candidate["provenance"], + evidence: [{ kind: "fused_from", ref: INTERNAL_OPS_MARKER }], + }; + expect(isApprovableFloored(c)).toBe(true); + }); + + it("does NOT floor a clean candidate (no marker on either carrier)", () => { + const clean: Pick = { + provenance: { + source: "x", + validated_against: "some grep target; another ref", + classification: { + sensitivity: "internal", + knowledge_type: "architecture", + audience: "all-staff", + validation_status: "source-verified", + confidence: "high", + provenance_class: "derived", + freshness: { as_of: "2026-06-08", re_verify_by: "2026-09-08" }, + }, + } as Candidate["provenance"], + evidence: [{ kind: "fused_from", ref: "github-issue:agui-adk:1732" }], + }; + expect(isApprovableFloored(clean)).toBe(false); + }); + + // Whole-token match, never substring: a marker that is a PREFIX of a longer + // token must NOT floor (mirrors the upstream hasFloorMarker semantics). + it("does NOT floor on a substring/prefix-only match (whole-token only)", () => { + const c: Pick = { + provenance: { + source: "x", + validated_against: `${INTERNAL_OPS_MARKER}-but-longer`, + classification: { + sensitivity: "internal", + knowledge_type: "architecture", + audience: "all-staff", + validation_status: "source-verified", + confidence: "high", + provenance_class: "derived", + freshness: { as_of: "2026-06-08", re_verify_by: "2026-09-08" }, + }, + } as Candidate["provenance"], + evidence: [], + }; + expect(isApprovableFloored(c)).toBe(false); + }); +}); + // ── S1 (Theme C.1): approvable is a persisted field on the storage input, and // toSeedEntryRow threads it through from the Candidate ────────────────────── describe("toSeedEntryRow persists the C.1 approvable field", () => { diff --git a/src/__tests__/atlas-validate.test.ts b/src/__tests__/atlas-validate.test.ts index 074bef5..14025a6 100644 --- a/src/__tests__/atlas-validate.test.ts +++ b/src/__tests__/atlas-validate.test.ts @@ -32,6 +32,7 @@ import type { ValidationContext } from "../atlas/validate.js"; import { recomputeRankScore } from "../atlas/canonicalize.js"; import { CandidateSchema, + INTERNAL_OPS_MARKER, RAG_NO_DELTA_MARKER, RESTATEMENT_MARKER, } from "../atlas/types.js"; @@ -1198,6 +1199,111 @@ describe("promoteValidation — composes the rag-dedup no-delta floor (dedicated }); }); +describe("promoteValidation — composes the audience-gate internal-ops floor (dedicated marker)", () => { + // The audience gate stamps INTERNAL_OPS_MARKER on a candidate it scoped to + // internal-ops (not shippable to the public corpus audience) and floors it to + // `approvable=false`. It follows the SAME dual carrier idiom the other floor + // markers use — provenance.validated_against and/or a `fused_from` evidence + // ref. validate must READ this marker via the S0 shared floor predicate so an + // internal-ops candidate whose symbols grep-verify stays approvable=false — + // the marker is a floor the source-verify recompute cannot lift. + + function withValidatedAgainst(c: Candidate, marker: string): Candidate { + return { + ...c, + provenance: { ...c.provenance, validated_against: marker }, + }; + } + + it("an internal-ops candidate (validated_against marker) with grep-verifiable symbols is approvable=false", async () => { + // `TwoLayerShim` IS declared in the fixture checkout → source-verifies → an + // architecture fact would normally recompute to approvable=true. The + // internal-ops floor marker must survive the recompute. + const base = makeCandidate({ + knowledge_type: "architecture", + validationTargets: ["TwoLayerShim"], + }); + const candidate = withValidatedAgainst(base, INTERNAL_OPS_MARKER); + const out = await promoteValidation(candidate, ctx); + // Status still promotes (display truth — the symbol really exists). + expect(out.provenance.classification.validation_status).toBe( + "source-verified", + ); + // The upstream floor survives — NOT clobbered back to true. + expect(out.approvable).toBe(false); + }); + + it("the internal-ops floor is recognized via the fused_from evidence ref alone", async () => { + const base = makeCandidate({ + knowledge_type: "architecture", + validationTargets: ["TwoLayerShim"], + }); + const candidate: Candidate = { + ...base, + evidence: [{ kind: "fused_from", ref: INTERNAL_OPS_MARKER }], + }; + const out = await promoteValidation(candidate, ctx); + expect(out.approvable).toBe(false); + }); + + it("the internal-ops marker is recognized among other '; '-joined validated_against tokens", async () => { + const base = makeCandidate({ + knowledge_type: "product", + validationTargets: ["TwoLayerShim"], + }); + const candidate = withValidatedAgainst( + base, + `rag-corpus-overlap:some-ref; ${INTERNAL_OPS_MARKER}`, + ); + const out = await promoteValidation(candidate, ctx); + expect(out.approvable).toBe(false); + }); + + it("an internal-ops candidate mapping to a GREEN pill (showcase-verifies) still stays approvable=false", async () => { + const base = makeCandidate({ + knowledge_type: "product", + title: "agentic-chat", + validationTargets: ["agentic-chat"], + }); + const candidate = withValidatedAgainst(base, INTERNAL_OPS_MARKER); + const out = await promoteValidation(candidate, ctx); + expect(out.provenance.classification.validation_status).toBe( + "showcase-verified", + ); + expect(out.approvable).toBe(false); + }); + + it("REGRESSION: the existing RESTATEMENT / RAG_NO_DELTA floors are unchanged", async () => { + // The shared predicate ORs all three markers; the two pre-existing floors + // must still fire exactly as before. + const restatement = withValidatedAgainst( + makeCandidate({ + knowledge_type: "architecture", + validationTargets: ["TwoLayerShim"], + }), + RESTATEMENT_MARKER, + ); + const noDelta = withValidatedAgainst( + makeCandidate({ + knowledge_type: "architecture", + validationTargets: ["TwoLayerShim"], + }), + RAG_NO_DELTA_MARKER, + ); + expect((await promoteValidation(restatement, ctx)).approvable).toBe(false); + expect((await promoteValidation(noDelta, ctx)).approvable).toBe(false); + }); + + it("CONTROL: a candidate with NO floor marker whose symbols grep-verify stays approvable=true", async () => { + const candidate = makeCandidate({ + knowledge_type: "architecture", + validationTargets: ["TwoLayerShim"], + }); + const out = await promoteValidation(candidate, ctx); + expect(out.approvable).toBe(true); + }); +}); + describe("promoteValidation — a restatement floor also floors the rank contribution (A.2)", () => { // The dominant rank factor (canonicalize's VALIDATION_WEIGHT) is derived from // validation_status. A RESTATEMENT-floored candidate (approvable=false) whose diff --git a/src/atlas/adapters/episodic.ts b/src/atlas/adapters/episodic.ts index f1d11fb..88c333b 100644 --- a/src/atlas/adapters/episodic.ts +++ b/src/atlas/adapters/episodic.ts @@ -29,6 +29,7 @@ import { mostRestrictiveSensitivity, Sensitivity, } from "../types.js"; +import { sanitizeEnvRefs } from "./sanitize-env-refs.js"; import type { AdapterContext, LeafAdapter } from "./types.js"; // ── Input unit ──────────────────────────────────────────────────────────────── @@ -167,6 +168,16 @@ export const episodicAdapter: LeafAdapter = { }, }; + // §3.3: sanitize the emitted content (and provenance.source) through the + // shared env-reference pass immediately before returning the fragment. The + // distiller reads raw transcript text — a machine-local path, session UUID, + // or private ref can survive into the distilled claim/source, so strip it + // here at fragment-production time before the contract parse. + const { content: sanitizedContent, source: sanitizedSource } = + sanitizeEnvRefs(fragment.content, fragment.provenance.source); + fragment.content = sanitizedContent; + fragment.provenance.source = sanitizedSource; + // Fail loud if the distillation+stamping did not yield a contract-valid // fragment (a bad LLM result in a knowledge-harvest is a defect to surface). return [CandidateFragmentSchema.parse(fragment)]; diff --git a/src/atlas/adapters/github.ts b/src/atlas/adapters/github.ts index 8d510fe..535d8e8 100644 --- a/src/atlas/adapters/github.ts +++ b/src/atlas/adapters/github.ts @@ -26,6 +26,7 @@ // LLM-backed adapter). import type { AdapterContext, LeafAdapter } from "./types.js"; +import { sanitizeEnvRefs } from "./sanitize-env-refs.js"; import { scanSensitivity } from "./sensitivity-scan.js"; import { extractValidationTargets } from "./validation-targets.js"; import type { CandidateFragment, Provenance, Sensitivity } from "../types.js"; @@ -202,6 +203,35 @@ export function distillBodyToContent(body: string | null | undefined): string { // dropped→dropped transition (a boilerplate-named heading inside the // section's still-open fence) it must be preserved, for the same reason. let inDroppedFence = false; + // When the outside-section blank-line recovery below ends an OPEN fence early + // (treating it as possibly unterminated), the parser is deliberately forced + // back OUTSIDE any fence so heading-parsing — and boilerplate stripping — + // resumes. But the line-based scan cannot know whether the fence markers that + // FOLLOW the recovery point are (a) that same fence's real trailing closer + // (a legitimate block with an internal blank line), or (b) the opener+closer + // of a later INDEPENDENT fenced block, or (c) nothing at all (truly + // unterminated). A bare ``` is byte-identical in all three, so it is not + // disambiguable line-by-line — and an earlier attempt (p3-fix-1) that tried + // to absorb only the "next" closer inverted parity in case (b): the new + // block's OPENER got absorbed (no `inFence = true`), so the block's real + // closer toggled `inFence` true while the parser was outside, and a trailing + // `## Test plan` was mis-kept. + // + // The principled resolution: the RELEVANT downstream consumer of `inFence` is + // ONLY the heading short-circuit (fence content is kept as literal prose + // whether or not `inFence` is set, because the outside-fence branch keeps + // every non-heading line too). Across all three cases the parser ends OUTSIDE + // a fence once the trailing markers are consumed, and every boilerplate + // heading of interest appears AFTER those markers. So once a recovery fires, + // we STAY logically outside for the rest of the body: every subsequent + // outside-fence ``` is kept as literal content and NEVER re-toggles parity. + // This satisfies all three cases uniformly (heading-parsing/stripping stays + // live to EOF, parity never inverts) at the cost of the same documented, + // accepted heuristic tradeoff — a `#`-shaped line inside a genuine post- + // recovery fenced block may parse as a heading. That over-keep/over-drop is + // strictly better than the parity inversion it replaces. The flag is sticky: + // it arms on recovery and stays armed to EOF. + let recoveredOutsideFence = false; for (const line of lines) { // Fenced code blocks are literal content: a `# …` line inside a fence is // (e.g.) a shell comment, not a markdown heading, and must neither toggle @@ -215,6 +245,21 @@ export function distillBodyToContent(body: string | null | undefined): string { inDroppedFence = !inDroppedFence; continue; } + // After an outside-section blank-line recovery has fired, stay logically + // outside a fence for the rest of the body: keep every subsequent + // outside-fence ``` as literal content WITHOUT toggling parity, so a + // later boilerplate heading is still parsed and stripped and parity can + // never invert (see `recoveredOutsideFence`). This uniformly covers a + // real block's trailing closer, an independent block's opener+closer, and + // a truly unterminated fence. It only applies while genuinely outside a + // fence — if a recovery has NOT fired, `recoveredOutsideFence` is false + // and the normal toggle below runs. + if (recoveredOutsideFence && !inFence) { + kept.push(line); + continue; + } + // A genuine fence open/close toggle (no recovery has fired since the last + // toggle balanced out). inFence = !inFence; kept.push(line); continue; @@ -234,7 +279,15 @@ export function distillBodyToContent(body: string | null | undefined): string { // the better failure than silently disabling it. This does not touch the // dropped-section fence (tracked in `inDroppedFence`) — the s4 case where // a fence INSIDE a dropped section drops its content is unaffected. - if (line.trim() === "") inFence = false; + if (line.trim() === "") { + inFence = false; + // Arm the sticky recovery: for the rest of the body, every outside- + // fence ``` is kept as literal content without re-toggling parity, so + // heading-parsing/stripping stays live and parity can never invert — + // whether this fence later closes, an independent block follows, or the + // fence is truly unterminated. See the flag's declaration. + recoveredOutsideFence = true; + } if (!droppingSection) kept.push(line); continue; } @@ -257,6 +310,16 @@ export function distillBodyToContent(body: string | null | undefined): string { if (inDroppedFence && line.trim() === "") { inDroppedFence = false; droppingSection = false; + // Arm the sticky recovery, symmetric with the outside-section branch + // (L282-290): a fence that OPENED inside this dropped section and is + // being treated as unterminated here means the parser is now logically + // OUTSIDE any fence. Left un-armed, a later ``` marker toggles `inFence` + // true while outside — inverting parity for the rest of the body, which + // over-keeps a subsequent boilerplate heading and can silently drop real + // trailing why/how prose to EOF. Arming keeps every post-recovery ``` as + // literal content without re-toggling parity, so heading-parsing and + // stripping stay live to EOF. See the flag's declaration. + recoveredOutsideFence = true; kept.push(line); continue; } @@ -489,6 +552,21 @@ function extractPullRequest( bareCredentialMentions: true, }); + // §3.3: sanitize the emitted content (and provenance.source) through the + // shared env-reference pass immediately before returning the fragment, so a + // machine-local path / session UUID / private ref in the distilled PR body + // is rewritten to its repo-relative tail / placeholder before it enters the + // pipeline. + const provenance = firstPassProvenance( + pr.htmlUrl, + pr.mergeCommitSha ?? null, + ctx.now, + sensitivity, + ); + const { content: sanitizedContent, source: sanitizedSource } = + sanitizeEnvRefs(content, provenance.source); + provenance.source = sanitizedSource; + return { sourcetype: "github-pr", // TRIMMED: the intake guard only trims fullName for its check; the @@ -506,13 +584,8 @@ function extractPullRequest( // checkouts). ref: baseBranch ?? unit.repo.defaultBranch, title: titleOrFallback(pr.title, `PR #${pr.number}`), - content, - provenance: firstPassProvenance( - pr.htmlUrl, - pr.mergeCommitSha ?? null, - ctx.now, - sensitivity, - ), + content: sanitizedContent, + provenance, // A.3: the WHAT-metadata header lifted off `content` is RETAINED here as a // provenance evidence entry — the facts (Repository/branch/commit/author/ // URL) are relocated, not dropped. Prepended so it reads as the fragment's @@ -575,6 +648,18 @@ function extractIssue( [issue.title, content, ...(unit.reviewThreads ?? [])].join("\n"), ); + // §3.3: sanitize the emitted content (and provenance.source) before return — + // same rationale as the PR path above. + const provenance = firstPassProvenance( + issue.htmlUrl, + null, + ctx.now, + sensitivity, + ); + const { content: sanitizedContent, source: sanitizedSource } = + sanitizeEnvRefs(content, provenance.source); + provenance.source = sanitizedSource; + return { sourcetype: "github-issue", // TRIMMED for the same canonical-key reason as the PR path; the shared @@ -584,8 +669,8 @@ function extractIssue( repo_url: unit.repo.cloneUrl, ref: unit.repo.defaultBranch, title: titleOrFallback(issue.title, `Issue #${issue.number}`), - content, - provenance: firstPassProvenance(issue.htmlUrl, null, ctx.now, sensitivity), + content: sanitizedContent, + provenance, // A.3: WHAT-metadata header retained as a provenance evidence entry (see // extractPullRequest). evidence: [ diff --git a/src/atlas/adapters/linear.ts b/src/atlas/adapters/linear.ts index a7aa53d..2fda885 100644 --- a/src/atlas/adapters/linear.ts +++ b/src/atlas/adapters/linear.ts @@ -19,6 +19,7 @@ // never mutates the input, takes no build-time dependency on the LLM seam. import type { CandidateFragment } from "../types.js"; +import { sanitizeEnvRefs } from "./sanitize-env-refs.js"; import { scanSensitivity } from "./sensitivity-scan.js"; import type { AdapterContext, LeafAdapter } from "./types.js"; @@ -180,6 +181,13 @@ export const linearAdapter: LeafAdapter = { }); } + // §3.3: sanitize the emitted content (and provenance.source) through the + // shared env-reference pass immediately before returning the fragment, so a + // machine-local path / session UUID / private ref in the distilled + // Problem/Why/Non-Goals prose is rewritten before it enters the pipeline. + const { content: sanitizedContent, source: sanitizedSource } = + sanitizeEnvRefs(content, "linear-doc"); + const fragment: CandidateFragment = { sourcetype: "linear-doc", subsystem, @@ -187,9 +195,9 @@ export const linearAdapter: LeafAdapter = { repo_url: undefined, ref: undefined, title: titleOrFallback(unit.title, `Linear doc ${unit.url}`), - content, + content: sanitizedContent, provenance: { - source: "linear-doc", + source: sanitizedSource, url: unit.url, date, validated_against: validatedAgainst, diff --git a/src/atlas/adapters/memory.ts b/src/atlas/adapters/memory.ts index b1cade9..dbb33bc 100644 --- a/src/atlas/adapters/memory.ts +++ b/src/atlas/adapters/memory.ts @@ -25,6 +25,7 @@ import type { KnowledgeType, Provenance, } from "../types.js"; +import { sanitizeEnvRefs } from "./sanitize-env-refs.js"; import { scanSensitivity } from "./sensitivity-scan.js"; import { extractValidationTargets } from "./validation-targets.js"; import type { AdapterContext, LeafAdapter } from "./types.js"; @@ -290,7 +291,6 @@ export const memoryAdapter: LeafAdapter = { const name = asString(frontmatter.name); const description = asString(frontmatter.description); - const originSessionId = asString(frontmatter.originSessionId); // KEEP/DROP gate. reference_/project_ always KEEP. feedback_ KEEPs only // operational/infra/codebase why-how. Unknown/absent prefix → DROP (the @@ -335,11 +335,27 @@ export const memoryAdapter: LeafAdapter = { // is SAFE and stays internal. const sensitivity = scanSensitivity(name, description, body); + // The provenance source is `memory:` with NO session suffix + // (spec §3.1-E2 / §3.3): the authoring session UUID is machine-local glue + // that resolves only for internal staff, so it must never enter the + // external corpus. `provenance.source` is externally persisted via + // toSeedEntryRow (spec §3.1), so it is leak-surface — the basename form + // keeps the traceable file identity without the private session id. The + // basename is the last path segment (extension retained), so a + // directory-qualified filename never leaks its machine-local prefix. + const provenanceBasename = unit.filename.split("/").pop() ?? unit.filename; + + // Sanitize the emitted content and provenance.source through the shared + // env-reference pass BEFORE returning the fragment (spec §3.3): a + // machine-local path or private ref embedded in a memory note's body/source + // is rewritten to its repo-relative tail / placeholder here, at + // fragment-production time. + const { content: sanitizedContent, source: sanitizedSource } = + sanitizeEnvRefs(content, `memory:${provenanceBasename}`); + const provenance: Provenance = { - // The session that authored the memory note is the primary source. - source: originSessionId - ? `memory:${unit.filename} (session ${originSessionId})` - : `memory:${unit.filename}`, + // The memory note is the primary source; identified by its basename. + source: sanitizedSource, date: isoDate(ctx.now), // description is the distilled human-written summary of the fact — the // single free-text provenance slot carries it forward for the reviewer. @@ -363,7 +379,7 @@ export const memoryAdapter: LeafAdapter = { source_name: unit.filename, // name is the already-distilled claim title — NOT the raw filename. title: name || slug, - content, + content: sanitizedContent, provenance, evidence: [], needsReview: false, diff --git a/src/atlas/adapters/notion.ts b/src/atlas/adapters/notion.ts index 0223bdd..a77bf9e 100644 --- a/src/atlas/adapters/notion.ts +++ b/src/atlas/adapters/notion.ts @@ -33,6 +33,7 @@ // nothing keeps an empty target list and stays human-gated (prose-aware). import type { AdapterContext, LeafAdapter } from "./types.js"; +import { sanitizeEnvRefs } from "./sanitize-env-refs.js"; import { scanSensitivity } from "./sensitivity-scan.js"; import { extractValidationTargets } from "./validation-targets.js"; import { mostRestrictiveSensitivity } from "../types.js"; @@ -328,6 +329,14 @@ function buildFragment( const provenance_class: ProvenanceClass = "primary"; const confidence: Confidence = "high"; + // §3.3: sanitize the emitted content (and provenance.source) through the + // shared env-reference pass immediately before returning the fragment. The + // E3 Notion arm rewrites any private notion.so page link in the decision body + // to `` in `content`, while `provenance.url` (the page URL) + // is DELIBERATELY retained as harvest attribution (spec §3.1-E3). + const { content: sanitizedContent, source: sanitizedSource } = + sanitizeEnvRefs(section.body.trim(), SOURCE_NAME); + return { sourcetype: "notion-doc", subsystem, @@ -335,9 +344,9 @@ function buildFragment( ...(unit.repo_url ? { repo_url: unit.repo_url } : {}), ...(unit.ref ? { ref: unit.ref } : {}), title, - content: section.body.trim(), + content: sanitizedContent, provenance: { - source: SOURCE_NAME, + source: sanitizedSource, url: unit.url, date: asOf, classification: { diff --git a/src/atlas/adapters/sanitize-env-refs.ts b/src/atlas/adapters/sanitize-env-refs.ts new file mode 100644 index 0000000..18bf45b --- /dev/null +++ b/src/atlas/adapters/sanitize-env-refs.ts @@ -0,0 +1,404 @@ +// Deterministic environment-reference sanitizer (spec §3.1, categories E1-E6). +// +// A pure, regex-only pass with NO I/O and NO LLM. Each adapter that emits +// externally-visible content calls this on the fragment's `content` (and +// `provenance.source` where applicable) before returning from extract(), so +// machine-local / session / private-infra glue never enters the external +// corpus. It is a fragment-production concern, not a pipeline stage — modeled +// structurally on sensitivity-scan.ts (module doctrine: cheap deterministic +// first, hardest safety guarantee). +// +// The discriminator (spec §3.2) is KEEP the product-portable specific, STRIP +// the environment glue: a machine-local prefix is glue, but the repo-relative +// tail (`src/atlas/x.ts`) is a portable specific an external builder can +// correlate with public docs, so E1 rewrites in place rather than dropping. +// +// IDEMPOTENCE is a hard contract: sanitizeEnvRefs(sanitizeEnvRefs(x)) must +// equal sanitizeEnvRefs(x) for every input. Each remediation replaces its +// match with a literal placeholder / a repo-relative tail that no longer +// matches the same pattern, so a second pass is a no-op. + +// Repo top-level segments we recognize when inferring a repo-relative tail from +// a machine-local absolute path (E1). Enumerated allowlist per spec §3.1 — kept +// deliberately small; extend by editing this list, not by loosening the match. +const REPO_TOP_LEVEL = ["src", "bin", "tests", "scripts", "deploy"]; + +// A path token runs from its leading boundary until the FIRST of: whitespace, +// quote, paren/bracket/brace/angle, backtick, or end-of-string — NOT the next +// slash (spec §3.1 E1; the F7 truncation guard depends on consuming the WHOLE +// path). It matches a machine-local absolute path (`/…` — the named +// /Users|home|root and /proj roots plus the general `/a/src/…` nested case the +// §8 anchor pins) or a `~/…` home path. +// +// LEADING BOUNDARY — a PRINCIPLED zero-width negative lookbehind, NOT a consuming +// allowlist (crfix6 systemic fix). The prior anchor was an ALLOWLIST char class +// `(^|[\s(`'"[\]{}])` that enumerated the chars permitted to precede a path. That +// allowlist OMITTED common path-preceding separators — `=`, `:`, `,`, `;` — so a +// machine-local path written as `CONFIG=/Users/jpr5/…`, `cwd=/Users/…`, +// `path: /Users/…`, or in a comma/semicolon list was NOT anchored at all and +// leaked its full username-bearing prefix UNSANITIZED (the exact leak this module +// exists to prevent). Enumerating separators is a losing game: every new +// separator (`|`, `&`, tab-vs-space, …) reopens the hole. +// +// So invert the polarity: instead of allowlisting the FEW chars that may precede a +// path, FORBID only the chars that must NOT — and let everything else (any present +// or FUTURE separator) legitimately precede a detected `/`- or `~/`-rooted path. +// `LEADING_BOUNDARY` forbids exactly ONE class, zero-width: PATH-/WORD-CONTINUATION +// chars `[A-Za-z0-9._~/-]` — a `/` preceded by one of these is MID-token (inside an +// already-formed path/word like `pkg/x` or `a/src/b`), not a fresh path anchor, so +// we must not re-anchor there. +// +// PLACEHOLDER TERMINATOR `>` IS DELIBERATELY NOT FORBIDDEN (crfix7 F1): the round-5 +// fix added `>` to this forbidden set so a `<…>` placeholder's trailing `>` could +// not be an E1 anchor. But that was the WRONG remedy and it OPENED A LEAK. E3/E5 run +// BEFORE E1 and emit placeholders (``, ``) that +// end in `>`. A real machine-local path appended to such a placeholder — e.g. the +// original `notion.so//Users/jpr5/x.ts` becomes `/Users/jpr5/x.ts` +// after E3 — was then preceded by `>`, so E1 SKIPPED it and leaked the username +// verbatim. Because this boundary is a ZERO-WIDTH lookbehind, it never consumes the +// `>`; allowing `>` to precede an anchor re-anchors E1 AFTER the placeholder without +// deleting the `>` — the placeholder stays intact, the appended path is sanitized, +// and idempotence holds (the emitted /placeholders contain no +// re-anchorable `/`-rooted path). The round-5 "delete the `/`" concern only applied +// to the round-4 CONSUMING allowlist; a zero-width lookbehind cannot delete anything. +// +// Being zero-width, the boundary neither consumes nor preserves a preceding char — +// there is no `lead` capture to re-emit — and it succeeds at start-of-string, so a +// path at position 0 still matches. A `/` inside a URL scheme (`https://…`) is +// preceded by `s` (a word char), so it is correctly rejected; Notion/Railway URLs +// are handled by E3/E5. +const LEADING_BOUNDARY = "(?` (``, +// ``) IS anchored and sanitized — the zero-width LEADING_BOUNDARY +// no longer forbids `>`, so the appended path is caught (no username leak) while the +// `>` itself is never consumed (placeholder stays intact). Angle brackets REMAIN in +// the trailing terminator char classes so a path still STOPS at a `<` and never +// absorbs a following `<…>` placeholder. +// +// Two rooted forms are recognized (CR round-3 finding 3): +// - a `/`-rooted absolute path MUST contain at least one interior `/` +// (a bare `/x` word is not a path) so ordinary prose slashes don't match; +// - a `~/`-rooted home path is machine-local by construction, so a home-root +// file with NO interior slash (`~/secret.env`, `~/.npmrc`) still matches and +// collapses to — the comment previously advertised `~/…` +// handling that the shared interior-slash requirement actually missed. +// Both forms consume the WHOLE path to end-of-token (the F7 truncation guard), +// and both share the same trailing-char rule below. +// +// The path may NOT end on a trailing prose-punctuation char (,.;:!?) — such a +// char is sentence punctuation that follows the path in prose, not part of the +// path tail, and absorbing it corrupts the content (CR finding 2). Interior +// punctuation is preserved: a filename period (`x.ts`) and a line:col suffix +// (`x.ts:12`) both survive because they are followed by more path chars. This +// is enforced by requiring the final matched char to be a non-terminal char +// class that excludes trailing punctuation. +const MACHINE_LOCAL_PATH = new RegExp( + `${LEADING_BOUNDARY}(\\/[^\\s()\`'"[\\]{}<>:/]+(?:\\/[^\\s()\`'"[\\]{}<>]*[^\\s()\`'"[\\]{}<>.,;:!?])+|~\\/[^\\s()\`'"[\\]{}<>:/]*(?:\\/[^\\s()\`'"[\\]{}<>]*)*[^\\s()\`'"[\\]{}<>.,;:!?])`, + "g", +); + +// A session UUID (UUIDv4 shape). E2 strips the surrounding "(session )" +// clause when present, else the bare UUID token. The clause form is matched +// first so the parenthetical wrapper is removed cleanly. +const SESSION_CLAUSE = + /\s*\(session\s+[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\)/gi; +const BARE_UUID = + /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi; + +// ── Shared host-boundary fragments (round-5 UNIFORM PASS) ────────────────── +// +// Every host/domain arm (NOTION_URL, NOTION_CONTEXT_ID, RAILWAY_PROJECT, +// RAILWAY_HOST) shares the SAME two boundary properties so no single arm is +// missed one at a time again: +// +// (i) LEFT BOUNDARY — a host may carry optional leading subdomain labels +// (`foo.`, `pr-1.myapp.`) which must be consumed as part of the match so +// they never leak in front of the placeholder; but it must NEVER match +// MID-LABEL (`evilnotion.so`, `xrailway.app`). The zero-width lookbehind +// `HOST_LEFT` forbids a preceding label char (letter/digit/hyphen/dot) so +// the match can only begin at a genuine host-label start, while the +// `LEADING_LABELS` fragment (matched INSIDE the arm) greedily eats any +// real subdomain labels that DO precede the anchor domain. +// +// (ii) CASE-INSENSITIVITY — DNS is case-insensitive and copy-pasted hosts +// carry uppercase (`MyApp.up.railway.app`, `Railway.app`), so every arm +// is compiled with the `i` flag. +// +// `HOST_LEFT` is a NEGATIVE LOOKBEHIND, not a consuming char class: a consuming +// left-anchor would either eat the preceding separator (leaking / corrupting +// it) or fail to reset for the next label. Being zero-width, it neither +// consumes nor leaks, and it correctly rejects `evilnotion.so` (the char before +// `notion` is `l`, a label char) while accepting ` notion.so`, `(notion.so`, +// `/notion.so`, and start-of-string. +const HOST_LEFT = "(?` slug tails and `…-` +// canonical ids), so every E3 arm must accept both or the un-matched form leaks +// its dangling internal prefix (CR findings 3 & 4). +const NOTION_ID = + "(?:[0-9a-f]{32}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"; + +// E3 Notion links. Content-only. A full notion.so URL (with a trailing 32-hex +// id or dashed-uuid, and with ZERO or more intermediate path segments so both +// `notion.so/` and `notion.so/team/Page-` collapse whole — CR finding +// 4), OR a bare id that appears in notion.so / notion.com host context. The +// scheme-less context arm ALSO allows zero-or-more intermediate segments +// (matching the full URL) so a multi-segment `notion.so/team/space/Page-` +// collapses WHOLE rather than leaking the internal page id or a dangling prefix +// (CR round-3 finding 2). A bare `/p/` is handled by a SEPARATE +// marker-gated arm (see NOTION_P_ID / NOTION_MARKER below). A bare 32-hex +// OUTSIDE Notion context (e.g. a commit SHA / blob hash in code) must NOT be +// stripped (spec OQ5) — so there is deliberately no context-free bare-hex arm, +// and the `/p/` arm is likewise gated on a same-string Notion marker. +// Each arm consumes the Notion context anchor along with the id, replacing the +// whole span with the placeholder (which no longer matches — idempotent). +// +// Both host arms consume ANY number of leading subdomain labels +// (`(?:[a-z0-9-]+\.)*`) immediately before `notion.so`/`notion.com`, not just +// `www`: a subdomained host (`foo.notion.so/`, `app.notion.com/`, +// `https://foo.notion.so/…`) must collapse WHOLE, or the leading label leaks in +// front of the placeholder (CR round-4 host hardening — the same multi-label +// leak class flagged on the Railway arm). Both are `gi`, so uppercase labels are +// handled too. +// The schemed-URL arm accepts BOTH `notion.so` and `notion.com` (crfix7 fold): +// NOTION_CONTEXT_ID already accepts both, so a schemed `https://notion.com/` +// that only NOTION_URL could match whole was otherwise bypassed here — the +// scheme-less NOTION_CONTEXT_ID arm then matched only `notion.com/` and left a +// dangling `https://` in front of the placeholder. Accepting `notion.com` here +// collapses the whole schemed URL (scheme included). +const NOTION_URL = new RegExp( + `${HOST_LEFT}https?:\\/\\/${LEADING_LABELS}(?:notion\\.so|notion\\.com)\\/(?:[a-zA-Z0-9-]+\\/)*(?:[a-zA-Z0-9-]*-)?${NOTION_ID}\\b`, + "gi", +); +// LEFT BOUNDARY (round-5 finding 2): without a left boundary this matched +// `notion.so`/`notion.com` MID-LABEL (`evilnotion.so/` => `evil<…>`), +// corrupting a non-Notion host. `HOST_LEFT` rejects a preceding label char so +// only a genuine host-label start (or a real leading subdomain consumed by +// `LEADING_LABELS`) matches. `evilnotion.so` no longer matches at all. +const NOTION_CONTEXT_ID = new RegExp( + `${HOST_LEFT}${LEADING_LABELS}(?:notion\\.so|notion\\.com)\\/(?:p\\/)?(?:[a-zA-Z0-9-]+\\/)*(?:[a-zA-Z0-9-]*-)?${NOTION_ID}\\b`, + "gi", +); +// A bare `/p/` path segment. On its own this is CONTEXT-FREE — it looks +// exactly like a machine-local absolute path (leading `/`, interior `/`) and +// stripping it unconditionally would violate spec OQ5 (no context-free bare-id +// strip) AND corrupt unrelated routes/paths that merely happen to be `/p/`. +// So this arm is applied ONLY when the SAME string carries a Notion context +// marker (see NOTION_MARKER) — i.e. it is a genuine Notion page link, not an +// incidental filesystem/route path. When there is no Notion context anywhere in +// the string, a `/p/` falls through to E1 and collapses to +// (no leaked id, no false Notion label). +const NOTION_P_ID = new RegExp( + `\\/p\\/(?:[a-zA-Z0-9-]*-)?${NOTION_ID}\\b`, + "gi", +); +// Notion context marker: the notion.so / notion.com host, or a standalone +// `notion` word. Its presence in the string authorizes the (otherwise +// context-free) bare `/p/` arm. `notion.so`/`notion.com` are matched with a +// leading domain boundary so `app.notion.com` still qualifies while an unrelated +// substring does not smuggle it in. +const NOTION_MARKER = /\bnotion(?:\.so|\.com)?\b/i; + +// E4 internal Slack channels — CLOSED allowlist only (spec §3.1 E4). An open +// `#[a-z][a-z0-9-]+` pattern over-fires on Markdown headings, hashtags, and +// English words (`#teamwork`, `#teams`, `# Infra`, "see the #infra section"), +// so this matches ONLY the three known private slugs as whole tokens (word +// boundary => `#deploys` does not fire inside `#deployserver`). +const SLACK_CHANNEL = /#(?:engr|oss-alerts|deploys)\b/g; + +// E5 internal Railway hosts. `railway.app/project/` and +// `.up.railway.app`. Content-only. Both arms share the UNIFORM +// host-boundary treatment (round-5): `HOST_LEFT` rejects a mid-label match, +// `LEADING_LABELS` consumes any leading subdomain labels WHOLE, and both are +// case-insensitive (`i`) because DNS is case-insensitive and copy-pasted hosts +// carry uppercase (`MyApp.up.railway.app`, `Railway.app/project/…`). +// +// RAILWAY_PROJECT (round-5 finding 1): round-5 hardened RAILWAY_HOST for +// multi-label leaks but SKIPPED the project arm, so a subdomained +// `pr-1.railway.app/project/xyz` / `foo.railway.app/project/my-id` leaked the +// leading label (`pr-1.`). `LEADING_LABELS` now consumes those +// leading labels, and `HOST_LEFT` blocks a mid-label match (`xrailway.app`). +// +// RAILWAY_HOST: `LEADING_LABELS` is zero-or-more (not `+`), so a label-less +// `up.railway.app` still collapses; a multi-label host +// (`pr-42.myapp.up.railway.app`) collapses WHOLE with no leading-label leak. +const RAILWAY_PROJECT = new RegExp( + `${HOST_LEFT}${LEADING_LABELS}railway\\.app\\/project\\/[0-9a-zA-Z-]+\\b`, + "gi", +); +const RAILWAY_HOST = new RegExp( + `${HOST_LEFT}${LEADING_LABELS}up\\.railway\\.app\\b`, + "gi", +); + +// E6 internal emails — ONLY the CopilotKit email arm (spec §3.1 E6). A bare +// `@handle` arm is deliberately absent: it would wreck `@copilotkit/react-core`, +// `@types/node`, `@Injectable`, `@copilotkit`. +const TEAM_EMAIL = /[a-z0-9._%+-]+@copilotkit\.ai\b/gi; + +// Home/user roots whose immediately-following segment is a machine-local +// username, NOT a repo-internal dir — so an anchor-dir token sitting there +// (`/Users/deploy/…`, `/home/src/…`) is machine-local structure, not a +// repo-relative tail, and must not anchor E1 (CR finding 1). +const HOME_ROOTS = ["Users", "home", "root"]; + +// From a machine-local absolute path, keep the substring starting at the FIRST +// occurrence of a recognized repo top-level segment to end-of-path; if none is +// present, return the literal ``. Splitting on "/" and scanning for +// the first recognized segment makes "first occurrence" deterministic on nested +// cases (`/a/src/b/src/c` => `src/b/src/c`, the first `src/`). +// +// The anchor must be a GENUINE repo-internal segment, never a machine-local +// one. The first anchorable segment depends on the root form: +// - a `/Users`|`/home`|`/root` root has a username at segment 2, so the first +// genuine repo segment is segment 3 (`/Users/deploy/proj/x.ts` must collapse +// to ``, NOT emit `deploy/…` — CR round-1 finding 1); +// - a `~/…` home root is ALREADY under the user's home at segment 1, so its +// first anchorable segment is segment 2 (`~/src/x.ts` is home structure and +// collapses; `~/proj/…/tests/foo.ts` still anchors at the later `tests`); +// - a plain `/…` root (any other top-level dir) can BE a repo anchor at +// segment 1 itself, so a path rooted directly at a top-level dir keeps its +// portable tail (`/src/x.ts` => `src/x.ts`, `/bin/y` => `bin/y` — CR round-3 +// finding 4), instead of over-redacting to ``. +// Segment 0 is always the leading `/`-marker or `~` and is never anchorable. +// The §8 anchor `/a/src/b/src/c.ts` still resolves to `src/b/src/c.ts`: it is a +// plain root (firstAnchorable 1), segment 1 (`a`) is not a recognized top-level +// dir, and the scan lands on the first genuine `src/` at segment 2. +function repoRelativeOrPlaceholder(absPath: string): string { + const segments = absPath.split("/"); + // Determine the first segment that may legitimately anchor a repo-relative + // tail, per the root-form rules above. + let firstAnchorable: number; + if (segments[0] === "~") { + // `~/…`: segment 1 is directly under the user's home (machine-local). + firstAnchorable = 2; + } else if (HOME_ROOTS.includes(segments[1])) { + // `/Users`|`/home`|`/root`: segment 2 is the username (machine-local). + firstAnchorable = 3; + } else { + // Plain `/…` root: segment 1 is itself a candidate repo top-level dir. + firstAnchorable = 1; + } + for (let i = firstAnchorable; i < segments.length; i++) { + if (REPO_TOP_LEVEL.includes(segments[i])) { + return segments.slice(i).join("/"); + } + } + return ""; +} + +// Apply E1 to a single string (used for both `content` and `provenance.source` +// per §3.1 E1). The leading boundary is a zero-width lookbehind (it consumes no +// preceding char), so the whole match IS the path — replace it in place. +function sanitizeMachineLocalPaths(input: string): string { + return input.replace(MACHINE_LOCAL_PATH, (absPath: string) => + repoRelativeOrPlaceholder(absPath), + ); +} + +// Apply E2 to a single string (content + provenance.source). Strip the +// "(session )" clause first, then any remaining bare UUID token. +function sanitizeSessionUuids(input: string): string { + return input.replace(SESSION_CLAUSE, "").replace(BARE_UUID, ""); +} + +/** + * Sanitize environment-specific references out of an externally-visible + * fragment. Pure and deterministic — no I/O, no LLM. Idempotent: + * `sanitizeEnvRefs(sanitizeEnvRefs(x)) === sanitizeEnvRefs(x)`. + * + * @param content the fragment's externally-visible body (E1-E6 applied) + * @param provenanceSource the fragment's provenance.source (E1 + E2 only — + * §3.1: provenance.source is externally persisted via toSeedEntryRow, so it + * gets machine-local-path + session-UUID stripping; E3-E6 are content-only) + */ +export function sanitizeEnvRefs( + content: string, + provenanceSource: string, +): { content: string; source: string } { + // --- content: E1-E6 --- + let out = content; + + // E3 Notion page ids/URLs (content only). Full-URL arm first (most specific), + // then the notion.so/notion.com-context id, then the `/p/` arm. + // + // ORDERING (crfix3): this block runs BEFORE E1 machine-local paths. A genuine + // Notion `/p/` link is a leading-`/` path with an interior `/`, so E1 + // would otherwise consume it as a machine-local path and collapse it to + // before the Notion arm ever saw it (the NOTION_P_ID arm was + // effectively DEAD for its own bare-path input). Running the context-anchored + // Notion arms first lets a real Notion link collapse to . + // + // The `/p/` arm is CONTEXT-ANCHORED (crfix3 / spec OQ5): it fires only when + // the string also carries a Notion marker, so a truly context-free `/p/` + // (no Notion anywhere) is NOT corrupted here — it falls through to E1 and + // collapses to (no leaked id, no false Notion label). Without the + // marker guard, ordering this arm before E1 would silently rewrite unrelated + // `/p/...` route/path segments — the exact context-free strip OQ5 forbids. + // + // The marker is evaluated against the PRE-replacement ORIGINAL `content`, not + // the already-partially-replaced `out` (CR round-4 finding). The gate's meaning + // is "did this string carry Notion context ANYWHERE?" — a fact about the + // original input. Testing the post-replacement `out` couples the gate's + // correctness to whether the placeholder happens to still contain the word + // "notion": if the only Notion marker was a notion.so URL that an earlier arm + // already collapsed, and the placeholder carried no `notion` token, the gate + // would wrongly read false and a genuine `/p/` link would fall through to + // E1 and be mislabeled . Reading the original removes that coupling. + // + // Run BEFORE the generic E2 UUID strip so a dashed-UUID Notion id is collapsed + // as a Notion link (not partially eaten by the bare-UUID arm). + const hadNotionContext = NOTION_MARKER.test(content); + out = out + .replace(NOTION_URL, "") + .replace(NOTION_CONTEXT_ID, ""); + if (hadNotionContext) { + out = out.replace(NOTION_P_ID, ""); + } + + // E5 internal Railway hosts. Runs BEFORE E1 (and before E2). Before E1 (crfix7 + // F1): E5 emits ``, and a genuine machine-local path appended + // to a railway host (`myapp.up.railway.app/Users/jpr5/x.ts` — after E5, + // `/Users/jpr5/x.ts`) must be caught by E1, exactly as a path + // appended to a Notion link is (E3 also runs before E1). If E1 ran first, that + // appended path would be preceded by a host word char (`p` of `.app`), skipped, + // and then orphaned after the emitted `>` with no later E1 pass — leaking the + // username on the first pass (idempotence would silently "fix" it on a 2nd pass, + // masking a real first-pass leak). Before E2: a dashed-UUID project id inside + // `railway.app/project/` is collapsed to as a whole, + // rather than the bare-UUID arm eating the id and leaving `railway.app/project/`. + out = out + .replace(RAILWAY_PROJECT, "") + .replace(RAILWAY_HOST, ""); + + // E1 machine-local paths. Runs AFTER the Notion arms AND the Railway arm so a + // genuine Notion `/p/` is already collapsed and any machine-local path + // appended to an emitted ``/`` placeholder is + // caught here (crfix7 F1); a context-free `/p/` (no Notion marker) reaches + // here and collapses to . + out = sanitizeMachineLocalPaths(out); + + // E2 session UUIDs. Runs after the structural URL/host arms so it only sees + // free-standing session UUIDs (in prose or a `(session )` clause). + out = sanitizeSessionUuids(out); + + // E4 internal Slack channels (closed allowlist). + out = out.replace(SLACK_CHANNEL, ""); + + // E6 internal emails. + out = out.replace(TEAM_EMAIL, ""); + + // --- provenance.source: E1 + E2 only (§3.1) --- + const source = sanitizeSessionUuids( + sanitizeMachineLocalPaths(provenanceSource), + ); + + return { content: out, source }; +} diff --git a/src/atlas/adapters/showcase.ts b/src/atlas/adapters/showcase.ts index 321f90d..76a2666 100644 --- a/src/atlas/adapters/showcase.ts +++ b/src/atlas/adapters/showcase.ts @@ -25,6 +25,7 @@ import type { CandidateFragment } from "../types.js"; import type { AdapterContext, LeafAdapter } from "./types.js"; +import { sanitizeEnvRefs } from "./sanitize-env-refs.js"; // ── Feature-registry shape (owned here; imported by S14 validate.ts) ─────────── // @@ -264,6 +265,16 @@ export const showcaseAdapter: LeafAdapter = { // feature, so a human sees quarantined/unknown ones. const validationTargets = allGreen ? [...features] : []; + // §3.3: sanitize the emitted content (and provenance.source) through the + // shared env-reference pass immediately before returning the fragment. The + // feature-support prose is registry-derived and low-risk, but the pass is + // applied uniformly across the fleet so no adapter is a leak gap. + const { content: sanitizedContent, source: sanitizedSource } = + sanitizeEnvRefs( + describeFeatureSupport(name, registry, features), + "showcase", + ); + const fragment: CandidateFragment = { sourcetype: "derived", subsystem: integration, @@ -278,9 +289,9 @@ export const showcaseAdapter: LeafAdapter = { // pill may be quarantined/unknown, so "supports N" would overclaim. The // count is the UNIQUE declared features. title: `${name} declares ${features.length} showcase feature(s)`, - content: describeFeatureSupport(name, registry, features), + content: sanitizedContent, provenance: { - source: "showcase", + source: sanitizedSource, url: manifest.repo_url, date: asOf, validated_against: "showcase/shared/feature-registry.json", diff --git a/src/atlas/adapters/source-comment.ts b/src/atlas/adapters/source-comment.ts index 657f9f0..e50c9b4 100644 --- a/src/atlas/adapters/source-comment.ts +++ b/src/atlas/adapters/source-comment.ts @@ -22,6 +22,7 @@ import type { CandidateFragment, EvidenceItem } from "../types.js"; import type { AdapterContext, LeafAdapter } from "./types.js"; +import { sanitizeEnvRefs } from "./sanitize-env-refs.js"; import { scanSensitivity } from "./sensitivity-scan.js"; // ── Unit shape ──────────────────────────────────────────────────────────────── @@ -287,6 +288,15 @@ export const sourceCommentAdapter: LeafAdapter = { { kind: "fused_from", ref: `source-comment:${anchor}` }, ]; + // §3.3: sanitize the emitted content (and provenance.source) through the + // shared env-reference pass immediately before returning the fragment, so a + // machine-local path / session UUID / private ref that survived into the + // distilled claim is rewritten before it enters the pipeline. (The + // file:line anchor on `validated_against` is repo-relative by contract and + // is not touched here.) + const { content: sanitizedContent, source: sanitizedSource } = + sanitizeEnvRefs(content, "source-comment"); + const fragment: CandidateFragment = { sourcetype: "agent-doc", subsystem, @@ -294,9 +304,9 @@ export const sourceCommentAdapter: LeafAdapter = { repo_url: unit.repoUrl, ref: unit.ref, title, - content, + content: sanitizedContent, provenance: { - source: "source-comment", + source: sanitizedSource, url: unit.sourceUrl, date: asOf, validated_against: anchor, diff --git a/src/atlas/audience-gate.ts b/src/atlas/audience-gate.ts new file mode 100644 index 0000000..4138571 --- /dev/null +++ b/src/atlas/audience-gate.ts @@ -0,0 +1,363 @@ +// Atlas audience/relevance gate (corpus-scoping spec §4) — the +// external-builder-vs-internal-ops core. +// +// Runs AFTER enforceDistillation, BEFORE dedupAgainstRagCorpus (wiring is S7). +// For each candidate the gate produces an `AudienceVerdict`: +// +// - relevant → PASS unchanged (answers a WHY/HOW/WHAT an external +// CopilotKit/Pathfinder builder would genuinely need). +// - borderline → set `needsReview=true` so the human gate reviews it +// (primarily internal but with some external utility). +// - internal-ops → stamp INTERNAL_OPS_MARKER (the S0 shared literal, from +// types.ts) onto a carrier the shared floor predicate reads, +// so the downstream approvability recompute floors it at +// `approvable=false`. NEVER dropped. +// +// NEVER-DROP: the output array is the SAME LENGTH as the input, in the same +// order, and every input candidate is preserved (an internal-ops candidate is +// marked non-approvable, never removed). The input candidates are never mutated +// — each returned candidate is a fresh object (structural spread), matching the +// pure-transform discipline of distillation-gate/canonicalize/validate. +// +// A cheap DETERMINISTIC pre-screen (§4.3, fail-restrictive) short-circuits the +// clear-cut cases WITHOUT calling the LLM judge — the SAME "cheap deterministic +// first, LLM second" pattern as exclude.ts's isCredentialRule/containsCredential +// pre-screen and distillation-gate's metadata-header pre-filter. The LLM judge is +// reserved for the genuinely ambiguous middle. + +import type { Candidate, KnowledgeType } from "./types.js"; +import { INTERNAL_OPS_MARKER } from "./types.js"; + +// The gate's verdict on a candidate's audience fit. A discriminated union on +// `kind` mirroring DistillationVerdict — the LLM seam in llm.ts (`judgeAudience`), +// this gate module, and any consumer narrow on the SAME `kind` discriminant. +export type AudienceVerdict = + | { kind: "relevant" } + | { kind: "borderline"; reason: string } + | { kind: "internal-ops"; reason: string }; + +// What the judge sees for one candidate. Kept structural (title/content + +// knowledge_type + subsystem) — the same narrow shape the llm.ts seam consumes — +// so the gate can hand a Candidate straight through after the deterministic +// pre-screen declines to classify it. Mirrors DistillationJudge. +export interface AudienceJudge { + judge( + c: Pick & { + knowledge_type: KnowledgeType; + subsystem: string; + }, + ): Promise; +} + +// Context handed to the gate: the LLM-backed judge seam. A struct (not a bare +// function) so the harvest driver can widen it later without changing the call +// signature, mirroring DistillationGateContext / RagDedupContext. +export interface AudienceGateContext { + judge: AudienceJudge; +} + +// The `"; "` carrier delimiter. `validated_against` is a `"; "`-joined WHOLE-token +// list, and the S0 shared floor predicate (hasFloorMarker) reads it by splitting +// on this exact sequence and matching a whole token. Kept in lock-step with +// distillation-gate's VALIDATED_AGAINST_DELIMITER and types.ts's hasFloorMarker. +const VALIDATED_AGAINST_DELIMITER = "; "; + +// Append `token` to the `"; "`-joined `provenance.validated_against` token list — +// the SAME carrier idiom distillation-gate/rag-dedup use and the S0 floor +// predicate reads (splitting on `"; "` and matching a WHOLE token). Idempotent on +// whole-token equality: if the token is already present it is not re-appended, so +// a re-run keeps the carrier byte-identical. Empty segments (from a malformed +// carrier with leading/trailing/adjacent delimiters) are filtered so they never +// accrue across re-runs. Returns a fresh provenance object; input never mutated. +function withValidatedAgainstToken( + c: Candidate, + token: string, +): Candidate["provenance"] { + const existing = c.provenance.validated_against; + const tokens = existing + ? existing.split(VALIDATED_AGAINST_DELIMITER).filter((t) => t.length > 0) + : []; + if (tokens.some((t) => t === token)) { + // Re-run no-op on the token, but still normalize away any empty segments a + // prior malformed carrier left behind (idempotent, whitespace-safe). + const normalized = tokens.join(VALIDATED_AGAINST_DELIMITER); + if (normalized === existing) { + return c.provenance; + } + return { ...c.provenance, validated_against: normalized }; + } + tokens.push(token); + return { + ...c.provenance, + validated_against: tokens.join(VALIDATED_AGAINST_DELIMITER), + }; +} + +// Strip any pre-existing INTERNAL_OPS_MARKER floor marker from a candidate on +// BOTH carrier idioms the S0 floor predicate reads (types.ts `hasFloorMarker`): +// a WHOLE `"; "`-delimited token in `provenance.validated_against`, AND a +// `fused_from` evidence ref whose `ref` is the WHOLE marker. Returns fresh +// `provenance` + `evidence` (input never mutated). The validated_against arm +// splits on the SAME `"; "` carrier delimiter and matches whole tokens (never a +// substring), filtering empty segments so a malformed carrier never accrues +// empties across re-runs; the evidence arm drops only the exact whole-ref +// `fused_from` marker item (never a substring, so co-resident refs survive) — +// both mirroring `hasFloorMarker`'s dual-carrier whole-token/whole-ref match and +// distillation-gate's `stripRestatementMarker`. +// +// VERDICT-FLIP HYGIENE (mirrors distillation-gate): a re-run can flip a candidate +// a PRIOR run scoped to internal-ops (carrying this floor marker) to a NON- +// internal-ops verdict (`relevant` or `borderline`). `processCandidatePipeline` +// runs the whole chain TWICE per harvest — once for `run --upsert` (runHarvest) +// and once for the approval artifact (buildArtifactCandidates) — and both must +// produce identical candidates; a non-deterministic judge flipping the verdict, +// or a carrier already holding the marker, must not leave the candidate floored. +// Leaving a stale marker on EITHER carrier would keep the S0 `isApprovableFloored` +// predicate (which validate + canonicalize both read) flooring `approvable=false` +// forever, silently defeating the flip. Applied UNIFORMLY on EVERY non-internal- +// ops verdict so no verdict-flip path can leave a stale floor. Only the internal- +// ops path (re)stamps the marker. Idempotent: a candidate with no stale marker on +// either carrier is returned with the same object references. +function stripInternalOpsMarker( + c: Candidate, +): Pick { + return { + provenance: stripMarkerFromValidatedAgainst(c), + evidence: stripMarkerFromEvidence(c), + }; +} + +// The `provenance.validated_against` arm of the dual-carrier strip. Returns the +// same provenance object when no stale marker is present (byte-identical carrier). +function stripMarkerFromValidatedAgainst( + c: Candidate, +): Candidate["provenance"] { + const existing = c.provenance.validated_against; + if (!existing) { + return c.provenance; + } + const kept = existing + .split(VALIDATED_AGAINST_DELIMITER) + .filter((t) => t.length > 0 && t !== INTERNAL_OPS_MARKER); + const rebuilt = kept.join(VALIDATED_AGAINST_DELIMITER); + if (rebuilt === existing) { + return c.provenance; + } + // `validated_against` is `.optional()`: an all-empty result must drop the field + // rather than persist an empty string, keeping the "no carrier" shape canonical. + const provenance = { ...c.provenance }; + if (rebuilt) { + provenance.validated_against = rebuilt; + } else { + delete provenance.validated_against; + } + return provenance; +} + +// The `fused_from` evidence-ref arm of the dual-carrier strip. Drops ONLY the +// exact whole-ref `fused_from` INTERNAL_OPS_MARKER item (matching hasFloorMarker's +// whole-ref semantics), leaving every other evidence item — including unrelated +// `fused_from` refs — intact. Returns the same array when no stale marker item is +// present so an untouched candidate keeps its evidence reference. +function stripMarkerFromEvidence(c: Candidate): Candidate["evidence"] { + const hasStaleMarker = c.evidence.some( + (e) => e.kind === "fused_from" && e.ref === INTERNAL_OPS_MARKER, + ); + if (!hasStaleMarker) { + return c.evidence; + } + return c.evidence.filter( + (e) => !(e.kind === "fused_from" && e.ref === INTERNAL_OPS_MARKER), + ); +} + +// ── Deterministic pre-screen (§4.3, fail-restrictive) ───────────────────────── + +// Clear internal-ops: an infra subsystem slug. Whole-string match on a leading +// infra family so `railway-web` matches but a hypothetical `not-railway` does not +// (`\b` word-boundary anchored to the START of the slug). +const INFRA_SUBSYSTEM_RE = /^(?:railway|ci|deploy|pr-closeout)-/; + +// Clear internal-ops: Railway/CI/deploy platform + action vocabulary in the body +// (only consulted when knowledge_type === "operational"). Two arms: a hosting +// platform name, and a deploy/PR-closeout action phrase. +const DEPLOY_PLATFORM_RE = /\b(?:railway|vercel|render|up\.railway\.app)\b/i; +const DEPLOY_ACTION_RE = /\b(?:redeployed|promoted to|merged PR #\d+)\b/i; + +// Clear relevant: knowledge types that are inherently external-builder-facing +// (architecture, design rationale, root cause, protocol, security) — the +// complement of the internal-ops-leaning operational/process families. +const CLEAR_RELEVANT_KNOWLEDGE_TYPES: ReadonlySet = + new Set([ + "architecture", + "design-rationale", + "root-cause", + "protocol", + "security", + ]); + +// A product-portable specific an external builder can look up, call, or configure +// (§3.2 discriminator). Because a clear-relevant match SHORT-CIRCUITS the gate and +// SKIPS the fail-restrictive judge, every arm must match ONLY a genuine specific — +// never ordinary prose. Each arm is scoped so an incidental token in internal-ops +// narrative (a "503 candidates" count, an "orchestrator (see …)" parenthetical) +// does NOT force the bypass: +// +// 1. HTTP-method endpoint — a REST verb immediately followed by a slash-rooted +// path (`POST /admin/reindex`). Verb + path together, not either alone. +// 2. HTTP status code IN GENUINE HTTP CONTEXT — a 4xx/5xx code only when a +// real HTTP status marker sits adjacent, so a bare "503 candidates" count +// (or the incidental prose words "code"/"error"/"status" near a number) is +// NOT matched. A marker is either an HTTP RESPONSE VERB (returns/returned/ +// responds/responded/HTTP) OR the two-word status NOUN PHRASE ("status +// code" / "error code" / "HTTP status") — never a bare "code"/"error"/ +// "status" on its own (finding 4 floor escape: those bare words matched +// ordinary internal-ops prose like "during code review we saw 503 +// candidates" and forced the clear-relevant bypass). Checked in both orders +// ("returns 401", "401 error code"). +// 3. Call-shaped API symbol — a plausible identifier immediately followed by +// `(` with NO intervening space, AND either camelCase/PascalCase (an interior +// uppercase letter, e.g. `useCoAgent(`, `runHarvest(`) OR wrapped in a +// backtick/code fence. This rejects prose parentheticals ("orchestrator (see +// …)" has a space) and bare lowercase call-shaped words ("promote()"). +// 4. Config-file key — a `.yaml`/`.yml`/`.json`/`.toml` extension. +// +// Deliberately conservative — it only GRANTS the bypass; anything it does not +// recognize falls through to the judge rather than being force-passed. +const HTTP_ENDPOINT_RE = /\b(?:GET|POST|PUT|PATCH|DELETE)\s+\/\S/; +// A genuine HTTP-status marker: an HTTP response VERB, or the two-word status +// NOUN PHRASE. Deliberately EXCLUDES bare "code"/"error"/"status" — those only +// qualify as part of "status code"/"error code"/"HTTP status", never alone (a +// bare word matches incidental prose and would force the bypass; finding 4). +const HTTP_STATUS_VERB = "returns?|returned|respond(?:s|ed)?|HTTP"; +const HTTP_STATUS_NOUN = "(?:status|error|HTTP)\\s+code|HTTP\\s+status"; +const HTTP_STATUS_IN_CONTEXT_RE = new RegExp( + // response verb within a short window before a 4xx/5xx code … + `\\b(?:${HTTP_STATUS_VERB})\\b[^.\\n]{0,24}?\\b[45]\\d{2}\\b` + + // … or the status noun phrase within a short window before the code … + `|\\b(?:${HTTP_STATUS_NOUN})\\b[^.\\n]{0,24}?\\b[45]\\d{2}\\b` + + // … or a 4xx/5xx code immediately followed by the status noun phrase + // ("401 error code" / "500 status code"). + `|\\b[45]\\d{2}\\b\\s+(?:${HTTP_STATUS_NOUN})\\b`, + "i", +); +// A camelCase/PascalCase identifier (has an interior uppercase) glued to `(` with +// no space, OR any backtick-fenced call-shaped identifier. +const CALL_SHAPED_SYMBOL_RE = + /\b[a-zA-Z_$][\w$]*[A-Z][\w$]*\(|`[a-zA-Z_$][\w$]*\s*\(/; +const CONFIG_KEY_RE = /\.(?:ya?ml|json|toml)\b/; + +function hasProductPortableSpecific(content: string): boolean { + return ( + HTTP_ENDPOINT_RE.test(content) || + HTTP_STATUS_IN_CONTEXT_RE.test(content) || + CALL_SHAPED_SYMBOL_RE.test(content) || + CONFIG_KEY_RE.test(content) + ); +} + +// Does the candidate's body/subsystem clear the internal-ops pre-screen? +function isClearInternalOps( + c: Pick & { + knowledge_type: KnowledgeType; + subsystem: string; + }, +): boolean { + if (INFRA_SUBSYSTEM_RE.test(c.subsystem)) { + return true; + } + return ( + c.knowledge_type === "operational" && + DEPLOY_PLATFORM_RE.test(c.content) && + DEPLOY_ACTION_RE.test(c.content) + ); +} + +// Does the candidate clear the relevant pre-screen (§4.3)? +function isClearRelevant( + c: Pick & { knowledge_type: KnowledgeType }, +): boolean { + return ( + CLEAR_RELEVANT_KNOWLEDGE_TYPES.has(c.knowledge_type) && + hasProductPortableSpecific(c.content) + ); +} + +// Enforce the audience/relevance gate over a candidate set. NEVER drops; +// same-length, same-order output; input never mutated. Each candidate is first +// run through the deterministic pre-screen; only the genuinely ambiguous middle +// reaches the injected LLM judge. Routed by verdict: +// - relevant → passed through unchanged (fresh object). +// - borderline → needsReview=true (human gate reviews it). +// - internal-ops → INTERNAL_OPS_MARKER stamped onto provenance.validated_against +// (the carrier the S0 floor predicate reads) so validate floors +// approvable=false and canonicalize floors the rank weight. +export async function enforceAudienceRelevance( + cands: Candidate[], + ctx: AudienceGateContext, +): Promise { + const out: Candidate[] = []; + for (const c of cands) { + const knowledge_type = c.provenance.classification.knowledge_type; + const screenInput = { + content: c.content, + knowledge_type, + subsystem: c.subsystem, + }; + + // Deterministic pre-screen first (fail-restrictive): clear internal-ops and + // clear relevant short-circuit WITHOUT the LLM judge. Internal-ops is checked + // first so a clear infra candidate never leaks through on a coincidental + // product-portable-specific match. + let verdict: AudienceVerdict; + if (isClearInternalOps(screenInput)) { + verdict = { + kind: "internal-ops", + reason: "deterministic pre-screen: infra/deploy", + }; + } else if (isClearRelevant(screenInput)) { + verdict = { kind: "relevant" }; + } else { + verdict = await ctx.judge.judge({ + title: c.title, + content: c.content, + knowledge_type, + subsystem: c.subsystem, + }); + } + + if (verdict.kind === "relevant") { + // Already external-builder-relevant — pass through as a fresh object so the + // input is never handed on by reference. Strip any stale INTERNAL_OPS_MARKER + // a PRIOR run left on EITHER carrier so a flip to relevant is not + // stale-floored (dual-carrier: validated_against + fused_from evidence). + out.push({ ...c, ...stripInternalOpsMarker(c) }); + continue; + } + + if (verdict.kind === "borderline") { + // Primarily internal but with some external utility → human review. Does + // NOT floor approvable — so strip any stale INTERNAL_OPS_MARKER a PRIOR run + // left on EITHER carrier (a flip from internal-ops to borderline must + // un-floor on validated_against + fused_from evidence alike). + out.push({ + ...c, + ...stripInternalOpsMarker(c), + needsReview: true, + }); + continue; + } + + // internal-ops: zero external-builder utility. Stamp the S0-read floor marker + // onto the carrier; the candidate is NOT dropped (it renders as a + // non-approvable note in the review artifact for the audit trail). This is the + // ONLY path that stamps INTERNAL_OPS_MARKER — every other verdict strips a + // stale one (verdict-flip hygiene); `withValidatedAgainstToken` is idempotent + // so a re-run that stays internal-ops keeps the carrier byte-identical. + out.push({ + ...c, + provenance: withValidatedAgainstToken(c, INTERNAL_OPS_MARKER), + }); + } + return out; +} diff --git a/src/atlas/canonicalize.ts b/src/atlas/canonicalize.ts index f2e8bb8..ba9ca70 100644 --- a/src/atlas/canonicalize.ts +++ b/src/atlas/canonicalize.ts @@ -24,12 +24,12 @@ // exclusion-rule engine (S13) remove rows (§10 bar 1). import { + APPROVABLE_FLOOR_MARKERS, BEHAVIOR_KNOWLEDGE_TYPES, buildCanonicalKey, dateToEpochMs, - RAG_NO_DELTA_MARKER, + isApprovableFloored, } from "./types.js"; -import { hasRestatementMarker } from "./validate.js"; import type { Candidate, CandidateFragment, @@ -188,26 +188,32 @@ function recency(fragment: CandidateFragment, now: number): number { // bounded boost (1 + log1p(count)) so a single strong fact is never buried under // a weakly-corroborated one purely on evidence count. // -// Rank-neutral §6.2 duplication marks: the rag-dedup gate stamps TWO kinds of -// `fused_from` evidence on a corpus-overlapping candidate, neither of which is -// corroboration for the claim — counting either would make a corpus DUPLICATE -// outrank its un-duplicated twin (inverting §6.2). Filter BOTH out of the depth -// count; genuine fused_from refs (aggregator provenance — canonical-key-shaped) -// still count. +// Rank-neutral §6.2 duplication / floor marks: several upstream gates stamp a +// `fused_from` evidence item that is NOT corroboration for the claim — counting +// any of them would make a floored/duplicate candidate outrank its genuine +// un-floored twin (inverting §6.2). Filter these out of the depth count; genuine +// fused_from refs (aggregator provenance — canonical-key-shaped) still count. // // 1. RAG_CORPUS_OVERLAP_REF_PREFIX: an AUDIT annotation about the corpus, -// appended on EVERY overlap verdict (delta included). -// 2. RAG_NO_DELTA_MARKER: the DEDICATED no-delta floor trace, stamped as a -// fused_from ref on a pure corpus duplicate (rag-dedup's floorNoDelta). It -// is a provenance floor, not evidence — the SAME constant the emitter and -// validate reader import, so the exclusion can never drift from the stamp. +// appended on EVERY overlap verdict (delta included). A PREFIX, not a whole +// marker, so it is matched separately from the floor-marker set. +// 2. APPROVABLE_FLOOR_MARKERS: the DEDICATED approvable-floor traces +// (RESTATEMENT / RAG_NO_DELTA / INTERNAL_OPS), each stamped as a `fused_from` +// ref on a floored candidate by its gate. All ride the IDENTICAL fused_from +// carrier (types.ts), so ALL must be excluded — not just the no-delta one. +// Use the SAME shared set computeRankScore's validation-weight floor reads +// (via isApprovableFloored), so the rank filter and the approvable floor +// pick up any FUTURE marker automatically and can never drift apart. +const FLOOR_MARKER_REFS: ReadonlySet = new Set( + APPROVABLE_FLOOR_MARKERS, +); function evidenceDepth(fragment: CandidateFragment): number { const corroborating = fragment.evidence.filter( (e) => !( e.kind === "fused_from" && (e.ref.startsWith(RAG_CORPUS_OVERLAP_REF_PREFIX) || - e.ref === RAG_NO_DELTA_MARKER) + FLOOR_MARKER_REFS.has(e.ref)) ), ); return 1 + Math.log1p(corroborating.length); @@ -222,17 +228,20 @@ export const RAG_CORPUS_OVERLAP_REF_PREFIX = "rag-corpus-overlap:"; function computeRankScore(fragment: CandidateFragment, now: number): number { const { validation_status, confidence } = fragment.provenance.classification; - // A RESTATEMENT-floored candidate (approvable=false; see validate.ts) carries - // no NEW verifiable claim. The validate gate still PROMOTES its - // validation_status when its symbols grep-verify (the status is display-truth - // — the symbols really do exist), but that promotion must NOT lift the rank: - // otherwise the restatement would OUT-RANK a genuine claim purely on the - // dominant validation weight, surfacing restatement noise above real why/how - // in the ranked artifact (§11.1). Floor the validation weight to `unverified` - // for a restatement, consistent with its approvable=false floor — the SAME - // predicate the approvability floor uses, so status-display and rank can never - // drift apart. - const validationWeight = hasRestatementMarker(fragment) + // An approvable-FLOORED candidate (approvable=false; see validate.ts) carries + // no NEW shippable verifiable claim — whether it is a distillation RESTATEMENT, + // a rag-dedup no-delta corpus DUPLICATE, or an INTERNAL_OPS audience-scoped + // fact (§6.2). The validate gate still PROMOTES its validation_status when its + // symbols grep-verify (the status is display-truth — the symbols really do + // exist), but that promotion must NOT lift the rank: otherwise the floored + // candidate would OUT-RANK a genuine claim purely on the dominant validation + // weight, surfacing floor noise above real why/how in the ranked artifact + // (§11.1). Floor the validation weight to `unverified` for ANY approvable-floor + // marker, consistent with the approvable=false floor — the SAME S0 shared + // predicate the approvability floor uses (isApprovableFloored over the complete + // APPROVABLE_FLOOR_MARKERS set), so status-display and rank can never drift + // apart and a new floor marker is picked up by both automatically. + const validationWeight = isApprovableFloored(fragment) ? VALIDATION_WEIGHT.unverified : VALIDATION_WEIGHT[validation_status]; return ( diff --git a/src/atlas/exclude.ts b/src/atlas/exclude.ts index 103d37d..9c9d6df 100644 --- a/src/atlas/exclude.ts +++ b/src/atlas/exclude.ts @@ -67,6 +67,12 @@ export const DEFAULT_EXCLUSION_RULES: ExclusionRule[] = [ kind: "english", text: "Exclude go-to-market or sales content that identifies a specific named customer, client, or account (deal details, customer engagements, account-specific commercial terms).", }, + // Internal operational trivia: Railway/CI deploy logs, PR-closeout records, + // internal-infra topology — no value to an external CopilotKit builder. + { + kind: "english", + text: "Exclude any candidate that is purely internal operational trivia — Railway/CI deploy logs, PR-closeout records, internal-infra topology — with zero value to an external CopilotKit builder.", + }, ]; // ── Deterministic credential pre-filter (D.2, fail-restrictive) ───────────────── diff --git a/src/atlas/harvest-cli.ts b/src/atlas/harvest-cli.ts index 1390ddc..c80c82e 100644 --- a/src/atlas/harvest-cli.ts +++ b/src/atlas/harvest-cli.ts @@ -58,6 +58,10 @@ import { enforceDistillation, type DistillationJudge, } from "./distillation-gate.js"; +import { + enforceAudienceRelevance, + type AudienceJudge, +} from "./audience-gate.js"; import { promoteValidation, type ValidationContext } from "./validate.js"; import { loadValidationContext } from "./validate-checkout.js"; import { @@ -154,6 +158,13 @@ export interface RunHarvestOptions { // backed judge (honors OPENAI_BASE_URL so tests can redirect to aimock); // tests inject a fixture judge here. judge?: DistillationJudge; + // The audience-relevance gate judge (external-builder-vs-internal-ops). NET-NEW + // top-level field, wired the SAME way as `judge`: an LLM seam, not a + // deterministic transform, so it lives here and not in `deps`. When omitted, + // `runHarvest` defaults to a real OpenAIDistiller-backed judge (honors + // OPENAI_BASE_URL so tests can redirect to aimock); tests inject a fixture + // judge here. + audienceJudge?: AudienceJudge; // Testing seam (see RunHarvestDeps). deps?: RunHarvestDeps; } @@ -203,6 +214,9 @@ export interface ProcessCandidatePipelineOptions { store: RunStore; // The A.1 distillation-gate judge (why-vs-what). judge: DistillationJudge; + // The audience-relevance gate judge (external-builder-vs-internal-ops). Runs + // AFTER the distillation gate, BEFORE rag-dedup (§4.5). + audienceJudge: AudienceJudge; // The live RAG-corpus lexical probe (rag-dedup pre-filter). ragClient: Pick; // Lexical verbatim-overlap threshold for the rag-dedup pre-filter. @@ -245,6 +259,19 @@ export async function processCandidatePipeline( judge: opts.judge, }); + // 4c. Audience-relevance gate (§4.5) — AFTER the distillation gate, BEFORE + // rag-dedup. Judges each candidate's external-builder fit: an internal-ops + // verdict stamps INTERNAL_OPS_MARKER (which validate reads as an + // approvable=false floor and canonicalize reads as a rank-weight floor), a + // borderline verdict sets needsReview=true, a relevant verdict passes + // untouched. NEVER drops; same-length output; input never mutated. Runs + // here (post-distillation) because distillation may rewrite a candidate + // into clearly-relevant why/how first, and there is no point RAG-probing a + // candidate we are about to mark internal-ops. + const audienced = await enforceAudienceRelevance(distilled, { + judge: opts.audienceJudge, + }); + // 5. RAG-dedup gate — BEFORE validate (spec §4). Detects corpus overlap // (lexical verbatim pre-filter + semantic pgvector retrieval) and RESOLVES // it via distill-to-delta. It is NO LONGER mark-only: on a SEMANTIC overlap @@ -258,9 +285,9 @@ export async function processCandidatePipeline( // candidate that survives the lexical pre-filter — one embedding call // apiece. Emit the estimated upper bound now that candidateCount is known, // so a large run's embedding spend is visible before it is incurred. - if (distilled.length > 0) { + if (audienced.length > 0) { console.warn( - `[rag-dedup] semantic dedup will embed up to ${distilled.length} candidate(s) ` + + `[rag-dedup] semantic dedup will embed up to ${audienced.length} candidate(s) ` + `(one /v1/embeddings call each for candidates surviving the lexical pre-filter)`, ); } @@ -271,7 +298,7 @@ export async function processCandidatePipeline( vectorSearch: opts.vectorSearch, distillDelta: opts.distillDelta, }; - const deduped = await dedup(distilled, ragCtx); + const deduped = await dedup(audienced, ragCtx); // 6. Validation gate — promote validation_status + enforce approvability. // validate can PROMOTE validation_status — the DOMINANT rank weight — so @@ -307,6 +334,7 @@ export async function processCandidatePipeline( // passed through unchanged. interface SharedPipelineSeamOverrides { judge?: DistillationJudge; + audienceJudge?: AudienceJudge; embed?: (text: string) => Promise; vectorSearch?: (vector: number[], k: number) => Promise; distillDelta?: ( @@ -317,8 +345,11 @@ interface SharedPipelineSeamOverrides { validate?: (cand: Candidate, ctx: ValidationContext) => Promise; } -function resolveSharedPipelineSeams(overrides: SharedPipelineSeamOverrides): { +export function resolveSharedPipelineSeams( + overrides: SharedPipelineSeamOverrides, +): { judge: DistillationJudge; + audienceJudge: AudienceJudge; embed: (text: string) => Promise; vectorSearch: (vector: number[], k: number) => Promise; distillDelta: ( @@ -330,14 +361,19 @@ function resolveSharedPipelineSeams(overrides: SharedPipelineSeamOverrides): { } { return { judge: overrides.judge ?? { - judge: (c) => buildLlm().judgeDistillation(c), + judge: (c) => __internalsForTest.buildLlm().judgeDistillation(c), }, - embed: overrides.embed ?? ((text: string) => buildLlm().embed(text)), + audienceJudge: overrides.audienceJudge ?? { + judge: (c) => __internalsForTest.buildLlm().judge(c), + }, + embed: + overrides.embed ?? + ((text: string) => __internalsForTest.buildLlm().embed(text)), vectorSearch: overrides.vectorSearch ?? defaultVectorSearch, distillDelta: overrides.distillDelta ?? ((cand: Candidate, overlaps: CorpusHit[]) => - buildLlm().distillDelta({ + __internalsForTest.buildLlm().distillDelta({ title: cand.title, content: cand.content, overlaps: overlaps.map((h) => ({ content: h.content })), @@ -402,6 +438,7 @@ export async function runHarvest( validationContext: opts.validationContext, ...resolveSharedPipelineSeams({ ...(opts.judge ? { judge: opts.judge } : {}), + ...(opts.audienceJudge ? { audienceJudge: opts.audienceJudge } : {}), ...(opts.embed ? { embed: opts.embed } : {}), ...(opts.vectorSearch ? { vectorSearch: opts.vectorSearch } : {}), ...(opts.distillDelta ? { distillDelta: opts.distillDelta } : {}), @@ -486,6 +523,12 @@ export interface BuildArtifactCandidatesOptions { // distillation gate `run --upsert` runs, or a restatement/rewritten candidate // diverges from the rows the approval page promises to match. judge?: DistillationJudge; + // The audience-relevance gate judge. Wired exactly like `judge`: injected by + // tests, else a real OpenAIDistiller-backed judge (honors OPENAI_BASE_URL). + // The artifact MUST run the SAME audience gate `run --upsert` runs, or an + // internal-ops candidate renders approvable=true on the page while the upsert + // writes approvable=false. + audienceJudge?: AudienceJudge; } // Build the ranked candidate set the approval artifact renders. It runs the @@ -530,6 +573,7 @@ export async function buildArtifactCandidates( validationContext: opts.validationContext, ...resolveSharedPipelineSeams({ ...(opts.judge ? { judge: opts.judge } : {}), + ...(opts.audienceJudge ? { audienceJudge: opts.audienceJudge } : {}), ...(opts.embed ? { embed: opts.embed } : {}), ...(opts.vectorSearch ? { vectorSearch: opts.vectorSearch } : {}), ...(opts.distillDelta ? { distillDelta: opts.distillDelta } : {}), @@ -632,11 +676,23 @@ function buildHttpClient(flags: { }); } -function buildLlm(): LlmDistiller { +// The canonical LLM constructor for the shared pipeline seams. OpenAIDistiller +// implements BOTH LlmDistiller (judge/embed/distillDelta) and AudienceJudge, so +// the return type advertises both: the audienceJudge seam builds its judge via +// this SAME factory as every other seam, rather than constructing its own +// distiller and silently bypassing any config buildLlm applies (or later gains). +function buildLlm(): LlmDistiller & AudienceJudge { // Reuses the openai dep; honors OPENAI_BASE_URL so it can be redirected. return new OpenAIDistiller(); } +// Indirection seam so tests can swap the LLM factory the shared-pipeline defaults +// resolve to WITHOUT constructing a real OpenAIDistiller (which needs a key). The +// defaults in resolveSharedPipelineSeams call `__internalsForTest.buildLlm()` +// (not the bare local `buildLlm`) so all four buildLlm-backed seams route through +// this ONE spyable slot. Production leaves it untouched → the real buildLlm. +export const __internalsForTest = { buildLlm }; + // Default vectorSearch seam: cosine top-k retrieval over the SAME chunks corpus // the indexer writes (db/queries.ts:searchChunks), mapped to the CorpusHit shape // the rag-dedup gate + distill-to-delta consume. `similarity` is 1 - cosine diff --git a/src/atlas/llm.ts b/src/atlas/llm.ts index 73e0836..566d5a9 100644 --- a/src/atlas/llm.ts +++ b/src/atlas/llm.ts @@ -30,6 +30,7 @@ import type { KnowledgeType, } from "./types.js"; import { mostRestrictiveSensitivity } from "./types.js"; +import type { AudienceJudge, AudienceVerdict } from "./audience-gate.js"; // ── Public types ────────────────────────────────────────────────────────────── @@ -81,6 +82,17 @@ export interface DistillationJudgeInput { knowledge_type: KnowledgeType; } +// What the audience judge sees for one candidate (corpus-scoping spec §4.4). +// Kept structural — the SAME narrow shape `AudienceJudge.judge` consumes in +// audience-gate.ts — so the gate can hand a Candidate straight through after its +// deterministic pre-screen declines to classify it. Mirrors DistillationJudgeInput. +export interface AudienceJudgeInput { + title: string; + content: string; + knowledge_type: KnowledgeType; + subsystem: string; +} + // What the distill-to-delta rewrite (Theme B fix (c)) sees for one candidate: its // title/content plus the overlapping corpus passages the semantic gate found. // The seam rewrites `content` down to the NET-NEW part the corpus does not @@ -228,8 +240,8 @@ const DISTILLATION_SYSTEM_PROMPT = `You are a WHY-vs-WHAT judge for an engineeri You are given ONE candidate knowledge entry (title + content + knowledge_type). Institutional-memory knowledge must explain the WHY / HOW behind a decision, root cause, architecture choice, or operational reality — NOT merely RESTATE the WHAT that is already obvious from metadata (which PR merged, what a file is named, that a component was added). Classify the candidate into EXACTLY ONE verdict: -- "distilled": already a why/how CLAIM (explains reasoning, tradeoffs, mechanism, or consequence). Keep as-is. A claim that already states a CONCRETE MECHANISM — specific endpoints/routes, HTTP status codes, error codes, named functions/methods/symbols, file paths, config keys, or specific numbers — is "distilled": keep it as-is; do NOT rewrite a concrete-mechanism claim up into higher-level rationale. -- "rewritten": the SUBSTANCE is salvageable but the current title/content just restates WHAT happened; a why/how claim can be extracted. Provide the rewrite. The rewrite MUST RETAIN every concrete verifiable detail present in the source — API endpoints/routes, HTTP status codes, error codes, function/method/symbol names, file paths, config keys, and specific numbers. Sharpen the claim by adding the WHY/HOW AROUND those specifics; NEVER drop, generalize, or paraphrase them away. (Concretely: rewriting "POST /admin/:op returns 401 via timingSafeEqual" into "authentication enhances security" is WRONG — the endpoint, the code, and the symbol were all dropped.) +- "distilled": already a why/how CLAIM (explains reasoning, tradeoffs, mechanism, or consequence). Keep as-is. A claim that already states a CONCRETE MECHANISM — specific endpoints/routes, HTTP status codes, error codes, named functions/methods/symbols, repo-relative file paths, config keys, or specific numbers — is "distilled": keep it as-is; do NOT rewrite a concrete-mechanism claim up into higher-level rationale. +- "rewritten": the SUBSTANCE is salvageable but the current title/content just restates WHAT happened; a why/how claim can be extracted. Provide the rewrite. The rewrite MUST RETAIN every concrete verifiable detail present in the source — API endpoints/routes, HTTP status codes, error codes, function/method/symbol names, repo-relative file paths, config keys, and specific numbers. Sharpen the claim by adding the WHY/HOW AROUND those specifics; NEVER drop, generalize, or paraphrase them away. (Concretely: rewriting "POST /admin/:op returns 401 via timingSafeEqual" into "authentication enhances security" is WRONG — the endpoint, the code, and the symbol were all dropped.) - "restatement": a PURE WHAT restatement (e.g. "adds X/Y/Z components", "PR #N merged", a stack/component inventory) that carries NO new reasoning or verifiable engineering claim. Cannot be salvaged into a why/how claim from the given text. Return JSON with EXACTLY this structure: @@ -241,11 +253,27 @@ Return JSON with EXACTLY this structure: } Rules: +- Machine-local absolute paths (e.g. /Users/…, /home/…, /proj/…), session UUIDs, private Notion page IDs, and internal Slack channel names are NOT product-portable specifics — do NOT treat their presence as making a claim 'distilled'. - Be conservative about "distilled": if the content only names WHAT (files, components, PRs) with no reasoning, it is NOT distilled. - Only choose "rewritten" when the given text ACTUALLY contains extractable why/how substance — do NOT invent reasoning that is not present. If nothing is salvageable, choose "restatement". - PRESERVE-SPECIFICS is mandatory on "rewritten": if you cannot produce a rewrite that keeps EVERY identifier/endpoint/status-code/error-code/symbol/path/config-key/number from the source, return "distilled" instead (pass the original through unchanged). Losing a verifiable specific is worse than leaving the prose slightly WHAT-flavored. - title/content are REQUIRED for "rewritten" and ignored for the other verdicts.`; +const AUDIENCE_SYSTEM_PROMPT = `You are an audience-relevance judge for an external-builder knowledge corpus. The audience is developers building products with CopilotKit / Pathfinder — they are NOT CopilotKit employees and do NOT have access to internal infra, private Slack, internal Notion pages, or our Railway/CI setup. + +Classify into EXACTLY ONE verdict: +- "relevant": answers a WHY/HOW/WHAT an external CopilotKit builder would genuinely need — architecture decisions, API/protocol/SDK behavior, configuration choices, root causes of observable behaviors. +- "borderline": primarily internal operational but contains some info an advanced external builder might want; send to human review. +- "internal-ops": purely internal operational trivia — deploy logs, PR-closeout records, CI/infra topology, internal Slack guidance, internal approval workflows. + +FAIL-RESTRICTIVE: when uncertain between "relevant" and "internal-ops", prefer "borderline". Only use "internal-ops" when the content has zero external-builder utility. + +Return JSON with EXACTLY this structure: +{ + "verdict": "", + "reason": "" +}`; + const DISTILL_DELTA_SYSTEM_PROMPT = `You are a knowledge-DELTA distiller for an engineering knowledge corpus. You are given ONE candidate knowledge entry (title + content) and one or more ALREADY-INDEXED corpus passages that overlap it. Re-seeding content the corpus already covers adds DUPLICATION, not knowledge. Your job is to rewrite the candidate's content down to ONLY the NET-NEW part — the reasoning, mechanism, consequence, or fact the overlapping passages do NOT already state. @@ -350,8 +378,35 @@ function coerceKnowledgeType(v: unknown): Classification["knowledge_type"] { ) { return normalized as Classification["knowledge_type"]; } - // Episodic distillations are explanatory by nature; default to design-rationale. - return "design-rationale"; + // Fail-restrictive default (finding 1 — floor escape). The prior default, + // "design-rationale", is a CLEAR-RELEVANT knowledge_type (audience-gate.ts's + // CLEAR_RELEVANT_KNOWLEDGE_TYPES). Coercing an UNRECOGNIZED/garbled model type + // to it silently granted a garbled — possibly internal-ops — candidate the + // clear-relevant pre-screen BYPASS, skipping the fail-restrictive judge and + // the internal-ops floor. The default must therefore be a type that is NOT + // clear-relevant, so an unrecognized candidate FALLS THROUGH to the judge + // (and can still be floored) instead of being force-passed. + // + // "ownership" is chosen deliberately over the schema-wide catch-all + // "operational" (classify.ts DEFAULT_KNOWLEDGE_TYPE): both are outside the + // clear-relevant set, but "operational" is ALSO in EXEMPT_KNOWLEDGE_TYPES, so + // it is APPROVABLE even while unverified — the wrong direction for a garbled, + // unclassifiable candidate. "ownership" is in BEHAVIOR_KNOWLEDGE_TYPES, so an + // unverified fragment (episodic fragments are always unverified) stays + // NON-approvable — the most restrictive direction on both axes (no bypass, no + // approvability). And we WARN, naming the discarded value, mirroring + // coerceEpisodicSensitivity — the sibling coercion that (unlike this one + // previously) already surfaced an unrecognized value instead of swallowing it. + const attempted = + v == null || (typeof v === "string" && v.trim() === "") + ? undefined + : JSON.stringify(v); + if (attempted !== undefined) { + console.warn( + `[atlas/llm] unrecognized model knowledge_type ${attempted} — defaulting to "ownership" (a NON-clear-relevant, non-approvable-when-unverified type so an unclassifiable candidate reaches the audience judge rather than taking the clear-relevant bypass)`, + ); + } + return "ownership"; } // The sensitivity enum values, mirrored from S0's Sensitivity. Used to validate @@ -399,7 +454,7 @@ function coerceEpisodicSensitivity(v: unknown): Classification["sensitivity"] { return "secret"; } -export class OpenAIDistiller implements LlmDistiller { +export class OpenAIDistiller implements LlmDistiller, AudienceJudge { private readonly client: OpenAI; private readonly model: string; private readonly embeddingModel: string; @@ -661,6 +716,100 @@ export class OpenAIDistiller implements LlmDistiller { ); } + async judgeAudience(candidate: AudienceJudgeInput): Promise { + const userPayload = JSON.stringify({ + title: candidate.title, + content: candidate.content, + knowledge_type: candidate.knowledge_type, + subsystem: candidate.subsystem, + }); + + const response = await this.client.chat.completions.create({ + model: this.model, + messages: [ + { role: "system", content: AUDIENCE_SYSTEM_PROMPT }, + { role: "user", content: userPayload }, + ], + response_format: { type: "json_object" }, + temperature: 0, + }); + + const parsed = parseJsonContent( + response.choices[0]?.message?.content, + "judgeAudience", + ); + + const verdict = asString(parsed.verdict)?.toLowerCase(); + const reason = asString(parsed.reason) ?? ""; + + if (verdict === "relevant") { + return { kind: "relevant" }; + } + if (verdict === "borderline") { + return { kind: "borderline", reason }; + } + // internal-ops is the only MULTI-TOKEN verdict, so a nondeterministic model + // routinely emits it with a non-canonical delimiter between the two tokens + // — a space, underscore, or hyphen, and just as easily a RUN of them (a + // double space, a tab, "internal - ops"). asString has already + // lowercased/trimmed via `verdict`; collapse ANY run of internal + // whitespace/underscore/hyphen delimiters to a single hyphen so every + // delimiter-run variant maps to the canonical internal-ops rather than + // slipping through to the borderline fallback below (which does NOT floor + // approvable=false) — a double-spaced internal-ops verdict must not escape + // the floor. + // + // Finding 3 (floor escape): match on the LEADING token/prefix, not + // exact-equality. A model routinely appends a trailing token to the two + // canonical ones — "internal-ops (trivia)", "internal-ops." — and exact + // equality let those fall through to the non-flooring borderline fallback. + // The collapse turns interior whitespace/underscore/hyphen runs into single + // hyphens; we then check the verdict STARTS WITH "internal-ops" at a token + // boundary (end-of-string, or a non-[a-z0-9] separator such as a space, + // "(", or "."). Anchoring at the start (rather than a bare `.includes`) + // avoids over-matching a genuinely DIFFERENT verdict that merely mentions + // the phrase mid-sentence ("relevant, not internal-ops"). + const collapsed = verdict?.replace(/[\s_-]+/g, "-"); + if (collapsed && /^internal-ops(?![a-z0-9])/.test(collapsed)) { + return { kind: "internal-ops", reason }; + } + + // NEVER-DROP, fail-restrictive: any OTHER unrecognized verdict falls back to + // borderline (keep for human review) rather than throwing. Mirrors how + // judgeDistillation salvages an ambiguous verdict to the conservative + // never-drop direction instead of failing the run loud. The model's reason + // (if any) carries through so a reviewer sees why it was ambiguous. + // + // Finding 2 (observability): WARN at the seam. Every SIBLING branch above + // either returns a recognized verdict or (previously) silently borderlined; + // a drifting model that keeps emitting unrecognized verdicts silently + // inflates the human-review queue with no operator signal. Warn (naming the + // discarded verdict, mirroring the coerce* warns) while KEEPING the + // never-throw behavior — the warn is diagnostic only, it does not change the + // returned verdict. + console.warn( + `[atlas/llm] judgeAudience: unrecognized audience verdict ${JSON.stringify( + parsed.verdict, + )} (expected relevant | borderline | internal-ops) — falling back to borderline (kept for human review)`, + ); + return { + kind: "borderline", + reason: + reason || + `model returned an unrecognized audience verdict ${JSON.stringify( + parsed.verdict, + )} (expected relevant | borderline | internal-ops); kept for human review`, + }; + } + + // AudienceJudge seam method (audience-gate.ts): the gate injects an + // OpenAIDistiller as its `AudienceJudge` and calls `.judge(...)`. Delegates to + // judgeAudience so the interface's method name and the seam's own method are + // one implementation — no drift between the two entry points. + judge(candidate: AudienceJudgeInput): Promise { + return this.judgeAudience(candidate); + } + async embed(text: string): Promise { const response = await this.client.embeddings.create({ model: this.embeddingModel, diff --git a/src/atlas/types.ts b/src/atlas/types.ts index 274b134..6cc3b65 100644 --- a/src/atlas/types.ts +++ b/src/atlas/types.ts @@ -91,6 +91,16 @@ export const RESTATEMENT_MARKER = "distillation:restatement"; // independently while the integration silently breaks. export const RAG_NO_DELTA_MARKER = "rag-dedup:no-delta"; +// The audience-scoping INTERNAL-OPS floor marker: the ONE literal the audience +// gate EMITS on a candidate it scoped to internal-ops (not shippable to the +// public corpus audience), and validate READS as a hard `approvable=false` +// floor the recompute cannot lift. Modeled on RESTATEMENT_MARKER / +// RAG_NO_DELTA_MARKER and carried the SAME way (a validated_against token +// and/or a `fused_from` evidence ref) so a reader recognizes all three idioms. +// Exported ONCE here so the emitter (audience gate) and the readers (validate's +// approvable floor + canonicalize's rank floor) import the SAME string. +export const INTERNAL_OPS_MARKER = "audience:internal-ops"; + // ── RAG-dedup semantic types (Theme B) ──────────────────────────────────────── // A single corpus passage returned by the SEMANTIC (pgvector cosine) retrieval @@ -252,6 +262,58 @@ export type Provenance = z.infer; export type CandidateFragment = z.infer; export type Candidate = z.infer; +// ── Shared approvable-floor markers (S0 FOUNDATION) ─────────────────────────── +// The COMPLETE set of DEDICATED floor markers an upstream gate stamps to force +// `approvable=false` on a candidate the recompute cannot lift. Owned HERE (next +// to the three marker literals) as the ONE source of truth so validate's +// approvable-floor and canonicalize's rank-floor read the SAME enumeration and +// can never drift. Any FUTURE gate that floors approvability just adds its +// dedicated marker to this array — both consumers pick it up automatically. +// +// - RESTATEMENT_MARKER (distillation gate): a pure restatement of already- +// indexed content — no NEW verifiable claim. +// - RAG_NO_DELTA_MARKER (rag-dedup no-delta gate): a pure corpus DUPLICATE +// with nothing net-new to re-seed. +// - INTERNAL_OPS_MARKER (audience gate): scoped to internal-ops, not +// shippable to the public corpus audience. +export const APPROVABLE_FLOOR_MARKERS = [ + RESTATEMENT_MARKER, + RAG_NO_DELTA_MARKER, + INTERNAL_OPS_MARKER, +] as const; + +// Whether the candidate carries a SPECIFIC dedicated floor `marker` on EITHER +// carrier idiom an upstream gate uses: a whole `"; "`-delimited token in +// `provenance.validated_against`, or a `fused_from` evidence ref. Whole-token / +// whole-ref matched (never a substring — one marker could be a prefix of +// another). This is the SHARED implementation of the whole-token match logic +// (mirrors validate's local `hasFloorMarker` byte-for-byte) so consumers read +// the floor from ONE place. +export function hasFloorMarker( + c: Pick, + marker: string, +): boolean { + const validatedAgainst = c.provenance.validated_against; + if ( + validatedAgainst && + validatedAgainst.split("; ").some((tok) => tok === marker) + ) { + return true; + } + return c.evidence.some((e) => e.kind === "fused_from" && e.ref === marker); +} + +// Whether the candidate is floored to `approvable=false` by ANY dedicated +// upstream floor marker in APPROVABLE_FLOOR_MARKERS. The ONE predicate both +// validate (approvable floor) and canonicalize (rank floor) import so the two +// floors are computed from the SAME marker set + match semantics and cannot +// drift. Cycle-free: this module imports nothing from validate/canonicalize. +export function isApprovableFloored( + c: Pick, +): boolean { + return APPROVABLE_FLOOR_MARKERS.some((marker) => hasFloorMarker(c, marker)); +} + // ── Canonical-key builder / parser ──────────────────────────────────────────── // Mirrors `subsystemHasNoDelimiter` above for the OTHER two key components: diff --git a/src/atlas/validate.ts b/src/atlas/validate.ts index 7c0dc55..ca3e8a7 100644 --- a/src/atlas/validate.ts +++ b/src/atlas/validate.ts @@ -35,8 +35,9 @@ import { lookupPill } from "./adapters/showcase.js"; import type { FeatureRegistry, PillStatus } from "./adapters/showcase.js"; import { BEHAVIOR_KNOWLEDGE_TYPES, - RAG_NO_DELTA_MARKER, RESTATEMENT_MARKER, + hasFloorMarker, + isApprovableFloored, } from "./types.js"; import type { Candidate, ValidationStatus } from "./types.js"; @@ -367,28 +368,6 @@ export function hasRestatementMarker( return hasFloorMarker(c, RESTATEMENT_MARKER); } -// Whether the candidate carries a DEDICATED floor marker `marker` on EITHER -// carrier idiom an upstream gate uses: a whole `"; "`-delimited token in -// `provenance.validated_against`, or a `fused_from` evidence ref. Whole-token / -// whole-ref matched (never a substring — one marker could be a prefix of -// another). Shared by the RESTATEMENT_MARKER reader (above) and the composed -// approvability floor (below), which check the two dedicated floor markers the -// upstream gates emit: distillation's RESTATEMENT_MARKER and rag-dedup's -// RAG_NO_DELTA_MARKER. -function hasFloorMarker( - c: Pick, - marker: string, -): boolean { - const validatedAgainst = c.provenance.validated_against; - if ( - validatedAgainst && - validatedAgainst.split("; ").some((tok) => tok === marker) - ) { - return true; - } - return c.evidence.some((e) => e.kind === "fused_from" && e.ref === marker); -} - // Promote a candidate through the validation ladder and enforce the binding // approvability rule. Returns a NEW Candidate; the input is not mutated. export async function promoteValidation( @@ -433,7 +412,7 @@ export async function promoteValidation( // COMPOSE upstream FLOORS — the recompute must never RAISE approvability above // a value an upstream GATE (not canonicalize's status rule) already floored it - // to. Two gates floor approvability for reasons the promoted status alone + // to. Several gates floor approvability for reasons the promoted status alone // cannot see, and EACH stamps a DEDICATED floor marker so the floor survives // this recompute: // @@ -441,23 +420,25 @@ export async function promoteValidation( // already-indexed content carries no NEW verifiable claim. // - RAG_NO_DELTA_MARKER (rag-dedup no-delta gate): a pure corpus DUPLICATE // with nothing net-new to re-seed (applyDistillDelta's no-delta floor). + // - INTERNAL_OPS_MARKER (audience gate): scoped to internal-ops, not + // shippable to the public corpus audience. // - // The old recompute honored ONLY the restatement marker, so a no-delta - // duplicate whose symbols grep-verify was clobbered back to approvable=true — - // silently defeating dedup's "duplicates aren't approvable" guarantee. The - // structural fix is to compose ALL dedicated floor markers GENERALLY. A - // dedicated marker is unambiguous: unlike the generic corpus-overlap - // ANNOTATION (stamped for EVERY overlap verdict, delta included, where the - // candidate stays approvable), a floor marker is emitted ONLY when the gate - // truly floors — so it fires the floor regardless of the incoming flag, and - // canonicalize's pure status-rule floor (which stamps NO marker, e.g. an - // unverified behavior fact) carries no floor here and is still LIFTED by - // `clearsValidationRule` on promotion, preserving the successfully-validated- - // behavior path. Any FUTURE gate that floors approvability just adds its - // dedicated marker to this set. - const upstreamFloored = - hasFloorMarker(c, RESTATEMENT_MARKER) || - hasFloorMarker(c, RAG_NO_DELTA_MARKER); + // An old recompute honored ONLY the restatement marker, so a floored + // duplicate/internal-ops candidate whose symbols grep-verify was clobbered + // back to approvable=true — silently defeating the upstream gate's guarantee. + // The structural fix composes ALL dedicated floor markers via the S0 shared + // `isApprovableFloored` predicate (the ONE source of truth over + // APPROVABLE_FLOOR_MARKERS, also read by canonicalize's rank floor — the two + // floors can never drift). A dedicated marker is unambiguous: unlike the + // generic corpus-overlap ANNOTATION (stamped for EVERY overlap verdict, delta + // included, where the candidate stays approvable), a floor marker is emitted + // ONLY when the gate truly floors — so it fires the floor regardless of the + // incoming flag, and canonicalize's pure status-rule floor (which stamps NO + // marker, e.g. an unverified behavior fact) carries no floor here and is still + // LIFTED by `clearsValidationRule` on promotion, preserving the successfully- + // validated-behavior path. Any FUTURE gate that floors approvability just adds + // its dedicated marker to APPROVABLE_FLOOR_MARKERS and both floors pick it up. + const upstreamFloored = isApprovableFloored(c); const approvable = clearsValidationRule && !upstreamFloored;