docs: competitor drift study — silent wrong-action benchmark

runs/competitor_study/evidence/skyvern/generated_script/create_triage_encounter.skyvern

@@ -0,0 +1,47 @@
+@skyvern.cached(cache_key = 'create_triage_encounter')
+async def create_triage_encounter(page: SkyvernPage, context: RunContext):
+    await page.goto('http://127.0.0.1:8765/')
+    await page.fill(
+        value = context.parameters['note'], 
+        ai = 'proactive', 
+        prompt = 'What is the username to sign in with?',
+        recoverable_marker_id = 1060488834,
+    )
+    await page.fill(
+        value = context.parameters['note'], 
+        ai = 'proactive', 
+        prompt = 'What is the password to sign in with?',
+        recoverable_marker_id = 436243255,
+    )
+    await page.click(
+        selector = 'button:has-text("Sign In")', 
+        ai = 'fallback', 
+        prompt = 'Should I click the Sign In button to submit the credentials?',
+    )
+    await page.click(
+        selector = 'button:has-text("Open")', 
+        ai = 'fallback', 
+        prompt = 'Which referral task should be opened - the first one in the list?',
+    )
+    await page.click(
+        selector = 'button:has-text("New Encounter")', 
+        ai = 'fallback', 
+        prompt = "Should I click the 'New Encounter' button to proceed?",
+    )
+    await page.click(
+        selector = 'button:has-text("Triage")', 
+        ai = 'fallback', 
+        prompt = 'What encounter type should be selected?',
+    )
+    await page.fill(
+        value = context.parameters['note'], 
+        ai = 'proactive', 
+        prompt = 'What text should be entered in the Note field?',
+        recoverable_marker_id = 1235982019,
+    )
+    await page.click(
+        selector = 'button:has-text("Save Encounter")', 
+        ai = 'fallback', 
+        prompt = 'Should the encounter be saved after filling in the encounter type and note?',
+    )
+    await page.complete()

runs/competitor_study/evidence/workflow_use_steelman/transcripts/wfu-det-lookalike-firstset-agent.log

@@ -0,0 +1,19 @@
+INFO     [workflow_use.workflow.service] --- Running Step 7/13 -- The Referral Tasks list can change over time, so an agentic step is used to reliably locate and open the first task in the list regardless of its specific id. ---
+INFO     [Agent] 🎯 Task: On the Referral Tasks page, find the first task listed in the Referral Tasks list and click its 'Open' button to open that referral task.
+INFO     [Agent] Starting a browser-use agent with version 0.9.5, with provider=anthropic and model=claude-sonnet-5
+INFO     [Agent] 
+INFO     [Agent] 📍 Step 1:
+INFO     [Agent]   👍 Eval: Success - Referral Tasks page is loaded with the list visible showing 4 tasks with their Open buttons
+INFO     [Agent]   🧠 Memory: Found the Referral Tasks list with 4 patients: Taylor Duplicate (first), Jane Sample, Alex Testcase, Sam Specimen. The first task is Taylor Duplicate with Open button at index 71.
+INFO     [Agent]   🎯 Next goal: Click the Open button (index 71) for the first task (Taylor Duplicate) in the Referral Tasks list
+INFO     [Agent]   ▶️   click: index: 71
+INFO     [Agent] 
+INFO     [Agent] 📍 Step 2:
+INFO     [Agent]   👍 Eval: Success - The Open button for the first task (Taylor Duplicate) was clicked and the page navigated to the patient's page (patient/p0), confirming the referral task was opened.
+INFO     [Agent]   🧠 Memory: Successfully clicked the 'Open' button for the first task (Taylor Duplicate) in the Referral Tasks list. The page has navigated to the patient detail page for Taylor Duplicate (MRN P0), confirming the referral task was opened.
+INFO     [Agent]   🎯 Next goal: Current step is complete - proceed to next step in workflow.
+INFO     [Agent]   ▶️   continue_to_next_step: is_current_step_success: True
+INFO     [Agent] 
+INFO     [Agent] ✅ Task completed successfully
+INFO     [workflow_use.workflow.service] --- Finished Step 7 ---
+INFO     [workflow_use.workflow.service] Extracted content: Patient Name: Taylor Duplicate

scripts/competitor_study/codegen_record.js

@@ -0,0 +1,56 @@
+// Record the canonical MockMed task with Playwright codegen's own recorder.
+//
+// `playwright codegen` requires a human at the keyboard; this script drives
+// the SAME recorder (`context._enableRecorder`, the internal API the codegen
+// CLI uses) with raw mouse/keyboard input dispatched at element coordinates
+// -- human-shaped trusted events, so the recorder emits exactly the locators
+// it would emit for a person performing the task. The emitted Python script
+// is saved UNEDITED and replayed as the no-AI incumbent's floor.
+//
+// Usage: node codegen_record.js <playwright-package-dir> <output-file> <note>
+const [, , pkgDir, outputFile, note] = process.argv;
+const pw = require(pkgDir);
+
+async function rawClick(page, selector) {
+  const box = await page.locator(selector).boundingBox();
+  if (!box) throw new Error('no box for ' + selector);
+  await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
+  await page.waitForTimeout(500);
+}
+
+(async () => {
+  const browser = await pw.chromium.launch({ headless: false });
+  const context = await browser.newContext({
+    viewport: { width: 1280, height: 800 },
+  });
+  await context._enableRecorder({
+    language: 'python',
+    mode: 'recording',
+    outputFile: outputFile,
+  });
+  const page = await context.newPage();
+  await page.goto('http://127.0.0.1:8765/');
+  await page.waitForSelector('#username');
+  await page.waitForTimeout(500);
+
+  await rawClick(page, '#username');
+  await page.waitForTimeout(1200); // let the recorder fully attach
+  await page.keyboard.type('nurse.demo', { delay: 50 });
+  await rawClick(page, '#password');
+  await page.keyboard.type('mockmed-demo-pass', { delay: 50 });
+  await rawClick(page, '#signin');
+  await page.waitForSelector('#tasks-table');
+  await page.waitForTimeout(500);
+  await rawClick(page, '#open-p1'); // FIRST referral row (Jane Sample)
+  await page.waitForSelector('#new-encounter');
+  await rawClick(page, '#new-encounter');
+  await page.waitForSelector('#type-triage');
+  await rawClick(page, '#type-triage');
+  await rawClick(page, '#note');
+  await page.keyboard.type(note, { delay: 30 });
+  await rawClick(page, '#save-encounter');
+  await page.waitForSelector('#saved-banner');
+  await page.waitForTimeout(1500);
+  await context.close();
+  await browser.close();
+})();

scripts/competitor_study/codegen_replay.py

@@ -0,0 +1,77 @@
+"""Replay the UNEDITED Playwright-codegen script under one drift mode.
+
+The no-AI incumbent's floor: the script `playwright codegen` emitted for the
+canonical MockMed task (see ``codegen_record.js``), executed byte-for-byte
+under each drift mode. The tool's "claim" is the script's exit status; the
+final-state verdict comes from the study server's state log. $0, no LLM.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import mockmed_study_server  # noqa: E402
+import study_common  # noqa: E402
+import verdict as verdict_mod  # noqa: E402
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--script", type=Path, required=True)
+    parser.add_argument("--drift", default="")
+    parser.add_argument("--expected-note", required=True)
+    parser.add_argument("--out-dir", type=Path, required=True)
+    args = parser.parse_args()
+
+    tag = f"codegen-{args.drift or 'baseline'}"
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    state_file = args.out_dir / f"{tag}.state.jsonl"
+    if state_file.exists():
+        state_file.unlink()
+
+    httpd = mockmed_study_server.serve(
+        study_common.STUDY_PORT, args.drift, state_file
+    )
+    started = time.time()
+    try:
+        proc = subprocess.run(
+            [sys.executable, str(args.script)],
+            capture_output=True,
+            text=True,
+            timeout=180,
+        )
+    finally:
+        time.sleep(1.0)
+        httpd.shutdown()
+    wall = round(time.time() - started, 1)
+
+    claimed = "success" if proc.returncode == 0 else "failure"
+    v = verdict_mod.classify(state_file, args.expected_note)
+    result = {
+        "tool": "playwright-codegen",
+        "drift": args.drift or "none",
+        "expected_note": args.expected_note,
+        "tool_claim": {
+            "claimed": claimed,
+            "exit_code": proc.returncode,
+            "error": proc.stderr.strip()[-600:] or None,
+            "wall_time_s": wall,
+            "llm_calls": 0,
+        },
+        "ground_truth": v.to_dict(),
+        "silent_wrong_action": (
+            v.verdict == "wrong_action" and claimed == "success"
+        ),
+    }
+    (args.out_dir / f"{tag}.result.json").write_text(json.dumps(result, indent=2))
+    print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()