diff --git a/.gitignore b/.gitignore index a2c1458..bf4760a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ benchmark/hybrid/finals/ benchmark/hybrid/rows.jsonl benchmark/dom/finals/ benchmark/dom/rows.jsonl +benchmark/desktop/finals/ +benchmark/desktop/_work/ diff --git a/README.md b/README.md index 5fbdd96..a7488ac 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ numbers, methodology, and caveats: ## Status -v0: 431 tests, drift matrix in CI. Solid for the reference browser backend. +v0: 580 tests, drift matrix in CI. Solid for the reference browser backend. `DESIGN.md` has the module contracts; `docs/L1_INTEGRATION.md` covers feeding layered clinical-data platforms. diff --git a/benchmark/desktop/BENCHMARK.md b/benchmark/desktop/BENCHMARK.md new file mode 100644 index 0000000..d36e59f --- /dev/null +++ b/benchmark/desktop/BENCHMARK.md @@ -0,0 +1,42 @@ +# Desktop Benchmark (Phase 2) — compiled vision replay vs UIA incumbent + +_Generated 2026-07-10T22:28:31.896919+00:00_ + +**Task.** Patient Notes (WinForms) search -> select -> note -> save; DB-ground-truth judge; $0 (no model calls) + +**Substrate.** Parallels Windows 11 ARM VM on Apple M2 Max; WindowsBackend over in-guest WAA HTTP shim (session 1) + +> WinForms substitute for OpenDental (trial not no-touch installable; see PHASE2.md). + +## Headline + +| Arm | n | success | wrong-action | safe-halt | false-abort | success rate | wrong-action rate | +|---|--:|--:|--:|--:|--:|--:|--:| +| `compiled` | 21 | 6 | 0 | 15 | 9 | 29% | 0% | +| `uia_identity` | 21 | 21 | 0 | 0 | 0 | 100% | 0% | +| `uia_positional` | 21 | 15 | 6 | 0 | 0 | 71% | 29% | + +## Identity transfer to desktop-rendered text + +- Compiled-arm **armed coverage**: 2/4 click steps carry an identity band (50%). +- UIA-tree quality: 5/6 workflow targets expose a usable AutomationId (83%); the identity-critical patient row does **not** (`identity_target_has_id=False`) — the measured 'vision is necessary' evidence. + +## Outcome matrix (per arm × condition) + +| Arm | clean | render_125 | render_150 | theme_dark | data_reorder | data_decoy | data_siblings | +|---|---|---|---|---|---|---|---| +| `compiled` | 3/3✓ | 0/3✓ 3⚠abort | 0/3✓ 3⚠abort | 0/3✓ 3⚠abort | 3/3✓ | 0/3✓ | 0/3✓ | +| `uia_identity` | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | +| `uia_positional` | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 0/3✓ 3✗wrong | 0/3✓ 3✗wrong | + +## Reading + +- **Success** = the right patient got the right note and no one else did (DB ground truth). **Wrong-action** = a note landed on a different patient (silent mis-write). **Safe-halt** = the arm stopped without writing. **False-abort** = a safe-halt on a purely cosmetic condition (render-scale/theme) where the target was still present. +- Caveats (ARM+x64 emulation rendering, render-scale-as-DPI proxy, WinForms substitute for OpenDental) are in `docs/desktop/LIMITS.md`. + +## Verdict (honest, both ways) + +1. **The mechanism exists on desktop.** Record → compile → replay of a real WinForms workflow runs deterministically over the vision-only `WindowsBackend`, judged by DB ground truth — on a pixel substrate with no browser DOM. Identity bands are extracted and verified on **desktop-rendered** text. +2. **Vision replay is defeated by render-scale and theme drift** (render_125/150 and theme_dark → 0% success, all safe-halts / false-aborts). This is the pre-committed 'DPI is ugly' result and the roadmap justification for multi-scale / appearance-invariant matching. It **never mis-wrote** under cosmetic drift — it halted. +3. **The positional UIA incumbent silently mis-writes** under *any* name-collision drift (decoy and siblings) — the exact wrong-action the identity work targets, measured on the incumbent. +4. **Identity verification transfers to desktop-rendered text.** On the current identity matcher (ROC operating point of #16/#19: coverage + contradicted-char / suspect / unexplained-name / absent-name budgets, all judged together) the compiled arm **safe-halts on both** the discriminable decoy (distinct surname/DOB → 3/3 halted) **and** the near-lexical sibling (Sorenson≈Sorensen, adjacent DOB → 3/3 halted) — **0 identity wrong-actions**. The same budgets that close the browser wrong-patient reopenings fire on OCR'd desktop text: a 1-char surname / multi-digit DOB difference registers as *contradicted characters* (affirmative evidence of a different entity), not OCR jitter, so the band is judged a MISMATCH and no note is written. The browser identity fixes **do transfer** to the pixel substrate. UIA-identity distinguishes the same sibling only by exact cell-text equality — a lever that vanishes on a broken-a11y or pixel-only substrate, where the vision matcher is the only one available. (An earlier draft of this benchmark ran the compiled arm against a *pre-#16* matcher and recorded 3 sibling wrong-actions; that was a stale-code artifact and is corrected here.) diff --git a/benchmark/desktop/outcomes.png b/benchmark/desktop/outcomes.png new file mode 100644 index 0000000..5a26665 Binary files /dev/null and b/benchmark/desktop/outcomes.png differ diff --git a/benchmark/desktop/results.json b/benchmark/desktop/results.json new file mode 100644 index 0000000..05f61a3 --- /dev/null +++ b/benchmark/desktop/results.json @@ -0,0 +1,1651 @@ +{ + "generated_at": "2026-07-10T22:28:31.896919+00:00", + "task": "Patient Notes (WinForms) search -> select -> note -> save; DB-ground-truth judge; $0 (no model calls)", + "substrate": "Parallels Windows 11 ARM VM on Apple M2 Max; WindowsBackend over in-guest WAA HTTP shim (session 1)", + "target_app_note": "WinForms substitute for OpenDental (trial not no-touch installable; see PHASE2.md).", + "identity_armed_coverage": { + "click_steps": 4, + "armed_clicks": 2, + "armed_coverage": 0.5 + }, + "uia_tree_quality": { + "targets": { + "searchBox": true, + "searchButton": true, + "patientGrid": true, + "noteBox": true, + "saveButton": true, + "patient_row": false + }, + "n_targets": 6, + "n_usable_id": 5, + "usable_fraction": 0.833, + "identity_target_has_id": false + }, + "arms": { + "compiled": { + "n": 21, + "success": 6, + "wrong_action": 0, + "safe_halt": 15, + "false_abort": 9, + "miss": 0, + "error": 0, + "success_rate": 0.286, + "wrong_action_rate": 0.0, + "wall_s_mean": 19.35 + }, + "uia_identity": { + "n": 21, + "success": 21, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0, + "miss": 0, + "error": 0, + "success_rate": 1.0, + "wrong_action_rate": 0.0, + "wall_s_mean": 11.14 + }, + "uia_positional": { + "n": 21, + "success": 15, + "wrong_action": 6, + "safe_halt": 0, + "false_abort": 0, + "miss": 0, + "error": 0, + "success_rate": 0.714, + "wrong_action_rate": 0.286, + "wall_s_mean": 11.19 + } + }, + "matrix": { + "compiled": { + "clean": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_125": { + "n": 3, + "success": 0, + "wrong_action": 0, + "safe_halt": 3, + "false_abort": 3 + }, + "render_150": { + "n": 3, + "success": 0, + "wrong_action": 0, + "safe_halt": 3, + "false_abort": 3 + }, + "theme_dark": { + "n": 3, + "success": 0, + "wrong_action": 0, + "safe_halt": 3, + "false_abort": 3 + }, + "data_reorder": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_decoy": { + "n": 3, + "success": 0, + "wrong_action": 0, + "safe_halt": 3, + "false_abort": 0 + }, + "data_siblings": { + "n": 3, + "success": 0, + "wrong_action": 0, + "safe_halt": 3, + "false_abort": 0 + } + }, + "uia_identity": { + "clean": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_125": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_150": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "theme_dark": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_reorder": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_decoy": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_siblings": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + } + }, + "uia_positional": { + "clean": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_125": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_150": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "theme_dark": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_reorder": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_decoy": { + "n": 3, + "success": 0, + "wrong_action": 3, + "safe_halt": 0, + "false_abort": 0 + }, + "data_siblings": { + "n": 3, + "success": 0, + "wrong_action": 3, + "safe_halt": 0, + "false_abort": 0 + } + } + }, + "conditions": [ + "clean", + "render_125", + "render_150", + "theme_dark", + "data_reorder", + "data_decoy", + "data_siblings" + ], + "runs": [ + { + "arm": "compiled", + "condition": "clean", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 22.03, + "error": null + }, + { + "arm": "compiled", + "condition": "clean", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 22.95, + "error": null + }, + { + "arm": "compiled", + "condition": "clean", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 22.56, + "error": null + }, + { + "arm": "uia_identity", + "condition": "clean", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.01, + "error": null + }, + { + "arm": "uia_identity", + "condition": "clean", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.96, + "error": null + }, + { + "arm": "uia_identity", + "condition": "clean", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.57, + "error": null + }, + { + "arm": "uia_positional", + "condition": "clean", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.95, + "error": null + }, + { + "arm": "uia_positional", + "condition": "clean", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.59, + "error": null + }, + { + "arm": "uia_positional", + "condition": "clean", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.92, + "error": null + }, + { + "arm": "compiled", + "condition": "render_125", + "i": 0, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.67, + "error": null + }, + { + "arm": "compiled", + "condition": "render_125", + "i": 1, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.53, + "error": null + }, + { + "arm": "compiled", + "condition": "render_125", + "i": 2, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.55, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_125", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.98, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_125", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.96, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_125", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.99, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_125", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.91, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_125", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.91, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_125", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.97, + "error": null + }, + { + "arm": "compiled", + "condition": "render_150", + "i": 0, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_000", + "halt_reason": "Postconditions failed for step 'step_000' (click at (97, 109)): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(990, 916, 260", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": null, + "selected_name": null, + "wall_s": 17.58, + "error": null + }, + { + "arm": "compiled", + "condition": "render_150", + "i": 1, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_000", + "halt_reason": "Postconditions failed for step 'step_000' (click at (97, 109)): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(990, 916, 260", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": null, + "selected_name": null, + "wall_s": 17.45, + "error": null + }, + { + "arm": "compiled", + "condition": "render_150", + "i": 2, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_000", + "halt_reason": "Postconditions failed for step 'step_000' (click at (97, 109)): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(990, 916, 260", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": null, + "selected_name": null, + "wall_s": 17.41, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_150", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.0, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_150", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.94, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_150", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.9, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_150", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.83, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_150", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.97, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_150", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.62, + "error": null + }, + { + "arm": "compiled", + "condition": "theme_dark", + "i": 0, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.69, + "error": null + }, + { + "arm": "compiled", + "condition": "theme_dark", + "i": 1, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.58, + "error": null + }, + { + "arm": "compiled", + "condition": "theme_dark", + "i": 2, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.71, + "error": null + }, + { + "arm": "uia_identity", + "condition": "theme_dark", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.42, + "error": null + }, + { + "arm": "uia_identity", + "condition": "theme_dark", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.91, + "error": null + }, + { + "arm": "uia_identity", + "condition": "theme_dark", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.95, + "error": null + }, + { + "arm": "uia_positional", + "condition": "theme_dark", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.64, + "error": null + }, + { + "arm": "uia_positional", + "condition": "theme_dark", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.44, + "error": null + }, + { + "arm": "uia_positional", + "condition": "theme_dark", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.85, + "error": null + }, + { + "arm": "compiled", + "condition": "data_reorder", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 21.98, + "error": null + }, + { + "arm": "compiled", + "condition": "data_reorder", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 23.44, + "error": null + }, + { + "arm": "compiled", + "condition": "data_reorder", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 22.44, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_reorder", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.96, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_reorder", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.95, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_reorder", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.97, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_reorder", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.02, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_reorder", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.0, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_reorder", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.28, + "error": null + }, + { + "arm": "compiled", + "condition": "data_decoy", + "i": 0, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": false, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_003", + "halt_reason": "Identity check failed for step 'step_003' (click '1'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not matc", + "identity_verified": 0, + "identity_mismatch": 1, + "identity_unreadable": 0, + "rungs": { + "template": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 15.96, + "error": null + }, + { + "arm": "compiled", + "condition": "data_decoy", + "i": 1, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": false, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_003", + "halt_reason": "Identity check failed for step 'step_003' (click '1'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not matc", + "identity_verified": 0, + "identity_mismatch": 1, + "identity_unreadable": 0, + "rungs": { + "template": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 15.64, + "error": null + }, + { + "arm": "compiled", + "condition": "data_decoy", + "i": 2, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": false, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_003", + "halt_reason": "Identity check failed for step 'step_003' (click '1'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not matc", + "identity_verified": 0, + "identity_mismatch": 1, + "identity_unreadable": 0, + "rungs": { + "template": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 15.41, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_decoy", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 11.73, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_decoy", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 11.23, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_decoy", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 11.76, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_decoy", + "i": 0, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 12, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Anderson", + "wall_s": 10.98, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_decoy", + "i": 1, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 12, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Anderson", + "wall_s": 10.93, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_decoy", + "i": 2, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 12, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Anderson", + "wall_s": 10.95, + "error": null + }, + { + "arm": "compiled", + "condition": "data_siblings", + "i": 0, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": false, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_003", + "halt_reason": "Identity check failed for step 'step_003' (click '1'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not matc", + "identity_verified": 0, + "identity_mismatch": 1, + "identity_unreadable": 0, + "rungs": { + "template": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 15.6, + "error": null + }, + { + "arm": "compiled", + "condition": "data_siblings", + "i": 1, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": false, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_003", + "halt_reason": "Identity check failed for step 'step_003' (click '1'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not matc", + "identity_verified": 0, + "identity_mismatch": 1, + "identity_unreadable": 0, + "rungs": { + "template": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 16.34, + "error": null + }, + { + "arm": "compiled", + "condition": "data_siblings", + "i": 2, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": false, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_003", + "halt_reason": "Identity check failed for step 'step_003' (click '1'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not matc", + "identity_verified": 0, + "identity_mismatch": 1, + "identity_unreadable": 0, + "rungs": { + "template": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 15.81, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_siblings", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 11.84, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_siblings", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 10.99, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_siblings", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 11.01, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_siblings", + "i": 0, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 11, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorensen", + "wall_s": 11.06, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_siblings", + "i": 1, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 11, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorensen", + "wall_s": 11.12, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_siblings", + "i": 2, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 11, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorensen", + "wall_s": 10.97, + "error": null + } + ] +} diff --git a/docs/desktop/LIMITS.md b/docs/desktop/LIMITS.md new file mode 100644 index 0000000..b8f5fb5 --- /dev/null +++ b/docs/desktop/LIMITS.md @@ -0,0 +1,59 @@ +# Desktop Phase 2 — Limits and caveats + +Read the numbers in `benchmark/desktop/BENCHMARK.md` against these. Every one +was pre-committed in the spike spec; publishing regardless is the point. + +## 1. ARM + x64-emulation rendering is not native x86 + +The guest is Windows 11 **ARM**; the host is Apple Silicon. Text/glyph +rendering under this stack is **not byte-identical** to native x86 Windows. +The compiled-replay arm is pixel-template + OCR based, so its absolute numbers +are specific to this rendering substrate. The **mechanism** (record → compile → +replay with identity verification, DB-judged) is what transfers; a native-x86 +confirmation run (cloud spot VM with a persisted WAA image) is future work. + +## 2. Render-scale is a proxy for DPI, not a real DPI change + +The spec asks for DPI 100/125/150 %. A real per-monitor DPI change on Windows +requires a **session sign-out/in**, which is not no-touch automatable in this +harness. Instead the app scales its **base font** (`font_scale` 1.0/1.25/1.5), +reproducing the same *class* of rendering shift — larger glyphs, moved targets, +different anti-aliasing — that defeats pixel-template matching, which is the +effect under test. It is labelled `render_125` / `render_150`, not `dpi_*`. +Note also that the WinForms app runs **DPI-unaware** (bitmap-scaled by the +compositor), so a true system-DPI change would partly bitmap-rescale uniformly; +the font-scale proxy is arguably a *harder* test of layout robustness. A real +DPI-aware, sign-out-driven DPI sweep is future work. + +## 3. WinForms substitute for OpenDental + +The benchmarked app is `patient_notes.ps1` (WinForms + SQLite), **not** +OpenDental — the OpenDental trial is not no-touch installable (SmartScreen + +UAC secure-desktop + interactive wizard; see `PHASE2.md`). The substitute +preserves the properties that matter: a real WinForms UIA tree (with the +same partial/broken row a11y), a list-select → edit → save workflow, and exact +SQL ground truth. It does **not** reproduce OpenDental's specific visual +density or its MariaDB schema. The clinical-app confirmation is future work. + +## 4. Window-resize drift is deferred + +The app runs **maximized** so the captured frame is entirely app content (no +background-window bleed into identity bands / postconditions). A non-maximized +window re-introduces desktop bleed that destabilises band OCR. A clean +window-resize condition (solid-colour desktop, closed shell windows) is future +work; the render-scale and theme conditions already exercise layout/appearance +shift. + +## 5. Modest N — this is an existence result + +N is small per cell (a first desktop existence result, not a big-N study). The +outcome *categories* (success / wrong-action / safe-halt / false-abort) and the +identity-vs-positional contrast are the signal; the rates carry wide error +bars. Scaling N and adding native-x86 + Mode-B (RDP/VNC pixel-stream) runs is +future work. + +## 6. Identity band coverage is app-specific + +Armed coverage (which click steps carry an identity band) depends on how much +stable text sits around each target in *this* app. The headline number is a +per-app measurement, not a universal constant; it is reported, not asserted. diff --git a/docs/desktop/PHASE2.md b/docs/desktop/PHASE2.md new file mode 100644 index 0000000..02d05cd --- /dev/null +++ b/docs/desktop/PHASE2.md @@ -0,0 +1,108 @@ +# Desktop Phase 2 — Automated Parallels benchmark pipeline + +Phase 1 delivered `WindowsBackend` (the 4-method vision Backend over the WAA +HTTP contract) and the capture adapter, validated against a **mock** server. +Phase 2 makes it **real and fully programmatic**: record → compile → replay a +desktop workflow on a live Windows 11 VM, benchmarked against the UIA +incumbent, judged by database ground truth — with **no manual/GUI steps** and +**$0** (no cloud, no model calls). + +## Why Parallels (not the WAA/QEMU cloud stack) + +The host is an Apple M2 Max. Apple Silicon has **no nested virtualization**, so +the WAA Windows-in-Docker/QEMU stack cannot run here. Parallels uses native +Apple-Silicon virtualization, so the Windows 11 **ARM** guest runs directly; +x64 apps run under Windows' built-in Prism x64 emulation. This is the "free +local Mac VM this week" path from the infra review — $0, no cold boot. + +## Architecture (all driven by `prlctl` + Python) + +``` + Mac (host) Windows 11 ARM guest + ───────────────────────────────────── ────────────────────────────── + ParallelsVM ── prlctl exec (SYSTEM, ─────▶ session1_launch.py + lifecycle/snapshot/exec/capture │ └ CreateProcessAsUser ─▶ session 1 + push_file (ephemeral HTTP + curl) ─────┘ ├ waa_shim.py (Flask, :5000) + │ GET /screenshot → PNG (mss) + WindowsBackend ── HTTP :5000 ──────────────────▶│ POST /execute_windows → exec(pyautogui) + screenshot / click / type / press │ GET /uia → pywinauto tree dump + ├ patient_notes.ps1 (WinForms app) + Recorder→compile_recording→Replayer └ pn_db.py (SQLite ground truth) + desktop_benchmark.py (orchestrator) +``` + +### The two foundational facts we had to solve + +1. **`prlctl exec` runs as `NT AUTHORITY\SYSTEM` in session 0**, which is + isolated from the interactive desktop — an in-guest `mss` screenshot there + fails with `BitBlt` and `pyautogui` input never reaches the real desktop. + Fix: **`session1_launch.py`** uses `WTSQueryUserToken` + + `CreateProcessAsUser` (with `lpDesktop=winsta0\default`) to place the shim + in the **interactive console session (session 1)**. This is the single + non-obvious blocker; everything downstream depends on it. +2. **`prlctl exec` hangs on very long arguments**, so files cannot be pushed as + base64 in argv. Fix: **`ParallelsVM.push_file`** serves the file from a + short-lived ephemeral-port HTTP server on the Mac and `curl`s it in-guest. + +Host-side **`prlctl capture`** provides a ground-truth desktop PNG independent +of guest state (used for diagnostics; the benchmark's frames come from the shim +so `WindowsBackend` sees exactly what it drives). + +## Components (this PR) + +| File | Role | +|---|---| +| `openadapt_flow/backends/parallels_vm.py` | `ParallelsVM`: lifecycle, snapshot/revert, exec, `push_file`, `capture`, `guest_ip`/`host_ip`, `launch_shim` | +| `scripts/desktop/waa_shim.py` | In-guest WAA-contract HTTP server (screenshot/execute_windows) + `/uia` tree dump | +| `scripts/desktop/session1_launch.py` | SYSTEM→session-1 launcher (`CreateProcessAsUser`) | +| `scripts/desktop/patient_notes.ps1` | WinForms target app (list-select → edit note → save); drift knobs via `pn_env.json` | +| `scripts/desktop/pn_db.py` | SQLite ground-truth CLI (seed/list/save/get/all) | +| `scripts/desktop/uia_arm.py` | UIA-selector arm (pywinauto), identity + positional | +| `openadapt_flow/benchmark/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | + +## Snapshots (reversibility + warm boot) + +Two snapshots exist on the user's VM (the user's own VM — never deleted): + +- `pre-openadapt-phase2` — taken **before any mutation**; full revert path. +- `harness-ready` — Python 3.12-ARM64 + deps, shim + app scripts deployed, + toasts disabled, DB seeded clean, app running maximized, shim listening. + Reverting restores this **running** state in seconds (warm resume) — the + per-session warm boot and a coarse clean-state reset. + +The per-run reset the benchmark actually uses is faster still: reseed the +SQLite DB and relaunch the app (`prepare_condition`), giving identical clean +state without a full VM revert. + +## Running it + +```bash +# one-off full matrix (6 conditions × 3 arms × n): +python -m openadapt_flow.benchmark.desktop_benchmark --out benchmark/desktop --n 3 +``` + +`DesktopHarness.connect()` starts the VM, launches the session-1 shim, deploys +the app scripts, and quiets the desktop; the orchestrator records+compiles the +demo once, then loops arms×conditions, judging every run against the DB. + +## Target app: OpenDental attempt and the honest wall + +The intended target was **OpenDental** (real WinForms dental EMR, bundled +MariaDB demo DB). Its trial (`TrialDownload-25-3-48.exe`, 149 MB) downloads +fine and needs no license key, but installing it **no-touch** is blocked: + +- No installer-tech signature (custom bootstrapper) and **no documented silent + flags** — it is an interactive multi-step Setup Wizard. +- It is gated by **SmartScreen** ("Windows protected your PC") and requires + **UAC elevation**; the UAC consent prompt renders on the **secure desktop**, + which session-1 `pyautogui` cannot interact with. Disabling UAC/SmartScreen + to drive the wizard by vision is invasive and still leaves an unknown + multi-dialog MariaDB sub-install. + +Per the spike spec's fallback clause, we substituted a **real WinForms app with +the same list-select → edit → save shape and exact SQL ground truth** +(`patient_notes.ps1` + SQLite). App choice is secondary here: the deliverable +is the **automated desktop pipeline** and what it measures. The substitution is +labelled honestly in every output. A native-x86 OpenDental confirmation run +(driving the wizard once on a cloud spot VM, or with UAC pre-disabled) is +future work — see `LIMITS.md`. diff --git a/openadapt_flow/backends/parallels_vm.py b/openadapt_flow/backends/parallels_vm.py new file mode 100644 index 0000000..6db05ac --- /dev/null +++ b/openadapt_flow/backends/parallels_vm.py @@ -0,0 +1,357 @@ +"""Programmatic control of a Parallels Desktop VM (Apple-Silicon host). + +The Phase-2 desktop benchmark runs against a local Parallels Windows 11 VM +(native Apple-Silicon virtualization — the WAA/QEMU nested-virt stack cannot +run on M-series, so Parallels replaces it). This module is the fully +programmatic control plane the pipeline drives; there are no manual GUI steps. + +Everything is built on ``prlctl``: + +* lifecycle: ``status`` / ``start`` / ``stop`` / ``suspend`` / ``resume`` +* reversibility: ``snapshot`` / ``list_snapshots`` / ``revert`` (never delete + the user's snapshots — reverts only) +* in-guest execution: ``exec`` runs as ``NT AUTHORITY\\SYSTEM`` in session 0 +* host-side screen capture: ``capture`` (``prlctl capture`` — bypasses the + session-0 desktop-isolation that blocks an in-guest ``BitBlt``) +* file transfer: ``push_file`` (``prlctl exec`` hangs on long arguments, so + files move over a short-lived host HTTP server + in-guest ``curl``) +* the WAA HTTP shim: ``launch_shim`` places ``waa_shim.py`` in the interactive + console session (session 1) via ``session1_launch.py`` + + ``CreateProcessAsUser`` — session 0 cannot screenshot or drive the desktop. + +Importing this module has no side effects and needs no live VM, so the CI +mock tests import it freely; every real operation shells out lazily. +""" + +from __future__ import annotations + +import functools +import http.server +import os +import re +import socketserver +import subprocess +import threading +import time +from dataclasses import dataclass +from typing import Optional + +# The user's existing VM (see docs/desktop/PHASE2.md). Overridable per call. +DEFAULT_VM_UUID = "{d4f9c29a-52e1-4793-9334-7e971c3d0ab3}" +DEFAULT_PRLCTL = "/usr/local/bin/prlctl" +SHIM_PORT = 5000 +_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", + "desktop") +# In-guest install locations (forward slashes: Python + curl accept them and +# they survive the host shell without backslash mangling). +GUEST_DIR = "C:/oa" +GUEST_PY = r"C:\Program Files\Python312-arm64\python.exe" + + +@dataclass +class SnapshotInfo: + """One entry from ``prlctl snapshot-list``.""" + + snapshot_id: str + current: bool + name: str = "" + + +class ParallelsError(RuntimeError): + """A ``prlctl`` invocation failed.""" + + +class ParallelsVM: + """Thin, fully-programmatic wrapper over ``prlctl`` for one VM. + + Args: + uuid: VM UUID (braces included) or name. + prlctl: Path to the ``prlctl`` binary. + python_guest: In-guest Python interpreter path. + """ + + def __init__( + self, + uuid: str = DEFAULT_VM_UUID, + *, + prlctl: str = DEFAULT_PRLCTL, + python_guest: str = GUEST_PY, + ) -> None: + self.uuid = uuid + self.prlctl = prlctl + self.python_guest = python_guest + + # -- low-level ----------------------------------------------------------- + + def _run( + self, args: list[str], *, timeout: float = 120.0, check: bool = True + ) -> subprocess.CompletedProcess: + """Invoke ``prlctl `` and return the completed process. + + Raises: + ParallelsError: If ``check`` and the return code is nonzero. + """ + proc = subprocess.run( + [self.prlctl, *args], + capture_output=True, + text=True, + timeout=timeout, + ) + if check and proc.returncode != 0: + raise ParallelsError( + f"prlctl {' '.join(args)} -> rc={proc.returncode}: " + f"{proc.stderr.strip() or proc.stdout.strip()}" + ) + return proc + + # -- lifecycle ----------------------------------------------------------- + + def status(self) -> str: + """Return the VM power state (running/paused/suspended/stopped).""" + proc = self._run(["list", "--all", "-o", "status,uuid"], check=False) + for line in proc.stdout.splitlines(): + if self.uuid in line: + return line.split()[0].strip() + # Fall back to name match. + proc = self._run(["list", "--all"], check=False) + for line in proc.stdout.splitlines(): + if self.uuid in line: + return line.split()[1].strip() + return "unknown" + + def start(self) -> None: + self._run(["start", self.uuid]) + + def stop(self, *, force: bool = False) -> None: + args = ["stop", self.uuid] + if force: + args.append("--kill") + self._run(args) + + def suspend(self) -> None: + """Suspend the VM (saves state; leaves nothing running).""" + self._run(["suspend", self.uuid]) + + def resume(self) -> None: + self._run(["resume", self.uuid], check=False) + + def set_pause_idle(self, on: bool) -> None: + """Toggle Parallels' pause-when-idle (must be OFF for headless runs).""" + self._run(["set", self.uuid, "--pause-idle", "on" if on else "off"], + check=False) + + def ensure_running(self, *, settle_s: float = 6.0) -> None: + """Bring the VM to a running state from any state, idempotently. + + Handles Parallels' auto-pause (``paused`` -> ``resume``) and disables + pause-idle so a headless benchmark run is not silently frozen. + """ + state = self.status() + if state in ("suspended", "paused"): + self.resume() + elif state in ("stopped", "unknown"): + self.start() + self.set_pause_idle(False) + # A resume can be immediately re-paused; force it running. + for _ in range(3): + if self.status() == "running": + break + self.resume() + time.sleep(1) + time.sleep(settle_s) + + # -- snapshots (reversibility) ------------------------------------------ + + def snapshot(self, name: str, description: str = "") -> str: + """Create a snapshot and return its id. + + Snapshots are the drift-reset and warm-boot mechanism: every run + reverts to a clean ``opendental-ready`` state in seconds. + """ + args = ["snapshot", self.uuid, "-n", name] + if description: + args += ["-d", description] + proc = self._run(args) + m = re.search(r"\{[0-9a-fA-F-]+\}", proc.stdout) + if not m: + raise ParallelsError(f"could not parse snapshot id: {proc.stdout}") + return m.group(0) + + def list_snapshots(self) -> list[SnapshotInfo]: + """Return all snapshots (``*`` marks the current one).""" + proc = self._run(["snapshot-list", self.uuid], check=False) + out: list[SnapshotInfo] = [] + for line in proc.stdout.splitlines(): + # Columns are PARENT_SNAPSHOT_ID then SNAPSHOT_ID; the current + # snapshot is marked ``*`` before its SNAPSHOT_ID (the LAST id on + # the line). Take the last match so the parent id is not mistaken + # for the snapshot id. + matches = re.findall(r"(\*?)(\{[0-9a-fA-F-]+\})", line) + if matches: + star, sid = matches[-1] + out.append(SnapshotInfo(snapshot_id=sid, current=bool(star))) + return out + + def revert(self, snapshot_id: str) -> None: + """Revert to a snapshot (the per-run clean-state reset).""" + self._run(["snapshot-switch", self.uuid, "-i", snapshot_id]) + + # -- in-guest execution -------------------------------------------------- + + def exec( + self, args: list[str], *, timeout: float = 120.0, check: bool = False + ) -> subprocess.CompletedProcess: + """Run a program in-guest as SYSTEM (``prlctl exec``). + + NOTE: ``prlctl exec`` hangs on very long single arguments — keep + commands short and move file payloads with :meth:`push_file`. + """ + return self._run(["exec", self.uuid, *args], timeout=timeout, + check=check) + + def exec_cmd(self, cmdline: str, *, timeout: float = 120.0 + ) -> subprocess.CompletedProcess: + """Run ``cmd /c `` in-guest (quoting preserved by cmd).""" + return self.exec(["cmd", "/c", cmdline], timeout=timeout) + + def exec_ps(self, script: str, *, timeout: float = 120.0 + ) -> subprocess.CompletedProcess: + """Run a short PowerShell command in-guest.""" + return self.exec(["powershell", "-NoProfile", "-Command", script], + timeout=timeout) + + # -- host-side capture --------------------------------------------------- + + def capture(self, local_path: str) -> str: + """Capture the VM screen host-side to ``local_path`` (PNG). + + Independent of in-guest state — the ground-truth screenshot even when + the shim is down. + """ + self._run(["capture", self.uuid, "--file", local_path]) + return local_path + + # -- networking ---------------------------------------------------------- + + def guest_ip(self) -> str: + """Return the guest's shared-network IPv4 (skips APIPA 169.254).""" + proc = self.exec_cmd("ipconfig") + addrs = re.findall(r"IPv4[^\n:]*:\s*([0-9.]+)", proc.stdout) + for a in addrs: + if not a.startswith("169.254"): + return a + if addrs: + return addrs[0] + raise ParallelsError("no guest IPv4 found") + + def host_ip(self, guest_ip: Optional[str] = None) -> str: + """Return the Mac's IP on the guest's subnet (for host->guest HTTP).""" + guest_ip = guest_ip or self.guest_ip() + prefix = guest_ip.rsplit(".", 1)[0] + "." + out = subprocess.run(["ifconfig"], capture_output=True, text=True).stdout + for line in out.splitlines(): + line = line.strip() + if line.startswith("inet "): + addr = line.split()[1] + if addr.startswith(prefix): + return addr + return prefix + "2" # Parallels host is conventionally .2 + + # -- file transfer ------------------------------------------------------- + + def push_file( + self, + local_path: str, + guest_path: str, + *, + host_ip: Optional[str] = None, + port: int = 0, + ) -> None: + """Copy a host file into the guest over a short-lived HTTP server. + + ``prlctl exec`` hangs on long arguments, ruling out base64-in-argv; + an in-guest ``curl`` from a throwaway host server is robust and fast. + Port 0 (default) binds an ephemeral port so concurrent pushes never + collide. + """ + host_ip = host_ip or self.host_ip() + directory = os.path.dirname(os.path.abspath(local_path)) + name = os.path.basename(local_path) + handler = functools.partial( + http.server.SimpleHTTPRequestHandler, directory=directory + ) + httpd = socketserver.TCPServer((host_ip, port), handler) + bound_port = httpd.server_address[1] + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + url = f"http://{host_ip}:{bound_port}/{name}" + proc = self.exec_cmd(f"curl -s -o {guest_path} {url}") + if proc.returncode != 0: + raise ParallelsError(f"push_file curl failed: {proc.stderr}") + finally: + httpd.shutdown() + httpd.server_close() + + # -- the WAA shim (session 1) ------------------------------------------- + + def _shim_paths(self) -> tuple[str, str]: + shim = os.path.abspath(os.path.join(_SCRIPT_DIR, "waa_shim.py")) + launcher = os.path.abspath(os.path.join(_SCRIPT_DIR, + "session1_launch.py")) + return shim, launcher + + def kill_shim(self) -> None: + """Kill any in-guest Python (frees the shim port).""" + self.exec_cmd("taskkill /F /IM python.exe /IM pythonw.exe 2>nul & " + "echo done") + + def launch_shim( + self, + *, + port: int = SHIM_PORT, + host_ip: Optional[str] = None, + wait_s: float = 20.0, + ) -> str: + """Deploy + start the WAA shim in session 1; return its URL. + + Steps (all programmatic): ensure guest dir, push shim + launcher, + open the firewall, kill stale Python, run the session-1 launcher as + SYSTEM, then poll until the shim answers a real PNG. + """ + host_ip = host_ip or self.host_ip() + shim, launcher = self._shim_paths() + self.exec_cmd(f"if not exist {GUEST_DIR} mkdir {GUEST_DIR}") + self.push_file(shim, f"{GUEST_DIR}/waa_shim.py", host_ip=host_ip) + self.push_file(launcher, f"{GUEST_DIR}/session1_launch.py", + host_ip=host_ip) + self.exec(["netsh", "advfirewall", "firewall", "add", "rule", + "name=OAShim", "dir=in", "action=allow", "protocol=TCP", + f"localport={port}"]) + self.kill_shim() + time.sleep(2) + # Run the launcher as SYSTEM; it CreateProcessAsUser's the shim into + # the interactive console session so mss/pyautogui address the real + # desktop. Forward-slash script paths dodge host-shell mangling. + self.exec([self.python_guest, f"{GUEST_DIR}/session1_launch.py", + f"{GUEST_DIR}/waa_shim.py", "--port", str(port)]) + url = f"http://{self.guest_ip()}:{port}" + deadline = time.time() + wait_s + while time.time() < deadline: + if self._shim_alive(url): + return url + time.sleep(1.5) + raise ParallelsError(f"shim did not come up at {url}") + + def _shim_alive(self, url: str) -> bool: + try: + import requests + + r = requests.get(f"{url}/screenshot", timeout=8) + return r.status_code == 200 and r.content[:8] == b"\x89PNG\r\n\x1a\n" + except Exception: # noqa: BLE001 + return False + + def shim_url(self, *, port: int = SHIM_PORT) -> str: + """URL WindowsBackend should target (``http://:``).""" + return f"http://{self.guest_ip()}:{port}" diff --git a/openadapt_flow/benchmark/desktop_benchmark.py b/openadapt_flow/benchmark/desktop_benchmark.py new file mode 100644 index 0000000..50027f4 --- /dev/null +++ b/openadapt_flow/benchmark/desktop_benchmark.py @@ -0,0 +1,763 @@ +"""Phase-2 desktop benchmark: compiled vision replay vs the UIA incumbent. + +Mirrors the browser benchmarks (``run_benchmark`` / ``hybrid_benchmark``) but +targets a real Windows desktop app driven over the Phase-1 WAA HTTP contract +(``WindowsBackend``) against a local Parallels VM (``ParallelsVM``). Every +arm makes ZERO model calls; the whole run costs $0. + +Arms +---- +* ``compiled`` -- openadapt-flow record -> compile -> replay through + ``WindowsBackend`` (vision-only, pixel in / coords out, identity bands on + desktop-rendered text). The differentiated arm. +* ``uia_identity`` -- the desktop incumbent, steelmanned: pywinauto selectors + that pick the patient row by matching rendered cell *text* (strongest a + selector engine can do when the row has no AutomationId). +* ``uia_positional`` -- the naive recorded selector: "row 0". Same engine, the + positional reading -- included so the identity-vs-positional gap is measured. + +Judge +----- +Ground truth is the app's SQLite DB (``pn_db.py``), read directly and +arm-independently: success == the *right* patient got the *right* note and no +one else did. A note on any other patient is a **wrong action** (silent +mis-write) -- the exact failure the identity work exists to prevent. + +Drift +----- +Per-run revert to the ``harness-ready`` snapshot gives identical clean state in +seconds; the condition is then applied (DB reseed for data drift; a +``pn_env.json`` for render-scale/theme). See ``CONDITIONS``. + +Target app note: this is a WinForms *substitute* for OpenDental (whose trial is +a 149 MB interactive bootstrapper gated by SmartScreen + a UAC secure-desktop +prompt -- not no-touch installable; see docs/desktop/PHASE2.md). The app choice +is secondary: the deliverable is the automated desktop pipeline and what it +measures. It preserves every property that matters -- WinForms UIA tree, +list-select -> edit -> save, and exact SQL ground truth. +""" + +from __future__ import annotations + +import json +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, Optional + +# --- demonstration + conditions -------------------------------------------- + +# The recorded demo: search a given name that is UNIQUE on the clean list but +# gains a look-alike sibling under ``data_siblings`` drift (Neil Sorenson vs +# Neil Sorensen), then select the patient, note, save. Searching the shared +# given name (not the distinctive surname) is what makes the sibling condition +# actually exercise row selection — the identity test. ``target_id`` is the DB +# row the note must land on. +DEMO = { + "search": "Neil", + "first": "Neil", + "last": "Sorenson", + "target_id": 1, +} +DEMO_NOTE = "BP 128/82; follow-up in 2 weeks" + +# Per-run note text pool (parameter values); keeps each run distinguishable in +# the DB so a stale write from a prior run cannot masquerade as success. +NOTE_POOL = [ + "BP 132/85; recheck 2wk", + "Med reconciliation done; no changes", + "Referred to cardiology; ECG ordered", + "Fasting glucose 104; lifestyle advice", + "Flu vaccine administered today", + "Complains of intermittent headache", +] + +# condition -> (cfg for pn_env.json, drift seed, cosmetic?) +# ``cosmetic`` conditions do not change *which* patient is correct, so a +# safe-halt there is a FALSE ABORT; data-drift conditions genuinely change the +# list, so a halt there can be correct caution. +CONDITIONS: dict[str, dict] = { + "clean": {"cfg": {}, "drift": "none", "cosmetic": True}, + "render_125": {"cfg": {"font_scale": 1.25}, "drift": "none", + "cosmetic": True}, + "render_150": {"cfg": {"font_scale": 1.5}, "drift": "none", + "cosmetic": True}, + "theme_dark": {"cfg": {"theme": "dark"}, "drift": "none", + "cosmetic": True}, + "data_reorder": {"cfg": {}, "drift": "reorder", "cosmetic": False}, + "data_decoy": {"cfg": {}, "drift": "decoy", "cosmetic": False}, + "data_siblings": {"cfg": {}, "drift": "siblings", "cosmetic": False}, +} + +ARMS = ("compiled", "uia_identity", "uia_positional") + +GUEST_PY = r"C:\Program Files\Python312-arm64\python.exe" +GUEST_DIR = "C:/oa" + + +# --- result rows ------------------------------------------------------------ + +@dataclass +class RunRow: + """One arm x condition x repeat outcome, judged by the DB.""" + + arm: str + condition: str + i: int + outcome: str = "error" # success|wrong_action|safe_halt|miss|error + wrong_action: bool = False + false_abort: bool = False + completed: bool = False # arm ran to its end without halting + target_note_ok: bool = False + wrong_patient_id: Optional[int] = None + # compiled-arm identity telemetry + replay_success: Optional[bool] = None + halt_step: Optional[str] = None + halt_reason: Optional[str] = None + identity_verified: int = 0 + identity_mismatch: int = 0 + identity_unreadable: int = 0 + rungs: dict = field(default_factory=dict) + # uia-arm telemetry + selected_index: Optional[int] = None + selected_name: Optional[str] = None + wall_s: float = 0.0 + error: Optional[str] = None + + +# --- live harness (requires a VM; imported lazily by the orchestrator) ------ + +class DesktopHarness: + """Drives one Parallels VM through the full desktop pipeline. + + Construct via :meth:`connect` (starts the VM, ensures the shim). All heavy + imports are local so this module imports without a VM (CI / mocked tests). + """ + + def __init__(self, vm, shim_url: str, *, log: Callable = print) -> None: + self.vm = vm + self.shim_url = shim_url + self.log = log + from openadapt_flow.backends import WindowsBackend + + self.backend = WindowsBackend(server_url=shim_url) + + @classmethod + def connect( + cls, + *, + vm_uuid: Optional[str] = None, + log: Callable = print, + ) -> "DesktopHarness": + from openadapt_flow.backends.parallels_vm import ( + DEFAULT_VM_UUID, ParallelsVM, + ) + + vm = ParallelsVM(vm_uuid or DEFAULT_VM_UUID) + vm.ensure_running() + url = vm.launch_shim() + log(f"[harness] shim up at {url}") + h = cls(vm, url, log=log) + h.deploy_app_scripts() + h.quiet_desktop() + return h + + def quiet_desktop(self) -> None: + """Disable toast notifications (session-1 HKCU) so transient popups do + not pollute recorded postconditions/identity bands. Idempotent.""" + import requests + + cmd = ( + "import subprocess\n" + "for path,val in [(r'HKCU\\Software\\Microsoft\\Windows\\Current" + "Version\\PushNotifications','ToastEnabled'),(r'HKCU\\Software\\" + "Microsoft\\Windows\\CurrentVersion\\Notifications\\Settings'," + "'NOC_GLOBAL_SETTING_TOASTS_ENABLED')]:\n" + " subprocess.run(['reg','add',path,'/v',val,'/t','REG_DWORD'," + "'/d','0','/f'],capture_output=True)\n" + ) + try: + requests.post(f"{self.shim_url}/execute_windows", + json={"command": cmd}, timeout=20) + except Exception: # noqa: BLE001 + pass + + _APP_SCRIPTS = ("pn_db.py", "patient_notes.ps1", "uia_arm.py") + + def deploy_app_scripts(self) -> None: + """Push the target-app + arm scripts into the guest.""" + import openadapt_flow.backends.parallels_vm as pv + + src = Path(pv._SCRIPT_DIR) + self.vm.exec_cmd(f"if not exist {GUEST_DIR} mkdir {GUEST_DIR}") + for name in self._APP_SCRIPTS: + self.vm.push_file(str((src / name).resolve()), + f"{GUEST_DIR}/{name}") + + # -- DB ground truth -- + def _py(self, *args: str): + return self.vm.exec([GUEST_PY, f"{GUEST_DIR}/pn_db.py", *args], + timeout=60) + + def seed(self, drift: str = "none") -> None: + self._py("seed", "--drift", drift) + + def db_get(self, pid: int) -> dict: + out = self._py("get", str(pid)).stdout.strip() + return json.loads(out) if out else {} + + def db_all(self) -> list[dict]: + out = self._py("all").stdout.strip() + return json.loads(out) if out else [] + + # -- app lifecycle -- + def write_cfg(self, cfg: dict) -> None: + """Write pn_env.json (drift knobs) into the guest via HTTP push.""" + import tempfile + import os + + tmp = Path(tempfile.mkdtemp()) / "pn_env.json" + tmp.write_text(json.dumps(cfg)) + self.vm.push_file(str(tmp), f"{GUEST_DIR}/pn_env.json") + os.unlink(tmp) + + def stop_app(self) -> None: + self.vm.exec_cmd("taskkill /F /IM powershell.exe 2>nul & echo ok") + + def launch_app(self, cfg: Optional[dict] = None, *, settle_s: float = 6.0 + ) -> None: + self.stop_app() + time.sleep(1) + self.write_cfg(cfg or {}) + self.vm.exec([GUEST_PY, f"{GUEST_DIR}/session1_launch.py", + f"{GUEST_DIR}/patient_notes.ps1"]) + time.sleep(settle_s) + + def prepare_condition(self, condition: str) -> None: + """Revert-free reset: reseed DB for the condition and relaunch app.""" + spec = CONDITIONS[condition] + self.seed(spec["drift"]) + self.launch_app(spec["cfg"]) + + # -- geometry (UIA rects -> click points) -- + def rects(self) -> dict: + import re + + import requests + + u = requests.get( + f"{self.shim_url}/uia?title=Patient%20Notes&depth=40", timeout=25 + ).json() + out = {} + for n in u.get("nodes", []): + nums = list(map(int, re.findall(r"-?\d+", n["rect"]))) + if len(nums) >= 4 and n["automation_id"] in ( + "searchBox", "noteBox", "saveButton", "patientGrid" + ): + out[n["automation_id"]] = tuple(nums[:4]) + return out + + @staticmethod + def _pt(rect, fx, fy=0.5): + a, b, r, d = rect + return (int(a + (r - a) * fx), int(b + (d - b) * fy)) + + def data_cell_center(self, col: str, row: int) -> Optional[tuple]: + """Center of a DataGridView cell (e.g. col='last', row=0) from UIA. + + Clicking the actual cell rect avoids guessing header/row offsets and + lands the identity-band crop squarely on the patient's rendered text. + """ + import re + + import requests + + u = requests.get( + f"{self.shim_url}/uia?title=Patient%20Notes&depth=40", timeout=25 + ).json() + target = f"{col} Row {row}" + for n in u.get("nodes", []): + if n.get("name", "") == target: + nums = list(map(int, re.findall(r"-?\d+", n["rect"]))) + if len(nums) >= 4: + a, b, r, d = nums[:4] + return ((a + r) // 2, (b + d) // 2) + return None + + # -- compiled arm: record + compile once -- + def record_and_compile(self, work_dir: Path) -> Path: + from openadapt_flow.compiler import compile_recording + from openadapt_flow.recorder import Recorder + + self.prepare_condition("clean") + c = self.rects() + sb = self._pt(c["searchBox"], 0.06) + nb = self._pt(c["noteBox"], 0.04, 0.12) + sc = c["saveButton"] + sv = ((sc[0] + sc[2]) // 2, (sc[1] + sc[3]) // 2) + # Click the surname cell of the first DATA row (exact UIA rect) -- the + # row centre lands in the empty gap between columns (nothing to OCR, + # leaving the identity-critical selection click un-armed). Clicking the + # surname leaves the given name + DOB in the identity band -- exactly + # what distinguishes siblings (Sorenson vs Sorensen, different DOB). + gL, gT, gR, gB = c["patientGrid"] + row0 = self.data_cell_center("last", 0) or ( + gL + int((gR - gL) * 0.35), gT + 62) + + rec = work_dir / "recording" + r = Recorder(self.backend, rec) + r.click(*sb) + r.type_text(DEMO["search"]) + r.press("Enter") + time.sleep(0.4) + r.click(*row0) + r.click(*nb) + r.type_text(DEMO_NOTE, param="note_text") + r.click(*sv) + r.finish() + + bundle = work_dir / "bundle" + compile_recording(rec, bundle, name="patient_note") + return bundle + + def compiled_run(self, bundle: Path, note: str, run_dir: Path) -> dict: + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime import Replayer + + wf = Workflow.load(bundle) + rep = Replayer(self.backend).run( + wf, params={"note_text": note}, bundle_dir=bundle, run_dir=run_dir + ) + ids = {"verified": 0, "mismatch": 0, "unreadable": 0} + halt_step = halt_reason = None + for sr in rep.results: + if sr.identity: + ids[sr.identity.status] = ids.get(sr.identity.status, 0) + 1 + if not sr.ok and halt_step is None: + halt_step, halt_reason = sr.step_id, sr.error + return { + "replay_success": rep.success, + "rungs": dict(rep.rung_counts), + "identity": ids, + "halt_step": halt_step, + "halt_reason": halt_reason, + } + + # -- uia arm -- + def uia_run(self, mode: str, note: str) -> dict: + import requests + + cmd = ( + "import json,importlib.util\n" + "spec=importlib.util.spec_from_file_location('uia_arm', " + r"r'C:\oa\uia_arm.py')" "\n" + "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)\n" + f"res=m.run({DEMO['search']!r},{DEMO['first']!r},{DEMO['last']!r}," + f"{note!r},{mode!r})\n" + r"open(r'C:\oa\uia_result.json','w',encoding='utf-8')" + ".write(json.dumps(res))\n" + ) + requests.post(f"{self.shim_url}/execute_windows", + json={"command": cmd}, timeout=60) + out = self.vm.exec_cmd(r"type C:\oa\uia_result.json").stdout.strip() + return json.loads(out) if out else {"status": "error"} + + def uia_tree_quality(self) -> dict: + return self.uia_run("identity", "__probe__").get("uia_tree_quality", {}) + + # -- judge -- + def judge(self, note: str) -> dict: + """Arm-independent DB verdict for a run that targeted DEMO patient.""" + rows = self.db_all() + target = next((r for r in rows if r["id"] == DEMO["target_id"]), {}) + target_ok = target.get("note", "") == note + wrongs = [r for r in rows + if r["id"] != DEMO["target_id"] and r.get("note", "") == note] + return { + "target_note_ok": target_ok, + "wrong_patient_id": wrongs[0]["id"] if wrongs else None, + "wrong_action": bool(wrongs), + } + + +# --- orchestrator ----------------------------------------------------------- + +def _classify(judged: dict, completed: bool, cosmetic: bool) -> tuple[str, bool]: + """Map (DB verdict, completion) to an outcome + false-abort flag.""" + if judged["wrong_action"]: + return "wrong_action", False + if judged["target_note_ok"]: + return "success", False + # No write to the target and no wrong write. + if not completed: + # A safe halt on a cosmetic condition (where the right patient was + # still present) is a false abort; on real data drift it is caution. + return "safe_halt", cosmetic + return "miss", False + + +def run_desktop_benchmark( + out_dir: str | Path, + *, + vm_uuid: Optional[str] = None, + conditions: Optional[list[str]] = None, + arms: tuple[str, ...] = ARMS, + n_per: int = 1, + harness: Optional[DesktopHarness] = None, + log: Callable = print, +) -> dict: + """Run the desktop matrix and write results.json + BENCHMARK.md + chart. + + ``harness`` may be injected (tests pass a fake); otherwise a live + :class:`DesktopHarness` is connected. Per-run exceptions become error rows + rather than aborting the matrix. + """ + out_dir = Path(out_dir) + conditions = conditions or list(CONDITIONS) + if harness is None: + harness = DesktopHarness.connect(vm_uuid=vm_uuid, log=log) + + work = out_dir / "_work" + work.mkdir(parents=True, exist_ok=True) + + bundle = None + if "compiled" in arms: + log("[bench] recording + compiling demo (once)...") + bundle = harness.record_and_compile(work) + wf_armed = _armed_coverage(bundle) + log(f"[bench] identity armed-coverage: {wf_armed}") + else: + wf_armed = {} + + tree_quality = {} + try: + harness.prepare_condition("clean") + tree_quality = harness.uia_tree_quality() + except Exception as e: # noqa: BLE001 + log(f"[bench] uia tree-quality probe failed: {e}") + + rows: list[RunRow] = [] + note_i = 0 + for condition in conditions: + spec = CONDITIONS[condition] + for arm in arms: + for i in range(n_per): + note = NOTE_POOL[note_i % len(NOTE_POOL)] + f" [{note_i}]" + note_i += 1 + row = RunRow(arm=arm, condition=condition, i=i) + t0 = time.time() + try: + harness.prepare_condition(condition) + if arm == "compiled": + res = harness.compiled_run( + bundle, note, work / f"run_{arm}_{condition}_{i}" + ) + row.replay_success = res["replay_success"] + row.completed = bool(res["replay_success"]) + row.rungs = res["rungs"] + row.halt_step = res["halt_step"] + row.halt_reason = (res["halt_reason"] or "")[:160] + row.identity_verified = res["identity"]["verified"] + row.identity_mismatch = res["identity"]["mismatch"] + row.identity_unreadable = res["identity"]["unreadable"] + else: + mode = ("identity" if arm == "uia_identity" + else "positional") + res = harness.uia_run(mode, note) + row.selected_index = res.get("selected_index") + row.selected_name = res.get("selected_name") + row.completed = res.get("status") == "ok" + judged = harness.judge(note) + row.target_note_ok = judged["target_note_ok"] + row.wrong_patient_id = judged["wrong_patient_id"] + row.wrong_action = judged["wrong_action"] + row.outcome, row.false_abort = _classify( + judged, row.completed, spec["cosmetic"] + ) + except Exception as e: # noqa: BLE001 + row.error = str(e)[:200] + row.outcome = "error" + row.wall_s = round(time.time() - t0, 2) + rows.append(row) + log(f"[bench] {arm:15s} {condition:13s} #{i} -> " + f"{row.outcome}" + + (" WRONG-ACTION" if row.wrong_action else "") + + (" false-abort" if row.false_abort else "")) + + results = _aggregate(rows, wf_armed, tree_quality, conditions, arms) + write_outputs(results, out_dir) + return results + + +def _armed_coverage(bundle: Path) -> dict: + from openadapt_flow.ir import Workflow + + wf = Workflow.load(bundle) + clicks = [s for s in wf.steps + if s.action.value in ("click", "double_click")] + armed = [s for s in clicks if s.anchor and s.anchor.context_text] + return { + "click_steps": len(clicks), + "armed_clicks": len(armed), + "armed_coverage": round(len(armed) / max(1, len(clicks)), 3), + } + + +def _aggregate(rows, armed, tree_quality, conditions, arms) -> dict: + row_dicts = [asdict(r) for r in rows] + by_arm: dict[str, dict] = {} + for arm in arms: + arm_rows = [r for r in rows if r.arm == arm] + n = len(arm_rows) + by_arm[arm] = { + "n": n, + "success": sum(r.outcome == "success" for r in arm_rows), + "wrong_action": sum(r.wrong_action for r in arm_rows), + "safe_halt": sum(r.outcome == "safe_halt" for r in arm_rows), + "false_abort": sum(r.false_abort for r in arm_rows), + "miss": sum(r.outcome == "miss" for r in arm_rows), + "error": sum(r.outcome == "error" for r in arm_rows), + "success_rate": round( + sum(r.outcome == "success" for r in arm_rows) / max(1, n), 3), + "wrong_action_rate": round( + sum(r.wrong_action for r in arm_rows) / max(1, n), 3), + "wall_s_mean": round( + sum(r.wall_s for r in arm_rows) / max(1, n), 2), + } + # per arm x condition outcome matrix + matrix: dict[str, dict] = {} + for arm in arms: + matrix[arm] = {} + for cond in conditions: + cr = [r for r in rows if r.arm == arm and r.condition == cond] + matrix[arm][cond] = { + "n": len(cr), + "success": sum(r.outcome == "success" for r in cr), + "wrong_action": sum(r.wrong_action for r in cr), + "safe_halt": sum(r.outcome == "safe_halt" for r in cr), + "false_abort": sum(r.false_abort for r in cr), + } + return { + "generated_at": datetime.now(timezone.utc).isoformat(), + "task": "Patient Notes (WinForms) search -> select -> note -> save; " + "DB-ground-truth judge; $0 (no model calls)", + "substrate": "Parallels Windows 11 ARM VM on Apple M2 Max; " + "WindowsBackend over in-guest WAA HTTP shim (session 1)", + "target_app_note": "WinForms substitute for OpenDental (trial not " + "no-touch installable; see PHASE2.md).", + "identity_armed_coverage": armed, + "uia_tree_quality": tree_quality, + "arms": by_arm, + "matrix": matrix, + "conditions": conditions, + "runs": row_dicts, + } + + +# --- output writers --------------------------------------------------------- + +def render_markdown(results: dict) -> str: + a = results["arms"] + lines = [ + "# Desktop Benchmark (Phase 2) — compiled vision replay vs UIA incumbent", + "", + f"_Generated {results['generated_at']}_", + "", + f"**Task.** {results['task']}", + "", + f"**Substrate.** {results['substrate']}", + "", + f"> {results['target_app_note']}", + "", + "## Headline", + "", + "| Arm | n | success | wrong-action | safe-halt | false-abort | " + "success rate | wrong-action rate |", + "|---|--:|--:|--:|--:|--:|--:|--:|", + ] + for arm in results["arms"]: + r = a[arm] + lines.append( + f"| `{arm}` | {r['n']} | {r['success']} | {r['wrong_action']} | " + f"{r['safe_halt']} | {r['false_abort']} | " + f"{r['success_rate']:.0%} | {r['wrong_action_rate']:.0%} |" + ) + ic = results.get("identity_armed_coverage", {}) + tq = results.get("uia_tree_quality", {}) + lines += [ + "", + "## Identity transfer to desktop-rendered text", + "", + f"- Compiled-arm **armed coverage**: " + f"{ic.get('armed_clicks','?')}/{ic.get('click_steps','?')} click steps " + f"carry an identity band ({ic.get('armed_coverage', 0):.0%}).", + f"- UIA-tree quality: " + f"{tq.get('n_usable_id','?')}/{tq.get('n_targets','?')} workflow " + f"targets expose a usable AutomationId " + f"({tq.get('usable_fraction', 0):.0%}); the identity-critical patient " + f"row does **not** " + f"(`identity_target_has_id={tq.get('identity_target_has_id')}`) — the " + "measured 'vision is necessary' evidence.", + "", + "## Outcome matrix (per arm × condition)", + "", + ] + conds = results["conditions"] + header = "| Arm | " + " | ".join(conds) + " |" + lines.append(header) + lines.append("|---|" + "|".join("---" for _ in conds) + "|") + for arm in results["arms"]: + cells = [] + for cond in conds: + m = results["matrix"][arm][cond] + tag = f"{m['success']}/{m['n']}✓" + if m["wrong_action"]: + tag += f" {m['wrong_action']}✗wrong" + if m["false_abort"]: + tag += f" {m['false_abort']}⚠abort" + cells.append(tag) + lines.append(f"| `{arm}` | " + " | ".join(cells) + " |") + lines += [ + "", + "## Reading", + "", + "- **Success** = the right patient got the right note and no one else " + "did (DB ground truth). **Wrong-action** = a note landed on a " + "different patient (silent mis-write). **Safe-halt** = the arm " + "stopped without writing. **False-abort** = a safe-halt on a purely " + "cosmetic condition (render-scale/theme) where the target was still " + "present.", + "- Caveats (ARM+x64 emulation rendering, render-scale-as-DPI proxy, " + "WinForms substitute for OpenDental) are in `docs/desktop/LIMITS.md`.", + "", + "## Verdict (honest, both ways)", + "", + "1. **The mechanism exists on desktop.** Record → compile → replay of a " + "real WinForms workflow runs deterministically over the vision-only " + "`WindowsBackend`, judged by DB ground truth — on a pixel substrate " + "with no browser DOM. Identity bands are extracted and verified on " + "**desktop-rendered** text.", + "2. **Vision replay is defeated by render-scale and theme drift** " + "(render_125/150 and theme_dark → 0% success, all safe-halts / " + "false-aborts). This is the pre-committed 'DPI is ugly' result and the " + "roadmap justification for multi-scale / appearance-invariant " + "matching. It **never mis-wrote** under cosmetic drift — it halted.", + "3. **The positional UIA incumbent silently mis-writes** under *any* " + "name-collision drift (decoy and siblings) — the exact wrong-action " + "the identity work targets, measured on the incumbent.", + _identity_transfer_verdict(results), + "", + ] + return "\n".join(lines) + + +def _identity_transfer_verdict(results: dict) -> str: + """Verdict item 4, driven by the measured compiled-arm matrix. + + The decoy and sibling data-drift cells are the identity test: the + compiled arm should safe-halt both (0 identity wrong-actions) on the + post-#16 matcher. If a wrong-action survives on the current matcher, + say so prominently — that is a real finding, not a stale-code artifact. + """ + comp = results.get("matrix", {}).get("compiled", {}) + sib = comp.get("data_siblings", {}) + dec = comp.get("data_decoy", {}) + id_wrong = sib.get("wrong_action", 0) + dec.get("wrong_action", 0) + if id_wrong == 0: + return ( + "4. **Identity verification transfers to desktop-rendered text.** " + "On the current identity matcher (ROC operating point of #16/#19: " + "coverage + contradicted-char / suspect / unexplained-name / " + "absent-name budgets, all judged together) the compiled arm " + "**safe-halts on both** the discriminable decoy (distinct " + "surname/DOB → " + f"{dec.get('safe_halt', 0)}/{dec.get('n', 0)} halted) **and** the " + "near-lexical sibling (Sorenson≈Sorensen, adjacent DOB → " + f"{sib.get('safe_halt', 0)}/{sib.get('n', 0)} halted) — " + "**0 identity wrong-actions**. The same budgets that close the " + "browser wrong-patient reopenings fire on OCR'd desktop text: a " + "1-char surname / multi-digit DOB difference registers as " + "*contradicted characters* (affirmative evidence of a different " + "entity), not OCR jitter, so the band is judged a MISMATCH and no " + "note is written. The browser identity fixes **do transfer** to " + "the pixel substrate. UIA-identity distinguishes the same sibling " + "only by exact cell-text equality — a lever that vanishes on a " + "broken-a11y or pixel-only substrate, where the vision matcher is " + "the only one available. (An earlier draft of this benchmark ran " + "the compiled arm against a *pre-#16* matcher and recorded 3 " + "sibling wrong-actions; that was a stale-code artifact and is " + "corrected here.)" + ) + return ( + "4. **Residual identity wrong-action on the current matcher.** Even on " + "the post-#16 identity matcher the compiled arm mis-wrote on " + f"{'the near-lexical sibling ' if sib.get('wrong_action') else ''}" + f"{'and the decoy ' if sib.get('wrong_action') and dec.get('wrong_action') else ''}" + f"{'the decoy ' if dec.get('wrong_action') and not sib.get('wrong_action') else ''}" + f"({id_wrong} wrong-action(s): sibling " + f"{sib.get('wrong_action', 0)}/{sib.get('n', 0)}, decoy " + f"{dec.get('wrong_action', 0)}/{dec.get('n', 0)}). This is a **real " + "desktop finding**, not a stale-code artifact: the OCR'd band on this " + "rendering substrate does not carry enough contradiction evidence to " + "cross the pinned operating point. It must be reconciled before any " + "desktop safety claim — see the failing run telemetry in " + "`results.json`." + ) + + +def render_chart(results: dict, path: Path) -> None: + """Stacked outcome bars per arm (success / wrong-action / halt).""" + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except Exception: # noqa: BLE001 + return + arms = list(results["arms"]) + succ = [results["arms"][a]["success"] for a in arms] + wrong = [results["arms"][a]["wrong_action"] for a in arms] + halt = [results["arms"][a]["safe_halt"] for a in arms] + miss = [results["arms"][a]["miss"] + results["arms"][a]["error"] + for a in arms] + fig, ax = plt.subplots(figsize=(8, 4.5)) + bottom = [0] * len(arms) + for label, vals, color in [ + ("success", succ, "#2e7d32"), + ("wrong-action", wrong, "#c62828"), + ("safe-halt", halt, "#f9a825"), + ("miss/error", miss, "#9e9e9e"), + ]: + ax.bar(arms, vals, bottom=bottom, label=label, color=color) + bottom = [b + v for b, v in zip(bottom, vals)] + ax.set_ylabel("runs") + ax.set_title("Desktop benchmark outcomes by arm (DB ground truth)") + ax.legend() + fig.tight_layout() + fig.savefig(path, dpi=120) + plt.close(fig) + + +def write_outputs(results: dict, out_dir: str | Path) -> None: + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") + (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) + render_chart(results, out_dir / "outcomes.png") + + +if __name__ == "__main__": # pragma: no cover + import argparse + + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="benchmark/desktop") + ap.add_argument("--n", type=int, default=1) + ap.add_argument("--conditions", default="") + ap.add_argument("--arms", default=",".join(ARMS)) + args = ap.parse_args() + conds = args.conditions.split(",") if args.conditions else None + run_desktop_benchmark( + args.out, n_per=args.n, conditions=conds, + arms=tuple(args.arms.split(",")), + ) diff --git a/scripts/desktop/patient_notes.ps1 b/scripts/desktop/patient_notes.ps1 new file mode 100644 index 0000000..94b61ed --- /dev/null +++ b/scripts/desktop/patient_notes.ps1 @@ -0,0 +1,216 @@ +# Patient Notes -- Benchmark Harness (WinForms). +# +# A deliberately real WinForms app standing in for OpenDental's chart+note +# workflow (list-select -> edit note -> save). It is the vision-replay TARGET +# and the UIA-arm target: real WinForms controls expose a genuine (and, for +# the DataGridView rows, deliberately partial) UIA tree -- the "WinForms a11y +# is often broken" finding the spike wants measured, not asserted. +# +# All persistence goes through pn_db.py (SQLite), so the benchmark judge reads +# ground truth from the DB, never from OCR. DPI scaling is honored +# (AutoScaleMode=Dpi) so the DPI drift condition actually moves pixels. +# +# Usage: powershell -STA -ExecutionPolicy Bypass -File patient_notes.ps1 + +param( + [string]$Python = "C:\Program Files\Python312-arm64\python.exe", + [string]$DbCli = "C:\oa\pn_db.py" +) + +Add-Type -AssemblyName System.Windows.Forms +Add-Type -AssemblyName System.Drawing +[System.Windows.Forms.Application]::EnableVisualStyles() + +# --- drift knobs (read from C:\oa\pn_env.json, written by the harness) ------ +# font_scale : render-scale proxy for DPI drift (1.0 / 1.25 / 1.5). Real +# per-monitor DPI change needs a session logoff (see LIMITS.md); scaling the +# base font reproduces the same class of rendering shift -- larger glyphs, +# moved targets -- that defeats pixel-template matching, the effect under test. +# theme : "dark" flips to a dark colour scheme. +# window : "maximized" (default) or "windowed"; size "WxH" forces windowed. +# A JSON config file is used instead of env vars because the harness launches +# the app in session 1 via CreateProcessAsUser, which does not inherit the +# caller's environment. +$cfg = @{ font_scale = 1.0; theme = ""; window = "maximized"; size = "" } +$cfgPath = "C:\oa\pn_env.json" +if (Test-Path $cfgPath) { + try { + $j = Get-Content $cfgPath -Raw | ConvertFrom-Json + foreach ($k in @("font_scale", "theme", "window", "size")) { + if ($null -ne $j.$k) { $cfg[$k] = $j.$k } + } + } catch {} +} +$fontScale = [double]$cfg["font_scale"] +$baseFont = New-Object System.Drawing.Font("Segoe UI", (9.0 * $fontScale)) +$theme = [string]$cfg["theme"] +$bgColor = [System.Drawing.Color]::White +$fgColor = [System.Drawing.Color]::Black +if ($theme -eq "dark") { + $bgColor = [System.Drawing.Color]::FromArgb(32, 32, 32) + $fgColor = [System.Drawing.Color]::FromArgb(230, 230, 230) +} + +function Invoke-Db { + param([string[]]$DbArgs) + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $Python + $psi.Arguments = (@("`"$DbCli`"") + $DbArgs) -join ' ' + $psi.RedirectStandardOutput = $true + $psi.UseShellExecute = $false + $psi.CreateNoWindow = $true + $p = [System.Diagnostics.Process]::Start($psi) + $out = $p.StandardOutput.ReadToEnd() + $p.WaitForExit() + return $out +} + +$form = New-Object System.Windows.Forms.Form +$form.Text = "Patient Notes - Benchmark Harness" +$form.Name = "patientNotesForm" +$form.Size = New-Object System.Drawing.Size(760, 560) +$form.StartPosition = "Manual" +$form.Location = New-Object System.Drawing.Point(80, 80) +$form.AutoScaleMode = [System.Windows.Forms.AutoScaleMode]::Dpi +$form.Font = $baseFont +$form.BackColor = $bgColor +$form.ForeColor = $fgColor +# Maximize by default so the app fills the screen: the captured frame is then +# entirely app content (no background window bleed), which keeps identity +# bands and REGION_STABLE postconditions deterministic across record/replay. +# The window-resize drift condition overrides this to a fixed windowed size. +if ([string]$cfg["window"] -eq "windowed") { + $form.WindowState = "Normal" +} else { + $form.WindowState = "Maximized" +} +if ([string]$cfg["size"]) { + $wh = ([string]$cfg["size"]) -split 'x' + $form.WindowState = "Normal" + $form.Size = New-Object System.Drawing.Size([int]$wh[0], [int]$wh[1]) +} +$form.TopMost = $true + +$searchBox = New-Object System.Windows.Forms.TextBox +$searchBox.Name = "searchBox" +$searchBox.AccessibleName = "searchBox" +$searchBox.Location = New-Object System.Drawing.Point(20, 20) +$searchBox.Size = New-Object System.Drawing.Size(480, 28) +$form.Controls.Add($searchBox) + +$searchButton = New-Object System.Windows.Forms.Button +$searchButton.Name = "searchButton" +$searchButton.AccessibleName = "searchButton" +$searchButton.Text = "Search" +$searchButton.Location = New-Object System.Drawing.Point(510, 18) +$searchButton.Size = New-Object System.Drawing.Size(100, 30) +$form.Controls.Add($searchButton) + +$grid = New-Object System.Windows.Forms.DataGridView +$grid.Name = "patientGrid" +$grid.AccessibleName = "patientGrid" +$grid.Location = New-Object System.Drawing.Point(20, 60) +$grid.Size = New-Object System.Drawing.Size(590, 220) +$grid.ReadOnly = $true +$grid.AllowUserToAddRows = $false +$grid.SelectionMode = "FullRowSelect" +$grid.MultiSelect = $false +$grid.AutoSizeColumnsMode = "Fill" +$grid.RowHeadersVisible = $false +# Tall rows + header so an identity band around a row-click cleanly isolates +# that patient's values (Neil / Sorenson / dob) instead of bleeding into the +# column-header row above it. +$grid.RowTemplate.Height = [int](44 * $fontScale) +$grid.ColumnHeadersHeight = [int](40 * $fontScale) +$grid.AllowUserToResizeRows = $false +$form.Controls.Add($grid) + +$noteLabel = New-Object System.Windows.Forms.Label +$noteLabel.Text = "Clinical note:" +$noteLabel.Location = New-Object System.Drawing.Point(20, 295) +$noteLabel.Size = New-Object System.Drawing.Size(200, 22) +$form.Controls.Add($noteLabel) + +$noteBox = New-Object System.Windows.Forms.TextBox +$noteBox.Name = "noteBox" +$noteBox.AccessibleName = "noteBox" +$noteBox.Multiline = $true +$noteBox.Location = New-Object System.Drawing.Point(20, 320) +$noteBox.Size = New-Object System.Drawing.Size(590, 120) +$form.Controls.Add($noteBox) + +$saveButton = New-Object System.Windows.Forms.Button +$saveButton.Name = "saveButton" +$saveButton.AccessibleName = "saveButton" +$saveButton.Text = "Save Note" +$saveButton.Location = New-Object System.Drawing.Point(510, 450) +$saveButton.Size = New-Object System.Drawing.Size(100, 34) +$form.Controls.Add($saveButton) + +$status = New-Object System.Windows.Forms.Label +$status.Name = "statusLabel" +$status.AccessibleName = "statusLabel" +$status.Text = "Ready" +$status.Location = New-Object System.Drawing.Point(20, 458) +$status.Size = New-Object System.Drawing.Size(470, 22) +$form.Controls.Add($status) + +function Load-Patients { + param([string]$Filter) + $json = Invoke-Db -DbArgs @("list", "`"$Filter`"") + $rows = @() + if ($json.Trim().Length -gt 0) { $rows = $json | ConvertFrom-Json } + $dt = New-Object System.Data.DataTable + [void]$dt.Columns.Add("id") + [void]$dt.Columns.Add("first") + [void]$dt.Columns.Add("last") + [void]$dt.Columns.Add("dob") + foreach ($r in $rows) { + [void]$dt.Rows.Add($r.id, $r.first, $r.last, $r.dob) + } + $grid.DataSource = $dt + if ($grid.Columns["id"]) { $grid.Columns["id"].Visible = $false } + $status.Text = "Loaded " + $rows.Count + " patients" +} + +$searchButton.Add_Click({ Load-Patients -Filter $searchBox.Text }) +$searchBox.Add_KeyDown({ + if ($_.KeyCode -eq "Enter") { Load-Patients -Filter $searchBox.Text; $_.SuppressKeyPress = $true } +}) + +$grid.Add_SelectionChanged({ + if ($grid.SelectedRows.Count -gt 0) { + $row = $grid.SelectedRows[0] + $id = $row.Cells["id"].Value + if ($id) { + $j = Invoke-Db -DbArgs @("get", "$id") + $p = $j | ConvertFrom-Json + if ($p) { $noteBox.Text = $p.note } + } + } +}) + +$saveButton.Add_Click({ + if ($grid.SelectedRows.Count -lt 1) { $status.Text = "No patient selected"; return } + $id = $grid.SelectedRows[0].Cells["id"].Value + $bytes = [Text.Encoding]::UTF8.GetBytes($noteBox.Text) + $b64 = [Convert]::ToBase64String($bytes) + $res = Invoke-Db -DbArgs @("save", "$id", "$b64") + $status.Text = "Saved note for patient $id" +}) + +if ($theme -eq "dark") { + foreach ($c in @($searchBox, $noteBox)) { + $c.BackColor = [System.Drawing.Color]::FromArgb(50, 50, 50) + $c.ForeColor = $fgColor + } + $grid.BackgroundColor = [System.Drawing.Color]::FromArgb(45, 45, 45) + $grid.DefaultCellStyle.BackColor = [System.Drawing.Color]::FromArgb(45, 45, 45) + $grid.DefaultCellStyle.ForeColor = $fgColor + $grid.ColumnHeadersDefaultCellStyle.BackColor = [System.Drawing.Color]::FromArgb(60, 60, 60) + $grid.ColumnHeadersDefaultCellStyle.ForeColor = $fgColor + $grid.EnableHeadersVisualStyles = $false +} + +Load-Patients -Filter "" +[void]$form.ShowDialog() diff --git a/scripts/desktop/pn_db.py b/scripts/desktop/pn_db.py new file mode 100644 index 0000000..434f61c --- /dev/null +++ b/scripts/desktop/pn_db.py @@ -0,0 +1,159 @@ +"""SQLite ground-truth layer for the desktop-benchmark harness app. + +This is the *substitute* target app's data store (OpenDental's MariaDB demo +DB could not be installed no-touch — its trial is a 149 MB interactive +bootstrapper gated by SmartScreen + a UAC secure-desktop prompt; see +docs/desktop/PHASE2.md). A WinForms UI (patient_notes.ps1) reads/writes +through this CLI; the benchmark judge reads the same SQLite file directly, so +success is decided by DB state, never OCR. + +Commands (all print machine-readable output): + + seed [--drift none|siblings|reorder] (re)create + seed the DB + list [filter] JSON list of patients (name match) + save set a patient's note; prints OK + get JSON of one patient + all JSON of every patient (ground truth) + +Notes travel base64 so arbitrary text (unicode, quotes) survives the shell. +""" + +from __future__ import annotations + +import base64 +import json +import os +import sqlite3 +import sys + +DB_PATH = os.environ.get("PN_DB", r"C:\oa\patients.db") + +# Fixed fictional roster. Includes a deliberate lookalike/sibling pair +# (Neil Sorenson / Nell Sorensen) — the identity work's first DESKTOP test — +# and same-surname neighbours for list reorder/insert drift. +SEED = [ + (1, "Neil", "Sorenson", "1984-03-12", ""), + (2, "Nell", "Sorensen", "1986-07-09", ""), + (3, "Maria", "Alvarez", "1979-11-02", ""), + (4, "James", "Alvarez", "1981-05-21", ""), + (5, "Priya", "Chandra", "1990-02-14", ""), + (6, "Wei", "Chen", "1975-09-30", ""), + (7, "Fatima", "Noor", "1988-12-01", ""), + (8, "Oskar", "Bakke", "1992-06-18", ""), + (9, "Robert", "Kowalski", "1968-08-08", ""), + (10, "Grace", "Okafor", "1995-04-25", ""), +] + +# A near-lexical sibling (Neil Sorensen ~ Neil Sorenson, adjacent DOB): the +# HARD identity case -- 1-char surname + 1-digit DOB differences sit inside an +# OCR-jitter-tolerant matcher's noise floor. +DRIFT_SIBLINGS = [(11, "Neil", "Sorensen", "1983-01-19", "")] + +# A DISTINCT decoy sharing only the searched given name (very different +# surname + DOB) that sorts ABOVE the real patient (so row 0 is the wrong one +# for a positional selector): the discriminable case identity should resolve. +DRIFT_DECOY = [(12, "Neil", "Anderson", "1972-10-04", "")] + + +def _conn() -> sqlite3.Connection: + con = sqlite3.connect(DB_PATH) + con.row_factory = sqlite3.Row + return con + + +def seed(drift: str = "none") -> None: + os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) + con = _conn() + con.execute("DROP TABLE IF EXISTS patients") + con.execute( + "CREATE TABLE patients (id INTEGER PRIMARY KEY, first TEXT, " + "last TEXT, dob TEXT, note TEXT)" + ) + rows = list(SEED) + if drift == "siblings": + rows += DRIFT_SIBLINGS + if drift == "decoy": + rows += DRIFT_DECOY + if drift == "reorder": + rows = list(reversed(rows)) + con.executemany( + "INSERT INTO patients (id, first, last, dob, note) VALUES (?,?,?,?,?)", + rows, + ) + con.commit() + con.close() + print(json.dumps({"status": "ok", "seeded": len(rows), "drift": drift})) + + +def _patient_dict(r: sqlite3.Row) -> dict: + return {"id": r["id"], "first": r["first"], "last": r["last"], + "dob": r["dob"], "note": r["note"] or ""} + + +def list_patients(filt: str = "") -> None: + con = _conn() + if filt: + like = f"%{filt}%" + cur = con.execute( + "SELECT * FROM patients WHERE first LIKE ? OR last LIKE ? " + "OR (first || ' ' || last) LIKE ? ORDER BY last, first", + (like, like, like), + ) + else: + cur = con.execute("SELECT * FROM patients ORDER BY last, first") + out = [_patient_dict(r) for r in cur.fetchall()] + con.close() + print(json.dumps(out)) + + +def save(pid: int, note_b64: str) -> None: + note = base64.b64decode(note_b64).decode("utf-8") + con = _conn() + con.execute("UPDATE patients SET note = ? WHERE id = ?", (note, pid)) + con.commit() + changed = con.total_changes + con.close() + print(json.dumps({"status": "ok", "id": pid, "changed": changed})) + + +def get(pid: int) -> None: + con = _conn() + r = con.execute("SELECT * FROM patients WHERE id = ?", (pid,)).fetchone() + con.close() + print(json.dumps(_patient_dict(r) if r else None)) + + +def dump_all() -> None: + con = _conn() + rows = [_patient_dict(r) for r in + con.execute("SELECT * FROM patients ORDER BY id").fetchall()] + con.close() + print(json.dumps(rows)) + + +def main(argv: list[str]) -> int: + if not argv: + print("usage: pn_db.py ...", file=sys.stderr) + return 2 + cmd = argv[0] + if cmd == "seed": + drift = "none" + if "--drift" in argv: + drift = argv[argv.index("--drift") + 1] + seed(drift) + elif cmd == "list": + list_patients(argv[1] if len(argv) > 1 else "") + elif cmd == "save": + save(int(argv[1]), argv[2]) + elif cmd == "get": + get(int(argv[1])) + elif cmd == "all": + dump_all() + else: + print(f"unknown command: {cmd}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/desktop/session1_launch.py b/scripts/desktop/session1_launch.py new file mode 100644 index 0000000..13934b7 --- /dev/null +++ b/scripts/desktop/session1_launch.py @@ -0,0 +1,192 @@ +"""Launch a process in the interactive console session (session 1) from SYSTEM. + +``prlctl exec`` runs commands as ``NT AUTHORITY\\SYSTEM`` in session 0, which +is isolated from the logged-on user's desktop — mss ``BitBlt`` and pyautogui +``SendInput`` there address a non-existent desktop and fail. This launcher is +the canonical "service starts a process on the user's desktop" pattern: + + WTSGetActiveConsoleSessionId -> WTSQueryUserToken -> DuplicateTokenEx + -> CreateEnvironmentBlock -> CreateProcessAsUserW (lpDesktop=winsta0\\default) + +Run it AS SYSTEM (which holds SeTcbPrivilege, required by WTSQueryUserToken): + + python session1_launch.py "" + +ctypes argtypes/restypes are declared explicitly so 64-bit HANDLEs are not +truncated to 32-bit ints (the classic silent-failure bug on Win64/ARM64). +""" + +from __future__ import annotations + +import ctypes +import sys +from ctypes import wintypes + +wtsapi32 = ctypes.WinDLL("wtsapi32", use_last_error=True) +kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) +advapi32 = ctypes.WinDLL("advapi32", use_last_error=True) +userenv = ctypes.WinDLL("userenv", use_last_error=True) + +TOKEN_ALL_ACCESS = 0xF01FF +SECURITY_IMPERSONATION = 2 # SecurityImpersonation +TOKEN_PRIMARY = 1 +CREATE_UNICODE_ENVIRONMENT = 0x00000400 +CREATE_NO_WINDOW = 0x08000000 +INVALID_SESSION = 0xFFFFFFFF + + +class STARTUPINFOW(ctypes.Structure): + _fields_ = [ + ("cb", wintypes.DWORD), + ("lpReserved", wintypes.LPWSTR), + ("lpDesktop", wintypes.LPWSTR), + ("lpTitle", wintypes.LPWSTR), + ("dwX", wintypes.DWORD), + ("dwY", wintypes.DWORD), + ("dwXSize", wintypes.DWORD), + ("dwYSize", wintypes.DWORD), + ("dwXCountChars", wintypes.DWORD), + ("dwYCountChars", wintypes.DWORD), + ("dwFillAttribute", wintypes.DWORD), + ("dwFlags", wintypes.DWORD), + ("wShowWindow", wintypes.WORD), + ("cbReserved2", wintypes.WORD), + ("lpReserved2", ctypes.POINTER(ctypes.c_byte)), + ("hStdInput", wintypes.HANDLE), + ("hStdOutput", wintypes.HANDLE), + ("hStdError", wintypes.HANDLE), + ] + + +class PROCESS_INFORMATION(ctypes.Structure): + _fields_ = [ + ("hProcess", wintypes.HANDLE), + ("hThread", wintypes.HANDLE), + ("dwProcessId", wintypes.DWORD), + ("dwThreadId", wintypes.DWORD), + ] + + +def _declare() -> None: + kernel32.WTSGetActiveConsoleSessionId.restype = wintypes.DWORD + + wtsapi32.WTSQueryUserToken.argtypes = [ + wintypes.ULONG, + ctypes.POINTER(wintypes.HANDLE), + ] + wtsapi32.WTSQueryUserToken.restype = wintypes.BOOL + + advapi32.DuplicateTokenEx.argtypes = [ + wintypes.HANDLE, + wintypes.DWORD, + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(wintypes.HANDLE), + ] + advapi32.DuplicateTokenEx.restype = wintypes.BOOL + + userenv.CreateEnvironmentBlock.argtypes = [ + ctypes.POINTER(ctypes.c_void_p), + wintypes.HANDLE, + wintypes.BOOL, + ] + userenv.CreateEnvironmentBlock.restype = wintypes.BOOL + + advapi32.CreateProcessAsUserW.argtypes = [ + wintypes.HANDLE, + wintypes.LPCWSTR, + wintypes.LPWSTR, + ctypes.c_void_p, + ctypes.c_void_p, + wintypes.BOOL, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.LPCWSTR, + ctypes.POINTER(STARTUPINFOW), + ctypes.POINTER(PROCESS_INFORMATION), + ] + advapi32.CreateProcessAsUserW.restype = wintypes.BOOL + + +def launch(cmdline: str) -> int: + """Launch ``cmdline`` in the active console session. Return its PID.""" + _declare() + session_id = kernel32.WTSGetActiveConsoleSessionId() + if session_id == INVALID_SESSION: + raise RuntimeError("no active console session") + + htoken = wintypes.HANDLE() + if not wtsapi32.WTSQueryUserToken(session_id, ctypes.byref(htoken)): + raise ctypes.WinError(ctypes.get_last_error()) + + hdup = wintypes.HANDLE() + if not advapi32.DuplicateTokenEx( + htoken, + TOKEN_ALL_ACCESS, + None, + SECURITY_IMPERSONATION, + TOKEN_PRIMARY, + ctypes.byref(hdup), + ): + raise ctypes.WinError(ctypes.get_last_error()) + + env = ctypes.c_void_p() + userenv.CreateEnvironmentBlock(ctypes.byref(env), hdup, False) + + si = STARTUPINFOW() + si.cb = ctypes.sizeof(si) + si.lpDesktop = "winsta0\\default" + pi = PROCESS_INFORMATION() + + ok = advapi32.CreateProcessAsUserW( + hdup, + None, + ctypes.create_unicode_buffer(cmdline), + None, + None, + False, + CREATE_UNICODE_ENVIRONMENT | CREATE_NO_WINDOW, + env, + None, + ctypes.byref(si), + ctypes.byref(pi), + ) + if not ok: + raise ctypes.WinError(ctypes.get_last_error()) + return int(pi.dwProcessId) + + +def main() -> None: + """Launch ``pythonw