From 9c915377b118837ddbbb9f88a6863ec88a8a27dc Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 10 Jul 2026 17:49:16 -0400 Subject: [PATCH 1/2] feat: automated Parallels desktop benchmark pipeline (Phase 2) Fully programmatic, $0 desktop benchmark on a local Parallels Windows 11 ARM VM (Apple Silicon has no nested virt, so the WAA/QEMU stack can't run here). No manual/GUI steps; no cloud; no model calls (ANTHROPIC_API_KEY unset). Control plane - openadapt_flow/backends/parallels_vm.py: ParallelsVM wraps prlctl for lifecycle/snapshot/revert/exec/capture, guest/host IP discovery, ephemeral- port file push (prlctl exec hangs on long args), and shim launch. - scripts/desktop/session1_launch.py: launches the shim in the interactive console session (session 1) via WTSQueryUserToken + CreateProcessAsUser -- prlctl exec lands in SYSTEM/session 0, where mss BitBlt and pyautogui input can't reach the desktop. This is the foundational blocker, solved. - scripts/desktop/waa_shim.py: in-guest WAA-contract HTTP shim (GET /screenshot PNG, POST /execute_windows exec, GET /uia tree dump), reusing the Phase-1 WindowsBackend contract unchanged. Target app + ground truth - scripts/desktop/patient_notes.ps1: real WinForms list-select->edit->save app (drift knobs via pn_env.json). Substitute for OpenDental, whose trial is a 149MB interactive bootstrapper gated by SmartScreen + a UAC secure-desktop prompt -- not no-touch installable (documented honestly in PHASE2.md/LIMITS). - scripts/desktop/pn_db.py: SQLite ground-truth CLI; the judge reads DB state, never OCR -- wrong-action detection is exact. - scripts/desktop/uia_arm.py: pywinauto UIA incumbent, identity + positional. Benchmark - openadapt_flow/benchmark/desktop_benchmark.py: 3 arms x 7 conditions (clean, render_125/150 as DPI proxy, theme_dark, data_reorder/decoy/siblings), record->compile->replay via WindowsBackend, DB judge, results.json + BENCHMARK.md + chart. Per-run reset by DB reseed + relaunch; harness-ready VM snapshot for warm boot. Findings (n=3/cell, DB ground truth): the record->compile->replay mechanism works on a real desktop with identity bands on desktop-rendered text; vision replay is defeated by render-scale/theme drift (0% -> safe-halt, never mis-writes); the positional UIA incumbent silently mis-writes on any name collision; identity catches a distinct decoy (safe-halt) but FALSE-VERIFIES a near-lexical sibling (Sorenson~Sorensen) -- the desktop analogue of the open browser wrong-action findings; UIA-tree quality 5/6, the identity-critical patient row has no AutomationId. Caveats (ARM+x64 emulation, render-scale-as- DPI proxy, WinForms substitute) in docs/desktop/LIMITS.md. Includes the Phase-1 WindowsBackend + capture adapter (rebased onto current main; previously only on feat/desktop-backend). Tests mock the VM/HTTP; full suite green. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01CKrVJJy5jWVCkXAqgUqtqZ --- .gitignore | 2 + benchmark/desktop/BENCHMARK.md | 42 + benchmark/desktop/outcomes.png | Bin 0 -> 28240 bytes benchmark/desktop/results.json | 1657 +++++++++++++++++ docs/desktop/LIMITS.md | 59 + docs/desktop/PHASE1.md | 194 ++ docs/desktop/PHASE2.md | 108 ++ openadapt_flow/adapters/__init__.py | 23 + openadapt_flow/adapters/capture.py | 419 +++++ openadapt_flow/backends/__init__.py | 7 +- openadapt_flow/backends/parallels_vm.py | 357 ++++ openadapt_flow/backends/windows_backend.py | 308 +++ openadapt_flow/benchmark/desktop_benchmark.py | 720 +++++++ pyproject.toml | 4 + scripts/desktop/patient_notes.ps1 | 216 +++ scripts/desktop/pn_db.py | 159 ++ scripts/desktop/session1_launch.py | 192 ++ scripts/desktop/uia_arm.py | 199 ++ scripts/desktop/waa_shim.py | 196 ++ tests/test_capture_adapter.py | 310 +++ tests/test_desktop_benchmark.py | 176 ++ tests/test_parallels_vm.py | 120 ++ tests/test_windows_backend.py | 464 +++++ 23 files changed, 5931 insertions(+), 1 deletion(-) create mode 100644 benchmark/desktop/BENCHMARK.md create mode 100644 benchmark/desktop/outcomes.png create mode 100644 benchmark/desktop/results.json create mode 100644 docs/desktop/LIMITS.md create mode 100644 docs/desktop/PHASE1.md create mode 100644 docs/desktop/PHASE2.md create mode 100644 openadapt_flow/adapters/__init__.py create mode 100644 openadapt_flow/adapters/capture.py create mode 100644 openadapt_flow/backends/parallels_vm.py create mode 100644 openadapt_flow/backends/windows_backend.py create mode 100644 openadapt_flow/benchmark/desktop_benchmark.py create mode 100644 scripts/desktop/patient_notes.ps1 create mode 100644 scripts/desktop/pn_db.py create mode 100644 scripts/desktop/session1_launch.py create mode 100644 scripts/desktop/uia_arm.py create mode 100644 scripts/desktop/waa_shim.py create mode 100644 tests/test_capture_adapter.py create mode 100644 tests/test_desktop_benchmark.py create mode 100644 tests/test_parallels_vm.py create mode 100644 tests/test_windows_backend.py diff --git a/.gitignore b/.gitignore index a2c1458..bf4760a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ benchmark/hybrid/finals/ benchmark/hybrid/rows.jsonl benchmark/dom/finals/ benchmark/dom/rows.jsonl +benchmark/desktop/finals/ +benchmark/desktop/_work/ diff --git a/benchmark/desktop/BENCHMARK.md b/benchmark/desktop/BENCHMARK.md new file mode 100644 index 0000000..025ff45 --- /dev/null +++ b/benchmark/desktop/BENCHMARK.md @@ -0,0 +1,42 @@ +# Desktop Benchmark (Phase 2) — compiled vision replay vs UIA incumbent + +_Generated 2026-07-10T21:48:06.669583+00:00_ + +**Task.** Patient Notes (WinForms) search -> select -> note -> save; DB-ground-truth judge; $0 (no model calls) + +**Substrate.** Parallels Windows 11 ARM VM on Apple M2 Max; WindowsBackend over in-guest WAA HTTP shim (session 1) + +> WinForms substitute for OpenDental (trial not no-touch installable; see PHASE2.md). + +## Headline + +| Arm | n | success | wrong-action | safe-halt | false-abort | success rate | wrong-action rate | +|---|--:|--:|--:|--:|--:|--:|--:| +| `compiled` | 21 | 6 | 3 | 12 | 9 | 29% | 14% | +| `uia_identity` | 21 | 21 | 0 | 0 | 0 | 100% | 0% | +| `uia_positional` | 21 | 15 | 6 | 0 | 0 | 71% | 29% | + +## Identity transfer to desktop-rendered text + +- Compiled-arm **armed coverage**: 2/4 click steps carry an identity band (50%). +- UIA-tree quality: 5/6 workflow targets expose a usable AutomationId (83%); the identity-critical patient row does **not** (`identity_target_has_id=False`) — the measured 'vision is necessary' evidence. + +## Outcome matrix (per arm × condition) + +| Arm | clean | render_125 | render_150 | theme_dark | data_reorder | data_decoy | data_siblings | +|---|---|---|---|---|---|---|---| +| `compiled` | 3/3✓ | 0/3✓ 3⚠abort | 0/3✓ 3⚠abort | 0/3✓ 3⚠abort | 3/3✓ | 0/3✓ | 0/3✓ 3✗wrong | +| `uia_identity` | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | +| `uia_positional` | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 3/3✓ | 0/3✓ 3✗wrong | 0/3✓ 3✗wrong | + +## Reading + +- **Success** = the right patient got the right note and no one else did (DB ground truth). **Wrong-action** = a note landed on a different patient (silent mis-write). **Safe-halt** = the arm stopped without writing. **False-abort** = a safe-halt on a purely cosmetic condition (render-scale/theme) where the target was still present. +- Caveats (ARM+x64 emulation rendering, render-scale-as-DPI proxy, WinForms substitute for OpenDental) are in `docs/desktop/LIMITS.md`. + +## Verdict (honest, both ways) + +1. **The mechanism exists on desktop.** Record → compile → replay of a real WinForms workflow runs deterministically over the vision-only `WindowsBackend`, judged by DB ground truth — on a pixel substrate with no browser DOM. Identity bands are extracted and verified on **desktop-rendered** text. +2. **Vision replay is defeated by render-scale and theme drift** (render_125/150 and theme_dark → 0% success, all safe-halts / false-aborts). This is the pre-committed 'DPI is ugly' result and the roadmap justification for multi-scale / appearance-invariant matching. It **never mis-wrote** under cosmetic drift — it halted. +3. **The positional UIA incumbent silently mis-writes** under *any* name-collision drift (decoy and siblings) — the exact wrong-action the identity work targets, measured on the incumbent. +4. **Identity verification catches a *discriminable* decoy** (distinct surname/DOB → compiled safe-halts) **but false-verifies a *near-lexical* sibling** (Sorenson≈Sorensen, adjacent DOB → compiled wrong-action): the browser-tuned matcher's OCR-jitter similarity tier (`TOKEN_SIM_RATIO`) treats a 1-char surname / 1-digit DOB difference as noise. This is the **desktop analogue of the open browser wrong-action findings** — the safety claim does **not** yet fully transfer to desktop; a stricter/desktop-tuned matcher (or an exact-field key for high-collision fields) is required. UIA-identity distinguishes the same sibling only because it does exact cell-text equality — an option that vanishes on a broken-a11y or pixel-only substrate, where the vision matcher is the only lever. diff --git a/benchmark/desktop/outcomes.png b/benchmark/desktop/outcomes.png new file mode 100644 index 0000000000000000000000000000000000000000..2eb5992acbccc17c808df5785e360095aa7a3bd4 GIT binary patch literal 28240 zcmc$`cT|&aw=EhhC@P>}`GJ6_6zNLupeQ9YMd?+kp-Lx#09I6_Nbgt(NDU=)2q-GO zC6YjBDmC;DA(Z%yOy0a@J!{Q1=UmV0yLuXoC)iIwAP`0^ z&D#bL$l*o^LmZirqN=63S&dF(AOCg%1Z?-2FyaukbiMtOjvFg(^Y^@c!Ld}#mZMjS2$Kp<{) zTDNaL2uPbBJ{EAcZ*ysxm4yYdSnp4FlU#rNmF7LwqjYDoPF|5defFeD@cFE3uin49 za_tTGoi~S`Ub}OXj$Hq8gyF~rvTIj@y5KI|>W=WkW=@%Y?HWn0mQavTaCATECCkA? z2fiJ`-H6g!I7NE}i9UNg^x$t5s*r=f6dyfgL;H_Vk<*aFv=^>VApg*Q+MGPnNc#yf z=Z4U32&DfI-F4c}#`FIN4sk0+;V>kKm5oi{evY1RYJ#Yxd~1qSe%!Ttw*-v}u3f&Y z6$s6g_k~V&r3+28CnmmRya;tZe|(VT_~7Z5!DpYhBFEgj(&S<;tI@-MJvo+!_nYg# zEM#2dgVm4ai?HiRx>xGhEgCOq7<1*$o7=UD-S9$+gkzWMWc_SWt?ctLo8FIiyrz3D zlY2jfQ9AV^vUQTL43yf(>ZQw5;`T`8F5YHuRm!FNj=FfOMH&U1||AZ5nDj`-C?!$r#{tIzrd`_ytlYufv?O8%0FG|ky z&-1leqJE7J`HF6eq}IY5gyE&_MJ?EL&j*82J1sJyFqPO{u70;3v7UzV{QdiPsppuw z+~Z%Nr!Fa5ICQ4)Htei~kGI7O{eXK6*YGw+U6O&xxRH1Y&C11jzkm4f;dR}Mjr9(R z?xe+uUX4q-eX|@&2#j1LpMHk$;bTlX#S@1ZK*(v@W8??HEo8B;q5pE2{#0~dzH$52 zD8(!LdsOsUAtPbamE#ai4l`X!l*(RoNvnkH2`_$3>tdRY^YPZY{dJ3;sz zyOeW)Tf@eHW2P2Z4_V$*U6b`09(x>vdqGj2a?C7$6m;5LUZ_kBkJJa+gViz_KyX`i z*xR8v$RRA8sD1rBKbQ01gLUvS3=FS+;=HgERHxEy&|$;0%%PJ9xxLn9PhP)E%$;>1 zVRFY?-Zf1jvzim>S!pb9od|??xdR3BG}r+Wb)*a*uFUH9oCb_!xCJ z=6R?#wqfhfN$(w(*7BUp;O$o79LMeqcV1G_yd~PY!F?k(;r;C433^8M+j9FBWdA%k zVNCB5&(>1nkLY@ig?O+BRP!(f48SG`h6*Oe^c_?CQL*u(~RPVm9CwjJ7P%+iYGgAV=%k8V(8 z9=w5rS#e1?n>m(PQC z$n$5XJ{s%xTyJ7l zN$OrpT308USGbx^1zJbBSvP2etfoefK73>;#4=N-fZWE&6}86;P5HvXWAM$=PVBme z4WgJ3(|ej{%es|a_Vz)%DlF74DpyXNyLPvcqY_-Mui{F}RrgH$_Jq!5u+Zg#Jm*PL zWxIWOhO67veUbi+p>#JyH8%6JFG{;6>91NLB{2r?I)|Ge3h~d>v>Gs_x%Vjke&5C=ExBO1JVf%8hH1d*)yRyr0 zjW5RS!Me9w!(w6mEXZ@5@Re~-bA3=-I=qxDs~&$DAP^BK{mhYN%2XzGh&}+Uj9x1Z zdS)&T-jMJMV$y?dC^nVgy3xV8Mfu$UerK@q_;f4XhaV03ap#@^3FtgzHsXyhD;pP= zd-ff5b$NDa*Xg=@PJhK*}7OCY9%#YJlME++@j|Fkd|@W;L_}l7F6Q@I0wJefH-s zokU}hjLH;;`Hdju_U-2@>t@!#txpXiV*{4H9Me_yMdTY5J{p>0$4sxRC&>mZy|->Q z*4a@i+F=Yu%2n)AxW*7%Jm372Oo(ruD=u<{?08Bb*Jt|fCFBg|X6hzgzfADuUt4xQ z>anf#{N>A(RXr7An3NH}0qNcct4v?yHYdy`CzD*AM+bKe-d#72#d?flmrsOBUB~-? zn)g~9y%4w?xsIZGf}}JV*xQZRT$o>q&$wU|ZD$$IF5~W6F)DBL>13$yB^P;JbaT?= zO|qKeu0_ z7)K3bioWGbF}kIY{awdMUL`-D;UeN0q|L8nl}`OHPmgvg_evsMw`x20itw4pk84SU z)m*3Ek2l;EqJJI_eF8Zd8ozfJW`^9(TWhVt5-)0WkEqb{YeKS^wel2W(n!HHbLGfS zI6bjQ06)PQR$P7Sow86ShTXaX)vBiV zVPAqFh*oWu#N9R_^lb*njuQ6c*rMlz2V}0jWWIazpUhQm?B9H`Qio@Gtc#}0%7MlU zCfitS8oIJ<2b4dRK|X#@ZCfX~9qlw@-=4}0kypVZgEoi#Q@bs4G^W!1M{dCQ_0s-9 zLremNh(7T`#*yOa%0%UBz4Q2EJXoB8vqy8fa#hcwR5rwq92Beub-*l+2JlPd%a}BEyW@L$rD? zWrKm%)t9Ugi!2qqGpgUrN>gj}tT$f^&3>`=L48y8dVfd$bZEhbEQ$Uuth)y~NJUa6 z5?9DMnJI}Cyf|qakeFra`<*XJqvjDlY}?Tt@f9UA{Ux7>zQJIPWFwO+Bz8e!)HUY` zfl+x-^NAYDjv!q6B}@O=ehZ-o=-@aKh#v(%p%PDIz1Oh>TT1k0~!6?F3usWPO*u9!#woc zdLtqIH1%8j-a-{uttwQN-$XUE8oruM_PK4!`FOhc+zcww>W!z>Jhw*u2Faxq zv;_DwZ}+NT8vh`gKFElUylmLoqb}+;tET9A@K`DYE;nP5)DCM8P7f!3VNyxh2+z#M zL|6JCC7J&0$On%Mq&@bD}TpYvGb!%LWql)EHSX1j~;?Fo;TH?iuImfztb zT@(YCFv@tQP+|5XWHlWsJzt+_-OVPt}=c?iqw_|4OJ$D&a+pMsrfwb-rVr)NCUCE>G2%?@+S z`;eP2g-9|xNPdoZeXe8n*$EFSG>Z2BW{Q_szWZ4@J9nAGbF9WjjsY1B&s4pxKMnBWwrFPxOyYm_IsU z^k}=o7Y>?GDb!m;Q@+e6GlT3VKQgx0Gq1eTz+?1|kJ3s0p9k?Q?OkY2X zuq`meTX>{l)-<%OF-J7>+dDIq%t23k!9Mwhp{JpI+KWl=_5_i887s4=8Mm`6L!K+; z?~|{yjhNtltFbGNrk(@nJsb}E>=<*3fpiG;N$fL2&J$+&*vb6rTS){|<9R zOyW=Az((&P6189c?Nm%MoxJwB9z>@7t?0y$Pyh<*KQSzz*hy5uMox4Zx4+ zNac*UJsJrd~eUCzJ@48U+cSqJ8bQFeh z&Lua>lh+Jgn~i+{+LY`kAPZ2&>@6?^W5U|+7<#UeL*4z6ZXwv_s*Ga{mvf8GF?ce~ z2LD{2skw6X>O_Rh$kME=_S~4%jh}_)#GrlBPhLL6oLK|w_Heo)9M|y)McO{C)}U6{ zHdmNZJ6lIk<5XHODlmCC;?it~?FlPJD8l%y&3w(C`HLtK7W)vz|$G!*$WPP?OXUtFR1%>wf9ttkNl}{^c3O|Cz^0t88H{YuHfeyH}*vw zobI*RH?NuOW2N29ngLn%Dd;@@uKG5P;&UzalSb6G5}N6Sq|fx-%pXEVDSFMZ>lS*s z1}Kij`ppc=K4=_+M#2asxy{cj>lnEdWuYT$Tf`DCB|LWd$Hf~=&$5V6kF{DNd~c8H zQAtL)o7-Nj#rlMpswGn7PsMQI3IFsHv@1IkcfR|kj+J#yZ@f+sw|_&?2{S%yzyb|@ z^59lx%S`dvjCtf5xiq2937vAockg3g6zIQfN$uoVE2Ln){?=92|G?1Gky6}6{3Z2KI=D>-&R);#pN{CaOFdjHpKHXUC5CM z>T!aGZE|Q)ZDL{i%rvX7h(J&GIzwq+ZX$nYrt&i3p`}l?*`E(6O*2D+Amgiu*^24= zkFEdxEHN0}Fy`ef-dAQQU2TZ35{csT?JRBjc9Nr+idRDVRGJ>}7egaYT>Y(M(!_MW_<5kxS-)(9kN-uLO{ zWdA+TDyPKQ6_$J$4Rf5*s|n6`wADuh&%(#5nSC3k5o;#6ipCmOCF3;{8a>b&hOQ^h z$`uWMZc-*!=agT>jF0}OFff#|RG3?F)Rg2FI08GH#^Z7*^hmZ!zgfh+{Bc-R5oGR8 z^YgyQV$2)&XY`EUm@tVUZj}X3UQ~L%xz951(<(7dJOT1 z*sVX|;tm~2<6TU@Ub5H#eZbiX)akn}0*R4l3V1_Ekwj6OL7>C2T|i|(%C1&Lw*w>I&S33804 zQm8@zo}|6_A7zWi>zZ1>b||Z~f+Ap1Rs8(L3*PU7G*yI~`}XbIEuh?^9yYynNGjao zhUgt)->GmRiV7GVr5%k925?!Ncnz3b+V=c_J*^h5iOp5);rl{rQU+{OIKJ2<>ixg%q+5YRjXIwJ}n zwA05JIGw0t;)(Nt&CXn*$B!zmaPGzT=6Y zO6lwOSeEAE_rbxV!6%3Ova z-^vo_&~fd{<;yXWm+4PSgf(3Tj8)7m5#0_%5FWLcrwod1ZZa@v?JUI|=taj>%~tZi zcx<>1@B}`791~xBdA;qy$J;SQV&dWkUz?gHzP>o=wJ;>t3i$Mo=ve7Sa5GgQIOlG& z744<)I6)qs+bdJuVt^d?qO45tnDQSy(9^GQC~AXqcihgEqbE$yS@Zax zw~?S7ShNICp!|Rpk5PoLB*BydH{x(8?RHRd#Xt#*JH{2zm>dkCPHEt#`#`mK&RM|+ zEjsv6qi560W6h~H(tOM5m%yv+x)3qv88LklEAmn7FNWu?9)H zl5=%}g@T2=KLBm}+SBsyr&RyoA9@o!2CkHaY-^n_Fu5ES$>~kq`8iOwGLR-$y1(@M z+G;SRiMiAdpM4Dv;>9h2vc5wpT_Rd{ivNjGPX&BC)gxKXBcsEz^mb&rBFJ?}-Di+f zp%Qj2+(7ZUm+c}@*cQVZ?o9$KWgJ{tHim##+8nZl`gW%)T$r!cT@pJb!J*PSWu%a7p2{vK@74sRtoe~&fWx6Gp3f#MP_zOTKA74kkW>XW+B9m#3% z2^mP1bc*h<##&&vmo;E3{kALLH%F=Q4zpe9e4S%zVQr;SE;_}Qls)Q1$o-t{_z-Ix zw`#s>p!e7hM)*X$F%J*VHy=HFIj`|o6d=`%Tw5J)Pwd_S?D%K9?u_ACuQ7#?fPktr zdEYiS5`ZN;{o=G+ej{1d_9cfR;z5O|>6L+nvoB$-bAJr;|6s->`&|cfqT+-NZYOjl zU6w0#kiw&_Y8{|9NL}xRuSa3TN_3o{*#o1l6y9VB2T6R0MTJfc6Q|%g?ZgBl#*5o= z`_D(zvsNGhzZ##cY>tMs&!QXLNr!6bf3%suE|JnzD1W0DG?gg{i+RhXL_)jKa=@?J zvJ0Y`Det+f=O4@Fkex&dFUmyX-G3U8@Q&SWLt3$EPYTVRKq;Y`kDOSRU(fUN}0h&^$G?c&~mFI%ZbR z)1Z-g6HBT?);`bH6*6p3j51DXsBo9MIb%Io(K@-mC*c;bj(Ym^sofM7sXV3fq4G?w zB!s{+T4LMW#;UJ)IhJ2v`dCc90ScPj5XQv$Mtt$hvtyk}SfCSy`%#wDtrccg<_b=N zGf@3nJyldYjy`(QuIuBSmBE*cD?8>MttcVm@z-IO5}2)m2fiKu0>jDXGuGllCI9FZ z>T8b|nrFjM^yuDY-p+^?O$G+04bnoD^NSNP6M7F*#VDJ+jE58u3;MZ7yTY z?P2&F?bK{t6f9jnM$F$mb^)Vttb!y*SbufA5@rwhzj>ccPGw=<6BjmWl&x`b0!BK) zb@2P@Yl~uVuZcIt2a9a(sF`k>>^&eE%F(1*om6R&AN*n|3qVhby%T<3E(f{N%J227 zkq-X*#f2eU@LI11kWQjnUYxw}NAh6NWtAg0-5vWL#RGr~Gbzia?8mzhJ4O}dyrCb0 zC_2n-Map|KLRuI9STXgZEEO0|GR6~?w_8p<9Cuz!Px*k&g4ZDfwJx0J`_vre-=10H zxtcvkY8ai`p~leDnH?%nV+Hr3C5Q|HBet0q(ZF^2hBTJt0W zjLl6s?}fA=kb8a>8R?q+M0?Jt~Q+by=UO`qYH{o(RTTk<|ndrXV+J2yleQW?iPdOaM|vB!*_{ARwBnD zIc_u^88%0~VK}dps#P?MX+07nZ>bQQXIh?Dx7DR!Q26K$`DabJIcW)>GZBW;?kp|O zahxAE#Zyx!cfN;9FL(dhQfP(&0q=W}6`Bo==aHMM#mkuh{f9*< z1A30`v9ku`RO30i>!HtYL*7==H->T^{`CM)e_$2~1yJUHfwum)eJ1@w5vej0pu&1l zH{2NP{<=|Y61g}3-0_s;<)EDAjVv`)D3A)%0?tg^MbVS_z)T=)6CY>gMJ6 z#V9+3T6pTBf{~xFIp5LM32&;fu;hbMhfYBl?uZ{7%*KP0_Azdws0x^Ep6Uew1D^Mr z=MS()=!1zr6YXB}+rE{sZL9pvks9B2x9u0&iQ|ad&nXQ?yM2;pWZY#{8obu?hU(yW zH!jPUQc?Wr#B%nMisZh$?9bYb$oDf~qM&!=oYE;S0mOj_Wfq4MrjYqjl9g z9y1Rr-!H|Y8>HK1xwu62qqszlx)2jA`tr3`-G}Yai5q3%#)UD?ZY0f&z{U70>ocvX zto&_avxH>(>Fq~lE8Qw6gZ6|;6s6DC&~4a^uQQE}cYfjUvGL^(YADr0vq{4mEmrtE z(2t|{7vcoO9dJCcMPS)=+3$JsGxwI$Spn@t(y2%F>({S5-jf~K&jaTAOWdh;J6p>R zBeuq>^rwmpZiBwV&3r(-?10#@2!>&=q?oQd+u1sqh&?yA?Ds7;wy=bC+;#+1&+bg=m zc5_);;%l8UpT>X3GpocUFkWo?UG6dgYRC9^?tR?mMB*^6V4(OW7pJ0$^DwfKV{2Y! zpgO2}ydy!EmLD@K%T0%Cyv8?2J7=eBd|9v$pD7O4*S-a*BpJzHY^<<3ANwOpiI2ZK z?cIcRut1KCTa1+Vq~`V-c>?V=XgB`;=E^oO#b9Pbw!Tz%AT~!X&e!Y%fC(F_DgkyH z?MN;q!9+3Z1fo@-t`$&U*sj~C#;pMr?W5vS<6#<RzE) zcH4r-MVXXM7issn7V}EG#<8e&bjZ%{f;aiRg}mWs-+erKvZga+{2K$D`r-mlAfdqE znzeE+Eo~?V*V(nEnPx|F3W1sMji`t4T@c1&cL{NPwRr~Ex}nj}@18$@u9^Gs z&IFlDNe4EQ@$6VS>__m*>$5Gf8b2a91OWJnUjt8ZRMe8^{>xMD?Ks?nd}yg-pPg|{ zr`ge3we|#q95bopa+@csW%n)xao@bM(y4Jt>je&_>u3b`o`{)3Y$j~qI8uEB4sAopU`}tYNgrtH!FAB6S7lQ36`}wseOGRdjb(I#0|4N_On^P|LilXhJeZeGx_w= zPBU{vq>#Rv{Vuwm)wMR^R;{4<$LEy5rRjT9;!@o1cR#%Z-4j8?4PWCsJhIqdOMLL! z4=G|^OJG}bmU1X8D(K5>mvV8~Ygj-x*zKpkw^+@gzJFL~#@C~F9CTm_fE@$EGwceR z;-QUWmx6iE|K#Dz@j0uBqHxC0)u~oi)L{`uN$;taGrWaTy4v=`*;5_l4dteQzjN4_MBJ4lA0sj|1< z8NsY7T$9=UwDif02H9@B_KS`u{VPBDVmjTv)WPlN0Sb`E^mQ32wBRB?V(j|M(%p5e zI2HSk6sFTNimy6TNtG&;Z>N~EIJM3#eIA(&zjydp%1ov&}B61WyIY9ceY1nzfL~h zOv>4g)a5*A5^iKjco^n`h?Vyp!sRfW5XYB?IqH*ww}|M5ge#O)?ahZx9VXRg2!)AG z8T+dCq$ct$wN7zNqJ6h>pZsunQ$Ezbz3mSfuN!7CPu++w>Q=PPQ>d)Ro5ocJ1t5^q zcgdsW`D`jCVRE>QM*Jw!_{3zpW1f<^q^k99r;L~N?zZ^K?k4IBfUHaVyZQ^c76Wl_ z^4libT9Y0&(O|tny_D4&+zlzFGsKd_{Ib0CP0Yg6F{a)0A8x357B8< z9?P0*t0_npmPgrP+r{Zxye$vc&qUrXa~qV-eUKaDI$UE2+@?u?Hr65r>|+#8h88~e z+iPGFI&?4oUSBNuecl!B7JDSm*B=pmXXpqsJI{bKG2sxG(&?)dvTS2_!+L_NbnPy9@WQKucfwg*ll}g(E{%j>(8hAXw_yU71xahOBttVvr_+n}BUSMG31~|p zz%9f+_nO^sE9=+|V^+wqtVZ-5`93OU8e=odV0XDS{ea2R67w++1E@_#Fj!3BOWK|A zEO5CiI^7vDq@lNOC2xAx2QFwW6l9&q6^AGT_aC6?G+SeMd9Ph-nJ*ix^k@fK`p0w` zKq9?4Cv<*nML$DP00+PAtg;~^&a#GSndc`; z$QW1<E))Y6QL99n z6Pz3FwzFvs8~7Ba*pJD@G%PCPrHyiwdh!zslH~BoMaDUL*>8$*zgD_M%*T7bJexr6 z?b@55tqi!Wm1&E7PQ~js?#0WL{rso?Pw}~Bj_t)Iwi6%Di;RCr5Saj8>g;C)G`4_} z87%nl!-tZ|^{v0a3uOr;z!D!AWi%(Vjag-1I7GyRn@z%QNe(?S-u6z721kVUckVkI z77B8jVXot)0@MYWL(3mXD=SmyB<{bUwDM=_6f4r6iYB6LvOAGuptKd^KjNu38?e9Y zwej_6rB8%&pI6}M0+z}HjCA4-e{Qq&l;@APq!^|U^Z89d#{cqpZ**leLy2#Tb7^aA zC0kqgBX4OR3G?nXKyzxyS0aQD`z|V0D%sfu?RYxY^ZOBBAn)seVw8K z&KiPkjl1zF4djO4?THJtUiN+uO7gw0@7|v~tWJC7zWHgj0ljM%vQ}YQvjBjn^`+I* zu+$J}QI2F5y*Yc0+c)1>W|ja6n7-(d#9|ljr@mgG{kqV|w&JO^VTleKek5RUVVEJo zc&qC?-4d|tc8um^Aljzjn5zZ1UYyV}F-c;03e1$(Xu8&eLNlo|-n2OhZs}(={)y}U zdBMj?Iy@M*Y6M>D1uPXowd@FK}>rP9doO z2tmJLBL%+rSa$8w>i+)5LcFBE{MfGTssn|Hin>g+F(xCFU>Qx_X`a|953Ap`q0FJ{ zxe*onotY{P;2eMCz@3RjBEU5s2fT_&&-Q>@dM&>4wv@Dw-Ct?yo?(|HjTGMClpn_z&5?@I;{=U#=<8PA zu(`MN)imrUz_kzt=7Ntcsc6zuvjM@;gsiUB)zij)fe!4h|_ri1bSOtJ*6RIeWzS-9WXdq{);18 z(O@bEoZRr|Eo9#CclkXu7Bf8eq$y0dCkItd{`xfyh7rr`1{m3jP0@sjQ^`5+!UZu4 z6TNgLq0x|?5{By{i`cc8*K~>#-LV>2%JfxMo5i8ZZsGg+7E@jCt`ZcT^}A~_N_nBT zDXw3qdt-X-qtzY&d?Y9uF;FTL)^zsZ^UiNbNfs2%JtZ>cFfrU5~ zvkzC*^@<2p)H%QB7#1&I73pTGihgtW7Z_$~DHA<>H&NKs0N4d=_ChM9@9uAySFVBL zn+W6-Ho(O*Jo@zMCR((8hFg1I#f2bD*&2(|Wmo}fdzQ_Yrz^nisX2;8vN0dvB7@uq ztPmFgcOBm948aGVf8m$ZE(LFWfLG%lM$&BQIYEbLCv1=%x_FeG_6l<6zp6ZcG0^|_ zfu;VgLi5V;)oSmFOfXoZAf$QgFVo^W38(^e4|!YW|g$hvriWP-I)@LtP=sN1ML(v%nHgiC|y`=m}prZ z^YZ`x#vsq>o@0%Qv`50>FlKGg7NfkXz1-V-1Y51UQu+G?joz*d(D?JcrZ8T$?KOjH zS5tu`G3z+Rtr&+UmHn1hd=3iW40oD*Ba&L-9$$Zq8M^I5&LRx^YmZn|e;ig8xj9lE z((n^Yb-(XR-E7#4VG=RZG;#V|u)V4;=ednhBxFZ0%*0o?i~x>MV4}z0)j|Q1z6-WLKX7$8Gf+=2Q%R^i-!S%MqNo-+ z(JJQ5Vfq%pzAx1v6Gg2PT+W=KfhP+rqA$pZTb~PV`}GKGfJXoPXeKITW2ox!_tne$O>;rY7J&l32V~CBeLC6U zrQVJ}kTR2Uz1BHHh;OwHaXaMOKCn7PgOurm_gb{iEvVjRpiLi!KFBpl7I$AP4DIno zj40rvG8EayR%$SALS;1JiCK}`Ek*L(_U4FCgfk&dQFPuLsbw3#=Nz4%X)e7>MKdJq z4_D*ku+HnQ&FS4wpFOjm0whxVl56VROi`)*@QBB7dHYWNAgZSPPL7CmE#XM(ofBst z)7QzB*|XOEUD?rs_H9YM^Ywi{`n(Y&1$?>gV;$!QxmQvryJa}<76N?nvTmM^%%@Db5+GMP@+KnFB6rQpD03@mkq``D2`zA@t??f(T`G{Iw@Fj=pR= z+3~Sk{Gok-zk@0RO6C9?=sKn#R%ZhZu^hZo3Fz)Cix{?wRPHbriW19rponJyEwj{j z)+iTDeZCoNmvN}I=7#VT^>pnMwQafc#Ao)I<>q z85C|*6xa2PzBSUmedwhKlu_I;FHU%NxY8q&V*b^BEBo7<9H+wv^Sul_SFYU3nC}zn zJKErP^PCf{nrU*+n`5qvW$DEI_ancD&NkI^OFicK-FW+;QmS&-1Y5e_QRjm_rrXkv z>IUvZEs#5S{*DlHL*xKYm1>W#Pvz{;uW z;FYY~)&CR6P;}nQn_$i_g*J4UAY#r3M(o7lD|QSgPlm_1x5Wvp z0N=Srat0V;ri{nx2bmE&Q)TRdC9gDi7s0^9lmM)XTD5-jF|I*^;FKVtx`%&7DQ|0z z1dK7ldfq>wDOl33{oJg4I!o*jv{k(sm>J|A`w`)tngK-GOoH$7pF1!x2nM_eP%x;- zLquCADkHaB0M{i7x^}5&(g&m428jc5=}{2D%r2txUA1{*oyfuJUY(TT=#t zL)S0kzR&m67V0;x?#~|pvgqxs6c(Eh%WAJXU^<)c0$_?jE2|Cct66^Yq>?T$nIwJx z=r&NQU#}$7Hv(n2y#ureS0-po48&#@eAY?B(9BXUaAVE@42TVYxltgkYalk}1OP`K z={W*UIH>`+fx^M(aWhN5n|ix0ff>1{PDCQ$ZT{5j(hd~5#n8g?t8xUP4F*o9gZ9Cs zaf`xY-BsF5<^!~qCH<*?PS4koejEC5I=p!KvIe!RuKh|Qii@*b1bkX`G?J~KL6wE{ z=vh5yG;d?k7xbNcHJ6Kj*fcd!cnxs)P*TC4oTe84b2laHdSD9UzI@b< zp;!jg!yAT4`kBg{wlxQC1^J&bIW!ObjVIxNm73oHgM&B3Xy>fj#sa=;1W>cbm^h;U zd=GDChk*On|CV>I?f2s(7%-Es%Wu%?9faHdWEEr!s2$=VdqJ$lW*B*IAd>Nji;F|4 zqhLCx|LuhWB?WU%Cq_>B2S&h%(4HvfpPj0VteG3E)CV`enrGwM@JtxEx3iH`lkh6FI@r?8(X zVlGVql*a^UZ5*~ffrw3JvE)48YTh>iq#P|kvkm`P?=$#C^EOeySeX-Pw+l?A%=T!~ z4jTINa^I0gUTEBPF>6?H-V--hnti}(#3}H1Kk3xpzz+*``J3)SW&UFN|2w-D{4Kzc z|DX3x_`i7;&ou?l0&N&3UErHz&->t(kGLUsNnBTb94>5fQ=X!s_q;FP*$9Co+=mu4_8K= zJb5w}gsmQEVt=I>fe8UN8TT|v816PO!CvlGrn+94JWKmbGz%&P(r;`gWoDhRGTG^} z$E6ZHG-YpDk1!OqtlH>P>X!)On5h=Q# z6G_`$g3t&D9ltvoSOj@~5oia2{JHWgPc@L{+NW`17{&|<$4D^t`a`}u`m);42oa>| ze{Gu~qjV3XBP9}?UpIvZnAi29>kifni=OJSM$$+na6~!?{{n{2HcS8Eb5r&(vL`6p zI#IYAWRT{3z$B|%N&=)3*k8M54<7P2>qd~t&zU|OW#!jv=~mfy{jnT@b|Dre|9PIMgqTF;Y^l#~vba6;Y2iUF=$D3o^->T2J_$zH{e+eyIziIwY>$sT z2b`YcvJfv6MJ9t;FJVY73k_Ft?TD0kLDj0au0)9){ z+7W8WN)w1tD9{Z~BL7l*d~41JYf+N~ zkp+k%=Bfr0p4Y`3O@wWif16Syt|5W?#uI<>%ReC zsGds8q)p}Gz*l@T`TM5c6Yp*>f#G;9pwHWb4O0w0sWZ|%GC&fC6& zH4}|7d>;s$95fFcZMPd>b)?y>-7`UyDcmrk`5S?|>1g8xF{?O$L}-go$I1#WJ=G}(vHr=V&Jf%GQ6 zXbDi&f4ccqkAHtH1w&lVt%MJbEs)Q60AwHnXZ{_?EOjT=fQOT2vcHin00h_qi)!%? z#s@cwc8S0<7YnxCsn*3u7BgK^7N+gWf^q7-1x*l6?#cfPxnPHBIHL{d@MG&`mbHGZ zAotmAE&cX1+laU*V@I180~zM^t5?yUV?Ste1`)&V1I)+91hh5HWsduN?<%Y>FTQ*B z&EkR45OT4SPV|2E+qIcKZS}YR{NAj)1Q=fO!|hmo<#%B4y#-*f2{6ZRg{SQ<@t`Rf z?32Q)5y`=)xd4LIXXag#4(w(7(yN1S>3|(t0gVvv)!o`FI<1^wMJztq=ljYpYY;sRoAgyZ}@W#F$`y z8WDYgqN7eJ02MP^a2?R~ykO^-*W&;BDBTj+Y{?$1EO$I`8)E zoY%LVPGCuj>jqAJk3Hw8pS19eqck-&b*e#(!J=CZRQUVM6(OQlbyaD~NVAxBFkRm{ z+SAi>Ez?v<-MZFKq<4}ihy&FqQmyMCta9@hixWbD9syd1nDJ<^4U*HdpZUhx_yj-LeG=iV0iHpwKNCP>cl0!AO!e0Fp6)A_5YXB3O2-sI3^txg``L z$x!6jtzse<6i6s+BtwA`EhNnTiheV**1Y%Q&GgLsGyi(^D$2Tb?|05Vd+)Q0mcdEM z983lfB*KCG5%7hAzZ?ec`cyi-VusW6n;F$?CHlJCLsAv3_WBm`C%Rfuh{~dsR zRq~?Xy$Sh()samml=BH5oo(dVp{X9`=K;olQHt6B!WCazEVO-*-yUR}iEuhmryc=f9Iu~|MQ2ooLdzsRSZWe9EfIbqn43>T4U#veUy~i;LF_h6~wbh76v=HWG@zwfo`#b_wn4H z@1v;rjJt`vR2!)|s#f6&kk0HdPS(O2|A%8y$dS4j+c5))Ef2w^z2 zuoFEWgI0wTLpI8mAaBImNTLt~p*Ot6_TW}!lf&7GNJQlL7czS>Te zWDO<5wXF@zK-A?KCG|gpV$0q+^Ea3Z<1r*TrpNC4Wn6X|M0q>{*ho`D^w=HG7DUvL)e)iF zGu_ufBbbtm{k1nhP_429$?1jC80dl!T6f@Gu#7#u8Jr<~b~c!rYNX%IF6TX95>G*S zUGRpa{tlsP(nxl35iHV4*1#%^;rc_7p($Kmr+%ql989s#Xjfog+OyR>TY@{(83IVI zb8X(TlTl8p!o)#4Nge`9rRIa-%&utaClF$51r1=Dk^B?WCopMI>)@=9?>- z%Sa%WDmaO{o+_m+4;CdYML59U;E4^$OKYbz%Yt(a{mZwS?=Ep`7<%pzo`F`oF=mq3 zgj~1CXwMwUt$Zse;QmaF(k<86oQ(s5evcZ{mXsLn^6flKY_|E7u$tI*yKKYg@Nmj)fGYV6&D8DuNKmW(^ zrT=_=`Rn&%{*pqeC0UH&L>EHnWcUUsMUL$JRW?+JI;21Hx$=(SL1{i?es3({a1mD( zk6G)1f|DiD6p%-G9Q3-x27~BfyHy5i-1yUPAM68L(>W@QZWG(d!^QV@<4&?6O`$IB zC#kIDNh6EIXlNuw^2l>reZ|XD`39uB-8DyE9mgF9XKw{>CXsNSZB7wevg8M>R|Vi4 z>LP4Gl9)O%h?BK>F_lX{k}dYO@*l`^Ah8NBH%~df!?X3~hFHCAm}R+M$a#$kX}_>r z&&!&;Mx$rN1-=VzzF+^CGn9oEr&Ulbu2nCN`)*0VF^M4HUa8c#L=J1{4UP5dHQ%nXqL=gM$vVDxPsGGNEbl@^OuJ%Y07?#l)a^(u&|eWcO`RVpv6kbqi|-k%mIC87Kq2#E(^%yht4ZUy+?b>)H^ zBjn7QPsU!}{x|(MrPLjZR6M+?AgYlO-C1gJL&D8H_B&cLU;>RUmCPo+5ZuB;`)*bKW4eBXPW!tc z8_<1!dzH<+Y|5o?qa2sfJ`*hN+X7|Or8(%hZ?nh^JUhheS1_0*^ZTGe3}t~CIIPCB zYq)OAJcbH#xC3Y+1j%6ggdW%8gGx%}uJx7Sp+n3Zl0Q1D@;z8vx0G_lSM`JI2LD?( zZrm7qMxH$n&h$??*ahAg17^KkM*n1~I@nO=_4)|tqOZa)Q?8^}x9Pl%5}@#1wVy}H zGdCb47qU!$soROahxB!dq4`*@OebLF(AdnO?vm#~C42@Vp3E-!f-DX5n5*4;coPQ- zV*O1@>+N14+SQkG0z&$_zH(n3>Ytp*4_Ssf_w?ke(iSMGd%1cSO+j|YgO<7}e;Iz` z()VWHPDt!H`&rZ?$eZ$!$}f>~;qFgYm$p1;p>bv>W9BD?-UNBKffe8f*5OR0p9V^- z>fjx;cs<||dm#>HN*l26gz8d){Y2TA@E9ni&brf}D+uu$sjB>qxA0GqaG#qkUD@+$ zg_W#oC`h$Z#_Iy$2yf7$gWy3(;B#xR2cj@{dX1hJnN6m|ftklBX^)0rF>%ykRQ-_c z9CyC{i6*h{)I2&&r#?>v!g)tz6ttb2%n1z_#7t!6!*bZMjdDR1NmjV9z%6ec&T8Zp z#>F4`)BdC{*nfwd(4aBbmW^>5SM)Nzf+GsyG10kt5{tV%5y;TL^yvhYhVWiZi2-2= zIc5uMjk3p$=B&*L9KXO|@vnY|p3o?X1tG?dk7<*lzB2Ju$RbgXsR>V}mlCb3`^~Hz zmv&^%21|Y$Y|OAfsCYW@iX*llK1gZII}8E+Es*tN3tw|&ahYHZVm%n2fIr`D^#ag~ zhJ8;gNR#H~*_N%2sh;HwN9FA7t6SOy7GDOgJy`ob%ggslyezV>X-$7jIU4`kKSxrS z-?~^BEEAn`9&{DLUCluZ#>SJ7Y0`Rk+PphmeW~G@%`d0R!fOK^yO^c0XCK7KED2t* zTo!Un0Kx(Y2}J@YpKg7jlUzGj)z0vD@cgpoM4uHE89Y%ny7)mbcj*-9x zlr6$ul&@cs>zEqVDpRy0q0YTW?ycD}?Sl2)gbb*%%%TMrk0>++O$A$FQh`|B$6^hS zavJH~5A1Wn%!E!SY6t9hX?sIF79&)HC8Da+!{JC#2wPck%9!QjH#OSJP51olF-da6 zP3Y|Tp3wo(jU#Adq0+D3DGj)&&48B!G%s1(8b2W%01MlDuU9d-oQT{9Z8>NtdOF2;bqvR>`ty zFN>(F0`aoTiD-S{xDU!|;0A}d2&A{Auh?u<`kk0q-z89wv2CwSOMn=}fH!1&bum1e zOoOLrs>XxH`ck<^_?41=)|Z^lXjh=cs>4WhJb)XIWeXeE+v4hM84JicaUex(eR9QR z)8{YWazWOiq2@56?fuu+9|0$S>hs6n4%49`G>{nkEIVp6X#U?F8Ze2Ly7@p!eQU0M zYsI$yuejRD@?WyrV^qCZUy5f5hYn=|YIaNz=`#+(Q3v&-^v<2tI}ks5Ab5+380yik zx{Z_5c>{&L!G}WP>h9y~iRcZo6n5is8nZ5%7x^}arD3G?$I>X4Rm*ysN>H5lMYM!g zkM(_E_C|M;R#iRKHO{5|UB^TX zw{UvOt7s_peWiPsXG&2g%hS8XXH4->j2b6+6lKI8|D2-Mo;~^vP_C6pqjdO{JG|5%F550;QUJkn&d)(Amcx!X0crgooXK4Uk6^$pEp_|;lzz}&~errsN-bH3Wv%2m7+h6K;l!vITxFr7`kib>$VEF!kOgi zr5wOzmzX~i_v(APTcB7e#_LFt5*4lk_+@#f^Yx(6ISzKLqNsx>?0DE@pg!3a zNf*u;{)3IBrM^hI!UO8hkLtenq0xuOA6Bql-x2N1)U&wTKi_Id)M8veLO!e=toCmX z<3-a`_Vivkk-5W^-MN0lt9G8g(AS(-m!99nO{}97%&F}4M=DK(=VCU-(!*uA=)R7#JI!QzGjqMpTi$9D|V) zS1dOpO~2KBt;FJ4);^1iklpQCP&tP~jlPVK5#RQG73!0dqu!PD_f_qw7J01v7EF0L z@?9^yj90t$sQwf@EG@IKk}F$$Tibhw%)A;fjfHy@ zKqehja=Wp$#SlwCZHQ#t&i!)UY-s>^@7jsl(d}?su;!fVNMJa8L4|i}s6XC3*-JII zGyDGfX1-Q@5*re9%zRCY&TH_aWfmAVBx-o?y~ys7JJ7h^-puS5%jDsvy3yCsb^H#W zQtD!X2g9Z}JH#Y37w^W&G2S(gu1 zbTkP4Z>FkqQso~z=UAR?tjCgfj{fts70{HiYuob%;JGZLcu4I)9kyQfkkyH@jClCD z=*DBX*$1)eo&Y7bL+#CT;VWMU<&-6K*uvN|uh({m zlF{ph4bE3LM0wy>B(m(-N}1J%EMp`3Q+qG>#`eEkt@F~==V_IHUJZchM2Xy3qQwuk}${gUEim+f?it-6Z12JCwor6!P^3C*OS`E!FY^b3Tm{JQaFe05R(uF=OoC=>XbezaIv?W;y1w?mX@U!`&w5*!EFUIY6)FNhnX zB{s-gr5WCQ>U4i6d=pQfP*2uIe^)TlladlWd!+lYPz8QKC7mZ9i-YVr|Bhr=Pedx^eP5c*O>g4jv@cRPi%?F2#Q85oGNZSA>JZLf`a zm$Q$PB}R)N(x3Ep(D1)JJ2Ol-w5wH?QH+vz2yg7fPq)a<%C+x4q{V#jgGFI7j!BaW z7lLA8OXDL0g*eeO6;%F|#-tZPm;4)Hn{rX~oycx!q|ClGZ<%!H;}RqH6AAj?xnY|Cw}yAGwXw7#5wEC->-^$^tPclmV^M+dg4 z47kb`lSFlWCH(=h{yXnT4j9?(7HS5}uNH zS=e(ckIiwD**7?1$6$-8kJg`}ucov0&e^4@nP817DQPN>LmRo7?8vLC(P=$Xm7?C0 zdY5mfNNdxY7l_K228N!C$O7FQDypg~o-r9=WaED`)0o-&l1w=WE9qDrEBt(TJr^wL z!F%?hmNASwPt{88;rLO^0@rCTBG#K7vb;dM7NNQ?F!7$?*K_!ko}fuBU^S5 zDmyi0pBPM%(gL6CLKnpkk8n?nBd4!aq>t=-iqBnmD|NJ*)_hh%5lA*n$^CK86()Ul z!;XNd7HPvLXRUjc^u>?cc|C_47@!2{s=`gO^)-|l@#y!)Koktcm<;JyBh3_})Nner)uzAKowI`rJ8?6*^>6%VS?Rm zG=Z0Q1Zm@xck^WVlo6o~&e5IWXCm``b>{>+yc|MB(S(=f91Tl6X~Rc`xIph929qa}9jd+6>OAD;oP}JaROd l|8cazUumzC@c2AUnKf8){`&rlp9y`cz3 select -> note -> save; DB-ground-truth judge; $0 (no model calls)", + "substrate": "Parallels Windows 11 ARM VM on Apple M2 Max; WindowsBackend over in-guest WAA HTTP shim (session 1)", + "target_app_note": "WinForms substitute for OpenDental (trial not no-touch installable; see PHASE2.md).", + "identity_armed_coverage": { + "click_steps": 4, + "armed_clicks": 2, + "armed_coverage": 0.5 + }, + "uia_tree_quality": { + "targets": { + "searchBox": true, + "searchButton": true, + "patientGrid": true, + "noteBox": true, + "saveButton": true, + "patient_row": false + }, + "n_targets": 6, + "n_usable_id": 5, + "usable_fraction": 0.833, + "identity_target_has_id": false + }, + "arms": { + "compiled": { + "n": 21, + "success": 6, + "wrong_action": 3, + "safe_halt": 12, + "false_abort": 9, + "miss": 0, + "error": 0, + "success_rate": 0.286, + "wrong_action_rate": 0.143, + "wall_s_mean": 20.69 + }, + "uia_identity": { + "n": 21, + "success": 21, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0, + "miss": 0, + "error": 0, + "success_rate": 1.0, + "wrong_action_rate": 0.0, + "wall_s_mean": 11.14 + }, + "uia_positional": { + "n": 21, + "success": 15, + "wrong_action": 6, + "safe_halt": 0, + "false_abort": 0, + "miss": 0, + "error": 0, + "success_rate": 0.714, + "wrong_action_rate": 0.286, + "wall_s_mean": 11.03 + } + }, + "matrix": { + "compiled": { + "clean": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_125": { + "n": 3, + "success": 0, + "wrong_action": 0, + "safe_halt": 3, + "false_abort": 3 + }, + "render_150": { + "n": 3, + "success": 0, + "wrong_action": 0, + "safe_halt": 3, + "false_abort": 3 + }, + "theme_dark": { + "n": 3, + "success": 0, + "wrong_action": 0, + "safe_halt": 3, + "false_abort": 3 + }, + "data_reorder": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_decoy": { + "n": 3, + "success": 0, + "wrong_action": 0, + "safe_halt": 3, + "false_abort": 0 + }, + "data_siblings": { + "n": 3, + "success": 0, + "wrong_action": 3, + "safe_halt": 0, + "false_abort": 0 + } + }, + "uia_identity": { + "clean": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_125": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_150": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "theme_dark": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_reorder": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_decoy": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_siblings": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + } + }, + "uia_positional": { + "clean": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_125": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "render_150": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "theme_dark": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_reorder": { + "n": 3, + "success": 3, + "wrong_action": 0, + "safe_halt": 0, + "false_abort": 0 + }, + "data_decoy": { + "n": 3, + "success": 0, + "wrong_action": 3, + "safe_halt": 0, + "false_abort": 0 + }, + "data_siblings": { + "n": 3, + "success": 0, + "wrong_action": 3, + "safe_halt": 0, + "false_abort": 0 + } + } + }, + "conditions": [ + "clean", + "render_125", + "render_150", + "theme_dark", + "data_reorder", + "data_decoy", + "data_siblings" + ], + "runs": [ + { + "arm": "compiled", + "condition": "clean", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 21.97, + "error": null + }, + { + "arm": "compiled", + "condition": "clean", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 23.0, + "error": null + }, + { + "arm": "compiled", + "condition": "clean", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 22.41, + "error": null + }, + { + "arm": "uia_identity", + "condition": "clean", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.91, + "error": null + }, + { + "arm": "uia_identity", + "condition": "clean", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.0, + "error": null + }, + { + "arm": "uia_identity", + "condition": "clean", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.97, + "error": null + }, + { + "arm": "uia_positional", + "condition": "clean", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.97, + "error": null + }, + { + "arm": "uia_positional", + "condition": "clean", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.0, + "error": null + }, + { + "arm": "uia_positional", + "condition": "clean", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.93, + "error": null + }, + { + "arm": "compiled", + "condition": "render_125", + "i": 0, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.81, + "error": null + }, + { + "arm": "compiled", + "condition": "render_125", + "i": 1, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.74, + "error": null + }, + { + "arm": "compiled", + "condition": "render_125", + "i": 2, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.78, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_125", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.23, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_125", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.99, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_125", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.92, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_125", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.93, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_125", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.0, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_125", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.95, + "error": null + }, + { + "arm": "compiled", + "condition": "render_150", + "i": 0, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.37, + "error": null + }, + { + "arm": "compiled", + "condition": "render_150", + "i": 1, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.34, + "error": null + }, + { + "arm": "compiled", + "condition": "render_150", + "i": 2, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.42, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_150", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.38, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_150", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.96, + "error": null + }, + { + "arm": "uia_identity", + "condition": "render_150", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.66, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_150", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.98, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_150", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.98, + "error": null + }, + { + "arm": "uia_positional", + "condition": "render_150", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.97, + "error": null + }, + { + "arm": "compiled", + "condition": "theme_dark", + "i": 0, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.92, + "error": null + }, + { + "arm": "compiled", + "condition": "theme_dark", + "i": 1, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.6, + "error": null + }, + { + "arm": "compiled", + "condition": "theme_dark", + "i": 2, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": true, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_001", + "halt_reason": "Postconditions failed for step 'step_001' (type 'Neil'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(16, 60, 106, 94) \u2014 r", + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "geometry": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 20.81, + "error": null + }, + { + "arm": "uia_identity", + "condition": "theme_dark", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.0, + "error": null + }, + { + "arm": "uia_identity", + "condition": "theme_dark", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.21, + "error": null + }, + { + "arm": "uia_identity", + "condition": "theme_dark", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.96, + "error": null + }, + { + "arm": "uia_positional", + "condition": "theme_dark", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.92, + "error": null + }, + { + "arm": "uia_positional", + "condition": "theme_dark", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.99, + "error": null + }, + { + "arm": "uia_positional", + "condition": "theme_dark", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.95, + "error": null + }, + { + "arm": "compiled", + "condition": "data_reorder", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 21.89, + "error": null + }, + { + "arm": "compiled", + "condition": "data_reorder", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 23.32, + "error": null + }, + { + "arm": "compiled", + "condition": "data_reorder", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 22.6, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_reorder", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.39, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_reorder", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.96, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_reorder", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.97, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_reorder", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 10.97, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_reorder", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.02, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_reorder", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorenson", + "wall_s": 11.0, + "error": null + }, + { + "arm": "compiled", + "condition": "data_decoy", + "i": 0, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": false, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_003", + "halt_reason": "Identity check failed for step 'step_003' (click '1'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not matc", + "identity_verified": 0, + "identity_mismatch": 1, + "identity_unreadable": 0, + "rungs": { + "template": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 15.3, + "error": null + }, + { + "arm": "compiled", + "condition": "data_decoy", + "i": 1, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": false, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_003", + "halt_reason": "Identity check failed for step 'step_003' (click '1'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not matc", + "identity_verified": 0, + "identity_mismatch": 1, + "identity_unreadable": 0, + "rungs": { + "template": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 15.62, + "error": null + }, + { + "arm": "compiled", + "condition": "data_decoy", + "i": 2, + "outcome": "safe_halt", + "wrong_action": false, + "false_abort": false, + "completed": false, + "target_note_ok": false, + "wrong_patient_id": null, + "replay_success": false, + "halt_step": "step_003", + "halt_reason": "Identity check failed for step 'step_003' (click '1'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not matc", + "identity_verified": 0, + "identity_mismatch": 1, + "identity_unreadable": 0, + "rungs": { + "template": 1 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 15.44, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_decoy", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 11.71, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_decoy", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 11.67, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_decoy", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 11.01, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_decoy", + "i": 0, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 12, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Anderson", + "wall_s": 11.02, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_decoy", + "i": 1, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 12, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Anderson", + "wall_s": 11.02, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_decoy", + "i": 2, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 12, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Anderson", + "wall_s": 10.95, + "error": null + }, + { + "arm": "compiled", + "condition": "data_siblings", + "i": 0, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 11, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 21.72, + "error": null + }, + { + "arm": "compiled", + "condition": "data_siblings", + "i": 1, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 11, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 22.87, + "error": null + }, + { + "arm": "compiled", + "condition": "data_siblings", + "i": 2, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 11, + "replay_success": true, + "halt_step": null, + "halt_reason": "", + "identity_verified": 2, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": { + "template": 4 + }, + "selected_index": null, + "selected_name": null, + "wall_s": 22.56, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_siblings", + "i": 0, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 10.99, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_siblings", + "i": 1, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 10.98, + "error": null + }, + { + "arm": "uia_identity", + "condition": "data_siblings", + "i": 2, + "outcome": "success", + "wrong_action": false, + "false_abort": false, + "completed": true, + "target_note_ok": true, + "wrong_patient_id": null, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 1, + "selected_name": "Neil Sorenson", + "wall_s": 11.01, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_siblings", + "i": 0, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 11, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorensen", + "wall_s": 11.01, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_siblings", + "i": 1, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 11, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorensen", + "wall_s": 11.04, + "error": null + }, + { + "arm": "uia_positional", + "condition": "data_siblings", + "i": 2, + "outcome": "wrong_action", + "wrong_action": true, + "false_abort": false, + "completed": true, + "target_note_ok": false, + "wrong_patient_id": 11, + "replay_success": null, + "halt_step": null, + "halt_reason": null, + "identity_verified": 0, + "identity_mismatch": 0, + "identity_unreadable": 0, + "rungs": {}, + "selected_index": 0, + "selected_name": "Neil Sorensen", + "wall_s": 10.99, + "error": null + } + ] +} diff --git a/docs/desktop/LIMITS.md b/docs/desktop/LIMITS.md new file mode 100644 index 0000000..b8f5fb5 --- /dev/null +++ b/docs/desktop/LIMITS.md @@ -0,0 +1,59 @@ +# Desktop Phase 2 — Limits and caveats + +Read the numbers in `benchmark/desktop/BENCHMARK.md` against these. Every one +was pre-committed in the spike spec; publishing regardless is the point. + +## 1. ARM + x64-emulation rendering is not native x86 + +The guest is Windows 11 **ARM**; the host is Apple Silicon. Text/glyph +rendering under this stack is **not byte-identical** to native x86 Windows. +The compiled-replay arm is pixel-template + OCR based, so its absolute numbers +are specific to this rendering substrate. The **mechanism** (record → compile → +replay with identity verification, DB-judged) is what transfers; a native-x86 +confirmation run (cloud spot VM with a persisted WAA image) is future work. + +## 2. Render-scale is a proxy for DPI, not a real DPI change + +The spec asks for DPI 100/125/150 %. A real per-monitor DPI change on Windows +requires a **session sign-out/in**, which is not no-touch automatable in this +harness. Instead the app scales its **base font** (`font_scale` 1.0/1.25/1.5), +reproducing the same *class* of rendering shift — larger glyphs, moved targets, +different anti-aliasing — that defeats pixel-template matching, which is the +effect under test. It is labelled `render_125` / `render_150`, not `dpi_*`. +Note also that the WinForms app runs **DPI-unaware** (bitmap-scaled by the +compositor), so a true system-DPI change would partly bitmap-rescale uniformly; +the font-scale proxy is arguably a *harder* test of layout robustness. A real +DPI-aware, sign-out-driven DPI sweep is future work. + +## 3. WinForms substitute for OpenDental + +The benchmarked app is `patient_notes.ps1` (WinForms + SQLite), **not** +OpenDental — the OpenDental trial is not no-touch installable (SmartScreen + +UAC secure-desktop + interactive wizard; see `PHASE2.md`). The substitute +preserves the properties that matter: a real WinForms UIA tree (with the +same partial/broken row a11y), a list-select → edit → save workflow, and exact +SQL ground truth. It does **not** reproduce OpenDental's specific visual +density or its MariaDB schema. The clinical-app confirmation is future work. + +## 4. Window-resize drift is deferred + +The app runs **maximized** so the captured frame is entirely app content (no +background-window bleed into identity bands / postconditions). A non-maximized +window re-introduces desktop bleed that destabilises band OCR. A clean +window-resize condition (solid-colour desktop, closed shell windows) is future +work; the render-scale and theme conditions already exercise layout/appearance +shift. + +## 5. Modest N — this is an existence result + +N is small per cell (a first desktop existence result, not a big-N study). The +outcome *categories* (success / wrong-action / safe-halt / false-abort) and the +identity-vs-positional contrast are the signal; the rates carry wide error +bars. Scaling N and adding native-x86 + Mode-B (RDP/VNC pixel-stream) runs is +future work. + +## 6. Identity band coverage is app-specific + +Armed coverage (which click steps carry an identity band) depends on how much +stable text sits around each target in *this* app. The headline number is a +per-app measurement, not a universal constant; it is reported, not asserted. diff --git a/docs/desktop/PHASE1.md b/docs/desktop/PHASE1.md new file mode 100644 index 0000000..034c028 --- /dev/null +++ b/docs/desktop/PHASE1.md @@ -0,0 +1,194 @@ +# Desktop Spike — Phase 1: WindowsBackend + Recording-Adapter Contract + +Phase 1 de-risks the desktop integration (spec §2.1–§2.2). It does **not** +run the benchmark matrix — that is Phase 2, gated on the identity matcher +settling (PRs #16/#17). No model calls, no agent arm, $0 API spend. + +## Infrastructure path taken: contract mock, not the live VM + +The spec assumes the Azure WAA pool VM. Reality check at execution time: +`az vm list` returned **zero VMs across all enabled subscriptions** — the +`waa-pool-XX` pool does not currently exist, and bringing one up means +`pool-create` plus a ~20–35 minute Windows install (plus VM spend). Per the +phase plan, the WindowsBackend was therefore built and validated against a +**local stateful mock of the WAA HTTP contract**; the real-VM smoke test is +deferred to Phase 2 as a small wiring step. + +The contract itself is taken from the authoritative client in +openadapt-evals (`openadapt_evals/training/standalone/waa_direct.py`, +the WAADirect pattern — plain HTTP, no adapter layer; the WAALiveAdapter is +known to crash the server): + +| Endpoint | Contract | +|---|---| +| `GET /screenshot` | raw PNG bytes via Flask `send_file()` — **not** base64 JSON; read `resp.content` | +| `POST /execute_windows` | `{"command": ""}`; the server runs `exec(command, ...)` with `pyautogui` importable | + +**Correction to project memory:** the note "WAA `/execute` needs a +`python -c "..."` wrapper" applies to a *different* endpoint. For +`/execute_windows` the WAADirect docstring is explicit: send bare Python +statements, do **not** wrap in `python -c`. This backend follows WAADirect +(and its tests `exec()` every emitted command to prove it is valid bare +Python). + +## WindowsBackend (`openadapt_flow/backends/windows_backend.py`) + +Implements the existing 4-method vision-only `Backend` protocol +(screenshot / click / type_text / press, + scroll) over the WAA HTTP API, +mirroring `PlaywrightBackend`'s shape. ~180 LOC. Design points: + +- **Vision-only, honestly.** It does *not* implement the optional + `StructuralBackend` observations (url / page_title / page_count): native + Windows has no cheap equivalent, so steps that would rely on structural + postconditions stay unverified (docs/LIMITS.md), exactly as the protocol + intends. The replayer probes these with `getattr(..., None)` — absence is + handled, not an error. +- **Viewport** is derived once from the first screenshot's PNG header + (IHDR), overridable in the constructor. No resolution assumption. +- **Command safety.** Typed text is embedded via `repr()` — a valid Python + literal, immune to quote/backslash injection (WAADirect's manual escaping + is a known sharp edge). +- **Non-ASCII typing.** `pyautogui.write` silently drops characters it + cannot type — a *silent wrong-write* mode. Non-ASCII text is therefore + routed through the clipboard: base64 → PowerShell `Set-Clipboard` → + Ctrl+V. ASCII text uses plain `pyautogui.write`. +- **Keys.** Playwright-style names/chords (what recordings and the replayer + emit, e.g. `Enter`, `ControlOrMeta+a`, `Meta+d`) are normalized to + pyautogui names (`enter`, `hotkey('ctrl','a')`, `hotkey('win','d')`); + `ControlOrMeta` resolves to Ctrl on Windows. +- **Scroll.** The protocol speaks pixels (Playwright wheel, +dy = view + down); pyautogui speaks wheel notches with the opposite vertical sign. + Conversion is 100 px/notch (approximate by design — replay's closed-loop + scroll re-resolves after every gesture, so the ratio is not load-bearing; + tune on the real VM in Phase 2). +- **Screenshots** retry (server momentarily unready), and payloads are + validated as PNG before being returned. +- `requests` is the only new dependency, behind the `windows` extra + (`pip install 'openadapt-flow[windows]'`); the `backends` package + re-exports lazily so nothing changes for browser-only users. + +### Protocol conformance result: zero compiler/replayer changes + +`tests/test_windows_backend.py::test_record_compile_replay_over_windows_backend` +runs the **unmodified** Recorder → compiler → Replayer stack over the +WindowsBackend against a stateful mock WAA app (a cv2-drawn 3-step +click → type-note → Enter workflow whose screen state advances only when +the emitted commands are correct, coordinate-checked): recording, bundle +compilation, and a successful replay (`report.success`, postconditions +verified by real OCR, typed-input verification passing) all work with no +changes to any existing module. The 4-method protocol held. + +No abstraction leaks found in Phase 1. Two soft spots to watch on the real +VM (not leaks, tuning): screenshot latency vs. the settle-detection +polling budget, and the px/notch scroll ratio. + +## Recording-adapter contract (`openadapt_flow/adapters/capture.py`) + +The spike's main integration risk (spec §2.2): desktop demonstrations come +from **openadapt-capture**, whose output format was reverse-engineered from +the sibling repo (`openadapt_capture/events.py`, `storage`), and converted +to the flow recording format the compiler consumes. + +### Input schema (openadapt-capture session directory) + +``` +/ + capture.db # SQLite + video.mp4 # screen video; frame wall-clock time = + # capture.video_start_time + frame_pts_seconds +``` + +`capture.db` tables (relevant columns): + +- `capture(id, started_at, ended_at, platform, screen_width, screen_height, + pixel_ratio, video_start_time, task_description, ...)` — one row. +- `events(timestamp REAL, type TEXT, data JSON, parent_id)` — raw streams + (`mouse.move/down/up`, `key.down/up`, `screen.frame`, `audio.chunk`) plus + *derived* action events produced by openadapt-capture's post-processing + (`process_events`): `mouse.singleclick`, `mouse.doubleclick`, + `mouse.drag`, `key.type`, `mouse.scroll`. + +### Output schema (openadapt-flow recording, unchanged) + +``` +/ + meta.json # {"id", "created_at", "viewport": [w,h], "app_url": null, + # "params": {...}, "source": "openadapt-capture", + # "task_description": ...} + events.jsonl # {"i":0,"kind":"click","x":..,"y":..,"t":..} etc. + frames/{i:04d}_before.png / _after.png +``` + +### Conversion rules + +| capture event | flow event | notes | +|---|---|---| +| `mouse.singleclick` (left) | `{"kind":"click"}` | non-left buttons: **rejected** (no flow equivalent) | +| `mouse.doubleclick` (left) | `{"kind":"double_click"}` | | +| `key.type {text}` | `{"kind":"type","text",...}` | marked `"param"` when the text equals a value in the caller's `params` map | +| `key.down` (named, non-modifier) | `{"kind":"key","key"}` | pynput names mapped to Playwright names (`enter`→`Enter`, `esc`→`Escape`, `up`→`ArrowUp`, ...); bare modifier presses skipped | +| `mouse.scroll {dx,dy}` | `{"kind":"scroll"}` | pynput notches (+dy = scroll up) → pixels at 100 px/notch with vertical sign flipped (+dy = view down); signs provisional until verified on hardware | +| `mouse.move`, `mouse.down/up`, `key.up`, `screen.frame`, `audio.chunk` | skipped | already merged into derived events / irrelevant to replay | + +**Loud-failure rules** (silent drops are wrong-action seeds): + +- Raw-only sessions (no derived events) → `ValueError`: run + openadapt-capture's `process_events` first; deriving actions is the + capture library's job, not the adapter's. +- `mouse.drag`, `key.shortcut`, and any *unknown* `mouse.*`/`key.*` type → + `ValueError` (converting would silently drop a user action). +- Unmapped named keys → `ValueError` naming the key. +- Two params demonstrating the same value → `ValueError` (ambiguous + marking). + +**Coordinate spaces.** Capture mouse coordinates are logical points +(pynput); video frames are physical pixels (Retina/HiDPI differ by +`pixel_ratio`). Flow requires event coordinates in frame-pixel space, so +points are scaled by `video_frame_width / capture.screen_width` — the video +is authoritative, not the stored ratio. + +**Frame selection.** For an event at wall-clock `T`: *before* = last video +frame at/before `T`; *after* = frame at `T + settle_s` (default 1.0 s), +clamped to just before the next event. This approximates the live +Recorder's perceptual-hash settle wait offline; if it proves too coarse on +real captures, the fix is a phash-based settle scan over the video segment +— same module, no format change. + +### Contract validation + +`tests/test_capture_adapter.py` builds a synthetic capture session — a real +SQLite `capture.db` in openadapt-capture's exact schema and a real +`video.mp4` (cv2-encoded, Retina-style 2× pixel ratio) — converts it, and +asserts: event mapping/order, recorder line-format parity, coordinate +scaling, before/after frame selection (classified against the app's state +ladder), param marking, and every loud-failure rule. The converted +recording is then compiled by the **unmodified** `compile_recording` +(click/type/key steps, param preserved). The format bridge is proven. + +## Proven vs deferred + +Proven in Phase 1 (all offline, mock-contract): + +- WindowsBackend implements the Backend protocol; commands match the WAA + wire format (every emitted command is `exec()`'d in tests the way the + server does it). +- Zero compiler/replayer changes needed — full record→compile→replay loop + over the WindowsBackend succeeds. +- capture→flow recording contract defined, implemented, and compiler- + accepted on a synthetic demonstration. + +Deferred to Phase 2 (needs VM / matcher settled): + +1. Bring up a WAA pool VM (`oa-vm pool-create` → `pool-wait`; never + `az vm restart`) and run the real 2–3-step smoke (Notepad-class app) + through the WindowsBackend: screenshot latency vs settle budget, scroll + px/notch, pyautogui key-name coverage, clipboard-paste path on real + PowerShell. (Note: the local `oa-vm` CLI itself is currently broken by a + NumPy 2 incompatibility in openadapt-evals' import chain — fix or use + raw `az` + SSH tunnel.) +2. Convert a *real* openadapt-capture recording (incl. `process_events` + derivation, real video timing) — the synthetic test pins the schema, not + capture's runtime quirks. +3. OpenDental install + DB ground truth, drift matrix, identity/false-abort + numbers on desktop rendering, UIA steelman arm, Mode B (RDP stream) — + the benchmark proper (spec §3–§7), after PRs #16/#17 land. diff --git a/docs/desktop/PHASE2.md b/docs/desktop/PHASE2.md new file mode 100644 index 0000000..02d05cd --- /dev/null +++ b/docs/desktop/PHASE2.md @@ -0,0 +1,108 @@ +# Desktop Phase 2 — Automated Parallels benchmark pipeline + +Phase 1 delivered `WindowsBackend` (the 4-method vision Backend over the WAA +HTTP contract) and the capture adapter, validated against a **mock** server. +Phase 2 makes it **real and fully programmatic**: record → compile → replay a +desktop workflow on a live Windows 11 VM, benchmarked against the UIA +incumbent, judged by database ground truth — with **no manual/GUI steps** and +**$0** (no cloud, no model calls). + +## Why Parallels (not the WAA/QEMU cloud stack) + +The host is an Apple M2 Max. Apple Silicon has **no nested virtualization**, so +the WAA Windows-in-Docker/QEMU stack cannot run here. Parallels uses native +Apple-Silicon virtualization, so the Windows 11 **ARM** guest runs directly; +x64 apps run under Windows' built-in Prism x64 emulation. This is the "free +local Mac VM this week" path from the infra review — $0, no cold boot. + +## Architecture (all driven by `prlctl` + Python) + +``` + Mac (host) Windows 11 ARM guest + ───────────────────────────────────── ────────────────────────────── + ParallelsVM ── prlctl exec (SYSTEM, ─────▶ session1_launch.py + lifecycle/snapshot/exec/capture │ └ CreateProcessAsUser ─▶ session 1 + push_file (ephemeral HTTP + curl) ─────┘ ├ waa_shim.py (Flask, :5000) + │ GET /screenshot → PNG (mss) + WindowsBackend ── HTTP :5000 ──────────────────▶│ POST /execute_windows → exec(pyautogui) + screenshot / click / type / press │ GET /uia → pywinauto tree dump + ├ patient_notes.ps1 (WinForms app) + Recorder→compile_recording→Replayer └ pn_db.py (SQLite ground truth) + desktop_benchmark.py (orchestrator) +``` + +### The two foundational facts we had to solve + +1. **`prlctl exec` runs as `NT AUTHORITY\SYSTEM` in session 0**, which is + isolated from the interactive desktop — an in-guest `mss` screenshot there + fails with `BitBlt` and `pyautogui` input never reaches the real desktop. + Fix: **`session1_launch.py`** uses `WTSQueryUserToken` + + `CreateProcessAsUser` (with `lpDesktop=winsta0\default`) to place the shim + in the **interactive console session (session 1)**. This is the single + non-obvious blocker; everything downstream depends on it. +2. **`prlctl exec` hangs on very long arguments**, so files cannot be pushed as + base64 in argv. Fix: **`ParallelsVM.push_file`** serves the file from a + short-lived ephemeral-port HTTP server on the Mac and `curl`s it in-guest. + +Host-side **`prlctl capture`** provides a ground-truth desktop PNG independent +of guest state (used for diagnostics; the benchmark's frames come from the shim +so `WindowsBackend` sees exactly what it drives). + +## Components (this PR) + +| File | Role | +|---|---| +| `openadapt_flow/backends/parallels_vm.py` | `ParallelsVM`: lifecycle, snapshot/revert, exec, `push_file`, `capture`, `guest_ip`/`host_ip`, `launch_shim` | +| `scripts/desktop/waa_shim.py` | In-guest WAA-contract HTTP server (screenshot/execute_windows) + `/uia` tree dump | +| `scripts/desktop/session1_launch.py` | SYSTEM→session-1 launcher (`CreateProcessAsUser`) | +| `scripts/desktop/patient_notes.ps1` | WinForms target app (list-select → edit note → save); drift knobs via `pn_env.json` | +| `scripts/desktop/pn_db.py` | SQLite ground-truth CLI (seed/list/save/get/all) | +| `scripts/desktop/uia_arm.py` | UIA-selector arm (pywinauto), identity + positional | +| `openadapt_flow/benchmark/desktop_benchmark.py` | Orchestrator: 3 arms × drift matrix, DB judge, results.json/BENCHMARK.md/chart | + +## Snapshots (reversibility + warm boot) + +Two snapshots exist on the user's VM (the user's own VM — never deleted): + +- `pre-openadapt-phase2` — taken **before any mutation**; full revert path. +- `harness-ready` — Python 3.12-ARM64 + deps, shim + app scripts deployed, + toasts disabled, DB seeded clean, app running maximized, shim listening. + Reverting restores this **running** state in seconds (warm resume) — the + per-session warm boot and a coarse clean-state reset. + +The per-run reset the benchmark actually uses is faster still: reseed the +SQLite DB and relaunch the app (`prepare_condition`), giving identical clean +state without a full VM revert. + +## Running it + +```bash +# one-off full matrix (6 conditions × 3 arms × n): +python -m openadapt_flow.benchmark.desktop_benchmark --out benchmark/desktop --n 3 +``` + +`DesktopHarness.connect()` starts the VM, launches the session-1 shim, deploys +the app scripts, and quiets the desktop; the orchestrator records+compiles the +demo once, then loops arms×conditions, judging every run against the DB. + +## Target app: OpenDental attempt and the honest wall + +The intended target was **OpenDental** (real WinForms dental EMR, bundled +MariaDB demo DB). Its trial (`TrialDownload-25-3-48.exe`, 149 MB) downloads +fine and needs no license key, but installing it **no-touch** is blocked: + +- No installer-tech signature (custom bootstrapper) and **no documented silent + flags** — it is an interactive multi-step Setup Wizard. +- It is gated by **SmartScreen** ("Windows protected your PC") and requires + **UAC elevation**; the UAC consent prompt renders on the **secure desktop**, + which session-1 `pyautogui` cannot interact with. Disabling UAC/SmartScreen + to drive the wizard by vision is invasive and still leaves an unknown + multi-dialog MariaDB sub-install. + +Per the spike spec's fallback clause, we substituted a **real WinForms app with +the same list-select → edit → save shape and exact SQL ground truth** +(`patient_notes.ps1` + SQLite). App choice is secondary here: the deliverable +is the **automated desktop pipeline** and what it measures. The substitution is +labelled honestly in every output. A native-x86 OpenDental confirmation run +(driving the wizard once on a cloud spot VM, or with UAC pre-disabled) is +future work — see `LIMITS.md`. diff --git a/openadapt_flow/adapters/__init__.py b/openadapt_flow/adapters/__init__.py new file mode 100644 index 0000000..209ce05 --- /dev/null +++ b/openadapt_flow/adapters/__init__.py @@ -0,0 +1,23 @@ +"""Adapters converting external demonstration sources into openadapt-flow +recordings (``meta.json`` + ``events.jsonl`` + ``frames/``). + +`convert_capture` is re-exported lazily so importing this package does not +pull in cv2 until the adapter is actually used. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from openadapt_flow.adapters.capture import convert_capture + +__all__ = ["convert_capture"] + + +def __getattr__(name: str) -> object: + if name == "convert_capture": + from openadapt_flow.adapters.capture import convert_capture + + return convert_capture + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/openadapt_flow/adapters/capture.py b/openadapt_flow/adapters/capture.py new file mode 100644 index 0000000..13151ee --- /dev/null +++ b/openadapt_flow/adapters/capture.py @@ -0,0 +1,419 @@ +"""openadapt-capture -> openadapt-flow recording adapter. + +This module is the *recording adapter contract* for desktop demonstrations +(docs/desktop/PHASE1.md): it converts an openadapt-capture session into the +recording format the compiler consumes (``meta.json`` + ``events.jsonl`` + +``frames/{i:04d}_before.png`` / ``_after.png``). + +Input contract (an openadapt-capture session directory): + + / + capture.db # SQLite: + # capture(id, started_at, ended_at, platform, + # screen_width, screen_height, pixel_ratio, + # video_start_time, task_description, ...) + # events(timestamp REAL, type TEXT, data JSON, ...) + video.mp4 # screen video; a frame's wall-clock time is + # capture.video_start_time + frame_pts_seconds + +The adapter consumes *derived* action events (openadapt-capture's +post-processing merges raw down/up/move streams into these): + + mouse.singleclick {x, y, button} -> {"kind": "click"} + mouse.doubleclick {x, y, button} -> {"kind": "double_click"} + key.type {text} -> {"kind": "type"} + key.down {key_name|key_char} -> {"kind": "key"} (named, + non-modifier keys only) + mouse.scroll {x, y, dx, dy} -> {"kind": "scroll"} + +Raw-only sessions (no derived events) are rejected: run openadapt-capture's +``process_events`` first. Deriving actions from raw streams is the capture +library's job, not this adapter's. + +Coordinate spaces: capture mouse coordinates are in *logical points* +(pynput); video frames are *physical pixels* (on Retina/HiDPI these differ +by ``pixel_ratio``). openadapt-flow requires event coordinates in the same +pixel space as the frames, so points are scaled by +``video_frame_width / capture.screen_width`` (empirical, not the stored +pixel_ratio — the video is authoritative). + +Frame selection: for an event at wall-clock time ``T``, the *before* frame +is the last video frame at or before ``T`` and the *after* frame is the +frame at ``T + settle_s``, clamped to just before the next event (an +approximation of the live Recorder's perceptual-hash settle wait; see +docs/desktop/PHASE1.md for the tradeoff). + +Scroll deltas: pynput reports wheel *notches* with positive ``dy`` = scroll +up, while the flow recording stores *pixels* with positive ``dy`` = view +down (Playwright wheel convention). Notches are converted at +``SCROLL_PIXELS_PER_NOTCH`` px/notch and the vertical sign is flipped. +""" + +from __future__ import annotations + +import json +import sqlite3 +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +# Wheel-notch -> pixel conversion (matches the WindowsBackend constant; the +# exact ratio is not load-bearing — replay's closed-loop scroll re-resolves +# after each gesture). +SCROLL_PIXELS_PER_NOTCH = 100 + +# Capture event types this adapter consumes. +_ACTION_TYPES = { + "mouse.singleclick", + "mouse.doubleclick", + "key.type", + "key.down", + "mouse.scroll", +} +# Derived action types with no flow equivalent: converting a demonstration +# that contains them would silently drop a user action (a wrong-action +# seed), so they fail the conversion loudly instead. +_REJECTED_ACTION_TYPES = { + "mouse.drag", + "key.shortcut", +} +# Raw / non-action streams, safely skipped (their information is already +# merged into the derived events or is irrelevant to replay). +_IGNORED_TYPES = { + "mouse.move", + "mouse.down", + "mouse.up", + "key.up", + "screen.frame", + "audio.chunk", +} + +# pynput key names (openadapt-capture ``key_name``) -> flow/Playwright names. +_KEY_NAME_MAP = { + "enter": "Enter", + "tab": "Tab", + "esc": "Escape", + "escape": "Escape", + "backspace": "Backspace", + "delete": "Delete", + "space": "Space", + "home": "Home", + "end": "End", + "page_up": "PageUp", + "page_down": "PageDown", + "up": "ArrowUp", + "down": "ArrowDown", + "left": "ArrowLeft", + "right": "ArrowRight", +} + +# Bare modifier presses carry no workflow meaning (their effect is only +# visible combined with another key, which capture merges elsewhere). +_MODIFIER_KEY_NAMES = { + "shift", "shift_l", "shift_r", + "ctrl", "ctrl_l", "ctrl_r", + "alt", "alt_l", "alt_r", "alt_gr", + "cmd", "cmd_l", "cmd_r", + "caps_lock", +} + + +@dataclass +class _Session: + """Metadata row from capture.db's ``capture`` table.""" + + started_at: float + screen_width: int + screen_height: int + pixel_ratio: float + video_start_time: Optional[float] + task_description: Optional[str] + + +class _Video: + """Random-access frame reader for the capture's screen video.""" + + def __init__(self, path: Path) -> None: + import cv2 + + self._cv2 = cv2 + self._cap = cv2.VideoCapture(str(path)) + if not self._cap.isOpened(): + raise ValueError(f"cannot open capture video: {path}") + self.fps = float(self._cap.get(cv2.CAP_PROP_FPS)) or 30.0 + self.frame_count = int(self._cap.get(cv2.CAP_PROP_FRAME_COUNT)) + self.width = int(self._cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + self.height = int(self._cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + self.duration_s = ( + self.frame_count / self.fps if self.frame_count else 0.0 + ) + + def frame_png_at(self, t_video: float) -> bytes: + """Return the frame nearest to video-time ``t_video`` as PNG bytes.""" + t = min(max(t_video, 0.0), max(self.duration_s - 1.0 / self.fps, 0.0)) + index = min( + int(round(t * self.fps)), + max(self.frame_count - 1, 0), + ) + self._cap.set(self._cv2.CAP_PROP_POS_FRAMES, index) + ok, frame = self._cap.read() + if not ok: + raise ValueError( + f"cannot read video frame at t={t_video:.3f}s (index {index})" + ) + ok, buf = self._cv2.imencode(".png", frame) + if not ok: # pragma: no cover - imencode failure is environmental + raise ValueError("cannot encode video frame as PNG") + return buf.tobytes() + + def close(self) -> None: + self._cap.release() + + +def _load_session(db_path: Path) -> _Session: + """Read the ``capture`` metadata row.""" + with sqlite3.connect(db_path) as conn: + conn.row_factory = sqlite3.Row + row = conn.execute("SELECT * FROM capture LIMIT 1").fetchone() + if row is None: + raise ValueError(f"no capture row in {db_path}") + keys = row.keys() + return _Session( + started_at=float(row["started_at"]), + screen_width=int(row["screen_width"]), + screen_height=int(row["screen_height"]), + pixel_ratio=float(row["pixel_ratio"]) if "pixel_ratio" in keys and row["pixel_ratio"] is not None else 1.0, + video_start_time=( + float(row["video_start_time"]) + if "video_start_time" in keys and row["video_start_time"] is not None + else None + ), + task_description=( + row["task_description"] if "task_description" in keys else None + ), + ) + + +def _load_action_events(db_path: Path) -> list[dict[str, Any]]: + """Read derived action events (timestamp order) from capture.db.""" + with sqlite3.connect(db_path) as conn: + rows = conn.execute( + "SELECT timestamp, type, data FROM events ORDER BY timestamp" + ).fetchall() + seen_types = {r[1] for r in rows} + rejected = sorted(seen_types & _REJECTED_ACTION_TYPES) + if rejected: + raise ValueError( + f"capture contains derived action types {rejected} that have no " + "openadapt-flow equivalent; converting would silently drop user " + "actions" + ) + # Unknown *input-stream* types are load-bearing (dropping one drops a + # user action); unknown auxiliary streams (window/browser/audio + # metadata) are not. + unknown_input = sorted( + t + for t in seen_types + if t.startswith(("mouse.", "key.")) + and t not in _ACTION_TYPES | _REJECTED_ACTION_TYPES | _IGNORED_TYPES + ) + if unknown_input: + raise ValueError( + f"capture contains unknown input event types {unknown_input}; " + "extend the adapter before converting" + ) + actions = [ + {"timestamp": float(r[0]), "type": r[1], **json.loads(r[2])} + for r in rows + if r[1] in _ACTION_TYPES + ] + if not actions: + raise ValueError( + "capture.db contains no derived action events " + f"(found types: {sorted(seen_types)}); run openadapt-capture's " + "process_events() to merge raw mouse/keyboard streams first" + ) + return actions + + +def _key_event_name(event: dict[str, Any]) -> Optional[str]: + """Flow key name for a ``key.down`` event, or None to skip it.""" + key_name = event.get("key_name") or event.get("canonical_key_name") + if key_name: + if key_name in _MODIFIER_KEY_NAMES: + return None + mapped = _KEY_NAME_MAP.get(key_name) + if mapped is None: + raise ValueError( + f"unmapped key.down key_name {key_name!r} at " + f"t={event['timestamp']:.3f}; extend _KEY_NAME_MAP" + ) + return mapped + char = event.get("key_char") or event.get("canonical_key_char") + return char or None + + +def convert_capture( + capture_dir: Path | str, + out_recording_dir: Path | str, + *, + params: Optional[dict[str, str]] = None, + settle_s: float = 1.0, +) -> Path: + """Convert an openadapt-capture session into a flow recording directory. + + Args: + capture_dir: Directory containing ``capture.db`` and ``video.mp4``. + out_recording_dir: Output recording directory (created if missing). + params: Optional ``{param_name: demonstrated_value}`` map. A ``type`` + event whose text equals a demonstrated value is marked as that + parameter (the compiler then treats it as per-run input and + lints against value leakage). + settle_s: Seconds after each action at which the *after* frame is + sampled (clamped to just before the next event). + + Returns: + The recording directory path (compile-ready). + + Raises: + FileNotFoundError: If ``capture.db`` or ``video.mp4`` is missing. + ValueError: On raw-only sessions, unmapped keys, non-left clicks, + multiple params demonstrating the same value, or an unreadable + video. + """ + capture_dir = Path(capture_dir) + out_dir = Path(out_recording_dir) + db_path = capture_dir / "capture.db" + video_path = capture_dir / "video.mp4" + for path in (db_path, video_path): + if not path.exists(): + raise FileNotFoundError(path) + + params = dict(params or {}) + value_to_param: dict[str, str] = {} + for name, value in params.items(): + if value in value_to_param: + raise ValueError( + f"params {value_to_param[value]!r} and {name!r} demonstrate " + "the same value; parameter marking would be ambiguous" + ) + value_to_param[value] = name + + session = _load_session(db_path) + events = _load_action_events(db_path) + video = _Video(video_path) + try: + # Capture coords are logical points; frames are physical pixels. + # The video is authoritative for the frame pixel space. + scale_x = video.width / session.screen_width + scale_y = video.height / session.screen_height + video_t0 = ( + session.video_start_time + if session.video_start_time is not None + else session.started_at + ) + + frames_dir = out_dir / "frames" + frames_dir.mkdir(parents=True, exist_ok=True) + events_path = out_dir / "events.jsonl" + + used_params: dict[str, str] = {} + lines: list[str] = [] + i = 0 + for j, event in enumerate(events): + line = _convert_event(event, scale_x, scale_y, value_to_param) + if line is None: # skippable (e.g. bare modifier press) + continue + if "param" in line: + used_params[line["param"]] = line["text"] + + t_event = event["timestamp"] - video_t0 + t_after = t_event + settle_s + if j + 1 < len(events): + next_t = events[j + 1]["timestamp"] - video_t0 + t_after = min(t_after, max(next_t - 1.0 / video.fps, t_event)) + (frames_dir / f"{i:04d}_before.png").write_bytes( + video.frame_png_at(t_event - 1.0 / video.fps) + ) + (frames_dir / f"{i:04d}_after.png").write_bytes( + video.frame_png_at(t_after) + ) + + line["i"] = i + line["t"] = round(event["timestamp"] - session.started_at, 3) + # Match the Recorder's key order ({"i", ...event, "t"}). + ordered = { + "i": line.pop("i"), + **{k: v for k, v in line.items() if k != "t"}, + "t": line["t"], + } + lines.append(json.dumps(ordered)) + i += 1 + + if i == 0: + raise ValueError( + "no convertible action events (all were skippable)" + ) + events_path.write_text("\n".join(lines) + "\n") + + meta = { + "id": uuid.uuid4().hex, + "created_at": datetime.now(timezone.utc).isoformat(), + "viewport": [video.width, video.height], + "app_url": None, + "params": used_params, + "source": "openadapt-capture", + "task_description": session.task_description, + } + (out_dir / "meta.json").write_text(json.dumps(meta, indent=2)) + return out_dir + finally: + video.close() + + +def _convert_event( + event: dict[str, Any], + scale_x: float, + scale_y: float, + value_to_param: dict[str, str], +) -> Optional[dict[str, Any]]: + """Convert one capture action event to a flow event line (sans i/t). + + Returns None for events that are valid but carry no workflow meaning + (bare modifier presses). + """ + etype = event["type"] + if etype in ("mouse.singleclick", "mouse.doubleclick"): + button = event.get("button", "left") + if button != "left": + raise ValueError( + f"{etype} with button={button!r} has no flow equivalent " + f"(t={event['timestamp']:.3f})" + ) + kind = "click" if etype == "mouse.singleclick" else "double_click" + return { + "kind": kind, + "x": int(round(event["x"] * scale_x)), + "y": int(round(event["y"] * scale_y)), + } + if etype == "key.type": + text = event["text"] + line: dict[str, Any] = {"kind": "type", "text": text} + if text in value_to_param: + line["param"] = value_to_param[text] + return line + if etype == "key.down": + key = _key_event_name(event) + if key is None: + return None + return {"kind": "key", "key": key} + if etype == "mouse.scroll": + # pynput: notches, +dy = scroll up. Flow: pixels, +dy = view down. + return { + "kind": "scroll", + "dx": int(round(event.get("dx", 0) * SCROLL_PIXELS_PER_NOTCH)), + "dy": int(round(-event.get("dy", 0) * SCROLL_PIXELS_PER_NOTCH)), + } + raise ValueError(f"unhandled capture event type: {etype!r}") diff --git a/openadapt_flow/backends/__init__.py b/openadapt_flow/backends/__init__.py index 1bdca7f..69f8e4e 100644 --- a/openadapt_flow/backends/__init__.py +++ b/openadapt_flow/backends/__init__.py @@ -10,8 +10,9 @@ if TYPE_CHECKING: # pragma: no cover from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.backends.windows_backend import WindowsBackend -__all__ = ["PlaywrightBackend"] +__all__ = ["PlaywrightBackend", "WindowsBackend"] def __getattr__(name: str) -> object: @@ -21,4 +22,8 @@ def __getattr__(name: str) -> object: ) return PlaywrightBackend + if name == "WindowsBackend": + from openadapt_flow.backends.windows_backend import WindowsBackend + + return WindowsBackend raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/openadapt_flow/backends/parallels_vm.py b/openadapt_flow/backends/parallels_vm.py new file mode 100644 index 0000000..6db05ac --- /dev/null +++ b/openadapt_flow/backends/parallels_vm.py @@ -0,0 +1,357 @@ +"""Programmatic control of a Parallels Desktop VM (Apple-Silicon host). + +The Phase-2 desktop benchmark runs against a local Parallels Windows 11 VM +(native Apple-Silicon virtualization — the WAA/QEMU nested-virt stack cannot +run on M-series, so Parallels replaces it). This module is the fully +programmatic control plane the pipeline drives; there are no manual GUI steps. + +Everything is built on ``prlctl``: + +* lifecycle: ``status`` / ``start`` / ``stop`` / ``suspend`` / ``resume`` +* reversibility: ``snapshot`` / ``list_snapshots`` / ``revert`` (never delete + the user's snapshots — reverts only) +* in-guest execution: ``exec`` runs as ``NT AUTHORITY\\SYSTEM`` in session 0 +* host-side screen capture: ``capture`` (``prlctl capture`` — bypasses the + session-0 desktop-isolation that blocks an in-guest ``BitBlt``) +* file transfer: ``push_file`` (``prlctl exec`` hangs on long arguments, so + files move over a short-lived host HTTP server + in-guest ``curl``) +* the WAA HTTP shim: ``launch_shim`` places ``waa_shim.py`` in the interactive + console session (session 1) via ``session1_launch.py`` + + ``CreateProcessAsUser`` — session 0 cannot screenshot or drive the desktop. + +Importing this module has no side effects and needs no live VM, so the CI +mock tests import it freely; every real operation shells out lazily. +""" + +from __future__ import annotations + +import functools +import http.server +import os +import re +import socketserver +import subprocess +import threading +import time +from dataclasses import dataclass +from typing import Optional + +# The user's existing VM (see docs/desktop/PHASE2.md). Overridable per call. +DEFAULT_VM_UUID = "{d4f9c29a-52e1-4793-9334-7e971c3d0ab3}" +DEFAULT_PRLCTL = "/usr/local/bin/prlctl" +SHIM_PORT = 5000 +_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", + "desktop") +# In-guest install locations (forward slashes: Python + curl accept them and +# they survive the host shell without backslash mangling). +GUEST_DIR = "C:/oa" +GUEST_PY = r"C:\Program Files\Python312-arm64\python.exe" + + +@dataclass +class SnapshotInfo: + """One entry from ``prlctl snapshot-list``.""" + + snapshot_id: str + current: bool + name: str = "" + + +class ParallelsError(RuntimeError): + """A ``prlctl`` invocation failed.""" + + +class ParallelsVM: + """Thin, fully-programmatic wrapper over ``prlctl`` for one VM. + + Args: + uuid: VM UUID (braces included) or name. + prlctl: Path to the ``prlctl`` binary. + python_guest: In-guest Python interpreter path. + """ + + def __init__( + self, + uuid: str = DEFAULT_VM_UUID, + *, + prlctl: str = DEFAULT_PRLCTL, + python_guest: str = GUEST_PY, + ) -> None: + self.uuid = uuid + self.prlctl = prlctl + self.python_guest = python_guest + + # -- low-level ----------------------------------------------------------- + + def _run( + self, args: list[str], *, timeout: float = 120.0, check: bool = True + ) -> subprocess.CompletedProcess: + """Invoke ``prlctl `` and return the completed process. + + Raises: + ParallelsError: If ``check`` and the return code is nonzero. + """ + proc = subprocess.run( + [self.prlctl, *args], + capture_output=True, + text=True, + timeout=timeout, + ) + if check and proc.returncode != 0: + raise ParallelsError( + f"prlctl {' '.join(args)} -> rc={proc.returncode}: " + f"{proc.stderr.strip() or proc.stdout.strip()}" + ) + return proc + + # -- lifecycle ----------------------------------------------------------- + + def status(self) -> str: + """Return the VM power state (running/paused/suspended/stopped).""" + proc = self._run(["list", "--all", "-o", "status,uuid"], check=False) + for line in proc.stdout.splitlines(): + if self.uuid in line: + return line.split()[0].strip() + # Fall back to name match. + proc = self._run(["list", "--all"], check=False) + for line in proc.stdout.splitlines(): + if self.uuid in line: + return line.split()[1].strip() + return "unknown" + + def start(self) -> None: + self._run(["start", self.uuid]) + + def stop(self, *, force: bool = False) -> None: + args = ["stop", self.uuid] + if force: + args.append("--kill") + self._run(args) + + def suspend(self) -> None: + """Suspend the VM (saves state; leaves nothing running).""" + self._run(["suspend", self.uuid]) + + def resume(self) -> None: + self._run(["resume", self.uuid], check=False) + + def set_pause_idle(self, on: bool) -> None: + """Toggle Parallels' pause-when-idle (must be OFF for headless runs).""" + self._run(["set", self.uuid, "--pause-idle", "on" if on else "off"], + check=False) + + def ensure_running(self, *, settle_s: float = 6.0) -> None: + """Bring the VM to a running state from any state, idempotently. + + Handles Parallels' auto-pause (``paused`` -> ``resume``) and disables + pause-idle so a headless benchmark run is not silently frozen. + """ + state = self.status() + if state in ("suspended", "paused"): + self.resume() + elif state in ("stopped", "unknown"): + self.start() + self.set_pause_idle(False) + # A resume can be immediately re-paused; force it running. + for _ in range(3): + if self.status() == "running": + break + self.resume() + time.sleep(1) + time.sleep(settle_s) + + # -- snapshots (reversibility) ------------------------------------------ + + def snapshot(self, name: str, description: str = "") -> str: + """Create a snapshot and return its id. + + Snapshots are the drift-reset and warm-boot mechanism: every run + reverts to a clean ``opendental-ready`` state in seconds. + """ + args = ["snapshot", self.uuid, "-n", name] + if description: + args += ["-d", description] + proc = self._run(args) + m = re.search(r"\{[0-9a-fA-F-]+\}", proc.stdout) + if not m: + raise ParallelsError(f"could not parse snapshot id: {proc.stdout}") + return m.group(0) + + def list_snapshots(self) -> list[SnapshotInfo]: + """Return all snapshots (``*`` marks the current one).""" + proc = self._run(["snapshot-list", self.uuid], check=False) + out: list[SnapshotInfo] = [] + for line in proc.stdout.splitlines(): + # Columns are PARENT_SNAPSHOT_ID then SNAPSHOT_ID; the current + # snapshot is marked ``*`` before its SNAPSHOT_ID (the LAST id on + # the line). Take the last match so the parent id is not mistaken + # for the snapshot id. + matches = re.findall(r"(\*?)(\{[0-9a-fA-F-]+\})", line) + if matches: + star, sid = matches[-1] + out.append(SnapshotInfo(snapshot_id=sid, current=bool(star))) + return out + + def revert(self, snapshot_id: str) -> None: + """Revert to a snapshot (the per-run clean-state reset).""" + self._run(["snapshot-switch", self.uuid, "-i", snapshot_id]) + + # -- in-guest execution -------------------------------------------------- + + def exec( + self, args: list[str], *, timeout: float = 120.0, check: bool = False + ) -> subprocess.CompletedProcess: + """Run a program in-guest as SYSTEM (``prlctl exec``). + + NOTE: ``prlctl exec`` hangs on very long single arguments — keep + commands short and move file payloads with :meth:`push_file`. + """ + return self._run(["exec", self.uuid, *args], timeout=timeout, + check=check) + + def exec_cmd(self, cmdline: str, *, timeout: float = 120.0 + ) -> subprocess.CompletedProcess: + """Run ``cmd /c `` in-guest (quoting preserved by cmd).""" + return self.exec(["cmd", "/c", cmdline], timeout=timeout) + + def exec_ps(self, script: str, *, timeout: float = 120.0 + ) -> subprocess.CompletedProcess: + """Run a short PowerShell command in-guest.""" + return self.exec(["powershell", "-NoProfile", "-Command", script], + timeout=timeout) + + # -- host-side capture --------------------------------------------------- + + def capture(self, local_path: str) -> str: + """Capture the VM screen host-side to ``local_path`` (PNG). + + Independent of in-guest state — the ground-truth screenshot even when + the shim is down. + """ + self._run(["capture", self.uuid, "--file", local_path]) + return local_path + + # -- networking ---------------------------------------------------------- + + def guest_ip(self) -> str: + """Return the guest's shared-network IPv4 (skips APIPA 169.254).""" + proc = self.exec_cmd("ipconfig") + addrs = re.findall(r"IPv4[^\n:]*:\s*([0-9.]+)", proc.stdout) + for a in addrs: + if not a.startswith("169.254"): + return a + if addrs: + return addrs[0] + raise ParallelsError("no guest IPv4 found") + + def host_ip(self, guest_ip: Optional[str] = None) -> str: + """Return the Mac's IP on the guest's subnet (for host->guest HTTP).""" + guest_ip = guest_ip or self.guest_ip() + prefix = guest_ip.rsplit(".", 1)[0] + "." + out = subprocess.run(["ifconfig"], capture_output=True, text=True).stdout + for line in out.splitlines(): + line = line.strip() + if line.startswith("inet "): + addr = line.split()[1] + if addr.startswith(prefix): + return addr + return prefix + "2" # Parallels host is conventionally .2 + + # -- file transfer ------------------------------------------------------- + + def push_file( + self, + local_path: str, + guest_path: str, + *, + host_ip: Optional[str] = None, + port: int = 0, + ) -> None: + """Copy a host file into the guest over a short-lived HTTP server. + + ``prlctl exec`` hangs on long arguments, ruling out base64-in-argv; + an in-guest ``curl`` from a throwaway host server is robust and fast. + Port 0 (default) binds an ephemeral port so concurrent pushes never + collide. + """ + host_ip = host_ip or self.host_ip() + directory = os.path.dirname(os.path.abspath(local_path)) + name = os.path.basename(local_path) + handler = functools.partial( + http.server.SimpleHTTPRequestHandler, directory=directory + ) + httpd = socketserver.TCPServer((host_ip, port), handler) + bound_port = httpd.server_address[1] + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + url = f"http://{host_ip}:{bound_port}/{name}" + proc = self.exec_cmd(f"curl -s -o {guest_path} {url}") + if proc.returncode != 0: + raise ParallelsError(f"push_file curl failed: {proc.stderr}") + finally: + httpd.shutdown() + httpd.server_close() + + # -- the WAA shim (session 1) ------------------------------------------- + + def _shim_paths(self) -> tuple[str, str]: + shim = os.path.abspath(os.path.join(_SCRIPT_DIR, "waa_shim.py")) + launcher = os.path.abspath(os.path.join(_SCRIPT_DIR, + "session1_launch.py")) + return shim, launcher + + def kill_shim(self) -> None: + """Kill any in-guest Python (frees the shim port).""" + self.exec_cmd("taskkill /F /IM python.exe /IM pythonw.exe 2>nul & " + "echo done") + + def launch_shim( + self, + *, + port: int = SHIM_PORT, + host_ip: Optional[str] = None, + wait_s: float = 20.0, + ) -> str: + """Deploy + start the WAA shim in session 1; return its URL. + + Steps (all programmatic): ensure guest dir, push shim + launcher, + open the firewall, kill stale Python, run the session-1 launcher as + SYSTEM, then poll until the shim answers a real PNG. + """ + host_ip = host_ip or self.host_ip() + shim, launcher = self._shim_paths() + self.exec_cmd(f"if not exist {GUEST_DIR} mkdir {GUEST_DIR}") + self.push_file(shim, f"{GUEST_DIR}/waa_shim.py", host_ip=host_ip) + self.push_file(launcher, f"{GUEST_DIR}/session1_launch.py", + host_ip=host_ip) + self.exec(["netsh", "advfirewall", "firewall", "add", "rule", + "name=OAShim", "dir=in", "action=allow", "protocol=TCP", + f"localport={port}"]) + self.kill_shim() + time.sleep(2) + # Run the launcher as SYSTEM; it CreateProcessAsUser's the shim into + # the interactive console session so mss/pyautogui address the real + # desktop. Forward-slash script paths dodge host-shell mangling. + self.exec([self.python_guest, f"{GUEST_DIR}/session1_launch.py", + f"{GUEST_DIR}/waa_shim.py", "--port", str(port)]) + url = f"http://{self.guest_ip()}:{port}" + deadline = time.time() + wait_s + while time.time() < deadline: + if self._shim_alive(url): + return url + time.sleep(1.5) + raise ParallelsError(f"shim did not come up at {url}") + + def _shim_alive(self, url: str) -> bool: + try: + import requests + + r = requests.get(f"{url}/screenshot", timeout=8) + return r.status_code == 200 and r.content[:8] == b"\x89PNG\r\n\x1a\n" + except Exception: # noqa: BLE001 + return False + + def shim_url(self, *, port: int = SHIM_PORT) -> str: + """URL WindowsBackend should target (``http://:``).""" + return f"http://{self.guest_ip()}:{port}" diff --git a/openadapt_flow/backends/windows_backend.py b/openadapt_flow/backends/windows_backend.py new file mode 100644 index 0000000..e0f91f0 --- /dev/null +++ b/openadapt_flow/backends/windows_backend.py @@ -0,0 +1,308 @@ +"""Windows backend over the WAA (Windows Agent Arena) HTTP API. + +Implements the `openadapt_flow.backend.Backend` protocol against the WAA +Flask server that runs inside the Windows VM (the WAADirect pattern from +openadapt-evals — plain HTTP, no adapter layer): + + GET /screenshot -> raw PNG bytes (Flask send_file, NOT base64 JSON) + POST /execute_windows -> server does exec(command, ...) with pyautogui + importable. The payload is + ``{"command": ""}`` — + NOT wrapped in ``python -c "..."``. + +The backend is vision-only by construction: PNG frames in, pixel-coordinate +input out. It deliberately does NOT implement the optional +`StructuralBackend` observations (url/title/page count) — native Windows has +no cheap equivalent, so those steps stay honestly unverified (docs/LIMITS.md). + +Typed text is embedded into the command via ``repr()`` (a valid Python +literal, immune to quoting bugs). Non-ASCII text cannot be typed by +pyautogui.write (it silently drops unknown characters — a wrong-write mode), +so it is routed through the Windows clipboard instead: the value travels +base64-encoded to PowerShell ``Set-Clipboard`` and is pasted with Ctrl+V. +""" + +from __future__ import annotations + +import base64 +import struct +from typing import Optional + +import requests + +DEFAULT_SERVER_URL = "http://localhost:5001" + +# Screenshot retry defaults (mirrors WAADirect in openadapt-evals). +SCREENSHOT_MAX_RETRIES = 3 +SCREENSHOT_RETRY_DELAY_S = 2.0 + +# The Backend protocol expresses scroll in pixels (a Playwright wheel +# gesture); pyautogui expresses it in wheel notches. One notch scrolls +# roughly this many pixels in common Windows apps. Approximate by design — +# to be tuned against the real VM (Phase 2); replay's closed-loop scroll +# re-resolves after each gesture, so the exact ratio is not load-bearing. +SCROLL_PIXELS_PER_NOTCH = 100 + +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + +# Playwright-style modifier names (as recorded / emitted by the replayer, +# e.g. 'ControlOrMeta+a') -> pyautogui key names. On Windows the Meta/Command +# key maps to the Windows key and ControlOrMeta resolves to Ctrl. +_MODIFIER_MAP = { + "ctrl": "ctrl", + "control": "ctrl", + "controlormeta": "ctrl", + "meta": "win", + "cmd": "win", + "command": "win", + "win": "win", + "alt": "alt", + "option": "alt", + "shift": "shift", +} + +# Playwright-style named keys -> pyautogui key names. +_NAMED_KEY_MAP = { + "enter": "enter", + "return": "enter", + "tab": "tab", + "escape": "esc", + "esc": "esc", + "backspace": "backspace", + "delete": "delete", + "space": "space", + "home": "home", + "end": "end", + "pageup": "pageup", + "pagedown": "pagedown", + "arrowup": "up", + "arrowdown": "down", + "arrowleft": "left", + "arrowright": "right", +} + + +def _png_size(png: bytes) -> tuple[int, int]: + """Return (width, height) parsed from a PNG's IHDR chunk. + + Args: + png: PNG file bytes. + + Returns: + (width, height) in pixels. + + Raises: + ValueError: If the bytes are not a PNG. + """ + if len(png) < 24 or not png.startswith(_PNG_SIGNATURE): + raise ValueError("not a PNG frame") + width, height = struct.unpack(">II", png[16:24]) + return int(width), int(height) + + +def _pyautogui_key(part: str) -> str: + """Map one Playwright-style key/modifier name to pyautogui's name.""" + lower = part.lower() + if lower in _MODIFIER_MAP: + return _MODIFIER_MAP[lower] + if lower in _NAMED_KEY_MAP: + return _NAMED_KEY_MAP[lower] + # Single characters and F-keys: pyautogui uses lowercase names. + return lower + + +def normalize_chord(key: str) -> list[str]: + """Normalize a key or ``+``-joined chord to pyautogui key names. + + Args: + key: e.g. ``'Enter'``, ``'ControlOrMeta+a'``, ``'Meta+d'``. + + Returns: + List of pyautogui key names, modifiers first (input order kept). + + Raises: + ValueError: If the chord is empty. + """ + parts = [p for p in key.split("+") if p] + if not parts: + raise ValueError(f"empty key chord: {key!r}") + return [_pyautogui_key(p) for p in parts] + + +class WindowsBackend: + """`Backend` implementation over the WAA HTTP API. + + Args: + server_url: WAA Flask server base URL (default matches the standard + SSH tunnel ``localhost:5001`` -> VM ``:5000``). + viewport: Optional (width, height) override. When omitted it is + derived once from the first screenshot's PNG header and cached. + type_interval_s: Inter-key delay for ``pyautogui.write``. + timeout_s: HTTP timeout per request. + screenshot_max_retries: Screenshot retry attempts (server may be + momentarily unready). + screenshot_retry_delay_s: Delay between screenshot retries. + session: Optional ``requests.Session`` (injected in tests). + """ + + def __init__( + self, + server_url: str = DEFAULT_SERVER_URL, + *, + viewport: Optional[tuple[int, int]] = None, + type_interval_s: float = 0.05, + timeout_s: float = 30.0, + screenshot_max_retries: int = SCREENSHOT_MAX_RETRIES, + screenshot_retry_delay_s: float = SCREENSHOT_RETRY_DELAY_S, + session: Optional[requests.Session] = None, + ) -> None: + self.server_url = server_url.rstrip("/") + self._viewport = viewport + self._type_interval_s = type_interval_s + self._timeout_s = timeout_s + self._screenshot_max_retries = max(1, int(screenshot_max_retries)) + self._screenshot_retry_delay_s = screenshot_retry_delay_s + self._session = session if session is not None else requests.Session() + + # -- Backend protocol ---------------------------------------------------- + + @property + def viewport(self) -> tuple[int, int]: + """(width, height) of the VM screen, derived from a screenshot.""" + if self._viewport is None: + self._viewport = _png_size(self.screenshot()) + return self._viewport + + def screenshot(self) -> bytes: + """Return the current frame as PNG bytes (with retries). + + WAA's ``/screenshot`` returns raw PNG via Flask ``send_file()`` — + read ``resp.content``, never ``resp.json()``. + + Raises: + RuntimeError: If all attempts fail or return a non-PNG payload. + """ + import time + + last_error: Optional[Exception] = None + for attempt in range(1, self._screenshot_max_retries + 1): + try: + resp = self._session.get( + f"{self.server_url}/screenshot", timeout=self._timeout_s + ) + if resp.status_code != 200: + raise RuntimeError( + f"screenshot HTTP {resp.status_code}: {resp.text[:200]}" + ) + png = resp.content + _png_size(png) # validates signature and header + return png + except Exception as e: # noqa: BLE001 - retried, then re-raised + last_error = e + if attempt < self._screenshot_max_retries: + time.sleep(self._screenshot_retry_delay_s) + raise RuntimeError( + f"screenshot failed after {self._screenshot_max_retries} attempts" + ) from last_error + + def click(self, x: int, y: int, *, double: bool = False) -> None: + """Click (or double-click) at pixel coordinates via pyautogui.""" + fn = "doubleClick" if double else "click" + self._execute(f"import pyautogui; pyautogui.{fn}({int(x)}, {int(y)})") + + def type_text(self, text: str) -> None: + """Type text into the currently focused element. + + ASCII text goes through ``pyautogui.write``. Text containing + characters pyautogui cannot type (it silently drops them — a + wrong-write failure mode) goes through the Windows clipboard: + base64 -> PowerShell ``Set-Clipboard`` -> Ctrl+V. + """ + if not text: + return + if text.isascii(): + self._execute( + "import pyautogui; " + f"pyautogui.write({text!r}, " + f"interval={self._type_interval_s})" + ) + return + b64 = base64.b64encode(text.encode("utf-8")).decode("ascii") + ps = ( + "Set-Clipboard -Value ([Text.Encoding]::UTF8.GetString(" + f"[Convert]::FromBase64String('{b64}')))" + ) + self._execute( + "\n".join( + [ + "import subprocess; import time; import pyautogui", + "subprocess.run(['powershell', '-NoProfile', '-Command', " + f"{ps!r}], capture_output=True)", + "time.sleep(0.2)", + "pyautogui.hotkey('ctrl', 'v')", + ] + ) + ) + + def press(self, key: str) -> None: + """Press a key or chord, e.g. ``'Enter'`` or ``'ControlOrMeta+a'``.""" + keys = normalize_chord(key) + if len(keys) == 1: + self._execute( + f"import pyautogui; pyautogui.press({keys[0]!r})" + ) + else: + args = ", ".join(repr(k) for k in keys) + self._execute(f"import pyautogui; pyautogui.hotkey({args})") + + def scroll(self, dx: int, dy: int) -> None: + """Dispatch a wheel gesture (Backend convention: positive ``dy`` + scrolls content up / view down). + + pyautogui's convention is inverted for vertical scrolling (positive + = view up), so ``dy`` changes sign; horizontal signs agree. + """ + parts: list[str] = ["import pyautogui"] + v = self._notches(dy) + h = self._notches(dx) + if v: + parts.append(f"pyautogui.scroll({-v})") + if h: + parts.append(f"pyautogui.hscroll({h})") + if len(parts) == 1: + return + self._execute("; ".join(parts)) + + # -- internals ------------------------------------------------------------- + + @staticmethod + def _notches(pixels: int) -> int: + """Convert a pixel delta to wheel notches (>=1 notch when nonzero).""" + if pixels == 0: + return 0 + notches = round(abs(pixels) / SCROLL_PIXELS_PER_NOTCH) + return max(1, notches) * (1 if pixels > 0 else -1) + + def _execute(self, command: str) -> None: + """POST bare Python statements to WAA's ``/execute_windows``. + + Raises: + RuntimeError: On a non-200 response. + """ + resp = self._session.post( + f"{self.server_url}/execute_windows", + json={"command": command}, + timeout=self._timeout_s, + ) + if resp.status_code != 200: + raise RuntimeError( + f"execute_windows HTTP {resp.status_code}: {resp.text[:200]}" + ) + + def probe(self) -> bool: + """True if the WAA server answers with a valid PNG screenshot.""" + try: + self.screenshot() + return True + except RuntimeError: + return False diff --git a/openadapt_flow/benchmark/desktop_benchmark.py b/openadapt_flow/benchmark/desktop_benchmark.py new file mode 100644 index 0000000..6c2f5ef --- /dev/null +++ b/openadapt_flow/benchmark/desktop_benchmark.py @@ -0,0 +1,720 @@ +"""Phase-2 desktop benchmark: compiled vision replay vs the UIA incumbent. + +Mirrors the browser benchmarks (``run_benchmark`` / ``hybrid_benchmark``) but +targets a real Windows desktop app driven over the Phase-1 WAA HTTP contract +(``WindowsBackend``) against a local Parallels VM (``ParallelsVM``). Every +arm makes ZERO model calls; the whole run costs $0. + +Arms +---- +* ``compiled`` -- openadapt-flow record -> compile -> replay through + ``WindowsBackend`` (vision-only, pixel in / coords out, identity bands on + desktop-rendered text). The differentiated arm. +* ``uia_identity`` -- the desktop incumbent, steelmanned: pywinauto selectors + that pick the patient row by matching rendered cell *text* (strongest a + selector engine can do when the row has no AutomationId). +* ``uia_positional`` -- the naive recorded selector: "row 0". Same engine, the + positional reading -- included so the identity-vs-positional gap is measured. + +Judge +----- +Ground truth is the app's SQLite DB (``pn_db.py``), read directly and +arm-independently: success == the *right* patient got the *right* note and no +one else did. A note on any other patient is a **wrong action** (silent +mis-write) -- the exact failure the identity work exists to prevent. + +Drift +----- +Per-run revert to the ``harness-ready`` snapshot gives identical clean state in +seconds; the condition is then applied (DB reseed for data drift; a +``pn_env.json`` for render-scale/theme). See ``CONDITIONS``. + +Target app note: this is a WinForms *substitute* for OpenDental (whose trial is +a 149 MB interactive bootstrapper gated by SmartScreen + a UAC secure-desktop +prompt -- not no-touch installable; see docs/desktop/PHASE2.md). The app choice +is secondary: the deliverable is the automated desktop pipeline and what it +measures. It preserves every property that matters -- WinForms UIA tree, +list-select -> edit -> save, and exact SQL ground truth. +""" + +from __future__ import annotations + +import json +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, Optional + +# --- demonstration + conditions -------------------------------------------- + +# The recorded demo: search a given name that is UNIQUE on the clean list but +# gains a look-alike sibling under ``data_siblings`` drift (Neil Sorenson vs +# Neil Sorensen), then select the patient, note, save. Searching the shared +# given name (not the distinctive surname) is what makes the sibling condition +# actually exercise row selection — the identity test. ``target_id`` is the DB +# row the note must land on. +DEMO = { + "search": "Neil", + "first": "Neil", + "last": "Sorenson", + "target_id": 1, +} +DEMO_NOTE = "BP 128/82; follow-up in 2 weeks" + +# Per-run note text pool (parameter values); keeps each run distinguishable in +# the DB so a stale write from a prior run cannot masquerade as success. +NOTE_POOL = [ + "BP 132/85; recheck 2wk", + "Med reconciliation done; no changes", + "Referred to cardiology; ECG ordered", + "Fasting glucose 104; lifestyle advice", + "Flu vaccine administered today", + "Complains of intermittent headache", +] + +# condition -> (cfg for pn_env.json, drift seed, cosmetic?) +# ``cosmetic`` conditions do not change *which* patient is correct, so a +# safe-halt there is a FALSE ABORT; data-drift conditions genuinely change the +# list, so a halt there can be correct caution. +CONDITIONS: dict[str, dict] = { + "clean": {"cfg": {}, "drift": "none", "cosmetic": True}, + "render_125": {"cfg": {"font_scale": 1.25}, "drift": "none", + "cosmetic": True}, + "render_150": {"cfg": {"font_scale": 1.5}, "drift": "none", + "cosmetic": True}, + "theme_dark": {"cfg": {"theme": "dark"}, "drift": "none", + "cosmetic": True}, + "data_reorder": {"cfg": {}, "drift": "reorder", "cosmetic": False}, + "data_decoy": {"cfg": {}, "drift": "decoy", "cosmetic": False}, + "data_siblings": {"cfg": {}, "drift": "siblings", "cosmetic": False}, +} + +ARMS = ("compiled", "uia_identity", "uia_positional") + +GUEST_PY = r"C:\Program Files\Python312-arm64\python.exe" +GUEST_DIR = "C:/oa" + + +# --- result rows ------------------------------------------------------------ + +@dataclass +class RunRow: + """One arm x condition x repeat outcome, judged by the DB.""" + + arm: str + condition: str + i: int + outcome: str = "error" # success|wrong_action|safe_halt|miss|error + wrong_action: bool = False + false_abort: bool = False + completed: bool = False # arm ran to its end without halting + target_note_ok: bool = False + wrong_patient_id: Optional[int] = None + # compiled-arm identity telemetry + replay_success: Optional[bool] = None + halt_step: Optional[str] = None + halt_reason: Optional[str] = None + identity_verified: int = 0 + identity_mismatch: int = 0 + identity_unreadable: int = 0 + rungs: dict = field(default_factory=dict) + # uia-arm telemetry + selected_index: Optional[int] = None + selected_name: Optional[str] = None + wall_s: float = 0.0 + error: Optional[str] = None + + +# --- live harness (requires a VM; imported lazily by the orchestrator) ------ + +class DesktopHarness: + """Drives one Parallels VM through the full desktop pipeline. + + Construct via :meth:`connect` (starts the VM, ensures the shim). All heavy + imports are local so this module imports without a VM (CI / mocked tests). + """ + + def __init__(self, vm, shim_url: str, *, log: Callable = print) -> None: + self.vm = vm + self.shim_url = shim_url + self.log = log + from openadapt_flow.backends import WindowsBackend + + self.backend = WindowsBackend(server_url=shim_url) + + @classmethod + def connect( + cls, + *, + vm_uuid: Optional[str] = None, + log: Callable = print, + ) -> "DesktopHarness": + from openadapt_flow.backends.parallels_vm import ( + DEFAULT_VM_UUID, ParallelsVM, + ) + + vm = ParallelsVM(vm_uuid or DEFAULT_VM_UUID) + vm.ensure_running() + url = vm.launch_shim() + log(f"[harness] shim up at {url}") + h = cls(vm, url, log=log) + h.deploy_app_scripts() + h.quiet_desktop() + return h + + def quiet_desktop(self) -> None: + """Disable toast notifications (session-1 HKCU) so transient popups do + not pollute recorded postconditions/identity bands. Idempotent.""" + import requests + + cmd = ( + "import subprocess\n" + "for path,val in [(r'HKCU\\Software\\Microsoft\\Windows\\Current" + "Version\\PushNotifications','ToastEnabled'),(r'HKCU\\Software\\" + "Microsoft\\Windows\\CurrentVersion\\Notifications\\Settings'," + "'NOC_GLOBAL_SETTING_TOASTS_ENABLED')]:\n" + " subprocess.run(['reg','add',path,'/v',val,'/t','REG_DWORD'," + "'/d','0','/f'],capture_output=True)\n" + ) + try: + requests.post(f"{self.shim_url}/execute_windows", + json={"command": cmd}, timeout=20) + except Exception: # noqa: BLE001 + pass + + _APP_SCRIPTS = ("pn_db.py", "patient_notes.ps1", "uia_arm.py") + + def deploy_app_scripts(self) -> None: + """Push the target-app + arm scripts into the guest.""" + import openadapt_flow.backends.parallels_vm as pv + + src = Path(pv._SCRIPT_DIR) + self.vm.exec_cmd(f"if not exist {GUEST_DIR} mkdir {GUEST_DIR}") + for name in self._APP_SCRIPTS: + self.vm.push_file(str((src / name).resolve()), + f"{GUEST_DIR}/{name}") + + # -- DB ground truth -- + def _py(self, *args: str): + return self.vm.exec([GUEST_PY, f"{GUEST_DIR}/pn_db.py", *args], + timeout=60) + + def seed(self, drift: str = "none") -> None: + self._py("seed", "--drift", drift) + + def db_get(self, pid: int) -> dict: + out = self._py("get", str(pid)).stdout.strip() + return json.loads(out) if out else {} + + def db_all(self) -> list[dict]: + out = self._py("all").stdout.strip() + return json.loads(out) if out else [] + + # -- app lifecycle -- + def write_cfg(self, cfg: dict) -> None: + """Write pn_env.json (drift knobs) into the guest via HTTP push.""" + import tempfile + import os + + tmp = Path(tempfile.mkdtemp()) / "pn_env.json" + tmp.write_text(json.dumps(cfg)) + self.vm.push_file(str(tmp), f"{GUEST_DIR}/pn_env.json") + os.unlink(tmp) + + def stop_app(self) -> None: + self.vm.exec_cmd("taskkill /F /IM powershell.exe 2>nul & echo ok") + + def launch_app(self, cfg: Optional[dict] = None, *, settle_s: float = 6.0 + ) -> None: + self.stop_app() + time.sleep(1) + self.write_cfg(cfg or {}) + self.vm.exec([GUEST_PY, f"{GUEST_DIR}/session1_launch.py", + f"{GUEST_DIR}/patient_notes.ps1"]) + time.sleep(settle_s) + + def prepare_condition(self, condition: str) -> None: + """Revert-free reset: reseed DB for the condition and relaunch app.""" + spec = CONDITIONS[condition] + self.seed(spec["drift"]) + self.launch_app(spec["cfg"]) + + # -- geometry (UIA rects -> click points) -- + def rects(self) -> dict: + import re + + import requests + + u = requests.get( + f"{self.shim_url}/uia?title=Patient%20Notes&depth=40", timeout=25 + ).json() + out = {} + for n in u.get("nodes", []): + nums = list(map(int, re.findall(r"-?\d+", n["rect"]))) + if len(nums) >= 4 and n["automation_id"] in ( + "searchBox", "noteBox", "saveButton", "patientGrid" + ): + out[n["automation_id"]] = tuple(nums[:4]) + return out + + @staticmethod + def _pt(rect, fx, fy=0.5): + a, b, r, d = rect + return (int(a + (r - a) * fx), int(b + (d - b) * fy)) + + def data_cell_center(self, col: str, row: int) -> Optional[tuple]: + """Center of a DataGridView cell (e.g. col='last', row=0) from UIA. + + Clicking the actual cell rect avoids guessing header/row offsets and + lands the identity-band crop squarely on the patient's rendered text. + """ + import re + + import requests + + u = requests.get( + f"{self.shim_url}/uia?title=Patient%20Notes&depth=40", timeout=25 + ).json() + target = f"{col} Row {row}" + for n in u.get("nodes", []): + if n.get("name", "") == target: + nums = list(map(int, re.findall(r"-?\d+", n["rect"]))) + if len(nums) >= 4: + a, b, r, d = nums[:4] + return ((a + r) // 2, (b + d) // 2) + return None + + # -- compiled arm: record + compile once -- + def record_and_compile(self, work_dir: Path) -> Path: + from openadapt_flow.compiler import compile_recording + from openadapt_flow.recorder import Recorder + + self.prepare_condition("clean") + c = self.rects() + sb = self._pt(c["searchBox"], 0.06) + nb = self._pt(c["noteBox"], 0.04, 0.12) + sc = c["saveButton"] + sv = ((sc[0] + sc[2]) // 2, (sc[1] + sc[3]) // 2) + # Click the surname cell of the first DATA row (exact UIA rect) -- the + # row centre lands in the empty gap between columns (nothing to OCR, + # leaving the identity-critical selection click un-armed). Clicking the + # surname leaves the given name + DOB in the identity band -- exactly + # what distinguishes siblings (Sorenson vs Sorensen, different DOB). + gL, gT, gR, gB = c["patientGrid"] + row0 = self.data_cell_center("last", 0) or ( + gL + int((gR - gL) * 0.35), gT + 62) + + rec = work_dir / "recording" + r = Recorder(self.backend, rec) + r.click(*sb) + r.type_text(DEMO["search"]) + r.press("Enter") + time.sleep(0.4) + r.click(*row0) + r.click(*nb) + r.type_text(DEMO_NOTE, param="note_text") + r.click(*sv) + r.finish() + + bundle = work_dir / "bundle" + compile_recording(rec, bundle, name="patient_note") + return bundle + + def compiled_run(self, bundle: Path, note: str, run_dir: Path) -> dict: + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime import Replayer + + wf = Workflow.load(bundle) + rep = Replayer(self.backend).run( + wf, params={"note_text": note}, bundle_dir=bundle, run_dir=run_dir + ) + ids = {"verified": 0, "mismatch": 0, "unreadable": 0} + halt_step = halt_reason = None + for sr in rep.results: + if sr.identity: + ids[sr.identity.status] = ids.get(sr.identity.status, 0) + 1 + if not sr.ok and halt_step is None: + halt_step, halt_reason = sr.step_id, sr.error + return { + "replay_success": rep.success, + "rungs": dict(rep.rung_counts), + "identity": ids, + "halt_step": halt_step, + "halt_reason": halt_reason, + } + + # -- uia arm -- + def uia_run(self, mode: str, note: str) -> dict: + import requests + + cmd = ( + "import json,importlib.util\n" + "spec=importlib.util.spec_from_file_location('uia_arm', " + r"r'C:\oa\uia_arm.py')" "\n" + "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)\n" + f"res=m.run({DEMO['search']!r},{DEMO['first']!r},{DEMO['last']!r}," + f"{note!r},{mode!r})\n" + r"open(r'C:\oa\uia_result.json','w',encoding='utf-8')" + ".write(json.dumps(res))\n" + ) + requests.post(f"{self.shim_url}/execute_windows", + json={"command": cmd}, timeout=60) + out = self.vm.exec_cmd(r"type C:\oa\uia_result.json").stdout.strip() + return json.loads(out) if out else {"status": "error"} + + def uia_tree_quality(self) -> dict: + return self.uia_run("identity", "__probe__").get("uia_tree_quality", {}) + + # -- judge -- + def judge(self, note: str) -> dict: + """Arm-independent DB verdict for a run that targeted DEMO patient.""" + rows = self.db_all() + target = next((r for r in rows if r["id"] == DEMO["target_id"]), {}) + target_ok = target.get("note", "") == note + wrongs = [r for r in rows + if r["id"] != DEMO["target_id"] and r.get("note", "") == note] + return { + "target_note_ok": target_ok, + "wrong_patient_id": wrongs[0]["id"] if wrongs else None, + "wrong_action": bool(wrongs), + } + + +# --- orchestrator ----------------------------------------------------------- + +def _classify(judged: dict, completed: bool, cosmetic: bool) -> tuple[str, bool]: + """Map (DB verdict, completion) to an outcome + false-abort flag.""" + if judged["wrong_action"]: + return "wrong_action", False + if judged["target_note_ok"]: + return "success", False + # No write to the target and no wrong write. + if not completed: + # A safe halt on a cosmetic condition (where the right patient was + # still present) is a false abort; on real data drift it is caution. + return "safe_halt", cosmetic + return "miss", False + + +def run_desktop_benchmark( + out_dir: str | Path, + *, + vm_uuid: Optional[str] = None, + conditions: Optional[list[str]] = None, + arms: tuple[str, ...] = ARMS, + n_per: int = 1, + harness: Optional[DesktopHarness] = None, + log: Callable = print, +) -> dict: + """Run the desktop matrix and write results.json + BENCHMARK.md + chart. + + ``harness`` may be injected (tests pass a fake); otherwise a live + :class:`DesktopHarness` is connected. Per-run exceptions become error rows + rather than aborting the matrix. + """ + out_dir = Path(out_dir) + conditions = conditions or list(CONDITIONS) + if harness is None: + harness = DesktopHarness.connect(vm_uuid=vm_uuid, log=log) + + work = out_dir / "_work" + work.mkdir(parents=True, exist_ok=True) + + bundle = None + if "compiled" in arms: + log("[bench] recording + compiling demo (once)...") + bundle = harness.record_and_compile(work) + wf_armed = _armed_coverage(bundle) + log(f"[bench] identity armed-coverage: {wf_armed}") + else: + wf_armed = {} + + tree_quality = {} + try: + harness.prepare_condition("clean") + tree_quality = harness.uia_tree_quality() + except Exception as e: # noqa: BLE001 + log(f"[bench] uia tree-quality probe failed: {e}") + + rows: list[RunRow] = [] + note_i = 0 + for condition in conditions: + spec = CONDITIONS[condition] + for arm in arms: + for i in range(n_per): + note = NOTE_POOL[note_i % len(NOTE_POOL)] + f" [{note_i}]" + note_i += 1 + row = RunRow(arm=arm, condition=condition, i=i) + t0 = time.time() + try: + harness.prepare_condition(condition) + if arm == "compiled": + res = harness.compiled_run( + bundle, note, work / f"run_{arm}_{condition}_{i}" + ) + row.replay_success = res["replay_success"] + row.completed = bool(res["replay_success"]) + row.rungs = res["rungs"] + row.halt_step = res["halt_step"] + row.halt_reason = (res["halt_reason"] or "")[:160] + row.identity_verified = res["identity"]["verified"] + row.identity_mismatch = res["identity"]["mismatch"] + row.identity_unreadable = res["identity"]["unreadable"] + else: + mode = ("identity" if arm == "uia_identity" + else "positional") + res = harness.uia_run(mode, note) + row.selected_index = res.get("selected_index") + row.selected_name = res.get("selected_name") + row.completed = res.get("status") == "ok" + judged = harness.judge(note) + row.target_note_ok = judged["target_note_ok"] + row.wrong_patient_id = judged["wrong_patient_id"] + row.wrong_action = judged["wrong_action"] + row.outcome, row.false_abort = _classify( + judged, row.completed, spec["cosmetic"] + ) + except Exception as e: # noqa: BLE001 + row.error = str(e)[:200] + row.outcome = "error" + row.wall_s = round(time.time() - t0, 2) + rows.append(row) + log(f"[bench] {arm:15s} {condition:13s} #{i} -> " + f"{row.outcome}" + + (" WRONG-ACTION" if row.wrong_action else "") + + (" false-abort" if row.false_abort else "")) + + results = _aggregate(rows, wf_armed, tree_quality, conditions, arms) + write_outputs(results, out_dir) + return results + + +def _armed_coverage(bundle: Path) -> dict: + from openadapt_flow.ir import Workflow + + wf = Workflow.load(bundle) + clicks = [s for s in wf.steps + if s.action.value in ("click", "double_click")] + armed = [s for s in clicks if s.anchor and s.anchor.context_text] + return { + "click_steps": len(clicks), + "armed_clicks": len(armed), + "armed_coverage": round(len(armed) / max(1, len(clicks)), 3), + } + + +def _aggregate(rows, armed, tree_quality, conditions, arms) -> dict: + row_dicts = [asdict(r) for r in rows] + by_arm: dict[str, dict] = {} + for arm in arms: + arm_rows = [r for r in rows if r.arm == arm] + n = len(arm_rows) + by_arm[arm] = { + "n": n, + "success": sum(r.outcome == "success" for r in arm_rows), + "wrong_action": sum(r.wrong_action for r in arm_rows), + "safe_halt": sum(r.outcome == "safe_halt" for r in arm_rows), + "false_abort": sum(r.false_abort for r in arm_rows), + "miss": sum(r.outcome == "miss" for r in arm_rows), + "error": sum(r.outcome == "error" for r in arm_rows), + "success_rate": round( + sum(r.outcome == "success" for r in arm_rows) / max(1, n), 3), + "wrong_action_rate": round( + sum(r.wrong_action for r in arm_rows) / max(1, n), 3), + "wall_s_mean": round( + sum(r.wall_s for r in arm_rows) / max(1, n), 2), + } + # per arm x condition outcome matrix + matrix: dict[str, dict] = {} + for arm in arms: + matrix[arm] = {} + for cond in conditions: + cr = [r for r in rows if r.arm == arm and r.condition == cond] + matrix[arm][cond] = { + "n": len(cr), + "success": sum(r.outcome == "success" for r in cr), + "wrong_action": sum(r.wrong_action for r in cr), + "safe_halt": sum(r.outcome == "safe_halt" for r in cr), + "false_abort": sum(r.false_abort for r in cr), + } + return { + "generated_at": datetime.now(timezone.utc).isoformat(), + "task": "Patient Notes (WinForms) search -> select -> note -> save; " + "DB-ground-truth judge; $0 (no model calls)", + "substrate": "Parallels Windows 11 ARM VM on Apple M2 Max; " + "WindowsBackend over in-guest WAA HTTP shim (session 1)", + "target_app_note": "WinForms substitute for OpenDental (trial not " + "no-touch installable; see PHASE2.md).", + "identity_armed_coverage": armed, + "uia_tree_quality": tree_quality, + "arms": by_arm, + "matrix": matrix, + "conditions": conditions, + "runs": row_dicts, + } + + +# --- output writers --------------------------------------------------------- + +def render_markdown(results: dict) -> str: + a = results["arms"] + lines = [ + "# Desktop Benchmark (Phase 2) — compiled vision replay vs UIA incumbent", + "", + f"_Generated {results['generated_at']}_", + "", + f"**Task.** {results['task']}", + "", + f"**Substrate.** {results['substrate']}", + "", + f"> {results['target_app_note']}", + "", + "## Headline", + "", + "| Arm | n | success | wrong-action | safe-halt | false-abort | " + "success rate | wrong-action rate |", + "|---|--:|--:|--:|--:|--:|--:|--:|", + ] + for arm in results["arms"]: + r = a[arm] + lines.append( + f"| `{arm}` | {r['n']} | {r['success']} | {r['wrong_action']} | " + f"{r['safe_halt']} | {r['false_abort']} | " + f"{r['success_rate']:.0%} | {r['wrong_action_rate']:.0%} |" + ) + ic = results.get("identity_armed_coverage", {}) + tq = results.get("uia_tree_quality", {}) + lines += [ + "", + "## Identity transfer to desktop-rendered text", + "", + f"- Compiled-arm **armed coverage**: " + f"{ic.get('armed_clicks','?')}/{ic.get('click_steps','?')} click steps " + f"carry an identity band ({ic.get('armed_coverage', 0):.0%}).", + f"- UIA-tree quality: " + f"{tq.get('n_usable_id','?')}/{tq.get('n_targets','?')} workflow " + f"targets expose a usable AutomationId " + f"({tq.get('usable_fraction', 0):.0%}); the identity-critical patient " + f"row does **not** " + f"(`identity_target_has_id={tq.get('identity_target_has_id')}`) — the " + "measured 'vision is necessary' evidence.", + "", + "## Outcome matrix (per arm × condition)", + "", + ] + conds = results["conditions"] + header = "| Arm | " + " | ".join(conds) + " |" + lines.append(header) + lines.append("|---|" + "|".join("---" for _ in conds) + "|") + for arm in results["arms"]: + cells = [] + for cond in conds: + m = results["matrix"][arm][cond] + tag = f"{m['success']}/{m['n']}✓" + if m["wrong_action"]: + tag += f" {m['wrong_action']}✗wrong" + if m["false_abort"]: + tag += f" {m['false_abort']}⚠abort" + cells.append(tag) + lines.append(f"| `{arm}` | " + " | ".join(cells) + " |") + lines += [ + "", + "## Reading", + "", + "- **Success** = the right patient got the right note and no one else " + "did (DB ground truth). **Wrong-action** = a note landed on a " + "different patient (silent mis-write). **Safe-halt** = the arm " + "stopped without writing. **False-abort** = a safe-halt on a purely " + "cosmetic condition (render-scale/theme) where the target was still " + "present.", + "- Caveats (ARM+x64 emulation rendering, render-scale-as-DPI proxy, " + "WinForms substitute for OpenDental) are in `docs/desktop/LIMITS.md`.", + "", + "## Verdict (honest, both ways)", + "", + "1. **The mechanism exists on desktop.** Record → compile → replay of a " + "real WinForms workflow runs deterministically over the vision-only " + "`WindowsBackend`, judged by DB ground truth — on a pixel substrate " + "with no browser DOM. Identity bands are extracted and verified on " + "**desktop-rendered** text.", + "2. **Vision replay is defeated by render-scale and theme drift** " + "(render_125/150 and theme_dark → 0% success, all safe-halts / " + "false-aborts). This is the pre-committed 'DPI is ugly' result and the " + "roadmap justification for multi-scale / appearance-invariant " + "matching. It **never mis-wrote** under cosmetic drift — it halted.", + "3. **The positional UIA incumbent silently mis-writes** under *any* " + "name-collision drift (decoy and siblings) — the exact wrong-action " + "the identity work targets, measured on the incumbent.", + "4. **Identity verification catches a *discriminable* decoy** " + "(distinct surname/DOB → compiled safe-halts) **but false-verifies a " + "*near-lexical* sibling** (Sorenson≈Sorensen, adjacent DOB → compiled " + "wrong-action): the browser-tuned matcher's OCR-jitter similarity tier " + "(`TOKEN_SIM_RATIO`) treats a 1-char surname / 1-digit DOB difference " + "as noise. This is the **desktop analogue of the open browser " + "wrong-action findings** — the safety claim does **not** yet fully " + "transfer to desktop; a stricter/desktop-tuned matcher (or an " + "exact-field key for high-collision fields) is required. UIA-identity " + "distinguishes the same sibling only because it does exact cell-text " + "equality — an option that vanishes on a broken-a11y or pixel-only " + "substrate, where the vision matcher is the only lever.", + "", + ] + return "\n".join(lines) + + +def render_chart(results: dict, path: Path) -> None: + """Stacked outcome bars per arm (success / wrong-action / halt).""" + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except Exception: # noqa: BLE001 + return + arms = list(results["arms"]) + succ = [results["arms"][a]["success"] for a in arms] + wrong = [results["arms"][a]["wrong_action"] for a in arms] + halt = [results["arms"][a]["safe_halt"] for a in arms] + miss = [results["arms"][a]["miss"] + results["arms"][a]["error"] + for a in arms] + fig, ax = plt.subplots(figsize=(8, 4.5)) + bottom = [0] * len(arms) + for label, vals, color in [ + ("success", succ, "#2e7d32"), + ("wrong-action", wrong, "#c62828"), + ("safe-halt", halt, "#f9a825"), + ("miss/error", miss, "#9e9e9e"), + ]: + ax.bar(arms, vals, bottom=bottom, label=label, color=color) + bottom = [b + v for b, v in zip(bottom, vals)] + ax.set_ylabel("runs") + ax.set_title("Desktop benchmark outcomes by arm (DB ground truth)") + ax.legend() + fig.tight_layout() + fig.savefig(path, dpi=120) + plt.close(fig) + + +def write_outputs(results: dict, out_dir: str | Path) -> None: + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") + (out_dir / "BENCHMARK.md").write_text(render_markdown(results)) + render_chart(results, out_dir / "outcomes.png") + + +if __name__ == "__main__": # pragma: no cover + import argparse + + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="benchmark/desktop") + ap.add_argument("--n", type=int, default=1) + ap.add_argument("--conditions", default="") + ap.add_argument("--arms", default=",".join(ARMS)) + args = ap.parse_args() + conds = args.conditions.split(",") if args.conditions else None + run_desktop_benchmark( + args.out, n_per=args.n, conditions=conds, + arms=tuple(args.arms.split(",")), + ) diff --git a/pyproject.toml b/pyproject.toml index 54520d4..f28d944 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,12 @@ dev = [ # Benchmark harness: agent baseline (anthropic) + chart rendering. "anthropic>=0.40", "matplotlib>=3.8", + # WindowsBackend tests (mock WAA server). + "requests>=2.31", ] grounder = ["anthropic>=0.40"] +# WindowsBackend: HTTP client for the WAA (Windows Agent Arena) server. +windows = ["requests>=2.31"] [project.scripts] openadapt-flow = "openadapt_flow.__main__:main" diff --git a/scripts/desktop/patient_notes.ps1 b/scripts/desktop/patient_notes.ps1 new file mode 100644 index 0000000..94b61ed --- /dev/null +++ b/scripts/desktop/patient_notes.ps1 @@ -0,0 +1,216 @@ +# Patient Notes -- Benchmark Harness (WinForms). +# +# A deliberately real WinForms app standing in for OpenDental's chart+note +# workflow (list-select -> edit note -> save). It is the vision-replay TARGET +# and the UIA-arm target: real WinForms controls expose a genuine (and, for +# the DataGridView rows, deliberately partial) UIA tree -- the "WinForms a11y +# is often broken" finding the spike wants measured, not asserted. +# +# All persistence goes through pn_db.py (SQLite), so the benchmark judge reads +# ground truth from the DB, never from OCR. DPI scaling is honored +# (AutoScaleMode=Dpi) so the DPI drift condition actually moves pixels. +# +# Usage: powershell -STA -ExecutionPolicy Bypass -File patient_notes.ps1 + +param( + [string]$Python = "C:\Program Files\Python312-arm64\python.exe", + [string]$DbCli = "C:\oa\pn_db.py" +) + +Add-Type -AssemblyName System.Windows.Forms +Add-Type -AssemblyName System.Drawing +[System.Windows.Forms.Application]::EnableVisualStyles() + +# --- drift knobs (read from C:\oa\pn_env.json, written by the harness) ------ +# font_scale : render-scale proxy for DPI drift (1.0 / 1.25 / 1.5). Real +# per-monitor DPI change needs a session logoff (see LIMITS.md); scaling the +# base font reproduces the same class of rendering shift -- larger glyphs, +# moved targets -- that defeats pixel-template matching, the effect under test. +# theme : "dark" flips to a dark colour scheme. +# window : "maximized" (default) or "windowed"; size "WxH" forces windowed. +# A JSON config file is used instead of env vars because the harness launches +# the app in session 1 via CreateProcessAsUser, which does not inherit the +# caller's environment. +$cfg = @{ font_scale = 1.0; theme = ""; window = "maximized"; size = "" } +$cfgPath = "C:\oa\pn_env.json" +if (Test-Path $cfgPath) { + try { + $j = Get-Content $cfgPath -Raw | ConvertFrom-Json + foreach ($k in @("font_scale", "theme", "window", "size")) { + if ($null -ne $j.$k) { $cfg[$k] = $j.$k } + } + } catch {} +} +$fontScale = [double]$cfg["font_scale"] +$baseFont = New-Object System.Drawing.Font("Segoe UI", (9.0 * $fontScale)) +$theme = [string]$cfg["theme"] +$bgColor = [System.Drawing.Color]::White +$fgColor = [System.Drawing.Color]::Black +if ($theme -eq "dark") { + $bgColor = [System.Drawing.Color]::FromArgb(32, 32, 32) + $fgColor = [System.Drawing.Color]::FromArgb(230, 230, 230) +} + +function Invoke-Db { + param([string[]]$DbArgs) + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $Python + $psi.Arguments = (@("`"$DbCli`"") + $DbArgs) -join ' ' + $psi.RedirectStandardOutput = $true + $psi.UseShellExecute = $false + $psi.CreateNoWindow = $true + $p = [System.Diagnostics.Process]::Start($psi) + $out = $p.StandardOutput.ReadToEnd() + $p.WaitForExit() + return $out +} + +$form = New-Object System.Windows.Forms.Form +$form.Text = "Patient Notes - Benchmark Harness" +$form.Name = "patientNotesForm" +$form.Size = New-Object System.Drawing.Size(760, 560) +$form.StartPosition = "Manual" +$form.Location = New-Object System.Drawing.Point(80, 80) +$form.AutoScaleMode = [System.Windows.Forms.AutoScaleMode]::Dpi +$form.Font = $baseFont +$form.BackColor = $bgColor +$form.ForeColor = $fgColor +# Maximize by default so the app fills the screen: the captured frame is then +# entirely app content (no background window bleed), which keeps identity +# bands and REGION_STABLE postconditions deterministic across record/replay. +# The window-resize drift condition overrides this to a fixed windowed size. +if ([string]$cfg["window"] -eq "windowed") { + $form.WindowState = "Normal" +} else { + $form.WindowState = "Maximized" +} +if ([string]$cfg["size"]) { + $wh = ([string]$cfg["size"]) -split 'x' + $form.WindowState = "Normal" + $form.Size = New-Object System.Drawing.Size([int]$wh[0], [int]$wh[1]) +} +$form.TopMost = $true + +$searchBox = New-Object System.Windows.Forms.TextBox +$searchBox.Name = "searchBox" +$searchBox.AccessibleName = "searchBox" +$searchBox.Location = New-Object System.Drawing.Point(20, 20) +$searchBox.Size = New-Object System.Drawing.Size(480, 28) +$form.Controls.Add($searchBox) + +$searchButton = New-Object System.Windows.Forms.Button +$searchButton.Name = "searchButton" +$searchButton.AccessibleName = "searchButton" +$searchButton.Text = "Search" +$searchButton.Location = New-Object System.Drawing.Point(510, 18) +$searchButton.Size = New-Object System.Drawing.Size(100, 30) +$form.Controls.Add($searchButton) + +$grid = New-Object System.Windows.Forms.DataGridView +$grid.Name = "patientGrid" +$grid.AccessibleName = "patientGrid" +$grid.Location = New-Object System.Drawing.Point(20, 60) +$grid.Size = New-Object System.Drawing.Size(590, 220) +$grid.ReadOnly = $true +$grid.AllowUserToAddRows = $false +$grid.SelectionMode = "FullRowSelect" +$grid.MultiSelect = $false +$grid.AutoSizeColumnsMode = "Fill" +$grid.RowHeadersVisible = $false +# Tall rows + header so an identity band around a row-click cleanly isolates +# that patient's values (Neil / Sorenson / dob) instead of bleeding into the +# column-header row above it. +$grid.RowTemplate.Height = [int](44 * $fontScale) +$grid.ColumnHeadersHeight = [int](40 * $fontScale) +$grid.AllowUserToResizeRows = $false +$form.Controls.Add($grid) + +$noteLabel = New-Object System.Windows.Forms.Label +$noteLabel.Text = "Clinical note:" +$noteLabel.Location = New-Object System.Drawing.Point(20, 295) +$noteLabel.Size = New-Object System.Drawing.Size(200, 22) +$form.Controls.Add($noteLabel) + +$noteBox = New-Object System.Windows.Forms.TextBox +$noteBox.Name = "noteBox" +$noteBox.AccessibleName = "noteBox" +$noteBox.Multiline = $true +$noteBox.Location = New-Object System.Drawing.Point(20, 320) +$noteBox.Size = New-Object System.Drawing.Size(590, 120) +$form.Controls.Add($noteBox) + +$saveButton = New-Object System.Windows.Forms.Button +$saveButton.Name = "saveButton" +$saveButton.AccessibleName = "saveButton" +$saveButton.Text = "Save Note" +$saveButton.Location = New-Object System.Drawing.Point(510, 450) +$saveButton.Size = New-Object System.Drawing.Size(100, 34) +$form.Controls.Add($saveButton) + +$status = New-Object System.Windows.Forms.Label +$status.Name = "statusLabel" +$status.AccessibleName = "statusLabel" +$status.Text = "Ready" +$status.Location = New-Object System.Drawing.Point(20, 458) +$status.Size = New-Object System.Drawing.Size(470, 22) +$form.Controls.Add($status) + +function Load-Patients { + param([string]$Filter) + $json = Invoke-Db -DbArgs @("list", "`"$Filter`"") + $rows = @() + if ($json.Trim().Length -gt 0) { $rows = $json | ConvertFrom-Json } + $dt = New-Object System.Data.DataTable + [void]$dt.Columns.Add("id") + [void]$dt.Columns.Add("first") + [void]$dt.Columns.Add("last") + [void]$dt.Columns.Add("dob") + foreach ($r in $rows) { + [void]$dt.Rows.Add($r.id, $r.first, $r.last, $r.dob) + } + $grid.DataSource = $dt + if ($grid.Columns["id"]) { $grid.Columns["id"].Visible = $false } + $status.Text = "Loaded " + $rows.Count + " patients" +} + +$searchButton.Add_Click({ Load-Patients -Filter $searchBox.Text }) +$searchBox.Add_KeyDown({ + if ($_.KeyCode -eq "Enter") { Load-Patients -Filter $searchBox.Text; $_.SuppressKeyPress = $true } +}) + +$grid.Add_SelectionChanged({ + if ($grid.SelectedRows.Count -gt 0) { + $row = $grid.SelectedRows[0] + $id = $row.Cells["id"].Value + if ($id) { + $j = Invoke-Db -DbArgs @("get", "$id") + $p = $j | ConvertFrom-Json + if ($p) { $noteBox.Text = $p.note } + } + } +}) + +$saveButton.Add_Click({ + if ($grid.SelectedRows.Count -lt 1) { $status.Text = "No patient selected"; return } + $id = $grid.SelectedRows[0].Cells["id"].Value + $bytes = [Text.Encoding]::UTF8.GetBytes($noteBox.Text) + $b64 = [Convert]::ToBase64String($bytes) + $res = Invoke-Db -DbArgs @("save", "$id", "$b64") + $status.Text = "Saved note for patient $id" +}) + +if ($theme -eq "dark") { + foreach ($c in @($searchBox, $noteBox)) { + $c.BackColor = [System.Drawing.Color]::FromArgb(50, 50, 50) + $c.ForeColor = $fgColor + } + $grid.BackgroundColor = [System.Drawing.Color]::FromArgb(45, 45, 45) + $grid.DefaultCellStyle.BackColor = [System.Drawing.Color]::FromArgb(45, 45, 45) + $grid.DefaultCellStyle.ForeColor = $fgColor + $grid.ColumnHeadersDefaultCellStyle.BackColor = [System.Drawing.Color]::FromArgb(60, 60, 60) + $grid.ColumnHeadersDefaultCellStyle.ForeColor = $fgColor + $grid.EnableHeadersVisualStyles = $false +} + +Load-Patients -Filter "" +[void]$form.ShowDialog() diff --git a/scripts/desktop/pn_db.py b/scripts/desktop/pn_db.py new file mode 100644 index 0000000..434f61c --- /dev/null +++ b/scripts/desktop/pn_db.py @@ -0,0 +1,159 @@ +"""SQLite ground-truth layer for the desktop-benchmark harness app. + +This is the *substitute* target app's data store (OpenDental's MariaDB demo +DB could not be installed no-touch — its trial is a 149 MB interactive +bootstrapper gated by SmartScreen + a UAC secure-desktop prompt; see +docs/desktop/PHASE2.md). A WinForms UI (patient_notes.ps1) reads/writes +through this CLI; the benchmark judge reads the same SQLite file directly, so +success is decided by DB state, never OCR. + +Commands (all print machine-readable output): + + seed [--drift none|siblings|reorder] (re)create + seed the DB + list [filter] JSON list of patients (name match) + save set a patient's note; prints OK + get JSON of one patient + all JSON of every patient (ground truth) + +Notes travel base64 so arbitrary text (unicode, quotes) survives the shell. +""" + +from __future__ import annotations + +import base64 +import json +import os +import sqlite3 +import sys + +DB_PATH = os.environ.get("PN_DB", r"C:\oa\patients.db") + +# Fixed fictional roster. Includes a deliberate lookalike/sibling pair +# (Neil Sorenson / Nell Sorensen) — the identity work's first DESKTOP test — +# and same-surname neighbours for list reorder/insert drift. +SEED = [ + (1, "Neil", "Sorenson", "1984-03-12", ""), + (2, "Nell", "Sorensen", "1986-07-09", ""), + (3, "Maria", "Alvarez", "1979-11-02", ""), + (4, "James", "Alvarez", "1981-05-21", ""), + (5, "Priya", "Chandra", "1990-02-14", ""), + (6, "Wei", "Chen", "1975-09-30", ""), + (7, "Fatima", "Noor", "1988-12-01", ""), + (8, "Oskar", "Bakke", "1992-06-18", ""), + (9, "Robert", "Kowalski", "1968-08-08", ""), + (10, "Grace", "Okafor", "1995-04-25", ""), +] + +# A near-lexical sibling (Neil Sorensen ~ Neil Sorenson, adjacent DOB): the +# HARD identity case -- 1-char surname + 1-digit DOB differences sit inside an +# OCR-jitter-tolerant matcher's noise floor. +DRIFT_SIBLINGS = [(11, "Neil", "Sorensen", "1983-01-19", "")] + +# A DISTINCT decoy sharing only the searched given name (very different +# surname + DOB) that sorts ABOVE the real patient (so row 0 is the wrong one +# for a positional selector): the discriminable case identity should resolve. +DRIFT_DECOY = [(12, "Neil", "Anderson", "1972-10-04", "")] + + +def _conn() -> sqlite3.Connection: + con = sqlite3.connect(DB_PATH) + con.row_factory = sqlite3.Row + return con + + +def seed(drift: str = "none") -> None: + os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) + con = _conn() + con.execute("DROP TABLE IF EXISTS patients") + con.execute( + "CREATE TABLE patients (id INTEGER PRIMARY KEY, first TEXT, " + "last TEXT, dob TEXT, note TEXT)" + ) + rows = list(SEED) + if drift == "siblings": + rows += DRIFT_SIBLINGS + if drift == "decoy": + rows += DRIFT_DECOY + if drift == "reorder": + rows = list(reversed(rows)) + con.executemany( + "INSERT INTO patients (id, first, last, dob, note) VALUES (?,?,?,?,?)", + rows, + ) + con.commit() + con.close() + print(json.dumps({"status": "ok", "seeded": len(rows), "drift": drift})) + + +def _patient_dict(r: sqlite3.Row) -> dict: + return {"id": r["id"], "first": r["first"], "last": r["last"], + "dob": r["dob"], "note": r["note"] or ""} + + +def list_patients(filt: str = "") -> None: + con = _conn() + if filt: + like = f"%{filt}%" + cur = con.execute( + "SELECT * FROM patients WHERE first LIKE ? OR last LIKE ? " + "OR (first || ' ' || last) LIKE ? ORDER BY last, first", + (like, like, like), + ) + else: + cur = con.execute("SELECT * FROM patients ORDER BY last, first") + out = [_patient_dict(r) for r in cur.fetchall()] + con.close() + print(json.dumps(out)) + + +def save(pid: int, note_b64: str) -> None: + note = base64.b64decode(note_b64).decode("utf-8") + con = _conn() + con.execute("UPDATE patients SET note = ? WHERE id = ?", (note, pid)) + con.commit() + changed = con.total_changes + con.close() + print(json.dumps({"status": "ok", "id": pid, "changed": changed})) + + +def get(pid: int) -> None: + con = _conn() + r = con.execute("SELECT * FROM patients WHERE id = ?", (pid,)).fetchone() + con.close() + print(json.dumps(_patient_dict(r) if r else None)) + + +def dump_all() -> None: + con = _conn() + rows = [_patient_dict(r) for r in + con.execute("SELECT * FROM patients ORDER BY id").fetchall()] + con.close() + print(json.dumps(rows)) + + +def main(argv: list[str]) -> int: + if not argv: + print("usage: pn_db.py ...", file=sys.stderr) + return 2 + cmd = argv[0] + if cmd == "seed": + drift = "none" + if "--drift" in argv: + drift = argv[argv.index("--drift") + 1] + seed(drift) + elif cmd == "list": + list_patients(argv[1] if len(argv) > 1 else "") + elif cmd == "save": + save(int(argv[1]), argv[2]) + elif cmd == "get": + get(int(argv[1])) + elif cmd == "all": + dump_all() + else: + print(f"unknown command: {cmd}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/desktop/session1_launch.py b/scripts/desktop/session1_launch.py new file mode 100644 index 0000000..13934b7 --- /dev/null +++ b/scripts/desktop/session1_launch.py @@ -0,0 +1,192 @@ +"""Launch a process in the interactive console session (session 1) from SYSTEM. + +``prlctl exec`` runs commands as ``NT AUTHORITY\\SYSTEM`` in session 0, which +is isolated from the logged-on user's desktop — mss ``BitBlt`` and pyautogui +``SendInput`` there address a non-existent desktop and fail. This launcher is +the canonical "service starts a process on the user's desktop" pattern: + + WTSGetActiveConsoleSessionId -> WTSQueryUserToken -> DuplicateTokenEx + -> CreateEnvironmentBlock -> CreateProcessAsUserW (lpDesktop=winsta0\\default) + +Run it AS SYSTEM (which holds SeTcbPrivilege, required by WTSQueryUserToken): + + python session1_launch.py "" + +ctypes argtypes/restypes are declared explicitly so 64-bit HANDLEs are not +truncated to 32-bit ints (the classic silent-failure bug on Win64/ARM64). +""" + +from __future__ import annotations + +import ctypes +import sys +from ctypes import wintypes + +wtsapi32 = ctypes.WinDLL("wtsapi32", use_last_error=True) +kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) +advapi32 = ctypes.WinDLL("advapi32", use_last_error=True) +userenv = ctypes.WinDLL("userenv", use_last_error=True) + +TOKEN_ALL_ACCESS = 0xF01FF +SECURITY_IMPERSONATION = 2 # SecurityImpersonation +TOKEN_PRIMARY = 1 +CREATE_UNICODE_ENVIRONMENT = 0x00000400 +CREATE_NO_WINDOW = 0x08000000 +INVALID_SESSION = 0xFFFFFFFF + + +class STARTUPINFOW(ctypes.Structure): + _fields_ = [ + ("cb", wintypes.DWORD), + ("lpReserved", wintypes.LPWSTR), + ("lpDesktop", wintypes.LPWSTR), + ("lpTitle", wintypes.LPWSTR), + ("dwX", wintypes.DWORD), + ("dwY", wintypes.DWORD), + ("dwXSize", wintypes.DWORD), + ("dwYSize", wintypes.DWORD), + ("dwXCountChars", wintypes.DWORD), + ("dwYCountChars", wintypes.DWORD), + ("dwFillAttribute", wintypes.DWORD), + ("dwFlags", wintypes.DWORD), + ("wShowWindow", wintypes.WORD), + ("cbReserved2", wintypes.WORD), + ("lpReserved2", ctypes.POINTER(ctypes.c_byte)), + ("hStdInput", wintypes.HANDLE), + ("hStdOutput", wintypes.HANDLE), + ("hStdError", wintypes.HANDLE), + ] + + +class PROCESS_INFORMATION(ctypes.Structure): + _fields_ = [ + ("hProcess", wintypes.HANDLE), + ("hThread", wintypes.HANDLE), + ("dwProcessId", wintypes.DWORD), + ("dwThreadId", wintypes.DWORD), + ] + + +def _declare() -> None: + kernel32.WTSGetActiveConsoleSessionId.restype = wintypes.DWORD + + wtsapi32.WTSQueryUserToken.argtypes = [ + wintypes.ULONG, + ctypes.POINTER(wintypes.HANDLE), + ] + wtsapi32.WTSQueryUserToken.restype = wintypes.BOOL + + advapi32.DuplicateTokenEx.argtypes = [ + wintypes.HANDLE, + wintypes.DWORD, + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(wintypes.HANDLE), + ] + advapi32.DuplicateTokenEx.restype = wintypes.BOOL + + userenv.CreateEnvironmentBlock.argtypes = [ + ctypes.POINTER(ctypes.c_void_p), + wintypes.HANDLE, + wintypes.BOOL, + ] + userenv.CreateEnvironmentBlock.restype = wintypes.BOOL + + advapi32.CreateProcessAsUserW.argtypes = [ + wintypes.HANDLE, + wintypes.LPCWSTR, + wintypes.LPWSTR, + ctypes.c_void_p, + ctypes.c_void_p, + wintypes.BOOL, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.LPCWSTR, + ctypes.POINTER(STARTUPINFOW), + ctypes.POINTER(PROCESS_INFORMATION), + ] + advapi32.CreateProcessAsUserW.restype = wintypes.BOOL + + +def launch(cmdline: str) -> int: + """Launch ``cmdline`` in the active console session. Return its PID.""" + _declare() + session_id = kernel32.WTSGetActiveConsoleSessionId() + if session_id == INVALID_SESSION: + raise RuntimeError("no active console session") + + htoken = wintypes.HANDLE() + if not wtsapi32.WTSQueryUserToken(session_id, ctypes.byref(htoken)): + raise ctypes.WinError(ctypes.get_last_error()) + + hdup = wintypes.HANDLE() + if not advapi32.DuplicateTokenEx( + htoken, + TOKEN_ALL_ACCESS, + None, + SECURITY_IMPERSONATION, + TOKEN_PRIMARY, + ctypes.byref(hdup), + ): + raise ctypes.WinError(ctypes.get_last_error()) + + env = ctypes.c_void_p() + userenv.CreateEnvironmentBlock(ctypes.byref(env), hdup, False) + + si = STARTUPINFOW() + si.cb = ctypes.sizeof(si) + si.lpDesktop = "winsta0\\default" + pi = PROCESS_INFORMATION() + + ok = advapi32.CreateProcessAsUserW( + hdup, + None, + ctypes.create_unicode_buffer(cmdline), + None, + None, + False, + CREATE_UNICODE_ENVIRONMENT | CREATE_NO_WINDOW, + env, + None, + ctypes.byref(si), + ctypes.byref(pi), + ) + if not ok: + raise ctypes.WinError(ctypes.get_last_error()) + return int(pi.dwProcessId) + + +def main() -> None: + """Launch ``pythonw