From 9bc19d114a12f09bfe5b56ecd17f1db7e269ffc4 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 13 Jul 2026 09:57:46 -0400 Subject: [PATCH 1/3] style: apply ruff format and import sorting to openadapt_flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical only — no logic changes. Normalizes formatting and import order across the package so the new ruff lint+format gate (added in the follow-up commit) is green. Verified: all 70 modules import, the core test subset (342 passed) and every quality gate stay green. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01CKrVJJy5jWVCkXAqgUqtqZ --- openadapt_flow/__main__.py | 73 +- openadapt_flow/adapters/capture.py | 30 +- openadapt_flow/backends/parallels_vm.py | 63 +- openadapt_flow/backends/rdp_backend.py | 4 +- openadapt_flow/backends/windows_backend.py | 4 +- openadapt_flow/bench.py | 4 +- openadapt_flow/benchmark/agent_baseline.py | 22 +- openadapt_flow/benchmark/chart_fonts.py | 7 +- openadapt_flow/benchmark/desktop_benchmark.py | 112 +-- openadapt_flow/benchmark/dom_arm.py | 195 +++--- openadapt_flow/benchmark/hybrid_benchmark.py | 263 +++---- openadapt_flow/benchmark/openemr_benchmark.py | 113 ++- openadapt_flow/benchmark/reliability.py | 17 +- .../benchmark/reliability_corpus.py | 34 +- openadapt_flow/benchmark/run_benchmark.py | 78 +-- openadapt_flow/benchmark/verify.py | 1 + openadapt_flow/compiler/codegen.py | 4 +- openadapt_flow/compiler/compile.py | 78 +-- openadapt_flow/emit/mcp_tool.py | 4 +- openadapt_flow/ir.py | 4 +- openadapt_flow/mockmed/server.py | 4 +- openadapt_flow/recorder.py | 9 +- openadapt_flow/report.py | 30 +- openadapt_flow/runtime/grounder.py | 26 +- openadapt_flow/runtime/heal.py | 7 +- openadapt_flow/runtime/identity.py | 133 ++-- openadapt_flow/runtime/remote_vlm.py | 6 +- openadapt_flow/runtime/replayer.py | 140 ++-- openadapt_flow/runtime/resolver.py | 11 +- openadapt_flow/services/vlm_service/app.py | 4 +- .../services/vlm_service/backends.py | 8 +- openadapt_flow/validation/adversary_corpus.py | 269 ++++++-- .../validation/adversary_corpus_v2.py | 72 +- .../validation/adversary_corpus_v3.py | 16 +- openadapt_flow/validation/dense_surface.py | 593 ++++++++++------ openadapt_flow/validation/identity_ladder.py | 332 ++++++--- openadapt_flow/validation/identity_roc.py | 136 ++-- .../validation/pixel_identity_probe.py | 647 ++++++++++++------ .../validation/vlm_identity_probe.py | 133 ++-- openadapt_flow/vision/match.py | 8 +- openadapt_flow/vision/ocr.py | 20 +- openadapt_flow/volatility.py | 34 +- 42 files changed, 2175 insertions(+), 1573 deletions(-) diff --git a/openadapt_flow/__main__.py b/openadapt_flow/__main__.py index 3b49e89..68206f2 100644 --- a/openadapt_flow/__main__.py +++ b/openadapt_flow/__main__.py @@ -63,9 +63,7 @@ def _cmd_demo_record(args: argparse.Namespace) -> int: def _cmd_compile(args: argparse.Namespace) -> int: from openadapt_flow.compiler import compile_recording - workflow = compile_recording( - Path(args.recording), Path(args.out), name=args.name - ) + workflow = compile_recording(Path(args.recording), Path(args.out), name=args.name) print( f"Compiled {len(workflow.steps)} steps into {args.out} " f"(workflow: {workflow.name!r})" @@ -150,18 +148,14 @@ def _cmd_replay(args: argparse.Namespace) -> int: backend, grounder=grounder, identity_vlm=appliance.identity_vlm if appliance else None, - state_verifier=( - appliance.state_verifier if appliance else None - ), + state_verifier=(appliance.state_verifier if appliance else None), ).run( workflow, params=params, bundle_dir=bundle, run_dir=run_dir, save_healed_to=( - Path(args.save_healed_to) - if args.save_healed_to - else None + Path(args.save_healed_to) if args.save_healed_to else None ), ) finally: @@ -215,9 +209,7 @@ def backend_factory(): finally: stop() - report_md = render_bench_report( - run_root / "bench.json", run_root / "BENCH.md" - ) + report_md = render_bench_report(run_root / "bench.json", run_root / "BENCH.md") print( f"Bench: {result['success_count']}/{result['n']} succeeded " f"(p50 {result['total_ms_p50']:.0f} ms) — {report_md}" @@ -284,20 +276,12 @@ def build_parser() -> argparse.ArgumentParser: default="Follow-up in 2 weeks; BP recheck.", help="Note text typed during the demo (recorded as a parameter)", ) - p.add_argument( - "--param-name", default="note", help="Parameter name for the note" - ) - p.add_argument( - "--drift", default=None, help="Comma-separated MockMed drift modes" - ) - p.add_argument( - "--headed", action="store_true", help="Run the browser headed" - ) + p.add_argument("--param-name", default="note", help="Parameter name for the note") + p.add_argument("--drift", default=None, help="Comma-separated MockMed drift modes") + p.add_argument("--headed", action="store_true", help="Run the browser headed") p.set_defaults(func=_cmd_demo_record) - p = sub.add_parser( - "compile", help="Compile a recording into a workflow bundle" - ) + p = sub.add_parser("compile", help="Compile a recording into a workflow bundle") p.add_argument("recording", help="Recording directory") p.add_argument("--out", required=True, help="Output bundle directory") p.add_argument("--name", required=True, help="Workflow name") @@ -314,10 +298,7 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument( "--url", default=None, - help=( - "URL of the target app (default: serve the bundled MockMed " - "demo app)" - ), + help=("URL of the target app (default: serve the bundled MockMed demo app)"), ) p.add_argument( "--drift", @@ -347,9 +328,7 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Write the healed bundle to this directory", ) - p.add_argument( - "--headed", action="store_true", help="Run the browser headed" - ) + p.add_argument("--headed", action="store_true", help="Run the browser headed") p.set_defaults(func=_cmd_replay) p = sub.add_parser( @@ -362,18 +341,14 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Comma-separated drift modes forwarded to the MockMed URL", ) - p.add_argument( - "--run-root", required=True, help="Directory for per-iteration runs" - ) + p.add_argument("--run-root", required=True, help="Directory for per-iteration runs") p.add_argument( "--param", action="append", metavar="K=V", help="Parameter substitution (repeatable)", ) - p.add_argument( - "--headed", action="store_true", help="Run the browser headed" - ) + p.add_argument("--headed", action="store_true", help="Run the browser headed") p.set_defaults(func=_cmd_bench) p = sub.add_parser( @@ -390,9 +365,7 @@ def build_parser() -> argparse.ArgumentParser: default=100, help="Compiled-replay iterations", ) - p.add_argument( - "--n-agent", type=int, default=20, help="Agent iterations" - ) + p.add_argument("--n-agent", type=int, default=20, help="Agent iterations") p.add_argument( "--out", default="benchmark/", @@ -403,27 +376,17 @@ def build_parser() -> argparse.ArgumentParser: default="Follow-up in 2 weeks; BP recheck.", help="Note text both arms enter", ) - p.add_argument( - "--headed", action="store_true", help="Run the browsers headed" - ) + p.add_argument("--headed", action="store_true", help="Run the browsers headed") p.set_defaults(func=_cmd_benchmark) - p = sub.add_parser( - "emit-skill", help="Emit an Agent Skills folder for a bundle" - ) + p = sub.add_parser("emit-skill", help="Emit an Agent Skills folder for a bundle") p.add_argument("bundle", help="Workflow bundle directory") - p.add_argument( - "--out", required=True, help="Parent directory for the skill folder" - ) + p.add_argument("--out", required=True, help="Parent directory for the skill folder") p.set_defaults(func=_cmd_emit_skill) - p = sub.add_parser( - "emit-mcp", help="Emit a standalone MCP server.py for a bundle" - ) + p = sub.add_parser("emit-mcp", help="Emit a standalone MCP server.py for a bundle") p.add_argument("bundle", help="Workflow bundle directory") - p.add_argument( - "--out", required=True, help="Path for the generated server.py" - ) + p.add_argument("--out", required=True, help="Path for the generated server.py") p.set_defaults(func=_cmd_emit_mcp) return parser diff --git a/openadapt_flow/adapters/capture.py b/openadapt_flow/adapters/capture.py index 3d19270..141cdaa 100644 --- a/openadapt_flow/adapters/capture.py +++ b/openadapt_flow/adapters/capture.py @@ -128,16 +128,28 @@ # Bare modifier presses carry no workflow meaning on their own (their effect is # only visible combined with another key). _MODIFIER_KEY_NAMES = { - "shift", "shift_l", "shift_r", - "ctrl", "ctrl_l", "ctrl_r", - "alt", "alt_l", "alt_r", "alt_gr", - "cmd", "cmd_l", "cmd_r", + "shift", + "shift_l", + "shift_r", + "ctrl", + "ctrl_l", + "ctrl_r", + "alt", + "alt_l", + "alt_r", + "alt_gr", + "cmd", + "cmd_l", + "cmd_r", "caps_lock", } # Modifiers that, combined with another key, form a shortcut/chord with no flow # equivalent (shift is excluded — shift+char is just a shifted character). _CHORD_MODIFIER_NAMES = _MODIFIER_KEY_NAMES - { - "shift", "shift_l", "shift_r", "caps_lock", + "shift", + "shift_l", + "shift_r", + "caps_lock", } @@ -293,9 +305,7 @@ def _convert_key_type( name = non_mods[0] mapped = _KEY_NAME_MAP.get(name.lower()) if mapped is None: - raise ValueError( - f"unmapped key {name!r} at t={ts:.3f}; extend _KEY_NAME_MAP" - ) + raise ValueError(f"unmapped key {name!r} at t={ts:.3f}; extend _KEY_NAME_MAP") flush_text() events.append({"kind": "key", "key": mapped, "_ts": ts}) @@ -382,9 +392,7 @@ def convert_capture( t_after = ts + settle_s if i + 1 < len(events): t_after = min(t_after, float(events[i + 1]["_ts"])) - after_img = session.get_frame_at( - t_after, tolerance=FRAME_TOLERANCE_S - ) + after_img = session.get_frame_at(t_after, tolerance=FRAME_TOLERANCE_S) if event["kind"] in ("click", "double_click") and before_img is None: raise ValueError( diff --git a/openadapt_flow/backends/parallels_vm.py b/openadapt_flow/backends/parallels_vm.py index 6db05ac..b160d83 100644 --- a/openadapt_flow/backends/parallels_vm.py +++ b/openadapt_flow/backends/parallels_vm.py @@ -40,8 +40,7 @@ DEFAULT_VM_UUID = "{d4f9c29a-52e1-4793-9334-7e971c3d0ab3}" DEFAULT_PRLCTL = "/usr/local/bin/prlctl" SHIM_PORT = 5000 -_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", - "desktop") +_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", "desktop") # In-guest install locations (forward slashes: Python + curl accept them and # they survive the host shell without backslash mangling). GUEST_DIR = "C:/oa" @@ -137,8 +136,9 @@ def resume(self) -> None: def set_pause_idle(self, on: bool) -> None: """Toggle Parallels' pause-when-idle (must be OFF for headless runs).""" - self._run(["set", self.uuid, "--pause-idle", "on" if on else "off"], - check=False) + self._run( + ["set", self.uuid, "--pause-idle", "on" if on else "off"], check=False + ) def ensure_running(self, *, settle_s: float = 6.0) -> None: """Bring the VM to a running state from any state, idempotently. @@ -206,19 +206,21 @@ def exec( NOTE: ``prlctl exec`` hangs on very long single arguments — keep commands short and move file payloads with :meth:`push_file`. """ - return self._run(["exec", self.uuid, *args], timeout=timeout, - check=check) + return self._run(["exec", self.uuid, *args], timeout=timeout, check=check) - def exec_cmd(self, cmdline: str, *, timeout: float = 120.0 - ) -> subprocess.CompletedProcess: + def exec_cmd( + self, cmdline: str, *, timeout: float = 120.0 + ) -> subprocess.CompletedProcess: """Run ``cmd /c `` in-guest (quoting preserved by cmd).""" return self.exec(["cmd", "/c", cmdline], timeout=timeout) - def exec_ps(self, script: str, *, timeout: float = 120.0 - ) -> subprocess.CompletedProcess: + def exec_ps( + self, script: str, *, timeout: float = 120.0 + ) -> subprocess.CompletedProcess: """Run a short PowerShell command in-guest.""" - return self.exec(["powershell", "-NoProfile", "-Command", script], - timeout=timeout) + return self.exec( + ["powershell", "-NoProfile", "-Command", script], timeout=timeout + ) # -- host-side capture --------------------------------------------------- @@ -297,14 +299,12 @@ def push_file( def _shim_paths(self) -> tuple[str, str]: shim = os.path.abspath(os.path.join(_SCRIPT_DIR, "waa_shim.py")) - launcher = os.path.abspath(os.path.join(_SCRIPT_DIR, - "session1_launch.py")) + launcher = os.path.abspath(os.path.join(_SCRIPT_DIR, "session1_launch.py")) return shim, launcher def kill_shim(self) -> None: """Kill any in-guest Python (frees the shim port).""" - self.exec_cmd("taskkill /F /IM python.exe /IM pythonw.exe 2>nul & " - "echo done") + self.exec_cmd("taskkill /F /IM python.exe /IM pythonw.exe 2>nul & echo done") def launch_shim( self, @@ -323,18 +323,35 @@ def launch_shim( shim, launcher = self._shim_paths() self.exec_cmd(f"if not exist {GUEST_DIR} mkdir {GUEST_DIR}") self.push_file(shim, f"{GUEST_DIR}/waa_shim.py", host_ip=host_ip) - self.push_file(launcher, f"{GUEST_DIR}/session1_launch.py", - host_ip=host_ip) - self.exec(["netsh", "advfirewall", "firewall", "add", "rule", - "name=OAShim", "dir=in", "action=allow", "protocol=TCP", - f"localport={port}"]) + self.push_file(launcher, f"{GUEST_DIR}/session1_launch.py", host_ip=host_ip) + self.exec( + [ + "netsh", + "advfirewall", + "firewall", + "add", + "rule", + "name=OAShim", + "dir=in", + "action=allow", + "protocol=TCP", + f"localport={port}", + ] + ) self.kill_shim() time.sleep(2) # Run the launcher as SYSTEM; it CreateProcessAsUser's the shim into # the interactive console session so mss/pyautogui address the real # desktop. Forward-slash script paths dodge host-shell mangling. - self.exec([self.python_guest, f"{GUEST_DIR}/session1_launch.py", - f"{GUEST_DIR}/waa_shim.py", "--port", str(port)]) + self.exec( + [ + self.python_guest, + f"{GUEST_DIR}/session1_launch.py", + f"{GUEST_DIR}/waa_shim.py", + "--port", + str(port), + ] + ) url = f"http://{self.guest_ip()}:{port}" deadline = time.time() + wait_s while time.time() < deadline: diff --git a/openadapt_flow/backends/rdp_backend.py b/openadapt_flow/backends/rdp_backend.py index 204b7f4..b6474d0 100644 --- a/openadapt_flow/backends/rdp_backend.py +++ b/openadapt_flow/backends/rdp_backend.py @@ -236,9 +236,7 @@ def screenshot(self) -> bytes: img.convert("RGB").save(buf, format="PNG") return buf.getvalue() - def wait_first_frame( - self, *, retries: int = 20, settle_s: float = 0.25 - ) -> bytes: + def wait_first_frame(self, *, retries: int = 20, settle_s: float = 0.25) -> bytes: """Poll :meth:`screenshot` until a non-blank frame, returning its PNG. The first frame(s) an RDP session paints are often a single-colour diff --git a/openadapt_flow/backends/windows_backend.py b/openadapt_flow/backends/windows_backend.py index 104669d..ef87774 100644 --- a/openadapt_flow/backends/windows_backend.py +++ b/openadapt_flow/backends/windows_backend.py @@ -398,9 +398,7 @@ def press(self, key: str) -> None: """Press a key or chord, e.g. ``'Enter'`` or ``'ControlOrMeta+a'``.""" keys = normalize_chord(key) if len(keys) == 1: - self._execute( - f"import pyautogui; pyautogui.press({keys[0]!r})" - ) + self._execute(f"import pyautogui; pyautogui.press({keys[0]!r})") else: args = ", ".join(repr(k) for k in keys) self._execute(f"import pyautogui; pyautogui.hotkey({args})") diff --git a/openadapt_flow/bench.py b/openadapt_flow/bench.py index 5fee445..22e654f 100644 --- a/openadapt_flow/bench.py +++ b/openadapt_flow/bench.py @@ -45,9 +45,7 @@ def _run_one( run_dir: Path, ) -> RunReport: replayer = replayer_cls(backend) - return replayer.run( - workflow, params=params, bundle_dir=bundle_dir, run_dir=run_dir - ) + return replayer.run(workflow, params=params, bundle_dir=bundle_dir, run_dir=run_dir) def run_bench( diff --git a/openadapt_flow/benchmark/agent_baseline.py b/openadapt_flow/benchmark/agent_baseline.py index 36c0ee2..4882d95 100644 --- a/openadapt_flow/benchmark/agent_baseline.py +++ b/openadapt_flow/benchmark/agent_baseline.py @@ -153,12 +153,12 @@ def triage_task_prompt(note_text: str) -> str: return ( "You are looking at MockMed, a demo clinic web app (fake data " "only). Complete this task:\n\n" - "1. Sign in with username \"nurse.demo\" and password " - "\"mockmed-demo-pass\".\n" + '1. Sign in with username "nurse.demo" and password ' + '"mockmed-demo-pass".\n' "2. Open the first referral task in the list.\n" "3. From the patient's page, create a New Encounter and choose the " - "type \"Triage\".\n" - f"4. Enter exactly this note in the Note field: \"{note_text}\"\n" + 'type "Triage".\n' + f'4. Enter exactly this note in the Note field: "{note_text}"\n' "5. Save the encounter.\n\n" "You are done when you are back on the patient's page and see the " "'Encounter saved' confirmation. Then stop and reply with a one-line " @@ -183,14 +183,14 @@ def openemr_task_prompt(note_text: str) -> str: return ( "You are looking at the OpenEMR public demo (a real EMR web app " "with fake demo patients only). Complete this task:\n\n" - "1. Sign in with username \"admin\" and password \"pass\".\n" + '1. Sign in with username "admin" and password "pass".\n' "2. Use the patient search box in the top bar to search for " - "\"Phil\" and open the chart of the patient \"Belford, Phil\".\n" + '"Phil" and open the chart of the patient "Belford, Phil".\n' "3. On the patient's dashboard, open the Patient Messages section " "(the Messages card — you will likely need to scroll down to " "find it).\n" "4. Add a new note and enter exactly this text as the note: " - f"\"{note_text}\"\n" + f'"{note_text}"\n' "5. Save it as a new message.\n\n" "You are done when you are back on the patient-message list and " "can see the new note. Then stop and reply with a one-line " @@ -464,9 +464,7 @@ def error(message: str) -> dict[str, Any]: } -def _truncate_screenshots( - messages: list[dict[str, Any]], keep: int -) -> None: +def _truncate_screenshots(messages: list[dict[str, Any]], keep: int) -> None: """Replace all but the last ``keep`` screenshot blocks with text stubs. Walks the ``tool_result`` blocks of user messages (the only place this @@ -648,9 +646,7 @@ def run_agent( ) continue actions += 1 - action_log.append( - f"{actions}: {dict(block.input) if block.input else {}}" - ) + action_log.append(f"{actions}: {dict(block.input) if block.input else {}}") results.append(_execute_action(backend, block)) messages.append({"role": "user", "content": results}) _truncate_screenshots(messages, keep_screenshots) diff --git a/openadapt_flow/benchmark/chart_fonts.py b/openadapt_flow/benchmark/chart_fonts.py index b261e5d..2067415 100644 --- a/openadapt_flow/benchmark/chart_fonts.py +++ b/openadapt_flow/benchmark/chart_fonts.py @@ -45,9 +45,7 @@ def configure_bundled_font() -> Any: matplotlib.use("Agg") from matplotlib import font_manager - font_path = ( - Path(matplotlib.get_data_path()) / "fonts" / "ttf" / "DejaVuSans.ttf" - ) + font_path = Path(matplotlib.get_data_path()) / "fonts" / "ttf" / "DejaVuSans.ttf" if font_path.is_file(): try: font_manager.fontManager.addfont(str(font_path)) @@ -57,8 +55,7 @@ def configure_bundled_font() -> Any: import matplotlib.pyplot as plt existing = [ - f for f in plt.rcParams.get("font.sans-serif", []) - if f != BUNDLED_FONT_NAME + f for f in plt.rcParams.get("font.sans-serif", []) if f != BUNDLED_FONT_NAME ] plt.rcParams["font.family"] = "sans-serif" plt.rcParams["font.sans-serif"] = [BUNDLED_FONT_NAME, *existing] diff --git a/openadapt_flow/benchmark/desktop_benchmark.py b/openadapt_flow/benchmark/desktop_benchmark.py index 3d4699c..d96f03b 100644 --- a/openadapt_flow/benchmark/desktop_benchmark.py +++ b/openadapt_flow/benchmark/desktop_benchmark.py @@ -78,15 +78,12 @@ # safe-halt there is a FALSE ABORT; data-drift conditions genuinely change the # list, so a halt there can be correct caution. CONDITIONS: dict[str, dict] = { - "clean": {"cfg": {}, "drift": "none", "cosmetic": True}, - "render_125": {"cfg": {"font_scale": 1.25}, "drift": "none", - "cosmetic": True}, - "render_150": {"cfg": {"font_scale": 1.5}, "drift": "none", - "cosmetic": True}, - "theme_dark": {"cfg": {"theme": "dark"}, "drift": "none", - "cosmetic": True}, - "data_reorder": {"cfg": {}, "drift": "reorder", "cosmetic": False}, - "data_decoy": {"cfg": {}, "drift": "decoy", "cosmetic": False}, + "clean": {"cfg": {}, "drift": "none", "cosmetic": True}, + "render_125": {"cfg": {"font_scale": 1.25}, "drift": "none", "cosmetic": True}, + "render_150": {"cfg": {"font_scale": 1.5}, "drift": "none", "cosmetic": True}, + "theme_dark": {"cfg": {"theme": "dark"}, "drift": "none", "cosmetic": True}, + "data_reorder": {"cfg": {}, "drift": "reorder", "cosmetic": False}, + "data_decoy": {"cfg": {}, "drift": "decoy", "cosmetic": False}, "data_siblings": {"cfg": {}, "drift": "siblings", "cosmetic": False}, } @@ -98,6 +95,7 @@ # --- result rows ------------------------------------------------------------ + @dataclass class RunRow: """One arm x condition x repeat outcome, judged by the DB.""" @@ -105,10 +103,10 @@ class RunRow: arm: str condition: str i: int - outcome: str = "error" # success|wrong_action|safe_halt|miss|error + outcome: str = "error" # success|wrong_action|safe_halt|miss|error wrong_action: bool = False false_abort: bool = False - completed: bool = False # arm ran to its end without halting + completed: bool = False # arm ran to its end without halting target_note_ok: bool = False wrong_patient_id: Optional[int] = None # compiled-arm identity telemetry @@ -128,6 +126,7 @@ class RunRow: # --- live harness (requires a VM; imported lazily by the orchestrator) ------ + class DesktopHarness: """Drives one Parallels VM through the full desktop pipeline. @@ -151,7 +150,8 @@ def connect( log: Callable = print, ) -> "DesktopHarness": from openadapt_flow.backends.parallels_vm import ( - DEFAULT_VM_UUID, ParallelsVM, + DEFAULT_VM_UUID, + ParallelsVM, ) vm = ParallelsVM(vm_uuid or DEFAULT_VM_UUID) @@ -178,8 +178,9 @@ def quiet_desktop(self) -> None: "'/d','0','/f'],capture_output=True)\n" ) try: - requests.post(f"{self.shim_url}/execute_windows", - json={"command": cmd}, timeout=20) + requests.post( + f"{self.shim_url}/execute_windows", json={"command": cmd}, timeout=20 + ) except Exception: # noqa: BLE001 pass @@ -192,13 +193,11 @@ def deploy_app_scripts(self) -> None: src = Path(pv._SCRIPT_DIR) self.vm.exec_cmd(f"if not exist {GUEST_DIR} mkdir {GUEST_DIR}") for name in self._APP_SCRIPTS: - self.vm.push_file(str((src / name).resolve()), - f"{GUEST_DIR}/{name}") + self.vm.push_file(str((src / name).resolve()), f"{GUEST_DIR}/{name}") # -- DB ground truth -- def _py(self, *args: str): - return self.vm.exec([GUEST_PY, f"{GUEST_DIR}/pn_db.py", *args], - timeout=60) + return self.vm.exec([GUEST_PY, f"{GUEST_DIR}/pn_db.py", *args], timeout=60) def seed(self, drift: str = "none") -> None: self._py("seed", "--drift", drift) @@ -214,8 +213,8 @@ def db_all(self) -> list[dict]: # -- app lifecycle -- def write_cfg(self, cfg: dict) -> None: """Write pn_env.json (drift knobs) into the guest via HTTP push.""" - import tempfile import os + import tempfile tmp = Path(tempfile.mkdtemp()) / "pn_env.json" tmp.write_text(json.dumps(cfg)) @@ -225,13 +224,17 @@ def write_cfg(self, cfg: dict) -> None: def stop_app(self) -> None: self.vm.exec_cmd("taskkill /F /IM powershell.exe 2>nul & echo ok") - def launch_app(self, cfg: Optional[dict] = None, *, settle_s: float = 6.0 - ) -> None: + def launch_app(self, cfg: Optional[dict] = None, *, settle_s: float = 6.0) -> None: self.stop_app() time.sleep(1) self.write_cfg(cfg or {}) - self.vm.exec([GUEST_PY, f"{GUEST_DIR}/session1_launch.py", - f"{GUEST_DIR}/patient_notes.ps1"]) + self.vm.exec( + [ + GUEST_PY, + f"{GUEST_DIR}/session1_launch.py", + f"{GUEST_DIR}/patient_notes.ps1", + ] + ) time.sleep(settle_s) def prepare_condition(self, condition: str) -> None: @@ -253,7 +256,10 @@ def rects(self) -> dict: for n in u.get("nodes", []): nums = list(map(int, re.findall(r"-?\d+", n["rect"]))) if len(nums) >= 4 and n["automation_id"] in ( - "searchBox", "noteBox", "saveButton", "patientGrid" + "searchBox", + "noteBox", + "saveButton", + "patientGrid", ): out[n["automation_id"]] = tuple(nums[:4]) return out @@ -302,8 +308,7 @@ def record_and_compile(self, work_dir: Path) -> Path: # surname leaves the given name + DOB in the identity band -- exactly # what distinguishes siblings (Sorenson vs Sorensen, different DOB). gL, gT, gR, gB = c["patientGrid"] - row0 = self.data_cell_center("last", 0) or ( - gL + int((gR - gL) * 0.35), gT + 62) + row0 = self.data_cell_center("last", 0) or (gL + int((gR - gL) * 0.35), gT + 62) rec = work_dir / "recording" r = Recorder(self.backend, rec) @@ -351,15 +356,17 @@ def uia_run(self, mode: str, note: str) -> dict: cmd = ( "import json,importlib.util\n" "spec=importlib.util.spec_from_file_location('uia_arm', " - r"r'C:\oa\uia_arm.py')" "\n" + r"r'C:\oa\uia_arm.py')" + "\n" "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)\n" f"res=m.run({DEMO['search']!r},{DEMO['first']!r},{DEMO['last']!r}," f"{note!r},{mode!r})\n" r"open(r'C:\oa\uia_result.json','w',encoding='utf-8')" ".write(json.dumps(res))\n" ) - requests.post(f"{self.shim_url}/execute_windows", - json={"command": cmd}, timeout=60) + requests.post( + f"{self.shim_url}/execute_windows", json={"command": cmd}, timeout=60 + ) out = self.vm.exec_cmd(r"type C:\oa\uia_result.json").stdout.strip() return json.loads(out) if out else {"status": "error"} @@ -372,8 +379,11 @@ def judge(self, note: str) -> dict: rows = self.db_all() target = next((r for r in rows if r["id"] == DEMO["target_id"]), {}) target_ok = target.get("note", "") == note - wrongs = [r for r in rows - if r["id"] != DEMO["target_id"] and r.get("note", "") == note] + wrongs = [ + r + for r in rows + if r["id"] != DEMO["target_id"] and r.get("note", "") == note + ] return { "target_note_ok": target_ok, "wrong_patient_id": wrongs[0]["id"] if wrongs else None, @@ -383,6 +393,7 @@ def judge(self, note: str) -> dict: # --- orchestrator ----------------------------------------------------------- + def _classify(judged: dict, completed: bool, cosmetic: bool) -> tuple[str, bool]: """Map (DB verdict, completion) to an outcome + false-abort flag.""" if judged["wrong_action"]: @@ -462,8 +473,7 @@ def run_desktop_benchmark( row.identity_mismatch = res["identity"]["mismatch"] row.identity_unreadable = res["identity"]["unreadable"] else: - mode = ("identity" if arm == "uia_identity" - else "positional") + mode = "identity" if arm == "uia_identity" else "positional" res = harness.uia_run(mode, note) row.selected_index = res.get("selected_index") row.selected_name = res.get("selected_name") @@ -480,10 +490,12 @@ def run_desktop_benchmark( row.outcome = "error" row.wall_s = round(time.time() - t0, 2) rows.append(row) - log(f"[bench] {arm:15s} {condition:13s} #{i} -> " + log( + f"[bench] {arm:15s} {condition:13s} #{i} -> " f"{row.outcome}" + (" WRONG-ACTION" if row.wrong_action else "") - + (" false-abort" if row.false_abort else "")) + + (" false-abort" if row.false_abort else "") + ) results = _aggregate(rows, wf_armed, tree_quality, conditions, arms) write_outputs(results, out_dir) @@ -494,8 +506,7 @@ def _armed_coverage(bundle: Path) -> dict: from openadapt_flow.ir import Workflow wf = Workflow.load(bundle) - clicks = [s for s in wf.steps - if s.action.value in ("click", "double_click")] + clicks = [s for s in wf.steps if s.action.value in ("click", "double_click")] armed = [s for s in clicks if s.anchor and s.anchor.context_text] return { "click_steps": len(clicks), @@ -519,11 +530,12 @@ def _aggregate(rows, armed, tree_quality, conditions, arms) -> dict: "miss": sum(r.outcome == "miss" for r in arm_rows), "error": sum(r.outcome == "error" for r in arm_rows), "success_rate": round( - sum(r.outcome == "success" for r in arm_rows) / max(1, n), 3), + sum(r.outcome == "success" for r in arm_rows) / max(1, n), 3 + ), "wrong_action_rate": round( - sum(r.wrong_action for r in arm_rows) / max(1, n), 3), - "wall_s_mean": round( - sum(r.wall_s for r in arm_rows) / max(1, n), 2), + sum(r.wrong_action for r in arm_rows) / max(1, n), 3 + ), + "wall_s_mean": round(sum(r.wall_s for r in arm_rows) / max(1, n), 2), } # per arm x condition outcome matrix matrix: dict[str, dict] = {} @@ -541,11 +553,11 @@ def _aggregate(rows, armed, tree_quality, conditions, arms) -> dict: return { "generated_at": datetime.now(timezone.utc).isoformat(), "task": "Patient Notes (WinForms) search -> select -> note -> save; " - "DB-ground-truth judge; $0 (no model calls)", + "DB-ground-truth judge; $0 (no model calls)", "substrate": "Parallels Windows 11 ARM VM on Apple M2 Max; " - "WindowsBackend over in-guest WAA HTTP shim (session 1)", + "WindowsBackend over in-guest WAA HTTP shim (session 1)", "target_app_note": "WinForms substitute for OpenDental (trial not " - "no-touch installable; see PHASE2.md).", + "no-touch installable; see PHASE2.md).", "identity_armed_coverage": armed, "uia_tree_quality": tree_quality, "arms": by_arm, @@ -557,6 +569,7 @@ def _aggregate(rows, armed, tree_quality, conditions, arms) -> dict: # --- output writers --------------------------------------------------------- + def render_markdown(results: dict) -> str: a = results["arms"] lines = [ @@ -590,10 +603,10 @@ def render_markdown(results: dict) -> str: "## Identity transfer to desktop-rendered text", "", f"- Compiled-arm **armed coverage**: " - f"{ic.get('armed_clicks','?')}/{ic.get('click_steps','?')} click steps " + f"{ic.get('armed_clicks', '?')}/{ic.get('click_steps', '?')} click steps " f"carry an identity band ({ic.get('armed_coverage', 0):.0%}).", f"- UIA-tree quality: " - f"{tq.get('n_usable_id','?')}/{tq.get('n_targets','?')} workflow " + f"{tq.get('n_usable_id', '?')}/{tq.get('n_targets', '?')} workflow " f"targets expose a usable AutomationId " f"({tq.get('usable_fraction', 0):.0%}); the identity-critical patient " f"row does **not** " @@ -715,8 +728,7 @@ def render_chart(results: dict, path: Path) -> None: succ = [results["arms"][a]["success"] for a in arms] wrong = [results["arms"][a]["wrong_action"] for a in arms] halt = [results["arms"][a]["safe_halt"] for a in arms] - miss = [results["arms"][a]["miss"] + results["arms"][a]["error"] - for a in arms] + miss = [results["arms"][a]["miss"] + results["arms"][a]["error"] for a in arms] fig, ax = plt.subplots(figsize=(8, 4.5)) bottom = [0] * len(arms) for label, vals, color in [ @@ -756,6 +768,8 @@ def write_outputs(results: dict, out_dir: str | Path) -> None: args = ap.parse_args() conds = args.conditions.split(",") if args.conditions else None run_desktop_benchmark( - args.out, n_per=args.n, conditions=conds, + args.out, + n_per=args.n, + conditions=conds, arms=tuple(args.arms.split(",")), ) diff --git a/openadapt_flow/benchmark/dom_arm.py b/openadapt_flow/benchmark/dom_arm.py index 120604d..a6c0e59 100644 --- a/openadapt_flow/benchmark/dom_arm.py +++ b/openadapt_flow/benchmark/dom_arm.py @@ -98,13 +98,20 @@ def note_for_slot(arm: str, slot: int) -> str: tag = _ARM_TAGS.get(arm, arm[:1].upper()) return f"{_NOTE_POOL[slot % len(_NOTE_POOL)]} [{tag}{slot:02d}]" + #: The perturbation drift modes from the validation suite (PR #12/#13), #: plus ``sort`` (a changed default sort order — every referral present, #: the recorded target no longer first) and ``typelabel`` (Triage segment #: relabeled AND reordered), both flag-gated in the MockMed app. PERTURBATIONS: tuple[str, ...] = ( - "lookalike", "missing", "grow", "sort", - "theme", "rename", "move", "typelabel", + "lookalike", + "missing", + "grow", + "sort", + "theme", + "rename", + "move", + "typelabel", ) # -- the DOM-selector script (the steelman) ------------------------------------ @@ -174,8 +181,7 @@ def dom_script( # cannot tell. Measured, not assumed. open_step = ( "open first referral", - lambda: page.get_by_role( - "button", name="Open").first.click(), + lambda: page.get_by_role("button", name="Open").first.click(), ) else: # Identity reading: scope the Open button to the row whose @@ -186,46 +192,53 @@ def dom_script( # the compiled arm's recorded identity band carries. open_step = ( "open referral by patient name", - lambda: page.get_by_role("row", name=target_patient) - .get_by_role("button", name="Open") - .click(), + lambda: ( + page.get_by_role("row", name=target_patient) + .get_by_role("button", name="Open") + .click() + ), ) return [ # Form fields by their explicit