diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..41f0ea7 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,12 @@ +# Default owner for everything in the repo. +* @abrichr + +# Safety-critical core: identity gate, resolution ladder, halt/postcondition +# logic. Changes here warrant extra scrutiny (never-false-accept invariant). +/openadapt_flow/runtime/identity.py @abrichr +/openadapt_flow/runtime/replayer.py @abrichr +/openadapt_flow/validation/ @abrichr + +# Release, CI, and supply-chain configuration. +/.github/ @abrichr +/pyproject.toml @abrichr diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..be6258e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,46 @@ +name: Bug report +description: Report something that behaves incorrectly. +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for the report. Please do **not** file security issues here — + see [SECURITY.md](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/SECURITY.md). + - type: textarea + id: what-happened + attributes: + label: What happened? + description: What did you expect, and what happened instead? + validations: + required: true + - type: textarea + id: repro + attributes: + label: Steps to reproduce + description: Ideally the exact `openadapt-flow` commands you ran. + placeholder: | + 1. openadapt-flow demo-record --out rec + 2. openadapt-flow compile rec --out bundle --name my-task + 3. openadapt-flow replay bundle + validations: + required: true + - type: textarea + id: report + attributes: + label: Run report / logs + description: Attach the run's REPORT.md or report.json if you have one (scrub any PHI/PII first). + - type: input + id: version + attributes: + label: openadapt-flow version + description: "Output of: python -c \"import openadapt_flow; print(openadapt_flow.__version__)\"" + validations: + required: true + - type: input + id: env + attributes: + label: OS + Python version + placeholder: "macOS 14 / Python 3.12" + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3b57029 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Security vulnerability + url: https://github.com/OpenAdaptAI/openadapt-flow/security/advisories/new + about: Please report security issues privately, not as public issues (see SECURITY.md). + - name: Question / discussion + url: https://github.com/OpenAdaptAI/openadapt-flow/discussions + about: Ask usage questions or discuss ideas here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..7b18a47 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,21 @@ +name: Feature request +description: Suggest an improvement or new capability. +labels: ["enhancement"] +body: + - type: textarea + id: problem + attributes: + label: Problem / use case + description: What are you trying to do, and what makes it hard today? + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed solution + description: What would you like to see? Alternatives you considered? + - type: textarea + id: context + attributes: + label: Additional context + description: Substrate (browser/desktop/RDP), scale, determinism/safety needs, etc. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..cdb6cfd --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,27 @@ +## What & why + + + +## Type of change + + + +- [ ] `fix:` bug fix +- [ ] `feat:` new feature +- [ ] `docs:` documentation only +- [ ] `ci:` / `chore:` / `refactor:` / `test:` (no user-facing behavior change) + +## Checklist + +- [ ] PR title uses Conventional Commit format +- [ ] `ruff check openadapt_flow` and `ruff format --check openadapt_flow` pass +- [ ] `mypy` passes +- [ ] `pytest -q` passes locally +- [ ] Tests added/updated for behavior changes +- [ ] Docs updated (README/DESIGN/docs) if behavior or contracts changed +- [ ] If this touches the identity gate / resolution ladder / halt logic, I + explained why the never-false-accept invariant still holds + +## Notes for reviewers + + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..df2b6d0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,27 @@ +# Dependabot: keep the supply chain fresh. GitHub Actions are pinned by commit +# SHA (see the workflows); Dependabot understands the trailing `# vX` comment +# and proposes SHA bumps with the new version, so pinning does not freeze us on +# stale, potentially-vulnerable action releases. +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] + commit-message: + prefix: ci + + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + # Group the minor/patch churn so we get one PR instead of a swarm; majors + # come as their own reviewable PRs. + groups: + python-minor: + update-types: ["minor", "patch"] + commit-message: + prefix: build diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a47fc9..aba8d3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,35 @@ concurrency: # version. Dependabot (.github/dependabot.yml) proposes SHA bumps. jobs: + # --- Lint + type check (ruff + mypy) ------------------------------------- + # Runs on PRs (and post-merge/nightly). Deliberately a SEPARATE job from the + # required `test` gate: it must NOT be wired as a dependency of `test` (that + # would leave `test` reporting a compound context and break branch + # protection, whose required context is exactly `test`). Add `lint` as its + # own required context in branch protection if you want it to block merges. + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install + run: pip install -e .[dev] + + - name: Ruff lint + run: ruff check openadapt_flow + + - name: Ruff format check + run: ruff format --check openadapt_flow tests + + - name: Mypy + run: mypy + # --- Required merge gate: FAST unit suite (ubuntu, single Python) --------- # This is the ONE required status check (branch protection context: "test"). # It MUST stay a single, non-matrix job named exactly `test` so its reported diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f1de543..76aa425 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,12 @@ name: Release and PyPI Publish # Prerequisites (repo/org settings): secrets.ADMIN_TOKEN (org-level, visibility # "all") to push the release commit/tag past branch protection; PyPI Trusted # Publishing for openadapt-flow with environment "pypi". +# +# Supply chain: GitHub Actions below are pinned to full commit SHAs (not mutable +# tags) with the human-readable version in a trailing comment; Dependabot +# (.github/dependabot.yml) proposes SHA bumps. Follow-up (settings change, not +# code): the org-wide ADMIN_TOKEN (visibility "all") should become a +# repo-scoped fine-grained token or GitHub App to narrow its blast radius. on: push: @@ -35,7 +41,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 token: ${{ secrets.ADMIN_TOKEN }} @@ -51,13 +57,13 @@ jobs: - name: Set up Python if: steps.check_skip.outputs.skip != 'true' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install uv if: steps.check_skip.outputs.skip != 'true' - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 # Semantic Release computes the version, bumps pyproject, tags, AND runs # `build_command` (uv build) to populate dist/. There is deliberately no @@ -65,17 +71,17 @@ jobs: - name: Python Semantic Release if: steps.check_skip.outputs.skip != 'true' id: release - uses: python-semantic-release/python-semantic-release@v9.15.2 + uses: python-semantic-release/python-semantic-release@7b3f71697ccfbaef884e1e754b6364e974b134cf # v9.15.2 with: github_token: ${{ secrets.ADMIN_TOKEN }} - name: Publish to PyPI if: steps.check_skip.outputs.skip != 'true' && steps.release.outputs.released == 'true' - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 - name: Publish GitHub Release if: steps.check_skip.outputs.skip != 'true' && steps.release.outputs.released == 'true' - uses: python-semantic-release/publish-action@v9.15.2 + uses: python-semantic-release/publish-action@b9c41d4b0754dee5a6c7188d42b33f66e3a8aafd # v9.15.2 with: github_token: ${{ secrets.ADMIN_TOKEN }} @@ -89,20 +95,20 @@ jobs: steps: - name: Checkout the requested ref - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ inputs.ref }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 - name: Build package run: uv build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..9deb1e5 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,67 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior: + +- The use of sexualized language or imagery, and sexual attention or advances of + any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement privately through +a [GitHub Security Advisory](https://github.com/OpenAdaptAI/openadapt-flow/security/advisories/new) +or by contacting the maintainer [@abrichr](https://github.com/abrichr). +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. + +[homepage]: https://www.contributor-covenant.org diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4945ce8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,64 @@ +# Contributing to openadapt-flow + +Thanks for your interest in improving openadapt-flow. This project compiles a +recorded GUI demonstration into a deterministic, self-healing, locally-run +script — so correctness, determinism, and honest measurement matter more here +than raw feature count. + +## Development setup + +```bash +git clone https://github.com/OpenAdaptAI/openadapt-flow && cd openadapt-flow +python -m venv .venv && source .venv/bin/activate +pip install -e '.[dev]' +playwright install chromium +pytest -q +``` + +Python 3.10–3.13 are supported and exercised in CI. + +## The checks CI runs (run them locally first) + +```bash +ruff check openadapt_flow # lint +ruff format --check openadapt_flow # format (drop --check to auto-apply) +mypy # type-check (config in pyproject.toml) +pytest -q # tests +``` + +- **Lint/format:** `ruff`. Config lives in `[tool.ruff]` in `pyproject.toml`. +- **Types:** `mypy` runs on the core package (not tests). It is deliberately + lenient today; a set of modules with known type debt is listed under + `[[tool.mypy.overrides]]`. Improving a module's annotations and removing it + from that list is a very welcome PR. +- **Coverage:** CI reports coverage for visibility. There is no hard floor yet, + but new code should come with tests. + +## Pull request guidelines + +- **Conventional Commits** for titles and commits: `feat:`, `fix:`, `perf:`, + `docs:`, `ci:`, `chore:`, `refactor:`, `test:`. Releases are automated from + these — `feat:` → minor, `fix:`/`perf:` → patch, `BREAKING CHANGE` → major. +- Keep PRs focused. Separate mechanical changes (formatting, renames) from + behavior changes so review stays legible. +- Add or update tests for any behavior change. The suite mocks browsers/servers + where it can, so most of it runs with no live VM. +- Update docs (`README.md`, `DESIGN.md`, `docs/`) when behavior or contracts + change. We prefer honest, measured claims — if something is experimental, say + so. + +## Safety-sensitive areas + +The identity gate, the resolution ladder, and the postcondition/halt logic are +the safety core: the whole value proposition is that the tool halts instead of +acting on the wrong target. Changes there deserve extra tests (see the +`test_identity_*`, `test_resolver*`, and `*_fuzz` suites) and a clear +explanation of why the never-false-accept invariant still holds. + +## Reporting security issues + +See [SECURITY.md](SECURITY.md) — do not file security problems as public issues. + +## Code of Conduct + +This project follows the [Contributor Covenant](CODE_OF_CONDUCT.md). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e29b8d0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,42 @@ +# Security Policy + +openadapt-flow is often deployed next to sensitive systems (it can drive +clinical and other regulated desktop workflows, and its `privacy` extra scrubs +PHI/PII on the persist/log paths). We take security reports seriously. + +## Reporting a vulnerability + +**Please do not open a public issue for security problems.** + +Report privately through GitHub's built-in channel: + +1. Go to the repository's **Security** tab → **Advisories** → + **Report a vulnerability** ([direct link](https://github.com/OpenAdaptAI/openadapt-flow/security/advisories/new)). +2. Describe the issue, the impact, and a minimal reproduction if you have one. + +This opens a private advisory visible only to you and the maintainers. If you +are unable to use that channel, open a public issue that contains **no +details** and asks a maintainer to open a private channel with you. + +## What to expect + +- We aim to acknowledge a report within **5 business days**. +- We will confirm the issue, determine affected versions, and prepare a fix. +- We will credit reporters who wish to be credited once a fix is released. + +## Scope notes specific to this project + +- **PHI/PII handling.** The compiled bundle and `report.json` intentionally + retain literal identifiers (for the identity check and audit trail) and are + protected by a documented boundary — see [docs/PRIVACY.md](docs/PRIVACY.md). + A report that these are exposed *outside* that boundary is in scope. +- **Identity crops to the on-prem VLM appliance** are deliberately not scrubbed; + the control there is on-prem-only + no-retention. Reports of retention or + off-prem transmission are in scope. +- **Supply chain.** GitHub Actions are pinned by commit SHA and dependency + updates flow through Dependabot; reports of a pinning gap are welcome. + +## Supported versions + +We support the latest released version on PyPI. Fixes are shipped forward; there +is no long-term-support branch at this stage. diff --git a/openadapt_flow/__init__.py b/openadapt_flow/__init__.py index a96a4e6..24cc6ef 100644 --- a/openadapt_flow/__init__.py +++ b/openadapt_flow/__init__.py @@ -1,6 +1,6 @@ """openadapt-flow: record once, compile, replay deterministically, heal on drift.""" -__version__ = "0.1.0" +__version__ = "0.13.0" from openadapt_flow.ir import ( # noqa: F401 ActionKind, diff --git a/openadapt_flow/__main__.py b/openadapt_flow/__main__.py index 0f0a1bb..7004679 100644 --- a/openadapt_flow/__main__.py +++ b/openadapt_flow/__main__.py @@ -89,9 +89,7 @@ def _cmd_demo_record(args: argparse.Namespace) -> int: def _cmd_compile(args: argparse.Namespace) -> int: from openadapt_flow.compiler import compile_recording - workflow = compile_recording( - Path(args.recording), Path(args.out), name=args.name - ) + workflow = compile_recording(Path(args.recording), Path(args.out), name=args.name) print( f"Compiled {len(workflow.steps)} steps into {args.out} " f"(workflow: {workflow.name!r})" @@ -176,9 +174,7 @@ def _cmd_replay(args: argparse.Namespace) -> int: backend, grounder=grounder, identity_vlm=appliance.identity_vlm if appliance else None, - state_verifier=( - appliance.state_verifier if appliance else None - ), + state_verifier=(appliance.state_verifier if appliance else None), # Normal replay prefers the deterministic structural rung. # ``--drift`` exists to DEMONSTRATE the visual healing ladder # on the bundled MockMed app, so it forces the visual floor @@ -191,9 +187,7 @@ def _cmd_replay(args: argparse.Namespace) -> int: bundle_dir=bundle, run_dir=run_dir, save_healed_to=( - Path(args.save_healed_to) - if args.save_healed_to - else None + Path(args.save_healed_to) if args.save_healed_to else None ), ) finally: @@ -247,9 +241,7 @@ def backend_factory(): finally: stop() - report_md = render_bench_report( - run_root / "bench.json", run_root / "BENCH.md" - ) + report_md = render_bench_report(run_root / "bench.json", run_root / "BENCH.md") print( f"Bench: {result['success_count']}/{result['n']} succeeded " f"(p50 {result['total_ms_p50']:.0f} ms) — {report_md}" @@ -394,9 +386,7 @@ def build_parser() -> argparse.ArgumentParser: "record", help="Record YOUR app interactively in a headed browser (--url)", ) - p.add_argument( - "--url", required=True, help="URL of the app to record against" - ) + p.add_argument("--url", required=True, help="URL of the app to record against") p.add_argument("--out", required=True, help="Recording output directory") p.add_argument( "--secret", @@ -438,20 +428,12 @@ def build_parser() -> argparse.ArgumentParser: default="Follow-up in 2 weeks; BP recheck.", help="Note text typed during the demo (recorded as a parameter)", ) - p.add_argument( - "--param-name", default="note", help="Parameter name for the note" - ) - p.add_argument( - "--drift", default=None, help="Comma-separated MockMed drift modes" - ) - p.add_argument( - "--headed", action="store_true", help="Run the browser headed" - ) + p.add_argument("--param-name", default="note", help="Parameter name for the note") + p.add_argument("--drift", default=None, help="Comma-separated MockMed drift modes") + p.add_argument("--headed", action="store_true", help="Run the browser headed") p.set_defaults(func=_cmd_demo_record) - p = sub.add_parser( - "compile", help="Compile a recording into a workflow bundle" - ) + p = sub.add_parser("compile", help="Compile a recording into a workflow bundle") p.add_argument("recording", help="Recording directory") p.add_argument("--out", required=True, help="Output bundle directory") p.add_argument("--name", required=True, help="Workflow name") @@ -468,10 +450,7 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument( "--url", default=None, - help=( - "URL of the target app (default: serve the bundled MockMed " - "demo app)" - ), + help=("URL of the target app (default: serve the bundled MockMed demo app)"), ) p.add_argument( "--drift", @@ -501,9 +480,7 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Write the healed bundle to this directory", ) - p.add_argument( - "--headed", action="store_true", help="Run the browser headed" - ) + p.add_argument("--headed", action="store_true", help="Run the browser headed") p.set_defaults(func=_cmd_replay) p = sub.add_parser( @@ -516,18 +493,14 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Comma-separated drift modes forwarded to the MockMed URL", ) - p.add_argument( - "--run-root", required=True, help="Directory for per-iteration runs" - ) + p.add_argument("--run-root", required=True, help="Directory for per-iteration runs") p.add_argument( "--param", action="append", metavar="K=V", help="Parameter substitution (repeatable)", ) - p.add_argument( - "--headed", action="store_true", help="Run the browser headed" - ) + p.add_argument("--headed", action="store_true", help="Run the browser headed") p.set_defaults(func=_cmd_bench) p = sub.add_parser( @@ -544,9 +517,7 @@ def build_parser() -> argparse.ArgumentParser: default=100, help="Compiled-replay iterations", ) - p.add_argument( - "--n-agent", type=int, default=20, help="Agent iterations" - ) + p.add_argument("--n-agent", type=int, default=20, help="Agent iterations") p.add_argument( "--out", default="benchmark/", @@ -557,9 +528,7 @@ def build_parser() -> argparse.ArgumentParser: default="Follow-up in 2 weeks; BP recheck.", help="Note text both arms enter", ) - p.add_argument( - "--headed", action="store_true", help="Run the browsers headed" - ) + p.add_argument("--headed", action="store_true", help="Run the browsers headed") p.set_defaults(func=_cmd_benchmark) p = sub.add_parser( @@ -588,9 +557,7 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument( "--policy", required=True, - help=( - "Policy YAML path, or a built-in name (permissive, clinical-write)" - ), + help=("Policy YAML path, or a built-in name (permissive, clinical-write)"), ) p.set_defaults(func=_cmd_certify) @@ -618,22 +585,14 @@ def build_parser() -> argparse.ArgumentParser: ) p.set_defaults(func=_cmd_disambiguate) - p = sub.add_parser( - "emit-skill", help="Emit an Agent Skills folder for a bundle" - ) + p = sub.add_parser("emit-skill", help="Emit an Agent Skills folder for a bundle") p.add_argument("bundle", help="Workflow bundle directory") - p.add_argument( - "--out", required=True, help="Parent directory for the skill folder" - ) + p.add_argument("--out", required=True, help="Parent directory for the skill folder") p.set_defaults(func=_cmd_emit_skill) - p = sub.add_parser( - "emit-mcp", help="Emit a standalone MCP server.py for a bundle" - ) + p = sub.add_parser("emit-mcp", help="Emit a standalone MCP server.py for a bundle") p.add_argument("bundle", help="Workflow bundle directory") - p.add_argument( - "--out", required=True, help="Path for the generated server.py" - ) + p.add_argument("--out", required=True, help="Path for the generated server.py") p.set_defaults(func=_cmd_emit_mcp) return parser diff --git a/openadapt_flow/adapters/capture.py b/openadapt_flow/adapters/capture.py index 3d19270..141cdaa 100644 --- a/openadapt_flow/adapters/capture.py +++ b/openadapt_flow/adapters/capture.py @@ -128,16 +128,28 @@ # Bare modifier presses carry no workflow meaning on their own (their effect is # only visible combined with another key). _MODIFIER_KEY_NAMES = { - "shift", "shift_l", "shift_r", - "ctrl", "ctrl_l", "ctrl_r", - "alt", "alt_l", "alt_r", "alt_gr", - "cmd", "cmd_l", "cmd_r", + "shift", + "shift_l", + "shift_r", + "ctrl", + "ctrl_l", + "ctrl_r", + "alt", + "alt_l", + "alt_r", + "alt_gr", + "cmd", + "cmd_l", + "cmd_r", "caps_lock", } # Modifiers that, combined with another key, form a shortcut/chord with no flow # equivalent (shift is excluded — shift+char is just a shifted character). _CHORD_MODIFIER_NAMES = _MODIFIER_KEY_NAMES - { - "shift", "shift_l", "shift_r", "caps_lock", + "shift", + "shift_l", + "shift_r", + "caps_lock", } @@ -293,9 +305,7 @@ def _convert_key_type( name = non_mods[0] mapped = _KEY_NAME_MAP.get(name.lower()) if mapped is None: - raise ValueError( - f"unmapped key {name!r} at t={ts:.3f}; extend _KEY_NAME_MAP" - ) + raise ValueError(f"unmapped key {name!r} at t={ts:.3f}; extend _KEY_NAME_MAP") flush_text() events.append({"kind": "key", "key": mapped, "_ts": ts}) @@ -382,9 +392,7 @@ def convert_capture( t_after = ts + settle_s if i + 1 < len(events): t_after = min(t_after, float(events[i + 1]["_ts"])) - after_img = session.get_frame_at( - t_after, tolerance=FRAME_TOLERANCE_S - ) + after_img = session.get_frame_at(t_after, tolerance=FRAME_TOLERANCE_S) if event["kind"] in ("click", "double_click") and before_img is None: raise ValueError( diff --git a/openadapt_flow/backend.py b/openadapt_flow/backend.py index b4fd0d9..0f690d3 100644 --- a/openadapt_flow/backend.py +++ b/openadapt_flow/backend.py @@ -8,7 +8,7 @@ from __future__ import annotations -from typing import Any, Optional, Protocol, TYPE_CHECKING, runtime_checkable +from typing import TYPE_CHECKING, Any, Optional, Protocol, runtime_checkable if TYPE_CHECKING: # pragma: no cover from openadapt_flow.ir import StructuralHandle, StructuralLocator @@ -44,6 +44,7 @@ def system_of_record(self) -> Optional[list[dict[str, Any]]]: """ ... + @runtime_checkable class StructuralBackend(Protocol): """Optional structural observations a backend MAY expose. @@ -166,9 +167,7 @@ class StructuralActionBackend(Protocol): that point (never raises) -- resolution then uses the visual ladder. """ - def structural_locator_at( - self, x: int, y: int - ) -> Optional["StructuralLocator"]: + def structural_locator_at(self, x: int, y: int) -> Optional["StructuralLocator"]: """Return a stable structural locator for the element at pixel (x, y). The coordinate space matches :meth:`Backend.click`. Returns None when diff --git a/openadapt_flow/backends/parallels_vm.py b/openadapt_flow/backends/parallels_vm.py index 6db05ac..b160d83 100644 --- a/openadapt_flow/backends/parallels_vm.py +++ b/openadapt_flow/backends/parallels_vm.py @@ -40,8 +40,7 @@ DEFAULT_VM_UUID = "{d4f9c29a-52e1-4793-9334-7e971c3d0ab3}" DEFAULT_PRLCTL = "/usr/local/bin/prlctl" SHIM_PORT = 5000 -_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", - "desktop") +_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "scripts", "desktop") # In-guest install locations (forward slashes: Python + curl accept them and # they survive the host shell without backslash mangling). GUEST_DIR = "C:/oa" @@ -137,8 +136,9 @@ def resume(self) -> None: def set_pause_idle(self, on: bool) -> None: """Toggle Parallels' pause-when-idle (must be OFF for headless runs).""" - self._run(["set", self.uuid, "--pause-idle", "on" if on else "off"], - check=False) + self._run( + ["set", self.uuid, "--pause-idle", "on" if on else "off"], check=False + ) def ensure_running(self, *, settle_s: float = 6.0) -> None: """Bring the VM to a running state from any state, idempotently. @@ -206,19 +206,21 @@ def exec( NOTE: ``prlctl exec`` hangs on very long single arguments — keep commands short and move file payloads with :meth:`push_file`. """ - return self._run(["exec", self.uuid, *args], timeout=timeout, - check=check) + return self._run(["exec", self.uuid, *args], timeout=timeout, check=check) - def exec_cmd(self, cmdline: str, *, timeout: float = 120.0 - ) -> subprocess.CompletedProcess: + def exec_cmd( + self, cmdline: str, *, timeout: float = 120.0 + ) -> subprocess.CompletedProcess: """Run ``cmd /c `` in-guest (quoting preserved by cmd).""" return self.exec(["cmd", "/c", cmdline], timeout=timeout) - def exec_ps(self, script: str, *, timeout: float = 120.0 - ) -> subprocess.CompletedProcess: + def exec_ps( + self, script: str, *, timeout: float = 120.0 + ) -> subprocess.CompletedProcess: """Run a short PowerShell command in-guest.""" - return self.exec(["powershell", "-NoProfile", "-Command", script], - timeout=timeout) + return self.exec( + ["powershell", "-NoProfile", "-Command", script], timeout=timeout + ) # -- host-side capture --------------------------------------------------- @@ -297,14 +299,12 @@ def push_file( def _shim_paths(self) -> tuple[str, str]: shim = os.path.abspath(os.path.join(_SCRIPT_DIR, "waa_shim.py")) - launcher = os.path.abspath(os.path.join(_SCRIPT_DIR, - "session1_launch.py")) + launcher = os.path.abspath(os.path.join(_SCRIPT_DIR, "session1_launch.py")) return shim, launcher def kill_shim(self) -> None: """Kill any in-guest Python (frees the shim port).""" - self.exec_cmd("taskkill /F /IM python.exe /IM pythonw.exe 2>nul & " - "echo done") + self.exec_cmd("taskkill /F /IM python.exe /IM pythonw.exe 2>nul & echo done") def launch_shim( self, @@ -323,18 +323,35 @@ def launch_shim( shim, launcher = self._shim_paths() self.exec_cmd(f"if not exist {GUEST_DIR} mkdir {GUEST_DIR}") self.push_file(shim, f"{GUEST_DIR}/waa_shim.py", host_ip=host_ip) - self.push_file(launcher, f"{GUEST_DIR}/session1_launch.py", - host_ip=host_ip) - self.exec(["netsh", "advfirewall", "firewall", "add", "rule", - "name=OAShim", "dir=in", "action=allow", "protocol=TCP", - f"localport={port}"]) + self.push_file(launcher, f"{GUEST_DIR}/session1_launch.py", host_ip=host_ip) + self.exec( + [ + "netsh", + "advfirewall", + "firewall", + "add", + "rule", + "name=OAShim", + "dir=in", + "action=allow", + "protocol=TCP", + f"localport={port}", + ] + ) self.kill_shim() time.sleep(2) # Run the launcher as SYSTEM; it CreateProcessAsUser's the shim into # the interactive console session so mss/pyautogui address the real # desktop. Forward-slash script paths dodge host-shell mangling. - self.exec([self.python_guest, f"{GUEST_DIR}/session1_launch.py", - f"{GUEST_DIR}/waa_shim.py", "--port", str(port)]) + self.exec( + [ + self.python_guest, + f"{GUEST_DIR}/session1_launch.py", + f"{GUEST_DIR}/waa_shim.py", + "--port", + str(port), + ] + ) url = f"http://{self.guest_ip()}:{port}" deadline = time.time() + wait_s while time.time() < deadline: diff --git a/openadapt_flow/backends/playwright_backend.py b/openadapt_flow/backends/playwright_backend.py index 880f57f..594d1bf 100644 --- a/openadapt_flow/backends/playwright_backend.py +++ b/openadapt_flow/backends/playwright_backend.py @@ -207,9 +207,7 @@ def structured_text_at(self, x: int, y: int) -> Optional[str]: # -- structural action (openadapt_flow.backend.StructuralActionBackend) -- - def structural_locator_at( - self, x: int, y: int - ) -> Optional[StructuralLocator]: + def structural_locator_at(self, x: int, y: int) -> Optional[StructuralLocator]: """Return a stable DOM locator for the element under (x, y). Walks from ``document.elementFromPoint`` to the nearest ACTIONABLE @@ -285,9 +283,7 @@ def locate_structural( if locator.selector: loc = self.page.locator(locator.selector) elif locator.role and locator.name: - loc = self.page.get_by_role( - locator.role, name=locator.name, exact=True - ) + loc = self.page.get_by_role(locator.role, name=locator.name, exact=True) if loc is None: return None if loc.count() != 1: diff --git a/openadapt_flow/backends/rdp_backend.py b/openadapt_flow/backends/rdp_backend.py index 204b7f4..b6474d0 100644 --- a/openadapt_flow/backends/rdp_backend.py +++ b/openadapt_flow/backends/rdp_backend.py @@ -236,9 +236,7 @@ def screenshot(self) -> bytes: img.convert("RGB").save(buf, format="PNG") return buf.getvalue() - def wait_first_frame( - self, *, retries: int = 20, settle_s: float = 0.25 - ) -> bytes: + def wait_first_frame(self, *, retries: int = 20, settle_s: float = 0.25) -> bytes: """Poll :meth:`screenshot` until a non-blank frame, returning its PNG. The first frame(s) an RDP session paints are often a single-colour diff --git a/openadapt_flow/backends/windows_backend.py b/openadapt_flow/backends/windows_backend.py index bf97ae6..1a7e8dd 100644 --- a/openadapt_flow/backends/windows_backend.py +++ b/openadapt_flow/backends/windows_backend.py @@ -383,9 +383,7 @@ def _read_structured_json(self, snippet: str) -> object: except Exception: return None - def structural_locator_at( - self, x: int, y: int - ) -> Optional[StructuralLocator]: + def structural_locator_at(self, x: int, y: int) -> Optional[StructuralLocator]: """Return a stable UIA locator for the control under (x, y). Runs a ``uiautomation.ControlFromPoint`` snippet on the VM: it climbs to @@ -578,9 +576,7 @@ def press(self, key: str) -> None: """Press a key or chord, e.g. ``'Enter'`` or ``'ControlOrMeta+a'``.""" keys = normalize_chord(key) if len(keys) == 1: - self._execute( - f"import pyautogui; pyautogui.press({keys[0]!r})" - ) + self._execute(f"import pyautogui; pyautogui.press({keys[0]!r})") else: args = ", ".join(repr(k) for k in keys) self._execute(f"import pyautogui; pyautogui.hotkey({args})") diff --git a/openadapt_flow/bench.py b/openadapt_flow/bench.py index 5fee445..22e654f 100644 --- a/openadapt_flow/bench.py +++ b/openadapt_flow/bench.py @@ -45,9 +45,7 @@ def _run_one( run_dir: Path, ) -> RunReport: replayer = replayer_cls(backend) - return replayer.run( - workflow, params=params, bundle_dir=bundle_dir, run_dir=run_dir - ) + return replayer.run(workflow, params=params, bundle_dir=bundle_dir, run_dir=run_dir) def run_bench( diff --git a/openadapt_flow/benchmark/agent_baseline.py b/openadapt_flow/benchmark/agent_baseline.py index 36c0ee2..4882d95 100644 --- a/openadapt_flow/benchmark/agent_baseline.py +++ b/openadapt_flow/benchmark/agent_baseline.py @@ -153,12 +153,12 @@ def triage_task_prompt(note_text: str) -> str: return ( "You are looking at MockMed, a demo clinic web app (fake data " "only). Complete this task:\n\n" - "1. Sign in with username \"nurse.demo\" and password " - "\"mockmed-demo-pass\".\n" + '1. Sign in with username "nurse.demo" and password ' + '"mockmed-demo-pass".\n' "2. Open the first referral task in the list.\n" "3. From the patient's page, create a New Encounter and choose the " - "type \"Triage\".\n" - f"4. Enter exactly this note in the Note field: \"{note_text}\"\n" + 'type "Triage".\n' + f'4. Enter exactly this note in the Note field: "{note_text}"\n' "5. Save the encounter.\n\n" "You are done when you are back on the patient's page and see the " "'Encounter saved' confirmation. Then stop and reply with a one-line " @@ -183,14 +183,14 @@ def openemr_task_prompt(note_text: str) -> str: return ( "You are looking at the OpenEMR public demo (a real EMR web app " "with fake demo patients only). Complete this task:\n\n" - "1. Sign in with username \"admin\" and password \"pass\".\n" + '1. Sign in with username "admin" and password "pass".\n' "2. Use the patient search box in the top bar to search for " - "\"Phil\" and open the chart of the patient \"Belford, Phil\".\n" + '"Phil" and open the chart of the patient "Belford, Phil".\n' "3. On the patient's dashboard, open the Patient Messages section " "(the Messages card — you will likely need to scroll down to " "find it).\n" "4. Add a new note and enter exactly this text as the note: " - f"\"{note_text}\"\n" + f'"{note_text}"\n' "5. Save it as a new message.\n\n" "You are done when you are back on the patient-message list and " "can see the new note. Then stop and reply with a one-line " @@ -464,9 +464,7 @@ def error(message: str) -> dict[str, Any]: } -def _truncate_screenshots( - messages: list[dict[str, Any]], keep: int -) -> None: +def _truncate_screenshots(messages: list[dict[str, Any]], keep: int) -> None: """Replace all but the last ``keep`` screenshot blocks with text stubs. Walks the ``tool_result`` blocks of user messages (the only place this @@ -648,9 +646,7 @@ def run_agent( ) continue actions += 1 - action_log.append( - f"{actions}: {dict(block.input) if block.input else {}}" - ) + action_log.append(f"{actions}: {dict(block.input) if block.input else {}}") results.append(_execute_action(backend, block)) messages.append({"role": "user", "content": results}) _truncate_screenshots(messages, keep_screenshots) diff --git a/openadapt_flow/benchmark/chart_fonts.py b/openadapt_flow/benchmark/chart_fonts.py index b261e5d..2067415 100644 --- a/openadapt_flow/benchmark/chart_fonts.py +++ b/openadapt_flow/benchmark/chart_fonts.py @@ -45,9 +45,7 @@ def configure_bundled_font() -> Any: matplotlib.use("Agg") from matplotlib import font_manager - font_path = ( - Path(matplotlib.get_data_path()) / "fonts" / "ttf" / "DejaVuSans.ttf" - ) + font_path = Path(matplotlib.get_data_path()) / "fonts" / "ttf" / "DejaVuSans.ttf" if font_path.is_file(): try: font_manager.fontManager.addfont(str(font_path)) @@ -57,8 +55,7 @@ def configure_bundled_font() -> Any: import matplotlib.pyplot as plt existing = [ - f for f in plt.rcParams.get("font.sans-serif", []) - if f != BUNDLED_FONT_NAME + f for f in plt.rcParams.get("font.sans-serif", []) if f != BUNDLED_FONT_NAME ] plt.rcParams["font.family"] = "sans-serif" plt.rcParams["font.sans-serif"] = [BUNDLED_FONT_NAME, *existing] diff --git a/openadapt_flow/benchmark/desktop_benchmark.py b/openadapt_flow/benchmark/desktop_benchmark.py index 3d4699c..d96f03b 100644 --- a/openadapt_flow/benchmark/desktop_benchmark.py +++ b/openadapt_flow/benchmark/desktop_benchmark.py @@ -78,15 +78,12 @@ # safe-halt there is a FALSE ABORT; data-drift conditions genuinely change the # list, so a halt there can be correct caution. CONDITIONS: dict[str, dict] = { - "clean": {"cfg": {}, "drift": "none", "cosmetic": True}, - "render_125": {"cfg": {"font_scale": 1.25}, "drift": "none", - "cosmetic": True}, - "render_150": {"cfg": {"font_scale": 1.5}, "drift": "none", - "cosmetic": True}, - "theme_dark": {"cfg": {"theme": "dark"}, "drift": "none", - "cosmetic": True}, - "data_reorder": {"cfg": {}, "drift": "reorder", "cosmetic": False}, - "data_decoy": {"cfg": {}, "drift": "decoy", "cosmetic": False}, + "clean": {"cfg": {}, "drift": "none", "cosmetic": True}, + "render_125": {"cfg": {"font_scale": 1.25}, "drift": "none", "cosmetic": True}, + "render_150": {"cfg": {"font_scale": 1.5}, "drift": "none", "cosmetic": True}, + "theme_dark": {"cfg": {"theme": "dark"}, "drift": "none", "cosmetic": True}, + "data_reorder": {"cfg": {}, "drift": "reorder", "cosmetic": False}, + "data_decoy": {"cfg": {}, "drift": "decoy", "cosmetic": False}, "data_siblings": {"cfg": {}, "drift": "siblings", "cosmetic": False}, } @@ -98,6 +95,7 @@ # --- result rows ------------------------------------------------------------ + @dataclass class RunRow: """One arm x condition x repeat outcome, judged by the DB.""" @@ -105,10 +103,10 @@ class RunRow: arm: str condition: str i: int - outcome: str = "error" # success|wrong_action|safe_halt|miss|error + outcome: str = "error" # success|wrong_action|safe_halt|miss|error wrong_action: bool = False false_abort: bool = False - completed: bool = False # arm ran to its end without halting + completed: bool = False # arm ran to its end without halting target_note_ok: bool = False wrong_patient_id: Optional[int] = None # compiled-arm identity telemetry @@ -128,6 +126,7 @@ class RunRow: # --- live harness (requires a VM; imported lazily by the orchestrator) ------ + class DesktopHarness: """Drives one Parallels VM through the full desktop pipeline. @@ -151,7 +150,8 @@ def connect( log: Callable = print, ) -> "DesktopHarness": from openadapt_flow.backends.parallels_vm import ( - DEFAULT_VM_UUID, ParallelsVM, + DEFAULT_VM_UUID, + ParallelsVM, ) vm = ParallelsVM(vm_uuid or DEFAULT_VM_UUID) @@ -178,8 +178,9 @@ def quiet_desktop(self) -> None: "'/d','0','/f'],capture_output=True)\n" ) try: - requests.post(f"{self.shim_url}/execute_windows", - json={"command": cmd}, timeout=20) + requests.post( + f"{self.shim_url}/execute_windows", json={"command": cmd}, timeout=20 + ) except Exception: # noqa: BLE001 pass @@ -192,13 +193,11 @@ def deploy_app_scripts(self) -> None: src = Path(pv._SCRIPT_DIR) self.vm.exec_cmd(f"if not exist {GUEST_DIR} mkdir {GUEST_DIR}") for name in self._APP_SCRIPTS: - self.vm.push_file(str((src / name).resolve()), - f"{GUEST_DIR}/{name}") + self.vm.push_file(str((src / name).resolve()), f"{GUEST_DIR}/{name}") # -- DB ground truth -- def _py(self, *args: str): - return self.vm.exec([GUEST_PY, f"{GUEST_DIR}/pn_db.py", *args], - timeout=60) + return self.vm.exec([GUEST_PY, f"{GUEST_DIR}/pn_db.py", *args], timeout=60) def seed(self, drift: str = "none") -> None: self._py("seed", "--drift", drift) @@ -214,8 +213,8 @@ def db_all(self) -> list[dict]: # -- app lifecycle -- def write_cfg(self, cfg: dict) -> None: """Write pn_env.json (drift knobs) into the guest via HTTP push.""" - import tempfile import os + import tempfile tmp = Path(tempfile.mkdtemp()) / "pn_env.json" tmp.write_text(json.dumps(cfg)) @@ -225,13 +224,17 @@ def write_cfg(self, cfg: dict) -> None: def stop_app(self) -> None: self.vm.exec_cmd("taskkill /F /IM powershell.exe 2>nul & echo ok") - def launch_app(self, cfg: Optional[dict] = None, *, settle_s: float = 6.0 - ) -> None: + def launch_app(self, cfg: Optional[dict] = None, *, settle_s: float = 6.0) -> None: self.stop_app() time.sleep(1) self.write_cfg(cfg or {}) - self.vm.exec([GUEST_PY, f"{GUEST_DIR}/session1_launch.py", - f"{GUEST_DIR}/patient_notes.ps1"]) + self.vm.exec( + [ + GUEST_PY, + f"{GUEST_DIR}/session1_launch.py", + f"{GUEST_DIR}/patient_notes.ps1", + ] + ) time.sleep(settle_s) def prepare_condition(self, condition: str) -> None: @@ -253,7 +256,10 @@ def rects(self) -> dict: for n in u.get("nodes", []): nums = list(map(int, re.findall(r"-?\d+", n["rect"]))) if len(nums) >= 4 and n["automation_id"] in ( - "searchBox", "noteBox", "saveButton", "patientGrid" + "searchBox", + "noteBox", + "saveButton", + "patientGrid", ): out[n["automation_id"]] = tuple(nums[:4]) return out @@ -302,8 +308,7 @@ def record_and_compile(self, work_dir: Path) -> Path: # surname leaves the given name + DOB in the identity band -- exactly # what distinguishes siblings (Sorenson vs Sorensen, different DOB). gL, gT, gR, gB = c["patientGrid"] - row0 = self.data_cell_center("last", 0) or ( - gL + int((gR - gL) * 0.35), gT + 62) + row0 = self.data_cell_center("last", 0) or (gL + int((gR - gL) * 0.35), gT + 62) rec = work_dir / "recording" r = Recorder(self.backend, rec) @@ -351,15 +356,17 @@ def uia_run(self, mode: str, note: str) -> dict: cmd = ( "import json,importlib.util\n" "spec=importlib.util.spec_from_file_location('uia_arm', " - r"r'C:\oa\uia_arm.py')" "\n" + r"r'C:\oa\uia_arm.py')" + "\n" "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)\n" f"res=m.run({DEMO['search']!r},{DEMO['first']!r},{DEMO['last']!r}," f"{note!r},{mode!r})\n" r"open(r'C:\oa\uia_result.json','w',encoding='utf-8')" ".write(json.dumps(res))\n" ) - requests.post(f"{self.shim_url}/execute_windows", - json={"command": cmd}, timeout=60) + requests.post( + f"{self.shim_url}/execute_windows", json={"command": cmd}, timeout=60 + ) out = self.vm.exec_cmd(r"type C:\oa\uia_result.json").stdout.strip() return json.loads(out) if out else {"status": "error"} @@ -372,8 +379,11 @@ def judge(self, note: str) -> dict: rows = self.db_all() target = next((r for r in rows if r["id"] == DEMO["target_id"]), {}) target_ok = target.get("note", "") == note - wrongs = [r for r in rows - if r["id"] != DEMO["target_id"] and r.get("note", "") == note] + wrongs = [ + r + for r in rows + if r["id"] != DEMO["target_id"] and r.get("note", "") == note + ] return { "target_note_ok": target_ok, "wrong_patient_id": wrongs[0]["id"] if wrongs else None, @@ -383,6 +393,7 @@ def judge(self, note: str) -> dict: # --- orchestrator ----------------------------------------------------------- + def _classify(judged: dict, completed: bool, cosmetic: bool) -> tuple[str, bool]: """Map (DB verdict, completion) to an outcome + false-abort flag.""" if judged["wrong_action"]: @@ -462,8 +473,7 @@ def run_desktop_benchmark( row.identity_mismatch = res["identity"]["mismatch"] row.identity_unreadable = res["identity"]["unreadable"] else: - mode = ("identity" if arm == "uia_identity" - else "positional") + mode = "identity" if arm == "uia_identity" else "positional" res = harness.uia_run(mode, note) row.selected_index = res.get("selected_index") row.selected_name = res.get("selected_name") @@ -480,10 +490,12 @@ def run_desktop_benchmark( row.outcome = "error" row.wall_s = round(time.time() - t0, 2) rows.append(row) - log(f"[bench] {arm:15s} {condition:13s} #{i} -> " + log( + f"[bench] {arm:15s} {condition:13s} #{i} -> " f"{row.outcome}" + (" WRONG-ACTION" if row.wrong_action else "") - + (" false-abort" if row.false_abort else "")) + + (" false-abort" if row.false_abort else "") + ) results = _aggregate(rows, wf_armed, tree_quality, conditions, arms) write_outputs(results, out_dir) @@ -494,8 +506,7 @@ def _armed_coverage(bundle: Path) -> dict: from openadapt_flow.ir import Workflow wf = Workflow.load(bundle) - clicks = [s for s in wf.steps - if s.action.value in ("click", "double_click")] + clicks = [s for s in wf.steps if s.action.value in ("click", "double_click")] armed = [s for s in clicks if s.anchor and s.anchor.context_text] return { "click_steps": len(clicks), @@ -519,11 +530,12 @@ def _aggregate(rows, armed, tree_quality, conditions, arms) -> dict: "miss": sum(r.outcome == "miss" for r in arm_rows), "error": sum(r.outcome == "error" for r in arm_rows), "success_rate": round( - sum(r.outcome == "success" for r in arm_rows) / max(1, n), 3), + sum(r.outcome == "success" for r in arm_rows) / max(1, n), 3 + ), "wrong_action_rate": round( - sum(r.wrong_action for r in arm_rows) / max(1, n), 3), - "wall_s_mean": round( - sum(r.wall_s for r in arm_rows) / max(1, n), 2), + sum(r.wrong_action for r in arm_rows) / max(1, n), 3 + ), + "wall_s_mean": round(sum(r.wall_s for r in arm_rows) / max(1, n), 2), } # per arm x condition outcome matrix matrix: dict[str, dict] = {} @@ -541,11 +553,11 @@ def _aggregate(rows, armed, tree_quality, conditions, arms) -> dict: return { "generated_at": datetime.now(timezone.utc).isoformat(), "task": "Patient Notes (WinForms) search -> select -> note -> save; " - "DB-ground-truth judge; $0 (no model calls)", + "DB-ground-truth judge; $0 (no model calls)", "substrate": "Parallels Windows 11 ARM VM on Apple M2 Max; " - "WindowsBackend over in-guest WAA HTTP shim (session 1)", + "WindowsBackend over in-guest WAA HTTP shim (session 1)", "target_app_note": "WinForms substitute for OpenDental (trial not " - "no-touch installable; see PHASE2.md).", + "no-touch installable; see PHASE2.md).", "identity_armed_coverage": armed, "uia_tree_quality": tree_quality, "arms": by_arm, @@ -557,6 +569,7 @@ def _aggregate(rows, armed, tree_quality, conditions, arms) -> dict: # --- output writers --------------------------------------------------------- + def render_markdown(results: dict) -> str: a = results["arms"] lines = [ @@ -590,10 +603,10 @@ def render_markdown(results: dict) -> str: "## Identity transfer to desktop-rendered text", "", f"- Compiled-arm **armed coverage**: " - f"{ic.get('armed_clicks','?')}/{ic.get('click_steps','?')} click steps " + f"{ic.get('armed_clicks', '?')}/{ic.get('click_steps', '?')} click steps " f"carry an identity band ({ic.get('armed_coverage', 0):.0%}).", f"- UIA-tree quality: " - f"{tq.get('n_usable_id','?')}/{tq.get('n_targets','?')} workflow " + f"{tq.get('n_usable_id', '?')}/{tq.get('n_targets', '?')} workflow " f"targets expose a usable AutomationId " f"({tq.get('usable_fraction', 0):.0%}); the identity-critical patient " f"row does **not** " @@ -715,8 +728,7 @@ def render_chart(results: dict, path: Path) -> None: succ = [results["arms"][a]["success"] for a in arms] wrong = [results["arms"][a]["wrong_action"] for a in arms] halt = [results["arms"][a]["safe_halt"] for a in arms] - miss = [results["arms"][a]["miss"] + results["arms"][a]["error"] - for a in arms] + miss = [results["arms"][a]["miss"] + results["arms"][a]["error"] for a in arms] fig, ax = plt.subplots(figsize=(8, 4.5)) bottom = [0] * len(arms) for label, vals, color in [ @@ -756,6 +768,8 @@ def write_outputs(results: dict, out_dir: str | Path) -> None: args = ap.parse_args() conds = args.conditions.split(",") if args.conditions else None run_desktop_benchmark( - args.out, n_per=args.n, conditions=conds, + args.out, + n_per=args.n, + conditions=conds, arms=tuple(args.arms.split(",")), ) diff --git a/openadapt_flow/benchmark/dom_arm.py b/openadapt_flow/benchmark/dom_arm.py index 120604d..a6c0e59 100644 --- a/openadapt_flow/benchmark/dom_arm.py +++ b/openadapt_flow/benchmark/dom_arm.py @@ -98,13 +98,20 @@ def note_for_slot(arm: str, slot: int) -> str: tag = _ARM_TAGS.get(arm, arm[:1].upper()) return f"{_NOTE_POOL[slot % len(_NOTE_POOL)]} [{tag}{slot:02d}]" + #: The perturbation drift modes from the validation suite (PR #12/#13), #: plus ``sort`` (a changed default sort order — every referral present, #: the recorded target no longer first) and ``typelabel`` (Triage segment #: relabeled AND reordered), both flag-gated in the MockMed app. PERTURBATIONS: tuple[str, ...] = ( - "lookalike", "missing", "grow", "sort", - "theme", "rename", "move", "typelabel", + "lookalike", + "missing", + "grow", + "sort", + "theme", + "rename", + "move", + "typelabel", ) # -- the DOM-selector script (the steelman) ------------------------------------ @@ -174,8 +181,7 @@ def dom_script( # cannot tell. Measured, not assumed. open_step = ( "open first referral", - lambda: page.get_by_role( - "button", name="Open").first.click(), + lambda: page.get_by_role("button", name="Open").first.click(), ) else: # Identity reading: scope the Open button to the row whose @@ -186,46 +192,53 @@ def dom_script( # the compiled arm's recorded identity band carries. open_step = ( "open referral by patient name", - lambda: page.get_by_role("row", name=target_patient) - .get_by_role("button", name="Open") - .click(), + lambda: ( + page.get_by_role("row", name=target_patient) + .get_by_role("button", name="Open") + .click() + ), ) return [ # Form fields by their explicit