diff --git a/openclaw_bench/backend.py b/openclaw_bench/backend.py index d5a3ca2..0638011 100644 --- a/openclaw_bench/backend.py +++ b/openclaw_bench/backend.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Protocol +from .container import runtime_kind_target from .models import BackendResponse, ModelSpec, TaskSpec @@ -39,10 +40,19 @@ def __init__( def _cmd(self, cwd: Path | None = None) -> list[str]: if not self.container: return ["openclaw"] + kind, target = runtime_kind_target(self.container) + if kind == "incus": + cmd = ["incus", "exec", target] + if cwd is not None: + cmd.extend(["--cwd", str(cwd)]) + cmd.append("--") + cmd.append("openclaw") + return cmd cmd = ["docker", "exec"] if cwd is not None: cmd.extend(["-w", str(cwd)]) - cmd.extend([self.container, "openclaw"]) + cmd.append(target) + cmd.append("openclaw") return cmd def smoke(self, model: ModelSpec, timeout_s: int) -> BackendResponse: @@ -437,8 +447,10 @@ def _ensure_workspace_agent( def _container_openclaw_cmd(container: str) -> list[str]: - return ["docker", "exec", container, "openclaw"] - + kind, target = runtime_kind_target(container) + if kind == "incus": + return ["incus", "exec", target, "--", "openclaw"] + return ["docker", "exec", target, "openclaw"] def _agent_id(session_id: str) -> str: safe = "".join(char if char.isalnum() or char == "-" else "-" for char in session_id.lower()) diff --git a/openclaw_bench/cli.py b/openclaw_bench/cli.py index 0e87d3e..43acd57 100644 --- a/openclaw_bench/cli.py +++ b/openclaw_bench/cli.py @@ -61,7 +61,7 @@ def main(argv: list[str] | None = None) -> int: default=60, help="Seconds to wait for the OpenClaw route smoke before running task attempts", ) - preflight_parser = subparsers.add_parser("preflight", help="Check whether a benchmark run can execute safely") + preflight_parser = subparsers.add_parser("preflight", help="Check benchmark suite/model readiness before a run") _add_common_run_args(preflight_parser) preflight_parser.add_argument("--json", action="store_true", help="Print machine-readable preflight output") preflight_parser.add_argument("--smoke-turn", action="store_true", help="Run a tiny OpenClaw model route turn for eligible models") @@ -73,7 +73,7 @@ def main(argv: list[str] | None = None) -> int: preflight_parser.add_argument("--smoke-timeout", type=int, default=60, help="Seconds to wait for each smoke turn") provider_preflight_parser = subparsers.add_parser( "provider-preflight", - help="Run the four verification gates against an OpenClaw profile + provider route.", + help="Verify an OpenClaw provider route: config, models list, health, and route smoke", ) provider_preflight_parser.add_argument("--profile", required=True) provider_preflight_parser.add_argument("--provider", required=True, choices=["vllm", "ollama", "llamacpp", "lmstudio"]) @@ -203,11 +203,17 @@ def _add_quickstart_init_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--no-validate", action="store_true", help="Write files without running openclaw config validate") parser.add_argument("--no-detect", action="store_true", help="Skip provider auto-detection; use --vllm-* flags or env vars.") parser.add_argument("--oc-runtime", default=None, help="Override OpenClaw runtime probe target (e.g., ssh:user@host).") + parser.add_argument( + "--probe-hosts", + default="127.0.0.1", + help="Comma-separated host addresses to probe; include bridge/LAN addresses for services not bound to loopback", + ) def _add_quickstart_lifecycle_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--openclaw-profile", default=DEFAULT_PROFILE) parser.add_argument("--timeout", type=int, default=60) + parser.add_argument("--oc-runtime", default=None, help="Run OpenClaw commands via runtime target (incus: or docker:)") def init_command(args: argparse.Namespace) -> int: @@ -226,6 +232,7 @@ def init_command(args: argparse.Namespace) -> int: providers=["vllm", "ollama", "llamacpp", "lmstudio"], probes=probes, home=probe_home, + probe_hosts=_parse_csv(args.probe_hosts), ) for finding in report.findings: print(f"finding: {finding}") @@ -235,7 +242,18 @@ def init_command(args: argparse.Namespace) -> int: "to specify one explicitly, or start a model server first" ) return 2 - detected = next((c for c in report.candidates if c.provider == "vllm"), report.candidates[0]) + detected = next((c for c in report.candidates if c.provider == "vllm"), None) + if detected is None: + detected_summary = ", ".join( + f"{candidate.provider}@{candidate.base_url}" + for candidate in report.candidates + ) + print( + "detected local provider(s) that oc-bench init cannot generate yet: " + f"{detected_summary}; supported init provider: vllm. " + "Pass --no-detect with --vllm-base-url/--vllm-model for an explicit vLLM route." + ) + return 2 # Inherit path: detection found an existing OC profile with the right wiring # already configured. Clone it verbatim and skip the generation flow that @@ -392,13 +410,21 @@ def _init_via_inheritance( def start_command(args: argparse.Namespace) -> int: - check = start_benchclaw_gateway(args.openclaw_profile, timeout_s=args.timeout) + check = start_benchclaw_gateway( + args.openclaw_profile, + container=getattr(args, "oc_runtime", None), + timeout_s=args.timeout, + ) print(f"{check.name}={check.status} {check.notes}") return 0 if check.status != "fail" else 1 def stop_command(args: argparse.Namespace) -> int: - check = stop_benchclaw_gateway(args.openclaw_profile, timeout_s=args.timeout) + check = stop_benchclaw_gateway( + args.openclaw_profile, + container=getattr(args, "oc_runtime", None), + timeout_s=args.timeout, + ) print(f"{check.name}={check.status} {check.notes}") return 0 if check.status != "fail" else 1 @@ -449,7 +475,11 @@ def quickstart_command(args: argparse.Namespace) -> int: return run_code finally: if args.stop_after and started: - stop_check = stop_benchclaw_gateway(args.openclaw_profile, timeout_s=30) + stop_check = stop_benchclaw_gateway( + args.openclaw_profile, + container=getattr(args, "oc_runtime", None), + timeout_s=30, + ) print(f"{stop_check.name}={stop_check.status} {stop_check.notes}") @@ -575,7 +605,7 @@ def run_command(args: argparse.Namespace) -> int: results = runner.run(config) finally: if stop_started_gateway: - stop_check = stop_openclaw_gateway(args.openclaw_profile, timeout_s=args.openclaw_gateway_timeout) + stop_check = stop_openclaw_gateway(args.openclaw_profile, args.openclaw_container, timeout_s=args.openclaw_gateway_timeout) print(f"{stop_check.name}={stop_check.status} {stop_check.notes}") failures = sum(1 for result in results if result.status != "pass") print(f"run_id={run_id}") diff --git a/openclaw_bench/container.py b/openclaw_bench/container.py index 937ed79..55793ad 100644 --- a/openclaw_bench/container.py +++ b/openclaw_bench/container.py @@ -24,6 +24,24 @@ def to_row(self) -> dict[str, str]: return {"name": self.name, "status": self.status, "notes": self.notes} +def runtime_kind_target(runtime: str) -> tuple[str, str]: + if ":" not in runtime: + return "docker", runtime + kind, _, target = runtime.partition(":") + kind = kind.strip().lower() + target = target.strip() + if kind not in {"docker", "incus"} or not target: + raise ValueError(f"unsupported OpenClaw runtime {runtime!r}; use docker: or incus:") + return kind, target + + +def runtime_exec_prefix(runtime: str) -> list[str]: + kind, target = runtime_kind_target(runtime) + if kind == "incus": + return ["incus", "exec", target, "--"] + return ["docker", "exec", target] + + def ensure_openclaw_container( *, container: str, diff --git a/openclaw_bench/preflight.py b/openclaw_bench/preflight.py index 5a71ad7..1c4b550 100644 --- a/openclaw_bench/preflight.py +++ b/openclaw_bench/preflight.py @@ -3,6 +3,7 @@ import json import os import signal +import shlex import shutil import subprocess import tempfile @@ -11,7 +12,7 @@ from pathlib import Path from .backend import OpenClawBackend -from .container import DEFAULT_GATEWAY_PORT, gateway_run_command +from .container import DEFAULT_GATEWAY_PORT, gateway_run_command, runtime_exec_prefix, runtime_kind_target from .models import ModelSpec, SuiteManifest, TaskSpec from .workspace import OPENCLAW_SEED_FILES, copy_fixture, seed_openclaw_workspace_files @@ -255,7 +256,7 @@ def _check_output_path(path: Path, name: str) -> PreflightCheck: def _check_container_path(path: Path, container: str, name: str) -> PreflightCheck: - cmd = ["docker", "exec", container, "test", "-d", str(path), "-a", "-w", str(path)] + cmd = [*runtime_exec_prefix(container), "test", "-d", str(path), "-a", "-w", str(path)] try: proc = subprocess.run(cmd, text=True, capture_output=True, timeout=10, check=False) except (OSError, subprocess.TimeoutExpired) as exc: @@ -345,13 +346,19 @@ def _check_openclaw( ) -> list[PreflightCheck]: checks = [] if container: - docker = shutil.which("docker") - if docker is None: - return [PreflightCheck("openclaw_cli", "fail", "docker executable not found for --openclaw-container")] - checks.append(PreflightCheck("openclaw_cli", "pass", f"docker exec {container} openclaw")) + kind, target = runtime_kind_target(container) + executable = shutil.which(kind) + if executable is None: + return [PreflightCheck("openclaw_cli", "fail", f"{kind} executable not found for --openclaw-container/--oc-runtime")] + checks.append(PreflightCheck("openclaw_cli", "pass", f"{kind} exec {target} openclaw")) else: executable = shutil.which("openclaw") if executable is None: + if local: + return [ + PreflightCheck("openclaw_cli", "fail", "openclaw executable not found"), + PreflightCheck("openclaw_gateway", "warn", "--openclaw-local skips gateway requirement"), + ] return [PreflightCheck("openclaw_cli", "fail", "openclaw executable not found")] checks.append(PreflightCheck("openclaw_cli", "pass", executable)) version_check = check_openclaw_version(container) @@ -634,21 +641,26 @@ def ensure_openclaw_gateway(profile: str, container: str | None = None, timeout_ ) -def stop_openclaw_gateway(profile: str, timeout_s: int = 30) -> PreflightCheck: - foreground = _stop_foreground_gateway(profile, timeout_s) +def stop_openclaw_gateway(profile: str, container: str | None = None, timeout_s: int = 30) -> PreflightCheck: + foreground = _stop_foreground_gateway(profile, timeout_s, container) if foreground.status == "pass": return PreflightCheck("openclaw_gateway_stop", "pass", foreground.notes) - cmd = ["openclaw", "--profile", profile, "gateway", "stop"] + cmd = [*_openclaw_cmd(container), "--profile", profile, "gateway", "stop"] try: proc = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout_s, check=False) except (OSError, subprocess.TimeoutExpired) as exc: return PreflightCheck("openclaw_gateway_stop", "fail", str(exc)) output = f"{proc.stdout}\n{proc.stderr}".strip() if proc.returncode != 0: + runtime_kill = _stop_runtime_gateway_process(profile, container, timeout_s) + if runtime_kill is not None and runtime_kill.status == "pass": + return PreflightCheck("openclaw_gateway_stop", "pass", runtime_kill.notes) notes = _trim_output(output) if foreground.status == "fail": notes = _join_gateway_notes(notes, foreground.notes) + if runtime_kill is not None and runtime_kill.status == "fail": + notes = _join_gateway_notes(notes, runtime_kill.notes) return PreflightCheck("openclaw_gateway_stop", "fail", notes) if foreground.status == "fail": return PreflightCheck("openclaw_gateway_stop", "fail", foreground.notes) @@ -674,7 +686,10 @@ def _join_gateway_notes(prefix: str, notes: str) -> str: def _openclaw_cmd(container: str | None = None) -> list[str]: if not container: return ["openclaw"] - return ["docker", "exec", container, "openclaw"] + kind, target = runtime_kind_target(container) + if kind == "incus": + return ["incus", "exec", target, "--", "openclaw"] + return ["docker", "exec", target, "openclaw"] def _start_gateway_process(profile: str, container: str | None = None, timeout_s: int = 60) -> PreflightCheck: @@ -687,7 +702,7 @@ def _start_gateway_process(profile: str, container: str | None = None, timeout_s output = f"{start.stdout}\n{start.stderr}".strip() if start.returncode != 0: return PreflightCheck("openclaw_gateway_start", "fail", _trim_output(output)) - return PreflightCheck("openclaw_gateway_start", "pass", _trim_output(output) or "started detached gateway in container") + return PreflightCheck("openclaw_gateway_start", "pass", _trim_output(output) or "started detached gateway in runtime") start_cmd = _gateway_start_cmd(profile, None) try: @@ -709,11 +724,34 @@ def _start_gateway_process(profile: str, container: str | None = None, timeout_s def _gateway_start_cmd(profile: str, container: str | None = None) -> list[str]: if container: - return ["docker", "exec", "-d", container, "sh", "-lc", gateway_run_command(profile, DEFAULT_GATEWAY_PORT)] + kind, target = runtime_kind_target(container) + command = gateway_run_command(profile, DEFAULT_GATEWAY_PORT) + if kind == "incus": + return [ + "incus", + "exec", + target, + "--", + "sh", + "-lc", + f"nohup {command} >{_runtime_gateway_log_path(profile)} 2>&1 PreflightCheck: +def _runtime_gateway_log_path(profile: str) -> str: + return f"/tmp/oc-bench-gateway-{_safe_path_part(profile)}.log" + + +def _stop_foreground_gateway(profile: str, timeout_s: int = 30, container: str | None = None) -> PreflightCheck: + if container: + kind, _ = runtime_kind_target(container) + return PreflightCheck( + "openclaw_gateway_foreground_stop", + "warn", + f"{kind} runtime gateways are stopped through the runtime OpenClaw command", + ) pid_path = _gateway_pid_path(profile) if not pid_path.exists(): return PreflightCheck("openclaw_gateway_foreground_stop", "warn", f"no foreground pid file at {pid_path}") @@ -743,6 +781,25 @@ def _stop_foreground_gateway(profile: str, timeout_s: int = 30) -> PreflightChec return PreflightCheck("openclaw_gateway_foreground_stop", "fail", f"foreground gateway pid {pid} did not stop within {timeout_s}s") +def _stop_runtime_gateway_process(profile: str, container: str | None, timeout_s: int) -> PreflightCheck | None: + if not container: + return None + kind, _ = runtime_kind_target(container) + pattern = f"openclaw.*--profile {shlex.quote(profile)}.*gateway.*run" + cmd = [*runtime_exec_prefix(container), "sh", "-lc", f"pkill -f {shlex.quote(pattern)}"] + try: + proc = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout_s, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + return PreflightCheck("openclaw_gateway_runtime_stop", "fail", str(exc)) + output = f"{proc.stdout}\n{proc.stderr}".strip() + if proc.returncode in {0, 1}: + note = f"stopped {kind} runtime gateway for profile {profile}" + if proc.returncode == 1: + note = f"no {kind} runtime gateway process matched profile {profile}" + return PreflightCheck("openclaw_gateway_runtime_stop", "pass", note) + return PreflightCheck("openclaw_gateway_runtime_stop", "fail", _trim_output(output)) + + def _gateway_pid_path(profile: str) -> Path: return Path(tempfile.gettempdir()) / f"openclaw-bench-gateway-{_safe_path_part(profile)}.pid" @@ -832,16 +889,18 @@ def _check_provider_health( url = base_url.rstrip("/") + "/models" if container is None: probe = LocalProbe() - elif container_kind == "docker": - probe = DockerExecProbe(container) - elif container_kind == "incus": - probe = IncusExecProbe(container) else: - return PreflightCheck( - "provider_health", - "fail", - f"unsupported container_kind={container_kind!r}; use incus or docker", - ) + container_kind, container = runtime_kind_target(container) + if container_kind == "docker": + probe = DockerExecProbe(container) + elif container_kind == "incus": + probe = IncusExecProbe(container) + else: + return PreflightCheck( + "provider_health", + "fail", + f"unsupported container_kind={container_kind!r}; use incus or docker", + ) try: result = probe.http_get( url, diff --git a/openclaw_bench/providers/detect.py b/openclaw_bench/providers/detect.py index 62a2661..02c5e22 100644 --- a/openclaw_bench/providers/detect.py +++ b/openclaw_bench/providers/detect.py @@ -2,6 +2,7 @@ import json import os +import subprocess import time from dataclasses import dataclass, field from pathlib import Path @@ -15,7 +16,7 @@ class ProviderCandidate: base_url: str models: list[str] probe_results: dict[str, ProbeResult] - source: str # "already_known" | "port_probe" + source: str # "already_known" | "port_probe" | "explicit" # Only set when source == "already_known": absolute path to the source # OC profile's openclaw.json. Lets the inherit path clone the full # profile config rather than regenerate it from scratch. @@ -141,7 +142,7 @@ def run_detection( candidates: list[ProviderCandidate] = [] findings: list[str] = [] - already_known = scan_existing_oc_profiles(home) + already_known = scan_existing_oc_profiles(home, probes=probes) known_by_provider = {c.provider: c for c in already_known} for provider in providers: @@ -221,7 +222,7 @@ def _probe_from_override(spec: str) -> Probe: ) -def scan_existing_oc_profiles(home: Path) -> list[ProviderCandidate]: +def scan_existing_oc_profiles(home: Path, probes: list | None = None) -> list[ProviderCandidate]: home = Path(home).expanduser() candidates: list[ProviderCandidate] = [] for profile_dir in sorted(home.glob(".openclaw*")): @@ -229,29 +230,83 @@ def scan_existing_oc_profiles(home: Path) -> list[ProviderCandidate]: if not config_path.is_file(): continue try: - payload = json.loads(config_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - continue - providers = (((payload or {}).get("models") or {}).get("providers") or {}) - for key, block in providers.items(): - if key not in KNOWN_PROVIDERS: - continue - base_url = (block or {}).get("baseUrl") - if not isinstance(base_url, str): - continue - model_ids: list[str] = [] - for entry in (block or {}).get("models") or []: - model_id = (entry or {}).get("id") if isinstance(entry, dict) else None - if isinstance(model_id, str): - model_ids.append(model_id) - candidates.append( - ProviderCandidate( - provider=key, - base_url=base_url, - models=model_ids, - probe_results={}, - source="already_known", + candidates.extend( + _candidates_from_config_text( + config_path.read_text(encoding="utf-8"), source_profile_path=str(config_path), ) ) + except (OSError, json.JSONDecodeError): + continue + for probe in probes or []: + candidates.extend(_scan_runtime_oc_profiles(probe)) + return candidates + + +def _candidates_from_config_text(text: str, *, source_profile_path: str | None = None) -> list[ProviderCandidate]: + payload = json.loads(text) + candidates: list[ProviderCandidate] = [] + providers = (((payload or {}).get("models") or {}).get("providers") or {}) + for key, block in providers.items(): + if key not in KNOWN_PROVIDERS: + continue + base_url = (block or {}).get("baseUrl") + if not isinstance(base_url, str): + continue + model_ids: list[str] = [] + for entry in (block or {}).get("models") or []: + model_id = (entry or {}).get("id") if isinstance(entry, dict) else None + if isinstance(model_id, str): + model_ids.append(model_id) + candidates.append( + ProviderCandidate( + provider=key, + base_url=base_url, + models=model_ids, + probe_results={}, + source="already_known", + source_profile_path=source_profile_path, + ) + ) return candidates + + +def _scan_runtime_oc_profiles(probe) -> list[ProviderCandidate]: + from .probes import DockerExecProbe, IncusExecProbe # noqa: PLC0415 + + if isinstance(probe, IncusExecProbe): + cmd = ["incus", "exec", probe.instance, "--", "sh", "-lc", _profile_scan_script()] + elif isinstance(probe, DockerExecProbe): + cmd = ["docker", "exec", probe.container, "sh", "-lc", _profile_scan_script()] + else: + return [] + try: + proc = subprocess.run(cmd, text=True, capture_output=True, timeout=10, check=False) + except (OSError, subprocess.TimeoutExpired): + return [] + if proc.returncode != 0: + return [] + candidates: list[ProviderCandidate] = [] + for text in _split_profile_scan_output(proc.stdout): + try: + candidates.extend(_candidates_from_config_text(text)) + except json.JSONDecodeError: + continue + return candidates + + +def _profile_scan_script() -> str: + return r''' +for base in "$HOME" /home/* /root; do + [ -d "$base" ] || continue + for f in "$base"/.openclaw*/openclaw.json; do + [ -f "$f" ] || continue + printf '\n---OC_BENCH_PROFILE---\n' + cat "$f" + done +done +''' + + +def _split_profile_scan_output(output: str) -> list[str]: + return [part.strip() for part in output.split("---OC_BENCH_PROFILE---") if part.strip()] diff --git a/openclaw_bench/quickstart.py b/openclaw_bench/quickstart.py index a15ff87..82e8458 100644 --- a/openclaw_bench/quickstart.py +++ b/openclaw_bench/quickstart.py @@ -109,7 +109,11 @@ def init_quickstart( detected_candidate: "ProviderCandidate | None" = None, ) -> QuickstartInitResult: provider_mode = normalize_provider_selection(providers) - if detected_candidate is not None and detected_candidate.provider == "vllm" and detected_candidate.models: + if detected_candidate is not None and detected_candidate.provider != "vllm": + raise ValueError(f"detected provider {detected_candidate.provider!r} is not supported by oc-bench init yet") + if detected_candidate is not None and not detected_candidate.models: + raise ValueError(f"detected provider {detected_candidate.provider!r} did not report any models") + if detected_candidate is not None and detected_candidate.provider == "vllm": vllm = VllmEndpoint( base_url=detected_candidate.base_url, model=detected_candidate.models[0], @@ -177,15 +181,23 @@ def init_quickstart( ) -def start_benchclaw_gateway(profile: str = DEFAULT_PROFILE, timeout_s: int = 60) -> PreflightCheck: - version = check_openclaw_version() +def start_benchclaw_gateway( + profile: str = DEFAULT_PROFILE, + container: str | None = None, + timeout_s: int = 60, +) -> PreflightCheck: + version = check_openclaw_version(container) if version.status == "fail": return version - return ensure_openclaw_gateway(profile, None, timeout_s=timeout_s) + return ensure_openclaw_gateway(profile, container, timeout_s=timeout_s) -def stop_benchclaw_gateway(profile: str = DEFAULT_PROFILE, timeout_s: int = 30) -> PreflightCheck: - return stop_openclaw_gateway(profile, timeout_s=timeout_s) +def stop_benchclaw_gateway( + profile: str = DEFAULT_PROFILE, + container: str | None = None, + timeout_s: int = 30, +) -> PreflightCheck: + return stop_openclaw_gateway(profile, container, timeout_s=timeout_s) def validate_benchclaw_config(profile: str, home: Path | None = None, timeout_s: int = 15) -> PreflightCheck: @@ -292,7 +304,7 @@ def _openclaw_config(providers: str, project_root: Path, port: int, agent: str, provider_config = {} plugin_entries = {} if providers in {"local", "both"}: - provider_config["vllm"] = _vllm_provider_config(vllm) + provider_config["vllm"] = _generate_vllm_route_config(vllm) plugin_entries["vllm"] = {"enabled": True} if providers in {"api", "both"}: provider_config["openai"] = _read_asset_json(project_root, "openclaw-config", "openai-provider.example.json") @@ -307,10 +319,7 @@ def _openclaw_config(providers: str, project_root: Path, port: int, agent: str, if providers in {"local", "both"}: agent_defaults["models"] = { vllm.route_model: { - "params": { - "maxTokens": vllm.max_tokens, - "chatTemplateKwargs": {"enable_thinking": False}, - } + "params": _vllm_agent_params(vllm) } } @@ -427,6 +436,35 @@ def _vllm_provider_config(vllm: VllmEndpoint) -> dict: } +def _vllm_candidate(vllm: VllmEndpoint) -> "ProviderCandidate": + from .providers.detect import ProviderCandidate # noqa: PLC0415 + + return ProviderCandidate( + provider="vllm", + base_url=vllm.base_url, + models=[vllm.model], + probe_results={}, + source="explicit", + ) + + +def _generate_vllm_route_config(vllm: VllmEndpoint) -> dict: + from .providers import vllm as vllm_provider # noqa: PLC0415 + + return vllm_provider.generate_route_config( + _vllm_candidate(vllm), + context=vllm.context, + max_tokens=vllm.max_tokens, + ) + + +def _vllm_agent_params(vllm: VllmEndpoint) -> dict: + from .providers import vllm as vllm_provider # noqa: PLC0415 + + params = vllm_provider.parameter_shaping(_vllm_candidate(vllm)) + return {"maxTokens": vllm.max_tokens, **params} + + def _openclaw_route_context(context: int) -> int: return max(context, OPENCLAW_MIN_MODEL_CONTEXT) diff --git a/openclaw_bench/scoring.py b/openclaw_bench/scoring.py index 01efb43..4dea3c7 100644 --- a/openclaw_bench/scoring.py +++ b/openclaw_bench/scoring.py @@ -38,10 +38,13 @@ def run_verify_command(workspace: Path, command: list[str], timeout_s: int = 60) def score_task(task: TaskSpec, workspace: Path, response: BackendResponse, changed: list[str], tests_passed: bool) -> tuple[float, str | None, int, str]: hallucinated = _hallucinated_paths(workspace, response) if hallucinated: + # Intent: fail answers that cite files the workspace does not contain. return 0.0, "hallucinated_file", hallucinated, "response referenced nonexistent workspace paths" if response.timed_out: + # Intent: separate backend timeouts from ordinary wrong or incomplete answers. return 0.0, "openclaw_timeout", hallucinated, "backend timed out" if response.error: + # Intent: preserve known backend failure taxonomy while quarantining unexpected errors. failure_type = response.error if response.error in FAILURE_TYPES else "unknown" return 0.0, failure_type, hallucinated, response.error @@ -110,6 +113,7 @@ def score_task(task: TaskSpec, workspace: Path, response: BackendResponse, chang # Intent: preserve strict compact JSON after a long enough tool chain to expose format drift. checks.extend(_score_format_drift_under_length(task, workspace, response, changed)) else: + # Intent: fail closed when a manifest names a task type the scorer does not implement. checks.append((False, f"unknown task type {task.task_type}")) if len(changed) > task.max_changed_files: @@ -676,7 +680,7 @@ def _failure_type(task: TaskSpec, failed_notes: list[str], tests_passed: bool, h return "instruction_violation" if "policy file" in text or "OpenClaw seed" in text or "changed_files missing target file" in text: return "instruction_violation" - if "action gate" in text or "decision did not match" in text or "preserved file" in text: + if "action gate" in text or "decision did not match" in text: return "instruction_violation" if "read-only task changed files" in text or "tests were edited" in text or "dependencies were edited" in text or "helper" in text: return "instruction_violation" diff --git a/tests/test_backend.py b/tests/test_backend.py index c29b0e6..e9f5df0 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -95,6 +95,16 @@ def test_openclaw_container_smoke_uses_docker_exec(self): self.assertEqual(cmd[:4], ["docker", "exec", "oc-bench-gateway", "openclaw"]) self.assertIn("--profile", cmd) + def test_openclaw_incus_runtime_smoke_uses_incus_exec(self): + backend = OpenClawBackend(profile="bench", container="incus:oc-stack") + model = ModelSpec(model_id="model", served_model_name="provider/model") + with patch("subprocess.Popen", return_value=_FakeProcess(returncode=0, stdout='{"text":"ok"}', stderr="")) as popen_mock: + response = backend.smoke(model, timeout_s=1) + self.assertIsNone(response.error) + cmd = popen_mock.call_args.args[0] + self.assertEqual(cmd[:5], ["incus", "exec", "oc-stack", "--", "openclaw"]) + self.assertIn("--profile", cmd) + def test_openclaw_error_counts_as_request_error(self): backend = OpenClawBackend(local=True) model = ModelSpec(model_id="model", served_model_name="provider/model") @@ -210,6 +220,21 @@ def test_openclaw_container_agent_run_sets_container_workdir(self): self.assertEqual(cmd[4:6], ["oc-bench-gateway", "openclaw"]) self.assertIsNone(popen_mock.call_args.kwargs["cwd"]) + def test_openclaw_incus_runtime_agent_run_sets_runtime_cwd(self): + backend = OpenClawBackend(profile="bench", container="incus:oc-stack") + model = ModelSpec(model_id="model", served_model_name="served", openclaw_model_name="vllm/served") + task = _Task() + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + workspace.mkdir() + with patch("subprocess.Popen", return_value=_FakeProcess(returncode=0, stdout='{"text":"ok"}', stderr="")) as popen_mock: + response = backend.run(model, task, workspace, "run-w000-task-abcdef", timeout_s=30) + self.assertIsNone(response.error) + cmd = popen_mock.call_args.args[0] + self.assertEqual(cmd[:5], ["incus", "exec", "oc-stack", "--cwd", str(workspace)]) + self.assertEqual(cmd[5:7], ["--", "openclaw"]) + self.assertIsNone(popen_mock.call_args.kwargs["cwd"]) + def test_workspace_agent_setup_failure_is_reported_before_attempt(self): backend = OpenClawBackend(profile="bench", workspace_agents=True) model = ModelSpec(model_id="model", served_model_name="served", openclaw_model_name="vllm/served") diff --git a/tests/test_container.py b/tests/test_container.py index 4831277..1cf4e4f 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -5,11 +5,17 @@ from pathlib import Path from unittest.mock import patch -from openclaw_bench.container import _docker_run_command, ensure_openclaw_container +from openclaw_bench.container import _docker_run_command, ensure_openclaw_container, runtime_exec_prefix, runtime_kind_target from openclaw_bench.models import ModelSpec class ContainerEnsureTests(unittest.TestCase): + def test_runtime_kind_target_treats_plain_container_as_docker(self): + self.assertEqual(runtime_kind_target("oc-bench-gateway"), ("docker", "oc-bench-gateway")) + + def test_runtime_exec_prefix_supports_incus(self): + self.assertEqual(runtime_exec_prefix("incus:oc-stack"), ["incus", "exec", "oc-stack", "--"]) + def test_existing_running_container_is_reused(self): completed = subprocess.CompletedProcess(args=[], returncode=0, stdout="true\n", stderr="") with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 81924bd..92cd8af 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -11,9 +11,11 @@ from openclaw_bench.preflight import ( OPENCLAW_PINNED_VERSION, _check_gateway, + _gateway_start_cmd, check_openclaw_version, ensure_openclaw_gateway, run_preflight, + stop_openclaw_gateway, ) ROOT = Path(__file__).resolve().parent.parent @@ -370,6 +372,48 @@ def test_ensure_gateway_respects_openclaw_container(self): self.assertIn("--profile bench gateway --port 19091", calls[1][-1]) self.assertIn("--verbose", calls[1][-1]) + def test_openclaw_version_uses_incus_runtime(self): + completed = subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=f"openclaw {OPENCLAW_PINNED_VERSION}\n", + stderr="", + ) + with patch("subprocess.run", return_value=completed) as run_mock: + check = check_openclaw_version("incus:oc-stack") + self.assertEqual(check.status, "pass") + self.assertEqual( + run_mock.call_args.args[0], + ["incus", "exec", "oc-stack", "--", "openclaw", "--version"], + ) + + def test_gateway_start_command_uses_incus_detached_shell(self): + cmd = _gateway_start_cmd("bench", "incus:oc-stack") + self.assertEqual(cmd[:5], ["incus", "exec", "oc-stack", "--", "sh"]) + self.assertIn("--profile bench gateway --port 19091", cmd[-1]) + self.assertIn("/tmp/oc-bench-gateway-bench.log", cmd[-1]) + self.assertIn("nohup", cmd[-1]) + self.assertIn("&", cmd[-1]) + + def test_stop_gateway_uses_incus_runtime_command(self): + stopped = subprocess.CompletedProcess(args=[], returncode=0, stdout="stopped\n", stderr="") + with patch("subprocess.run", return_value=stopped) as run_mock: + check = stop_openclaw_gateway("bench", "incus:oc-stack") + self.assertEqual(check.status, "pass") + self.assertEqual( + run_mock.call_args.args[0], + ["incus", "exec", "oc-stack", "--", "openclaw", "--profile", "bench", "gateway", "stop"], + ) + + def test_stop_gateway_falls_back_to_runtime_process_kill(self): + stop_failed = subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr="not running") + killed = subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + with patch("subprocess.run", side_effect=[stop_failed, killed]) as run_mock: + check = stop_openclaw_gateway("bench", "incus:oc-stack") + self.assertEqual(check.status, "pass") + self.assertEqual(run_mock.call_args_list[1].args[0][:5], ["incus", "exec", "oc-stack", "--", "sh"]) + self.assertIn("pkill -f", run_mock.call_args_list[1].args[0][-1]) + def test_container_preflight_checks_container_openclaw_and_route_smoke(self): suite = load_suite(ROOT / "manifests" / "openclaw-agent-core.json") models = [ diff --git a/tests/test_providers_detect.py b/tests/test_providers_detect.py index 0e8ab8c..20052aa 100644 --- a/tests/test_providers_detect.py +++ b/tests/test_providers_detect.py @@ -76,6 +76,37 @@ def test_scan_handles_unknown_provider_keys_gracefully(self): entries = scan_existing_oc_profiles(home) self.assertEqual(entries, []) + def test_scan_reads_incus_runtime_profiles(self): + output = """ +---OC_BENCH_PROFILE--- +{"models":{"providers":{"vllm":{"baseUrl":"http://10.68.198.1:8000/v1","models":[{"id":"gpt-oss-20b"}]}}}} +""" + completed = unittest.mock.Mock(returncode=0, stdout=output, stderr="") + from openclaw_bench.providers.probes import IncusExecProbe + + with tempfile.TemporaryDirectory() as tmp: + with unittest.mock.patch("subprocess.run", return_value=completed) as run_mock: + entries = scan_existing_oc_profiles(Path(tmp), probes=[IncusExecProbe("oc-stack")]) + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0].provider, "vllm") + self.assertEqual(entries[0].models, ["gpt-oss-20b"]) + self.assertEqual(run_mock.call_args.args[0][:4], ["incus", "exec", "oc-stack", "--"]) + + def test_scan_reads_docker_runtime_profiles(self): + output = """ +---OC_BENCH_PROFILE--- +{"models":{"providers":{"ollama":{"baseUrl":"http://127.0.0.1:11434","models":[{"id":"llama3.1:8b"}]}}}} +""" + completed = unittest.mock.Mock(returncode=0, stdout=output, stderr="") + from openclaw_bench.providers.probes import DockerExecProbe + + with tempfile.TemporaryDirectory() as tmp: + with unittest.mock.patch("subprocess.run", return_value=completed) as run_mock: + entries = scan_existing_oc_profiles(Path(tmp), probes=[DockerExecProbe("oc")]) + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0].provider, "ollama") + self.assertEqual(run_mock.call_args.args[0][:3], ["docker", "exec", "oc"]) + from unittest.mock import MagicMock @@ -186,6 +217,24 @@ def test_port_probe_runs_when_nothing_already_known(self): self.assertEqual(report.candidates[0].source, "port_probe") self.assertEqual(report.candidates[0].models, ["gpt-oss-20b"]) + def test_runtime_already_known_short_circuits_port_probe(self): + output = """ +---OC_BENCH_PROFILE--- +{"models":{"providers":{"vllm":{"baseUrl":"http://10.68.198.1:8000/v1","models":[{"id":"gpt-oss-20b"}]}}}} +""" + completed = unittest.mock.Mock(returncode=0, stdout=output, stderr="") + from openclaw_bench.providers.probes import IncusExecProbe + + with tempfile.TemporaryDirectory() as tmp: + host = MagicMock() + host.name = "host" + runtime = IncusExecProbe("oc-stack") + with unittest.mock.patch("subprocess.run", return_value=completed): + report = run_detection(providers=["vllm"], probes=[host, runtime], home=Path(tmp)) + self.assertEqual(len(report.candidates), 1) + self.assertEqual(report.candidates[0].source, "already_known") + host.http_get.assert_not_called() + def test_zero_candidates_yields_empty_report(self): with tempfile.TemporaryDirectory() as tmp: home = Path(tmp) diff --git a/tests/test_providers_init_integration.py b/tests/test_providers_init_integration.py index 2375de2..8783c1e 100644 --- a/tests/test_providers_init_integration.py +++ b/tests/test_providers_init_integration.py @@ -1,6 +1,8 @@ import json import tempfile import unittest +from contextlib import redirect_stdout +import io from pathlib import Path from unittest.mock import patch @@ -67,6 +69,76 @@ def test_no_detect_falls_back_to_env_var_defaults(self): config["models"]["providers"]["vllm"]["models"][0]["id"], "fallback-model" ) + def test_no_detect_vllm_flags_use_provider_parameter_shaping(self): + with tempfile.TemporaryDirectory() as tmp: + home = Path(tmp) / "home" + bench = Path(tmp) / "bench" + exit_code = cli_main( + [ + "init", + "--providers", "local", + "--no-detect", + "--vllm-base-url", "http://10.68.198.1:8000/v1", + "--vllm-model", "gpt-oss-20b", + "--bench-root", str(bench), + "--config-home", str(home), + "--gateway-port", "19223", + "--no-validate", + ] + ) + self.assertEqual(exit_code, 0) + config = json.loads((home / ".openclaw-benchclaw" / "openclaw.json").read_text()) + params = config["agents"]["defaults"]["models"]["vllm/gpt-oss-20b"]["params"] + self.assertEqual(params["extra_body"], {"reasoning_effort": "low"}) + + def test_init_rejects_detect_only_provider_without_default_vllm_fallback(self): + candidate = ProviderCandidate( + provider="ollama", + base_url="http://127.0.0.1:11434", + models=["llama3.1:8b"], + probe_results={"host": _ok("{}")}, + source="port_probe", + ) + report = DetectionReport(candidates=(candidate,), findings=()) + with tempfile.TemporaryDirectory() as tmp: + home = Path(tmp) / "home" + bench = Path(tmp) / "bench" + buf = io.StringIO() + with patch("openclaw_bench.cli.run_detection", return_value=report), redirect_stdout(buf): + exit_code = cli_main( + [ + "init", + "--providers", "local", + "--bench-root", str(bench), + "--config-home", str(home), + "--gateway-port", "19225", + "--no-validate", + ] + ) + self.assertEqual(exit_code, 2) + self.assertFalse((home / ".openclaw-benchclaw" / "openclaw.json").exists()) + self.assertIn("cannot generate yet", buf.getvalue()) + self.assertIn("ollama", buf.getvalue()) + + def test_init_forwards_probe_hosts_to_detection(self): + report = DetectionReport(candidates=(), findings=()) + with tempfile.TemporaryDirectory() as tmp: + home = Path(tmp) / "home" + bench = Path(tmp) / "bench" + with patch("openclaw_bench.cli.run_detection", return_value=report) as detect_mock: + cli_main( + [ + "init", + "--providers", "local", + "--bench-root", str(bench), + "--config-home", str(home), + "--gateway-port", "19226", + "--probe-hosts", "127.0.0.1,10.68.198.1", + "--no-validate", + ] + ) + self.assertEqual(detect_mock.call_args.kwargs["probe_hosts"], ["127.0.0.1", "10.68.198.1"]) + def test_zero_candidates_with_detect_aborts_with_clear_error(self): report = DetectionReport(candidates=(), findings=()) with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/test_providers_preflight.py b/tests/test_providers_preflight.py index c384884..7cd9ad6 100644 --- a/tests/test_providers_preflight.py +++ b/tests/test_providers_preflight.py @@ -132,3 +132,20 @@ def test_authenticated_200_returns_pass(self): "http://10.68.198.1:8000/v1", None, 5, provider="vllm" ) self.assertEqual(check.status, "pass") + + def test_plain_container_name_uses_docker_probe(self): + success = ProbeResult( + ok=True, status_code=200, body='{"data":[]}', probe_name="docker:oc", error=None + ) + with patch("openclaw_bench.providers.probes.DockerExecProbe") as fake_probe_cls: + fake_probe = MagicMock() + fake_probe.http_get.return_value = success + fake_probe_cls.return_value = fake_probe + check = _check_provider_health( + "http://10.68.198.1:8000/v1", + "oc", + 5, + provider="vllm", + ) + self.assertEqual(check.status, "pass") + fake_probe_cls.assert_called_once_with("oc") diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py index 7acb2be..20bbddf 100644 --- a/tests/test_quickstart.py +++ b/tests/test_quickstart.py @@ -9,6 +9,7 @@ from openclaw_bench.cli import init_command, quickstart_command from openclaw_bench.preflight import PreflightCheck +from openclaw_bench.providers import ProviderCandidate from openclaw_bench.quickstart import ( DEFAULT_PROFILE, OPENCLAW_CONFIG_VERSION, @@ -64,7 +65,11 @@ def test_init_generates_isolated_benchclaw_config_and_starter_manifests(self): self.assertTrue(config["meta"]["lastTouchedAt"].endswith("Z")) self.assertEqual( config["agents"]["defaults"]["models"]["vllm/gpt-oss-20b-nvfp4-smoke"]["params"], - {"maxTokens": 256, "chatTemplateKwargs": {"enable_thinking": False}}, + { + "maxTokens": 256, + "chatTemplateKwargs": {"enable_thinking": False}, + "extra_body": {"reasoning_effort": "low"}, + }, ) model_config = json.loads(result.paths.model_config_path.read_text(encoding="utf-8")) @@ -116,6 +121,27 @@ def test_init_can_target_external_small_vllm_endpoint(self): self.assertEqual(metadata["vllm_context"], 2048) self.assertEqual(metadata["openclaw_route_context"], OPENCLAW_MIN_MODEL_CONTEXT) + def test_init_rejects_detected_provider_without_generator(self): + candidate = ProviderCandidate( + provider="ollama", + base_url="http://127.0.0.1:11434", + models=["llama3.1:8b"], + probe_results={}, + source="port_probe", + ) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + with self.assertRaisesRegex(ValueError, "not supported"): + init_quickstart( + providers="local", + project_root=ROOT, + bench_root=root / "bench", + home=root / "home", + port=19222, + validate=False, + detected_candidate=candidate, + ) + def test_init_refuses_to_overwrite_without_force(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -197,7 +223,7 @@ def test_quickstart_orchestrates_init_start_preflight_run_and_optional_stop(self start_mock.assert_called_once_with("benchclaw", timeout_s=60) preflight_mock.assert_called_once() run_mock.assert_called_once() - stop_mock.assert_called_once_with("benchclaw", timeout_s=30) + stop_mock.assert_called_once_with("benchclaw", container=None, timeout_s=30) self.assertEqual(run_mock.call_args.args[0].run_id, "starter") self.assertEqual(preflight_mock.call_args.args[0].fixtures_root, str(Path(tmp) / "bench" / "fixtures")) self.assertIn("result_path=", stdout.getvalue())