diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index be28fb5..1f174ad 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.3.2", + "version": "0.3.3", "keywords": [ "agentops", "evaluation", diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index be28fb5..1f174ad 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.3.2", + "version": "0.3.3", "keywords": [ "agentops", "evaluation", diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bb4259..8deaa3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,73 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] -## [0.3.2] - 2026-05-31 +## [0.3.3] - 2026-05-31 + +### Changed +- **Runtime dependencies now have upper bounds so a future SDK major release + cannot silently break installs.** `pyproject.toml` previously declared every + Azure-SDK dependency with only a lower bound (e.g. `azure-ai-projects>=2.0.1`), + so `pip install agentops-accelerator` could resolve `azure-ai-projects 3.x` + the day after that ships and break the agent-definition serialization (the + exact failure mode that produced the `invalid_payload — Required properties + ["kind"] are not present` regression below). Each Azure SDK dependency + (`azure-ai-projects`, `azure-ai-evaluation`, `azure-identity`, `azure-monitor-*`, + `azure-mgmt-*`) is now constrained to its current major. `pandas`, `fastapi`, + `uvicorn`, `httpx`, and `markdown` are similarly capped to their next major. + `cryptography` is intentionally left unbounded so security patches can flow + through without a coordinated AgentOps release. Lift any of these bounds via + an explicit PR that exercises the new SDK against `tests/`. + +- **`agentops workflow generate` now stamps the installed agentops version + into generated CI/CD templates instead of always installing from + `git+...@main`.** Every generated `agentops-pr.yml`, `agentops-deploy-*.yml`, + `agentops-watchdog.yml` (and their Azure DevOps pipeline equivalents) used to + contain `pip install "agentops-accelerator[...] @ git+https://github.com/Azure/agentops.git@main"`, + with no version pin and a stale "NOTE: pinned to GitHub main until the next + package release" comment. User CI runs were therefore non-reproducible: the + same workflow file pulled different agentops snapshots day to day, which is + how PO's recorded tutorial took a hard SDK regression mid-record. The + generator now writes a literal `==X.Y.Z` pin derived from the agentops version + currently installed on the machine running `agentops workflow generate` — so + a user who generates workflows against AgentOps `0.3.3` always installs + `agentops-accelerator==0.3.3` on every CI run, and `agentops-accelerator` + brings exact-major Azure SDKs along (per the upper bounds above). Editable + installs (versions carrying a local segment like `+gabcdef` or marked + `.devN`) keep the `@main` fallback so contributors testing template changes + still get a resolvable install. Existing user workflows are unaffected until + the user re-runs `agentops workflow generate --force` against a release of + AgentOps that ships this change. ### Fixed +- **Doctor regression check no longer flags the previous PR run as "current" + in CI.** The results-history loader (`agent/sources/results_history.py`) + was reading the wrong fields from `results.json` and excluding + `.agentops/results/latest/` from the candidate list. Three coordinated + schema-alignment fixes restore correctness: + 1. `_summarize` now reads top-level `aggregate_metrics` first (the field + the orchestrator actually writes, per `core/results.py`), then falls + back to legacy `metrics`/`run_metrics`. Previously the loader looked + only at the legacy fields, so every freshly-written local + `RunSummary` had `metrics = {}` and the regression check could never + see the current run's metrics. + 2. `_summarize` now reads `summary.overall_passed` first when deriving + the `run_pass` flag, then falls back to the legacy `summary.run_pass` + / `metrics.run_pass` shapes. + 3. `_summarize` now orders runs by `timestamp` → `finished_at` → + `started_at` → `created_at` → `summary.timestamp`. The previous list + omitted `finished_at`/`started_at`, which are the two fields + `results.json` actually contains, so every loaded run defaulted to + epoch-zero ordering. + 4. `_collect_local_runs` now includes `.agentops/results/latest/` when it + is the only local results directory. In CI, generated workflows run + `agentops eval run --output .agentops/results/latest` and write + nowhere else; the old loader unconditionally skipped `latest/` for + dev-mode dedup, so in CI `local_runs` was always empty. With cloud + listing trailing behind by seconds (eventual consistency), the + regression check would then compute `latest = previous_run` and + blame the just-completed candidate's coherence/groundedness on the + prior PR. Dev-mode dedup is preserved: when a timestamped sibling + exists, `latest/` is still skipped. - **Prompt-agent deploy: `stage` no longer fails with `Required properties ["kind"] are not present` against `azure-ai-projects` 2.x.** `_copy_definition` previously called `.copy()` on the typed `PromptAgentDefinition` returned by `get_version`. In SDK 1.x that diff --git a/plugins/agentops/package.json b/plugins/agentops/package.json index 5c48b3e..0486920 100644 --- a/plugins/agentops/package.json +++ b/plugins/agentops/package.json @@ -2,7 +2,7 @@ "name": "agentops-accelerator", "displayName": "AgentOps Accelerator — Skills for GitHub Copilot", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.3.2", + "version": "0.3.3", "publisher": "AgentOpsAccelerator", "icon": "icon.png", "license": "MIT", diff --git a/plugins/agentops/plugin.json b/plugins/agentops/plugin.json index 98ab434..f5174e9 100644 --- a/plugins/agentops/plugin.json +++ b/plugins/agentops/plugin.json @@ -1,7 +1,7 @@ { "name": "agentops-accelerator", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.3.2", + "version": "0.3.3", "author": { "name": "AgentOps Accelerator", "url": "https://github.com/Azure/agentops" diff --git a/pyproject.toml b/pyproject.toml index 493b7cc..6974c5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,29 +12,29 @@ dependencies = [ "typer>=0.12,<1.0", "pydantic>=2,<3", "ruamel.yaml>=0.18,<1.0", - "azure-ai-projects>=2.0.1", + "azure-ai-projects>=2.0.1,<3.0", ] license = { file = "LICENSE" } [project.optional-dependencies] mcp = ["mcp>=1.0,<2"] foundry = [ - "azure-ai-evaluation>=1.0", - "azure-identity>=1.17", - "azure-monitor-opentelemetry>=1.6", - "pandas>=2.0", + "azure-ai-evaluation>=1.0,<2.0", + "azure-identity>=1.17,<2.0", + "azure-monitor-opentelemetry>=1.6,<2.0", + "pandas>=2.0,<3.0", ] agent = [ - "fastapi>=0.110", - "uvicorn[standard]>=0.30", - "httpx>=0.27", + "fastapi>=0.110,<1.0", + "uvicorn[standard]>=0.30,<1.0", + "httpx>=0.27,<1.0", "cryptography>=42", - "markdown>=3.6", - "azure-monitor-query>=1.3", - "azure-monitor-opentelemetry>=1.6", - "azure-identity>=1.17", - "azure-mgmt-cognitiveservices>=13.5", - "azure-mgmt-monitor>=6.0", + "markdown>=3.6,<4.0", + "azure-monitor-query>=1.3,<2.0", + "azure-monitor-opentelemetry>=1.6,<2.0", + "azure-identity>=1.17,<2.0", + "azure-mgmt-cognitiveservices>=13.5,<14.0", + "azure-mgmt-monitor>=6.0,<7.0", ] [project.scripts] @@ -66,7 +66,7 @@ where = ["src"] [dependency-groups] dev = [ - "azure-ai-evaluation>=1.0", + "azure-ai-evaluation>=1.0,<2.0", "mypy>=1.19.1", "pre-commit>=4.0", "pytest>=8.0", diff --git a/src/agentops/agent/sources/results_history.py b/src/agentops/agent/sources/results_history.py index 966fb13..72c5f3e 100644 --- a/src/agentops/agent/sources/results_history.py +++ b/src/agentops/agent/sources/results_history.py @@ -74,7 +74,12 @@ def _summarize(path: Path) -> Optional[RunSummary]: if not isinstance(data, dict): return None - metrics_raw = data.get("metrics") or data.get("run_metrics") or {} + metrics_raw = ( + data.get("aggregate_metrics") + or data.get("metrics") + or data.get("run_metrics") + or {} + ) metrics: Dict[str, float] = {} if isinstance(metrics_raw, dict): for key, value in metrics_raw.items(): @@ -85,9 +90,11 @@ def _summarize(path: Path) -> Optional[RunSummary]: summary = data.get("summary") or {} run_pass: Optional[bool] = None - if isinstance(summary, dict) and "run_pass" in summary: + if isinstance(summary, dict) and "overall_passed" in summary: + run_pass = bool(summary["overall_passed"]) + elif isinstance(summary, dict) and "run_pass" in summary: run_pass = bool(summary["run_pass"]) - elif "run_pass" in metrics_raw: + elif isinstance(metrics_raw, dict) and "run_pass" in metrics_raw: try: run_pass = bool(float(metrics_raw["run_pass"])) except (TypeError, ValueError): @@ -105,6 +112,8 @@ def _summarize(path: Path) -> Optional[RunSummary]: timestamp_raw = ( data.get("timestamp") + or data.get("finished_at") + or data.get("started_at") or data.get("created_at") or (summary.get("timestamp") if isinstance(summary, dict) else None) ) @@ -184,14 +193,23 @@ def _collect_local_runs( return [] candidates: List[Path] = [] + latest_target: Optional[Path] = None for child in base.iterdir(): if not child.is_dir(): continue + target = child / "results.json" + if not target.is_file(): + continue if child.name == "latest": + latest_target = target continue - target = child / "results.json" - if target.is_file(): - candidates.append(target) + candidates.append(target) + # CI workflows write directly to `.agentops/results/latest/` with no + # timestamped sibling. Include the `latest/` entry only when it is the + # sole local result so dev-mode runs (which already have a timestamped + # sibling) are not double-counted. + if not candidates and latest_target is not None: + candidates.append(latest_target) summaries: List[RunSummary] = [] for path in candidates: diff --git a/src/agentops/cli/app.py b/src/agentops/cli/app.py index 3eeb6c5..edeff00 100644 --- a/src/agentops/cli/app.py +++ b/src/agentops/cli/app.py @@ -14,7 +14,7 @@ from pathlib import Path from textwrap import wrap from collections.abc import Sequence -from typing import Annotated, Optional +from typing import Annotated, Any, Optional import typer @@ -1554,7 +1554,8 @@ def _prompt(question: str, default: Optional[str]) -> str: } def _on_answer(field_name: str, value: str) -> None: - partial = WizardAnswers(**{field_name: value}) + partial_kwargs: dict[str, Any] = {field_name: value} + partial = WizardAnswers(**partial_kwargs) try: partial_result = apply_answers( workspace, diff --git a/src/agentops/services/cicd.py b/src/agentops/services/cicd.py index 5b87947..d9b711d 100644 --- a/src/agentops/services/cicd.py +++ b/src/agentops/services/cicd.py @@ -27,6 +27,42 @@ _CLOUD_EVAL_CONFIG_NAME = ".agentops.cloud.yaml" _CI_EVAL_OUTPUT = ".agentops/results/latest" +# Git ref used by the dev/editable-install fallback when stamping the agentops +# package version into generated CI/CD templates. Kept here so tests can target +# the same constant without parsing template output. +AGENTOPS_DEV_INSTALL_SPEC = " @ git+https://github.com/Azure/agentops.git@main" + + +def _agentops_install_spec(version: str | None = None) -> str: + """Return the pip version-spec suffix used in generated workflows. + + For a clean public release (e.g. ``0.3.2``, ``0.3.2.post1``, ``0.3.2rc1``) + we pin the exact version so user CI runs are reproducible regardless of + when they are triggered. For editable or dev installs (versions carrying + a local segment such as ``+gabcdef`` or marked as dev-releases) we fall + back to ``@main`` so contributors testing template changes still get a + resolvable install. + """ + + if version is None: + from agentops import __version__ + + resolved = __version__ + else: + resolved = version + try: + from packaging.version import InvalidVersion, Version + except ImportError: # pragma: no cover - packaging ships with pip/setuptools + return AGENTOPS_DEV_INSTALL_SPEC + + try: + parsed = Version(resolved) + except InvalidVersion: + return AGENTOPS_DEV_INSTALL_SPEC + + if parsed.local is not None or parsed.is_devrelease: + return AGENTOPS_DEV_INSTALL_SPEC + return f"=={resolved}" # CI/CD platforms supported by ``agentops workflow generate``. PLATFORMS: Tuple[str, ...] = ("github", "azure-devops") @@ -648,7 +684,10 @@ def generate_cicd_workflows( continue seen.add(kind) result.kinds.append(kind) - substitutions: dict[str, str] = {"__DOCTOR_GATE__": doctor_gate} + substitutions: dict[str, str] = { + "__DOCTOR_GATE__": doctor_gate, + "__AGENTOPS_INSTALL_SPEC__": _agentops_install_spec(), + } eval_config = ( "${{ inputs.config || 'agentops.yaml' }}" if platform == "github" and kind == "pr" diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-dev-azd.yml b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-dev-azd.yml index f542678..fd677fd 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-dev-azd.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-dev-azd.yml @@ -74,7 +74,7 @@ __AILZ_PREFLIGHT_COMMAND__ versionSpec: "3.11" - bash: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit __EVAL_TASKS__ diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-dev.yml b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-dev.yml index 32cccec..ffb8464 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-dev.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-dev.yml @@ -50,7 +50,7 @@ stages: - bash: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit __EVAL_TASKS__ diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prod-azd.yml b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prod-azd.yml index adf3f12..51a6c7e 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prod-azd.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prod-azd.yml @@ -74,7 +74,7 @@ __AILZ_PREFLIGHT_COMMAND__ versionSpec: "3.11" - bash: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit __EVAL_TASKS__ diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prod.yml b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prod.yml index 8f5ecdc..c61d6c6 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prod.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prod.yml @@ -48,7 +48,7 @@ stages: - bash: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit __EVAL_TASKS__ diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prompt-agent.yml b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prompt-agent.yml index 1ecdb2c..46c49d3 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prompt-agent.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-prompt-agent.yml @@ -48,7 +48,7 @@ stages: scriptLocation: inlineScript inlineScript: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" python -m agentops.pipeline.prompt_deploy stage \ --config "$(AGENTOPS_CONFIG)" \ --environment "$(TARGET_ENVIRONMENT)" \ @@ -93,7 +93,7 @@ stages: - bash: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit __EVAL_TASKS__ @@ -153,7 +153,7 @@ __EVAL_TASKS__ versionSpec: "3.11" - bash: | - python -m pip install "agentops-accelerator @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator__AGENTOPS_INSTALL_SPEC__" python -m agentops.pipeline.prompt_deploy summarize \ --deployment ".agentops/deployments/foundry-agent.json" \ --environment "$(TARGET_ENVIRONMENT)" diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-qa-azd.yml b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-qa-azd.yml index c6d576d..84c590a 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-qa-azd.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-qa-azd.yml @@ -74,7 +74,7 @@ __AILZ_PREFLIGHT_COMMAND__ versionSpec: "3.11" - bash: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit __EVAL_TASKS__ diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-qa.yml b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-qa.yml index 6eaf494..5095bc1 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-deploy-qa.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-deploy-qa.yml @@ -46,7 +46,7 @@ stages: - bash: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit __EVAL_TASKS__ diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-pr-prompt-agent.yml b/src/agentops/templates/pipelines/azuredevops/agentops-pr-prompt-agent.yml index 0936dcf..02dfdf0 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-pr-prompt-agent.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-pr-prompt-agent.yml @@ -72,7 +72,7 @@ stages: scriptLocation: inlineScript inlineScript: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" python -m agentops.pipeline.prompt_deploy stage \ --config "$(AGENTOPS_CONFIG)" \ --environment "$(TARGET_ENVIRONMENT)" \ @@ -113,7 +113,7 @@ stages: - bash: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit __EVAL_TASKS__ diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-pr.yml b/src/agentops/templates/pipelines/azuredevops/agentops-pr.yml index 8cf3648..cb19f45 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-pr.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-pr.yml @@ -53,8 +53,7 @@ stages: - bash: | python -m pip install --upgrade pip - # NOTE: pinned to GitHub main until the next package release includes this flow. - python -m pip install "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit __EVAL_TASKS__ diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-watchdog.yml b/src/agentops/templates/pipelines/azuredevops/agentops-watchdog.yml index 186497f..d42062b 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-watchdog.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-watchdog.yml @@ -65,7 +65,7 @@ stages: scriptLocation: inlineScript inlineScript: | python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" agentops doctor --workspace . --out .agentops/agent/report.md \ --severity-fail critical --evidence-pack env: diff --git a/src/agentops/templates/workflows/agentops-deploy-dev-azd.yml b/src/agentops/templates/workflows/agentops-deploy-dev-azd.yml index c217bf6..d49e9d3 100644 --- a/src/agentops/templates/workflows/agentops-deploy-dev-azd.yml +++ b/src/agentops/templates/workflows/agentops-deploy-dev-azd.yml @@ -114,7 +114,7 @@ __AILZ_PREFLIGHT_COMMAND__ - name: Install AgentOps Toolkit run: | - uv pip install --system "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" __EVAL_STEPS__ diff --git a/src/agentops/templates/workflows/agentops-deploy-dev.yml b/src/agentops/templates/workflows/agentops-deploy-dev.yml index a598dfc..1e77df2 100644 --- a/src/agentops/templates/workflows/agentops-deploy-dev.yml +++ b/src/agentops/templates/workflows/agentops-deploy-dev.yml @@ -66,9 +66,7 @@ jobs: - name: Install AgentOps Toolkit run: | - # NOTE: pinned to GitHub main until the next package release includes this flow. - # Switch to `uv pip install --system "agentops-accelerator[foundry]"` after release. - uv pip install --system "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" __EVAL_STEPS__ diff --git a/src/agentops/templates/workflows/agentops-deploy-prod-azd.yml b/src/agentops/templates/workflows/agentops-deploy-prod-azd.yml index cfa9c2d..df0d1bc 100644 --- a/src/agentops/templates/workflows/agentops-deploy-prod-azd.yml +++ b/src/agentops/templates/workflows/agentops-deploy-prod-azd.yml @@ -106,7 +106,7 @@ __AILZ_PREFLIGHT_COMMAND__ enable-cache: false - name: Install AgentOps Toolkit - run: uv pip install --system "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + run: uv pip install --system "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" __EVAL_STEPS__ - name: Generate release evidence if: always() diff --git a/src/agentops/templates/workflows/agentops-deploy-prod.yml b/src/agentops/templates/workflows/agentops-deploy-prod.yml index 2dd3bb9..039cdad 100644 --- a/src/agentops/templates/workflows/agentops-deploy-prod.yml +++ b/src/agentops/templates/workflows/agentops-deploy-prod.yml @@ -73,9 +73,7 @@ jobs: - name: Install AgentOps Toolkit run: | - # NOTE: pinned to GitHub main until the next package release includes this flow. - # Switch to `uv pip install --system "agentops-accelerator[foundry,agent]"` after release. - uv pip install --system "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" __EVAL_STEPS__ diff --git a/src/agentops/templates/workflows/agentops-deploy-prompt-agent.yml b/src/agentops/templates/workflows/agentops-deploy-prompt-agent.yml index 76e724f..9154107 100644 --- a/src/agentops/templates/workflows/agentops-deploy-prompt-agent.yml +++ b/src/agentops/templates/workflows/agentops-deploy-prompt-agent.yml @@ -60,7 +60,7 @@ jobs: - name: Install AgentOps Toolkit run: | - uv pip install --system "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" - name: Create candidate agent version from prompt_file env: @@ -117,7 +117,7 @@ jobs: - name: Install AgentOps Toolkit run: | - uv pip install --system "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" __EVAL_STEPS__ @@ -181,7 +181,7 @@ __EVAL_STEPS__ - name: Install AgentOps Toolkit run: | - python -m pip install "agentops-accelerator @ git+https://github.com/Azure/agentops.git@main" + python -m pip install "agentops-accelerator__AGENTOPS_INSTALL_SPEC__" - name: Mark candidate as deployed run: | diff --git a/src/agentops/templates/workflows/agentops-deploy-qa-azd.yml b/src/agentops/templates/workflows/agentops-deploy-qa-azd.yml index d7422c3..af75371 100644 --- a/src/agentops/templates/workflows/agentops-deploy-qa-azd.yml +++ b/src/agentops/templates/workflows/agentops-deploy-qa-azd.yml @@ -109,7 +109,7 @@ __AILZ_PREFLIGHT_COMMAND__ enable-cache: false - name: Install AgentOps Toolkit - run: uv pip install --system "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + run: uv pip install --system "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" __EVAL_STEPS__ - name: Upload AgentOps results if: always() diff --git a/src/agentops/templates/workflows/agentops-deploy-qa.yml b/src/agentops/templates/workflows/agentops-deploy-qa.yml index 2ae78e0..5f437b7 100644 --- a/src/agentops/templates/workflows/agentops-deploy-qa.yml +++ b/src/agentops/templates/workflows/agentops-deploy-qa.yml @@ -66,9 +66,7 @@ jobs: - name: Install AgentOps Toolkit run: | - # NOTE: pinned to GitHub main until the next package release includes this flow. - # Switch to `uv pip install --system "agentops-accelerator[foundry]"` after release. - uv pip install --system "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry]__AGENTOPS_INSTALL_SPEC__" __EVAL_STEPS__ diff --git a/src/agentops/templates/workflows/agentops-pr-prompt-agent.yml b/src/agentops/templates/workflows/agentops-pr-prompt-agent.yml index 9cbcb04..81e069a 100644 --- a/src/agentops/templates/workflows/agentops-pr-prompt-agent.yml +++ b/src/agentops/templates/workflows/agentops-pr-prompt-agent.yml @@ -76,7 +76,7 @@ jobs: - name: Install AgentOps Toolkit run: | - uv pip install --system "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" - name: Create candidate agent version from prompt_file env: @@ -133,7 +133,7 @@ jobs: - name: Install AgentOps Toolkit run: | - uv pip install --system "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" __EVAL_STEPS__ diff --git a/src/agentops/templates/workflows/agentops-pr.yml b/src/agentops/templates/workflows/agentops-pr.yml index a5d907e..7792974 100644 --- a/src/agentops/templates/workflows/agentops-pr.yml +++ b/src/agentops/templates/workflows/agentops-pr.yml @@ -69,9 +69,7 @@ jobs: - name: Install AgentOps Toolkit run: | - # NOTE: pinned to GitHub main until the next package release includes this flow. - # Switch to `uv pip install --system "agentops-accelerator[foundry,agent]"` after release. - uv pip install --system "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" __EVAL_STEPS__ diff --git a/src/agentops/templates/workflows/agentops-watchdog.yml b/src/agentops/templates/workflows/agentops-watchdog.yml index 3ce7077..168b973 100644 --- a/src/agentops/templates/workflows/agentops-watchdog.yml +++ b/src/agentops/templates/workflows/agentops-watchdog.yml @@ -62,7 +62,7 @@ jobs: - name: Install AgentOps Toolkit run: | - uv pip install --system "agentops-accelerator[foundry,agent] @ git+https://github.com/Azure/agentops.git@main" + uv pip install --system "agentops-accelerator[foundry,agent]__AGENTOPS_INSTALL_SPEC__" # First run has no prior artifact. Using `pattern:` instead of # `name:` makes actions/download-artifact return success with zero diff --git a/tests/unit/test_agent_results_history.py b/tests/unit/test_agent_results_history.py index 769f0ae..4584976 100644 --- a/tests/unit/test_agent_results_history.py +++ b/tests/unit/test_agent_results_history.py @@ -70,6 +70,147 @@ def test_collect_results_history_disabled(tmp_path: Path) -> None: assert history.diagnostics["status"] == "disabled" +def test_collect_results_history_loads_latest_only_in_ci(tmp_path: Path) -> None: + """CI writes directly to .agentops/results/latest/ without a sibling dir. + + The loader must include `latest/` when no other timestamped dirs exist so + the regression check sees the just-completed run as `latest` instead of + falling back to a stale Foundry-listing entry. + """ + workspace = tmp_path + results = workspace / ".agentops" / "results" + latest_dir = results / "latest" + latest_dir.mkdir(parents=True) + payload = { + "version": 1, + "started_at": "2026-05-31T12:54:00+00:00", + "finished_at": "2026-05-31T12:54:04+00:00", + "aggregate_metrics": { + "coherence": 5.0, + "similarity": 5.0, + }, + "summary": { + "overall_passed": True, + "items_total": 3, + "items_passed_all": 3, + }, + } + (latest_dir / "results.json").write_text(json.dumps(payload), encoding="utf-8") + + config = ResultsHistorySourceConfig( + enabled=True, path=".agentops/results", lookback_runs=10 + ) + history = collect_results_history(workspace, config) + + assert len(history.runs) == 1 + run = history.runs[0] + assert run.run_id == "latest" + assert run.metrics == {"coherence": 5.0, "similarity": 5.0} + assert run.run_pass is True + assert run.items_total == 3 + assert run.timestamp is not None + assert run.timestamp.year == 2026 + + +def test_collect_results_history_prefers_timestamped_over_latest(tmp_path: Path) -> None: + """In dev mode both `latest/` and a timestamped dir exist (same run). + + The loader must skip `latest/` so the regression check does not see the + same run under two different keys. + """ + workspace = tmp_path + results = workspace / ".agentops" / "results" + # Timestamped dir wins; `latest/` is its sibling pointer. + _write_run(results, "2026-05-31-12-54", "2026-05-31T12:54:00Z", {"coherence": 5.0}) + _write_run(results, "latest", "2026-05-31T12:54:00Z", {"coherence": 5.0}) + + config = ResultsHistorySourceConfig( + enabled=True, path=".agentops/results", lookback_runs=10 + ) + history = collect_results_history(workspace, config) + + assert [r.run_id for r in history.runs] == ["2026-05-31-12-54"] + + +def test_collect_results_history_reads_aggregate_metrics_field(tmp_path: Path) -> None: + """The orchestrator writes `aggregate_metrics`, not `metrics`/`run_metrics`.""" + workspace = tmp_path + results = workspace / ".agentops" / "results" + run_dir = results / "run-1" + run_dir.mkdir(parents=True) + payload = { + "version": 1, + "started_at": "2026-05-30T10:00:00+00:00", + "finished_at": "2026-05-30T10:00:30+00:00", + "aggregate_metrics": {"coherence": 4.5, "fluency": 4.0}, + "summary": { + "overall_passed": True, + "items_total": 2, + "items_passed_all": 2, + }, + } + (run_dir / "results.json").write_text(json.dumps(payload), encoding="utf-8") + + config = ResultsHistorySourceConfig( + enabled=True, path=".agentops/results", lookback_runs=10 + ) + history = collect_results_history(workspace, config) + + assert len(history.runs) == 1 + assert history.runs[0].metrics == {"coherence": 4.5, "fluency": 4.0} + assert history.runs[0].run_pass is True + + +def test_collect_results_history_orders_by_finished_at(tmp_path: Path) -> None: + """`finished_at` should be preferred over `started_at` for ordering.""" + workspace = tmp_path + results = workspace / ".agentops" / "results" + + # run-a started later but finished earlier (e.g., shorter run). + run_a = results / "run-a" + run_a.mkdir(parents=True) + (run_a / "results.json").write_text( + json.dumps( + { + "started_at": "2026-05-30T10:05:00+00:00", + "finished_at": "2026-05-30T10:05:10+00:00", + "aggregate_metrics": {"coherence": 4.0}, + "summary": { + "overall_passed": True, + "items_total": 1, + "items_passed_all": 1, + }, + } + ), + encoding="utf-8", + ) + # run-b started earlier but finished later (long-running). + run_b = results / "run-b" + run_b.mkdir(parents=True) + (run_b / "results.json").write_text( + json.dumps( + { + "started_at": "2026-05-30T10:00:00+00:00", + "finished_at": "2026-05-30T11:00:00+00:00", + "aggregate_metrics": {"coherence": 3.0}, + "summary": { + "overall_passed": True, + "items_total": 1, + "items_passed_all": 1, + }, + } + ), + encoding="utf-8", + ) + + config = ResultsHistorySourceConfig( + enabled=True, path=".agentops/results", lookback_runs=10 + ) + history = collect_results_history(workspace, config) + + assert [r.run_id for r in history.runs] == ["run-a", "run-b"] + + def test_collect_results_history_falls_back_to_foundry_cloud( tmp_path: Path, monkeypatch ) -> None: diff --git a/tests/unit/test_cicd.py b/tests/unit/test_cicd.py index 51753d6..6a93572 100644 --- a/tests/unit/test_cicd.py +++ b/tests/unit/test_cicd.py @@ -1007,3 +1007,99 @@ def test_azure_devops_prompt_agent_pr_template_propagates_doctor_gate(tmp_path: assert f"--severity-fail {gate}" in content assert "__DOCTOR_GATE__" not in content + +# --------------------------------------------------------------------------- +# Version-pinning of the AgentOps install spec +# --------------------------------------------------------------------------- + +def test_agentops_install_spec_pins_clean_release() -> None: + from agentops.services.cicd import _agentops_install_spec + + assert _agentops_install_spec("0.3.2") == "==0.3.2" + assert _agentops_install_spec("1.0.0") == "==1.0.0" + + +def test_agentops_install_spec_pins_post_and_rc_releases() -> None: + from agentops.services.cicd import _agentops_install_spec + + # PEP 440 post and rc releases are public-installable and should pin. + assert _agentops_install_spec("0.3.2.post1") == "==0.3.2.post1" + assert _agentops_install_spec("0.4.0rc1") == "==0.4.0rc1" + + +def test_agentops_install_spec_falls_back_for_dev_or_local_installs() -> None: + from agentops.services.cicd import ( + AGENTOPS_DEV_INSTALL_SPEC, + _agentops_install_spec, + ) + + # setuptools-scm dev versions and editable installs must fall back to + # git+main so contributors can still install from their checkout. + assert _agentops_install_spec("0.3.3.dev1") == AGENTOPS_DEV_INSTALL_SPEC + assert _agentops_install_spec("0.3.2+gabcdef") == AGENTOPS_DEV_INSTALL_SPEC + assert _agentops_install_spec("0.0.0-dev") == AGENTOPS_DEV_INSTALL_SPEC + + +def test_workflow_install_lines_pin_to_release_version(tmp_path: Path) -> None: + """Generated workflows must pin agentops to a release version, not @main.""" + + from agentops.services import cicd + + # Force a clean release version so the substitution is deterministic. + original = cicd._agentops_install_spec + cicd._agentops_install_spec = lambda version=None: "==9.9.9" + try: + generate_cicd_workflows(tmp_path, kinds=list(DEFAULT_KINDS), force=True) + generate_cicd_workflows( + tmp_path, + platform="azure-devops", + kinds=list(DEFAULT_KINDS), + force=True, + ) + finally: + cicd._agentops_install_spec = original + + expected_files = ( + tmp_path / _PR_PATH, + tmp_path / _DEV_PATH, + tmp_path / _QA_PATH, + tmp_path / _PROD_PATH, + tmp_path / ".azuredevops/pipelines/agentops-pr.yml", + tmp_path / ".azuredevops/pipelines/agentops-deploy-dev.yml", + tmp_path / ".azuredevops/pipelines/agentops-deploy-qa.yml", + tmp_path / ".azuredevops/pipelines/agentops-deploy-prod.yml", + ) + for path in expected_files: + content = path.read_text(encoding="utf-8") + assert "__AGENTOPS_INSTALL_SPEC__" not in content, ( + f"{path} still has the placeholder" + ) + assert "git+https://github.com/Azure/agentops.git@main" not in content, ( + f"{path} still installs from @main" + ) + assert "agentops-accelerator" in content + assert "==9.9.9" in content, ( + f"{path} does not pin to the substituted release version" + ) + + +def test_workflow_install_lines_fall_back_to_main_for_dev_installs(tmp_path: Path) -> None: + """Editable / setuptools-scm installs keep the git+main fallback.""" + + from agentops.services import cicd + + original = cicd._agentops_install_spec + cicd._agentops_install_spec = ( + lambda version=None: cicd.AGENTOPS_DEV_INSTALL_SPEC + ) + try: + generate_cicd_workflows(tmp_path, kinds=["pr", "dev"], force=True) + finally: + cicd._agentops_install_spec = original + + for path in (tmp_path / _PR_PATH, tmp_path / _DEV_PATH): + content = path.read_text(encoding="utf-8") + assert "__AGENTOPS_INSTALL_SPEC__" not in content + assert " @ git+https://github.com/Azure/agentops.git@main" in content + +