Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions scripts/verify.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ function withoutCanonicalTemplateVocabulary(text) {
return text
.replaceAll(/driving-parity/giu, "")
.replaceAll(/driving parity/giu, "")
.replaceAll(/parity progress/giu, "");
.replaceAll(/parity progress/giu, "")
.replaceAll(/visual-parity/giu, "")
.replaceAll(/visual parity/giu, "");
}

const leakPatterns = [
Expand Down Expand Up @@ -266,6 +268,7 @@ const jsFiles = [
...walkFiles(resolve(repoRoot, "skills/burnlist/scripts"), (path) => path.endsWith(".mjs")),
resolve(repoRoot, "skills/burnlist/dashboard/differential-testing-progress-chart.js"),
resolve(repoRoot, "skills/burnlist/dashboard/differential-testing-renderer.js"),
resolve(repoRoot, "skills/burnlist/dashboard/visual-parity-renderer.js"),
].sort();

for (const file of jsFiles) {
Expand Down Expand Up @@ -406,10 +409,11 @@ assertSourceExcludes("skills/burnlist/scripts/burnlist-dashboard-server.mjs", '"
assertSourceExcludes("skills/burnlist/dashboard/src/app.tsx", '"/targets"', "React dashboard still exposes the removed Targets route.");
assertSourceExcludes("skills/burnlist/scripts/oven-contract.mjs", '"target"', "Oven contract still accepts the removed Target widget.");
assertSkillSet(["burnlist"]);
assertBuiltInOvenSet(["checklist", "differential-testing", "streaming-diff"]);
assertBuiltInOvenSet(["checklist", "differential-testing", "streaming-diff", "visual-parity"]);
assertBuiltInOven("checklist", "Checklist");
assertBuiltInOven("differential-testing", "Differential Testing");
assertBuiltInOven("streaming-diff", "Streaming Diff");
assertBuiltInOven("visual-parity", "Visual Parity");
assertDifferentialTestingContractAssets();
assertPublishablePackage();

Expand All @@ -426,6 +430,7 @@ run(process.execPath, [
"skills/burnlist/scripts/repo-state.test.mjs",
"skills/burnlist/scripts/streaming-diff-contract.test.mjs",
"skills/burnlist/scripts/streaming-diff-server.test.mjs",
"skills/burnlist/scripts/visual-parity.test.mjs",
]);

run(process.execPath, ["scripts/register-skills.mjs", "--force-global", "--dry-run"], {
Expand Down
51 changes: 32 additions & 19 deletions skills/burnlist/dashboard/differential-testing-progress-chart.js
Original file line number Diff line number Diff line change
Expand Up @@ -362,20 +362,20 @@ export function renderDifferentialTestingProgressChart(svg, history, { mode = "f
return segments.filter((candidate, index) =>
index === segments.length - 1 || candidate.some((point) => failedFieldValueForPoint(point) > 0));
};
const chartSegmentPath = (series, valueAccessor = valueForPoint, yAccessor = plotY) => {
const chartSegmentPath = (series, valueAccessor = valueForPoint, yAccessor = plotY, xAccessor = x) => {
const [first, ...rest] = series;
const commands = ["M " + x(first.time).toFixed(1) + " " + yAccessor(valueAccessor(first)).toFixed(1)];
const commands = ["M " + xAccessor(first.time).toFixed(1) + " " + yAccessor(valueAccessor(first)).toFixed(1)];
let previous = first;
for (const point of rest) {
const pointX = x(point.time).toFixed(1);
const pointX = xAccessor(point.time).toFixed(1);
commands.push("L " + pointX + " " + yAccessor(valueAccessor(previous)).toFixed(1));
commands.push("L " + pointX + " " + yAccessor(valueAccessor(point)).toFixed(1));
previous = point;
}
return commands.join(" ");
};
const chartPath = (series, valueAccessor = valueForPoint, yAccessor = plotY) =>
chartSegments(series).map((segment) => chartSegmentPath(segment, valueAccessor, yAccessor)).join(" ");
const chartPath = (series, valueAccessor = valueForPoint, yAccessor = plotY, xAccessor = x) =>
chartSegments(series).map((segment) => chartSegmentPath(segment, valueAccessor, yAccessor, xAccessor)).join(" ");
const chartAreaPath = (series, valueAccessor = valueForPoint) => {
const baselineY = plotY(valueMin).toFixed(1);
return chartSegments(series).map((segment) => {
Expand All @@ -400,7 +400,8 @@ export function renderDifferentialTestingProgressChart(svg, history, { mode = "f
"Z",
].join(" ");
}).join(" ");
const path = isFailedChart ? "" : chartPath(fallback, progressValueForPoint);
const progressLineX = (time) => Math.min(width - seriesPad.right - 1, x(time));
const path = isFailedChart ? "" : chartPath(fallback, progressValueForPoint, plotY, progressLineX);
const areaPath = isFailedChart ? "" : chartAreaPath(fallback);
const last = fallback.at(-1);
const failedFieldTitleForPoint = (point) => {
Expand Down Expand Up @@ -460,10 +461,11 @@ export function renderDifferentialTestingProgressChart(svg, history, { mode = "f
svg.classList.toggle("delta-chart", isDeltaChart);
svg.classList.toggle(
"goal-reached",
isFailedChart &&
(isFailedChart &&
!isDeltaChart &&
failedFieldSeries.length > 0 &&
failedFieldValueForPoint(failedFieldSeries.at(-1)) <= 0,
failedFieldValueForPoint(failedFieldSeries.at(-1)) <= 0) ||
(!isFailedChart && progressValueForPoint(last) <= 0),
);
svg.dataset.plotLeft = String(seriesPad.left);
svg.dataset.plotRight = String(width - seriesPad.right);
Expand All @@ -474,13 +476,14 @@ export function renderDifferentialTestingProgressChart(svg, history, { mode = "f
svg.dataset.timeScale = compactTimeline ? "compact" : "all";
svg.replaceChildren();
if (!isFailedChart && isDrivingParityPage) {
const remainingArea = document.createElementNS("http://www.w3.org/2000/svg", "rect");
const remainingArea = document.createElementNS("http://www.w3.org/2000/svg", "path");
remainingArea.setAttribute("class", "progress-remaining-area");
remainingArea.setAttribute("x", String(seriesPad.left));
remainingArea.setAttribute("y", String(pad.top));
remainingArea.setAttribute("width", String(innerWidth));
remainingArea.setAttribute("height", String(innerHeight));
remainingArea.setAttribute("d", areaPath);
svg.append(remainingArea);
const completedArea = document.createElementNS("http://www.w3.org/2000/svg", "path");
completedArea.setAttribute("class", "progress-area");
completedArea.setAttribute("d", chartCeilingAreaPath(fallback, progressValueForPoint));
svg.append(completedArea);
}
const yTicks = isDeltaChart
? [0, 1, 2, 3, 4].map((index) => valueMin + ((valueMax - valueMin) * index) / 4)
Expand Down Expand Up @@ -529,10 +532,12 @@ export function renderDifferentialTestingProgressChart(svg, history, { mode = "f
axis.setAttribute("y2", String(height - pad.bottom));
if (!isDrivingParityPage) svg.append(axis);
if (!isFailedChart) {
const area = document.createElementNS("http://www.w3.org/2000/svg", "path");
area.setAttribute("class", "progress-area");
area.setAttribute("d", isDrivingParityPage ? chartCeilingAreaPath(fallback, progressValueForPoint) : areaPath);
svg.append(area);
if (!isDrivingParityPage) {
const area = document.createElementNS("http://www.w3.org/2000/svg", "path");
area.setAttribute("class", "progress-area");
area.setAttribute("d", areaPath);
svg.append(area);
}
const line = document.createElementNS("http://www.w3.org/2000/svg", "path");
line.setAttribute("class", "progress-line");
line.setAttribute("d", path);
Expand Down Expand Up @@ -794,7 +799,15 @@ export function renderDifferentialTestingFrameDeltaChart(svg, metrics) {
const measuredHeight = Math.round(svg.getBoundingClientRect().height || svg.clientHeight || parseFloat(getComputedStyle(svg).height) || 200);
const height = Math.max(160, measuredHeight);
svg.setAttribute("viewBox", "0 0 " + width + " " + height);
svg.setAttribute("aria-label", "Largest signed candidate minus reference residual by frame; display capped at the 98th percentile with clipped values marked");
const valueLabel = typeof metrics?.valueLabel === "string" && metrics.valueLabel.trim()
? metrics.valueLabel.trim()
: "signed residual";
svg.setAttribute(
"aria-label",
typeof metrics?.ariaLabel === "string" && metrics.ariaLabel.trim()
? metrics.ariaLabel.trim()
: "Largest signed candidate minus reference residual by frame; display capped at the 98th percentile with clipped values marked",
);
svg.classList.remove("range-zoomable", "failed-chart", "goal-reached");
svg.classList.add("delta-chart");
delete svg.dataset.domainMin;
Expand Down Expand Up @@ -956,7 +969,7 @@ export function renderDifferentialTestingFrameDeltaChart(svg, metrics) {
: `M${(markerX - 3).toFixed(1)},${(markerY - 5).toFixed(1)}L${markerX.toFixed(1)},${markerY.toFixed(1)}L${(markerX + 3).toFixed(1)},${(markerY - 5).toFixed(1)}Z`,
});
const title = svgElement("title");
title.textContent = `${value > 0 ? "+" : ""}${formatRatio(value)} signed residual`;
title.textContent = `${value > 0 ? "+" : ""}${formatRatio(value)} ${valueLabel}`;
marker.append(title);
svg.append(marker);
});
Expand Down
48 changes: 32 additions & 16 deletions skills/burnlist/dashboard/differential-testing-renderer.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ function escapeHtml(value) {
.replaceAll("'", "&#39;");
}

export function differentialTestingLoadingMarkup() {
export function differentialTestingLoadingMarkup({ ovenName = "Differential Testing" } = {}) {
const title = `<div class="driving-parity-kpi-item driving-parity-kpi-title-item">
<span class="driving-parity-kpi-heading differential-scenario-heading">Scenario</span>
<span class="driving-parity-kpi-title-subtitle"><span class="differential-scenario-control"><select aria-label="Differential Testing scenario" disabled><option selected>Loading scenario…</option></select></span></span>
Expand All @@ -28,7 +28,7 @@ export function differentialTestingLoadingMarkup() {
.replace('<h2 id="driving-parity-summary" class="driving-parity-summary" hidden></h2>', '<h2 id="driving-parity-summary" class="driving-parity-summary">Fields List</h2>')
.replace('<div id="driving-parity-controls" class="driving-parity-controls" hidden>', '<div id="driving-parity-controls" class="driving-parity-controls">');
return `<div class="differential-testing-loading" aria-busy="true">
<span class="differential-loading-sr" role="status">Loading Differential Testing</span>
<span class="differential-loading-sr" role="status">Loading ${escapeHtml(ovenName)}</span>
<div class="differential-testing-loading-visual" aria-hidden="true" inert>${template}</div>
</div>`;
}
Expand Down Expand Up @@ -160,6 +160,9 @@ export function mountDifferentialTestingDashboard(root, oven, payload, {
onFieldViewChange = () => {},
fieldPage = null,
frameDeltaMetrics = null,
detailCellId = "field-details",
detailRenderer = null,
initialProgressChart = "delta",
templateOnly = false,
} = {}) {
if (templateOnly) return templateHtml();
Expand All @@ -170,7 +173,7 @@ export function mountDifferentialTestingDashboard(root, oven, payload, {
const telemetryAvailability = differentialTelemetryAvailability(payload);
const state = {
chart: "current",
progressChart: "delta",
progressChart: initialProgressChart,
sort: fieldPage?.sort ?? (telemetryAvailability.status === "comparable" ? "changed" : "default"),
filter: fieldPage?.filter ?? "all",
search: fieldPage?.search ?? "",
Expand Down Expand Up @@ -1593,7 +1596,7 @@ export function mountDifferentialTestingDashboard(root, oven, payload, {
existingHeaderStatus?.remove();
if (!state.oven || !state.payload) return;
const cells = new Map(state.oven.detail.cells.map((cell) => [cell.id, cell]));
const title = cells.get("title"), burns = cells.get("burns"), fields = cells.get("fields"), frames = cells.get("frames"), progressCell = cells.get("progress"), logCell = cells.get("log"), details = cells.get("field-details");
const title = cells.get("title"), burns = cells.get("burns"), fields = cells.get("fields"), frames = cells.get("frames"), progressCell = cells.get("progress"), logCell = cells.get("log"), details = cells.get(detailCellId);
if (![title, burns, fields, frames, progressCell, logCell, details].every(Boolean)) { root.innerHTML = '<div class="empty">Differential Testing Oven layout is incomplete.</div>'; return; }
const titleText = String(title.title || state.oven.name || "Differential Testing");
if (state.payload.scenarioCatalog?.selectedScenarioId === null && state.payload.scenarioCatalog?.scenarios?.length === 0) {
Expand Down Expand Up @@ -1652,6 +1655,11 @@ export function mountDifferentialTestingDashboard(root, oven, payload, {
.replace('<div class="rows-view" id="hybrid-rows"></div>', `<div class="rows-view" id="hybrid-rows">${fieldRows(page)}</div>`)
.replace(/<div id="driving-parity-pagination" class="driving-parity-controls driving-parity-pagination" hidden>[\s\S]*?<\/div>\n <\/main>/u, `${paginationHtml}\n </main>`);
root.innerHTML = html;
if (detailRenderer) {
const detailRows = root.querySelector("#hybrid-rows");
if (detailRows) detailRows.innerHTML = detailRenderer({ cell: details, oven: state.oven, payload: state.payload });
root.querySelector("#driving-parity-pagination")?.remove();
}
const overviewTimestamp = root.querySelector("#differential-overview-time");
const refreshStatusElement = root.querySelector("#differential-refresh-status");
const dashboardNav = typeof document === "undefined" ? null : document.querySelector(".dashboard-primary-nav");
Expand All @@ -1662,7 +1670,7 @@ export function mountDifferentialTestingDashboard(root, oven, payload, {
paintWaffles();
renderProgressChart();
const search = root.querySelector("#driving-parity-field-search"), pageSize = root.querySelector("#driving-parity-page-size");
search.value = state.search;
if (search) search.value = state.search;
if (pageSize) pageSize.value = String(state.pageSize);
}
function requestFieldView({ refocusSearch = false } = {}) {
Expand Down Expand Up @@ -1806,13 +1814,17 @@ export function startDifferentialTestingLiveUpdates(root, {
locationImpl = globalThis.location,
historyImpl = globalThis.history,
mount = mountDifferentialTestingDashboard,
ovenId = "differential-testing",
ovenName = "Differential Testing",
scenarioParam = "scenario",
payloadTransform = (json) => differentialPagedPayload(json.payload, json.fieldPage),
refreshMs = DIFFERENTIAL_TESTING_REFRESH_MS,
onError = (error, hasDashboard) => {
if (!hasDashboard) root.innerHTML = `<div class="empty">${escapeHtml(String(error?.message || error))}</div>`;
else console.error("Could not refresh Differential Testing data.", error);
else console.error(`Could not refresh ${ovenName} data.`, error);
},
} = {}) {
root.innerHTML = differentialTestingLoadingMarkup();
root.innerHTML = differentialTestingLoadingMarkup({ ovenName });
let oven = null;
let dashboard = null;
let payloadRevision = "";
Expand All @@ -1825,13 +1837,16 @@ export function startDifferentialTestingLiveUpdates(root, {
const payloadCache = new Map();
let fieldViewQuery = null;
let selectedScenarioId = (() => {
try { return new URLSearchParams(locationImpl?.search || "").get("scenario") || ""; }
try { return new URLSearchParams(locationImpl?.search || "").get(scenarioParam) || ""; }
catch { return ""; }
})();

const payloadUrl = (scenarioId) => {
const searchParams = new URLSearchParams();
if (scenarioId) searchParams.set("scenario", scenarioId);
if (scenarioId) {
if (scenarioParam === "scenario") searchParams.set("scenario", scenarioId);
else searchParams.set(scenarioParam, scenarioId);
}
if (fieldViewQuery) {
searchParams.set("search", fieldViewQuery.search);
searchParams.set("filter", fieldViewQuery.filter);
Expand All @@ -1840,7 +1855,7 @@ export function startDifferentialTestingLiveUpdates(root, {
searchParams.set("pageSize", String(fieldViewQuery.pageSize));
}
const query = searchParams.toString();
return `/api/oven-data/differential-testing${query ? `?${query}` : ""}`;
return `/api/oven-data/${ovenId}${query ? `?${query}` : ""}`;
};

const read = async (url, key, fallbackMessage) => {
Expand All @@ -1857,13 +1872,13 @@ export function startDifferentialTestingLiveUpdates(root, {
...(cached?.etag ? { headers: { "If-None-Match": cached.etag } } : {}),
});
if (response.status === 304) {
if (!cached) throw new Error("Differential Testing data returned 304 before an initial payload was loaded.");
if (!cached) throw new Error(`${ovenName} data returned 304 before an initial payload was loaded.`);
return { ...cached, notModified: true };
}
const json = await response.json();
if (!response.ok) throw new Error(json.error || "Could not load Differential Testing data.");
if (!response.ok) throw new Error(json.error || `Could not load ${ovenName} data.`);
const result = {
payload: differentialPagedPayload(json.payload, json.fieldPage),
payload: payloadTransform(json),
transport: json.transport ?? null,
fieldPage: json.fieldPage ?? null,
frameDeltaMetrics: json.frameDeltaMetrics ?? null,
Expand All @@ -1890,7 +1905,7 @@ export function startDifferentialTestingLiveUpdates(root, {
const requestPayloadUrl = payloadUrl(requestScenarioId);
try {
const [nextOven, payloadResult] = await Promise.all([
oven ?? read("/api/ovens/differential-testing", "oven", "Could not load Differential Testing Oven."),
oven ?? read(`/api/ovens/${ovenId}`, "oven", `Could not load ${ovenName} Oven.`),
readPayload(requestPayloadUrl),
]);
if (stopped || requestGeneration !== scenarioGeneration) return;
Expand Down Expand Up @@ -1940,8 +1955,9 @@ export function startDifferentialTestingLiveUpdates(root, {
fieldViewQuery = null;
payloadCache.clear();
try {
const nextUrl = new URL(locationImpl?.href || "/ovens/differential-testing/view", "http://localhost");
nextUrl.searchParams.set("scenario", scenarioId);
const nextUrl = new URL(locationImpl?.href || `/ovens/${ovenId}/view`, "http://localhost");
if (scenarioParam === "scenario") nextUrl.searchParams.set("scenario", scenarioId);
else nextUrl.searchParams.set(scenarioParam, scenarioId);
historyImpl?.replaceState?.(null, "", `${nextUrl.pathname}${nextUrl.search}${nextUrl.hash}`);
} catch {}
return refresh();
Expand Down
6 changes: 4 additions & 2 deletions skills/burnlist/dashboard/differential-testing.css
Original file line number Diff line number Diff line change
Expand Up @@ -920,8 +920,10 @@ body.driving-parity-view .dashboard-header {
.chart .progress-dot { fill: var(--done); }
.driving-parity-view .chart .progress-remaining-area { fill: var(--red-subtle); }
.driving-parity-view .chart .progress-area { fill: rgba(97,211,148,.10); }
.driving-parity-view .chart .progress-line { stroke: var(--status-green); }
.driving-parity-view .chart .progress-dot { fill: var(--status-green); }
.driving-parity-view .chart .progress-line { stroke: var(--status-red); stroke-width: 1.6; stroke-linecap: butt; stroke-linejoin: miter; opacity: .8; vector-effect: non-scaling-stroke; }
.driving-parity-view .chart .progress-dot { fill: var(--status-red); opacity: .8; }
.driving-parity-view .chart.goal-reached .progress-line { stroke: var(--status-green); stroke-width: 1.55; }
.driving-parity-view .chart.goal-reached .progress-dot { fill: var(--status-green); }
.chart .failed-fields-pass-area { fill: rgba(97,211,148,.08); }
.chart .failed-fields-area { fill: var(--red-subtle); }
.chart .failed-fields-line { fill: none; stroke: var(--red); stroke-width: 1.6; stroke-linecap: butt; stroke-linejoin: round; opacity: .8; }
Expand Down
Loading