From f0bde42bfed4625c343455b6bd7b6c493d2b39ac Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:06:05 +1000 Subject: [PATCH] ci: type-check examples/** and testing/** to catch call-site regressions (#820) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test:pr`/`test:ci` excluded examples/** and testing/** from every target, so `test:types` never checked the apps where the library is actually consumed — call-site type regressions (e.g. a provider summarize adapter not assignable to `summarize()`) slipped through CI. Because those surfaces were never type-checked, they had also accumulated ~80 latent type errors. CI wiring: - test:pr / test:ci now run a second pass — `test:types` over examples/** and testing/** — after the existing packages-only run. Heavy targets (build/lib/...) stay excluded; only the cheap, high-value type check is added. - Add `test:types:examples` convenience script and document the gate in CLAUDE.md. Coverage: - Add a `test:types` target to every example/testing project that lacked one (e2e, panel, react-native-chat, react-native-smoke). ts-angular-chat type-checks templates via `ngc`, resolved from an existing @angular/build peer (no new dep, no lockfile change). Fixes (examples/testing drift from the current API; nothing under packages/**): - ts-react-chat, ts-solid-chat, ts-code-mode-web: MediaPrompt, maxTokens→modelOptions, ContentPart[] narrowing, transport XOR, onResult result-type inference (#848). - testing/e2e: drop poisoning `as never` model casts, fix adapter/tool typings, OpenRouter summarize httpClient header injection. - testing/panel: AnyAdapter factory maps, AG-UI EventType migration, remove dead app.config.ts (Vinxi-era), rewrite createEventRecording to emit AG-UI StreamChunks. Guard: - packages/ai-grok/tests/summarize-callsite-type-safety.test.ts asserts the grokSummarize → summarize() contract in an included package — a positive assertion now that the options-shape fix (#854) has landed, so a regression surfaces as a real type error instead of silently. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 5 +- examples/ts-angular-chat/package.json | 3 +- .../ts-angular-chat/scripts/typecheck.mjs | 28 ++ examples/ts-angular-chat/src/app.component.ts | 2 +- .../src/lib/tool-result-content.ts | 31 ++ .../src/routes/_banking-demo/banking-demo.tsx | 3 +- .../_database-demo/api.database-demo.ts | 3 +- .../routes/_database-demo/database-demo.tsx | 3 +- .../src/routes/_home/api.product-regular.ts | 3 +- .../src/routes/_home/index.tsx | 5 +- .../routes/_npm-github-chat/api.codemode.ts | 3 +- .../_npm-github-chat/npm-github-chat.tsx | 3 +- .../src/routes/_reporting/api.reports.ts | 3 +- .../src/routes/_reporting/reporting-agent.tsx | 3 +- .../src/routes/generations.image.tsx | 11 +- .../routes/generations.structured-output.tsx | 3 - .../src/routes/generations.video.tsx | 11 +- examples/ts-react-chat/src/routes/index.tsx | 12 +- examples/ts-react-native-chat/package.json | 1 + examples/ts-react-native-chat/tsconfig.json | 15 +- examples/ts-solid-chat/src/routes/index.tsx | 3 +- package.json | 5 +- .../summarize-callsite-type-safety.test.ts | 37 ++ testing/e2e/package.json | 1 + testing/e2e/src/routes/$provider/index.tsx | 1 + .../e2e/src/routes/api.arktype-tool-wire.ts | 12 +- .../routes/api.openai-shell-skills-wire.ts | 2 +- testing/e2e/src/routes/api.openrouter-cost.ts | 10 +- .../routes/api.openrouter-web-tools-wire.ts | 12 +- testing/e2e/src/routes/api.otel-usage.ts | 2 +- testing/e2e/src/routes/api.summarize.ts | 25 +- .../routes/api.tool-call-lifecycle-wire.ts | 1 + testing/e2e/src/routes/tools-test.tsx | 9 +- testing/panel/README.md | 8 +- testing/panel/app.config.ts | 12 - testing/panel/package.json | 1 + testing/panel/src/components/Header.tsx | 1 + testing/panel/src/lib/recording.ts | 460 +++++++++++------- testing/panel/src/routes/api.image.ts | 26 +- .../panel/src/routes/api.simulator-chat.ts | 186 ++++--- testing/panel/src/routes/api.structured.ts | 48 +- testing/panel/src/routes/api.summarize.ts | 48 +- testing/panel/src/routes/api.transcription.ts | 2 +- testing/panel/src/routes/api.video.ts | 2 +- testing/panel/src/routes/stream-debugger.tsx | 69 ++- testing/panel/tests/basic-inference.spec.ts | 6 +- testing/panel/tests/helpers.ts | 6 +- testing/react-native-smoke/package.json | 1 + 48 files changed, 662 insertions(+), 485 deletions(-) create mode 100644 examples/ts-angular-chat/scripts/typecheck.mjs create mode 100644 examples/ts-code-mode-web/src/lib/tool-result-content.ts create mode 100644 packages/ai-grok/tests/summarize-callsite-type-safety.test.ts delete mode 100644 testing/panel/app.config.ts diff --git a/CLAUDE.md b/CLAUDE.md index f1fa41281..c5a83e51e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -230,7 +230,7 @@ The single canonical command is: pnpm test:pr ``` -This runs the exact target set the `PR` workflow runs in CI (`nx affected --targets=test:sherif,test:knip,test:docs,test:eslint,test:lib,test:types,test:build,build --exclude=examples/**,testing/**`). +This runs the exact target set the `PR` workflow runs in CI: first `nx affected --targets=test:sherif,test:knip,test:docs,test:kiira,test:eslint,test:lib,test:types,test:build,build --exclude=examples/**,testing/**` (the heavy targets, packages only), then a second `nx affected --targets=test:types --projects=examples/**,testing/**` pass that **does** type-check the example apps and `testing/` packages. The example/testing surfaces are deliberately excluded from the heavy targets (build/lib/etc.) but included for `test:types`, because call-site type regressions often only manifest where the library is actually consumed (see issue #820). Use `pnpm test:types:examples` to run just that second pass locally. If you can't run `test:pr` (e.g. it's too slow on your machine), at minimum run each of these and confirm they're green before pushing: @@ -238,7 +238,8 @@ If you can't run `test:pr` (e.g. it's too slow on your machine), at minimum run - `pnpm test:knip` — unused dependencies - `pnpm test:docs` — doc link verification - `pnpm test:eslint` — lint -- `pnpm test:types` — typecheck +- `pnpm test:types` — typecheck (packages) +- `pnpm test:types:examples` — typecheck the example apps + `testing/` packages - `pnpm test:lib` — unit tests - `pnpm test:build` — build artifact verification - `pnpm build` — build all affected packages diff --git a/examples/ts-angular-chat/package.json b/examples/ts-angular-chat/package.json index d012f27c2..96df3a5ec 100644 --- a/examples/ts-angular-chat/package.json +++ b/examples/ts-angular-chat/package.json @@ -6,7 +6,8 @@ "scripts": { "dev": "vite", "build": "vite build", - "preview": "vite preview" + "preview": "vite preview", + "test:types": "node scripts/typecheck.mjs" }, "dependencies": { "@angular/common": "^21.2.0", diff --git a/examples/ts-angular-chat/scripts/typecheck.mjs b/examples/ts-angular-chat/scripts/typecheck.mjs new file mode 100644 index 000000000..024bde0a4 --- /dev/null +++ b/examples/ts-angular-chat/scripts/typecheck.mjs @@ -0,0 +1,28 @@ +// Type-check this Angular example, including template type-checking. +// +// Angular template type-checking (`strictTemplates`) requires the Angular +// compiler (`ngc`), not plain `tsc`. `ngc` ships in `@angular/compiler-cli`, +// which is a *peer* dependency of `@angular/build` (a direct devDependency of +// this example). We resolve it from there instead of declaring it directly so +// CI's frozen lockfile install stays unchanged. +import { createRequire } from 'node:module' +import { spawnSync } from 'node:child_process' +import { dirname, join } from 'node:path' + +const require = createRequire(import.meta.url) + +// `@angular/compiler-cli` is resolvable from `@angular/build`'s location. +const buildPkg = require.resolve('@angular/build/package.json') +const cliPkgPath = require.resolve('@angular/compiler-cli/package.json', { + paths: [buildPkg], +}) +const cliPkg = require(cliPkgPath) +const ngc = join(dirname(cliPkgPath), cliPkg.bin.ngc) + +const { status } = spawnSync( + process.execPath, + [ngc, '-p', 'tsconfig.app.json', '--noEmit'], + { stdio: 'inherit' }, +) + +process.exit(status ?? 1) diff --git a/examples/ts-angular-chat/src/app.component.ts b/examples/ts-angular-chat/src/app.component.ts index 994c3e685..8dc12652d 100644 --- a/examples/ts-angular-chat/src/app.component.ts +++ b/examples/ts-angular-chat/src/app.component.ts @@ -187,7 +187,7 @@ export class AppComponent { /** A message is worth rendering if it has visible text or a tool call. */ isRenderable(message: { - parts: ReadonlyArray<{ type: string; content?: string }> + parts: ReadonlyArray<{ type: string; content?: unknown }> }): boolean { return message.parts.some( (part) => diff --git a/examples/ts-code-mode-web/src/lib/tool-result-content.ts b/examples/ts-code-mode-web/src/lib/tool-result-content.ts new file mode 100644 index 000000000..c0b604dd3 --- /dev/null +++ b/examples/ts-code-mode-web/src/lib/tool-result-content.ts @@ -0,0 +1,31 @@ +import type { UIMessage } from '@tanstack/ai-react' + +type ToolResultPart = Extract< + UIMessage['parts'][number], + { type: 'tool-result' } +> + +/** `string | Array` — a tool result's raw content. */ +type ToolResultContent = ToolResultPart['content'] + +type ContentPartItem = Exclude[number] + +/** + * Reduce a tool-result part's `content` to a plain string for rendering. + * + * Tool results carry `string | Array` (multimodal results are + * normalized to an array of content parts upstream). These demos render tool + * results as plain strings, so array content is flattened to the concatenation + * of its text parts; non-text parts (image, audio, video, document) have no + * string form here and are skipped. + */ +export function toolResultContentToString(content: ToolResultContent): string { + if (typeof content === 'string') return content + return content + .filter( + (part): part is Extract => + part.type === 'text', + ) + .map((part) => part.content) + .join('') +} diff --git a/examples/ts-code-mode-web/src/routes/_banking-demo/banking-demo.tsx b/examples/ts-code-mode-web/src/routes/_banking-demo/banking-demo.tsx index 4dbdff18b..d0ab65e1d 100644 --- a/examples/ts-code-mode-web/src/routes/_banking-demo/banking-demo.tsx +++ b/examples/ts-code-mode-web/src/routes/_banking-demo/banking-demo.tsx @@ -18,6 +18,7 @@ import { } from '@/components/reports/useReportSSE' import ChatInput from '@/components/ChatInput' import { Header } from '@/components' +import { toolResultContentToString } from '@/lib/tool-result-content' import { applyUIEvent, applyUIUpdates } from '@/lib/reports/apply-event' import type { RefreshResult, @@ -159,7 +160,7 @@ function Messages({ messages }: { messages: Array }) { for (const p of message.parts) { if (p.type === 'tool-result') { toolResults.set(p.toolCallId, { - content: p.content, + content: toolResultContentToString(p.content), state: p.state, error: p.error, }) diff --git a/examples/ts-code-mode-web/src/routes/_database-demo/api.database-demo.ts b/examples/ts-code-mode-web/src/routes/_database-demo/api.database-demo.ts index acf79386c..8bcd7e15a 100644 --- a/examples/ts-code-mode-web/src/routes/_database-demo/api.database-demo.ts +++ b/examples/ts-code-mode-web/src/routes/_database-demo/api.database-demo.ts @@ -17,6 +17,7 @@ import type { AnyTextAdapter, ServerTool, StreamChunk } from '@tanstack/ai' import type { IsolateDriver } from '@tanstack/ai-code-mode' import { databaseTools, getSchemaInfoTool } from '@/lib/tools/database-tools' +import { maxTokensModelOptions } from '@/lib/max-tokens-model-options' type Provider = 'anthropic' | 'openai' | 'gemini' @@ -284,7 +285,7 @@ export const Route = createFileRoute( systemPrompts, agentLoopStrategy: maxIterations(15), abortController, - maxTokens: 8192, + modelOptions: maxTokensModelOptions(rawAdapter, 8192), }) const instrumentedStream = wrapWithTimingEvents(stream, rawAdapter) diff --git a/examples/ts-code-mode-web/src/routes/_database-demo/database-demo.tsx b/examples/ts-code-mode-web/src/routes/_database-demo/database-demo.tsx index ca2a204af..4062871be 100644 --- a/examples/ts-code-mode-web/src/routes/_database-demo/database-demo.tsx +++ b/examples/ts-code-mode-web/src/routes/_database-demo/database-demo.tsx @@ -23,6 +23,7 @@ import type { VMEvent } from '@/components' import { CodeBlock, ExecutionResult, JavaScriptVM, Header } from '@/components' import ChatInput from '@/components/ChatInput' import { formatDuration } from '@/lib/efficiency' +import { toolResultContentToString } from '@/lib/tool-result-content' export const Route = createFileRoute('/_database-demo/database-demo' as any)({ component: DatabaseDemoPage, @@ -378,7 +379,7 @@ function Messages({ for (const p of message.parts) { if (p.type === 'tool-result') { toolResults.set(p.toolCallId, { - content: p.content, + content: toolResultContentToString(p.content), state: p.state, error: p.error, }) diff --git a/examples/ts-code-mode-web/src/routes/_home/api.product-regular.ts b/examples/ts-code-mode-web/src/routes/_home/api.product-regular.ts index da6a0d2ad..00620e3b0 100644 --- a/examples/ts-code-mode-web/src/routes/_home/api.product-regular.ts +++ b/examples/ts-code-mode-web/src/routes/_home/api.product-regular.ts @@ -5,6 +5,7 @@ import { openaiText } from '@tanstack/ai-openai' import { geminiText } from '@tanstack/ai-gemini' import type { AnyTextAdapter, StreamChunk } from '@tanstack/ai' import { productTools } from '@/lib/tools/product-tools' +import { maxTokensModelOptions } from '@/lib/max-tokens-model-options' type Provider = 'anthropic' | 'openai' | 'gemini' @@ -90,7 +91,7 @@ export const Route = createFileRoute('/_home/api/product-regular')({ ], agentLoopStrategy: maxIterations(30), abortController, - maxTokens: 8192, + modelOptions: maxTokensModelOptions(adapter, 8192), }) const requestStartTimeMs = Date.now() diff --git a/examples/ts-code-mode-web/src/routes/_home/index.tsx b/examples/ts-code-mode-web/src/routes/_home/index.tsx index 319824e90..97c98fd90 100644 --- a/examples/ts-code-mode-web/src/routes/_home/index.tsx +++ b/examples/ts-code-mode-web/src/routes/_home/index.tsx @@ -18,6 +18,7 @@ import { parsePartialJSON } from '@tanstack/ai' import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' import type { VMEvent } from '@/components' import { CodeBlock, ExecutionResult, JavaScriptVM, Header } from '@/components' +import { toolResultContentToString } from '@/lib/tool-result-content' export const Route = createFileRoute('/_home/')({ component: ProductDemoPage, @@ -792,7 +793,7 @@ function CodeModePanel({ for (const p of message.parts) { if (p.type === 'tool-result') { toolResults.set(p.toolCallId, { - content: p.content, + content: toolResultContentToString(p.content), state: p.state, error: p.error, }) @@ -1088,7 +1089,7 @@ function RegularToolsPanel({ for (const part of message.parts) { if (part.type === 'tool-result') { toolResults.set(part.toolCallId, { - content: part.content, + content: toolResultContentToString(part.content), state: part.state, error: part.error, }) diff --git a/examples/ts-code-mode-web/src/routes/_npm-github-chat/api.codemode.ts b/examples/ts-code-mode-web/src/routes/_npm-github-chat/api.codemode.ts index 6b3b4c162..a5054e29b 100644 --- a/examples/ts-code-mode-web/src/routes/_npm-github-chat/api.codemode.ts +++ b/examples/ts-code-mode-web/src/routes/_npm-github-chat/api.codemode.ts @@ -8,6 +8,7 @@ import type { AnyTextAdapter, StreamChunk } from '@tanstack/ai' import { allTools } from '@/lib/tools' import { CODE_MODE_SYSTEM_PROMPT } from '@/lib/prompts' import { exportConversationToPdfTool } from '@/lib/tools/export-pdf-tool' +import { maxTokensModelOptions } from '@/lib/max-tokens-model-options' type Provider = 'anthropic' | 'openai' | 'gemini' @@ -121,7 +122,7 @@ export const Route = createFileRoute('/_npm-github-chat/api/codemode')({ agentLoopStrategy: maxIterations(15), abortController, // Increase max tokens to allow for complex code generation - maxTokens: 8192, + modelOptions: maxTokensModelOptions(adapter, 8192), }) const requestStartTimeMs = Date.now() diff --git a/examples/ts-code-mode-web/src/routes/_npm-github-chat/npm-github-chat.tsx b/examples/ts-code-mode-web/src/routes/_npm-github-chat/npm-github-chat.tsx index 2d8e213ab..66e3f1355 100644 --- a/examples/ts-code-mode-web/src/routes/_npm-github-chat/npm-github-chat.tsx +++ b/examples/ts-code-mode-web/src/routes/_npm-github-chat/npm-github-chat.tsx @@ -27,6 +27,7 @@ import { } from '@/components' import { NpmDataSidebar } from '@/components/NpmDataSidebar' import { exportConversationToPdfTool } from '@/lib/tools/export-pdf-tool' +import { toolResultContentToString } from '@/lib/tool-result-content' export const Route = createFileRoute('/_npm-github-chat/npm-github-chat')({ component: CodeModePage, @@ -252,7 +253,7 @@ function Messages({ for (const p of message.parts) { if (p.type === 'tool-result') { toolResults.set(p.toolCallId, { - content: p.content, + content: toolResultContentToString(p.content), state: p.state, error: p.error, }) diff --git a/examples/ts-code-mode-web/src/routes/_reporting/api.reports.ts b/examples/ts-code-mode-web/src/routes/_reporting/api.reports.ts index 02cf73cb1..2cf4d761c 100644 --- a/examples/ts-code-mode-web/src/routes/_reporting/api.reports.ts +++ b/examples/ts-code-mode-web/src/routes/_reporting/api.reports.ts @@ -10,6 +10,7 @@ import { allTools } from '@/lib/tools' import { CODE_MODE_SYSTEM_PROMPT, REPORTS_SYSTEM_PROMPT } from '@/lib/prompts' import { reportTools } from '@/lib/reports/tools' import { createReportBindings } from '@/lib/reports/create-report-bindings' +import { maxTokensModelOptions } from '@/lib/max-tokens-model-options' type Provider = 'anthropic' | 'openai' | 'gemini' @@ -79,7 +80,7 @@ export const Route = createFileRoute('/_reporting/api/reports' as any)({ ], agentLoopStrategy: maxIterations(20), abortController, - maxTokens: 8192, + modelOptions: maxTokensModelOptions(adapter, 8192), }) const sseStream = toServerSentEventsStream(stream, abortController) diff --git a/examples/ts-code-mode-web/src/routes/_reporting/reporting-agent.tsx b/examples/ts-code-mode-web/src/routes/_reporting/reporting-agent.tsx index 591259ee8..09d5999a3 100644 --- a/examples/ts-code-mode-web/src/routes/_reporting/reporting-agent.tsx +++ b/examples/ts-code-mode-web/src/routes/_reporting/reporting-agent.tsx @@ -26,6 +26,7 @@ import { usePersistedReports, } from '@/components/reports' import type { Report, UIEvent } from '@/lib/reports/types' +import { toolResultContentToString } from '@/lib/tool-result-content' export const Route = createFileRoute('/_reporting/reporting-agent')({ component: ReportingAgentPage, @@ -251,7 +252,7 @@ function Messages({ for (const p of message.parts) { if (p.type === 'tool-result') { toolResults.set(p.toolCallId, { - content: p.content, + content: toolResultContentToString(p.content), state: p.state, error: p.error, }) diff --git a/examples/ts-react-chat/src/routes/generations.image.tsx b/examples/ts-react-chat/src/routes/generations.image.tsx index 38c6838a6..044570498 100644 --- a/examples/ts-react-chat/src/routes/generations.image.tsx +++ b/examples/ts-react-chat/src/routes/generations.image.tsx @@ -3,6 +3,7 @@ import { createFileRoute } from '@tanstack/react-router' import { useGenerateImage } from '@tanstack/ai-react' import type { UseGenerateImageReturn } from '@tanstack/ai-react' import { fetchServerSentEvents } from '@tanstack/ai-client' +import { resolveMediaPrompt } from '@tanstack/ai' import { generateImageFn, generateImageStreamFn } from '../lib/server-fns' function StreamingImageGeneration() { @@ -29,7 +30,10 @@ function DirectImageGeneration() { const [numberOfImages, setNumberOfImages] = useState(1) const hookReturn = useGenerateImage({ - fetcher: (input) => generateImageFn({ data: input }), + fetcher: (input) => + generateImageFn({ + data: { ...input, prompt: resolveMediaPrompt(input.prompt).text }, + }), }) return ( @@ -48,7 +52,10 @@ function ServerFnImageGeneration() { const [numberOfImages, setNumberOfImages] = useState(1) const hookReturn = useGenerateImage({ - fetcher: (input) => generateImageStreamFn({ data: input }), + fetcher: (input) => + generateImageStreamFn({ + data: { ...input, prompt: resolveMediaPrompt(input.prompt).text }, + }), }) return ( diff --git a/examples/ts-react-chat/src/routes/generations.structured-output.tsx b/examples/ts-react-chat/src/routes/generations.structured-output.tsx index c11d3f143..e3b824925 100644 --- a/examples/ts-react-chat/src/routes/generations.structured-output.tsx +++ b/examples/ts-react-chat/src/routes/generations.structured-output.tsx @@ -254,9 +254,6 @@ function StructuredOutputPage() { outputSchema: GuitarRecommendationSchema, connection: fetchServerSentEvents('/api/structured-output'), forwardedProps: { provider, model, stream }, - devtools: { - outputKind: 'structured', - }, onChunk: handleChunk, onError: (err) => { setError(err.message) diff --git a/examples/ts-react-chat/src/routes/generations.video.tsx b/examples/ts-react-chat/src/routes/generations.video.tsx index 792d975a9..d9e2683f0 100644 --- a/examples/ts-react-chat/src/routes/generations.video.tsx +++ b/examples/ts-react-chat/src/routes/generations.video.tsx @@ -3,6 +3,7 @@ import { createFileRoute } from '@tanstack/react-router' import { useGenerateVideo } from '@tanstack/ai-react' import type { UseGenerateVideoReturn } from '@tanstack/ai-react' import { fetchServerSentEvents } from '@tanstack/ai-client' +import { resolveMediaPrompt } from '@tanstack/ai' import { generateVideoFn, generateVideoStreamFn } from '../lib/server-fns' function StreamingVideoGeneration() { @@ -21,7 +22,10 @@ function DirectVideoGeneration() { const [prompt, setPrompt] = useState('') const hookReturn = useGenerateVideo({ - fetcher: (input) => generateVideoFn({ data: input }), + fetcher: (input) => + generateVideoFn({ + data: { ...input, prompt: resolveMediaPrompt(input.prompt).text }, + }), }) return ( @@ -33,7 +37,10 @@ function ServerFnVideoGeneration() { const [prompt, setPrompt] = useState('') const hookReturn = useGenerateVideo({ - fetcher: (input) => generateVideoStreamFn({ data: input }), + fetcher: (input) => + generateVideoStreamFn({ + data: { ...input, prompt: resolveMediaPrompt(input.prompt).text }, + }), }) return ( diff --git a/examples/ts-react-chat/src/routes/index.tsx b/examples/ts-react-chat/src/routes/index.tsx index 47f16f002..2c2450b01 100644 --- a/examples/ts-react-chat/src/routes/index.tsx +++ b/examples/ts-react-chat/src/routes/index.tsx @@ -27,7 +27,7 @@ import { import { clientTools } from '@tanstack/ai-client' import { ThinkingPart } from '@tanstack/ai-react-ui' import type { UIMessage } from '@tanstack/ai-react' -import type { ContentPart, TranscriptionResult } from '@tanstack/ai' +import type { ContentPart } from '@tanstack/ai' import type { GeminiInteractionsCustomEventValue } from '@tanstack/ai-gemini/experimental' import type { ModelOption } from '@/lib/model-selection' import GuitarRecommendation from '@/components/example-GuitarRecommendation' @@ -428,18 +428,14 @@ function ChatPage() { // Voice input: record from the mic, transcribe via /api/transcribe, then drop // the text into the composer for the user to review/edit/send. (Text chat // models don't accept raw audio; transcription is the path that works.) - // NOTE: the explicit type arg works around an inference bug in the generation - // hooks' `onResult` (same root cause as the recorder's `onComplete`, now - // fixed there) — without it, `r` is implicitly `any` and the call won't - // typecheck under strict mode. + // `onResult`'s `r` infers as `TranscriptionResult` from the hook (no explicit + // type arg needed — the generation hooks' result-type inference handles it). // Surface voice-input failures (permission denied, recorder error, // transcription error) to the user rather than only logging them — a silent // mic button is the worst outcome. const [recordError, setRecordError] = useState(null) - const { generate: transcribe, isLoading: isTranscribing } = useTranscription< - (r: TranscriptionResult) => void - >({ + const { generate: transcribe, isLoading: isTranscribing } = useTranscription({ connection: fetchServerSentEvents('/api/transcribe'), onResult: (r) => setInput((prev) => (prev ? `${prev} ${r.text}` : r.text)), // A failed transcription (network/provider) is just as silent as a mic diff --git a/examples/ts-react-native-chat/package.json b/examples/ts-react-native-chat/package.json index 41d31afd6..e9e2e456f 100644 --- a/examples/ts-react-native-chat/package.json +++ b/examples/ts-react-native-chat/package.json @@ -18,6 +18,7 @@ "dev:app": "node -e \"const { spawnSync } = require('node:child_process'); const env = { ...process.env, EXPO_NO_DOTENV: '1' }; delete env.OPENAI_API_KEY; delete env.OPENAI_MODEL; const args = ['exec', 'expo', 'start', '--lan', '--clear']; const result = process.platform === 'win32' ? spawnSync(env.ComSpec ?? 'cmd.exe', ['/d', '/s', '/c', 'pnpm.cmd', ...args], { stdio: 'inherit', env }) : spawnSync('pnpm', args, { stdio: 'inherit', env }); process.exit(result.status ?? 1)\"", "test:dev-script": "node --test scripts/dev.test.mjs", "typecheck": "tsc --noEmit", + "test:types": "tsc --noEmit", "smoke:server": "tsx scripts/smoke-server.ts", "smoke:expo": "node -e \"const { spawnSync } = require('node:child_process'); const env = { ...process.env, EXPO_NO_DOTENV: '1' }; delete env.OPENAI_API_KEY; delete env.OPENAI_MODEL; const args = ['exec', 'expo', 'export', '--platform', 'ios', '--no-bytecode', '--output-dir', '.expo-example-dist', '--max-workers', '0']; const result = process.platform === 'win32' ? spawnSync(env.ComSpec ?? 'cmd.exe', ['/d', '/s', '/c', 'pnpm.cmd', ...args], { stdio: 'inherit', env }) : spawnSync('pnpm', args, { stdio: 'inherit', env }); process.exit(result.status ?? 1)\"", "verify:react-resolution": "node scripts/verify-react-resolution.mjs", diff --git a/examples/ts-react-native-chat/tsconfig.json b/examples/ts-react-native-chat/tsconfig.json index 087b17064..5839a2853 100644 --- a/examples/ts-react-native-chat/tsconfig.json +++ b/examples/ts-react-native-chat/tsconfig.json @@ -6,20 +6,7 @@ "moduleResolution": "Bundler", "allowImportingTsExtensions": true, "noEmit": true, - "types": ["node", "react"], - "paths": { - "@tanstack/ai": ["../../packages/ai/src/index.ts"], - "@tanstack/ai/adapters": ["../../packages/ai/src/activities/index.ts"], - "@tanstack/ai/adapter-internals": [ - "../../packages/ai/src/adapter-internals.ts" - ], - "@tanstack/ai/client": ["../../packages/ai/src/client.ts"], - "@tanstack/ai-client": ["../../packages/ai-client/src/index.ts"], - "@tanstack/ai-utils": ["../../packages/ai-utils/src/index.ts"], - "@tanstack/ai-openai": ["../../packages/ai-openai/src/index.ts"], - "@tanstack/ai-react": ["../../packages/ai-react/src/index.ts"], - "@tanstack/openai-base": ["../../packages/openai-base/src/index.ts"] - } + "types": ["node", "react"] }, "include": [ "index.ts", diff --git a/examples/ts-solid-chat/src/routes/index.tsx b/examples/ts-solid-chat/src/routes/index.tsx index b8ee0cdb3..45f0f3657 100644 --- a/examples/ts-solid-chat/src/routes/index.tsx +++ b/examples/ts-solid-chat/src/routes/index.tsx @@ -310,8 +310,7 @@ function ChatPage() { const { messages, sendMessage, isLoading, addToolApprovalResponse, stop } = useChat({ - connection: chatOptions.connection, - tools: clientTools, + ...chatOptions, onChunk: (chunk: any) => { setChunks((prev) => [...prev, chunk]) }, diff --git a/package.json b/package.json index f821b130a..7f9c98d3f 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,8 @@ "clean": "pnpm --filter \"./packages/**\" run clean", "clean:all": "git clean -fdx --exclude=\"!.env\"", "test": "pnpm run test:ci", - "test:pr": "pnpm run test:react-native && nx affected --targets=test:sherif,test:knip,test:docs,test:kiira,test:eslint,test:lib,test:types,test:build,build --exclude=examples/**,testing/**", - "test:ci": "pnpm run test:react-native && nx run-many --targets=test:sherif,test:knip,test:docs,test:kiira,test:eslint,test:lib,test:types,test:build,build --exclude=examples/**,testing/**", + "test:pr": "pnpm run test:react-native && nx affected --targets=test:sherif,test:knip,test:docs,test:kiira,test:eslint,test:lib,test:types,test:build,build --exclude=examples/**,testing/** && pnpm run test:types:examples", + "test:ci": "pnpm run test:react-native && nx run-many --targets=test:sherif,test:knip,test:docs,test:kiira,test:eslint,test:lib,test:types,test:build,build --exclude=examples/**,testing/** && pnpm run test:types:examples", "test:eslint": "nx affected --target=test:eslint --exclude=examples/**,testing/**", "test:sherif": "sherif", "test:lib": "nx affected --targets=test:lib --exclude=examples/**,testing/**", @@ -23,6 +23,7 @@ "test:coverage": "nx affected --targets=test:coverage --exclude=examples/**,testing/**", "test:build": "nx affected --target=test:build --exclude=examples/**,testing/**", "test:types": "nx affected --targets=test:types --exclude=examples/**,testing/**", + "test:types:examples": "nx run-many --targets=test:types --projects=examples/**,testing/**", "test:knip": "knip", "test:docs": "node scripts/verify-links.ts", "test:kiira": "kiira check", diff --git a/packages/ai-grok/tests/summarize-callsite-type-safety.test.ts b/packages/ai-grok/tests/summarize-callsite-type-safety.test.ts new file mode 100644 index 000000000..8ddc7b68f --- /dev/null +++ b/packages/ai-grok/tests/summarize-callsite-type-safety.test.ts @@ -0,0 +1,37 @@ +/** + * Call-site type-safety guard for `summarize({ adapter: grokSummarize(...) })`. + * + * Why this lives here: the assignability of a provider's summarize adapter to + * the `summarize()` `adapter` param is only checked when `summarize()` is + * actually *called* — merely constructing the adapter (as the rest of + * `grok-adapter.test.ts` does) never instantiates the + * `TAdapter extends SummarizeAdapter` constraint. This file is + * an *included*-package guard so the per-provider adapter -> activity contract + * is exercised by CI without depending on the (excluded) example/testing apps. + * See issue #820 (the CI gap this closes). + * + * These are POSITIVE assertions: `grokSummarize(...)` must be assignable to the + * `summarize()` `adapter` param for every current Grok model. They originally + * tracked the known options-shape bug (#821) with `@ts-expect-error`; that fix + * landed (#854 removed the index signature from the Grok options), so a + * regression now surfaces as a real type error here instead of silently. + * + * Compile-time only: `_callSiteTypeChecks` is never invoked, so no adapter is + * constructed and no network call is made — the assertions exist purely to make + * the call-site constraint visible to `tsc`. + */ +import { describe, expect, it } from 'vitest' +import { summarize } from '@tanstack/ai' +import { grokSummarize } from '../src/adapters/summarize' + +// Never invoked — compile-time call-site assertions only. +function _callSiteTypeChecks() { + void summarize({ adapter: grokSummarize('grok-4.3'), text: '' }) + void summarize({ adapter: grokSummarize('grok-build-0.1'), text: '' }) +} + +describe('grokSummarize -> summarize() call-site contract', () => { + it('keeps the compile-time guard wired (see _callSiteTypeChecks)', () => { + expect(typeof _callSiteTypeChecks).toBe('function') + }) +}) diff --git a/testing/e2e/package.json b/testing/e2e/package.json index a2af416a8..caae171e8 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -5,6 +5,7 @@ "scripts": { "dev": "vite dev --port 3010", "build": "vite build", + "test:types": "tsc --noEmit", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", "record": "LLMOCK_RECORD=true pnpm dev", diff --git a/testing/e2e/src/routes/$provider/index.tsx b/testing/e2e/src/routes/$provider/index.tsx index fb1bdaa15..94e72acba 100644 --- a/testing/e2e/src/routes/$provider/index.tsx +++ b/testing/e2e/src/routes/$provider/index.tsx @@ -28,6 +28,7 @@ function ProviderPage() { testId: undefined, aimockPort: undefined, mode: undefined, + persistence: undefined, }} className="p-3 bg-gray-800/50 border border-gray-700 rounded-lg hover:border-orange-500/40 transition-colors text-center text-sm" > diff --git a/testing/e2e/src/routes/api.arktype-tool-wire.ts b/testing/e2e/src/routes/api.arktype-tool-wire.ts index f2f615097..f0c9c3837 100644 --- a/testing/e2e/src/routes/api.arktype-tool-wire.ts +++ b/testing/e2e/src/routes/api.arktype-tool-wire.ts @@ -47,14 +47,10 @@ export const Route = createFileRoute('/api/arktype-tool-wire')({ }) } - const adapter = createOpenRouterText( - 'openai/gpt-4o' as never, - DUMMY_KEY, - { - serverURL: `${LLMOCK_DEFAULT_BASE}/v1`, - httpClient, - }, - ) + const adapter = createOpenRouterText('openai/gpt-4o', DUMMY_KEY, { + serverURL: `${LLMOCK_DEFAULT_BASE}/v1`, + httpClient, + }) try { for await (const _ of chat({ diff --git a/testing/e2e/src/routes/api.openai-shell-skills-wire.ts b/testing/e2e/src/routes/api.openai-shell-skills-wire.ts index 58e1180a4..ce5dcdb23 100644 --- a/testing/e2e/src/routes/api.openai-shell-skills-wire.ts +++ b/testing/e2e/src/routes/api.openai-shell-skills-wire.ts @@ -158,7 +158,7 @@ export const Route = createFileRoute('/api/openai-shell-skills-wire')({ }) } - const adapter = createOpenaiChat('gpt-4o', DUMMY_KEY, { + const adapter = createOpenaiChat('gpt-5.2', DUMMY_KEY, { fetch: capturingFetch, }) diff --git a/testing/e2e/src/routes/api.openrouter-cost.ts b/testing/e2e/src/routes/api.openrouter-cost.ts index 8c0fe56cf..6ff83add8 100644 --- a/testing/e2e/src/routes/api.openrouter-cost.ts +++ b/testing/e2e/src/routes/api.openrouter-cost.ts @@ -16,13 +16,9 @@ export const Route = createFileRoute('/api/openrouter-cost')({ server: { handlers: { POST: async () => { - const adapter = createOpenRouterText( - 'openai/gpt-4o' as never, - DUMMY_KEY, - { - serverURL: `${LLMOCK_DEFAULT_BASE}/openrouter-cost/v1`, - }, - ) + const adapter = createOpenRouterText('openai/gpt-4o', DUMMY_KEY, { + serverURL: `${LLMOCK_DEFAULT_BASE}/openrouter-cost/v1`, + }) let usage: Record | undefined try { diff --git a/testing/e2e/src/routes/api.openrouter-web-tools-wire.ts b/testing/e2e/src/routes/api.openrouter-web-tools-wire.ts index cfe96883e..953ddedf5 100644 --- a/testing/e2e/src/routes/api.openrouter-web-tools-wire.ts +++ b/testing/e2e/src/routes/api.openrouter-web-tools-wire.ts @@ -40,14 +40,10 @@ export const Route = createFileRoute('/api/openrouter-web-tools-wire')({ }) } - const adapter = createOpenRouterText( - 'openai/gpt-4o' as never, - DUMMY_KEY, - { - serverURL: `${LLMOCK_DEFAULT_BASE}/v1`, - httpClient, - }, - ) + const adapter = createOpenRouterText('openai/gpt-4o', DUMMY_KEY, { + serverURL: `${LLMOCK_DEFAULT_BASE}/v1`, + httpClient, + }) try { for await (const _ of chat({ diff --git a/testing/e2e/src/routes/api.otel-usage.ts b/testing/e2e/src/routes/api.otel-usage.ts index 0171fdd2b..367d83715 100644 --- a/testing/e2e/src/routes/api.otel-usage.ts +++ b/testing/e2e/src/routes/api.otel-usage.ts @@ -130,7 +130,7 @@ export const Route = createFileRoute('/api/otel-usage')({ const adapter = provider === 'openrouter' - ? createOpenRouterText('openai/gpt-4o' as never, DUMMY_KEY, { + ? createOpenRouterText('openai/gpt-4o', DUMMY_KEY, { serverURL: `${LLMOCK_DEFAULT_BASE}/openrouter-cost/v1`, }) : createOpenaiChatCompletions('gpt-4o', DUMMY_KEY, { diff --git a/testing/e2e/src/routes/api.summarize.ts b/testing/e2e/src/routes/api.summarize.ts index 6ba0e2e37..a5eff2226 100644 --- a/testing/e2e/src/routes/api.summarize.ts +++ b/testing/e2e/src/routes/api.summarize.ts @@ -6,6 +6,7 @@ import { createGeminiSummarize } from '@tanstack/ai-gemini' import { createOllamaSummarize } from '@tanstack/ai-ollama' import { createGrokSummarize } from '@tanstack/ai-grok' import { createOpenRouterSummarize } from '@tanstack/ai-openrouter' +import { HTTPClient } from '@openrouter/sdk' import type { Provider } from '@/lib/types' const LLMOCK_BASE = process.env.LLMOCK_URL || 'http://127.0.0.1:4010' @@ -24,6 +25,26 @@ function testHeaders(testId?: string): Record | undefined { return testId ? { 'X-Test-Id': testId } : undefined } +/** + * The OpenRouter SDK exposes no raw `headers` option, so test-bucket headers + * (`X-Test-Id`) are injected via an `HTTPClient` `beforeRequest` hook — the + * same pattern the OpenRouter wire-format routes use. + */ +function openRouterHttpClient( + headers?: Record, +): HTTPClient | undefined { + if (!headers) return undefined + const httpClient = new HTTPClient() + httpClient.addHook('beforeRequest', (req) => { + const next = new Request(req) + for (const [key, value] of Object.entries(headers)) { + next.headers.set(key, value) + } + return next + }) + return httpClient +} + function createSummarizeAdapter( provider: Provider, aimockPort?: number, @@ -59,12 +80,12 @@ function createSummarizeAdapter( openrouter: () => createOpenRouterSummarize('openai/gpt-4o', DUMMY_KEY, { serverURL: openaiUrl(aimockPort), - headers, + httpClient: openRouterHttpClient(headers), }), 'openrouter-responses': () => createOpenRouterSummarize('openai/gpt-4o', DUMMY_KEY, { serverURL: openaiUrl(aimockPort), - headers, + httpClient: openRouterHttpClient(headers), }), } return factories[provider]?.() diff --git a/testing/e2e/src/routes/api.tool-call-lifecycle-wire.ts b/testing/e2e/src/routes/api.tool-call-lifecycle-wire.ts index e822cdcc0..fc9c55ea7 100644 --- a/testing/e2e/src/routes/api.tool-call-lifecycle-wire.ts +++ b/testing/e2e/src/routes/api.tool-call-lifecycle-wire.ts @@ -121,6 +121,7 @@ function createServerToolAdapter(): AnyTextAdapter { timestamp: Date.now(), } }, + structuredOutput: async () => ({ data: {}, rawText: '{}' }), } } diff --git a/testing/e2e/src/routes/tools-test.tsx b/testing/e2e/src/routes/tools-test.tsx index ad4f58cf0..ddc8fbcf9 100644 --- a/testing/e2e/src/routes/tools-test.tsx +++ b/testing/e2e/src/routes/tools-test.tsx @@ -1,6 +1,7 @@ import { useState, useCallback, useRef, useEffect, useMemo } from 'react' import { createFileRoute } from '@tanstack/react-router' import { useChat, fetchServerSentEvents } from '@tanstack/ai-react' +import type { UIMessage } from '@tanstack/ai-react' import { modelMessagesToUIMessages, toolDefinition, @@ -195,8 +196,14 @@ function ToolsTestPage() { // Create tracked tools (memoized since addEvent is stable) const clientTools = useRef(createTrackedTools(addEvent)).current + // The fixture intentionally carries a `getWeather` tool call that isn't part + // of `clientTools`, so the generic message type can't be narrowed to the + // client-tool union — cast to the message type `useChat` infers from `tools`. const initialMessages = useMemo( - () => createHistoryFixtureMessages(historyFixture), + () => + createHistoryFixtureMessages(historyFixture) as Array< + UIMessage + >, [historyFixture], ) diff --git a/testing/panel/README.md b/testing/panel/README.md index c18534427..a56095052 100644 --- a/testing/panel/README.md +++ b/testing/panel/README.md @@ -22,9 +22,9 @@ Then open http://localhost:3010 ## Creating Trace Files -Trace files are automatically created when you use the chat interface with a `traceId`. The panel subscribes to `aiEventClient` events to record all stream activity. +Trace files are automatically created when you use the chat interface with a `traceId`. The `api.chat` route wraps the adapter with `wrapAdapterForRecording`, which writes a `ChunkRecording` (see `src/lib/recording.ts`) to a file when the stream completes. -You can also create trace files programmatically by subscribing to events: +You can also create trace files programmatically by subscribing to `aiEventClient` events with `createEventRecording`, which reconstructs the same `ChunkRecording` (AG-UI `StreamChunk`s) from the devtools event stream: ```typescript import { createEventRecording } from '@/lib/recording' @@ -36,15 +36,13 @@ const recording = createEventRecording('tmp/my-trace.json', 'my-trace-id') recording.stop() ``` -The recording utility automatically captures: +Either path captures: - Stream chunks (content, tool calls, tool results, done, errors, thinking) - Final accumulated content - Tool calls and their results - Finish reason -Or capture traces from the test panel and save them. - ## Trace Format ```json diff --git a/testing/panel/app.config.ts b/testing/panel/app.config.ts deleted file mode 100644 index 1d505309d..000000000 --- a/testing/panel/app.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { defineConfig } from '@tanstack/react-start/config' -import viteTsConfigPaths from 'vite-tsconfig-paths' - -export default defineConfig({ - vite: { - plugins: () => [ - viteTsConfigPaths({ - projects: ['./tsconfig.json'], - }), - ], - }, -}) diff --git a/testing/panel/package.json b/testing/panel/package.json index 38bfc2558..1b2902242 100644 --- a/testing/panel/package.json +++ b/testing/panel/package.json @@ -6,6 +6,7 @@ "dev": "vite dev --port 3010", "build": "vite build", "preview": "vite preview", + "test:types": "tsc --noEmit", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui" }, diff --git a/testing/panel/src/components/Header.tsx b/testing/panel/src/components/Header.tsx index 1dd8f20a6..21e94f96f 100644 --- a/testing/panel/src/components/Header.tsx +++ b/testing/panel/src/components/Header.tsx @@ -72,6 +72,7 @@ export default function Header() { setIsOpen(false)} className="flex items-center gap-3 p-3 rounded-lg hover:bg-gray-800 transition-colors mb-2" activeProps={{ diff --git a/testing/panel/src/lib/recording.ts b/testing/panel/src/lib/recording.ts index 8ecd9aade..a814340ed 100644 --- a/testing/panel/src/lib/recording.ts +++ b/testing/panel/src/lib/recording.ts @@ -1,11 +1,14 @@ import * as fs from 'node:fs/promises' import * as path from 'node:path' +import { EventType } from '@tanstack/ai' import { aiEventClient as baseAiEventClient } from '@tanstack/ai-event-client' import type { AIDevtoolsEventMap } from '@tanstack/ai-event-client' -import type { StreamChunk } from '@tanstack/ai' +import type { StreamChunk, TokenUsage } from '@tanstack/ai' /** - * Recording data structure matching the old format + * Recording data structure matching the trace format produced by + * `wrapAdapterForRecording` in `routes/api.chat.ts` and consumed by the stream + * debugger. Each `chunk` is a `StreamChunk` (an AG-UI protocol event). */ export interface RecordedToolCall { id: string @@ -31,6 +34,161 @@ export interface ChunkRecording { } } +type FinishReason = 'stop' | 'length' | 'content_filter' | 'tool_calls' | null + +const normalizeFinishReason = (finishReason: string | null): FinishReason => { + if ( + finishReason === 'stop' || + finishReason === 'length' || + finishReason === 'content_filter' || + finishReason === 'tool_calls' + ) { + return finishReason + } + return 'stop' +} + +// --------------------------------------------------------------------------- +// AG-UI event constructors. +// +// The devtools event stream is already decomposed into per-kind chunks +// (`text:chunk:content`, `text:chunk:tool-call`, ...). We reconstruct the +// equivalent AG-UI `StreamChunk`s — including the START/END boundary events the +// StreamProcessor needs to open and close messages and tool calls — so the +// recorded trace replays the same way an adapter-recorded one (from +// `wrapAdapterForRecording`) does. +// --------------------------------------------------------------------------- + +const textMessageStart = ( + messageId: string, + model: string, + timestamp: number, +): StreamChunk => ({ + type: EventType.TEXT_MESSAGE_START, + messageId, + role: 'assistant', + model, + timestamp, +}) + +const textMessageContent = ( + messageId: string, + content: string, + delta: string | undefined, + model: string, + timestamp: number, +): StreamChunk => ({ + type: EventType.TEXT_MESSAGE_CONTENT, + messageId, + delta: delta ?? '', + content, + model, + timestamp, +}) + +const textMessageEnd = ( + messageId: string, + model: string, + timestamp: number, +): StreamChunk => ({ + type: EventType.TEXT_MESSAGE_END, + messageId, + model, + timestamp, +}) + +const toolCallStart = ( + toolCallId: string, + toolName: string, + index: number, + model: string, + timestamp: number, +): StreamChunk => ({ + type: EventType.TOOL_CALL_START, + toolCallId, + toolCallName: toolName, + toolName, + index, + model, + timestamp, +}) + +const toolCallArgs = ( + toolCallId: string, + delta: string, + args: string, + model: string, + timestamp: number, +): StreamChunk => ({ + type: EventType.TOOL_CALL_ARGS, + toolCallId, + delta, + args, + model, + timestamp, +}) + +const toolCallResult = ( + messageId: string, + toolCallId: string, + content: string, + model: string, + timestamp: number, +): StreamChunk => ({ + type: EventType.TOOL_CALL_RESULT, + messageId, + toolCallId, + content, + role: 'tool', + model, + timestamp, +}) + +const runFinished = ( + runId: string, + threadId: string, + finishReason: string | null, + usage: TokenUsage | undefined, + model: string, + timestamp: number, +): StreamChunk => ({ + type: EventType.RUN_FINISHED, + runId, + threadId, + finishReason: normalizeFinishReason(finishReason), + usage, + model, + timestamp, +}) + +const runError = ( + runId: string, + threadId: string, + message: string, + model: string, + timestamp: number, +): StreamChunk => ({ + type: EventType.RUN_ERROR, + runId, + threadId, + message, + model, + timestamp, +}) + +const reasoningContent = ( + messageId: string, + delta: string | undefined, + model: string, + timestamp: number, +): StreamChunk => ({ + type: EventType.REASONING_MESSAGE_CONTENT, + messageId, + delta: delta ?? '', + model, + timestamp, +}) + /** * Creates an event-based recording that subscribes to aiEventClient events * and saves recordings to a file when a stream completes. @@ -51,148 +209,55 @@ export function createEventRecording( stop: () => void getStreamId: () => string | undefined } { - // Track active streams and their data - const activeStreams = new Map< - string, - { - streamId: string - requestId: string - model: string - provider: string - chunks: Array<{ - chunk: StreamChunk - timestamp: number - index: number - }> - accumulatedContent: string - toolCalls: Map - finishReason: string | null - traceId?: string - } - >() + type RecordedChunk = { chunk: StreamChunk; timestamp: number; index: number } + + // Per-stream recording state, including the AG-UI message/tool ids and the + // boundary flags used to emit START/END events exactly once. + interface StreamState { + streamId: string + requestId: string + model: string + provider: string + runId: string + threadId: string + messageId: string + reasoningId: string + messageStarted: boolean + startedToolCalls: Set + chunks: Array + accumulatedContent: string + toolCalls: Map + finishReason: string | null + } + + const activeStreams = new Map() // Track which streamId belongs to this recording (if traceId is provided) let recordingStreamId: string | undefined let chunkIndex = 0 - // Helper to reconstruct StreamChunk from events - const createContentChunk = ( - content: string, - delta: string | undefined, - model: string, - timestamp: number, - id: string, - ): StreamChunk => ({ - type: 'content', - content, - delta: delta ?? '', - model, - timestamp, - id, - }) - - const createToolCallChunk = ( - toolCallId: string, - toolName: string, - index: number, - arguments_: string, - model: string, + const pushChunk = ( + stream: StreamState, + chunk: StreamChunk, timestamp: number, - id: string, - ): StreamChunk => ({ - type: 'tool_call', - toolCall: { - id: toolCallId, - type: 'function', - function: { - name: toolName, - arguments: arguments_, - }, - }, - index, - model, - timestamp, - id, - }) - - const createToolResultChunk = ( - toolCallId: string, - result: string, - model: string, - timestamp: number, - id: string, - ): StreamChunk => ({ - type: 'tool_result', - toolCallId, - content: result, - model, - timestamp, - id, - }) - - type FinishReason = 'stop' | 'length' | 'content_filter' | 'tool_calls' | null - - const normalizeFinishReason = (finishReason: string | null): FinishReason => { - if ( - finishReason === 'stop' || - finishReason === 'length' || - finishReason === 'content_filter' || - finishReason === 'tool_calls' - ) { - return finishReason - } - return 'stop' + ): void => { + stream.chunks.push({ chunk, timestamp, index: chunkIndex++ }) } - const createDoneChunk = ( - finishReason: string | null, - usage?: { - promptTokens: number - completionTokens: number - totalTokens: number - }, - model?: string, - timestamp?: number, - id?: string, - ): StreamChunk => ({ - type: 'done', - finishReason: normalizeFinishReason(finishReason), - usage, - model: model ?? 'unknown', - timestamp: timestamp ?? Date.now(), - id: id ?? `done-${Date.now()}`, - }) - - const createErrorChunk = ( - error: string, - model: string, - timestamp: number, - id: string, - ): StreamChunk => ({ - type: 'error', - error: { - message: error, - }, - model, - timestamp, - id, - }) - - const createThinkingChunk = ( - content: string, - delta: string | undefined, - model: string, + // Ensures a TEXT_MESSAGE_START has been emitted before any content/end. + const ensureMessageStarted = ( + stream: StreamState, timestamp: number, - id: string, - ): StreamChunk => ({ - type: 'thinking', - content, - delta: delta ?? '', - model, - timestamp, - id, - }) + ): void => { + if (stream.messageStarted) return + stream.messageStarted = true + pushChunk( + stream, + textMessageStart(stream.messageId, stream.model, timestamp), + timestamp, + ) + } type DevtoolsEventHandler = (event: { payload: AIDevtoolsEventMap[TEventName] }) => void @@ -219,11 +284,16 @@ export function createEventRecording( requestId, model, provider, + runId: requestId || `run-${streamId}`, + threadId: `thread-${streamId}`, + messageId: `msg-${streamId}`, + reasoningId: `reasoning-${streamId}`, + messageStarted: false, + startedToolCalls: new Set(), chunks: [], accumulatedContent: '', toolCalls: new Map(), finishReason: null, - traceId: undefined, }) const optionsTraceId = options?.traceId @@ -255,19 +325,19 @@ export function createEventRecording( const stream = activeStreams.get(streamId) if (stream) { stream.accumulatedContent = content - const resolvedModel = model ?? 'unknown' - const chunkId = `chunk-${chunkIndex}` - stream.chunks.push({ - chunk: createContentChunk( + const resolvedModel = model ?? stream.model + ensureMessageStarted(stream, timestamp) + pushChunk( + stream, + textMessageContent( + stream.messageId, content, delta, resolvedModel, timestamp, - chunkId, ), timestamp, - index: chunkIndex++, - }) + ) } }, { withEventTarget: false }, @@ -289,21 +359,27 @@ export function createEventRecording( if (!shouldRecord(streamId)) return const stream = activeStreams.get(streamId) if (stream) { - const resolvedModel = model ?? 'unknown' - const chunkId = `chunk-${chunkIndex}` - stream.chunks.push({ - chunk: createToolCallChunk( - toolCallId, - toolName, - index, - args, - resolvedModel, + const resolvedModel = model ?? stream.model + // Emit a TOOL_CALL_START the first time we see this tool call. + if (!stream.startedToolCalls.has(toolCallId)) { + stream.startedToolCalls.add(toolCallId) + pushChunk( + stream, + toolCallStart( + toolCallId, + toolName, + index, + resolvedModel, + timestamp, + ), timestamp, - chunkId, - ), + ) + } + pushChunk( + stream, + toolCallArgs(toolCallId, args, args, resolvedModel, timestamp), timestamp, - index: chunkIndex++, - }) + ) // Store tool call info for final recording (update arguments as they stream) const existing = stream.toolCalls.get(toolCallId) if (existing) { @@ -329,19 +405,18 @@ export function createEventRecording( if (!shouldRecord(streamId)) return const stream = activeStreams.get(streamId) if (stream) { - const resolvedModel = model ?? 'unknown' - const chunkId = `chunk-${chunkIndex}` - stream.chunks.push({ - chunk: createToolResultChunk( + const resolvedModel = model ?? stream.model + pushChunk( + stream, + toolCallResult( + stream.messageId, toolCallId, result, resolvedModel, timestamp, - chunkId, ), timestamp, - index: chunkIndex++, - }) + ) } }, { withEventTarget: false }, @@ -356,19 +431,28 @@ export function createEventRecording( const stream = activeStreams.get(streamId) if (stream) { stream.finishReason = finishReason || null - const resolvedModel = model ?? 'unknown' - const chunkId = `chunk-${chunkIndex}` - stream.chunks.push({ - chunk: createDoneChunk( + const resolvedModel = model ?? stream.model + // Close the open text message (if any) before finishing the run. + if (stream.messageStarted) { + pushChunk( + stream, + textMessageEnd(stream.messageId, resolvedModel, timestamp), + timestamp, + ) + stream.messageStarted = false + } + pushChunk( + stream, + runFinished( + stream.runId, + stream.threadId, finishReason, usage, resolvedModel, timestamp, - chunkId, ), timestamp, - index: chunkIndex++, - }) + ) } }, { withEventTarget: false }, @@ -382,13 +466,18 @@ export function createEventRecording( if (!shouldRecord(streamId)) return const stream = activeStreams.get(streamId) if (stream) { - const resolvedModel = model ?? 'unknown' - const chunkId = `chunk-${chunkIndex}` - stream.chunks.push({ - chunk: createErrorChunk(error, resolvedModel, timestamp, chunkId), + const resolvedModel = model ?? stream.model + pushChunk( + stream, + runError( + stream.runId, + stream.threadId, + error, + resolvedModel, + timestamp, + ), timestamp, - index: chunkIndex++, - }) + ) } }, { withEventTarget: false }, @@ -398,29 +487,22 @@ export function createEventRecording( const unsubscribeThinking = aiEventClient.on( 'text:chunk:thinking', (event) => { - const { streamId, content, delta, timestamp, model } = event.payload + const { streamId, delta, timestamp, model } = event.payload if (!shouldRecord(streamId)) return const stream = activeStreams.get(streamId) if (stream) { - const resolvedModel = model ?? 'unknown' - const chunkId = `chunk-${chunkIndex}` - stream.chunks.push({ - chunk: createThinkingChunk( - content, - delta, - resolvedModel, - timestamp, - chunkId, - ), + const resolvedModel = model ?? stream.model + pushChunk( + stream, + reasoningContent(stream.reasoningId, delta, resolvedModel, timestamp), timestamp, - index: chunkIndex++, - }) + ) } }, { withEventTarget: false }, ) - // Subscribe to text:request:completed to get final tool calls + // Subscribe to text:request:completed to get final content + finish reason const unsubscribeChatCompleted = aiEventClient.on( 'text:request:completed', (event) => { diff --git a/testing/panel/src/routes/api.image.ts b/testing/panel/src/routes/api.image.ts index efb3c7022..e9c4aeda4 100644 --- a/testing/panel/src/routes/api.image.ts +++ b/testing/panel/src/routes/api.image.ts @@ -1,8 +1,9 @@ import { createFileRoute } from '@tanstack/react-router' -import { generateImage, createImageOptions } from '@tanstack/ai' +import { generateImage } from '@tanstack/ai' import { geminiImage } from '@tanstack/ai-gemini' import { openaiImage } from '@tanstack/ai-openai' import { openRouterImage } from '@tanstack/ai-openrouter' +import type { AnyImageAdapter } from '@tanstack/ai' type Provider = 'openai' | 'gemini' | 'openrouter' @@ -24,30 +25,21 @@ export const Route = createFileRoute('/api/image')({ data.model || body.model || defaultModels[provider] try { - const adapterConfig = { - gemini: () => - createImageOptions({ - adapter: geminiImage(model as any), - }), - openai: () => - createImageOptions({ - adapter: openaiImage(model as any), - }), - openrouter: () => - createImageOptions({ - adapter: openRouterImage(model as any), - }), + const adapterConfig: Record AnyImageAdapter> = { + gemini: () => geminiImage(model as any), + openai: () => openaiImage(model as any), + openrouter: () => openRouterImage(model as any), } - // Get typed adapter options using createImageOptions pattern - const options = adapterConfig[provider]() + // Select the provider's image adapter + const adapter = adapterConfig[provider]() console.log( `>> image generation with model: ${model} on provider: ${provider}`, ) const result = await generateImage({ - ...options, + adapter, prompt, numberOfImages, size, diff --git a/testing/panel/src/routes/api.simulator-chat.ts b/testing/panel/src/routes/api.simulator-chat.ts index 1d62e691f..c971b97bb 100644 --- a/testing/panel/src/routes/api.simulator-chat.ts +++ b/testing/panel/src/routes/api.simulator-chat.ts @@ -1,6 +1,11 @@ import { createFileRoute } from '@tanstack/react-router' -import { chat, maxIterations, toServerSentEventsResponse } from '@tanstack/ai' -import type { AIAdapter, ChatOptions, StreamChunk } from '@tanstack/ai' +import { + EventType, + chat, + maxIterations, + toServerSentEventsResponse, +} from '@tanstack/ai' +import type { ModelMessage, StreamChunk } from '@tanstack/ai' import { clientServerTool, @@ -66,51 +71,91 @@ const VALID_TOOLS = new Set([ 'clientServerToolWithApproval', ]) +const MODEL = 'simulator-v1' + /** * Simulated LLM adapter that: * - Echoes messages back if no tool calls detected * - Parses tool call syntax and generates appropriate chunks + * + * Emits the standard AG-UI lifecycle events (`RUN_STARTED`, + * `TEXT_MESSAGE_*` / `TOOL_CALL_*`, `RUN_FINISHED`) that the chat engine and + * StreamProcessor consume. The return is cast to `any` at the call site, so + * this only needs to be a structurally valid streaming text source. */ -function createSimulatorAdapter(): AIAdapter { +function createSimulatorAdapter() { + // Stream a text message (start -> content deltas -> end) under one run. + async function* streamText( + text: string, + delayMs: number, + ): AsyncIterable { + const timestamp = Date.now() + const messageId = `sim-msg-${timestamp}` + + yield { + type: EventType.TEXT_MESSAGE_START, + messageId, + model: MODEL, + timestamp, + role: 'assistant', + } + + let accumulated = '' + for (const char of text) { + accumulated += char + yield { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId, + model: MODEL, + timestamp, + delta: char, + content: accumulated, + } + // Small delay for streaming effect + await new Promise((resolve) => setTimeout(resolve, delayMs)) + } + + yield { + type: EventType.TEXT_MESSAGE_END, + messageId, + model: MODEL, + timestamp: Date.now(), + } + } + return { name: 'simulator', - models: ['simulator-v1'] as const, - _modelProviderOptionsByName: {}, + model: MODEL, - async *chatStream(options: ChatOptions): AsyncIterable { + async *chatStream(options: { + messages: Array + }): AsyncIterable { const messages = options.messages const lastMessage = messages[messages.length - 1] + const runId = `sim-run-${Date.now()}` + const threadId = `sim-thread-${Date.now()}` + + yield { + type: EventType.RUN_STARTED, + runId, + threadId, + model: MODEL, + timestamp: Date.now(), + } + // Check if this is a tool result - if so, acknowledge it if (lastMessage?.role === 'tool') { - const timestamp = Date.now() - const id = `sim-${timestamp}` - - // Generate acknowledgment response const content = 'Tool execution completed. The result has been processed.' - // Stream content character by character for realistic effect - let accumulated = '' - for (const char of content) { - accumulated += char - yield { - type: 'content', - id, - model: 'simulator-v1', - timestamp, - delta: char, - content: accumulated, - role: 'assistant', - } - // Small delay for streaming effect - await new Promise((resolve) => setTimeout(resolve, 10)) - } + yield* streamText(content, 10) yield { - type: 'done', - id, - model: 'simulator-v1', + type: EventType.RUN_FINISHED, + runId, + threadId, + model: MODEL, timestamp: Date.now(), finishReason: 'stop', usage: { @@ -137,34 +182,17 @@ function createSimulatorAdapter(): AIAdapter { const toolCalls = parseToolCalls(userContent) const validToolCalls = toolCalls.filter((tc) => VALID_TOOLS.has(tc.name)) - const timestamp = Date.now() - const id = `sim-${timestamp}` - if (validToolCalls.length === 0) { // No tool calls - echo the message back const echoContent = `[Echo] ${userContent}` - // Stream content character by character - let accumulated = '' - for (const char of echoContent) { - accumulated += char - yield { - type: 'content', - id, - model: 'simulator-v1', - timestamp, - delta: char, - content: accumulated, - role: 'assistant', - } - // Small delay for streaming effect - await new Promise((resolve) => setTimeout(resolve, 15)) - } + yield* streamText(echoContent, 15) yield { - type: 'done', - id, - model: 'simulator-v1', + type: EventType.RUN_FINISHED, + runId, + threadId, + model: MODEL, timestamp: Date.now(), finishReason: 'stop', usage: { @@ -175,52 +203,59 @@ function createSimulatorAdapter(): AIAdapter { } } else { // Generate tool calls + const timestamp = Date.now() for (let i = 0; i < validToolCalls.length; i++) { const tc = validToolCalls[i] const toolCallId = `call-${timestamp}-${i}` const argsJson = JSON.stringify(tc.arguments) + yield { + type: EventType.TOOL_CALL_START, + toolCallId, + toolCallName: tc.name, + toolName: tc.name, + model: MODEL, + timestamp, + index: i, + } + // Stream tool call arguments character by character - // The arguments field contains the DELTA (incremental), not accumulated + let argsAccumulated = '' for (const char of argsJson) { + argsAccumulated += char yield { - type: 'tool_call', - id, - model: 'simulator-v1', + type: EventType.TOOL_CALL_ARGS, + toolCallId, + model: MODEL, timestamp, - toolCall: { - id: toolCallId, - type: 'function', - function: { - name: tc.name, - arguments: char, // Delta only, not accumulated - }, - }, - index: i, + delta: char, + args: argsAccumulated, } // Small delay for streaming effect await new Promise((resolve) => setTimeout(resolve, 5)) } + + yield { + type: EventType.TOOL_CALL_END, + toolCallId, + toolCallName: tc.name, + toolName: tc.name, + model: MODEL, + timestamp: Date.now(), + } } yield { - type: 'done', - id, - model: 'simulator-v1', + type: EventType.RUN_FINISHED, + runId, + threadId, + model: MODEL, timestamp: Date.now(), finishReason: 'tool_calls', usage: { promptTokens: 10, completionTokens: 50, totalTokens: 60 }, } } }, - - async summarize() { - throw new Error('Summarize not supported in simulator') - }, - - async createEmbeddings() { - throw new Error('Embeddings not supported in simulator') - }, } } @@ -241,7 +276,6 @@ export const Route = createFileRoute('/api/simulator-chat')({ const stream = chat({ adapter: adapter as any, - model: 'simulator-v1', tools: [ // Server tools with implementations serverTool, diff --git a/testing/panel/src/routes/api.structured.ts b/testing/panel/src/routes/api.structured.ts index 2bd9f877f..2dca0bf5a 100644 --- a/testing/panel/src/routes/api.structured.ts +++ b/testing/panel/src/routes/api.structured.ts @@ -1,5 +1,5 @@ import { createFileRoute } from '@tanstack/react-router' -import { chat, createChatOptions } from '@tanstack/ai' +import { chat } from '@tanstack/ai' import { anthropicText } from '@tanstack/ai-anthropic' import { geminiText } from '@tanstack/ai-gemini' import { grokText } from '@tanstack/ai-grok' @@ -7,6 +7,7 @@ import { openaiText } from '@tanstack/ai-openai' import { ollamaText } from '@tanstack/ai-ollama' import { openRouterText } from '@tanstack/ai-openrouter' import { z } from 'zod' +import type { AnyTextAdapter } from '@tanstack/ai' type Provider = | 'openai' @@ -75,37 +76,20 @@ export const Route = createFileRoute('/api/structured')({ // Determine the actual model being used const actualModel = model || defaultModels[provider] - // Pre-define typed adapter configurations with full type inference - // Model is passed to the adapter factory function for type-safe autocomplete - const adapterConfig = { - anthropic: () => - createChatOptions({ - adapter: anthropicText(actualModel as any), - }), - gemini: () => - createChatOptions({ - adapter: geminiText(actualModel as any), - }), - grok: () => - createChatOptions({ - adapter: grokText(actualModel as any), - }), - ollama: () => - createChatOptions({ - adapter: ollamaText(actualModel as any), - }), - openai: () => - createChatOptions({ - adapter: openaiText(actualModel as any), - }), - openrouter: () => - createChatOptions({ - adapter: openRouterText(actualModel as any), - }), + // Pre-define adapter factories per provider. The map is typed as the + // text-adapter union so each provider's adapter unifies cleanly when + // handed to chat() below. + const adapterConfig: Record AnyTextAdapter> = { + anthropic: () => anthropicText(actualModel as any), + gemini: () => geminiText(actualModel as any), + grok: () => grokText(actualModel as any), + ollama: () => ollamaText(actualModel as any), + openai: () => openaiText(actualModel as any), + openrouter: () => openRouterText(actualModel as any), } - // Get typed adapter options using createChatOptions pattern - const options = adapterConfig[provider]() + // Select the provider's text adapter + const adapter = adapterConfig[provider]() console.log( `>> ${mode} output with model: ${actualModel} on provider: ${provider}`, @@ -114,7 +98,7 @@ export const Route = createFileRoute('/api/structured')({ if (mode === 'structured') { // Structured output mode - returns validated object const result = await chat({ - ...options, + adapter, messages: [ { role: 'user', @@ -139,7 +123,7 @@ export const Route = createFileRoute('/api/structured')({ } else { // One-shot markdown mode - returns streamed text const markdown = await chat({ - ...options, + adapter, stream: false, messages: [ { diff --git a/testing/panel/src/routes/api.summarize.ts b/testing/panel/src/routes/api.summarize.ts index 7dce8279b..0cbfbd1fe 100644 --- a/testing/panel/src/routes/api.summarize.ts +++ b/testing/panel/src/routes/api.summarize.ts @@ -1,11 +1,12 @@ import { createFileRoute } from '@tanstack/react-router' -import { summarize, createSummarizeOptions } from '@tanstack/ai' +import { summarize } from '@tanstack/ai' import { anthropicSummarize } from '@tanstack/ai-anthropic' import { geminiSummarize } from '@tanstack/ai-gemini' import { grokSummarize } from '@tanstack/ai-grok' import { openaiSummarize } from '@tanstack/ai-openai' import { ollamaSummarize } from '@tanstack/ai-ollama' import { openRouterSummarize } from '@tanstack/ai-openrouter' +import type { AnySummarizeAdapter } from '@tanstack/ai' type Provider = | 'openai' @@ -45,37 +46,20 @@ export const Route = createFileRoute('/api/summarize')({ // Determine the actual model being used const actualModel = model || defaultModels[provider] - // Pre-define typed adapter configurations with full type inference - // Model is passed to the adapter factory function for type-safe autocomplete - const adapterConfig = { - anthropic: () => - createSummarizeOptions({ - adapter: anthropicSummarize(actualModel as any), - }), - gemini: () => - createSummarizeOptions({ - adapter: geminiSummarize(actualModel as any), - }), - grok: () => - createSummarizeOptions({ - adapter: grokSummarize(actualModel as any), - }), - ollama: () => - createSummarizeOptions({ - adapter: ollamaSummarize(actualModel), - }), - openai: () => - createSummarizeOptions({ - adapter: openaiSummarize(actualModel as any), - }), - openrouter: () => - createSummarizeOptions({ - adapter: openRouterSummarize(actualModel as any), - }), + // Pre-define adapter factories per provider. The map is typed as the + // adapter union so each provider's summarize adapter unifies cleanly + // when handed to summarize() below. + const adapterConfig: Record AnySummarizeAdapter> = { + anthropic: () => anthropicSummarize(actualModel as any), + gemini: () => geminiSummarize(actualModel as any), + grok: () => grokSummarize(actualModel as any), + ollama: () => ollamaSummarize(actualModel as any), + openai: () => openaiSummarize(actualModel as any), + openrouter: () => openRouterSummarize(actualModel as any), } - // Get typed adapter options using createSummarizeOptions pattern - const options = adapterConfig[provider]() + // Select the provider's summarize adapter + const adapter = adapterConfig[provider]() console.log( `>> summarize with model: ${actualModel} on provider: ${provider} (stream: ${stream})`, @@ -88,7 +72,7 @@ export const Route = createFileRoute('/api/summarize')({ async start(controller) { try { const streamResult = summarize({ - ...options, + adapter, text, maxLength, style, @@ -131,7 +115,7 @@ export const Route = createFileRoute('/api/summarize')({ // Non-streaming mode const result = await summarize({ - ...options, + adapter, text, maxLength, style, diff --git a/testing/panel/src/routes/api.transcription.ts b/testing/panel/src/routes/api.transcription.ts index be6fa8f32..f3f66b278 100644 --- a/testing/panel/src/routes/api.transcription.ts +++ b/testing/panel/src/routes/api.transcription.ts @@ -26,7 +26,7 @@ export const Route = createFileRoute('/api/transcription')({ } try { - const adapter = openaiTranscription(model) + const adapter = openaiTranscription(model as any) // Prepare audio data let audioData: string | File diff --git a/testing/panel/src/routes/api.video.ts b/testing/panel/src/routes/api.video.ts index 3f5316aa8..3799cfaf7 100644 --- a/testing/panel/src/routes/api.video.ts +++ b/testing/panel/src/routes/api.video.ts @@ -61,7 +61,7 @@ export const Route = createFileRoute('/api/video')({ return new Response( JSON.stringify({ action: 'status', - jobId: result.jobId, + jobId, status: result.status, progress: result.progress, error: result.error, diff --git a/testing/panel/src/routes/stream-debugger.tsx b/testing/panel/src/routes/stream-debugger.tsx index 35c697205..5f5b72286 100644 --- a/testing/panel/src/routes/stream-debugger.tsx +++ b/testing/panel/src/routes/stream-debugger.tsx @@ -10,7 +10,11 @@ import { SkipForward, Upload, } from 'lucide-react' -import { StreamProcessor, uiMessageToModelMessages } from '@tanstack/ai' +import { + EventType, + StreamProcessor, + uiMessageToModelMessages, +} from '@tanstack/ai' import type { ChunkRecording, @@ -189,13 +193,10 @@ function TestPanel() { }) .then((traceData) => { const chunkRecording: ChunkRecording = { - id: traceData.id, + version: '1.0', timestamp: traceData.timestamp, - metadata: { - provider: traceData.provider, - model: traceData.model, - messages: traceData.messages, - }, + model: traceData.model, + provider: traceData.provider, chunks: traceData.chunks, } loadRecording(chunkRecording) @@ -224,17 +225,15 @@ function TestPanel() { const nextIndex = currentChunkIndex + 1 if (nextIndex >= recording.chunks.length) return - if (!processorRef.current) { - createProcessor() - } + const processor = processorRef.current ?? createProcessor() const chunk = recording.chunks[nextIndex]?.chunk if (chunk) { - processorRef.current?.processChunk(chunk) + processor.processChunk(chunk) setCurrentChunkIndex(nextIndex) // Update result with current processor state - const state = processorRef.current.getState() + const state = processor.getState() // Convert toolCalls Map to array const toolCallsArray = Array.from(state.toolCalls.values()).map((tc) => ({ @@ -409,13 +408,10 @@ ${JSON.stringify(report.processorState, null, 2)} .then((traceData) => { // Convert trace data to ChunkRecording format const chunkRecording: ChunkRecording = { - id: traceData.id, + version: '1.0', timestamp: traceData.timestamp, - metadata: { - provider: traceData.provider, - model: traceData.model, - messages: traceData.messages, - }, + model: traceData.model, + provider: traceData.provider, chunks: traceData.chunks, } setRecording(chunkRecording) @@ -640,30 +636,33 @@ function ChunkItem({ isProcessed: boolean }) { const typeColors: Record = { - content: 'text-green-400', - tool_call: 'text-blue-400', - tool_result: 'text-purple-400', - done: 'text-yellow-400', - error: 'text-red-400', - thinking: 'text-cyan-400', - 'approval-requested': 'text-orange-400', - 'tool-input-available': 'text-pink-400', + [EventType.TEXT_MESSAGE_CONTENT]: 'text-green-400', + [EventType.TOOL_CALL_START]: 'text-blue-400', + [EventType.TOOL_CALL_ARGS]: 'text-blue-400', + [EventType.TOOL_CALL_END]: 'text-blue-400', + [EventType.TOOL_CALL_RESULT]: 'text-purple-400', + [EventType.RUN_FINISHED]: 'text-yellow-400', + [EventType.RUN_ERROR]: 'text-red-400', + [EventType.REASONING_MESSAGE_CONTENT]: 'text-cyan-400', + [EventType.CUSTOM]: 'text-orange-400', } const getSummary = (chunk: StreamChunk): string => { switch (chunk.type) { - case 'content': + case EventType.TEXT_MESSAGE_CONTENT: return `δ="${chunk.delta?.slice(0, 30) ?? ''}${(chunk.delta?.length ?? 0) > 30 ? '...' : ''}"` - case 'tool_call': - return `${chunk.toolCall.function.name}[${chunk.index}]` - case 'tool_result': + case EventType.TOOL_CALL_START: + return `${chunk.toolCallName}[${chunk.index ?? 0}]` + case EventType.TOOL_CALL_ARGS: + return `δ="${chunk.delta?.slice(0, 30) ?? ''}..."` + case EventType.TOOL_CALL_RESULT: return `${chunk.toolCallId}` - case 'done': - return `${chunk.finishReason}` - case 'thinking': + case EventType.RUN_FINISHED: + return `${chunk.finishReason ?? ''}` + case EventType.REASONING_MESSAGE_CONTENT: return `δ="${chunk.delta?.slice(0, 20) ?? ''}..."` - case 'error': - return chunk.error.message.slice(0, 30) + case EventType.RUN_ERROR: + return chunk.message?.slice(0, 30) ?? '' default: return '' } diff --git a/testing/panel/tests/basic-inference.spec.ts b/testing/panel/tests/basic-inference.spec.ts index b7a796553..50f7ef87f 100644 --- a/testing/panel/tests/basic-inference.spec.ts +++ b/testing/panel/tests/basic-inference.spec.ts @@ -7,11 +7,7 @@ */ import { test, expect } from '@playwright/test' -import { - PROVIDERS, - isProviderAvailable, - getInferenceCapableProviders, -} from './vendor-config' +import { PROVIDERS, isProviderAvailable } from './vendor-config' import { goToChatPage, selectProvider, diff --git a/testing/panel/tests/helpers.ts b/testing/panel/tests/helpers.ts index 60dfb5746..4b32e5bf4 100644 --- a/testing/panel/tests/helpers.ts +++ b/testing/panel/tests/helpers.ts @@ -3,7 +3,7 @@ */ import type { Page, APIRequestContext } from '@playwright/test' -import type { ProviderId, ProviderConfig } from './vendor-config' +import type { ProviderId } from './vendor-config' /** * Select a provider/model from the model dropdown on the chat page @@ -126,10 +126,6 @@ export async function waitForResponse( } else { // Loading might have been too fast or there's an error // Wait for either an assistant message or an error to appear - const messagesJson = page - .locator('pre') - .filter({ hasText: '"role"' }) - .first() try { // Wait for the messages JSON to contain an assistant message await page.waitForFunction( diff --git a/testing/react-native-smoke/package.json b/testing/react-native-smoke/package.json index d4b2d4235..399199bc1 100644 --- a/testing/react-native-smoke/package.json +++ b/testing/react-native-smoke/package.json @@ -14,6 +14,7 @@ }, "scripts": { "typecheck": "tsc --noEmit", + "test:types": "tsc --noEmit", "smoke:imports": "tsx scripts/assert-import-surface.ts", "smoke:expo": "expo export --platform ios --no-bytecode --output-dir .expo-smoke-dist --max-workers 0 && tsx scripts/assert-bundle-output.ts .expo-smoke-dist", "smoke:metro": "pnpm smoke:expo",