diff --git a/src/provider/language-model.ts b/src/provider/language-model.ts index 7cdcd62..57d2700 100644 --- a/src/provider/language-model.ts +++ b/src/provider/language-model.ts @@ -76,7 +76,9 @@ export class CursorLanguageModel implements LanguageModelV3 { readonly specificationVersion = "v3" as const; readonly modelId: string; readonly provider: string; - // Images are passed inline as base64 data, so no URLs are fetched natively. + // The local Cursor agent has no attachment channel, so file parts (images, + // directories, other media) are noted as text in message-map rather than + // fetched or attached; no URLs are resolved natively. readonly supportedUrls: Record = {}; constructor( diff --git a/src/provider/message-map.ts b/src/provider/message-map.ts index c5e5d53..36fdab6 100644 --- a/src/provider/message-map.ts +++ b/src/provider/message-map.ts @@ -1,5 +1,9 @@ -import type { LanguageModelV3Prompt } from "@ai-sdk/provider"; -import type { SDKImage, SDKUserMessage } from "@cursor/sdk"; +import { fileURLToPath } from "node:url"; +import type { + LanguageModelV3FilePart, + LanguageModelV3Prompt, +} from "@ai-sdk/provider"; +import type { SDKUserMessage } from "@cursor/sdk"; /** * Caps on inlined tool payloads in the flattened transcript. Tool outputs are @@ -32,17 +36,19 @@ function truncate(text: string, cap: number): string { * The Cursor agent keeps its own per-agent conversation memory, but opencode * re-sends the whole history each turn. To stay correct without double-counting * context, we create a fresh agent per turn (see language-model.ts) and flatten - * the entire prompt into one transcript message. Images from the final user - * turn are attached natively so multimodal models can see them. + * the entire prompt into one transcript message. File attachments (images and + * other files alike) are noted as text rather than attached natively: the + * Cursor LOCAL SDK agent — the only backend this chat path uses — cannot accept + * images in any form (URL images throw "URL images are only supported for cloud + * SDK agents"; inline base64 images fail the run with an empty `status:"error"`, + * surfacing as "Cursor run ended with status error" on an `@image` mention). */ export function promptToCursorMessage( prompt: LanguageModelV3Prompt, ): SDKUserMessage { const lines: string[] = []; - const images: SDKImage[] = []; - prompt.forEach((message, index) => { - const isLast = index === prompt.length - 1; + prompt.forEach((message) => { switch (message.role) { case "system": lines.push(`# System\n${message.content}`); @@ -51,16 +57,10 @@ export function promptToCursorMessage( const text: string[] = []; for (const part of message.content) { if (part.type === "text") text.push(part.text); - else if ( - part.type === "file" && - part.mediaType.startsWith("image/") - ) { - const image = fileToImage(part.data, part.mediaType); - // Only attach images natively for the final user turn; earlier ones - // are referenced by transcript order. - if (isLast && image) images.push(image); - text.push("[image attached]"); - } + // File parts (images and other files) can't be forwarded to the + // local Cursor agent, so note them as text instead of dropping + // them — the agent still learns a file was referenced. + else if (part.type === "file") text.push(fileNote(part)); } lines.push(`# User\n${text.join("\n")}`); break; @@ -96,27 +96,56 @@ export function promptToCursorMessage( } }); - const out: SDKUserMessage = { text: lines.join("\n\n") }; - if (images.length > 0) out.images = images; - return out; + return { text: lines.join("\n\n") }; } -function fileToImage( - data: string | Uint8Array | URL, - mediaType: string, -): SDKImage | undefined { - if (data instanceof URL) return { url: data.toString() }; - if (typeof data === "string") { - // Either a URL or already-base64 encoded data. - if (/^https?:\/\//i.test(data)) return { url: data }; - return { data, mimeType: mediaType }; - } - if (data instanceof Uint8Array) { - return { data: Buffer.from(data).toString("base64"), mimeType: mediaType }; - } +/** + * A short text note standing in for a file attachment that can't be forwarded + * to the local Cursor agent. + * + * opencode hands `@`-mentions to the provider as file parts; `text/plain` and + * directory mentions are already inlined as text upstream, so what reaches the + * provider here is images and other media/binaries. The Cursor LOCAL SDK agent + * (the only backend this chat path uses) cannot accept any of them: + * - `{ url }` images throw `ConfigurationError: URL images are only supported + * for cloud SDK agents`, + * - `{ data, mimeType }` inline-base64 images fail the run with an empty + * `status:"error"` (the "Cursor run ended with status error" a user hits on + * an `@image` mention — regardless of model). + * So rather than attaching — and failing the whole turn — we note the file as + * text. The run completes and the agent still learns a file was referenced. + */ +function fileNote(part: LanguageModelV3FilePart): string { + const name = part.filename ?? describeSource(part.data) ?? "file"; + return `[attached file: ${name} (${part.mediaType}) — not forwarded to Cursor]`; +} + +/** Matches strings with a URL scheme followed by `://` (http, https, file, …). */ +const URL_SCHEME = /^[a-z][a-z0-9+.-]*:\/\//i; + +function describeSource(data: string | Uint8Array | URL): string | undefined { + // file:// sources become filesystem paths: the agent runs locally with + // workspace tools that operate on paths, so a path is actionable where a + // file:// href is not. + if (data instanceof URL) + return data.protocol === "file:" ? fileUrlToPath(data) : data.href; + // Only accept strings that look like a URL. AI-SDK file parts may carry + // raw base64 in `data` (no `data:` prefix); returning that verbatim would + // inline the entire blob into the note text. + if (typeof data === "string" && URL_SCHEME.test(data)) + return data.startsWith("file://") ? fileUrlToPath(data) : data; return undefined; } +/** Convert a file:// URL to a filesystem path, falling back to the href when invalid. */ +function fileUrlToPath(url: string | URL): string { + try { + return fileURLToPath(url); + } catch { + return typeof url === "string" ? url : url.href; + } +} + /** * Extract only the final user turn as a Cursor message. Used when resuming a * pooled agent that already remembers the prior conversation, so we send just @@ -130,17 +159,10 @@ export function latestUserMessage( if (!last || last.role !== "user") return undefined; const text: string[] = []; - const images: SDKImage[] = []; for (const part of last.content) { if (part.type === "text") text.push(part.text); - else if (part.type === "file" && part.mediaType.startsWith("image/")) { - const image = fileToImage(part.data, part.mediaType); - if (image) images.push(image); - text.push("[image attached]"); - } + else if (part.type === "file") text.push(fileNote(part)); } - const out: SDKUserMessage = { text: text.join("\n") }; - if (images.length > 0) out.images = images; - return out; + return { text: text.join("\n") }; } diff --git a/test/message-map.test.ts b/test/message-map.test.ts index 02a7a2b..b382546 100644 --- a/test/message-map.test.ts +++ b/test/message-map.test.ts @@ -1,10 +1,27 @@ import { describe, expect, it } from "vitest"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import type { LanguageModelV3Prompt } from "@ai-sdk/provider"; import { latestUserMessage, promptToCursorMessage, } from "../src/provider/message-map.js"; +/** Write a temp file and return its absolute path + file:// URL string. */ +function tempFile(name: string, bytes: Buffer): { path: string; url: string } { + const dir = mkdtempSync(join(tmpdir(), "cursor-msgmap-")); + const path = join(dir, name); + writeFileSync(path, bytes); + return { path, url: pathToFileURL(path).href }; +} + +const PNG_BYTES = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + "base64", +); + describe("promptToCursorMessage", () => { it("flattens a multi-role conversation into a transcript", () => { const prompt: LanguageModelV3Prompt = [ @@ -21,24 +38,32 @@ describe("promptToCursorMessage", () => { expect(msg.images).toBeUndefined(); }); - it("attaches images from the final user turn as base64 data", () => { + // Cursor's LOCAL SDK agent (the only backend the chat path uses) cannot + // accept images in any form: `{url}` throws "URL images are only supported + // for cloud SDK agents", and `{data,mimeType}` base64 fails the run with an + // empty `status:"error"` (the "Cursor run ended with status error" a user + // hits on an `@image` mention). So we never populate `images`; instead every + // file part is surfaced as a text note so the run always completes. + it("never attaches images; notes an image file part as text", () => { const bytes = new Uint8Array([1, 2, 3, 4]); const prompt: LanguageModelV3Prompt = [ { role: "user", content: [ { type: "text", text: "Describe this" }, - { type: "file", data: bytes, mediaType: "image/png" }, + { + type: "file", + data: bytes, + mediaType: "image/png", + filename: "shot.png", + }, ], }, ]; const msg = promptToCursorMessage(prompt); - expect(msg.images).toHaveLength(1); - expect(msg.images![0]).toEqual({ - data: Buffer.from(bytes).toString("base64"), - mimeType: "image/png", - }); - expect(msg.text).toContain("[image attached]"); + expect(msg.images).toBeUndefined(); + expect(msg.text).toContain("shot.png"); + expect(msg.text).toContain("image/png"); }); it("includes tool outputs (truncated) instead of dropping them", () => { @@ -90,7 +115,7 @@ describe("promptToCursorMessage", () => { expect(msg.text.length).toBeLessThan(3000); }); - it("passes through image URLs", () => { + it("notes an http(s) image URL as text without attaching it", () => { const prompt: LanguageModelV3Prompt = [ { role: "user", @@ -99,12 +124,164 @@ describe("promptToCursorMessage", () => { type: "file", data: "https://example.com/a.png", mediaType: "image/png", + filename: "a.png", + }, + ], + }, + ]; + const msg = promptToCursorMessage(prompt); + expect(msg.images).toBeUndefined(); + expect(msg.text).toContain("a.png"); + }); + + it("notes a file:// image URL (URL object) as text, never as an image", () => { + const { url } = tempFile("px.png", PNG_BYTES); + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { type: "text", text: "look" }, + { + type: "file", + data: new URL(url), + mediaType: "image/png", + filename: "px.png", + }, + ], + }, + ]; + const msg = promptToCursorMessage(prompt); + expect(msg.images).toBeUndefined(); + expect(msg.text).toContain("px.png"); + }); + + it("notes a data: URI image as text, never as an image", () => { + const b64 = PNG_BYTES.toString("base64"); + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { + type: "file", + data: `data:image/png;base64,${b64}`, + mediaType: "image/png", + filename: "inline.png", }, ], }, ]; const msg = promptToCursorMessage(prompt); - expect(msg.images![0]).toEqual({ url: "https://example.com/a.png" }); + expect(msg.images).toBeUndefined(); + expect(msg.text).toContain("inline.png"); + }); + + it("notes a non-image file attachment as text instead of dropping it", () => { + const { url } = tempFile("doc.pdf", Buffer.from("%PDF-1.4\n")); + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { type: "text", text: "summarize" }, + { + type: "file", + data: new URL(url), + mediaType: "application/pdf", + filename: "doc.pdf", + }, + ], + }, + ]; + const msg = promptToCursorMessage(prompt); + expect(msg.images).toBeUndefined(); + expect(msg.text).toContain("doc.pdf"); + expect(msg.text).toContain("application/pdf"); + }); + + it("notes a directory @-mention as text instead of failing the turn", () => { + const { url } = tempFile("readme.md", Buffer.from("# hi\n")); + const dirUrl = url.slice(0, url.lastIndexOf("/")); // parent dir file:// URL + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { type: "text", text: "what's in here?" }, + { + type: "file", + data: new URL(dirUrl), + mediaType: "application/x-directory", + filename: "src", + }, + ], + }, + ]; + const msg = promptToCursorMessage(prompt); + expect(msg.images).toBeUndefined(); + expect(msg.text).toContain("src"); + expect(msg.text).toContain("application/x-directory"); + }); + + it("falls back to the filesystem path when a file:// part has no filename", () => { + const { path, url } = tempFile("noname.bin", Buffer.from([0, 1, 2])); + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { + type: "file", + data: new URL(url), + mediaType: "application/octet-stream", + }, + ], + }, + ]; + const msg = promptToCursorMessage(prompt); + expect(msg.images).toBeUndefined(); + // No filename → the note identifies the file by its filesystem path + // (not the file:// href) so the local agent's workspace tools can act + // on it directly. + expect(msg.text).toContain(path); + expect(msg.text).not.toContain("file://"); + expect(msg.text).toContain("application/octet-stream"); + }); + + it("uses an http URL string as the name when a file part has no filename", () => { + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { + type: "file", + data: "https://example.com/pic.png", + mediaType: "image/png", + }, + ], + }, + ]; + const msg = promptToCursorMessage(prompt); + expect(msg.images).toBeUndefined(); + expect(msg.text).toContain("https://example.com/pic.png"); + }); + + it("never inlines raw base64 string data; falls back to a generic name", () => { + const b64 = PNG_BYTES.toString("base64"); + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { + type: "file", + // Raw base64 with no data: prefix and no filename — must NOT + // end up verbatim in the note text. + data: b64, + mediaType: "image/png", + }, + ], + }, + ]; + const msg = promptToCursorMessage(prompt); + expect(msg.images).toBeUndefined(); + expect(msg.text).not.toContain(b64); + expect(msg.text).toContain("[attached file: file (image/png)"); }); }); @@ -126,4 +303,69 @@ describe("latestUserMessage", () => { ]; expect(latestUserMessage(prompt)).toBeUndefined(); }); + + it("notes an image in the final user turn as text, never as an image", () => { + const { url } = tempFile("px.png", PNG_BYTES); + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { type: "text", text: "look" }, + { + type: "file", + data: new URL(url), + mediaType: "image/png", + filename: "px.png", + }, + ], + }, + ]; + const msg = latestUserMessage(prompt); + expect(msg?.images).toBeUndefined(); + expect(msg?.text).toContain("px.png"); + }); + + it("notes a non-image attachment in the final user turn as text", () => { + const { url } = tempFile("doc.pdf", Buffer.from("%PDF-1.4\n")); + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { type: "text", text: "summarize" }, + { + type: "file", + data: new URL(url), + mediaType: "application/pdf", + filename: "doc.pdf", + }, + ], + }, + ]; + const msg = latestUserMessage(prompt); + expect(msg?.images).toBeUndefined(); + expect(msg?.text).toContain("doc.pdf"); + }); + + it("notes a directory @-mention in the final user turn as text", () => { + const { url } = tempFile("readme.md", Buffer.from("# hi\n")); + const dirUrl = url.slice(0, url.lastIndexOf("/")); + const prompt: LanguageModelV3Prompt = [ + { + role: "user", + content: [ + { type: "text", text: "what's in here?" }, + { + type: "file", + data: new URL(dirUrl), + mediaType: "application/x-directory", + filename: "src", + }, + ], + }, + ]; + const msg = latestUserMessage(prompt); + expect(msg?.images).toBeUndefined(); + expect(msg?.text).toContain("src"); + expect(msg?.text).toContain("application/x-directory"); + }); });