diff --git a/EXTENSIONS.md b/EXTENSIONS.md index 1104e9526..10e96eae3 100644 --- a/EXTENSIONS.md +++ b/EXTENSIONS.md @@ -215,6 +215,10 @@ api.playSound("sounds/click.mp3", { volume: 0.8 }); api.log("hello", payload); ``` +Sounds played by export-time cursor interaction handlers, such as `cursor:click`, are +included in MP4 exports. Sounds played outside that capture window are not added +automatically. + ### Read-only Queries ```js diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 9ed087523..65d8ceec3 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -138,6 +138,7 @@ import { saveEditorPresets, serializeEditorPresetSnapshot, } from "./editorPreferences"; +import { collectExtensionAudioRegionsForExport } from "./extensionExportAudio"; import ProjectBrowserDialog, { type ProjectLibraryEntry } from "./ProjectBrowserDialog"; import { hasUnsavedProjectChanges } from "./projectDirtyState"; import { @@ -3347,7 +3348,9 @@ export default function VideoEditor() { const handlePreviewSkipBack = useCallback(() => { const currentMs = timelinePlayheadTime * 1000; const keyframes = timelineRef.current?.keyframes ?? []; - const previous = [...keyframes].reverse().find((keyframe) => keyframe.time < currentMs - 50); + const previous = [...keyframes] + .reverse() + .find((keyframe) => keyframe.time < currentMs - 50); handleSeek(previous ? previous.time / 1000 : Math.max(0, timelinePlayheadTime - 5)); }, [handleSeek, timelinePlayheadTime]); @@ -3355,9 +3358,7 @@ export default function VideoEditor() { const currentMs = timelinePlayheadTime * 1000; const keyframes = timelineRef.current?.keyframes ?? []; const next = keyframes.find((keyframe) => keyframe.time > currentMs + 50); - handleSeek( - next ? next.time / 1000 : Math.min(timelineDuration, timelinePlayheadTime + 5), - ); + handleSeek(next ? next.time / 1000 : Math.min(timelineDuration, timelinePlayheadTime + 5)); }, [handleSeek, timelineDuration, timelinePlayheadTime]); const handleSelectZoom = useCallback((id: string | null) => { @@ -4356,6 +4357,16 @@ export default function VideoEditor() { selectedClipId !== null ? audio.selectedClipSourceAudioTrackSettings : audio.activeSourceAudioTrackSettings; + const extensionAudioRegions = collectExtensionAudioRegionsForExport( + extensionHost, + effectiveCursorTelemetry, + ); + const audioRegionsForExport = + extensionAudioRegions.length > 0 + ? [...audioRegions, ...extensionAudioRegions].sort( + (a, b) => a.startMs - b.startMs || a.id.localeCompare(b.id), + ) + : audioRegions; const exporterConfig = { videoUrl: videoPath, @@ -4426,7 +4437,7 @@ export default function VideoEditor() { cursorClickBounceDuration, cursorSway, frame, - audioRegions, + audioRegions: audioRegionsForExport, clipRegions, sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath: @@ -5148,10 +5159,7 @@ export default function VideoEditor() { volume={ audio.shouldMutePreviewVideo || audio.isCurrentClipMuted ? 0 - : Math.max( - 0, - Math.min(1, previewVolume * audio.embeddedSourcePreviewGain), - ) + : Math.max(0, Math.min(1, previewVolume * audio.embeddedSourcePreviewGain)) } suspendRendering={suspendRendering} /> diff --git a/src/components/video-editor/extensionExportAudio.test.ts b/src/components/video-editor/extensionExportAudio.test.ts new file mode 100644 index 000000000..7aec93229 --- /dev/null +++ b/src/components/video-editor/extensionExportAudio.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it, vi } from "vitest"; +import type { ExtensionExportAudioCue } from "@/lib/extensions/extensionHost"; +import { + collectExtensionAudioRegionsForExport, + extensionAudioCuesToRegions, + isExportableCursorInteraction, +} from "./extensionExportAudio"; +import type { CursorTelemetryPoint } from "./types"; + +describe("extension export audio", () => { + it("captures extension sounds for cursor interactions that preview treats as clicks", () => { + const cues: ExtensionExportAudioCue[] = []; + let activeTimeMs = 0; + const host = { + beginExportAudioCapture: vi.fn(), + setExportAudioCaptureTime: vi.fn((timeMs: number) => { + activeTimeMs = timeMs; + }), + emitEvent: vi.fn((event: { type: string }) => { + if (event.type === "cursor:click") { + cues.push({ + id: `cue-${cues.length}`, + extensionId: "com.test.clicks", + timeMs: activeTimeMs, + audioPath: `file:///sounds/click-${cues.length}.mp3`, + volume: 0.8, + }); + } + }), + finishExportAudioCapture: vi.fn(() => cues), + cancelExportAudioCapture: vi.fn(), + }; + const telemetry: CursorTelemetryPoint[] = [ + { timeMs: 100, cx: 0.1, cy: 0.2, interactionType: "move" }, + { timeMs: 250, cx: 0.3, cy: 0.4, interactionType: "click" }, + { timeMs: 400, cx: 0.5, cy: 0.6, interactionType: "mouseup" }, + { timeMs: 550, cx: 0.7, cy: 0.8 }, + ]; + + const regions = collectExtensionAudioRegionsForExport(host, telemetry, 300); + + expect(host.beginExportAudioCapture).toHaveBeenCalledTimes(1); + expect(host.setExportAudioCaptureTime).toHaveBeenNthCalledWith(1, 250); + expect(host.setExportAudioCaptureTime).toHaveBeenNthCalledWith(2, 400); + expect(host.emitEvent).toHaveBeenCalledTimes(2); + expect(host.emitEvent).toHaveBeenNthCalledWith(1, { + type: "cursor:click", + timeMs: 250, + data: { cx: 0.3, cy: 0.4, interactionType: "click" }, + }); + expect(host.cancelExportAudioCapture).not.toHaveBeenCalled(); + expect(regions).toEqual([ + { + id: "extension-audio-cue-0", + startMs: 250, + endMs: 550, + audioPath: "file:///sounds/click-0.mp3", + volume: 0.8, + normalize: false, + }, + { + id: "extension-audio-cue-1", + startMs: 400, + endMs: 700, + audioPath: "file:///sounds/click-1.mp3", + volume: 0.8, + normalize: false, + }, + ]); + }); + + it("cancels capture if event collection fails", () => { + const error = new Error("event failed"); + const host = { + beginExportAudioCapture: vi.fn(), + setExportAudioCaptureTime: vi.fn(), + emitEvent: vi.fn(() => { + throw error; + }), + finishExportAudioCapture: vi.fn(), + cancelExportAudioCapture: vi.fn(), + }; + + expect(() => + collectExtensionAudioRegionsForExport(host, [ + { timeMs: 100, cx: 0.5, cy: 0.5, interactionType: "click" }, + ]), + ).toThrow(error); + expect(host.cancelExportAudioCapture).toHaveBeenCalledTimes(1); + expect(host.finishExportAudioCapture).not.toHaveBeenCalled(); + }); + + it("guards cursor interaction and cue duration boundaries", () => { + expect(isExportableCursorInteraction({ timeMs: 0, cx: 0, cy: 0 })).toBe(false); + expect( + isExportableCursorInteraction({ + timeMs: 0, + cx: 0, + cy: 0, + interactionType: "move", + }), + ).toBe(false); + expect( + isExportableCursorInteraction({ + timeMs: 0, + cx: 0, + cy: 0, + interactionType: "right-click", + }), + ).toBe(true); + + expect( + extensionAudioCuesToRegions( + [ + { + id: "cue", + extensionId: "com.test.clicks", + timeMs: 10, + audioPath: "file:///sounds/click.mp3", + volume: 0.5, + }, + ], + 0, + )[0], + ).toMatchObject({ startMs: 10, endMs: 11 }); + }); +}); diff --git a/src/components/video-editor/extensionExportAudio.ts b/src/components/video-editor/extensionExportAudio.ts new file mode 100644 index 000000000..5c86b2ec1 --- /dev/null +++ b/src/components/video-editor/extensionExportAudio.ts @@ -0,0 +1,80 @@ +import type { ExtensionExportAudioCue, ExtensionHost } from "@/lib/extensions/extensionHost"; +import type { ExtensionEvent } from "@/lib/extensions/types"; +import type { AudioRegion, CursorTelemetryPoint } from "./types"; + +export const DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS = 500; + +type ExtensionAudioCaptureHost = Pick< + ExtensionHost, + | "beginExportAudioCapture" + | "setExportAudioCaptureTime" + | "emitEvent" + | "finishExportAudioCapture" + | "cancelExportAudioCapture" +>; + +/** + * Match preview playback by treating every non-move cursor interaction as a click event. + */ +export function isExportableCursorInteraction(point: CursorTelemetryPoint): boolean { + return Boolean(point.interactionType && point.interactionType !== "move"); +} + +/** + * Build the extension event payload emitted during export audio cue collection. + */ +function createCursorClickEvent(point: CursorTelemetryPoint): ExtensionEvent { + return { + type: "cursor:click", + timeMs: point.timeMs, + data: { + cx: point.cx, + cy: point.cy, + interactionType: point.interactionType, + }, + }; +} + +/** + * Convert captured extension sound cues into temporary audio regions for the exporter. + */ +export function extensionAudioCuesToRegions( + cues: ExtensionExportAudioCue[], + durationMs = DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS, +): AudioRegion[] { + const cueDurationMs = Number.isFinite(durationMs) && durationMs > 0 ? durationMs : 1; + + return cues.map((cue) => ({ + id: `extension-audio-${cue.id}`, + startMs: cue.timeMs, + endMs: cue.timeMs + cueDurationMs, + audioPath: cue.audioPath, + volume: cue.volume, + normalize: false, + })); +} + +/** + * Emit export-time cursor interaction events and collect resulting extension audio regions. + */ +export function collectExtensionAudioRegionsForExport( + extensionHost: ExtensionAudioCaptureHost, + cursorTelemetry: CursorTelemetryPoint[], + durationMs = DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS, +): AudioRegion[] { + extensionHost.beginExportAudioCapture(); + + try { + for (const point of cursorTelemetry) { + if (!isExportableCursorInteraction(point)) continue; + + extensionHost.setExportAudioCaptureTime(point.timeMs); + extensionHost.emitEvent(createCursorClickEvent(point)); + } + + return extensionAudioCuesToRegions(extensionHost.finishExportAudioCapture(), durationMs); + } catch (error) { + extensionHost.cancelExportAudioCapture(); + throw error; + } +} diff --git a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts index 8e1cc6bf5..ab1af945e 100644 --- a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts +++ b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts @@ -6,7 +6,7 @@ import { ZOOM_OUT_EARLY_START_MS, } from "./constants"; import { clampFocusToScale } from "./focusUtils"; -import { clamp01, cubicBezier, easeOutZoom } from "./mathUtils"; +import { clamp01, easeOutZoom } from "./mathUtils"; const CHAINED_ZOOM_PAN_GAP_MS = 1350; const CONNECTED_ZOOM_PAN_DURATION_MS = 1000; @@ -34,14 +34,6 @@ type ConnectedPanTransition = { endScale: number; }; -function lerp(start: number, end: number, amount: number) { - return start + (end - start) * amount; -} - -function easeConnectedPan(value: number) { - return cubicBezier(0.1, 0.0, 0.2, 1.0, value); -} - export function computeRegionStrength( region: ZoomRegion, timeMs: number, @@ -79,13 +71,6 @@ export function computeRegionStrength( return 1 - easeOutZoom(progress); } -function getLinearFocus(start: ZoomFocus, end: ZoomFocus, amount: number): ZoomFocus { - return { - cx: lerp(start.cx, end.cx, amount), - cy: lerp(start.cy, end.cy, amount), - }; -} - function getResolvedFocus(region: ZoomRegion, zoomScale: number): ZoomFocus { return clampFocusToScale(region.focus, zoomScale); } @@ -199,44 +184,6 @@ function getConnectedRegionHold(timeMs: number, connectedPairs: ConnectedRegionP return null; } -function getConnectedRegionTransition(connectedPairs: ConnectedRegionPair[], timeMs: number) { - for (const pair of connectedPairs) { - const { currentRegion, nextRegion, transitionStart, transitionEnd } = pair; - - if (timeMs < transitionStart || timeMs > transitionEnd) { - continue; - } - - const transitionProgress = easeConnectedPan( - clamp01((timeMs - transitionStart) / Math.max(1, transitionEnd - transitionStart)), - ); - const currentScale = ZOOM_DEPTH_SCALES[currentRegion.depth]; - const nextScale = ZOOM_DEPTH_SCALES[nextRegion.depth]; - const transitionScale = lerp(currentScale, nextScale, transitionProgress); - const currentFocus = getResolvedFocus(currentRegion, currentScale); - const nextFocus = getResolvedFocus(nextRegion, nextScale); - const transitionFocus = getLinearFocus(currentFocus, nextFocus, transitionProgress); - - return { - region: { - ...nextRegion, - focus: transitionFocus, - }, - strength: 1, - blendedScale: transitionScale, - transition: { - progress: transitionProgress, - startFocus: currentFocus, - endFocus: nextFocus, - startScale: currentScale, - endScale: nextScale, - }, - }; - } - - return null; -} - export function findDominantRegion( regions: ZoomRegion[], timeMs: number, diff --git a/src/lib/extensions/extensionHost.test.ts b/src/lib/extensions/extensionHost.test.ts new file mode 100644 index 000000000..a3ee8e3b5 --- /dev/null +++ b/src/lib/extensions/extensionHost.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "vitest"; +import { ExtensionHost } from "./extensionHost"; +import type { RecordlyExtensionAPI } from "./types"; + +interface ExtensionHostPrivateApi { + createAPI( + extensionId: string, + extensionPath: string, + permissions: string[], + disposables: (() => void)[], + ): RecordlyExtensionAPI; +} + +function createAudioApi(host: ExtensionHost): RecordlyExtensionAPI { + return (host as unknown as ExtensionHostPrivateApi).createAPI( + "com.test.click-sound", + "/tmp/recordly-extension", + ["audio"], + [], + ); +} + +describe("ExtensionHost export audio capture", () => { + it("captures playSound calls as export audio cues", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(1234.4); + const stop = api.playSound("sounds/click.mp3", { volume: 0.8 }); + const cues = host.finishExportAudioCapture(); + + expect(typeof stop).toBe("function"); + expect(cues).toEqual([ + { + id: "com.test.click-sound-sound-0", + extensionId: "com.test.click-sound", + timeMs: 1234, + audioPath: "file:///tmp/recordly-extension/sounds/click.mp3", + volume: 0.8, + }, + ]); + }); + + it("lets a captured sound be stopped before export regions are finalized", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(100); + const stop = api.playSound("sounds/click.mp3"); + stop(); + + expect(host.finishExportAudioCapture()).toEqual([]); + }); + + it("ignores export playSound calls until an export capture time is set", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + api.playSound("sounds/click.mp3"); + + expect(host.finishExportAudioCapture()).toEqual([]); + }); + + it("clamps negative export capture times to zero", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(-10); + api.playSound("sounds/click.mp3"); + const cues = host.finishExportAudioCapture(); + + expect(cues).toEqual([ + { + id: "com.test.click-sound-sound-0", + extensionId: "com.test.click-sound", + timeMs: 0, + audioPath: "file:///tmp/recordly-extension/sounds/click.mp3", + volume: 1, + }, + ]); + }); + + it("does not capture sounds for non-finite export capture times", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(Number.NaN); + api.playSound("sounds/click.mp3"); + host.setExportAudioCaptureTime(Number.POSITIVE_INFINITY); + api.playSound("sounds/click.mp3"); + + expect(host.finishExportAudioCapture()).toEqual([]); + }); + + it("defaults non-finite sound volumes before capturing export audio cues", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(100); + api.playSound("sounds/click.mp3", { volume: Number.NaN }); + host.setExportAudioCaptureTime(200); + api.playSound("sounds/click.mp3", { volume: Number.POSITIVE_INFINITY }); + const cues = host.finishExportAudioCapture(); + + expect(cues).toEqual([ + { + id: "com.test.click-sound-sound-0", + extensionId: "com.test.click-sound", + timeMs: 100, + audioPath: "file:///tmp/recordly-extension/sounds/click.mp3", + volume: 1, + }, + { + id: "com.test.click-sound-sound-1", + extensionId: "com.test.click-sound", + timeMs: 200, + audioPath: "file:///tmp/recordly-extension/sounds/click.mp3", + volume: 1, + }, + ]); + }); +}); diff --git a/src/lib/extensions/extensionHost.ts b/src/lib/extensions/extensionHost.ts index d0683e7e3..e33904419 100644 --- a/src/lib/extensions/extensionHost.ts +++ b/src/lib/extensions/extensionHost.ts @@ -122,6 +122,14 @@ interface ActiveExtension { disposables: (() => void)[]; } +export interface ExtensionExportAudioCue { + id: string; + extensionId: string; + timeMs: number; + audioPath: string; + volume: number; +} + /** * The Extension Host manages all loaded extensions and provides * access to their registered hooks, effects, and settings. @@ -147,6 +155,10 @@ export class ExtensionHost { private fullSettingsStore: Record> | null = null; private persistTimeout: ReturnType | null = null; private iconPathCache = new Map(); + private exportAudioCues: ExtensionExportAudioCue[] | null = null; + private exportAudioCanceledCueIds: Set | null = null; + private exportAudioCaptureTimeMs: number | null = null; + private exportAudioCueCounter = 0; // Shared playback/project state — set by the app, queried by extensions private _videoInfo: { width: number; height: number; durationMs: number; fps: number } | null = @@ -357,6 +369,51 @@ export class ExtensionHost { } } + /** + * Start collecting extension-triggered audio cues for an MP4 export pass. + */ + beginExportAudioCapture(): void { + this.exportAudioCues = []; + this.exportAudioCanceledCueIds = new Set(); + this.exportAudioCaptureTimeMs = null; + this.exportAudioCueCounter = 0; + } + + /** + * Set the source-timeline timestamp assigned to subsequently captured sounds. + */ + setExportAudioCaptureTime(timeMs: number | null): void { + if (!this.exportAudioCues) return; + this.exportAudioCaptureTimeMs = + typeof timeMs === "number" && Number.isFinite(timeMs) + ? Math.max(0, Math.round(timeMs)) + : null; + } + + /** + * Stop capture and return uncanceled extension audio cues. + */ + finishExportAudioCapture(): ExtensionExportAudioCue[] { + const canceledCueIds = this.exportAudioCanceledCueIds; + const cues = + canceledCueIds && canceledCueIds.size > 0 + ? (this.exportAudioCues ?? []).filter((cue) => !canceledCueIds.has(cue.id)) + : (this.exportAudioCues ?? []); + this.exportAudioCues = null; + this.exportAudioCanceledCueIds = null; + this.exportAudioCaptureTimeMs = null; + return cues; + } + + /** + * Abort export audio capture and discard all pending cues. + */ + cancelExportAudioCapture(): void { + this.exportAudioCues = null; + this.exportAudioCanceledCueIds = null; + this.exportAudioCaptureTimeMs = null; + } + // --------------------------------------------------------------------------- // Queries // --------------------------------------------------------------------------- @@ -874,10 +931,33 @@ export class ExtensionHost { playSound(relativePath: string, options?: { volume?: number }): () => void { requirePermission("audio", "playSound"); - const audio = new Audio( - resolveExtensionRelativeFileUrl(extensionPath, relativePath), - ); - audio.volume = Math.max(0, Math.min(1, options?.volume ?? 1)); + const audioPath = resolveExtensionRelativeFileUrl(extensionPath, relativePath); + const requestedVolume = options?.volume; + const volume = + typeof requestedVolume === "number" && Number.isFinite(requestedVolume) + ? Math.max(0, Math.min(1, requestedVolume)) + : 1; + + if (host.exportAudioCues) { + let cue: ExtensionExportAudioCue | null = null; + if (host.exportAudioCaptureTimeMs !== null) { + cue = { + id: `${extensionId}-sound-${host.exportAudioCueCounter++}`, + extensionId, + timeMs: host.exportAudioCaptureTimeMs, + audioPath, + volume, + }; + host.exportAudioCues.push(cue); + } + return () => { + if (!cue || !host.exportAudioCanceledCueIds) return; + host.exportAudioCanceledCueIds.add(cue.id); + }; + } + + const audio = new Audio(audioPath); + audio.volume = volume; audio.play().catch((err) => { console.warn(`[ext:${extensionId}] Failed to play sound:`, err); });