From b09478e605816ec02b6b2cf1debd6eaf1e55a59c Mon Sep 17 00:00:00 2001 From: Hasbi1605 <185914665+Hasbi1605@users.noreply.github.com> Date: Sun, 7 Jun 2026 15:54:29 +0700 Subject: [PATCH 1/4] Include extension click sounds in MP4 exports --- EXTENSIONS.md | 2 + src/components/video-editor/VideoEditor.tsx | 24 ++-- .../video-editor/extensionExportAudio.test.ts | 127 ++++++++++++++++++ .../video-editor/extensionExportAudio.ts | 68 ++++++++++ .../videoPlayback/zoomRegionUtils.ts | 55 +------- src/lib/extensions/extensionHost.test.ts | 66 +++++++++ src/lib/extensions/extensionHost.ts | 65 ++++++++- 7 files changed, 340 insertions(+), 67 deletions(-) create mode 100644 src/components/video-editor/extensionExportAudio.test.ts create mode 100644 src/components/video-editor/extensionExportAudio.ts create mode 100644 src/lib/extensions/extensionHost.test.ts diff --git a/EXTENSIONS.md b/EXTENSIONS.md index 1104e9526..67c57ca3a 100644 --- a/EXTENSIONS.md +++ b/EXTENSIONS.md @@ -215,6 +215,8 @@ api.playSound("sounds/click.mp3", { volume: 0.8 }); api.log("hello", payload); ``` +Sounds played from cursor interaction handlers are also included in MP4 exports. + ### Read-only Queries ```js diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 9ed087523..ea27ed1fc 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -138,6 +138,7 @@ import { saveEditorPresets, serializeEditorPresetSnapshot, } from "./editorPreferences"; +import { collectExtensionAudioRegionsForExport } from "./extensionExportAudio"; import ProjectBrowserDialog, { type ProjectLibraryEntry } from "./ProjectBrowserDialog"; import { hasUnsavedProjectChanges } from "./projectDirtyState"; import { @@ -3347,7 +3348,9 @@ export default function VideoEditor() { const handlePreviewSkipBack = useCallback(() => { const currentMs = timelinePlayheadTime * 1000; const keyframes = timelineRef.current?.keyframes ?? []; - const previous = [...keyframes].reverse().find((keyframe) => keyframe.time < currentMs - 50); + const previous = [...keyframes] + .reverse() + .find((keyframe) => keyframe.time < currentMs - 50); handleSeek(previous ? previous.time / 1000 : Math.max(0, timelinePlayheadTime - 5)); }, [handleSeek, timelinePlayheadTime]); @@ -3355,9 +3358,7 @@ export default function VideoEditor() { const currentMs = timelinePlayheadTime * 1000; const keyframes = timelineRef.current?.keyframes ?? []; const next = keyframes.find((keyframe) => keyframe.time > currentMs + 50); - handleSeek( - next ? next.time / 1000 : Math.min(timelineDuration, timelinePlayheadTime + 5), - ); + handleSeek(next ? next.time / 1000 : Math.min(timelineDuration, timelinePlayheadTime + 5)); }, [handleSeek, timelineDuration, timelinePlayheadTime]); const handleSelectZoom = useCallback((id: string | null) => { @@ -4356,6 +4357,14 @@ export default function VideoEditor() { selectedClipId !== null ? audio.selectedClipSourceAudioTrackSettings : audio.activeSourceAudioTrackSettings; + const extensionAudioRegions = collectExtensionAudioRegionsForExport( + extensionHost, + effectiveCursorTelemetry, + ); + const audioRegionsForExport = + extensionAudioRegions.length > 0 + ? [...audioRegions, ...extensionAudioRegions] + : audioRegions; const exporterConfig = { videoUrl: videoPath, @@ -4426,7 +4435,7 @@ export default function VideoEditor() { cursorClickBounceDuration, cursorSway, frame, - audioRegions, + audioRegions: audioRegionsForExport, clipRegions, sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath: @@ -5148,10 +5157,7 @@ export default function VideoEditor() { volume={ audio.shouldMutePreviewVideo || audio.isCurrentClipMuted ? 0 - : Math.max( - 0, - Math.min(1, previewVolume * audio.embeddedSourcePreviewGain), - ) + : Math.max(0, Math.min(1, previewVolume * audio.embeddedSourcePreviewGain)) } suspendRendering={suspendRendering} /> diff --git a/src/components/video-editor/extensionExportAudio.test.ts b/src/components/video-editor/extensionExportAudio.test.ts new file mode 100644 index 000000000..7aec93229 --- /dev/null +++ b/src/components/video-editor/extensionExportAudio.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it, vi } from "vitest"; +import type { ExtensionExportAudioCue } from "@/lib/extensions/extensionHost"; +import { + collectExtensionAudioRegionsForExport, + extensionAudioCuesToRegions, + isExportableCursorInteraction, +} from "./extensionExportAudio"; +import type { CursorTelemetryPoint } from "./types"; + +describe("extension export audio", () => { + it("captures extension sounds for cursor interactions that preview treats as clicks", () => { + const cues: ExtensionExportAudioCue[] = []; + let activeTimeMs = 0; + const host = { + beginExportAudioCapture: vi.fn(), + setExportAudioCaptureTime: vi.fn((timeMs: number) => { + activeTimeMs = timeMs; + }), + emitEvent: vi.fn((event: { type: string }) => { + if (event.type === "cursor:click") { + cues.push({ + id: `cue-${cues.length}`, + extensionId: "com.test.clicks", + timeMs: activeTimeMs, + audioPath: `file:///sounds/click-${cues.length}.mp3`, + volume: 0.8, + }); + } + }), + finishExportAudioCapture: vi.fn(() => cues), + cancelExportAudioCapture: vi.fn(), + }; + const telemetry: CursorTelemetryPoint[] = [ + { timeMs: 100, cx: 0.1, cy: 0.2, interactionType: "move" }, + { timeMs: 250, cx: 0.3, cy: 0.4, interactionType: "click" }, + { timeMs: 400, cx: 0.5, cy: 0.6, interactionType: "mouseup" }, + { timeMs: 550, cx: 0.7, cy: 0.8 }, + ]; + + const regions = collectExtensionAudioRegionsForExport(host, telemetry, 300); + + expect(host.beginExportAudioCapture).toHaveBeenCalledTimes(1); + expect(host.setExportAudioCaptureTime).toHaveBeenNthCalledWith(1, 250); + expect(host.setExportAudioCaptureTime).toHaveBeenNthCalledWith(2, 400); + expect(host.emitEvent).toHaveBeenCalledTimes(2); + expect(host.emitEvent).toHaveBeenNthCalledWith(1, { + type: "cursor:click", + timeMs: 250, + data: { cx: 0.3, cy: 0.4, interactionType: "click" }, + }); + expect(host.cancelExportAudioCapture).not.toHaveBeenCalled(); + expect(regions).toEqual([ + { + id: "extension-audio-cue-0", + startMs: 250, + endMs: 550, + audioPath: "file:///sounds/click-0.mp3", + volume: 0.8, + normalize: false, + }, + { + id: "extension-audio-cue-1", + startMs: 400, + endMs: 700, + audioPath: "file:///sounds/click-1.mp3", + volume: 0.8, + normalize: false, + }, + ]); + }); + + it("cancels capture if event collection fails", () => { + const error = new Error("event failed"); + const host = { + beginExportAudioCapture: vi.fn(), + setExportAudioCaptureTime: vi.fn(), + emitEvent: vi.fn(() => { + throw error; + }), + finishExportAudioCapture: vi.fn(), + cancelExportAudioCapture: vi.fn(), + }; + + expect(() => + collectExtensionAudioRegionsForExport(host, [ + { timeMs: 100, cx: 0.5, cy: 0.5, interactionType: "click" }, + ]), + ).toThrow(error); + expect(host.cancelExportAudioCapture).toHaveBeenCalledTimes(1); + expect(host.finishExportAudioCapture).not.toHaveBeenCalled(); + }); + + it("guards cursor interaction and cue duration boundaries", () => { + expect(isExportableCursorInteraction({ timeMs: 0, cx: 0, cy: 0 })).toBe(false); + expect( + isExportableCursorInteraction({ + timeMs: 0, + cx: 0, + cy: 0, + interactionType: "move", + }), + ).toBe(false); + expect( + isExportableCursorInteraction({ + timeMs: 0, + cx: 0, + cy: 0, + interactionType: "right-click", + }), + ).toBe(true); + + expect( + extensionAudioCuesToRegions( + [ + { + id: "cue", + extensionId: "com.test.clicks", + timeMs: 10, + audioPath: "file:///sounds/click.mp3", + volume: 0.5, + }, + ], + 0, + )[0], + ).toMatchObject({ startMs: 10, endMs: 11 }); + }); +}); diff --git a/src/components/video-editor/extensionExportAudio.ts b/src/components/video-editor/extensionExportAudio.ts new file mode 100644 index 000000000..eda6cffd9 --- /dev/null +++ b/src/components/video-editor/extensionExportAudio.ts @@ -0,0 +1,68 @@ +import type { ExtensionExportAudioCue, ExtensionHost } from "@/lib/extensions/extensionHost"; +import type { ExtensionEvent } from "@/lib/extensions/types"; +import type { AudioRegion, CursorTelemetryPoint } from "./types"; + +export const DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS = 500; + +type ExtensionAudioCaptureHost = Pick< + ExtensionHost, + | "beginExportAudioCapture" + | "setExportAudioCaptureTime" + | "emitEvent" + | "finishExportAudioCapture" + | "cancelExportAudioCapture" +>; + +export function isExportableCursorInteraction(point: CursorTelemetryPoint): boolean { + return Boolean(point.interactionType && point.interactionType !== "move"); +} + +function createCursorClickEvent(point: CursorTelemetryPoint): ExtensionEvent { + return { + type: "cursor:click", + timeMs: point.timeMs, + data: { + cx: point.cx, + cy: point.cy, + interactionType: point.interactionType, + }, + }; +} + +export function extensionAudioCuesToRegions( + cues: ExtensionExportAudioCue[], + durationMs = DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS, +): AudioRegion[] { + const cueDurationMs = Number.isFinite(durationMs) && durationMs > 0 ? durationMs : 1; + + return cues.map((cue) => ({ + id: `extension-audio-${cue.id}`, + startMs: cue.timeMs, + endMs: cue.timeMs + cueDurationMs, + audioPath: cue.audioPath, + volume: cue.volume, + normalize: false, + })); +} + +export function collectExtensionAudioRegionsForExport( + extensionHost: ExtensionAudioCaptureHost, + cursorTelemetry: CursorTelemetryPoint[], + durationMs = DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS, +): AudioRegion[] { + extensionHost.beginExportAudioCapture(); + + try { + for (const point of cursorTelemetry) { + if (!isExportableCursorInteraction(point)) continue; + + extensionHost.setExportAudioCaptureTime(point.timeMs); + extensionHost.emitEvent(createCursorClickEvent(point)); + } + + return extensionAudioCuesToRegions(extensionHost.finishExportAudioCapture(), durationMs); + } catch (error) { + extensionHost.cancelExportAudioCapture(); + throw error; + } +} diff --git a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts index 8e1cc6bf5..ab1af945e 100644 --- a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts +++ b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts @@ -6,7 +6,7 @@ import { ZOOM_OUT_EARLY_START_MS, } from "./constants"; import { clampFocusToScale } from "./focusUtils"; -import { clamp01, cubicBezier, easeOutZoom } from "./mathUtils"; +import { clamp01, easeOutZoom } from "./mathUtils"; const CHAINED_ZOOM_PAN_GAP_MS = 1350; const CONNECTED_ZOOM_PAN_DURATION_MS = 1000; @@ -34,14 +34,6 @@ type ConnectedPanTransition = { endScale: number; }; -function lerp(start: number, end: number, amount: number) { - return start + (end - start) * amount; -} - -function easeConnectedPan(value: number) { - return cubicBezier(0.1, 0.0, 0.2, 1.0, value); -} - export function computeRegionStrength( region: ZoomRegion, timeMs: number, @@ -79,13 +71,6 @@ export function computeRegionStrength( return 1 - easeOutZoom(progress); } -function getLinearFocus(start: ZoomFocus, end: ZoomFocus, amount: number): ZoomFocus { - return { - cx: lerp(start.cx, end.cx, amount), - cy: lerp(start.cy, end.cy, amount), - }; -} - function getResolvedFocus(region: ZoomRegion, zoomScale: number): ZoomFocus { return clampFocusToScale(region.focus, zoomScale); } @@ -199,44 +184,6 @@ function getConnectedRegionHold(timeMs: number, connectedPairs: ConnectedRegionP return null; } -function getConnectedRegionTransition(connectedPairs: ConnectedRegionPair[], timeMs: number) { - for (const pair of connectedPairs) { - const { currentRegion, nextRegion, transitionStart, transitionEnd } = pair; - - if (timeMs < transitionStart || timeMs > transitionEnd) { - continue; - } - - const transitionProgress = easeConnectedPan( - clamp01((timeMs - transitionStart) / Math.max(1, transitionEnd - transitionStart)), - ); - const currentScale = ZOOM_DEPTH_SCALES[currentRegion.depth]; - const nextScale = ZOOM_DEPTH_SCALES[nextRegion.depth]; - const transitionScale = lerp(currentScale, nextScale, transitionProgress); - const currentFocus = getResolvedFocus(currentRegion, currentScale); - const nextFocus = getResolvedFocus(nextRegion, nextScale); - const transitionFocus = getLinearFocus(currentFocus, nextFocus, transitionProgress); - - return { - region: { - ...nextRegion, - focus: transitionFocus, - }, - strength: 1, - blendedScale: transitionScale, - transition: { - progress: transitionProgress, - startFocus: currentFocus, - endFocus: nextFocus, - startScale: currentScale, - endScale: nextScale, - }, - }; - } - - return null; -} - export function findDominantRegion( regions: ZoomRegion[], timeMs: number, diff --git a/src/lib/extensions/extensionHost.test.ts b/src/lib/extensions/extensionHost.test.ts new file mode 100644 index 000000000..bc3d9e62e --- /dev/null +++ b/src/lib/extensions/extensionHost.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { ExtensionHost } from "./extensionHost"; +import type { RecordlyExtensionAPI } from "./types"; + +interface ExtensionHostPrivateApi { + createAPI( + extensionId: string, + extensionPath: string, + permissions: string[], + disposables: (() => void)[], + ): RecordlyExtensionAPI; +} + +function createAudioApi(host: ExtensionHost): RecordlyExtensionAPI { + return (host as unknown as ExtensionHostPrivateApi).createAPI( + "com.test.click-sound", + "/tmp/recordly-extension", + ["audio"], + [], + ); +} + +describe("ExtensionHost export audio capture", () => { + it("captures playSound calls as export audio cues", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(1234.4); + const stop = api.playSound("sounds/click.mp3", { volume: 0.8 }); + const cues = host.finishExportAudioCapture(); + + expect(typeof stop).toBe("function"); + expect(cues).toEqual([ + { + id: "com.test.click-sound-sound-0", + extensionId: "com.test.click-sound", + timeMs: 1234, + audioPath: "file:///tmp/recordly-extension/sounds/click.mp3", + volume: 0.8, + }, + ]); + }); + + it("lets a captured sound be stopped before export regions are finalized", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(100); + const stop = api.playSound("sounds/click.mp3"); + stop(); + + expect(host.finishExportAudioCapture()).toEqual([]); + }); + + it("ignores export playSound calls until an export capture time is set", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + api.playSound("sounds/click.mp3"); + + expect(host.finishExportAudioCapture()).toEqual([]); + }); +}); diff --git a/src/lib/extensions/extensionHost.ts b/src/lib/extensions/extensionHost.ts index d0683e7e3..774f08a6a 100644 --- a/src/lib/extensions/extensionHost.ts +++ b/src/lib/extensions/extensionHost.ts @@ -122,6 +122,14 @@ interface ActiveExtension { disposables: (() => void)[]; } +export interface ExtensionExportAudioCue { + id: string; + extensionId: string; + timeMs: number; + audioPath: string; + volume: number; +} + /** * The Extension Host manages all loaded extensions and provides * access to their registered hooks, effects, and settings. @@ -147,6 +155,9 @@ export class ExtensionHost { private fullSettingsStore: Record> | null = null; private persistTimeout: ReturnType | null = null; private iconPathCache = new Map(); + private exportAudioCues: ExtensionExportAudioCue[] | null = null; + private exportAudioCaptureTimeMs: number | null = null; + private exportAudioCueCounter = 0; // Shared playback/project state — set by the app, queried by extensions private _videoInfo: { width: number; height: number; durationMs: number; fps: number } | null = @@ -357,6 +368,32 @@ export class ExtensionHost { } } + beginExportAudioCapture(): void { + this.exportAudioCues = []; + this.exportAudioCaptureTimeMs = null; + this.exportAudioCueCounter = 0; + } + + setExportAudioCaptureTime(timeMs: number | null): void { + if (!this.exportAudioCues) return; + this.exportAudioCaptureTimeMs = + typeof timeMs === "number" && Number.isFinite(timeMs) + ? Math.max(0, Math.round(timeMs)) + : null; + } + + finishExportAudioCapture(): ExtensionExportAudioCue[] { + const cues = this.exportAudioCues ?? []; + this.exportAudioCues = null; + this.exportAudioCaptureTimeMs = null; + return cues; + } + + cancelExportAudioCapture(): void { + this.exportAudioCues = null; + this.exportAudioCaptureTimeMs = null; + } + // --------------------------------------------------------------------------- // Queries // --------------------------------------------------------------------------- @@ -874,10 +911,30 @@ export class ExtensionHost { playSound(relativePath: string, options?: { volume?: number }): () => void { requirePermission("audio", "playSound"); - const audio = new Audio( - resolveExtensionRelativeFileUrl(extensionPath, relativePath), - ); - audio.volume = Math.max(0, Math.min(1, options?.volume ?? 1)); + const audioPath = resolveExtensionRelativeFileUrl(extensionPath, relativePath); + const volume = Math.max(0, Math.min(1, options?.volume ?? 1)); + + if (host.exportAudioCues) { + let cue: ExtensionExportAudioCue | null = null; + if (host.exportAudioCaptureTimeMs !== null) { + cue = { + id: `${extensionId}-sound-${host.exportAudioCueCounter++}`, + extensionId, + timeMs: host.exportAudioCaptureTimeMs, + audioPath, + volume, + }; + host.exportAudioCues.push(cue); + } + return () => { + if (!cue || !host.exportAudioCues) return; + const index = host.exportAudioCues.indexOf(cue); + if (index >= 0) host.exportAudioCues.splice(index, 1); + }; + } + + const audio = new Audio(audioPath); + audio.volume = volume; audio.play().catch((err) => { console.warn(`[ext:${extensionId}] Failed to play sound:`, err); }); From d2b41b507562114b1119f7f77ff4957981f4e6e1 Mon Sep 17 00:00:00 2001 From: Hasbi1605 <185914665+Hasbi1605@users.noreply.github.com> Date: Sun, 7 Jun 2026 16:06:45 +0700 Subject: [PATCH 2/4] Address Copilot export audio review --- EXTENSIONS.md | 4 +++- src/components/video-editor/VideoEditor.tsx | 4 +++- src/lib/extensions/extensionHost.test.ts | 24 +++++++++++++++++++++ src/lib/extensions/extensionHost.ts | 15 +++++++++---- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/EXTENSIONS.md b/EXTENSIONS.md index 67c57ca3a..10e96eae3 100644 --- a/EXTENSIONS.md +++ b/EXTENSIONS.md @@ -215,7 +215,9 @@ api.playSound("sounds/click.mp3", { volume: 0.8 }); api.log("hello", payload); ``` -Sounds played from cursor interaction handlers are also included in MP4 exports. +Sounds played by export-time cursor interaction handlers, such as `cursor:click`, are +included in MP4 exports. Sounds played outside that capture window are not added +automatically. ### Read-only Queries diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index ea27ed1fc..65d8ceec3 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -4363,7 +4363,9 @@ export default function VideoEditor() { ); const audioRegionsForExport = extensionAudioRegions.length > 0 - ? [...audioRegions, ...extensionAudioRegions] + ? [...audioRegions, ...extensionAudioRegions].sort( + (a, b) => a.startMs - b.startMs || a.id.localeCompare(b.id), + ) : audioRegions; const exporterConfig = { diff --git a/src/lib/extensions/extensionHost.test.ts b/src/lib/extensions/extensionHost.test.ts index bc3d9e62e..6b994e888 100644 --- a/src/lib/extensions/extensionHost.test.ts +++ b/src/lib/extensions/extensionHost.test.ts @@ -63,4 +63,28 @@ describe("ExtensionHost export audio capture", () => { expect(host.finishExportAudioCapture()).toEqual([]); }); + + it("clamps negative export capture times to zero", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(-10); + api.playSound("sounds/click.mp3"); + + expect(host.finishExportAudioCapture()[0]).toMatchObject({ timeMs: 0 }); + }); + + it("does not capture sounds for non-finite export capture times", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(Number.NaN); + api.playSound("sounds/click.mp3"); + host.setExportAudioCaptureTime(Number.POSITIVE_INFINITY); + api.playSound("sounds/click.mp3"); + + expect(host.finishExportAudioCapture()).toEqual([]); + }); }); diff --git a/src/lib/extensions/extensionHost.ts b/src/lib/extensions/extensionHost.ts index 774f08a6a..64a319b47 100644 --- a/src/lib/extensions/extensionHost.ts +++ b/src/lib/extensions/extensionHost.ts @@ -156,6 +156,7 @@ export class ExtensionHost { private persistTimeout: ReturnType | null = null; private iconPathCache = new Map(); private exportAudioCues: ExtensionExportAudioCue[] | null = null; + private exportAudioCanceledCueIds: Set | null = null; private exportAudioCaptureTimeMs: number | null = null; private exportAudioCueCounter = 0; @@ -370,6 +371,7 @@ export class ExtensionHost { beginExportAudioCapture(): void { this.exportAudioCues = []; + this.exportAudioCanceledCueIds = new Set(); this.exportAudioCaptureTimeMs = null; this.exportAudioCueCounter = 0; } @@ -383,14 +385,20 @@ export class ExtensionHost { } finishExportAudioCapture(): ExtensionExportAudioCue[] { - const cues = this.exportAudioCues ?? []; + const canceledCueIds = this.exportAudioCanceledCueIds; + const cues = + canceledCueIds && canceledCueIds.size > 0 + ? (this.exportAudioCues ?? []).filter((cue) => !canceledCueIds.has(cue.id)) + : (this.exportAudioCues ?? []); this.exportAudioCues = null; + this.exportAudioCanceledCueIds = null; this.exportAudioCaptureTimeMs = null; return cues; } cancelExportAudioCapture(): void { this.exportAudioCues = null; + this.exportAudioCanceledCueIds = null; this.exportAudioCaptureTimeMs = null; } @@ -927,9 +935,8 @@ export class ExtensionHost { host.exportAudioCues.push(cue); } return () => { - if (!cue || !host.exportAudioCues) return; - const index = host.exportAudioCues.indexOf(cue); - if (index >= 0) host.exportAudioCues.splice(index, 1); + if (!cue || !host.exportAudioCanceledCueIds) return; + host.exportAudioCanceledCueIds.add(cue.id); }; } From e5809b7f49874b5d8e1d8b63433b3c0e45065cdf Mon Sep 17 00:00:00 2001 From: Hasbi1605 <185914665+Hasbi1605@users.noreply.github.com> Date: Sun, 7 Jun 2026 16:24:49 +0700 Subject: [PATCH 3/4] Harden extension export audio volume handling --- src/lib/extensions/extensionHost.test.ts | 40 +++++++++++++++++++++++- src/lib/extensions/extensionHost.ts | 6 +++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/lib/extensions/extensionHost.test.ts b/src/lib/extensions/extensionHost.test.ts index 6b994e888..a3ee8e3b5 100644 --- a/src/lib/extensions/extensionHost.test.ts +++ b/src/lib/extensions/extensionHost.test.ts @@ -71,8 +71,17 @@ describe("ExtensionHost export audio capture", () => { host.beginExportAudioCapture(); host.setExportAudioCaptureTime(-10); api.playSound("sounds/click.mp3"); + const cues = host.finishExportAudioCapture(); - expect(host.finishExportAudioCapture()[0]).toMatchObject({ timeMs: 0 }); + expect(cues).toEqual([ + { + id: "com.test.click-sound-sound-0", + extensionId: "com.test.click-sound", + timeMs: 0, + audioPath: "file:///tmp/recordly-extension/sounds/click.mp3", + volume: 1, + }, + ]); }); it("does not capture sounds for non-finite export capture times", () => { @@ -87,4 +96,33 @@ describe("ExtensionHost export audio capture", () => { expect(host.finishExportAudioCapture()).toEqual([]); }); + + it("defaults non-finite sound volumes before capturing export audio cues", () => { + const host = new ExtensionHost(); + const api = createAudioApi(host); + + host.beginExportAudioCapture(); + host.setExportAudioCaptureTime(100); + api.playSound("sounds/click.mp3", { volume: Number.NaN }); + host.setExportAudioCaptureTime(200); + api.playSound("sounds/click.mp3", { volume: Number.POSITIVE_INFINITY }); + const cues = host.finishExportAudioCapture(); + + expect(cues).toEqual([ + { + id: "com.test.click-sound-sound-0", + extensionId: "com.test.click-sound", + timeMs: 100, + audioPath: "file:///tmp/recordly-extension/sounds/click.mp3", + volume: 1, + }, + { + id: "com.test.click-sound-sound-1", + extensionId: "com.test.click-sound", + timeMs: 200, + audioPath: "file:///tmp/recordly-extension/sounds/click.mp3", + volume: 1, + }, + ]); + }); }); diff --git a/src/lib/extensions/extensionHost.ts b/src/lib/extensions/extensionHost.ts index 64a319b47..1763cdd1b 100644 --- a/src/lib/extensions/extensionHost.ts +++ b/src/lib/extensions/extensionHost.ts @@ -920,7 +920,11 @@ export class ExtensionHost { playSound(relativePath: string, options?: { volume?: number }): () => void { requirePermission("audio", "playSound"); const audioPath = resolveExtensionRelativeFileUrl(extensionPath, relativePath); - const volume = Math.max(0, Math.min(1, options?.volume ?? 1)); + const requestedVolume = options?.volume; + const volume = + typeof requestedVolume === "number" && Number.isFinite(requestedVolume) + ? Math.max(0, Math.min(1, requestedVolume)) + : 1; if (host.exportAudioCues) { let cue: ExtensionExportAudioCue | null = null; From 12a2e4c85850e63ae7a8957381fcaef105866fac Mon Sep 17 00:00:00 2001 From: Hasbi1605 <185914665+Hasbi1605@users.noreply.github.com> Date: Sun, 7 Jun 2026 16:34:00 +0700 Subject: [PATCH 4/4] Document extension export audio helpers --- src/components/video-editor/extensionExportAudio.ts | 12 ++++++++++++ src/lib/extensions/extensionHost.ts | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/components/video-editor/extensionExportAudio.ts b/src/components/video-editor/extensionExportAudio.ts index eda6cffd9..5c86b2ec1 100644 --- a/src/components/video-editor/extensionExportAudio.ts +++ b/src/components/video-editor/extensionExportAudio.ts @@ -13,10 +13,16 @@ type ExtensionAudioCaptureHost = Pick< | "cancelExportAudioCapture" >; +/** + * Match preview playback by treating every non-move cursor interaction as a click event. + */ export function isExportableCursorInteraction(point: CursorTelemetryPoint): boolean { return Boolean(point.interactionType && point.interactionType !== "move"); } +/** + * Build the extension event payload emitted during export audio cue collection. + */ function createCursorClickEvent(point: CursorTelemetryPoint): ExtensionEvent { return { type: "cursor:click", @@ -29,6 +35,9 @@ function createCursorClickEvent(point: CursorTelemetryPoint): ExtensionEvent { }; } +/** + * Convert captured extension sound cues into temporary audio regions for the exporter. + */ export function extensionAudioCuesToRegions( cues: ExtensionExportAudioCue[], durationMs = DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS, @@ -45,6 +54,9 @@ export function extensionAudioCuesToRegions( })); } +/** + * Emit export-time cursor interaction events and collect resulting extension audio regions. + */ export function collectExtensionAudioRegionsForExport( extensionHost: ExtensionAudioCaptureHost, cursorTelemetry: CursorTelemetryPoint[], diff --git a/src/lib/extensions/extensionHost.ts b/src/lib/extensions/extensionHost.ts index 1763cdd1b..e33904419 100644 --- a/src/lib/extensions/extensionHost.ts +++ b/src/lib/extensions/extensionHost.ts @@ -369,6 +369,9 @@ export class ExtensionHost { } } + /** + * Start collecting extension-triggered audio cues for an MP4 export pass. + */ beginExportAudioCapture(): void { this.exportAudioCues = []; this.exportAudioCanceledCueIds = new Set(); @@ -376,6 +379,9 @@ export class ExtensionHost { this.exportAudioCueCounter = 0; } + /** + * Set the source-timeline timestamp assigned to subsequently captured sounds. + */ setExportAudioCaptureTime(timeMs: number | null): void { if (!this.exportAudioCues) return; this.exportAudioCaptureTimeMs = @@ -384,6 +390,9 @@ export class ExtensionHost { : null; } + /** + * Stop capture and return uncanceled extension audio cues. + */ finishExportAudioCapture(): ExtensionExportAudioCue[] { const canceledCueIds = this.exportAudioCanceledCueIds; const cues = @@ -396,6 +405,9 @@ export class ExtensionHost { return cues; } + /** + * Abort export audio capture and discard all pending cues. + */ cancelExportAudioCapture(): void { this.exportAudioCues = null; this.exportAudioCanceledCueIds = null;