From 259189e671f467259c56b90f04a08822ad45b866 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 30 Jun 2026 15:09:53 +0200 Subject: [PATCH 1/2] fix(recording): parallelize screen/mic capture and fade in mic gain (#57) The mic track was lagging and clicking at the start of recordings, then running visibly behind the video track throughout. Three things combined to produce the symptom on the browser capture path: - Screen and mic were acquired sequentially in startRecording, so the mic track started capturing only after the screen track had been running for the full IPC + permission roundtrip. - The AudioContext (which routes the mic to the recorder's destination stream) was constructed only after both streams were in hand, so any audio the mic captured before then was dropped on the floor. - The mic gain was a constant 1.4 with no fade-in, so the very first packet produced a click/pop from the echo-canceller warm-up. Two changes: - Capture screen and microphone with Promise.all so they both start at the same wall-clock instant. - Extract the AudioContext + GainNode + MediaStreamDestination wiring into src/lib/audioMix.ts and add a 20 ms linearRampToValueAtTime on the mic gain from 0 -> MIC_GAIN_BOOST, so the destination stream's first audio packet doesn't click. The native Windows and macOS paths are unaffected (they already anchor audio to the first video frame). --- src/hooks/useScreenRecorder.ts | 133 ++++++++++++++++---------------- src/lib/audioMix.test.ts | 136 +++++++++++++++++++++++++++++++++ src/lib/audioMix.ts | 56 ++++++++++++++ 3 files changed, 261 insertions(+), 64 deletions(-) create mode 100644 src/lib/audioMix.test.ts create mode 100644 src/lib/audioMix.ts diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index ce0e0b77f..02ddad066 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -2,6 +2,7 @@ import { fixWebmDuration } from "@fix-webm-duration/fix"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { useScopedT } from "@/contexts/I18nContext"; +import { MIC_GAIN_BOOST, mixAudioTracks } from "@/lib/audioMix"; import { type NativeMacRecordingRequest, parseMacDisplayIdFromSourceId, @@ -44,7 +45,6 @@ const WEBCAM_FILE_SUFFIX = "-webcam"; const AUDIO_BITRATE_VOICE = 128_000; const AUDIO_BITRATE_SYSTEM = 192_000; -const MIC_GAIN_BOOST = 1.4; const WEBCAM_TARGET_FRAME_RATE = 30; type UseScreenRecorderReturn = { @@ -1151,23 +1151,27 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } - let screenMediaStream: MediaStream; - const platform = await window.electronAPI.getPlatform(); + // Capture screen + microphone in parallel: the gap between the two + // `getUserMedia` calls is the dominant source of the mic-vs-video lag at the + // start of the recording (issue #57). + const screenCapture = (async (): Promise => { + const platform = await window.electronAPI.getPlatform(); + + if (platform === "win32") { + // getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the + // pre-selected source. Editable cursor mode excludes the system cursor so + // the editor can render a replacement; system mode bakes it into the video. + return navigator.mediaDevices.getDisplayMedia({ + video: { + cursor: cursorCaptureMode === "editable-overlay" ? "never" : "always", + width: { max: TARGET_WIDTH }, + height: { max: TARGET_HEIGHT }, + frameRate: { ideal: TARGET_FRAME_RATE }, + } as MediaTrackConstraints, + audio: systemAudioEnabled, + } as DisplayMediaStreamOptions); + } - if (platform === "win32") { - // getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the - // pre-selected source. Editable cursor mode excludes the system cursor so - // the editor can render a replacement; system mode bakes it into the video. - screenMediaStream = await navigator.mediaDevices.getDisplayMedia({ - video: { - cursor: cursorCaptureMode === "editable-overlay" ? "never" : "always", - width: { max: TARGET_WIDTH }, - height: { max: TARGET_HEIGHT }, - frameRate: { ideal: TARGET_FRAME_RATE }, - } as MediaTrackConstraints, - audio: systemAudioEnabled, - } as DisplayMediaStreamOptions); - } else { const videoConstraints = { mandatory: { chromeMediaSource: CHROME_MEDIA_SOURCE, @@ -1181,7 +1185,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (systemAudioEnabled) { try { - screenMediaStream = await navigator.mediaDevices.getUserMedia({ + return navigator.mediaDevices.getUserMedia({ audio: { mandatory: { chromeMediaSource: CHROME_MEDIA_SOURCE, @@ -1193,48 +1197,55 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } catch (audioErr) { console.warn("System audio capture failed, falling back to video-only:", audioErr); toast.error(t("recording.systemAudioUnavailable")); - screenMediaStream = await navigator.mediaDevices.getUserMedia({ + return navigator.mediaDevices.getUserMedia({ audio: false, video: videoConstraints, } as unknown as MediaStreamConstraints); } - } else { - screenMediaStream = await navigator.mediaDevices.getUserMedia({ - audio: false, - video: videoConstraints, - } as unknown as MediaStreamConstraints); } - } - screenStream.current = screenMediaStream; + + return navigator.mediaDevices.getUserMedia({ + audio: false, + video: videoConstraints, + } as unknown as MediaStreamConstraints); + })(); + + const micCapture: Promise = microphoneEnabled + ? (async () => { + try { + return await navigator.mediaDevices.getUserMedia({ + audio: microphoneDeviceId + ? { + deviceId: { exact: microphoneDeviceId }, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + } + : { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + video: false, + }); + } catch (audioError) { + console.warn("Failed to get microphone access:", audioError); + toast.error(t("recording.microphoneDenied")); + setMicrophoneEnabled(false); + return null; + } + })() + : Promise.resolve(null); + + const [screenMediaStream, micMediaStream] = await Promise.all([screenCapture, micCapture]); if (!isCountdownRunActive(countdownRunToken)) { teardownMedia(); return; } - if (microphoneEnabled) { - try { - microphoneStream.current = await navigator.mediaDevices.getUserMedia({ - audio: microphoneDeviceId - ? { - deviceId: { exact: microphoneDeviceId }, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - } - : { - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - }, - video: false, - }); - } catch (audioError) { - console.warn("Failed to get microphone access:", audioError); - toast.error(t("recording.microphoneDenied")); - setMicrophoneEnabled(false); - } - } + screenStream.current = screenMediaStream; + microphoneStream.current = micMediaStream; if (!isCountdownRunActive(countdownRunToken)) { teardownMedia(); @@ -1277,21 +1288,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const systemAudioTrack = screenMediaStream.getAudioTracks()[0]; const micAudioTrack = microphoneStream.current?.getAudioTracks()[0]; - if (systemAudioTrack && micAudioTrack) { - const ctx = new AudioContext(); - mixingContext.current = ctx; - const systemSource = ctx.createMediaStreamSource(new MediaStream([systemAudioTrack])); - const micSource = ctx.createMediaStreamSource(new MediaStream([micAudioTrack])); - const micGain = ctx.createGain(); - micGain.gain.value = MIC_GAIN_BOOST; - const destination = ctx.createMediaStreamDestination(); - systemSource.connect(destination); - micSource.connect(micGain).connect(destination); - stream.current.addTrack(destination.stream.getAudioTracks()[0]); - } else if (systemAudioTrack) { - stream.current.addTrack(systemAudioTrack); - } else if (micAudioTrack) { - stream.current.addTrack(micAudioTrack); + const { context: mixingCtx, track: mixedTrack } = mixAudioTracks({ + systemAudioTrack, + micAudioTrack, + }); + if (mixingCtx) { + mixingContext.current = mixingCtx; + } + if (mixedTrack) { + stream.current.addTrack(mixedTrack); } try { diff --git a/src/lib/audioMix.test.ts b/src/lib/audioMix.test.ts new file mode 100644 index 000000000..d997fbe24 --- /dev/null +++ b/src/lib/audioMix.test.ts @@ -0,0 +1,136 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { MIC_FADE_IN_S, MIC_GAIN_BOOST, mixAudioTracks } from "./audioMix"; + +class FakeAudioParam { + value = 1; + setValueAtTime = vi.fn((v: number) => { + this.value = v; + }); + linearRampToValueAtTime = vi.fn((v: number) => { + this.value = v; + }); +} + +class FakeGainNode { + gain = new FakeAudioParam(); + // Real AudioNode.connect returns the destination so it can be chained. + connect = vi.fn((destination: T) => destination); +} + +class FakeMediaStreamAudioSourceNode { + connect = vi.fn((destination: T) => destination); +} + +class FakeAudioContext { + currentTime = 0.5; + destination: FakeMediaStreamDestination; + createGain = vi.fn(() => new FakeGainNode()); + createMediaStreamSource = vi.fn(() => new FakeMediaStreamAudioSourceNode()); + createMediaStreamDestination = vi.fn(() => new FakeMediaStreamDestination()); + close = vi.fn(); + constructor() { + this.destination = new FakeMediaStreamDestination(); + } +} + +class FakeMediaStreamDestination { + stream = { getAudioTracks: () => [{ kind: "audio", id: "dest-track" }] }; + connect = vi.fn((destination: T) => destination); +} + +const stubAudioContext = (currentTime = 0.5): FakeAudioContext => { + const instance = new FakeAudioContext(); + instance.currentTime = currentTime; + // Returning a non-this object from a constructor replaces the constructed + // instance, so every `new AudioContext()` yields the same spy bundle. + class StubAudioContext { + constructor() { + return instance as unknown as StubAudioContext; + } + } + vi.stubGlobal("AudioContext", StubAudioContext); + return instance; +}; + +const stubMediaStream = () => { + class FakeMediaStream { + tracks: MediaStreamTrack[]; + constructor(tracks: MediaStreamTrack[]) { + this.tracks = tracks; + } + } + vi.stubGlobal("MediaStream", FakeMediaStream); +}; + +const track = (id: string) => ({ kind: "audio", id }) as unknown as MediaStreamTrack; + +describe("mixAudioTracks", () => { + beforeEach(() => { + stubMediaStream(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("returns the system track verbatim and no context when only system audio is present", () => { + const system = track("sys"); + const result = mixAudioTracks({ systemAudioTrack: system, micAudioTrack: null }); + expect(result.context).toBeNull(); + expect(result.track).toBe(system); + }); + + it("returns the mic track verbatim and no context when only mic is present", () => { + const mic = track("mic"); + const result = mixAudioTracks({ systemAudioTrack: null, micAudioTrack: mic }); + expect(result.context).toBeNull(); + expect(result.track).toBe(mic); + }); + + it("returns null track and no context when neither track is present", () => { + const result = mixAudioTracks({ systemAudioTrack: null, micAudioTrack: null }); + expect(result.context).toBeNull(); + expect(result.track).toBeNull(); + }); + + it("creates an AudioContext, fades the mic gain from 0 to MIC_GAIN_BOOST over MIC_FADE_IN_S, and returns the destination track when both inputs are present", () => { + const fakeCtx = stubAudioContext(0.5); + + const system = track("sys"); + const mic = track("mic"); + const result = mixAudioTracks({ systemAudioTrack: system, micAudioTrack: mic }); + + expect(result.context).toBe(fakeCtx); + expect(fakeCtx.createGain).toHaveBeenCalledTimes(1); + expect(fakeCtx.createMediaStreamSource).toHaveBeenCalledTimes(2); + expect(fakeCtx.createMediaStreamDestination).toHaveBeenCalledTimes(1); + + const [gain] = fakeCtx.createGain.mock.results.map((r) => r.value) as FakeGainNode[]; + expect(gain.gain.setValueAtTime).toHaveBeenCalledWith(0, 0.5); + expect(gain.gain.linearRampToValueAtTime).toHaveBeenCalledWith( + MIC_GAIN_BOOST, + 0.5 + MIC_FADE_IN_S, + ); + + const [systemSource, micSource] = fakeCtx.createMediaStreamSource.mock.results.map( + (r) => r.value, + ) as FakeMediaStreamAudioSourceNode[]; + const [destination] = fakeCtx.createMediaStreamDestination.mock.results.map( + (r) => r.value, + ) as FakeMediaStreamDestination[]; + expect(systemSource.connect).toHaveBeenCalledWith(destination); + expect(micSource.connect).toHaveBeenCalledWith(gain); + expect(gain.connect).toHaveBeenCalledWith(destination); + + expect(result.track).toEqual({ kind: "audio", id: "dest-track" }); + }); + + it("honors AudioContext.currentTime=0 so the fade-in starts at the very first audio packet", () => { + const fakeCtx = stubAudioContext(0); + const result = mixAudioTracks({ systemAudioTrack: track("sys"), micAudioTrack: track("mic") }); + const [gain] = fakeCtx.createGain.mock.results.map((r) => r.value) as FakeGainNode[]; + expect(gain.gain.setValueAtTime).toHaveBeenCalledWith(0, 0); + expect(gain.gain.linearRampToValueAtTime).toHaveBeenCalledWith(MIC_GAIN_BOOST, MIC_FADE_IN_S); + expect(result.context).toBe(fakeCtx); + }); +}); diff --git a/src/lib/audioMix.ts b/src/lib/audioMix.ts new file mode 100644 index 000000000..f614a67bc --- /dev/null +++ b/src/lib/audioMix.ts @@ -0,0 +1,56 @@ +/** + * Mic gain applied when mixing the microphone with system audio into the recorder's + * destination stream. Boosted slightly above unity so the voice is clearly audible + * over typical system-audio levels. + */ +export const MIC_GAIN_BOOST = 1.4; + +/** + * Duration of the mic fade-in applied at the start of a recording. Masks the + * click/pop that the mic produces at its first packet (echo-canceller warm-up, + * DC offset, gain-stage step) without audibly muting speech. + */ +export const MIC_FADE_IN_S = 0.02; + +export type MixAudioTracksInput = { + systemAudioTrack?: MediaStreamTrack | null | undefined; + micAudioTrack?: MediaStreamTrack | null | undefined; +}; + +export type MixAudioTracksResult = { + /** AudioContext created to mix the two tracks. `null` when no mixing was needed. */ + context: AudioContext | null; + /** The track to add to the recorder's MediaStream, or `null` when neither input was given. */ + track: MediaStreamTrack | null; +}; + +/** + * Combine system audio and the microphone into a single audio track for the recorder. + * + * - Both tracks present: builds an AudioContext + GainNode + MediaStreamDestination + * that mixes them, applying a short fade-in on the mic gain so the very first packet + * doesn't click. Caller owns the returned `context` and must `.close()` it on teardown. + * - Exactly one track present: returns it verbatim, no AudioContext created. + * - Neither present: returns `{ context: null, track: null }`. + */ +export function mixAudioTracks({ + systemAudioTrack, + micAudioTrack, +}: MixAudioTracksInput): MixAudioTracksResult { + if (systemAudioTrack && micAudioTrack) { + const context = new AudioContext(); + const systemSource = context.createMediaStreamSource(new MediaStream([systemAudioTrack])); + const micSource = context.createMediaStreamSource(new MediaStream([micAudioTrack])); + const micGain = context.createGain(); + micGain.gain.setValueAtTime(0, context.currentTime); + micGain.gain.linearRampToValueAtTime(MIC_GAIN_BOOST, context.currentTime + MIC_FADE_IN_S); + const destination = context.createMediaStreamDestination(); + systemSource.connect(destination); + micSource.connect(micGain).connect(destination); + return { context, track: destination.stream.getAudioTracks()[0] }; + } + return { + context: null, + track: systemAudioTrack ?? micAudioTrack ?? null, + }; +} From d31aecbf1a600d89b64f73d2f4cb0325a40277b4 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Wed, 15 Jul 2026 10:00:06 +0200 Subject: [PATCH 2/2] fix(recording): stop orphaned capture streams and fade in mic-only recordings Addresses two review findings on the parallel screen/mic capture path: - Stream leak on early exit: after `Promise.all`, the cancellation branch (and a rejected screen capture) ran before the local streams were copied into `screenStream.current` / `microphoneStream.current`, so `teardownMedia()` could not stop them and the screen/mic indicator stayed on. Assign the refs before the cancel check, and stop the parallel mic stream if the screen capture rejects. - Mic-only click: the anti-click fade-in only ran when both system audio and mic were present, so mic-only recordings (a common mode) still clicked at the first packet. Route the mic through the GainNode fade-in in that case too, ramping to unity so mic-only loudness is unchanged (the 1.4x boost still applies only when the mic competes with system audio). tsc + biome clean; 295/295 vitest pass (mic-only test updated). Co-Authored-By: Claude Opus 4.8 --- src/hooks/useScreenRecorder.ts | 21 +++++++++++++----- src/lib/audioMix.test.ts | 28 +++++++++++++++++++++--- src/lib/audioMix.ts | 40 +++++++++++++++++++--------------- 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 02ddad066..9aaa0e243 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -1237,13 +1237,24 @@ export function useScreenRecorder(): UseScreenRecorderReturn { })() : Promise.resolve(null); - const [screenMediaStream, micMediaStream] = await Promise.all([screenCapture, micCapture]); - - if (!isCountdownRunActive(countdownRunToken)) { - teardownMedia(); - return; + // Await both in-flight captures. If the screen capture rejects it would + // otherwise orphan a mic stream that resolved in parallel (leaving the + // screen/mic indicator on), so stop that stream before rethrowing. + let screenMediaStream: MediaStream; + try { + screenMediaStream = await screenCapture; + } catch (error) { + void micCapture + .then((micStream) => micStream?.getTracks().forEach((track) => track.stop())) + .catch(() => { + // Mic capture itself failed too; nothing left to stop. + }); + throw error; } + const micMediaStream = await micCapture; + // Assign the refs before the cancellation check below so teardownMedia() can + // stop the freshly acquired streams if the countdown was cancelled mid-capture. screenStream.current = screenMediaStream; microphoneStream.current = micMediaStream; diff --git a/src/lib/audioMix.test.ts b/src/lib/audioMix.test.ts index d997fbe24..d9ce11e78 100644 --- a/src/lib/audioMix.test.ts +++ b/src/lib/audioMix.test.ts @@ -80,11 +80,33 @@ describe("mixAudioTracks", () => { expect(result.track).toBe(system); }); - it("returns the mic track verbatim and no context when only mic is present", () => { + it("fades a mic-only recording in from 0 to unity gain through its own AudioContext, with no system source", () => { + const fakeCtx = stubAudioContext(0.5); + const mic = track("mic"); const result = mixAudioTracks({ systemAudioTrack: null, micAudioTrack: mic }); - expect(result.context).toBeNull(); - expect(result.track).toBe(mic); + + expect(result.context).toBe(fakeCtx); + expect(fakeCtx.createGain).toHaveBeenCalledTimes(1); + // Only the mic is routed through a source node; there is no system audio. + expect(fakeCtx.createMediaStreamSource).toHaveBeenCalledTimes(1); + expect(fakeCtx.createMediaStreamDestination).toHaveBeenCalledTimes(1); + + const [gain] = fakeCtx.createGain.mock.results.map((r) => r.value) as FakeGainNode[]; + // Ramps to unity (1), not MIC_GAIN_BOOST, so mic-only loudness is unchanged. + expect(gain.gain.setValueAtTime).toHaveBeenCalledWith(0, 0.5); + expect(gain.gain.linearRampToValueAtTime).toHaveBeenCalledWith(1, 0.5 + MIC_FADE_IN_S); + + const [micSource] = fakeCtx.createMediaStreamSource.mock.results.map( + (r) => r.value, + ) as FakeMediaStreamAudioSourceNode[]; + const [destination] = fakeCtx.createMediaStreamDestination.mock.results.map( + (r) => r.value, + ) as FakeMediaStreamDestination[]; + expect(micSource.connect).toHaveBeenCalledWith(gain); + expect(gain.connect).toHaveBeenCalledWith(destination); + + expect(result.track).toEqual({ kind: "audio", id: "dest-track" }); }); it("returns null track and no context when neither track is present", () => { diff --git a/src/lib/audioMix.ts b/src/lib/audioMix.ts index f614a67bc..1329c59cd 100644 --- a/src/lib/audioMix.ts +++ b/src/lib/audioMix.ts @@ -27,30 +27,36 @@ export type MixAudioTracksResult = { /** * Combine system audio and the microphone into a single audio track for the recorder. * - * - Both tracks present: builds an AudioContext + GainNode + MediaStreamDestination - * that mixes them, applying a short fade-in on the mic gain so the very first packet - * doesn't click. Caller owns the returned `context` and must `.close()` it on teardown. - * - Exactly one track present: returns it verbatim, no AudioContext created. + * Whenever a mic track is present it is routed through an AudioContext + GainNode with + * a short fade-in so its first packet doesn't click — mic-only recordings (a common + * mode) get the same anti-click treatment as mixed ones, so `context` is non-null + * whenever a mic track is present. The mic is boosted above unity only when it has to + * sit over system audio; on its own it rides at unity so mic-only loudness is unchanged. + * Caller owns the returned `context` and must `.close()` it on teardown. + * + * - System audio only: returned verbatim, no AudioContext created (no warm-up click). * - Neither present: returns `{ context: null, track: null }`. */ export function mixAudioTracks({ systemAudioTrack, micAudioTrack, }: MixAudioTracksInput): MixAudioTracksResult { - if (systemAudioTrack && micAudioTrack) { - const context = new AudioContext(); + if (!micAudioTrack) { + return { context: null, track: systemAudioTrack ?? null }; + } + + const context = new AudioContext(); + const destination = context.createMediaStreamDestination(); + if (systemAudioTrack) { const systemSource = context.createMediaStreamSource(new MediaStream([systemAudioTrack])); - const micSource = context.createMediaStreamSource(new MediaStream([micAudioTrack])); - const micGain = context.createGain(); - micGain.gain.setValueAtTime(0, context.currentTime); - micGain.gain.linearRampToValueAtTime(MIC_GAIN_BOOST, context.currentTime + MIC_FADE_IN_S); - const destination = context.createMediaStreamDestination(); systemSource.connect(destination); - micSource.connect(micGain).connect(destination); - return { context, track: destination.stream.getAudioTracks()[0] }; } - return { - context: null, - track: systemAudioTrack ?? micAudioTrack ?? null, - }; + // Unity when the mic is on its own; boosted only when competing with system audio. + const micTargetGain = systemAudioTrack ? MIC_GAIN_BOOST : 1; + const micSource = context.createMediaStreamSource(new MediaStream([micAudioTrack])); + const micGain = context.createGain(); + micGain.gain.setValueAtTime(0, context.currentTime); + micGain.gain.linearRampToValueAtTime(micTargetGain, context.currentTime + MIC_FADE_IN_S); + micSource.connect(micGain).connect(destination); + return { context, track: destination.stream.getAudioTracks()[0] }; }