diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 656509763..8eb97c23c 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -2,6 +2,7 @@ import { fixWebmDuration } from "@fix-webm-duration/fix"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { useScopedT } from "@/contexts/I18nContext"; +import { MIC_GAIN_BOOST, mixAudioTracks } from "@/lib/audioMix"; import { type NativeMacRecordingRequest, parseMacDisplayIdFromSourceId, @@ -45,7 +46,6 @@ const WEBCAM_FILE_SUFFIX = "-webcam"; const AUDIO_BITRATE_VOICE = 128_000; const AUDIO_BITRATE_SYSTEM = 192_000; -const MIC_GAIN_BOOST = 1.4; const WEBCAM_TARGET_FRAME_RATE = 30; type UseScreenRecorderReturn = { @@ -1181,23 +1181,27 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } - let screenMediaStream: MediaStream; - const platform = await window.electronAPI.getPlatform(); + // Capture screen + microphone in parallel: the gap between the two + // `getUserMedia` calls is the dominant source of the mic-vs-video lag at the + // start of the recording (issue #57). + const screenCapture = (async (): Promise => { + const platform = await window.electronAPI.getPlatform(); + + if (platform === "win32") { + // getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the + // pre-selected source. Editable cursor mode excludes the system cursor so + // the editor can render a replacement; system mode bakes it into the video. + return navigator.mediaDevices.getDisplayMedia({ + video: { + cursor: cursorCaptureMode === "editable-overlay" ? "never" : "always", + width: { max: TARGET_WIDTH }, + height: { max: TARGET_HEIGHT }, + frameRate: { ideal: TARGET_FRAME_RATE }, + } as MediaTrackConstraints, + audio: systemAudioEnabled, + } as DisplayMediaStreamOptions); + } - if (platform === "win32") { - // getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the - // pre-selected source. Editable cursor mode excludes the system cursor so - // the editor can render a replacement; system mode bakes it into the video. - screenMediaStream = await navigator.mediaDevices.getDisplayMedia({ - video: { - cursor: cursorCaptureMode === "editable-overlay" ? "never" : "always", - width: { max: TARGET_WIDTH }, - height: { max: TARGET_HEIGHT }, - frameRate: { ideal: TARGET_FRAME_RATE }, - } as MediaTrackConstraints, - audio: systemAudioEnabled, - } as DisplayMediaStreamOptions); - } else { const videoConstraints = { mandatory: { chromeMediaSource: CHROME_MEDIA_SOURCE, @@ -1211,7 +1215,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (systemAudioEnabled) { try { - screenMediaStream = await navigator.mediaDevices.getUserMedia({ + return navigator.mediaDevices.getUserMedia({ audio: { mandatory: { chromeMediaSource: CHROME_MEDIA_SOURCE, @@ -1223,48 +1227,66 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } catch (audioErr) { console.warn("System audio capture failed, falling back to video-only:", audioErr); toast.error(t("recording.systemAudioUnavailable")); - screenMediaStream = await navigator.mediaDevices.getUserMedia({ + return navigator.mediaDevices.getUserMedia({ audio: false, video: videoConstraints, } as unknown as MediaStreamConstraints); } - } else { - screenMediaStream = await navigator.mediaDevices.getUserMedia({ - audio: false, - video: videoConstraints, - } as unknown as MediaStreamConstraints); } - } - screenStream.current = screenMediaStream; - if (!isCountdownRunActive(countdownRunToken)) { - teardownMedia(); - return; - } + return navigator.mediaDevices.getUserMedia({ + audio: false, + video: videoConstraints, + } as unknown as MediaStreamConstraints); + })(); - if (microphoneEnabled) { - try { - microphoneStream.current = await navigator.mediaDevices.getUserMedia({ - audio: microphoneDeviceId - ? { - deviceId: { exact: microphoneDeviceId }, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - } - : { - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - }, - video: false, + const micCapture: Promise = microphoneEnabled + ? (async () => { + try { + return await navigator.mediaDevices.getUserMedia({ + audio: microphoneDeviceId + ? { + deviceId: { exact: microphoneDeviceId }, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + } + : { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + video: false, + }); + } catch (audioError) { + console.warn("Failed to get microphone access:", audioError); + toast.error(t("recording.microphoneDenied")); + setMicrophoneEnabled(false); + return null; + } + })() + : Promise.resolve(null); + + // Await both in-flight captures. If the screen capture rejects it would + // otherwise orphan a mic stream that resolved in parallel (leaving the + // screen/mic indicator on), so stop that stream before rethrowing. + let screenMediaStream: MediaStream; + try { + screenMediaStream = await screenCapture; + } catch (error) { + void micCapture + .then((micStream) => micStream?.getTracks().forEach((track) => track.stop())) + .catch(() => { + // Mic capture itself failed too; nothing left to stop. }); - } catch (audioError) { - console.warn("Failed to get microphone access:", audioError); - toast.error(t("recording.microphoneDenied")); - setMicrophoneEnabled(false); - } + throw error; } + const micMediaStream = await micCapture; + + // Assign the refs before the cancellation check below so teardownMedia() can + // stop the freshly acquired streams if the countdown was cancelled mid-capture. + screenStream.current = screenMediaStream; + microphoneStream.current = micMediaStream; if (!isCountdownRunActive(countdownRunToken)) { teardownMedia(); @@ -1307,21 +1329,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const systemAudioTrack = screenMediaStream.getAudioTracks()[0]; const micAudioTrack = microphoneStream.current?.getAudioTracks()[0]; - if (systemAudioTrack && micAudioTrack) { - const ctx = new AudioContext(); - mixingContext.current = ctx; - const systemSource = ctx.createMediaStreamSource(new MediaStream([systemAudioTrack])); - const micSource = ctx.createMediaStreamSource(new MediaStream([micAudioTrack])); - const micGain = ctx.createGain(); - micGain.gain.value = MIC_GAIN_BOOST; - const destination = ctx.createMediaStreamDestination(); - systemSource.connect(destination); - micSource.connect(micGain).connect(destination); - stream.current.addTrack(destination.stream.getAudioTracks()[0]); - } else if (systemAudioTrack) { - stream.current.addTrack(systemAudioTrack); - } else if (micAudioTrack) { - stream.current.addTrack(micAudioTrack); + const { context: mixingCtx, track: mixedTrack } = mixAudioTracks({ + systemAudioTrack, + micAudioTrack, + }); + if (mixingCtx) { + mixingContext.current = mixingCtx; + } + if (mixedTrack) { + stream.current.addTrack(mixedTrack); } try { diff --git a/src/lib/audioMix.test.ts b/src/lib/audioMix.test.ts new file mode 100644 index 000000000..d9ce11e78 --- /dev/null +++ b/src/lib/audioMix.test.ts @@ -0,0 +1,158 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { MIC_FADE_IN_S, MIC_GAIN_BOOST, mixAudioTracks } from "./audioMix"; + +class FakeAudioParam { + value = 1; + setValueAtTime = vi.fn((v: number) => { + this.value = v; + }); + linearRampToValueAtTime = vi.fn((v: number) => { + this.value = v; + }); +} + +class FakeGainNode { + gain = new FakeAudioParam(); + // Real AudioNode.connect returns the destination so it can be chained. + connect = vi.fn((destination: T) => destination); +} + +class FakeMediaStreamAudioSourceNode { + connect = vi.fn((destination: T) => destination); +} + +class FakeAudioContext { + currentTime = 0.5; + destination: FakeMediaStreamDestination; + createGain = vi.fn(() => new FakeGainNode()); + createMediaStreamSource = vi.fn(() => new FakeMediaStreamAudioSourceNode()); + createMediaStreamDestination = vi.fn(() => new FakeMediaStreamDestination()); + close = vi.fn(); + constructor() { + this.destination = new FakeMediaStreamDestination(); + } +} + +class FakeMediaStreamDestination { + stream = { getAudioTracks: () => [{ kind: "audio", id: "dest-track" }] }; + connect = vi.fn((destination: T) => destination); +} + +const stubAudioContext = (currentTime = 0.5): FakeAudioContext => { + const instance = new FakeAudioContext(); + instance.currentTime = currentTime; + // Returning a non-this object from a constructor replaces the constructed + // instance, so every `new AudioContext()` yields the same spy bundle. + class StubAudioContext { + constructor() { + return instance as unknown as StubAudioContext; + } + } + vi.stubGlobal("AudioContext", StubAudioContext); + return instance; +}; + +const stubMediaStream = () => { + class FakeMediaStream { + tracks: MediaStreamTrack[]; + constructor(tracks: MediaStreamTrack[]) { + this.tracks = tracks; + } + } + vi.stubGlobal("MediaStream", FakeMediaStream); +}; + +const track = (id: string) => ({ kind: "audio", id }) as unknown as MediaStreamTrack; + +describe("mixAudioTracks", () => { + beforeEach(() => { + stubMediaStream(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("returns the system track verbatim and no context when only system audio is present", () => { + const system = track("sys"); + const result = mixAudioTracks({ systemAudioTrack: system, micAudioTrack: null }); + expect(result.context).toBeNull(); + expect(result.track).toBe(system); + }); + + it("fades a mic-only recording in from 0 to unity gain through its own AudioContext, with no system source", () => { + const fakeCtx = stubAudioContext(0.5); + + const mic = track("mic"); + const result = mixAudioTracks({ systemAudioTrack: null, micAudioTrack: mic }); + + expect(result.context).toBe(fakeCtx); + expect(fakeCtx.createGain).toHaveBeenCalledTimes(1); + // Only the mic is routed through a source node; there is no system audio. + expect(fakeCtx.createMediaStreamSource).toHaveBeenCalledTimes(1); + expect(fakeCtx.createMediaStreamDestination).toHaveBeenCalledTimes(1); + + const [gain] = fakeCtx.createGain.mock.results.map((r) => r.value) as FakeGainNode[]; + // Ramps to unity (1), not MIC_GAIN_BOOST, so mic-only loudness is unchanged. + expect(gain.gain.setValueAtTime).toHaveBeenCalledWith(0, 0.5); + expect(gain.gain.linearRampToValueAtTime).toHaveBeenCalledWith(1, 0.5 + MIC_FADE_IN_S); + + const [micSource] = fakeCtx.createMediaStreamSource.mock.results.map( + (r) => r.value, + ) as FakeMediaStreamAudioSourceNode[]; + const [destination] = fakeCtx.createMediaStreamDestination.mock.results.map( + (r) => r.value, + ) as FakeMediaStreamDestination[]; + expect(micSource.connect).toHaveBeenCalledWith(gain); + expect(gain.connect).toHaveBeenCalledWith(destination); + + expect(result.track).toEqual({ kind: "audio", id: "dest-track" }); + }); + + it("returns null track and no context when neither track is present", () => { + const result = mixAudioTracks({ systemAudioTrack: null, micAudioTrack: null }); + expect(result.context).toBeNull(); + expect(result.track).toBeNull(); + }); + + it("creates an AudioContext, fades the mic gain from 0 to MIC_GAIN_BOOST over MIC_FADE_IN_S, and returns the destination track when both inputs are present", () => { + const fakeCtx = stubAudioContext(0.5); + + const system = track("sys"); + const mic = track("mic"); + const result = mixAudioTracks({ systemAudioTrack: system, micAudioTrack: mic }); + + expect(result.context).toBe(fakeCtx); + expect(fakeCtx.createGain).toHaveBeenCalledTimes(1); + expect(fakeCtx.createMediaStreamSource).toHaveBeenCalledTimes(2); + expect(fakeCtx.createMediaStreamDestination).toHaveBeenCalledTimes(1); + + const [gain] = fakeCtx.createGain.mock.results.map((r) => r.value) as FakeGainNode[]; + expect(gain.gain.setValueAtTime).toHaveBeenCalledWith(0, 0.5); + expect(gain.gain.linearRampToValueAtTime).toHaveBeenCalledWith( + MIC_GAIN_BOOST, + 0.5 + MIC_FADE_IN_S, + ); + + const [systemSource, micSource] = fakeCtx.createMediaStreamSource.mock.results.map( + (r) => r.value, + ) as FakeMediaStreamAudioSourceNode[]; + const [destination] = fakeCtx.createMediaStreamDestination.mock.results.map( + (r) => r.value, + ) as FakeMediaStreamDestination[]; + expect(systemSource.connect).toHaveBeenCalledWith(destination); + expect(micSource.connect).toHaveBeenCalledWith(gain); + expect(gain.connect).toHaveBeenCalledWith(destination); + + expect(result.track).toEqual({ kind: "audio", id: "dest-track" }); + }); + + it("honors AudioContext.currentTime=0 so the fade-in starts at the very first audio packet", () => { + const fakeCtx = stubAudioContext(0); + const result = mixAudioTracks({ systemAudioTrack: track("sys"), micAudioTrack: track("mic") }); + const [gain] = fakeCtx.createGain.mock.results.map((r) => r.value) as FakeGainNode[]; + expect(gain.gain.setValueAtTime).toHaveBeenCalledWith(0, 0); + expect(gain.gain.linearRampToValueAtTime).toHaveBeenCalledWith(MIC_GAIN_BOOST, MIC_FADE_IN_S); + expect(result.context).toBe(fakeCtx); + }); +}); diff --git a/src/lib/audioMix.ts b/src/lib/audioMix.ts new file mode 100644 index 000000000..1329c59cd --- /dev/null +++ b/src/lib/audioMix.ts @@ -0,0 +1,62 @@ +/** + * Mic gain applied when mixing the microphone with system audio into the recorder's + * destination stream. Boosted slightly above unity so the voice is clearly audible + * over typical system-audio levels. + */ +export const MIC_GAIN_BOOST = 1.4; + +/** + * Duration of the mic fade-in applied at the start of a recording. Masks the + * click/pop that the mic produces at its first packet (echo-canceller warm-up, + * DC offset, gain-stage step) without audibly muting speech. + */ +export const MIC_FADE_IN_S = 0.02; + +export type MixAudioTracksInput = { + systemAudioTrack?: MediaStreamTrack | null | undefined; + micAudioTrack?: MediaStreamTrack | null | undefined; +}; + +export type MixAudioTracksResult = { + /** AudioContext created to mix the two tracks. `null` when no mixing was needed. */ + context: AudioContext | null; + /** The track to add to the recorder's MediaStream, or `null` when neither input was given. */ + track: MediaStreamTrack | null; +}; + +/** + * Combine system audio and the microphone into a single audio track for the recorder. + * + * Whenever a mic track is present it is routed through an AudioContext + GainNode with + * a short fade-in so its first packet doesn't click — mic-only recordings (a common + * mode) get the same anti-click treatment as mixed ones, so `context` is non-null + * whenever a mic track is present. The mic is boosted above unity only when it has to + * sit over system audio; on its own it rides at unity so mic-only loudness is unchanged. + * Caller owns the returned `context` and must `.close()` it on teardown. + * + * - System audio only: returned verbatim, no AudioContext created (no warm-up click). + * - Neither present: returns `{ context: null, track: null }`. + */ +export function mixAudioTracks({ + systemAudioTrack, + micAudioTrack, +}: MixAudioTracksInput): MixAudioTracksResult { + if (!micAudioTrack) { + return { context: null, track: systemAudioTrack ?? null }; + } + + const context = new AudioContext(); + const destination = context.createMediaStreamDestination(); + if (systemAudioTrack) { + const systemSource = context.createMediaStreamSource(new MediaStream([systemAudioTrack])); + systemSource.connect(destination); + } + // Unity when the mic is on its own; boosted only when competing with system audio. + const micTargetGain = systemAudioTrack ? MIC_GAIN_BOOST : 1; + const micSource = context.createMediaStreamSource(new MediaStream([micAudioTrack])); + const micGain = context.createGain(); + micGain.gain.setValueAtTime(0, context.currentTime); + micGain.gain.linearRampToValueAtTime(micTargetGain, context.currentTime + MIC_FADE_IN_S); + micSource.connect(micGain).connect(destination); + return { context, track: destination.stream.getAudioTracks()[0] }; +}