diff --git a/locales/en/app.json b/locales/en/app.json index 7f2e2647f..82b2ce663 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -206,6 +206,11 @@ "audio_tab": { "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play.", "effect_volume_label": "Sound effect volume", + "mic_cutoff_description": "Sound quieter than the cutoff volume will not be sent to other participants.", + "mic_cutoff_header": "Microphone cutoff", + "mic_cutoff_label": "Mute microphone input below a volume cutoff", + "mic_cutoff_not_supported": "(Microphone cutoff is not supported by this browser.)", + "mic_cutoff_threshold_label": "Cutoff volume", "rnnoise_header": "Noise suppression", "rnnoise_label": "Enable enhanced noise suppression (RNNoise)", "rnnoise_not_supported": "(Enhanced noise suppression is not supported by this browser.)", diff --git a/src/Slider.module.css b/src/Slider.module.css index 44ec838db..9f34233e4 100644 --- a/src/Slider.module.css +++ b/src/Slider.module.css @@ -40,6 +40,22 @@ Please see LICENSE in the repository root for full details. background: var(--cpd-color-bg-action-primary-disabled); } +.level { + position: absolute; + block-size: 100%; + inline-size: 0; + border-radius: var(--cpd-radius-pill-effect); + background: var(--cpd-color-icon-quaternary); + transition: + inline-size 0.06s linear, + background-color 0.1s ease; + pointer-events: none; +} + +.levelAbove { + background: var(--cpd-color-icon-success-primary); +} + .handle { display: block; block-size: var(--cpd-space-4x); diff --git a/src/Slider.tsx b/src/Slider.tsx index 29f9ef426..834d564cb 100644 --- a/src/Slider.tsx +++ b/src/Slider.tsx @@ -36,6 +36,12 @@ interface Props { * displayed as a percentage. */ tooltipFormatter?: (value: number) => string; + /** + * Live input level to render as a meter inside the track, as a fraction + * (0-1) of the slider's range. The meter is highlighted while the level + * exceeds the slider's current value. + */ + level?: number; } /** @@ -52,6 +58,7 @@ export const Slider: FC = ({ step, disabled, tooltipFormatter, + level, }) => { const onValueChange = useCallback( ([v]: number[]) => onValueChangeProp(v), @@ -61,6 +68,8 @@ export const Slider: FC = ({ ([v]: number[]) => onValueCommitProp?.(v), [onValueCommitProp], ); + const levelAboveValue = + level !== undefined && max > min && level >= (value - min) / (max - min); return ( = ({ > + {level !== undefined && ( +
+ )} {/* Note: This is expected not to be visible on mobile.*/} , ) => boolean; -} { - let ProcessorCtor: - | (new () => { - process: ( - inputs: Float32Array[][], - outputs: Float32Array[][], - params?: Record, - ) => boolean; - }) - | undefined; + port: { + postMessage: ReturnType; + onmessage: ((event: MessageEvent) => void) | null; + }; +}; + +function instantiateWorkletProcessor(workletCode: string): TestWorklet { + let ProcessorCtor: (new () => TestWorklet) | undefined; class TestAudioWorkletProcessor { public readonly port = { @@ -171,16 +170,7 @@ function instantiateWorkletProcessor(workletCode: string): { } const registerProcessor = vi.fn( - ( - _name: string, - ctor: new () => { - process: ( - inputs: Float32Array[][], - outputs: Float32Array[][], - params?: Record, - ) => boolean; - }, - ) => { + (_name: string, ctor: new () => TestWorklet) => { ProcessorCtor = ctor; }, ); @@ -200,6 +190,26 @@ function instantiateWorkletProcessor(workletCode: string): { return new ProcessorCtor(); } +const GATE_TEST_FRAME_SIZE = 480; + +function sendWorkletMessage(worklet: TestWorklet, data: unknown): void { + worklet.port.onmessage?.({ data } as MessageEvent); +} + +/** + * Feeds one full 480-sample frame of constant amplitude through the worklet + * and returns the corresponding output frame. + */ +function processGateFrame( + worklet: TestWorklet, + amplitude: number, +): Float32Array { + const input = new Float32Array(GATE_TEST_FRAME_SIZE).fill(amplitude); + const output = new Float32Array(GATE_TEST_FRAME_SIZE); + worklet.process([[input]], [[output]], {}); + return output; +} + describe("RNNoiseProcessor", () => { beforeEach(() => { vi.stubGlobal( @@ -652,4 +662,265 @@ describe("RNNoiseProcessor", () => { // init() must have aborted after seeing destroyed=true; no track exposed expect(processor.processedTrack).toBeUndefined(); }); + + it("posts denoise and gate configuration to the worklet on init", async () => { + const t = createTestContext(); + vi.stubGlobal( + "AudioWorkletNode", + class { + constructor() { + return t.workletNode; + } + }, + ); + const processor = new RNNoiseProcessor("balanced", false, { + enabled: true, + thresholdDb: -35, + }); + + await processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + expect(t.workletNode.port.postMessage).toHaveBeenCalledWith({ + type: "denoise", + enabled: false, + }); + expect(t.workletNode.port.postMessage).toHaveBeenCalledWith({ + type: "gate", + enabled: true, + thresholdDb: -35, + }); + }); + + it("updates the gate configuration at runtime", async () => { + const t = createTestContext(); + vi.stubGlobal( + "AudioWorkletNode", + class { + constructor() { + return t.workletNode; + } + }, + ); + const processor = new RNNoiseProcessor(); + + await processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + processor.setGateConfig({ enabled: true, thresholdDb: -50 }); + + expect(t.workletNode.port.postMessage).toHaveBeenCalledWith({ + type: "gate", + enabled: true, + thresholdDb: -50, + }); + }); + + it("allows non-48kHz sample rates when only the gate is enabled", async () => { + const t = createTestContext(44100); + vi.stubGlobal( + "AudioWorkletNode", + class { + constructor() { + return t.workletNode; + } + }, + ); + const processor = new RNNoiseProcessor("conservative", false, { + enabled: true, + thresholdDb: -40, + }); + + await processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + expect(t.addModule).toHaveBeenCalledOnce(); + expect(processor.processedTrack).toBe(t.processedTrack); + }); + + it("throws when enabling denoising on a non-48kHz context", async () => { + const t = createTestContext(44100); + vi.stubGlobal( + "AudioWorkletNode", + class { + constructor() { + return t.workletNode; + } + }, + ); + const processor = new RNNoiseProcessor("conservative", false, { + enabled: true, + thresholdDb: -40, + }); + + await processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + expect(() => processor.setDenoiseEnabled(true)).toThrow("48000Hz"); + }); + + describe("microphone cutoff gate", () => { + it("passes input above the cutoff through unattenuated", () => { + const worklet = instantiateWorkletProcessor(getGeneratedWorkletCode()); + sendWorkletMessage(worklet, { + type: "gate", + enabled: true, + thresholdDb: -40, + }); + + // 0.5 amplitude ≈ -6dBFS, well above the -40dB cutoff + let output: Float32Array = new Float32Array(GATE_TEST_FRAME_SIZE); + for (let i = 0; i < 5; i++) { + output = processGateFrame(worklet, 0.5); + } + + for (const sample of output) { + expect(sample).toBeCloseTo(0.5, 6); + } + }); + + it("mutes sustained input below the cutoff", () => { + const worklet = instantiateWorkletProcessor(getGeneratedWorkletCode()); + sendWorkletMessage(worklet, { + type: "gate", + enabled: true, + thresholdDb: -40, + }); + + // 0.0001 amplitude ≈ -80dBFS, well below the -46dB close threshold + let output: Float32Array = new Float32Array(GATE_TEST_FRAME_SIZE); + for (let i = 0; i < 50; i++) { + output = processGateFrame(worklet, 0.0001); + } + + const maxAbs = output.reduce((max, s) => Math.max(max, Math.abs(s)), 0); + expect(maxAbs).toBeLessThan(1e-6); + }); + + it("reopens quickly when the level rises above the cutoff", () => { + const worklet = instantiateWorkletProcessor(getGeneratedWorkletCode()); + sendWorkletMessage(worklet, { + type: "gate", + enabled: true, + thresholdDb: -40, + }); + + // Close the gate with sustained quiet input, then speak + for (let i = 0; i < 50; i++) { + processGateFrame(worklet, 0.0001); + } + let output: Float32Array = new Float32Array(GATE_TEST_FRAME_SIZE); + for (let i = 0; i < 5; i++) { + output = processGateFrame(worklet, 0.5); + } + + expect(output[output.length - 1]).toBeGreaterThan(0.49); + }); + + it("holds the gate open briefly after the level drops", () => { + const worklet = instantiateWorkletProcessor(getGeneratedWorkletCode()); + sendWorkletMessage(worklet, { + type: "gate", + enabled: true, + thresholdDb: -40, + }); + + // One loud frame arms the hold, then quiet frames within the hold + // window must still pass through unattenuated. + processGateFrame(worklet, 0.5); + let output: Float32Array = new Float32Array(GATE_TEST_FRAME_SIZE); + for (let i = 0; i < 10; i++) { + output = processGateFrame(worklet, 0.0001); + } + + expect(output[output.length - 1]).toBeCloseTo(0.0001, 6); + }); + + it("becomes transparent again when the gate is disabled", () => { + const worklet = instantiateWorkletProcessor(getGeneratedWorkletCode()); + sendWorkletMessage(worklet, { + type: "gate", + enabled: true, + thresholdDb: -40, + }); + + // Close the gate, then disable it; passthrough must resume immediately + for (let i = 0; i < 50; i++) { + processGateFrame(worklet, 0.0001); + } + sendWorkletMessage(worklet, { + type: "gate", + enabled: false, + thresholdDb: -40, + }); + const output = processGateFrame(worklet, 0.0001); + + expect(output[output.length - 1]).toBeCloseTo(0.0001, 6); + }); + + it("reports the measured input level while processing", () => { + const worklet = instantiateWorkletProcessor(getGeneratedWorkletCode()); + sendWorkletMessage(worklet, { + type: "gate", + enabled: true, + thresholdDb: -40, + }); + + // Level reports aggregate 5 frames; process enough for one report + for (let i = 0; i < 5; i++) { + processGateFrame(worklet, 0.5); + } + + const levelReports = worklet.port.postMessage.mock.calls + .map(([message]) => message as { type: string; rmsDb?: number }) + .filter((message) => message.type === "level"); + expect(levelReports.length).toBeGreaterThan(0); + // Constant 0.5 amplitude has an RMS of 0.5 ≈ -6.02dBFS + expect(levelReports[0].rmsDb).toBeCloseTo(-6.02, 1); + }); + }); + + it("publishes worklet level reports and clears them on destroy", async () => { + const t = createTestContext(); + vi.stubGlobal( + "AudioWorkletNode", + class { + constructor() { + return t.workletNode; + } + }, + ); + const processor = new RNNoiseProcessor(); + + await processor.init({ + kind: Track.Kind.Audio, + track: t.track, + audioContext: t.audioContext, + }); + + const onmessage = ( + t.workletNode.port as unknown as { + onmessage: (event: MessageEvent) => void; + } + ).onmessage; + expect(onmessage).toBeTypeOf("function"); + + onmessage({ data: { type: "level", rmsDb: -42 } } as MessageEvent); + expect(microphoneInputLevelDb$.value).toBe(-42); + + await processor.destroy(); + expect(microphoneInputLevelDb$.value).toBeNull(); + }); }); diff --git a/src/audio/RNNoiseProcessor.ts b/src/audio/RNNoiseProcessor.ts index 023758abd..adcae8292 100644 --- a/src/audio/RNNoiseProcessor.ts +++ b/src/audio/RNNoiseProcessor.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { logger } from "matrix-js-sdk/lib/logger"; +import { BehaviorSubject } from "rxjs"; import type { AudioProcessorOptions, @@ -13,6 +14,16 @@ import type { TrackProcessor, } from "livekit-client"; import type { RNNoiseSuppressionPreset } from "./rnnoiseTypes"; +import type { Behavior } from "../state/Behavior"; +import { + GATE_CLOSE_MS, + GATE_HOLD_FRAMES, + GATE_HYSTERESIS_DB, + GATE_OPEN_MS, + LEVEL_REPORT_FRAMES, + MIC_CUTOFF_DEFAULT_DB, + type MicrophoneGateConfig, +} from "./microphoneGate"; import rnnoiseWorkletModuleUrl from "./RNNoiseWorkletModule.ts?worker&url"; /** @@ -27,6 +38,16 @@ const DEFAULT_RNNOISE_PRESET: RNNoiseSuppressionPreset = "conservative"; const workletRegistrations = new WeakMap>(); const warnedUnsupportedSampleRates = new Set(); +const _microphoneInputLevelDb$ = new BehaviorSubject(null); +/** + * The current microphone input level in dBFS, measured by the active + * microphone audio worklet (peak frame RMS per ~50ms window), or null while + * no processor is running. Used by the settings UI to render a live level + * meter next to the microphone cutoff slider. + */ +export const microphoneInputLevelDb$: Behavior = + _microphoneInputLevelDb$; + type RNNoiseSupportGlobal = typeof globalThis & { AudioWorklet?: { prototype?: { @@ -97,6 +118,12 @@ const FRAME_SIZE = ${RNNOISE_SAMPLE_LENGTH}; const RING_SIZE = FRAME_SIZE * 3; // Enough headroom for buffering const SAMPLE_RATE = ${RNNOISE_REQUIRED_SAMPLE_RATE}; const DEFAULT_PRESET = "${DEFAULT_RNNOISE_PRESET}"; +const GATE_HYSTERESIS_DB = ${GATE_HYSTERESIS_DB}; +const GATE_HOLD_FRAMES = ${GATE_HOLD_FRAMES}; +const GATE_OPEN_MS = ${GATE_OPEN_MS}; +const GATE_CLOSE_MS = ${GATE_CLOSE_MS}; +const MIC_CUTOFF_DEFAULT_DB = ${MIC_CUTOFF_DEFAULT_DB}; +const LEVEL_REPORT_FRAMES = ${LEVEL_REPORT_FRAMES}; const PRESETS = { conservative: { maxAttenuationDb: 4, @@ -133,6 +160,7 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { // Ring buffers this._inBuf = new Float32Array(RING_SIZE); this._outBuf = new Float32Array(RING_SIZE); + this._frameBuf = new Float32Array(FRAME_SIZE); this._inW = 0; // input write position this._inR = 0; // input read position this._outW = 0; // output write position @@ -140,8 +168,19 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this._currentGain = 1; this._targetGain = 1; this._holdFrames = 0; + this._denoiseEnabled = true; + this._gateEnabled = false; + this._gateOpenThresholdDb = MIC_CUTOFF_DEFAULT_DB; + this._gateCloseThresholdDb = MIC_CUTOFF_DEFAULT_DB - GATE_HYSTERESIS_DB; + this._gateHoldFrames = 0; + this._gateCurrentGain = 1; + this._gateTargetGain = 1; + this._levelReportCounter = 0; + this._levelReportMaxRms = 0; this._setPreset(DEFAULT_PRESET); + this._gateOpenStep = this._smoothingStepFromMs(GATE_OPEN_MS); + this._gateCloseStep = this._smoothingStepFromMs(GATE_CLOSE_MS); this._initRNNoise(); @@ -150,6 +189,10 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this._cleanup(); } else if (event.data.type === 'preset') { this._setPreset(event.data.preset); + } else if (event.data.type === 'denoise') { + this._setDenoiseEnabled(event.data.enabled === true); + } else if (event.data.type === 'gate') { + this._setGateConfig(event.data.enabled, event.data.thresholdDb); } }; } @@ -172,6 +215,62 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this._releaseStep = this._smoothingStepFromMs(config.releaseMs); } + _setDenoiseEnabled(enabled) { + this._denoiseEnabled = enabled; + if (!enabled) { + // Release any RNNoise attenuation so the stage becomes transparent. + this._targetGain = 1; + this._holdFrames = 0; + } + } + + _setGateConfig(enabled, thresholdDb) { + this._gateEnabled = enabled === true; + if (typeof thresholdDb === 'number' && Number.isFinite(thresholdDb)) { + this._gateOpenThresholdDb = thresholdDb; + this._gateCloseThresholdDb = thresholdDb - GATE_HYSTERESIS_DB; + } + if (!this._gateEnabled) { + this._gateTargetGain = 1; + this._gateCurrentGain = 1; + this._gateHoldFrames = 0; + } + } + + _reportLevel(frameRms) { + if (frameRms > this._levelReportMaxRms) { + this._levelReportMaxRms = frameRms; + } + this._levelReportCounter += 1; + if (this._levelReportCounter < LEVEL_REPORT_FRAMES) return; + + const rmsDb = this._levelReportMaxRms > 0 + ? 20 * Math.log10(this._levelReportMaxRms) + : -200; + this.port.postMessage({ type: 'level', rmsDb }); + this._levelReportCounter = 0; + this._levelReportMaxRms = 0; + } + + _updateGateGain(frameRms) { + if (!this._gateEnabled) { + this._gateTargetGain = 1; + return; + } + + const rmsDb = frameRms > 0 ? 20 * Math.log10(frameRms) : -200; + if (rmsDb >= this._gateOpenThresholdDb) { + this._gateHoldFrames = GATE_HOLD_FRAMES; + this._gateTargetGain = 1; + } else if (this._gateHoldFrames > 0) { + this._gateHoldFrames -= 1; + } else if (rmsDb < this._gateCloseThresholdDb) { + this._gateTargetGain = 0; + } + // Between the close and open thresholds the gate keeps its previous + // state (hysteresis). + } + _updateTargetGain(vadProbability) { if (vadProbability >= this._openThreshold) { this._holdFrames = this._holdFramesConfig; @@ -233,32 +332,58 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this._destroyed = true; } - _processRNNoiseFrame() { - const heapIdx = this._pcmBuf >> 2; // byte offset → float32 index + _processFrame() { + const useDenoise = this._denoiseEnabled && this._ready; + let sumSquares = 0; - // Copy from input ring buffer to WASM heap, scaling to int16 range - for (let i = 0; i < FRAME_SIZE; i++) { - this._heapF32[heapIdx + i] = - this._inBuf[(this._inR + i) % RING_SIZE] * 32768.0; + if (useDenoise) { + const heapIdx = this._pcmBuf >> 2; // byte offset → float32 index + + // Copy from input ring buffer to WASM heap, scaling to int16 range + for (let i = 0; i < FRAME_SIZE; i++) { + this._heapF32[heapIdx + i] = + this._inBuf[(this._inR + i) % RING_SIZE] * 32768.0; + } + + // Run RNNoise denoising (in-place) + const vadProbability = this._module._rnnoise_process_frame( + this._state, this._pcmBuf, this._pcmBuf + ); + this._updateTargetGain(vadProbability); + + for (let i = 0; i < FRAME_SIZE; i++) { + const sample = this._heapF32[heapIdx + i] / 32768.0; + this._frameBuf[i] = sample; + sumSquares += sample * sample; + } + } else { + for (let i = 0; i < FRAME_SIZE; i++) { + const sample = this._inBuf[(this._inR + i) % RING_SIZE]; + this._frameBuf[i] = sample; + sumSquares += sample * sample; + } } this._inR = (this._inR + FRAME_SIZE) % RING_SIZE; - // Run RNNoise denoising (in-place) - const vadProbability = this._module._rnnoise_process_frame( - this._state, this._pcmBuf, this._pcmBuf - ); - this._updateTargetGain(vadProbability); + const frameRms = Math.sqrt(sumSquares / FRAME_SIZE); + this._updateGateGain(frameRms); + this._reportLevel(frameRms); - // Copy from WASM heap to output ring buffer, scaling back to float range. - // Apply additional conservative attenuation between speech segments. + // Copy to the output ring buffer, applying the smoothed RNNoise + // attenuation and the noise-gate gain. for (let i = 0; i < FRAME_SIZE; i++) { const smoothingStep = this._targetGain < this._currentGain ? this._attenuateStep : this._releaseStep; this._currentGain += (this._targetGain - this._currentGain) * smoothingStep; + const gateStep = this._gateTargetGain < this._gateCurrentGain + ? this._gateCloseStep + : this._gateOpenStep; + this._gateCurrentGain += + (this._gateTargetGain - this._gateCurrentGain) * gateStep; this._outBuf[(this._outW + i) % RING_SIZE] = - (this._heapF32[heapIdx + i] / 32768.0) * this._currentGain; + this._frameBuf[i] * this._currentGain * this._gateCurrentGain; } this._outW = (this._outW + FRAME_SIZE) % RING_SIZE; } @@ -284,8 +409,11 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { const blockSize = output.length; const channelCount = inputChannels.length; - if (!this._ready) { - // Pass through until RNNoise is ready, with deterministic mono downmix. + const framePathActive = + (this._denoiseEnabled && this._ready) || this._gateEnabled; + if (!framePathActive) { + // Pass through when no processing stage is active (e.g. RNNoise not + // ready yet), with deterministic mono downmix. for (let i = 0; i < blockSize; i++) { output[i] = this._mixInputChannels(inputChannels, i, channelCount); } @@ -304,7 +432,7 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { // Process complete frames while (this._ringAvailable(this._inW, this._inR) >= FRAME_SIZE) { - this._processRNNoiseFrame(); + this._processFrame(); } // Read from output ring buffer @@ -336,7 +464,8 @@ export function createRNNoiseWorkletCodeForTesting( /** * A LiveKit TrackProcessor that applies RNNoise-based noise suppression - * to a local audio track via an AudioWorklet. + * and/or a level-based noise gate (microphone cutoff volume) to a local + * audio track via an AudioWorklet. * * The RNNoise WASM binary is lazy-loaded only when the processor is * initialized, keeping the main bundle small. @@ -353,12 +482,21 @@ export class RNNoiseProcessor implements TrackProcessor< private destinationNode?: MediaStreamAudioDestinationNode; private destroyed = false; private preset: RNNoiseSuppressionPreset; + private denoiseEnabled: boolean; + private gate: MicrophoneGateConfig; private lastAudioContext?: AudioContext; public constructor( preset: RNNoiseSuppressionPreset = DEFAULT_RNNOISE_PRESET, + denoiseEnabled = true, + gate: MicrophoneGateConfig = { + enabled: false, + thresholdDb: MIC_CUTOFF_DEFAULT_DB, + }, ) { this.preset = preset; + this.denoiseEnabled = denoiseEnabled; + this.gate = { ...gate }; } private async ensureWorkletRegistered( @@ -387,7 +525,11 @@ export class RNNoiseProcessor implements TrackProcessor< this.destroyed = false; const { audioContext, track } = opts; - if (audioContext.sampleRate !== RNNOISE_REQUIRED_SAMPLE_RATE) { + // RNNoise is trained for 48kHz audio; the gate works at any rate. + if ( + this.denoiseEnabled && + audioContext.sampleRate !== RNNOISE_REQUIRED_SAMPLE_RATE + ) { warnUnsupportedSampleRate(audioContext.sampleRate); throw createUnsupportedSampleRateError(audioContext.sampleRate); } @@ -418,7 +560,28 @@ export class RNNoiseProcessor implements TrackProcessor< this.sourceNode = sourceNode; this.workletNode = workletNode; this.destinationNode = destinationNode; + this.workletNode.port.onmessage = (event: MessageEvent): void => { + const data = event.data as { + type?: string; + rmsDb?: number; + message?: string; + }; + if (data.type === "level" && typeof data.rmsDb === "number") { + _microphoneInputLevelDb$.next(data.rmsDb); + } else if (data.type === "error") { + logger.warn("Microphone audio worklet error", data.message); + } + }; this.workletNode.port.postMessage({ type: "preset", preset: this.preset }); + this.workletNode.port.postMessage({ + type: "denoise", + enabled: this.denoiseEnabled, + }); + this.workletNode.port.postMessage({ + type: "gate", + enabled: this.gate.enabled, + thresholdDb: this.gate.thresholdDb, + }); this.processedTrack = destinationNode.stream.getAudioTracks()[0]; this.lastAudioContext = audioContext; } @@ -460,6 +623,7 @@ export class RNNoiseProcessor implements TrackProcessor< this.workletNode = undefined; this.destinationNode = undefined; this.processedTrack = undefined; + _microphoneInputLevelDb$.next(null); await Promise.resolve(); } @@ -470,4 +634,35 @@ export class RNNoiseProcessor implements TrackProcessor< preset: this.preset, }); } + + /** + * Toggles the RNNoise denoising stage without tearing down the worklet. + * Throws when enabling denoising on an AudioContext whose sample rate + * RNNoise does not support. + */ + public setDenoiseEnabled(enabled: boolean): void { + if ( + enabled && + !this.denoiseEnabled && + this.lastAudioContext !== undefined && + this.lastAudioContext.sampleRate !== RNNOISE_REQUIRED_SAMPLE_RATE + ) { + warnUnsupportedSampleRate(this.lastAudioContext.sampleRate); + throw createUnsupportedSampleRateError(this.lastAudioContext.sampleRate); + } + this.denoiseEnabled = enabled; + this.workletNode?.port.postMessage({ type: "denoise", enabled }); + } + + /** + * Updates the level-based noise gate (microphone cutoff volume). + */ + public setGateConfig(gate: MicrophoneGateConfig): void { + this.gate = { ...gate }; + this.workletNode?.port.postMessage({ + type: "gate", + enabled: gate.enabled, + thresholdDb: gate.thresholdDb, + }); + } } diff --git a/src/audio/RNNoiseWorkletModule.ts b/src/audio/RNNoiseWorkletModule.ts index 3f59e2922..9f0b83014 100644 --- a/src/audio/RNNoiseWorkletModule.ts +++ b/src/audio/RNNoiseWorkletModule.ts @@ -8,6 +8,14 @@ Please see LICENSE in the repository root for full details. import createRNNWasmModuleSync from "@jitsi/rnnoise-wasm/dist/rnnoise-sync.js"; import type { RNNoiseSuppressionPreset } from "./rnnoiseTypes"; +import { + GATE_CLOSE_MS, + GATE_HOLD_FRAMES, + GATE_HYSTERESIS_DB, + GATE_OPEN_MS, + LEVEL_REPORT_FRAMES, + MIC_CUTOFF_DEFAULT_DB, +} from "./microphoneGate"; declare abstract class AudioWorkletProcessor { protected constructor(options?: AudioWorkletNodeOptions); @@ -43,7 +51,9 @@ type RNNoiseModule = { type WorkletMessage = | { type: "destroy" } - | { type: "preset"; preset: RNNoiseSuppressionPreset }; + | { type: "preset"; preset: RNNoiseSuppressionPreset } + | { type: "denoise"; enabled: boolean } + | { type: "gate"; enabled: boolean; thresholdDb: number }; const PRESETS: Record = { conservative: { @@ -81,6 +91,7 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { private destroyed = false; private readonly inBuf = new Float32Array(RING_SIZE); private readonly outBuf = new Float32Array(RING_SIZE); + private readonly frameBuf = new Float32Array(FRAME_SIZE); private inW = 0; private inR = 0; private outW = 0; @@ -88,6 +99,17 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { private currentGain = 1; private targetGain = 1; private holdFrames = 0; + private denoiseEnabled = true; + private gateEnabled = false; + private gateOpenThresholdDb = MIC_CUTOFF_DEFAULT_DB; + private gateCloseThresholdDb = MIC_CUTOFF_DEFAULT_DB - GATE_HYSTERESIS_DB; + private gateHoldFrames = 0; + private gateCurrentGain = 1; + private gateTargetGain = 1; + private gateOpenStep = 1; + private gateCloseStep = 1; + private levelReportCounter = 0; + private levelReportMaxRms = 0; private maxAttenuationDb = PRESETS[DEFAULT_PRESET].maxAttenuationDb; private openThreshold = PRESETS[DEFAULT_PRESET].openThreshold; private closeThreshold = PRESETS[DEFAULT_PRESET].closeThreshold; @@ -103,6 +125,8 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { super(); this.setPreset(DEFAULT_PRESET); + this.gateOpenStep = this.smoothingStepFromMs(GATE_OPEN_MS); + this.gateCloseStep = this.smoothingStepFromMs(GATE_CLOSE_MS); this.initRNNoise(); this.port.onmessage = (event: MessageEvent): void => { @@ -110,6 +134,10 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this.cleanup(); } else if (event.data.type === "preset" && isPreset(event.data.preset)) { this.setPreset(event.data.preset); + } else if (event.data.type === "denoise") { + this.setDenoiseEnabled(event.data.enabled === true); + } else if (event.data.type === "gate") { + this.setGateConfig(event.data.enabled, event.data.thresholdDb); } }; } @@ -132,6 +160,63 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this.releaseStep = this.smoothingStepFromMs(config.releaseMs); } + private setDenoiseEnabled(enabled: boolean): void { + this.denoiseEnabled = enabled; + if (!enabled) { + // Release any RNNoise attenuation so the stage becomes transparent. + this.targetGain = 1; + this.holdFrames = 0; + } + } + + private setGateConfig(enabled: unknown, thresholdDb: unknown): void { + this.gateEnabled = enabled === true; + if (typeof thresholdDb === "number" && Number.isFinite(thresholdDb)) { + this.gateOpenThresholdDb = thresholdDb; + this.gateCloseThresholdDb = thresholdDb - GATE_HYSTERESIS_DB; + } + if (!this.gateEnabled) { + this.gateTargetGain = 1; + this.gateCurrentGain = 1; + this.gateHoldFrames = 0; + } + } + + private reportLevel(frameRms: number): void { + if (frameRms > this.levelReportMaxRms) { + this.levelReportMaxRms = frameRms; + } + this.levelReportCounter += 1; + if (this.levelReportCounter < LEVEL_REPORT_FRAMES) return; + + const rmsDb = + this.levelReportMaxRms > 0 + ? 20 * Math.log10(this.levelReportMaxRms) + : -200; + this.port.postMessage({ type: "level", rmsDb }); + this.levelReportCounter = 0; + this.levelReportMaxRms = 0; + } + + private updateGateGain(frameRms: number): void { + if (!this.gateEnabled) { + this.gateTargetGain = 1; + return; + } + + const rmsDb = frameRms > 0 ? 20 * Math.log10(frameRms) : -200; + if (rmsDb >= this.gateOpenThresholdDb) { + this.gateHoldFrames = GATE_HOLD_FRAMES; + this.gateTargetGain = 1; + } else if (this.gateHoldFrames > 0) { + this.gateHoldFrames -= 1; + } else if (rmsDb < this.gateCloseThresholdDb) { + this.gateTargetGain = 0; + } + // Between the close and open thresholds the gate keeps its previous + // state (hysteresis). + } + private updateTargetGain(vadProbability: number): void { if (vadProbability >= this.openThreshold) { this.holdFrames = this.holdFramesConfig; @@ -197,35 +282,50 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { this.destroyed = true; } - private processRNNoiseFrame(): void { - if ( - !this.module || - this.state === null || - this.pcmBuf === undefined || - !this.heapF32 - ) { - return; - } + private processFrame(): void { + const { module, state, pcmBuf, heapF32 } = this; + const useDenoise = + this.denoiseEnabled && + this.ready && + module !== undefined && + state !== null && + pcmBuf !== undefined && + heapF32 !== undefined; - const heapIdx = this.pcmBuf >> 2; + let sumSquares = 0; - for (let i = 0; i < FRAME_SIZE; i++) { - this.heapF32[heapIdx + i] = - this.inBuf[(this.inR + i) % RING_SIZE] * 32768; + if (useDenoise) { + const heapIdx = pcmBuf >> 2; + + for (let i = 0; i < FRAME_SIZE; i++) { + heapF32[heapIdx + i] = this.inBuf[(this.inR + i) % RING_SIZE] * 32768; + } + + const rnnoiseProcessFrame = module["_rnnoise_process_frame"] as ( + state: number, + input: number, + output: number, + ) => number; + const vadProbability = rnnoiseProcessFrame(state, pcmBuf, pcmBuf); + this.updateTargetGain(vadProbability); + + for (let i = 0; i < FRAME_SIZE; i++) { + const sample = heapF32[heapIdx + i] / 32768; + this.frameBuf[i] = sample; + sumSquares += sample * sample; + } + } else { + for (let i = 0; i < FRAME_SIZE; i++) { + const sample = this.inBuf[(this.inR + i) % RING_SIZE]; + this.frameBuf[i] = sample; + sumSquares += sample * sample; + } } this.inR = (this.inR + FRAME_SIZE) % RING_SIZE; - const rnnoiseProcessFrame = this.module["_rnnoise_process_frame"] as ( - state: number, - input: number, - output: number, - ) => number; - const vadProbability = rnnoiseProcessFrame( - this.state, - this.pcmBuf, - this.pcmBuf, - ); - this.updateTargetGain(vadProbability); + const frameRms = Math.sqrt(sumSquares / FRAME_SIZE); + this.updateGateGain(frameRms); + this.reportLevel(frameRms); for (let i = 0; i < FRAME_SIZE; i++) { const smoothingStep = @@ -233,8 +333,14 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { ? this.attenuateStep : this.releaseStep; this.currentGain += (this.targetGain - this.currentGain) * smoothingStep; + const gateStep = + this.gateTargetGain < this.gateCurrentGain + ? this.gateCloseStep + : this.gateOpenStep; + this.gateCurrentGain += + (this.gateTargetGain - this.gateCurrentGain) * gateStep; this.outBuf[(this.outW + i) % RING_SIZE] = - (this.heapF32[heapIdx + i] / 32768) * this.currentGain; + this.frameBuf[i] * this.currentGain * this.gateCurrentGain; } this.outW = (this.outW + FRAME_SIZE) % RING_SIZE; } @@ -263,7 +369,9 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { const blockSize = output.length; const channelCount = inputChannels.length; - if (!this.ready) { + const framePathActive = + (this.denoiseEnabled && this.ready) || this.gateEnabled; + if (!framePathActive) { for (let i = 0; i < blockSize; i++) { output[i] = this.mixInputChannels(inputChannels, i, channelCount); } @@ -280,7 +388,7 @@ class RNNoiseWorkletProcessor extends AudioWorkletProcessor { } while (this.ringAvailable(this.inW, this.inR) >= FRAME_SIZE) { - this.processRNNoiseFrame(); + this.processFrame(); } const outAvailable = this.ringAvailable(this.outW, this.outR); diff --git a/src/audio/microphoneGate.ts b/src/audio/microphoneGate.ts new file mode 100644 index 000000000..c1455d417 --- /dev/null +++ b/src/audio/microphoneGate.ts @@ -0,0 +1,46 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * Range and default of the microphone cutoff volume, in dBFS. Microphone + * input whose level stays below the cutoff is muted by a noise gate in the + * audio worklet. + */ +export const MIC_CUTOFF_MIN_DB = -60; +export const MIC_CUTOFF_MAX_DB = -10; +export const MIC_CUTOFF_DEFAULT_DB = -40; + +/** + * The gate closes only once the level falls this far below the cutoff, so it + * doesn't flutter when the level hovers around the threshold. + */ +export const GATE_HYSTERESIS_DB = 6; +/** + * Number of 10ms frames the gate stays open after the level last exceeded + * the cutoff, so it doesn't clip the ends of words. + */ +export const GATE_HOLD_FRAMES = 30; +/** + * Gain smoothing time constants for opening and closing the gate. Opening is + * fast to avoid clipping the start of speech; closing is slower to avoid + * audible pumping. + */ +export const GATE_OPEN_MS = 5; +export const GATE_CLOSE_MS = 40; + +/** + * How many 10ms frames to aggregate before the worklet reports the measured + * input level (peak frame RMS within the window) to the main thread, for the + * live meter shown next to the cutoff slider. + */ +export const LEVEL_REPORT_FRAMES = 5; + +export interface MicrophoneGateConfig { + enabled: boolean; + /** Level in dBFS below which microphone input is muted. */ + thresholdDb: number; +} diff --git a/src/settings/SettingsModal.test.tsx b/src/settings/SettingsModal.test.tsx index c4a93e437..702647e4d 100644 --- a/src/settings/SettingsModal.test.tsx +++ b/src/settings/SettingsModal.test.tsx @@ -14,10 +14,13 @@ import type { MatrixClient } from "matrix-js-sdk"; import type { ReactNode } from "react"; import { SettingsModal } from "./SettingsModal"; import { + micCutoffEnabled, + micCutoffThresholdDb, rnnoiseNoiseSuppression, rnnoiseNoiseSuppressionPreset, } from "./settings"; import { supportsRNNoiseProcessor } from "../audio/RNNoiseProcessor"; +import { MIC_CUTOFF_DEFAULT_DB } from "../audio/microphoneGate"; const { mockRequestDeviceNames } = vi.hoisted(() => ({ mockRequestDeviceNames: vi.fn(), @@ -133,6 +136,8 @@ describe("SettingsModal RNNoise controls", () => { mockRequestDeviceNames.mockClear(); rnnoiseNoiseSuppressionPreset.setValue("conservative"); rnnoiseNoiseSuppression.setValue(false); + micCutoffEnabled.setValue(false); + micCutoffThresholdDb.setValue(MIC_CUTOFF_DEFAULT_DB); vi.mocked(supportsRNNoiseProcessor).mockReturnValue(true); }); @@ -186,4 +191,43 @@ describe("SettingsModal RNNoise controls", () => { localStorage.getItem("matrix-setting-rnnoise-noise-suppression"), ).toBe("true"); }); + + it("shows the cutoff volume slider only when microphone cutoff is enabled", async () => { + const user = userEvent.setup(); + renderSettingsModal(); + + const checkbox = screen.getByLabelText( + "Mute microphone input below a volume cutoff", + ); + expect(checkbox).not.toBeChecked(); + // Only the sound effect volume slider is present initially + expect(screen.getAllByRole("slider")).toHaveLength(1); + expect(screen.queryByText(/Cutoff volume/)).not.toBeInTheDocument(); + + await user.click(checkbox); + + expect(micCutoffEnabled.getValue()).toBe(true); + expect(localStorage.getItem("matrix-setting-mic-cutoff-enabled")).toBe( + "true", + ); + expect(screen.getByText(/Cutoff volume/)).toBeInTheDocument(); + expect(screen.getAllByRole("slider")).toHaveLength(2); + }); + + it("disables microphone cutoff when AudioWorklet support is unavailable", () => { + vi.mocked(supportsRNNoiseProcessor).mockReturnValue(false); + micCutoffEnabled.setValue(true); + + renderSettingsModal(); + + const checkbox = screen.getByLabelText( + "Mute microphone input below a volume cutoff", + ); + expect(checkbox).toBeDisabled(); + expect(checkbox).not.toBeChecked(); + expect( + screen.getByText("(Microphone cutoff is not supported by this browser.)"), + ).toBeInTheDocument(); + expect(screen.queryByText(/Cutoff volume/)).not.toBeInTheDocument(); + }); }); diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index ea4695574..3172df347 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -56,6 +56,8 @@ import { type VideoCodec, rnnoiseNoiseSuppression as rnnoiseNoiseSuppressionSetting, rnnoiseNoiseSuppressionPreset as rnnoiseNoiseSuppressionPresetSetting, + micCutoffEnabled as micCutoffEnabledSetting, + micCutoffThresholdDb as micCutoffThresholdDbSetting, } from "./settings"; import { PreferencesSettingsTab } from "./PreferencesSettingsTab"; import { Slider } from "../Slider"; @@ -66,11 +68,15 @@ import { FieldRow, InputField } from "../input/Input"; import { useSubmitRageshake } from "./submit-rageshake"; import { useUrlParams } from "../UrlParams"; import { useBehavior } from "../useBehavior"; -import { supportsRNNoiseProcessor } from "../audio/RNNoiseProcessor"; +import { + microphoneInputLevelDb$, + supportsRNNoiseProcessor, +} from "../audio/RNNoiseProcessor"; import { type RNNoiseSuppressionPreset, rnnoiseSuppressionPresets, } from "../audio/rnnoiseTypes"; +import { MIC_CUTOFF_MAX_DB, MIC_CUTOFF_MIN_DB } from "../audio/microphoneGate"; type SettingsTab = | "audio" @@ -320,6 +326,72 @@ export const SettingsModal: FC = ({ ); }; + const MicrophoneCutoffSettings: React.FC = (): ReactNode => { + const supported = supportsRNNoiseProcessor(); + const [cutoffEnabled, setCutoffEnabled] = useSetting( + micCutoffEnabledSetting, + ); + const [cutoffDb, setCutoffDb] = useSetting(micCutoffThresholdDbSetting); + const [cutoffDbRaw, setCutoffDbRaw] = useState(cutoffDb); + const effectiveCutoffEnabled = supported && !!cutoffEnabled; + + // Live input level from the mic worklet, mapped onto the slider's range. + // Only flows while the processor is attached (i.e. during a call). + const inputLevelDb = useBehavior(microphoneInputLevelDb$); + const inputLevelFraction = + inputLevelDb === null + ? 0 + : Math.max( + 0, + Math.min( + 1, + (inputLevelDb - MIC_CUTOFF_MIN_DB) / + (MIC_CUTOFF_MAX_DB - MIC_CUTOFF_MIN_DB), + ), + ); + + return ( + <> +

{t("settings.audio_tab.mic_cutoff_header")}

+ + setCutoffEnabled(e.target.checked)} + disabled={!supported} + /> + + {effectiveCutoffEnabled && ( +
+ + `${v} dB`} + level={inputLevelFraction} + /> +
+ )} + + ); + }; + const AudioProcessingSettings: React.FC = (): ReactNode => { const [echoCancellation, setEchoCancellation] = useSetting( echoCancellationSetting, @@ -481,6 +553,8 @@ export const SettingsModal: FC = ({
+ + diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 8c7f01c89..d09a55838 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -9,6 +9,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject } from "rxjs"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; +import { MIC_CUTOFF_DEFAULT_DB } from "../audio/microphoneGate"; import type { RNNoiseSuppressionPreset } from "../audio/rnnoiseTypes"; import { type Behavior } from "../state/Behavior"; import { useBehavior } from "../useBehavior"; @@ -127,6 +128,15 @@ export const rnnoiseNoiseSuppressionPreset = "conservative", ); +export const micCutoffEnabled = new Setting( + "mic-cutoff-enabled", + false, +); +export const micCutoffThresholdDb = new Setting( + "mic-cutoff-threshold-db", + MIC_CUTOFF_DEFAULT_DB, +); + export const showHandRaisedTimer = new Setting( "hand-raised-show-timer", false, diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index 73fa4714d..a2d7baa94 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -28,10 +28,13 @@ import { Publisher } from "./Publisher"; import { type Connection } from "../remoteMembers/Connection"; import { type MuteStates } from "../../MuteStates"; import { + micCutoffEnabled, + micCutoffThresholdDb, rnnoiseNoiseSuppression, rnnoiseNoiseSuppressionPreset, } from "../../../settings/settings"; import type { RNNoiseProcessor } from "../../../audio/RNNoiseProcessor"; +import { MIC_CUTOFF_DEFAULT_DB } from "../../../audio/microphoneGate"; let scope: ObservableScope; @@ -390,6 +393,8 @@ describe("Publisher", () => { vi.unstubAllGlobals(); rnnoiseNoiseSuppression.setValue(false); rnnoiseNoiseSuppressionPreset.setValue("conservative"); + micCutoffEnabled.setValue(false); + micCutoffThresholdDb.setValue(MIC_CUTOFF_DEFAULT_DB); }); it("enabling setting applies RNNoise processor on microphone track", async () => { @@ -614,6 +619,40 @@ describe("Publisher", () => { ); }); + it("keeps the processor and only disables the gate when cutoff is turned off while RNNoise is on", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { getProcessor: () => unknown }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + rnnoiseNoiseSuppression.setValue(true); + micCutoffEnabled.setValue(true); + for (let i = 0; i < 5; i++) { + await flushPromises(); + } + + const processor = micTrack.getProcessor() as RNNoiseProcessor; + expect(processor).toBeDefined(); + const setGateConfigSpy = vi.spyOn(processor, "setGateConfig"); + vi.mocked(micTrack.stopProcessor).mockClear(); + + micCutoffEnabled.setValue(false); + await flushPromises(); + + expect(setGateConfigSpy).toHaveBeenCalledWith( + expect.objectContaining({ enabled: false }), + ); + expect(micTrack.stopProcessor).not.toHaveBeenCalled(); + }); + it("updates active RNNoise processor preset when preset setting changes", async () => { const micTrack = createMockLocalTrack( Track.Source.Microphone, @@ -640,6 +679,117 @@ describe("Publisher", () => { expect(setPresetSpy).toHaveBeenCalledWith("strong"); }); }); + + describe("Microphone cutoff", () => { + beforeEach(() => { + vi.stubGlobal("AudioWorkletNode", class AudioWorkletNode {}); + vi.stubGlobal( + "AudioWorklet", + class AudioWorklet { + public async addModule(): Promise { + await Promise.resolve(); + } + }, + ); + vi.stubGlobal( + "MediaStreamAudioDestinationNode", + class MediaStreamAudioDestinationNode {}, + ); + vi.stubGlobal( + "MediaStreamAudioSourceNode", + class MediaStreamAudioSourceNode {}, + ); + micCutoffEnabled.setValue(false); + micCutoffThresholdDb.setValue(MIC_CUTOFF_DEFAULT_DB); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + micCutoffEnabled.setValue(false); + micCutoffThresholdDb.setValue(MIC_CUTOFF_DEFAULT_DB); + }); + + function publishMicTrack(): LocalTrack & { + setProcessor: (...args: unknown[]) => Promise; + stopProcessor: () => void; + restartTrack: (...args: unknown[]) => void; + getProcessor: () => unknown; + } { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { + setProcessor: (...args: unknown[]) => Promise; + stopProcessor: () => void; + restartTrack: (...args: unknown[]) => void; + getProcessor: () => unknown; + }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + return micTrack; + } + + it("enabling cutoff applies the processor even when RNNoise is disabled", async () => { + const micTrack = publishMicTrack(); + + micCutoffEnabled.setValue(true); + await flushPromises(); + + expect(micTrack.setProcessor).toHaveBeenCalledOnce(); + // The gate doesn't change capture constraints, so no restart is needed + expect(micTrack.restartTrack).not.toHaveBeenCalled(); + }); + + it("disabling cutoff removes the processor when RNNoise is off", async () => { + const micTrack = publishMicTrack(); + + micCutoffEnabled.setValue(true); + await flushPromises(); + micCutoffEnabled.setValue(false); + await flushPromises(); + + expect(micTrack.setProcessor).toHaveBeenCalledOnce(); + expect(micTrack.stopProcessor).toHaveBeenCalledOnce(); + }); + + it("updates the gate config on the active processor when the threshold changes", async () => { + const micTrack = publishMicTrack(); + + micCutoffEnabled.setValue(true); + await flushPromises(); + + const processor = micTrack.getProcessor() as RNNoiseProcessor; + const setGateConfigSpy = vi.spyOn(processor, "setGateConfig"); + + micCutoffThresholdDb.setValue(-30); + await flushPromises(); + + expect(setGateConfigSpy).toHaveBeenCalledWith({ + enabled: true, + thresholdDb: -30, + }); + }); + + it("auto-disables the cutoff setting when processor setup fails", async () => { + const micTrack = publishMicTrack(); + vi.mocked(micTrack.setProcessor).mockRejectedValueOnce( + new Error("gate setup failed"), + ); + + micCutoffEnabled.setValue(true); + for (let i = 0; i < 5; i++) { + await flushPromises(); + } + + expect(micCutoffEnabled.getValue()).toBe(false); + }); + }); }); describe("Bug fix", () => { diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 043c624ee..03af3bc17 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -44,11 +44,14 @@ import { import { shouldEnableNativeNoiseSuppression } from "../../../audio/noiseSuppressionPolicy.ts"; import { echoCancellationSetting, + micCutoffEnabled, + micCutoffThresholdDb, noiseSuppressionSetting, rnnoiseNoiseSuppression, rnnoiseNoiseSuppressionPreset, } from "../../../settings/settings.ts"; import type { RNNoiseSuppressionPreset } from "../../../audio/rnnoiseTypes.ts"; +import type { MicrophoneGateConfig } from "../../../audio/microphoneGate.ts"; /** * A wrapper for a Connection object. @@ -485,41 +488,63 @@ export class Publisher { microphoneTrack$, rnnoiseNoiseSuppression.value$, rnnoiseNoiseSuppressionPreset.value$, + micCutoffEnabled.value$, + micCutoffThresholdDb.value$, ]) .pipe( scope.bind(), + // Changes to the RNNoise enabled setting are deliberately ignored + // here; they need a track restart and are handled by + // observeRNNoiseSettingRestart. distinctUntilChanged( - ([aTrack, _aEnabled, aPreset], [bTrack, _bEnabled, bPreset]) => { - return aTrack === bTrack && aPreset === bPreset; + ( + [aTrack, _aEnabled, aPreset, aGateEnabled, aGateThresholdDb], + [bTrack, _bEnabled, bPreset, bGateEnabled, bGateThresholdDb], + ) => { + return ( + aTrack === bTrack && + aPreset === bPreset && + aGateEnabled === bGateEnabled && + aGateThresholdDb === bGateThresholdDb + ); }, ), ) - .subscribe(([microphoneTrack, rnnoiseEnabled, rnnoisePreset]) => { - const rnnoiseSupported = supportsRNNoiseProcessor(); - if (!microphoneTrack || !rnnoiseSupported) { - this.rnnoisePolicySyncedTrack = microphoneTrack; - return; - } + .subscribe( + ([ + microphoneTrack, + rnnoiseEnabled, + rnnoisePreset, + gateEnabled, + gateThresholdDb, + ]) => { + const rnnoiseSupported = supportsRNNoiseProcessor(); + if (!microphoneTrack || !rnnoiseSupported) { + this.rnnoisePolicySyncedTrack = microphoneTrack; + return; + } - const isNewMicrophoneTrack = - microphoneTrack !== this.rnnoisePolicySyncedTrack; - this.rnnoisePolicySyncedTrack = microphoneTrack; + const isNewMicrophoneTrack = + microphoneTrack !== this.rnnoisePolicySyncedTrack; + this.rnnoisePolicySyncedTrack = microphoneTrack; - this.enqueueRNNoiseOperation(async () => { - if (rnnoiseEnabled && isNewMicrophoneTrack) { - await this.restartMicrophoneTrackForNoiseSuppressionPolicy( + this.enqueueRNNoiseOperation(async () => { + if (rnnoiseEnabled && isNewMicrophoneTrack) { + await this.restartMicrophoneTrackForNoiseSuppressionPolicy( + microphoneTrack, + devices, + rnnoiseEnabled, + ); + } + await this.syncRNNoiseProcessor( microphoneTrack, - devices, rnnoiseEnabled, + rnnoisePreset, + { enabled: gateEnabled, thresholdDb: gateThresholdDb }, ); - } - await this.syncRNNoiseProcessor( - microphoneTrack, - rnnoiseEnabled, - rnnoisePreset, - ); - }); - }); + }); + }, + ); } private observeRNNoiseSettingRestart( @@ -546,6 +571,10 @@ export class Publisher { audioTrack, rnnoiseEnabled && rnnoiseSupported, rnnoiseNoiseSuppressionPreset.getValue(), + { + enabled: micCutoffEnabled.getValue(), + thresholdDb: micCutoffThresholdDb.getValue(), + }, ); }); }); @@ -586,33 +615,43 @@ export class Publisher { microphoneTrack: LocalAudioTrack, rnnoiseEnabled: boolean, rnnoisePreset: RNNoiseSuppressionPreset, + gate: MicrophoneGateConfig, ): Promise { try { const processor = microphoneTrack.getProcessor(); - const rnnoiseActive = processor?.name === "rnnoise-noise-suppression"; + const processorActive = processor?.name === "rnnoise-noise-suppression"; const rnnoiseProcessor = processor instanceof RNNoiseProcessor ? processor : undefined; - if (rnnoiseEnabled) { + if (rnnoiseEnabled || gate.enabled) { if (rnnoiseProcessor) { rnnoiseProcessor.setPreset(rnnoisePreset); + rnnoiseProcessor.setDenoiseEnabled(rnnoiseEnabled); + rnnoiseProcessor.setGateConfig(gate); return; } - if (rnnoiseActive) { + if (processorActive) { await microphoneTrack.stopProcessor(); } - await microphoneTrack.setProcessor(new RNNoiseProcessor(rnnoisePreset)); - } else if (rnnoiseActive) { + await microphoneTrack.setProcessor( + new RNNoiseProcessor(rnnoisePreset, rnnoiseEnabled, gate), + ); + } else if (processorActive) { await microphoneTrack.stopProcessor(); } } catch (e) { - this.logger.error("Failed to apply RNNoise microphone processor", e); + this.logger.error("Failed to apply microphone audio processor", e); if (rnnoiseEnabled && rnnoiseNoiseSuppression.getValue()) { this.logger.warn( "Disabling RNNoise setting after processor setup failure", ); rnnoiseNoiseSuppression.setValue(false); + } else if (gate.enabled && micCutoffEnabled.getValue()) { + this.logger.warn( + "Disabling microphone cutoff setting after processor setup failure", + ); + micCutoffEnabled.setValue(false); } } }