From 92c2f578e28a2201518678c5104aaf4aaeadef64 Mon Sep 17 00:00:00 2001 From: Myx Date: Fri, 14 Aug 2026 03:19:29 +0200 Subject: [PATCH] fix(voice): route media on evidence and switch devices without dropping the call Outgoing voice was gated on the observer's roster copy of the remote user's voice state, which is signaling gossip. The signal server broadcasts `user_left` for any socket it declares dead, so a suspended laptop or a flaky hop wiped that copy and the observer detached its microphone from a peer that never left the channel - a silent member with no way back through the UI. `decideVoicePathRouting` now closes a path only on positive evidence: we left voice, the peer itself reported another channel or none, or the connection is gone. Missing gossip holds an established path instead. Opening still needs confirmation, so a guess never starts sending; the same rule gates playback, camera video, and the microphone a new connection puts in its first offer. Peers are also asked for their voice state when a connection or data channel comes up, so a rebuilt path re-confirms itself. Alongside it, the microphone can be switched mid-call: capture moves to a device service and rules, the live track is swapped with `replaceTrack` so the session is never renegotiated, and the speaking indicator follows the new stream. --- .../voice/live-input-device-change.spec.ts | 293 ++++++++++ .../voice/long-session-voice-soak.spec.ts | 151 +++++ .../voice/roster-loss-preserves-voice.spec.ts | 252 ++++++++ e2e/tests/voice/turn-relay-only-voice.spec.ts | 156 +++++ toju-app/public/i18n/catalog/settings.json | 3 +- toju-app/public/i18n/catalog/voice.json | 4 + .../facades/voice-connection.facade.ts | 8 + .../services/voice-activity.service.ts | 21 +- .../services/voice-playback.service.spec.ts | 32 +- .../services/voice-playback.service.ts | 30 +- .../src/app/domains/voice-session/README.md | 21 + .../voice-audio-device.service.spec.ts | 198 +++++++ .../services/voice-audio-device.service.ts | 167 ++++++ .../audio-device-selection.rules.spec.ts | 109 ++++ .../logic/audio-device-selection.rules.ts | 91 +++ .../logic/voice-path-routing.rules.spec.ts | 160 +++++ .../domain/logic/voice-path-routing.rules.ts | 112 ++++ .../floating-voice-controls.component.ts | 31 +- .../voice-controls.component.ts | 102 +--- .../src/app/domains/voice-session/index.ts | 4 + .../voice-settings.component.html | 24 +- .../voice-settings.component.ts | 64 +- .../realtime/media/media.manager.spec.ts | 547 ++++++++++++++++++ .../realtime/media/media.manager.ts | 466 +++++++++++---- .../media/voice-session-controller.ts | 7 + .../connection/create-peer-connection.spec.ts | 171 ++++++ .../connection/create-peer-connection.ts | 13 + .../connection/negotiation.ts | 5 +- .../messaging/data-channel.spec.ts | 27 + .../messaging/data-channel.ts | 86 ++- .../peer-connection-manager/shared.ts | 15 + .../realtime/realtime-session.service.ts | 83 ++- .../realtime/streams/peer-media-facade.ts | 13 +- 33 files changed, 3149 insertions(+), 317 deletions(-) create mode 100644 e2e/tests/voice/live-input-device-change.spec.ts create mode 100644 e2e/tests/voice/long-session-voice-soak.spec.ts create mode 100644 e2e/tests/voice/roster-loss-preserves-voice.spec.ts create mode 100644 e2e/tests/voice/turn-relay-only-voice.spec.ts create mode 100644 toju-app/src/app/domains/voice-session/application/services/voice-audio-device.service.spec.ts create mode 100644 toju-app/src/app/domains/voice-session/application/services/voice-audio-device.service.ts create mode 100644 toju-app/src/app/domains/voice-session/domain/logic/audio-device-selection.rules.spec.ts create mode 100644 toju-app/src/app/domains/voice-session/domain/logic/audio-device-selection.rules.ts create mode 100644 toju-app/src/app/domains/voice-session/domain/logic/voice-path-routing.rules.spec.ts create mode 100644 toju-app/src/app/domains/voice-session/domain/logic/voice-path-routing.rules.ts create mode 100644 toju-app/src/app/infrastructure/realtime/media/media.manager.spec.ts create mode 100644 toju-app/src/app/infrastructure/realtime/peer-connection-manager/connection/create-peer-connection.spec.ts diff --git a/e2e/tests/voice/live-input-device-change.spec.ts b/e2e/tests/voice/live-input-device-change.spec.ts new file mode 100644 index 0000000..c29b88f --- /dev/null +++ b/e2e/tests/voice/live-input-device-change.spec.ts @@ -0,0 +1,293 @@ +import { expect, type Page } from '@playwright/test'; +import { test, type Client } from '../../fixtures/multi-client'; +import { countCreatedPeerConnections } from '../../helpers/peer-role'; +import { openSettingsDetailPage } from '../../helpers/settings-modal'; +import { installDeterministicVoiceSettings } from '../../helpers/voice-session'; +import { + dumpRtcDiagnostics, + installAutoResumeAudioContext, + installWebRTCTracking, + waitForAllPeerAudioFlow, + waitForAudioStatsPresent, + waitForConnectedPeerCount, + waitForOpenDataChannelCount +} from '../../helpers/webrtc-helpers'; +import { ChatRoomPage } from '../../pages/chat-room.page'; +import { RegisterPage } from '../../pages/register.page'; +import { ServerSearchPage } from '../../pages/server-search.page'; + +interface VoiceClient extends Client { + displayName: string; + username: string; +} + +const USER_PASSWORD = 'TestPass123!'; +const VOICE_CHANNEL = 'General'; + +test.describe('Live audio device change', () => { + test('switching the microphone mid-call keeps both directions of audio alive', async ({ createClient }) => { + test.setTimeout(240_000); + + const clients = await createVoicePair(createClient, `Mic Swap ${Date.now()}`); + const [alice, bob] = clients; + + await assertMeshAudio(clients, 'initial two-user voice'); + + const connectionsBefore = { + alice: await countCreatedPeerConnections(alice.page), + bob: await countCreatedPeerConnections(bob.page) + }; + const sentTracksBefore = await readOutboundAudioTrackIds(alice.page); + + expect(sentTracksBefore, 'Alice should be sending audio before the switch').toHaveLength(1); + + await test.step('Alice picks a different microphone from voice settings', async () => { + await openVoiceSettings(alice.page); + + const alternateDeviceId = await readAlternateInputDeviceId(alice.page); + + await startVoiceStateWatch(alice.page); + await alice.page.getByTestId('voice-settings-input-device').selectOption(alternateDeviceId); + + // The swap re-captures the microphone; give it a moment before reading senders. + await expect + .poll(async () => (await readOutboundAudioTrackIds(alice.page))[0], { timeout: 20_000 }) + .not.toBe(sentTracksBefore[0]); + }); + + await test.step('The session was never interrupted', async () => { + const drops = await stopVoiceStateWatch(alice.page); + + expect(drops, 'Alice left and rejoined voice instead of swapping the track').toBe(0); + + expect( + await countCreatedPeerConnections(alice.page), + 'Alice rebuilt her peer connection to change microphone' + ).toBe(connectionsBefore.alice); + + expect( + await countCreatedPeerConnections(bob.page), + 'Bob rebuilt his peer connection because Alice changed microphone' + ).toBe(connectionsBefore.bob); + }); + + await test.step('Audio still flows both ways on the new microphone', async () => { + await waitForConnectedPeerCount(alice.page, 1, 30_000); + await waitForConnectedPeerCount(bob.page, 1, 30_000); + await assertMeshAudio(clients, 'after microphone switch'); + }); + }); + + test('switching the speaker mid-call keeps remote audio playing', async ({ createClient }) => { + test.setTimeout(240_000); + + const clients = await createVoicePair(createClient, `Speaker Swap ${Date.now()}`); + const [alice] = clients; + + await assertMeshAudio(clients, 'initial two-user voice'); + + await openVoiceSettings(alice.page); + + const alternateDeviceId = await readAlternateOutputDeviceId(alice.page); + + test.skip(alternateDeviceId === null, 'This browser exposes no audio output devices'); + + await startVoiceStateWatch(alice.page); + await alice.page.getByTestId('voice-settings-output-device').selectOption(alternateDeviceId as string); + + await expect + .poll(async () => readPreferredOutputDeviceId(alice.page), { timeout: 20_000 }) + .toBe(alternateDeviceId === '' ? 'default' : alternateDeviceId); + + expect(await stopVoiceStateWatch(alice.page), 'Changing the speaker dropped Alice out of voice').toBe(0); + + await assertMeshAudio(clients, 'after speaker switch'); + }); +}); + +async function openVoiceSettings(page: Page): Promise { + await openSettingsDetailPage(page, 'voice'); + await expect(page.getByTestId('voice-settings-input-device')).toBeVisible({ timeout: 10_000 }); +} + +/** The picker value to switch to: any real device, else the system-default entry. */ +async function readAlternateInputDeviceId(page: Page): Promise { + const select = page.getByTestId('voice-settings-input-device'); + const currentValue = await select.inputValue(); + const values = await select.locator('option').evaluateAll( + (options) => options.map((option) => (option as HTMLOptionElement).value) + ); + const alternate = values.find((value) => value !== currentValue); + + if (alternate === undefined) { + throw new Error(`The microphone picker only offers "${currentValue}", so no switch can be made`); + } + + return alternate; +} + +async function readAlternateOutputDeviceId(page: Page): Promise { + const select = page.getByTestId('voice-settings-output-device'); + const currentValue = await select.inputValue(); + const values = await select.locator('option').evaluateAll( + (options) => options.map((option) => (option as HTMLOptionElement).value) + ); + + return values.find((value) => value !== currentValue) ?? null; +} + +/** The audio track ids this page is currently sending, one per peer connection. */ +async function readOutboundAudioTrackIds(page: Page): Promise<(string | null)[]> { + return await page.evaluate(() => { + const connections = (window as unknown as { __rtcConnections?: RTCPeerConnection[] }).__rtcConnections ?? []; + + return connections + .filter((connection) => connection.connectionState === 'connected') + .map((connection) => { + const audioSender = connection + .getSenders() + .find((sender) => sender.track?.kind === 'audio'); + + return audioSender?.track?.id ?? null; + }); + }); +} + +async function readPreferredOutputDeviceId(page: Page): Promise { + return await page.evaluate(() => { + interface AngularDebugApi { + getComponent: (element: Element) => Record; + } + interface PlaybackShape { preferredOutputDeviceId?: string } + + const host = document.querySelector('app-voice-settings'); + const debugApi = (window as { ng?: AngularDebugApi }).ng; + + if (!host || !debugApi?.getComponent) { + return null; + } + + const playback = debugApi.getComponent(host)['voicePlayback'] as PlaybackShape | undefined; + + return playback?.preferredOutputDeviceId ?? null; + }); +} + +/** + * Start counting moments where this client considered itself out of voice. + * A device change that tears the session down and rebuilds it registers here, + * even when the end state looks healthy again. + */ +async function startVoiceStateWatch(page: Page): Promise { + await page.evaluate(() => { + interface AngularDebugApi { + getComponent: (element: Element) => Record; + } + interface VoiceStateShape { isConnected?: boolean } + interface UserShape { voiceState?: VoiceStateShape } + + const watchWindow = window as unknown as { __voiceDrops?: number; __voiceWatch?: number }; + + watchWindow.__voiceDrops = 0; + watchWindow.__voiceWatch = window.setInterval(() => { + const host = document.querySelector('app-rooms-side-panel'); + const debugApi = (window as { ng?: AngularDebugApi }).ng; + + if (!host || !debugApi?.getComponent) { + return; + } + + const component = debugApi.getComponent(host); + const currentUser = (component['currentUser'] as (() => UserShape | null) | undefined)?.() ?? null; + + if (currentUser?.voiceState?.isConnected === false) { + watchWindow.__voiceDrops = (watchWindow.__voiceDrops ?? 0) + 1; + } + }, 100); + }); +} + +async function stopVoiceStateWatch(page: Page): Promise { + return await page.evaluate(() => { + const watchWindow = window as unknown as { __voiceDrops?: number; __voiceWatch?: number }; + + if (watchWindow.__voiceWatch !== undefined) { + window.clearInterval(watchWindow.__voiceWatch); + watchWindow.__voiceWatch = undefined; + } + + return watchWindow.__voiceDrops ?? 0; + }); +} + +async function createVoicePair( + createClient: () => Promise, + serverName: string +): Promise { + const clients: VoiceClient[] = []; + + for (let index = 0; index < 2; index++) { + const client = await createClient(); + + await installDeterministicVoiceSettings(client.page); + await installWebRTCTracking(client.page); + await installAutoResumeAudioContext(client.page); + + clients.push({ + ...client, + displayName: `Device Voice ${index + 1}`, + username: `device_voice_${Date.now()}_${index + 1}` + }); + } + + await test.step('Register both clients', async () => { + for (const client of clients) { + const registerPage = new RegisterPage(client.page); + + await registerPage.goto(); + await registerPage.register(client.username, client.displayName, USER_PASSWORD); + await expect(client.page).toHaveURL(/\/dashboard/, { timeout: 20_000 }); + } + }); + + await test.step('Create and join the server', async () => { + await new ServerSearchPage(clients[0].page).createServer(serverName, { + description: 'Live audio device change test' + }); + + await expect(clients[0].page).toHaveURL(/\/room\//, { timeout: 20_000 }); + + await new ServerSearchPage(clients[1].page).joinServerFromSearch(serverName); + await expect(clients[1].page).toHaveURL(/\/room\//, { timeout: 20_000 }); + }); + + await test.step('Join both clients to voice', async () => { + await new ChatRoomPage(clients[0].page).ensureVoiceChannelExists(VOICE_CHANNEL); + + for (const client of clients) { + const room = new ChatRoomPage(client.page); + + await room.joinVoiceChannel(VOICE_CHANNEL); + await expect(room.voiceControls).toBeVisible({ timeout: 20_000 }); + } + + for (const client of clients) { + await waitForConnectedPeerCount(client.page, 1, 90_000); + await waitForOpenDataChannelCount(client.page, 1, 90_000); + await waitForAudioStatsPresent(client.page, 30_000); + } + }); + + return clients; +} + +async function assertMeshAudio(clients: readonly VoiceClient[], label: string): Promise { + for (const client of clients) { + try { + await waitForAllPeerAudioFlow(client.page, 1, 60_000); + } catch (error) { + console.log(`[${client.displayName} ${label} RTC]\n${await dumpRtcDiagnostics(client.page)}`); + throw error; + } + } +} diff --git a/e2e/tests/voice/long-session-voice-soak.spec.ts b/e2e/tests/voice/long-session-voice-soak.spec.ts new file mode 100644 index 0000000..31295ec --- /dev/null +++ b/e2e/tests/voice/long-session-voice-soak.spec.ts @@ -0,0 +1,151 @@ +import { expect, type Page } from '@playwright/test'; +import { test } from '../../fixtures/multi-client'; +import { countCreatedPeerConnections } from '../../helpers/peer-role'; +import { createVoicePairInNewServer, type VoicePairClient } from '../../helpers/voice-session'; +import { + dumpRtcDiagnostics, + getOpenDataChannelCount, + getPerPeerAudioStats +} from '../../helpers/webrtc-helpers'; + +type PeerAudioStats = Awaited>; + +/** Override for a quick check or a longer leak hunt: `SOAK_MINUTES=2 npx playwright test ...`. */ +const SOAK_MINUTES = Number(process.env['SOAK_MINUTES'] ?? 30); +const SAMPLE_INTERVAL_MS = 30_000; + +interface ResourceSnapshot { + audioElements: number; + heapMb: number; + remoteTracks: number; +} + +/** + * A long call must not accumulate anything. Structural counters are the honest leak + * signal here - a churning recovery loop shows up as extra remote tracks or audio + * elements long before heap bytes say anything conclusive. + */ +async function readResources(page: Page): Promise { + return page.evaluate(() => { + interface HeapCapablePerformance extends Performance { + memory?: { usedJSHeapSize: number }; + } + + const usedHeap = (performance as HeapCapablePerformance).memory?.usedJSHeapSize ?? 0; + const remoteTracks = (window as unknown as { __rtcRemoteTracks?: unknown[] }).__rtcRemoteTracks ?? []; + + return { + audioElements: document.querySelectorAll('audio').length, + heapMb: Math.round(usedHeap / (1_024 * 1_024)), + remoteTracks: remoteTracks.length + }; + }); +} + +function describeStats(stats: PeerAudioStats): string { + return stats + .map((stat) => `${stat.connectionState} in=${stat.inboundPackets} out=${stat.outboundPackets}`) + .join(' | ') || 'no peers'; +} + +test.describe('Long voice session', () => { + test(`carries audio for ${SOAK_MINUTES} minutes without stalling, rebuilding, or accumulating`, async ({ + createClient + }) => { + const soakMs = SOAK_MINUTES * 60_000; + + test.setTimeout(soakMs + 300_000); + + const clients = await createVoicePairInNewServer(createClient, `Voice Soak ${Date.now()}`, { + namePrefix: 'Soak Voice' + }); + const baselineConnections = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page))); + + expect(baselineConnections, 'each client should start with exactly one peer connection').toEqual([1, 1]); + + const baselineResources = await Promise.all(clients.map((client) => readResources(client.page))); + const previousStats: PeerAudioStats[] = await Promise.all( + clients.map((client) => getPerPeerAudioStats(client.page)) + ); + const deadline = Date.now() + soakMs; + const startedAt = Date.now(); + + let sampleIndex = 0; + + while (Date.now() < deadline) { + await clients[0].page.waitForTimeout(SAMPLE_INTERVAL_MS); + sampleIndex++; + + const elapsedSeconds = Math.round((Date.now() - startedAt) / 1_000); + + for (let index = 0; index < clients.length; index++) { + const client = clients[index]; + + await assertClientStillHealthy(client, previousStats[index], elapsedSeconds); + previousStats[index] = await getPerPeerAudioStats(client.page); + } + + const resources = await Promise.all(clients.map((current) => readResources(current.page))); + + console.log( + `[soak] sample ${sampleIndex} at +${elapsedSeconds}s: ` + + clients + .map((client, index) => `${client.displayName} heap=${resources[index].heapMb}MB` + + ` audio=${resources[index].audioElements} tracks=${resources[index].remoteTracks}`) + .join(', ') + ); + } + + await test.step('Nothing accumulated over the session', async () => { + const finalResources = await Promise.all(clients.map((client) => readResources(client.page))); + + for (let index = 0; index < clients.length; index++) { + const baseline = baselineResources[index]; + const final = finalResources[index]; + const label = clients[index].displayName; + + // A stable call fires `track` once per remote track; repeats mean the media path + // was torn down and rebuilt behind the assertions above. + expect(final.remoteTracks, `${label} gained remote tracks during the soak`).toBe(baseline.remoteTracks); + expect(final.audioElements, `${label} accumulated audio elements`).toBeLessThanOrEqual(baseline.audioElements + 1); + expect( + final.heapMb, + `${label} heap grew from ${baseline.heapMb}MB to ${final.heapMb}MB` + ).toBeLessThan(baseline.heapMb * 3 + 200); + } + }); + }); +}); + +async function assertClientStillHealthy( + client: VoicePairClient, + previous: PeerAudioStats, + elapsedSeconds: number +): Promise { + const label = `${client.displayName} at +${elapsedSeconds}s`; + + try { + const current = await getPerPeerAudioStats(client.page); + const connected = current.filter((stat) => stat.connectionState === 'connected'); + + expect(connected, `${label}: expected exactly one connected peer, saw ${describeStats(current)}`).toHaveLength(1); + + const before = previous[0]; + const now = current[0]; + + expect(now.inboundPackets, `${label}: inbound audio stalled`).toBeGreaterThan(before.inboundPackets); + expect(now.outboundPackets, `${label}: outbound audio stalled`).toBeGreaterThan(before.outboundPackets); + + // A rebuild would restore audio within a sample or two, so the flow assertions above + // cannot see it. Only the creation count can. + expect( + await countCreatedPeerConnections(client.page), + `${label}: the peer connection was rebuilt mid-call` + ).toBe(1); + + expect(await getOpenDataChannelCount(client.page), `${label}: the control channel is not open`).toBe(1); + } catch (error) { + console.log(`[soak] ${label} diagnostics:\n${await dumpRtcDiagnostics(client.page)}`); + throw error; + } +} diff --git a/e2e/tests/voice/roster-loss-preserves-voice.spec.ts b/e2e/tests/voice/roster-loss-preserves-voice.spec.ts new file mode 100644 index 0000000..a736703 --- /dev/null +++ b/e2e/tests/voice/roster-loss-preserves-voice.spec.ts @@ -0,0 +1,252 @@ +import { type Page } from '@playwright/test'; +import { test } from '../../fixtures/multi-client'; +import { createVoicePairInNewServer, type VoicePairClient } from '../../helpers/voice-session'; +import { + dumpRtcDiagnostics, + getAudioStatsDelta, + waitForConnectedPeerCount, + waitForOpenDataChannelCount +} from '../../helpers/webrtc-helpers'; + +/** + * The signal server pings every 30s and gives up on a socket 45s after the last pong, + * so it needs up to 75s to declare a client dead and broadcast `user_left`. + */ +const DEAD_SOCKET_HOLD_MS = 95_000; + +/** + * Outgoing voice used to be gated on the observer's roster copy of the remote user's + * voice state, which is signaling gossip. The signal server broadcasts `user_left` for + * any socket it declares dead, so a sleeping laptop, a flaky wifi hop, or a dropped + * socket wiped that copy - and the observer cut its microphone to a peer that never + * left the channel. + */ +test.describe('Losing a peer from the roster must not silence the call', () => { + // The roster wipe is injected directly, because reproducing it through a real outage + // depends on whether the observer notices the dead transport before `user_left` + // arrives - the reducer keeps the voice state while a live peer transport exists. + test('keeps sending to a peer the roster forgot', async ({ createClient }) => { + test.setTimeout(300_000); + + const clients = await createVoicePairInNewServer( + createClient, + `Roster Wipe Voice ${Date.now()}`, + { namePrefix: 'Roster Wipe' } + ); + const [peer, observer] = clients; + + for (const client of clients) { + await assertTwoWayAudio(client, 'before the roster wipe'); + } + + await test.step('The observer is told the peer left the server', async () => { + const wipedUserId = await wipeRemoteVoiceMembersFromRoster(observer.page); + + test.info().annotations.push({ type: 'wiped user', description: wipedUserId }); + await waitForNoRemoteVoiceMembersInRoster(observer.page, 15_000); + }); + + // Nothing about the media plane changed, so the peer must not lose a single second of + // audio. Checking only the end state would hide the cut: the peer keeps sending voice + // heartbeats, so the roster heals itself moments later. + await test.step('The peer never stops receiving the observer microphone', async () => { + await assertUninterruptedInboundAudio(peer, 10); + }); + }); + + /** + * The sleep/wake shape without a suspend: one client loses its signal socket long + * enough for the server to declare it dead, and its peer connections die with it. When + * everything returns the peer re-identifies with no voice state attached, so asking the + * peer over the rebuilt data channel is the only thing that can confirm it is still in + * our channel. + * + * `recovery-preserves-media.spec.ts` cannot reach this: killing the server leaves + * nobody to broadcast `user_left`. + */ + test('restores two-way voice after the server declares one client dead', async ({ createClient }) => { + test.setTimeout(600_000); + + const clients = await createVoicePairInNewServer( + createClient, + `Roster Loss Voice ${Date.now()}`, + { namePrefix: 'Roster Loss' } + ); + const [droppedClient, observer] = clients; + + for (const client of clients) { + await assertTwoWayAudio(client, 'before the outage'); + } + + await test.step('One client loses its signal socket and its peer connections', async () => { + await droppedClient.context.setOffline(true); + await closeTrackedPeerConnections(droppedClient.page); + await observer.page.waitForTimeout(DEAD_SOCKET_HOLD_MS); + }); + + await test.step('Both clients are two-way again once the socket returns', async () => { + await droppedClient.context.setOffline(false); + + for (const client of clients) { + await waitForConnectedPeerCount(client.page, 1, 180_000); + await waitForOpenDataChannelCount(client.page, 1, 180_000); + } + + for (const client of clients) { + await assertTwoWayAudio(client, 'after the socket returned', 90_000); + } + }); + }); +}); + +/** Fail unless the client both sends and receives voice packets within the timeout. */ +async function assertTwoWayAudio( + client: VoicePairClient, + label: string, + timeoutMs = 60_000 +): Promise { + const deadline = Date.now() + timeoutMs; + + let outboundPacketsDelta = 0; + let inboundPacketsDelta = 0; + + while (Date.now() < deadline) { + ({ outboundPacketsDelta, inboundPacketsDelta } = await getAudioStatsDelta(client.page, 3_000)); + + if (outboundPacketsDelta > 0 && inboundPacketsDelta > 0) { + return; + } + } + + throw new Error( + `${client.displayName} is not two-way ${label}: sent ${outboundPacketsDelta}, ` + + `received ${inboundPacketsDelta} packets in the last sample.\n` + + await dumpRtcDiagnostics(client.page) + ); +} + +/** + * Fail if the client goes even one second without receiving voice packets. Peers gossip + * their voice state every 5s, so a torn-down microphone comes back on its own - only a + * continuous sample can tell that the audio never stopped. + */ +async function assertUninterruptedInboundAudio( + client: VoicePairClient, + seconds: number +): Promise { + for (let sample = 1; sample <= seconds; sample++) { + const { inboundPacketsDelta } = await getAudioStatsDelta(client.page, 1_000); + + if (inboundPacketsDelta === 0) { + throw new Error( + `${client.displayName} stopped receiving voice ${sample}s after the roster wipe.\n` + + await dumpRtcDiagnostics(client.page) + ); + } + } +} + +/** Kill the media plane the way a suspend does, leaving the peer to notice on its own. */ +async function closeTrackedPeerConnections(page: Page): Promise { + await page.evaluate(() => { + const connections = (window as { __rtcConnections?: RTCPeerConnection[] }).__rtcConnections ?? []; + + for (const connection of connections) { + connection.close(); + } + }); +} + +/** + * Replay what the signal server does when it declares a socket dead: tell this client the + * remote user left the server, with no live transport recorded. Returns the wiped user id. + */ +async function wipeRemoteVoiceMembersFromRoster(page: Page): Promise { + return page.evaluate(() => { + interface RosterUser { + id?: string; + oderId?: string; + peerId?: string; + voiceState?: { isConnected?: boolean }; + } + interface StoreLike { + dispatch: (action: { type: string } & Record) => void; + } + interface AngularDebugApi { + getComponent: (element: Element) => Record; + } + + const host = document.querySelector('app-rooms-side-panel'); + const debugApi = (window as { ng?: AngularDebugApi }).ng; + + if (!host || !debugApi?.getComponent) { + throw new Error('Angular debug API is unavailable, cannot reach the store'); + } + + const component = debugApi.getComponent(host); + const store = component['store'] as StoreLike | undefined; + const users = (component['onlineUsers'] as (() => RosterUser[]) | undefined)?.() ?? []; + const currentUser = (component['currentUser'] as (() => RosterUser | null) | undefined)?.() ?? null; + const currentRoom = (component['currentRoom'] as (() => { id?: string } | null) | undefined)?.() ?? null; + const remoteVoiceUser = users.find((user) => + user.voiceState?.isConnected === true + && user.id !== currentUser?.id + && user.oderId !== currentUser?.oderId); + + if (!store || !remoteVoiceUser?.id || !currentRoom?.id) { + throw new Error('No remote voice member to wipe from the roster'); + } + + store.dispatch({ + type: '[Users] User Left', + userId: remoteVoiceUser.id, + serverId: currentRoom.id, + connectedPeerIds: [] + }); + + return remoteVoiceUser.id; + }); +} + +/** Wait until no remote user in the client's roster claims to be in voice. */ +async function waitForNoRemoteVoiceMembersInRoster(page: Page, timeout: number): Promise { + await page.waitForFunction( + () => { + interface RosterUser { + id?: string; + oderId?: string; + peerId?: string; + voiceState?: { isConnected?: boolean }; + } + interface AngularDebugApi { + getComponent: (element: Element) => Record; + } + + const host = document.querySelector('app-rooms-side-panel'); + const debugApi = (window as { ng?: AngularDebugApi }).ng; + + if (!host || !debugApi?.getComponent) { + return false; + } + + const component = debugApi.getComponent(host); + const users = (component['onlineUsers'] as (() => RosterUser[]) | undefined)?.() ?? []; + const currentUser = (component['currentUser'] as (() => RosterUser | null) | undefined)?.() ?? null; + const selfIds = new Set([ + currentUser?.id, + currentUser?.oderId, + currentUser?.peerId + ].filter(Boolean)); + + return users + .filter((user) => ![ + user.id, + user.oderId, + user.peerId + ].some((id) => !!id && selfIds.has(id))) + .every((user) => user.voiceState?.isConnected !== true); + }, + undefined, + { timeout } + ); +} diff --git a/e2e/tests/voice/turn-relay-only-voice.spec.ts b/e2e/tests/voice/turn-relay-only-voice.spec.ts new file mode 100644 index 0000000..b5dcfe7 --- /dev/null +++ b/e2e/tests/voice/turn-relay-only-voice.spec.ts @@ -0,0 +1,156 @@ +import { expect } from '@playwright/test'; +import { test, type Client } from '../../fixtures/multi-client'; +import { + forceRelayOnlyIce, + getRelayIceConfigs, + seedTurnOnlyIceServers, + waitForRelayedCandidatePairs, + type TurnCredentials +} from '../../helpers/turn-relay'; +import { + isDockerAvailable, + startTurnServer, + type TurnServerHandle +} from '../../helpers/turn-server'; +import { installDeterministicVoiceSettings } from '../../helpers/voice-session'; +import { + installAutoResumeAudioContext, + installWebRTCTracking, + waitForAllPeerAudioFlow, + waitForAudioStatsPresent, + waitForConnectedPeerCount, + waitForOpenDataChannelCount +} from '../../helpers/webrtc-helpers'; +import { ChatRoomPage } from '../../pages/chat-room.page'; +import { RegisterPage } from '../../pages/register.page'; +import { ServerSearchPage } from '../../pages/server-search.page'; + +const USER_PASSWORD = 'TestPass123!'; +const VOICE_CHANNEL = 'General'; + +/** + * Symmetric NAT gives a browser no usable direct path, so the whole call has to + * ride a TURN relay. `iceTransportPolicy: 'relay'` reproduces that without any + * network trickery: host and server-reflexive candidates are thrown away, and + * only the TURN server the app was configured with is left. + */ +test.describe('Relay-only voice', () => { + let turnServer: TurnServerHandle | null = null; + + test.beforeAll(async () => { + if (!await isDockerAvailable()) { + return; + } + + turnServer = await startTurnServer(); + }); + + test.afterAll(async () => { + await turnServer?.stop(); + turnServer = null; + }); + + test('two users hear each other with every direct path removed', async ({ createClient }) => { + test.skip(!turnServer, 'Relay-only voice needs Docker to run a local coturn.'); + test.setTimeout(240_000); + + const turn = turnServer as TurnServerHandle; + const clients = await createRelayOnlyVoicePair(createClient, turn, `Relay Only Voice ${Date.now()}`); + + await test.step('Both ends settled on a TURN relay, not a direct path', async () => { + for (const client of clients) { + const pairs = await waitForRelayedCandidatePairs(client.page, 1, 60_000); + + // A direct pair here would mean the policy leaked and the test proved nothing. + for (const pair of pairs) { + expect(pair.localCandidateType, 'a peer connection escaped the relay-only policy').toBe('relay'); + } + } + }); + + await test.step('Audio flows both ways through the relay', async () => { + for (const client of clients) { + await waitForAllPeerAudioFlow(client.page, 1, 60_000); + } + }); + }); +}); + +async function createRelayOnlyVoicePair( + createClient: () => Promise, + turn: TurnCredentials, + serverName: string +): Promise { + const clients: Client[] = []; + const credentials: { username: string; displayName: string }[] = []; + + for (let index = 0; index < 2; index++) { + const client = await createClient(); + + await installDeterministicVoiceSettings(client.page); + await installWebRTCTracking(client.page); + await forceRelayOnlyIce(client.page); + await seedTurnOnlyIceServers(client.page, turn); + await installAutoResumeAudioContext(client.page); + + clients.push(client); + credentials.push({ + displayName: `Relay Voice ${index + 1}`, + username: `relay_voice_${Date.now()}_${index + 1}` + }); + } + + await test.step('Register both clients', async () => { + for (const [index, client] of clients.entries()) { + const registerPage = new RegisterPage(client.page); + + await registerPage.goto(); + await registerPage.register( + credentials[index].username, + credentials[index].displayName, + USER_PASSWORD + ); + + await expect(client.page).toHaveURL(/\/dashboard/, { timeout: 20_000 }); + } + }); + + await test.step('Create and join the server', async () => { + await new ServerSearchPage(clients[0].page).createServer(serverName, { + description: 'Relay-only voice test' + }); + + await expect(clients[0].page).toHaveURL(/\/room\//, { timeout: 20_000 }); + + await new ServerSearchPage(clients[1].page).joinServerFromSearch(serverName); + await expect(clients[1].page).toHaveURL(/\/room\//, { timeout: 20_000 }); + }); + + await test.step('Join both clients to voice', async () => { + await new ChatRoomPage(clients[0].page).ensureVoiceChannelExists(VOICE_CHANNEL); + + for (const client of clients) { + const room = new ChatRoomPage(client.page); + + await room.joinVoiceChannel(VOICE_CHANNEL); + await expect(room.voiceControls).toBeVisible({ timeout: 20_000 }); + } + + for (const [index, client] of clients.entries()) { + try { + await waitForConnectedPeerCount(client.page, 1, 90_000); + await waitForOpenDataChannelCount(client.page, 1, 90_000); + await waitForAudioStatsPresent(client.page, 30_000); + } catch (error) { + // No TURN server in the config looks exactly like a failed relay from the + // outside, so show what the app actually handed to WebRTC. + const configs = await getRelayIceConfigs(client.page); + + console.log(`[relay client ${index + 1} ice configs] ${JSON.stringify(configs.slice(0, 3))}`); + throw error; + } + } + }); + + return clients; +} diff --git a/toju-app/public/i18n/catalog/settings.json b/toju-app/public/i18n/catalog/settings.json index 29440fb..a0f16f1 100644 --- a/toju-app/public/i18n/catalog/settings.json +++ b/toju-app/public/i18n/catalog/settings.json @@ -181,7 +181,8 @@ "microphone": "Microphone", "speaker": "Speaker", "microphoneFallback": "Microphone {{index}}", - "speakerFallback": "Speaker {{index}}" + "speakerFallback": "Speaker {{index}}", + "systemDefault": "System default" }, "volume": { "title": "Volume", diff --git a/toju-app/public/i18n/catalog/voice.json b/toju-app/public/i18n/catalog/voice.json index a68cfae..f172358 100644 --- a/toju-app/public/i18n/catalog/voice.json +++ b/toju-app/public/i18n/catalog/voice.json @@ -7,6 +7,10 @@ "retry": "Retry", "failedConnect": "Failed to connect voice session." }, + "devices": { + "inputFellBack": "Your microphone was disconnected. Switched to the system default.", + "outputFellBack": "Your speaker was disconnected. Switched to the system default." + }, "floating": { "backToServer": "Back to {{server}}", "voiceFallback": "Voice", diff --git a/toju-app/src/app/domains/voice-connection/application/facades/voice-connection.facade.ts b/toju-app/src/app/domains/voice-connection/application/facades/voice-connection.facade.ts index 3815c29..a9f0b50 100644 --- a/toju-app/src/app/domains/voice-connection/application/facades/voice-connection.facade.ts +++ b/toju-app/src/app/domains/voice-connection/application/facades/voice-connection.facade.ts @@ -81,6 +81,10 @@ export class VoiceConnectionFacade { await this.realtime.setLocalStream(stream); } + async switchInputDevice(deviceId: string): Promise { + return await this.realtime.switchInputDevice(deviceId); + } + toggleMute(muted?: boolean): void { this.realtime.toggleMute(muted); } @@ -120,4 +124,8 @@ export class VoiceConnectionFacade { syncOutgoingVoiceRouting(allowedPeerIds: string[]): void { this.realtime.syncOutgoingVoiceRouting(allowedPeerIds); } + + mayHearPeerVoice(peerId: string, hasEstablishedPlayback: boolean): boolean { + return this.realtime.mayHearPeerVoice(peerId, hasEstablishedPlayback); + } } diff --git a/toju-app/src/app/domains/voice-connection/application/services/voice-activity.service.ts b/toju-app/src/app/domains/voice-connection/application/services/voice-activity.service.ts index 38b81a4..517869b 100644 --- a/toju-app/src/app/domains/voice-connection/application/services/voice-activity.service.ts +++ b/toju-app/src/app/domains/voice-connection/application/services/voice-activity.service.ts @@ -27,7 +27,7 @@ import { import { Subscription } from 'rxjs'; import { VoiceConnectionFacade } from '../facades/voice-connection.facade'; import { DebuggingService } from '../../../../core/services/debugging.service'; -/* eslint-disable @typescript-eslint/member-ordering, @typescript-eslint/prefer-for-of, max-statements-per-line */ +/* eslint-disable @typescript-eslint/prefer-for-of, max-statements-per-line */ const SPEAKING_THRESHOLD = 0.015; const SILENT_FRAME_GRACE = 8; @@ -50,6 +50,8 @@ export class VoiceActivityService implements OnDestroy { private readonly debugging = inject(DebuggingService); private readonly tracked = new Map(); + /** The id the local microphone is tracked under, so it can follow a device switch. */ + private localMicUserId: string | null = null; private animFrameId: number | null = null; private readonly subs: Subscription[] = []; private readonly _speakingMap = signal>(new Map()); @@ -84,13 +86,30 @@ export class VoiceActivityService implements OnDestroy { } trackLocalMic(userId: string, stream: MediaStream): void { + this.localMicUserId = userId; this.trackStream(userId, stream); } untrackLocalMic(userId: string): void { + if (this.localMicUserId === userId) { + this.localMicUserId = null; + } + this.untrackStream(userId); } + /** + * Point the local speaking indicator at a replacement microphone stream, + * so switching devices mid-call does not leave it watching a dead track. + */ + refreshLocalMicStream(stream: MediaStream): void { + if (!this.localMicUserId) { + return; + } + + this.trackStream(this.localMicUserId, stream); + } + isSpeaking(userId: string): Signal { const entry = this.tracked.get(userId); diff --git a/toju-app/src/app/domains/voice-connection/application/services/voice-playback.service.spec.ts b/toju-app/src/app/domains/voice-connection/application/services/voice-playback.service.spec.ts index 6505bb7..b5f08a8 100644 --- a/toju-app/src/app/domains/voice-connection/application/services/voice-playback.service.spec.ts +++ b/toju-app/src/app/domains/voice-connection/application/services/voice-playback.service.spec.ts @@ -13,10 +13,12 @@ import { VoiceConnectionFacade } from '../facades/voice-connection.facade'; import { VoicePlaybackService } from './voice-playback.service'; let audioContextCount = 0; +let createdGainNodes: { gain: { value: number } }[] = []; describe('VoicePlaybackService', () => { beforeEach(() => { audioContextCount = 0; + createdGainNodes = []; installAudioDomMocks(); installLocalStorageMock(); }); @@ -94,6 +96,27 @@ describe('VoicePlaybackService', () => { expect(audioContextCount).toBe(1); expect(context.service.getUserVolume('peer-1')).toBe(100); }); + + // The roster is gossip. A peer we already receive audio from stays audible unless the + // voice session has positive evidence it left, otherwise a dropped socket silences a + // member who never went anywhere. + it('keeps a peer audible while the voice session holds the path open', () => { + const context = createServiceContext({ isVoiceConnected: true }); + + context.service.handleRemoteStream('peer-1', createMockAudioStream(['track-a']), connectedOptions()); + + expect(context.voiceConnection.mayHearPeerVoice).toHaveBeenCalledWith('peer-1', true); + expect(createdGainNodes.at(-1)?.gain.value).toBe(1); + }); + + it('mutes a peer the voice session reports as gone from our channel', () => { + const context = createServiceContext({ isVoiceConnected: true }); + + context.voiceConnection.mayHearPeerVoice.mockReturnValue(false); + context.service.handleRemoteStream('peer-1', createMockAudioStream(['track-a']), connectedOptions()); + + expect(createdGainNodes.at(-1)?.gain.value).toBe(0); + }); }); interface ServiceContext { @@ -106,6 +129,7 @@ interface ServiceContext { getRemoteVoiceStream: ReturnType; getConnectedPeers: ReturnType; syncOutgoingVoiceRouting: ReturnType; + mayHearPeerVoice: ReturnType; }; remoteStream$: Subject<{ peerId: string; stream: MediaStream }>; } @@ -119,7 +143,8 @@ function createServiceContext(options: { isVoiceConnected?: boolean } = {}): Ser onPeerDisconnected: new Subject(), getRemoteVoiceStream: vi.fn(() => null), getConnectedPeers: vi.fn(() => []), - syncOutgoingVoiceRouting: vi.fn() + syncOutgoingVoiceRouting: vi.fn(), + mayHearPeerVoice: vi.fn(() => true) }; const screenShare = { isScreenShareRemotePlaybackSuppressed: signal(false), @@ -237,10 +262,13 @@ function installAudioDomMocks(): void { } createGain() { - return { + const gainNode = { gain: { value: 0 }, connect: vi.fn() }; + + createdGainNodes.push(gainNode); + return gainNode; } createMediaStreamSource() { diff --git a/toju-app/src/app/domains/voice-connection/application/services/voice-playback.service.ts b/toju-app/src/app/domains/voice-connection/application/services/voice-playback.service.ts index a7fbd83..d2ff2bd 100644 --- a/toju-app/src/app/domains/voice-connection/application/services/voice-playback.service.ts +++ b/toju-app/src/app/domains/voice-connection/application/services/voice-playback.service.ts @@ -7,7 +7,6 @@ import { Store } from '@ngrx/store'; import { STORAGE_KEY_USER_VOLUMES } from '../../../../core/constants'; import { jsonStorage } from '../../../../infrastructure/persistence/json-storage.service'; import { ScreenShareFacade } from '../../../../domains/screen-share'; -import { User } from '../../../../shared-kernel'; import { selectAllUsers, selectCurrentUser } from '../../../../store/users/users.selectors'; import { VoiceConnectionFacade } from '../facades/voice-connection.facade'; @@ -331,7 +330,7 @@ export class VoicePlaybackService { if (!pipeline) return; - if (this.deafened || this.captureEchoSuppressed || this.isUserMuted(peerId) || !this.isPeerInCurrentVoiceRoom(peerId)) { + if (this.deafened || this.captureEchoSuppressed || this.isUserMuted(peerId) || !this.mayHearPeer(peerId)) { pipeline.gainNode.gain.value = 0; return; } @@ -346,22 +345,17 @@ export class VoicePlaybackService { this.peerPipelines.forEach((_pipeline, peerId) => this.applyGain(peerId)); } - private isPeerInCurrentVoiceRoom(peerId: string): boolean { - const localVoiceState = this.currentUser()?.voiceState; - - if (!localVoiceState?.isConnected || !localVoiceState.roomId || !localVoiceState.serverId) { - return false; - } - - const remoteVoiceState = this.findUserForPeer(peerId)?.voiceState; - - return !!remoteVoiceState?.isConnected - && remoteVoiceState.roomId === localVoiceState.roomId - && remoteVoiceState.serverId === localVoiceState.serverId; - } - - private findUserForPeer(peerId: string): User | undefined { - return this.allUsers().find((user) => user.id === peerId || user.oderId === peerId || user.peerId === peerId); + /** + * Whether this peer's audio may be audible. + * + * The roster is gossip: the signal server broadcasts `user_left` for any socket it + * declares dead, which used to mute a peer still sitting in our channel. Playback + * therefore asks the voice session, which weighs the roster against what the peer + * itself reported over its data channel, and holds a stream we already receive when + * neither confirms nor denies. + */ + private mayHearPeer(peerId: string): boolean { + return this.voiceConnection.mayHearPeerVoice(peerId, this.rawRemoteStreams.has(peerId)); } private syncOutgoingVoiceRouting(): void { diff --git a/toju-app/src/app/domains/voice-session/README.md b/toju-app/src/app/domains/voice-session/README.md index ff08fcd..b0f050d 100644 --- a/toju-app/src/app/domains/voice-session/README.md +++ b/toju-app/src/app/domains/voice-session/README.md @@ -12,10 +12,14 @@ voice-session/ │ ├── facades/ │ │ └── voice-session.facade.ts Tracks active voice session, drives floating controls │ └── services/ +│ ├── voice-audio-device.service.ts Microphone/speaker selection, live apply, devicechange fallback │ └── voice-workspace.service.ts Workspace mode (hidden/expanded/minimized), focused stream, mini-window position │ ├── domain/ │ ├── logic/ +│ │ ├── audio-device-selection.rules.ts Device fallback decisions + getUserMedia constraints +│ │ ├── stream-indicator.rules.ts Whether a user's LIVE indicator shows +│ │ ├── voice-path-routing.rules.ts Whether a voice path with a peer may carry audio │ │ └── voice-session.logic.ts isViewingVoiceSessionServer, buildVoiceSessionRoom │ └── models/ │ └── voice-session.model.ts VoiceSessionInfo interface @@ -92,6 +96,12 @@ Each install has a stable `clientInstanceId` (`ClientInstanceService`). `VoiceSt Rules live in `domain/logic/client-voice-session.rules.ts`. +Every media path to a peer — the microphone we send, the camera we send, and the audio we play back — is decided per peer by `decideVoicePathRouting` (`domain/logic/voice-path-routing.rules.ts`). `MediaManager.syncVoiceRouting()` and `syncCameraRouting()` call it on every routing pass, `mayHearPeerVoice()` reuses it for playback gain, and `mayOpenVoicePathToPeer()` answers the same question for a peer connection being built, so `createPeerConnection` cannot put the microphone in a first offer that routing would refuse. A peer therefore cannot be audible while muted-by-routing, or keep our camera after losing our microphone, or start receiving either one because a connection happened to be created while we were in voice. + +Presence in the roster or the peer's own `voice-state` message opens a path; only positive evidence closes one — we left voice, the peer said it is not in our channel, or the peer connection is closed. Everything else is `hold`: an already-negotiated path stays up. A peer missing from the roster is not evidence it left voice, because the signal server broadcasts `user_left` for any socket it declares dead; treating that as a departure used to detach the microphone from a peer still sitting in the channel, leaving one side of the call permanently silent. A path that was never negotiated still stays closed until something confirms the peer, so a guess can never start sending the microphone. When the peer and the roster disagree, the peer's own report wins unless it is older than the roster claim. + +Regression cover: `voice-path-routing.rules.spec.ts`, the routing, first-offer, and camera cases in `media.manager.spec.ts`, the gate case in `create-peer-connection.spec.ts`, and `e2e/tests/voice/roster-loss-preserves-voice.spec.ts`. + Remote voice playback is scoped to the active voice channel, not the whole server. Users stay connected to the shared peer mesh for text, presence, and screen-share control, but voice transport and playback only stay active for peers whose `voiceState.roomId` and `voiceState.serverId` match the local user's current voice session. Owners and admins can also move connected users between voice channels from the room sidebar by dragging a user onto a different voice channel. The moved client updates its local heartbeat and voice-session metadata to the new channel, so routing, floating controls, and occupancy stay in sync after the move. @@ -112,6 +122,8 @@ stateDiagram-v2 Minimized --> Hidden: voice session ends ``` +A user's LIVE indicator is decided by `shouldShowStreamIndicator` (`domain/logic/stream-indicator.rules.ts`) from the observed user's state alone — whether they are in a voice channel, what they announced over the peer plane, and any live track. The observer's own voice session is deliberately not part of that decision, so someone sharing alone is visible to everyone in the server; clicking the badge from outside the channel joins that channel first and then focuses the stream. + The minimized mode renders a draggable mini-window. Its position is tracked in `miniWindowPosition` and clamped to viewport bounds on resize. `focusedStreamId` controls which live stream gets the widescreen treatment in expanded mode, using feature-level stream IDs such as `screen:` or `camera:`. ## Voice settings @@ -132,3 +144,12 @@ Settings are stored in localStorage under a single JSON key. All values are vali | includeSystemAudio | `false` | boolean | `loadVoiceSettingsFromStorage()` and `saveVoiceSettingsToStorage(patch)` are the only entry points. The save function merges the patch with the current stored value so callers only need to pass changed fields. + +## Audio devices: one owner, applied live + +`VoiceAudioDeviceService` owns `inputDevice` and `outputDevice`. Every surface that offers a picker — the settings modal today — calls it instead of writing storage itself, so a picker cannot be live in one place and inert in another. + +- **Changing the microphone never leaves voice.** The service calls `VoiceConnectionFacade.switchInputDevice()`, which re-captures the mic and swaps the track into the existing peer senders via `replaceTrack`. There is no disconnect, no rejoin broadcast, and no SDP renegotiation, so remote peers keep the same audio track. Proven by `e2e/tests/voice/live-input-device-change.spec.ts`. +- **Changing the speaker** re-applies the sink to every live playback pipeline (`VoicePlaybackService.applyOutputDevice`). +- **A device disappearing** falls back to the system default and sets `deviceNotice` (a translation key) for the UI. An empty device list is treated as missing evidence, never as an unplugged device: browsers report no devices before the microphone permission is granted, and Firefox never enumerates audio outputs. +- **Mute and deafen are not owned here.** `MediaManager` holds them and the state reaches the UI through `VoiceConnectionFacade.isMuted` / `isDeafened`. Components must read those signals rather than keeping a local copy, otherwise two control surfaces disagree after a swap or a rejoin. diff --git a/toju-app/src/app/domains/voice-session/application/services/voice-audio-device.service.spec.ts b/toju-app/src/app/domains/voice-session/application/services/voice-audio-device.service.spec.ts new file mode 100644 index 0000000..fd9b029 --- /dev/null +++ b/toju-app/src/app/domains/voice-session/application/services/voice-audio-device.service.spec.ts @@ -0,0 +1,198 @@ +import { Injector, runInInjectionContext } from '@angular/core'; +import { + afterEach, + beforeEach, + describe, + expect, + it, + vi +} from 'vitest'; +import { VoiceAudioDeviceService } from './voice-audio-device.service'; +import { + VoiceActivityService, + VoiceConnectionFacade, + VoicePlaybackService +} from '../../../voice-connection'; +import { saveVoiceSettingsToStorage, loadVoiceSettingsFromStorage } from '../../infrastructure/util/voice-settings-storage.util'; + +const BUILT_IN_MIC = { kind: 'audioinput', deviceId: 'default', label: 'Built-in Mic' }; +const HEADSET_MIC = { kind: 'audioinput', deviceId: 'headset', label: 'Headset' }; +const SPEAKERS = { kind: 'audiooutput', deviceId: 'speakers', label: 'Speakers' }; +const HDMI_OUTPUT = { kind: 'audiooutput', deviceId: 'hdmi', label: 'HDMI Output' }; + +interface DeviceHarness { + service: VoiceAudioDeviceService; + voiceConnection: { + isVoiceConnected: ReturnType; + switchInputDevice: ReturnType; + }; + voicePlayback: { applyOutputDevice: ReturnType }; + voiceActivity: { refreshLocalMicStream: ReturnType }; + emitDeviceChange: () => Promise; + setDevices: (devices: { kind: string; deviceId: string; label: string }[]) => void; +} + +describe('VoiceAudioDeviceService', () => { + let harness: DeviceHarness; + + beforeEach(() => { + harness = createHarness(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('lists the microphones and speakers the browser reports', async () => { + await harness.service.refreshDevices(); + + expect(harness.service.inputDevices().map((device) => device.deviceId)).toEqual(['default', 'headset']); + expect(harness.service.outputDevices().map((device) => device.deviceId)).toEqual(['speakers']); + }); + + it('switches the live microphone instead of rejoining voice', async () => { + harness.voiceConnection.isVoiceConnected.mockReturnValue(true); + + await harness.service.selectInputDevice('headset'); + + expect(harness.voiceConnection.switchInputDevice).toHaveBeenCalledWith('headset'); + expect(harness.service.selectedInputDeviceId()).toBe('headset'); + expect(loadVoiceSettingsFromStorage().inputDevice).toBe('headset'); + }); + + it('keeps the local speaking indicator on the new microphone stream', async () => { + const replacementStream = { id: 'replacement' } as MediaStream; + + harness.voiceConnection.isVoiceConnected.mockReturnValue(true); + harness.voiceConnection.switchInputDevice.mockResolvedValue(replacementStream); + + await harness.service.selectInputDevice('headset'); + + expect(harness.voiceActivity.refreshLocalMicStream).toHaveBeenCalledWith(replacementStream); + }); + + it('applies a speaker choice to the live playback pipelines', async () => { + await harness.service.selectOutputDevice('speakers'); + + expect(harness.voicePlayback.applyOutputDevice).toHaveBeenCalledWith('speakers'); + expect(loadVoiceSettingsFromStorage().outputDevice).toBe('speakers'); + }); + + it('falls back to the system default when the selected microphone is unplugged', async () => { + harness.voiceConnection.isVoiceConnected.mockReturnValue(true); + + await harness.service.selectInputDevice('headset'); + harness.voiceConnection.switchInputDevice.mockClear(); + + harness.setDevices([BUILT_IN_MIC, SPEAKERS]); + + await harness.emitDeviceChange(); + + expect(harness.voiceConnection.switchInputDevice).toHaveBeenCalledWith(''); + expect(harness.service.selectedInputDeviceId()).toBe(''); + expect(harness.service.deviceNotice()).toBe('voice.devices.inputFellBack'); + }); + + it('falls back to the system default when the selected speaker is unplugged', async () => { + await harness.service.selectOutputDevice('speakers'); + harness.voicePlayback.applyOutputDevice.mockClear(); + + harness.setDevices([BUILT_IN_MIC, HDMI_OUTPUT]); + + await harness.emitDeviceChange(); + + expect(harness.voicePlayback.applyOutputDevice).toHaveBeenCalledWith(''); + expect(harness.service.deviceNotice()).toBe('voice.devices.outputFellBack'); + }); + + it('does not report a speaker fallback when the browser lists no speakers at all', async () => { + await harness.service.selectOutputDevice('speakers'); + harness.voicePlayback.applyOutputDevice.mockClear(); + + harness.setDevices([BUILT_IN_MIC]); + + await harness.emitDeviceChange(); + + expect(harness.voicePlayback.applyOutputDevice).not.toHaveBeenCalled(); + expect(harness.service.deviceNotice()).toBeNull(); + }); + + it('leaves a selection alone when a device list arrives empty', async () => { + harness.voiceConnection.isVoiceConnected.mockReturnValue(true); + + await harness.service.selectInputDevice('headset'); + harness.voiceConnection.switchInputDevice.mockClear(); + + harness.setDevices([]); + + await harness.emitDeviceChange(); + + expect(harness.voiceConnection.switchInputDevice).not.toHaveBeenCalled(); + expect(harness.service.selectedInputDeviceId()).toBe('headset'); + expect(harness.service.deviceNotice()).toBeNull(); + }); + + it('records a device choice made before joining voice without capturing', async () => { + harness.voiceConnection.isVoiceConnected.mockReturnValue(false); + + await harness.service.selectInputDevice('headset'); + + expect(harness.voiceConnection.switchInputDevice).not.toHaveBeenCalled(); + expect(loadVoiceSettingsFromStorage().inputDevice).toBe('headset'); + }); +}); + +function createHarness(): DeviceHarness { + saveVoiceSettingsToStorage({ inputDevice: '', outputDevice: '' }); + + let devices = [ + BUILT_IN_MIC, + HEADSET_MIC, + SPEAKERS + ]; + + const listeners: (() => void)[] = []; + + vi.stubGlobal('navigator', { + mediaDevices: { + enumerateDevices: vi.fn(async () => devices), + addEventListener: vi.fn((type: string, listener: () => void) => { + if (type === 'devicechange') { + listeners.push(listener); + } + }), + removeEventListener: vi.fn() + } + }); + + const voiceConnection = { + isVoiceConnected: vi.fn(() => false), + switchInputDevice: vi.fn(async () => null) + }; + const voicePlayback = { applyOutputDevice: vi.fn() }; + const voiceActivity = { refreshLocalMicStream: vi.fn() }; + const injector = Injector.create({ + providers: [ + { provide: VoiceConnectionFacade, useValue: voiceConnection }, + { provide: VoicePlaybackService, useValue: voicePlayback }, + { provide: VoiceActivityService, useValue: voiceActivity } + ] + }); + const service = runInInjectionContext(injector, () => new VoiceAudioDeviceService()); + + return { + service, + voiceConnection, + voicePlayback, + voiceActivity, + emitDeviceChange: async () => { + listeners.forEach((listener) => listener()); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }, + setDevices: (next) => { + devices = next; + } + }; +} diff --git a/toju-app/src/app/domains/voice-session/application/services/voice-audio-device.service.ts b/toju-app/src/app/domains/voice-session/application/services/voice-audio-device.service.ts new file mode 100644 index 0000000..ccfc12e --- /dev/null +++ b/toju-app/src/app/domains/voice-session/application/services/voice-audio-device.service.ts @@ -0,0 +1,167 @@ +/** + * Owns the audio-device selection for voice: which microphone and speaker the + * user picked, persisting it, and applying it to a live session. + * + * Both the in-channel voice controls and the settings modal go through this + * service, so a picker cannot be a dead control in one surface and live in the + * other, and a device change never costs the user their voice session. + */ +import { + Injectable, + OnDestroy, + computed, + inject, + signal +} from '@angular/core'; +import { + VoiceActivityService, + VoiceConnectionFacade, + VoicePlaybackService +} from '../../../voice-connection'; +import { SYSTEM_DEFAULT_AUDIO_DEVICE_ID, resolveAudioDeviceSelection } from '../../domain/logic/audio-device-selection.rules'; +import { loadVoiceSettingsFromStorage, saveVoiceSettingsToStorage } from '../../infrastructure/util/voice-settings-storage.util'; + +export interface AudioDeviceOption { + deviceId: string; + label: string; +} + +/** Translation keys shown when a device disappears underneath the user. */ +export const INPUT_FALLBACK_NOTICE = 'voice.devices.inputFellBack'; +export const OUTPUT_FALLBACK_NOTICE = 'voice.devices.outputFellBack'; + +@Injectable({ providedIn: 'root' }) +export class VoiceAudioDeviceService implements OnDestroy { + private readonly voiceConnection = inject(VoiceConnectionFacade); + private readonly voicePlayback = inject(VoicePlaybackService); + private readonly voiceActivity = inject(VoiceActivityService); + + private readonly _inputDevices = signal([]); + private readonly _outputDevices = signal([]); + private readonly _selectedInputDeviceId = signal(SYSTEM_DEFAULT_AUDIO_DEVICE_ID); + private readonly _selectedOutputDeviceId = signal(SYSTEM_DEFAULT_AUDIO_DEVICE_ID); + private readonly _deviceNotice = signal(null); + + readonly inputDevices = computed(() => this._inputDevices()); + readonly outputDevices = computed(() => this._outputDevices()); + readonly selectedInputDeviceId = computed(() => this._selectedInputDeviceId()); + readonly selectedOutputDeviceId = computed(() => this._selectedOutputDeviceId()); + /** A translation key when a selected device vanished, otherwise `null`. */ + readonly deviceNotice = computed(() => this._deviceNotice()); + + private readonly handleDeviceChange = (): void => { + void this.refreshDevices({ reconcileSelection: true }); + }; + + constructor() { + const settings = loadVoiceSettingsFromStorage(); + + this._selectedInputDeviceId.set(settings.inputDevice); + this._selectedOutputDeviceId.set(settings.outputDevice); + + navigator.mediaDevices?.addEventListener?.('devicechange', this.handleDeviceChange); + } + + ngOnDestroy(): void { + navigator.mediaDevices?.removeEventListener?.('devicechange', this.handleDeviceChange); + } + + /** Re-read the browser device lists, optionally repairing a stale selection. */ + async refreshDevices(options: { reconcileSelection?: boolean } = {}): Promise { + if (!navigator.mediaDevices?.enumerateDevices) { + return; + } + + let devices: MediaDeviceInfo[]; + + try { + devices = await navigator.mediaDevices.enumerateDevices(); + } catch { + return; + } + + this._inputDevices.set(toDeviceOptions(devices, 'audioinput')); + this._outputDevices.set(toDeviceOptions(devices, 'audiooutput')); + + if (options.reconcileSelection) { + await this.reconcileSelection(); + } + } + + /** Pick a microphone, applying it to a live session when there is one. */ + async selectInputDevice(deviceId: string): Promise { + this._deviceNotice.set(null); + await this.applyInputDevice(deviceId); + } + + /** Pick a speaker, applying it to the live playback pipelines. */ + async selectOutputDevice(deviceId: string): Promise { + this._deviceNotice.set(null); + this.applyOutputDevice(deviceId); + } + + /** Re-apply the stored speaker choice to the live playback pipelines. */ + applySelectedOutputDevice(): void { + this.voicePlayback.applyOutputDevice(this._selectedOutputDeviceId()); + } + + dismissDeviceNotice(): void { + this._deviceNotice.set(null); + } + + private async applyInputDevice(deviceId: string): Promise { + this._selectedInputDeviceId.set(deviceId); + saveVoiceSettingsToStorage({ inputDevice: deviceId }); + + if (!this.voiceConnection.isVoiceConnected()) { + return; + } + + const stream = await this.voiceConnection.switchInputDevice(deviceId); + + if (stream) { + this.voiceActivity.refreshLocalMicStream(stream); + } + } + + private applyOutputDevice(deviceId: string): void { + this._selectedOutputDeviceId.set(deviceId); + saveVoiceSettingsToStorage({ outputDevice: deviceId }); + this.voicePlayback.applyOutputDevice(deviceId); + } + + /** + * Repair selections whose device is gone. An empty device list is treated as + * missing evidence, not as an unplugged device. + */ + private async reconcileSelection(): Promise { + const input = resolveAudioDeviceSelection({ + selectedDeviceId: this._selectedInputDeviceId(), + availableDeviceIds: this._inputDevices().map((device) => device.deviceId) + }); + const output = resolveAudioDeviceSelection({ + selectedDeviceId: this._selectedOutputDeviceId(), + availableDeviceIds: this._outputDevices().map((device) => device.deviceId) + }); + + if (input.didFallBack) { + await this.applyInputDevice(input.deviceId); + this._deviceNotice.set(INPUT_FALLBACK_NOTICE); + } + + if (output.didFallBack) { + this.applyOutputDevice(output.deviceId); + this._deviceNotice.set(OUTPUT_FALLBACK_NOTICE); + } + } +} + +/** + * Devices without an id cannot be requested, and the empty id already means + * "system default" in the picker, so they are dropped. + */ +function toDeviceOptions(devices: MediaDeviceInfo[], kind: MediaDeviceKind): AudioDeviceOption[] { + return devices + .filter((device) => device.kind === kind && !!device.deviceId) + .map((device) => ({ deviceId: device.deviceId, label: device.label })); +} diff --git a/toju-app/src/app/domains/voice-session/domain/logic/audio-device-selection.rules.spec.ts b/toju-app/src/app/domains/voice-session/domain/logic/audio-device-selection.rules.spec.ts new file mode 100644 index 0000000..6685ff6 --- /dev/null +++ b/toju-app/src/app/domains/voice-session/domain/logic/audio-device-selection.rules.spec.ts @@ -0,0 +1,109 @@ +import { + describe, + expect, + it +} from 'vitest'; +import { + SYSTEM_DEFAULT_AUDIO_DEVICE_ID, + buildMicrophoneConstraints, + isDeviceUnavailableError, + resolveAudioDeviceSelection +} from './audio-device-selection.rules'; + +describe('resolveAudioDeviceSelection', () => { + it('keeps a selection that is still present in the device list', () => { + const outcome = resolveAudioDeviceSelection({ + selectedDeviceId: 'headset', + availableDeviceIds: [ + 'default', + 'headset', + 'webcam-mic' + ] + }); + + expect(outcome).toEqual({ deviceId: 'headset', didFallBack: false }); + }); + + it('reports the system default when nothing is selected', () => { + const outcome = resolveAudioDeviceSelection({ + selectedDeviceId: SYSTEM_DEFAULT_AUDIO_DEVICE_ID, + availableDeviceIds: ['default', 'headset'] + }); + + expect(outcome).toEqual({ deviceId: SYSTEM_DEFAULT_AUDIO_DEVICE_ID, didFallBack: false }); + }); + + it('falls back to the system default when the selected device disappeared', () => { + const outcome = resolveAudioDeviceSelection({ + selectedDeviceId: 'headset', + availableDeviceIds: ['default', 'webcam-mic'] + }); + + expect(outcome).toEqual({ deviceId: SYSTEM_DEFAULT_AUDIO_DEVICE_ID, didFallBack: true }); + }); + + it('keeps the selection when the device list is empty, because an empty list is no evidence', () => { + const outcome = resolveAudioDeviceSelection({ + selectedDeviceId: 'headset', + availableDeviceIds: [] + }); + + expect(outcome).toEqual({ deviceId: 'headset', didFallBack: false }); + }); + + it('keeps the selection when the list only holds unlabelled placeholder ids', () => { + const outcome = resolveAudioDeviceSelection({ + selectedDeviceId: 'headset', + availableDeviceIds: ['', ''] + }); + + expect(outcome).toEqual({ deviceId: 'headset', didFallBack: false }); + }); +}); + +describe('buildMicrophoneConstraints', () => { + // A bare id is only a preference: the browser may return the current default + // instead, which makes the device picker look broken on real hardware. + it('requires the chosen device exactly', () => { + const constraints = buildMicrophoneConstraints({ + deviceId: 'headset', + browserNoiseSuppression: false + }); + + expect(constraints.audio).toMatchObject({ deviceId: { exact: 'headset' } }); + expect(constraints.video).toBe(false); + }); + + it('omits deviceId entirely when following the system default', () => { + const constraints = buildMicrophoneConstraints({ + deviceId: SYSTEM_DEFAULT_AUDIO_DEVICE_ID, + browserNoiseSuppression: true + }); + + expect(constraints.audio).not.toHaveProperty('deviceId'); + }); + + it('passes the browser noise-suppression choice through', () => { + expect( + buildMicrophoneConstraints({ deviceId: '', browserNoiseSuppression: true }).audio + ).toMatchObject({ noiseSuppression: true, echoCancellation: true, autoGainControl: true }); + + expect( + buildMicrophoneConstraints({ deviceId: '', browserNoiseSuppression: false }).audio + ).toMatchObject({ noiseSuppression: false }); + }); +}); + +describe('isDeviceUnavailableError', () => { + it('recognises the errors that mean the exact device cannot be used', () => { + expect(isDeviceUnavailableError({ name: 'OverconstrainedError' })).toBe(true); + expect(isDeviceUnavailableError({ name: 'NotFoundError' })).toBe(true); + }); + + // A denied permission must keep failing instead of silently opening another mic. + it('does not treat a denied permission or an unknown failure as a missing device', () => { + expect(isDeviceUnavailableError({ name: 'NotAllowedError' })).toBe(false); + expect(isDeviceUnavailableError(new Error('boom'))).toBe(false); + expect(isDeviceUnavailableError(null)).toBe(false); + }); +}); diff --git a/toju-app/src/app/domains/voice-session/domain/logic/audio-device-selection.rules.ts b/toju-app/src/app/domains/voice-session/domain/logic/audio-device-selection.rules.ts new file mode 100644 index 0000000..c9d9b5c --- /dev/null +++ b/toju-app/src/app/domains/voice-session/domain/logic/audio-device-selection.rules.ts @@ -0,0 +1,91 @@ +/** + * Pure decisions about which audio device to capture from or play to. + * + * The browser device list is unreliable evidence: it is empty before the + * microphone permission is granted and can hold unlabelled placeholder + * entries, so "my device is not in the list" only means the device is gone + * when the list itself is trustworthy. + */ + +/** The stored value that means "follow whatever the operating system picked". */ +export const SYSTEM_DEFAULT_AUDIO_DEVICE_ID = ''; + +export interface AudioDeviceSelectionInput { + /** The user's saved choice, or {@link SYSTEM_DEFAULT_AUDIO_DEVICE_ID}. */ + readonly selectedDeviceId: string; + /** Device ids the browser currently reports for this device kind. */ + readonly availableDeviceIds: readonly string[]; +} + +export interface AudioDeviceSelectionOutcome { + /** The device id to use now. */ + readonly deviceId: string; + /** True only when a real device list proved the saved choice is gone. */ + readonly didFallBack: boolean; +} + +export interface MicrophoneConstraintsInput { + readonly deviceId: string; + /** Whether the browser's own noise suppression should run (off when RNNoise handles it). */ + readonly browserNoiseSuppression: boolean; +} + +/** + * Decide which device to use, falling back to the system default when the + * saved choice is provably gone. + */ +export function resolveAudioDeviceSelection( + input: AudioDeviceSelectionInput +): AudioDeviceSelectionOutcome { + const { selectedDeviceId } = input; + + if (!selectedDeviceId) { + return { deviceId: SYSTEM_DEFAULT_AUDIO_DEVICE_ID, didFallBack: false }; + } + + const knownDeviceIds = input.availableDeviceIds.filter((deviceId) => !!deviceId); + + if (knownDeviceIds.length === 0) { + return { deviceId: selectedDeviceId, didFallBack: false }; + } + + if (knownDeviceIds.includes(selectedDeviceId)) { + return { deviceId: selectedDeviceId, didFallBack: false }; + } + + return { deviceId: SYSTEM_DEFAULT_AUDIO_DEVICE_ID, didFallBack: true }; +} + +/** + * Build the `getUserMedia` constraints for the microphone. + * + * `deviceId` is `exact`, because a bare id is only a preference the browser may + * ignore - against real hardware it can hand back the current default, so the + * picker appears to do nothing. Callers must handle `OverconstrainedError` by + * retrying with the system default (see `isDeviceUnavailableError`). + */ +export function buildMicrophoneConstraints( + input: MicrophoneConstraintsInput +): MediaStreamConstraints { + const audio: MediaTrackConstraints = { + echoCancellation: true, + noiseSuppression: input.browserNoiseSuppression, + autoGainControl: true + }; + + if (input.deviceId) { + audio.deviceId = { exact: input.deviceId }; + } + + return { audio, video: false }; +} + +/** + * Whether a `getUserMedia` rejection means "that exact device is not usable", + * as opposed to a denied permission or a missing API, which must keep failing. + */ +export function isDeviceUnavailableError(error: unknown): boolean { + const name = (error as { name?: string } | null)?.name; + + return name === 'OverconstrainedError' || name === 'NotFoundError'; +} diff --git a/toju-app/src/app/domains/voice-session/domain/logic/voice-path-routing.rules.spec.ts b/toju-app/src/app/domains/voice-session/domain/logic/voice-path-routing.rules.spec.ts new file mode 100644 index 0000000..7a86067 --- /dev/null +++ b/toju-app/src/app/domains/voice-session/domain/logic/voice-path-routing.rules.spec.ts @@ -0,0 +1,160 @@ +import { + describe, + expect, + it +} from 'vitest'; +import { + decideVoicePathRouting, + isSameVoiceChannel, + type VoicePathRoutingInput +} from './voice-path-routing.rules'; + +const OUR_CHANNEL = { roomId: 'voice-1', serverId: 'server-1' }; + +function input(overrides: Partial = {}): VoicePathRoutingInput { + return { + hasLocalVoice: true, + localVoiceChannel: OUR_CHANNEL, + rosterPresenceAt: 1_000, + peerReport: null, + hasEstablishedPath: true, + isPeerConnectionClosed: false, + ...overrides + }; +} + +describe('decideVoicePathRouting', () => { + it('opens the path to a peer the roster lists in our channel', () => { + expect(decideVoicePathRouting(input())).toBe('open'); + }); + + it('opens the path to a peer that reports our channel itself', () => { + const decision = decideVoicePathRouting(input({ + rosterPresenceAt: null, + peerReport: { + at: 2_000, + isConnected: true, + ...OUR_CHANNEL + } + })); + + expect(decision).toBe('open'); + }); + + // The bug this file exists for: a dead socket makes the server broadcast + // `user_left`, which wipes the roster copy of a peer that never left voice. + it('holds an established path when the roster no longer knows the peer', () => { + const decision = decideVoicePathRouting(input({ + rosterPresenceAt: null, + peerReport: null, + hasEstablishedPath: true + })); + + expect(decision).toBe('hold'); + }); + + it('never opens a new path for a peer nothing confirms', () => { + const decision = decideVoicePathRouting(input({ + rosterPresenceAt: null, + peerReport: null, + hasEstablishedPath: false + })); + + expect(decision).toBe('close'); + }); + + it('closes the path when the peer answers that it is not in voice', () => { + const decision = decideVoicePathRouting(input({ + rosterPresenceAt: null, + peerReport: { + at: 2_000, + isConnected: false + } + })); + + expect(decision).toBe('close'); + }); + + it('closes the path when the peer reports another voice channel', () => { + const decision = decideVoicePathRouting(input({ + rosterPresenceAt: null, + peerReport: { + at: 2_000, + isConnected: true, + roomId: 'voice-2', + serverId: 'server-1' + } + })); + + expect(decision).toBe('close'); + }); + + it('lets the peer own word overrule a roster that still lists it', () => { + const decision = decideVoicePathRouting(input({ + rosterPresenceAt: 1_000, + peerReport: { + at: 1_500, + isConnected: false + } + })); + + expect(decision).toBe('close'); + }); + + it('ignores a departure report the roster already superseded', () => { + const decision = decideVoicePathRouting(input({ + rosterPresenceAt: 2_000, + peerReport: { + at: 1_500, + isConnected: false + } + })); + + expect(decision).toBe('open'); + }); + + it('closes every path once we leave voice ourselves', () => { + expect(decideVoicePathRouting(input({ hasLocalVoice: false }))).toBe('close'); + }); + + // Voice starts before the channel is recorded, so an unknown local channel is not + // a reason to cut anything - it just cannot contradict the peer. + it('falls back to the roster while our own channel is still unknown', () => { + expect(decideVoicePathRouting(input({ localVoiceChannel: null }))).toBe('open'); + + const unverifiableReport = decideVoicePathRouting(input({ + localVoiceChannel: null, + rosterPresenceAt: null, + peerReport: { + at: 2_000, + isConnected: true, + roomId: 'voice-9', + serverId: 'server-9' + } + })); + + expect(unverifiableReport).toBe('hold'); + }); + + it('closes the path when the peer connection can no longer carry media', () => { + const decision = decideVoicePathRouting(input({ + isPeerConnectionClosed: true, + peerReport: { + at: 2_000, + isConnected: true, + ...OUR_CHANNEL + } + })); + + expect(decision).toBe('close'); + }); +}); + +describe('isSameVoiceChannel', () => { + it('matches only when both room and server are known and equal', () => { + expect(isSameVoiceChannel(OUR_CHANNEL, { ...OUR_CHANNEL })).toBe(true); + expect(isSameVoiceChannel(OUR_CHANNEL, { roomId: 'voice-2', serverId: 'server-1' })).toBe(false); + expect(isSameVoiceChannel(OUR_CHANNEL, { roomId: 'voice-1' })).toBe(false); + expect(isSameVoiceChannel(null, OUR_CHANNEL)).toBe(false); + }); +}); diff --git a/toju-app/src/app/domains/voice-session/domain/logic/voice-path-routing.rules.ts b/toju-app/src/app/domains/voice-session/domain/logic/voice-path-routing.rules.ts new file mode 100644 index 0000000..f49e2e3 --- /dev/null +++ b/toju-app/src/app/domains/voice-session/domain/logic/voice-path-routing.rules.ts @@ -0,0 +1,112 @@ +/** + * Pure decisions about whether a voice path with one peer may stay open - both the + * microphone we send and the audio we play back. + * + * Voice used to be gated purely on the observer's roster copy of the remote user's + * voice state, which is signaling gossip. When a socket dies the signal server + * broadcasts `user_left`, the roster copy is wiped, and the observer then cut the + * media path to a peer that never left the channel - a silent member with no way + * back through the UI. + * + * Missing gossip about a peer is not evidence the peer left voice, so it must never + * close a negotiated path. The reverse is not symmetric: a path we never opened stays + * closed until something positively confirms the peer is in our channel, so a guess + * can never start sending our microphone to anyone. + * + * The peer's own voice report (data channel `voice-state`) mirrors its local voice + * truth, so it outranks the roster when the two disagree and the report is not older + * than the roster claim. + */ + +/** The voice channel a client is in: a channel of a server, or a direct call. */ +export interface VoiceChannelRef { + readonly roomId?: string; + readonly serverId?: string; +} + +/** What a peer last told us about its own voice membership. */ +export interface PeerVoiceReport extends VoiceChannelRef { + /** When the peer told us, in `Date.now()` milliseconds. */ + readonly at: number; + readonly isConnected: boolean; +} + +export interface VoicePathRoutingInput { + /** We are in voice ourselves. */ + readonly hasLocalVoice: boolean; + /** The voice channel we are in, from our own state - never from gossip. */ + readonly localVoiceChannel: VoiceChannelRef | null; + /** When the roster last started listing this peer in our channel, or `null` when it does not. */ + readonly rosterPresenceAt: number | null; + /** The peer's own last voice report, or `null` when it never sent one. */ + readonly peerReport: PeerVoiceReport | null; + /** A negotiated media path with this peer already exists. */ + readonly hasEstablishedPath: boolean; + /** The peer connection can no longer carry media at all. */ + readonly isPeerConnectionClosed: boolean; +} + +/** + * - `open`: the peer is confirmed in our channel; carry voice. + * - `hold`: nothing confirms or denies, so an established path stays up. + * - `close`: positive evidence that this path must not carry voice. + */ +export type VoicePathRouting = 'open' | 'hold' | 'close'; + +/** Whether two channel references point at the same voice channel. */ +export function isSameVoiceChannel(left: VoiceChannelRef | null, right: VoiceChannelRef | null): boolean { + if (!left?.roomId || !left.serverId || !right?.roomId || !right.serverId) { + return false; + } + + return left.roomId === right.roomId && left.serverId === right.serverId; +} + +/** Decide whether a voice path with this peer may carry audio right now. */ +export function decideVoicePathRouting(input: VoicePathRoutingInput): VoicePathRouting { + const { + hasLocalVoice, + localVoiceChannel, + rosterPresenceAt, + peerReport, + hasEstablishedPath, + isPeerConnectionClosed + } = input; + + // We left voice, or the transport is gone: both are positive evidence. + if (!hasLocalVoice) { + return 'close'; + } + + if (isPeerConnectionClosed) { + return 'close'; + } + + // Our own channel can still be unset while voice starts, and an unknown channel + // cannot contradict anything the peer says about the channel it is in. + const knowsLocalChannel = !!localVoiceChannel?.roomId && !!localVoiceChannel.serverId; + const reportsOurChannel = !!peerReport + && peerReport.isConnected + && knowsLocalChannel + && isSameVoiceChannel(peerReport, localVoiceChannel); + + if (reportsOurChannel) { + return 'open'; + } + + const reportsAway = !!peerReport + && (!peerReport.isConnected || (knowsLocalChannel && !isSameVoiceChannel(peerReport, localVoiceChannel))); + const rosterOutranksReport = rosterPresenceAt !== null + && (!reportsAway || peerReport.at < rosterPresenceAt); + + if (rosterOutranksReport) { + return 'open'; + } + + if (reportsAway) { + return 'close'; + } + + // Nothing confirms and nothing denies. Only an already-negotiated path survives that. + return hasEstablishedPath ? 'hold' : 'close'; +} diff --git a/toju-app/src/app/domains/voice-session/feature/floating-voice-controls/floating-voice-controls.component.ts b/toju-app/src/app/domains/voice-session/feature/floating-voice-controls/floating-voice-controls.component.ts index 496b3d2..f877cf9 100644 --- a/toju-app/src/app/domains/voice-session/feature/floating-voice-controls/floating-voice-controls.component.ts +++ b/toju-app/src/app/domains/voice-session/feature/floating-voice-controls/floating-voice-controls.component.ts @@ -1,4 +1,4 @@ -/* eslint-disable @typescript-eslint/member-ordering, @typescript-eslint/no-unused-vars */ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { Component, inject, @@ -20,6 +20,7 @@ import { } from '@ng-icons/lucide'; import { VoiceSessionFacade } from '../../application/facades/voice-session.facade'; +import { VoiceAudioDeviceService } from '../../application/services/voice-audio-device.service'; import { loadVoiceSettingsFromStorage, saveVoiceSettingsToStorage } from '../../infrastructure/util/voice-settings-storage.util'; import { VoiceConnectionFacade } from '../../../../domains/voice-connection'; import { VoicePlaybackService } from '../../../../domains/voice-connection'; @@ -70,6 +71,7 @@ export class FloatingVoiceControlsComponent implements OnInit { readonly showScreenShareButton = computed(() => !this.viewport.isMobile() && !this.mobilePlatform.isNativeMobile()); private readonly voiceSessionService = inject(VoiceSessionFacade); private readonly voicePlayback = inject(VoicePlaybackService); + private readonly audioDevices = inject(VoiceAudioDeviceService); private readonly store = inject(Store); private readonly appI18n = inject(AppI18nService); @@ -80,29 +82,23 @@ export class FloatingVoiceControlsComponent implements OnInit { voiceSession = this.voiceSessionService.voiceSession; isConnected = computed(() => this.webrtcService.isVoiceConnected()); - isMuted = signal(false); - isDeafened = signal(false); + /** Same media-layer truth the in-channel controls read, so the two never disagree. */ + isMuted = computed(() => this.webrtcService.isMuted()); + isDeafened = computed(() => this.webrtcService.isDeafened()); isScreenSharing = this.screenShareService.isScreenSharing; includeSystemAudio = signal(false); screenShareQuality = signal('balanced'); askScreenShareQuality = signal(true); showScreenShareQualityDialog = signal(false); - /** Sync local mute/deafen state from the WebRTC service on init. */ ngOnInit(): void { - // Sync mute/deafen state from webrtc service - this.isMuted.set(this.webrtcService.isMuted()); - this.isDeafened.set(this.webrtcService.isDeafened()); this.syncScreenShareSettings(); const settings = loadVoiceSettingsFromStorage(); this.voicePlayback.updateOutputVolume(settings.outputVolume / 100); this.voicePlayback.updateDeafened(this.isDeafened()); - - if (settings.outputDevice) { - this.voicePlayback.applyOutputDevice(settings.outputDevice); - } + this.audioDevices.applySelectedOutputDevice(); } backToServerTitle(): string { @@ -118,8 +114,7 @@ export class FloatingVoiceControlsComponent implements OnInit { /** Toggle microphone mute and broadcast the updated voice state. */ toggleMute(): void { - this.isMuted.update((current) => !current); - this.webrtcService.toggleMute(this.isMuted()); + this.webrtcService.toggleMute(!this.isMuted()); // Broadcast mute state change this.webrtcService.broadcastMessage({ @@ -136,13 +131,11 @@ export class FloatingVoiceControlsComponent implements OnInit { /** Toggle deafen state (muting audio output) and broadcast the updated voice state. */ toggleDeafen(): void { - this.isDeafened.update((current) => !current); - this.webrtcService.toggleDeafen(this.isDeafened()); + this.webrtcService.toggleDeafen(!this.isDeafened()); this.voicePlayback.updateDeafened(this.isDeafened()); // When deafening, also mute if (this.isDeafened() && !this.isMuted()) { - this.isMuted.set(true); this.webrtcService.toggleMute(true); } @@ -211,7 +204,7 @@ export class FloatingVoiceControlsComponent implements OnInit { // Disable voice this.webrtcService.disableVoice(); this.voicePlayback.teardownAll(); - this.voicePlayback.updateDeafened(false); + this.voicePlayback.updateDeafened(this.isDeafened()); // Update user voice state in store const user = this.currentUser(); @@ -229,10 +222,6 @@ export class FloatingVoiceControlsComponent implements OnInit { // End voice session this.voiceSessionService.endSession(); - - // Reset local state - this.isMuted.set(false); - this.isDeafened.set(false); } /** Return the CSS classes for the compact control button based on active state. */ diff --git a/toju-app/src/app/domains/voice-session/feature/voice-controls/voice-controls.component.ts b/toju-app/src/app/domains/voice-session/feature/voice-controls/voice-controls.component.ts index a4d0c84..830f2e9 100644 --- a/toju-app/src/app/domains/voice-session/feature/voice-controls/voice-controls.component.ts +++ b/toju-app/src/app/domains/voice-session/feature/voice-controls/voice-controls.component.ts @@ -1,4 +1,4 @@ -/* eslint-disable @typescript-eslint/member-ordering, @typescript-eslint/no-unused-vars, complexity */ +/* eslint-disable @typescript-eslint/no-unused-vars, complexity */ import { Component, ElementRef, @@ -24,6 +24,8 @@ import { } from '@ng-icons/lucide'; import { VoiceSessionFacade } from '../../application/facades/voice-session.facade'; +import { VoiceAudioDeviceService } from '../../application/services/voice-audio-device.service'; +import { buildMicrophoneConstraints } from '../../domain/logic/audio-device-selection.rules'; import { loadVoiceSettingsFromStorage, saveVoiceSettingsToStorage } from '../../infrastructure/util/voice-settings-storage.util'; import { VoiceActivityService, VoiceConnectionFacade } from '../../../../domains/voice-connection'; import { PlaybackOptions, VoicePlaybackService } from '../../../../domains/voice-connection'; @@ -43,11 +45,6 @@ import { } from '../../../../shared'; import { APP_TRANSLATE_IMPORTS, AppI18nService } from '../../../../core/i18n'; -interface AudioDevice { - deviceId: string; - label: string; -} - @Component({ selector: 'app-voice-controls', standalone: true, @@ -79,6 +76,7 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { private readonly webrtcService = inject(VoiceConnectionFacade); private readonly screenShareService = inject(ScreenShareFacade); private readonly voiceSessionService = inject(VoiceSessionFacade); + private readonly audioDevices = inject(VoiceAudioDeviceService); private readonly voiceActivity = inject(VoiceActivityService); private readonly voicePlayback = inject(VoicePlaybackService); private readonly store = inject(Store); @@ -104,8 +102,9 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { return this.appI18n.instant(message); }); - isMuted = signal(false); - isDeafened = signal(false); + /** Mute and deafen are owned by the media layer; every surface reads the same truth. */ + isMuted = computed(() => this.webrtcService.isMuted()); + isDeafened = computed(() => this.webrtcService.isDeafened()); isCameraEnabled = computed(() => this.webrtcService.isCameraEnabled()); isScreenSharing = this.screenShareService.isScreenSharing; showSettings = signal(false); @@ -123,10 +122,6 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { this.profileCard.open(this.hostEl.nativeElement, user, { placement: 'above', editable: true }); } - inputDevices = signal([]); - outputDevices = signal([]); - selectedInputDevice = signal(''); - selectedOutputDevice = signal(''); inputVolume = signal(100); outputVolume = signal(100); audioBitrate = signal(96); @@ -145,7 +140,7 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { }; } async ngOnInit(): Promise { - await this.loadAudioDevices(); + await this.audioDevices.refreshDevices(); // Load persisted voice settings and apply this.loadSettings(); @@ -158,24 +153,6 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { } } - async loadAudioDevices(): Promise { - try { - if (!navigator.mediaDevices?.enumerateDevices) { - return; - } - - const devices = await navigator.mediaDevices.enumerateDevices(); - - this.inputDevices.set( - devices.filter((device) => device.kind === 'audioinput').map((device) => ({ deviceId: device.deviceId, label: device.label })) - ); - - this.outputDevices.set( - devices.filter((device) => device.kind === 'audiooutput').map((device) => ({ deviceId: device.deviceId, label: device.label })) - ); - } catch (_error) {} - } - async connect(): Promise { try { // Require signaling connectivity first @@ -195,13 +172,12 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { return; } - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - deviceId: this.selectedInputDevice() || undefined, - echoCancellation: true, - noiseSuppression: !this.noiseReduction() - } - }); + const stream = await navigator.mediaDevices.getUserMedia( + buildMicrophoneConstraints({ + deviceId: this.audioDevices.selectedInputDeviceId(), + browserNoiseSuppression: !this.noiseReduction() + }) + ); await this.webrtcService.setLocalStream(stream); @@ -305,7 +281,7 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { // Disable voice (stops audio tracks but keeps peer connections open for chat) this.webrtcService.disableVoice(); this.voicePlayback.teardownAll(); - this.voicePlayback.updateDeafened(false); + this.voicePlayback.updateDeafened(this.isDeafened()); const user = this.currentUser(); @@ -333,14 +309,10 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { // End voice session for floating controls this.voiceSessionService.endSession(); - - this.isMuted.set(false); - this.isDeafened.set(false); } toggleMute(): void { - this.isMuted.update((current) => !current); - this.webrtcService.toggleMute(this.isMuted()); + this.webrtcService.toggleMute(!this.isMuted()); // Update local store so the side panel reflects the mute state const user = this.currentUser(); @@ -372,14 +344,11 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { } toggleDeafen(): void { - this.isDeafened.update((current) => !current); - this.webrtcService.toggleDeafen(this.isDeafened()); - + this.webrtcService.toggleDeafen(!this.isDeafened()); this.voicePlayback.updateDeafened(this.isDeafened()); // When deafening, also mute if (this.isDeafened() && !this.isMuted()) { - this.isMuted.set(true); this.webrtcService.toggleMute(true); } @@ -492,28 +461,6 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { this.showSettings.set(false); } - onInputDeviceChange(event: Event): void { - const select = event.target as HTMLSelectElement; - - this.selectedInputDevice.set(select.value); - - // Reconnect with new device if connected - if (this.isConnected()) { - this.disconnect(); - this.connect(); - } - - this.saveSettings(); - } - - onOutputDeviceChange(event: Event): void { - const select = event.target as HTMLSelectElement; - - this.selectedOutputDevice.set(select.value); - this.applyOutputDevice(); - this.saveSettings(); - } - onInputVolumeChange(event: Event): void { const input = event.target as HTMLInputElement; @@ -567,8 +514,6 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { private loadSettings(): void { const settings = loadVoiceSettingsFromStorage(); - this.selectedInputDevice.set(settings.inputDevice); - this.selectedOutputDevice.set(settings.outputDevice); this.inputVolume.set(settings.inputVolume); this.outputVolume.set(settings.outputVolume); this.audioBitrate.set(settings.audioBitrate); @@ -581,8 +526,6 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { private saveSettings(): void { saveVoiceSettingsToStorage({ - inputDevice: this.selectedInputDevice(), - outputDevice: this.selectedOutputDevice(), inputVolume: this.inputVolume(), outputVolume: this.outputVolume(), audioBitrate: this.audioBitrate(), @@ -601,22 +544,13 @@ export class VoiceControlsComponent implements OnInit, OnDestroy { this.webrtcService.setInputVolume(this.inputVolume() / 100); this.webrtcService.setAudioBitrate(this.audioBitrate()); this.webrtcService.setLatencyProfile(this.latencyProfile()); - this.applyOutputDevice(); + this.audioDevices.applySelectedOutputDevice(); // Always sync the desired noise-reduction preference (even before // a mic stream exists - the flag will be honoured on connect). this.webrtcService.toggleNoiseReduction(this.noiseReduction()); } catch {} } - private async applyOutputDevice(): Promise { - const deviceId = this.selectedOutputDevice(); - - if (!deviceId) - return; - - this.voicePlayback.applyOutputDevice(deviceId); - } - private syncScreenShareSettings(): void { const settings = loadVoiceSettingsFromStorage(); diff --git a/toju-app/src/app/domains/voice-session/index.ts b/toju-app/src/app/domains/voice-session/index.ts index b4d0ff6..c7ea238 100644 --- a/toju-app/src/app/domains/voice-session/index.ts +++ b/toju-app/src/app/domains/voice-session/index.ts @@ -1,7 +1,11 @@ export * from './application/facades/voice-session.facade'; +export * from './application/services/voice-audio-device.service'; export * from './application/services/voice-client-takeover.service'; export * from './application/services/voice-workspace.service'; +export * from './domain/logic/audio-device-selection.rules'; export * from './domain/logic/client-voice-session.rules'; +export * from './domain/logic/stream-indicator.rules'; +export * from './domain/logic/voice-path-routing.rules'; export * from './domain/models/voice-session.model'; export * from './infrastructure/util/voice-settings-storage.util'; diff --git a/toju-app/src/app/features/settings/settings-modal/voice-settings/voice-settings.component.html b/toju-app/src/app/features/settings/settings-modal/voice-settings/voice-settings.component.html index b31f72f..708d149 100644 --- a/toju-app/src/app/features/settings/settings-modal/voice-settings/voice-settings.component.html +++ b/toju-app/src/app/features/settings/settings-modal/voice-settings/voice-settings.component.html @@ -6,8 +6,16 @@ name="lucideMic" class="w-5 h-5 text-muted-foreground" /> -

'settings.voice.devices.title' | translate

+

{{ 'settings.voice.devices.title' | translate }}

+ @if (deviceNotice(); as notice) { +

+ {{ notice | translate }} +

+ }