Files
Toju/toju-app/src/app/domains/voice-connection/application/services/voice-playback.service.ts
T
myxelium 92c2f578e2 fix(voice): route media on evidence and switch devices without dropping the call
Outgoing voice was gated on the observer's roster copy of the remote user's
voice state, which is signaling gossip. The signal server broadcasts
`user_left` for any socket it declares dead, so a suspended laptop or a flaky
hop wiped that copy and the observer detached its microphone from a peer that
never left the channel - a silent member with no way back through the UI.

`decideVoicePathRouting` now closes a path only on positive evidence: we left
voice, the peer itself reported another channel or none, or the connection is
gone. Missing gossip holds an established path instead. Opening still needs
confirmation, so a guess never starts sending; the same rule gates playback,
camera video, and the microphone a new connection puts in its first offer.
Peers are also asked for their voice state when a connection or data channel
comes up, so a rebuilt path re-confirms itself.

Alongside it, the microphone can be switched mid-call: capture moves to a
device service and rules, the live track is swapped with `replaceTrack` so
the session is never renegotiated, and the speaking indicator follows the new
stream.
2026-08-14 03:19:29 +02:00

481 lines
14 KiB
TypeScript

import {
Injectable,
effect,
inject
} from '@angular/core';
import { Store } from '@ngrx/store';
import { STORAGE_KEY_USER_VOLUMES } from '../../../../core/constants';
import { jsonStorage } from '../../../../infrastructure/persistence/json-storage.service';
import { ScreenShareFacade } from '../../../../domains/screen-share';
import { selectAllUsers, selectCurrentUser } from '../../../../store/users/users.selectors';
import { VoiceConnectionFacade } from '../facades/voice-connection.facade';
export interface PlaybackOptions {
isConnected: boolean;
outputVolume: number;
isDeafened: boolean;
}
/**
* Per-peer Web Audio pipeline that routes the remote MediaStream
* through a GainNode so volume can be amplified beyond 100% (up to 200%).
*
* Chrome/Electron workaround: a muted HTMLAudioElement is attached to
* the stream first so that `createMediaStreamSource` actually outputs
* audio. The priming element itself is silent; audible output is routed
* through a separate output element fed by
* `GainNode -> MediaStreamDestination` so output-device switching stays
* reliable during Linux screen sharing.
*/
interface PeerAudioPipeline {
audioElement: HTMLAudioElement;
outputElement: HTMLAudioElement;
context: AudioContext;
sourceNodes: MediaStreamAudioSourceNode[];
gainNode: GainNode;
}
@Injectable({ providedIn: 'root' })
export class VoicePlaybackService {
private readonly store = inject(Store);
private readonly jsonStorage = jsonStorage;
private readonly voiceConnection = inject(VoiceConnectionFacade);
private readonly screenShare = inject(ScreenShareFacade);
private readonly allUsers = this.store.selectSignal(selectAllUsers);
private readonly currentUser = this.store.selectSignal(selectCurrentUser);
private peerPipelines = new Map<string, PeerAudioPipeline>();
private pendingRemoteStreams = new Map<string, MediaStream>();
private rawRemoteStreams = new Map<string, MediaStream>();
private userVolumes = new Map<string, number>();
private userMuted = new Map<string, boolean>();
private preferredOutputDeviceId = 'default';
private temporaryOutputDeviceId: string | null = null;
private masterVolume = 1;
private deafened = false;
private captureEchoSuppressed = false;
constructor() {
this.loadPersistedVolumes();
effect(() => {
this.captureEchoSuppressed = this.screenShare.isScreenShareRemotePlaybackSuppressed();
this.recalcAllGains();
});
effect(() => {
this.temporaryOutputDeviceId = this.screenShare.forceDefaultRemotePlaybackOutput()
? 'default'
: null;
void this.applyEffectiveOutputDeviceToAllPipelines();
});
effect(() => {
this.syncOutgoingVoiceRouting();
this.recalcAllGains();
});
this.voiceConnection.onRemoteStream.subscribe(({ peerId }) => {
const voiceStream = this.voiceConnection.getRemoteVoiceStream(peerId);
if (!voiceStream) {
this.removeRemoteAudio(peerId);
return;
}
this.handleRemoteStream(peerId, voiceStream, this.buildPlaybackOptions());
});
this.voiceConnection.onVoiceConnected.subscribe(() => {
const options = this.buildPlaybackOptions(true);
this.playPendingStreams(options);
this.ensureAllRemoteStreamsPlaying(options);
});
this.voiceConnection.onPeerDisconnected.subscribe((peerId) => {
this.removeRemoteAudio(peerId);
});
}
handleRemoteStream(peerId: string, stream: MediaStream, options: PlaybackOptions): void {
if (!options.isConnected) {
this.pendingRemoteStreams.set(peerId, stream);
return;
}
if (!this.hasAudio(stream)) {
this.rawRemoteStreams.delete(peerId);
this.removePipeline(peerId);
return;
}
this.masterVolume = options.outputVolume;
this.deafened = options.isDeafened;
if (this.shouldReusePipeline(peerId, stream)) {
this.rawRemoteStreams.set(peerId, stream);
this.applyGain(peerId);
return;
}
this.removePipeline(peerId);
this.rawRemoteStreams.set(peerId, stream);
this.createPipeline(peerId, stream);
}
removeRemoteAudio(peerId: string): void {
this.pendingRemoteStreams.delete(peerId);
this.rawRemoteStreams.delete(peerId);
this.removePipeline(peerId);
}
playPendingStreams(options: PlaybackOptions): void {
if (!options.isConnected)
return;
this.pendingRemoteStreams.forEach((stream, peerId) => this.handleRemoteStream(peerId, stream, options));
this.pendingRemoteStreams.clear();
}
ensureAllRemoteStreamsPlaying(options: PlaybackOptions): void {
if (!options.isConnected)
return;
const peers = this.voiceConnection.getConnectedPeers();
for (const peerId of peers) {
const stream = this.voiceConnection.getRemoteVoiceStream(peerId);
if (!stream || !this.hasAudio(stream)) {
continue;
}
if (this.shouldReusePipeline(peerId, stream)) {
this.masterVolume = options.outputVolume;
this.deafened = options.isDeafened;
this.rawRemoteStreams.set(peerId, stream);
this.applyGain(peerId);
continue;
}
this.handleRemoteStream(peerId, stream, options);
}
}
updateOutputVolume(volume: number): void {
this.masterVolume = volume;
this.recalcAllGains();
}
updateDeafened(isDeafened: boolean): void {
this.deafened = isDeafened;
this.recalcAllGains();
}
getUserVolume(peerId: string): number {
return this.userVolumes.get(peerId) ?? 100;
}
setUserVolume(peerId: string, volume: number): void {
const clamped = Math.max(0, Math.min(200, volume));
this.userVolumes.set(peerId, clamped);
this.applyGain(peerId);
this.persistVolumes();
}
isUserMuted(peerId: string): boolean {
return this.userMuted.get(peerId) ?? false;
}
setUserMuted(peerId: string, muted: boolean): void {
this.userMuted.set(peerId, muted);
this.applyGain(peerId);
this.persistVolumes();
}
applyOutputDevice(deviceId: string): void {
this.preferredOutputDeviceId = deviceId || 'default';
void this.applyEffectiveOutputDeviceToAllPipelines();
}
teardownAll(): void {
this.peerPipelines.forEach((_pipeline, peerId) => this.removePipeline(peerId));
this.peerPipelines.clear();
this.rawRemoteStreams.clear();
this.pendingRemoteStreams.clear();
}
private buildPlaybackOptions(forceConnected = this.voiceConnection.isVoiceConnected()): PlaybackOptions {
return {
isConnected: forceConnected,
outputVolume: this.masterVolume,
isDeafened: this.deafened
};
}
/**
* Build the Web Audio graph for a remote peer:
*
* remoteStream
* ↓
* muted <audio> element (Chrome workaround - primes the stream)
* ↓
* MediaStreamSource -> GainNode -> MediaStreamDestination -> output <audio>
*/
private createPipeline(peerId: string, stream: MediaStream): void {
// Chromium/Electron needs a muted <audio> element before Web Audio can read the stream.
const audioEl = new Audio();
const outputEl = new Audio();
const audioTracks = stream.getAudioTracks().filter((track) => track.readyState === 'live');
audioEl.srcObject = stream;
audioEl.muted = true;
audioEl.play().catch(() => {});
const ctx = new AudioContext();
const gainNode = ctx.createGain();
const mediaDestination = ctx.createMediaStreamDestination();
const sourceNodes = audioTracks.map((track) => ctx.createMediaStreamSource(new MediaStream([track])));
sourceNodes.forEach((sourceNode) => sourceNode.connect(gainNode));
gainNode.connect(mediaDestination);
outputEl.srcObject = mediaDestination.stream;
outputEl.muted = false;
outputEl.volume = 1;
outputEl.play().catch(() => {});
const pipeline: PeerAudioPipeline = {
audioElement: audioEl,
outputElement: outputEl,
context: ctx,
sourceNodes,
gainNode
};
this.peerPipelines.set(peerId, pipeline);
this.applyGain(peerId);
void this.applyEffectiveOutputDeviceToPipeline(pipeline);
}
private async applyEffectiveOutputDeviceToAllPipelines(): Promise<void> {
await Promise.all(Array.from(this.peerPipelines.values(), (pipeline) =>
this.applyEffectiveOutputDeviceToPipeline(pipeline)
));
}
private async applyEffectiveOutputDeviceToPipeline(pipeline: PeerAudioPipeline): Promise<void> {
const deviceId = this.getEffectiveOutputDeviceId();
if (!deviceId) {
return;
}
// eslint-disable-next-line
const anyAudio = pipeline.outputElement as any;
const tasks: Promise<unknown>[] = [];
if (typeof anyAudio.setSinkId === 'function') {
tasks.push(anyAudio.setSinkId(deviceId).catch(() => undefined));
}
if (tasks.length > 0) {
await Promise.all(tasks);
}
}
private getEffectiveOutputDeviceId(): string {
return this.temporaryOutputDeviceId ?? this.preferredOutputDeviceId;
}
private removePipeline(peerId: string): void {
const pipeline = this.peerPipelines.get(peerId);
if (!pipeline)
return;
try {
pipeline.gainNode.disconnect();
} catch {
// nodes may already be disconnected
}
pipeline.sourceNodes.forEach((sourceNode) => {
try {
sourceNode.disconnect();
} catch {
// nodes may already be disconnected
}
});
pipeline.audioElement.srcObject = null;
pipeline.audioElement.remove();
pipeline.outputElement.srcObject = null;
pipeline.outputElement.remove();
if (pipeline.context.state !== 'closed') {
pipeline.context.close().catch(() => {});
}
this.peerPipelines.delete(peerId);
}
private applyGain(peerId: string): void {
const pipeline = this.peerPipelines.get(peerId);
if (!pipeline)
return;
if (this.deafened || this.captureEchoSuppressed || this.isUserMuted(peerId) || !this.mayHearPeer(peerId)) {
pipeline.gainNode.gain.value = 0;
return;
}
const userVol = this.getUserVolume(peerId) / 100; // 0.0-2.0
const effective = this.masterVolume * userVol;
pipeline.gainNode.gain.value = effective;
}
private recalcAllGains(): void {
this.peerPipelines.forEach((_pipeline, peerId) => this.applyGain(peerId));
}
/**
* Whether this peer's audio may be audible.
*
* The roster is gossip: the signal server broadcasts `user_left` for any socket it
* declares dead, which used to mute a peer still sitting in our channel. Playback
* therefore asks the voice session, which weighs the roster against what the peer
* itself reported over its data channel, and holds a stream we already receive when
* neither confirms nor denies.
*/
private mayHearPeer(peerId: string): boolean {
return this.voiceConnection.mayHearPeerVoice(peerId, this.rawRemoteStreams.has(peerId));
}
private syncOutgoingVoiceRouting(): void {
const localVoiceState = this.currentUser()?.voiceState;
if (!localVoiceState?.isConnected || !localVoiceState.roomId || !localVoiceState.serverId) {
this.voiceConnection.syncOutgoingVoiceRouting([]);
return;
}
const allowedPeerIds = new Set<string>();
for (const user of this.allUsers()) {
const voiceState = user.voiceState;
if (
!voiceState?.isConnected
|| voiceState.roomId !== localVoiceState.roomId
|| voiceState.serverId !== localVoiceState.serverId
) {
continue;
}
if (user.id) {
allowedPeerIds.add(user.id);
}
if (user.oderId) {
allowedPeerIds.add(user.oderId);
}
if (user.peerId) {
allowedPeerIds.add(user.peerId);
}
}
this.voiceConnection.syncOutgoingVoiceRouting(Array.from(allowedPeerIds));
}
private persistVolumes(): void {
try {
const data: Record<string, { volume: number; muted: boolean }> = {};
this.userVolumes.forEach((vol, id) => {
data[id] = { volume: vol, muted: this.userMuted.get(id) ?? false };
});
// Also persist any muted-only entries
this.userMuted.forEach((muted, id) => {
if (!data[id]) {
data[id] = { volume: 100, muted };
}
});
this.jsonStorage.write(STORAGE_KEY_USER_VOLUMES, data);
} catch {
// storage not available
}
}
private loadPersistedVolumes(): void {
const data = this.jsonStorage.read<Record<string, { volume: number; muted: boolean }> | null>(
STORAGE_KEY_USER_VOLUMES,
null
);
if (!data) {
return;
}
Object.entries(data).forEach(([id, entry]) => {
if (typeof entry.volume === 'number') {
this.userVolumes.set(id, entry.volume);
}
if (entry.muted) {
this.userMuted.set(id, true);
}
});
}
private hasAudio(stream: MediaStream): boolean {
return stream.getAudioTracks().length > 0;
}
/**
* Remote composite-stream notifications (camera, screen share, renegotiation)
* can arrive without any change to the underlying voice audio tracks.
* Rebuilding the Web Audio graph in that case only churns AudioContexts.
*/
private shouldReusePipeline(peerId: string, stream: MediaStream): boolean {
const trackedRaw = this.rawRemoteStreams.get(peerId);
return this.peerPipelines.has(peerId)
&& !!trackedRaw
&& this.streamsShareLiveAudioTracks(trackedRaw, stream);
}
private streamsShareLiveAudioTracks(previous: MediaStream, next: MediaStream): boolean {
const previousTrackIds = this.getLiveAudioTrackIds(previous);
const nextTrackIds = this.getLiveAudioTrackIds(next);
if (previousTrackIds.length === 0 || nextTrackIds.length === 0) {
return false;
}
if (previousTrackIds.length !== nextTrackIds.length) {
return false;
}
const previousIds = new Set(previousTrackIds);
return nextTrackIds.every((trackId) => previousIds.has(trackId));
}
private getLiveAudioTrackIds(stream: MediaStream): string[] {
return stream
.getAudioTracks()
.filter((track) => track.readyState === 'live')
.map((track) => track.id);
}
}