myxelium/Toju
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects 2 Releases 33 Wiki Activity

feat: Add webcam basic support

Browse Source
This commit is contained in:
SocksOnHead
2026-03-30 03:10:44 +02:00
parent 727059fb52
commit b7d4bf20e3
40 changed files with 1042 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

305
toju-app/src/app/infrastructure/realtime/media/media.manager.ts
View File

@@ -1,7 +1,7 @@
/* eslint-disable @typescript-eslint/member-ordering, @typescript-eslint/no-unused-vars,, id-length */
/* eslint-disable @typescript-eslint/member-ordering, @typescript-eslint/no-unused-vars, */
/**
* Manages local voice media: getUserMedia, mute, deafen,
* attaching/detaching audio tracks to peer connections, bitrate tuning,
* Manages local voice and camera media: getUserMedia, mute, deafen,
* attaching/detaching tracks to peer connections, bitrate tuning,
* and optional RNNoise-based noise reduction.
*/
import { Subject } from 'rxjs';
@@ -24,6 +24,7 @@ import {
VOLUME_MAX,
VOICE_HEARTBEAT_INTERVAL_MS,
DEFAULT_DISPLAY_NAME,
P2P_TYPE_CAMERA_STATE,
P2P_TYPE_VOICE_STATE
} from '../realtime.constants';
@@ -40,6 +41,8 @@ export interface MediaManagerCallbacks {
/** Get identify credentials (for broadcasting). */
getIdentifyOderId(): string;
getIdentifyDisplayName(): string;
/** Push the current local camera state back into service-level signals. */
setCameraEnabled?(enabled: boolean): void;
}
export class MediaManager {
@@ -53,6 +56,9 @@ export class MediaManager {
*/
private rawMicStream: MediaStream | null = null;
/** The dedicated local camera stream, always captured without audio. */
private localCameraStream: MediaStream | null = null;
/** Remote audio output volume (0-1). */
private remoteAudioVolume = VOLUME_MAX;
@@ -86,6 +92,7 @@ export class MediaManager {
private isVoiceActive = false;
private isMicMuted = false;
private isSelfDeafened = false;
private isCameraActive = false;
/** Current voice channel room ID (set when joining voice). */
private currentVoiceRoomId: string | undefined;
@@ -118,6 +125,10 @@ export class MediaManager {
getRawMicStream(): MediaStream | null {
return this.rawMicStream;
}
/** Returns the current local camera stream, or `null` if the camera is disabled. */
getLocalCameraStream(): MediaStream | null {
return this.localCameraStream;
}
/** Whether voice is currently active (mic captured). */
getIsVoiceActive(): boolean {
return this.isVoiceActive;
@@ -130,6 +141,10 @@ export class MediaManager {
getIsSelfDeafened(): boolean {
return this.isSelfDeafened;
}
/** Whether the local camera is currently active. */
getIsCameraActive(): boolean {
return this.isCameraActive;
}
/** Current remote audio output volume (normalised 0-1). */
getRemoteAudioVolume(): number {
return this.remoteAudioVolume;
@@ -156,10 +171,12 @@ export class MediaManager {
this.allowedVoicePeerIds = nextAllowed;
this.syncVoiceRouting();
this.syncCameraRouting();
}
refreshVoiceRouting(): void {
this.syncVoiceRouting();
this.syncCameraRouting();
}
/**
@@ -229,6 +246,7 @@ export class MediaManager {
* The peer connections themselves are kept alive.
*/
disableVoice(): void {
this.disableCamera();
this.noiseReduction.disable();
this.teardownInputGain();
@@ -285,6 +303,78 @@ export class MediaManager {
this.voiceConnected$.next();
}
/**
* Request camera access and bind the resulting video track to peers in the
* active voice channel. Audio is explicitly disabled for this capture.
*/
async enableCamera(): Promise<MediaStream> {
if (!this.isVoiceActive) {
throw new Error('Voice must be active before enabling the camera.');
}
try {
this.stopLocalCameraStream();
const mediaConstraints: MediaStreamConstraints = {
audio: false,
video: true
};
this.logger.info('getUserMedia camera constraints', mediaConstraints);
if (!navigator.mediaDevices?.getUserMedia) {
throw new Error(
'navigator.mediaDevices is not available. '
+ 'This requires a secure context (HTTPS or localhost). '
+ 'If accessing from an external device, use HTTPS.'
);
}
const stream = await navigator.mediaDevices.getUserMedia(mediaConstraints);
const cameraTrack = stream.getVideoTracks()[0];
if (!cameraTrack) {
stream.getTracks().forEach((track) => track.stop());
throw new Error('Camera capture did not return a video track.');
}
cameraTrack.onended = () => {
if (this.isCameraActive) {
this.disableCamera();
}
};
this.localCameraStream = stream;
this.isCameraActive = true;
this.callbacks.setCameraEnabled?.(true);
this.logger.attachTrackDiagnostics(cameraTrack, 'localCamera');
this.logger.logStream('localCamera', stream);
this.syncCameraRouting();
this.broadcastCameraState();
return stream;
} catch (error) {
this.logger.error('Failed to get camera media', error);
throw error;
}
}
/** Stop camera capture and remove camera senders from every peer. */
disableCamera(): void {
if (!this.localCameraStream && !this.isCameraActive) {
return;
}
this.stopLocalCameraStream();
this.isCameraActive = false;
this.callbacks.setCameraEnabled?.(false);
this.syncCameraRouting();
this.broadcastCameraState();
}
/**
* Toggle the local microphone mute state.
*
@@ -366,43 +456,41 @@ export class MediaManager {
/**
* Set the output volume for remote audio.
*
* @param volume - Normalised value: 0 = silent, 1 = 100%, up to 2 = 200%.
* @param volume - Normalized value: 0 = silent, 1 = 100%, up to 2 = 200%.
*/
setOutputVolume(volume: number): void {
this.remoteAudioVolume = Math.max(VOLUME_MIN, Math.min(2, volume));
}
/**
* Set the input (microphone) volume.
* Set the input microphone gain.
*
* If a local stream is active the gain node is updated in real time.
* If no stream exists yet the value is stored and applied on connect.
* If a local stream is already active the gain node is updated immediately.
* Otherwise the value is stored and applied the next time voice starts.
*
* @param volume - Normalised 0-1 (0 = silent, 1 = 100%).
* @param volume - Normalized 0-1 value.
*/
setInputVolume(volume: number): void {
this.inputGainVolume = Math.max(0, Math.min(1, volume));
if (this.inputGainNode) {
// Pipeline already exists - just update the gain value
this.inputGainNode.gain.value = this.inputGainVolume;
} else if (this.localMediaStream) {
// Stream is active but gain pipeline hasn't been created yet
return;
}
if (this.localMediaStream) {
this.applyInputGainToCurrentStream();
this.bindLocalTracksToAllPeers();
}
}
/** Get current input gain value (0-1). */
/** Return the current input gain value. */
getInputVolume(): number {
return this.inputGainVolume;
}
/**
* Set the maximum audio bitrate on every active peer's audio sender.
*
* The value is clamped between {@link AUDIO_BITRATE_MIN_BPS} and
* {@link AUDIO_BITRATE_MAX_BPS}.
* Set the maximum audio bitrate on every active peer audio sender.
*
* @param kbps - Target bitrate in kilobits per second.
*/
@@ -413,15 +501,16 @@ export class MediaManager {
);
this.callbacks.getActivePeers().forEach(async (peerData) => {
const sender =
peerData.audioSender ||
peerData.connection.getSenders().find((s) => s.track?.kind === TRACK_KIND_AUDIO);
const sender = peerData.audioSender
|| peerData.connection.getSenders().find((candidate) => candidate.track?.kind === TRACK_KIND_AUDIO);
if (!sender?.track)
if (!sender?.track) {
return;
}
if (peerData.connection.signalingState !== 'stable')
if (peerData.connection.signalingState !== 'stable') {
return;
}
let params: RTCRtpSendParameters;
@@ -447,7 +536,7 @@ export class MediaManager {
/**
* Apply a named latency profile that maps to a predefined bitrate.
*
* @param profile - One of `'low'`, `'balanced'`, or `'high'`.
* @param profile - One of `low`, `balanced`, or `high`.
*/
async setLatencyProfile(profile: LatencyProfile): Promise<void> {
await this.setAudioBitrate(LATENCY_PROFILE_BITRATES[profile]);
@@ -491,59 +580,10 @@ export class MediaManager {
}
}
/**
* Bind local audio/video tracks to all existing peer transceivers.
* Restores transceiver direction to sendrecv if previously set to recvonly
* (which happens when disableVoice calls removeTrack).
*/
/** Bind any active local mic/camera tracks to the current peer set. */
private bindLocalTracksToAllPeers(): void {
const peers = this.callbacks.getActivePeers();
if (!this.localMediaStream)
return;
const localStream = this.localMediaStream;
const localAudioTrack = localStream.getAudioTracks()[0] || null;
const localVideoTrack = localStream.getVideoTracks()[0] || null;
peers.forEach((peerData, peerId) => {
if (localAudioTrack) {
if (this.allowedVoicePeerIds.has(peerId)) {
this.attachVoiceTrackToPeer(peerId, peerData, localStream, localAudioTrack);
} else {
this.detachVoiceTrackFromPeer(peerData);
}
}
if (localVideoTrack) {
const videoTransceiver = this.getOrCreateReusableTransceiver(peerData, TRACK_KIND_VIDEO, {
preferredSender: peerData.videoSender,
excludedSenders: [peerData.screenVideoSender]
});
const videoSender = videoTransceiver.sender;
peerData.videoSender = videoSender;
if (
videoTransceiver &&
(videoTransceiver.direction === TRANSCEIVER_RECV_ONLY ||
videoTransceiver.direction === TRANSCEIVER_INACTIVE)
) {
videoTransceiver.direction = TRANSCEIVER_SEND_RECV;
}
if (typeof videoSender.setStreams === 'function') {
videoSender.setStreams(localStream);
}
videoSender
.replaceTrack(localVideoTrack)
.then(() => this.logger.info('video replaceTrack ok', { peerId }))
.catch((error) => this.logger.error('video replaceTrack failed', error));
}
this.callbacks.renegotiate(peerId);
});
this.syncVoiceRouting();
this.syncCameraRouting();
}
private syncVoiceRouting(): void {
@@ -562,6 +602,22 @@ export class MediaManager {
});
}
private syncCameraRouting(): void {
const peers = this.callbacks.getActivePeers();
const localCameraStream = this.localCameraStream;
const localCameraTrack = localCameraStream?.getVideoTracks()[0] || null;
peers.forEach((peerData, peerId) => {
const didChange = localCameraStream && localCameraTrack && this.allowedVoicePeerIds.has(peerId)
? this.attachCameraTrackToPeer(peerId, peerData, localCameraStream, localCameraTrack)
: this.detachCameraTrackFromPeer(peerData, peerId);
if (didChange) {
void this.callbacks.renegotiate(peerId);
}
});
}
private attachVoiceTrackToPeer(
peerId: string,
peerData: PeerData,
@@ -613,6 +669,78 @@ export class MediaManager {
return true;
}
private attachCameraTrackToPeer(
peerId: string,
peerData: PeerData,
localStream: MediaStream,
localCameraTrack: MediaStreamTrack
): boolean {
const videoTransceiver = this.getOrCreateReusableTransceiver(peerData, TRACK_KIND_VIDEO, {
preferredSender: peerData.videoSender,
excludedSenders: [peerData.screenVideoSender]
});
const videoSender = videoTransceiver.sender;
const needsDirectionRestore = videoTransceiver.direction === TRANSCEIVER_RECV_ONLY
|| videoTransceiver.direction === TRANSCEIVER_INACTIVE;
const needsTrackReplace = videoSender.track !== localCameraTrack;
peerData.videoSender = videoSender;
if (!needsDirectionRestore && !needsTrackReplace) {
return false;
}
if (needsDirectionRestore) {
videoTransceiver.direction = TRANSCEIVER_SEND_RECV;
}
if (typeof videoSender.setStreams === 'function') {
videoSender.setStreams(localStream);
}
if (needsTrackReplace) {
videoSender
.replaceTrack(localCameraTrack)
.then(() => this.logger.info('camera replaceTrack ok', { peerId }))
.catch((error) => this.logger.error('camera replaceTrack failed', error));
}
return true;
}
private detachCameraTrackFromPeer(peerData: PeerData, peerId: string): boolean {
const videoSender = peerData.videoSender
?? peerData.connection.getSenders().find((sender) => sender !== peerData.screenVideoSender && sender.track?.kind === TRACK_KIND_VIDEO);
const videoTransceiver = videoSender
? peerData.connection.getTransceivers().find((transceiver) => transceiver.sender === videoSender)
: undefined;
if (!videoTransceiver) {
return false;
}
peerData.videoSender = videoTransceiver.sender;
const hasTrack = !!videoTransceiver.sender.track;
const needsDirectionReset = videoTransceiver.direction === TRANSCEIVER_SEND_RECV;
if (!hasTrack && !needsDirectionReset) {
return false;
}
if (hasTrack) {
videoTransceiver.sender.replaceTrack(null)
.then(() => this.logger.info('camera replaceTrack cleared', { peerId }))
.catch((error) => this.logger.error('Failed to clear camera sender track', error, { peerId }));
}
if (needsDirectionReset) {
videoTransceiver.direction = TRANSCEIVER_RECV_ONLY;
}
return true;
}
private areSetsEqual(left: Set<string>, right: Set<string>): boolean {
if (left.size !== right.size) {
return false;
@@ -690,6 +818,19 @@ export class MediaManager {
});
}
/** Broadcast the local camera state to all connected peers. */
private broadcastCameraState(): void {
const oderId = this.callbacks.getIdentifyOderId();
const displayName = this.callbacks.getIdentifyDisplayName();
this.callbacks.broadcastMessage({
type: P2P_TYPE_CAMERA_STATE,
oderId,
displayName,
isCameraEnabled: this.isCameraActive
});
}
// -- Input gain helpers --
/**
@@ -764,6 +905,22 @@ export class MediaManager {
this.preGainStream = null;
}
private stopLocalCameraStream(): void {
if (!this.localCameraStream) {
return;
}
this.localCameraStream.getTracks().forEach((track) => {
if (track.kind === TRACK_KIND_VIDEO) {
track.onended = null;
}
track.stop();
});
this.localCameraStream = null;
}
/** Clean up all resources. */
destroy(): void {
this.teardownInputGain();

21
toju-app/src/app/infrastructure/realtime/media/screen-share.manager.ts
View File

@@ -403,18 +403,17 @@ export class ScreenShareManager {
this.logger.attachTrackDiagnostics(screenVideoTrack, `screenVideo:${peerId}`);
let videoSender = peerData.videoSender || peerData.connection.getSenders().find((sender) => sender.track?.kind === TRACK_KIND_VIDEO);
let screenVideoSender = peerData.screenVideoSender;
if (!videoSender) {
if (!screenVideoSender) {
const videoTransceiver = peerData.connection.addTransceiver(TRACK_KIND_VIDEO, {
direction: TRANSCEIVER_SEND_RECV
});
videoSender = videoTransceiver.sender;
peerData.videoSender = videoSender;
screenVideoSender = videoTransceiver.sender;
} else {
const videoTransceiver = peerData.connection.getTransceivers().find(
(transceiver) => transceiver.sender === videoSender
(transceiver) => transceiver.sender === screenVideoSender
);
if (videoTransceiver?.direction === TRANSCEIVER_RECV_ONLY) {
@@ -422,16 +421,16 @@ export class ScreenShareManager {
}
}
peerData.screenVideoSender = videoSender;
peerData.screenVideoSender = screenVideoSender;
if (typeof videoSender.setStreams === 'function') {
videoSender.setStreams(this.activeScreenStream);
if (typeof screenVideoSender.setStreams === 'function') {
screenVideoSender.setStreams(this.activeScreenStream);
}
videoSender.replaceTrack(screenVideoTrack)
screenVideoSender.replaceTrack(screenVideoTrack)
.then(() => {
this.logger.info('screen video replaceTrack ok', { peerId });
void this.applyScreenShareVideoParameters(videoSender, preset, peerId);
void this.applyScreenShareVideoParameters(screenVideoSender, preset, peerId);
})
.catch((error) => this.logger.error('screen video replaceTrack failed', error));
@@ -474,7 +473,7 @@ export class ScreenShareManager {
private detachScreenTracksFromPeer(peerData: PeerData, peerId: string): void {
const transceivers = peerData.connection.getTransceivers();
const videoTransceiver = transceivers.find(
(transceiver) => transceiver.sender === peerData.videoSender || transceiver.sender === peerData.screenVideoSender
(transceiver) => transceiver.sender === peerData.screenVideoSender
);
const screenAudioTransceiver = transceivers.find(
(transceiver) => transceiver.sender === peerData.screenAudioSender