fix(realtime): survive signal outages and per-server identities

Peer recovery burned its whole retry budget while signaling was down, then
dropped the tracker with no re-arm, so a peer stayed dead until an unrelated
roster event healed it. Recovery now waits for a usable transport before
spending an attempt.

Initiator election also compared a home actor id against foreign roster ids,
which is not antisymmetric across signal servers - both sides offered, or
neither did. Election moves into `peer-role.rules` and compares ids only
within one signal server's identity space.
This commit is contained in:
2026-08-14 03:19:29 +02:00
parent f9e8538c80
commit a83f5aa750
14 changed files with 1176 additions and 64 deletions
@@ -0,0 +1,188 @@
import { expect } from '@playwright/test';
import { test, type Client } from '../../fixtures/multi-client';
import { expectDashboardReady } from '../../helpers/dashboard';
import {
countCreatedPeerConnections,
expectExactlyOneInitiatorPerPair,
readPeerRoleEdges,
type PeerRoleEdge
} from '../../helpers/peer-role';
import { installTestServerEndpoints, type SeededEndpointInput } from '../../helpers/seed-test-endpoint';
import { startTestServer } from '../../helpers/test-server';
import {
installDeterministicVoiceSettings,
joinRoomFromSearch,
joinVoiceChannelUntilConnected,
openSavedRoomByName
} from '../../helpers/voice-session';
import { waitForVoiceRosterCount } from '../../helpers/voice-roster';
import {
dumpRtcDiagnostics,
installAutoResumeAudioContext,
installWebRTCTracking,
waitForAllPeerAudioFlow,
waitForAudioStatsPresent,
waitForPeerConnected
} from '../../helpers/webrtc-helpers';
import { ChatRoomPage } from '../../pages/chat-room.page';
import { RegisterPage } from '../../pages/register.page';
import { ServerSearchPage } from '../../pages/server-search.page';
const SIGNAL_A_ID = 'e2e-cross-signal-a';
const SIGNAL_B_ID = 'e2e-cross-signal-b';
const VOICE_CHANNEL = 'General';
const USER_PASSWORD = 'TestPass123!';
const USER_COUNT = 4;
const EXPECTED_REMOTE_PEERS = USER_COUNT - 1;
interface TestUser {
username: string;
displayName: string;
/** Signal server this human registered on - their home identity space. */
homeSignalId: string;
}
type TestClient = Client & { user: TestUser };
test.describe('Cross-signal WebRTC identity', () => {
test.describe.configure({ timeout: 600_000 });
test('elects exactly one initiator per pair when peers have different home signal servers', async ({
createClient,
testServer
}) => {
const signalB = await startTestServer();
try {
const suffix = `cross_signal_${Date.now()}`;
const roomName = `Cross Signal Voice ${suffix}`;
const endpoints: SeededEndpointInput[] = [
{
id: SIGNAL_A_ID,
name: 'E2E Signal A',
url: testServer.url,
isActive: true,
status: 'online'
},
{
id: SIGNAL_B_ID,
name: 'E2E Signal B',
url: signalB.url,
isActive: true,
status: 'online'
}
];
// The room is hosted on signal B. Two humans are at home there and two are
// foreign guests, so most pairs must compare a foreign actor id against a
// foreign actor id - never a home id against one.
const users: TestUser[] = [
{ username: `host_${suffix}`, displayName: 'Cross Host', homeSignalId: SIGNAL_B_ID },
{ username: `native_${suffix}`, displayName: 'Cross Native', homeSignalId: SIGNAL_B_ID },
{ username: `guest_a_${suffix}`, displayName: 'Cross Guest A', homeSignalId: SIGNAL_A_ID },
{ username: `guest_b_${suffix}`, displayName: 'Cross Guest B', homeSignalId: SIGNAL_A_ID }
];
const clients: TestClient[] = [];
for (const user of users) {
const client = await createClient();
await installTestServerEndpoints(client.context, endpoints);
await installDeterministicVoiceSettings(client.page);
await installWebRTCTracking(client.context);
await installAutoResumeAudioContext(client.page);
clients.push({ ...client, user });
}
const [host] = clients;
await test.step('Each human registers on their own home signal server', async () => {
for (const client of clients) {
const register = new RegisterPage(client.page);
await register.goto();
await register.serverSelect.selectOption(client.user.homeSignalId);
await register.register(client.user.username, client.user.displayName, USER_PASSWORD);
await expectDashboardReady(client.page);
}
});
await test.step('The host creates the voice room on signal B', async () => {
await new ServerSearchPage(host.page).createServer(roomName, {
description: 'Cross-signal initiator election coverage',
sourceId: SIGNAL_B_ID
});
await expect(host.page).toHaveURL(/\/room\//, { timeout: 20_000 });
await new ChatRoomPage(host.page).ensureVoiceChannelExists(VOICE_CHANNEL);
});
await test.step('Everyone else joins the room, provisioning a foreign account when needed', async () => {
for (const client of clients.slice(1)) {
await joinRoomFromSearch(client.page, roomName);
}
await openSavedRoomByName(host.page, roomName);
});
// Everyone reconnects at once, so every pair elects its initiator from the same
// roster snapshot. Staggered arrivals let one side's 1s fallback-offer timer
// serialize negotiation, which hides a wrong comparison; a reconnect storm - a
// signal blip, or a channel everyone piles into - does not.
await test.step('All four reconnect simultaneously', async () => {
await Promise.all(clients.map(async (client) => {
await client.page.reload({ waitUntil: 'domcontentloaded' });
await openSavedRoomByName(client.page, roomName);
}));
});
await test.step('All four join the same voice channel simultaneously', async () => {
await Promise.all(clients.map((client) =>
joinVoiceChannelUntilConnected(client.page, VOICE_CHANNEL)
));
for (const client of clients) {
await waitForVoiceRosterCount(client.page, VOICE_CHANNEL, USER_COUNT);
}
});
await test.step('Every pair carries bidirectional audio', async () => {
await Promise.all(clients.map((client) => waitForPeerConnected(client.page, 90_000)));
await Promise.all(clients.map((client) => waitForAudioStatsPresent(client.page, 30_000)));
for (const client of clients) {
try {
await waitForAllPeerAudioFlow(client.page, EXPECTED_REMOTE_PEERS, 120_000);
} catch (error) {
console.log(`[${client.user.displayName} RTC]\n${await dumpRtcDiagnostics(client.page)}`);
throw error;
}
}
});
await test.step('Exactly one side of every pair elected itself initiator', async () => {
const edgesByClient: Record<string, PeerRoleEdge[]> = {};
for (const client of clients) {
edgesByClient[client.user.displayName] = (await readPeerRoleEdges(client.page))
.filter((edge) => edge.connectionState === 'connected');
}
// Comparing a home id against a foreign actor id is not antisymmetric, so both
// peers could offer (glare) or neither could until a takeover timer fired.
expectExactlyOneInitiatorPerPair(edgesByClient);
});
await test.step('No peer had to be rebuilt to reach that state', async () => {
for (const client of clients) {
expect(
await countCreatedPeerConnections(client.page),
`${client.user.displayName} rebuilt a peer connection instead of connecting on the first offer`
).toBe(EXPECTED_REMOTE_PEERS);
}
});
} finally {
await signalB.stop();
}
});
});
@@ -0,0 +1,230 @@
import { expect, type Page } from '@playwright/test';
import { test, type Client } from '../../fixtures/multi-client';
import { countCreatedPeerConnections } from '../../helpers/peer-role';
import { installDeterministicVoiceSettings } from '../../helpers/voice-session';
import {
closeOpenDataChannels,
dumpRtcDiagnostics,
getOpenDataChannelCount,
installAutoResumeAudioContext,
installWebRTCTracking,
waitForAllPeerAudioFlow,
waitForAudioStatsPresent,
waitForConnectedPeerCount,
waitForOpenDataChannelCount
} from '../../helpers/webrtc-helpers';
import { ChatRoomPage } from '../../pages/chat-room.page';
import { RegisterPage } from '../../pages/register.page';
import { ServerSearchPage } from '../../pages/server-search.page';
interface VoiceClient extends Client {
displayName: string;
username: string;
}
const USER_PASSWORD = 'TestPass123!';
const VOICE_CHANNEL = 'General';
/** 12 reconnect attempts at 5s - the whole budget fits inside this outage. */
const OUTAGE_HOLD_MS = 70_000;
test.describe('Recovery preserves live media', () => {
test('replaces a dead control channel without rebuilding the peer connection', async ({ createClient }) => {
test.setTimeout(240_000);
const clients = await createVoicePair(createClient, `DC Soft Replace ${Date.now()}`);
const [alice, bob] = clients;
await assertMeshAudio(clients, 'initial two-user voice');
const connectionsBefore = {
alice: await countCreatedPeerConnections(alice.page),
bob: await countCreatedPeerConnections(bob.page)
};
expect(connectionsBefore.alice).toBe(1);
expect(connectionsBefore.bob).toBe(1);
await test.step('The control channel is replaced on the same connection', async () => {
const closed = await closeOpenDataChannels(alice.page);
expect(closed).toBeGreaterThan(0);
await waitForOpenDataChannelCount(alice.page, 1, 60_000);
await waitForOpenDataChannelCount(bob.page, 1, 60_000);
// A rebuild would construct a second RTCPeerConnection on both sides, taking voice,
// camera, and screen share down with the control channel.
expect(
await countCreatedPeerConnections(alice.page),
'Alice rebuilt her peer connection instead of replacing the control channel'
).toBe(connectionsBefore.alice);
expect(
await countCreatedPeerConnections(bob.page),
'Bob rebuilt his peer connection instead of adopting the replacement control channel'
).toBe(connectionsBefore.bob);
});
await test.step('Audio never had to be renegotiated', async () => {
await waitForConnectedPeerCount(alice.page, 1, 30_000);
await waitForConnectedPeerCount(bob.page, 1, 30_000);
await assertMeshAudio(clients, 'after control-channel replacement');
});
});
// This covers the user-visible half: an outage that outlives the 12-attempt reconnect
// budget must not end the call. It cannot isolate the attempt accounting, because the
// roster resync on signaling reconnect re-peers anyway - `peer-recovery.spec.ts` owns
// the deterministic proof that a deferred attempt costs nothing.
test('keeps voice alive across a signal outage longer than the reconnect budget', async ({
createClient,
testServer
}) => {
test.setTimeout(480_000);
const clients = await createVoicePair(createClient, `Signal Outage Voice ${Date.now()}`);
await assertMeshAudio(clients, 'initial two-user voice');
const connectionsBefore = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
expect(connectionsBefore).toEqual([1, 1]);
await test.step('The signal server goes away for longer than the reconnect budget', async () => {
await testServer.kill();
for (const client of clients) {
await waitForSignalingConnected(client.page, false, 60_000);
}
await clients[0].page.waitForTimeout(OUTAGE_HOLD_MS);
});
await test.step('Peer media is unaffected by the signaling outage', async () => {
await assertMeshAudio(clients, 'during signal outage');
});
await test.step('Voice is still healthy once signaling returns', async () => {
await testServer.start();
for (const client of clients) {
await waitForSignalingConnected(client.page, true, 120_000);
}
for (const client of clients) {
await waitForConnectedPeerCount(client.page, 1, 90_000);
await waitForOpenDataChannelCount(client.page, 1, 90_000);
}
await assertMeshAudio(clients, 'after signaling recovery');
});
await test.step('The call was never rebuilt behind the user back', async () => {
// Media never depended on the signal server, so the roster resync must adopt the
// living peer connection. Rebuilding it would drop audio for a beat and reset
// screen share - invisible to the assertions above, which only re-check the end state.
const connectionsAfter = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
expect(connectionsAfter, 'a client rebuilt its peer connection when signaling came back').toEqual(
connectionsBefore
);
});
});
});
async function createVoicePair(
createClient: () => Promise<Client>,
serverName: string
): Promise<VoiceClient[]> {
const clients: VoiceClient[] = [];
for (let index = 0; index < 2; index++) {
const client = await createClient();
await installDeterministicVoiceSettings(client.page);
await installWebRTCTracking(client.page);
await installAutoResumeAudioContext(client.page);
clients.push({
...client,
displayName: `Recovery Voice ${index + 1}`,
username: `recovery_voice_${Date.now()}_${index + 1}`
});
}
await test.step('Register both clients', async () => {
for (const client of clients) {
const registerPage = new RegisterPage(client.page);
await registerPage.goto();
await registerPage.register(client.username, client.displayName, USER_PASSWORD);
await expect(client.page).toHaveURL(/\/dashboard/, { timeout: 20_000 });
}
});
await test.step('Create and join the server', async () => {
await new ServerSearchPage(clients[0].page).createServer(serverName, {
description: 'Recovery keeps live media test'
});
await expect(clients[0].page).toHaveURL(/\/room\//, { timeout: 20_000 });
await new ServerSearchPage(clients[1].page).joinServerFromSearch(serverName);
await expect(clients[1].page).toHaveURL(/\/room\//, { timeout: 20_000 });
});
await test.step('Join both clients to voice', async () => {
await new ChatRoomPage(clients[0].page).ensureVoiceChannelExists(VOICE_CHANNEL);
for (const client of clients) {
const room = new ChatRoomPage(client.page);
await room.joinVoiceChannel(VOICE_CHANNEL);
await expect(room.voiceControls).toBeVisible({ timeout: 20_000 });
}
for (const client of clients) {
await waitForConnectedPeerCount(client.page, 1, 90_000);
await waitForOpenDataChannelCount(client.page, 1, 90_000);
await waitForAudioStatsPresent(client.page, 30_000);
}
});
return clients;
}
async function assertMeshAudio(clients: readonly VoiceClient[], label: string): Promise<void> {
for (const client of clients) {
try {
await waitForAllPeerAudioFlow(client.page, 1, 60_000);
} catch (error) {
console.log(`[${client.displayName} ${label} data channels] ${await getOpenDataChannelCount(client.page)}`);
console.log(`[${client.displayName} ${label} RTC]\n${await dumpRtcDiagnostics(client.page)}`);
throw error;
}
}
}
/** Wait until the client's own view of its signaling connection matches `connected`. */
async function waitForSignalingConnected(page: Page, connected: boolean, timeout: number): Promise<void> {
await page.waitForFunction(
(expected) => {
interface AngularDebugApi {
getComponent: (element: Element) => Record<string, unknown>;
}
const host = document.querySelector('app-rooms-side-panel');
const debugApi = (window as { ng?: AngularDebugApi }).ng;
if (!host || !debugApi?.getComponent) {
return false;
}
const realtime = debugApi.getComponent(host)['realtime'] as { isConnected?: () => boolean } | undefined;
return realtime?.isConnected?.() === expected;
},
connected,
{ timeout }
);
}
@@ -121,6 +121,8 @@ Each signaling URL gets its own `SignalingManager` (one WebSocket each). `Signal
**Identify-before-join invariant.** The server drops any `join_server` / `view_server` that arrives on a connection that has not yet `identify`-ed, so a join that races ahead of identify is silently lost and the user never appears in the presence roster. On every (re)connect, `SignalingManager.reIdentifyAndRejoin` therefore re-`identify`s and only then re-joins. For this to work the manager's credential lookup (`SignalingTransportHandler.getIdentifyCredentialsForSignalUrl`) must resolve a credential as soon as one exists for that signal URL — it falls back to the credential store when the per-URL identify cache has not been populated yet. Do not narrow that lookup to only the in-memory cache; doing so lets a fresh socket emit a join before any identify and reintroduces the dropped-presence bug.
**Per-signal identity invariant.** One human has a separate account - and therefore a separate actor id - on every signal server they are provisioned on. Roster ids, `fromUserId`s, and peer-map keys are always actor ids *of the signal server they arrived from*, so anything that compares them against a local id must use the local actor id for that same server: `SignalingTransportHandler.getLocalOderIdForSignalUrl` in the signaling handler (which already has the `signalUrl`), and `getIdentifyCredentialsForPeer` in the peer manager (which resolves it through `ServerSignalingCoordinator.getPeerSignalUrl`). Comparing a home id against a foreign actor id is not antisymmetric: each side compares its own home id against the other's foreign id, so both peers can elect themselves initiator (glare) or neither can until the 5s non-initiator takeover fires — the "some users can't hear each other" failure when several people join or reconnect at once. `getLocalOderIdForSignalUrl` deliberately returns `null` instead of falling back to the home id; the election call sites defer and retry, and a wrong-space id is worse than a late one. The same rule covers self-filtering (your own foreign roster entry does not match your home id, so the client would try to peer with itself) and the `oderId` announced in voice/camera/screen-share payloads, which must be the id the receiving peer knows you by. Elect roles only through `peer-role.rules.ts`.
The server side enforces this ordering too: `handleWebSocketMessage` serializes messages per connection (a promise chain keyed by connection id in `server/src/websocket/handler.ts`). `identify` awaits a session-token DB lookup, and `identify` + `join_server` sent back-to-back often arrive in the same TCP segment; without serialization the join was evaluated mid-identify, rejected with `auth_required`, and the room membership was silently lost — the client then never received `user_joined` / `chat_message` broadcasts for that room even though same-account `account_sync` kept working, which is exactly the "chats don't sync for multi-client users" failure mode. Do not process websocket messages for one connection concurrently.
Room affinity is authoritative at this layer as well. The renderer repairs each room's saved `sourceId` / `sourceUrl` from server-directory responses and routes `join_server`, `view_server`, and room-scoped signaling traffic to that room's signaling URL first. If that route fails, alternate endpoints can be tried temporarily, but server-scoped raw messages are no longer broadcast to every connected signaling manager when the route is unknown.
@@ -241,26 +243,39 @@ stateDiagram-v2
Disconnected --> Connected: recovers within 10s
Disconnected --> Failed: grace period expires
Failed --> Reconnecting: schedule reconnect (every 5s)
Reconnecting --> Reconnecting: signaling down - attempt deferred, budget untouched
Reconnecting --> Connected: new offer accepted
Reconnecting --> GaveUp: 12 attempts failed
Reconnecting --> RecoveryFailed: 12 attempts spent
RecoveryFailed --> Reconnecting: signaling reconnected, budget reset
Connected --> Closed: leave / cleanup
GaveUp --> [*]
RecoveryFailed --> [*]
Closed --> [*]
```
When a peer connection enters `disconnected`, a 10-second grace period starts. If it recovers on its own (network blip), nothing happens. If it reaches `failed`, the connection is torn down and a reconnect loop starts. A fresh `RTCPeerConnection` is created every 5 seconds, up to 12 attempts; only the deterministically elected initiator sends a reconnect offer, while the other side waits for that offer.
The 12-attempt budget only pays for attempts that can actually reach the peer. While signaling is disconnected the loop keeps ticking but defers instead of spending an attempt, so a signal-server outage cannot silently exhaust recovery. When the budget really does run out, `schedulePeerReconnect` stops the loop and `peerRecoveryStatus$` emits `{ status: 'failed', attempts }` (exposed as `WebRTCService.onPeerRecoveryStatus`). The tracker entry is kept and marked `recoveryFailed`, so when signaling reconnects the session calls `resumeStalledPeerRecovery()`, which resets the budget, emits `{ status: 'retrying' }`, and re-arms the loop.
## Data channel
A single ordered data channel carries all peer-to-peer messages: chat events, attachment chunks, profile-avatar summary/request/full/chunk events, voice/screen state broadcasts, voice-channel move control events, state requests, pings, and screen share control.
Back-pressure is handled with a high-water mark (4 MB) and low-water mark (1 MB). `sendToPeerBuffered()` waits for the buffer to drain before sending, which matters during file transfers.
`sendToPeer()` returns whether the payload entered an open data channel. A peer can sit in `getConnectedPeers()` - that list is filled when `RTCPeerConnection` reaches `connected` - while its data channel is closed or still opening, so callers that reason about peer state (sync rounds, call rings) must check the result instead of assuming the send landed.
Profile avatar sync follows attachment-style chunk transport plus server-icon-style version handshakes: sender announces avatar/profile versions, receiver requests only when either remote version is newer, then sender streams ordered base64 chunks when avatar bytes are needed and still uses the same full-event path for profile-only updates.
Every 5 seconds a PING message is sent to each peer. The peer responds with PONG carrying the original timestamp, and the round-trip latency is stored in a signal.
Data-channel failures are treated as control-plane failures. When an open channel reports a non-fatal error, the client requests a fresh voice-state snapshot over that same channel. When the channel is already closed there is no recovery on it, so the peer manager acts immediately: the deterministic initiator renegotiates a new `RTCDataChannel` on the existing `RTCPeerConnection` (preserving audio/video transport), the non-initiator briefly waits for that replacement and then forces a full peer rebuild if it does not arrive, and a peer whose `RTCPeerConnection` is no longer in `connected` state is recreated immediately through the normal deterministic reconnect path. A closing-but-not-yet-closed channel still waits a short grace period in case the underlying transport flips back. Either way, the rebuild heals chat, state sync, voice, camera, and screen-share transport together instead of preserving a media connection whose control channel can no longer coordinate peer state.
Data-channel failures are treated as control-plane failures. When an open channel reports a non-fatal error, the client requests a fresh voice-state snapshot over that same channel. When the channel is already closed there is no recovery on it, so `repairUnavailableDataChannel` picks the least destructive repair that can work:
- **`RTCPeerConnection` still `connected`, and we are the elected initiator** — `replaceDataChannel` closes the dead channel and creates a fresh `RTCDataChannel` on the same connection. No SDP renegotiation is needed because the SCTP transport is already up, so audio, camera, and screen share keep flowing. The remote side's `ondatachannel` adopts the replacement (its own primary channel is closed) and `setupDataChannel` re-wires it.
- **Still `connected`, and we are not the initiator** — wait for that replacement instead of creating a second channel. The channel's `onopen` clears the recovery timer, which is what marks the repair as successful.
- **Replacement never opens within `DATA_CHANNEL_RECOVERY_GRACE_MS`** — fall back to the full rebuild: track the peer as disconnected, remove it, and reconnect through the normal deterministic path. Same fallback when the replacement cannot be created at all.
- **Connection no longer `connected`, or the peer's scoped identity is unknown** — rebuild immediately; there is no live connection to preserve, and without a scoped actor id the initiator election would be a guess.
A closing-but-not-yet-closed channel still waits a short grace period first in case the underlying transport flips back. The rebuild path heals chat, state sync, voice, camera, and screen-share transport together; the replacement path avoids paying for that teardown when only the control channel died.
## Media pipeline
@@ -287,7 +302,9 @@ graph LR
`MediaManager` grabs the mic with `getUserMedia`, optionally pipes it through the RNNoise AudioWorklet for noise reduction (48 kHz, loaded from `rnnoise-worklet.js`), optionally runs it through a `GainNode` for input volume control, and then routes the resulting audio track only to peers that currently belong to the same active voice channel. The same manager also owns camera capture as a separate video-only stream, attaches it to its own video transceiver, and applies the same voice-channel routing rules so webcam video only reaches peers in the active voice room.
Mute just disables the audio track (`track.enabled = false`), the connection stays up. Deafen suppresses incoming audio playback on the local side.
Mute just disables the audio track (`track.enabled = false`), the connection stays up. Deafen suppresses incoming audio playback on the local side. `MediaManager` is the single owner of both flags; the UI reads them back through `VoiceConnectionFacade.isMuted` / `isDeafened` instead of tracking its own copy.
`enableVoice()` captures the microphone the user chose (`setPreferredInputDeviceId`, seeded from stored voice settings), so joining voice from the channel list uses the same device as joining from the voice controls. `switchInputDevice(deviceId)` moves a live session to another microphone: it re-captures, rebuilds the denoise/gain chain, swaps the track into the existing senders, and only then stops the previous device. A plain track swap on an already negotiated sender reports **no** renegotiation, so changing microphone mid-call costs no SDP exchange and no glare risk; renegotiation is still triggered when a transceiver had to be created or its direction restored.
Because peers stay connected across the server for shared state and chat, voice-channel isolation is enforced in both transport and playback: outgoing mic audio is only attached to peers whose voice membership matches the local user's current channel, and remote voice audio plus join/leave cues are only active when the remote peer's announced `voiceState.roomId` and `voiceState.serverId` match the local user's current voice channel.
@@ -1,4 +1,4 @@
/* eslint-disable @typescript-eslint/member-ordering */
import { ChatEvent } from '../../../shared-kernel';
import { DATA_CHANNEL_LABEL } from '../realtime.constants';
import { recordDebugNetworkDownloadRates } from '../logging/debug-network-metrics';
@@ -15,7 +15,9 @@ import {
} from './connection/negotiation';
import {
broadcastCurrentStates,
broadcastIdentityScopedMessage,
broadcastMessage,
PeerScopedIdentity,
sendCurrentStatesToPeer,
sendToPeer,
sendToPeerBuffered,
@@ -32,6 +34,7 @@ import {
removePeer as removeManagedPeer,
requestVoiceStateFromPeer,
resetConnectedPeers,
resumeStalledPeerRecovery,
scheduleDataChannelRecovery,
schedulePeerDisconnectRecovery,
schedulePeerReconnect,
@@ -91,6 +94,10 @@ export class PeerConnectionManager {
readonly peerConnected$ = this.state.peerConnected$;
readonly peerDisconnected$ = this.state.peerDisconnected$;
/** Emitted when peer recovery gives up on a peer, or re-arms it. */
readonly peerRecoveryStatus$ = this.state.peerRecoveryStatus$;
readonly remoteStream$ = this.state.remoteStream$;
readonly messageReceived$ = this.state.messageReceived$;
@@ -181,10 +188,20 @@ export class PeerConnectionManager {
}
/**
* Send a ChatEvent to a specific peer's data channel.
* Broadcast a self-identifying ChatEvent, rebuilt per peer so each peer receives the
* actor id it knows us by on its own signal server.
*/
sendToPeer(peerId: string, event: ChatEvent): void {
sendToPeer(this.context, peerId, event);
broadcastIdentityScopedMessage(buildEvent: (identity: PeerScopedIdentity) => ChatEvent): void {
broadcastIdentityScopedMessage(this.context, buildEvent);
}
/**
* Send a ChatEvent to a specific peer's data channel.
*
* @returns whether the payload reached the peer's open data channel.
*/
sendToPeer(peerId: string, event: ChatEvent): boolean {
return sendToPeer(this.context, peerId, event);
}
/**
@@ -225,6 +242,11 @@ export class PeerConnectionManager {
clearAllPeerReconnectTimers(this.state);
}
/** Retry every peer whose reconnect budget ran out, now that signaling works again. */
resumeStalledPeerRecovery(): void {
resumeStalledPeerRecovery(this.context, this.recoveryHandlers);
}
/** Return a snapshot copy of the currently-connected peer IDs. */
getConnectedPeerIds(): string[] {
return getConnectedPeerIds(this.state);
@@ -247,6 +269,7 @@ export class PeerConnectionManager {
this.closeAllPeers();
this.peerConnected$.complete();
this.peerDisconnected$.complete();
this.peerRecoveryStatus$.complete();
this.remoteStream$.complete();
this.messageReceived$.complete();
this.connectedPeersChanged$.complete();
@@ -1,5 +1,10 @@
import { DATA_CHANNEL_RECOVERY_GRACE_MS, DATA_CHANNEL_STATE_OPEN } from '../../realtime.constants';
import type { PeerData } from '../../realtime.types';
import {
DATA_CHANNEL_RECOVERY_GRACE_MS,
DATA_CHANNEL_STATE_OPEN,
PEER_RECONNECT_INTERVAL_MS,
PEER_RECONNECT_MAX_ATTEMPTS
} from '../../realtime.constants';
import type { PeerData, PeerRecoveryStatusEvent } from '../../realtime.types';
import {
createPeerConnectionManagerState,
PeerConnectionManagerContext,
@@ -8,7 +13,10 @@ import {
import {
closeAllPeers,
removePeer,
scheduleDataChannelRecovery
resumeStalledPeerRecovery,
scheduleDataChannelRecovery,
schedulePeerReconnect,
trackDisconnectedPeer
} from './peer-recovery';
describe('peer recovery', () => {
@@ -51,7 +59,25 @@ describe('peer recovery', () => {
expect(context.state.remotePeerCameraStreams.size).toBe(0);
});
it('recreates a peer immediately when the data channel is already closed', () => {
it('replaces the data channel on the live connection instead of tearing down media', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
const peerData = createPeerData(channel, 'connected');
context.state.activePeerConnections.set('bob', peerData);
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.replaceDataChannel).toHaveBeenCalledWith('bob', channel);
expect(handlers.removePeer).not.toHaveBeenCalled();
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
expect(context.state.activePeerConnections.get('bob')?.connection).toBe(peerData.connection);
});
it('keeps the connection when the replacement data channel opens', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
@@ -62,13 +88,38 @@ describe('peer recovery', () => {
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
const replaced = context.state.activePeerConnections.get('bob');
if (replaced) {
replaced.dataChannel = createDataChannel(DATA_CHANNEL_STATE_OPEN);
}
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.removePeer).not.toHaveBeenCalled();
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
});
it('rebuilds the peer when the replacement data channel never opens', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
context.state.activePeerConnections.set('bob', createPeerData(channel, 'connected'));
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.removePeer).not.toHaveBeenCalled();
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
expect(handlers.createAndSendOffer).toHaveBeenCalledWith('bob');
expect(context.state.dataChannelRecoveryTimers.has('bob')).toBe(false);
});
it('waits a short grace period before recreating a peer with a closing data channel', () => {
it('waits a short grace period before repairing a closing data channel', () => {
vi.useFakeTimers();
const channel = createDataChannel('closing');
@@ -80,14 +131,28 @@ describe('peer recovery', () => {
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS - 1);
expect(handlers.removePeer).not.toHaveBeenCalled();
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
vi.advanceTimersByTime(1);
expect(handlers.replaceDataChannel).toHaveBeenCalledWith('bob', channel);
expect(handlers.removePeer).not.toHaveBeenCalled();
});
it('rebuilds instead of replacing the channel when the connection is no longer connected', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
context.state.activePeerConnections.set('bob', createPeerData(channel, 'disconnected'));
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
expect(handlers.createAndSendOffer).toHaveBeenCalledWith('bob');
});
it('does not recreate a peer when a replacement data channel is adopted before the grace expires', () => {
@@ -127,7 +192,7 @@ describe('peer recovery', () => {
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
});
it('recreates a connected non-initiator peer and waits for the remote initiator offer', () => {
it('lets a non-initiator wait for the remote replacement channel before rebuilding', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
@@ -137,12 +202,143 @@ describe('peer recovery', () => {
context.state.activePeerConnections.set('bob', createPeerData(channel, 'connected', false));
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
expect(handlers.removePeer).not.toHaveBeenCalled();
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', false);
expect(handlers.createAndSendOffer).not.toHaveBeenCalled();
});
it('adopts a remote replacement channel without rebuilding the non-initiator peer', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
const context = createContext('zoe');
const handlers = createRecoveryHandlers(context);
const peerData = createPeerData(channel, 'connected', false);
context.state.activePeerConnections.set('bob', peerData);
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
peerData.dataChannel = createDataChannel(DATA_CHANNEL_STATE_OPEN);
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.removePeer).not.toHaveBeenCalled();
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
});
it('elects the channel repair role from the actor id on the peer signal server', () => {
vi.useFakeTimers();
// Home id sorts before "bob", but on the signal server that routes bob this human is
// "zoe-foreign" and sorts after - so the remote side owns the replacement, not us.
const channel = createDataChannel('closed');
const context = createContext('aa-home', { bob: 'zoe-foreign' });
const handlers = createRecoveryHandlers(context);
context.state.activePeerConnections.set('bob', createPeerData(channel, 'connected'));
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', false);
expect(handlers.createAndSendOffer).not.toHaveBeenCalled();
});
it('does not spend reconnect attempts while signaling is disconnected', () => {
vi.useFakeTimers();
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
let signalingConnected = false;
context.callbacks.isSignalingConnected = vi.fn(() => signalingConnected);
trackDisconnectedPeer(context.state, 'bob');
schedulePeerReconnect(context, 'bob', handlers);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * (PEER_RECONNECT_MAX_ATTEMPTS + 2));
expect(context.state.disconnectedPeerTracker.get('bob')?.reconnectAttempts).toBe(0);
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
expect(context.state.peerReconnectTimers.has('bob')).toBe(true);
signalingConnected = true;
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS);
expect(context.state.disconnectedPeerTracker.get('bob')?.reconnectAttempts).toBe(1);
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
});
it('publishes a failed recovery state when every attempt is spent', () => {
vi.useFakeTimers();
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
const statuses: PeerRecoveryStatusEvent[] = [];
context.state.peerRecoveryStatus$.subscribe((event) => statuses.push(event));
trackDisconnectedPeer(context.state, 'bob');
schedulePeerReconnect(context, 'bob', handlers);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * PEER_RECONNECT_MAX_ATTEMPTS);
expect(statuses).toEqual([]);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS);
expect(statuses).toEqual([{ attempts: PEER_RECONNECT_MAX_ATTEMPTS, peerId: 'bob', status: 'failed' }]);
expect(context.state.peerReconnectTimers.has('bob')).toBe(false);
expect(context.state.disconnectedPeerTracker.get('bob')?.recoveryFailed).toBe(true);
});
it('re-arms failed peer recovery when signaling comes back', () => {
vi.useFakeTimers();
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
const statuses: PeerRecoveryStatusEvent[] = [];
context.state.peerRecoveryStatus$.subscribe((event) => statuses.push(event));
trackDisconnectedPeer(context.state, 'bob');
schedulePeerReconnect(context, 'bob', handlers);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * (PEER_RECONNECT_MAX_ATTEMPTS + 1));
handlers.createPeerConnection.mockClear();
resumeStalledPeerRecovery(context, handlers);
expect(statuses.at(-1)).toEqual({ attempts: 0, peerId: 'bob', status: 'retrying' });
expect(context.state.disconnectedPeerTracker.get('bob')?.recoveryFailed).toBe(false);
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
expect(context.state.peerReconnectTimers.has('bob')).toBe(true);
});
it('does not re-arm failed peer recovery while signaling is still down', () => {
vi.useFakeTimers();
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
trackDisconnectedPeer(context.state, 'bob');
schedulePeerReconnect(context, 'bob', handlers);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * (PEER_RECONNECT_MAX_ATTEMPTS + 1));
handlers.createPeerConnection.mockClear();
context.callbacks.isSignalingConnected = vi.fn(() => false);
resumeStalledPeerRecovery(context, handlers);
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
expect(context.state.disconnectedPeerTracker.get('bob')?.recoveryFailed).toBe(true);
});
it('waits for the remote initiator when a non-connected peer needs full reconnect', () => {
vi.useFakeTimers();
@@ -160,7 +356,16 @@ describe('peer recovery', () => {
});
});
function createContext(localOderId: string): PeerConnectionManagerContext {
function createContext(
localOderId: string,
localActorIdByPeerId: Record<string, string> = {}
): PeerConnectionManagerContext {
const credentialsFor = (oderId: string) => ({
displayName: oderId,
oderId,
token: 'session-token'
});
return {
logger: {
error: vi.fn(),
@@ -171,9 +376,12 @@ function createContext(localOderId: string): PeerConnectionManagerContext {
} as unknown as PeerConnectionManagerContext['logger'],
callbacks: {
getIceServers: vi.fn(() => []),
getIdentifyCredentials: vi.fn(() => ({ oderId: localOderId, token: 'session-token', displayName: localOderId })),
getIdentifyCredentials: vi.fn(() => credentialsFor(localOderId)),
getIdentifyCredentialsForPeer: vi.fn((peerId: string) =>
credentialsFor(localActorIdByPeerId[peerId] ?? localOderId)),
getLocalMediaStream: vi.fn(() => null),
getLocalPeerId: vi.fn(() => localOderId),
mayOpenVoicePathToPeer: vi.fn(() => true),
getVoiceStateSnapshot: vi.fn(() => ({
isConnected: true,
isMuted: false,
@@ -7,6 +7,7 @@ import {
PEER_RECONNECT_INTERVAL_MS,
PEER_RECONNECT_MAX_ATTEMPTS
} from '../../realtime.constants';
import { shouldInitiatePeerConnection } from '../../peer-role.rules';
import {
PeerConnectionManagerContext,
PeerConnectionManagerState,
@@ -205,6 +206,15 @@ export function scheduleDataChannelRecovery(
state.dataChannelRecoveryTimers.set(peerId, timer);
}
/**
* Repair a failed control channel with the least destructive option available.
*
* While the `RTCPeerConnection` is still connected the channel is replaced on that same
* connection, so voice, camera, and screen share keep flowing. Only the deterministically
* elected initiator creates the replacement; the other side adopts the incoming channel.
* A full peer rebuild is the fallback when the connection itself is gone, when the
* replacement cannot be created, or when the replacement never opens.
*/
function repairUnavailableDataChannel(
context: PeerConnectionManagerContext,
peerId: string,
@@ -212,7 +222,7 @@ function repairUnavailableDataChannel(
reason: string,
handlers: RecoveryHandlers
): void {
const { logger, state } = context;
const { callbacks, logger, state } = context;
const peerData = state.activePeerConnections.get(peerId);
if (!peerData || peerData.dataChannel !== channel)
@@ -221,11 +231,103 @@ function repairUnavailableDataChannel(
if (peerData.dataChannel?.readyState === DATA_CHANNEL_STATE_OPEN)
return;
logger.warn('[data-channel] Recreating peer transport after control channel failure', {
if (peerData.connection.connectionState !== CONNECTION_STATE_CONNECTED) {
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
return;
}
const localOderId = callbacks.getIdentifyCredentialsForPeer(peerId)?.oderId ?? null;
if (!localOderId) {
logger.warn('[data-channel] Logical identity unknown; rebuilding peer instead of replacing channel', {
peerId,
reason
});
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
return;
}
if (shouldInitiatePeerConnection(localOderId, peerId)) {
logger.warn('[data-channel] Replacing control channel on the live connection', {
channelLabel: channel.label,
peerId,
reason
});
if (!handlers.replaceDataChannel(peerId, channel)) {
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
return;
}
watchDataChannelReplacement(context, peerId, reason, handlers);
return;
}
logger.info('[data-channel] Waiting for the remote replacement control channel', {
channelLabel: channel.label,
connectionState: peerData.connection.connectionState,
localOderId,
peerId,
readyState: peerData.dataChannel?.readyState ?? null,
reason
});
watchDataChannelReplacement(context, peerId, reason, handlers);
}
/** Rebuild the peer if the replacement control channel does not open within the grace period. */
function watchDataChannelReplacement(
context: PeerConnectionManagerContext,
peerId: string,
reason: string,
handlers: RecoveryHandlers
): void {
const { logger, state } = context;
clearDataChannelRecoveryTimer(state, peerId);
const timer = setTimeout(() => {
state.dataChannelRecoveryTimers.delete(peerId);
const latestPeerData = state.activePeerConnections.get(peerId);
if (!latestPeerData)
return;
if (latestPeerData.dataChannel?.readyState === DATA_CHANNEL_STATE_OPEN) {
logger.info('[data-channel] Replacement control channel is open; peer kept alive', {
peerId,
reason
});
return;
}
logger.warn('[data-channel] Replacement control channel never opened; rebuilding peer', {
connectionState: latestPeerData.connection.connectionState,
peerId,
readyState: latestPeerData.dataChannel?.readyState ?? null,
reason
});
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
}, DATA_CHANNEL_RECOVERY_GRACE_MS);
state.dataChannelRecoveryTimers.set(peerId, timer);
}
function rebuildPeerAfterDataChannelFailure(
context: PeerConnectionManagerContext,
peerId: string,
reason: string,
handlers: RecoveryHandlers
): void {
const { logger, state } = context;
const peerData = state.activePeerConnections.get(peerId);
logger.warn('[data-channel] Recreating peer transport after control channel failure', {
connectionState: peerData?.connection.connectionState ?? null,
peerId,
readyState: peerData?.dataChannel?.readyState ?? null,
reason
});
@@ -299,30 +401,84 @@ export function schedulePeerReconnect(
return;
}
// An attempt costs nothing while the socket is down and there is no way to send an
// offer, so the budget must only be spent on attempts that can actually reach the peer.
if (!callbacks.isSignalingConnected()) {
logger.info('Deferring P2P reconnect - no signaling connection', {
peerId,
attemptsSpent: info.reconnectAttempts
});
return;
}
if (info.reconnectAttempts >= PEER_RECONNECT_MAX_ATTEMPTS) {
failPeerRecovery(context, peerId, info.reconnectAttempts);
return;
}
info.reconnectAttempts++;
logger.info('P2P reconnect attempt', {
peerId,
attempt: info.reconnectAttempts
});
if (info.reconnectAttempts >= PEER_RECONNECT_MAX_ATTEMPTS) {
logger.info('P2P reconnect max attempts reached', { peerId });
clearPeerReconnectTimer(state, peerId);
state.disconnectedPeerTracker.delete(peerId);
return;
}
if (!callbacks.isSignalingConnected()) {
logger.info('Skipping P2P reconnect - no signaling connection', { peerId });
return;
}
attemptPeerReconnect(context, peerId, handlers);
}, PEER_RECONNECT_INTERVAL_MS);
state.peerReconnectTimers.set(peerId, timer);
}
/**
* Stop retrying a peer and say so. The tracker entry is kept (marked failed) so
* `resumeStalledPeerRecovery` can re-arm it once signaling is usable again.
*/
function failPeerRecovery(
context: PeerConnectionManagerContext,
peerId: string,
attempts: number
): void {
const { logger, state } = context;
const info = state.disconnectedPeerTracker.get(peerId);
logger.warn('P2P reconnect gave up', { attempts, peerId });
clearPeerReconnectTimer(state, peerId);
if (info) {
info.recoveryFailed = true;
}
state.peerRecoveryStatus$.next({ attempts, peerId, status: 'failed' });
}
/**
* Re-arm every peer whose recovery gave up. Called when signaling reconnects, so a peer
* that ran out of attempts during an outage is retried instead of staying dead silently.
*/
export function resumeStalledPeerRecovery(
context: PeerConnectionManagerContext,
handlers: RecoveryHandlers
): void {
const { callbacks, logger, state } = context;
if (!callbacks.isSignalingConnected())
return;
state.disconnectedPeerTracker.forEach((info, peerId) => {
if (!info.recoveryFailed)
return;
logger.info('Re-arming P2P recovery after signaling returned', { peerId });
info.recoveryFailed = false;
info.reconnectAttempts = 0;
state.peerRecoveryStatus$.next({ attempts: 0, peerId, status: 'retrying' });
attemptPeerReconnect(context, peerId, handlers);
schedulePeerReconnect(context, peerId, handlers);
});
}
export function attemptPeerReconnect(
context: PeerConnectionManagerContext,
peerId: string,
@@ -334,7 +490,7 @@ export function attemptPeerReconnect(
handlers.removePeer(peerId, { preserveReconnectState: true });
}
const localOderId = callbacks.getIdentifyCredentials()?.oderId ?? null;
const localOderId = callbacks.getIdentifyCredentialsForPeer(peerId)?.oderId ?? null;
if (!localOderId) {
logger.info('Skipping reconnect offer until logical identity is ready', { peerId });
@@ -342,7 +498,7 @@ export function attemptPeerReconnect(
return;
}
const shouldInitiate = peerId !== localOderId && localOderId < peerId;
const shouldInitiate = shouldInitiatePeerConnection(localOderId, peerId);
handlers.createPeerConnection(peerId, shouldInitiate);
@@ -0,0 +1,55 @@
import { isPoliteOnOfferCollision, shouldInitiatePeerConnection } from './peer-role.rules';
describe('peer role election', () => {
it('elects exactly one initiator for every pair of distinct actor ids', () => {
const actorIds = [
'alice@signal-a',
'alice@signal-b',
'bob@signal-a',
'bob@signal-b',
'0f3c-uuid-lower',
'ZZZ-uppercase'
];
for (const local of actorIds) {
for (const remote of actorIds) {
if (local === remote)
continue;
const initiators = [shouldInitiatePeerConnection(local, remote), shouldInitiatePeerConnection(remote, local)].filter(Boolean);
expect(initiators, `${local} <-> ${remote}`).toHaveLength(1);
}
}
});
it('never initiates against itself', () => {
expect(shouldInitiatePeerConnection('alice@signal-a', 'alice@signal-a')).toBe(false);
});
it('does not initiate while the local actor id is unknown', () => {
expect(shouldInitiatePeerConnection(null, 'bob@signal-a')).toBe(false);
expect(shouldInitiatePeerConnection(' ', 'bob@signal-a')).toBe(false);
expect(shouldInitiatePeerConnection('alice@signal-a', undefined)).toBe(false);
});
it('makes exactly one side impolite on an offer collision', () => {
const local = 'alice@signal-a';
const remote = 'bob@signal-a';
expect(isPoliteOnOfferCollision(local, remote)).toBe(!isPoliteOnOfferCollision(remote, local));
});
it('makes the elected initiator the impolite side', () => {
const initiator = 'alice@signal-a';
const responder = 'bob@signal-a';
expect(shouldInitiatePeerConnection(initiator, responder)).toBe(true);
expect(isPoliteOnOfferCollision(initiator, responder)).toBe(false);
expect(isPoliteOnOfferCollision(responder, initiator)).toBe(true);
});
it('yields on collision when the local actor id is unknown', () => {
expect(isPoliteOnOfferCollision(null, 'bob@signal-a')).toBe(true);
});
});
@@ -0,0 +1,37 @@
/**
* Deterministic WebRTC role election.
*
* Both ids must come from the **same signal server's identity space**. One human has a
* separate account - and therefore a separate actor id - on every signal server they are
* provisioned on, so comparing a home id against a foreign actor id is not antisymmetric:
* each side compares its own home id against the other's foreign id, and both can elect
* themselves initiator (glare) or neither can (no offer ever sent).
*/
export function shouldInitiatePeerConnection(
localActorId: string | null | undefined,
remoteActorId: string | null | undefined
): boolean {
const local = localActorId?.trim();
const remote = remoteActorId?.trim();
if (!local || !remote)
return false;
if (local === remote)
return false;
return local < remote;
}
/**
* Perfect-negotiation politeness, derived from the same election so the two can never
* disagree: the elected initiator is the impolite side and keeps its offer, the other
* side rolls back. Falls back to polite when the local actor id is unknown, so an
* unidentified client yields instead of deadlocking the negotiation.
*/
export function isPoliteOnOfferCollision(
localActorId: string | null | undefined,
remoteActorId: string | null | undefined
): boolean {
return !shouldInitiatePeerConnection(localActorId, remoteActorId);
}
@@ -62,6 +62,21 @@ export interface DisconnectedPeerEntry {
lastSeenTimestamp: number;
/** Number of reconnect attempts made so far. */
reconnectAttempts: number;
/** `true` once every attempt was spent; the entry is kept so recovery can be re-armed. */
recoveryFailed?: boolean;
}
/** Whether peer recovery gave up or is trying again. */
export type PeerRecoveryStatus = 'failed' | 'retrying';
/** Emitted when peer recovery gives up on a peer, or re-arms after signaling returns. */
export interface PeerRecoveryStatusEvent {
/** The remote peer this recovery state belongs to. */
peerId: string;
/** Current recovery status. */
status: PeerRecoveryStatus;
/** Reconnect attempts spent so far (reset to 0 when re-armed). */
attempts: number;
}
/** Snapshot of current voice / screen state (broadcast to peers). */
@@ -41,6 +41,122 @@ describe('IncomingSignalingMessageHandler user_left handling', () => {
expect(context.peerManager.removePeer).toHaveBeenCalledWith('peer-a');
expect(context.coordinator.getPeerSignalUrl('peer-a')).toBeUndefined();
});
it('preserves a peer that is still shared through another signal scope', () => {
const context = createHandlerContext({ voiceConnected: false });
context.coordinator.trackPeerInServer('peer-a', 'server-a', 'ws://signal-a');
context.coordinator.trackPeerInServer('peer-a', 'server-b', 'ws://signal-b');
context.peerManager.activePeerConnections.set('peer-a', createPeerData('connected', 'open'));
context.handler.handleMessage(
{
type: 'user_left',
oderId: 'peer-a',
serverId: 'server-a'
},
'ws://signal-a'
);
expect(context.peerManager.removePeer).not.toHaveBeenCalled();
expect(context.coordinator.getPeerSignalUrl('peer-a')).toBe('ws://signal-b');
expect(context.coordinator.hasTrackedPeerServers('peer-a')).toBe(true);
});
});
describe('IncomingSignalingMessageHandler cross-signal initiator election', () => {
const HOME_SIGNAL_URL = 'ws://signal-home';
const FOREIGN_SIGNAL_URL = 'ws://signal-foreign';
it('elects the initiator from the actor ids of the signal server the peer is on', () => {
// On the foreign server this human is "zz-alice-foreign" even though their home
// account is "aa-alice-home". The peer is "mm-bob-foreign" there, so the foreign
// comparison must elect the peer - not us, as the home id would.
const context = createHandlerContext({
localOderIdBySignalUrl: {
[HOME_SIGNAL_URL]: 'aa-alice-home',
[FOREIGN_SIGNAL_URL]: 'zz-alice-foreign'
},
voiceConnected: false
});
context.handler.handleMessage(
{
serverId: 'server-1',
type: 'server_users',
users: [{ displayName: 'Bob', oderId: 'mm-bob-foreign' }]
},
FOREIGN_SIGNAL_URL
);
expect(context.peerManager.createPeerConnection).not.toHaveBeenCalled();
expect(context.peerManager.createAndSendOffer).not.toHaveBeenCalled();
});
it('initiates when the local actor id on that signal server sorts first', () => {
const context = createHandlerContext({
localOderIdBySignalUrl: {
[HOME_SIGNAL_URL]: 'zz-alice-home',
[FOREIGN_SIGNAL_URL]: 'aa-alice-foreign'
},
voiceConnected: false
});
context.handler.handleMessage(
{
serverId: 'server-1',
type: 'server_users',
users: [{ displayName: 'Bob', oderId: 'mm-bob-foreign' }]
},
FOREIGN_SIGNAL_URL
);
expect(context.peerManager.createPeerConnection).toHaveBeenCalledWith('mm-bob-foreign', true);
expect(context.peerManager.createAndSendOffer).toHaveBeenCalledWith('mm-bob-foreign');
});
it('skips its own foreign roster entry instead of peering with itself', () => {
const context = createHandlerContext({
localOderIdBySignalUrl: {
[HOME_SIGNAL_URL]: 'aa-alice-home',
[FOREIGN_SIGNAL_URL]: 'zz-alice-foreign'
},
voiceConnected: false
});
context.handler.handleMessage(
{
serverId: 'server-1',
type: 'server_users',
users: [{ displayName: 'Alice', oderId: 'zz-alice-foreign' }]
},
FOREIGN_SIGNAL_URL
);
expect(context.peerManager.createPeerConnection).not.toHaveBeenCalled();
expect(context.coordinator.getPeerSignalUrl('zz-alice-foreign')).toBeUndefined();
});
it('ignores an offer echoed back from its own foreign identity', () => {
const context = createHandlerContext({
localOderIdBySignalUrl: {
[HOME_SIGNAL_URL]: 'aa-alice-home',
[FOREIGN_SIGNAL_URL]: 'zz-alice-foreign'
},
voiceConnected: false
});
context.handler.handleMessage(
{
fromUserId: 'zz-alice-foreign',
payload: { sdp: { sdp: 'v=0', type: 'offer' } },
type: 'offer'
},
FOREIGN_SIGNAL_URL
);
expect(context.peerManager.handleOffer).not.toHaveBeenCalled();
});
});
interface HandlerContext {
@@ -48,11 +164,17 @@ interface HandlerContext {
handler: IncomingSignalingMessageHandler;
peerManager: PeerConnectionManager & {
activePeerConnections: Map<string, PeerData>;
createAndSendOffer: ReturnType<typeof vi.fn>;
createPeerConnection: ReturnType<typeof vi.fn>;
handleOffer: ReturnType<typeof vi.fn>;
removePeer: ReturnType<typeof vi.fn>;
};
}
function createHandlerContext(options: { voiceConnected: boolean }): HandlerContext {
function createHandlerContext(options: {
voiceConnected: boolean;
localOderIdBySignalUrl?: Record<string, string>;
}): HandlerContext {
const coordinator = new ServerSignalingCoordinator<unknown>({
createManager: vi.fn(),
handleConnectionStatus: vi.fn(),
@@ -61,11 +183,15 @@ function createHandlerContext(options: { voiceConnected: boolean }): HandlerCont
});
const peerManager = {
activePeerConnections: new Map<string, PeerData>(),
createAndSendOffer: vi.fn(async () => undefined),
createPeerConnection: vi.fn(),
handleOffer: vi.fn(),
removePeer: vi.fn()
} as unknown as HandlerContext['peerManager'];
const localOderIdBySignalUrl = options.localOderIdBySignalUrl ?? { 'ws://signal-a': 'local-user' };
const handler = new IncomingSignalingMessageHandler({
getEffectiveServerId: () => 'server-1',
getLocalOderId: () => 'local-user',
getLocalOderIdForSignalUrl: (signalUrl: string) => localOderIdBySignalUrl[signalUrl] ?? null,
isVoiceConnected: () => options.voiceConnected,
logger: {
error: vi.fn(),
@@ -11,6 +11,7 @@ import {
SIGNALING_TYPE_USER_LEFT
} from '../realtime.constants';
import { PeerConnectionManager } from '../peer-connection-manager/peer-connection.manager';
import { shouldInitiatePeerConnection } from '../peer-role.rules';
import { ServerSignalingCoordinator } from './server-signaling-coordinator';
import { WebRTCLogger } from '../logging/webrtc-logger';
@@ -43,7 +44,12 @@ interface IncomingSignalingMessageHandlerDependencies {
peerManager: PeerConnectionManager;
signalingCoordinator: ServerSignalingCoordinator<IncomingSignalingMessage>;
logger: WebRTCLogger;
getLocalOderId(): string | null;
/**
* Local actor id in the identity space of `signalUrl`. Roster ids and `fromUserId`s
* arrive per signal server, so every comparison against them must use the local id
* for that same server.
*/
getLocalOderIdForSignalUrl(signalUrl: string): string | null;
getEffectiveServerId(): string | null;
isVoiceConnected(): boolean;
setServerTime(serverTime: number): void;
@@ -128,7 +134,7 @@ export class IncomingSignalingMessageHandler {
private handleServerUsersSignalingMessage(message: IncomingSignalingMessage, signalUrl: string): void {
const users = Array.isArray(message.users) ? message.users : [];
const localOderId = this.dependencies.getLocalOderId();
const localOderId = this.dependencies.getLocalOderIdForSignalUrl(signalUrl);
this.dependencies.logger.info('Server users', {
count: users.length,
@@ -245,7 +251,7 @@ export class IncomingSignalingMessageHandler {
}
private handleUserJoinedSignalingMessage(message: IncomingSignalingMessage, signalUrl: string): void {
if (message.oderId && message.oderId === this.dependencies.getLocalOderId()) {
if (this.isLocalActorOnSignal(message.oderId, signalUrl)) {
return;
}
@@ -281,7 +287,7 @@ export class IncomingSignalingMessageHandler {
}
private handleUserLeftSignalingMessage(message: IncomingSignalingMessage, signalUrl: string): void {
if (message.oderId && message.oderId === this.dependencies.getLocalOderId()) {
if (this.isLocalActorOnSignal(message.oderId, signalUrl)) {
return;
}
@@ -326,7 +332,7 @@ export class IncomingSignalingMessageHandler {
if (!fromUserId || !sdp)
return;
if (fromUserId === this.dependencies.getLocalOderId())
if (this.isLocalActorOnSignal(fromUserId, signalUrl))
return;
this.clearUserJoinedFallbackOffer(fromUserId);
@@ -350,7 +356,7 @@ export class IncomingSignalingMessageHandler {
if (!fromUserId || !sdp)
return;
if (fromUserId === this.dependencies.getLocalOderId())
if (this.isLocalActorOnSignal(fromUserId, signalUrl))
return;
this.clearUserJoinedFallbackOffer(fromUserId);
@@ -366,7 +372,7 @@ export class IncomingSignalingMessageHandler {
if (!fromUserId || !candidate)
return;
if (fromUserId === this.dependencies.getLocalOderId())
if (this.isLocalActorOnSignal(fromUserId, signalUrl))
return;
this.clearUserJoinedFallbackOffer(fromUserId);
@@ -390,7 +396,7 @@ export class IncomingSignalingMessageHandler {
const timer = setTimeout(() => {
this.userJoinedFallbackTimers.delete(peerId);
const localOderId = this.dependencies.getLocalOderId();
const localOderId = this.dependencies.getLocalOderIdForSignalUrl(signalUrl);
const existing = this.dependencies.peerManager.activePeerConnections.get(peerId);
if (this.hasActivePeerConnection(existing)) {
@@ -525,14 +531,15 @@ export class IncomingSignalingMessageHandler {
this.userJoinedFallbackTimers.delete(peerId);
}
private shouldInitiatePeer(peerId: string, localOderId: string | null = this.dependencies.getLocalOderId()): boolean {
if (!localOderId)
private shouldInitiatePeer(peerId: string, localOderId: string | null): boolean {
return shouldInitiatePeerConnection(localOderId, peerId);
}
private isLocalActorOnSignal(candidateId: string | undefined, signalUrl: string): boolean {
if (!candidateId)
return false;
if (peerId === localOderId)
return false;
return localOderId < peerId;
return candidateId === this.dependencies.getLocalOderIdForSignalUrl(signalUrl);
}
private hasActivePeerConnection(peer: PeerData | undefined): boolean {
@@ -4,7 +4,7 @@ import { SIGNALING_TYPE_IDENTIFY } from '../realtime.constants';
describe('SignalingTransportHandler identify', () => {
const SIGNAL_URL = 'ws://signal.example.com:3001';
function createHandler(resolved: ResolvedSignalCredential | null) {
function createHandler(resolved: ResolvedSignalCredential | null, resolveCredential = vi.fn(() => resolved)) {
const sentMessages: Record<string, unknown>[] = [];
const manager = {
isSocketOpen: () => true,
@@ -20,12 +20,12 @@ describe('SignalingTransportHandler identify', () => {
signalingCoordinator: coordinator,
logger: { warn: () => undefined, error: () => undefined, info: () => undefined } as never,
getLocalPeerId: () => 'local-peer',
resolveCredential: () => resolved,
resolveCredential,
getHomeCredential: () => resolved,
getClientInstanceId: () => 'device-a'
});
return { handler, sentMessages };
return { handler, resolveCredential, sentMessages };
}
it('uses the freshly supplied display name over the stale stored credential', () => {
@@ -68,4 +68,24 @@ describe('SignalingTransportHandler identify', () => {
expect(message['displayName']).toBe('Alice');
}
});
it('resolves fresh-socket identify credentials from the per-signal credential store', () => {
const resolveCredential = vi.fn(() => ({
userId: 'foreign-actor-1',
token: 'foreign-token-1',
displayName: 'Alice Foreign',
homeSignalServerUrl: 'https://signal.home.example'
}));
const { handler } = createHandler(null, resolveCredential);
expect(handler.getIdentifyCredentialsForSignalUrl(SIGNAL_URL)).toEqual({
oderId: 'foreign-actor-1',
token: 'foreign-token-1',
displayName: 'Alice Foreign',
homeSignalServerUrl: 'https://signal.home.example',
clientInstanceId: 'device-a'
});
expect(resolveCredential).toHaveBeenCalledWith(SIGNAL_URL);
});
});
@@ -71,12 +71,24 @@ export class SignalingTransportHandler<TMessage> {
};
}
getIdentifyOderId(): string {
return this.getIdentifyCredentials()?.oderId || this.dependencies.getLocalPeerId();
/**
* Local actor id in the identity space of one signal server. Returns null rather than
* falling back to the home id: a wrong-space id would break the antisymmetry of peer
* role election, and callers already defer and retry when the id is not resolvable yet.
*/
getLocalOderIdForSignalUrl(signalUrl: string): string | null {
return this.getIdentifyCredentialsForSignalUrl(signalUrl)?.oderId ?? null;
}
getIdentifyDisplayName(): string {
return this.getIdentifyCredentials()?.displayName || DEFAULT_DISPLAY_NAME;
/** Identify credentials in the identity space of the signal server routing this peer. */
getIdentifyCredentialsForPeer(peerId: string): IdentifyCredentials | null {
const peerSignalUrl = this.dependencies.signalingCoordinator.getPeerSignalUrl(peerId);
if (!peerSignalUrl) {
return null;
}
return this.getIdentifyCredentialsForSignalUrl(peerSignalUrl);
}
getIdentifyDescription(): string | undefined {
@@ -33,6 +33,7 @@ class MockWebSocket {
shouldFailSend = false;
autoAckKeepalive = true;
readonly sentPayloads: Record<string, unknown>[] = [];
readonly url: string;
readyState = MockWebSocket.CONNECTING;
onopen: MockSocketHandler | null = null;
@@ -59,13 +60,15 @@ class MockWebSocket {
throw new Error('mock send failure');
}
const message = JSON.parse(payload) as Record<string, unknown>;
this.sentPayloads.push(message);
if (!this.autoAckKeepalive) {
return;
}
try {
const message = JSON.parse(payload) as { type?: string };
if (message.type === 'keepalive') {
this.onmessage?.({ data: JSON.stringify({ type: 'keepalive_ack' }) });
}
@@ -231,6 +234,21 @@ describe('SignalingManager reconnection', () => {
});
});
it('sends identify before every authenticated join and view on a fresh socket', () => {
const manager = createManager();
manager.connect(serverUrl).subscribe();
MockWebSocket.instances[0]?.simulateOpen();
const authenticatedTypes = MockWebSocket.instances[0]?.sentPayloads.map((message) => message['type']).filter((type) => type !== 'keepalive');
expect(authenticatedTypes).toEqual([
'identify',
'join_server',
'view_server'
]);
});
it('does not force reconnect on servers that never send keepalive acknowledgements', () => {
const manager = createManager();