fix(realtime): survive signal outages and per-server identities

Peer recovery burned its whole retry budget while signaling was down, then
dropped the tracker with no re-arm, so a peer stayed dead until an unrelated
roster event healed it. Recovery now waits for a usable transport before
spending an attempt.

Initiator election also compared a home actor id against foreign roster ids,
which is not antisymmetric across signal servers - both sides offered, or
neither did. Election moves into `peer-role.rules` and compares ids only
within one signal server's identity space.
This commit is contained in:
2026-08-14 03:19:29 +02:00
parent f9e8538c80
commit a83f5aa750
14 changed files with 1176 additions and 64 deletions
@@ -1,4 +1,4 @@
/* eslint-disable @typescript-eslint/member-ordering */
import { ChatEvent } from '../../../shared-kernel';
import { DATA_CHANNEL_LABEL } from '../realtime.constants';
import { recordDebugNetworkDownloadRates } from '../logging/debug-network-metrics';
@@ -15,7 +15,9 @@ import {
} from './connection/negotiation';
import {
broadcastCurrentStates,
broadcastIdentityScopedMessage,
broadcastMessage,
PeerScopedIdentity,
sendCurrentStatesToPeer,
sendToPeer,
sendToPeerBuffered,
@@ -32,6 +34,7 @@ import {
removePeer as removeManagedPeer,
requestVoiceStateFromPeer,
resetConnectedPeers,
resumeStalledPeerRecovery,
scheduleDataChannelRecovery,
schedulePeerDisconnectRecovery,
schedulePeerReconnect,
@@ -91,6 +94,10 @@ export class PeerConnectionManager {
readonly peerConnected$ = this.state.peerConnected$;
readonly peerDisconnected$ = this.state.peerDisconnected$;
/** Emitted when peer recovery gives up on a peer, or re-arms it. */
readonly peerRecoveryStatus$ = this.state.peerRecoveryStatus$;
readonly remoteStream$ = this.state.remoteStream$;
readonly messageReceived$ = this.state.messageReceived$;
@@ -181,10 +188,20 @@ export class PeerConnectionManager {
}
/**
* Send a ChatEvent to a specific peer's data channel.
* Broadcast a self-identifying ChatEvent, rebuilt per peer so each peer receives the
* actor id it knows us by on its own signal server.
*/
sendToPeer(peerId: string, event: ChatEvent): void {
sendToPeer(this.context, peerId, event);
broadcastIdentityScopedMessage(buildEvent: (identity: PeerScopedIdentity) => ChatEvent): void {
broadcastIdentityScopedMessage(this.context, buildEvent);
}
/**
* Send a ChatEvent to a specific peer's data channel.
*
* @returns whether the payload reached the peer's open data channel.
*/
sendToPeer(peerId: string, event: ChatEvent): boolean {
return sendToPeer(this.context, peerId, event);
}
/**
@@ -225,6 +242,11 @@ export class PeerConnectionManager {
clearAllPeerReconnectTimers(this.state);
}
/** Retry every peer whose reconnect budget ran out, now that signaling works again. */
resumeStalledPeerRecovery(): void {
resumeStalledPeerRecovery(this.context, this.recoveryHandlers);
}
/** Return a snapshot copy of the currently-connected peer IDs. */
getConnectedPeerIds(): string[] {
return getConnectedPeerIds(this.state);
@@ -247,6 +269,7 @@ export class PeerConnectionManager {
this.closeAllPeers();
this.peerConnected$.complete();
this.peerDisconnected$.complete();
this.peerRecoveryStatus$.complete();
this.remoteStream$.complete();
this.messageReceived$.complete();
this.connectedPeersChanged$.complete();
@@ -1,5 +1,10 @@
import { DATA_CHANNEL_RECOVERY_GRACE_MS, DATA_CHANNEL_STATE_OPEN } from '../../realtime.constants';
import type { PeerData } from '../../realtime.types';
import {
DATA_CHANNEL_RECOVERY_GRACE_MS,
DATA_CHANNEL_STATE_OPEN,
PEER_RECONNECT_INTERVAL_MS,
PEER_RECONNECT_MAX_ATTEMPTS
} from '../../realtime.constants';
import type { PeerData, PeerRecoveryStatusEvent } from '../../realtime.types';
import {
createPeerConnectionManagerState,
PeerConnectionManagerContext,
@@ -8,7 +13,10 @@ import {
import {
closeAllPeers,
removePeer,
scheduleDataChannelRecovery
resumeStalledPeerRecovery,
scheduleDataChannelRecovery,
schedulePeerReconnect,
trackDisconnectedPeer
} from './peer-recovery';
describe('peer recovery', () => {
@@ -51,7 +59,25 @@ describe('peer recovery', () => {
expect(context.state.remotePeerCameraStreams.size).toBe(0);
});
it('recreates a peer immediately when the data channel is already closed', () => {
it('replaces the data channel on the live connection instead of tearing down media', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
const peerData = createPeerData(channel, 'connected');
context.state.activePeerConnections.set('bob', peerData);
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.replaceDataChannel).toHaveBeenCalledWith('bob', channel);
expect(handlers.removePeer).not.toHaveBeenCalled();
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
expect(context.state.activePeerConnections.get('bob')?.connection).toBe(peerData.connection);
});
it('keeps the connection when the replacement data channel opens', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
@@ -62,13 +88,38 @@ describe('peer recovery', () => {
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
const replaced = context.state.activePeerConnections.get('bob');
if (replaced) {
replaced.dataChannel = createDataChannel(DATA_CHANNEL_STATE_OPEN);
}
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.removePeer).not.toHaveBeenCalled();
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
});
it('rebuilds the peer when the replacement data channel never opens', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
context.state.activePeerConnections.set('bob', createPeerData(channel, 'connected'));
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.removePeer).not.toHaveBeenCalled();
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
expect(handlers.createAndSendOffer).toHaveBeenCalledWith('bob');
expect(context.state.dataChannelRecoveryTimers.has('bob')).toBe(false);
});
it('waits a short grace period before recreating a peer with a closing data channel', () => {
it('waits a short grace period before repairing a closing data channel', () => {
vi.useFakeTimers();
const channel = createDataChannel('closing');
@@ -80,14 +131,28 @@ describe('peer recovery', () => {
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS - 1);
expect(handlers.removePeer).not.toHaveBeenCalled();
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
vi.advanceTimersByTime(1);
expect(handlers.replaceDataChannel).toHaveBeenCalledWith('bob', channel);
expect(handlers.removePeer).not.toHaveBeenCalled();
});
it('rebuilds instead of replacing the channel when the connection is no longer connected', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
context.state.activePeerConnections.set('bob', createPeerData(channel, 'disconnected'));
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
expect(handlers.createAndSendOffer).toHaveBeenCalledWith('bob');
});
it('does not recreate a peer when a replacement data channel is adopted before the grace expires', () => {
@@ -127,7 +192,7 @@ describe('peer recovery', () => {
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
});
it('recreates a connected non-initiator peer and waits for the remote initiator offer', () => {
it('lets a non-initiator wait for the remote replacement channel before rebuilding', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
@@ -137,12 +202,143 @@ describe('peer recovery', () => {
context.state.activePeerConnections.set('bob', createPeerData(channel, 'connected', false));
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
expect(handlers.removePeer).not.toHaveBeenCalled();
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', false);
expect(handlers.createAndSendOffer).not.toHaveBeenCalled();
});
it('adopts a remote replacement channel without rebuilding the non-initiator peer', () => {
vi.useFakeTimers();
const channel = createDataChannel('closed');
const context = createContext('zoe');
const handlers = createRecoveryHandlers(context);
const peerData = createPeerData(channel, 'connected', false);
context.state.activePeerConnections.set('bob', peerData);
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
peerData.dataChannel = createDataChannel(DATA_CHANNEL_STATE_OPEN);
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.removePeer).not.toHaveBeenCalled();
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
});
it('elects the channel repair role from the actor id on the peer signal server', () => {
vi.useFakeTimers();
// Home id sorts before "bob", but on the signal server that routes bob this human is
// "zoe-foreign" and sorts after - so the remote side owns the replacement, not us.
const channel = createDataChannel('closed');
const context = createContext('aa-home', { bob: 'zoe-foreign' });
const handlers = createRecoveryHandlers(context);
context.state.activePeerConnections.set('bob', createPeerData(channel, 'connected'));
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', false);
expect(handlers.createAndSendOffer).not.toHaveBeenCalled();
});
it('does not spend reconnect attempts while signaling is disconnected', () => {
vi.useFakeTimers();
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
let signalingConnected = false;
context.callbacks.isSignalingConnected = vi.fn(() => signalingConnected);
trackDisconnectedPeer(context.state, 'bob');
schedulePeerReconnect(context, 'bob', handlers);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * (PEER_RECONNECT_MAX_ATTEMPTS + 2));
expect(context.state.disconnectedPeerTracker.get('bob')?.reconnectAttempts).toBe(0);
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
expect(context.state.peerReconnectTimers.has('bob')).toBe(true);
signalingConnected = true;
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS);
expect(context.state.disconnectedPeerTracker.get('bob')?.reconnectAttempts).toBe(1);
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
});
it('publishes a failed recovery state when every attempt is spent', () => {
vi.useFakeTimers();
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
const statuses: PeerRecoveryStatusEvent[] = [];
context.state.peerRecoveryStatus$.subscribe((event) => statuses.push(event));
trackDisconnectedPeer(context.state, 'bob');
schedulePeerReconnect(context, 'bob', handlers);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * PEER_RECONNECT_MAX_ATTEMPTS);
expect(statuses).toEqual([]);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS);
expect(statuses).toEqual([{ attempts: PEER_RECONNECT_MAX_ATTEMPTS, peerId: 'bob', status: 'failed' }]);
expect(context.state.peerReconnectTimers.has('bob')).toBe(false);
expect(context.state.disconnectedPeerTracker.get('bob')?.recoveryFailed).toBe(true);
});
it('re-arms failed peer recovery when signaling comes back', () => {
vi.useFakeTimers();
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
const statuses: PeerRecoveryStatusEvent[] = [];
context.state.peerRecoveryStatus$.subscribe((event) => statuses.push(event));
trackDisconnectedPeer(context.state, 'bob');
schedulePeerReconnect(context, 'bob', handlers);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * (PEER_RECONNECT_MAX_ATTEMPTS + 1));
handlers.createPeerConnection.mockClear();
resumeStalledPeerRecovery(context, handlers);
expect(statuses.at(-1)).toEqual({ attempts: 0, peerId: 'bob', status: 'retrying' });
expect(context.state.disconnectedPeerTracker.get('bob')?.recoveryFailed).toBe(false);
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
expect(context.state.peerReconnectTimers.has('bob')).toBe(true);
});
it('does not re-arm failed peer recovery while signaling is still down', () => {
vi.useFakeTimers();
const context = createContext('alice');
const handlers = createRecoveryHandlers(context);
trackDisconnectedPeer(context.state, 'bob');
schedulePeerReconnect(context, 'bob', handlers);
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * (PEER_RECONNECT_MAX_ATTEMPTS + 1));
handlers.createPeerConnection.mockClear();
context.callbacks.isSignalingConnected = vi.fn(() => false);
resumeStalledPeerRecovery(context, handlers);
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
expect(context.state.disconnectedPeerTracker.get('bob')?.recoveryFailed).toBe(true);
});
it('waits for the remote initiator when a non-connected peer needs full reconnect', () => {
vi.useFakeTimers();
@@ -160,7 +356,16 @@ describe('peer recovery', () => {
});
});
function createContext(localOderId: string): PeerConnectionManagerContext {
function createContext(
localOderId: string,
localActorIdByPeerId: Record<string, string> = {}
): PeerConnectionManagerContext {
const credentialsFor = (oderId: string) => ({
displayName: oderId,
oderId,
token: 'session-token'
});
return {
logger: {
error: vi.fn(),
@@ -171,9 +376,12 @@ function createContext(localOderId: string): PeerConnectionManagerContext {
} as unknown as PeerConnectionManagerContext['logger'],
callbacks: {
getIceServers: vi.fn(() => []),
getIdentifyCredentials: vi.fn(() => ({ oderId: localOderId, token: 'session-token', displayName: localOderId })),
getIdentifyCredentials: vi.fn(() => credentialsFor(localOderId)),
getIdentifyCredentialsForPeer: vi.fn((peerId: string) =>
credentialsFor(localActorIdByPeerId[peerId] ?? localOderId)),
getLocalMediaStream: vi.fn(() => null),
getLocalPeerId: vi.fn(() => localOderId),
mayOpenVoicePathToPeer: vi.fn(() => true),
getVoiceStateSnapshot: vi.fn(() => ({
isConnected: true,
isMuted: false,
@@ -7,6 +7,7 @@ import {
PEER_RECONNECT_INTERVAL_MS,
PEER_RECONNECT_MAX_ATTEMPTS
} from '../../realtime.constants';
import { shouldInitiatePeerConnection } from '../../peer-role.rules';
import {
PeerConnectionManagerContext,
PeerConnectionManagerState,
@@ -205,6 +206,15 @@ export function scheduleDataChannelRecovery(
state.dataChannelRecoveryTimers.set(peerId, timer);
}
/**
* Repair a failed control channel with the least destructive option available.
*
* While the `RTCPeerConnection` is still connected the channel is replaced on that same
* connection, so voice, camera, and screen share keep flowing. Only the deterministically
* elected initiator creates the replacement; the other side adopts the incoming channel.
* A full peer rebuild is the fallback when the connection itself is gone, when the
* replacement cannot be created, or when the replacement never opens.
*/
function repairUnavailableDataChannel(
context: PeerConnectionManagerContext,
peerId: string,
@@ -212,7 +222,7 @@ function repairUnavailableDataChannel(
reason: string,
handlers: RecoveryHandlers
): void {
const { logger, state } = context;
const { callbacks, logger, state } = context;
const peerData = state.activePeerConnections.get(peerId);
if (!peerData || peerData.dataChannel !== channel)
@@ -221,11 +231,103 @@ function repairUnavailableDataChannel(
if (peerData.dataChannel?.readyState === DATA_CHANNEL_STATE_OPEN)
return;
logger.warn('[data-channel] Recreating peer transport after control channel failure', {
if (peerData.connection.connectionState !== CONNECTION_STATE_CONNECTED) {
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
return;
}
const localOderId = callbacks.getIdentifyCredentialsForPeer(peerId)?.oderId ?? null;
if (!localOderId) {
logger.warn('[data-channel] Logical identity unknown; rebuilding peer instead of replacing channel', {
peerId,
reason
});
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
return;
}
if (shouldInitiatePeerConnection(localOderId, peerId)) {
logger.warn('[data-channel] Replacing control channel on the live connection', {
channelLabel: channel.label,
peerId,
reason
});
if (!handlers.replaceDataChannel(peerId, channel)) {
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
return;
}
watchDataChannelReplacement(context, peerId, reason, handlers);
return;
}
logger.info('[data-channel] Waiting for the remote replacement control channel', {
channelLabel: channel.label,
connectionState: peerData.connection.connectionState,
localOderId,
peerId,
readyState: peerData.dataChannel?.readyState ?? null,
reason
});
watchDataChannelReplacement(context, peerId, reason, handlers);
}
/** Rebuild the peer if the replacement control channel does not open within the grace period. */
function watchDataChannelReplacement(
context: PeerConnectionManagerContext,
peerId: string,
reason: string,
handlers: RecoveryHandlers
): void {
const { logger, state } = context;
clearDataChannelRecoveryTimer(state, peerId);
const timer = setTimeout(() => {
state.dataChannelRecoveryTimers.delete(peerId);
const latestPeerData = state.activePeerConnections.get(peerId);
if (!latestPeerData)
return;
if (latestPeerData.dataChannel?.readyState === DATA_CHANNEL_STATE_OPEN) {
logger.info('[data-channel] Replacement control channel is open; peer kept alive', {
peerId,
reason
});
return;
}
logger.warn('[data-channel] Replacement control channel never opened; rebuilding peer', {
connectionState: latestPeerData.connection.connectionState,
peerId,
readyState: latestPeerData.dataChannel?.readyState ?? null,
reason
});
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
}, DATA_CHANNEL_RECOVERY_GRACE_MS);
state.dataChannelRecoveryTimers.set(peerId, timer);
}
function rebuildPeerAfterDataChannelFailure(
context: PeerConnectionManagerContext,
peerId: string,
reason: string,
handlers: RecoveryHandlers
): void {
const { logger, state } = context;
const peerData = state.activePeerConnections.get(peerId);
logger.warn('[data-channel] Recreating peer transport after control channel failure', {
connectionState: peerData?.connection.connectionState ?? null,
peerId,
readyState: peerData?.dataChannel?.readyState ?? null,
reason
});
@@ -299,30 +401,84 @@ export function schedulePeerReconnect(
return;
}
// An attempt costs nothing while the socket is down and there is no way to send an
// offer, so the budget must only be spent on attempts that can actually reach the peer.
if (!callbacks.isSignalingConnected()) {
logger.info('Deferring P2P reconnect - no signaling connection', {
peerId,
attemptsSpent: info.reconnectAttempts
});
return;
}
if (info.reconnectAttempts >= PEER_RECONNECT_MAX_ATTEMPTS) {
failPeerRecovery(context, peerId, info.reconnectAttempts);
return;
}
info.reconnectAttempts++;
logger.info('P2P reconnect attempt', {
peerId,
attempt: info.reconnectAttempts
});
if (info.reconnectAttempts >= PEER_RECONNECT_MAX_ATTEMPTS) {
logger.info('P2P reconnect max attempts reached', { peerId });
clearPeerReconnectTimer(state, peerId);
state.disconnectedPeerTracker.delete(peerId);
return;
}
if (!callbacks.isSignalingConnected()) {
logger.info('Skipping P2P reconnect - no signaling connection', { peerId });
return;
}
attemptPeerReconnect(context, peerId, handlers);
}, PEER_RECONNECT_INTERVAL_MS);
state.peerReconnectTimers.set(peerId, timer);
}
/**
* Stop retrying a peer and say so. The tracker entry is kept (marked failed) so
* `resumeStalledPeerRecovery` can re-arm it once signaling is usable again.
*/
function failPeerRecovery(
context: PeerConnectionManagerContext,
peerId: string,
attempts: number
): void {
const { logger, state } = context;
const info = state.disconnectedPeerTracker.get(peerId);
logger.warn('P2P reconnect gave up', { attempts, peerId });
clearPeerReconnectTimer(state, peerId);
if (info) {
info.recoveryFailed = true;
}
state.peerRecoveryStatus$.next({ attempts, peerId, status: 'failed' });
}
/**
* Re-arm every peer whose recovery gave up. Called when signaling reconnects, so a peer
* that ran out of attempts during an outage is retried instead of staying dead silently.
*/
export function resumeStalledPeerRecovery(
context: PeerConnectionManagerContext,
handlers: RecoveryHandlers
): void {
const { callbacks, logger, state } = context;
if (!callbacks.isSignalingConnected())
return;
state.disconnectedPeerTracker.forEach((info, peerId) => {
if (!info.recoveryFailed)
return;
logger.info('Re-arming P2P recovery after signaling returned', { peerId });
info.recoveryFailed = false;
info.reconnectAttempts = 0;
state.peerRecoveryStatus$.next({ attempts: 0, peerId, status: 'retrying' });
attemptPeerReconnect(context, peerId, handlers);
schedulePeerReconnect(context, peerId, handlers);
});
}
export function attemptPeerReconnect(
context: PeerConnectionManagerContext,
peerId: string,
@@ -334,7 +490,7 @@ export function attemptPeerReconnect(
handlers.removePeer(peerId, { preserveReconnectState: true });
}
const localOderId = callbacks.getIdentifyCredentials()?.oderId ?? null;
const localOderId = callbacks.getIdentifyCredentialsForPeer(peerId)?.oderId ?? null;
if (!localOderId) {
logger.info('Skipping reconnect offer until logical identity is ready', { peerId });
@@ -342,7 +498,7 @@ export function attemptPeerReconnect(
return;
}
const shouldInitiate = peerId !== localOderId && localOderId < peerId;
const shouldInitiate = shouldInitiatePeerConnection(localOderId, peerId);
handlers.createPeerConnection(peerId, shouldInitiate);