fix(realtime): survive signal outages and per-server identities
Peer recovery burned its whole retry budget while signaling was down, then dropped the tracker with no re-arm, so a peer stayed dead until an unrelated roster event healed it. Recovery now waits for a usable transport before spending an attempt. Initiator election also compared a home actor id against foreign roster ids, which is not antisymmetric across signal servers - both sides offered, or neither did. Election moves into `peer-role.rules` and compares ids only within one signal server's identity space.
This commit is contained in:
+27
-4
@@ -1,4 +1,4 @@
|
||||
/* eslint-disable @typescript-eslint/member-ordering */
|
||||
|
||||
import { ChatEvent } from '../../../shared-kernel';
|
||||
import { DATA_CHANNEL_LABEL } from '../realtime.constants';
|
||||
import { recordDebugNetworkDownloadRates } from '../logging/debug-network-metrics';
|
||||
@@ -15,7 +15,9 @@ import {
|
||||
} from './connection/negotiation';
|
||||
import {
|
||||
broadcastCurrentStates,
|
||||
broadcastIdentityScopedMessage,
|
||||
broadcastMessage,
|
||||
PeerScopedIdentity,
|
||||
sendCurrentStatesToPeer,
|
||||
sendToPeer,
|
||||
sendToPeerBuffered,
|
||||
@@ -32,6 +34,7 @@ import {
|
||||
removePeer as removeManagedPeer,
|
||||
requestVoiceStateFromPeer,
|
||||
resetConnectedPeers,
|
||||
resumeStalledPeerRecovery,
|
||||
scheduleDataChannelRecovery,
|
||||
schedulePeerDisconnectRecovery,
|
||||
schedulePeerReconnect,
|
||||
@@ -91,6 +94,10 @@ export class PeerConnectionManager {
|
||||
|
||||
readonly peerConnected$ = this.state.peerConnected$;
|
||||
readonly peerDisconnected$ = this.state.peerDisconnected$;
|
||||
|
||||
/** Emitted when peer recovery gives up on a peer, or re-arms it. */
|
||||
readonly peerRecoveryStatus$ = this.state.peerRecoveryStatus$;
|
||||
|
||||
readonly remoteStream$ = this.state.remoteStream$;
|
||||
readonly messageReceived$ = this.state.messageReceived$;
|
||||
|
||||
@@ -181,10 +188,20 @@ export class PeerConnectionManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a ChatEvent to a specific peer's data channel.
|
||||
* Broadcast a self-identifying ChatEvent, rebuilt per peer so each peer receives the
|
||||
* actor id it knows us by on its own signal server.
|
||||
*/
|
||||
sendToPeer(peerId: string, event: ChatEvent): void {
|
||||
sendToPeer(this.context, peerId, event);
|
||||
broadcastIdentityScopedMessage(buildEvent: (identity: PeerScopedIdentity) => ChatEvent): void {
|
||||
broadcastIdentityScopedMessage(this.context, buildEvent);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a ChatEvent to a specific peer's data channel.
|
||||
*
|
||||
* @returns whether the payload reached the peer's open data channel.
|
||||
*/
|
||||
sendToPeer(peerId: string, event: ChatEvent): boolean {
|
||||
return sendToPeer(this.context, peerId, event);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -225,6 +242,11 @@ export class PeerConnectionManager {
|
||||
clearAllPeerReconnectTimers(this.state);
|
||||
}
|
||||
|
||||
/** Retry every peer whose reconnect budget ran out, now that signaling works again. */
|
||||
resumeStalledPeerRecovery(): void {
|
||||
resumeStalledPeerRecovery(this.context, this.recoveryHandlers);
|
||||
}
|
||||
|
||||
/** Return a snapshot copy of the currently-connected peer IDs. */
|
||||
getConnectedPeerIds(): string[] {
|
||||
return getConnectedPeerIds(this.state);
|
||||
@@ -247,6 +269,7 @@ export class PeerConnectionManager {
|
||||
this.closeAllPeers();
|
||||
this.peerConnected$.complete();
|
||||
this.peerDisconnected$.complete();
|
||||
this.peerRecoveryStatus$.complete();
|
||||
this.remoteStream$.complete();
|
||||
this.messageReceived$.complete();
|
||||
this.connectedPeersChanged$.complete();
|
||||
|
||||
+221
-13
@@ -1,5 +1,10 @@
|
||||
import { DATA_CHANNEL_RECOVERY_GRACE_MS, DATA_CHANNEL_STATE_OPEN } from '../../realtime.constants';
|
||||
import type { PeerData } from '../../realtime.types';
|
||||
import {
|
||||
DATA_CHANNEL_RECOVERY_GRACE_MS,
|
||||
DATA_CHANNEL_STATE_OPEN,
|
||||
PEER_RECONNECT_INTERVAL_MS,
|
||||
PEER_RECONNECT_MAX_ATTEMPTS
|
||||
} from '../../realtime.constants';
|
||||
import type { PeerData, PeerRecoveryStatusEvent } from '../../realtime.types';
|
||||
import {
|
||||
createPeerConnectionManagerState,
|
||||
PeerConnectionManagerContext,
|
||||
@@ -8,7 +13,10 @@ import {
|
||||
import {
|
||||
closeAllPeers,
|
||||
removePeer,
|
||||
scheduleDataChannelRecovery
|
||||
resumeStalledPeerRecovery,
|
||||
scheduleDataChannelRecovery,
|
||||
schedulePeerReconnect,
|
||||
trackDisconnectedPeer
|
||||
} from './peer-recovery';
|
||||
|
||||
describe('peer recovery', () => {
|
||||
@@ -51,7 +59,25 @@ describe('peer recovery', () => {
|
||||
expect(context.state.remotePeerCameraStreams.size).toBe(0);
|
||||
});
|
||||
|
||||
it('recreates a peer immediately when the data channel is already closed', () => {
|
||||
it('replaces the data channel on the live connection instead of tearing down media', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const channel = createDataChannel('closed');
|
||||
const context = createContext('alice');
|
||||
const handlers = createRecoveryHandlers(context);
|
||||
const peerData = createPeerData(channel, 'connected');
|
||||
|
||||
context.state.activePeerConnections.set('bob', peerData);
|
||||
|
||||
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
|
||||
|
||||
expect(handlers.replaceDataChannel).toHaveBeenCalledWith('bob', channel);
|
||||
expect(handlers.removePeer).not.toHaveBeenCalled();
|
||||
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
|
||||
expect(context.state.activePeerConnections.get('bob')?.connection).toBe(peerData.connection);
|
||||
});
|
||||
|
||||
it('keeps the connection when the replacement data channel opens', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const channel = createDataChannel('closed');
|
||||
@@ -62,13 +88,38 @@ describe('peer recovery', () => {
|
||||
|
||||
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
|
||||
|
||||
const replaced = context.state.activePeerConnections.get('bob');
|
||||
|
||||
if (replaced) {
|
||||
replaced.dataChannel = createDataChannel(DATA_CHANNEL_STATE_OPEN);
|
||||
}
|
||||
|
||||
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
|
||||
|
||||
expect(handlers.removePeer).not.toHaveBeenCalled();
|
||||
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rebuilds the peer when the replacement data channel never opens', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const channel = createDataChannel('closed');
|
||||
const context = createContext('alice');
|
||||
const handlers = createRecoveryHandlers(context);
|
||||
|
||||
context.state.activePeerConnections.set('bob', createPeerData(channel, 'connected'));
|
||||
|
||||
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
|
||||
expect(handlers.removePeer).not.toHaveBeenCalled();
|
||||
|
||||
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
|
||||
|
||||
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
|
||||
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
|
||||
expect(handlers.createAndSendOffer).toHaveBeenCalledWith('bob');
|
||||
expect(context.state.dataChannelRecoveryTimers.has('bob')).toBe(false);
|
||||
});
|
||||
|
||||
it('waits a short grace period before recreating a peer with a closing data channel', () => {
|
||||
it('waits a short grace period before repairing a closing data channel', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const channel = createDataChannel('closing');
|
||||
@@ -80,14 +131,28 @@ describe('peer recovery', () => {
|
||||
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
|
||||
|
||||
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS - 1);
|
||||
expect(handlers.removePeer).not.toHaveBeenCalled();
|
||||
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
|
||||
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
|
||||
|
||||
vi.advanceTimersByTime(1);
|
||||
|
||||
expect(handlers.replaceDataChannel).toHaveBeenCalledWith('bob', channel);
|
||||
expect(handlers.removePeer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rebuilds instead of replacing the channel when the connection is no longer connected', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const channel = createDataChannel('closed');
|
||||
const context = createContext('alice');
|
||||
const handlers = createRecoveryHandlers(context);
|
||||
|
||||
context.state.activePeerConnections.set('bob', createPeerData(channel, 'disconnected'));
|
||||
|
||||
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
|
||||
|
||||
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
|
||||
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
|
||||
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
|
||||
expect(handlers.createAndSendOffer).toHaveBeenCalledWith('bob');
|
||||
});
|
||||
|
||||
it('does not recreate a peer when a replacement data channel is adopted before the grace expires', () => {
|
||||
@@ -127,7 +192,7 @@ describe('peer recovery', () => {
|
||||
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('recreates a connected non-initiator peer and waits for the remote initiator offer', () => {
|
||||
it('lets a non-initiator wait for the remote replacement channel before rebuilding', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const channel = createDataChannel('closed');
|
||||
@@ -137,12 +202,143 @@ describe('peer recovery', () => {
|
||||
context.state.activePeerConnections.set('bob', createPeerData(channel, 'connected', false));
|
||||
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
|
||||
|
||||
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
|
||||
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
|
||||
expect(handlers.removePeer).not.toHaveBeenCalled();
|
||||
|
||||
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
|
||||
|
||||
expect(handlers.removePeer).toHaveBeenCalledWith('bob', { preserveReconnectState: true });
|
||||
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', false);
|
||||
expect(handlers.createAndSendOffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('adopts a remote replacement channel without rebuilding the non-initiator peer', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const channel = createDataChannel('closed');
|
||||
const context = createContext('zoe');
|
||||
const handlers = createRecoveryHandlers(context);
|
||||
const peerData = createPeerData(channel, 'connected', false);
|
||||
|
||||
context.state.activePeerConnections.set('bob', peerData);
|
||||
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
|
||||
|
||||
peerData.dataChannel = createDataChannel(DATA_CHANNEL_STATE_OPEN);
|
||||
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
|
||||
|
||||
expect(handlers.removePeer).not.toHaveBeenCalled();
|
||||
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('elects the channel repair role from the actor id on the peer signal server', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
// Home id sorts before "bob", but on the signal server that routes bob this human is
|
||||
// "zoe-foreign" and sorts after - so the remote side owns the replacement, not us.
|
||||
const channel = createDataChannel('closed');
|
||||
const context = createContext('aa-home', { bob: 'zoe-foreign' });
|
||||
const handlers = createRecoveryHandlers(context);
|
||||
|
||||
context.state.activePeerConnections.set('bob', createPeerData(channel, 'connected'));
|
||||
scheduleDataChannelRecovery(context, 'bob', channel, 'close', handlers);
|
||||
|
||||
expect(handlers.replaceDataChannel).not.toHaveBeenCalled();
|
||||
|
||||
vi.advanceTimersByTime(DATA_CHANNEL_RECOVERY_GRACE_MS);
|
||||
|
||||
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', false);
|
||||
expect(handlers.createAndSendOffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not spend reconnect attempts while signaling is disconnected', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const context = createContext('alice');
|
||||
const handlers = createRecoveryHandlers(context);
|
||||
|
||||
let signalingConnected = false;
|
||||
|
||||
context.callbacks.isSignalingConnected = vi.fn(() => signalingConnected);
|
||||
|
||||
trackDisconnectedPeer(context.state, 'bob');
|
||||
schedulePeerReconnect(context, 'bob', handlers);
|
||||
|
||||
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * (PEER_RECONNECT_MAX_ATTEMPTS + 2));
|
||||
|
||||
expect(context.state.disconnectedPeerTracker.get('bob')?.reconnectAttempts).toBe(0);
|
||||
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
|
||||
expect(context.state.peerReconnectTimers.has('bob')).toBe(true);
|
||||
|
||||
signalingConnected = true;
|
||||
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS);
|
||||
|
||||
expect(context.state.disconnectedPeerTracker.get('bob')?.reconnectAttempts).toBe(1);
|
||||
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
|
||||
});
|
||||
|
||||
it('publishes a failed recovery state when every attempt is spent', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const context = createContext('alice');
|
||||
const handlers = createRecoveryHandlers(context);
|
||||
const statuses: PeerRecoveryStatusEvent[] = [];
|
||||
|
||||
context.state.peerRecoveryStatus$.subscribe((event) => statuses.push(event));
|
||||
|
||||
trackDisconnectedPeer(context.state, 'bob');
|
||||
schedulePeerReconnect(context, 'bob', handlers);
|
||||
|
||||
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * PEER_RECONNECT_MAX_ATTEMPTS);
|
||||
expect(statuses).toEqual([]);
|
||||
|
||||
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS);
|
||||
|
||||
expect(statuses).toEqual([{ attempts: PEER_RECONNECT_MAX_ATTEMPTS, peerId: 'bob', status: 'failed' }]);
|
||||
|
||||
expect(context.state.peerReconnectTimers.has('bob')).toBe(false);
|
||||
expect(context.state.disconnectedPeerTracker.get('bob')?.recoveryFailed).toBe(true);
|
||||
});
|
||||
|
||||
it('re-arms failed peer recovery when signaling comes back', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const context = createContext('alice');
|
||||
const handlers = createRecoveryHandlers(context);
|
||||
const statuses: PeerRecoveryStatusEvent[] = [];
|
||||
|
||||
context.state.peerRecoveryStatus$.subscribe((event) => statuses.push(event));
|
||||
|
||||
trackDisconnectedPeer(context.state, 'bob');
|
||||
schedulePeerReconnect(context, 'bob', handlers);
|
||||
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * (PEER_RECONNECT_MAX_ATTEMPTS + 1));
|
||||
|
||||
handlers.createPeerConnection.mockClear();
|
||||
resumeStalledPeerRecovery(context, handlers);
|
||||
|
||||
expect(statuses.at(-1)).toEqual({ attempts: 0, peerId: 'bob', status: 'retrying' });
|
||||
expect(context.state.disconnectedPeerTracker.get('bob')?.recoveryFailed).toBe(false);
|
||||
expect(handlers.createPeerConnection).toHaveBeenCalledWith('bob', true);
|
||||
expect(context.state.peerReconnectTimers.has('bob')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not re-arm failed peer recovery while signaling is still down', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
const context = createContext('alice');
|
||||
const handlers = createRecoveryHandlers(context);
|
||||
|
||||
trackDisconnectedPeer(context.state, 'bob');
|
||||
schedulePeerReconnect(context, 'bob', handlers);
|
||||
vi.advanceTimersByTime(PEER_RECONNECT_INTERVAL_MS * (PEER_RECONNECT_MAX_ATTEMPTS + 1));
|
||||
|
||||
handlers.createPeerConnection.mockClear();
|
||||
context.callbacks.isSignalingConnected = vi.fn(() => false);
|
||||
resumeStalledPeerRecovery(context, handlers);
|
||||
|
||||
expect(handlers.createPeerConnection).not.toHaveBeenCalled();
|
||||
expect(context.state.disconnectedPeerTracker.get('bob')?.recoveryFailed).toBe(true);
|
||||
});
|
||||
|
||||
it('waits for the remote initiator when a non-connected peer needs full reconnect', () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
@@ -160,7 +356,16 @@ describe('peer recovery', () => {
|
||||
});
|
||||
});
|
||||
|
||||
function createContext(localOderId: string): PeerConnectionManagerContext {
|
||||
function createContext(
|
||||
localOderId: string,
|
||||
localActorIdByPeerId: Record<string, string> = {}
|
||||
): PeerConnectionManagerContext {
|
||||
const credentialsFor = (oderId: string) => ({
|
||||
displayName: oderId,
|
||||
oderId,
|
||||
token: 'session-token'
|
||||
});
|
||||
|
||||
return {
|
||||
logger: {
|
||||
error: vi.fn(),
|
||||
@@ -171,9 +376,12 @@ function createContext(localOderId: string): PeerConnectionManagerContext {
|
||||
} as unknown as PeerConnectionManagerContext['logger'],
|
||||
callbacks: {
|
||||
getIceServers: vi.fn(() => []),
|
||||
getIdentifyCredentials: vi.fn(() => ({ oderId: localOderId, token: 'session-token', displayName: localOderId })),
|
||||
getIdentifyCredentials: vi.fn(() => credentialsFor(localOderId)),
|
||||
getIdentifyCredentialsForPeer: vi.fn((peerId: string) =>
|
||||
credentialsFor(localActorIdByPeerId[peerId] ?? localOderId)),
|
||||
getLocalMediaStream: vi.fn(() => null),
|
||||
getLocalPeerId: vi.fn(() => localOderId),
|
||||
mayOpenVoicePathToPeer: vi.fn(() => true),
|
||||
getVoiceStateSnapshot: vi.fn(() => ({
|
||||
isConnected: true,
|
||||
isMuted: false,
|
||||
|
||||
+174
-18
@@ -7,6 +7,7 @@ import {
|
||||
PEER_RECONNECT_INTERVAL_MS,
|
||||
PEER_RECONNECT_MAX_ATTEMPTS
|
||||
} from '../../realtime.constants';
|
||||
import { shouldInitiatePeerConnection } from '../../peer-role.rules';
|
||||
import {
|
||||
PeerConnectionManagerContext,
|
||||
PeerConnectionManagerState,
|
||||
@@ -205,6 +206,15 @@ export function scheduleDataChannelRecovery(
|
||||
state.dataChannelRecoveryTimers.set(peerId, timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Repair a failed control channel with the least destructive option available.
|
||||
*
|
||||
* While the `RTCPeerConnection` is still connected the channel is replaced on that same
|
||||
* connection, so voice, camera, and screen share keep flowing. Only the deterministically
|
||||
* elected initiator creates the replacement; the other side adopts the incoming channel.
|
||||
* A full peer rebuild is the fallback when the connection itself is gone, when the
|
||||
* replacement cannot be created, or when the replacement never opens.
|
||||
*/
|
||||
function repairUnavailableDataChannel(
|
||||
context: PeerConnectionManagerContext,
|
||||
peerId: string,
|
||||
@@ -212,7 +222,7 @@ function repairUnavailableDataChannel(
|
||||
reason: string,
|
||||
handlers: RecoveryHandlers
|
||||
): void {
|
||||
const { logger, state } = context;
|
||||
const { callbacks, logger, state } = context;
|
||||
const peerData = state.activePeerConnections.get(peerId);
|
||||
|
||||
if (!peerData || peerData.dataChannel !== channel)
|
||||
@@ -221,11 +231,103 @@ function repairUnavailableDataChannel(
|
||||
if (peerData.dataChannel?.readyState === DATA_CHANNEL_STATE_OPEN)
|
||||
return;
|
||||
|
||||
logger.warn('[data-channel] Recreating peer transport after control channel failure', {
|
||||
if (peerData.connection.connectionState !== CONNECTION_STATE_CONNECTED) {
|
||||
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
|
||||
return;
|
||||
}
|
||||
|
||||
const localOderId = callbacks.getIdentifyCredentialsForPeer(peerId)?.oderId ?? null;
|
||||
|
||||
if (!localOderId) {
|
||||
logger.warn('[data-channel] Logical identity unknown; rebuilding peer instead of replacing channel', {
|
||||
peerId,
|
||||
reason
|
||||
});
|
||||
|
||||
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
|
||||
return;
|
||||
}
|
||||
|
||||
if (shouldInitiatePeerConnection(localOderId, peerId)) {
|
||||
logger.warn('[data-channel] Replacing control channel on the live connection', {
|
||||
channelLabel: channel.label,
|
||||
peerId,
|
||||
reason
|
||||
});
|
||||
|
||||
if (!handlers.replaceDataChannel(peerId, channel)) {
|
||||
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
|
||||
return;
|
||||
}
|
||||
|
||||
watchDataChannelReplacement(context, peerId, reason, handlers);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info('[data-channel] Waiting for the remote replacement control channel', {
|
||||
channelLabel: channel.label,
|
||||
connectionState: peerData.connection.connectionState,
|
||||
localOderId,
|
||||
peerId,
|
||||
readyState: peerData.dataChannel?.readyState ?? null,
|
||||
reason
|
||||
});
|
||||
|
||||
watchDataChannelReplacement(context, peerId, reason, handlers);
|
||||
}
|
||||
|
||||
/** Rebuild the peer if the replacement control channel does not open within the grace period. */
|
||||
function watchDataChannelReplacement(
|
||||
context: PeerConnectionManagerContext,
|
||||
peerId: string,
|
||||
reason: string,
|
||||
handlers: RecoveryHandlers
|
||||
): void {
|
||||
const { logger, state } = context;
|
||||
|
||||
clearDataChannelRecoveryTimer(state, peerId);
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
state.dataChannelRecoveryTimers.delete(peerId);
|
||||
|
||||
const latestPeerData = state.activePeerConnections.get(peerId);
|
||||
|
||||
if (!latestPeerData)
|
||||
return;
|
||||
|
||||
if (latestPeerData.dataChannel?.readyState === DATA_CHANNEL_STATE_OPEN) {
|
||||
logger.info('[data-channel] Replacement control channel is open; peer kept alive', {
|
||||
peerId,
|
||||
reason
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
logger.warn('[data-channel] Replacement control channel never opened; rebuilding peer', {
|
||||
connectionState: latestPeerData.connection.connectionState,
|
||||
peerId,
|
||||
readyState: latestPeerData.dataChannel?.readyState ?? null,
|
||||
reason
|
||||
});
|
||||
|
||||
rebuildPeerAfterDataChannelFailure(context, peerId, reason, handlers);
|
||||
}, DATA_CHANNEL_RECOVERY_GRACE_MS);
|
||||
|
||||
state.dataChannelRecoveryTimers.set(peerId, timer);
|
||||
}
|
||||
|
||||
function rebuildPeerAfterDataChannelFailure(
|
||||
context: PeerConnectionManagerContext,
|
||||
peerId: string,
|
||||
reason: string,
|
||||
handlers: RecoveryHandlers
|
||||
): void {
|
||||
const { logger, state } = context;
|
||||
const peerData = state.activePeerConnections.get(peerId);
|
||||
|
||||
logger.warn('[data-channel] Recreating peer transport after control channel failure', {
|
||||
connectionState: peerData?.connection.connectionState ?? null,
|
||||
peerId,
|
||||
readyState: peerData?.dataChannel?.readyState ?? null,
|
||||
reason
|
||||
});
|
||||
|
||||
@@ -299,30 +401,84 @@ export function schedulePeerReconnect(
|
||||
return;
|
||||
}
|
||||
|
||||
// An attempt costs nothing while the socket is down and there is no way to send an
|
||||
// offer, so the budget must only be spent on attempts that can actually reach the peer.
|
||||
if (!callbacks.isSignalingConnected()) {
|
||||
logger.info('Deferring P2P reconnect - no signaling connection', {
|
||||
peerId,
|
||||
attemptsSpent: info.reconnectAttempts
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (info.reconnectAttempts >= PEER_RECONNECT_MAX_ATTEMPTS) {
|
||||
failPeerRecovery(context, peerId, info.reconnectAttempts);
|
||||
return;
|
||||
}
|
||||
|
||||
info.reconnectAttempts++;
|
||||
logger.info('P2P reconnect attempt', {
|
||||
peerId,
|
||||
attempt: info.reconnectAttempts
|
||||
});
|
||||
|
||||
if (info.reconnectAttempts >= PEER_RECONNECT_MAX_ATTEMPTS) {
|
||||
logger.info('P2P reconnect max attempts reached', { peerId });
|
||||
clearPeerReconnectTimer(state, peerId);
|
||||
state.disconnectedPeerTracker.delete(peerId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!callbacks.isSignalingConnected()) {
|
||||
logger.info('Skipping P2P reconnect - no signaling connection', { peerId });
|
||||
return;
|
||||
}
|
||||
|
||||
attemptPeerReconnect(context, peerId, handlers);
|
||||
}, PEER_RECONNECT_INTERVAL_MS);
|
||||
|
||||
state.peerReconnectTimers.set(peerId, timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop retrying a peer and say so. The tracker entry is kept (marked failed) so
|
||||
* `resumeStalledPeerRecovery` can re-arm it once signaling is usable again.
|
||||
*/
|
||||
function failPeerRecovery(
|
||||
context: PeerConnectionManagerContext,
|
||||
peerId: string,
|
||||
attempts: number
|
||||
): void {
|
||||
const { logger, state } = context;
|
||||
const info = state.disconnectedPeerTracker.get(peerId);
|
||||
|
||||
logger.warn('P2P reconnect gave up', { attempts, peerId });
|
||||
clearPeerReconnectTimer(state, peerId);
|
||||
|
||||
if (info) {
|
||||
info.recoveryFailed = true;
|
||||
}
|
||||
|
||||
state.peerRecoveryStatus$.next({ attempts, peerId, status: 'failed' });
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-arm every peer whose recovery gave up. Called when signaling reconnects, so a peer
|
||||
* that ran out of attempts during an outage is retried instead of staying dead silently.
|
||||
*/
|
||||
export function resumeStalledPeerRecovery(
|
||||
context: PeerConnectionManagerContext,
|
||||
handlers: RecoveryHandlers
|
||||
): void {
|
||||
const { callbacks, logger, state } = context;
|
||||
|
||||
if (!callbacks.isSignalingConnected())
|
||||
return;
|
||||
|
||||
state.disconnectedPeerTracker.forEach((info, peerId) => {
|
||||
if (!info.recoveryFailed)
|
||||
return;
|
||||
|
||||
logger.info('Re-arming P2P recovery after signaling returned', { peerId });
|
||||
|
||||
info.recoveryFailed = false;
|
||||
info.reconnectAttempts = 0;
|
||||
state.peerRecoveryStatus$.next({ attempts: 0, peerId, status: 'retrying' });
|
||||
|
||||
attemptPeerReconnect(context, peerId, handlers);
|
||||
schedulePeerReconnect(context, peerId, handlers);
|
||||
});
|
||||
}
|
||||
|
||||
export function attemptPeerReconnect(
|
||||
context: PeerConnectionManagerContext,
|
||||
peerId: string,
|
||||
@@ -334,7 +490,7 @@ export function attemptPeerReconnect(
|
||||
handlers.removePeer(peerId, { preserveReconnectState: true });
|
||||
}
|
||||
|
||||
const localOderId = callbacks.getIdentifyCredentials()?.oderId ?? null;
|
||||
const localOderId = callbacks.getIdentifyCredentialsForPeer(peerId)?.oderId ?? null;
|
||||
|
||||
if (!localOderId) {
|
||||
logger.info('Skipping reconnect offer until logical identity is ready', { peerId });
|
||||
@@ -342,7 +498,7 @@ export function attemptPeerReconnect(
|
||||
return;
|
||||
}
|
||||
|
||||
const shouldInitiate = peerId !== localOderId && localOderId < peerId;
|
||||
const shouldInitiate = shouldInitiatePeerConnection(localOderId, peerId);
|
||||
|
||||
handlers.createPeerConnection(peerId, shouldInitiate);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user