fix(realtime): survive signal outages and per-server identities
Peer recovery burned its whole retry budget while signaling was down, then dropped the tracker with no re-arm, so a peer stayed dead until an unrelated roster event healed it. Recovery now waits for a usable transport before spending an attempt. Initiator election also compared a home actor id against foreign roster ids, which is not antisymmetric across signal servers - both sides offered, or neither did. Election moves into `peer-role.rules` and compares ids only within one signal server's identity space.
This commit is contained in:
@@ -0,0 +1,230 @@
|
||||
import { expect, type Page } from '@playwright/test';
|
||||
import { test, type Client } from '../../fixtures/multi-client';
|
||||
import { countCreatedPeerConnections } from '../../helpers/peer-role';
|
||||
import { installDeterministicVoiceSettings } from '../../helpers/voice-session';
|
||||
import {
|
||||
closeOpenDataChannels,
|
||||
dumpRtcDiagnostics,
|
||||
getOpenDataChannelCount,
|
||||
installAutoResumeAudioContext,
|
||||
installWebRTCTracking,
|
||||
waitForAllPeerAudioFlow,
|
||||
waitForAudioStatsPresent,
|
||||
waitForConnectedPeerCount,
|
||||
waitForOpenDataChannelCount
|
||||
} from '../../helpers/webrtc-helpers';
|
||||
import { ChatRoomPage } from '../../pages/chat-room.page';
|
||||
import { RegisterPage } from '../../pages/register.page';
|
||||
import { ServerSearchPage } from '../../pages/server-search.page';
|
||||
|
||||
interface VoiceClient extends Client {
|
||||
displayName: string;
|
||||
username: string;
|
||||
}
|
||||
|
||||
const USER_PASSWORD = 'TestPass123!';
|
||||
const VOICE_CHANNEL = 'General';
|
||||
/** 12 reconnect attempts at 5s - the whole budget fits inside this outage. */
|
||||
const OUTAGE_HOLD_MS = 70_000;
|
||||
|
||||
test.describe('Recovery preserves live media', () => {
|
||||
test('replaces a dead control channel without rebuilding the peer connection', async ({ createClient }) => {
|
||||
test.setTimeout(240_000);
|
||||
|
||||
const clients = await createVoicePair(createClient, `DC Soft Replace ${Date.now()}`);
|
||||
const [alice, bob] = clients;
|
||||
|
||||
await assertMeshAudio(clients, 'initial two-user voice');
|
||||
|
||||
const connectionsBefore = {
|
||||
alice: await countCreatedPeerConnections(alice.page),
|
||||
bob: await countCreatedPeerConnections(bob.page)
|
||||
};
|
||||
|
||||
expect(connectionsBefore.alice).toBe(1);
|
||||
expect(connectionsBefore.bob).toBe(1);
|
||||
|
||||
await test.step('The control channel is replaced on the same connection', async () => {
|
||||
const closed = await closeOpenDataChannels(alice.page);
|
||||
|
||||
expect(closed).toBeGreaterThan(0);
|
||||
|
||||
await waitForOpenDataChannelCount(alice.page, 1, 60_000);
|
||||
await waitForOpenDataChannelCount(bob.page, 1, 60_000);
|
||||
|
||||
// A rebuild would construct a second RTCPeerConnection on both sides, taking voice,
|
||||
// camera, and screen share down with the control channel.
|
||||
expect(
|
||||
await countCreatedPeerConnections(alice.page),
|
||||
'Alice rebuilt her peer connection instead of replacing the control channel'
|
||||
).toBe(connectionsBefore.alice);
|
||||
|
||||
expect(
|
||||
await countCreatedPeerConnections(bob.page),
|
||||
'Bob rebuilt his peer connection instead of adopting the replacement control channel'
|
||||
).toBe(connectionsBefore.bob);
|
||||
});
|
||||
|
||||
await test.step('Audio never had to be renegotiated', async () => {
|
||||
await waitForConnectedPeerCount(alice.page, 1, 30_000);
|
||||
await waitForConnectedPeerCount(bob.page, 1, 30_000);
|
||||
await assertMeshAudio(clients, 'after control-channel replacement');
|
||||
});
|
||||
});
|
||||
|
||||
// This covers the user-visible half: an outage that outlives the 12-attempt reconnect
|
||||
// budget must not end the call. It cannot isolate the attempt accounting, because the
|
||||
// roster resync on signaling reconnect re-peers anyway - `peer-recovery.spec.ts` owns
|
||||
// the deterministic proof that a deferred attempt costs nothing.
|
||||
test('keeps voice alive across a signal outage longer than the reconnect budget', async ({
|
||||
createClient,
|
||||
testServer
|
||||
}) => {
|
||||
test.setTimeout(480_000);
|
||||
|
||||
const clients = await createVoicePair(createClient, `Signal Outage Voice ${Date.now()}`);
|
||||
|
||||
await assertMeshAudio(clients, 'initial two-user voice');
|
||||
|
||||
const connectionsBefore = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
|
||||
|
||||
expect(connectionsBefore).toEqual([1, 1]);
|
||||
|
||||
await test.step('The signal server goes away for longer than the reconnect budget', async () => {
|
||||
await testServer.kill();
|
||||
|
||||
for (const client of clients) {
|
||||
await waitForSignalingConnected(client.page, false, 60_000);
|
||||
}
|
||||
|
||||
await clients[0].page.waitForTimeout(OUTAGE_HOLD_MS);
|
||||
});
|
||||
|
||||
await test.step('Peer media is unaffected by the signaling outage', async () => {
|
||||
await assertMeshAudio(clients, 'during signal outage');
|
||||
});
|
||||
|
||||
await test.step('Voice is still healthy once signaling returns', async () => {
|
||||
await testServer.start();
|
||||
|
||||
for (const client of clients) {
|
||||
await waitForSignalingConnected(client.page, true, 120_000);
|
||||
}
|
||||
|
||||
for (const client of clients) {
|
||||
await waitForConnectedPeerCount(client.page, 1, 90_000);
|
||||
await waitForOpenDataChannelCount(client.page, 1, 90_000);
|
||||
}
|
||||
|
||||
await assertMeshAudio(clients, 'after signaling recovery');
|
||||
});
|
||||
|
||||
await test.step('The call was never rebuilt behind the user back', async () => {
|
||||
// Media never depended on the signal server, so the roster resync must adopt the
|
||||
// living peer connection. Rebuilding it would drop audio for a beat and reset
|
||||
// screen share - invisible to the assertions above, which only re-check the end state.
|
||||
const connectionsAfter = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
|
||||
|
||||
expect(connectionsAfter, 'a client rebuilt its peer connection when signaling came back').toEqual(
|
||||
connectionsBefore
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
async function createVoicePair(
|
||||
createClient: () => Promise<Client>,
|
||||
serverName: string
|
||||
): Promise<VoiceClient[]> {
|
||||
const clients: VoiceClient[] = [];
|
||||
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const client = await createClient();
|
||||
|
||||
await installDeterministicVoiceSettings(client.page);
|
||||
await installWebRTCTracking(client.page);
|
||||
await installAutoResumeAudioContext(client.page);
|
||||
|
||||
clients.push({
|
||||
...client,
|
||||
displayName: `Recovery Voice ${index + 1}`,
|
||||
username: `recovery_voice_${Date.now()}_${index + 1}`
|
||||
});
|
||||
}
|
||||
|
||||
await test.step('Register both clients', async () => {
|
||||
for (const client of clients) {
|
||||
const registerPage = new RegisterPage(client.page);
|
||||
|
||||
await registerPage.goto();
|
||||
await registerPage.register(client.username, client.displayName, USER_PASSWORD);
|
||||
await expect(client.page).toHaveURL(/\/dashboard/, { timeout: 20_000 });
|
||||
}
|
||||
});
|
||||
|
||||
await test.step('Create and join the server', async () => {
|
||||
await new ServerSearchPage(clients[0].page).createServer(serverName, {
|
||||
description: 'Recovery keeps live media test'
|
||||
});
|
||||
|
||||
await expect(clients[0].page).toHaveURL(/\/room\//, { timeout: 20_000 });
|
||||
|
||||
await new ServerSearchPage(clients[1].page).joinServerFromSearch(serverName);
|
||||
await expect(clients[1].page).toHaveURL(/\/room\//, { timeout: 20_000 });
|
||||
});
|
||||
|
||||
await test.step('Join both clients to voice', async () => {
|
||||
await new ChatRoomPage(clients[0].page).ensureVoiceChannelExists(VOICE_CHANNEL);
|
||||
|
||||
for (const client of clients) {
|
||||
const room = new ChatRoomPage(client.page);
|
||||
|
||||
await room.joinVoiceChannel(VOICE_CHANNEL);
|
||||
await expect(room.voiceControls).toBeVisible({ timeout: 20_000 });
|
||||
}
|
||||
|
||||
for (const client of clients) {
|
||||
await waitForConnectedPeerCount(client.page, 1, 90_000);
|
||||
await waitForOpenDataChannelCount(client.page, 1, 90_000);
|
||||
await waitForAudioStatsPresent(client.page, 30_000);
|
||||
}
|
||||
});
|
||||
|
||||
return clients;
|
||||
}
|
||||
|
||||
async function assertMeshAudio(clients: readonly VoiceClient[], label: string): Promise<void> {
|
||||
for (const client of clients) {
|
||||
try {
|
||||
await waitForAllPeerAudioFlow(client.page, 1, 60_000);
|
||||
} catch (error) {
|
||||
console.log(`[${client.displayName} ${label} data channels] ${await getOpenDataChannelCount(client.page)}`);
|
||||
console.log(`[${client.displayName} ${label} RTC]\n${await dumpRtcDiagnostics(client.page)}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Wait until the client's own view of its signaling connection matches `connected`. */
|
||||
async function waitForSignalingConnected(page: Page, connected: boolean, timeout: number): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
(expected) => {
|
||||
interface AngularDebugApi {
|
||||
getComponent: (element: Element) => Record<string, unknown>;
|
||||
}
|
||||
|
||||
const host = document.querySelector('app-rooms-side-panel');
|
||||
const debugApi = (window as { ng?: AngularDebugApi }).ng;
|
||||
|
||||
if (!host || !debugApi?.getComponent) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const realtime = debugApi.getComponent(host)['realtime'] as { isConnected?: () => boolean } | undefined;
|
||||
|
||||
return realtime?.isConnected?.() === expected;
|
||||
},
|
||||
connected,
|
||||
{ timeout }
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user