Files
Toju/e2e/tests/voice/recovery-preserves-media.spec.ts
myxelium a83f5aa750 fix(realtime): survive signal outages and per-server identities
Peer recovery burned its whole retry budget while signaling was down, then
dropped the tracker with no re-arm, so a peer stayed dead until an unrelated
roster event healed it. Recovery now waits for a usable transport before
spending an attempt.

Initiator election also compared a home actor id against foreign roster ids,
which is not antisymmetric across signal servers - both sides offered, or
neither did. Election moves into `peer-role.rules` and compares ids only
within one signal server's identity space.
2026-08-14 03:19:29 +02:00

231 lines
8.3 KiB
TypeScript

import { expect, type Page } from '@playwright/test';
import { test, type Client } from '../../fixtures/multi-client';
import { countCreatedPeerConnections } from '../../helpers/peer-role';
import { installDeterministicVoiceSettings } from '../../helpers/voice-session';
import {
closeOpenDataChannels,
dumpRtcDiagnostics,
getOpenDataChannelCount,
installAutoResumeAudioContext,
installWebRTCTracking,
waitForAllPeerAudioFlow,
waitForAudioStatsPresent,
waitForConnectedPeerCount,
waitForOpenDataChannelCount
} from '../../helpers/webrtc-helpers';
import { ChatRoomPage } from '../../pages/chat-room.page';
import { RegisterPage } from '../../pages/register.page';
import { ServerSearchPage } from '../../pages/server-search.page';
interface VoiceClient extends Client {
displayName: string;
username: string;
}
const USER_PASSWORD = 'TestPass123!';
const VOICE_CHANNEL = 'General';
/** 12 reconnect attempts at 5s - the whole budget fits inside this outage. */
const OUTAGE_HOLD_MS = 70_000;
test.describe('Recovery preserves live media', () => {
test('replaces a dead control channel without rebuilding the peer connection', async ({ createClient }) => {
test.setTimeout(240_000);
const clients = await createVoicePair(createClient, `DC Soft Replace ${Date.now()}`);
const [alice, bob] = clients;
await assertMeshAudio(clients, 'initial two-user voice');
const connectionsBefore = {
alice: await countCreatedPeerConnections(alice.page),
bob: await countCreatedPeerConnections(bob.page)
};
expect(connectionsBefore.alice).toBe(1);
expect(connectionsBefore.bob).toBe(1);
await test.step('The control channel is replaced on the same connection', async () => {
const closed = await closeOpenDataChannels(alice.page);
expect(closed).toBeGreaterThan(0);
await waitForOpenDataChannelCount(alice.page, 1, 60_000);
await waitForOpenDataChannelCount(bob.page, 1, 60_000);
// A rebuild would construct a second RTCPeerConnection on both sides, taking voice,
// camera, and screen share down with the control channel.
expect(
await countCreatedPeerConnections(alice.page),
'Alice rebuilt her peer connection instead of replacing the control channel'
).toBe(connectionsBefore.alice);
expect(
await countCreatedPeerConnections(bob.page),
'Bob rebuilt his peer connection instead of adopting the replacement control channel'
).toBe(connectionsBefore.bob);
});
await test.step('Audio never had to be renegotiated', async () => {
await waitForConnectedPeerCount(alice.page, 1, 30_000);
await waitForConnectedPeerCount(bob.page, 1, 30_000);
await assertMeshAudio(clients, 'after control-channel replacement');
});
});
// This covers the user-visible half: an outage that outlives the 12-attempt reconnect
// budget must not end the call. It cannot isolate the attempt accounting, because the
// roster resync on signaling reconnect re-peers anyway - `peer-recovery.spec.ts` owns
// the deterministic proof that a deferred attempt costs nothing.
test('keeps voice alive across a signal outage longer than the reconnect budget', async ({
createClient,
testServer
}) => {
test.setTimeout(480_000);
const clients = await createVoicePair(createClient, `Signal Outage Voice ${Date.now()}`);
await assertMeshAudio(clients, 'initial two-user voice');
const connectionsBefore = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
expect(connectionsBefore).toEqual([1, 1]);
await test.step('The signal server goes away for longer than the reconnect budget', async () => {
await testServer.kill();
for (const client of clients) {
await waitForSignalingConnected(client.page, false, 60_000);
}
await clients[0].page.waitForTimeout(OUTAGE_HOLD_MS);
});
await test.step('Peer media is unaffected by the signaling outage', async () => {
await assertMeshAudio(clients, 'during signal outage');
});
await test.step('Voice is still healthy once signaling returns', async () => {
await testServer.start();
for (const client of clients) {
await waitForSignalingConnected(client.page, true, 120_000);
}
for (const client of clients) {
await waitForConnectedPeerCount(client.page, 1, 90_000);
await waitForOpenDataChannelCount(client.page, 1, 90_000);
}
await assertMeshAudio(clients, 'after signaling recovery');
});
await test.step('The call was never rebuilt behind the user back', async () => {
// Media never depended on the signal server, so the roster resync must adopt the
// living peer connection. Rebuilding it would drop audio for a beat and reset
// screen share - invisible to the assertions above, which only re-check the end state.
const connectionsAfter = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
expect(connectionsAfter, 'a client rebuilt its peer connection when signaling came back').toEqual(
connectionsBefore
);
});
});
});
async function createVoicePair(
createClient: () => Promise<Client>,
serverName: string
): Promise<VoiceClient[]> {
const clients: VoiceClient[] = [];
for (let index = 0; index < 2; index++) {
const client = await createClient();
await installDeterministicVoiceSettings(client.page);
await installWebRTCTracking(client.page);
await installAutoResumeAudioContext(client.page);
clients.push({
...client,
displayName: `Recovery Voice ${index + 1}`,
username: `recovery_voice_${Date.now()}_${index + 1}`
});
}
await test.step('Register both clients', async () => {
for (const client of clients) {
const registerPage = new RegisterPage(client.page);
await registerPage.goto();
await registerPage.register(client.username, client.displayName, USER_PASSWORD);
await expect(client.page).toHaveURL(/\/dashboard/, { timeout: 20_000 });
}
});
await test.step('Create and join the server', async () => {
await new ServerSearchPage(clients[0].page).createServer(serverName, {
description: 'Recovery keeps live media test'
});
await expect(clients[0].page).toHaveURL(/\/room\//, { timeout: 20_000 });
await new ServerSearchPage(clients[1].page).joinServerFromSearch(serverName);
await expect(clients[1].page).toHaveURL(/\/room\//, { timeout: 20_000 });
});
await test.step('Join both clients to voice', async () => {
await new ChatRoomPage(clients[0].page).ensureVoiceChannelExists(VOICE_CHANNEL);
for (const client of clients) {
const room = new ChatRoomPage(client.page);
await room.joinVoiceChannel(VOICE_CHANNEL);
await expect(room.voiceControls).toBeVisible({ timeout: 20_000 });
}
for (const client of clients) {
await waitForConnectedPeerCount(client.page, 1, 90_000);
await waitForOpenDataChannelCount(client.page, 1, 90_000);
await waitForAudioStatsPresent(client.page, 30_000);
}
});
return clients;
}
async function assertMeshAudio(clients: readonly VoiceClient[], label: string): Promise<void> {
for (const client of clients) {
try {
await waitForAllPeerAudioFlow(client.page, 1, 60_000);
} catch (error) {
console.log(`[${client.displayName} ${label} data channels] ${await getOpenDataChannelCount(client.page)}`);
console.log(`[${client.displayName} ${label} RTC]\n${await dumpRtcDiagnostics(client.page)}`);
throw error;
}
}
}
/** Wait until the client's own view of its signaling connection matches `connected`. */
async function waitForSignalingConnected(page: Page, connected: boolean, timeout: number): Promise<void> {
await page.waitForFunction(
(expected) => {
interface AngularDebugApi {
getComponent: (element: Element) => Record<string, unknown>;
}
const host = document.querySelector('app-rooms-side-panel');
const debugApi = (window as { ng?: AngularDebugApi }).ng;
if (!host || !debugApi?.getComponent) {
return false;
}
const realtime = debugApi.getComponent(host)['realtime'] as { isConnected?: () => boolean } | undefined;
return realtime?.isConnected?.() === expected;
},
connected,
{ timeout }
);
}