Peer recovery burned its whole retry budget while signaling was down, then dropped the tracker with no re-arm, so a peer stayed dead until an unrelated roster event healed it. Recovery now waits for a usable transport before spending an attempt. Initiator election also compared a home actor id against foreign roster ids, which is not antisymmetric across signal servers - both sides offered, or neither did. Election moves into `peer-role.rules` and compares ids only within one signal server's identity space.
231 lines
8.3 KiB
TypeScript
231 lines
8.3 KiB
TypeScript
import { expect, type Page } from '@playwright/test';
|
|
import { test, type Client } from '../../fixtures/multi-client';
|
|
import { countCreatedPeerConnections } from '../../helpers/peer-role';
|
|
import { installDeterministicVoiceSettings } from '../../helpers/voice-session';
|
|
import {
|
|
closeOpenDataChannels,
|
|
dumpRtcDiagnostics,
|
|
getOpenDataChannelCount,
|
|
installAutoResumeAudioContext,
|
|
installWebRTCTracking,
|
|
waitForAllPeerAudioFlow,
|
|
waitForAudioStatsPresent,
|
|
waitForConnectedPeerCount,
|
|
waitForOpenDataChannelCount
|
|
} from '../../helpers/webrtc-helpers';
|
|
import { ChatRoomPage } from '../../pages/chat-room.page';
|
|
import { RegisterPage } from '../../pages/register.page';
|
|
import { ServerSearchPage } from '../../pages/server-search.page';
|
|
|
|
interface VoiceClient extends Client {
|
|
displayName: string;
|
|
username: string;
|
|
}
|
|
|
|
const USER_PASSWORD = 'TestPass123!';
|
|
const VOICE_CHANNEL = 'General';
|
|
/** 12 reconnect attempts at 5s - the whole budget fits inside this outage. */
|
|
const OUTAGE_HOLD_MS = 70_000;
|
|
|
|
test.describe('Recovery preserves live media', () => {
|
|
test('replaces a dead control channel without rebuilding the peer connection', async ({ createClient }) => {
|
|
test.setTimeout(240_000);
|
|
|
|
const clients = await createVoicePair(createClient, `DC Soft Replace ${Date.now()}`);
|
|
const [alice, bob] = clients;
|
|
|
|
await assertMeshAudio(clients, 'initial two-user voice');
|
|
|
|
const connectionsBefore = {
|
|
alice: await countCreatedPeerConnections(alice.page),
|
|
bob: await countCreatedPeerConnections(bob.page)
|
|
};
|
|
|
|
expect(connectionsBefore.alice).toBe(1);
|
|
expect(connectionsBefore.bob).toBe(1);
|
|
|
|
await test.step('The control channel is replaced on the same connection', async () => {
|
|
const closed = await closeOpenDataChannels(alice.page);
|
|
|
|
expect(closed).toBeGreaterThan(0);
|
|
|
|
await waitForOpenDataChannelCount(alice.page, 1, 60_000);
|
|
await waitForOpenDataChannelCount(bob.page, 1, 60_000);
|
|
|
|
// A rebuild would construct a second RTCPeerConnection on both sides, taking voice,
|
|
// camera, and screen share down with the control channel.
|
|
expect(
|
|
await countCreatedPeerConnections(alice.page),
|
|
'Alice rebuilt her peer connection instead of replacing the control channel'
|
|
).toBe(connectionsBefore.alice);
|
|
|
|
expect(
|
|
await countCreatedPeerConnections(bob.page),
|
|
'Bob rebuilt his peer connection instead of adopting the replacement control channel'
|
|
).toBe(connectionsBefore.bob);
|
|
});
|
|
|
|
await test.step('Audio never had to be renegotiated', async () => {
|
|
await waitForConnectedPeerCount(alice.page, 1, 30_000);
|
|
await waitForConnectedPeerCount(bob.page, 1, 30_000);
|
|
await assertMeshAudio(clients, 'after control-channel replacement');
|
|
});
|
|
});
|
|
|
|
// This covers the user-visible half: an outage that outlives the 12-attempt reconnect
|
|
// budget must not end the call. It cannot isolate the attempt accounting, because the
|
|
// roster resync on signaling reconnect re-peers anyway - `peer-recovery.spec.ts` owns
|
|
// the deterministic proof that a deferred attempt costs nothing.
|
|
test('keeps voice alive across a signal outage longer than the reconnect budget', async ({
|
|
createClient,
|
|
testServer
|
|
}) => {
|
|
test.setTimeout(480_000);
|
|
|
|
const clients = await createVoicePair(createClient, `Signal Outage Voice ${Date.now()}`);
|
|
|
|
await assertMeshAudio(clients, 'initial two-user voice');
|
|
|
|
const connectionsBefore = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
|
|
|
|
expect(connectionsBefore).toEqual([1, 1]);
|
|
|
|
await test.step('The signal server goes away for longer than the reconnect budget', async () => {
|
|
await testServer.kill();
|
|
|
|
for (const client of clients) {
|
|
await waitForSignalingConnected(client.page, false, 60_000);
|
|
}
|
|
|
|
await clients[0].page.waitForTimeout(OUTAGE_HOLD_MS);
|
|
});
|
|
|
|
await test.step('Peer media is unaffected by the signaling outage', async () => {
|
|
await assertMeshAudio(clients, 'during signal outage');
|
|
});
|
|
|
|
await test.step('Voice is still healthy once signaling returns', async () => {
|
|
await testServer.start();
|
|
|
|
for (const client of clients) {
|
|
await waitForSignalingConnected(client.page, true, 120_000);
|
|
}
|
|
|
|
for (const client of clients) {
|
|
await waitForConnectedPeerCount(client.page, 1, 90_000);
|
|
await waitForOpenDataChannelCount(client.page, 1, 90_000);
|
|
}
|
|
|
|
await assertMeshAudio(clients, 'after signaling recovery');
|
|
});
|
|
|
|
await test.step('The call was never rebuilt behind the user back', async () => {
|
|
// Media never depended on the signal server, so the roster resync must adopt the
|
|
// living peer connection. Rebuilding it would drop audio for a beat and reset
|
|
// screen share - invisible to the assertions above, which only re-check the end state.
|
|
const connectionsAfter = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
|
|
|
|
expect(connectionsAfter, 'a client rebuilt its peer connection when signaling came back').toEqual(
|
|
connectionsBefore
|
|
);
|
|
});
|
|
});
|
|
});
|
|
|
|
async function createVoicePair(
|
|
createClient: () => Promise<Client>,
|
|
serverName: string
|
|
): Promise<VoiceClient[]> {
|
|
const clients: VoiceClient[] = [];
|
|
|
|
for (let index = 0; index < 2; index++) {
|
|
const client = await createClient();
|
|
|
|
await installDeterministicVoiceSettings(client.page);
|
|
await installWebRTCTracking(client.page);
|
|
await installAutoResumeAudioContext(client.page);
|
|
|
|
clients.push({
|
|
...client,
|
|
displayName: `Recovery Voice ${index + 1}`,
|
|
username: `recovery_voice_${Date.now()}_${index + 1}`
|
|
});
|
|
}
|
|
|
|
await test.step('Register both clients', async () => {
|
|
for (const client of clients) {
|
|
const registerPage = new RegisterPage(client.page);
|
|
|
|
await registerPage.goto();
|
|
await registerPage.register(client.username, client.displayName, USER_PASSWORD);
|
|
await expect(client.page).toHaveURL(/\/dashboard/, { timeout: 20_000 });
|
|
}
|
|
});
|
|
|
|
await test.step('Create and join the server', async () => {
|
|
await new ServerSearchPage(clients[0].page).createServer(serverName, {
|
|
description: 'Recovery keeps live media test'
|
|
});
|
|
|
|
await expect(clients[0].page).toHaveURL(/\/room\//, { timeout: 20_000 });
|
|
|
|
await new ServerSearchPage(clients[1].page).joinServerFromSearch(serverName);
|
|
await expect(clients[1].page).toHaveURL(/\/room\//, { timeout: 20_000 });
|
|
});
|
|
|
|
await test.step('Join both clients to voice', async () => {
|
|
await new ChatRoomPage(clients[0].page).ensureVoiceChannelExists(VOICE_CHANNEL);
|
|
|
|
for (const client of clients) {
|
|
const room = new ChatRoomPage(client.page);
|
|
|
|
await room.joinVoiceChannel(VOICE_CHANNEL);
|
|
await expect(room.voiceControls).toBeVisible({ timeout: 20_000 });
|
|
}
|
|
|
|
for (const client of clients) {
|
|
await waitForConnectedPeerCount(client.page, 1, 90_000);
|
|
await waitForOpenDataChannelCount(client.page, 1, 90_000);
|
|
await waitForAudioStatsPresent(client.page, 30_000);
|
|
}
|
|
});
|
|
|
|
return clients;
|
|
}
|
|
|
|
async function assertMeshAudio(clients: readonly VoiceClient[], label: string): Promise<void> {
|
|
for (const client of clients) {
|
|
try {
|
|
await waitForAllPeerAudioFlow(client.page, 1, 60_000);
|
|
} catch (error) {
|
|
console.log(`[${client.displayName} ${label} data channels] ${await getOpenDataChannelCount(client.page)}`);
|
|
console.log(`[${client.displayName} ${label} RTC]\n${await dumpRtcDiagnostics(client.page)}`);
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Wait until the client's own view of its signaling connection matches `connected`. */
|
|
async function waitForSignalingConnected(page: Page, connected: boolean, timeout: number): Promise<void> {
|
|
await page.waitForFunction(
|
|
(expected) => {
|
|
interface AngularDebugApi {
|
|
getComponent: (element: Element) => Record<string, unknown>;
|
|
}
|
|
|
|
const host = document.querySelector('app-rooms-side-panel');
|
|
const debugApi = (window as { ng?: AngularDebugApi }).ng;
|
|
|
|
if (!host || !debugApi?.getComponent) {
|
|
return false;
|
|
}
|
|
|
|
const realtime = debugApi.getComponent(host)['realtime'] as { isConnected?: () => boolean } | undefined;
|
|
|
|
return realtime?.isConnected?.() === expected;
|
|
},
|
|
connected,
|
|
{ timeout }
|
|
);
|
|
}
|