fix(realtime): survive signal outages and per-server identities
Peer recovery burned its whole retry budget while signaling was down, then dropped the tracker with no re-arm, so a peer stayed dead until an unrelated roster event healed it. Recovery now waits for a usable transport before spending an attempt. Initiator election also compared a home actor id against foreign roster ids, which is not antisymmetric across signal servers - both sides offered, or neither did. Election moves into `peer-role.rules` and compares ids only within one signal server's identity space.
This commit is contained in:
@@ -0,0 +1,188 @@
|
||||
import { expect } from '@playwright/test';
|
||||
import { test, type Client } from '../../fixtures/multi-client';
|
||||
import { expectDashboardReady } from '../../helpers/dashboard';
|
||||
import {
|
||||
countCreatedPeerConnections,
|
||||
expectExactlyOneInitiatorPerPair,
|
||||
readPeerRoleEdges,
|
||||
type PeerRoleEdge
|
||||
} from '../../helpers/peer-role';
|
||||
import { installTestServerEndpoints, type SeededEndpointInput } from '../../helpers/seed-test-endpoint';
|
||||
import { startTestServer } from '../../helpers/test-server';
|
||||
import {
|
||||
installDeterministicVoiceSettings,
|
||||
joinRoomFromSearch,
|
||||
joinVoiceChannelUntilConnected,
|
||||
openSavedRoomByName
|
||||
} from '../../helpers/voice-session';
|
||||
import { waitForVoiceRosterCount } from '../../helpers/voice-roster';
|
||||
import {
|
||||
dumpRtcDiagnostics,
|
||||
installAutoResumeAudioContext,
|
||||
installWebRTCTracking,
|
||||
waitForAllPeerAudioFlow,
|
||||
waitForAudioStatsPresent,
|
||||
waitForPeerConnected
|
||||
} from '../../helpers/webrtc-helpers';
|
||||
import { ChatRoomPage } from '../../pages/chat-room.page';
|
||||
import { RegisterPage } from '../../pages/register.page';
|
||||
import { ServerSearchPage } from '../../pages/server-search.page';
|
||||
|
||||
const SIGNAL_A_ID = 'e2e-cross-signal-a';
|
||||
const SIGNAL_B_ID = 'e2e-cross-signal-b';
|
||||
const VOICE_CHANNEL = 'General';
|
||||
const USER_PASSWORD = 'TestPass123!';
|
||||
const USER_COUNT = 4;
|
||||
const EXPECTED_REMOTE_PEERS = USER_COUNT - 1;
|
||||
|
||||
interface TestUser {
|
||||
username: string;
|
||||
displayName: string;
|
||||
/** Signal server this human registered on - their home identity space. */
|
||||
homeSignalId: string;
|
||||
}
|
||||
|
||||
type TestClient = Client & { user: TestUser };
|
||||
|
||||
test.describe('Cross-signal WebRTC identity', () => {
|
||||
test.describe.configure({ timeout: 600_000 });
|
||||
|
||||
test('elects exactly one initiator per pair when peers have different home signal servers', async ({
|
||||
createClient,
|
||||
testServer
|
||||
}) => {
|
||||
const signalB = await startTestServer();
|
||||
|
||||
try {
|
||||
const suffix = `cross_signal_${Date.now()}`;
|
||||
const roomName = `Cross Signal Voice ${suffix}`;
|
||||
const endpoints: SeededEndpointInput[] = [
|
||||
{
|
||||
id: SIGNAL_A_ID,
|
||||
name: 'E2E Signal A',
|
||||
url: testServer.url,
|
||||
isActive: true,
|
||||
status: 'online'
|
||||
},
|
||||
{
|
||||
id: SIGNAL_B_ID,
|
||||
name: 'E2E Signal B',
|
||||
url: signalB.url,
|
||||
isActive: true,
|
||||
status: 'online'
|
||||
}
|
||||
];
|
||||
// The room is hosted on signal B. Two humans are at home there and two are
|
||||
// foreign guests, so most pairs must compare a foreign actor id against a
|
||||
// foreign actor id - never a home id against one.
|
||||
const users: TestUser[] = [
|
||||
{ username: `host_${suffix}`, displayName: 'Cross Host', homeSignalId: SIGNAL_B_ID },
|
||||
{ username: `native_${suffix}`, displayName: 'Cross Native', homeSignalId: SIGNAL_B_ID },
|
||||
{ username: `guest_a_${suffix}`, displayName: 'Cross Guest A', homeSignalId: SIGNAL_A_ID },
|
||||
{ username: `guest_b_${suffix}`, displayName: 'Cross Guest B', homeSignalId: SIGNAL_A_ID }
|
||||
];
|
||||
const clients: TestClient[] = [];
|
||||
|
||||
for (const user of users) {
|
||||
const client = await createClient();
|
||||
|
||||
await installTestServerEndpoints(client.context, endpoints);
|
||||
await installDeterministicVoiceSettings(client.page);
|
||||
await installWebRTCTracking(client.context);
|
||||
await installAutoResumeAudioContext(client.page);
|
||||
|
||||
clients.push({ ...client, user });
|
||||
}
|
||||
|
||||
const [host] = clients;
|
||||
|
||||
await test.step('Each human registers on their own home signal server', async () => {
|
||||
for (const client of clients) {
|
||||
const register = new RegisterPage(client.page);
|
||||
|
||||
await register.goto();
|
||||
await register.serverSelect.selectOption(client.user.homeSignalId);
|
||||
await register.register(client.user.username, client.user.displayName, USER_PASSWORD);
|
||||
await expectDashboardReady(client.page);
|
||||
}
|
||||
});
|
||||
|
||||
await test.step('The host creates the voice room on signal B', async () => {
|
||||
await new ServerSearchPage(host.page).createServer(roomName, {
|
||||
description: 'Cross-signal initiator election coverage',
|
||||
sourceId: SIGNAL_B_ID
|
||||
});
|
||||
|
||||
await expect(host.page).toHaveURL(/\/room\//, { timeout: 20_000 });
|
||||
await new ChatRoomPage(host.page).ensureVoiceChannelExists(VOICE_CHANNEL);
|
||||
});
|
||||
|
||||
await test.step('Everyone else joins the room, provisioning a foreign account when needed', async () => {
|
||||
for (const client of clients.slice(1)) {
|
||||
await joinRoomFromSearch(client.page, roomName);
|
||||
}
|
||||
|
||||
await openSavedRoomByName(host.page, roomName);
|
||||
});
|
||||
|
||||
// Everyone reconnects at once, so every pair elects its initiator from the same
|
||||
// roster snapshot. Staggered arrivals let one side's 1s fallback-offer timer
|
||||
// serialize negotiation, which hides a wrong comparison; a reconnect storm - a
|
||||
// signal blip, or a channel everyone piles into - does not.
|
||||
await test.step('All four reconnect simultaneously', async () => {
|
||||
await Promise.all(clients.map(async (client) => {
|
||||
await client.page.reload({ waitUntil: 'domcontentloaded' });
|
||||
await openSavedRoomByName(client.page, roomName);
|
||||
}));
|
||||
});
|
||||
|
||||
await test.step('All four join the same voice channel simultaneously', async () => {
|
||||
await Promise.all(clients.map((client) =>
|
||||
joinVoiceChannelUntilConnected(client.page, VOICE_CHANNEL)
|
||||
));
|
||||
|
||||
for (const client of clients) {
|
||||
await waitForVoiceRosterCount(client.page, VOICE_CHANNEL, USER_COUNT);
|
||||
}
|
||||
});
|
||||
|
||||
await test.step('Every pair carries bidirectional audio', async () => {
|
||||
await Promise.all(clients.map((client) => waitForPeerConnected(client.page, 90_000)));
|
||||
await Promise.all(clients.map((client) => waitForAudioStatsPresent(client.page, 30_000)));
|
||||
|
||||
for (const client of clients) {
|
||||
try {
|
||||
await waitForAllPeerAudioFlow(client.page, EXPECTED_REMOTE_PEERS, 120_000);
|
||||
} catch (error) {
|
||||
console.log(`[${client.user.displayName} RTC]\n${await dumpRtcDiagnostics(client.page)}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await test.step('Exactly one side of every pair elected itself initiator', async () => {
|
||||
const edgesByClient: Record<string, PeerRoleEdge[]> = {};
|
||||
|
||||
for (const client of clients) {
|
||||
edgesByClient[client.user.displayName] = (await readPeerRoleEdges(client.page))
|
||||
.filter((edge) => edge.connectionState === 'connected');
|
||||
}
|
||||
|
||||
// Comparing a home id against a foreign actor id is not antisymmetric, so both
|
||||
// peers could offer (glare) or neither could until a takeover timer fired.
|
||||
expectExactlyOneInitiatorPerPair(edgesByClient);
|
||||
});
|
||||
|
||||
await test.step('No peer had to be rebuilt to reach that state', async () => {
|
||||
for (const client of clients) {
|
||||
expect(
|
||||
await countCreatedPeerConnections(client.page),
|
||||
`${client.user.displayName} rebuilt a peer connection instead of connecting on the first offer`
|
||||
).toBe(EXPECTED_REMOTE_PEERS);
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
await signalB.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,230 @@
|
||||
import { expect, type Page } from '@playwright/test';
|
||||
import { test, type Client } from '../../fixtures/multi-client';
|
||||
import { countCreatedPeerConnections } from '../../helpers/peer-role';
|
||||
import { installDeterministicVoiceSettings } from '../../helpers/voice-session';
|
||||
import {
|
||||
closeOpenDataChannels,
|
||||
dumpRtcDiagnostics,
|
||||
getOpenDataChannelCount,
|
||||
installAutoResumeAudioContext,
|
||||
installWebRTCTracking,
|
||||
waitForAllPeerAudioFlow,
|
||||
waitForAudioStatsPresent,
|
||||
waitForConnectedPeerCount,
|
||||
waitForOpenDataChannelCount
|
||||
} from '../../helpers/webrtc-helpers';
|
||||
import { ChatRoomPage } from '../../pages/chat-room.page';
|
||||
import { RegisterPage } from '../../pages/register.page';
|
||||
import { ServerSearchPage } from '../../pages/server-search.page';
|
||||
|
||||
interface VoiceClient extends Client {
|
||||
displayName: string;
|
||||
username: string;
|
||||
}
|
||||
|
||||
const USER_PASSWORD = 'TestPass123!';
|
||||
const VOICE_CHANNEL = 'General';
|
||||
/** 12 reconnect attempts at 5s - the whole budget fits inside this outage. */
|
||||
const OUTAGE_HOLD_MS = 70_000;
|
||||
|
||||
test.describe('Recovery preserves live media', () => {
|
||||
test('replaces a dead control channel without rebuilding the peer connection', async ({ createClient }) => {
|
||||
test.setTimeout(240_000);
|
||||
|
||||
const clients = await createVoicePair(createClient, `DC Soft Replace ${Date.now()}`);
|
||||
const [alice, bob] = clients;
|
||||
|
||||
await assertMeshAudio(clients, 'initial two-user voice');
|
||||
|
||||
const connectionsBefore = {
|
||||
alice: await countCreatedPeerConnections(alice.page),
|
||||
bob: await countCreatedPeerConnections(bob.page)
|
||||
};
|
||||
|
||||
expect(connectionsBefore.alice).toBe(1);
|
||||
expect(connectionsBefore.bob).toBe(1);
|
||||
|
||||
await test.step('The control channel is replaced on the same connection', async () => {
|
||||
const closed = await closeOpenDataChannels(alice.page);
|
||||
|
||||
expect(closed).toBeGreaterThan(0);
|
||||
|
||||
await waitForOpenDataChannelCount(alice.page, 1, 60_000);
|
||||
await waitForOpenDataChannelCount(bob.page, 1, 60_000);
|
||||
|
||||
// A rebuild would construct a second RTCPeerConnection on both sides, taking voice,
|
||||
// camera, and screen share down with the control channel.
|
||||
expect(
|
||||
await countCreatedPeerConnections(alice.page),
|
||||
'Alice rebuilt her peer connection instead of replacing the control channel'
|
||||
).toBe(connectionsBefore.alice);
|
||||
|
||||
expect(
|
||||
await countCreatedPeerConnections(bob.page),
|
||||
'Bob rebuilt his peer connection instead of adopting the replacement control channel'
|
||||
).toBe(connectionsBefore.bob);
|
||||
});
|
||||
|
||||
await test.step('Audio never had to be renegotiated', async () => {
|
||||
await waitForConnectedPeerCount(alice.page, 1, 30_000);
|
||||
await waitForConnectedPeerCount(bob.page, 1, 30_000);
|
||||
await assertMeshAudio(clients, 'after control-channel replacement');
|
||||
});
|
||||
});
|
||||
|
||||
// This covers the user-visible half: an outage that outlives the 12-attempt reconnect
|
||||
// budget must not end the call. It cannot isolate the attempt accounting, because the
|
||||
// roster resync on signaling reconnect re-peers anyway - `peer-recovery.spec.ts` owns
|
||||
// the deterministic proof that a deferred attempt costs nothing.
|
||||
test('keeps voice alive across a signal outage longer than the reconnect budget', async ({
|
||||
createClient,
|
||||
testServer
|
||||
}) => {
|
||||
test.setTimeout(480_000);
|
||||
|
||||
const clients = await createVoicePair(createClient, `Signal Outage Voice ${Date.now()}`);
|
||||
|
||||
await assertMeshAudio(clients, 'initial two-user voice');
|
||||
|
||||
const connectionsBefore = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
|
||||
|
||||
expect(connectionsBefore).toEqual([1, 1]);
|
||||
|
||||
await test.step('The signal server goes away for longer than the reconnect budget', async () => {
|
||||
await testServer.kill();
|
||||
|
||||
for (const client of clients) {
|
||||
await waitForSignalingConnected(client.page, false, 60_000);
|
||||
}
|
||||
|
||||
await clients[0].page.waitForTimeout(OUTAGE_HOLD_MS);
|
||||
});
|
||||
|
||||
await test.step('Peer media is unaffected by the signaling outage', async () => {
|
||||
await assertMeshAudio(clients, 'during signal outage');
|
||||
});
|
||||
|
||||
await test.step('Voice is still healthy once signaling returns', async () => {
|
||||
await testServer.start();
|
||||
|
||||
for (const client of clients) {
|
||||
await waitForSignalingConnected(client.page, true, 120_000);
|
||||
}
|
||||
|
||||
for (const client of clients) {
|
||||
await waitForConnectedPeerCount(client.page, 1, 90_000);
|
||||
await waitForOpenDataChannelCount(client.page, 1, 90_000);
|
||||
}
|
||||
|
||||
await assertMeshAudio(clients, 'after signaling recovery');
|
||||
});
|
||||
|
||||
await test.step('The call was never rebuilt behind the user back', async () => {
|
||||
// Media never depended on the signal server, so the roster resync must adopt the
|
||||
// living peer connection. Rebuilding it would drop audio for a beat and reset
|
||||
// screen share - invisible to the assertions above, which only re-check the end state.
|
||||
const connectionsAfter = await Promise.all(clients.map((client) => countCreatedPeerConnections(client.page)));
|
||||
|
||||
expect(connectionsAfter, 'a client rebuilt its peer connection when signaling came back').toEqual(
|
||||
connectionsBefore
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
async function createVoicePair(
|
||||
createClient: () => Promise<Client>,
|
||||
serverName: string
|
||||
): Promise<VoiceClient[]> {
|
||||
const clients: VoiceClient[] = [];
|
||||
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const client = await createClient();
|
||||
|
||||
await installDeterministicVoiceSettings(client.page);
|
||||
await installWebRTCTracking(client.page);
|
||||
await installAutoResumeAudioContext(client.page);
|
||||
|
||||
clients.push({
|
||||
...client,
|
||||
displayName: `Recovery Voice ${index + 1}`,
|
||||
username: `recovery_voice_${Date.now()}_${index + 1}`
|
||||
});
|
||||
}
|
||||
|
||||
await test.step('Register both clients', async () => {
|
||||
for (const client of clients) {
|
||||
const registerPage = new RegisterPage(client.page);
|
||||
|
||||
await registerPage.goto();
|
||||
await registerPage.register(client.username, client.displayName, USER_PASSWORD);
|
||||
await expect(client.page).toHaveURL(/\/dashboard/, { timeout: 20_000 });
|
||||
}
|
||||
});
|
||||
|
||||
await test.step('Create and join the server', async () => {
|
||||
await new ServerSearchPage(clients[0].page).createServer(serverName, {
|
||||
description: 'Recovery keeps live media test'
|
||||
});
|
||||
|
||||
await expect(clients[0].page).toHaveURL(/\/room\//, { timeout: 20_000 });
|
||||
|
||||
await new ServerSearchPage(clients[1].page).joinServerFromSearch(serverName);
|
||||
await expect(clients[1].page).toHaveURL(/\/room\//, { timeout: 20_000 });
|
||||
});
|
||||
|
||||
await test.step('Join both clients to voice', async () => {
|
||||
await new ChatRoomPage(clients[0].page).ensureVoiceChannelExists(VOICE_CHANNEL);
|
||||
|
||||
for (const client of clients) {
|
||||
const room = new ChatRoomPage(client.page);
|
||||
|
||||
await room.joinVoiceChannel(VOICE_CHANNEL);
|
||||
await expect(room.voiceControls).toBeVisible({ timeout: 20_000 });
|
||||
}
|
||||
|
||||
for (const client of clients) {
|
||||
await waitForConnectedPeerCount(client.page, 1, 90_000);
|
||||
await waitForOpenDataChannelCount(client.page, 1, 90_000);
|
||||
await waitForAudioStatsPresent(client.page, 30_000);
|
||||
}
|
||||
});
|
||||
|
||||
return clients;
|
||||
}
|
||||
|
||||
async function assertMeshAudio(clients: readonly VoiceClient[], label: string): Promise<void> {
|
||||
for (const client of clients) {
|
||||
try {
|
||||
await waitForAllPeerAudioFlow(client.page, 1, 60_000);
|
||||
} catch (error) {
|
||||
console.log(`[${client.displayName} ${label} data channels] ${await getOpenDataChannelCount(client.page)}`);
|
||||
console.log(`[${client.displayName} ${label} RTC]\n${await dumpRtcDiagnostics(client.page)}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Wait until the client's own view of its signaling connection matches `connected`. */
|
||||
async function waitForSignalingConnected(page: Page, connected: boolean, timeout: number): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
(expected) => {
|
||||
interface AngularDebugApi {
|
||||
getComponent: (element: Element) => Record<string, unknown>;
|
||||
}
|
||||
|
||||
const host = document.querySelector('app-rooms-side-panel');
|
||||
const debugApi = (window as { ng?: AngularDebugApi }).ng;
|
||||
|
||||
if (!host || !debugApi?.getComponent) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const realtime = debugApi.getComponent(host)['realtime'] as { isConnected?: () => boolean } | undefined;
|
||||
|
||||
return realtime?.isConnected?.() === expected;
|
||||
},
|
||||
connected,
|
||||
{ timeout }
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user