Repair connectivity correctly v1

e2e/helpers/webrtc-helpers.ts

@@ -11,6 +11,7 @@ import { type Page } from '@playwright/test';
 export async function installWebRTCTracking(page: Page): Promise<void> {
   await page.addInitScript(() => {
     const connections: RTCPeerConnection[] = [];
+    const dataChannels: RTCDataChannel[] = [];
     const syntheticMediaResources: {
       audioCtx: AudioContext;
       source?: AudioScheduledSourceNode;
@@ -18,20 +19,40 @@ export async function installWebRTCTracking(page: Page): Promise<void> {
     }[] = [];
 
     (window as any).__rtcConnections = connections;
+    (window as any).__rtcDataChannels = dataChannels;
     (window as any).__rtcRemoteTracks = [] as { kind: string; id: string; readyState: string }[];
     (window as any).__rtcSyntheticMediaResources = syntheticMediaResources;
 
     const OriginalRTCPeerConnection = window.RTCPeerConnection;
+    const trackDataChannel = (channel: RTCDataChannel) => {
+      if (dataChannels.includes(channel)) {
+        return;
+      }
+
+      dataChannels.push(channel);
+    };
 
     (window as any).RTCPeerConnection = function(this: RTCPeerConnection, ...args: any[]) {
       const pc: RTCPeerConnection = new OriginalRTCPeerConnection(...args);
+      const originalCreateDataChannel = pc.createDataChannel.bind(pc);
 
       connections.push(pc);
 
+      pc.createDataChannel = ((label: string, options?: RTCDataChannelInit) => {
+        const channel = originalCreateDataChannel(label, options);
+
+        trackDataChannel(channel);
+        return channel;
+      }) as RTCPeerConnection['createDataChannel'];
+
       pc.addEventListener('connectionstatechange', () => {
         (window as any).__lastRtcState = pc.connectionState;
       });
 
+      pc.addEventListener('datachannel', (event: RTCDataChannelEvent) => {
+        trackDataChannel(event.channel);
+      });
+
       pc.addEventListener('track', (event: RTCTrackEvent) => {
         (window as any).__rtcRemoteTracks.push({
           kind: event.track.kind,
@@ -211,6 +232,66 @@ export async function waitForConnectedPeerCount(page: Page, expectedCount: numbe
   );
 }
 
+/** Returns the number of tracked RTCDataChannels in the open state. */
+export async function getOpenDataChannelCount(page: Page): Promise<number> {
+  return page.evaluate(
+    () => ((window as any).__rtcDataChannels as RTCDataChannel[] | undefined)?.filter(
+      (channel) => channel.readyState === 'open'
+    ).length ?? 0
+  );
+}
+
+/** Wait until the expected number of tracked RTCDataChannels are open. */
+export async function waitForOpenDataChannelCount(page: Page, expectedCount: number, timeout = 45_000): Promise<void> {
+  await page.waitForFunction(
+    (count) => ((window as any).__rtcDataChannels as RTCDataChannel[] | undefined)?.filter(
+      (channel) => channel.readyState === 'open'
+    ).length === count,
+    expectedCount,
+    { timeout }
+  );
+}
+
+/** Close every currently-open RTCDataChannel and return how many were closed. */
+export async function closeOpenDataChannels(page: Page): Promise<number> {
+  return page.evaluate(() => {
+    const channels = ((window as any).__rtcDataChannels as RTCDataChannel[] | undefined) ?? [];
+
+    let closed = 0;
+
+    for (const channel of channels) {
+      if (channel.readyState !== 'open') {
+        continue;
+      }
+
+      channel.close();
+      closed++;
+    }
+
+    return closed;
+  });
+}
+
+/** Dispatch a synthetic data-channel error event on each open channel. */
+export async function dispatchDataChannelErrors(page: Page): Promise<number> {
+  return page.evaluate(() => {
+    const channels = ((window as any).__rtcDataChannels as RTCDataChannel[] | undefined) ?? [];
+
+    let dispatched = 0;
+
+    for (const channel of channels) {
+      if (channel.readyState !== 'open') {
+        continue;
+      }
+
+      channel.dispatchEvent(new Event('error'));
+      dispatched++;
+    }
+
+    return dispatched;
+  });
+}
+
 /**
  * Resume all suspended AudioContext instances created by the synthetic
  * media patch.  Uses CDP `Runtime.evaluate` with `userGesture: true` so

e2e/tests/voice/data-channel-recovery.spec.ts

@@ -0,0 +1,181 @@
+import { expect, type Page } from '@playwright/test';
+import { test, type Client } from '../../fixtures/multi-client';
+import {
+  closeOpenDataChannels,
+  dispatchDataChannelErrors,
+  dumpRtcDiagnostics,
+  getOpenDataChannelCount,
+  installAutoResumeAudioContext,
+  installWebRTCTracking,
+  waitForAllPeerAudioFlow,
+  waitForAudioStatsPresent,
+  waitForConnectedPeerCount,
+  waitForOpenDataChannelCount
+} from '../../helpers/webrtc-helpers';
+import { RegisterPage } from '../../pages/register.page';
+import { ServerSearchPage } from '../../pages/server-search.page';
+import { ChatRoomPage } from '../../pages/chat-room.page';
+
+interface VoiceClient extends Client {
+  displayName: string;
+  username: string;
+}
+
+const USER_PASSWORD = 'TestPass123!';
+const VOICE_CHANNEL = 'General';
+
+test.describe('Voice data-channel recovery', () => {
+  test('keeps two users hearing each other after a data-channel error and close', async ({ createClient }) => {
+    test.setTimeout(240_000);
+
+    const clients = await createVoiceScenario(createClient, 2, `DC Recovery Duo ${Date.now()}`);
+    const [alice, bob] = clients;
+
+    await assertMeshAudio(clients, 1, 'initial two-user voice');
+
+    await test.step('A non-fatal data-channel error does not interrupt audio', async () => {
+      const dispatched = await dispatchDataChannelErrors(alice.page);
+
+      expect(dispatched).toBeGreaterThan(0);
+      await waitForOpenDataChannelCount(alice.page, 1, 15_000);
+      await waitForOpenDataChannelCount(bob.page, 1, 15_000);
+      await assertMeshAudio(clients, 1, 'after synthetic data-channel error');
+    });
+
+    await test.step('A closed data channel is rebuilt and audio resumes both ways', async () => {
+      const closed = await closeOpenDataChannels(alice.page);
+
+      expect(closed).toBeGreaterThan(0);
+      await waitForConnectedPeerCount(alice.page, 1, 60_000);
+      await waitForConnectedPeerCount(bob.page, 1, 60_000);
+      await waitForOpenDataChannelCount(alice.page, 1, 60_000);
+      await waitForOpenDataChannelCount(bob.page, 1, 60_000);
+      await assertMeshAudio(clients, 1, 'after data-channel close recovery');
+    });
+  });
+
+  test('heals a three-user voice mesh when one client loses every data channel', async ({ createClient }) => {
+    test.setTimeout(300_000);
+
+    const clients = await createVoiceScenario(createClient, 3, `DC Recovery Trio ${Date.now()}`);
+    const bob = clients[1];
+
+    await assertMeshAudio(clients, 2, 'initial three-user mesh');
+
+    await test.step('Bob loses all control channels and the full mesh recovers', async () => {
+      const closed = await closeOpenDataChannels(bob.page);
+
+      expect(closed).toBe(2);
+
+      for (const client of clients) {
+        await waitForConnectedPeerCount(client.page, 2, 90_000);
+        await waitForOpenDataChannelCount(client.page, 2, 90_000);
+      }
+
+      await assertMeshAudio(clients, 2, 'after full control-channel recovery');
+    });
+  });
+});
+
+async function createVoiceScenario(
+  createClient: () => Promise<Client>,
+  userCount: number,
+  serverName: string
+): Promise<VoiceClient[]> {
+  const clients: VoiceClient[] = [];
+
+  for (let index = 0; index < userCount; index++) {
+    const client = await createClient();
+    const displayName = `DC Voice ${index + 1}`;
+
+    await installDeterministicVoiceSettings(client.page);
+    await installWebRTCTracking(client.page);
+    await installAutoResumeAudioContext(client.page);
+
+    clients.push({
+      ...client,
+      displayName,
+      username: `dc_voice_${Date.now()}_${index + 1}`
+    });
+  }
+
+  await test.step('Register clients', async () => {
+    for (const client of clients) {
+      const registerPage = new RegisterPage(client.page);
+
+      await registerPage.goto();
+      await registerPage.register(client.username, client.displayName, USER_PASSWORD);
+      await expect(client.page).toHaveURL(/\/search/, { timeout: 20_000 });
+    }
+  });
+
+  await test.step('Create and join server', async () => {
+    const hostSearch = new ServerSearchPage(clients[0].page);
+
+    await hostSearch.createServer(serverName, { description: 'Data-channel recovery voice test' });
+    await expect(clients[0].page).toHaveURL(/\/room\//, { timeout: 20_000 });
+
+    for (const client of clients.slice(1)) {
+      const searchPage = new ServerSearchPage(client.page);
+
+      await searchPage.joinServerFromSearch(serverName);
+      await expect(client.page).toHaveURL(/\/room\//, { timeout: 20_000 });
+    }
+  });
+
+  await test.step('Join everyone to voice', async () => {
+    const hostRoom = new ChatRoomPage(clients[0].page);
+
+    await hostRoom.ensureVoiceChannelExists(VOICE_CHANNEL);
+
+    for (const client of clients) {
+      const room = new ChatRoomPage(client.page);
+
+      await room.joinVoiceChannel(VOICE_CHANNEL);
+      await expect(room.voiceControls).toBeVisible({ timeout: 20_000 });
+    }
+
+    const expectedRemotePeers = clients.length - 1;
+
+    for (const client of clients) {
+      await waitForConnectedPeerCount(client.page, expectedRemotePeers, 90_000);
+      await waitForOpenDataChannelCount(client.page, expectedRemotePeers, 90_000);
+      await waitForAudioStatsPresent(client.page, 30_000);
+    }
+  });
+
+  return clients;
+}
+
+async function installDeterministicVoiceSettings(page: Page): Promise<void> {
+  await page.addInitScript(() => {
+    localStorage.setItem('metoyou_voice_settings', JSON.stringify({
+      inputVolume: 100,
+      outputVolume: 100,
+      audioBitrate: 96,
+      latencyProfile: 'balanced',
+      includeSystemAudio: false,
+      noiseReduction: false,
+      screenShareQuality: 'balanced',
+      askScreenShareQuality: false
+    }));
+  });
+}
+
+async function assertMeshAudio(
+  clients: readonly VoiceClient[],
+  expectedRemotePeers: number,
+  label: string
+): Promise<void> {
+  for (const client of clients) {
+    try {
+      await waitForAllPeerAudioFlow(client.page, expectedRemotePeers, 60_000);
+    } catch (error) {
+      const dataChannelCount = await getOpenDataChannelCount(client.page);
+
+      console.log(`[${client.displayName} ${label} data channels] ${dataChannelCount}`);
+      console.log(`[${client.displayName} ${label} RTC]\n${await dumpRtcDiagnostics(client.page)}`);
+      throw error;
+    }
+  }
+}

e2e/tests/voice/direct-call.spec.ts

@@ -1,6 +1,7 @@
 import { expect, type Page } from '@playwright/test';
 import { test, type Client } from '../../fixtures/multi-client';
 import {
+  closeOpenDataChannels,
   dumpRtcDiagnostics,
   installAutoResumeAudioContext,
   installWebRTCTracking,
@@ -8,6 +9,7 @@ import {
   waitForAudioFlow,
   waitForAudioStatsPresent,
   waitForConnectedPeerCount,
+  waitForOpenDataChannelCount,
   waitForInboundVideoFlow,
   waitForOutboundVideoFlow,
   waitForPeerConnected
@@ -371,6 +373,35 @@ test.describe('Direct private calls', () => {
     });
   });
 
+  test('keeps private-call audio flowing after the data channel closes', async ({ createClient }) => {
+    const scenario = await createDirectCallScenario(createClient);
+
+    await test.step('Alice starts a private call and Bob joins', async () => {
+      await startCallFromSearch(scenario.alice.page, scenario.bobUserId, 'Bob');
+      await scenario.bob.page.getByRole('button', { name: 'Open private call' }).click();
+      await expect(scenario.bob.page).toHaveURL(/\/call\//, { timeout: 20_000 });
+      await scenario.bob.page.getByRole('button', { name: 'Join call' }).click();
+      await expect(scenario.bob.page.getByRole('button', { name: 'Leave call' })).toBeVisible({ timeout: 20_000 });
+
+      await waitForConnectedPeerCount(scenario.alice.page, 1, 45_000);
+      await waitForConnectedPeerCount(scenario.bob.page, 1, 45_000);
+      await waitForOpenDataChannelCount(scenario.alice.page, 1, 45_000);
+      await waitForOpenDataChannelCount(scenario.bob.page, 1, 45_000);
+      await waitForAllPeerAudioFlow(scenario.alice.page, 1, 45_000);
+      await waitForAllPeerAudioFlow(scenario.bob.page, 1, 45_000);
+    });
+
+    await test.step('Data-channel recovery keeps the call audible', async () => {
+      const closed = await closeOpenDataChannels(scenario.alice.page);
+
+      expect(closed).toBeGreaterThan(0);
+      await waitForOpenDataChannelCount(scenario.alice.page, 1, 60_000);
+      await waitForOpenDataChannelCount(scenario.bob.page, 1, 60_000);
+      await waitForAllPeerAudioFlow(scenario.alice.page, 1, 60_000);
+      await waitForAllPeerAudioFlow(scenario.bob.page, 1, 60_000);
+    });
+  });
+
   test('missing and ended private calls do not leave stale call controls behind', async ({ createClient }) => {
     const scenario = await createDirectCallScenario(createClient);