Fix UI freeze by using MediaStreamTrackProcessor for audio capture (restored PCM)

src/main/index.ts

@@ -77,6 +77,12 @@ app.whenReady().then(() => {
         network?.sendEncodedVideoChunk(payload.chunk, payload.isKeyFrame, payload.timestamp, payload.streamType);
     });
 
+    // Original simple PCM audio sending
+    ipcMain.on('send-audio-frame', (_, { frame }) => {
+        network?.sendAudioFrame(new Uint8Array(frame));
+    });
+
+    // Opus encoded audio (keeping for compatibility)
     ipcMain.on('send-audio-chunk', (_, payload) => {
         network?.sendEncodedAudioChunk(payload.chunk, payload.timestamp);
     });

src/main/network.ts

@@ -146,7 +146,11 @@ export class NetworkManager extends EventEmitter {
                 this.safeSend('chat-message', msg.data);
                 break;
             case 'UpdateStream':
-                this.safeSend('peer-stream-update', msg.data);
+                // Ignore stream updates for self (we manage local state directly)
+                if (msg.data.user_id !== this.userId) {
+                    console.log(`[Network] Peer Stream Update: User=${msg.data.user_id} Type=${msg.data.media_type} Active=${msg.data.active}`);
+                    this.safeSend('peer-stream-update', msg.data);
+                }
                 break;
             case 'Error':
                 console.error('WS Error Msg:', msg.data);
@@ -221,54 +225,8 @@ export class NetworkManager extends EventEmitter {
         const payload = msg.subarray(HEADER_SIZE);
 
         if (mediaType === MediaType.Audio) {
-            // Audio can be fragmented now (PCM)
-            this.safeSend('video-chunk', { // Use 'video-chunk' handler in renderer for reassembly? 
-                // Wait, App.tsx has separate 'audio-chunk' which doesn't reassemble.
-                // We need to reassemble here or change App.tsx.
-                // Reassembling in main process is easier or reusing video logic.
-
-                // Let's use 'audio-chunk' but we need to pass frag info?
-                // No, App.tsx 'audio-chunk' handler just decodes immediately.
-                // It expects a full frame.
-
-                // We MUST reassemble here or update App.tsx.
-                // Updating App.tsx to use the reassembler for Audio is cleaner.
-                // But 'video-chunk' in App.tsx calls 'handleIncomingVideoFragment' which uses 'MediaEngine.decodeVideoChunk'.
-
-                // Option: Treat Audio as "Video" for transport, but with streamType='audio'?
-                // MediaType.Audio is distinct.
-
-                // Let's implement reassembly here in NetworkManager?
-                // Or update App.tsx to use 'handleIncomingVideoFragment' for audio too?
-                // 'handleIncomingVideoFragment' does `decodeVideoChunk`.
-
-                // Let's change App.tsx to have `handleIncomingAudioFragment`?
-                // Or just reassemble here. UDP reassembly in Node.js is fine.
-
-                // ACtually, App.tsx's `handleIncomingVideoFragment` is nice.
-                // Let's emit 'audio-fragment' and add a handler in App.tsx.
-                user_id: userId,
-                data: payload,
-                seq: this.audioSeq, // Wait, seq is in packet
-                ts: timestamp,
-                fidx: fragIdx,
-                fcnt: fragCnt,
-                isKeyFrame,
-                streamType: 'audio'
-                // We can't use 'video-chunk' channel because it calls decodeVideoChunk.
-            });
-
-            // Actually, let's just send it to 'audio-fragment' channel
-            this.safeSend('audio-fragment', {
-                user_id: userId,
-                data: payload,
-                seq: seq, // We need valid seq from packet
-                ts: timestamp,
-                fidx: fragIdx,
-                fcnt: fragCnt,
-                isKeyFrame
-            });
-
+            // Original simple approach - just forward to renderer (PCM)
+            this.safeSend('audio-frame', { user_id: userId, data: payload });
         } else if (mediaType === MediaType.Video || mediaType === MediaType.Screen) {
             // Differentiate based on MediaType
             const streamType = mediaType === MediaType.Screen ? 'screen' : 'video';
@@ -292,10 +250,15 @@ export class NetworkManager extends EventEmitter {
     private safeSend(channel: string, data: any) {
         if (this.mainWindow && !this.mainWindow.isDestroyed() && this.mainWindow.webContents) {
             try {
+                if (channel === 'audio-fragment') {
+                    console.log(`[Network] safeSend audio-fragment to renderer, data size=${data.data?.length}`);
+                }
                 this.mainWindow.webContents.send(channel, data);
             } catch (e) {
                 console.error(`Failed to send ${channel} to renderer:`, e);
             }
+        } else {
+            console.warn(`[Network] Cannot send ${channel}: mainWindow not ready`);
         }
     }
 
@@ -344,6 +307,26 @@ export class NetworkManager extends EventEmitter {
         }
     }
 
+    // Simple audio frame sending (raw PCM) - matches original working implementation
+    sendAudioFrame(frame: Uint8Array) {
+        if (!this.udp) return;
+
+        const header = Buffer.alloc(HEADER_SIZE);
+        header.writeUInt8(1, 0); // Version
+        header.writeUInt8(MediaType.Audio, 1);
+        header.writeUInt32LE(this.userId, 2);
+        header.writeUInt32LE(this.audioSeq++, 6);
+        header.writeBigUInt64LE(BigInt(Date.now()), 10);
+        header.writeUInt16LE(0, 18); // Frag idx
+        header.writeUInt16LE(1, 20); // Frag cnt
+        header.writeUInt16LE(0, 22); // Flags
+
+        const packet = Buffer.concat([header, Buffer.from(frame)]);
+
+        // Send directly via pacer queue
+        this.udpQueue.push(packet);
+    }
+
     sendEncodedAudioChunk(chunk: Uint8Array, timestamp: number) {
         if (!this.udp) {
             console.warn('[Network] UDP Socket not ready for Audio');