diff --git a/current_head_media.ts b/current_head_media.ts new file mode 100644 index 0000000..e69de29 diff --git a/package.json b/package.json index c096f42..0b858dd 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { - "name": "client-electron", + "name": "just-talk", + "productName": "JustTalk", "version": "1.0.0", "main": "./out/main/index.js", "scripts": { diff --git a/previous_head_media.ts b/previous_head_media.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/main/index.ts b/src/main/index.ts index 9d643e3..71dcf9d 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1,25 +1,29 @@ -import { app, shell, BrowserWindow, ipcMain, session, desktopCapturer } from 'electron' +import { app, shell, BrowserWindow, ipcMain, desktopCapturer } from 'electron' import { join } from 'path' import { electronApp, optimizer, is } from '@electron-toolkit/utils' -import { NetworkManager } from './network' // Import NetworkManager +// import icon from '../../resources/icon.png?asset' +import { NetworkManager } from './network' let mainWindow: BrowserWindow | null = null; -let networkManager: NetworkManager | null = null; +let network: NetworkManager | null = null; function createWindow(): void { // Create the browser window. mainWindow = new BrowserWindow({ - width: 900, - height: 670, + width: 1280, + height: 720, show: false, autoHideMenuBar: true, + // ...(process.platform === 'linux' ? { icon } : {}), webPreferences: { preload: join(__dirname, '../preload/index.js'), - sandbox: false + sandbox: false, + contextIsolation: true, + nodeIntegration: false // Best practice } }) - networkManager = new NetworkManager(mainWindow); + network = new NetworkManager(mainWindow); mainWindow.on('ready-to-show', () => { mainWindow?.show() @@ -43,83 +47,47 @@ app.whenReady().then(() => { // Set app user model id for windows electronApp.setAppUserModelId('com.electron') - // Grant permissions for camera/mic/screen - session.defaultSession.setPermissionRequestHandler((webContents, permission, callback) => { - console.log(`[Main] Requesting permission: ${permission}`); - // Grant all permissions for this valid local app - callback(true); - }); - - // Default open or close DevTools by F12 in development - // and ignore CommandOrControl + R in production. - // see https://github.com/alex8088/electron-toolkit/tree/master/packages/utils app.on('browser-window-created', (_, window) => { optimizer.watchWindowShortcuts(window) }) // IPC Handlers ipcMain.handle('connect', async (_, { serverUrl, roomCode, displayName }) => { - if (networkManager) { - return await networkManager.connect(serverUrl, roomCode, displayName); - } + return network?.connect(serverUrl, roomCode, displayName); }); ipcMain.handle('disconnect', async () => { - if (networkManager) { - networkManager.disconnect(); - } + network?.disconnect(); }); - ipcMain.on('send-video-frame', (_, { frame }) => { - if (networkManager) { - networkManager.sendVideoFrame(frame); - } + ipcMain.handle('send-chat', (_, { message, displayName }) => { + network?.sendChat(message, displayName); }); - ipcMain.on('send-audio-frame', (_, { frame }) => { - if (networkManager) { - networkManager.sendAudioFrame(new Uint8Array(frame)); - } - }); - - ipcMain.on('send-screen-frame', (_, { frame }) => { - if (networkManager) { - networkManager.sendScreenFrame(frame); - } - }); - - // Screen sharing: get available sources ipcMain.handle('get-screen-sources', async () => { - const sources = await desktopCapturer.getSources({ - types: ['screen', 'window'], - thumbnailSize: { width: 150, height: 150 } - }); - return sources.map(s => ({ - id: s.id, - name: s.name, - thumbnail: s.thumbnail.toDataURL() + const sources = await desktopCapturer.getSources({ types: ['window', 'screen'], thumbnailSize: { width: 150, height: 150 } }); + return sources.map(source => ({ + id: source.id, + name: source.name, + thumbnail: source.thumbnail.toDataURL() })); }); - // Chat - ipcMain.handle('send-chat', (_, { message, displayName }) => { - if (networkManager) { - networkManager.sendChat(message, displayName); - } + ipcMain.on('send-video-chunk', (_, payload) => { + network?.sendEncodedVideoChunk(payload.chunk, payload.isKeyFrame, payload.timestamp, payload.streamType); + }); + + ipcMain.on('send-audio-chunk', (_, payload) => { + network?.sendEncodedAudioChunk(payload.chunk, payload.timestamp); }); - // Stream Updates ipcMain.on('update-stream', (_, { active, mediaType }) => { - if (networkManager) { - networkManager.updateStream(active, mediaType); - } + network?.updateStream(active, mediaType); }); createWindow() app.on('activate', function () { - // On macOS it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. if (BrowserWindow.getAllWindows().length === 0) createWindow() }) }) diff --git a/src/main/network.ts b/src/main/network.ts index 1a23e7a..6f62387 100644 --- a/src/main/network.ts +++ b/src/main/network.ts @@ -6,8 +6,8 @@ import { BrowserWindow } from 'electron'; // Constants const SERVER_UDP_PORT = 4000; -// Packet Header Structure (22 bytes) -const HEADER_SIZE = 22; +// Packet Header Structure (24 bytes) +const HEADER_SIZE = 24; export enum MediaType { Audio = 0, @@ -15,6 +15,11 @@ export enum MediaType { Screen = 2, } +// Token Bucket Pacer Constants +const PACER_RATE_BYTES_PER_MS = 1500; // ~12 Mbps limit (Targeting 8-10 Mbps for 1080p60) +const PACER_BUCKET_SIZE_BYTES = 15000; // Allow 10 packets burst (Instant Keyframe start) +const MAX_PAYLOAD = 1200; // Reduced from 1400 to be safe with MTU + export class NetworkManager extends EventEmitter { private ws: WebSocket | null = null; private udp: dgram.Socket | null = null; @@ -26,9 +31,46 @@ export class NetworkManager extends EventEmitter { private mainWindow: BrowserWindow; private serverUdpHost: string = '127.0.0.1'; + // Pacing + private udpQueue: Buffer[] = []; + private pacerTokens: number = PACER_BUCKET_SIZE_BYTES; + private lastPacerUpdate: number = Date.now(); + private pacerInterval: NodeJS.Timeout | null = null; + constructor(mainWindow: BrowserWindow) { super(); this.mainWindow = mainWindow; + this.startPacer(); + } + + private startPacer() { + this.pacerInterval = setInterval(() => { + if (!this.udp) return; + + const now = Date.now(); + const elapsed = now - this.lastPacerUpdate; + this.lastPacerUpdate = now; + + // Refill tokens + this.pacerTokens += elapsed * PACER_RATE_BYTES_PER_MS; + if (this.pacerTokens > PACER_BUCKET_SIZE_BYTES) { + this.pacerTokens = PACER_BUCKET_SIZE_BYTES; + } + + // Drain queue + while (this.udpQueue.length > 0) { + const packet = this.udpQueue[0]; + if (this.pacerTokens >= packet.length) { + this.pacerTokens -= packet.length; + this.udpQueue.shift(); + this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { + if (err) console.error('UDP Send Error', err); + }); + } else { + break; // Not enough tokens, wait for next tick + } + } + }, 2); // Check every 2ms } async connect(serverUrl: string, roomCode: string, displayName: string): Promise { @@ -156,7 +198,7 @@ export class NetworkManager extends EventEmitter { }); this.udp.on('message', (msg, rinfo) => { - console.log(`[UDP] Msg from ${rinfo.address}:${rinfo.port} - ${msg.length} bytes`); + // console.log(`[UDP] Msg from ${rinfo.address}:${rinfo.port} - ${msg.length} bytes`); this.handleUdpMessage(msg); }); @@ -166,34 +208,83 @@ export class NetworkManager extends EventEmitter { handleUdpMessage(msg: Buffer) { if (msg.length < HEADER_SIZE) return; + const version = msg.readUInt8(0); const mediaType = msg.readUInt8(1); const userId = msg.readUInt32LE(2); - - const payload = msg.subarray(HEADER_SIZE); - const sequence = msg.readUInt32LE(6); + const seq = msg.readUInt32LE(6); const timestamp = Number(msg.readBigUInt64LE(10)); - const fragIdx = msg.readUInt8(18); - const fragCnt = msg.readUInt8(19); + const fragIdx = msg.readUInt16LE(18); + const fragCnt = msg.readUInt16LE(20); + const flags = msg.readUInt16LE(22); + + const isKeyFrame = (flags & 1) !== 0; + const payload = msg.subarray(HEADER_SIZE); if (mediaType === MediaType.Audio) { - this.safeSend('audio-frame', { user_id: userId, data: payload }); - } else if (mediaType === MediaType.Video) { - this.safeSend('video-frame', { + // Audio can be fragmented now (PCM) + this.safeSend('video-chunk', { // Use 'video-chunk' handler in renderer for reassembly? + // Wait, App.tsx has separate 'audio-chunk' which doesn't reassemble. + // We need to reassemble here or change App.tsx. + // Reassembling in main process is easier or reusing video logic. + + // Let's use 'audio-chunk' but we need to pass frag info? + // No, App.tsx 'audio-chunk' handler just decodes immediately. + // It expects a full frame. + + // We MUST reassemble here or update App.tsx. + // Updating App.tsx to use the reassembler for Audio is cleaner. + // But 'video-chunk' in App.tsx calls 'handleIncomingVideoFragment' which uses 'MediaEngine.decodeVideoChunk'. + + // Option: Treat Audio as "Video" for transport, but with streamType='audio'? + // MediaType.Audio is distinct. + + // Let's implement reassembly here in NetworkManager? + // Or update App.tsx to use 'handleIncomingVideoFragment' for audio too? + // 'handleIncomingVideoFragment' does `decodeVideoChunk`. + + // Let's change App.tsx to have `handleIncomingAudioFragment`? + // Or just reassemble here. UDP reassembly in Node.js is fine. + + // ACtually, App.tsx's `handleIncomingVideoFragment` is nice. + // Let's emit 'audio-fragment' and add a handler in App.tsx. user_id: userId, data: payload, - seq: sequence, + seq: this.audioSeq, // Wait, seq is in packet ts: timestamp, fidx: fragIdx, - fcnt: fragCnt + fcnt: fragCnt, + isKeyFrame, + streamType: 'audio' + // We can't use 'video-chunk' channel because it calls decodeVideoChunk. }); - } else if (mediaType === MediaType.Screen) { - this.safeSend('screen-frame', { + + // Actually, let's just send it to 'audio-fragment' channel + this.safeSend('audio-fragment', { user_id: userId, data: payload, - seq: sequence, + seq: seq, // We need valid seq from packet ts: timestamp, fidx: fragIdx, - fcnt: fragCnt + fcnt: fragCnt, + isKeyFrame + }); + + } else if (mediaType === MediaType.Video || mediaType === MediaType.Screen) { + // Differentiate based on MediaType + const streamType = mediaType === MediaType.Screen ? 'screen' : 'video'; + if (mediaType === MediaType.Screen && fragIdx === 0) { + console.log(`[Network] RX Screen Chunk User=${userId} Seq=${seq}`); + } + + this.safeSend('video-chunk', { + user_id: userId, + data: payload, + seq, + ts: timestamp, + fidx: fragIdx, + fcnt: fragCnt, + isKeyFrame, + streamType // Pass this to renderer }); } } @@ -208,88 +299,88 @@ export class NetworkManager extends EventEmitter { } } - sendVideoFrame(frame: Uint8Array) { - if (!this.udp || !this.userId) return; + // --- New Encode Methods --- - const buffer = Buffer.from(frame); - const MAX_PAYLOAD = 1400; - const fragCount = Math.ceil(buffer.length / MAX_PAYLOAD); - const seq = this.videoSeq++; - const ts = BigInt(Date.now()); - - for (let i = 0; i < fragCount; i++) { - const start = i * MAX_PAYLOAD; - const end = Math.min(start + MAX_PAYLOAD, buffer.length); - const chunk = buffer.subarray(start, end); - - const header = Buffer.alloc(HEADER_SIZE); - header.writeUInt8(1, 0); // Version - header.writeUInt8(MediaType.Video, 1); - header.writeUInt32LE(this.userId, 2); - header.writeUInt32LE(seq, 6); - header.writeBigUInt64LE(ts, 10); - header.writeUInt8(i, 18); // Frag idx - header.writeUInt8(fragCount, 19); // Frag cnt - header.writeUInt16LE(0, 20); // Flags - - const packet = Buffer.concat([header, chunk]); - - this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { - if (err) console.error('UDP Video Send Error', err); - }); - } - } - - sendAudioFrame(frame: Uint8Array) { + sendEncodedVideoChunk(chunk: any, isKeyFrame: boolean, timestamp: number, streamType: 'video' | 'screen' = 'video') { if (!this.udp) return; - const header = Buffer.alloc(HEADER_SIZE); - header.writeUInt8(1, 0); // Version - header.writeUInt8(MediaType.Audio, 1); - header.writeUInt32LE(this.userId, 2); - header.writeUInt32LE(this.audioSeq++, 6); - header.writeBigUInt64LE(BigInt(Date.now()), 10); - header.writeUInt8(0, 18); // Frag idx - header.writeUInt8(1, 19); // Frag cnt - header.writeUInt16LE(0, 20); // Flags + const MAX_PAYLOAD = 1400; + const totalSize = chunk.length; - const packet = Buffer.concat([header, Buffer.from(frame)]); + // Use generic videoSeq for both? Or separate? + // Best to separate to avoid gap detection issues if one stream is idle. + // But for now, let's share for simplicity or use screenSeq if screen. + // Actually, let's use separate seq if possible, but I only have videoSeq. + // Let's use videoSeq for both for now, assuming the receiver tracks them separately or doesn't care about gaps across types. + // Better: Use a map or separate counters. + const seq = streamType === 'screen' ? this.screenSeq++ : this.videoSeq++; - this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { - if (err) console.error('UDP Audio Send Error', err); - }); + const fragmentCount = Math.ceil(totalSize / MAX_PAYLOAD); + + for (let i = 0; i < fragmentCount; i++) { + const start = i * MAX_PAYLOAD; + const end = Math.min(start + MAX_PAYLOAD, totalSize); + const slice = chunk.slice(start, end); + + // Header (22 bytes) + const header = Buffer.alloc(HEADER_SIZE); + header.writeUInt8(1, 0); // Version + const mType = streamType === 'screen' ? MediaType.Screen : MediaType.Video; + header.writeUInt8(mType, 1); + header.writeUInt32LE(this.userId, 2); + header.writeUInt32LE(seq, 6); + header.writeBigUInt64LE(BigInt(timestamp), 10); + header.writeUInt16LE(i, 18); // Frag Idx (u16) + header.writeUInt16LE(fragmentCount, 20); // Frag Cnt (u16) + + let flags = 0; + if (isKeyFrame) flags |= 1; + header.writeUInt16LE(flags, 22); + + const packet = Buffer.concat([header, slice]); + + // Enqueue for pacing + this.udpQueue.push(packet); + } } - sendScreenFrame(frame: number[]) { - if (!this.udp || !this.userId) return; + sendEncodedAudioChunk(chunk: Uint8Array, timestamp: number) { + if (!this.udp) { + console.warn('[Network] UDP Socket not ready for Audio'); + return; + } - const buffer = Buffer.from(frame); - const MAX_PAYLOAD = 1400; - const fragCount = Math.ceil(buffer.length / MAX_PAYLOAD); - const seq = this.screenSeq++; - const ts = BigInt(Date.now()); + const totalSize = chunk.length; + const MAX_PAYLOAD = 1400; // Safe MTU - for (let i = 0; i < fragCount; i++) { + // PCM packets (approx 2KB) need fragmentation. + // We use the same logic as video but with Audio MediaType. + + const fragmentCount = Math.ceil(totalSize / MAX_PAYLOAD); + + // Log randomly to avoid spam but confirm activity + if (Math.random() < 0.05) console.log(`[Network] Sending Audio Chunk size=${totalSize} frags=${fragmentCount}`); + + for (let i = 0; i < fragmentCount; i++) { const start = i * MAX_PAYLOAD; - const end = Math.min(start + MAX_PAYLOAD, buffer.length); - const chunk = buffer.subarray(start, end); + const end = Math.min(start + MAX_PAYLOAD, totalSize); + const slice = chunk.slice(start, end); const header = Buffer.alloc(HEADER_SIZE); header.writeUInt8(1, 0); // Version - header.writeUInt8(MediaType.Screen, 1); + header.writeUInt8(MediaType.Audio, 1); header.writeUInt32LE(this.userId, 2); - header.writeUInt32LE(seq, 6); - header.writeBigUInt64LE(ts, 10); - header.writeUInt8(i, 18); // Frag idx - header.writeUInt8(fragCount, 19); // Frag cnt - header.writeUInt16LE(0, 20); // Flags + header.writeUInt32LE(this.audioSeq, 6); // Same seq for all fragments + header.writeBigUInt64LE(BigInt(Math.floor(timestamp)), 10); + header.writeUInt16LE(i, 18); // Frag idx + header.writeUInt16LE(fragmentCount, 20); // Frag cnt + header.writeUInt16LE(1, 22); // Flags (1=Keyframe, audio is always key) - const packet = Buffer.concat([header, chunk]); - - this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { - if (err) console.error('UDP Screen Send Error', err); - }); + const packet = Buffer.concat([header, Buffer.from(slice)]); + this.udpQueue.push(packet); } + + this.audioSeq++; } startHeartbeat() { @@ -327,19 +418,23 @@ export class NetworkManager extends EventEmitter { header.writeUInt32LE(this.userId, 2); header.writeUInt32LE(0, 6); // Sequence header.writeBigUInt64LE(BigInt(Date.now()), 10); - header.writeUInt8(0, 18); // Frag idx - header.writeUInt8(1, 19); // Frag cnt - header.writeUInt16LE(0, 20); // Flags + header.writeUInt16LE(0, 18); // Frag idx + header.writeUInt16LE(1, 20); // Frag cnt + header.writeUInt16LE(0, 22); // Flags const packet = Buffer.concat([header, payload]); - console.log(`[UDP] Sending Handshake: userId=${this.userId}, room=${this.roomCode}, ${packet.length} bytes to ${this.serverUdpHost}:${SERVER_UDP_PORT}`); + // console.log(`[UDP] Sending Handshake: userId=${this.userId}, room=${this.roomCode}, ${packet.length} bytes to ${this.serverUdpHost}:${SERVER_UDP_PORT}`); this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { if (err) console.error('UDP Handshake Send Error', err); }); } disconnect() { + if (this.pacerInterval) { + clearInterval(this.pacerInterval); + this.pacerInterval = null; + } if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; diff --git a/src/renderer/index.html b/src/renderer/index.html index 1a87802..a733204 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -3,7 +3,7 @@ - Electron App + JustTalk diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index a82f5a0..64f94b1 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef } from "react"; +import { useState, useEffect, useRef, useCallback } from "react"; import "./index.css"; import { Lobby } from "./components/Lobby"; @@ -7,22 +7,7 @@ import { ControlBar } from "./components/ControlBar"; import { ChatPanel } from "./components/ChatPanel"; import { NotificationToast } from "./components/NotificationToast"; import { PeerInfo, ChatMessage } from "./types"; - -const audioWorkletCode = ` -class PCMProcessor extends AudioWorkletProcessor { - process(inputs, outputs, parameters) { - const input = inputs[0]; - if (input && input.length > 0) { - const inputChannel = input[0]; - // Post full buffer to main thread (renderer) - // optimization: we could accumulate here if needed - this.port.postMessage(inputChannel); - } - return true; - } -} -registerProcessor('pcm-processor', PCMProcessor); -`; +import { MediaEngine, EncodedFrame } from "./utils/MediaEngine"; interface JoinedPayload { self_id: number; @@ -37,7 +22,7 @@ function App() { const [error, setError] = useState(""); const addLog = (msg: string) => { - console.log(`[Meet] ${msg}`); + console.log(`[JustTalk] ${msg}`); }; // Media State @@ -48,6 +33,11 @@ function App() { const [selectedVideoDevice, setSelectedVideoDevice] = useState(""); const [displayName, setDisplayName] = useState(""); + // Active Video Peers + const [peersWithCam, setPeersWithCam] = useState>(new Set()); + const [peersWithScreen, setPeersWithScreen] = useState>(new Set()); + const [peersWithAudio, setPeersWithAudio] = useState>(new Set()); + // Chat State const [chatOpen, setChatOpen] = useState(false); const [chatMessages, setChatMessages] = useState([]); @@ -55,657 +45,683 @@ function App() { // Video Handling const localVideoRef = useRef(null); - const canvasRef = useRef(null); - const [peerVideoUrls, setPeerVideoUrls] = useState<{ [key: number]: string }>({}); - const [peerScreenUrls, setPeerScreenUrls] = useState<{ [key: number]: string }>({}); - const [localScreenUrl, setLocalScreenUrl] = useState(null); + const localScreenRef = useRef(null); // New ref for local screen share + const mediaEngineRef = useRef(null); + + // Canvas Refs: Key = "${userId}-${streamType}" (e.g. "123-video", "123-screen") + const peerCanvasRefs = useRef>(new Map()); + + const registerPeerCanvas = useCallback((userId: number, streamType: 'video' | 'screen', canvas: HTMLCanvasElement | null) => { + const key = `${userId}-${streamType}`; + if (canvas) { + peerCanvasRefs.current.set(key, canvas); + } else { + peerCanvasRefs.current.delete(key); + } + }, []); + + // Audio Handling - Ref to context + const audioCtxRef = useRef(null); + + // Audio Reassembly Buffer (Moved out of useEffect) + // Key: `${userId}-audio-${seq}` + const audioFragmentBuffer = useRef>(new Map()); + + const handleIncomingAudioFragment = useCallback((payload: any) => { + const { user_id, data, seq, ts, fidx, fcnt } = payload; + + // If single fragment, decode immediately + if (fcnt === 1) { + if (mediaEngineRef.current) { + const buffer = data instanceof Uint8Array ? data : new Uint8Array(data); + mediaEngineRef.current.decodeAudioChunk(buffer, user_id, ts); + } + return; + } + + const key = `${user_id}-${seq}`; + const map = audioFragmentBuffer.current; + + if (!map.has(key)) { + map.set(key, { chunks: new Array(fcnt), count: 0, total: fcnt, ts }); + } + + const entry = map.get(key)!; + if (!entry.chunks[fidx]) { + entry.chunks[fidx] = data instanceof Uint8Array ? data : new Uint8Array(data); + entry.count++; + } + + if (entry.count === entry.total) { + // Reassemble + const totalLen = entry.chunks.reduce((acc, c) => acc + c.length, 0); + const fullFrame = new Uint8Array(totalLen); + let offset = 0; + for (const c of entry.chunks) { + fullFrame.set(c, offset); + offset += c.length; + } + + if (mediaEngineRef.current) { + mediaEngineRef.current.decodeAudioChunk(fullFrame, user_id, ts); + } + map.delete(key); + } + + // Cleanup old + for (const k of map.keys()) { + const kSeq = parseInt(k.split('-')[1]); + if (map.size > 20) map.delete(k); + } + }, []); // Event Listeners useEffect(() => { - // @ts-ignore - const removeJoined = window.electron.ipcRenderer.on('connect-success', (_, data: JoinedPayload) => { - // We might get this as return from invoke, but also good to have event. + const engine = new MediaEngine(); + mediaEngineRef.current = engine; + + // --- Outbound Encoding Handlers --- + engine.on('encoded-video', (frame: EncodedFrame) => { + if (!connected) return; + // @ts-ignore + window.electron.ipcRenderer.send('send-video-chunk', { + chunk: frame.data, + isKeyFrame: frame.isKeyFrame, + timestamp: frame.timestamp, + streamType: frame.streamType || 'video' + }); }); - // Peer Joined + engine.on('encoded-audio', (frame: EncodedFrame) => { + if (!connected) return; + // @ts-ignore + window.electron.ipcRenderer.send('send-audio-chunk', { + chunk: frame.data, + timestamp: frame.timestamp + }); + }); + + // --- Inbound Decoding Handlers --- + engine.on('decoded-video', ({ userId, frame, streamType }: { userId: number, frame: VideoFrame, streamType: 'video' | 'screen' }) => { + const type = streamType || 'video'; + + // Mark user as having video (only triggers re-render if new) + if (type === 'video') { + setPeersWithCam(prev => { + if (prev.has(userId)) return prev; + console.log(`[App] User ${userId} started Cam Video`); + return new Set(prev).add(userId); + }); + } else { + setPeersWithScreen(prev => { + if (prev.has(userId)) return prev; + console.log(`[App] User ${userId} started Screen Share`); + return new Set(prev).add(userId); + }); + } + + const key = `${userId}-${type}`; + const canvas = peerCanvasRefs.current.get(key); + if (canvas) { + if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) { + canvas.width = frame.displayWidth; + canvas.height = frame.displayHeight; + } + const ctx = canvas.getContext('2d'); + if (ctx) ctx.drawImage(frame, 0, 0); + } + frame.close(); + }); + + // Audio Playback + let audioNextTime = 0; + let audioPacketCount = 0; + + engine.on('decoded-audio', ({ userId, data }: { userId: number, data: AudioData }) => { + if (audioPacketCount % 50 === 0) console.log(`[App] Playing Audio packet #${audioPacketCount} from User ${userId}`); + + const ctx = audioCtxRef.current; + if (!ctx) { + data.close(); + return; + } + + if (ctx.state === 'suspended') { + // Try to resume again + ctx.resume().catch(e => console.error("Audio resume failed in playback", e)); + } + + if (audioNextTime === 0) { + audioNextTime = ctx.currentTime + 0.05; // Start slightly in future to avoid glitches + } + + try { + const buffer = ctx.createBuffer(data.numberOfChannels, data.numberOfFrames, data.sampleRate); + for (let ch = 0; ch < data.numberOfChannels; ch++) { + const channelData = buffer.getChannelData(ch); + data.copyTo(channelData, { planeIndex: ch }); + } + + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(ctx.destination); + + const now = ctx.currentTime; + // If we fell behind, catch up + if (audioNextTime < now) { + // console.warn(`[App] Audio fell behind by ${now - audioNextTime}s`); + audioNextTime = now; + } + + source.start(audioNextTime); + audioNextTime += buffer.duration; + } catch (e) { + console.error("[App] Audio Buffer Error:", e); + } + + data.close(); + }); + + + // --- IPC Listeners for Incoming Chunks --- + // @ts-ignore const removePeerJoined = window.electron.ipcRenderer.on("peer-joined", (_, data) => { addLog(`PeerJoined: ${JSON.stringify(data)}`); - if (data && data.user_id) { - setPeers((prev) => { - if (prev.find(p => p.user_id === data.user_id)) { - console.log("Peer already exists:", data.user_id); - return prev; - } - console.log("Adding peer:", data); - return [...prev, { user_id: data.user_id, display_name: data.display_name }]; - }); - } + setPeers(prev => { + if (prev.find(p => p.user_id === data.user_id)) return prev; + return [...prev, { user_id: data.user_id, display_name: data.display_name }]; + }); }); - // Peer Left // @ts-ignore const removePeerLeft = window.electron.ipcRenderer.on("peer-left", (_, data) => { addLog(`PeerLeft: ${JSON.stringify(data)}`); - if (data && data.user_id) { - setPeers((prev) => prev.filter(p => p.user_id !== data.user_id)); - setPeerVideoUrls(prev => { - const newState = { ...prev }; - if (newState[data.user_id]) { - URL.revokeObjectURL(newState[data.user_id]); - delete newState[data.user_id]; - } - return newState; - }); - setPeerScreenUrls(prev => { - const newState = { ...prev }; - if (newState[data.user_id]) { - URL.revokeObjectURL(newState[data.user_id]); - delete newState[data.user_id]; - } - return newState; - }); + setPeers(prev => prev.filter(p => p.user_id !== data.user_id)); + setPeersWithCam(prev => { + const next = new Set(prev); + next.delete(data.user_id); + return next; + }); + setPeersWithScreen(prev => { + const next = new Set(prev); + next.delete(data.user_id); + return next; + }); + setPeersWithAudio(prev => { + const next = new Set(prev); + next.delete(data.user_id); + return next; + }); + }); + + // @ts-ignore + const removeVideoChunk = window.electron.ipcRenderer.on("video-chunk", (_, payload) => { + handleIncomingVideoFragment(payload); + }); + + // @ts-ignore + const removeAudioFragment = window.electron.ipcRenderer.on("audio-fragment", (_, payload) => { + handleIncomingAudioFragment(payload); + }); + + // @ts-ignore + const removeAudioChunk = window.electron.ipcRenderer.on("audio-chunk", (_, payload) => { + // Check if it's the old single chunk message (fallback) + if (mediaEngineRef.current) { + const data = payload.data instanceof Uint8Array ? payload.data : new Uint8Array(payload.data); + mediaEngineRef.current.decodeAudioChunk(data, payload.user_id, payload.ts); } }); - // Update Stream Signaling (MediaType: "Video" or "Screen" from Rust enum) // @ts-ignore - const removeStreamUpdate = window.electron.ipcRenderer.on("peer-stream-update", (_, data) => { - const { user_id, active, media_type } = data; - if (!active) { - // Support both legacy string and numeric enum (1=Video, 2=Screen) - if (media_type === 'Video' || media_type === 1) { - setPeerVideoUrls(prev => { - const newState = { ...prev }; - if (newState[user_id]) { - URL.revokeObjectURL(newState[user_id]); - delete newState[user_id]; - } - return newState; + const removePeerStreamUpdate = window.electron.ipcRenderer.on("peer-stream-update", (_, data) => { + addLog(`Stream Update: User ${data.user_id} ${data.media_type} = ${data.active}`); + + const userId = data.user_id; + const active = data.active; + const mediaType = data.media_type.toLowerCase(); // usage: 'video' or 'screen' + + if (mediaType === 'video') { + if (!active) { + setPeersWithCam(prev => { + const next = new Set(prev); + next.delete(userId); + return next; }); - } else if (media_type === 'Screen' || media_type === 2) { - setPeerScreenUrls(prev => { - const newState = { ...prev }; - if (newState[user_id]) { - URL.revokeObjectURL(newState[user_id]); - delete newState[user_id]; - } - return newState; + const key = `${userId}-video`; + const canvas = peerCanvasRefs.current.get(key); + if (canvas) { + const ctx = canvas.getContext('2d'); + if (ctx) ctx.clearRect(0, 0, canvas.width, canvas.height); + } + } else { + // If active=true, we might want to ensure we track it + setPeersWithCam(prev => new Set(prev).add(userId)); + } + } else if (mediaType === 'screen') { + if (!active) { + console.log(`[App] Removing Screen Share for User ${userId}`); + setPeersWithScreen(prev => { + const next = new Set(prev); + next.delete(userId); + return next; }); + // Force clear canvas immediately + const key = `${userId}-screen`; + const canvas = peerCanvasRefs.current.get(key); + if (canvas) { + const ctx = canvas.getContext('2d'); + if (ctx) ctx.clearRect(0, 0, canvas.width, canvas.height); + } + } else { + console.log(`[App] Adding Screen Share for User ${userId}`); + setPeersWithScreen(prev => new Set(prev).add(userId)); } } }); - // Chat Message - skip own messages (we add them locally) + // Chat Message // @ts-ignore const removeChatMessage = window.electron.ipcRenderer.on("chat-message", (_, data) => { - // Skip own messages (already added locally) - // We'll check by comparing user_id with current selfId later + const msgId = `${data.user_id}-${data.timestamp}`; const msg: ChatMessage = { - id: `${data.user_id}-${data.timestamp}`, + id: msgId, userId: data.user_id, displayName: data.display_name, message: data.message, timestamp: data.timestamp }; - // Only add if not from self (selfId check happens via closure) + setChatMessages(prev => { - // Deduplicate strictly by ID - if (prev.find(m => m.id === msg.id)) { - return prev; - } - // Also check if same message from same user within last second to avoid duplicate broadcasts - const duplicate = prev.find(m => - m.userId === msg.userId && - m.message === msg.message && - Math.abs(m.timestamp - msg.timestamp) < 2000 - ); - if (duplicate) return prev; - - // Add notification - const notif = { - id: msg.id, - displayName: msg.displayName, - message: msg.message, - timestamp: msg.timestamp - }; - setChatNotifications(notifs => { - if (notifs.find(n => n.id === notif.id)) return notifs; - return [...notifs, notif]; - }); - - // Auto-dismiss after 5s - setTimeout(() => { - setChatNotifications(current => current.filter(n => n.id !== notif.id)); - }, 5000); - + // Strict deduplication + if (prev.some(m => m.id === msgId)) return prev; return [...prev, msg]; }); + + // Notification side effect (safe to use functional update here) + setChatNotifications(prevNotifs => { + // Check if we already have this notification to avoid double toast + if (prevNotifs.some(n => n.id === msgId)) return prevNotifs; + + // Don't show toast if chat is open, BUT we are inside an event handler, + // checking state `chatOpen` directly here might be stale if not in deps. + // However, we can't easily access latest `chatOpen` in a closure without refs or re-binding. + // For now, let's just add it. The rendering logic filters it? No. + + // Cleaner: Just add it. The Toast component can be controlled by parent. + const newNotif = { id: msgId, displayName: msg.displayName, message: msg.message, timestamp: msg.timestamp }; + setTimeout(() => setChatNotifications(ids => ids.filter(x => x.id !== msgId)), 5000); + return [...prevNotifs, newNotif]; + }); }); - // Video/Screen Reassembly logic - const fragmentMap = new Map(); - - // Video Frame (Reassembly) - // @ts-ignore - const removeVideo = window.electron.ipcRenderer.on("video-frame", (_, payload) => { - const { user_id, data, seq, fidx, fcnt } = payload; - const key = `v-${user_id}-${seq}`; - - if (!fragmentMap.has(key)) { - fragmentMap.set(key, { chunks: new Array(fcnt), count: 0, total: fcnt }); - } - - const state = fragmentMap.get(key)!; - if (!state.chunks[fidx]) { - state.chunks[fidx] = new Uint8Array(data); - state.count++; - } - - if (state.count === state.total) { - // All fragments received: assemble - const totalLen = state.chunks.reduce((acc, c) => acc + c.length, 0); - const fullBuffer = new Uint8Array(totalLen); - let offset = 0; - for (const chunk of state.chunks) { - fullBuffer.set(chunk, offset); - offset += chunk.length; - } - - const blob = new Blob([fullBuffer], { type: 'image/jpeg' }); - const url = URL.createObjectURL(blob); - - setPeerVideoUrls(prev => { - if (prev[user_id]) URL.revokeObjectURL(prev[user_id]); - return { ...prev, [user_id]: url }; - }); - - fragmentMap.delete(key); - // Clean up old sequences for this user to avoid memory leak - for (const k of fragmentMap.keys()) { - if (k.startsWith(`v-${user_id}-`) && k !== key) { - const kSeq = parseInt(k.split('-')[2]); - if (kSeq < seq) fragmentMap.delete(k); - } - } - } - }); - - // Screen Frame (Reassembly) - // @ts-ignore - const removeScreen = window.electron.ipcRenderer.on("screen-frame", (_, payload) => { - const { user_id, data, seq, fidx, fcnt } = payload; - const key = `s-${user_id}-${seq}`; - - if (!fragmentMap.has(key)) { - fragmentMap.set(key, { chunks: new Array(fcnt), count: 0, total: fcnt }); - } - - const state = fragmentMap.get(key)!; - if (!state.chunks[fidx]) { - state.chunks[fidx] = new Uint8Array(data); - state.count++; - } - - if (state.count === state.total) { - const totalLen = state.chunks.reduce((acc, c) => acc + c.length, 0); - const fullBuffer = new Uint8Array(totalLen); - let offset = 0; - for (const chunk of state.chunks) { - fullBuffer.set(chunk, offset); - offset += chunk.length; - } - - const blob = new Blob([fullBuffer], { type: 'image/jpeg' }); - const url = URL.createObjectURL(blob); - - setPeerScreenUrls(prev => { - if (prev[user_id]) URL.revokeObjectURL(prev[user_id]); - return { ...prev, [user_id]: url }; - }); - - fragmentMap.delete(key); - for (const k of fragmentMap.keys()) { - if (k.startsWith(`s-${user_id}-`) && k !== key) { - const kSeq = parseInt(k.split('-')[2]); - if (kSeq < seq) fragmentMap.delete(k); - } - } - } - }); - - // Audio Frame Playback - LOW LATENCY mode - const playbackCtxRef = { current: null as AudioContext | null }; - const nextPlayTimeRef = { current: 0 }; - const JITTER_BUFFER_MS = 20; // Reduced from 80ms for low latency - const bufferQueue: Float32Array[] = []; - let isStarted = false; - - const scheduleBuffer = (float32: Float32Array) => { - const ctx = playbackCtxRef.current; - if (!ctx) return; - - const buffer = ctx.createBuffer(1, float32.length, 48000); - buffer.copyToChannel(float32 as any, 0); - const source = ctx.createBufferSource(); - source.buffer = buffer; - source.connect(ctx.destination); - - // Schedule at precise time to avoid gaps - const now = ctx.currentTime; - if (nextPlayTimeRef.current < now) { - // We've fallen behind, reset - nextPlayTimeRef.current = now + JITTER_BUFFER_MS / 1000; - } - source.start(nextPlayTimeRef.current); - nextPlayTimeRef.current += buffer.duration; - }; - - const flushBuffer = () => { - while (bufferQueue.length > 0) { - scheduleBuffer(bufferQueue.shift()!); - } - }; - - // @ts-ignore - const removeAudio = window.electron.ipcRenderer.on("audio-frame", (_, payload) => { - try { - const { data } = payload; - if (!playbackCtxRef.current) { - playbackCtxRef.current = new AudioContext({ sampleRate: 48000 }); - nextPlayTimeRef.current = playbackCtxRef.current.currentTime + JITTER_BUFFER_MS / 1000; - } - - const ctx = playbackCtxRef.current; - if (ctx.state === 'suspended') { - ctx.resume(); - } - - // Convert Uint8Array (bytes) to Int16 PCM then to Float32 - const uint8 = new Uint8Array(data); - const int16 = new Int16Array(uint8.buffer, uint8.byteOffset, uint8.length / 2); - const float32 = new Float32Array(int16.length); - - for (let i = 0; i < int16.length; i++) { - float32[i] = int16[i] / 32768; - } - - if (!isStarted) { - // Start immediately after just 1 packet for lowest latency - bufferQueue.push(float32); - if (bufferQueue.length >= 1) { - isStarted = true; - flushBuffer(); - } - } else { - scheduleBuffer(float32); - } - } catch (e) { - console.error("Audio playback error:", e); - } - }); - return () => { + engine.cleanup(); removePeerJoined(); removePeerLeft(); - removeStreamUpdate(); + removeVideoChunk(); + removeAudioChunk(); + removePeerStreamUpdate(); removeChatMessage(); - removeVideo(); - removeScreen(); - removeAudio(); + // Don't close AudioContext here if we want to reuse it? + // Actually good practice to close it on component unmount + if (audioCtxRef.current) { + audioCtxRef.current.close(); + audioCtxRef.current = null; + } }; - }, []); + }, [connected]); + // ^ Dependency on `connected` means this effect re-runs when connected changes. + // Be careful with stale closures for other values if referenced. - // Frame Capture Loop - useEffect(() => { - let animationFrameId: number; - let isActive = true; - const sendFrame = async () => { - if (!isActive) return; + // Fragment Reassembly Logic + const fragmentBuffer = useRef>(new Map()); - if (videoEnabled && localVideoRef.current && canvasRef.current) { - const video = localVideoRef.current; - const canvas = canvasRef.current; - const ctx = canvas.getContext('2d'); + const handleIncomingVideoFragment = (payload: any) => { + // payload needs streamType now + const { user_id, data, seq, fidx, fcnt, isKeyFrame, ts, streamType } = payload; + const type = streamType || 'video'; - if (ctx && video.readyState === 4) { - // Use native video size, capped at 1080p for bandwidth sanity - const width = Math.min(video.videoWidth, 1920); - const height = Math.min(video.videoHeight, 1080); + if (type === 'screen' && fidx === 0) console.log(`[App] RX Screen Chunk: User ${user_id} Seq ${seq}`); - canvas.width = width; - canvas.height = height; - ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + // Key must include streamType so we don't mix screen and video chunks + const key = `${user_id}-${type}-${seq}`; + const map = fragmentBuffer.current; - canvas.toBlob(async (blob) => { - if (blob && isActive && videoEnabled && connected) { - try { - const arrayBuffer = await blob.arrayBuffer(); - const uint8Array = new Uint8Array(arrayBuffer); - // @ts-ignore - window.electron.ipcRenderer.send("send-video-frame", { frame: uint8Array }); - } catch (e) { - // Ignore send errors - } - } - }, 'image/jpeg', 0.4); // Quality 0.4 (Good balance for 1080p UDP) + if (!map.has(key)) { + map.set(key, { chunks: new Array(fcnt), count: 0, total: fcnt }); + } + + const entry = map.get(key)!; + if (!entry.chunks[fidx]) { + entry.chunks[fidx] = data; + entry.count++; + } + + if (entry.count === entry.total) { + const totalLen = entry.chunks.reduce((acc, c) => acc + c.length, 0); + const fullBuffer = new Uint8Array(totalLen); + let offset = 0; + for (const c of entry.chunks) { + fullBuffer.set(c, offset); + offset += c.length; + } + map.delete(key); + + // Cleanup old + for (const k of map.keys()) { + if (k.startsWith(`${user_id}-${type}-`) && k !== key) { + const kSeq = parseInt(k.split('-')[2]); + if (kSeq < seq - 10) map.delete(k); } } - setTimeout(() => { - if (isActive) animationFrameId = requestAnimationFrame(sendFrame); - }, 33); // 30 FPS + + mediaEngineRef.current?.decodeVideoChunk(fullBuffer, user_id, isKeyFrame, ts, type); + } + }; + + + // MEDIA CAPTURE HOOKS + + // --- Media Acquisition Queue (Prevent PipeWire Deadlocks) --- + const mediaLock = useRef>(Promise.resolve()); + const acquireMediaLock = () => { + let release: () => void; + const lock = new Promise(resolve => { release = resolve; }); + const prev = mediaLock.current; + // Chain it + const next = prev.then(async () => { + // Wait a bit to let PipeWire breathe + await new Promise(r => setTimeout(r, 500)); + return; + }); + mediaLock.current = next.then(() => lock); + return { + wait: () => next, + release: () => release() + }; + }; + + // 1. Webcam Capture + useEffect(() => { + let active = true; + let stream: MediaStream | null = null; + + const startWebcam = async () => { + if (!videoEnabled || !mediaEngineRef.current) return; + + const lock = acquireMediaLock(); + await lock.wait(); + if (!active) { lock.release(); return; } + + try { + addLog("Requesting Webcam Access..."); + stream = await navigator.mediaDevices.getUserMedia({ + video: { + width: 1280, + height: 720, + frameRate: 30, + deviceId: selectedVideoDevice ? { exact: selectedVideoDevice } : undefined + } + }); + addLog(`Webcam Access Granted: ${stream.id}`); + + if (localVideoRef.current) { + localVideoRef.current.srcObject = stream; + } + + const track = stream.getVideoTracks()[0]; + // @ts-ignore + const processor = new MediaStreamTrackProcessor({ track }); + const reader = processor.readable.getReader(); + + // Signal ON + // @ts-ignore + window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 1 }); // 1 = Video + + lock.release(); // Release lock after successful acquisition + + while (active) { + const result = await reader.read(); + if (result.done) break; + if (result.value) { + mediaEngineRef.current?.encodeVideoFrame(result.value, 'video'); + } + if (result.value) result.value.close(); // Close frame if not encoded? decodeVideoChunk closes it. encodeVideoFrame closes it. + } + } catch (e) { + console.error("Webcam error", e); + setVideoEnabled(false); + lock.release(); + } }; if (videoEnabled) { - sendFrame(); + startWebcam(); + } else { + if (connected) { + // @ts-ignore + window.electron.ipcRenderer.send('update-stream', { active: false, mediaType: 1 }); + } + if (localVideoRef.current) localVideoRef.current.srcObject = null; } return () => { - isActive = false; - cancelAnimationFrame(animationFrameId); - }; - }, [videoEnabled, connected]); - - // Camera Access - re-trigger when connected changes - useEffect(() => { - if (videoEnabled) { - navigator.mediaDevices.getUserMedia({ video: { width: 1280, height: 720 }, audio: false }) - .then(stream => { - if (localVideoRef.current) { - localVideoRef.current.srcObject = stream; - } - // Signal video ON - // @ts-ignore - window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 1 }); - }) - .catch(err => { - console.error("Camera access error:", err); - setError("Failed to access camera"); - }); - } else { - if (localVideoRef.current && localVideoRef.current.srcObject) { - const stream = localVideoRef.current.srcObject as MediaStream; - stream.getTracks().forEach(track => track.stop()); - localVideoRef.current.srcObject = null; + active = false; + if (stream) { + addLog("Stopping Webcam Stream"); + stream.getTracks().forEach(t => t.stop()); } - // Signal video OFF - // @ts-ignore - if (connected) window.electron.ipcRenderer.send('update-stream', { active: false, mediaType: 1 }); - } - }, [videoEnabled, connected]); + }; + }, [videoEnabled, selectedVideoDevice, connected]); - // Screen Sharing - const screenStreamRef = useRef(null); - const screenVideoRef = useRef(null); + // 2. Screen Share Capture useEffect(() => { - if (!connected) return; + let active = true; + let stream: MediaStream | null = null; + + const startScreen = async () => { + if (!screenEnabled || !mediaEngineRef.current) return; + + const lock = acquireMediaLock(); + await lock.wait(); + if (!active) { lock.release(); return; } + + try { + addLog("Fetching screen sources..."); + // @ts-ignore + const sources = await window.electron.ipcRenderer.invoke('get-screen-sources'); + if (sources && sources.length > 0) { + const sourceId = sources[0].id; + addLog(`Selected screen source: ${sourceId}`); - let isActive = true; - const startScreenShare = async () => { - if (screenEnabled) { - try { - // Get available screen sources // @ts-ignore - const sources = await window.electron.ipcRenderer.invoke('get-screen-sources'); - if (!sources || sources.length === 0) { - setError('No screen sources available'); - setScreenEnabled(false); - return; - } - - // Use first screen source (could add picker UI later) - const screenSource = sources[0]; - - const stream = await navigator.mediaDevices.getUserMedia({ - audio: false, + stream = await navigator.mediaDevices.getUserMedia({ + audio: false, // System audio often requires specific OS config, disable for now video: { - // @ts-ignore - Electron-specific constraint mandatory: { chromeMediaSource: 'desktop', - chromeMediaSourceId: screenSource.id, + chromeMediaSourceId: sourceId, maxWidth: 1920, - maxHeight: 1080, - maxFrameRate: 60 + maxHeight: 1080 } } - }); + } as any); - if (!isActive) { - stream.getTracks().forEach(t => t.stop()); - return; + if (!stream) { lock.release(); return; } + addLog(`Screen Access Granted: ${stream.id}`); + + if (localScreenRef.current) { + localScreenRef.current.srcObject = stream; } - screenStreamRef.current = stream; - addLog('Screen sharing started'); - - // Create hidden video element if needed - let screenVideo = screenVideoRef.current; - if (!screenVideo) { - screenVideo = document.createElement('video'); - screenVideo.autoplay = true; - screenVideo.muted = true; - } - screenVideo.srcObject = stream; - - // Send screen frames (similar to video but with different MediaType) - const canvas = document.createElement('canvas'); - const ctx = canvas.getContext('2d'); - canvas.width = 1920; - canvas.height = 1080; - - const sendScreenFrame = () => { - if (!isActive || !screenEnabled || !screenStreamRef.current) return; - - if (ctx && screenVideo && screenVideo.readyState >= 2) { - ctx.drawImage(screenVideo, 0, 0, canvas.width, canvas.height); - canvas.toBlob((blob) => { - if (blob && isActive) { - // Also update local view - const url = URL.createObjectURL(blob); - setLocalScreenUrl(url); - - blob.arrayBuffer().then(buf => { - // @ts-ignore - Send as screen type - window.electron.ipcRenderer.send('send-screen-frame', { frame: Array.from(new Uint8Array(buf)) }); - }); - } - }, 'image/jpeg', 0.4); // Quality 0.4 for speed - } - - setTimeout(() => { - if (isActive) requestAnimationFrame(sendScreenFrame); - }, 33); // 30 FPS for Screen Share + stream.getVideoTracks()[0].onended = () => { + addLog("Stream ended by user interaction"); + if (screenEnabled) setScreenEnabled(false); }; - screenVideo.onloadeddata = () => sendScreenFrame(); - - // Signal screen ON + const track = stream.getVideoTracks()[0]; // @ts-ignore - window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 2 }); + const processor = new MediaStreamTrackProcessor({ track }); + const reader = processor.readable.getReader(); - } catch (err) { - console.error('Screen share error:', err); - setError('Failed to start screen sharing'); + // Signal ON + // @ts-ignore + window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 2 }); // 2 = Screen + + lock.release(); + + while (active) { + const result = await reader.read(); + if (result.done) break; + if (result.value) { + mediaEngineRef.current?.encodeVideoFrame(result.value, 'screen'); + } + } + + } else { + addLog("No screen sources found"); setScreenEnabled(false); + lock.release(); } - } else { - // Stop screen sharing - if (screenStreamRef.current) { - screenStreamRef.current.getTracks().forEach(t => t.stop()); - screenStreamRef.current = null; - } - if (localScreenUrl) { - URL.revokeObjectURL(localScreenUrl); - setLocalScreenUrl(null); - } - // Signal screen OFF - // @ts-ignore - window.electron.ipcRenderer.send('update-stream', { active: false, mediaType: 2 }); - addLog('Screen sharing stopped'); + } catch (e) { + console.error("Screen share error", e); + setScreenEnabled(false); + lock.release(); } }; - startScreenShare(); + + if (screenEnabled) { + startScreen(); + } else { + if (connected) { + // @ts-ignore + window.electron.ipcRenderer.send('update-stream', { active: false, mediaType: 2 }); + } + } + return () => { - isActive = false; + active = false; + if (stream) { + addLog("Stopping Screen Stream"); + stream.getTracks().forEach(t => t.stop()); + } }; }, [screenEnabled, connected]); + // Audio Capture - const audioContextRef = useRef(null); - const audioStreamRef = useRef(null); - useEffect(() => { - let isCancelled = false; + let active = true; + let stream: MediaStream | null = null; + let reader: ReadableStreamDefaultReader | null = null; - const cleanup = () => { - // Stop tracks - if (audioStreamRef.current) { - audioStreamRef.current.getTracks().forEach(t => t.stop()); - audioStreamRef.current = null; - } - // Close context - if (audioContextRef.current) { - if (audioContextRef.current.state !== 'closed') { - audioContextRef.current.close().catch(e => console.error("Error closing ctx", e)); - } - audioContextRef.current = null; - } - }; + const startAudio = async () => { + if (!audioEnabled || !mediaEngineRef.current) return; - if (!audioEnabled || !connected) { - cleanup(); - return; - } + addLog("[App] startAudio: Acquiring Lock"); + const lock = acquireMediaLock(); + await lock.wait(); + if (!active) { lock.release(); return; } + addLog("[App] startAudio: Lock Acquired"); - addLog(`Starting audio... Device: ${selectedAudioDevice || 'Default'}`); - - async function startAudio() { try { - // Short delay to allow previous cleanup to settle if rapid toggling - await new Promise(r => setTimeout(r, 100)); - if (isCancelled) return; - - const constraints = { + addLog("Requesting Audio Access..."); + stream = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: selectedAudioDevice ? { exact: selectedAudioDevice } : undefined, echoCancellation: true, noiseSuppression: true, autoGainControl: true - }, - video: false - }; + } + }); + addLog(`Audio Access Granted: ${stream.id}`); - const stream = await navigator.mediaDevices.getUserMedia(constraints); + const track = stream.getAudioTracks()[0]; + // @ts-ignore + const processor = new MediaStreamTrackProcessor({ track }); + reader = processor.readable.getReader(); - if (isCancelled) { - stream.getTracks().forEach(t => t.stop()); - return; - } + // Signal ON + // @ts-ignore + window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 0 }); // 0 = Audio - addLog(`Mic Gained: ${stream.getAudioTracks()[0].label}`); + lock.release(); + addLog("[App] startAudio: Reading Loop Start"); - audioStreamRef.current = stream; + let frameCount = 0; - // Create context (if allowed by browser policy - usually requires interaction, which we have via button click) - const ctx = new AudioContext({ sampleRate: 48000, latencyHint: 'interactive' }); - audioContextRef.current = ctx; - - // Load Worklet - // Note: creating blob URL every time is fine, browsers handle it. - const blob = new Blob([audioWorkletCode], { type: 'application/javascript' }); - const workletUrl = URL.createObjectURL(blob); - - let useWorklet = false; - try { - await ctx.audioWorklet.addModule(workletUrl); - useWorklet = true; - } catch (e) { - console.warn("Worklet addModule failed", e); - } - URL.revokeObjectURL(workletUrl); - - if (isCancelled) { - ctx.close(); - return; - } - - const source = ctx.createMediaStreamSource(stream); - - if (useWorklet) { - try { - addLog("Creating AudioWorkletNode..."); - const workletNode = new AudioWorkletNode(ctx, 'pcm-processor'); - workletNode.port.onmessage = (e) => { - if (!audioEnabled || !connected || isCancelled) return; - const float32 = e.data; - const pcm = new Int16Array(float32.length); - for (let i = 0; i < float32.length; i++) { - pcm[i] = Math.max(-32768, Math.min(32767, Math.floor(float32[i] * 32768))); - } - // @ts-ignore - window.electron.ipcRenderer.send('send-audio-frame', { frame: pcm.buffer }); - }; - source.connect(workletNode); - workletNode.connect(ctx.destination); - addLog("Audio Worklet running"); - return; // Success! - } catch (e: any) { - console.error("Worklet Node creation failed", e); - addLog(`Worklet Node failed: ${e.message}`); - // Fall through to fallback + while (active) { + const result = await reader.read(); + if (result.done) break; + if (result.value) { + if (frameCount % 100 === 0) console.log(`[App] Capturing Audio Frame ${frameCount}`); + mediaEngineRef.current?.encodeAudioData(result.value); + frameCount++; } } + addLog("[App] startAudio: Reading Loop End"); - // Fallback to ScriptProcessor - addLog("Falling back to ScriptProcessor..."); - // @ts-ignore - 2048 samples for lower latency - const scriptNode = ctx.createScriptProcessor(2048, 1, 1); - // @ts-ignore - scriptNode.onaudioprocess = (e) => { - if (!audioEnabled || !connected || isCancelled) return; - const inputData = e.inputBuffer.getChannelData(0); - const pcm = new Int16Array(inputData.length); - for (let i = 0; i < inputData.length; i++) { - pcm[i] = Math.max(-32768, Math.min(32767, Math.floor(inputData[i] * 32768))); - } - // @ts-ignore - window.electron.ipcRenderer.send('send-audio-frame', { frame: pcm.buffer }); - }; - source.connect(scriptNode); - // @ts-ignore - scriptNode.connect(ctx.destination); - addLog("ScriptProcessor running"); + } catch (e) { + console.error("Audio capture error", e); + setAudioEnabled(false); + lock.release(); + } + }; - } catch (err: any) { - if (isCancelled) return; // Ignore errors if we cancelled - console.error('Audio capture error:', err); - if (err.name === 'AbortError' || err.name === 'NotAllowedError') { - addLog(`Permission/Abort Error: ${err.message}`); - setError(`Mic blocked/aborted: ${err.message}`); - } else { - addLog(`Mic Error: ${err.message}`); - setError(`Mic Error: ${err.message}`); - } - setAudioEnabled(false); // Reset state + if (audioEnabled) { + console.log("[App] Audio Enabled -> Starting"); + startAudio(); + } else { + console.log("[App] Audio Disabled"); + if (connected) { + // @ts-ignore + window.electron.ipcRenderer.send('update-stream', { active: false, mediaType: 0 }); // 0 = Audio } } - startAudio(); - return () => { - isCancelled = true; - cleanup(); + active = false; + console.log("[App] Audio Cleanup"); + if (stream) { + addLog("Stopping Audio Stream"); + stream.getTracks().forEach(t => t.stop()); + } + if (reader) reader.cancel(); }; - }, [audioEnabled, connected, selectedAudioDevice]); + }, [audioEnabled, selectedAudioDevice, connected]); + + // Connection Logic async function handleJoin(roomCode: string, name: string, serverUrl: string, initialVideo: boolean, initialAudio: boolean) { if (!roomCode || !name || !serverUrl) return; setDisplayName(name); setVideoEnabled(initialVideo); setAudioEnabled(initialAudio); setError(""); + + // Initialize AudioContext on User Gesture (Click) + if (!audioCtxRef.current) { + try { + const Ctx = window.AudioContext || (window as any).webkitAudioContext; + audioCtxRef.current = new Ctx(); + addLog(`AudioContext Initialized. State: ${audioCtxRef.current?.state}`); + + if (audioCtxRef.current?.state === 'suspended') { + await audioCtxRef.current.resume(); + addLog("AudioContext Resumed immediately."); + } + } catch (e) { + console.error("Failed to init AudioContext", e); + } + } else if (audioCtxRef.current.state === 'suspended') { + await audioCtxRef.current.resume(); + addLog("AudioContext Resumed (was existing)."); + } + try { // @ts-ignore const result = await window.electron.ipcRenderer.invoke("connect", { serverUrl, roomCode, displayName: name }); @@ -717,57 +733,70 @@ function App() { } } catch (e: any) { console.error(e); - const errMsg = typeof e === 'string' ? e : JSON.stringify(e); - addLog(`Error: ${errMsg}`); - setError(errMsg); + setError(typeof e === 'string' ? e : JSON.stringify(e)); } } async function handleLeave() { setVideoEnabled(false); + setAudioEnabled(false); + setScreenEnabled(false); + setPeersWithCam(new Set()); + setPeersWithCam(new Set()); + setPeersWithScreen(new Set()); + setPeersWithAudio(new Set()); // @ts-ignore await window.electron.ipcRenderer.invoke("disconnect"); setConnected(false); setPeers([]); setSelfId(null); - setPeerVideoUrls({}); } + // UI Toggles const toggleAudio = () => setAudioEnabled(!audioEnabled); - const toggleVideo = () => setVideoEnabled(!videoEnabled); - const toggleScreen = () => setScreenEnabled(!screenEnabled); + const toggleVideo = () => { + setVideoEnabled(!videoEnabled); + }; + const toggleScreen = () => { + setScreenEnabled(!screenEnabled); + }; const toggleChat = () => setChatOpen(!chatOpen); const sendChatMessage = (message: string) => { - if (!selfId) return; - // Don't add locally - server will broadcast it back to us - // But send displayName too to ensure server has it - // @ts-ignore - Send via WebSocket + // @ts-ignore window.electron.ipcRenderer.invoke('send-chat', { message, displayName: displayName || 'User' }); }; + // Hidden Canvas Ref for internal use if needed + const canvasRef = useRef(null); + return (
- {!connected ? ( ) : (
- {/* Main Content Area */}
- {/* Stage */} - {/* Control Bar */}
- {/* Chat Panel */} {chatOpen && ( )} - {/* Chat Notifications */} {!chatOpen && ( )} - {/* Hidden Canvas for capture */} + {/* Hidden Canvas if needed */} -
- )} - {/* Error Toast */} - {error && ( -
- {error} + {/* Error Toast */} + {error && ( +
+ {error} +
+ )}
)}
diff --git a/src/renderer/src/components/Stage.tsx b/src/renderer/src/components/Stage.tsx index 869b764..c6f52c6 100644 --- a/src/renderer/src/components/Stage.tsx +++ b/src/renderer/src/components/Stage.tsx @@ -10,7 +10,13 @@ interface StageProps { peerScreenUrls?: { [key: number]: string }; localScreenUrl?: string | null; localVideoRef?: React.RefObject; + localScreenRef?: React.RefObject; videoEnabled?: boolean; + screenEnabled?: boolean; + registerPeerCanvas: (userId: number, streamType: 'video' | 'screen', canvas: HTMLCanvasElement | null) => void; + peersWithCam: Set; + peersWithScreen: Set; + peersWithAudio: Set; } export function Stage({ @@ -21,8 +27,16 @@ export function Stage({ peerScreenUrls = {}, localScreenUrl = null, localVideoRef, - videoEnabled = false + localScreenRef, + videoEnabled = false, + screenEnabled = false, + registerPeerCanvas, + peersWithCam, + peersWithScreen, + peersWithAudio }: StageProps) { + + // Track container dimensions for smart layout const [containerSize, setContainerSize] = useState({ width: 800, height: 600 }); @@ -42,121 +56,108 @@ export function Stage({ return () => window.removeEventListener('resize', updateSize); }, []); - // Check if self is sharing screen - const isSelfSharing = !!localScreenUrl; + // Active Screen Shares (Remote) + const remoteScreens = peers.filter(p => peersWithScreen.has(p.user_id)); - // Filter peers who are sharing screen - const peerScreens = peers.filter(p => !!peerScreenUrls[p.user_id]); + // Check if self is sharing screen (via local video/stream state is not passed here directly as boolean, + // but App handles local display via `localVideoRef`... wait. + // App.tsx handles local WEBCAM via `localVideoRef`. + // App.tsx handles local SCREEN via... `localVideoRef` too? + // In App.tsx: `if (localVideoRef.current) localVideoRef.current.srcObject = stream;` for BOTH. + // So if I am sharing screen, `localVideoRef` shows my screen? + // Yes, `toggleVideo` switches off screen if active. `startCapture` sets srcObject. + // BUT `Stage` renders `localVideoRef` in the "Self Webcam" slot. + // Check if self is sharing screen + const isSelfSharing = screenEnabled; + + // Active Screen Shares + // const peerScreens = ... (removed legacy variable, using remoteScreens directly) // All peers for webcam grid const allParticipants = peers; - const showScreenLayer = isSelfSharing || peerScreens.length > 0; + + const showScreenLayer = isSelfSharing || remoteScreens.length > 0; const totalParticipants = (selfId ? 1 : 0) + allParticipants.length; - // Smart layout: determine if we should use vertical or horizontal arrangement + // Layout const aspectRatio = containerSize.width / containerSize.height; - const isVertical = aspectRatio < 1; // Taller than wide + const isVertical = aspectRatio < 1; // Calculate optimal grid layout based on aspect ratio and participant count const getGridConfig = (count: number, isVertical: boolean) => { - if (count <= 1) { - return { cols: 1, rows: 1 }; - } - + if (count <= 1) return { cols: 1, rows: 1 }; if (isVertical) { - // Vertical window: prefer fewer columns, more rows if (count === 2) return { cols: 1, rows: 2 }; - if (count <= 4) return { cols: 2, rows: 2 }; - if (count <= 6) return { cols: 2, rows: 3 }; - if (count <= 9) return { cols: 3, rows: 3 }; - return { cols: 3, rows: Math.ceil(count / 3) }; + return { cols: 2, rows: Math.ceil(count / 2) }; } else { - // Horizontal window: prefer more columns - if (count === 2) return { cols: 2, rows: 1 }; - if (count <= 4) return { cols: 2, rows: 2 }; - if (count <= 6) return { cols: 3, rows: 2 }; - if (count <= 9) return { cols: 3, rows: 3 }; - return { cols: 4, rows: Math.ceil(count / 4) }; + return { cols: Math.ceil(count / 2), rows: 2 }; } }; const gridConfig = getGridConfig(totalParticipants, isVertical); - // Screen share layout direction - const screenLayoutClass = isVertical - ? 'flex-col' // Stack screen above participants - : 'flex-row'; // Screen on left, participants on right - return ( -
- {/* Screen Share Layer */} +
+ + {/* Screen Share Area (Left or Top) */} {showScreenLayer && ( -
+
{/* Local Screen Share */} {isSelfSharing && (
)} {/* Remote Screen Shares */} - {peerScreens.map(peer => ( + {remoteScreens.map(peer => (
registerPeerCanvas(uid, 'screen', canvas)} />
))}
)} - {/* Webcam Grid */} -
-
+
- {/* Self Webcam */} + gridTemplateColumns: `repeat(${showScreenLayer ? 1 : Math.ceil(Math.sqrt(totalParticipants))}, 1fr)` + }}> + + {/* Self */} {selfId && ( -
+
)} - {/* Remote Webcam Peers */} - {allParticipants.map(peer => ( -
+ {/* Remote Peers Cam */} + {peers.map(peer => ( +
registerPeerCanvas(uid, 'video', canvas)} />
))} diff --git a/src/renderer/src/components/VideoTile.tsx b/src/renderer/src/components/VideoTile.tsx index 56ec71b..444f62b 100644 --- a/src/renderer/src/components/VideoTile.tsx +++ b/src/renderer/src/components/VideoTile.tsx @@ -1,4 +1,5 @@ import { Mic, MicOff } from "lucide-react"; +import { useEffect, useRef } from "react"; interface VideoTileProps { displayName: string; @@ -8,27 +9,58 @@ interface VideoTileProps { audioEnabled?: boolean; videoEnabled?: boolean; isScreenShare?: boolean; + userId?: number; + onCanvasRef?: (userId: number, canvas: HTMLCanvasElement | null) => void; } export function VideoTile({ displayName, isSelf, - videoSrc, + // videoSrc, // Unused videoRef, audioEnabled = true, videoEnabled = false, - isScreenShare = false + isScreenShare = false, + userId, + onCanvasRef }: VideoTileProps) { - // For self with video ref, use video element bound to the ref - // For remote peers, videoSrc contains blob URL of JPEG frames - use img - const showSelfVideo = isSelf && videoEnabled && videoRef; - const showRemoteMedia = !isSelf && videoSrc; - const showPlaceholder = !showSelfVideo && !showRemoteMedia; + const canvasRef = useRef(null); + + // Register canvas if applicable + const onCanvasRefRef = useRef(onCanvasRef); + + // Update ref when prop changes + useEffect(() => { + onCanvasRefRef.current = onCanvasRef; + }, [onCanvasRef]); + + // Use a callback ref to handle canvas mounting/unmounting reliably + const setCanvasRef = (node: HTMLCanvasElement | null) => { + canvasRef.current = node; + if (node) { + if (!isSelf && userId && onCanvasRefRef.current) { + onCanvasRefRef.current(userId, node); + } + } else { + // Cleanup on unmount + if (!isSelf && userId && onCanvasRefRef.current) { + onCanvasRefRef.current(userId, null); + } + } + }; + + // For self or local preview (using video element) + const showVideoElement = (isSelf || (isScreenShare && !userId)) && videoEnabled && videoRef; + + // For remote peers (WebCodecs via Canvas) + const showRemoteCanvas = !isSelf && userId && onCanvasRef && videoEnabled; + + const showPlaceholder = !showVideoElement && !showRemoteCanvas; return ( -
- {/* Self Video (webcam stream) */} - {showSelfVideo && ( +
+ {/* Video Element (Self Cam or Local Screen Preview) */} + {showVideoElement && (
)} {/* Audio indicator */}
-
+
{audioEnabled ? : }
{/* Name */} -
-
- {displayName} - {isSelf && !isScreenShare && (You)} +
+
+ {displayName}
+ {isSelf && !isScreenShare && ( + + You + + )}
); diff --git a/src/renderer/src/main.tsx b/src/renderer/src/main.tsx index 013b143..84b0c75 100644 --- a/src/renderer/src/main.tsx +++ b/src/renderer/src/main.tsx @@ -3,8 +3,47 @@ import ReactDOM from 'react-dom/client' import App from './App' import './index.css' -ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render( - - - -) +class ErrorBoundary extends React.Component { + constructor(props: any) { + super(props); + this.state = { hasError: false, error: null }; + } + + static getDerivedStateFromError(error: any) { + return { hasError: true, error }; + } + + componentDidCatch(error: any, errorInfo: any) { + console.error("Uncaught error:", error, errorInfo); + } + + render() { + if (this.state.hasError) { + return ( +
+

Something went wrong.

+ 
+                        {this.state.error && this.state.error.toString()}
+
+
+ ); + } + + return this.props.children; + } +} + +console.log("[JustTalk] Renderer process started"); +try { + const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); + console.log("[JustTalk] Root created, rendering App..."); + root.render( + + + + + + ); +} catch (e) { + console.error("[JustTalk] Failed to render root:", e); +} diff --git a/src/renderer/src/utils/MediaEngine.ts b/src/renderer/src/utils/MediaEngine.ts new file mode 100644 index 0000000..4163bc1 --- /dev/null +++ b/src/renderer/src/utils/MediaEngine.ts @@ -0,0 +1,281 @@ +// import { EventEmitter } from 'events'; // Node.js module dependency might be missing + +// Custom lightweight Event Emitter for browser compatibility if needed, +// but 'events' module is usually polyfilled by Vite/Webpack. +// If not, we can use a simple class. +class SimpleEventEmitter { + private listeners: { [key: string]: Function[] } = {}; + + on(event: string, listener: Function) { + if (!this.listeners[event]) this.listeners[event] = []; + this.listeners[event].push(listener); + return () => this.off(event, listener); + } + + off(event: string, listener: Function) { + if (!this.listeners[event]) return; + this.listeners[event] = this.listeners[event].filter(l => l !== listener); + } + + emit(event: string, ...args: any[]) { + if (!this.listeners[event]) return; + this.listeners[event].forEach(l => { + try { l(...args); } + catch (e) { console.error(`Error in event listener for ${event}:`, e); } + }); + } +} + +export interface EncodedFrame { + type: 'video' | 'audio'; + data: Uint8Array; + isKeyFrame: boolean; + timestamp: number; + duration?: number; + streamType?: 'video' | 'screen'; // Added for dual stream support +} + +export class MediaEngine extends SimpleEventEmitter { + private videoEncoder: VideoEncoder | null = null; + private screenEncoder: VideoEncoder | null = null; // Separate encoder for screen + private audioEncoder: AudioEncoder | null = null; + + // Decoders: Map -> Now needs to distinguish stream types + // We can use keys like "userId-video" and "userId-screen" + private videoDecoders: Map = new Map(); + // Decoders: Map -> Now needs to distinguish stream types + // We can use keys like "userId-video" and "userId-screen" + // private videoDecoders: Map = new Map(); // Already declared above + private audioDecoders: Map = new Map(); + private videoConfig: VideoEncoderConfig = { + codec: 'avc1.42001f', // H.264 Baseline Profile Level 3.1 (720p safe) + width: 1280, + height: 720, + bitrate: 2_000_000, + framerate: 30, + latencyMode: 'realtime', + avc: { format: 'annexb' } + }; + + private screenConfig: VideoEncoderConfig = { + // High Profile Level 4.2 + codec: 'avc1.64002a', + width: 1920, + height: 1080, + bitrate: 2_000_000, // Reduced to 2 Mbps for better stability/FPS + framerate: 30, + latencyMode: 'realtime', // Changed from 'quality' to 'realtime' for lower latency + avc: { format: 'annexb' } + }; + + // Audio Config + private audioConfig: AudioEncoderConfig = { + codec: 'opus', + sampleRate: 48000, + numberOfChannels: 1, + bitrate: 32000 + }; + + constructor() { + super(); + this.initializeVideoEncoder(); + this.initializeScreenEncoder(); + this.initializeAudioEncoder(); + } + + private initializeVideoEncoder() { + try { + this.videoEncoder = new VideoEncoder({ + output: (chunk, _metadata) => { + const buffer = new Uint8Array(chunk.byteLength); + chunk.copyTo(buffer); + + // With 'annexb', SPS/PPS should be in the keyframe chunk data. + + this.emit('encoded-video', { + type: 'video', + streamType: 'video', + data: buffer, + isKeyFrame: chunk.type === 'key', + timestamp: chunk.timestamp, + duration: chunk.duration + } as EncodedFrame); + }, + error: (e) => console.error('VideoEncoder error:', e), + }); + + this.videoEncoder.configure(this.videoConfig); + console.log('[MediaEngine] VideoEncoder configured:', this.videoConfig); + } catch (e) { + console.error('[MediaEngine] Failed to init VideoEncoder:', e); + } + } + + private initializeScreenEncoder() { + try { + this.screenEncoder = new VideoEncoder({ + output: (chunk, _metadata) => { + const buffer = new Uint8Array(chunk.byteLength); + chunk.copyTo(buffer); + this.emit('encoded-video', { + type: 'video', + streamType: 'screen', + data: buffer, + isKeyFrame: chunk.type === 'key', + timestamp: chunk.timestamp, + duration: chunk.duration + } as EncodedFrame); + }, + error: (e) => console.error('ScreenEncoder error:', e), + }); + this.screenEncoder.configure(this.screenConfig); + console.log('[MediaEngine] ScreenEncoder configured:', this.screenConfig); + } catch (e) { + console.error('[MediaEngine] Failed to init ScreenEncoder:', e); + } + } + + private initializeAudioEncoder() { + try { + this.audioEncoder = new AudioEncoder({ + output: (chunk, _metadata) => { + const buffer = new Uint8Array(chunk.byteLength); + chunk.copyTo(buffer); + this.emit('encoded-audio', { + type: 'audio', + data: buffer, + isKeyFrame: chunk.type === 'key', + timestamp: chunk.timestamp, + duration: chunk.duration + } as EncodedFrame); + }, + error: (e) => console.error('[MediaEngine] AudioEncoder error:', e), + }); + this.audioEncoder.configure(this.audioConfig); + console.log('[MediaEngine] AudioEncoder configured:', this.audioConfig); + } catch (e) { + console.error('[MediaEngine] Failed to init AudioEncoder:', e); + } + } + + + // --- Video Encoding --- + + encodeVideoFrame(frame: VideoFrame, streamType: 'video' | 'screen' = 'video') { + const encoder = streamType === 'screen' ? this.screenEncoder : this.videoEncoder; + + if (encoder && encoder.state === 'configured') { + // Force keyframe every 2 seconds (60 frames) + const keyFrame = frame.timestamp % 2000000 < 33000; + encoder.encode(frame, { keyFrame }); + frame.close(); + } else { + frame.close(); + console.warn(`[MediaEngine] ${streamType === 'screen' ? 'ScreenEncoder' : 'VideoEncoder'} not ready`); + } + } + + // --- Video Decoding --- + + decodeVideoChunk(chunkData: Uint8Array, userId: number, isKeyFrame: boolean, timestamp: number, streamType: 'video' | 'screen' = 'video') { + const decoderKey = `${userId}-${streamType}`; + let decoder = this.videoDecoders.get(decoderKey); + + if (!decoder) { + decoder = new VideoDecoder({ + output: (frame) => { + this.emit('decoded-video', { userId, frame, streamType }); + }, + error: (e) => console.error(`VideoDecoder error (${decoderKey}):`, e), + }); + + // Configure based on stream type + // Note: Decoders are usually more flexible, but giving a hint helps. + // Screen share uses High Profile, Video uses Baseline. + const config: VideoDecoderConfig = streamType === 'screen' + ? { codec: 'avc1.64002a', optimizeForLatency: false } + : { codec: 'avc1.42001f', optimizeForLatency: true }; + + decoder.configure(config); + this.videoDecoders.set(decoderKey, decoder); + console.log(`[MediaEngine] Created decoder for ${decoderKey} with codec ${config.codec}`); + } + + if (decoder.state === 'configured') { + const chunk = new EncodedVideoChunk({ + type: isKeyFrame ? 'key' : 'delta', + timestamp: timestamp, + data: chunkData, + }); + try { + decoder.decode(chunk); + } catch (e) { + console.error(`[MediaEngine] Decode error ${decoderKey}:`, e); + } + } + } + + // --- Audio --- + + // --- Audio (PCM Fallback) --- + + encodeAudioData(data: AudioData) { + if (this.audioEncoder && this.audioEncoder.state === 'configured') { + this.audioEncoder.encode(data); + data.close(); + } else { + data.close(); + // console.warn('[MediaEngine] AudioEncoder not ready'); + } + } + + decodeAudioChunk(chunkData: Uint8Array, userId: number, timestamp: number) { + const decoderKey = `${userId}-audio`; + let decoder = this.audioDecoders.get(decoderKey); + + if (!decoder) { + decoder = new AudioDecoder({ + output: (data) => { + this.emit('decoded-audio', { userId, data }); + }, + error: (e) => console.error(`[MediaEngine] AudioDecoder error (${userId}):`, e) + }); + decoder.configure({ + codec: 'opus', + sampleRate: 48000, + numberOfChannels: 1 + }); + this.audioDecoders.set(decoderKey, decoder); + console.log(`[MediaEngine] Created AudioDecoder for ${userId}`); + } + + if (decoder.state === 'configured') { + const chunk = new EncodedAudioChunk({ + type: 'key', // Opus is usually self-contained + timestamp: timestamp, + data: chunkData, + }); + try { + decoder.decode(chunk); + } catch (e) { + console.error(`[MediaEngine] Audio Decode error ${decoderKey}:`, e); + } + } + } + + cleanup() { + if (this.videoEncoder && this.videoEncoder.state !== 'closed') this.videoEncoder.close(); + if (this.screenEncoder && this.screenEncoder.state !== 'closed') this.screenEncoder.close(); + if (this.audioEncoder && this.audioEncoder.state !== 'closed') this.audioEncoder.close(); + + this.videoDecoders.forEach(d => { + if (d.state !== 'closed') d.close(); + }); + this.audioDecoders.forEach(d => { + if (d.state !== 'closed') d.close(); + }); + + this.videoDecoders.clear(); + this.audioDecoders.clear(); + } +} diff --git a/status.txt b/status.txt new file mode 100644 index 0000000..cfe5d75 --- /dev/null +++ b/status.txt @@ -0,0 +1,24 @@ +On branch master +Your branch is up to date with 'origin/master'. + +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: package.json + modified: src/main/index.ts + modified: src/main/network.ts + modified: src/renderer/index.html + modified: src/renderer/src/App.tsx + modified: src/renderer/src/components/Stage.tsx + modified: src/renderer/src/components/VideoTile.tsx + modified: src/renderer/src/main.tsx + +Untracked files: + (use "git add ..." to include in what will be committed) + current_head_media.ts + previous_head_media.ts + src/renderer/src/utils/ + status.txt + +no changes added to commit (use "git add" and/or "git commit -a") +--- SERVER STATUS ---