diff --git a/client_commit.txt b/client_commit.txt deleted file mode 100644 index b13ea13..0000000 --- a/client_commit.txt +++ /dev/null @@ -1,5 +0,0 @@ -commit 4bd20fc9887876086319874d5be51a14bfcfc978 -Author: Sarthak -Date: Mon Feb 9 17:06:56 2026 +0530 - - chore: revert audio logic to webcodecs diff --git a/current_head_media.ts b/current_head_media.ts deleted file mode 100644 index e69de29..0000000 diff --git a/package.json b/package.json index 0b858dd..c096f42 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,5 @@ { - "name": "just-talk", - "productName": "JustTalk", + "name": "client-electron", "version": "1.0.0", "main": "./out/main/index.js", "scripts": { diff --git a/previous_head_media.ts b/previous_head_media.ts deleted file mode 100644 index e69de29..0000000 diff --git a/src/main/index.ts b/src/main/index.ts index 71dcf9d..9d643e3 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1,29 +1,25 @@ -import { app, shell, BrowserWindow, ipcMain, desktopCapturer } from 'electron' +import { app, shell, BrowserWindow, ipcMain, session, desktopCapturer } from 'electron' import { join } from 'path' import { electronApp, optimizer, is } from '@electron-toolkit/utils' -// import icon from '../../resources/icon.png?asset' -import { NetworkManager } from './network' +import { NetworkManager } from './network' // Import NetworkManager let mainWindow: BrowserWindow | null = null; -let network: NetworkManager | null = null; +let networkManager: NetworkManager | null = null; function createWindow(): void { // Create the browser window. mainWindow = new BrowserWindow({ - width: 1280, - height: 720, + width: 900, + height: 670, show: false, autoHideMenuBar: true, - // ...(process.platform === 'linux' ? { icon } : {}), webPreferences: { preload: join(__dirname, '../preload/index.js'), - sandbox: false, - contextIsolation: true, - nodeIntegration: false // Best practice + sandbox: false } }) - network = new NetworkManager(mainWindow); + networkManager = new NetworkManager(mainWindow); mainWindow.on('ready-to-show', () => { mainWindow?.show() @@ -47,47 +43,83 @@ app.whenReady().then(() => { // Set app user model id for windows electronApp.setAppUserModelId('com.electron') + // Grant permissions for camera/mic/screen + session.defaultSession.setPermissionRequestHandler((webContents, permission, callback) => { + console.log(`[Main] Requesting permission: ${permission}`); + // Grant all permissions for this valid local app + callback(true); + }); + + // Default open or close DevTools by F12 in development + // and ignore CommandOrControl + R in production. + // see https://github.com/alex8088/electron-toolkit/tree/master/packages/utils app.on('browser-window-created', (_, window) => { optimizer.watchWindowShortcuts(window) }) // IPC Handlers ipcMain.handle('connect', async (_, { serverUrl, roomCode, displayName }) => { - return network?.connect(serverUrl, roomCode, displayName); + if (networkManager) { + return await networkManager.connect(serverUrl, roomCode, displayName); + } }); ipcMain.handle('disconnect', async () => { - network?.disconnect(); + if (networkManager) { + networkManager.disconnect(); + } }); - ipcMain.handle('send-chat', (_, { message, displayName }) => { - network?.sendChat(message, displayName); + ipcMain.on('send-video-frame', (_, { frame }) => { + if (networkManager) { + networkManager.sendVideoFrame(frame); + } }); + ipcMain.on('send-audio-frame', (_, { frame }) => { + if (networkManager) { + networkManager.sendAudioFrame(new Uint8Array(frame)); + } + }); + + ipcMain.on('send-screen-frame', (_, { frame }) => { + if (networkManager) { + networkManager.sendScreenFrame(frame); + } + }); + + // Screen sharing: get available sources ipcMain.handle('get-screen-sources', async () => { - const sources = await desktopCapturer.getSources({ types: ['window', 'screen'], thumbnailSize: { width: 150, height: 150 } }); - return sources.map(source => ({ - id: source.id, - name: source.name, - thumbnail: source.thumbnail.toDataURL() + const sources = await desktopCapturer.getSources({ + types: ['screen', 'window'], + thumbnailSize: { width: 150, height: 150 } + }); + return sources.map(s => ({ + id: s.id, + name: s.name, + thumbnail: s.thumbnail.toDataURL() })); }); - ipcMain.on('send-video-chunk', (_, payload) => { - network?.sendEncodedVideoChunk(payload.chunk, payload.isKeyFrame, payload.timestamp, payload.streamType); - }); - - ipcMain.on('send-audio-chunk', (_, payload) => { - network?.sendEncodedAudioChunk(payload.chunk, payload.timestamp); + // Chat + ipcMain.handle('send-chat', (_, { message, displayName }) => { + if (networkManager) { + networkManager.sendChat(message, displayName); + } }); + // Stream Updates ipcMain.on('update-stream', (_, { active, mediaType }) => { - network?.updateStream(active, mediaType); + if (networkManager) { + networkManager.updateStream(active, mediaType); + } }); createWindow() app.on('activate', function () { + // On macOS it's common to re-create a window in the app when the + // dock icon is clicked and there are no other windows open. if (BrowserWindow.getAllWindows().length === 0) createWindow() }) }) diff --git a/src/main/network.ts b/src/main/network.ts index 6f62387..1a23e7a 100644 --- a/src/main/network.ts +++ b/src/main/network.ts @@ -6,8 +6,8 @@ import { BrowserWindow } from 'electron'; // Constants const SERVER_UDP_PORT = 4000; -// Packet Header Structure (24 bytes) -const HEADER_SIZE = 24; +// Packet Header Structure (22 bytes) +const HEADER_SIZE = 22; export enum MediaType { Audio = 0, @@ -15,11 +15,6 @@ export enum MediaType { Screen = 2, } -// Token Bucket Pacer Constants -const PACER_RATE_BYTES_PER_MS = 1500; // ~12 Mbps limit (Targeting 8-10 Mbps for 1080p60) -const PACER_BUCKET_SIZE_BYTES = 15000; // Allow 10 packets burst (Instant Keyframe start) -const MAX_PAYLOAD = 1200; // Reduced from 1400 to be safe with MTU - export class NetworkManager extends EventEmitter { private ws: WebSocket | null = null; private udp: dgram.Socket | null = null; @@ -31,46 +26,9 @@ export class NetworkManager extends EventEmitter { private mainWindow: BrowserWindow; private serverUdpHost: string = '127.0.0.1'; - // Pacing - private udpQueue: Buffer[] = []; - private pacerTokens: number = PACER_BUCKET_SIZE_BYTES; - private lastPacerUpdate: number = Date.now(); - private pacerInterval: NodeJS.Timeout | null = null; - constructor(mainWindow: BrowserWindow) { super(); this.mainWindow = mainWindow; - this.startPacer(); - } - - private startPacer() { - this.pacerInterval = setInterval(() => { - if (!this.udp) return; - - const now = Date.now(); - const elapsed = now - this.lastPacerUpdate; - this.lastPacerUpdate = now; - - // Refill tokens - this.pacerTokens += elapsed * PACER_RATE_BYTES_PER_MS; - if (this.pacerTokens > PACER_BUCKET_SIZE_BYTES) { - this.pacerTokens = PACER_BUCKET_SIZE_BYTES; - } - - // Drain queue - while (this.udpQueue.length > 0) { - const packet = this.udpQueue[0]; - if (this.pacerTokens >= packet.length) { - this.pacerTokens -= packet.length; - this.udpQueue.shift(); - this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { - if (err) console.error('UDP Send Error', err); - }); - } else { - break; // Not enough tokens, wait for next tick - } - } - }, 2); // Check every 2ms } async connect(serverUrl: string, roomCode: string, displayName: string): Promise { @@ -198,7 +156,7 @@ export class NetworkManager extends EventEmitter { }); this.udp.on('message', (msg, rinfo) => { - // console.log(`[UDP] Msg from ${rinfo.address}:${rinfo.port} - ${msg.length} bytes`); + console.log(`[UDP] Msg from ${rinfo.address}:${rinfo.port} - ${msg.length} bytes`); this.handleUdpMessage(msg); }); @@ -208,83 +166,34 @@ export class NetworkManager extends EventEmitter { handleUdpMessage(msg: Buffer) { if (msg.length < HEADER_SIZE) return; - const version = msg.readUInt8(0); const mediaType = msg.readUInt8(1); const userId = msg.readUInt32LE(2); - const seq = msg.readUInt32LE(6); - const timestamp = Number(msg.readBigUInt64LE(10)); - const fragIdx = msg.readUInt16LE(18); - const fragCnt = msg.readUInt16LE(20); - const flags = msg.readUInt16LE(22); - const isKeyFrame = (flags & 1) !== 0; const payload = msg.subarray(HEADER_SIZE); + const sequence = msg.readUInt32LE(6); + const timestamp = Number(msg.readBigUInt64LE(10)); + const fragIdx = msg.readUInt8(18); + const fragCnt = msg.readUInt8(19); if (mediaType === MediaType.Audio) { - // Audio can be fragmented now (PCM) - this.safeSend('video-chunk', { // Use 'video-chunk' handler in renderer for reassembly? - // Wait, App.tsx has separate 'audio-chunk' which doesn't reassemble. - // We need to reassemble here or change App.tsx. - // Reassembling in main process is easier or reusing video logic. - - // Let's use 'audio-chunk' but we need to pass frag info? - // No, App.tsx 'audio-chunk' handler just decodes immediately. - // It expects a full frame. - - // We MUST reassemble here or update App.tsx. - // Updating App.tsx to use the reassembler for Audio is cleaner. - // But 'video-chunk' in App.tsx calls 'handleIncomingVideoFragment' which uses 'MediaEngine.decodeVideoChunk'. - - // Option: Treat Audio as "Video" for transport, but with streamType='audio'? - // MediaType.Audio is distinct. - - // Let's implement reassembly here in NetworkManager? - // Or update App.tsx to use 'handleIncomingVideoFragment' for audio too? - // 'handleIncomingVideoFragment' does `decodeVideoChunk`. - - // Let's change App.tsx to have `handleIncomingAudioFragment`? - // Or just reassemble here. UDP reassembly in Node.js is fine. - - // ACtually, App.tsx's `handleIncomingVideoFragment` is nice. - // Let's emit 'audio-fragment' and add a handler in App.tsx. + this.safeSend('audio-frame', { user_id: userId, data: payload }); + } else if (mediaType === MediaType.Video) { + this.safeSend('video-frame', { user_id: userId, data: payload, - seq: this.audioSeq, // Wait, seq is in packet + seq: sequence, ts: timestamp, fidx: fragIdx, - fcnt: fragCnt, - isKeyFrame, - streamType: 'audio' - // We can't use 'video-chunk' channel because it calls decodeVideoChunk. + fcnt: fragCnt }); - - // Actually, let's just send it to 'audio-fragment' channel - this.safeSend('audio-fragment', { + } else if (mediaType === MediaType.Screen) { + this.safeSend('screen-frame', { user_id: userId, data: payload, - seq: seq, // We need valid seq from packet + seq: sequence, ts: timestamp, fidx: fragIdx, - fcnt: fragCnt, - isKeyFrame - }); - - } else if (mediaType === MediaType.Video || mediaType === MediaType.Screen) { - // Differentiate based on MediaType - const streamType = mediaType === MediaType.Screen ? 'screen' : 'video'; - if (mediaType === MediaType.Screen && fragIdx === 0) { - console.log(`[Network] RX Screen Chunk User=${userId} Seq=${seq}`); - } - - this.safeSend('video-chunk', { - user_id: userId, - data: payload, - seq, - ts: timestamp, - fidx: fragIdx, - fcnt: fragCnt, - isKeyFrame, - streamType // Pass this to renderer + fcnt: fragCnt }); } } @@ -299,88 +208,88 @@ export class NetworkManager extends EventEmitter { } } - // --- New Encode Methods --- - - sendEncodedVideoChunk(chunk: any, isKeyFrame: boolean, timestamp: number, streamType: 'video' | 'screen' = 'video') { - if (!this.udp) return; + sendVideoFrame(frame: Uint8Array) { + if (!this.udp || !this.userId) return; + const buffer = Buffer.from(frame); const MAX_PAYLOAD = 1400; - const totalSize = chunk.length; + const fragCount = Math.ceil(buffer.length / MAX_PAYLOAD); + const seq = this.videoSeq++; + const ts = BigInt(Date.now()); - // Use generic videoSeq for both? Or separate? - // Best to separate to avoid gap detection issues if one stream is idle. - // But for now, let's share for simplicity or use screenSeq if screen. - // Actually, let's use separate seq if possible, but I only have videoSeq. - // Let's use videoSeq for both for now, assuming the receiver tracks them separately or doesn't care about gaps across types. - // Better: Use a map or separate counters. - const seq = streamType === 'screen' ? this.screenSeq++ : this.videoSeq++; - - const fragmentCount = Math.ceil(totalSize / MAX_PAYLOAD); - - for (let i = 0; i < fragmentCount; i++) { + for (let i = 0; i < fragCount; i++) { const start = i * MAX_PAYLOAD; - const end = Math.min(start + MAX_PAYLOAD, totalSize); - const slice = chunk.slice(start, end); + const end = Math.min(start + MAX_PAYLOAD, buffer.length); + const chunk = buffer.subarray(start, end); - // Header (22 bytes) const header = Buffer.alloc(HEADER_SIZE); header.writeUInt8(1, 0); // Version - const mType = streamType === 'screen' ? MediaType.Screen : MediaType.Video; - header.writeUInt8(mType, 1); + header.writeUInt8(MediaType.Video, 1); header.writeUInt32LE(this.userId, 2); header.writeUInt32LE(seq, 6); - header.writeBigUInt64LE(BigInt(timestamp), 10); - header.writeUInt16LE(i, 18); // Frag Idx (u16) - header.writeUInt16LE(fragmentCount, 20); // Frag Cnt (u16) + header.writeBigUInt64LE(ts, 10); + header.writeUInt8(i, 18); // Frag idx + header.writeUInt8(fragCount, 19); // Frag cnt + header.writeUInt16LE(0, 20); // Flags - let flags = 0; - if (isKeyFrame) flags |= 1; - header.writeUInt16LE(flags, 22); + const packet = Buffer.concat([header, chunk]); - const packet = Buffer.concat([header, slice]); - - // Enqueue for pacing - this.udpQueue.push(packet); + this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { + if (err) console.error('UDP Video Send Error', err); + }); } } - sendEncodedAudioChunk(chunk: Uint8Array, timestamp: number) { - if (!this.udp) { - console.warn('[Network] UDP Socket not ready for Audio'); - return; - } + sendAudioFrame(frame: Uint8Array) { + if (!this.udp) return; - const totalSize = chunk.length; - const MAX_PAYLOAD = 1400; // Safe MTU + const header = Buffer.alloc(HEADER_SIZE); + header.writeUInt8(1, 0); // Version + header.writeUInt8(MediaType.Audio, 1); + header.writeUInt32LE(this.userId, 2); + header.writeUInt32LE(this.audioSeq++, 6); + header.writeBigUInt64LE(BigInt(Date.now()), 10); + header.writeUInt8(0, 18); // Frag idx + header.writeUInt8(1, 19); // Frag cnt + header.writeUInt16LE(0, 20); // Flags - // PCM packets (approx 2KB) need fragmentation. - // We use the same logic as video but with Audio MediaType. + const packet = Buffer.concat([header, Buffer.from(frame)]); - const fragmentCount = Math.ceil(totalSize / MAX_PAYLOAD); + this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { + if (err) console.error('UDP Audio Send Error', err); + }); + } - // Log randomly to avoid spam but confirm activity - if (Math.random() < 0.05) console.log(`[Network] Sending Audio Chunk size=${totalSize} frags=${fragmentCount}`); + sendScreenFrame(frame: number[]) { + if (!this.udp || !this.userId) return; - for (let i = 0; i < fragmentCount; i++) { + const buffer = Buffer.from(frame); + const MAX_PAYLOAD = 1400; + const fragCount = Math.ceil(buffer.length / MAX_PAYLOAD); + const seq = this.screenSeq++; + const ts = BigInt(Date.now()); + + for (let i = 0; i < fragCount; i++) { const start = i * MAX_PAYLOAD; - const end = Math.min(start + MAX_PAYLOAD, totalSize); - const slice = chunk.slice(start, end); + const end = Math.min(start + MAX_PAYLOAD, buffer.length); + const chunk = buffer.subarray(start, end); const header = Buffer.alloc(HEADER_SIZE); header.writeUInt8(1, 0); // Version - header.writeUInt8(MediaType.Audio, 1); + header.writeUInt8(MediaType.Screen, 1); header.writeUInt32LE(this.userId, 2); - header.writeUInt32LE(this.audioSeq, 6); // Same seq for all fragments - header.writeBigUInt64LE(BigInt(Math.floor(timestamp)), 10); - header.writeUInt16LE(i, 18); // Frag idx - header.writeUInt16LE(fragmentCount, 20); // Frag cnt - header.writeUInt16LE(1, 22); // Flags (1=Keyframe, audio is always key) + header.writeUInt32LE(seq, 6); + header.writeBigUInt64LE(ts, 10); + header.writeUInt8(i, 18); // Frag idx + header.writeUInt8(fragCount, 19); // Frag cnt + header.writeUInt16LE(0, 20); // Flags - const packet = Buffer.concat([header, Buffer.from(slice)]); - this.udpQueue.push(packet); + const packet = Buffer.concat([header, chunk]); + + this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { + if (err) console.error('UDP Screen Send Error', err); + }); } - - this.audioSeq++; } startHeartbeat() { @@ -418,23 +327,19 @@ export class NetworkManager extends EventEmitter { header.writeUInt32LE(this.userId, 2); header.writeUInt32LE(0, 6); // Sequence header.writeBigUInt64LE(BigInt(Date.now()), 10); - header.writeUInt16LE(0, 18); // Frag idx - header.writeUInt16LE(1, 20); // Frag cnt - header.writeUInt16LE(0, 22); // Flags + header.writeUInt8(0, 18); // Frag idx + header.writeUInt8(1, 19); // Frag cnt + header.writeUInt16LE(0, 20); // Flags const packet = Buffer.concat([header, payload]); - // console.log(`[UDP] Sending Handshake: userId=${this.userId}, room=${this.roomCode}, ${packet.length} bytes to ${this.serverUdpHost}:${SERVER_UDP_PORT}`); + console.log(`[UDP] Sending Handshake: userId=${this.userId}, room=${this.roomCode}, ${packet.length} bytes to ${this.serverUdpHost}:${SERVER_UDP_PORT}`); this.udp.send(packet, SERVER_UDP_PORT, this.serverUdpHost, (err) => { if (err) console.error('UDP Handshake Send Error', err); }); } disconnect() { - if (this.pacerInterval) { - clearInterval(this.pacerInterval); - this.pacerInterval = null; - } if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; diff --git a/src/renderer/index.html b/src/renderer/index.html index a733204..1a87802 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -3,7 +3,7 @@ - JustTalk + Electron App diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 64f94b1..a82f5a0 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef, useCallback } from "react"; +import { useState, useEffect, useRef } from "react"; import "./index.css"; import { Lobby } from "./components/Lobby"; @@ -7,7 +7,22 @@ import { ControlBar } from "./components/ControlBar"; import { ChatPanel } from "./components/ChatPanel"; import { NotificationToast } from "./components/NotificationToast"; import { PeerInfo, ChatMessage } from "./types"; -import { MediaEngine, EncodedFrame } from "./utils/MediaEngine"; + +const audioWorkletCode = ` +class PCMProcessor extends AudioWorkletProcessor { + process(inputs, outputs, parameters) { + const input = inputs[0]; + if (input && input.length > 0) { + const inputChannel = input[0]; + // Post full buffer to main thread (renderer) + // optimization: we could accumulate here if needed + this.port.postMessage(inputChannel); + } + return true; + } +} +registerProcessor('pcm-processor', PCMProcessor); +`; interface JoinedPayload { self_id: number; @@ -22,7 +37,7 @@ function App() { const [error, setError] = useState(""); const addLog = (msg: string) => { - console.log(`[JustTalk] ${msg}`); + console.log(`[Meet] ${msg}`); }; // Media State @@ -33,11 +48,6 @@ function App() { const [selectedVideoDevice, setSelectedVideoDevice] = useState(""); const [displayName, setDisplayName] = useState(""); - // Active Video Peers - const [peersWithCam, setPeersWithCam] = useState>(new Set()); - const [peersWithScreen, setPeersWithScreen] = useState>(new Set()); - const [peersWithAudio, setPeersWithAudio] = useState>(new Set()); - // Chat State const [chatOpen, setChatOpen] = useState(false); const [chatMessages, setChatMessages] = useState([]); @@ -45,683 +55,657 @@ function App() { // Video Handling const localVideoRef = useRef(null); - const localScreenRef = useRef(null); // New ref for local screen share - const mediaEngineRef = useRef(null); - - // Canvas Refs: Key = "${userId}-${streamType}" (e.g. "123-video", "123-screen") - const peerCanvasRefs = useRef>(new Map()); - - const registerPeerCanvas = useCallback((userId: number, streamType: 'video' | 'screen', canvas: HTMLCanvasElement | null) => { - const key = `${userId}-${streamType}`; - if (canvas) { - peerCanvasRefs.current.set(key, canvas); - } else { - peerCanvasRefs.current.delete(key); - } - }, []); - - // Audio Handling - Ref to context - const audioCtxRef = useRef(null); - - // Audio Reassembly Buffer (Moved out of useEffect) - // Key: `${userId}-audio-${seq}` - const audioFragmentBuffer = useRef>(new Map()); - - const handleIncomingAudioFragment = useCallback((payload: any) => { - const { user_id, data, seq, ts, fidx, fcnt } = payload; - - // If single fragment, decode immediately - if (fcnt === 1) { - if (mediaEngineRef.current) { - const buffer = data instanceof Uint8Array ? data : new Uint8Array(data); - mediaEngineRef.current.decodeAudioChunk(buffer, user_id, ts); - } - return; - } - - const key = `${user_id}-${seq}`; - const map = audioFragmentBuffer.current; - - if (!map.has(key)) { - map.set(key, { chunks: new Array(fcnt), count: 0, total: fcnt, ts }); - } - - const entry = map.get(key)!; - if (!entry.chunks[fidx]) { - entry.chunks[fidx] = data instanceof Uint8Array ? data : new Uint8Array(data); - entry.count++; - } - - if (entry.count === entry.total) { - // Reassemble - const totalLen = entry.chunks.reduce((acc, c) => acc + c.length, 0); - const fullFrame = new Uint8Array(totalLen); - let offset = 0; - for (const c of entry.chunks) { - fullFrame.set(c, offset); - offset += c.length; - } - - if (mediaEngineRef.current) { - mediaEngineRef.current.decodeAudioChunk(fullFrame, user_id, ts); - } - map.delete(key); - } - - // Cleanup old - for (const k of map.keys()) { - const kSeq = parseInt(k.split('-')[1]); - if (map.size > 20) map.delete(k); - } - }, []); + const canvasRef = useRef(null); + const [peerVideoUrls, setPeerVideoUrls] = useState<{ [key: number]: string }>({}); + const [peerScreenUrls, setPeerScreenUrls] = useState<{ [key: number]: string }>({}); + const [localScreenUrl, setLocalScreenUrl] = useState(null); // Event Listeners useEffect(() => { - const engine = new MediaEngine(); - mediaEngineRef.current = engine; - - // --- Outbound Encoding Handlers --- - engine.on('encoded-video', (frame: EncodedFrame) => { - if (!connected) return; - // @ts-ignore - window.electron.ipcRenderer.send('send-video-chunk', { - chunk: frame.data, - isKeyFrame: frame.isKeyFrame, - timestamp: frame.timestamp, - streamType: frame.streamType || 'video' - }); + // @ts-ignore + const removeJoined = window.electron.ipcRenderer.on('connect-success', (_, data: JoinedPayload) => { + // We might get this as return from invoke, but also good to have event. }); - engine.on('encoded-audio', (frame: EncodedFrame) => { - if (!connected) return; - // @ts-ignore - window.electron.ipcRenderer.send('send-audio-chunk', { - chunk: frame.data, - timestamp: frame.timestamp - }); - }); - - // --- Inbound Decoding Handlers --- - engine.on('decoded-video', ({ userId, frame, streamType }: { userId: number, frame: VideoFrame, streamType: 'video' | 'screen' }) => { - const type = streamType || 'video'; - - // Mark user as having video (only triggers re-render if new) - if (type === 'video') { - setPeersWithCam(prev => { - if (prev.has(userId)) return prev; - console.log(`[App] User ${userId} started Cam Video`); - return new Set(prev).add(userId); - }); - } else { - setPeersWithScreen(prev => { - if (prev.has(userId)) return prev; - console.log(`[App] User ${userId} started Screen Share`); - return new Set(prev).add(userId); - }); - } - - const key = `${userId}-${type}`; - const canvas = peerCanvasRefs.current.get(key); - if (canvas) { - if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) { - canvas.width = frame.displayWidth; - canvas.height = frame.displayHeight; - } - const ctx = canvas.getContext('2d'); - if (ctx) ctx.drawImage(frame, 0, 0); - } - frame.close(); - }); - - // Audio Playback - let audioNextTime = 0; - let audioPacketCount = 0; - - engine.on('decoded-audio', ({ userId, data }: { userId: number, data: AudioData }) => { - if (audioPacketCount % 50 === 0) console.log(`[App] Playing Audio packet #${audioPacketCount} from User ${userId}`); - - const ctx = audioCtxRef.current; - if (!ctx) { - data.close(); - return; - } - - if (ctx.state === 'suspended') { - // Try to resume again - ctx.resume().catch(e => console.error("Audio resume failed in playback", e)); - } - - if (audioNextTime === 0) { - audioNextTime = ctx.currentTime + 0.05; // Start slightly in future to avoid glitches - } - - try { - const buffer = ctx.createBuffer(data.numberOfChannels, data.numberOfFrames, data.sampleRate); - for (let ch = 0; ch < data.numberOfChannels; ch++) { - const channelData = buffer.getChannelData(ch); - data.copyTo(channelData, { planeIndex: ch }); - } - - const source = ctx.createBufferSource(); - source.buffer = buffer; - source.connect(ctx.destination); - - const now = ctx.currentTime; - // If we fell behind, catch up - if (audioNextTime < now) { - // console.warn(`[App] Audio fell behind by ${now - audioNextTime}s`); - audioNextTime = now; - } - - source.start(audioNextTime); - audioNextTime += buffer.duration; - } catch (e) { - console.error("[App] Audio Buffer Error:", e); - } - - data.close(); - }); - - - // --- IPC Listeners for Incoming Chunks --- - + // Peer Joined // @ts-ignore const removePeerJoined = window.electron.ipcRenderer.on("peer-joined", (_, data) => { addLog(`PeerJoined: ${JSON.stringify(data)}`); - setPeers(prev => { - if (prev.find(p => p.user_id === data.user_id)) return prev; - return [...prev, { user_id: data.user_id, display_name: data.display_name }]; - }); + if (data && data.user_id) { + setPeers((prev) => { + if (prev.find(p => p.user_id === data.user_id)) { + console.log("Peer already exists:", data.user_id); + return prev; + } + console.log("Adding peer:", data); + return [...prev, { user_id: data.user_id, display_name: data.display_name }]; + }); + } }); + // Peer Left // @ts-ignore const removePeerLeft = window.electron.ipcRenderer.on("peer-left", (_, data) => { addLog(`PeerLeft: ${JSON.stringify(data)}`); - setPeers(prev => prev.filter(p => p.user_id !== data.user_id)); - setPeersWithCam(prev => { - const next = new Set(prev); - next.delete(data.user_id); - return next; - }); - setPeersWithScreen(prev => { - const next = new Set(prev); - next.delete(data.user_id); - return next; - }); - setPeersWithAudio(prev => { - const next = new Set(prev); - next.delete(data.user_id); - return next; - }); - }); - - // @ts-ignore - const removeVideoChunk = window.electron.ipcRenderer.on("video-chunk", (_, payload) => { - handleIncomingVideoFragment(payload); - }); - - // @ts-ignore - const removeAudioFragment = window.electron.ipcRenderer.on("audio-fragment", (_, payload) => { - handleIncomingAudioFragment(payload); - }); - - // @ts-ignore - const removeAudioChunk = window.electron.ipcRenderer.on("audio-chunk", (_, payload) => { - // Check if it's the old single chunk message (fallback) - if (mediaEngineRef.current) { - const data = payload.data instanceof Uint8Array ? payload.data : new Uint8Array(payload.data); - mediaEngineRef.current.decodeAudioChunk(data, payload.user_id, payload.ts); + if (data && data.user_id) { + setPeers((prev) => prev.filter(p => p.user_id !== data.user_id)); + setPeerVideoUrls(prev => { + const newState = { ...prev }; + if (newState[data.user_id]) { + URL.revokeObjectURL(newState[data.user_id]); + delete newState[data.user_id]; + } + return newState; + }); + setPeerScreenUrls(prev => { + const newState = { ...prev }; + if (newState[data.user_id]) { + URL.revokeObjectURL(newState[data.user_id]); + delete newState[data.user_id]; + } + return newState; + }); } }); + // Update Stream Signaling (MediaType: "Video" or "Screen" from Rust enum) // @ts-ignore - const removePeerStreamUpdate = window.electron.ipcRenderer.on("peer-stream-update", (_, data) => { - addLog(`Stream Update: User ${data.user_id} ${data.media_type} = ${data.active}`); - - const userId = data.user_id; - const active = data.active; - const mediaType = data.media_type.toLowerCase(); // usage: 'video' or 'screen' - - if (mediaType === 'video') { - if (!active) { - setPeersWithCam(prev => { - const next = new Set(prev); - next.delete(userId); - return next; + const removeStreamUpdate = window.electron.ipcRenderer.on("peer-stream-update", (_, data) => { + const { user_id, active, media_type } = data; + if (!active) { + // Support both legacy string and numeric enum (1=Video, 2=Screen) + if (media_type === 'Video' || media_type === 1) { + setPeerVideoUrls(prev => { + const newState = { ...prev }; + if (newState[user_id]) { + URL.revokeObjectURL(newState[user_id]); + delete newState[user_id]; + } + return newState; }); - const key = `${userId}-video`; - const canvas = peerCanvasRefs.current.get(key); - if (canvas) { - const ctx = canvas.getContext('2d'); - if (ctx) ctx.clearRect(0, 0, canvas.width, canvas.height); - } - } else { - // If active=true, we might want to ensure we track it - setPeersWithCam(prev => new Set(prev).add(userId)); - } - } else if (mediaType === 'screen') { - if (!active) { - console.log(`[App] Removing Screen Share for User ${userId}`); - setPeersWithScreen(prev => { - const next = new Set(prev); - next.delete(userId); - return next; + } else if (media_type === 'Screen' || media_type === 2) { + setPeerScreenUrls(prev => { + const newState = { ...prev }; + if (newState[user_id]) { + URL.revokeObjectURL(newState[user_id]); + delete newState[user_id]; + } + return newState; }); - // Force clear canvas immediately - const key = `${userId}-screen`; - const canvas = peerCanvasRefs.current.get(key); - if (canvas) { - const ctx = canvas.getContext('2d'); - if (ctx) ctx.clearRect(0, 0, canvas.width, canvas.height); - } - } else { - console.log(`[App] Adding Screen Share for User ${userId}`); - setPeersWithScreen(prev => new Set(prev).add(userId)); } } }); - // Chat Message + // Chat Message - skip own messages (we add them locally) // @ts-ignore const removeChatMessage = window.electron.ipcRenderer.on("chat-message", (_, data) => { - const msgId = `${data.user_id}-${data.timestamp}`; + // Skip own messages (already added locally) + // We'll check by comparing user_id with current selfId later const msg: ChatMessage = { - id: msgId, + id: `${data.user_id}-${data.timestamp}`, userId: data.user_id, displayName: data.display_name, message: data.message, timestamp: data.timestamp }; - + // Only add if not from self (selfId check happens via closure) setChatMessages(prev => { - // Strict deduplication - if (prev.some(m => m.id === msgId)) return prev; + // Deduplicate strictly by ID + if (prev.find(m => m.id === msg.id)) { + return prev; + } + // Also check if same message from same user within last second to avoid duplicate broadcasts + const duplicate = prev.find(m => + m.userId === msg.userId && + m.message === msg.message && + Math.abs(m.timestamp - msg.timestamp) < 2000 + ); + if (duplicate) return prev; + + // Add notification + const notif = { + id: msg.id, + displayName: msg.displayName, + message: msg.message, + timestamp: msg.timestamp + }; + setChatNotifications(notifs => { + if (notifs.find(n => n.id === notif.id)) return notifs; + return [...notifs, notif]; + }); + + // Auto-dismiss after 5s + setTimeout(() => { + setChatNotifications(current => current.filter(n => n.id !== notif.id)); + }, 5000); + return [...prev, msg]; }); + }); - // Notification side effect (safe to use functional update here) - setChatNotifications(prevNotifs => { - // Check if we already have this notification to avoid double toast - if (prevNotifs.some(n => n.id === msgId)) return prevNotifs; + // Video/Screen Reassembly logic + const fragmentMap = new Map(); - // Don't show toast if chat is open, BUT we are inside an event handler, - // checking state `chatOpen` directly here might be stale if not in deps. - // However, we can't easily access latest `chatOpen` in a closure without refs or re-binding. - // For now, let's just add it. The rendering logic filters it? No. + // Video Frame (Reassembly) + // @ts-ignore + const removeVideo = window.electron.ipcRenderer.on("video-frame", (_, payload) => { + const { user_id, data, seq, fidx, fcnt } = payload; + const key = `v-${user_id}-${seq}`; - // Cleaner: Just add it. The Toast component can be controlled by parent. - const newNotif = { id: msgId, displayName: msg.displayName, message: msg.message, timestamp: msg.timestamp }; - setTimeout(() => setChatNotifications(ids => ids.filter(x => x.id !== msgId)), 5000); - return [...prevNotifs, newNotif]; - }); + if (!fragmentMap.has(key)) { + fragmentMap.set(key, { chunks: new Array(fcnt), count: 0, total: fcnt }); + } + + const state = fragmentMap.get(key)!; + if (!state.chunks[fidx]) { + state.chunks[fidx] = new Uint8Array(data); + state.count++; + } + + if (state.count === state.total) { + // All fragments received: assemble + const totalLen = state.chunks.reduce((acc, c) => acc + c.length, 0); + const fullBuffer = new Uint8Array(totalLen); + let offset = 0; + for (const chunk of state.chunks) { + fullBuffer.set(chunk, offset); + offset += chunk.length; + } + + const blob = new Blob([fullBuffer], { type: 'image/jpeg' }); + const url = URL.createObjectURL(blob); + + setPeerVideoUrls(prev => { + if (prev[user_id]) URL.revokeObjectURL(prev[user_id]); + return { ...prev, [user_id]: url }; + }); + + fragmentMap.delete(key); + // Clean up old sequences for this user to avoid memory leak + for (const k of fragmentMap.keys()) { + if (k.startsWith(`v-${user_id}-`) && k !== key) { + const kSeq = parseInt(k.split('-')[2]); + if (kSeq < seq) fragmentMap.delete(k); + } + } + } + }); + + // Screen Frame (Reassembly) + // @ts-ignore + const removeScreen = window.electron.ipcRenderer.on("screen-frame", (_, payload) => { + const { user_id, data, seq, fidx, fcnt } = payload; + const key = `s-${user_id}-${seq}`; + + if (!fragmentMap.has(key)) { + fragmentMap.set(key, { chunks: new Array(fcnt), count: 0, total: fcnt }); + } + + const state = fragmentMap.get(key)!; + if (!state.chunks[fidx]) { + state.chunks[fidx] = new Uint8Array(data); + state.count++; + } + + if (state.count === state.total) { + const totalLen = state.chunks.reduce((acc, c) => acc + c.length, 0); + const fullBuffer = new Uint8Array(totalLen); + let offset = 0; + for (const chunk of state.chunks) { + fullBuffer.set(chunk, offset); + offset += chunk.length; + } + + const blob = new Blob([fullBuffer], { type: 'image/jpeg' }); + const url = URL.createObjectURL(blob); + + setPeerScreenUrls(prev => { + if (prev[user_id]) URL.revokeObjectURL(prev[user_id]); + return { ...prev, [user_id]: url }; + }); + + fragmentMap.delete(key); + for (const k of fragmentMap.keys()) { + if (k.startsWith(`s-${user_id}-`) && k !== key) { + const kSeq = parseInt(k.split('-')[2]); + if (kSeq < seq) fragmentMap.delete(k); + } + } + } + }); + + // Audio Frame Playback - LOW LATENCY mode + const playbackCtxRef = { current: null as AudioContext | null }; + const nextPlayTimeRef = { current: 0 }; + const JITTER_BUFFER_MS = 20; // Reduced from 80ms for low latency + const bufferQueue: Float32Array[] = []; + let isStarted = false; + + const scheduleBuffer = (float32: Float32Array) => { + const ctx = playbackCtxRef.current; + if (!ctx) return; + + const buffer = ctx.createBuffer(1, float32.length, 48000); + buffer.copyToChannel(float32 as any, 0); + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(ctx.destination); + + // Schedule at precise time to avoid gaps + const now = ctx.currentTime; + if (nextPlayTimeRef.current < now) { + // We've fallen behind, reset + nextPlayTimeRef.current = now + JITTER_BUFFER_MS / 1000; + } + source.start(nextPlayTimeRef.current); + nextPlayTimeRef.current += buffer.duration; + }; + + const flushBuffer = () => { + while (bufferQueue.length > 0) { + scheduleBuffer(bufferQueue.shift()!); + } + }; + + // @ts-ignore + const removeAudio = window.electron.ipcRenderer.on("audio-frame", (_, payload) => { + try { + const { data } = payload; + if (!playbackCtxRef.current) { + playbackCtxRef.current = new AudioContext({ sampleRate: 48000 }); + nextPlayTimeRef.current = playbackCtxRef.current.currentTime + JITTER_BUFFER_MS / 1000; + } + + const ctx = playbackCtxRef.current; + if (ctx.state === 'suspended') { + ctx.resume(); + } + + // Convert Uint8Array (bytes) to Int16 PCM then to Float32 + const uint8 = new Uint8Array(data); + const int16 = new Int16Array(uint8.buffer, uint8.byteOffset, uint8.length / 2); + const float32 = new Float32Array(int16.length); + + for (let i = 0; i < int16.length; i++) { + float32[i] = int16[i] / 32768; + } + + if (!isStarted) { + // Start immediately after just 1 packet for lowest latency + bufferQueue.push(float32); + if (bufferQueue.length >= 1) { + isStarted = true; + flushBuffer(); + } + } else { + scheduleBuffer(float32); + } + } catch (e) { + console.error("Audio playback error:", e); + } }); return () => { - engine.cleanup(); removePeerJoined(); removePeerLeft(); - removeVideoChunk(); - removeAudioChunk(); - removePeerStreamUpdate(); + removeStreamUpdate(); removeChatMessage(); - // Don't close AudioContext here if we want to reuse it? - // Actually good practice to close it on component unmount - if (audioCtxRef.current) { - audioCtxRef.current.close(); - audioCtxRef.current = null; - } + removeVideo(); + removeScreen(); + removeAudio(); }; - }, [connected]); - // ^ Dependency on `connected` means this effect re-runs when connected changes. - // Be careful with stale closures for other values if referenced. + }, []); - - // Fragment Reassembly Logic - const fragmentBuffer = useRef>(new Map()); - - const handleIncomingVideoFragment = (payload: any) => { - // payload needs streamType now - const { user_id, data, seq, fidx, fcnt, isKeyFrame, ts, streamType } = payload; - const type = streamType || 'video'; - - if (type === 'screen' && fidx === 0) console.log(`[App] RX Screen Chunk: User ${user_id} Seq ${seq}`); - - // Key must include streamType so we don't mix screen and video chunks - const key = `${user_id}-${type}-${seq}`; - const map = fragmentBuffer.current; - - if (!map.has(key)) { - map.set(key, { chunks: new Array(fcnt), count: 0, total: fcnt }); - } - - const entry = map.get(key)!; - if (!entry.chunks[fidx]) { - entry.chunks[fidx] = data; - entry.count++; - } - - if (entry.count === entry.total) { - const totalLen = entry.chunks.reduce((acc, c) => acc + c.length, 0); - const fullBuffer = new Uint8Array(totalLen); - let offset = 0; - for (const c of entry.chunks) { - fullBuffer.set(c, offset); - offset += c.length; - } - map.delete(key); - - // Cleanup old - for (const k of map.keys()) { - if (k.startsWith(`${user_id}-${type}-`) && k !== key) { - const kSeq = parseInt(k.split('-')[2]); - if (kSeq < seq - 10) map.delete(k); - } - } - - mediaEngineRef.current?.decodeVideoChunk(fullBuffer, user_id, isKeyFrame, ts, type); - } - }; - - - // MEDIA CAPTURE HOOKS - - // --- Media Acquisition Queue (Prevent PipeWire Deadlocks) --- - const mediaLock = useRef>(Promise.resolve()); - const acquireMediaLock = () => { - let release: () => void; - const lock = new Promise(resolve => { release = resolve; }); - const prev = mediaLock.current; - // Chain it - const next = prev.then(async () => { - // Wait a bit to let PipeWire breathe - await new Promise(r => setTimeout(r, 500)); - return; - }); - mediaLock.current = next.then(() => lock); - return { - wait: () => next, - release: () => release() - }; - }; - - // 1. Webcam Capture + // Frame Capture Loop useEffect(() => { - let active = true; - let stream: MediaStream | null = null; + let animationFrameId: number; + let isActive = true; - const startWebcam = async () => { - if (!videoEnabled || !mediaEngineRef.current) return; + const sendFrame = async () => { + if (!isActive) return; - const lock = acquireMediaLock(); - await lock.wait(); - if (!active) { lock.release(); return; } + if (videoEnabled && localVideoRef.current && canvasRef.current) { + const video = localVideoRef.current; + const canvas = canvasRef.current; + const ctx = canvas.getContext('2d'); - try { - addLog("Requesting Webcam Access..."); - stream = await navigator.mediaDevices.getUserMedia({ - video: { - width: 1280, - height: 720, - frameRate: 30, - deviceId: selectedVideoDevice ? { exact: selectedVideoDevice } : undefined - } - }); - addLog(`Webcam Access Granted: ${stream.id}`); + if (ctx && video.readyState === 4) { + // Use native video size, capped at 1080p for bandwidth sanity + const width = Math.min(video.videoWidth, 1920); + const height = Math.min(video.videoHeight, 1080); - if (localVideoRef.current) { - localVideoRef.current.srcObject = stream; + canvas.width = width; + canvas.height = height; + ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + + canvas.toBlob(async (blob) => { + if (blob && isActive && videoEnabled && connected) { + try { + const arrayBuffer = await blob.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + // @ts-ignore + window.electron.ipcRenderer.send("send-video-frame", { frame: uint8Array }); + } catch (e) { + // Ignore send errors + } + } + }, 'image/jpeg', 0.4); // Quality 0.4 (Good balance for 1080p UDP) } - - const track = stream.getVideoTracks()[0]; - // @ts-ignore - const processor = new MediaStreamTrackProcessor({ track }); - const reader = processor.readable.getReader(); - - // Signal ON - // @ts-ignore - window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 1 }); // 1 = Video - - lock.release(); // Release lock after successful acquisition - - while (active) { - const result = await reader.read(); - if (result.done) break; - if (result.value) { - mediaEngineRef.current?.encodeVideoFrame(result.value, 'video'); - } - if (result.value) result.value.close(); // Close frame if not encoded? decodeVideoChunk closes it. encodeVideoFrame closes it. - } - } catch (e) { - console.error("Webcam error", e); - setVideoEnabled(false); - lock.release(); } + setTimeout(() => { + if (isActive) animationFrameId = requestAnimationFrame(sendFrame); + }, 33); // 30 FPS }; if (videoEnabled) { - startWebcam(); - } else { - if (connected) { - // @ts-ignore - window.electron.ipcRenderer.send('update-stream', { active: false, mediaType: 1 }); - } - if (localVideoRef.current) localVideoRef.current.srcObject = null; + sendFrame(); } return () => { - active = false; - if (stream) { - addLog("Stopping Webcam Stream"); - stream.getTracks().forEach(t => t.stop()); - } + isActive = false; + cancelAnimationFrame(animationFrameId); }; - }, [videoEnabled, selectedVideoDevice, connected]); + }, [videoEnabled, connected]); - - // 2. Screen Share Capture + // Camera Access - re-trigger when connected changes useEffect(() => { - let active = true; - let stream: MediaStream | null = null; - - const startScreen = async () => { - if (!screenEnabled || !mediaEngineRef.current) return; - - const lock = acquireMediaLock(); - await lock.wait(); - if (!active) { lock.release(); return; } - - try { - addLog("Fetching screen sources..."); - // @ts-ignore - const sources = await window.electron.ipcRenderer.invoke('get-screen-sources'); - if (sources && sources.length > 0) { - const sourceId = sources[0].id; - addLog(`Selected screen source: ${sourceId}`); - + if (videoEnabled) { + navigator.mediaDevices.getUserMedia({ video: { width: 1280, height: 720 }, audio: false }) + .then(stream => { + if (localVideoRef.current) { + localVideoRef.current.srcObject = stream; + } + // Signal video ON // @ts-ignore - stream = await navigator.mediaDevices.getUserMedia({ - audio: false, // System audio often requires specific OS config, disable for now + window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 1 }); + }) + .catch(err => { + console.error("Camera access error:", err); + setError("Failed to access camera"); + }); + } else { + if (localVideoRef.current && localVideoRef.current.srcObject) { + const stream = localVideoRef.current.srcObject as MediaStream; + stream.getTracks().forEach(track => track.stop()); + localVideoRef.current.srcObject = null; + } + // Signal video OFF + // @ts-ignore + if (connected) window.electron.ipcRenderer.send('update-stream', { active: false, mediaType: 1 }); + } + }, [videoEnabled, connected]); + + // Screen Sharing + const screenStreamRef = useRef(null); + const screenVideoRef = useRef(null); + + useEffect(() => { + if (!connected) return; + + let isActive = true; + const startScreenShare = async () => { + if (screenEnabled) { + try { + // Get available screen sources + // @ts-ignore + const sources = await window.electron.ipcRenderer.invoke('get-screen-sources'); + if (!sources || sources.length === 0) { + setError('No screen sources available'); + setScreenEnabled(false); + return; + } + + // Use first screen source (could add picker UI later) + const screenSource = sources[0]; + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: false, video: { + // @ts-ignore - Electron-specific constraint mandatory: { chromeMediaSource: 'desktop', - chromeMediaSourceId: sourceId, + chromeMediaSourceId: screenSource.id, maxWidth: 1920, - maxHeight: 1080 + maxHeight: 1080, + maxFrameRate: 60 } } - } as any); + }); - if (!stream) { lock.release(); return; } - addLog(`Screen Access Granted: ${stream.id}`); - - if (localScreenRef.current) { - localScreenRef.current.srcObject = stream; + if (!isActive) { + stream.getTracks().forEach(t => t.stop()); + return; } - stream.getVideoTracks()[0].onended = () => { - addLog("Stream ended by user interaction"); - if (screenEnabled) setScreenEnabled(false); + screenStreamRef.current = stream; + addLog('Screen sharing started'); + + // Create hidden video element if needed + let screenVideo = screenVideoRef.current; + if (!screenVideo) { + screenVideo = document.createElement('video'); + screenVideo.autoplay = true; + screenVideo.muted = true; + } + screenVideo.srcObject = stream; + + // Send screen frames (similar to video but with different MediaType) + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + canvas.width = 1920; + canvas.height = 1080; + + const sendScreenFrame = () => { + if (!isActive || !screenEnabled || !screenStreamRef.current) return; + + if (ctx && screenVideo && screenVideo.readyState >= 2) { + ctx.drawImage(screenVideo, 0, 0, canvas.width, canvas.height); + canvas.toBlob((blob) => { + if (blob && isActive) { + // Also update local view + const url = URL.createObjectURL(blob); + setLocalScreenUrl(url); + + blob.arrayBuffer().then(buf => { + // @ts-ignore - Send as screen type + window.electron.ipcRenderer.send('send-screen-frame', { frame: Array.from(new Uint8Array(buf)) }); + }); + } + }, 'image/jpeg', 0.4); // Quality 0.4 for speed + } + + setTimeout(() => { + if (isActive) requestAnimationFrame(sendScreenFrame); + }, 33); // 30 FPS for Screen Share }; - const track = stream.getVideoTracks()[0]; + screenVideo.onloadeddata = () => sendScreenFrame(); + + // Signal screen ON // @ts-ignore - const processor = new MediaStreamTrackProcessor({ track }); - const reader = processor.readable.getReader(); + window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 2 }); - // Signal ON - // @ts-ignore - window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 2 }); // 2 = Screen - - lock.release(); - - while (active) { - const result = await reader.read(); - if (result.done) break; - if (result.value) { - mediaEngineRef.current?.encodeVideoFrame(result.value, 'screen'); - } - } - - } else { - addLog("No screen sources found"); + } catch (err) { + console.error('Screen share error:', err); + setError('Failed to start screen sharing'); setScreenEnabled(false); - lock.release(); } - } catch (e) { - console.error("Screen share error", e); - setScreenEnabled(false); - lock.release(); - } - }; - - if (screenEnabled) { - startScreen(); - } else { - if (connected) { + } else { + // Stop screen sharing + if (screenStreamRef.current) { + screenStreamRef.current.getTracks().forEach(t => t.stop()); + screenStreamRef.current = null; + } + if (localScreenUrl) { + URL.revokeObjectURL(localScreenUrl); + setLocalScreenUrl(null); + } + // Signal screen OFF // @ts-ignore window.electron.ipcRenderer.send('update-stream', { active: false, mediaType: 2 }); + addLog('Screen sharing stopped'); } - } - + }; + startScreenShare(); return () => { - active = false; - if (stream) { - addLog("Stopping Screen Stream"); - stream.getTracks().forEach(t => t.stop()); - } + isActive = false; }; }, [screenEnabled, connected]); - // Audio Capture + const audioContextRef = useRef(null); + const audioStreamRef = useRef(null); + useEffect(() => { - let active = true; - let stream: MediaStream | null = null; - let reader: ReadableStreamDefaultReader | null = null; + let isCancelled = false; - const startAudio = async () => { - if (!audioEnabled || !mediaEngineRef.current) return; + const cleanup = () => { + // Stop tracks + if (audioStreamRef.current) { + audioStreamRef.current.getTracks().forEach(t => t.stop()); + audioStreamRef.current = null; + } + // Close context + if (audioContextRef.current) { + if (audioContextRef.current.state !== 'closed') { + audioContextRef.current.close().catch(e => console.error("Error closing ctx", e)); + } + audioContextRef.current = null; + } + }; - addLog("[App] startAudio: Acquiring Lock"); - const lock = acquireMediaLock(); - await lock.wait(); - if (!active) { lock.release(); return; } - addLog("[App] startAudio: Lock Acquired"); + if (!audioEnabled || !connected) { + cleanup(); + return; + } + addLog(`Starting audio... Device: ${selectedAudioDevice || 'Default'}`); + + async function startAudio() { try { - addLog("Requesting Audio Access..."); - stream = await navigator.mediaDevices.getUserMedia({ + // Short delay to allow previous cleanup to settle if rapid toggling + await new Promise(r => setTimeout(r, 100)); + if (isCancelled) return; + + const constraints = { audio: { deviceId: selectedAudioDevice ? { exact: selectedAudioDevice } : undefined, echoCancellation: true, noiseSuppression: true, autoGainControl: true - } - }); - addLog(`Audio Access Granted: ${stream.id}`); + }, + video: false + }; - const track = stream.getAudioTracks()[0]; - // @ts-ignore - const processor = new MediaStreamTrackProcessor({ track }); - reader = processor.readable.getReader(); + const stream = await navigator.mediaDevices.getUserMedia(constraints); - // Signal ON - // @ts-ignore - window.electron.ipcRenderer.send('update-stream', { active: true, mediaType: 0 }); // 0 = Audio + if (isCancelled) { + stream.getTracks().forEach(t => t.stop()); + return; + } - lock.release(); - addLog("[App] startAudio: Reading Loop Start"); + addLog(`Mic Gained: ${stream.getAudioTracks()[0].label}`); - let frameCount = 0; + audioStreamRef.current = stream; - while (active) { - const result = await reader.read(); - if (result.done) break; - if (result.value) { - if (frameCount % 100 === 0) console.log(`[App] Capturing Audio Frame ${frameCount}`); - mediaEngineRef.current?.encodeAudioData(result.value); - frameCount++; + // Create context (if allowed by browser policy - usually requires interaction, which we have via button click) + const ctx = new AudioContext({ sampleRate: 48000, latencyHint: 'interactive' }); + audioContextRef.current = ctx; + + // Load Worklet + // Note: creating blob URL every time is fine, browsers handle it. + const blob = new Blob([audioWorkletCode], { type: 'application/javascript' }); + const workletUrl = URL.createObjectURL(blob); + + let useWorklet = false; + try { + await ctx.audioWorklet.addModule(workletUrl); + useWorklet = true; + } catch (e) { + console.warn("Worklet addModule failed", e); + } + URL.revokeObjectURL(workletUrl); + + if (isCancelled) { + ctx.close(); + return; + } + + const source = ctx.createMediaStreamSource(stream); + + if (useWorklet) { + try { + addLog("Creating AudioWorkletNode..."); + const workletNode = new AudioWorkletNode(ctx, 'pcm-processor'); + workletNode.port.onmessage = (e) => { + if (!audioEnabled || !connected || isCancelled) return; + const float32 = e.data; + const pcm = new Int16Array(float32.length); + for (let i = 0; i < float32.length; i++) { + pcm[i] = Math.max(-32768, Math.min(32767, Math.floor(float32[i] * 32768))); + } + // @ts-ignore + window.electron.ipcRenderer.send('send-audio-frame', { frame: pcm.buffer }); + }; + source.connect(workletNode); + workletNode.connect(ctx.destination); + addLog("Audio Worklet running"); + return; // Success! + } catch (e: any) { + console.error("Worklet Node creation failed", e); + addLog(`Worklet Node failed: ${e.message}`); + // Fall through to fallback } } - addLog("[App] startAudio: Reading Loop End"); - } catch (e) { - console.error("Audio capture error", e); - setAudioEnabled(false); - lock.release(); - } - }; - - if (audioEnabled) { - console.log("[App] Audio Enabled -> Starting"); - startAudio(); - } else { - console.log("[App] Audio Disabled"); - if (connected) { + // Fallback to ScriptProcessor + addLog("Falling back to ScriptProcessor..."); + // @ts-ignore - 2048 samples for lower latency + const scriptNode = ctx.createScriptProcessor(2048, 1, 1); // @ts-ignore - window.electron.ipcRenderer.send('update-stream', { active: false, mediaType: 0 }); // 0 = Audio + scriptNode.onaudioprocess = (e) => { + if (!audioEnabled || !connected || isCancelled) return; + const inputData = e.inputBuffer.getChannelData(0); + const pcm = new Int16Array(inputData.length); + for (let i = 0; i < inputData.length; i++) { + pcm[i] = Math.max(-32768, Math.min(32767, Math.floor(inputData[i] * 32768))); + } + // @ts-ignore + window.electron.ipcRenderer.send('send-audio-frame', { frame: pcm.buffer }); + }; + source.connect(scriptNode); + // @ts-ignore + scriptNode.connect(ctx.destination); + addLog("ScriptProcessor running"); + + } catch (err: any) { + if (isCancelled) return; // Ignore errors if we cancelled + console.error('Audio capture error:', err); + if (err.name === 'AbortError' || err.name === 'NotAllowedError') { + addLog(`Permission/Abort Error: ${err.message}`); + setError(`Mic blocked/aborted: ${err.message}`); + } else { + addLog(`Mic Error: ${err.message}`); + setError(`Mic Error: ${err.message}`); + } + setAudioEnabled(false); // Reset state } } + startAudio(); + return () => { - active = false; - console.log("[App] Audio Cleanup"); - if (stream) { - addLog("Stopping Audio Stream"); - stream.getTracks().forEach(t => t.stop()); - } - if (reader) reader.cancel(); + isCancelled = true; + cleanup(); }; - }, [audioEnabled, selectedAudioDevice, connected]); + }, [audioEnabled, connected, selectedAudioDevice]); - - // Connection Logic async function handleJoin(roomCode: string, name: string, serverUrl: string, initialVideo: boolean, initialAudio: boolean) { if (!roomCode || !name || !serverUrl) return; setDisplayName(name); setVideoEnabled(initialVideo); setAudioEnabled(initialAudio); setError(""); - - // Initialize AudioContext on User Gesture (Click) - if (!audioCtxRef.current) { - try { - const Ctx = window.AudioContext || (window as any).webkitAudioContext; - audioCtxRef.current = new Ctx(); - addLog(`AudioContext Initialized. State: ${audioCtxRef.current?.state}`); - - if (audioCtxRef.current?.state === 'suspended') { - await audioCtxRef.current.resume(); - addLog("AudioContext Resumed immediately."); - } - } catch (e) { - console.error("Failed to init AudioContext", e); - } - } else if (audioCtxRef.current.state === 'suspended') { - await audioCtxRef.current.resume(); - addLog("AudioContext Resumed (was existing)."); - } - try { // @ts-ignore const result = await window.electron.ipcRenderer.invoke("connect", { serverUrl, roomCode, displayName: name }); @@ -733,70 +717,57 @@ function App() { } } catch (e: any) { console.error(e); - setError(typeof e === 'string' ? e : JSON.stringify(e)); + const errMsg = typeof e === 'string' ? e : JSON.stringify(e); + addLog(`Error: ${errMsg}`); + setError(errMsg); } } async function handleLeave() { setVideoEnabled(false); - setAudioEnabled(false); - setScreenEnabled(false); - setPeersWithCam(new Set()); - setPeersWithCam(new Set()); - setPeersWithScreen(new Set()); - setPeersWithAudio(new Set()); // @ts-ignore await window.electron.ipcRenderer.invoke("disconnect"); setConnected(false); setPeers([]); setSelfId(null); + setPeerVideoUrls({}); } - // UI Toggles const toggleAudio = () => setAudioEnabled(!audioEnabled); - const toggleVideo = () => { - setVideoEnabled(!videoEnabled); - }; - const toggleScreen = () => { - setScreenEnabled(!screenEnabled); - }; + const toggleVideo = () => setVideoEnabled(!videoEnabled); + const toggleScreen = () => setScreenEnabled(!screenEnabled); const toggleChat = () => setChatOpen(!chatOpen); const sendChatMessage = (message: string) => { - // @ts-ignore + if (!selfId) return; + // Don't add locally - server will broadcast it back to us + // But send displayName too to ensure server has it + // @ts-ignore - Send via WebSocket window.electron.ipcRenderer.invoke('send-chat', { message, displayName: displayName || 'User' }); }; - // Hidden Canvas Ref for internal use if needed - const canvasRef = useRef(null); - return (
+ {!connected ? ( ) : (
+ {/* Main Content Area */}
+ {/* Stage */} + {/* Control Bar */}
+ {/* Chat Panel */} {chatOpen && ( )} + {/* Chat Notifications */} {!chatOpen && ( )} - {/* Hidden Canvas if needed */} + {/* Hidden Canvas for capture */} +
+ )} - {/* Error Toast */} - {error && ( -
- {error} -
- )} + {/* Error Toast */} + {error && ( +
+ {error}
)}
diff --git a/src/renderer/src/components/Stage.tsx b/src/renderer/src/components/Stage.tsx index c6f52c6..869b764 100644 --- a/src/renderer/src/components/Stage.tsx +++ b/src/renderer/src/components/Stage.tsx @@ -10,13 +10,7 @@ interface StageProps { peerScreenUrls?: { [key: number]: string }; localScreenUrl?: string | null; localVideoRef?: React.RefObject; - localScreenRef?: React.RefObject; videoEnabled?: boolean; - screenEnabled?: boolean; - registerPeerCanvas: (userId: number, streamType: 'video' | 'screen', canvas: HTMLCanvasElement | null) => void; - peersWithCam: Set; - peersWithScreen: Set; - peersWithAudio: Set; } export function Stage({ @@ -27,16 +21,8 @@ export function Stage({ peerScreenUrls = {}, localScreenUrl = null, localVideoRef, - localScreenRef, - videoEnabled = false, - screenEnabled = false, - registerPeerCanvas, - peersWithCam, - peersWithScreen, - peersWithAudio + videoEnabled = false }: StageProps) { - - // Track container dimensions for smart layout const [containerSize, setContainerSize] = useState({ width: 800, height: 600 }); @@ -56,108 +42,121 @@ export function Stage({ return () => window.removeEventListener('resize', updateSize); }, []); - // Active Screen Shares (Remote) - const remoteScreens = peers.filter(p => peersWithScreen.has(p.user_id)); - - // Check if self is sharing screen (via local video/stream state is not passed here directly as boolean, - // but App handles local display via `localVideoRef`... wait. - // App.tsx handles local WEBCAM via `localVideoRef`. - // App.tsx handles local SCREEN via... `localVideoRef` too? - // In App.tsx: `if (localVideoRef.current) localVideoRef.current.srcObject = stream;` for BOTH. - // So if I am sharing screen, `localVideoRef` shows my screen? - // Yes, `toggleVideo` switches off screen if active. `startCapture` sets srcObject. - // BUT `Stage` renders `localVideoRef` in the "Self Webcam" slot. // Check if self is sharing screen - const isSelfSharing = screenEnabled; + const isSelfSharing = !!localScreenUrl; - // Active Screen Shares - // const peerScreens = ... (removed legacy variable, using remoteScreens directly) + // Filter peers who are sharing screen + const peerScreens = peers.filter(p => !!peerScreenUrls[p.user_id]); // All peers for webcam grid const allParticipants = peers; - - const showScreenLayer = isSelfSharing || remoteScreens.length > 0; + const showScreenLayer = isSelfSharing || peerScreens.length > 0; const totalParticipants = (selfId ? 1 : 0) + allParticipants.length; - // Layout + // Smart layout: determine if we should use vertical or horizontal arrangement const aspectRatio = containerSize.width / containerSize.height; - const isVertical = aspectRatio < 1; + const isVertical = aspectRatio < 1; // Taller than wide // Calculate optimal grid layout based on aspect ratio and participant count const getGridConfig = (count: number, isVertical: boolean) => { - if (count <= 1) return { cols: 1, rows: 1 }; + if (count <= 1) { + return { cols: 1, rows: 1 }; + } + if (isVertical) { + // Vertical window: prefer fewer columns, more rows if (count === 2) return { cols: 1, rows: 2 }; - return { cols: 2, rows: Math.ceil(count / 2) }; + if (count <= 4) return { cols: 2, rows: 2 }; + if (count <= 6) return { cols: 2, rows: 3 }; + if (count <= 9) return { cols: 3, rows: 3 }; + return { cols: 3, rows: Math.ceil(count / 3) }; } else { - return { cols: Math.ceil(count / 2), rows: 2 }; + // Horizontal window: prefer more columns + if (count === 2) return { cols: 2, rows: 1 }; + if (count <= 4) return { cols: 2, rows: 2 }; + if (count <= 6) return { cols: 3, rows: 2 }; + if (count <= 9) return { cols: 3, rows: 3 }; + return { cols: 4, rows: Math.ceil(count / 4) }; } }; const gridConfig = getGridConfig(totalParticipants, isVertical); - return ( -
+ // Screen share layout direction + const screenLayoutClass = isVertical + ? 'flex-col' // Stack screen above participants + : 'flex-row'; // Screen on left, participants on right - {/* Screen Share Area (Left or Top) */} + return ( +
+ {/* Screen Share Layer */} {showScreenLayer && ( -
+
{/* Local Screen Share */} {isSelfSharing && (
)} {/* Remote Screen Shares */} - {remoteScreens.map(peer => ( + {peerScreens.map(peer => (
registerPeerCanvas(uid, 'screen', canvas)} />
))}
)} - {/* Webcam Grid (Right or Bottom) */} -
-
+
- - {/* Self */} + gridTemplateColumns: showScreenLayer + ? '1fr' + : `repeat(${gridConfig.cols}, 1fr)`, + gridTemplateRows: showScreenLayer + ? 'auto' + : `repeat(${gridConfig.rows}, 1fr)`, + justifyContent: 'center', + alignContent: 'center' + }} + > + {/* Self Webcam */} {selfId && ( -
+
)} - {/* Remote Peers Cam */} - {peers.map(peer => ( -
+ {/* Remote Webcam Peers */} + {allParticipants.map(peer => ( +
registerPeerCanvas(uid, 'video', canvas)} + audioEnabled={true} + videoEnabled={!!peerVideoUrls[peer.user_id]} + videoSrc={peerVideoUrls[peer.user_id]} />
))} diff --git a/src/renderer/src/components/VideoTile.tsx b/src/renderer/src/components/VideoTile.tsx index 444f62b..56ec71b 100644 --- a/src/renderer/src/components/VideoTile.tsx +++ b/src/renderer/src/components/VideoTile.tsx @@ -1,5 +1,4 @@ import { Mic, MicOff } from "lucide-react"; -import { useEffect, useRef } from "react"; interface VideoTileProps { displayName: string; @@ -9,58 +8,27 @@ interface VideoTileProps { audioEnabled?: boolean; videoEnabled?: boolean; isScreenShare?: boolean; - userId?: number; - onCanvasRef?: (userId: number, canvas: HTMLCanvasElement | null) => void; } export function VideoTile({ displayName, isSelf, - // videoSrc, // Unused + videoSrc, videoRef, audioEnabled = true, videoEnabled = false, - isScreenShare = false, - userId, - onCanvasRef + isScreenShare = false }: VideoTileProps) { - const canvasRef = useRef(null); - - // Register canvas if applicable - const onCanvasRefRef = useRef(onCanvasRef); - - // Update ref when prop changes - useEffect(() => { - onCanvasRefRef.current = onCanvasRef; - }, [onCanvasRef]); - - // Use a callback ref to handle canvas mounting/unmounting reliably - const setCanvasRef = (node: HTMLCanvasElement | null) => { - canvasRef.current = node; - if (node) { - if (!isSelf && userId && onCanvasRefRef.current) { - onCanvasRefRef.current(userId, node); - } - } else { - // Cleanup on unmount - if (!isSelf && userId && onCanvasRefRef.current) { - onCanvasRefRef.current(userId, null); - } - } - }; - - // For self or local preview (using video element) - const showVideoElement = (isSelf || (isScreenShare && !userId)) && videoEnabled && videoRef; - - // For remote peers (WebCodecs via Canvas) - const showRemoteCanvas = !isSelf && userId && onCanvasRef && videoEnabled; - - const showPlaceholder = !showVideoElement && !showRemoteCanvas; + // For self with video ref, use video element bound to the ref + // For remote peers, videoSrc contains blob URL of JPEG frames - use img + const showSelfVideo = isSelf && videoEnabled && videoRef; + const showRemoteMedia = !isSelf && videoSrc; + const showPlaceholder = !showSelfVideo && !showRemoteMedia; return ( -
- {/* Video Element (Self Cam or Local Screen Preview) */} - {showVideoElement && ( +
+ {/* Self Video (webcam stream) */} + {showSelfVideo && (
)} {/* Audio indicator */}
-
+
{audioEnabled ? : }
{/* Name */} -
-
- {displayName} +
+
+ {displayName} + {isSelf && !isScreenShare && (You)}
- {isSelf && !isScreenShare && ( - - You - - )}
); diff --git a/src/renderer/src/main.tsx b/src/renderer/src/main.tsx index 84b0c75..013b143 100644 --- a/src/renderer/src/main.tsx +++ b/src/renderer/src/main.tsx @@ -3,47 +3,8 @@ import ReactDOM from 'react-dom/client' import App from './App' import './index.css' -class ErrorBoundary extends React.Component { - constructor(props: any) { - super(props); - this.state = { hasError: false, error: null }; - } - - static getDerivedStateFromError(error: any) { - return { hasError: true, error }; - } - - componentDidCatch(error: any, errorInfo: any) { - console.error("Uncaught error:", error, errorInfo); - } - - render() { - if (this.state.hasError) { - return ( -
-

Something went wrong.

- 
-                        {this.state.error && this.state.error.toString()}
-
-
- ); - } - - return this.props.children; - } -} - -console.log("[JustTalk] Renderer process started"); -try { - const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); - console.log("[JustTalk] Root created, rendering App..."); - root.render( - - - - - - ); -} catch (e) { - console.error("[JustTalk] Failed to render root:", e); -} +ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render( + + + +) diff --git a/src/renderer/src/utils/MediaEngine.ts b/src/renderer/src/utils/MediaEngine.ts deleted file mode 100644 index 4163bc1..0000000 --- a/src/renderer/src/utils/MediaEngine.ts +++ /dev/null @@ -1,281 +0,0 @@ -// import { EventEmitter } from 'events'; // Node.js module dependency might be missing - -// Custom lightweight Event Emitter for browser compatibility if needed, -// but 'events' module is usually polyfilled by Vite/Webpack. -// If not, we can use a simple class. -class SimpleEventEmitter { - private listeners: { [key: string]: Function[] } = {}; - - on(event: string, listener: Function) { - if (!this.listeners[event]) this.listeners[event] = []; - this.listeners[event].push(listener); - return () => this.off(event, listener); - } - - off(event: string, listener: Function) { - if (!this.listeners[event]) return; - this.listeners[event] = this.listeners[event].filter(l => l !== listener); - } - - emit(event: string, ...args: any[]) { - if (!this.listeners[event]) return; - this.listeners[event].forEach(l => { - try { l(...args); } - catch (e) { console.error(`Error in event listener for ${event}:`, e); } - }); - } -} - -export interface EncodedFrame { - type: 'video' | 'audio'; - data: Uint8Array; - isKeyFrame: boolean; - timestamp: number; - duration?: number; - streamType?: 'video' | 'screen'; // Added for dual stream support -} - -export class MediaEngine extends SimpleEventEmitter { - private videoEncoder: VideoEncoder | null = null; - private screenEncoder: VideoEncoder | null = null; // Separate encoder for screen - private audioEncoder: AudioEncoder | null = null; - - // Decoders: Map -> Now needs to distinguish stream types - // We can use keys like "userId-video" and "userId-screen" - private videoDecoders: Map = new Map(); - // Decoders: Map -> Now needs to distinguish stream types - // We can use keys like "userId-video" and "userId-screen" - // private videoDecoders: Map = new Map(); // Already declared above - private audioDecoders: Map = new Map(); - private videoConfig: VideoEncoderConfig = { - codec: 'avc1.42001f', // H.264 Baseline Profile Level 3.1 (720p safe) - width: 1280, - height: 720, - bitrate: 2_000_000, - framerate: 30, - latencyMode: 'realtime', - avc: { format: 'annexb' } - }; - - private screenConfig: VideoEncoderConfig = { - // High Profile Level 4.2 - codec: 'avc1.64002a', - width: 1920, - height: 1080, - bitrate: 2_000_000, // Reduced to 2 Mbps for better stability/FPS - framerate: 30, - latencyMode: 'realtime', // Changed from 'quality' to 'realtime' for lower latency - avc: { format: 'annexb' } - }; - - // Audio Config - private audioConfig: AudioEncoderConfig = { - codec: 'opus', - sampleRate: 48000, - numberOfChannels: 1, - bitrate: 32000 - }; - - constructor() { - super(); - this.initializeVideoEncoder(); - this.initializeScreenEncoder(); - this.initializeAudioEncoder(); - } - - private initializeVideoEncoder() { - try { - this.videoEncoder = new VideoEncoder({ - output: (chunk, _metadata) => { - const buffer = new Uint8Array(chunk.byteLength); - chunk.copyTo(buffer); - - // With 'annexb', SPS/PPS should be in the keyframe chunk data. - - this.emit('encoded-video', { - type: 'video', - streamType: 'video', - data: buffer, - isKeyFrame: chunk.type === 'key', - timestamp: chunk.timestamp, - duration: chunk.duration - } as EncodedFrame); - }, - error: (e) => console.error('VideoEncoder error:', e), - }); - - this.videoEncoder.configure(this.videoConfig); - console.log('[MediaEngine] VideoEncoder configured:', this.videoConfig); - } catch (e) { - console.error('[MediaEngine] Failed to init VideoEncoder:', e); - } - } - - private initializeScreenEncoder() { - try { - this.screenEncoder = new VideoEncoder({ - output: (chunk, _metadata) => { - const buffer = new Uint8Array(chunk.byteLength); - chunk.copyTo(buffer); - this.emit('encoded-video', { - type: 'video', - streamType: 'screen', - data: buffer, - isKeyFrame: chunk.type === 'key', - timestamp: chunk.timestamp, - duration: chunk.duration - } as EncodedFrame); - }, - error: (e) => console.error('ScreenEncoder error:', e), - }); - this.screenEncoder.configure(this.screenConfig); - console.log('[MediaEngine] ScreenEncoder configured:', this.screenConfig); - } catch (e) { - console.error('[MediaEngine] Failed to init ScreenEncoder:', e); - } - } - - private initializeAudioEncoder() { - try { - this.audioEncoder = new AudioEncoder({ - output: (chunk, _metadata) => { - const buffer = new Uint8Array(chunk.byteLength); - chunk.copyTo(buffer); - this.emit('encoded-audio', { - type: 'audio', - data: buffer, - isKeyFrame: chunk.type === 'key', - timestamp: chunk.timestamp, - duration: chunk.duration - } as EncodedFrame); - }, - error: (e) => console.error('[MediaEngine] AudioEncoder error:', e), - }); - this.audioEncoder.configure(this.audioConfig); - console.log('[MediaEngine] AudioEncoder configured:', this.audioConfig); - } catch (e) { - console.error('[MediaEngine] Failed to init AudioEncoder:', e); - } - } - - - // --- Video Encoding --- - - encodeVideoFrame(frame: VideoFrame, streamType: 'video' | 'screen' = 'video') { - const encoder = streamType === 'screen' ? this.screenEncoder : this.videoEncoder; - - if (encoder && encoder.state === 'configured') { - // Force keyframe every 2 seconds (60 frames) - const keyFrame = frame.timestamp % 2000000 < 33000; - encoder.encode(frame, { keyFrame }); - frame.close(); - } else { - frame.close(); - console.warn(`[MediaEngine] ${streamType === 'screen' ? 'ScreenEncoder' : 'VideoEncoder'} not ready`); - } - } - - // --- Video Decoding --- - - decodeVideoChunk(chunkData: Uint8Array, userId: number, isKeyFrame: boolean, timestamp: number, streamType: 'video' | 'screen' = 'video') { - const decoderKey = `${userId}-${streamType}`; - let decoder = this.videoDecoders.get(decoderKey); - - if (!decoder) { - decoder = new VideoDecoder({ - output: (frame) => { - this.emit('decoded-video', { userId, frame, streamType }); - }, - error: (e) => console.error(`VideoDecoder error (${decoderKey}):`, e), - }); - - // Configure based on stream type - // Note: Decoders are usually more flexible, but giving a hint helps. - // Screen share uses High Profile, Video uses Baseline. - const config: VideoDecoderConfig = streamType === 'screen' - ? { codec: 'avc1.64002a', optimizeForLatency: false } - : { codec: 'avc1.42001f', optimizeForLatency: true }; - - decoder.configure(config); - this.videoDecoders.set(decoderKey, decoder); - console.log(`[MediaEngine] Created decoder for ${decoderKey} with codec ${config.codec}`); - } - - if (decoder.state === 'configured') { - const chunk = new EncodedVideoChunk({ - type: isKeyFrame ? 'key' : 'delta', - timestamp: timestamp, - data: chunkData, - }); - try { - decoder.decode(chunk); - } catch (e) { - console.error(`[MediaEngine] Decode error ${decoderKey}:`, e); - } - } - } - - // --- Audio --- - - // --- Audio (PCM Fallback) --- - - encodeAudioData(data: AudioData) { - if (this.audioEncoder && this.audioEncoder.state === 'configured') { - this.audioEncoder.encode(data); - data.close(); - } else { - data.close(); - // console.warn('[MediaEngine] AudioEncoder not ready'); - } - } - - decodeAudioChunk(chunkData: Uint8Array, userId: number, timestamp: number) { - const decoderKey = `${userId}-audio`; - let decoder = this.audioDecoders.get(decoderKey); - - if (!decoder) { - decoder = new AudioDecoder({ - output: (data) => { - this.emit('decoded-audio', { userId, data }); - }, - error: (e) => console.error(`[MediaEngine] AudioDecoder error (${userId}):`, e) - }); - decoder.configure({ - codec: 'opus', - sampleRate: 48000, - numberOfChannels: 1 - }); - this.audioDecoders.set(decoderKey, decoder); - console.log(`[MediaEngine] Created AudioDecoder for ${userId}`); - } - - if (decoder.state === 'configured') { - const chunk = new EncodedAudioChunk({ - type: 'key', // Opus is usually self-contained - timestamp: timestamp, - data: chunkData, - }); - try { - decoder.decode(chunk); - } catch (e) { - console.error(`[MediaEngine] Audio Decode error ${decoderKey}:`, e); - } - } - } - - cleanup() { - if (this.videoEncoder && this.videoEncoder.state !== 'closed') this.videoEncoder.close(); - if (this.screenEncoder && this.screenEncoder.state !== 'closed') this.screenEncoder.close(); - if (this.audioEncoder && this.audioEncoder.state !== 'closed') this.audioEncoder.close(); - - this.videoDecoders.forEach(d => { - if (d.state !== 'closed') d.close(); - }); - this.audioDecoders.forEach(d => { - if (d.state !== 'closed') d.close(); - }); - - this.videoDecoders.clear(); - this.audioDecoders.clear(); - } -} diff --git a/status.txt b/status.txt deleted file mode 100644 index cfe5d75..0000000 --- a/status.txt +++ /dev/null @@ -1,24 +0,0 @@ -On branch master -Your branch is up to date with 'origin/master'. - -Changes not staged for commit: - (use "git add ..." to update what will be committed) - (use "git restore ..." to discard changes in working directory) - modified: package.json - modified: src/main/index.ts - modified: src/main/network.ts - modified: src/renderer/index.html - modified: src/renderer/src/App.tsx - modified: src/renderer/src/components/Stage.tsx - modified: src/renderer/src/components/VideoTile.tsx - modified: src/renderer/src/main.tsx - -Untracked files: - (use "git add ..." to include in what will be committed) - current_head_media.ts - previous_head_media.ts - src/renderer/src/utils/ - status.txt - -no changes added to commit (use "git add" and/or "git commit -a") ---- SERVER STATUS ---