diff --git a/web/backend/src/MediaPlayer.ts b/web/backend/src/MediaPlayer.ts index e899ad6..34216e2 100644 --- a/web/backend/src/MediaPlayer.ts +++ b/web/backend/src/MediaPlayer.ts @@ -375,29 +375,27 @@ export class MediaPlayer { siteName: "Screen Sharing", }); - // Special options for mpv to better handle screen sharing (AI recommended...) + // Start from mpv's maintained low-latency profile, then keep only a + // couple of frames of demuxed data ready for smooth 60 fps playback. await this.loadFile(url, "replace", false, [ - "demuxer-lavf-o=fflags=+nobuffer+discardcorrupt", // Reduce buffering and discard corrupt frames - "demuxer-lavf-o=analyzeduration=100000", // Reduce analyze duration - "demuxer-lavf-o=probesize=1000000", // Reduce probe size - "untimed=yes", // Ignore timing info - "cache=no", // Disable cache - "force-seekable=yes", // Force seekable - "no-cache=yes", // Disable cache - "demuxer-max-bytes=500K", // Limit demuxer buffer - "demuxer-readahead-secs=0.1", // Reduce readahead - "hr-seek=no", // Disable high-res seeking - "video-sync=display-resample", // Better sync mode - "video-latency-hacks=yes", // Enable latency hacks - "audio-sync=yes", // Enable audio sync - "audio-buffer=0.1", // Reduce audio buffer - "audio-channels=stereo", // Force stereo audio - "audio-samplerate=44100", // Match sample rate - "audio-format=s16", // Use 16-bit audio + "profile=low-latency", + "cache=no", + "force-seekable=no", + "demuxer-readahead-secs=0.05", + "demuxer-max-bytes=256K", + "demuxer-mkv-probe-start-time=no", + "vd-lavc-threads=2", + "vd-lavc-fast=yes", + "audio-buffer=0", + "pulse-buffer=20", + "pipewire-buffer=20", + "alsa-buffer-time=20000", + "initial-audio-sync=no", ]); - // Make sure it's playing - setTimeout(() => this.play(), 100); + // loadfile normally unpauses, but make that explicit without adding the + // previous fixed 100 ms startup delay. + await this.play(); } public async play() { @@ -475,13 +473,18 @@ export class MediaPlayer { private async loadFile(url: string, mode: string, fetchMetadata: boolean = true, options: string[] = []) { this.lastLoadCandidate = url; this.playbackErrors.delete(url); - this.modify(UserEvent.PlaylistUpdate, () => this.writeCommand("loadfile", [url, mode, "-1", options.join(',')])); + const loadPromise = this.modify( + UserEvent.PlaylistUpdate, + () => this.writeCommand("loadfile", [url, mode, "-1", options.join(',')]), + ); if (fetchMetadata && this.shouldFetchLinkPreview(url)) { this.fetchMetadataAndNotify(url).catch(error => { console.warn(`Failed to fetch metadata for ${this.loggableUrl(url)}:`, error); }); } + + await loadPromise; } private async modify(event: UserEvent, func: () => Promise): Promise { diff --git a/web/backend/src/server.ts b/web/backend/src/server.ts index 2d456c9..ab07920 100644 --- a/web/backend/src/server.ts +++ b/web/backend/src/server.ts @@ -26,6 +26,7 @@ import { searchInvidious, fetchThumbnail } from "./InvidiousAPI"; import { PlaylistItem } from './types'; import { PassThrough } from "stream"; import { AddressInfo } from "net"; +import { WebSocket } from "ws"; const app = express(); app.use(express.json()); @@ -43,6 +44,21 @@ const mediaPlayer = new MediaPlayer(); // Create a shared stream that both endpoints can access let activeScreenshareStream: PassThrough | null = null; let activeScreenshareMimeType: string | null = null; +let activeScreenshareSocket: WebSocket | null = null; +const screenshareStreamsClosedByViewer = new WeakSet(); + +const SCREENSHARE_STREAM_HIGH_WATER_MARK = 32 * 1024; + +const getScreenshareContentType = (mimeType: unknown): string => { + if (typeof mimeType !== "string") { + return "application/octet-stream"; + } + + const contentType = mimeType.split(";", 1)[0].trim().toLowerCase(); + return contentType === "video/webm" || contentType === "video/mp4" + ? contentType + : "application/octet-stream"; +}; const withErrorHandling = (func: (req: any, res: any) => Promise) => { return async (req: any, res: any) => { @@ -158,75 +174,104 @@ apiRouter.ws("/events", (ws, req) => { // using the fact that QueueCube/MPV is based all around streaming URLs, so the active // screenshare stream manifests as just another URL to play. apiRouter.get("/screenshareStream", withErrorHandling(async (req, res) => { - res.setHeader("Content-Type", activeScreenshareMimeType || "video/mp4"); + const screenshareStream = activeScreenshareStream; + if (!screenshareStream) { + res.status(503).send("No active screen sharing session"); + return; + } + + req.socket.setNoDelay(true); + res.setHeader("Content-Type", activeScreenshareMimeType || "application/octet-stream"); res.setHeader("Cache-Control", "no-cache, no-store, must-revalidate"); res.setHeader("Pragma", "no-cache"); res.setHeader("Expires", "0"); res.setHeader("Connection", "keep-alive"); res.setHeader("Transfer-Encoding", "chunked"); + res.setHeader("Accept-Ranges", "none"); + res.flushHeaders(); - if (!activeScreenshareStream) { - res.status(503).send("No active screen sharing session"); - return; - } - - // Handle client disconnection - req.on('close', () => { + res.on('close', () => { + screenshareStream.unpipe(res); console.log("Screenshare viewer disconnected"); + + // If mpv goes away unexpectedly, stop capture instead of leaving an + // unconsumed stream to build backpressure forever. + if (activeScreenshareStream === screenshareStream && !screenshareStream.writableEnded) { + screenshareStreamsClosedByViewer.add(screenshareStream); + activeScreenshareSocket?.close(1011, "Screen sharing viewer disconnected"); + } }); - // Configure stream for low latency - activeScreenshareStream.setMaxListeners(0); - - // Pipe with immediate flush - activeScreenshareStream.pipe(res, { end: false }); + screenshareStream.pipe(res); })); apiRouter.ws("/screenshare", (ws, req) => { - const mimeType = req.query.mimeType as string; + const mimeType = getScreenshareContentType(req.query.mimeType); console.log("Screen sharing client connected with mimeType: " + mimeType); - ws.binaryType = "arraybuffer"; + req.socket.setNoDelay(true); - let firstChunk = false; + // Supersede an old sender without allowing its close handler to tear down + // the new session. + const previousSocket = activeScreenshareSocket; + const previousStream = activeScreenshareStream; + activeScreenshareSocket = ws; + activeScreenshareMimeType = mimeType; + activeScreenshareStream = new PassThrough({ + highWaterMark: SCREENSHARE_STREAM_HIGH_WATER_MARK, + allowHalfOpen: false, + }); + const screenshareStream = activeScreenshareStream; + let waitingForDrain = false; - // Configure WebSocket for low latency - ws.setMaxListeners(0); - ws.binaryType = "arraybuffer"; + previousStream?.end(); + if (previousSocket && previousSocket.readyState === WebSocket.OPEN) { + previousSocket.close(1000, "A newer screen sharing session started"); + } + + // Start mpv while the browser starts its recorder, instead of waiting for a + // whole encoded chunk before beginning the HTTP connection and demux setup. + const port = (server.address() as AddressInfo).port; + const url = `http://127.0.0.1:${port}/api/screenshareStream`; + console.log(`Starting screen share stream at ${url}`); + mediaPlayer.initiateScreenSharing(url).catch((error: unknown) => { + console.error("Failed to start screen sharing:", error); + if (activeScreenshareSocket === ws) { + ws.close(1011, "Unable to start the media player"); + } + }); ws.on('message', (data: any) => { const buffer = data instanceof Buffer ? data : Buffer.from(data); - - if (!firstChunk) { - firstChunk = true; - const port = (server.address() as AddressInfo).port; - const url = `http://localhost:${port}/api/screenshareStream`; - console.log(`Starting screen share stream at ${url}`); - - // Create new shared stream with immediate flush - activeScreenshareStream = new PassThrough({ - highWaterMark: 1024 * 1024, // 1MB buffer - allowHalfOpen: false - }); - - activeScreenshareStream.write(buffer); - mediaPlayer.initiateScreenSharing(url); - } else if (activeScreenshareStream) { - // Write with immediate flush - activeScreenshareStream.write(buffer, () => { - activeScreenshareStream?.cork(); - activeScreenshareStream?.uncork(); + if (activeScreenshareStream !== screenshareStream) { + return; + } + + if (!screenshareStream.write(buffer) && !waitingForDrain) { + waitingForDrain = true; + ws.pause(); + screenshareStream.once('drain', () => { + waitingForDrain = false; + if (activeScreenshareStream === screenshareStream && ws.readyState === WebSocket.OPEN) { + ws.resume(); + } }); } }); ws.on('close', () => { console.log("Screen sharing client disconnected"); - if (activeScreenshareStream) { - activeScreenshareStream.end(); + if (activeScreenshareStream === screenshareStream) { + screenshareStream.end(); activeScreenshareStream = null; + activeScreenshareMimeType = null; + activeScreenshareSocket = null; + if (!screenshareStreamsClosedByViewer.has(screenshareStream)) { + mediaPlayer.stop().catch((error: unknown) => { + console.error("Failed to stop screen sharing:", error); + }); + } } - mediaPlayer.stop(); }); }); diff --git a/web/frontend/src/api/player.tsx b/web/frontend/src/api/player.tsx index 7740df9..2328967 100644 --- a/web/frontend/src/api/player.tsx +++ b/web/frontend/src/api/player.tsx @@ -203,7 +203,9 @@ export const API = { startScreenShare(mimeType: string): WebSocket { const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws'; - const ws = new WebSocket(`${protocol}://${window.location.host}/api/screenshare?mimeType=${mimeType}`); + const ws = new WebSocket( + `${protocol}://${window.location.host}/api/screenshare?mimeType=${encodeURIComponent(mimeType)}`, + ); return ws; } }; diff --git a/web/frontend/src/hooks/useScreenShare.ts b/web/frontend/src/hooks/useScreenShare.ts index f45106c..cb02817 100644 --- a/web/frontend/src/hooks/useScreenShare.ts +++ b/web/frontend/src/hooks/useScreenShare.ts @@ -1,6 +1,17 @@ import { useState, useEffect, useRef, useCallback } from 'react'; import { API } from '../api/player'; +const TARGET_FRAME_RATE = 60; +const MAX_CAPTURE_WIDTH = 1920; +const MAX_CAPTURE_HEIGHT = 1080; +const RECORDER_TIMESLICE_MS = 16; +const MIN_VIDEO_BIT_RATE = 2_500_000; +const MAX_VIDEO_BIT_RATE = 8_000_000; +const VIDEO_BITS_PER_PIXEL = 0.065; +const SOCKET_BUFFER_TARGET_MS = 75; +const MIN_SOCKET_BUFFER_BYTES = 32 * 1024; +const MIN_RESUME_BUFFER_BYTES = 8 * 1024; + interface UseScreenShareResult { isScreenSharing: boolean; isScreenSharingSupported: boolean; @@ -8,15 +19,21 @@ interface UseScreenShareResult { stopScreenShare: () => void; } -function getBestSupportedMimeType() { - // Ordered by preference (best first) - all of these include audio+video - const mimeTypes = [ - 'video/webm;codecs=vp9,opus', // Best quality, good compression - 'video/webm;codecs=vp8,opus', // Good fallback, well supported - 'video/webm;codecs=h264,opus', // Better compatibility with some systems - 'video/mp4;codecs=h264,aac', // Good for Safari but may not be supported for MediaRecorder - 'video/webm', // Generic fallback (browser will choose codecs) - 'video/mp4' // Last resort +function getBestSupportedMimeType(hasAudio: boolean): string | undefined { + // VP8 is substantially cheaper to encode than VP9 on most systems. Prefer it + // for real-time capture, then use hardware-friendly H.264 where available. + const mimeTypes = hasAudio ? [ + 'video/webm;codecs=vp8,opus', + 'video/mp4;codecs=avc1.42E01E,mp4a.40.2', + 'video/webm;codecs=vp9,opus', + 'video/webm', + 'video/mp4', + ] : [ + 'video/webm;codecs=vp8', + 'video/mp4;codecs=avc1.42E01E', + 'video/webm;codecs=vp9', + 'video/webm', + 'video/mp4', ]; // Find the first supported mimetype @@ -27,15 +44,42 @@ function getBestSupportedMimeType() { } } - // If none are supported, return null or a basic fallback console.warn('No preferred mime types supported by this browser'); - return 'video/webm'; // Most basic fallback + return undefined; +} + +function getTargetVideoBitRate(track: MediaStreamTrack): number { + const settings = track.getSettings(); + const width = settings.width ?? MAX_CAPTURE_WIDTH; + const height = settings.height ?? MAX_CAPTURE_HEIGHT; + const frameRate = Math.min(settings.frameRate ?? TARGET_FRAME_RATE, TARGET_FRAME_RATE); + const target = width * height * frameRate * VIDEO_BITS_PER_PIXEL; + + return Math.round(Math.min(MAX_VIDEO_BIT_RATE, Math.max(MIN_VIDEO_BIT_RATE, target))); +} + +function getSocketBufferLimits(recorder: MediaRecorder) { + const bytesPerSecond = (recorder.videoBitsPerSecond + recorder.audioBitsPerSecond) / 8; + const maximum = Math.max( + MIN_SOCKET_BUFFER_BYTES, + Math.ceil(bytesPerSecond * SOCKET_BUFFER_TARGET_MS / 1000), + ); + + return { + maximum, + resumeAt: Math.max(MIN_RESUME_BUFFER_BYTES, Math.floor(maximum / 4)), + }; } export const useScreenShare = (): UseScreenShareResult => { const [isScreenSharing, setIsScreenSharing] = useState(false); const [isScreenSharingSupported, setIsScreenSharingSupported] = useState(false); const screenShareSocketRef = useRef(null); + const mediaRecorderRef = useRef(null); + const mediaStreamRef = useRef(null); + const backpressureTimerRef = useRef(null); + const isStartingRef = useRef(false); + const sessionGenerationRef = useRef(0); // Check if screen sharing is supported useEffect(() => { @@ -47,80 +91,205 @@ export const useScreenShare = (): UseScreenShareResult => { }, []); const stopScreenShare = useCallback(() => { - if (screenShareSocketRef.current) { - screenShareSocketRef.current.close(); - screenShareSocketRef.current = null; + sessionGenerationRef.current += 1; + isStartingRef.current = false; + + if (backpressureTimerRef.current !== null) { + window.clearInterval(backpressureTimerRef.current); + backpressureTimerRef.current = null; } + + const recorder = mediaRecorderRef.current; + mediaRecorderRef.current = null; + if (recorder) { + recorder.ondataavailable = null; + recorder.onerror = null; + if (recorder.state !== 'inactive') { + recorder.stop(); + } + } + + const mediaStream = mediaStreamRef.current; + mediaStreamRef.current = null; + mediaStream?.getTracks().forEach((track) => { + track.onended = null; + track.stop(); + }); + + const socket = screenShareSocketRef.current; + screenShareSocketRef.current = null; + if (socket) { + socket.onopen = null; + socket.onclose = null; + socket.onerror = null; + if (socket.readyState === WebSocket.CONNECTING || socket.readyState === WebSocket.OPEN) { + socket.close(); + } + } + setIsScreenSharing(false); }, []); const startScreenShare = useCallback(async () => { + if (isStartingRef.current || screenShareSocketRef.current) { + return; + } + + isStartingRef.current = true; + const sessionGeneration = ++sessionGenerationRef.current; + try { const mediaStream = await navigator.mediaDevices.getDisplayMedia({ - video: true, + video: { + frameRate: { ideal: TARGET_FRAME_RATE, max: TARGET_FRAME_RATE }, + width: { max: MAX_CAPTURE_WIDTH }, + height: { max: MAX_CAPTURE_HEIGHT }, + }, audio: true, }); - let mimeType = getBestSupportedMimeType(); - console.log('Using MIME type:', mimeType); + // getDisplayMedia cannot be programmatically dismissed. If the user + // stopped or restarted while its picker was open, discard this result. + if (sessionGenerationRef.current !== sessionGeneration) { + mediaStream.getTracks().forEach((track) => track.stop()); + return; + } - const mediaRecorder = new MediaRecorder(mediaStream, { - mimeType: mimeType, - videoBitsPerSecond: 2500000, // 2.5 Mbps - audioBitsPerSecond: 128000, // 128 kbps + mediaStreamRef.current = mediaStream; + + const videoTrack = mediaStream.getVideoTracks()[0]; + if (!videoTrack) { + throw new Error('The selected source did not provide a video track'); + } + + // Favor smooth motion over static-text quality when the browser needs to + // trade capture resolution for frame rate. + videoTrack.contentHint = 'motion'; + mediaStream.getAudioTracks().forEach((track) => { + track.contentHint = 'music'; }); + const hasAudio = mediaStream.getAudioTracks().length > 0; + const preferredMimeType = getBestSupportedMimeType(hasAudio); + const recorderOptions: MediaRecorderOptions = { + videoBitsPerSecond: getTargetVideoBitRate(videoTrack), + }; + + if (preferredMimeType) { + recorderOptions.mimeType = preferredMimeType; + } + if (hasAudio) { + recorderOptions.audioBitsPerSecond = 128_000; + } + + const mediaRecorder = new MediaRecorder(mediaStream, recorderOptions); + mediaRecorderRef.current = mediaRecorder; + const mimeType = mediaRecorder.mimeType || preferredMimeType || 'application/octet-stream'; + const socketBufferLimits = getSocketBufferLimits(mediaRecorder); + console.log( + `Screen sharing at up to ${TARGET_FRAME_RATE} fps, ${Math.round(mediaRecorder.videoBitsPerSecond / 1000)} kbps, using ${mimeType}`, + ); + // Connect to WebSocket - screenShareSocketRef.current = API.startScreenShare(mimeType); + const socket = API.startScreenShare(mimeType); + screenShareSocketRef.current = socket; + isStartingRef.current = false; + + const resumeRecorderWhenDrained = () => { + if (screenShareSocketRef.current !== socket || mediaRecorderRef.current !== mediaRecorder) { + if (backpressureTimerRef.current !== null) { + window.clearInterval(backpressureTimerRef.current); + backpressureTimerRef.current = null; + } + return; + } + + if (socket.bufferedAmount <= socketBufferLimits.resumeAt && mediaRecorder.state === 'paused') { + mediaRecorder.resume(); + if (backpressureTimerRef.current !== null) { + window.clearInterval(backpressureTimerRef.current); + backpressureTimerRef.current = null; + } + } + }; + + const pauseRecorderForBackpressure = () => { + if (mediaRecorder.state === 'recording') { + mediaRecorder.pause(); + } + + if (backpressureTimerRef.current === null) { + backpressureTimerRef.current = window.setInterval( + resumeRecorderWhenDrained, + RECORDER_TIMESLICE_MS, + ); + } + }; // Set up WebSocket event handlers - screenShareSocketRef.current.onopen = () => { + socket.onopen = () => { + if (screenShareSocketRef.current !== socket) { + socket.close(); + return; + } + console.log('Screen sharing WebSocket connected'); setIsScreenSharing(true); - - mediaRecorder.start(100); + mediaRecorder.start(RECORDER_TIMESLICE_MS); }; - screenShareSocketRef.current.onclose = () => { + socket.onclose = () => { console.log('Screen sharing WebSocket closed'); - setIsScreenSharing(false); - - // Stop all tracks when WebSocket is closed - mediaStream.getTracks().forEach(track => track.stop()); + if (screenShareSocketRef.current === socket) { + stopScreenShare(); + } }; - screenShareSocketRef.current.onerror = (error) => { + socket.onerror = (error) => { console.error('Screen sharing WebSocket error:', error); - setIsScreenSharing(false); - - // Stop all tracks on error - mediaStream.getTracks().forEach(track => track.stop()); + if (screenShareSocketRef.current === socket) { + stopScreenShare(); + } + }; + + mediaRecorder.onerror = (error) => { + console.error('Screen sharing recorder error:', error); + if (mediaRecorderRef.current === mediaRecorder) { + stopScreenShare(); + } }; // Send data over WebSocket when available mediaRecorder.ondataavailable = (event) => { - if (event.data && event.data.size > 0 && screenShareSocketRef.current && screenShareSocketRef.current.readyState === WebSocket.OPEN) { - screenShareSocketRef.current.send(event.data); + if (event.data.size > 0 && screenShareSocketRef.current === socket && socket.readyState === WebSocket.OPEN) { + socket.send(event.data); + + // Browser WebSockets otherwise queue without a limit, turning a brief + // network slowdown into permanently stale playback. Pausing capture + // bounds that queue and resumes at the live edge once it drains. + if (socket.bufferedAmount >= socketBufferLimits.maximum) { + pauseRecorderForBackpressure(); + } } }; // Handle stream ending (user clicks "Stop sharing") - mediaStream.getVideoTracks()[0].onended = () => { - if (screenShareSocketRef.current) { - screenShareSocketRef.current.close(); - screenShareSocketRef.current = null; - } - setIsScreenSharing(false); - }; + videoTrack.onended = stopScreenShare; } catch (error) { - console.error('Error starting screen share:', error); - setIsScreenSharing(false); + if (sessionGenerationRef.current === sessionGeneration) { + console.error('Error starting screen share:', error); + stopScreenShare(); + } } - }, []); + }, [stopScreenShare]); + + useEffect(() => { + return () => stopScreenShare(); + }, [stopScreenShare]); const toggleScreenShare = useCallback(async () => { - if (screenShareSocketRef.current) { + if (screenShareSocketRef.current || isStartingRef.current) { stopScreenShare(); } else { await startScreenShare(); @@ -133,4 +302,4 @@ export const useScreenShare = (): UseScreenShareResult => { toggleScreenShare, stopScreenShare }; -}; \ No newline at end of file +};