Optimize screen sharing latency and frame rate

web/backend/src/MediaPlayer.ts

@@ -375,29 +375,27 @@ export class MediaPlayer {
             siteName: "Screen Sharing",
         });
 
-        // Special options for mpv to better handle screen sharing (AI recommended...)
+        // Start from mpv's maintained low-latency profile, then keep only a
+        // couple of frames of demuxed data ready for smooth 60 fps playback.
         await this.loadFile(url, "replace", false, [
-            "demuxer-lavf-o=fflags=+nobuffer+discardcorrupt",  // Reduce buffering and discard corrupt frames
+            "profile=low-latency",
-            "demuxer-lavf-o=analyzeduration=100000",           // Reduce analyze duration
+            "cache=no",
-            "demuxer-lavf-o=probesize=1000000",                // Reduce probe size
+            "force-seekable=no",
-            "untimed=yes",                                     // Ignore timing info
+            "demuxer-readahead-secs=0.05",
-            "cache=no",                                        // Disable cache
+            "demuxer-max-bytes=256K",
-            "force-seekable=yes",                              // Force seekable
+            "demuxer-mkv-probe-start-time=no",
-            "no-cache=yes",                                    // Disable cache
+            "vd-lavc-threads=2",
-            "demuxer-max-bytes=500K",                          // Limit demuxer buffer
+            "vd-lavc-fast=yes",
-            "demuxer-readahead-secs=0.1",                      // Reduce readahead
+            "audio-buffer=0",
-            "hr-seek=no",                                      // Disable high-res seeking
+            "pulse-buffer=20",
-            "video-sync=display-resample",                     // Better sync mode
+            "pipewire-buffer=20",
-            "video-latency-hacks=yes",                         // Enable latency hacks
+            "alsa-buffer-time=20000",
-            "audio-sync=yes",                                  // Enable audio sync
+            "initial-audio-sync=no",
-            "audio-buffer=0.1",                                // Reduce audio buffer
-            "audio-channels=stereo",                           // Force stereo audio
-            "audio-samplerate=44100",                          // Match sample rate
-            "audio-format=s16",                                // Use 16-bit audio
         ]);
 
-        // Make sure it's playing
+        // loadfile normally unpauses, but make that explicit without adding the
-        setTimeout(() => this.play(), 100);
+        // previous fixed 100 ms startup delay.
+        await this.play();
     }
 
     public async play() {
@@ -475,13 +473,18 @@ export class MediaPlayer {
     private async loadFile(url: string, mode: string, fetchMetadata: boolean = true, options: string[] = []) {
         this.lastLoadCandidate = url;
         this.playbackErrors.delete(url);
-        this.modify(UserEvent.PlaylistUpdate, () => this.writeCommand("loadfile", [url, mode, "-1", options.join(',')]));
+        const loadPromise = this.modify(
+            UserEvent.PlaylistUpdate,
+            () => this.writeCommand("loadfile", [url, mode, "-1", options.join(',')]),
+        );
 
         if (fetchMetadata && this.shouldFetchLinkPreview(url)) {
             this.fetchMetadataAndNotify(url).catch(error => {
                 console.warn(`Failed to fetch metadata for ${this.loggableUrl(url)}:`, error);
             });
         }
+
+        await loadPromise;
     }
 
     private async modify<T>(event: UserEvent, func: () => Promise<T>): Promise<T> {

web/backend/src/server.ts

@@ -26,6 +26,7 @@ import { searchInvidious, fetchThumbnail } from "./InvidiousAPI";
 import { PlaylistItem } from './types';
 import { PassThrough } from "stream";
 import { AddressInfo } from "net";
+import { WebSocket } from "ws";
 
 const app = express();
 app.use(express.json());
@@ -43,6 +44,21 @@ const mediaPlayer = new MediaPlayer();
 // Create a shared stream that both endpoints can access
 let activeScreenshareStream: PassThrough | null = null;
 let activeScreenshareMimeType: string | null = null;
+let activeScreenshareSocket: WebSocket | null = null;
+const screenshareStreamsClosedByViewer = new WeakSet<PassThrough>();
+
+const SCREENSHARE_STREAM_HIGH_WATER_MARK = 32 * 1024;
+
+const getScreenshareContentType = (mimeType: unknown): string => {
+    if (typeof mimeType !== "string") {
+        return "application/octet-stream";
+    }
+
+    const contentType = mimeType.split(";", 1)[0].trim().toLowerCase();
+    return contentType === "video/webm" || contentType === "video/mp4"
+        ? contentType
+        : "application/octet-stream";
+};
 
 const withErrorHandling = (func: (req: any, res: any) => Promise<any>) => {
     return async (req: any, res: any) => {
@@ -158,75 +174,104 @@ apiRouter.ws("/events", (ws, req) => {
 // using the fact that QueueCube/MPV is based all around streaming URLs, so the active 
 // screenshare stream manifests as just another URL to play. 
 apiRouter.get("/screenshareStream", withErrorHandling(async (req, res) => {
-    res.setHeader("Content-Type", activeScreenshareMimeType || "video/mp4");
+    const screenshareStream = activeScreenshareStream;
+    if (!screenshareStream) {
+        res.status(503).send("No active screen sharing session");
+        return;
+    }
+
+    req.socket.setNoDelay(true);
+    res.setHeader("Content-Type", activeScreenshareMimeType || "application/octet-stream");
     res.setHeader("Cache-Control", "no-cache, no-store, must-revalidate");
     res.setHeader("Pragma", "no-cache");
     res.setHeader("Expires", "0");
     res.setHeader("Connection", "keep-alive");
     res.setHeader("Transfer-Encoding", "chunked");
+    res.setHeader("Accept-Ranges", "none");
+    res.flushHeaders();
 
-    if (!activeScreenshareStream) {
+    res.on('close', () => {
-        res.status(503).send("No active screen sharing session");
+        screenshareStream.unpipe(res);
-        return;
-    }
-
-    // Handle client disconnection
-    req.on('close', () => {
         console.log("Screenshare viewer disconnected");
+
+        // If mpv goes away unexpectedly, stop capture instead of leaving an
+        // unconsumed stream to build backpressure forever.
+        if (activeScreenshareStream === screenshareStream && !screenshareStream.writableEnded) {
+            screenshareStreamsClosedByViewer.add(screenshareStream);
+            activeScreenshareSocket?.close(1011, "Screen sharing viewer disconnected");
+        }
     });
 
-    // Configure stream for low latency
+    screenshareStream.pipe(res);
-    activeScreenshareStream.setMaxListeners(0);
-    
-    // Pipe with immediate flush
-    activeScreenshareStream.pipe(res, { end: false });
 }));
 
 apiRouter.ws("/screenshare", (ws, req) => {
-    const mimeType = req.query.mimeType as string;
+    const mimeType = getScreenshareContentType(req.query.mimeType);
     console.log("Screen sharing client connected with mimeType: " + mimeType);
-    ws.binaryType = "arraybuffer";
+    req.socket.setNoDelay(true);
 
-    let firstChunk = false;
+    // Supersede an old sender without allowing its close handler to tear down
+    // the new session.
+    const previousSocket = activeScreenshareSocket;
+    const previousStream = activeScreenshareStream;
+    activeScreenshareSocket = ws;
+    activeScreenshareMimeType = mimeType;
+    activeScreenshareStream = new PassThrough({
+        highWaterMark: SCREENSHARE_STREAM_HIGH_WATER_MARK,
+        allowHalfOpen: false,
+    });
+    const screenshareStream = activeScreenshareStream;
+    let waitingForDrain = false;
 
-    // Configure WebSocket for low latency
+    previousStream?.end();
-    ws.setMaxListeners(0);
+    if (previousSocket && previousSocket.readyState === WebSocket.OPEN) {
-    ws.binaryType = "arraybuffer";
+        previousSocket.close(1000, "A newer screen sharing session started");
+    }
+
+    // Start mpv while the browser starts its recorder, instead of waiting for a
+    // whole encoded chunk before beginning the HTTP connection and demux setup.
+    const port = (server.address() as AddressInfo).port;
+    const url = `http://127.0.0.1:${port}/api/screenshareStream`;
+    console.log(`Starting screen share stream at ${url}`);
+    mediaPlayer.initiateScreenSharing(url).catch((error: unknown) => {
+        console.error("Failed to start screen sharing:", error);
+        if (activeScreenshareSocket === ws) {
+            ws.close(1011, "Unable to start the media player");
+        }
+    });
 
     ws.on('message', (data: any) => {
         const buffer = data instanceof Buffer ? data : Buffer.from(data);
 
-        if (!firstChunk) {
+        if (activeScreenshareStream !== screenshareStream) {
-            firstChunk = true;
+            return;
+        }
 
-            const port = (server.address() as AddressInfo).port;
+        if (!screenshareStream.write(buffer) && !waitingForDrain) {
-            const url = `http://localhost:${port}/api/screenshareStream`;
+            waitingForDrain = true;
-            console.log(`Starting screen share stream at ${url}`);
+            ws.pause();
-            
+            screenshareStream.once('drain', () => {
-            // Create new shared stream with immediate flush
+                waitingForDrain = false;
-            activeScreenshareStream = new PassThrough({
+                if (activeScreenshareStream === screenshareStream && ws.readyState === WebSocket.OPEN) {
-                highWaterMark: 1024 * 1024, // 1MB buffer
+                    ws.resume();
-                allowHalfOpen: false
+                }
-            });
-            
-            activeScreenshareStream.write(buffer);
-            mediaPlayer.initiateScreenSharing(url);
-        } else if (activeScreenshareStream) {
-            // Write with immediate flush
-            activeScreenshareStream.write(buffer, () => {
-                activeScreenshareStream?.cork();
-                activeScreenshareStream?.uncork();
             });
         }
     });
 
     ws.on('close', () => {
         console.log("Screen sharing client disconnected");
-        if (activeScreenshareStream) {
+        if (activeScreenshareStream === screenshareStream) {
-            activeScreenshareStream.end();
+            screenshareStream.end();
             activeScreenshareStream = null;
+            activeScreenshareMimeType = null;
+            activeScreenshareSocket = null;
+            if (!screenshareStreamsClosedByViewer.has(screenshareStream)) {
+                mediaPlayer.stop().catch((error: unknown) => {
+                    console.error("Failed to stop screen sharing:", error);
+                });
+            }
         }
-        mediaPlayer.stop();
     });
 });
 

web/frontend/src/api/player.tsx

@@ -203,7 +203,9 @@ export const API = {
 
   startScreenShare(mimeType: string): WebSocket {
     const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
-    const ws = new WebSocket(`${protocol}://${window.location.host}/api/screenshare?mimeType=${mimeType}`);
+    const ws = new WebSocket(
+      `${protocol}://${window.location.host}/api/screenshare?mimeType=${encodeURIComponent(mimeType)}`,
+    );
     return ws;
   }
 };

web/frontend/src/hooks/useScreenShare.ts

@@ -1,6 +1,17 @@
 import { useState, useEffect, useRef, useCallback } from 'react';
 import { API } from '../api/player';
 
+const TARGET_FRAME_RATE = 60;
+const MAX_CAPTURE_WIDTH = 1920;
+const MAX_CAPTURE_HEIGHT = 1080;
+const RECORDER_TIMESLICE_MS = 16;
+const MIN_VIDEO_BIT_RATE = 2_500_000;
+const MAX_VIDEO_BIT_RATE = 8_000_000;
+const VIDEO_BITS_PER_PIXEL = 0.065;
+const SOCKET_BUFFER_TARGET_MS = 75;
+const MIN_SOCKET_BUFFER_BYTES = 32 * 1024;
+const MIN_RESUME_BUFFER_BYTES = 8 * 1024;
+
 interface UseScreenShareResult {
   isScreenSharing: boolean;
   isScreenSharingSupported: boolean;
@@ -8,15 +19,21 @@ interface UseScreenShareResult {
   stopScreenShare: () => void;
 }
 
-function getBestSupportedMimeType() {
+function getBestSupportedMimeType(hasAudio: boolean): string | undefined {
-  // Ordered by preference (best first) - all of these include audio+video
+  // VP8 is substantially cheaper to encode than VP9 on most systems. Prefer it
-  const mimeTypes = [
+  // for real-time capture, then use hardware-friendly H.264 where available.
-    'video/webm;codecs=vp9,opus',     // Best quality, good compression
+  const mimeTypes = hasAudio ? [
-    'video/webm;codecs=vp8,opus',     // Good fallback, well supported
+    'video/webm;codecs=vp8,opus',
-    'video/webm;codecs=h264,opus',    // Better compatibility with some systems
+    'video/mp4;codecs=avc1.42E01E,mp4a.40.2',
-    'video/mp4;codecs=h264,aac',      // Good for Safari but may not be supported for MediaRecorder
+    'video/webm;codecs=vp9,opus',
-    'video/webm',                     // Generic fallback (browser will choose codecs)
+    'video/webm',
-    'video/mp4'                       // Last resort
+    'video/mp4',
+  ] : [
+    'video/webm;codecs=vp8',
+    'video/mp4;codecs=avc1.42E01E',
+    'video/webm;codecs=vp9',
+    'video/webm',
+    'video/mp4',
   ];
 
   // Find the first supported mimetype
@@ -27,15 +44,42 @@ function getBestSupportedMimeType() {
     }
   }
 
-  // If none are supported, return null or a basic fallback
   console.warn('No preferred mime types supported by this browser');
-  return 'video/webm'; // Most basic fallback
+  return undefined;
+}
+
+function getTargetVideoBitRate(track: MediaStreamTrack): number {
+  const settings = track.getSettings();
+  const width = settings.width ?? MAX_CAPTURE_WIDTH;
+  const height = settings.height ?? MAX_CAPTURE_HEIGHT;
+  const frameRate = Math.min(settings.frameRate ?? TARGET_FRAME_RATE, TARGET_FRAME_RATE);
+  const target = width * height * frameRate * VIDEO_BITS_PER_PIXEL;
+
+  return Math.round(Math.min(MAX_VIDEO_BIT_RATE, Math.max(MIN_VIDEO_BIT_RATE, target)));
+}
+
+function getSocketBufferLimits(recorder: MediaRecorder) {
+  const bytesPerSecond = (recorder.videoBitsPerSecond + recorder.audioBitsPerSecond) / 8;
+  const maximum = Math.max(
+    MIN_SOCKET_BUFFER_BYTES,
+    Math.ceil(bytesPerSecond * SOCKET_BUFFER_TARGET_MS / 1000),
+  );
+
+  return {
+    maximum,
+    resumeAt: Math.max(MIN_RESUME_BUFFER_BYTES, Math.floor(maximum / 4)),
+  };
 }
 
 export const useScreenShare = (): UseScreenShareResult => {
   const [isScreenSharing, setIsScreenSharing] = useState(false);
   const [isScreenSharingSupported, setIsScreenSharingSupported] = useState(false);
   const screenShareSocketRef = useRef<WebSocket | null>(null);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const mediaStreamRef = useRef<MediaStream | null>(null);
+  const backpressureTimerRef = useRef<number | null>(null);
+  const isStartingRef = useRef(false);
+  const sessionGenerationRef = useRef(0);
 
   // Check if screen sharing is supported
   useEffect(() => {
@@ -47,80 +91,205 @@ export const useScreenShare = (): UseScreenShareResult => {
   }, []);
 
   const stopScreenShare = useCallback(() => {
-    if (screenShareSocketRef.current) {
+    sessionGenerationRef.current += 1;
-      screenShareSocketRef.current.close();
+    isStartingRef.current = false;
-      screenShareSocketRef.current = null;
+
+    if (backpressureTimerRef.current !== null) {
+      window.clearInterval(backpressureTimerRef.current);
+      backpressureTimerRef.current = null;
     }
+
+    const recorder = mediaRecorderRef.current;
+    mediaRecorderRef.current = null;
+    if (recorder) {
+      recorder.ondataavailable = null;
+      recorder.onerror = null;
+      if (recorder.state !== 'inactive') {
+        recorder.stop();
+      }
+    }
+
+    const mediaStream = mediaStreamRef.current;
+    mediaStreamRef.current = null;
+    mediaStream?.getTracks().forEach((track) => {
+      track.onended = null;
+      track.stop();
+    });
+
+    const socket = screenShareSocketRef.current;
+    screenShareSocketRef.current = null;
+    if (socket) {
+      socket.onopen = null;
+      socket.onclose = null;
+      socket.onerror = null;
+      if (socket.readyState === WebSocket.CONNECTING || socket.readyState === WebSocket.OPEN) {
+        socket.close();
+      }
+    }
+
     setIsScreenSharing(false);
   }, []);
 
   const startScreenShare = useCallback(async () => {
+    if (isStartingRef.current || screenShareSocketRef.current) {
+      return;
+    }
+
+    isStartingRef.current = true;
+    const sessionGeneration = ++sessionGenerationRef.current;
+
     try {
       const mediaStream = await navigator.mediaDevices.getDisplayMedia({
-        video: true,
+        video: {
+          frameRate: { ideal: TARGET_FRAME_RATE, max: TARGET_FRAME_RATE },
+          width: { max: MAX_CAPTURE_WIDTH },
+          height: { max: MAX_CAPTURE_HEIGHT },
+        },
         audio: true,
       });
 
-      let mimeType = getBestSupportedMimeType();
+      // getDisplayMedia cannot be programmatically dismissed. If the user
-      console.log('Using MIME type:', mimeType);
+      // stopped or restarted while its picker was open, discard this result.
+      if (sessionGenerationRef.current !== sessionGeneration) {
+        mediaStream.getTracks().forEach((track) => track.stop());
+        return;
+      }
 
-      const mediaRecorder = new MediaRecorder(mediaStream, {
+      mediaStreamRef.current = mediaStream;
-        mimeType: mimeType,
+
-        videoBitsPerSecond: 2500000, // 2.5 Mbps
+      const videoTrack = mediaStream.getVideoTracks()[0];
-        audioBitsPerSecond: 128000,  // 128 kbps
+      if (!videoTrack) {
+        throw new Error('The selected source did not provide a video track');
+      }
+
+      // Favor smooth motion over static-text quality when the browser needs to
+      // trade capture resolution for frame rate.
+      videoTrack.contentHint = 'motion';
+      mediaStream.getAudioTracks().forEach((track) => {
+        track.contentHint = 'music';
       });
 
+      const hasAudio = mediaStream.getAudioTracks().length > 0;
+      const preferredMimeType = getBestSupportedMimeType(hasAudio);
+      const recorderOptions: MediaRecorderOptions = {
+        videoBitsPerSecond: getTargetVideoBitRate(videoTrack),
+      };
+
+      if (preferredMimeType) {
+        recorderOptions.mimeType = preferredMimeType;
+      }
+      if (hasAudio) {
+        recorderOptions.audioBitsPerSecond = 128_000;
+      }
+
+      const mediaRecorder = new MediaRecorder(mediaStream, recorderOptions);
+      mediaRecorderRef.current = mediaRecorder;
+      const mimeType = mediaRecorder.mimeType || preferredMimeType || 'application/octet-stream';
+      const socketBufferLimits = getSocketBufferLimits(mediaRecorder);
+      console.log(
+        `Screen sharing at up to ${TARGET_FRAME_RATE} fps, ${Math.round(mediaRecorder.videoBitsPerSecond / 1000)} kbps, using ${mimeType}`,
+      );
+
       // Connect to WebSocket
-      screenShareSocketRef.current = API.startScreenShare(mimeType);
+      const socket = API.startScreenShare(mimeType);
+      screenShareSocketRef.current = socket;
+      isStartingRef.current = false;
+
+      const resumeRecorderWhenDrained = () => {
+        if (screenShareSocketRef.current !== socket || mediaRecorderRef.current !== mediaRecorder) {
+          if (backpressureTimerRef.current !== null) {
+            window.clearInterval(backpressureTimerRef.current);
+            backpressureTimerRef.current = null;
+          }
+          return;
+        }
+
+        if (socket.bufferedAmount <= socketBufferLimits.resumeAt && mediaRecorder.state === 'paused') {
+          mediaRecorder.resume();
+          if (backpressureTimerRef.current !== null) {
+            window.clearInterval(backpressureTimerRef.current);
+            backpressureTimerRef.current = null;
+          }
+        }
+      };
+
+      const pauseRecorderForBackpressure = () => {
+        if (mediaRecorder.state === 'recording') {
+          mediaRecorder.pause();
+        }
+
+        if (backpressureTimerRef.current === null) {
+          backpressureTimerRef.current = window.setInterval(
+            resumeRecorderWhenDrained,
+            RECORDER_TIMESLICE_MS,
+          );
+        }
+      };
 
       // Set up WebSocket event handlers
-      screenShareSocketRef.current.onopen = () => {
+      socket.onopen = () => {
+        if (screenShareSocketRef.current !== socket) {
+          socket.close();
+          return;
+        }
+
         console.log('Screen sharing WebSocket connected');
         setIsScreenSharing(true);
-
+        mediaRecorder.start(RECORDER_TIMESLICE_MS);
-        mediaRecorder.start(100);
       };
 
-      screenShareSocketRef.current.onclose = () => {
+      socket.onclose = () => {
         console.log('Screen sharing WebSocket closed');
-        setIsScreenSharing(false);
+        if (screenShareSocketRef.current === socket) {
-
+          stopScreenShare();
-        // Stop all tracks when WebSocket is closed
+        }
-        mediaStream.getTracks().forEach(track => track.stop());
       };
 
-      screenShareSocketRef.current.onerror = (error) => {
+      socket.onerror = (error) => {
         console.error('Screen sharing WebSocket error:', error);
-        setIsScreenSharing(false);
+        if (screenShareSocketRef.current === socket) {
+          stopScreenShare();
+        }
+      };
 
-        // Stop all tracks on error
+      mediaRecorder.onerror = (error) => {
-        mediaStream.getTracks().forEach(track => track.stop());
+        console.error('Screen sharing recorder error:', error);
+        if (mediaRecorderRef.current === mediaRecorder) {
+          stopScreenShare();
+        }
       };
 
       // Send data over WebSocket when available
       mediaRecorder.ondataavailable = (event) => {
-        if (event.data && event.data.size > 0 && screenShareSocketRef.current && screenShareSocketRef.current.readyState === WebSocket.OPEN) {
+        if (event.data.size > 0 && screenShareSocketRef.current === socket && socket.readyState === WebSocket.OPEN) {
-          screenShareSocketRef.current.send(event.data);
+          socket.send(event.data);
+
+          // Browser WebSockets otherwise queue without a limit, turning a brief
+          // network slowdown into permanently stale playback. Pausing capture
+          // bounds that queue and resumes at the live edge once it drains.
+          if (socket.bufferedAmount >= socketBufferLimits.maximum) {
+            pauseRecorderForBackpressure();
+          }
         }
       };
 
       // Handle stream ending (user clicks "Stop sharing")
-      mediaStream.getVideoTracks()[0].onended = () => {
+      videoTrack.onended = stopScreenShare;
-        if (screenShareSocketRef.current) {
-          screenShareSocketRef.current.close();
-          screenShareSocketRef.current = null;
-        }
-        setIsScreenSharing(false);
-      };
 
     } catch (error) {
-      console.error('Error starting screen share:', error);
+      if (sessionGenerationRef.current === sessionGeneration) {
-      setIsScreenSharing(false);
+        console.error('Error starting screen share:', error);
+        stopScreenShare();
+      }
     }
-  }, []);
+  }, [stopScreenShare]);
+
+  useEffect(() => {
+    return () => stopScreenShare();
+  }, [stopScreenShare]);
 
   const toggleScreenShare = useCallback(async () => {
-    if (screenShareSocketRef.current) {
+    if (screenShareSocketRef.current || isStartingRef.current) {
       stopScreenShare();
     } else {
       await startScreenShare();