buzzert/QueueCube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize screen sharing latency and frame rate

Browse Source
This commit is contained in:
buzzert
2026-08-29 20:10:55 -07:00
parent ec99065cdf
commit a846d9b4ec
4 changed files with 332 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/backend/src/MediaPlayer.ts
+24 -21
View File
@@ -375,29 +375,27 @@ export class MediaPlayer {
siteName: "Screen Sharing",
});
// Special options for mpv to better handle screen sharing (AI recommended...)
// Start from mpv's maintained low-latency profile, then keep only a
// couple of frames of demuxed data ready for smooth 60 fps playback.
await this.loadFile(url, "replace", false, [
"demuxer-lavf-o=fflags=+nobuffer+discardcorrupt", // Reduce buffering and discard corrupt frames
"demuxer-lavf-o=analyzeduration=100000", // Reduce analyze duration
"demuxer-lavf-o=probesize=1000000", // Reduce probe size
"untimed=yes", // Ignore timing info
"cache=no", // Disable cache
"force-seekable=yes", // Force seekable
"no-cache=yes", // Disable cache
"demuxer-max-bytes=500K", // Limit demuxer buffer
"demuxer-readahead-secs=0.1", // Reduce readahead
"hr-seek=no", // Disable high-res seeking
"video-sync=display-resample", // Better sync mode
"video-latency-hacks=yes", // Enable latency hacks
"audio-sync=yes", // Enable audio sync
"audio-buffer=0.1", // Reduce audio buffer
"audio-channels=stereo", // Force stereo audio
"audio-samplerate=44100", // Match sample rate
"audio-format=s16", // Use 16-bit audio
"profile=low-latency",
"cache=no",
"force-seekable=no",
"demuxer-readahead-secs=0.05",
"demuxer-max-bytes=256K",
"demuxer-mkv-probe-start-time=no",
"vd-lavc-threads=2",
"vd-lavc-fast=yes",
"audio-buffer=0",
"pulse-buffer=20",
"pipewire-buffer=20",
"alsa-buffer-time=20000",
"initial-audio-sync=no",
]);
// Make sure it's playing
setTimeout(() => this.play(), 100);
// loadfile normally unpauses, but make that explicit without adding the
// previous fixed 100 ms startup delay.
await this.play();
}
public async play() {
@@ -475,13 +473,18 @@ export class MediaPlayer {
private async loadFile(url: string, mode: string, fetchMetadata: boolean = true, options: string[] = []) {
this.lastLoadCandidate = url;
this.playbackErrors.delete(url);
this.modify(UserEvent.PlaylistUpdate, () => this.writeCommand("loadfile", [url, mode, "-1", options.join(',')]));
const loadPromise = this.modify(
UserEvent.PlaylistUpdate,
() => this.writeCommand("loadfile", [url, mode, "-1", options.join(',')]),
);
if (fetchMetadata && this.shouldFetchLinkPreview(url)) {
this.fetchMetadataAndNotify(url).catch(error => {
console.warn(`Failed to fetch metadata for ${this.loggableUrl(url)}:`, error);
});
}
await loadPromise;
}
private async modify<T>(event: UserEvent, func: () => Promise<T>): Promise<T> {
web/backend/src/server.ts
+87 -42
View File
@@ -26,6 +26,7 @@ import { searchInvidious, fetchThumbnail } from "./InvidiousAPI";
import { PlaylistItem } from './types';
import { PassThrough } from "stream";
import { AddressInfo } from "net";
import { WebSocket } from "ws";
const app = express();
app.use(express.json());
@@ -43,6 +44,21 @@ const mediaPlayer = new MediaPlayer();
// Create a shared stream that both endpoints can access
let activeScreenshareStream: PassThrough | null = null;
let activeScreenshareMimeType: string | null = null;
let activeScreenshareSocket: WebSocket | null = null;
const screenshareStreamsClosedByViewer = new WeakSet<PassThrough>();
const SCREENSHARE_STREAM_HIGH_WATER_MARK = 32 * 1024;
const getScreenshareContentType = (mimeType: unknown): string => {
if (typeof mimeType !== "string") {
return "application/octet-stream";
}
const contentType = mimeType.split(";", 1)[0].trim().toLowerCase();
return contentType === "video/webm" || contentType === "video/mp4"
? contentType
: "application/octet-stream";
};
const withErrorHandling = (func: (req: any, res: any) => Promise<any>) => {
return async (req: any, res: any) => {
@@ -158,75 +174,104 @@ apiRouter.ws("/events", (ws, req) => {
// using the fact that QueueCube/MPV is based all around streaming URLs, so the active
// screenshare stream manifests as just another URL to play.
apiRouter.get("/screenshareStream", withErrorHandling(async (req, res) => {
res.setHeader("Content-Type", activeScreenshareMimeType || "video/mp4");
const screenshareStream = activeScreenshareStream;
if (!screenshareStream) {
res.status(503).send("No active screen sharing session");
return;
}
req.socket.setNoDelay(true);
res.setHeader("Content-Type", activeScreenshareMimeType || "application/octet-stream");
res.setHeader("Cache-Control", "no-cache, no-store, must-revalidate");
res.setHeader("Pragma", "no-cache");
res.setHeader("Expires", "0");
res.setHeader("Connection", "keep-alive");
res.setHeader("Transfer-Encoding", "chunked");
res.setHeader("Accept-Ranges", "none");
res.flushHeaders();
if (!activeScreenshareStream) {
res.status(503).send("No active screen sharing session");
return;
}
// Handle client disconnection
req.on('close', () => {
res.on('close', () => {
screenshareStream.unpipe(res);
console.log("Screenshare viewer disconnected");
// If mpv goes away unexpectedly, stop capture instead of leaving an
// unconsumed stream to build backpressure forever.
if (activeScreenshareStream === screenshareStream && !screenshareStream.writableEnded) {
screenshareStreamsClosedByViewer.add(screenshareStream);
activeScreenshareSocket?.close(1011, "Screen sharing viewer disconnected");
}
});
// Configure stream for low latency
activeScreenshareStream.setMaxListeners(0);
// Pipe with immediate flush
activeScreenshareStream.pipe(res, { end: false });
screenshareStream.pipe(res);
}));
apiRouter.ws("/screenshare", (ws, req) => {
const mimeType = req.query.mimeType as string;
const mimeType = getScreenshareContentType(req.query.mimeType);
console.log("Screen sharing client connected with mimeType: " + mimeType);
ws.binaryType = "arraybuffer";
req.socket.setNoDelay(true);
let firstChunk = false;
// Supersede an old sender without allowing its close handler to tear down
// the new session.
const previousSocket = activeScreenshareSocket;
const previousStream = activeScreenshareStream;
activeScreenshareSocket = ws;
activeScreenshareMimeType = mimeType;
activeScreenshareStream = new PassThrough({
highWaterMark: SCREENSHARE_STREAM_HIGH_WATER_MARK,
allowHalfOpen: false,
});
const screenshareStream = activeScreenshareStream;
let waitingForDrain = false;
// Configure WebSocket for low latency
ws.setMaxListeners(0);
ws.binaryType = "arraybuffer";
previousStream?.end();
if (previousSocket && previousSocket.readyState === WebSocket.OPEN) {
previousSocket.close(1000, "A newer screen sharing session started");
}
// Start mpv while the browser starts its recorder, instead of waiting for a
// whole encoded chunk before beginning the HTTP connection and demux setup.
const port = (server.address() as AddressInfo).port;
const url = `http://127.0.0.1:${port}/api/screenshareStream`;
console.log(`Starting screen share stream at ${url}`);
mediaPlayer.initiateScreenSharing(url).catch((error: unknown) => {
console.error("Failed to start screen sharing:", error);
if (activeScreenshareSocket === ws) {
ws.close(1011, "Unable to start the media player");
}
});
ws.on('message', (data: any) => {
const buffer = data instanceof Buffer ? data : Buffer.from(data);
if (!firstChunk) {
firstChunk = true;
const port = (server.address() as AddressInfo).port;
const url = `http://localhost:${port}/api/screenshareStream`;
console.log(`Starting screen share stream at ${url}`);
// Create new shared stream with immediate flush
activeScreenshareStream = new PassThrough({
highWaterMark: 1024 * 1024, // 1MB buffer
allowHalfOpen: false
});
activeScreenshareStream.write(buffer);
mediaPlayer.initiateScreenSharing(url);
} else if (activeScreenshareStream) {
// Write with immediate flush
activeScreenshareStream.write(buffer, () => {
activeScreenshareStream?.cork();
activeScreenshareStream?.uncork();
if (activeScreenshareStream !== screenshareStream) {
return;
}
if (!screenshareStream.write(buffer) && !waitingForDrain) {
waitingForDrain = true;
ws.pause();
screenshareStream.once('drain', () => {
waitingForDrain = false;
if (activeScreenshareStream === screenshareStream && ws.readyState === WebSocket.OPEN) {
ws.resume();
}
});
}
});
ws.on('close', () => {
console.log("Screen sharing client disconnected");
if (activeScreenshareStream) {
activeScreenshareStream.end();
if (activeScreenshareStream === screenshareStream) {
screenshareStream.end();
activeScreenshareStream = null;
activeScreenshareMimeType = null;
activeScreenshareSocket = null;
if (!screenshareStreamsClosedByViewer.has(screenshareStream)) {
mediaPlayer.stop().catch((error: unknown) => {
console.error("Failed to stop screen sharing:", error);
});
}
}
mediaPlayer.stop();
});
});