Improve frame sync telemetry

public/app.js

@@ -36,6 +36,8 @@ const elements = {
 
 const context = elements.canvas.getContext('2d', { alpha: false, desynchronized: true });
 const FRAME_LATE_GRACE_SECONDS = 0.25;
+const FRAME_CLOCK_SEND_INTERVAL_MS = 250;
+const CLIENT_TELEMETRY_INTERVAL_MS = 5000;
 const MAX_PENDING_FRAME_QUEUE_SECONDS = 2;
 const MAX_DECODED_FRAME_QUEUE_SECONDS = 3;
 const MIN_PENDING_FRAME_QUEUE = 12;
@@ -73,8 +75,11 @@ const state = {
   frameWatchdogTimer: 0,
   lastFramePacketAt: 0,
   lastFramePaintedAt: 0,
+  lastFrameClockSentAt: 0,
+  lastClientTelemetrySentAt: 0,
   lastPlaybackRestartAt: 0,
   lastRenderErrorAt: 0,
+  clientTelemetry: createClientTelemetry(),
 };
 
 void loadEntryLists();
@@ -259,6 +264,7 @@ elements.audio.addEventListener('pause', () => {
 });
 
 elements.audio.addEventListener('playing', () => {
+  noteClientTelemetry('audioPlaying');
   elements.loader.hidden = state.frameCount > 0;
   clearPlayerMessage();
   scheduleControlsHide();
@@ -266,6 +272,7 @@ elements.audio.addEventListener('playing', () => {
 
 elements.audio.addEventListener('waiting', () => {
   if (state.session) {
+    noteClientTelemetry('audioWaiting');
     elements.loader.hidden = false;
   }
 });
@@ -275,12 +282,15 @@ elements.audio.addEventListener('timeupdate', () => {
 });
 
 elements.audio.addEventListener('ended', () => {
+  noteClientTelemetry('audioEnded');
   syncPlayhead();
   setControlsVisible(true);
 });
 
 elements.audio.addEventListener('error', () => {
   if (state.session) {
+    noteClientTelemetry('audioErrors');
+    sendClientTelemetry({ force: true, reason: 'audio_error' });
     showPlayerMessage('Audio failed');
     setControlsVisible(true);
   }
@@ -335,6 +345,7 @@ function connectPlaybackStreams() {
   state.streamGeneration = streamGeneration;
   state.lastPlaybackRestartAt = now;
   state.lastFramePacketAt = now;
+  state.lastFrameClockSentAt = 0;
 
   if (state.websocket) {
     state.websocket.close(1000, 'client reconnecting');
@@ -446,6 +457,8 @@ function restartPlaybackStreams(reason = 'frame_stalled') {
   }
 
   console.warn(`Restarting playback streams: ${reason}`);
+  noteClientTelemetry('playbackRestarts');
+  sendClientTelemetry({ force: true, reason });
 
   if (state.seekable && Number.isFinite(state.duration) && state.duration > 0) {
     state.lastPlaybackRestartAt = Date.now();
@@ -491,6 +504,12 @@ function handleFramePacket(packet, streamGeneration) {
     return;
   }
 
+  if (isLateFrame(timestamp)) {
+    noteLateFrameTelemetry('lateFramePackets', timestamp);
+    state.lastFramePacketAt = Date.now();
+    return;
+  }
+
   state.lastFramePacketAt = Date.now();
   state.pendingFrames.push({ timestamp, jpeg: new Uint8Array(packet, 8), streamGeneration });
   trimPendingFrameQueue();
@@ -521,6 +540,70 @@ function handleControlMessage(rawMessage) {
   }
 }
 
+function sendFrameClock() {
+  if (!state.session || !state.websocket || state.websocket.readyState !== WebSocket.OPEN || elements.audio.readyState === 0) {
+    return;
+  }
+
+  const now = Date.now();
+
+  if (now - state.lastFrameClockSentAt < FRAME_CLOCK_SEND_INTERVAL_MS) {
+    return;
+  }
+
+  const currentTime = Number.isFinite(elements.audio.currentTime) ? elements.audio.currentTime : 0;
+
+  state.lastFrameClockSentAt = now;
+
+  try {
+    state.websocket.send(JSON.stringify({ type: 'clock', currentTime }));
+  } catch {
+    state.lastFrameClockSentAt = 0;
+  }
+}
+
+function sendClientTelemetry({ force = false, reason = 'periodic' } = {}) {
+  if (!state.websocket || state.websocket.readyState !== WebSocket.OPEN) {
+    return;
+  }
+
+  const now = Date.now();
+
+  if (!force && now - state.lastClientTelemetrySentAt < CLIENT_TELEMETRY_INTERVAL_MS) {
+    return;
+  }
+
+  state.lastClientTelemetrySentAt = now;
+
+  const currentTime = Number.isFinite(elements.audio.currentTime) ? elements.audio.currentTime : 0;
+  const telemetry = state.clientTelemetry;
+  const payload = {
+    type: 'telemetry',
+    reason,
+    currentTime,
+    paused: elements.audio.paused,
+    readyState: elements.audio.readyState,
+    pendingFrames: state.pendingFrames.length,
+    decodedFrames: state.frames.length,
+    paintedFrames: state.frameCount,
+    lastFramePacketAgeMs: state.lastFramePacketAt > 0 ? now - state.lastFramePacketAt : null,
+    lastFramePaintAgeMs: state.lastFramePaintedAt > 0 ? now - state.lastFramePaintedAt : null,
+    hidden: document.hidden,
+    online: navigator.onLine,
+    counters: telemetry.counters,
+    max: telemetry.max,
+  };
+
+  state.clientTelemetry = createClientTelemetry();
+
+  try {
+    state.websocket.send(JSON.stringify(payload));
+  } catch {
+    state.clientTelemetry = mergeClientTelemetry(state.clientTelemetry, telemetry);
+    state.lastClientTelemetrySentAt = 0;
+  }
+}
+
 function startRenderLoop() {
   if (state.raf) {
     return;
@@ -528,6 +611,8 @@ function startRenderLoop() {
 
   const render = () => {
     try {
+      sendFrameClock();
+      sendClientTelemetry();
       drawReadyFrames();
       syncPlayhead();
     } catch (error) {
@@ -549,7 +634,7 @@ function drawReadyFrames() {
     return;
   }
 
-  dropLateDecodedFrames({ keepNewest: true });
+  dropLateDecodedFrames();
 
   const frameLeadSeconds = 1 / Math.max(1, state.session.options.fps);
   const targetTime = elements.audio.currentTime + frameLeadSeconds;
@@ -559,6 +644,7 @@ function drawReadyFrames() {
     const frame = state.frames.shift();
 
     if (frameToDraw) {
+      noteClientTelemetry('readyFramesSkippedForNewer');
       releaseImage(frameToDraw.bitmap);
     }
 
@@ -575,6 +661,8 @@ function drawReadyFrames() {
 
   state.currentBitmap = frameToDraw.bitmap;
   drawBitmap(frameToDraw.bitmap);
+  noteClientTelemetry('framesPainted');
+  noteFrameLagTelemetry('paintLagMs', frameToDraw.timestamp);
   state.lastFramePaintedAt = Date.now();
   elements.loader.hidden = true;
   clearPlayerMessage();
@@ -601,7 +689,7 @@ async function pumpFrameDecodeQueue() {
 
   try {
     while (state.pendingFrames.length > 0) {
-      dropLatePendingFrames({ keepNewest: true });
+      dropLatePendingFrames();
 
       const frame = state.pendingFrames.shift();
 
@@ -610,10 +698,12 @@ async function pumpFrameDecodeQueue() {
       }
 
       if (frame.streamGeneration !== state.streamGeneration) {
+        noteClientTelemetry('staleGenerationPendingFrames');
         continue;
       }
 
-      if (isLateFrame(frame.timestamp) && state.pendingFrames.length > 0) {
+      if (isLateFrame(frame.timestamp)) {
+        noteLateFrameTelemetry('latePendingFramesBeforeDecode', frame.timestamp);
         continue;
       }
 
@@ -621,16 +711,19 @@ async function pumpFrameDecodeQueue() {
 
       try {
         bitmap = await decodeImageWithTimeout(new Blob([frame.jpeg], { type: 'image/jpeg' }));
-      } catch {
+      } catch (error) {
+        noteClientTelemetry(error.message === 'Image decode timed out.' ? 'imageDecodeTimeouts' : 'imageDecodeFailures');
         continue;
       }
 
       if (frame.streamGeneration !== state.streamGeneration) {
+        noteClientTelemetry('staleGenerationDecodedFrames');
         releaseImage(bitmap);
         continue;
       }
 
-      if (isLateFrame(frame.timestamp) && (state.pendingFrames.length > 0 || state.frames.length > 0)) {
+      if (isLateFrame(frame.timestamp)) {
+        noteLateFrameTelemetry('latePendingFramesAfterDecode', frame.timestamp);
         releaseImage(bitmap);
         continue;
       }
@@ -651,18 +744,20 @@ async function pumpFrameDecodeQueue() {
 }
 
 function trimPendingFrameQueue() {
-  dropLatePendingFrames({ keepNewest: true });
+  dropLatePendingFrames();
 
   const maxQueuedFrames = getFrameQueueLimit(MAX_PENDING_FRAME_QUEUE_SECONDS, MIN_PENDING_FRAME_QUEUE);
   const overflow = state.pendingFrames.length - maxQueuedFrames;
 
   if (overflow > 0) {
+    noteClientTelemetry('pendingQueueOverflowFrames', overflow);
+    noteClientTelemetryMax('pendingQueuePeakFrames', state.pendingFrames.length);
     state.pendingFrames.splice(0, overflow);
   }
 }
 
 function trimFrameQueue() {
-  dropLateDecodedFrames({ keepNewest: true });
+  dropLateDecodedFrames();
 
   const maxQueuedFrames = getFrameQueueLimit(MAX_DECODED_FRAME_QUEUE_SECONDS, MIN_DECODED_FRAME_QUEUE);
   const overflow = state.frames.length - maxQueuedFrames;
@@ -672,6 +767,8 @@ function trimFrameQueue() {
   }
 
   const removed = state.frames.splice(0, overflow);
+  noteClientTelemetry('decodedQueueOverflowFrames', overflow);
+  noteClientTelemetryMax('decodedQueuePeakFrames', state.frames.length + overflow);
 
   for (const frame of removed) {
     releaseImage(frame.bitmap);
@@ -687,6 +784,8 @@ function dropLatePendingFrames({ keepNewest = false } = {}) {
   }
 
   if (removeCount > 0) {
+    noteClientTelemetry('latePendingFrames', removeCount);
+    noteFrameLagTelemetry('latePendingFramesMaxLagMs', state.pendingFrames[0]?.timestamp);
     state.pendingFrames.splice(0, removeCount);
   }
 }
@@ -704,6 +803,8 @@ function dropLateDecodedFrames({ keepNewest = false } = {}) {
   }
 
   const removed = state.frames.splice(0, removeCount);
+  noteClientTelemetry('lateDecodedFrames', removeCount);
+  noteFrameLagTelemetry('lateDecodedFramesMaxLagMs', removed[0]?.timestamp);
 
   for (const frame of removed) {
     releaseImage(frame.bitmap);
@@ -724,7 +825,57 @@ function isLateFrame(timestamp) {
   return timestamp < currentTime - FRAME_LATE_GRACE_SECONDS;
 }
 
+function noteLateFrameTelemetry(name, timestamp, count = 1) {
+  noteClientTelemetry(name, count);
+  noteFrameLagTelemetry(`${name}MaxLagMs`, timestamp);
+}
+
+function noteFrameLagTelemetry(name, timestamp) {
+  if (!Number.isFinite(timestamp) || elements.audio.readyState === 0) {
+    return;
+  }
+
+  const currentTime = Number.isFinite(elements.audio.currentTime) ? elements.audio.currentTime : 0;
+  noteClientTelemetryMax(name, Math.max(0, currentTime - timestamp) * 1000);
+}
+
+function createClientTelemetry() {
+  return {
+    counters: {},
+    max: {},
+  };
+}
+
+function noteClientTelemetry(name, count = 1) {
+  if (!state.clientTelemetry || !Number.isFinite(count) || count <= 0) {
+    return;
+  }
+
+  state.clientTelemetry.counters[name] = (state.clientTelemetry.counters[name] ?? 0) + count;
+}
+
+function noteClientTelemetryMax(name, value) {
+  if (!state.clientTelemetry || !Number.isFinite(value)) {
+    return;
+  }
+
+  state.clientTelemetry.max[name] = Math.max(state.clientTelemetry.max[name] ?? 0, Math.round(value));
+}
+
+function mergeClientTelemetry(target, source) {
+  for (const [name, count] of Object.entries(source.counters)) {
+    target.counters[name] = (target.counters[name] ?? 0) + count;
+  }
+
+  for (const [name, value] of Object.entries(source.max)) {
+    target.max[name] = Math.max(target.max[name] ?? 0, value);
+  }
+
+  return target;
+}
+
 function stopSession({ showEntry: shouldShowEntry = true } = {}) {
+  sendClientTelemetry({ force: true, reason: 'stop_session' });
   state.generation += 1;
   state.streamGeneration += 1;
   state.session = null;
@@ -736,6 +887,9 @@ function stopSession({ showEntry: shouldShowEntry = true } = {}) {
   state.seekPreviewTime = 0;
   state.lastFramePacketAt = 0;
   state.lastFramePaintedAt = 0;
+  state.lastFrameClockSentAt = 0;
+  state.lastClientTelemetrySentAt = 0;
+  state.clientTelemetry = createClientTelemetry();
   clearHideControlsTimer();
   clearFrameWatchdog();
 
@@ -808,7 +962,7 @@ async function decodeImageWithTimeout(blob) {
   window.clearTimeout(timeoutId);
 
   if (!image) {
-    throw new Error('Image decode failed.');
+    throw new Error(timedOut ? 'Image decode timed out.' : 'Image decode failed.');
   }
 
   return image;