Improve frame stall recovery

public/app.js

@@ -74,6 +74,7 @@ const state = {
   lastFramePacketAt: 0,
   lastFramePaintedAt: 0,
   lastPlaybackRestartAt: 0,
+  lastRenderErrorAt: 0,
 };
 
 void loadEntryLists();
@@ -286,8 +287,18 @@ elements.audio.addEventListener('error', () => {
 });
 
 window.addEventListener('online', () => {
-  if (state.session && !elements.audio.paused && isFrameStreamStale(FRAME_STALL_RESET_MS)) {
-    restartPlaybackStreams();
+  const stallReason = getFrameStallReason();
+
+  if (state.session && !elements.audio.paused && stallReason) {
+    restartPlaybackStreams(`network_recovered_${stallReason}`);
+  }
+});
+
+document.addEventListener('visibilitychange', () => {
+  const stallReason = getFrameStallReason();
+
+  if (!document.hidden && state.session && !elements.audio.paused && stallReason) {
+    restartPlaybackStreams(`visible_${stallReason}`);
   }
 });
 
@@ -402,26 +413,40 @@ function checkFrameWatchdog() {
     return;
   }
 
-  const resetAfterMs = state.lastFramePaintedAt > 0 ? FRAME_STALL_RESET_MS : FRAME_STARTUP_STALL_RESET_MS;
+  const stallReason = getFrameStallReason();
 
-  if (!isFrameStreamStale(resetAfterMs)) {
+  if (!stallReason) {
     return;
   }
 
-  restartPlaybackStreams();
+  restartPlaybackStreams(stallReason);
 }
 
-function isFrameStreamStale(resetAfterMs) {
+function getFrameStallReason() {
   const now = Date.now();
-  const lastFrameAt = state.lastFramePacketAt || state.lastPlaybackRestartAt || now;
-  return now - lastFrameAt >= resetAfterMs && now - state.lastPlaybackRestartAt >= PLAYBACK_RESTART_COOLDOWN_MS;
+
+  if (!state.session || state.isSeeking || elements.audio.paused || elements.audio.ended || elements.audio.readyState === 0) {
+    return '';
   }
 
-function restartPlaybackStreams() {
+  if (now - state.lastPlaybackRestartAt < PLAYBACK_RESTART_COOLDOWN_MS) {
+    return '';
+  }
+
+  if (state.lastFramePaintedAt < state.lastPlaybackRestartAt) {
+    return now - state.lastPlaybackRestartAt >= FRAME_STARTUP_STALL_RESET_MS ? 'frame_startup_stalled' : '';
+  }
+
+  return now - state.lastFramePaintedAt >= FRAME_STALL_RESET_MS ? 'frame_paint_stalled' : '';
+}
+
+function restartPlaybackStreams(reason = 'frame_stalled') {
   if (!state.session) {
     return;
   }
 
+  console.warn(`Restarting playback streams: ${reason}`);
+
   if (state.seekable && Number.isFinite(state.duration) && state.duration > 0) {
     state.lastPlaybackRestartAt = Date.now();
     void seekTo(getVisiblePlaybackTime());
@@ -462,7 +487,7 @@ function handleFramePacket(packet, streamGeneration) {
 
   const timestamp = new DataView(packet, 0, 8).getFloat64(0, true);
 
-  if (streamGeneration !== state.streamGeneration || isLateFrame(timestamp)) {
+  if (streamGeneration !== state.streamGeneration) {
     return;
   }
 
@@ -502,9 +527,18 @@ function startRenderLoop() {
   }
 
   const render = () => {
+    try {
       drawReadyFrames();
       syncPlayhead();
+    } catch (error) {
+      logRenderError(error);
+    } finally {
+      if (state.session) {
         state.raf = requestAnimationFrame(render);
+      } else {
+        state.raf = 0;
+      }
+    }
   };
 
   state.raf = requestAnimationFrame(render);
@@ -515,7 +549,7 @@ function drawReadyFrames() {
     return;
   }
 
-  dropLateDecodedFrames();
+  dropLateDecodedFrames({ keepNewest: true });
 
   const frameLeadSeconds = 1 / Math.max(1, state.session.options.fps);
   const targetTime = elements.audio.currentTime + frameLeadSeconds;
@@ -567,7 +601,7 @@ async function pumpFrameDecodeQueue() {
 
   try {
     while (state.pendingFrames.length > 0) {
-      dropLatePendingFrames();
+      dropLatePendingFrames({ keepNewest: true });
 
       const frame = state.pendingFrames.shift();
 
@@ -575,7 +609,11 @@ async function pumpFrameDecodeQueue() {
         return;
       }
 
-      if (frame.streamGeneration !== state.streamGeneration || isLateFrame(frame.timestamp)) {
+      if (frame.streamGeneration !== state.streamGeneration) {
+        continue;
+      }
+
+      if (isLateFrame(frame.timestamp) && state.pendingFrames.length > 0) {
         continue;
       }
 
@@ -587,7 +625,12 @@ async function pumpFrameDecodeQueue() {
         continue;
       }
 
-      if (frame.streamGeneration !== state.streamGeneration || isLateFrame(frame.timestamp)) {
+      if (frame.streamGeneration !== state.streamGeneration) {
+        releaseImage(bitmap);
+        continue;
+      }
+
+      if (isLateFrame(frame.timestamp) && (state.pendingFrames.length > 0 || state.frames.length > 0)) {
         releaseImage(bitmap);
         continue;
       }
@@ -608,7 +651,7 @@ async function pumpFrameDecodeQueue() {
 }
 
 function trimPendingFrameQueue() {
-  dropLatePendingFrames();
+  dropLatePendingFrames({ keepNewest: true });
 
   const maxQueuedFrames = getFrameQueueLimit(MAX_PENDING_FRAME_QUEUE_SECONDS, MIN_PENDING_FRAME_QUEUE);
   const overflow = state.pendingFrames.length - maxQueuedFrames;
@@ -619,7 +662,7 @@ function trimPendingFrameQueue() {
 }
 
 function trimFrameQueue() {
-  dropLateDecodedFrames();
+  dropLateDecodedFrames({ keepNewest: true });
 
   const maxQueuedFrames = getFrameQueueLimit(MAX_DECODED_FRAME_QUEUE_SECONDS, MIN_DECODED_FRAME_QUEUE);
   const overflow = state.frames.length - maxQueuedFrames;
@@ -635,10 +678,11 @@ function trimFrameQueue() {
   }
 }
 
-function dropLatePendingFrames() {
+function dropLatePendingFrames({ keepNewest = false } = {}) {
   let removeCount = 0;
+  const removableFrames = keepNewest ? Math.max(0, state.pendingFrames.length - 1) : state.pendingFrames.length;
 
-  while (removeCount < state.pendingFrames.length && isLateFrame(state.pendingFrames[removeCount].timestamp)) {
+  while (removeCount < removableFrames && isLateFrame(state.pendingFrames[removeCount].timestamp)) {
     removeCount += 1;
   }
 
@@ -647,10 +691,11 @@ function dropLatePendingFrames() {
   }
 }
 
-function dropLateDecodedFrames() {
+function dropLateDecodedFrames({ keepNewest = false } = {}) {
   let removeCount = 0;
+  const removableFrames = keepNewest ? Math.max(0, state.frames.length - 1) : state.frames.length;
 
-  while (removeCount < state.frames.length && isLateFrame(state.frames[removeCount].timestamp)) {
+  while (removeCount < removableFrames && isLateFrame(state.frames[removeCount].timestamp)) {
     removeCount += 1;
   }
 
@@ -793,6 +838,17 @@ function releaseImage(image) {
   }
 }
 
+function logRenderError(error) {
+  const now = Date.now();
+
+  if (now - state.lastRenderErrorAt < 5000) {
+    return;
+  }
+
+  state.lastRenderErrorAt = now;
+  console.warn('Frame render loop error', error);
+}
+
 async function seekTo(value) {
   if (!state.session || !state.seekable) {
     state.isSeeking = false;