diff --git a/README.md b/README.md index bf74dd7..1e0f34c 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,13 @@ to find remote session IDs. Closing the window and reattaching to the local session restores its mixed tab bar. The **+** button creates a local tab in this workspace. Splitting a remote pane creates another shell on its SSH host. +If an established SSH connection drops, psyTerm keeps its tabs and tries to +reattach to the same remote session. The active tab shows the retry status and +a **Cancel** button. Retries start after one second and back off to a maximum +of 30 seconds between attempts. Other local tabs stay usable. Cancel detaches +the interrupted SSH tabs, or closes the window for a standalone SSH session. +You can reattach later if the remote daemon still has the session. + To detach, hover over a remote tab and click **×**, or focus it and choose **Detach remote session** in the command palette. If that SSH session has several tabs, they detach together. Its shells, running jobs, and scrollback @@ -195,8 +202,8 @@ in a window. The local daemon remembers these targets until it exits. If you detach a nested SSH session, run the command on the host that opened it. You can also select a session explicitly with `psyterm --remote user@host --attach ID`. -The local daemon must be running this build to close tabs, remember detached -sessions, and pass SSH settings to new windows. Existing protocol-v6 sessions +The local daemon must be running this build to reconnect SSH tabs, remember +detached sessions, and pass SSH settings to new windows. Existing protocol-v6 sessions on the SSH host can stay running. SSH uses `BatchMode=yes`. Configure key or agent authentication and trust the diff --git a/src/chrome.rs b/src/chrome.rs index bc581f8..3926c9e 100644 --- a/src/chrome.rs +++ b/src/chrome.rs @@ -60,6 +60,7 @@ pub struct ChromeTab { pub label: SharedString, pub remote: Option, pub active: bool, + pub reconnecting: bool, pub editor: Option>, } @@ -112,15 +113,19 @@ pub fn header( ); } if let Some(destination) = tab.remote { + let host = remote_hostname(&destination); + let subtitle = if tab.reconnecting { + format!("{host} · Reconnecting") + } else { + host.to_owned() + }; label = label.child( div() .h(px(12.)) .line_height(px(12.)) .text_size(px(10.)) .text_color(rgb(PANE_FOCUS)) - .child(middle_label( - remote_hostname(&destination).to_owned().into(), - )), + .child(middle_label(subtitle.into())), ); } tab_strip = tab_strip.child( @@ -314,6 +319,7 @@ mod tests { label: "Shell".into(), remote: None, active: true, + reconnecting: false, editor: None, }, ChromeTab { @@ -321,6 +327,7 @@ mod tests { label: "Build".into(), remote: Some("user@work".into()), active: false, + reconnecting: true, editor: None, }, ], diff --git a/src/client.rs b/src/client.rs index d0d3174..5cf6888 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,7 +1,11 @@ //! Display-only client transport. No terminal parser, PTY, or shell lives here. +mod transport; +use transport::{ActiveTransport, Supervisor, Wire}; + use crate::protocol::{ - self, ClientMessage, RemoteSessionInfo, RemoteWindow, ServerMessage, SessionIdentity, - SessionInfo, SessionRoute, SessionSummary, SessionTarget, Snapshot, SshOptions, TabId, + self, ClientMessage, ReconnectStatus, RemoteSessionInfo, RemoteWindow, ServerMessage, + SessionIdentity, SessionInfo, SessionRoute, SessionSummary, SessionTarget, Snapshot, + SshOptions, TabId, }; use anyhow::{Context, Result, bail, ensure}; use std::{ @@ -26,6 +30,8 @@ pub struct Update { pub clipboard: VecDeque, pub error: Option, pub disconnected: bool, + /// A change in the SSH retry state; the inner None clears the status. + pub reconnect: Option>, } pub struct Updates { @@ -111,7 +117,7 @@ impl RemoteListing { socket: PathBuf, target: RemoteWindow, frame: &Snapshot, - commands: mpsc::Sender, + connection: std::sync::Weak, ) -> Result { let identity = frame.lineage[0]; // Verified in greet(). let latest = Arc::new(Mutex::new(RemoteWindowListing { @@ -131,7 +137,11 @@ impl RemoteListing { changes, published, || { - let _ = commands.send(ClientMessage::ListSessions); + if let Some(owner) = connection.upgrade() + && let Some(active) = owner.active.lock().unwrap().as_ref() + { + let _ = active.commands.send(ClientMessage::ListSessions); + } }, ) { log::warn!("Could not list SSH window on local server: {error:#}"); @@ -161,20 +171,19 @@ impl RemoteListing { struct Connection { launcher_socket: PathBuf, remote: Option, - commands: mpsc::Sender, - socket: Option, - ssh: Mutex>, + active: Mutex>, + cancelled: Arc, + worker: std::sync::OnceLock, } + impl Drop for Connection { fn drop(&mut self) { - // Detach closes this transport, never a server-owned PTY. - if let Some(socket) = &self.socket { - let _ = socket.shutdown(Shutdown::Both); - } - if let Some(mut child) = self.ssh.lock().unwrap().take() { - let _ = child.kill(); - let _ = child.wait(); + self.cancelled.store(true, Ordering::Release); + // Wake either a sleeping retry or a reader on the current transport. + if let Some(worker) = self.worker.get() { + worker.unpark(); } + self.active.get_mut().unwrap().take(); } } @@ -233,102 +242,15 @@ impl Client { options: SshOptions, cancelled: Option>, ) -> Result<(Self, Updates)> { - let (session, expected) = match session { - SessionTarget::Remembered(identity) => ( - SessionTarget::Attach(Some(identity.session)), - Some(identity), - ), - other => (other, None), - }; - let (mut reader, mut writer, socket, mut ssh): (_, Box, _, _) = - if let Some(remote) = remote { - validate_destination(remote)?; - let executable = options.executable.as_deref().unwrap_or("psyterm"); - let attach = if matches!(session, SessionTarget::Attach(_)) { - " --attach" - } else { - "" - }; - let remote_command = format!("exec {} --bridge{attach}", shell_quote(executable)); - let mut ssh_command = Command::new("ssh"); - if let Some(socket) = &options.agent_socket { - ssh_command.env("SSH_AUTH_SOCK", socket); - } else { - ssh_command.env_remove("SSH_AUTH_SOCK"); - } - // A socketpair allows a bounded, cancellable greeting even when the - // SSH connection succeeds but the remote bridge stalls. - let (reader, stdout) = UnixStream::pair()?; - let mut child = ssh_command - .args([ - "-T", - "-o", - "BatchMode=yes", - "-o", - "ConnectTimeout=10", - "--", - remote, - &remote_command, - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::from(OwnedFd::from(stdout))) - .stderr(Stdio::piped()) - .spawn() - .context("start SSH terminal transport")?; - let writer = Box::new(child.stdin.take().unwrap()) as Box; - (reader, writer, None, Some(child)) - } else { - let stream = - UnixStream::connect(launcher_socket).context("connect to psyTerm server")?; - crate::launcher::verify_peer(&stream)?; - ( - stream.try_clone()?, - Box::new(stream.try_clone()?) as Box, - Some(stream), - None, - ) - }; - let stderr = ssh.as_mut().and_then(|child| child.stderr.take()); - let (commands, receive) = mpsc::channel(); - let (signal, notifications) = async_channel::bounded(1); - let pending = Arc::new(Mutex::new(Update::default())); - let connection = Arc::new(Connection { - launcher_socket: launcher_socket.to_owned(), - remote: remote.map(|destination| RemoteWindow { - destination: destination.into(), - ssh: options, - }), - commands, - socket, - ssh: Mutex::new(ssh), + let direct_window = remote.is_some() && cancelled.is_none(); + let cancelled = cancelled.unwrap_or_default(); + let target = remote.map(|destination| RemoteWindow { + destination: destination.into(), + ssh: options, }); - let diagnostic = stderr.map(ssh_diagnostics).transpose()?; - reader.set_read_timeout(Some(Duration::from_millis(150)))?; - let mut greeting = GreetingReader { - reader: &mut reader, - deadline: Instant::now() + Duration::from_secs(if remote.is_some() { 30 } else { 5 }), - cancelled: cancelled.as_deref(), - }; - let first_frame = match greet( - &mut greeting, - &mut writer, - session, - remote.is_some(), - expected, - ) { - Ok(greeting) => greeting, - Err(error) => { - drop(connection); - if let Some(detail) = diagnostic - .and_then(|rx| rx.recv_timeout(Duration::from_millis(150)).ok()) - .filter(|s| !s.is_empty()) - { - bail!("{error:#}: {detail}"); - } - return Err(error); - } - }; - let remote_session = first_frame + let wire = Wire::open(launcher_socket, target.as_ref(), session, &cancelled)?; + let remote_session = wire + .frame .as_ref() .filter(|_| remote.is_some()) .map(|frame| { @@ -338,54 +260,30 @@ impl Client { directory: Mutex::default(), }) }); - // Mounted SSH connections are already listed by their owning daemon. - let listing = if cancelled.is_none() - && let (Some(target), Some(frame)) = (&connection.remote, &first_frame) - && !frame.tabs.is_empty() - { - RemoteListing::start( - launcher_socket.to_owned(), - target.clone(), - frame, - connection.commands.clone(), - ) - .inspect_err(|error| log::warn!("Could not start SSH window listing: {error:#}")) - .ok() - } else { - None - }; - if let Some(frame) = first_frame { - pending.lock().unwrap().snapshot = Some(Arc::new(frame)); - let _ = signal.try_send(()); - } - reader.set_read_timeout(None)?; - let errors = pending.clone(); - let notify = signal.clone(); - thread::Builder::new() - .name("psyterm-client-writer".into()) - .spawn(move || { - let mut writer = writer; - while let Ok(message) = receive.recv() { - if let Err(error) = protocol::write_message(&mut writer, &message) { - errors.lock().unwrap().error = Some(format!("Connection lost: {error:#}")); - let _ = notify.try_send(()); - break; - } - } - })?; - let incoming = pending.clone(); - let reader_session = remote_session.clone(); - thread::Builder::new() + let connection = Arc::new(Connection { + launcher_socket: launcher_socket.to_owned(), + remote: target, + active: Mutex::default(), + cancelled, + worker: std::sync::OnceLock::new(), + }); + let (signal, notifications) = async_channel::bounded(1); + let pending = Arc::new(Mutex::new(Update::default())); + let mut supervisor = Supervisor::new( + &connection, + remote_session.clone(), + direct_window, + pending.clone(), + signal, + ); + let reader = supervisor.activate(wire, false)?; + let worker = thread::Builder::new() .name("psyterm-client-reader".into()) - .spawn(move || { - read_updates( - reader, - &incoming, - signal, - reader_session.as_deref(), - listing, - ) - })?; + .spawn(move || supervisor.run(reader))?; + connection + .worker + .set(worker.thread().clone()) + .expect("one reader per client"); Ok(( Self { connection, @@ -416,7 +314,9 @@ impl Client { return Vec::new(); }; let mut directory = remote.directory.lock().unwrap(); - directory.refresh(&self.connection.commands); + if let Some(active) = self.connection.active.lock().unwrap().as_ref() { + directory.refresh(&active.commands); + } directory.sessions.clone() } @@ -484,7 +384,11 @@ impl Client { } pub fn send(&self, message: ClientMessage) -> Result<()> { - self.connection + let active = self.connection.active.lock().unwrap(); + let Some(active) = active.as_ref() else { + bail!("server transport is offline"); + }; + active .commands .send(message) .context("server transport is closed") @@ -532,11 +436,11 @@ fn read_updates( pending: &Mutex, signal: async_channel::Sender<()>, remote_session: Option<&RemoteSession>, - mut listing: Option, -) { + mut listing: Option<&mut RemoteListing>, +) -> Result<()> { loop { - let message = protocol::read_message::(&mut reader); - if let Ok(ServerMessage::Sessions(sessions)) = message { + let message = protocol::read_message::(&mut reader)?; + if let ServerMessage::Sessions(sessions) = message { if let Some(remote) = remote_session { let mut directory = remote.directory.lock().unwrap(); directory.received(sessions); @@ -548,7 +452,7 @@ fn read_updates( } let mut update = pending.lock().unwrap(); match message { - Ok(ServerMessage::Snapshot(snapshot)) => { + ServerMessage::Snapshot(snapshot) => { match validate_snapshot(&snapshot) { Ok(()) => { if let Some(session) = remote_session { @@ -570,35 +474,29 @@ fn read_updates( } } } - Ok(ServerMessage::Clipboard(text)) => { + ServerMessage::Clipboard(text) => { if update.clipboard.len() < 16 { update.clipboard.push_back(text); } } - Ok(ServerMessage::Error(error)) => update.error = Some(error), - Ok(ServerMessage::Closing) => { + ServerMessage::Error(error) => update.error = Some(error), + ServerMessage::Closing => { if let Some(session) = remote_session { session.live.store(false, Ordering::Release); } update.disconnected = true; update.error = Some("Server stopped. Shell sessions have ended.".into()); } - Ok(ServerMessage::Pong) => {} - Ok(_) => { + ServerMessage::Pong => {} + _ => { update.error = Some("Unexpected server response after session handshake".into()); } - Err(error) => { - update.disconnected = true; - update.error = Some(format!( - "Disconnected — sessions remain on the server. {error:#}" - )); - } } let disconnected = update.disconnected; drop(update); let _ = signal.try_send(()); if disconnected { - break; + return Ok(()); } } } @@ -938,7 +836,8 @@ mod tests { let pending = Mutex::new(Update::default()); let (signal, received) = async_channel::bounded(1); thread::scope(|threads| { - threads.spawn(|| read_updates(client, &pending, signal, session, None)); + let reader = + threads.spawn(|| read_updates(client, &pending, signal, session, None)); let mut send_snapshot = |snapshot| { protocol::write_message(&mut server, &ServerMessage::Snapshot(snapshot)) .unwrap(); @@ -979,8 +878,8 @@ mod tests { assert!(!resumed.disconnected); server.shutdown(Shutdown::Write).unwrap(); - received.recv_blocking().unwrap(); - assert!(pending.lock().unwrap().disconnected); + let error = reader.join().unwrap().unwrap_err(); + assert!(error.downcast_ref::().is_some()); }); } } @@ -1010,7 +909,7 @@ mod tests { let mut bytes = Vec::new(); protocol::write_message(&mut bytes, &last_message).unwrap(); let (signal, _updates) = async_channel::bounded(1); - read_updates( + let _ = read_updates( bytes.as_slice(), &Mutex::new(Update::default()), signal, diff --git a/src/client/transport.rs b/src/client/transport.rs new file mode 100644 index 0000000..43ca7d3 --- /dev/null +++ b/src/client/transport.rs @@ -0,0 +1,413 @@ +//! Socket/SSH lifetimes. Each successful attachment gets a fresh command queue. +use super::*; +use std::sync::Weak; + +pub(super) struct ActiveTransport { + pub commands: mpsc::Sender, + interrupt: UnixStream, +} + +impl Drop for ActiveTransport { + fn drop(&mut self) { + let _ = self.interrupt.shutdown(Shutdown::Both); + } +} + +struct SshChild(Child); + +impl Drop for SshChild { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +pub(super) struct Wire { + reader: UnixStream, + writer: Box, + child: Option, + pub frame: Option, +} + +impl Wire { + pub fn open( + socket: &Path, + remote: Option<&RemoteWindow>, + target: SessionTarget, + cancelled: &AtomicBool, + ) -> Result { + let (target, expected) = match target { + SessionTarget::Remembered(identity) => ( + SessionTarget::Attach(Some(identity.session)), + Some(identity), + ), + other => (other, None), + }; + let (mut reader, mut writer, mut child): (_, Box, _) = + if let Some(remote) = remote { + validate_destination(&remote.destination)?; + let executable = remote.ssh.executable.as_deref().unwrap_or("psyterm"); + let attach = if matches!(target, SessionTarget::Attach(_)) { + " --attach" + } else { + "" + }; + let command = format!("exec {} --bridge{attach}", shell_quote(executable)); + let mut ssh = Command::new("ssh"); + if let Some(agent) = &remote.ssh.agent_socket { + ssh.env("SSH_AUTH_SOCK", agent); + } else { + ssh.env_remove("SSH_AUTH_SOCK"); + } + // The socketpair also makes partial SSH greetings cancellable. + let (reader, stdout) = UnixStream::pair()?; + let mut child = SshChild( + ssh.args([ + "-T", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=10", + "-o", + "ServerAliveInterval=15", + "-o", + "ServerAliveCountMax=3", + "--", + &remote.destination, + &command, + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::from(OwnedFd::from(stdout))) + .stderr(Stdio::piped()) + .spawn() + .context("start SSH terminal transport")?, + ); + (reader, Box::new(child.0.stdin.take().unwrap()), Some(child)) + } else { + let stream = UnixStream::connect(socket).context("connect to psyTerm server")?; + crate::launcher::verify_peer(&stream)?; + (stream.try_clone()?, Box::new(stream), None) + }; + let diagnostic = child + .as_mut() + .and_then(|child| child.0.stderr.take()) + .map(ssh_diagnostics) + .transpose()?; + reader.set_read_timeout(Some(Duration::from_millis(150)))?; + let mut greeting = GreetingReader { + reader: &mut reader, + deadline: Instant::now() + Duration::from_secs(if remote.is_some() { 30 } else { 5 }), + cancelled: Some(cancelled), + }; + let frame = match greet( + &mut greeting, + &mut writer, + target, + remote.is_some(), + expected, + ) { + Ok(frame) => frame, + Err(error) => { + drop(child); + if let Some(detail) = diagnostic + .and_then(|rx| rx.recv_timeout(Duration::from_millis(150)).ok()) + .filter(|s| !s.is_empty()) + { + // A v6 bridge with --attach exits before Hello when its daemon + // is gone. This is distinct from ssh being unable to reach it. + if expected.is_some() + && detail + .trim_start_matches("Error: ") + .starts_with("no sessions to attach to at ") + { + bail!("the remote daemon with this session has stopped"); + } + return Err(error.context(detail)); + } + return Err(error); + } + }; + reader.set_read_timeout(None)?; + Ok(Self { + reader, + writer, + child, + frame, + }) + } +} + +pub(super) struct Reader { + socket: UnixStream, + // Reap each ssh process on the supervisor thread, including failed handshakes. + _child: Option, +} + +pub(super) struct Supervisor { + owner: Weak, + cancelled: Arc, + remote: Option>, + direct_window: bool, + listing: Option, + pending: Arc>, + signal: async_channel::Sender<()>, +} + +impl Supervisor { + pub fn new( + owner: &Arc, + remote: Option>, + direct_window: bool, + pending: Arc>, + signal: async_channel::Sender<()>, + ) -> Self { + Self { + owner: Arc::downgrade(owner), + cancelled: owner.cancelled.clone(), + remote, + direct_window, + listing: None, + pending, + signal, + } + } + + pub fn activate(&mut self, wire: Wire, reattached: bool) -> Result { + let owner = self.owner.upgrade().context("connection cancelled")?; + let Wire { + reader, + mut writer, + child, + frame, + } = wire; + let (commands, receive) = mpsc::channel(); + let interrupt = reader.try_clone()?; + let wake_reader = reader.try_clone()?; + thread::Builder::new() + .name("psyterm-client-writer".into()) + .spawn(move || { + while let Ok(message) = receive.recv() { + if protocol::write_message(&mut writer, &message).is_err() { + // Only the reader reports a lost transport. Wake it even if + // the write half failed before the next incoming frame. + let _ = wake_reader.shutdown(Shutdown::Both); + break; + } + } + })?; + *owner.active.lock().unwrap() = Some(ActiveTransport { + commands, + interrupt, + }); + if let Some(frame) = frame { + if let Some(remote) = &self.remote { + remote.live.store(!frame.tabs.is_empty(), Ordering::Release); + } + if let Some(listing) = &mut self.listing { + listing.update(&frame); + } else if self.direct_window + && let Some(target) = &owner.remote + && !frame.tabs.is_empty() + { + self.listing = RemoteListing::start( + owner.launcher_socket.clone(), + target.clone(), + &frame, + self.owner.clone(), + ) + .inspect_err(|error| log::warn!("Could not start SSH window listing: {error:#}")) + .ok(); + } + let mut update = self.pending.lock().unwrap(); + update.snapshot = Some(Arc::new(frame)); + if reattached { + update.reconnect = Some(None); + } + drop(update); + let _ = self.signal.try_send(()); + } + Ok(Reader { + socket: reader, + _child: child, + }) + } + + pub fn run(mut self, mut reader: Reader) { + let mut attempt: u32 = 0; + loop { + let connected = Instant::now(); + let result = read_updates( + &mut reader.socket, + &self.pending, + self.signal.clone(), + self.remote.as_deref(), + self.listing.as_mut(), + ); + if let Some(owner) = self.owner.upgrade() { + owner.active.lock().unwrap().take(); + } + drop(reader); + if self.cancelled.load(Ordering::Acquire) { + return; + } + let Err(error) = result else { return }; + let Some(remote) = self.remote.as_ref() else { + self.finish(format!( + "Disconnected — sessions remain on the server. {error:#}" + )); + return; + }; + if !remote.live.load(Ordering::Acquire) { + return; + } + let identity = remote.identity; + if error.downcast_ref::().is_none() { + self.finish(format!("Could not read the SSH session: {error:#}")); + return; + } + *remote.directory.lock().unwrap() = RemoteDirectory::default(); + if let Some(listing) = &mut self.listing { + listing.update_sessions(&[]); + } + // A flapping transport retains its backoff even if a greeting succeeded. + if connected.elapsed() >= Duration::from_secs(30) { + attempt = 0; + } + loop { + attempt = attempt.saturating_add(1); + let delay_secs = retry_delay(attempt); + self.set_status(ReconnectStatus::Waiting { + attempt, + delay_secs, + }); + let deadline = Instant::now() + Duration::from_secs(delay_secs.into()); + while !self.cancelled.load(Ordering::Acquire) && Instant::now() < deadline { + thread::park_timeout(deadline.saturating_duration_since(Instant::now())); + } + if self.cancelled.load(Ordering::Acquire) { + return; + } + self.set_status(ReconnectStatus::Connecting { attempt }); + let Some(owner) = self.owner.upgrade() else { + return; + }; + let path = owner.launcher_socket.clone(); + let target = owner.remote.clone(); + drop(owner); // Closing a window can cancel a handshake in progress. + let result = Wire::open( + &path, + target.as_ref(), + SessionTarget::Remembered(identity), + &self.cancelled, + ); + if self.cancelled.load(Ordering::Acquire) { + return; + } + match result.and_then(|wire| self.activate(wire, true)) { + Ok(attached) => { + reader = attached; + break; + } + Err(error) if error.downcast_ref::().is_some() => { + log::debug!("SSH reconnect attempt {attempt}: {error:#}"); + } + Err(error) => { + if let Some(remote) = &self.remote { + remote.live.store(false, Ordering::Release); + } + self.finish(format!("Could not reattach to the SSH session: {error:#}")); + return; + } + } + } + } + } + + fn set_status(&self, status: ReconnectStatus) { + self.pending.lock().unwrap().reconnect = Some(Some(status)); + let _ = self.signal.try_send(()); + } + + fn finish(&self, message: String) { + let mut update = self.pending.lock().unwrap(); + update.disconnected = true; + update.reconnect = Some(None); + update.error = Some(message); + drop(update); + let _ = self.signal.try_send(()); + } +} + +fn retry_delay(attempt: u32) -> u32 { + (1 << attempt.saturating_sub(1).min(5)).min(30) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn dropping_a_session_cancels_its_wait_and_discards_offline_input() { + let (socket, peer) = UnixStream::pair().unwrap(); + let wire = Wire { + writer: Box::new(socket.try_clone().unwrap()), + reader: socket, + child: None, + frame: None, + }; + let owner = Arc::new(Connection { + launcher_socket: PathBuf::new(), + remote: None, + active: Mutex::default(), + cancelled: Arc::default(), + worker: std::sync::OnceLock::new(), + }); + let remote = Arc::new(RemoteSession { + identity: SessionIdentity { + server: [1; 16], + session: 7, + }, + live: AtomicBool::new(true), + directory: Mutex::default(), + }); + let client = Client { + connection: owner.clone(), + remote_session: Some(remote.clone()), + }; + let (signal, changes) = async_channel::bounded(1); + let pending = Arc::new(Mutex::new(Update::default())); + let mut supervisor = Supervisor::new(&owner, Some(remote), false, pending.clone(), signal); + let reader = supervisor.activate(wire, false).unwrap(); + let (done, stopped) = mpsc::channel(); + let worker = thread::spawn(move || { + supervisor.run(reader); + done.send(()).unwrap(); + }); + owner.worker.set(worker.thread().clone()).unwrap(); + drop(owner); + + drop(peer); + changes.recv_blocking().unwrap(); + let update = std::mem::take(&mut *pending.lock().unwrap()); + assert_eq!( + update.reconnect, + Some(Some(ReconnectStatus::Waiting { + attempt: 1, + delay_secs: 1 + })) + ); + assert!(!update.disconnected && update.error.is_none()); + assert!(client.input(1, b"do not replay".to_vec()).is_err()); + + drop(client); + stopped.recv_timeout(Duration::from_secs(5)).unwrap(); + worker.join().unwrap(); + assert!(changes.recv_blocking().is_err()); + } + + #[test] + fn retries_have_a_minimum_delay_and_a_bounded_exponential_backoff() { + let delays: Vec<_> = (1..=8).map(retry_delay).collect(); + assert_eq!(delays, [1, 2, 4, 8, 16, 30, 30, 30]); + assert_eq!(retry_delay(u32::MAX), 30); + } +} diff --git a/src/protocol.rs b/src/protocol.rs index 347ab4f..fc999cb 100644 --- a/src/protocol.rs +++ b/src/protocol.rs @@ -343,6 +343,27 @@ pub enum ServerMessage { }, } +/// Local transport state. Mounted tabs convey the description in a v6 connecting tab. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReconnectStatus { + Waiting { attempt: u32, delay_secs: u32 }, + Connecting { attempt: u32 }, +} + +impl ReconnectStatus { + pub fn description(self, host: &str) -> String { + match self { + Self::Waiting { + attempt, + delay_secs, + } => { + format!("Reconnecting to {host} · attempt {attempt} after {delay_secs}s") + } + Self::Connecting { attempt } => format!("Reconnecting to {host} · attempt {attempt}…"), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Snapshot { /// This session and the other sessions contributing tabs to its layout. diff --git a/src/server.rs b/src/server.rs index c4b1184..22effc2 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1581,6 +1581,9 @@ impl SessionState { self.update_titles(); self.revision += 1; let mut tabs = self.tabs.clone(); + for remote in self.remotes.values() { + remote.show_reconnect(&mut tabs); + } if let Some(remote) = self .remotes .values() diff --git a/src/server/remote.rs b/src/server/remote.rs index e48c8a2..c5d78a6 100644 --- a/src/server/remote.rs +++ b/src/server/remote.rs @@ -24,6 +24,7 @@ pub(super) struct RemoteSession { panes: HashMap, latest: Option>, attaching: Option, + reconnect: Option, requested_tab: Option, clipboard: VecDeque<(PaneId, Weak)>, } @@ -100,6 +101,11 @@ impl RemoteSession { } pub(super) fn send(&self, command: ClientMessage) -> Result<()> { + // Commands such as keystrokes or paste must never be replayed into the + // resumed shell. Local tab focus and explicit detach still work. + if self.reconnect.is_some() { + return Ok(()); + } self.connection .as_ref() .context("SSH session is connecting")? @@ -109,6 +115,10 @@ impl RemoteSession { // Keep one copy in flight on this connection. An attached session can // contain multiple SSH hosts whose clipboard replies arrive out of order. fn copy(&mut self, pane: PaneId, output: &Arc) -> Result<()> { + if self.reconnect.is_some() { + output.reliable(ServerMessage::Clipboard(String::new())); + return Ok(()); + } if self.clipboard.len() >= 16 { output.reliable(ServerMessage::Clipboard(String::new())); anyhow::bail!("too many pending clipboard requests"); @@ -147,7 +157,25 @@ impl RemoteSession { } } + // Project retrying tabs as v6 connecting placeholders. Keep their mappings + // and last good frame intact, even when another view joins during the outage. + pub(super) fn show_reconnect(&self, tabs: &mut [TabInfo]) { + let Some(status) = &self.reconnect else { + return; + }; + for tab in tabs.iter_mut().filter(|tab| self.owns_tab(tab.id)) { + tab.remote = Some(self.target.destination.clone()); + tab.title = status.description(&self.target.destination); + tab.panes.clear(); + tab.stacks.clear(); + tab.active_pane = 0; + } + } + pub(super) fn visible_panes(&self, tab: &mut TabInfo) -> Vec { + if tab.is_connecting() { + return Vec::new(); + } let Some(frame) = self.latest.as_ref() else { return Vec::new(); }; @@ -415,7 +443,7 @@ impl SessionState { index + 1, TabInfo { id: group, - title: "Connecting…".into(), + title: format!("Connecting to {}…", target.destination), remote: Some(target.destination.clone()), manual_title: None, panes: Vec::new(), @@ -439,6 +467,7 @@ impl SessionState { panes: HashMap::new(), latest: None, attaching, + reconnect: None, requested_tab: None, clipboard: VecDeque::new(), }, @@ -475,20 +504,22 @@ impl SessionState { .context("tab is not attached to an SSH session")?; // A nested SSH tab is owned by the upstream workspace. Detach it // there so the enclosing SSH session and its other tabs stay open. - if let Some(nested) = remote.nested_tab(tab) { + if remote.reconnect.is_none() + && let Some(nested) = remote.nested_tab(tab) + { remote.send(ClientMessage::DetachRemote { tab: nested.id })?; return Ok(None); } - let target = remote.connection.as_ref().and_then(|client| { - let identity = client - .remote_identity() - .or_else(|| remote.latest.as_ref()?.lineage.first().copied())?; - Some(DetachedRemote { + let target = remote + .latest + .as_ref() + .and_then(|frame| frame.lineage.first().copied()) + .or_else(|| remote.connection.as_ref()?.remote_identity()) + .map(|identity| DetachedRemote { destination: remote.target.destination.clone(), identity, executable: remote.target.ssh.executable.clone(), - }) - }); + }); self.remove_remote(group); Ok(target) } @@ -616,6 +647,23 @@ impl Server { } RemoteEvent::Ended(error) => (true, error), RemoteEvent::Updated(update) => { + if let Some(status) = update.reconnect { + remote.reconnect = status; + state.dirty = true; + if remote.reconnect.is_some() { + remote.cancel_copies(); + } else if let Some(tab) = remote.requested_tab.take() + && remote.owns_tab(state.active_tab) + && update.snapshot.as_ref().is_some_and(|frame| { + frame.active_tab != tab && frame.tabs.iter().any(|t| t.id == tab) + }) + && remote.send(ClientMessage::FocusTab { tab }).is_ok() + { + // Tab switches made while offline stay local until the + // recovered daemon confirms the tab still exists. + remote.requested_tab = Some(tab); + } + } for text in update.clipboard { if let Err(error) = remote.finish_copy(text) { log::warn!("SSH clipboard: {error}"); @@ -1004,17 +1052,9 @@ mod tests { assert_eq!(frame.panes[0].id, top); } - #[test] - fn discarded_ssh_frame_keeps_the_mounted_workspace_and_accepts_the_next_frame() { - let mut upstream = session(2); - upstream.new_tab(None).unwrap(); - let mut local = session(1); - local.new_tab(None).unwrap(); - let origin = local.active_tab; - let group = reserve(&mut local, 0); - apply(&mut local, group, upstream.snapshot()).unwrap(); + fn server_with_session(local: SessionState) -> Server { let id = local.id; - let mut server = Server { + Server { identity: local.identity.server, socket: local.socket.clone(), shell: None, @@ -1025,7 +1065,117 @@ mod tests { last_detached_remote: None, last_frame: Instant::now(), last_cwd_refresh: Instant::now(), + } + } + + #[test] + fn reconnecting_mount_keeps_tab_ids_and_restores_panes_without_stealing_focus() { + let mut upstream = session(2); + upstream.new_tab(None).unwrap(); + upstream.new_tab(None).unwrap(); + let baseline = upstream.snapshot(); + let mut local = session(1); + local.new_tab(None).unwrap(); + let local_tab = local.active_tab; + let group = reserve(&mut local, 0); + apply(&mut local, group, baseline.clone()).unwrap(); + let before = local.snapshot(); + let ids = before.tabs.iter().map(|tab| tab.id).collect::>(); + let id = local.id; + let mut server = server_with_session(local); + let waiting = ReconnectStatus::Waiting { + attempt: 2, + delay_secs: 2, }; + server.remote_event( + id, + group, + RemoteEvent::Updated(Update { + reconnect: Some(Some(waiting)), + ..Update::default() + }), + ); + + let local = server.sessions.get_mut(&id).unwrap(); + let waiting_frame = local.snapshot(); + crate::client::validate_snapshot(&waiting_frame).unwrap(); + assert!( + waiting_frame + .tabs + .iter() + .map(|tab| tab.id) + .eq(ids.iter().copied()) + ); + assert!( + waiting_frame.tabs[1..].iter().all(|tab| { + tab.is_connecting() && tab.title == waiting.description("user@work") + }) + ); + assert!(waiting_frame.panes.is_empty()); + let offline_pane = before.active_tab().unwrap().active_pane; + let clipboard = Arc::new(Outbox::default()); + local + .command(ClientMessage::Copy { pane: offline_pane }, &clipboard) + .unwrap(); + assert!( + matches!(clipboard.next(), Some(ServerMessage::Clipboard(text)) if text.is_empty()) + ); + local.focus_tab(ids[1]).unwrap(); + assert_eq!(local.active_tab, ids[1]); + local.focus_tab(local_tab).unwrap(); + local.dirty = false; + + server.remote_event( + id, + group, + RemoteEvent::Updated(Update { + reconnect: Some(None), + snapshot: Some(Arc::new(baseline)), + ..Update::default() + }), + ); + let local = server.sessions.get_mut(&id).unwrap(); + assert!(local.dirty); // Even an identical layout must replace the retry placeholder. + assert_eq!(local.active_tab, local_tab); + local.focus_tab(before.active_tab).unwrap(); + let after = local.snapshot(); + assert_eq!(after.tabs, before.tabs); + assert_eq!(after.panes[0].id, offline_pane); + } + + #[test] + fn cancelling_a_nested_tab_while_its_outer_host_is_offline_remembers_the_outer_session() { + let mut upstream = session(2); + upstream.new_tab(None).unwrap(); + let mut nested = upstream.snapshot(); + nested.tabs[0].remote = Some("inner".into()); + let mut local = session(1); + local.new_tab(None).unwrap(); + let local_tab = local.active_tab; + let group = reserve(&mut local, 0); + apply(&mut local, group, nested).unwrap(); + let mount = local.remotes.get_mut(&group).unwrap(); + mount.reconnect = Some(ReconnectStatus::Connecting { attempt: 1 }); + let cancelled = mount.cancelled.clone(); + let target = local.detach_remote(group).unwrap().unwrap(); + assert_eq!(target.identity, upstream.identity); + assert_eq!(target.destination, "user@work"); + assert!(cancelled.load(Ordering::Acquire)); + assert_eq!(local.tabs.len(), 1); + assert_eq!(local.active_tab, local_tab); + } + + #[test] + fn discarded_ssh_frame_keeps_the_mounted_workspace_and_accepts_the_next_frame() { + let mut upstream = session(2); + upstream.new_tab(None).unwrap(); + let mut local = session(1); + local.new_tab(None).unwrap(); + let origin = local.active_tab; + let group = reserve(&mut local, 0); + apply(&mut local, group, upstream.snapshot()).unwrap(); + let id = local.id; + let mut server = server_with_session(local); server.remote_event( id, group, diff --git a/src/terminal.rs b/src/terminal.rs index 2885aeb..d33c6f7 100644 --- a/src/terminal.rs +++ b/src/terminal.rs @@ -1,4 +1,5 @@ //! Display-only workspace and panes. All terminal and multiplexer state is remote. +mod reconnect; mod session_picker; use crate::{ @@ -13,8 +14,8 @@ use crate::{ palette::CommandPalette, pane_name::RenameField, protocol::{ - ClientMessage, Direction, PaneSnapshot, ScrollKind, SelectionCommand, SelectionKind, - SelectionSide, Snapshot, StackPaneInfo, VerticalDirection, + ClientMessage, Direction, PaneSnapshot, ReconnectStatus, ScrollKind, SelectionCommand, + SelectionKind, SelectionSide, Snapshot, StackPaneInfo, VerticalDirection, }, render::{RenderCell, Renderer}, transition::Slide, @@ -49,6 +50,8 @@ pub struct Workspace { session_directory: SessionDirectory, session_refresh: Option>, disconnected: bool, + reconnect: Option, + interrupted_titles: HashMap, tab_rename: Option, outgoing: HashMap>, outgoing_snapshot: Option>, @@ -229,6 +232,10 @@ impl Workspace { else { return; }; + if self.reconnect.is_some() { + window.remove_window(); + return; + } let message = if tab.remote.is_some() { ClientMessage::DetachRemote { tab: id } } else if self.client.remote().is_some() { @@ -245,6 +252,17 @@ impl Workspace { fn execute(&mut self, action: CommandAction, window: &mut Window, cx: &mut Context) { use CommandAction::*; + if self.reconnect.is_some() { + match action { + DetachRemote | Detach | ClosePane => { + window.remove_window(); + return; + } + TogglePalette | SwitchSession(_) | NewWindow | IncreaseFontSize + | DecreaseFontSize => {} + _ => return, + } + } let pane = self.focused_pane(); let pane_id = pane.as_ref().map(|pane| pane.read(cx).data().id); let message = match action { @@ -397,6 +415,9 @@ impl Workspace { cx.write_to_clipboard(ClipboardItem::new_string(text)); } view.disconnected |= update.disconnected; + if let Some(status) = update.reconnect { + view.update_reconnect(status, window, cx); + } if let Some(error) = update.error { view.error = Some(error); } @@ -410,6 +431,9 @@ impl Workspace { old.active_tab != snapshot.active_tab || old.tabs != snapshot.tabs }); + if metadata_changed { + view.update_interrupted_titles(&snapshot); + } let cancelled_tab_rename = view.tab_rename.as_ref().is_some_and(|rename| { snapshot.active_tab != rename.tab @@ -418,8 +442,9 @@ impl Workspace { if cancelled_tab_rename { view.tab_rename = None; } - let focus_allowed = - view.palette.is_none() && view.tab_rename.is_none(); + let focus_allowed = view.palette.is_none() + && view.tab_rename.is_none() + && view.reconnect.is_none(); if let Some(direction) = view .snapshot .as_ref() @@ -498,6 +523,7 @@ impl Workspace { window.focus(&pane.focus); } pane.transitioning = view.transition.is_some(); + pane.read_only = view.reconnect.is_some(); pane }); view.panes.insert(data.id, pane); @@ -544,6 +570,8 @@ impl Workspace { session_directory: SessionDirectory::Loading, session_refresh: None, disconnected: false, + reconnect: None, + interrupted_titles: HashMap::new(), tab_rename: None, outgoing: HashMap::new(), outgoing_snapshot: None, @@ -586,7 +614,14 @@ impl Render for Workspace { .iter() .map(|tab| chrome::ChromeTab { id: tab.id, - label: tab.title.clone().into(), + label: self + .interrupted_titles + .get(&tab.id) + .unwrap_or(&tab.title) + .clone() + .into(), + reconnecting: self.reconnect.is_some() + || self.interrupted_titles.contains_key(&tab.id), remote: tab .remote .as_deref() @@ -636,18 +671,6 @@ impl Render for Workspace { let mut incoming = scene(incoming_offset).id("incoming-scene"); if let Some(snapshot) = &self.snapshot { incoming = pane_scene(snapshot, &self.panes, incoming_offset, Some(&self.client)); - if let Some(host) = snapshot - .active_tab() - .filter(|tab| tab.is_connecting()) - .and_then(|tab| tab.remote.as_ref()) - { - incoming = incoming.child( - div() - .p_4() - .text_color(rgb(chrome::MUTED)) - .child(format!("Connecting to {host}…")), - ); - } } else { incoming = incoming.child( div() @@ -657,6 +680,9 @@ impl Render for Workspace { ); } body = body.child(incoming); + if let Some((tab, message)) = self.connection_notice() { + body = body.child(self.connection_panel(tab, message, cx)); + } let mut root = div() .relative() .track_focus(&self.focus) @@ -841,7 +867,7 @@ struct TerminalPane { snapshot: Arc, index: usize, active: bool, - // Frozen outgoing scenes are display-only; no PTY resize, focus or input. + // Outgoing scenes and disconnected SSH panes are display-only. read_only: bool, transitioning: bool, focus: FocusHandle, diff --git a/src/terminal/reconnect.rs b/src/terminal/reconnect.rs new file mode 100644 index 0000000..7ca07e1 --- /dev/null +++ b/src/terminal/reconnect.rs @@ -0,0 +1,135 @@ +use super::*; + +impl Workspace { + pub(super) fn update_reconnect( + &mut self, + status: Option, + window: &mut Window, + cx: &mut Context, + ) { + let was_offline = self.reconnect.is_some(); + self.reconnect = status; + if was_offline == status.is_some() { + return; + } + let read_only = status.is_some() || self.disconnected; + for pane in self.panes.values() { + pane.update(cx, |pane, cx| { + pane.read_only = read_only; + pane.selecting = false; + pane.preedit.clear(); + pane.rename = None; + pane.requested_size = None; + cx.notify(); + }); + } + self.tab_rename = None; + if self.palette.is_none() { + if read_only { + window.focus(&self.focus); + } else { + self.restore_focus(window, cx); + } + } + } + + pub(super) fn update_interrupted_titles(&mut self, snapshot: &Snapshot) { + self.interrupted_titles.retain(|id, _| { + snapshot + .tabs + .iter() + .any(|tab| tab.id == *id && tab.is_connecting()) + }); + if let Some(previous) = &self.snapshot { + for tab in snapshot.tabs.iter().filter(|tab| tab.is_connecting()) { + if let Some(old) = previous + .tabs + .iter() + .find(|old| old.id == tab.id && !old.is_connecting()) + { + self.interrupted_titles.insert(tab.id, old.title.clone()); + } + } + } + } + + pub(super) fn connection_notice(&self) -> Option<(u64, String)> { + let tab = self.snapshot.as_ref()?.active_tab()?; + // A direct connection owns the whole window, including any nested SSH tabs. + if let Some(status) = self.reconnect { + return Some((tab.id, status.description(self.client.remote()?))); + } + if !tab.is_connecting() { + return None; + } + let text = if tab.title == "Connecting…" { + // Initial placeholder from an earlier v6 daemon. + format!("Connecting to {}…", tab.remote.as_deref()?) + } else { + tab.title.clone() + }; + Some((tab.id, text)) + } + + pub(super) fn connection_panel( + &self, + tab: u64, + message: String, + cx: &mut Context, + ) -> Div { + div() + .absolute() + .inset_0() + .flex() + .items_center() + .justify_center() + .px_4() + .bg(rgba(0x06060f99)) + .on_mouse_down(MouseButton::Left, |_, _, cx| cx.stop_propagation()) + .on_scroll_wheel(|_, _, cx| cx.stop_propagation()) + .child( + div() + .flex() + .items_center() + .gap_3() + .px_4() + .py_3() + .border_1() + .border_color(rgb(chrome::BORDER)) + .bg(rgb(chrome::PANEL_BACKGROUND)) + .font_family(".SystemUIFont") + .text_size(px(13.)) + .text_color(rgb(chrome::TEXT)) + .child( + div() + .flex_shrink_0() + .w(px(3.)) + .h(px(22.)) + .bg(rgb(chrome::PANE_FOCUS)), + ) + .child(div().min_w_0().child(message)) + .child( + div() + .id("cancel-ssh-connection") + .flex_shrink_0() + .px_3() + .py_1() + .border_1() + .border_color(rgb(chrome::BORDER)) + .bg(rgb(chrome::CONTROL_BACKGROUND)) + .hover(|style| style.bg(rgb(chrome::HOVER_BACKGROUND))) + .cursor_pointer() + .child("Cancel") + .on_click(cx.listener(move |view, _, window, cx| { + cx.stop_propagation(); + if view + .connection_notice() + .is_some_and(|(active, _)| active == tab) + { + view.close_tab(tab, window, cx); + } + })), + ), + ) + } +}