Reconnect interrupted SSH sessions with cancellable backoff

This commit is contained in:
James Magahern
2026-11-17 20:46:45 -08:00
parent 804ab007ee
commit 17246675ed
9 changed files with 882 additions and 221 deletions
+9 -2
View File
@@ -183,6 +183,13 @@ to find remote session IDs. Closing the window and reattaching to the local
session restores its mixed tab bar. The **+** button creates a local tab in
this workspace. Splitting a remote pane creates another shell on its SSH host.
If an established SSH connection drops, psyTerm keeps its tabs and tries to
reattach to the same remote session. The active tab shows the retry status and
a **Cancel** button. Retries start after one second and back off to a maximum
of 30 seconds between attempts. Other local tabs stay usable. Cancel detaches
the interrupted SSH tabs, or closes the window for a standalone SSH session.
You can reattach later if the remote daemon still has the session.
To detach, hover over a remote tab and click **×**, or focus it and choose
**Detach remote session** in the command palette. If that SSH session has
several tabs, they detach together. Its shells, running jobs, and scrollback
@@ -195,8 +202,8 @@ in a window. The local daemon remembers these targets until it exits.
If you detach a nested SSH session, run the command on the host that opened it.
You can also select a session explicitly with `psyterm --remote user@host --attach ID`.
The local daemon must be running this build to close tabs, remember detached
sessions, and pass SSH settings to new windows. Existing protocol-v6 sessions
The local daemon must be running this build to reconnect SSH tabs, remember
detached sessions, and pass SSH settings to new windows. Existing protocol-v6 sessions
on the SSH host can stay running.
SSH uses `BatchMode=yes`. Configure key or agent authentication and trust the
+10 -3
View File
@@ -60,6 +60,7 @@ pub struct ChromeTab {
pub label: SharedString,
pub remote: Option<String>,
pub active: bool,
pub reconnecting: bool,
pub editor: Option<Entity<RenameField>>,
}
@@ -112,15 +113,19 @@ pub fn header(
);
}
if let Some(destination) = tab.remote {
let host = remote_hostname(&destination);
let subtitle = if tab.reconnecting {
format!("{host} · Reconnecting")
} else {
host.to_owned()
};
label = label.child(
div()
.h(px(12.))
.line_height(px(12.))
.text_size(px(10.))
.text_color(rgb(PANE_FOCUS))
.child(middle_label(
remote_hostname(&destination).to_owned().into(),
)),
.child(middle_label(subtitle.into())),
);
}
tab_strip = tab_strip.child(
@@ -314,6 +319,7 @@ mod tests {
label: "Shell".into(),
remote: None,
active: true,
reconnecting: false,
editor: None,
},
ChromeTab {
@@ -321,6 +327,7 @@ mod tests {
label: "Build".into(),
remote: Some("user@work".into()),
active: false,
reconnecting: true,
editor: None,
},
],
+78 -179
View File
@@ -1,7 +1,11 @@
//! Display-only client transport. No terminal parser, PTY, or shell lives here.
mod transport;
use transport::{ActiveTransport, Supervisor, Wire};
use crate::protocol::{
self, ClientMessage, RemoteSessionInfo, RemoteWindow, ServerMessage, SessionIdentity,
SessionInfo, SessionRoute, SessionSummary, SessionTarget, Snapshot, SshOptions, TabId,
self, ClientMessage, ReconnectStatus, RemoteSessionInfo, RemoteWindow, ServerMessage,
SessionIdentity, SessionInfo, SessionRoute, SessionSummary, SessionTarget, Snapshot,
SshOptions, TabId,
};
use anyhow::{Context, Result, bail, ensure};
use std::{
@@ -26,6 +30,8 @@ pub struct Update {
pub clipboard: VecDeque<String>,
pub error: Option<String>,
pub disconnected: bool,
/// A change in the SSH retry state; the inner None clears the status.
pub reconnect: Option<Option<ReconnectStatus>>,
}
pub struct Updates {
@@ -111,7 +117,7 @@ impl RemoteListing {
socket: PathBuf,
target: RemoteWindow,
frame: &Snapshot,
commands: mpsc::Sender<ClientMessage>,
connection: std::sync::Weak<Connection>,
) -> Result<Self> {
let identity = frame.lineage[0]; // Verified in greet().
let latest = Arc::new(Mutex::new(RemoteWindowListing {
@@ -131,7 +137,11 @@ impl RemoteListing {
changes,
published,
|| {
let _ = commands.send(ClientMessage::ListSessions);
if let Some(owner) = connection.upgrade()
&& let Some(active) = owner.active.lock().unwrap().as_ref()
{
let _ = active.commands.send(ClientMessage::ListSessions);
}
},
) {
log::warn!("Could not list SSH window on local server: {error:#}");
@@ -161,20 +171,19 @@ impl RemoteListing {
struct Connection {
launcher_socket: PathBuf,
remote: Option<RemoteWindow>,
commands: mpsc::Sender<ClientMessage>,
socket: Option<UnixStream>,
ssh: Mutex<Option<Child>>,
active: Mutex<Option<ActiveTransport>>,
cancelled: Arc<AtomicBool>,
worker: std::sync::OnceLock<thread::Thread>,
}
impl Drop for Connection {
fn drop(&mut self) {
// Detach closes this transport, never a server-owned PTY.
if let Some(socket) = &self.socket {
let _ = socket.shutdown(Shutdown::Both);
}
if let Some(mut child) = self.ssh.lock().unwrap().take() {
let _ = child.kill();
let _ = child.wait();
self.cancelled.store(true, Ordering::Release);
// Wake either a sleeping retry or a reader on the current transport.
if let Some(worker) = self.worker.get() {
worker.unpark();
}
self.active.get_mut().unwrap().take();
}
}
@@ -233,102 +242,15 @@ impl Client {
options: SshOptions,
cancelled: Option<Arc<AtomicBool>>,
) -> Result<(Self, Updates)> {
let (session, expected) = match session {
SessionTarget::Remembered(identity) => (
SessionTarget::Attach(Some(identity.session)),
Some(identity),
),
other => (other, None),
};
let (mut reader, mut writer, socket, mut ssh): (_, Box<dyn Write + Send>, _, _) =
if let Some(remote) = remote {
validate_destination(remote)?;
let executable = options.executable.as_deref().unwrap_or("psyterm");
let attach = if matches!(session, SessionTarget::Attach(_)) {
" --attach"
} else {
""
};
let remote_command = format!("exec {} --bridge{attach}", shell_quote(executable));
let mut ssh_command = Command::new("ssh");
if let Some(socket) = &options.agent_socket {
ssh_command.env("SSH_AUTH_SOCK", socket);
} else {
ssh_command.env_remove("SSH_AUTH_SOCK");
}
// A socketpair allows a bounded, cancellable greeting even when the
// SSH connection succeeds but the remote bridge stalls.
let (reader, stdout) = UnixStream::pair()?;
let mut child = ssh_command
.args([
"-T",
"-o",
"BatchMode=yes",
"-o",
"ConnectTimeout=10",
"--",
remote,
&remote_command,
])
.stdin(Stdio::piped())
.stdout(Stdio::from(OwnedFd::from(stdout)))
.stderr(Stdio::piped())
.spawn()
.context("start SSH terminal transport")?;
let writer = Box::new(child.stdin.take().unwrap()) as Box<dyn Write + Send>;
(reader, writer, None, Some(child))
} else {
let stream =
UnixStream::connect(launcher_socket).context("connect to psyTerm server")?;
crate::launcher::verify_peer(&stream)?;
(
stream.try_clone()?,
Box::new(stream.try_clone()?) as Box<dyn Write + Send>,
Some(stream),
None,
)
};
let stderr = ssh.as_mut().and_then(|child| child.stderr.take());
let (commands, receive) = mpsc::channel();
let (signal, notifications) = async_channel::bounded(1);
let pending = Arc::new(Mutex::new(Update::default()));
let connection = Arc::new(Connection {
launcher_socket: launcher_socket.to_owned(),
remote: remote.map(|destination| RemoteWindow {
destination: destination.into(),
ssh: options,
}),
commands,
socket,
ssh: Mutex::new(ssh),
let direct_window = remote.is_some() && cancelled.is_none();
let cancelled = cancelled.unwrap_or_default();
let target = remote.map(|destination| RemoteWindow {
destination: destination.into(),
ssh: options,
});
let diagnostic = stderr.map(ssh_diagnostics).transpose()?;
reader.set_read_timeout(Some(Duration::from_millis(150)))?;
let mut greeting = GreetingReader {
reader: &mut reader,
deadline: Instant::now() + Duration::from_secs(if remote.is_some() { 30 } else { 5 }),
cancelled: cancelled.as_deref(),
};
let first_frame = match greet(
&mut greeting,
&mut writer,
session,
remote.is_some(),
expected,
) {
Ok(greeting) => greeting,
Err(error) => {
drop(connection);
if let Some(detail) = diagnostic
.and_then(|rx| rx.recv_timeout(Duration::from_millis(150)).ok())
.filter(|s| !s.is_empty())
{
bail!("{error:#}: {detail}");
}
return Err(error);
}
};
let remote_session = first_frame
let wire = Wire::open(launcher_socket, target.as_ref(), session, &cancelled)?;
let remote_session = wire
.frame
.as_ref()
.filter(|_| remote.is_some())
.map(|frame| {
@@ -338,54 +260,30 @@ impl Client {
directory: Mutex::default(),
})
});
// Mounted SSH connections are already listed by their owning daemon.
let listing = if cancelled.is_none()
&& let (Some(target), Some(frame)) = (&connection.remote, &first_frame)
&& !frame.tabs.is_empty()
{
RemoteListing::start(
launcher_socket.to_owned(),
target.clone(),
frame,
connection.commands.clone(),
)
.inspect_err(|error| log::warn!("Could not start SSH window listing: {error:#}"))
.ok()
} else {
None
};
if let Some(frame) = first_frame {
pending.lock().unwrap().snapshot = Some(Arc::new(frame));
let _ = signal.try_send(());
}
reader.set_read_timeout(None)?;
let errors = pending.clone();
let notify = signal.clone();
thread::Builder::new()
.name("psyterm-client-writer".into())
.spawn(move || {
let mut writer = writer;
while let Ok(message) = receive.recv() {
if let Err(error) = protocol::write_message(&mut writer, &message) {
errors.lock().unwrap().error = Some(format!("Connection lost: {error:#}"));
let _ = notify.try_send(());
break;
}
}
})?;
let incoming = pending.clone();
let reader_session = remote_session.clone();
thread::Builder::new()
let connection = Arc::new(Connection {
launcher_socket: launcher_socket.to_owned(),
remote: target,
active: Mutex::default(),
cancelled,
worker: std::sync::OnceLock::new(),
});
let (signal, notifications) = async_channel::bounded(1);
let pending = Arc::new(Mutex::new(Update::default()));
let mut supervisor = Supervisor::new(
&connection,
remote_session.clone(),
direct_window,
pending.clone(),
signal,
);
let reader = supervisor.activate(wire, false)?;
let worker = thread::Builder::new()
.name("psyterm-client-reader".into())
.spawn(move || {
read_updates(
reader,
&incoming,
signal,
reader_session.as_deref(),
listing,
)
})?;
.spawn(move || supervisor.run(reader))?;
connection
.worker
.set(worker.thread().clone())
.expect("one reader per client");
Ok((
Self {
connection,
@@ -416,7 +314,9 @@ impl Client {
return Vec::new();
};
let mut directory = remote.directory.lock().unwrap();
directory.refresh(&self.connection.commands);
if let Some(active) = self.connection.active.lock().unwrap().as_ref() {
directory.refresh(&active.commands);
}
directory.sessions.clone()
}
@@ -484,7 +384,11 @@ impl Client {
}
pub fn send(&self, message: ClientMessage) -> Result<()> {
self.connection
let active = self.connection.active.lock().unwrap();
let Some(active) = active.as_ref() else {
bail!("server transport is offline");
};
active
.commands
.send(message)
.context("server transport is closed")
@@ -532,11 +436,11 @@ fn read_updates(
pending: &Mutex<Update>,
signal: async_channel::Sender<()>,
remote_session: Option<&RemoteSession>,
mut listing: Option<RemoteListing>,
) {
mut listing: Option<&mut RemoteListing>,
) -> Result<()> {
loop {
let message = protocol::read_message::<ServerMessage>(&mut reader);
if let Ok(ServerMessage::Sessions(sessions)) = message {
let message = protocol::read_message::<ServerMessage>(&mut reader)?;
if let ServerMessage::Sessions(sessions) = message {
if let Some(remote) = remote_session {
let mut directory = remote.directory.lock().unwrap();
directory.received(sessions);
@@ -548,7 +452,7 @@ fn read_updates(
}
let mut update = pending.lock().unwrap();
match message {
Ok(ServerMessage::Snapshot(snapshot)) => {
ServerMessage::Snapshot(snapshot) => {
match validate_snapshot(&snapshot) {
Ok(()) => {
if let Some(session) = remote_session {
@@ -570,35 +474,29 @@ fn read_updates(
}
}
}
Ok(ServerMessage::Clipboard(text)) => {
ServerMessage::Clipboard(text) => {
if update.clipboard.len() < 16 {
update.clipboard.push_back(text);
}
}
Ok(ServerMessage::Error(error)) => update.error = Some(error),
Ok(ServerMessage::Closing) => {
ServerMessage::Error(error) => update.error = Some(error),
ServerMessage::Closing => {
if let Some(session) = remote_session {
session.live.store(false, Ordering::Release);
}
update.disconnected = true;
update.error = Some("Server stopped. Shell sessions have ended.".into());
}
Ok(ServerMessage::Pong) => {}
Ok(_) => {
ServerMessage::Pong => {}
_ => {
update.error = Some("Unexpected server response after session handshake".into());
}
Err(error) => {
update.disconnected = true;
update.error = Some(format!(
"Disconnected — sessions remain on the server. {error:#}"
));
}
}
let disconnected = update.disconnected;
drop(update);
let _ = signal.try_send(());
if disconnected {
break;
return Ok(());
}
}
}
@@ -938,7 +836,8 @@ mod tests {
let pending = Mutex::new(Update::default());
let (signal, received) = async_channel::bounded(1);
thread::scope(|threads| {
threads.spawn(|| read_updates(client, &pending, signal, session, None));
let reader =
threads.spawn(|| read_updates(client, &pending, signal, session, None));
let mut send_snapshot = |snapshot| {
protocol::write_message(&mut server, &ServerMessage::Snapshot(snapshot))
.unwrap();
@@ -979,8 +878,8 @@ mod tests {
assert!(!resumed.disconnected);
server.shutdown(Shutdown::Write).unwrap();
received.recv_blocking().unwrap();
assert!(pending.lock().unwrap().disconnected);
let error = reader.join().unwrap().unwrap_err();
assert!(error.downcast_ref::<io::Error>().is_some());
});
}
}
@@ -1010,7 +909,7 @@ mod tests {
let mut bytes = Vec::new();
protocol::write_message(&mut bytes, &last_message).unwrap();
let (signal, _updates) = async_channel::bounded(1);
read_updates(
let _ = read_updates(
bytes.as_slice(),
&Mutex::new(Update::default()),
signal,
+413
View File
@@ -0,0 +1,413 @@
//! Socket/SSH lifetimes. Each successful attachment gets a fresh command queue.
use super::*;
use std::sync::Weak;
pub(super) struct ActiveTransport {
pub commands: mpsc::Sender<ClientMessage>,
interrupt: UnixStream,
}
impl Drop for ActiveTransport {
fn drop(&mut self) {
let _ = self.interrupt.shutdown(Shutdown::Both);
}
}
struct SshChild(Child);
impl Drop for SshChild {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
pub(super) struct Wire {
reader: UnixStream,
writer: Box<dyn Write + Send>,
child: Option<SshChild>,
pub frame: Option<Snapshot>,
}
impl Wire {
pub fn open(
socket: &Path,
remote: Option<&RemoteWindow>,
target: SessionTarget,
cancelled: &AtomicBool,
) -> Result<Self> {
let (target, expected) = match target {
SessionTarget::Remembered(identity) => (
SessionTarget::Attach(Some(identity.session)),
Some(identity),
),
other => (other, None),
};
let (mut reader, mut writer, mut child): (_, Box<dyn Write + Send>, _) =
if let Some(remote) = remote {
validate_destination(&remote.destination)?;
let executable = remote.ssh.executable.as_deref().unwrap_or("psyterm");
let attach = if matches!(target, SessionTarget::Attach(_)) {
" --attach"
} else {
""
};
let command = format!("exec {} --bridge{attach}", shell_quote(executable));
let mut ssh = Command::new("ssh");
if let Some(agent) = &remote.ssh.agent_socket {
ssh.env("SSH_AUTH_SOCK", agent);
} else {
ssh.env_remove("SSH_AUTH_SOCK");
}
// The socketpair also makes partial SSH greetings cancellable.
let (reader, stdout) = UnixStream::pair()?;
let mut child = SshChild(
ssh.args([
"-T",
"-o",
"BatchMode=yes",
"-o",
"ConnectTimeout=10",
"-o",
"ServerAliveInterval=15",
"-o",
"ServerAliveCountMax=3",
"--",
&remote.destination,
&command,
])
.stdin(Stdio::piped())
.stdout(Stdio::from(OwnedFd::from(stdout)))
.stderr(Stdio::piped())
.spawn()
.context("start SSH terminal transport")?,
);
(reader, Box::new(child.0.stdin.take().unwrap()), Some(child))
} else {
let stream = UnixStream::connect(socket).context("connect to psyTerm server")?;
crate::launcher::verify_peer(&stream)?;
(stream.try_clone()?, Box::new(stream), None)
};
let diagnostic = child
.as_mut()
.and_then(|child| child.0.stderr.take())
.map(ssh_diagnostics)
.transpose()?;
reader.set_read_timeout(Some(Duration::from_millis(150)))?;
let mut greeting = GreetingReader {
reader: &mut reader,
deadline: Instant::now() + Duration::from_secs(if remote.is_some() { 30 } else { 5 }),
cancelled: Some(cancelled),
};
let frame = match greet(
&mut greeting,
&mut writer,
target,
remote.is_some(),
expected,
) {
Ok(frame) => frame,
Err(error) => {
drop(child);
if let Some(detail) = diagnostic
.and_then(|rx| rx.recv_timeout(Duration::from_millis(150)).ok())
.filter(|s| !s.is_empty())
{
// A v6 bridge with --attach exits before Hello when its daemon
// is gone. This is distinct from ssh being unable to reach it.
if expected.is_some()
&& detail
.trim_start_matches("Error: ")
.starts_with("no sessions to attach to at ")
{
bail!("the remote daemon with this session has stopped");
}
return Err(error.context(detail));
}
return Err(error);
}
};
reader.set_read_timeout(None)?;
Ok(Self {
reader,
writer,
child,
frame,
})
}
}
pub(super) struct Reader {
socket: UnixStream,
// Reap each ssh process on the supervisor thread, including failed handshakes.
_child: Option<SshChild>,
}
pub(super) struct Supervisor {
owner: Weak<Connection>,
cancelled: Arc<AtomicBool>,
remote: Option<Arc<RemoteSession>>,
direct_window: bool,
listing: Option<RemoteListing>,
pending: Arc<Mutex<Update>>,
signal: async_channel::Sender<()>,
}
impl Supervisor {
pub fn new(
owner: &Arc<Connection>,
remote: Option<Arc<RemoteSession>>,
direct_window: bool,
pending: Arc<Mutex<Update>>,
signal: async_channel::Sender<()>,
) -> Self {
Self {
owner: Arc::downgrade(owner),
cancelled: owner.cancelled.clone(),
remote,
direct_window,
listing: None,
pending,
signal,
}
}
pub fn activate(&mut self, wire: Wire, reattached: bool) -> Result<Reader> {
let owner = self.owner.upgrade().context("connection cancelled")?;
let Wire {
reader,
mut writer,
child,
frame,
} = wire;
let (commands, receive) = mpsc::channel();
let interrupt = reader.try_clone()?;
let wake_reader = reader.try_clone()?;
thread::Builder::new()
.name("psyterm-client-writer".into())
.spawn(move || {
while let Ok(message) = receive.recv() {
if protocol::write_message(&mut writer, &message).is_err() {
// Only the reader reports a lost transport. Wake it even if
// the write half failed before the next incoming frame.
let _ = wake_reader.shutdown(Shutdown::Both);
break;
}
}
})?;
*owner.active.lock().unwrap() = Some(ActiveTransport {
commands,
interrupt,
});
if let Some(frame) = frame {
if let Some(remote) = &self.remote {
remote.live.store(!frame.tabs.is_empty(), Ordering::Release);
}
if let Some(listing) = &mut self.listing {
listing.update(&frame);
} else if self.direct_window
&& let Some(target) = &owner.remote
&& !frame.tabs.is_empty()
{
self.listing = RemoteListing::start(
owner.launcher_socket.clone(),
target.clone(),
&frame,
self.owner.clone(),
)
.inspect_err(|error| log::warn!("Could not start SSH window listing: {error:#}"))
.ok();
}
let mut update = self.pending.lock().unwrap();
update.snapshot = Some(Arc::new(frame));
if reattached {
update.reconnect = Some(None);
}
drop(update);
let _ = self.signal.try_send(());
}
Ok(Reader {
socket: reader,
_child: child,
})
}
pub fn run(mut self, mut reader: Reader) {
let mut attempt: u32 = 0;
loop {
let connected = Instant::now();
let result = read_updates(
&mut reader.socket,
&self.pending,
self.signal.clone(),
self.remote.as_deref(),
self.listing.as_mut(),
);
if let Some(owner) = self.owner.upgrade() {
owner.active.lock().unwrap().take();
}
drop(reader);
if self.cancelled.load(Ordering::Acquire) {
return;
}
let Err(error) = result else { return };
let Some(remote) = self.remote.as_ref() else {
self.finish(format!(
"Disconnected — sessions remain on the server. {error:#}"
));
return;
};
if !remote.live.load(Ordering::Acquire) {
return;
}
let identity = remote.identity;
if error.downcast_ref::<io::Error>().is_none() {
self.finish(format!("Could not read the SSH session: {error:#}"));
return;
}
*remote.directory.lock().unwrap() = RemoteDirectory::default();
if let Some(listing) = &mut self.listing {
listing.update_sessions(&[]);
}
// A flapping transport retains its backoff even if a greeting succeeded.
if connected.elapsed() >= Duration::from_secs(30) {
attempt = 0;
}
loop {
attempt = attempt.saturating_add(1);
let delay_secs = retry_delay(attempt);
self.set_status(ReconnectStatus::Waiting {
attempt,
delay_secs,
});
let deadline = Instant::now() + Duration::from_secs(delay_secs.into());
while !self.cancelled.load(Ordering::Acquire) && Instant::now() < deadline {
thread::park_timeout(deadline.saturating_duration_since(Instant::now()));
}
if self.cancelled.load(Ordering::Acquire) {
return;
}
self.set_status(ReconnectStatus::Connecting { attempt });
let Some(owner) = self.owner.upgrade() else {
return;
};
let path = owner.launcher_socket.clone();
let target = owner.remote.clone();
drop(owner); // Closing a window can cancel a handshake in progress.
let result = Wire::open(
&path,
target.as_ref(),
SessionTarget::Remembered(identity),
&self.cancelled,
);
if self.cancelled.load(Ordering::Acquire) {
return;
}
match result.and_then(|wire| self.activate(wire, true)) {
Ok(attached) => {
reader = attached;
break;
}
Err(error) if error.downcast_ref::<io::Error>().is_some() => {
log::debug!("SSH reconnect attempt {attempt}: {error:#}");
}
Err(error) => {
if let Some(remote) = &self.remote {
remote.live.store(false, Ordering::Release);
}
self.finish(format!("Could not reattach to the SSH session: {error:#}"));
return;
}
}
}
}
}
fn set_status(&self, status: ReconnectStatus) {
self.pending.lock().unwrap().reconnect = Some(Some(status));
let _ = self.signal.try_send(());
}
fn finish(&self, message: String) {
let mut update = self.pending.lock().unwrap();
update.disconnected = true;
update.reconnect = Some(None);
update.error = Some(message);
drop(update);
let _ = self.signal.try_send(());
}
}
fn retry_delay(attempt: u32) -> u32 {
(1 << attempt.saturating_sub(1).min(5)).min(30)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn dropping_a_session_cancels_its_wait_and_discards_offline_input() {
let (socket, peer) = UnixStream::pair().unwrap();
let wire = Wire {
writer: Box::new(socket.try_clone().unwrap()),
reader: socket,
child: None,
frame: None,
};
let owner = Arc::new(Connection {
launcher_socket: PathBuf::new(),
remote: None,
active: Mutex::default(),
cancelled: Arc::default(),
worker: std::sync::OnceLock::new(),
});
let remote = Arc::new(RemoteSession {
identity: SessionIdentity {
server: [1; 16],
session: 7,
},
live: AtomicBool::new(true),
directory: Mutex::default(),
});
let client = Client {
connection: owner.clone(),
remote_session: Some(remote.clone()),
};
let (signal, changes) = async_channel::bounded(1);
let pending = Arc::new(Mutex::new(Update::default()));
let mut supervisor = Supervisor::new(&owner, Some(remote), false, pending.clone(), signal);
let reader = supervisor.activate(wire, false).unwrap();
let (done, stopped) = mpsc::channel();
let worker = thread::spawn(move || {
supervisor.run(reader);
done.send(()).unwrap();
});
owner.worker.set(worker.thread().clone()).unwrap();
drop(owner);
drop(peer);
changes.recv_blocking().unwrap();
let update = std::mem::take(&mut *pending.lock().unwrap());
assert_eq!(
update.reconnect,
Some(Some(ReconnectStatus::Waiting {
attempt: 1,
delay_secs: 1
}))
);
assert!(!update.disconnected && update.error.is_none());
assert!(client.input(1, b"do not replay".to_vec()).is_err());
drop(client);
stopped.recv_timeout(Duration::from_secs(5)).unwrap();
worker.join().unwrap();
assert!(changes.recv_blocking().is_err());
}
#[test]
fn retries_have_a_minimum_delay_and_a_bounded_exponential_backoff() {
let delays: Vec<_> = (1..=8).map(retry_delay).collect();
assert_eq!(delays, [1, 2, 4, 8, 16, 30, 30, 30]);
assert_eq!(retry_delay(u32::MAX), 30);
}
}
+21
View File
@@ -343,6 +343,27 @@ pub enum ServerMessage {
},
}
/// Local transport state. Mounted tabs convey the description in a v6 connecting tab.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReconnectStatus {
Waiting { attempt: u32, delay_secs: u32 },
Connecting { attempt: u32 },
}
impl ReconnectStatus {
pub fn description(self, host: &str) -> String {
match self {
Self::Waiting {
attempt,
delay_secs,
} => {
format!("Reconnecting to {host} · attempt {attempt} after {delay_secs}s")
}
Self::Connecting { attempt } => format!("Reconnecting to {host} · attempt {attempt}…"),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Snapshot {
/// This session and the other sessions contributing tabs to its layout.
+3
View File
@@ -1581,6 +1581,9 @@ impl SessionState {
self.update_titles();
self.revision += 1;
let mut tabs = self.tabs.clone();
for remote in self.remotes.values() {
remote.show_reconnect(&mut tabs);
}
if let Some(remote) = self
.remotes
.values()
+169 -19
View File
@@ -24,6 +24,7 @@ pub(super) struct RemoteSession {
panes: HashMap<PaneId, PaneId>,
latest: Option<Arc<Snapshot>>,
attaching: Option<SessionIdentity>,
reconnect: Option<ReconnectStatus>,
requested_tab: Option<TabId>,
clipboard: VecDeque<(PaneId, Weak<Outbox>)>,
}
@@ -100,6 +101,11 @@ impl RemoteSession {
}
pub(super) fn send(&self, command: ClientMessage) -> Result<()> {
// Commands such as keystrokes or paste must never be replayed into the
// resumed shell. Local tab focus and explicit detach still work.
if self.reconnect.is_some() {
return Ok(());
}
self.connection
.as_ref()
.context("SSH session is connecting")?
@@ -109,6 +115,10 @@ impl RemoteSession {
// Keep one copy in flight on this connection. An attached session can
// contain multiple SSH hosts whose clipboard replies arrive out of order.
fn copy(&mut self, pane: PaneId, output: &Arc<Outbox>) -> Result<()> {
if self.reconnect.is_some() {
output.reliable(ServerMessage::Clipboard(String::new()));
return Ok(());
}
if self.clipboard.len() >= 16 {
output.reliable(ServerMessage::Clipboard(String::new()));
anyhow::bail!("too many pending clipboard requests");
@@ -147,7 +157,25 @@ impl RemoteSession {
}
}
// Project retrying tabs as v6 connecting placeholders. Keep their mappings
// and last good frame intact, even when another view joins during the outage.
pub(super) fn show_reconnect(&self, tabs: &mut [TabInfo]) {
let Some(status) = &self.reconnect else {
return;
};
for tab in tabs.iter_mut().filter(|tab| self.owns_tab(tab.id)) {
tab.remote = Some(self.target.destination.clone());
tab.title = status.description(&self.target.destination);
tab.panes.clear();
tab.stacks.clear();
tab.active_pane = 0;
}
}
pub(super) fn visible_panes(&self, tab: &mut TabInfo) -> Vec<PaneSnapshot> {
if tab.is_connecting() {
return Vec::new();
}
let Some(frame) = self.latest.as_ref() else {
return Vec::new();
};
@@ -415,7 +443,7 @@ impl SessionState {
index + 1,
TabInfo {
id: group,
title: "Connecting…".into(),
title: format!("Connecting to {}…", target.destination),
remote: Some(target.destination.clone()),
manual_title: None,
panes: Vec::new(),
@@ -439,6 +467,7 @@ impl SessionState {
panes: HashMap::new(),
latest: None,
attaching,
reconnect: None,
requested_tab: None,
clipboard: VecDeque::new(),
},
@@ -475,20 +504,22 @@ impl SessionState {
.context("tab is not attached to an SSH session")?;
// A nested SSH tab is owned by the upstream workspace. Detach it
// there so the enclosing SSH session and its other tabs stay open.
if let Some(nested) = remote.nested_tab(tab) {
if remote.reconnect.is_none()
&& let Some(nested) = remote.nested_tab(tab)
{
remote.send(ClientMessage::DetachRemote { tab: nested.id })?;
return Ok(None);
}
let target = remote.connection.as_ref().and_then(|client| {
let identity = client
.remote_identity()
.or_else(|| remote.latest.as_ref()?.lineage.first().copied())?;
Some(DetachedRemote {
let target = remote
.latest
.as_ref()
.and_then(|frame| frame.lineage.first().copied())
.or_else(|| remote.connection.as_ref()?.remote_identity())
.map(|identity| DetachedRemote {
destination: remote.target.destination.clone(),
identity,
executable: remote.target.ssh.executable.clone(),
})
});
});
self.remove_remote(group);
Ok(target)
}
@@ -616,6 +647,23 @@ impl Server {
}
RemoteEvent::Ended(error) => (true, error),
RemoteEvent::Updated(update) => {
if let Some(status) = update.reconnect {
remote.reconnect = status;
state.dirty = true;
if remote.reconnect.is_some() {
remote.cancel_copies();
} else if let Some(tab) = remote.requested_tab.take()
&& remote.owns_tab(state.active_tab)
&& update.snapshot.as_ref().is_some_and(|frame| {
frame.active_tab != tab && frame.tabs.iter().any(|t| t.id == tab)
})
&& remote.send(ClientMessage::FocusTab { tab }).is_ok()
{
// Tab switches made while offline stay local until the
// recovered daemon confirms the tab still exists.
remote.requested_tab = Some(tab);
}
}
for text in update.clipboard {
if let Err(error) = remote.finish_copy(text) {
log::warn!("SSH clipboard: {error}");
@@ -1004,17 +1052,9 @@ mod tests {
assert_eq!(frame.panes[0].id, top);
}
#[test]
fn discarded_ssh_frame_keeps_the_mounted_workspace_and_accepts_the_next_frame() {
let mut upstream = session(2);
upstream.new_tab(None).unwrap();
let mut local = session(1);
local.new_tab(None).unwrap();
let origin = local.active_tab;
let group = reserve(&mut local, 0);
apply(&mut local, group, upstream.snapshot()).unwrap();
fn server_with_session(local: SessionState) -> Server {
let id = local.id;
let mut server = Server {
Server {
identity: local.identity.server,
socket: local.socket.clone(),
shell: None,
@@ -1025,7 +1065,117 @@ mod tests {
last_detached_remote: None,
last_frame: Instant::now(),
last_cwd_refresh: Instant::now(),
}
}
#[test]
fn reconnecting_mount_keeps_tab_ids_and_restores_panes_without_stealing_focus() {
let mut upstream = session(2);
upstream.new_tab(None).unwrap();
upstream.new_tab(None).unwrap();
let baseline = upstream.snapshot();
let mut local = session(1);
local.new_tab(None).unwrap();
let local_tab = local.active_tab;
let group = reserve(&mut local, 0);
apply(&mut local, group, baseline.clone()).unwrap();
let before = local.snapshot();
let ids = before.tabs.iter().map(|tab| tab.id).collect::<Vec<_>>();
let id = local.id;
let mut server = server_with_session(local);
let waiting = ReconnectStatus::Waiting {
attempt: 2,
delay_secs: 2,
};
server.remote_event(
id,
group,
RemoteEvent::Updated(Update {
reconnect: Some(Some(waiting)),
..Update::default()
}),
);
let local = server.sessions.get_mut(&id).unwrap();
let waiting_frame = local.snapshot();
crate::client::validate_snapshot(&waiting_frame).unwrap();
assert!(
waiting_frame
.tabs
.iter()
.map(|tab| tab.id)
.eq(ids.iter().copied())
);
assert!(
waiting_frame.tabs[1..].iter().all(|tab| {
tab.is_connecting() && tab.title == waiting.description("user@work")
})
);
assert!(waiting_frame.panes.is_empty());
let offline_pane = before.active_tab().unwrap().active_pane;
let clipboard = Arc::new(Outbox::default());
local
.command(ClientMessage::Copy { pane: offline_pane }, &clipboard)
.unwrap();
assert!(
matches!(clipboard.next(), Some(ServerMessage::Clipboard(text)) if text.is_empty())
);
local.focus_tab(ids[1]).unwrap();
assert_eq!(local.active_tab, ids[1]);
local.focus_tab(local_tab).unwrap();
local.dirty = false;
server.remote_event(
id,
group,
RemoteEvent::Updated(Update {
reconnect: Some(None),
snapshot: Some(Arc::new(baseline)),
..Update::default()
}),
);
let local = server.sessions.get_mut(&id).unwrap();
assert!(local.dirty); // Even an identical layout must replace the retry placeholder.
assert_eq!(local.active_tab, local_tab);
local.focus_tab(before.active_tab).unwrap();
let after = local.snapshot();
assert_eq!(after.tabs, before.tabs);
assert_eq!(after.panes[0].id, offline_pane);
}
#[test]
fn cancelling_a_nested_tab_while_its_outer_host_is_offline_remembers_the_outer_session() {
let mut upstream = session(2);
upstream.new_tab(None).unwrap();
let mut nested = upstream.snapshot();
nested.tabs[0].remote = Some("inner".into());
let mut local = session(1);
local.new_tab(None).unwrap();
let local_tab = local.active_tab;
let group = reserve(&mut local, 0);
apply(&mut local, group, nested).unwrap();
let mount = local.remotes.get_mut(&group).unwrap();
mount.reconnect = Some(ReconnectStatus::Connecting { attempt: 1 });
let cancelled = mount.cancelled.clone();
let target = local.detach_remote(group).unwrap().unwrap();
assert_eq!(target.identity, upstream.identity);
assert_eq!(target.destination, "user@work");
assert!(cancelled.load(Ordering::Acquire));
assert_eq!(local.tabs.len(), 1);
assert_eq!(local.active_tab, local_tab);
}
#[test]
fn discarded_ssh_frame_keeps_the_mounted_workspace_and_accepts_the_next_frame() {
let mut upstream = session(2);
upstream.new_tab(None).unwrap();
let mut local = session(1);
local.new_tab(None).unwrap();
let origin = local.active_tab;
let group = reserve(&mut local, 0);
apply(&mut local, group, upstream.snapshot()).unwrap();
let id = local.id;
let mut server = server_with_session(local);
server.remote_event(
id,
group,
+44 -18
View File
@@ -1,4 +1,5 @@
//! Display-only workspace and panes. All terminal and multiplexer state is remote.
mod reconnect;
mod session_picker;
use crate::{
@@ -13,8 +14,8 @@ use crate::{
palette::CommandPalette,
pane_name::RenameField,
protocol::{
ClientMessage, Direction, PaneSnapshot, ScrollKind, SelectionCommand, SelectionKind,
SelectionSide, Snapshot, StackPaneInfo, VerticalDirection,
ClientMessage, Direction, PaneSnapshot, ReconnectStatus, ScrollKind, SelectionCommand,
SelectionKind, SelectionSide, Snapshot, StackPaneInfo, VerticalDirection,
},
render::{RenderCell, Renderer},
transition::Slide,
@@ -49,6 +50,8 @@ pub struct Workspace {
session_directory: SessionDirectory,
session_refresh: Option<Task<()>>,
disconnected: bool,
reconnect: Option<ReconnectStatus>,
interrupted_titles: HashMap<u64, String>,
tab_rename: Option<TabRename>,
outgoing: HashMap<u64, Entity<TerminalPane>>,
outgoing_snapshot: Option<Arc<Snapshot>>,
@@ -229,6 +232,10 @@ impl Workspace {
else {
return;
};
if self.reconnect.is_some() {
window.remove_window();
return;
}
let message = if tab.remote.is_some() {
ClientMessage::DetachRemote { tab: id }
} else if self.client.remote().is_some() {
@@ -245,6 +252,17 @@ impl Workspace {
fn execute(&mut self, action: CommandAction, window: &mut Window, cx: &mut Context<Self>) {
use CommandAction::*;
if self.reconnect.is_some() {
match action {
DetachRemote | Detach | ClosePane => {
window.remove_window();
return;
}
TogglePalette | SwitchSession(_) | NewWindow | IncreaseFontSize
| DecreaseFontSize => {}
_ => return,
}
}
let pane = self.focused_pane();
let pane_id = pane.as_ref().map(|pane| pane.read(cx).data().id);
let message = match action {
@@ -397,6 +415,9 @@ impl Workspace {
cx.write_to_clipboard(ClipboardItem::new_string(text));
}
view.disconnected |= update.disconnected;
if let Some(status) = update.reconnect {
view.update_reconnect(status, window, cx);
}
if let Some(error) = update.error {
view.error = Some(error);
}
@@ -410,6 +431,9 @@ impl Workspace {
old.active_tab != snapshot.active_tab
|| old.tabs != snapshot.tabs
});
if metadata_changed {
view.update_interrupted_titles(&snapshot);
}
let cancelled_tab_rename =
view.tab_rename.as_ref().is_some_and(|rename| {
snapshot.active_tab != rename.tab
@@ -418,8 +442,9 @@ impl Workspace {
if cancelled_tab_rename {
view.tab_rename = None;
}
let focus_allowed =
view.palette.is_none() && view.tab_rename.is_none();
let focus_allowed = view.palette.is_none()
&& view.tab_rename.is_none()
&& view.reconnect.is_none();
if let Some(direction) = view
.snapshot
.as_ref()
@@ -498,6 +523,7 @@ impl Workspace {
window.focus(&pane.focus);
}
pane.transitioning = view.transition.is_some();
pane.read_only = view.reconnect.is_some();
pane
});
view.panes.insert(data.id, pane);
@@ -544,6 +570,8 @@ impl Workspace {
session_directory: SessionDirectory::Loading,
session_refresh: None,
disconnected: false,
reconnect: None,
interrupted_titles: HashMap::new(),
tab_rename: None,
outgoing: HashMap::new(),
outgoing_snapshot: None,
@@ -586,7 +614,14 @@ impl Render for Workspace {
.iter()
.map(|tab| chrome::ChromeTab {
id: tab.id,
label: tab.title.clone().into(),
label: self
.interrupted_titles
.get(&tab.id)
.unwrap_or(&tab.title)
.clone()
.into(),
reconnecting: self.reconnect.is_some()
|| self.interrupted_titles.contains_key(&tab.id),
remote: tab
.remote
.as_deref()
@@ -636,18 +671,6 @@ impl Render for Workspace {
let mut incoming = scene(incoming_offset).id("incoming-scene");
if let Some(snapshot) = &self.snapshot {
incoming = pane_scene(snapshot, &self.panes, incoming_offset, Some(&self.client));
if let Some(host) = snapshot
.active_tab()
.filter(|tab| tab.is_connecting())
.and_then(|tab| tab.remote.as_ref())
{
incoming = incoming.child(
div()
.p_4()
.text_color(rgb(chrome::MUTED))
.child(format!("Connecting to {host}…")),
);
}
} else {
incoming = incoming.child(
div()
@@ -657,6 +680,9 @@ impl Render for Workspace {
);
}
body = body.child(incoming);
if let Some((tab, message)) = self.connection_notice() {
body = body.child(self.connection_panel(tab, message, cx));
}
let mut root = div()
.relative()
.track_focus(&self.focus)
@@ -841,7 +867,7 @@ struct TerminalPane {
snapshot: Arc<Snapshot>,
index: usize,
active: bool,
// Frozen outgoing scenes are display-only; no PTY resize, focus or input.
// Outgoing scenes and disconnected SSH panes are display-only.
read_only: bool,
transitioning: bool,
focus: FocusHandle,
+135
View File
@@ -0,0 +1,135 @@
use super::*;
impl Workspace {
pub(super) fn update_reconnect(
&mut self,
status: Option<ReconnectStatus>,
window: &mut Window,
cx: &mut Context<Self>,
) {
let was_offline = self.reconnect.is_some();
self.reconnect = status;
if was_offline == status.is_some() {
return;
}
let read_only = status.is_some() || self.disconnected;
for pane in self.panes.values() {
pane.update(cx, |pane, cx| {
pane.read_only = read_only;
pane.selecting = false;
pane.preedit.clear();
pane.rename = None;
pane.requested_size = None;
cx.notify();
});
}
self.tab_rename = None;
if self.palette.is_none() {
if read_only {
window.focus(&self.focus);
} else {
self.restore_focus(window, cx);
}
}
}
pub(super) fn update_interrupted_titles(&mut self, snapshot: &Snapshot) {
self.interrupted_titles.retain(|id, _| {
snapshot
.tabs
.iter()
.any(|tab| tab.id == *id && tab.is_connecting())
});
if let Some(previous) = &self.snapshot {
for tab in snapshot.tabs.iter().filter(|tab| tab.is_connecting()) {
if let Some(old) = previous
.tabs
.iter()
.find(|old| old.id == tab.id && !old.is_connecting())
{
self.interrupted_titles.insert(tab.id, old.title.clone());
}
}
}
}
pub(super) fn connection_notice(&self) -> Option<(u64, String)> {
let tab = self.snapshot.as_ref()?.active_tab()?;
// A direct connection owns the whole window, including any nested SSH tabs.
if let Some(status) = self.reconnect {
return Some((tab.id, status.description(self.client.remote()?)));
}
if !tab.is_connecting() {
return None;
}
let text = if tab.title == "Connecting…" {
// Initial placeholder from an earlier v6 daemon.
format!("Connecting to {}…", tab.remote.as_deref()?)
} else {
tab.title.clone()
};
Some((tab.id, text))
}
pub(super) fn connection_panel(
&self,
tab: u64,
message: String,
cx: &mut Context<Self>,
) -> Div {
div()
.absolute()
.inset_0()
.flex()
.items_center()
.justify_center()
.px_4()
.bg(rgba(0x06060f99))
.on_mouse_down(MouseButton::Left, |_, _, cx| cx.stop_propagation())
.on_scroll_wheel(|_, _, cx| cx.stop_propagation())
.child(
div()
.flex()
.items_center()
.gap_3()
.px_4()
.py_3()
.border_1()
.border_color(rgb(chrome::BORDER))
.bg(rgb(chrome::PANEL_BACKGROUND))
.font_family(".SystemUIFont")
.text_size(px(13.))
.text_color(rgb(chrome::TEXT))
.child(
div()
.flex_shrink_0()
.w(px(3.))
.h(px(22.))
.bg(rgb(chrome::PANE_FOCUS)),
)
.child(div().min_w_0().child(message))
.child(
div()
.id("cancel-ssh-connection")
.flex_shrink_0()
.px_3()
.py_1()
.border_1()
.border_color(rgb(chrome::BORDER))
.bg(rgb(chrome::CONTROL_BACKGROUND))
.hover(|style| style.bg(rgb(chrome::HOVER_BACKGROUND)))
.cursor_pointer()
.child("Cancel")
.on_click(cx.listener(move |view, _, window, cx| {
cx.stop_propagation();
if view
.connection_notice()
.is_some_and(|(active, _)| active == tab)
{
view.close_tab(tab, window, cx);
}
})),
),
)
}
}