fix: keep remote sessions alive across disconnects

12 files changed, 326 insertions, 41 deletions

diff --git a/src/api/schema/server.rs b/src/api/schema/server.rs
index 8bace6e0..7ceda633 100644
--- a/src/api/schema/server.rs
+++ b/src/api/schema/server.rs
@@ -16,4 +16,6 @@ pub struct ServerLiveHandoffParams {
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
 pub struct ServerCapabilities {
     pub live_handoff: bool,
+    #[serde(default)]
+    pub detached_server_daemon: bool,
 }
diff --git a/src/api/schema/tests.rs b/src/api/schema/tests.rs
index 01ec9baa..5106fafe 100644
--- a/src/api/schema/tests.rs
+++ b/src/api/schema/tests.rs
@@ -465,7 +465,10 @@ fn success_response_round_trips() {
         result: ResponseResult::Pong {
             version: "0.1.2".into(),
             protocol: 6,
-            capabilities: Some(ServerCapabilities { live_handoff: true }),
+            capabilities: Some(ServerCapabilities {
+                live_handoff: true,
+                detached_server_daemon: true,
+            }),
         },
     };
 
diff --git a/src/api/server.rs b/src/api/server.rs
index d01f4199..37b464ac 100644
--- a/src/api/server.rs
+++ b/src/api/server.rs
@@ -62,6 +62,7 @@ pub fn start_server(
         event_hub,
         Some(ServerCapabilities {
             live_handoff: crate::platform::capabilities().live_handoff,
+            detached_server_daemon: crate::platform::current_process_is_detached_server_daemon(),
         }),
     )
 }
@@ -743,7 +744,10 @@ mod tests {
                 method: Method::Ping(crate::api::schema::PingParams::default()),
             },
             &tx,
-            Some(ServerCapabilities { live_handoff: true }),
+            Some(ServerCapabilities {
+                live_handoff: true,
+                detached_server_daemon: true,
+            }),
         );
 
         let parsed: SuccessResponse = serde_json::from_str(&response).unwrap();
diff --git a/src/cli/status.rs b/src/cli/status.rs
index 348746c5..49b3c129 100644
--- a/src/cli/status.rs
+++ b/src/cli/status.rs
@@ -241,6 +241,7 @@ struct ServerStatusJson {
 #[derive(Serialize)]
 struct ServerCapabilitiesJson {
     live_handoff: bool,
+    detached_server_daemon: bool,
 }
 
 #[derive(Serialize)]
@@ -273,6 +274,7 @@ fn server_status_json(server: &ServerRuntimeStatus) -> ServerStatusJson {
                 .as_ref()
                 .map(|capabilities| ServerCapabilitiesJson {
                     live_handoff: capabilities.live_handoff,
+                    detached_server_daemon: capabilities.detached_server_daemon,
                 }),
             compatible: protocol.map(|value| value == crate::protocol::PROTOCOL_VERSION),
             socket: api::socket_path().display().to_string(),
diff --git a/src/client/mod.rs b/src/client/mod.rs
index 59b7b114..4e0a54ec 100644
--- a/src/client/mod.rs
+++ b/src/client/mod.rs
@@ -280,7 +280,14 @@ impl std::fmt::Display for ClientError {
                 Ok(())
             }
             ClientError::ConnectionLost(err) => {
-                write!(f, "lost connection to server: {err}")
+                if let Ok(reattach_command) = std::env::var(crate::remote::REATTACH_COMMAND_ENV_VAR)
+                {
+                    write!(f, "lost connection to remote Herdr: {err}")?;
+                    write!(f, "\nIf the remote server survived the SSH or network drop, its panes may still be running.")?;
+                    write!(f, "\nRun `{reattach_command}` to reattach")
+                } else {
+                    write!(f, "lost connection to server: {err}")
+                }
             }
             ClientError::Protocol(err) => {
                 write!(f, "protocol error: {err}")
@@ -2372,6 +2379,8 @@ mod tests {
 
     #[test]
     fn client_error_display_connection_lost() {
+        let _guard = env_lock().lock().unwrap();
+        let _env = EnvVarsRemovedGuard::new(&[crate::remote::REATTACH_COMMAND_ENV_VAR]);
         let err =
             ClientError::ConnectionLost(io::Error::new(io::ErrorKind::BrokenPipe, "broken pipe"));
         let msg = err.to_string();
@@ -2382,6 +2391,30 @@ mod tests {
     }
 
     #[test]
+    fn client_error_display_remote_connection_lost_has_reattach_hint() {
+        let _guard = env_lock().lock().unwrap();
+        let _remote_env = EnvVarGuard::set(
+            crate::remote::REATTACH_COMMAND_ENV_VAR,
+            "herdr --remote host --session work",
+        );
+        let err =
+            ClientError::ConnectionLost(io::Error::new(io::ErrorKind::BrokenPipe, "broken pipe"));
+        let msg = err.to_string();
+        assert!(
+            msg.contains("lost connection to remote Herdr"),
+            "should mention remote connection loss: {msg}"
+        );
+        assert!(
+            msg.contains("panes may still be running"),
+            "should explain possible persistence: {msg}"
+        );
+        assert!(
+            msg.contains("Run `herdr --remote host --session work` to reattach"),
+            "should show remote reattach command: {msg}"
+        );
+    }
+
+    #[test]
     fn sound_from_notify_message_maps_done() {
         assert_eq!(
             sound_from_notify_message("agent done"),
diff --git a/src/platform/fallback.rs b/src/platform/fallback.rs
index b27f181e..25bc761c 100644
--- a/src/platform/fallback.rs
+++ b/src/platform/fallback.rs
@@ -1,4 +1,5 @@
 use std::path::PathBuf;
+use std::process::Command;
 
 use super::{ClipboardImage, ForegroundJob, Signal};
 
@@ -6,6 +7,14 @@ use super::{ClipboardImage, ForegroundJob, Signal};
 pub fn raise_server_nofile_limit() {}
 
 /// Unsupported platform stub.
+pub fn detach_server_daemon_command(_command: &mut Command) {}
+
+/// Unsupported platform stub.
+pub fn current_process_is_detached_server_daemon() -> bool {
+    false
+}
+
+/// Unsupported platform stub.
 pub fn foreground_job(_child_pid: u32) -> Option<ForegroundJob> {
     None
 }
diff --git a/src/platform/mod.rs b/src/platform/mod.rs
index 0b5f9ecb..169b7c1c 100644
--- a/src/platform/mod.rs
+++ b/src/platform/mod.rs
@@ -40,6 +40,25 @@ pub(crate) const fn capabilities() -> PlatformCapabilities {
     }
 }
 
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+pub fn detach_server_daemon_command(command: &mut std::process::Command) {
+    use std::os::unix::process::CommandExt;
+
+    unsafe {
+        command.pre_exec(|| {
+            if libc::setsid() < 0 {
+                return Err(std::io::Error::last_os_error());
+            }
+            Ok(())
+        });
+    }
+}
+
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+pub fn current_process_is_detached_server_daemon() -> bool {
+    unsafe { libc::getsid(0) == libc::getpid() }
+}
+
 #[cfg(unix)]
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct ClipboardCommand {
diff --git a/src/platform/windows.rs b/src/platform/windows.rs
index 12a8345f..e7a77e35 100644
--- a/src/platform/windows.rs
+++ b/src/platform/windows.rs
@@ -46,6 +46,12 @@ struct WindowsProcessEntry {
 
 pub fn raise_server_nofile_limit() {}
 
+pub fn detach_server_daemon_command(_command: &mut std::process::Command) {}
+
+pub fn current_process_is_detached_server_daemon() -> bool {
+    false
+}
+
 pub fn foreground_job(child_pid: u32) -> Option<ForegroundJob> {
     let entries = snapshot_processes();
     select_pane_foreground_job(child_pid, &entries)
diff --git a/src/remote/unix.rs b/src/remote/unix.rs
index c6432626..b3c7192d 100644
--- a/src/remote/unix.rs
+++ b/src/remote/unix.rs
@@ -956,6 +956,7 @@ enum RemoteServerStatus {
         version: Option<String>,
         protocol: Option<u32>,
         live_handoff: bool,
+        detached_server_daemon: bool,
     },
     NotRunning,
 }
@@ -963,10 +964,18 @@ enum RemoteServerStatus {
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum RemoteServerRestartReason {
     ProtocolMismatch,
+    DaemonDetachMissing,
     BinaryUpdated,
     VersionMismatch,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum RemoteInstallRunningServerPlan {
+    KeepRunning,
+    LiveHandoff,
+    StopRequired(RemoteServerRestartReason),
+}
+
 fn ensure_remote_server_ready(
     ssh: &RemoteSsh,
     remote_herdr: &RemoteHerdr,
@@ -979,14 +988,18 @@ fn ensure_remote_server_ready(
         version,
         protocol,
         live_handoff,
+        detached_server_daemon,
     } = status
     else {
         return Ok(());
     };
 
-    let Some(reason) =
-        remote_server_restart_reason(version.as_deref(), protocol, remote_binary_changed)
-    else {
+    let Some(reason) = remote_server_restart_reason(
+        version.as_deref(),
+        protocol,
+        detached_server_daemon,
+        remote_binary_changed,
+    ) else {
         return Ok(());
     };
 
@@ -1014,17 +1027,21 @@ fn ensure_remote_server_ready(
 fn remote_server_restart_reason(
     version: Option<&str>,
     protocol: Option<u32>,
+    detached_server_daemon: bool,
     remote_binary_changed: bool,
 ) -> Option<RemoteServerRestartReason> {
     if protocol != Some(CURRENT_PROTOCOL) {
         return Some(RemoteServerRestartReason::ProtocolMismatch);
     }
-    if remote_binary_changed {
-        return Some(RemoteServerRestartReason::BinaryUpdated);
+    if !detached_server_daemon {
+        return Some(RemoteServerRestartReason::DaemonDetachMissing);
     }
     if version != Some(current_version().as_str()) {
         return Some(RemoteServerRestartReason::VersionMismatch);
     }
+    if remote_binary_changed {
+        return Some(RemoteServerRestartReason::BinaryUpdated);
+    }
     None
 }
 
@@ -1062,23 +1079,48 @@ fn confirm_remote_install_with_running_server(
     };
     let RemoteServerStatus::Running {
         version,
-        protocol: _,
+        protocol,
         live_handoff,
-    } = status
+        detached_server_daemon,
+    } = &status
     else {
         return Ok(false);
     };
+    let plan = remote_install_running_server_plan(
+        version.as_deref(),
+        *protocol,
+        *detached_server_daemon,
+        true,
+        *live_handoff,
+        live_handoff_enabled,
+    );
+
+    if plan == RemoteInstallRunningServerPlan::KeepRunning {
+        if io::stdin().is_terminal() {
+            eprintln!("remote herdr server on {target} is already compatible:");
+            eprintln!("  server: v{}", version_label(version.as_deref()));
+            eprintln!(
+                "Herdr will install {} without stopping the running remote server.",
+                current_version()
+            );
+        }
+        return Ok(false);
+    }
+
     if !io::stdin().is_terminal() {
-        if live_handoff_enabled && live_handoff {
-            return Ok(false);
+        match plan {
+            RemoteInstallRunningServerPlan::LiveHandoff => return Ok(false),
+            RemoteInstallRunningServerPlan::StopRequired(_) => {
+                return Err(io::Error::other(format!(
+                    "remote herdr server on {target} is running v{}; run from an interactive terminal to approve stopping it for the update",
+                    version_label(version.as_deref())
+                )));
+            }
+            RemoteInstallRunningServerPlan::KeepRunning => return Ok(false),
         }
-        return Err(io::Error::other(format!(
-            "remote herdr server on {target} is running v{}; run from an interactive terminal to approve stopping it for the update",
-            version_label(version.as_deref())
-        )));
     }
 
-    if live_handoff_enabled && live_handoff {
+    if plan == RemoteInstallRunningServerPlan::LiveHandoff {
         eprintln!("remote herdr server on {target} is currently running:");
         eprintln!("  server: v{}", version_label(version.as_deref()));
         eprintln!(
@@ -1114,6 +1156,30 @@ fn confirm_remote_install_with_running_server(
     Ok(true)
 }
 
+fn remote_install_running_server_plan(
+    version: Option<&str>,
+    protocol: Option<u32>,
+    detached_server_daemon: bool,
+    remote_binary_changed: bool,
+    live_handoff: bool,
+    live_handoff_enabled: bool,
+) -> RemoteInstallRunningServerPlan {
+    let Some(reason) = remote_server_restart_reason(
+        version,
+        protocol,
+        detached_server_daemon,
+        remote_binary_changed,
+    ) else {
+        return RemoteInstallRunningServerPlan::KeepRunning;
+    };
+
+    if live_handoff_enabled && live_handoff {
+        return RemoteInstallRunningServerPlan::LiveHandoff;
+    }
+
+    RemoteInstallRunningServerPlan::StopRequired(reason)
+}
+
 fn remote_server_status(
     ssh: &RemoteSsh,
     remote_herdr: &RemoteHerdr,
@@ -1144,6 +1210,8 @@ struct RemoteServerStatusJson {
 #[derive(Debug, Deserialize)]
 struct RemoteServerCapabilitiesJson {
     live_handoff: bool,
+    #[serde(default)]
+    detached_server_daemon: bool,
 }
 
 fn parse_client_status_json(status: &str) -> Option<RemoteClientStatusJson> {
@@ -1160,12 +1228,17 @@ fn parse_remote_server_status_json(status: &str) -> io::Result<RemoteServerStatu
         return Ok(RemoteServerStatus::NotRunning);
     }
 
+    let capabilities = parsed.capabilities;
+
     Ok(RemoteServerStatus::Running {
         version: parsed.version,
         protocol: parsed.protocol,
-        live_handoff: parsed
-            .capabilities
+        live_handoff: capabilities
+            .as_ref()
             .is_some_and(|capabilities| capabilities.live_handoff),
+        detached_server_daemon: capabilities
+            .as_ref()
+            .is_some_and(|capabilities| capabilities.detached_server_daemon),
     })
 }
 
@@ -1199,6 +1272,11 @@ fn confirm_remote_server_stop(
         RemoteServerRestartReason::ProtocolMismatch => {
             eprintln!("the remote server must stop before this client can attach.");
         }
+        RemoteServerRestartReason::DaemonDetachMissing => {
+            eprintln!(
+                "the remote server was started by a herdr build that may not survive SSH connection loss. restart it so network drops disconnect only this client."
+            );
+        }
         RemoteServerRestartReason::BinaryUpdated => {
             eprintln!(
                 "the remote herdr binary was installed or replaced. restart the remote server so it uses the prepared binary."
@@ -2446,19 +2524,20 @@ mod tests {
     fn parse_remote_server_status_json_reads_running_server() {
         assert_eq!(
             parse_remote_server_status_json(
-                r#"{"status":"running","running":true,"version":"0.6.0","protocol":8,"capabilities":{"live_handoff":true}}"#
+                r#"{"status":"running","running":true,"version":"0.6.0","protocol":8,"capabilities":{"live_handoff":true,"detached_server_daemon":true}}"#
             )
             .unwrap(),
             RemoteServerStatus::Running {
                 version: Some("0.6.0".into()),
                 protocol: Some(8),
-                live_handoff: true
+                live_handoff: true,
+                detached_server_daemon: true
             }
         );
     }
 
     #[test]
-    fn parse_remote_server_status_json_treats_missing_capability_as_no_handoff() {
+    fn parse_remote_server_status_json_treats_missing_capability_as_old_server() {
         assert_eq!(
             parse_remote_server_status_json(
                 r#"{"status":"running","running":true,"version":"0.6.0","protocol":8}"#
@@ -2467,7 +2546,8 @@ mod tests {
             RemoteServerStatus::Running {
                 version: Some("0.6.0".into()),
                 protocol: Some(8),
-                live_handoff: false
+                live_handoff: false,
+                detached_server_daemon: false
             }
         );
     }
@@ -2637,15 +2717,46 @@ mod tests {
     #[test]
     fn remote_server_restart_reason_requires_stop_for_protocol_mismatch() {
         assert_eq!(
-            remote_server_restart_reason(Some(&current_version()), Some(0), false),
+            remote_server_restart_reason(Some(&current_version()), Some(0), true, false),
             Some(RemoteServerRestartReason::ProtocolMismatch)
         );
     }
 
     #[test]
-    fn remote_server_restart_reason_offers_restart_after_binary_update() {
+    fn remote_server_restart_reason_allows_unchanged_compatible_server() {
+        assert_eq!(
+            remote_server_restart_reason(
+                Some(&current_version()),
+                Some(CURRENT_PROTOCOL),
+                true,
+                false
+            ),
+            None
+        );
+    }
+
+    #[test]
+    fn remote_server_restart_reason_requires_restart_for_old_daemon() {
+        assert_eq!(
+            remote_server_restart_reason(
+                Some(&current_version()),
+                Some(CURRENT_PROTOCOL),
+                false,
+                false
+            ),
+            Some(RemoteServerRestartReason::DaemonDetachMissing)
+        );
+    }
+
+    #[test]
+    fn remote_server_restart_reason_requires_restart_after_helper_update() {
         assert_eq!(
-            remote_server_restart_reason(Some(&current_version()), Some(CURRENT_PROTOCOL), true),
+            remote_server_restart_reason(
+                Some(&current_version()),
+                Some(CURRENT_PROTOCOL),
+                true,
+                true
+            ),
             Some(RemoteServerRestartReason::BinaryUpdated)
         );
     }
@@ -2653,11 +2764,11 @@ mod tests {
     #[test]
     fn remote_server_restart_reason_offers_restart_for_version_mismatch() {
         assert_eq!(
-            remote_server_restart_reason(Some("0.0.0"), Some(CURRENT_PROTOCOL), false),
+            remote_server_restart_reason(Some("0.0.0"), Some(CURRENT_PROTOCOL), true, false),
             Some(RemoteServerRestartReason::VersionMismatch)
         );
         assert_eq!(
-            remote_server_restart_reason(None, Some(CURRENT_PROTOCOL), false),
+            remote_server_restart_reason(None, Some(CURRENT_PROTOCOL), true, false),
             Some(RemoteServerRestartReason::VersionMismatch)
         );
     }
@@ -2665,12 +2776,96 @@ mod tests {
     #[test]
     fn remote_server_restart_reason_allows_current_server() {
         assert_eq!(
-            remote_server_restart_reason(Some(&current_version()), Some(CURRENT_PROTOCOL), false),
+            remote_server_restart_reason(
+                Some(&current_version()),
+                Some(CURRENT_PROTOCOL),
+                true,
+                false
+            ),
             None
         );
     }
 
     #[test]
+    fn remote_install_plan_keeps_compatible_running_server() {
+        assert_eq!(
+            remote_install_running_server_plan(
+                Some(&current_version()),
+                Some(CURRENT_PROTOCOL),
+                true,
+                false,
+                false,
+                false
+            ),
+            RemoteInstallRunningServerPlan::KeepRunning
+        );
+    }
+
+    #[test]
+    fn remote_install_plan_requires_stop_for_old_daemon() {
+        assert_eq!(
+            remote_install_running_server_plan(
+                Some(&current_version()),
+                Some(CURRENT_PROTOCOL),
+                false,
+                true,
+                false,
+                false
+            ),
+            RemoteInstallRunningServerPlan::StopRequired(
+                RemoteServerRestartReason::DaemonDetachMissing
+            )
+        );
+    }
+
+    #[test]
+    fn remote_install_plan_requires_stop_after_helper_update() {
+        assert_eq!(
+            remote_install_running_server_plan(
+                Some(&current_version()),
+                Some(CURRENT_PROTOCOL),
+                true,
+                true,
+                false,
+                false
+            ),
+            RemoteInstallRunningServerPlan::StopRequired(RemoteServerRestartReason::BinaryUpdated)
+        );
+    }
+
+    #[test]
+    fn remote_install_plan_requires_stop_for_incompatible_running_server() {
+        assert_eq!(
+            remote_install_running_server_plan(
+                Some("0.0.0"),
+                Some(CURRENT_PROTOCOL),
+                true,
+                true,
+                false,
+                false
+            ),
+            RemoteInstallRunningServerPlan::StopRequired(
+                RemoteServerRestartReason::VersionMismatch
+            )
+        );
+    }
+
+    #[test]
+    fn remote_install_plan_uses_live_handoff_for_incompatible_running_server() {
+        assert_eq!(
+            remote_install_running_server_plan(
+                Some("0.0.0"),
+                Some(CURRENT_PROTOCOL),
+                true,
+                true,
+                true,
+                true
+            ),
+            RemoteInstallRunningServerPlan::LiveHandoff
+        );
+    }
+
+    #[test]
     fn install_source_description_uses_override_binary() {
         let platform = RemotePlatform {
             os: "linux",
diff --git a/src/server/autodetect.rs b/src/server/autodetect.rs
index f40be988..0c9e0cde 100644
--- a/src/server/autodetect.rs
+++ b/src/server/autodetect.rs
@@ -9,8 +9,6 @@
 //! (escape hatch for users who want the traditional single-process behavior).
 
 use std::io;
-#[cfg(unix)]
-use std::os::unix::process::CommandExt;
 use std::path::Path;
 use std::path::PathBuf;
 use std::process::Command;
@@ -218,12 +216,7 @@ fn build_server_daemon_command(exe: PathBuf) -> Command {
         .stdin(std::process::Stdio::null())
         .stdout(std::process::Stdio::null())
         .stderr(std::process::Stdio::null());
-    #[cfg(unix)]
-    {
-        // Create a new process group so the server survives the parent's exit
-        // and doesn't receive SIGHUP when the client's terminal closes.
-        command.process_group(0);
-    }
+    crate::platform::detach_server_daemon_command(&mut command);
 
     match std::env::current_dir() {
         Ok(cwd) => {
@@ -384,6 +377,24 @@ mod tests {
         }));
     }
 
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn server_daemon_detach_creates_new_session() {
+        let mut command = Command::new("sh");
+        command.arg("-c").arg(
+            r#"sid=$(ps -o sid= -p $$ | tr -d ' ')
+test "$sid" = "$$"
+"#,
+        );
+        crate::platform::detach_server_daemon_command(&mut command);
+
+        let status = command.status().unwrap();
+        assert!(
+            status.success(),
+            "detached server child should be its own session leader"
+        );
+    }
+
     #[test]
     fn is_server_listening_returns_true_for_live_socket() {
         let dir = unique_test_dir("live");
diff --git a/src/server/handoff.rs b/src/server/handoff.rs
index 816537ab..2be36c2a 100644
--- a/src/server/handoff.rs
+++ b/src/server/handoff.rs
@@ -5,8 +5,6 @@ use std::os::fd::{AsRawFd, RawFd};
 #[cfg(unix)]
 use std::os::unix::net::{UnixListener, UnixStream};
 #[cfg(unix)]
-use std::os::unix::process::CommandExt;
-#[cfg(unix)]
 use std::path::{Path, PathBuf};
 #[cfg(unix)]
 use std::process::{Child, Command};
@@ -79,10 +77,10 @@ pub(crate) fn spawn_handoff_import(
         .arg("--handoff-import")
         .arg(socket_path)
         .arg(token)
-        .process_group(0)
         .stdin(std::process::Stdio::null())
         .stdout(std::process::Stdio::null())
         .stderr(std::process::Stdio::null());
+    crate::platform::detach_server_daemon_command(&mut command);
     command.spawn().map_err(|err| {
         io::Error::new(
             err.kind(),
diff --git a/src/update.rs b/src/update.rs
index bceef7b0..8ef77840 100644
--- a/src/update.rs
+++ b/src/update.rs
@@ -2732,7 +2732,10 @@ mod tests {
             server: crate::api::RuntimeStatus {
                 version: Some("0.6.2".to_string()),
                 protocol: Some(76),
-                capabilities: Some(crate::api::schema::ServerCapabilities { live_handoff: true }),
+                capabilities: Some(crate::api::schema::ServerCapabilities {
+                    live_handoff: true,
+                    detached_server_daemon: true,
+                }),
             },
         };