feat: add stable client endpoint compatibility (#3509)

* feat: add stable client endpoint compatibility

* fix: keep portable-pty patch checks composable

* ci: run maintenance checks in pull requests

* fix: reject endpoint errors from terminal clients

* feat: keep compatible servers running across updates

1 files changed, 164 insertions, 46 deletions

diff --git a/src/update.rs b/src/update.rs
index a15c030f..879726fc 100644
--- a/src/update.rs
+++ b/src/update.rs
@@ -197,6 +197,8 @@ impl AssetRef {
 #[derive(Deserialize)]
 struct UpdateManifest {
     version: String,
+    #[cfg(not(windows))]
+    endpoint_generation: Option<u32>,
     /// Thin-client protocol spoken by this release, when advertised by the manifest.
     #[cfg(not(windows))]
     protocol: Option<u32>,
@@ -236,6 +238,7 @@ struct PreviewManifest {
     commit: String,
     built_at: String,
     protocol: u32,
+    endpoint_generation: Option<u32>,
     notes: String,
     assets: BTreeMap<String, AssetRef>,
     #[serde(default)]
@@ -248,6 +251,7 @@ struct PreviewBuildMetadata {
     commit: String,
     built_at: String,
     protocol: u32,
+    endpoint_generation: Option<u32>,
     assets: BTreeMap<String, AssetRef>,
 }
 
@@ -306,6 +310,8 @@ struct ReleaseInfo {
     commit: Option<String>,
     #[cfg(not(windows))]
     target_protocol: Option<u32>,
+    #[cfg(not(windows))]
+    target_endpoint_generation: Option<u32>,
     download_url: String,
     sha256: Option<String>,
     #[cfg(windows)]
@@ -415,6 +421,8 @@ fn release_info_from_manifest(manifest: &UpdateManifest) -> Result<Option<Releas
         commit: None,
         #[cfg(not(windows))]
         target_protocol: manifest.protocol,
+        #[cfg(not(windows))]
+        target_endpoint_generation: manifest.endpoint_generation,
         download_url,
         sha256: Some(sha256),
         #[cfg(windows)]
@@ -475,6 +483,7 @@ fn release_info_from_preview_manifest(
             || archived.commit != manifest.commit
             || archived.built_at != manifest.built_at
             || archived.protocol != manifest.protocol
+            || archived.endpoint_generation != manifest.endpoint_generation
         {
             tracing::warn!(
                 build_id,
@@ -502,6 +511,8 @@ fn release_info_from_preview_manifest(
         commit: Some(manifest.commit.clone()),
         #[cfg(not(windows))]
         target_protocol: Some(manifest.protocol),
+        #[cfg(not(windows))]
+        target_endpoint_generation: manifest.endpoint_generation,
         download_url,
         sha256: asset.sha256.clone(),
         #[cfg(windows)]
@@ -825,10 +836,14 @@ fn update_requires_server_restart(
     server: &crate::api::RuntimeStatus,
     release: &ReleaseInfo,
 ) -> bool {
-    match (server.protocol, release.target_protocol) {
-        (Some(server_protocol), Some(target_protocol)) => server_protocol != target_protocol,
-        _ => true,
-    }
+    let Some(target_generation) = release.target_endpoint_generation else {
+        return true;
+    };
+    server
+        .capabilities
+        .as_ref()
+        .and_then(|capabilities| capabilities.endpoint_protocol_generation)
+        != Some(target_generation)
 }
 
 #[cfg(not(windows))]
@@ -903,6 +918,7 @@ enum RunningServerUpdateAction {
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 #[cfg(not(windows))]
 enum RunningServerUpdateOutcome {
+    CompatibleServerKept,
     RestartDeferred,
     Stopped,
     LiveHandoffComplete,
@@ -1229,7 +1245,12 @@ fn prompt_to_complete_plain_update(
     decisions: &[RunningServerUpdateDecision],
     release: &ReleaseInfo,
 ) -> Result<bool, String> {
-    if decisions.is_empty() {
+    let plans: Vec<&RunningServerUpdatePlan> = decisions
+        .iter()
+        .filter(|decision| decision.plan.requires_server_restart)
+        .map(|decision| &decision.plan)
+        .collect();
+    if plans.is_empty() {
         return Ok(true);
     }
 
@@ -1237,8 +1258,6 @@ fn prompt_to_complete_plain_update(
         return Ok(false);
     }
 
-    let plans: Vec<&RunningServerUpdatePlan> =
-        decisions.iter().map(|decision| &decision.plan).collect();
     let (singular, plural) = target_group_nouns(&plans);
     let noun = if plans.len() == 1 { singular } else { plural };
     eprintln!(
@@ -1288,7 +1307,9 @@ fn mark_plain_update_stop_decisions(
     decisions
         .into_iter()
         .map(|mut decision| {
-            decision.action = RunningServerUpdateAction::StopOldServer;
+            if decision.plan.requires_server_restart {
+                decision.action = RunningServerUpdateAction::StopOldServer;
+            }
             decision
         })
         .collect()
@@ -1719,7 +1740,10 @@ fn apply_running_session_update_decisions(
 
     for decision in decisions {
         let outcome = match decision.action {
-            RunningServerUpdateAction::None => RunningServerUpdateOutcome::RestartDeferred,
+            RunningServerUpdateAction::None if decision.plan.requires_server_restart => {
+                RunningServerUpdateOutcome::RestartDeferred
+            }
+            RunningServerUpdateAction::None => RunningServerUpdateOutcome::CompatibleServerKept,
             RunningServerUpdateAction::StopOldServer => {
                 stop_running_server_for_update(&decision.plan)?;
                 RunningServerUpdateOutcome::Stopped
@@ -1760,6 +1784,24 @@ fn print_running_session_update_outcomes(
 
     for outcome in outcomes {
         match outcome.outcome {
+            RunningServerUpdateOutcome::CompatibleServerKept => {
+                eprintln!(
+                    "{} {} kept running server v{}.",
+                    outcome.target_noun,
+                    outcome.session_label,
+                    version_label(outcome.server_version.as_deref())
+                );
+                match &outcome.attach_command {
+                    Some(command) => eprintln!(
+                        "Run `{command}` to reconnect with the updated client. Restart the server later only if you need server-side changes from {}.",
+                        release.label()
+                    ),
+                    None => eprintln!(
+                        "Reconnect with the same socket override to use the updated client. Restart the server later only if you need server-side changes from {}.",
+                        release.label()
+                    ),
+                }
+            }
             RunningServerUpdateOutcome::LiveHandoffComplete => {
                 if let Some(command) = &outcome.attach_command {
                     eprintln!(
@@ -1858,19 +1900,19 @@ pub(crate) fn update_install_command() -> &'static str {
 pub(crate) fn update_install_instruction(install_command: &str) -> String {
     match install_command {
         HERDR_UPDATE_COMMAND => {
-            "detach, run `herdr update`, then follow its restart guidance".to_string()
+            "detach, run `herdr update`, then run Herdr again to reconnect".to_string()
         }
         HOMEBREW_UPDATE_COMMAND => {
-            "detach, run `brew update && brew upgrade herdr`, then restart this Herdr session when ready".to_string()
+            "detach, run `brew update && brew upgrade herdr`, then run Herdr again to reconnect"
+                .to_string()
         }
         MISE_UPDATE_COMMAND => {
-            "detach, run `mise upgrade herdr`, then restart this Herdr session when ready"
-                .to_string()
+            "detach, run `mise upgrade herdr`, then run Herdr again to reconnect".to_string()
         }
         NIX_UPDATE_COMMAND => {
-            "detach, update through Nix, then restart this Herdr session when ready".to_string()
+            "detach, update through Nix, then run Herdr again to reconnect".to_string()
         }
-        command => format!("detach, run `{command}`, then restart this Herdr session when ready"),
+        command => format!("detach, run `{command}`, then run Herdr again to reconnect"),
     }
 }
 
@@ -2146,7 +2188,7 @@ pub fn self_update(options: SelfUpdateOptions) -> Result<Version, String> {
         eprintln!("installed {}", release.label());
         print_outdated_integration_notice_with_updated_binary(&updated_exe);
         eprintln!(
-            "Restart any running Herdr sessions to use {}.",
+            "Start Herdr again to use the updated client. Running servers remain active; restart them later only if you need server-side changes from {}.",
             release.label()
         );
     }
@@ -2432,6 +2474,9 @@ mod tests {
             build_id: None,
             commit: None,
             target_protocol,
+            target_endpoint_generation: Some(
+                crate::protocol::endpoint::ENDPOINT_PROTOCOL_GENERATION,
+            ),
             download_url: "https://example.com/herdr".to_string(),
             sha256: None,
             notes_body: "### Changed\n- One".to_string(),
@@ -2711,15 +2756,15 @@ mod tests {
     fn update_install_instruction_distinguishes_install_from_restart() {
         assert_eq!(
             update_install_instruction(HERDR_UPDATE_COMMAND),
-            "detach, run `herdr update`, then follow its restart guidance"
+            "detach, run `herdr update`, then run Herdr again to reconnect"
         );
         assert_eq!(
             update_install_instruction(HOMEBREW_UPDATE_COMMAND),
-            "detach, run `brew update && brew upgrade herdr`, then restart this Herdr session when ready"
+            "detach, run `brew update && brew upgrade herdr`, then run Herdr again to reconnect"
         );
         assert_eq!(
             update_install_instruction(MISE_UPDATE_COMMAND),
-            "detach, run `mise upgrade herdr`, then restart this Herdr session when ready"
+            "detach, run `mise upgrade herdr`, then run Herdr again to reconnect"
         );
     }
 
@@ -2796,41 +2841,60 @@ mod tests {
     }
 
     #[test]
-    fn update_requires_server_restart_when_target_protocol_differs_or_unknown() {
-        let server = crate::api::RuntimeStatus {
+    fn update_requires_server_restart_only_without_the_endpoint_baseline() {
+        let release = fake_release("0.5.6", Some(4));
+        let compatible = crate::api::RuntimeStatus {
             version: Some("0.5.5".to_string()),
             protocol: Some(2),
-            capabilities: None,
+            capabilities: Some(crate::api::schema::ServerCapabilities {
+                live_handoff: true,
+                detached_server_daemon: true,
+                endpoint_protocol_generation: Some(
+                    crate::protocol::endpoint::ENDPOINT_PROTOCOL_GENERATION,
+                ),
+            }),
         };
-        let compatible_release = ReleaseInfo {
-            version: Version::parse("0.5.6").unwrap(),
-            identity: "0.5.6".to_string(),
-            channel: UpdateChannel::Stable,
-            build_id: None,
-            commit: None,
-            target_protocol: Some(2),
-            download_url: "https://example.com/herdr".to_string(),
-            sha256: None,
-            notes_body: "### Changed\n- One".to_string(),
+        let missing_baseline = crate::api::RuntimeStatus {
+            capabilities: None,
+            ..compatible.clone()
         };
-        let incompatible_release = ReleaseInfo {
-            target_protocol: Some(4),
-            ..compatible_release.clone()
+        let incompatible_generation = crate::api::RuntimeStatus {
+            capabilities: Some(crate::api::schema::ServerCapabilities {
+                endpoint_protocol_generation: Some(
+                    crate::protocol::endpoint::ENDPOINT_PROTOCOL_GENERATION + 1,
+                ),
+                ..compatible.capabilities.clone().unwrap()
+            }),
+            ..compatible.clone()
         };
+
+        assert!(!update_requires_server_restart(&compatible, &release));
+        assert!(update_requires_server_restart(&missing_baseline, &release));
+        assert!(update_requires_server_restart(
+            &incompatible_generation,
+            &release
+        ));
+
         let unknown_release = ReleaseInfo {
-            target_protocol: None,
-            ..compatible_release.clone()
+            target_endpoint_generation: None,
+            ..release.clone()
         };
-
-        assert!(!update_requires_server_restart(
-            &server,
-            &compatible_release
+        assert!(update_requires_server_restart(
+            &compatible,
+            &unknown_release
         ));
         assert!(update_requires_server_restart(
-            &server,
-            &incompatible_release
+            &missing_baseline,
+            &unknown_release
         ));
-        assert!(update_requires_server_restart(&server, &unknown_release));
+
+        let future_release = ReleaseInfo {
+            target_endpoint_generation: Some(
+                crate::protocol::endpoint::ENDPOINT_PROTOCOL_GENERATION + 1,
+            ),
+            ..release
+        };
+        assert!(update_requires_server_restart(&compatible, &future_release));
     }
 
     #[test]
@@ -2857,6 +2921,9 @@ mod tests {
                 capabilities: Some(crate::api::schema::ServerCapabilities {
                     live_handoff: true,
                     detached_server_daemon: true,
+                    endpoint_protocol_generation: Some(
+                        crate::protocol::endpoint::ENDPOINT_PROTOCOL_GENERATION,
+                    ),
                 }),
             },
         };
@@ -2970,9 +3037,8 @@ mod tests {
         let work_client_socket = crate::session::client_socket_path_for(Some("work"));
         fs::create_dir_all(work_client_socket.parent().unwrap()).unwrap();
         let work_client_listener = UnixListener::bind(&work_client_socket).unwrap();
-        let release = fake_release("9.8.7", Some(77));
 
-        let err = plan_running_server_updates(&release).unwrap_err();
+        let err = plan_running_server_updates(&fake_release("9.8.7", Some(77))).unwrap_err();
 
         drop(work_client_listener);
         let _ = fs::remove_dir_all(config_home);
@@ -3051,6 +3117,9 @@ mod tests {
             build_id: None,
             commit: None,
             target_protocol: Some(3),
+            target_endpoint_generation: Some(
+                crate::protocol::endpoint::ENDPOINT_PROTOCOL_GENERATION,
+            ),
             download_url: "https://example.com/herdr".to_string(),
             sha256: None,
             notes_body: "### Changed\n- One".to_string(),
@@ -3085,6 +3154,50 @@ mod tests {
     }
 
     #[test]
+    fn noninteractive_plain_update_completes_with_compatible_server() {
+        assert!(
+            !io::stdin().is_terminal(),
+            "this test relies on noninteractive test stdin"
+        );
+        let release = fake_release("9.8.7", Some(77));
+        let plan = RunningServerUpdatePlan {
+            target: RunningUpdateTarget {
+                name: Some("work".to_string()),
+                label: "work".to_string(),
+                stop_command: "herdr session stop work".to_string(),
+                attach_command: Some("herdr session attach work".to_string()),
+                socket_path: crate::session::api_socket_path_for(Some("work")),
+                client_socket_path: crate::session::client_socket_path_for(Some("work")),
+                must_be_running: true,
+            },
+            requires_server_restart: false,
+            server: crate::api::RuntimeStatus {
+                version: Some("9.8.6".to_string()),
+                protocol: Some(76),
+                capabilities: Some(crate::api::schema::ServerCapabilities {
+                    live_handoff: true,
+                    detached_server_daemon: true,
+                    endpoint_protocol_generation: Some(
+                        crate::protocol::endpoint::ENDPOINT_PROTOCOL_GENERATION,
+                    ),
+                }),
+            },
+        };
+        let decisions = confirm_running_server_update_action(
+            vec![plan],
+            &release,
+            SelfUpdateOptions {
+                live_handoff: false,
+            },
+        )
+        .unwrap();
+
+        assert!(prompt_to_complete_plain_update(&decisions, &release).unwrap());
+        let decisions = mark_plain_update_stop_decisions(decisions);
+        assert_eq!(decisions[0].action, RunningServerUpdateAction::None);
+    }
+
+    #[test]
     fn client_protocol_server_is_running_at_detects_live_socket() {
         let socket_path = unique_test_socket_path("client-live");
         let listener = UnixListener::bind(&socket_path).unwrap();
@@ -3186,6 +3299,9 @@ mod tests {
             build_id: None,
             commit: None,
             target_protocol: Some(77),
+            target_endpoint_generation: Some(
+                crate::protocol::endpoint::ENDPOINT_PROTOCOL_GENERATION,
+            ),
             download_url: "https://example.com/herdr".to_string(),
             sha256: None,
             notes_body: "### Changed\n- One".to_string(),
@@ -3296,6 +3412,7 @@ mod tests {
         let json = "{\n\
             \"version\": \"0.2.0\",\n\
             \"protocol\": 4,\n\
+            \"endpoint_generation\": 1,\n\
             \"notes\": \"### Changed\\n- One\",\n\
             \"announcement\": {\n\
                 \"id\": \"keymap-v2\",\n\
@@ -3310,6 +3427,7 @@ mod tests {
         let manifest: UpdateManifest = serde_json::from_str(json).unwrap();
         assert_eq!(manifest.version, "0.2.0");
         assert_eq!(manifest.protocol, Some(4));
+        assert_eq!(manifest.endpoint_generation, Some(1));
         assert_eq!(manifest.assets.len(), 2);
         assert_eq!(
             manifest