fix: reuse ssh authentication and add machine recovery commands (#4490)

* fix: reuse ssh authentication and add in-app mfa recovery

refs #3763

* refactor: move ssh recovery to machine cli

refs #3763

12 files changed, 252 insertions, 13 deletions

diff --git a/src/client/endpoint/supervisor.rs b/src/client/endpoint/supervisor.rs
index a5154156..ff86ac77 100644
--- a/src/client/endpoint/supervisor.rs
+++ b/src/client/endpoint/supervisor.rs
@@ -210,7 +210,13 @@ impl EndpointSupervisors {
                 state.online_since.get_or_insert(now);
                 state.next_attempt = None;
             }
-            ClientEndpointStatus::Attention | ClientEndpointStatus::Disabled => {
+            ClientEndpointStatus::Attention => {
+                state.online_since = None;
+                // Authentication or configuration may be repaired outside this client.
+                state.next_attempt =
+                    (!endpoint_id.is_local()).then_some(now + Duration::from_secs(30));
+            }
+            ClientEndpointStatus::Disabled => {
                 state.online_since = None;
                 state.next_attempt = None;
             }
@@ -278,11 +284,11 @@ fn connect_once(
             (stream, Box::new(()))
         }
         ConnectTarget::Ssh(profile) => {
-            let connected = crate::remote::connect_saved_ssh(profile.id.as_str(), &profile.target, &profile.session).map_err(|error| {
-                if failure_needs_attention(&error) {
-                    std::io::Error::new(error.kind(), format!("{error}. Run `{}` interactively to approve setup, then restart this client", crate::remote::saved_ssh_bootstrap_command(&profile.target, &profile.session)))
-                } else { error }
-            })?;
+            let connected = crate::remote::connect_saved_ssh(
+                profile.id.as_str(),
+                &profile.target,
+                &profile.session,
+            )?;
             (connected.stream, Box::new(connected.bridge))
         }
     };
@@ -517,7 +523,7 @@ mod tests {
     }
 
     #[test]
-    fn ssh_recovery_rejects_stale_generations_and_stops_retries_for_attention() {
+    fn ssh_recovery_rejects_stale_generations_and_rechecks_attention() {
         let now = Instant::now();
         let mut supervisors = EndpointSupervisors::new(&[profile()], now);
         let endpoint_id = ClientEndpointId::Ssh(profile().id);
@@ -535,6 +541,9 @@ mod tests {
             Some(now + INITIAL_RETRY_DELAY)
         );
         assert!(supervisors.record_status(&endpoint_id, 4, ClientEndpointStatus::Attention, now));
-        assert!(supervisors.endpoints[&endpoint_id].next_attempt.is_none());
+        assert_eq!(
+            supervisors.endpoints[&endpoint_id].next_attempt,
+            Some(now + Duration::from_secs(30))
+        );
     }
 }
diff --git a/src/client/mod.rs b/src/client/mod.rs
index 68e569f9..9ad51a36 100644
--- a/src/client/mod.rs
+++ b/src/client/mod.rs
@@ -1184,6 +1184,7 @@ async fn run_client_loop(
                     }
                     let unavailable = state.shell.as_mut().and_then(|shell| {
                         shell.set_endpoint_status(&endpoint_id, status);
+                        shell.set_machine_diagnostic(&endpoint_id, message.clone());
                         (status == endpoint::ClientEndpointStatus::Attention
                             && shell.endpoint_is_active(&endpoint_id))
                         .then(|| format!("{}: {message}", shell.endpoint_label(&endpoint_id)))
diff --git a/src/client/shell.rs b/src/client/shell.rs
index 27d30df9..2d180443 100644
--- a/src/client/shell.rs
+++ b/src/client/shell.rs
@@ -3,6 +3,7 @@ use std::collections::{HashMap, HashSet, VecDeque};
 mod actions;
 mod agent_sidebar;
 mod aggregate_navigation;
+mod machine_diagnostics;
 mod workspace_navigation;
 use workspace_navigation::{PendingWorkspaceHighlight, WorkspaceNavigationTarget};
 mod composition;
diff --git a/src/client/shell/composition.rs b/src/client/shell/composition.rs
index 4da21e77..9b710dd5 100644
--- a/src/client/shell/composition.rs
+++ b/src/client/shell/composition.rs
@@ -55,6 +55,7 @@ impl ClientShellState {
                     == Some(ClientEndpointStatus::Online)
         });
         let mut render_state = render::ShellRenderState {
+            machine_diagnostics: &self.machine_diagnostics,
             endpoints: &self.endpoints,
             active_endpoint_id: &self.active_endpoint_id,
             collapsed_endpoints: &self.collapsed_endpoints,
@@ -134,6 +135,15 @@ impl ClientShellState {
             &self.config.keybinds,
             &self.config.palette,
         );
+        if let Some(notice) = &self.visible_endpoint_notice {
+            self.hits.notification_toast = endpoint_notices::render_notice(
+                &mut buffer,
+                Rect::new(0, 0, cols, rows),
+                notice,
+                1,
+                &self.config.palette,
+            );
+        }
         FrameData::from_ratatui_buffer_with_hyperlinks(&buffer, None, &[])
     }
 
@@ -197,6 +207,7 @@ impl ClientShellState {
             snapshot,
             &self.config,
             render::ShellRenderState {
+                machine_diagnostics: &self.machine_diagnostics,
                 endpoints: &self.endpoints,
                 active_endpoint_id: &self.active_endpoint_id,
                 collapsed_endpoints: &self.collapsed_endpoints,
diff --git a/src/client/shell/endpoint_sidebar.rs b/src/client/shell/endpoint_sidebar.rs
index c4f701c3..96982487 100644
--- a/src/client/shell/endpoint_sidebar.rs
+++ b/src/client/shell/endpoint_sidebar.rs
@@ -90,12 +90,26 @@ pub(super) fn render_collapsed(
                     palette.overlay0
                 }),
             );
+            let mut status_badge = Rect::default();
             if !endpoint.endpoint_id.is_local() {
                 let (glyph, _, color) = endpoint_status_presentation(endpoint.status, palette);
-                put_right_text(buffer, rect, rect.y, glyph, Style::default().fg(color));
+                let width = display_width(glyph).min(rect.width);
+                status_badge = Rect::new(rect.right().saturating_sub(width), rect.y, width, 1);
+                put_right_text(
+                    buffer,
+                    rect,
+                    rect.y,
+                    glyph,
+                    state.machine_diagnostics.badge_style(
+                        endpoint,
+                        palette,
+                        Style::default().fg(color),
+                    ),
+                );
             }
             hits.machines.push(MachineHit {
                 rect,
+                status_badge,
                 collapse_toggle: Rect::new(rect.x, rect.y, u16::from(rect.width > 1), 1),
                 endpoint_id: endpoint.endpoint_id.clone(),
             });
@@ -394,16 +408,18 @@ pub(super) fn render_expanded(
                 let rect = Rect::new(body.x, y, content_width, 1);
                 let collapsed = state.collapsed_endpoints.contains(&endpoint.endpoint_id);
                 let marker = if collapsed { "▸" } else { "▾" };
-                render_endpoint_row(
+                let status_badge = render_endpoint_row(
                     buffer,
                     rect,
                     marker,
                     endpoint,
                     collapsed && &endpoint.endpoint_id == state.active_endpoint_id,
+                    state.machine_diagnostics,
                     palette,
                 );
                 hits.machines.push(MachineHit {
                     rect,
+                    status_badge,
                     collapse_toggle: Rect::new(
                         rect.x.saturating_add(1),
                         rect.y,
@@ -577,8 +593,9 @@ fn render_endpoint_row(
     marker: &str,
     endpoint: &ClientShellEndpoint,
     highlighted: bool,
+    auth: &super::machine_diagnostics::MachineDiagnostics,
     palette: &Palette,
-) {
+) -> Rect {
     if highlighted {
         buffer.set_style(rect, Style::default().bg(palette.active_row_bg));
     }
@@ -588,7 +605,11 @@ fn render_endpoint_row(
     } else {
         state
     };
-    let signal = if endpoint.endpoint_id.is_local() {
+    let signal = if auth.required_for(endpoint) {
+        "! auth".to_owned()
+    } else if endpoint.status == ClientEndpointStatus::Attention {
+        "! error".to_owned()
+    } else if endpoint.endpoint_id.is_local() {
         String::new()
     } else if state.is_empty() {
         glyph.to_owned()
@@ -612,5 +633,17 @@ fn render_endpoint_row(
             )
             .add_modifier(Modifier::BOLD),
     );
-    put_right_text(buffer, rect, rect.y, &signal, Style::default().fg(color));
+    put_right_text(
+        buffer,
+        rect,
+        rect.y,
+        &signal,
+        auth.badge_style(endpoint, palette, Style::default().fg(color)),
+    );
+    Rect::new(
+        rect.right().saturating_sub(signal_width),
+        rect.y,
+        signal_width,
+        1,
+    )
 }
diff --git a/src/client/shell/endpoints.rs b/src/client/shell/endpoints.rs
index dd6fa3cb..fff28b71 100644
--- a/src/client/shell/endpoints.rs
+++ b/src/client/shell/endpoints.rs
@@ -26,6 +26,7 @@ pub(crate) struct ClientShellEndpoint {
 
 pub(super) struct MachineHit {
     pub(super) rect: Rect,
+    pub(super) status_badge: Rect,
     pub(super) collapse_toggle: Rect,
     pub(super) endpoint_id: ClientEndpointId,
 }
@@ -108,6 +109,7 @@ impl ClientShellState {
     }
 
     pub(crate) fn retire_endpoint(&mut self, endpoint_id: &ClientEndpointId) {
+        self.clear_machine_diagnostic(endpoint_id);
         if endpoint_id == &self.active_endpoint_id {
             self.pending_workspace_highlight = None;
         }
@@ -134,6 +136,12 @@ impl ClientShellState {
         endpoint_id: &ClientEndpointId,
         status: ClientEndpointStatus,
     ) {
+        if matches!(
+            status,
+            ClientEndpointStatus::Online | ClientEndpointStatus::Disabled
+        ) {
+            self.clear_machine_diagnostic(endpoint_id);
+        }
         if endpoint_id == &self.active_endpoint_id && status != ClientEndpointStatus::Online {
             self.pending_workspace_highlight = None;
         }
diff --git a/src/client/shell/input.rs b/src/client/shell/input.rs
index f1e7ea25..0e14e18c 100644
--- a/src/client/shell/input.rs
+++ b/src/client/shell/input.rs
@@ -163,6 +163,9 @@ impl ClientShellState {
             outcome.repaint = true;
         }
         for event in events {
+            if self.handle_machine_badge_event(&event, &mut outcome) {
+                continue;
+            }
             if let Some(update) = host_theme_update(&event) {
                 push_host_theme_update(&mut outcome.requests, update);
             }
diff --git a/src/client/shell/machine_diagnostics.rs b/src/client/shell/machine_diagnostics.rs
new file mode 100644
index 00000000..67983024
--- /dev/null
+++ b/src/client/shell/machine_diagnostics.rs
@@ -0,0 +1,130 @@
+use super::*;
+use crate::raw_input::RawInputEvent;
+use crossterm::event::{MouseButton, MouseEventKind};
+
+#[derive(Default)]
+pub(super) struct MachineDiagnostics {
+    errors: HashMap<ClientEndpointId, String>,
+    hover: Option<ClientEndpointId>,
+}
+
+impl MachineDiagnostics {
+    pub(super) fn required_for(&self, endpoint: &ClientShellEndpoint) -> bool {
+        self.errors
+            .get(&endpoint.endpoint_id)
+            .is_some_and(|message| crate::remote::ssh_error_requires_authentication(message))
+    }
+
+    pub(super) fn badge_style(
+        &self,
+        endpoint: &ClientShellEndpoint,
+        palette: &Palette,
+        style: Style,
+    ) -> Style {
+        if self.hover.as_ref() == Some(&endpoint.endpoint_id)
+            && self.errors.contains_key(&endpoint.endpoint_id)
+        {
+            style
+                .bg(palette.active_row_bg)
+                .add_modifier(Modifier::REVERSED)
+        } else {
+            style
+        }
+    }
+}
+
+impl ClientShellState {
+    pub(crate) fn set_machine_diagnostic(&mut self, id: &ClientEndpointId, message: String) {
+        if !id.is_local() {
+            self.machine_diagnostics.errors.insert(
+                id.clone(),
+                message
+                    .chars()
+                    .filter(|c| !c.is_control() || *c == '\n')
+                    .take(4096)
+                    .collect(),
+            );
+        }
+    }
+
+    pub(super) fn clear_machine_diagnostic(&mut self, id: &ClientEndpointId) {
+        self.machine_diagnostics.errors.remove(id);
+    }
+
+    pub(super) fn handle_machine_badge_event(
+        &mut self,
+        event: &RawInputEvent,
+        outcome: &mut ClientShellInput,
+    ) -> bool {
+        let RawInputEvent::Mouse(mouse) = event else {
+            return false;
+        };
+        if self.overlay.is_some()
+            || self.popup_pending
+            || self.hits.popup.is_some()
+            || contains(self.hits.notification_toast, (mouse.column, mouse.row))
+        {
+            return false;
+        }
+        let hit = self
+            .hits
+            .machines
+            .iter()
+            .find(|hit| {
+                contains(hit.status_badge, (mouse.column, mouse.row))
+                    && self
+                        .machine_diagnostics
+                        .errors
+                        .contains_key(&hit.endpoint_id)
+            })
+            .map(|hit| hit.endpoint_id.clone());
+        if mouse.kind == MouseEventKind::Moved {
+            if self.machine_diagnostics.hover != hit {
+                self.machine_diagnostics.hover = hit;
+                outcome.repaint = true;
+            }
+            return false;
+        }
+        let Some(id) = hit else {
+            return false;
+        };
+        if mouse.kind == MouseEventKind::Up(MouseButton::Left) {
+            return true;
+        }
+        if mouse.kind != MouseEventKind::Down(MouseButton::Left) {
+            return false;
+        }
+        let Some(error) = self.machine_diagnostics.errors.get(&id) else {
+            return true;
+        };
+        let ClientEndpointId::Ssh(profile_id) = &id else {
+            return true;
+        };
+        let command = if crate::remote::ssh_error_requires_authentication(error) {
+            format!("herdr machine reconnect {profile_id}")
+        } else {
+            format!("herdr machine status {profile_id}")
+        };
+        let code = format!("machine-diagnostic:{}", profile_id);
+        // An explicit click can reopen its diagnostic, but must not replace another notice.
+        if self
+            .visible_endpoint_notice
+            .as_ref()
+            .is_some_and(|notice| notice.key.code != code)
+        {
+            return true;
+        }
+        self.visible_endpoint_notice = Some(ClientVisibleEndpointNotice {
+            key: ClientEndpointNoticeKey {
+                boot_id: "machine".into(),
+                kind: ClientEndpointNoticeKind::Unavailable,
+                code,
+            },
+            title: format!("{}: {command}", self.endpoint_label(&id)),
+            body: error.clone(),
+            deadline: std::time::Instant::now() + std::time::Duration::from_secs(15),
+        });
+        outcome.repaint = true;
+        true
+    }
+}
diff --git a/src/client/shell/render.rs b/src/client/shell/render.rs
index 80387019..82cb2ac4 100644
--- a/src/client/shell/render.rs
+++ b/src/client/shell/render.rs
@@ -230,6 +230,7 @@ pub(super) fn render_mode_bar(
 }
 
 pub(super) struct ShellRenderState<'a> {
+    pub(super) machine_diagnostics: &'a super::machine_diagnostics::MachineDiagnostics,
     pub(super) endpoints: &'a [ClientShellEndpoint],
     pub(super) active_endpoint_id: &'a ClientEndpointId,
     pub(super) collapsed_endpoints: &'a HashSet<ClientEndpointId>,
diff --git a/src/client/shell/state.rs b/src/client/shell/state.rs
index 2eaf18bf..0d6937e0 100644
--- a/src/client/shell/state.rs
+++ b/src/client/shell/state.rs
@@ -844,6 +844,7 @@ pub(super) struct ClientCopyModeState {
 }
 
 pub(crate) struct ClientShellState {
+    pub(super) machine_diagnostics: super::machine_diagnostics::MachineDiagnostics,
     pub(super) config: ClientShellConfig,
     pub(super) snapshot: Option<Box<ClientShellSnapshot>>,
     pub(super) active_snapshot_generation: Option<u64>,
@@ -1007,6 +1008,7 @@ impl ClientShellState {
                 .extend(saved.collapsed_groups);
         }
         Self {
+            machine_diagnostics: Default::default(),
             config,
             snapshot: None,
             active_snapshot_generation: None,
diff --git a/src/client/shell/tests/endpoints.rs b/src/client/shell/tests/endpoints.rs
index 4434e119..f327f8c1 100644
--- a/src/client/shell/tests/endpoints.rs
+++ b/src/client/shell/tests/endpoints.rs
@@ -73,6 +73,45 @@ fn state_with_remote() -> (ClientShellState, ClientEndpointId) {
     (state, endpoint_id)
 }
 
+#[test]
+fn machine_diagnostic_badge_reopens_notice_without_collapsing_machine() {
+    let (mut state, id) = state_with_remote();
+    state.set_endpoint_status(&id, ClientEndpointStatus::Attention);
+    state.set_machine_diagnostic(&id, "Permission denied (keyboard-interactive)".into());
+    for _ in 0..2 {
+        state.compose(120, 40).unwrap();
+        let hit = state
+            .hits
+            .machines
+            .iter()
+            .find(|hit| hit.endpoint_id == id)
+            .unwrap();
+        let mouse = MouseEvent {
+            kind: MouseEventKind::Down(MouseButton::Left),
+            column: hit.status_badge.x,
+            row: hit.status_badge.y,
+            modifiers: KeyModifiers::NONE,
+        };
+        let outcome = state.handle_raw_events(vec![RawInputEvent::Mouse(mouse)]);
+        assert!(outcome.repaint);
+        assert!(!state.collapsed_endpoints.contains(&id));
+        let notice = state.visible_endpoint_notice.take().unwrap();
+        assert!(notice.body.contains("Permission denied"));
+        assert!(notice
+            .title
+            .contains("herdr machine reconnect 0123456789abcdef0123456789abcdef"));
+    }
+    state.set_endpoint_status(&id, ClientEndpointStatus::Online);
+    state.compose(120, 40).unwrap();
+    assert!(!state.machine_diagnostics.required_for(
+        state
+            .endpoints
+            .iter()
+            .find(|endpoint| endpoint.endpoint_id == id)
+            .unwrap()
+    ));
+}
+
 fn state_with_scrollable_agents() -> (ClientShellState, ClientEndpointId) {
     let (mut state, remote) = state_with_remote();
     for endpoint_id in [ClientEndpointId::Local, remote.clone()] {
diff --git a/src/client/shell_runtime.rs b/src/client/shell_runtime.rs
index 2e8d1c72..3843654f 100644
--- a/src/client/shell_runtime.rs
+++ b/src/client/shell_runtime.rs
@@ -597,6 +597,7 @@ pub(super) fn handle_endpoint_attention(
             shell.cancel_endpoint_request(&request_id);
         }
         shell.set_endpoint_status(endpoint_id, endpoint::ClientEndpointStatus::Attention);
+        shell.set_machine_diagnostic(endpoint_id, message.clone());
         endpoint_was_active.then(|| format!("{}: {message}", shell.endpoint_label(endpoint_id)))
     });
     if let Some(message) = unavailable {