fix: stagger restored agent startups (#4487)

* fix: stagger restored agent startups

refs #4102

* fix: report startup delay changes requiring restart

refs #4102

10 files changed, 182 insertions, 15 deletions

diff --git a/docs/next/website/src/content/docs/configuration.mdx b/docs/next/website/src/content/docs/configuration.mdx
index c54c79a9..3dae7168 100644
--- a/docs/next/website/src/content/docs/configuration.mdx
+++ b/docs/next/website/src/content/docs/configuration.mdx
@@ -532,8 +532,11 @@ Herdr resumes supported Agent conversations after a server restart by default:
 ```toml
 [session]
 resume_agents_on_restore = true
+startup_per_agent_delay_ms = 100
 ```
 
+Automatic restores start one agent at a time, 100 milliseconds apart by default, to reduce startup contention. The first eligible agent starts immediately. Set `startup_per_agent_delay_ms = 0` to disable spacing. Manual launches and explicit terminal attaches are unaffected. Restart the Herdr server to apply changes.
+
 Only panes with a valid native session reference from an official integration can resume; other panes restore as normal shells. See [Session state and restore](/docs/session-state/) for supported Agents and persistence behavior.
 
 ## IME cursor tracking
diff --git a/docs/next/website/src/content/docs/ja/configuration.mdx b/docs/next/website/src/content/docs/ja/configuration.mdx
index 12802e84..65b683f8 100644
--- a/docs/next/website/src/content/docs/ja/configuration.mdx
+++ b/docs/next/website/src/content/docs/ja/configuration.mdx
@@ -528,8 +528,11 @@ Herdr はデフォルトで、サーバーの再起動後に対応する Agent �
 ```toml
 [session]
 resume_agents_on_restore = true
+startup_per_agent_delay_ms = 100
 ```
 
+起動時の競合を減らすため、自動復元ではデフォルトで 100 ミリ秒間隔でエージェントを 1 つずつ起動します。起動条件を満たした最初のエージェントはすぐに起動します。`startup_per_agent_delay_ms = 0` で間隔を無効にできます。手動起動と明示的なターミナルへの接続には影響しません。変更を適用するには Herdr サーバーを再起動してください。
+
 公式インテグレーションからの有効なネイティブセッション参照を持つペインだけが再開できます。その他のペインは通常のシェルとして復元されます。対応する Agent と永続化の挙動については、[セッション状態と復元](/docs/session-state/)を参照してください。
 
 ## IME カーソルトラッキング
diff --git a/docs/next/website/src/content/docs/zh-cn/configuration.mdx b/docs/next/website/src/content/docs/zh-cn/configuration.mdx
index 9c1d5b2d..8b8738d6 100644
--- a/docs/next/website/src/content/docs/zh-cn/configuration.mdx
+++ b/docs/next/website/src/content/docs/zh-cn/configuration.mdx
@@ -528,8 +528,11 @@ kitty_graphics = false
 ```toml
 [session]
 resume_agents_on_restore = true
+startup_per_agent_delay_ms = 100
 ```
 
+为减少启动时的资源争用，自动恢复默认每隔 100 毫秒启动一个智能体。第一个满足条件的智能体会立即启动。设置 `startup_per_agent_delay_ms = 0` 可禁用间隔。手动启动和显式连接终端不受影响。修改后需重启 Herdr 服务器。
+
 只有通过官方集成获得有效原生会话引用的窗格才能恢复；其他窗格会作为普通 shell 恢复。有关受支持的智能体和持久化行为，请参阅[会话状态与恢复](/docs/session-state/)。
 
 ## IME 光标跟踪
diff --git a/docs/next/website/src/data/config-reference.json b/docs/next/website/src/data/config-reference.json
index ece9a833..76318e3a 100644
--- a/docs/next/website/src/data/config-reference.json
+++ b/docs/next/website/src/data/config-reference.json
@@ -1429,6 +1429,12 @@
           "type": "boolean",
           "default": "true",
           "description": "Resume supported AI-agent panes into their native conversation sessions when restoring a Herdr session."
+        },
+        {
+          "key": "session.startup_per_agent_delay_ms",
+          "type": "integer",
+          "default": "100",
+          "description": "Milliseconds between automatic agent restores after a server restart. The first agent starts without this delay; 0 disables spacing. Does not delay manual launches or explicit terminal attaches. Restart the server to apply."
         }
       ]
     },
diff --git a/src/app/agent_resume.rs b/src/app/agent_resume.rs
index 9bf2157f..f088138c 100644
--- a/src/app/agent_resume.rs
+++ b/src/app/agent_resume.rs
@@ -25,14 +25,19 @@ impl App {
     pub(crate) fn sync_pending_agent_resume_deadline(&mut self, now: Instant) {
         if !self.has_pending_agent_resumes() {
             self.pending_agent_resume_deadline = None;
+            self.next_agent_resume_at = None;
             return;
         }
         if self.pending_agent_resume_candidates().is_empty() {
             self.pending_agent_resume_deadline = None;
             return;
         }
-        self.pending_agent_resume_deadline
-            .get_or_insert(now + super::PENDING_AGENT_RESUME_THEME_WAIT);
+        if let Some(next) = self.next_agent_resume_at {
+            self.pending_agent_resume_deadline = Some(next);
+        } else {
+            self.pending_agent_resume_deadline
+                .get_or_insert(now + super::PENDING_AGENT_RESUME_THEME_WAIT);
+        }
     }
 
     pub(crate) fn pending_agent_resume_due(&self, now: Instant) -> bool {
@@ -40,7 +45,15 @@ impl App {
             .is_some_and(|deadline| now >= deadline)
     }
 
-    pub(crate) fn start_pending_agent_resumes(&mut self, allow_empty_theme: bool) -> bool {
+    pub(crate) fn start_pending_agent_resumes(
+        &mut self,
+        now: Instant,
+        allow_empty_theme: bool,
+    ) -> bool {
+        // Geometry/theme events can also enter here; they must not bypass spacing.
+        if self.next_agent_resume_at.is_some_and(|next| now < next) {
+            return false;
+        }
         let pending = self.pending_agent_resume_candidates();
         let mut changed = false;
         for PendingAgentResumeCandidate {
@@ -64,6 +77,11 @@ impl App {
                 cols,
                 allow_empty_theme,
             );
+            if changed && !self.startup_per_agent_delay.is_zero() {
+                self.next_agent_resume_at = Some(now + self.startup_per_agent_delay);
+                self.pending_agent_resume_deadline = self.next_agent_resume_at;
+                break;
+            }
         }
 
         if changed {
@@ -72,6 +90,9 @@ impl App {
         if !self.has_pending_agent_resumes() || self.pending_agent_resume_candidates().is_empty() {
             self.pending_agent_resume_deadline = None;
         }
+        if !self.has_pending_agent_resumes() {
+            self.next_agent_resume_at = None;
+        }
         changed
     }
 
@@ -375,6 +396,87 @@ mod tests {
         )
     }
 
+    #[tokio::test]
+    async fn pending_agent_resume_spacing_survives_events_and_failed_restores() {
+        for delay_ms in [100, 250, 0] {
+            let config: crate::config::Config = toml::from_str(&format!(
+                "[session]\nstartup_per_agent_delay_ms = {delay_ms}"
+            ))
+            .unwrap();
+            let (_api_tx, api_rx) = tokio::sync::mpsc::unbounded_channel();
+            let mut app = App::new(
+                &config,
+                crate::app::AppPolicy::TEST,
+                None,
+                api_rx,
+                crate::api::EventHub::default(),
+            );
+            app.state.workspaces = (0..4)
+                .map(|_| crate::workspace::Workspace::test_new("restore"))
+                .collect();
+            app.state.active = Some(0);
+            app.state.view.terminal_area = Rect::new(0, 0, 100, 30);
+            app.state.ensure_test_terminals();
+            let missing = std::env::current_dir()
+                .unwrap()
+                .join("__missing_resume_cwd__");
+            assert!(!missing.exists());
+            for terminal in app.state.terminals.values_mut() {
+                terminal.cwd = missing.clone();
+                terminal.pending_agent_resume_plan = Some(crate::agent_resume::AgentResumePlan {
+                    agent: "codex".into(),
+                    argv: vec!["codex".into()],
+                    dedupe_key: terminal.id.to_string(),
+                });
+            }
+            let now = Instant::now();
+            app.sync_pending_agent_resume_deadline(now);
+            assert!(!app.start_pending_agent_resumes(now, false));
+            assert!(app.start_pending_agent_resumes(now, true));
+            if delay_ms != 0 {
+                let next = now + std::time::Duration::from_millis(delay_ms);
+                assert_eq!(
+                    app.state
+                        .terminals
+                        .values()
+                        .filter(|t| t.restore_error.is_some())
+                        .count(),
+                    1
+                );
+                // Geometry changes clear the wakeup, but must preserve the launch gap.
+                app.pending_agent_resume_deadline = None;
+                app.sync_pending_agent_resume_deadline(now);
+                assert_eq!(app.pending_agent_resume_deadline, Some(next));
+                assert!(!app
+                    .start_pending_agent_resumes(next - std::time::Duration::from_millis(1), true));
+                // A late wakeup must not release every overdue agent in a burst.
+                for processed in 2..=4 {
+                    let late = now + std::time::Duration::from_secs(processed * 10);
+                    assert!(app.start_pending_agent_resumes(late, true));
+                    assert_eq!(
+                        app.state
+                            .terminals
+                            .values()
+                            .filter(|t| t.restore_error.is_some())
+                            .count(),
+                        processed as usize
+                    );
+                }
+            }
+            assert!(!app.has_pending_agent_resumes());
+            assert!(app.pending_agent_resume_deadline.is_none());
+            assert!(app.next_agent_resume_at.is_none());
+            assert_eq!(
+                app.state
+                    .terminals
+                    .values()
+                    .filter(|t| t.restore_error.is_some())
+                    .count(),
+                4
+            );
+        }
+    }
+
     #[cfg(unix)]
     fn long_running_test_argv() -> Vec<String> {
         vec!["/bin/sh".into(), "-c".into(), "sleep 5".into()]
@@ -458,7 +560,7 @@ mod tests {
             dedupe_key: "herdr:codex\0codex\0Id\0codex-session".into(),
         });
 
-        assert!(!app.start_pending_agent_resumes(false));
+        assert!(!app.start_pending_agent_resumes(Instant::now(), false));
         assert!(app.terminal_runtimes.get(&terminal_id).is_none());
 
         app.state.host_terminal_theme = crate::terminal_theme::TerminalTheme {
@@ -475,7 +577,7 @@ mod tests {
             ..Default::default()
         };
 
-        assert!(app.start_pending_agent_resumes(false));
+        assert!(app.start_pending_agent_resumes(Instant::now(), false));
         assert!(app.terminal_runtimes.get(&terminal_id).is_some());
         let terminal = app
             .state
@@ -537,8 +639,8 @@ mod tests {
         });
 
         app.sync_pending_agent_resume_deadline(std::time::Instant::now());
-        assert!(!app.start_pending_agent_resumes(false));
-        assert!(app.start_pending_agent_resumes(true));
+        assert!(!app.start_pending_agent_resumes(Instant::now(), false));
+        assert!(app.start_pending_agent_resumes(Instant::now(), true));
         assert!(app.terminal_runtimes.get(&terminal_id).is_some());
 
         for (_, runtime) in app.terminal_runtimes.drain() {
@@ -590,8 +692,14 @@ mod tests {
         app.pending_agent_resume_deadline =
             Some(std::time::Instant::now() - std::time::Duration::from_millis(1));
 
-        assert!(app.start_pending_agent_resumes(false));
+        let now = Instant::now();
+        assert!(app.start_pending_agent_resumes(now, false));
         assert!(app.terminal_runtimes.get(&active_terminal).is_some());
+        assert!(app.terminal_runtimes.get(&hidden_terminal).is_none());
+        assert!(!app.start_pending_agent_resumes(now, true));
+        assert!(
+            app.start_pending_agent_resumes(now + std::time::Duration::from_millis(100), false,)
+        );
         assert!(app.terminal_runtimes.get(&hidden_terminal).is_some());
         assert!(
             app.pending_agent_resume_deadline.is_none(),
@@ -651,7 +759,7 @@ mod tests {
             dedupe_key: "herdr:codex\0codex\0Id\0inactive-tab-session".into(),
         });
 
-        assert!(app.start_pending_agent_resumes(false));
+        assert!(app.start_pending_agent_resumes(Instant::now(), false));
         assert!(app.terminal_runtimes.get(&inactive_terminal).is_some());
         assert!(
             app.state
@@ -712,7 +820,7 @@ mod tests {
             dedupe_key: "herdr:codex\0codex\0Id\0zoom-hidden-session".into(),
         });
 
-        assert!(app.start_pending_agent_resumes(false));
+        assert!(app.start_pending_agent_resumes(Instant::now(), false));
         assert!(app.terminal_runtimes.get(&hidden_terminal).is_some());
         assert!(
             app.state
@@ -772,7 +880,7 @@ mod tests {
 
         app.sync_pending_agent_resume_deadline(std::time::Instant::now());
         assert!(app.pending_agent_resume_deadline.is_some());
-        assert!(app.start_pending_agent_resumes(false));
+        assert!(app.start_pending_agent_resumes(Instant::now(), false));
         assert!(app.terminal_runtimes.get(&previous_terminal).is_some());
         assert!(
             app.state
@@ -831,7 +939,7 @@ mod tests {
             dedupe_key: "herdr:codex\0codex\0Id\0codex-session".into(),
         });
 
-        assert!(app.start_pending_agent_resumes(false));
+        assert!(app.start_pending_agent_resumes(Instant::now(), false));
         assert_eq!(
             app.terminal_runtimes
                 .get(&terminal_id)
diff --git a/src/app/mod.rs b/src/app/mod.rs
index 757a7f66..7ff3040b 100644
--- a/src/app/mod.rs
+++ b/src/app/mod.rs
@@ -136,6 +136,8 @@ pub struct App {
     pub(crate) loaded_host_cursor: crate::config::HostCursorModeConfig,
     pub(crate) agent_metadata_deadline: Option<Instant>,
     pub(crate) pending_agent_resume_deadline: Option<Instant>,
+    startup_per_agent_delay: Duration,
+    next_agent_resume_at: Option<Instant>,
     pub(crate) session_save_deadline: Option<Instant>,
     pub(crate) session_save_thread: Option<std::thread::JoinHandle<()>>,
     session_writer: Arc<std::sync::Mutex<crate::persist::SessionWriter>>,
@@ -600,6 +602,10 @@ impl App {
             loaded_host_cursor: config.ui.host_cursor,
             agent_metadata_deadline: None,
             pending_agent_resume_deadline: None,
+            startup_per_agent_delay: Duration::from_millis(
+                config.session.startup_per_agent_delay_ms.into(),
+            ),
+            next_agent_resume_at: None,
             session_save_deadline: None,
             session_save_thread: None,
             session_writer,
@@ -857,6 +863,16 @@ impl App {
             }
         }
 
+        if !invalid_section("session")
+            && Duration::from_millis(config.session.startup_per_agent_delay_ms.into())
+                != self.startup_per_agent_delay
+        {
+            diagnostics.push(
+                "session.startup_per_agent_delay_ms changes require restarting Herdr; kept current setting"
+                    .into(),
+            );
+        }
+
         let graphics_config_valid = !invalid_section("terminal")
             && (config.terminal.kitty_graphics.is_some() || !invalid_section("experimental"));
         if graphics_config_valid
@@ -1761,6 +1777,26 @@ mod tests {
     }
 
     #[test]
+    fn reload_config_reports_startup_delay_requires_restart() {
+        let mut app = test_app();
+        let mut config = Config::default();
+        let report = app.apply_live_config(&config, &[], &[], false);
+        assert_eq!(report.status, crate::config::ConfigReloadStatus::Applied);
+
+        config.session.startup_per_agent_delay_ms = 250;
+        let report = app.apply_live_config(&config, &[], &[], false);
+        assert_eq!(report.status, crate::config::ConfigReloadStatus::Partial);
+        assert_eq!(app.startup_per_agent_delay, Duration::from_millis(100));
+        assert_eq!(report.diagnostics, vec![
+            "session.startup_per_agent_delay_ms changes require restarting Herdr; kept current setting"
+        ]);
+
+        let report = app.apply_live_config(&config, &[], &["session".into()], false);
+        assert!(report.diagnostics.is_empty());
+        assert_eq!(app.startup_per_agent_delay, Duration::from_millis(100));
+    }
+
+    #[test]
     fn reload_config_requests_client_reload_for_key_only_change() {
         let _guard = config_env_lock().lock().unwrap();
         let path = temp_config_path("reload-config-key-only");
diff --git a/src/config/model.rs b/src/config/model.rs
index 930df026..ce59eb2e 100644
--- a/src/config/model.rs
+++ b/src/config/model.rs
@@ -268,12 +268,15 @@ pub struct SessionConfig {
     /// Resume supported AI-agent panes into their native conversation sessions
     /// when restoring a Herdr session. Default: true.
     pub resume_agents_on_restore: bool,
+    /// Milliseconds between automatic agent restores. Zero disables spacing.
+    pub startup_per_agent_delay_ms: u32,
 }
 
 impl Default for SessionConfig {
     fn default() -> Self {
         Self {
             resume_agents_on_restore: true,
+            startup_per_agent_delay_ms: 100,
         }
     }
 }
@@ -1397,13 +1400,16 @@ new_cwd = "~/Projects"
     fn resume_agents_on_restore_defaults_on_and_parses() {
         let default_config = Config::default();
         assert!(default_config.session.resume_agents_on_restore);
+        assert_eq!(default_config.session.startup_per_agent_delay_ms, 100);
 
         let toml = r#"
 [session]
 resume_agents_on_restore = false
+startup_per_agent_delay_ms = 0
 "#;
         let config: Config = toml::from_str(toml).unwrap();
         assert!(!config.session.resume_agents_on_restore);
+        assert_eq!(config.session.startup_per_agent_delay_ms, 0);
     }
 
     #[test]
diff --git a/src/main.rs b/src/main.rs
index 5d7582cc..d5275529 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -393,6 +393,8 @@ const DEFAULT_CONFIG: &str = r##"# herdr configuration
 # Resume supported AI-agent panes into their native conversation sessions after
 # a Herdr server restart. Requires official integrations that report session refs.
 # resume_agents_on_restore = true
+# Milliseconds between automatic agent restores; 0 starts them without spacing.
+# startup_per_agent_delay_ms = 100
 
 [remote]
 # Whether herdr manages the ssh config used for `herdr --remote`.
diff --git a/src/server/headless.rs b/src/server/headless.rs
index c5a644ce..0aed2a37 100644
--- a/src/server/headless.rs
+++ b/src/server/headless.rs
@@ -808,7 +808,7 @@ impl HeadlessServer {
         self.app.sync_pending_agent_resume_deadline(now);
         if self
             .app
-            .start_pending_agent_resumes(self.app.pending_agent_resume_due(now))
+            .start_pending_agent_resumes(now, self.app.pending_agent_resume_due(now))
         {
             for client in self.clients.values_mut() {
                 client.request_recompute();
@@ -3445,7 +3445,7 @@ impl HeadlessServer {
             self.app.sync_pending_agent_resume_deadline(now);
             changed |= self
                 .app
-                .start_pending_agent_resumes(self.app.pending_agent_resume_due(now));
+                .start_pending_agent_resumes(now, self.app.pending_agent_resume_due(now));
         }
         changed
     }
diff --git a/src/server/headless/client_views.rs b/src/server/headless/client_views.rs
index 62fd04e4..c2e3c2d5 100644
--- a/src/server/headless/client_views.rs
+++ b/src/server/headless/client_views.rs
@@ -527,7 +527,7 @@ impl HeadlessServer {
         self.app.sync_pending_agent_resume_deadline(now);
         if self
             .app
-            .start_pending_agent_resumes(self.app.pending_agent_resume_due(now))
+            .start_pending_agent_resumes(now, self.app.pending_agent_resume_due(now))
         {
             for client in self.clients.values_mut() {
                 client.request_recompute();