diff options
| author | Can Celik <ogulcancelik@gmail.com> | 2026-09-14 03:55:53 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-09-14 03:55:53 +0300 |
| commit | c77af1892ff121736ecb103b32d504d6f1b31805 (patch) | |
| tree | f3012dfa1720ddbb5c2e789c3544fe0915f9c426 /src/platform | |
| parent | 21c1f8cf856affb875b5ab4302225a63f4563bdd (diff) | |
fix: clean up idle ssh bridges and back off flapping connections (#4083)
* fix: clean up idle ssh bridges and back off flapping connections
* chore: keep ssh liveness smoke tooling local
Diffstat (limited to 'src/platform')
| -rw-r--r-- | src/platform/fallback.rs | 8 | ||||
| -rw-r--r-- | src/platform/linux.rs | 2 | ||||
| -rw-r--r-- | src/platform/macos.rs | 2 | ||||
| -rw-r--r-- | src/platform/mod.rs | 7 | ||||
| -rw-r--r-- | src/platform/remote_bridge.rs | 135 | ||||
| -rw-r--r-- | src/platform/remote_bridge_tests.rs | 168 | ||||
| -rw-r--r-- | src/platform/unix_common.rs | 28 | ||||
| -rw-r--r-- | src/platform/windows.rs | 5 |
8 files changed, 348 insertions, 7 deletions
diff --git a/src/platform/fallback.rs b/src/platform/fallback.rs index a533796f..21d13ba0 100644 --- a/src/platform/fallback.rs +++ b/src/platform/fallback.rs @@ -13,7 +13,13 @@ pub(crate) fn set_default_plugin_pane_pwd( ) { } -pub(crate) fn forward_remote_bridge_stdio(stream: crate::ipc::LocalStream) -> std::io::Result<()> { +#[cfg(unix)] +pub(super) const REMOTE_BRIDGE_CLOCK: libc::clockid_t = libc::CLOCK_MONOTONIC; + +pub(crate) fn forward_remote_bridge_stdio( + stream: crate::ipc::LocalStream, + _idle_timeout: bool, +) -> std::io::Result<()> { use interprocess::TryClone as _; let mut stdout = std::io::stdout().lock(); diff --git a/src/platform/linux.rs b/src/platform/linux.rs index d563512d..7ea0a2c6 100644 --- a/src/platform/linux.rs +++ b/src/platform/linux.rs @@ -7,6 +7,8 @@ use std::{ sync::OnceLock, }; +pub(super) const REMOTE_BRIDGE_CLOCK: libc::clockid_t = libc::CLOCK_BOOTTIME; + use super::{ read_limited_reader, ClipboardCommand, ClipboardImage, ForegroundJob, ForegroundProcess, LimitedRead, Signal, diff --git a/src/platform/macos.rs b/src/platform/macos.rs index 422bbfca..4e86deab 100644 --- a/src/platform/macos.rs +++ b/src/platform/macos.rs @@ -7,6 +7,8 @@ use std::process::{Command, Stdio}; use std::ptr::NonNull; use std::sync::OnceLock; +pub(super) const REMOTE_BRIDGE_CLOCK: libc::clockid_t = libc::CLOCK_MONOTONIC; + use super::{ read_limited_reader, ClipboardCommand, ClipboardImage, ForegroundJob, ForegroundProcess, LimitedRead, Signal, diff --git a/src/platform/mod.rs b/src/platform/mod.rs index 3efad45e..a578b42d 100644 --- a/src/platform/mod.rs +++ b/src/platform/mod.rs @@ -266,6 +266,13 @@ pub(crate) struct RemoteSshConfigPaths { pub(crate) multiplexing: bool, } +pub(crate) const REMOTE_BRIDGE_IDLE_TIMEOUT_SUPPORTED: bool = + cfg!(any(target_os = "linux", target_os = "macos")); + +#[cfg(unix)] +mod remote_bridge; +#[cfg(all(test, unix))] +mod remote_bridge_tests; #[cfg(unix)] mod unix_common; #[cfg(unix)] diff --git a/src/platform/remote_bridge.rs b/src/platform/remote_bridge.rs new file mode 100644 index 00000000..a8db4d29 --- /dev/null +++ b/src/platform/remote_bridge.rs @@ -0,0 +1,135 @@ +use std::io::{self, Read, Write}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + mpsc, Arc, +}; +use std::time::Duration; + +pub(crate) const IDLE_TIMEOUT: Duration = Duration::from_secs(60); + +fn now() -> io::Result<u64> { + let mut time = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // This clock includes suspend, so short maintenance wakes can reap old bridges. + if unsafe { libc::clock_gettime(super::REMOTE_BRIDGE_CLOCK, &mut time) } != 0 { + return Err(io::Error::last_os_error()); + } + Ok((time.tv_sec as u64) * 1_000_000_000 + time.tv_nsec as u64) +} + +#[derive(Clone)] +pub(super) struct Activity { + last: Arc<AtomicU64>, + _stop: mpsc::Sender<()>, +} + +impl Activity { + pub(super) fn start(timeout: Duration) -> io::Result<Self> { + let last = Arc::new(AtomicU64::new(now()?)); + let watched = Arc::clone(&last); + let (stop, stopped) = mpsc::channel(); + std::thread::Builder::new() + .name("ssh-bridge-liveness".into()) + .spawn(move || { + loop { + match stopped.recv_timeout(timeout.min(Duration::from_secs(1))) { + Ok(()) | Err(mpsc::RecvTimeoutError::Disconnected) => return, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + let expired = now().map_or(true, |now| idle_expired(&watched, now, timeout)); + if expired { + // Only the dedicated bridge process owns this watchdog. Returning from a + // blocked copy cannot guarantee shutdown, and joining it could hang forever. + std::process::exit(1); + } + } + })?; + Ok(Self { last, _stop: stop }) + } + + fn record(&self) -> io::Result<()> { + self.last.fetch_max(now()?, Ordering::Relaxed); + Ok(()) + } +} + +fn idle_expired(last: &AtomicU64, now: u64, timeout: Duration) -> bool { + Duration::from_nanos(now.saturating_sub(last.load(Ordering::Relaxed))) >= timeout +} + +pub(super) struct TrackedIo<T> { + inner: T, + activity: Option<Activity>, +} + +impl<T> TrackedIo<T> { + pub(super) fn new(inner: T, activity: Option<Activity>) -> Self { + Self { inner, activity } + } + + fn progressed(&self, count: usize) -> io::Result<()> { + if count > 0 { + if let Some(activity) = &self.activity { + activity.record()?; + } + } + Ok(()) + } +} + +impl<T: Read> Read for TrackedIo<T> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> { + let count = self.inner.read(buffer)?; + self.progressed(count)?; + Ok(count) + } +} + +impl<T: Write> Write for TrackedIo<T> { + fn write(&mut self, buffer: &[u8]) -> io::Result<usize> { + let count = self.inner.write(buffer)?; + self.progressed(count)?; + Ok(count) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn idle_deadline_counts_elapsed_sleep_and_only_positive_io_renews_it() { + let (stop, _stopped) = mpsc::channel(); + let last = Arc::new(AtomicU64::new(0)); + let activity = Activity { + last: Arc::clone(&last), + _stop: stop, + }; + assert!(idle_expired( + &last, + IDLE_TIMEOUT.as_nanos() as u64, + IDLE_TIMEOUT + )); + let mut reader = TrackedIo::new(&b"output"[..], Some(activity.clone())); + reader.read_exact(&mut [0; 6]).unwrap(); + let after_read = last.load(Ordering::Relaxed); + assert!(after_read > 0); + assert!(!idle_expired(&last, after_read, IDLE_TIMEOUT)); + assert_eq!(reader.read(&mut [0; 1]).unwrap(), 0); + assert_eq!(last.load(Ordering::Relaxed), after_read); + let mut writer = TrackedIo::new(Vec::new(), Some(activity)); + writer.write_all(b"ping").unwrap(); + assert!(last.load(Ordering::Relaxed) >= after_read); + assert!(idle_expired( + &last, + last.load(Ordering::Relaxed) + 120_000_000_000, + IDLE_TIMEOUT + )); + } +} diff --git a/src/platform/remote_bridge_tests.rs b/src/platform/remote_bridge_tests.rs new file mode 100644 index 00000000..ef4aade8 --- /dev/null +++ b/src/platform/remote_bridge_tests.rs @@ -0,0 +1,168 @@ +use std::io::{Read, Write}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::PathBuf; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +const TIMEOUT: Duration = Duration::from_millis(300); + +#[test] +fn bridge_child() { + let Some(path) = std::env::var_os("HERDR_BRIDGE_TEST_SOCKET") else { + return; + }; + let stream = crate::ipc::connect_local_stream(&PathBuf::from(path)).unwrap(); + let timeout = (std::env::var_os("HERDR_BRIDGE_TEST_LEGACY").is_none()).then_some(TIMEOUT); + super::unix_common::forward_remote_bridge_stdio_with_timeout(stream, timeout).unwrap(); +} + +struct Bridge { + child: Child, + stream: UnixStream, + path: PathBuf, +} + +impl Bridge { + fn start(legacy: bool) -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + let path = std::env::temp_dir().join(format!( + "hbl-{}-{}.sock", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let listener = UnixListener::bind(&path).unwrap(); + listener.set_nonblocking(true).unwrap(); + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "platform::remote_bridge_tests::bridge_child", + "--nocapture", + ]) + .env("HERDR_BRIDGE_TEST_SOCKET", &path) + .env_remove("HERDR_BRIDGE_TEST_LEGACY") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + if legacy { + command.env("HERDR_BRIDGE_TEST_LEGACY", "1"); + } + let mut child = command.spawn().unwrap(); + let deadline = Instant::now() + Duration::from_secs(3); + let stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) + if error.kind() == std::io::ErrorKind::WouldBlock + && Instant::now() < deadline => + { + std::thread::sleep(Duration::from_millis(5)); + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + let _ = std::fs::remove_file(&path); + panic!("bridge did not connect: {error}"); + } + } + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_millis(100))) + .unwrap(); + Self { + child, + stream, + path, + } + } + + fn wait(&mut self) -> ExitStatus { + let deadline = Instant::now() + Duration::from_secs(3); + loop { + if let Some(status) = self.child.try_wait().unwrap() { + return status; + } + assert!(Instant::now() < deadline, "bridge did not exit"); + std::thread::sleep(Duration::from_millis(10)); + } + } + + fn finish(&mut self) -> String { + drop(self.child.stdin.take()); + let mut input = Vec::new(); + self.stream.read_to_end(&mut input).unwrap(); + self.stream + .write_all(b"final-output-after-stdin-eof") + .unwrap(); + self.stream.shutdown(std::net::Shutdown::Write).unwrap(); + assert!(self.wait().success()); + let mut output = String::new(); + self.child + .stdout + .take() + .unwrap() + .read_to_string(&mut output) + .unwrap(); + output + } +} + +impl Drop for Bridge { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = std::fs::remove_file(&self.path); + } +} + +#[test] +fn bridge_expires_when_silent_or_stdout_is_blocked() { + for blocked in [false, true] { + let mut bridge = Bridge::start(false); + if blocked { + let buffer = vec![b'x'; 64 * 1024]; + let deadline = Instant::now() + Duration::from_secs(2); + while bridge.stream.write_all(&buffer).is_ok() { + assert!(Instant::now() < deadline, "failed to fill bridge stdout"); + } + } + assert_eq!(bridge.wait().code(), Some(1)); + } +} + +#[test] +fn bridge_preserves_one_way_progress_and_drains_after_stdin_eof() { + for upload in [false, true] { + let mut bridge = Bridge::start(false); + for _ in 0..12 { + if upload { + bridge + .child + .stdin + .as_mut() + .unwrap() + .write_all(b"ping") + .unwrap(); + bridge.stream.read_exact(&mut [0; 4]).unwrap(); + } else { + bridge.stream.write_all(b"output").unwrap(); + } + std::thread::sleep(Duration::from_millis(60)); + assert!(bridge.child.try_wait().unwrap().is_none()); + } + assert!(bridge.finish().contains("final-output-after-stdin-eof")); + } +} + +#[test] +fn legacy_bridge_has_no_idle_deadline() { + let mut bridge = Bridge::start(true); + std::thread::sleep(TIMEOUT * 2); + assert!(bridge.child.try_wait().unwrap().is_none()); + assert!(bridge.finish().contains("final-output-after-stdin-eof")); +} diff --git a/src/platform/unix_common.rs b/src/platform/unix_common.rs index a633f283..a937b863 100644 --- a/src/platform/unix_common.rs +++ b/src/platform/unix_common.rs @@ -121,15 +121,33 @@ pub(crate) fn wait_client_stream_readable(stream: &crate::ipc::LocalStream) -> s Ok(()) } -pub(crate) fn forward_remote_bridge_stdio(stream: crate::ipc::LocalStream) -> std::io::Result<()> { +pub(crate) fn forward_remote_bridge_stdio( + stream: crate::ipc::LocalStream, + idle_timeout: bool, +) -> std::io::Result<()> { + forward_remote_bridge_stdio_with_timeout( + stream, + idle_timeout.then_some(super::remote_bridge::IDLE_TIMEOUT), + ) +} + +pub(super) fn forward_remote_bridge_stdio_with_timeout( + stream: crate::ipc::LocalStream, + idle_timeout: Option<std::time::Duration>, +) -> std::io::Result<()> { + use super::remote_bridge::{Activity, TrackedIo}; use interprocess::TryClone as _; - let mut stdout = std::io::stdout().lock(); - let mut socket_to_stdout = stream.try_clone()?; + let activity = idle_timeout.map(Activity::start).transpose()?; + let mut stdout = TrackedIo::new(std::io::stdout().lock(), activity.clone()); + let mut socket_to_stdout = TrackedIo::new(stream.try_clone()?, activity.clone()); let mut stdin_to_socket = stream; let _upload = std::thread::spawn(move || { - let mut stdin = std::io::stdin(); - let _ = copy_flush(&mut stdin, &mut stdin_to_socket); + let mut stdin = TrackedIo::new(std::io::stdin(), activity.clone()); + let _ = copy_flush( + &mut stdin, + &mut TrackedIo::new(&mut stdin_to_socket, activity), + ); let crate::ipc::LocalStream::UdSocket(stream) = stdin_to_socket; let _ = stream.inner().shutdown(std::net::Shutdown::Write); }); diff --git a/src/platform/windows.rs b/src/platform/windows.rs index 8344be2d..879778cc 100644 --- a/src/platform/windows.rs +++ b/src/platform/windows.rs @@ -52,7 +52,10 @@ pub(crate) fn wait_client_stream_readable( Ok(()) } -pub(crate) fn forward_remote_bridge_stdio(stream: crate::ipc::LocalStream) -> std::io::Result<()> { +pub(crate) fn forward_remote_bridge_stdio( + stream: crate::ipc::LocalStream, + _idle_timeout: bool, +) -> std::io::Result<()> { use interprocess::TryClone as _; use std::sync::atomic::{AtomicBool, Ordering}; |
