diff options
| author | Ogulcan Celik <ogulcancelik@gmail.com> | 2026-03-23 00:36:52 +0300 |
|---|---|---|
| committer | Ogulcan Celik <ogulcancelik@gmail.com> | 2026-03-27 21:37:05 +0300 |
| commit | a57b97285c3308fbdcc167ab3da9ae1dd78767b6 (patch) | |
| tree | 25a0eb4df100ea9b09c7551fa7cc25f8d86027ef /tests | |
initial release
terminal workspace manager for AI coding agents.
single rust binary, runs inside your existing terminal.
workspaces, tiled panes, intelligent agent state detection.
https://herdr.dev
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/integration/ext/herdr-test.ts | 146 | ||||
| -rwxr-xr-x | tests/integration/pdeathsig.py | 44 | ||||
| -rw-r--r-- | tests/integration/results/.gitignore | 2 | ||||
| -rwxr-xr-x | tests/integration/run_all.sh | 116 | ||||
| -rwxr-xr-x | tests/integration/run_spec.sh | 126 | ||||
| -rw-r--r-- | tests/integration/specs/create_workspace.md | 12 | ||||
| -rw-r--r-- | tests/integration/specs/sidebar_collapse.md | 13 | ||||
| -rw-r--r-- | tests/integration/specs/split_and_navigate.md | 11 | ||||
| -rw-r--r-- | tests/integration/specs/workspace_switching.md | 12 | ||||
| -rw-r--r-- | tests/integration/system.md | 48 |
10 files changed, 530 insertions, 0 deletions
diff --git a/tests/integration/ext/herdr-test.ts b/tests/integration/ext/herdr-test.ts new file mode 100644 index 00000000..4b6c17d5 --- /dev/null +++ b/tests/integration/ext/herdr-test.ts @@ -0,0 +1,146 @@ +/** + * Minimal tmux extension for herdr integration testing. + * + * Two actions: send and read. No wait, no pane creation, no discovery loops. + * Talks directly to a pane tagged @pi_name=herdr. + */ + +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import { StringEnum } from "@mariozechner/pi-ai"; +import { Text } from "@mariozechner/pi-tui"; +import { Type } from "@sinclair/typebox"; + +function stripAnsi(text: string): string { + return text + .replace(/\x1b\].*?(?:\x07|\x1b\\)/g, "") + .replace(/\x1bPtmux;.*?\x1b\\/g, "") + .replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "") + .replace(/\x1b[^[\]P]/g, "") + .replace(/\r/g, ""); +} + +export default function (pi: ExtensionAPI) { + let herdrPaneId: string | null = null; + let windowId: string | null = null; + + pi.on("session_start", async () => { + if (!process.env.TMUX) return; + + try { + const winResult = await pi.exec("tmux", [ + "display-message", "-p", "-t", process.env.TMUX_PANE || "", + "#{window_id}", + ]); + if (winResult.code === 0) windowId = winResult.stdout.trim(); + + if (windowId) { + const listResult = await pi.exec("tmux", [ + "list-panes", "-t", windowId, + "-F", "#{pane_id}\t#{@pi_name}", + ]); + if (listResult.code === 0) { + for (const line of listResult.stdout.trim().split("\n")) { + const [paneId, name] = line.split("\t"); + if (name === "herdr") { + herdrPaneId = paneId; + break; + } + } + } + } + } catch {} + }); + + pi.registerTool({ + name: "herdr", + label: "herdr", + description: + "Interact with the herdr instance under test. " + + "Actions: send (send keys or text to herdr), read (capture herdr screen).", + parameters: Type.Object({ + action: StringEnum(["send", "read"] as const, { + description: "Action: send or read", + }), + keys: Type.Optional( + Type.String({ + description: + "Keys to send, space-separated (for send). Examples: C-s, Enter, n, v, Escape", + }), + ), + text: Type.Optional( + Type.String({ description: "Literal text to type (for send). Sent as-is." }), + ), + lines: Type.Optional( + Type.Number({ description: "Scrollback lines to capture (for read, default: 50)" }), + ), + }), + + async execute(_toolCallId, params) { + if (!herdrPaneId) { + throw new Error("herdr pane not found. Is herdr running in this tmux window?"); + } + + switch (params.action) { + case "send": { + if (!params.keys && !params.text) { + throw new Error("send requires 'keys' or 'text'"); + } + if (params.text) { + await pi.exec("tmux", [ + "send-keys", "-l", "-t", herdrPaneId, params.text, + ]); + } + if (params.keys) { + const keyArgs = params.keys.split(/\s+/).filter(Boolean); + await pi.exec("tmux", [ + "send-keys", "-t", herdrPaneId, ...keyArgs, + ]); + } + const desc = [ + params.text && `"${params.text}"`, + params.keys, + ].filter(Boolean).join(" + "); + return { + content: [{ type: "text", text: `Sent: ${desc}` }], + }; + } + + case "read": { + const lines = params.lines ?? 50; + const result = await pi.exec("tmux", [ + "capture-pane", "-t", herdrPaneId, "-p", "-S", `-${lines}`, + ]); + if (result.code !== 0) { + throw new Error(`capture-pane failed: ${result.stderr}`); + } + let output = stripAnsi(result.stdout); + output = output.replace(/\n+$/, "\n"); + return { + content: [{ type: "text", text: output }], + }; + } + + default: + throw new Error(`Unknown action: ${params.action}`); + } + }, + + renderCall(args, theme) { + const action = args.action || "?"; + let t = theme.fg("toolTitle", theme.bold("herdr ")); + t += theme.fg("accent", action); + if (args.text) t += theme.fg("dim", ` "${args.text}"`); + if (args.keys) t += theme.fg("dim", ` ${args.keys}`); + return new Text(t, 0, 0); + }, + + renderResult(result, _opts, theme) { + const c = result.content?.[0]; + const text = c?.type === "text" ? c.text : ""; + if (text.length > 200) { + return new Text(theme.fg("dim", text.slice(0, 200) + "…"), 0, 0); + } + return new Text(theme.fg("dim", text), 0, 0); + }, + }); +} diff --git a/tests/integration/pdeathsig.py b/tests/integration/pdeathsig.py new file mode 100755 index 00000000..d4a25ac7 --- /dev/null +++ b/tests/integration/pdeathsig.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Exec a command with PR_SET_PDEATHSIG — receive a signal when parent dies. + +Usage: pdeathsig.py SIGNAL COMMAND [ARGS...] + +Fallback for systems without `setpriv --pdeathsig`. +""" +import ctypes +import ctypes.util +import os +import signal +import sys + +PR_SET_PDEATHSIG = 1 + +SIGNALS = { + "TERM": signal.SIGTERM, "SIGTERM": signal.SIGTERM, + "INT": signal.SIGINT, "SIGINT": signal.SIGINT, + "HUP": signal.SIGHUP, "SIGHUP": signal.SIGHUP, + "KILL": signal.SIGKILL, "SIGKILL": signal.SIGKILL, +} + +def main() -> int: + if len(sys.argv) < 3: + print(f"usage: {sys.argv[0]} SIGNAL COMMAND [ARGS...]", file=sys.stderr) + return 2 + sig_name = sys.argv[1] + argv = sys.argv[2:] + if sig_name not in SIGNALS: + print(f"unknown signal: {sig_name}", file=sys.stderr) + return 2 + sig = SIGNALS[sig_name] + libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6", use_errno=True) + if libc.prctl(PR_SET_PDEATHSIG, sig, 0, 0, 0) != 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + # Race fix: parent may have died before prctl took effect + if os.getppid() == 1: + os.kill(os.getpid(), sig) + os.execvp(argv[0], argv) + return 127 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/results/.gitignore b/tests/integration/results/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/tests/integration/results/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/integration/run_all.sh b/tests/integration/run_all.sh new file mode 100755 index 00000000..cd373af4 --- /dev/null +++ b/tests/integration/run_all.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# Run all herdr integration tests in parallel. +# +# Each spec gets its own tmux server (isolated socket). +# Workers are launched with PDEATHSIG so they get SIGTERM if this script dies. +# A stale sweep cleans up sockets from any previous crashed run. +# +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SPECS_DIR="$SCRIPT_DIR/specs" +RESULTS_DIR="$SCRIPT_DIR/results" +TMUX_ROOT="${TMPDIR:-/tmp}/herdr-test-sockets" + +mkdir -p "$RESULTS_DIR" "$TMUX_ROOT" + +# Clean previous results +rm -f "$RESULTS_DIR"/*.json "$RESULTS_DIR"/*.txt 2>/dev/null + +# Sweep stale tmux sockets from crashed previous runs +for dir in "$TMUX_ROOT"/*/; do + [ -d "$dir" ] || continue + owner_pid="" + [ -f "$dir/owner.pid" ] && owner_pid="$(<"$dir/owner.pid")" + # If owning worker is still alive, leave it + if [ -n "$owner_pid" ] && kill -0 "$owner_pid" 2>/dev/null; then + continue + fi + # Dead owner — kill the tmux server and remove the dir + [ -S "$dir/tmux.sock" ] && tmux -S "$dir/tmux.sock" kill-server 2>/dev/null || true + rm -rf "$dir" +done + +# Collect specs +SPECS=("$SPECS_DIR"/*.md) +if [ ${#SPECS[@]} -eq 0 ]; then + echo "No test specs found in $SPECS_DIR" + exit 0 +fi + +echo "Running ${#SPECS[@]} integration tests..." + +# Detect PDEATHSIG support +PDEATHSIG_CMD=() +if command -v setpriv >/dev/null 2>&1 && setpriv --help 2>&1 | grep -q -- '--pdeathsig'; then + PDEATHSIG_CMD=(setpriv --pdeathsig TERM --) +elif [ -f "$SCRIPT_DIR/pdeathsig.py" ]; then + PDEATHSIG_CMD=(python3 "$SCRIPT_DIR/pdeathsig.py" TERM) +fi + +# Launch all specs in parallel — directly from this shell (no subshell wrapper) +PIDS=() +NAMES=() + +for spec in "${SPECS[@]}"; do + name="$(basename "$spec" .md)" + NAMES+=("$name") + + "${PDEATHSIG_CMD[@]}" "$SCRIPT_DIR/run_spec.sh" "$spec" "$RESULTS_DIR" "$TMUX_ROOT" & + PIDS+=($!) +done + +# Wait for all, ignoring individual exit codes (we read results from files) +for pid in "${PIDS[@]}"; do + wait "$pid" 2>/dev/null || true +done + +echo "" + +# Print summary from result files +PASSED=0 +FAILED=0 +SKIPPED=0 + +for name in "${NAMES[@]}"; do + result_file="$RESULTS_DIR/$name.json" + + if [[ ! -f "$result_file" ]]; then + echo " ? $name (no result file)" + FAILED=$((FAILED + 1)) + continue + fi + + result=$(python3 -c "import sys,json; print(json.load(sys.stdin).get('result','unknown'))" < "$result_file" 2>/dev/null || echo "unknown") + + case "$result" in + pass) + echo " ✓ $name" + PASSED=$((PASSED + 1)) + ;; + skip) + echo " - $name (skipped)" + SKIPPED=$((SKIPPED + 1)) + ;; + *) + echo " ✗ $name" + python3 -c " +import sys, json +data = json.load(sys.stdin) +for c in data.get('checks', []): + if not c.get('pass', True): + print(f' ✗ {c[\"name\"]}: {c.get(\"detail\", \"\")}') +notes = data.get('notes', '') +if notes and data.get('result') == 'error': + print(f' {notes}') +" < "$result_file" 2>/dev/null + FAILED=$((FAILED + 1)) + ;; + esac +done + +echo "" +echo "$PASSED passed, $FAILED failed, $SKIPPED skipped" + +[ $FAILED -eq 0 ] diff --git a/tests/integration/run_spec.sh b/tests/integration/run_spec.sh new file mode 100755 index 00000000..43056f6e --- /dev/null +++ b/tests/integration/run_spec.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# +# Run a single herdr integration test spec via pi. +# +# Usage: ./run_spec.sh <spec.md> [results_dir] [tmux_socket_root] +# +# Each invocation creates its own tmux server (isolated socket). +# Cleanup kills the entire server — no orphaned sessions possible. +# +set -euo pipefail + +SPEC="$1" +SPEC_NAME="$(basename "$SPEC" .md)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +RESULTS_DIR="${2:-$SCRIPT_DIR/results}" +TMUX_ROOT="${3:-${TMPDIR:-/tmp}/herdr-test-sockets}" +SYSTEM_PROMPT_FILE="$SCRIPT_DIR/system.md" +HERDR="$PROJECT_DIR/target/release/herdr" +EXTENSION="$SCRIPT_DIR/ext/herdr-test.ts" +MODEL="${HERDR_TEST_MODEL:-minimax/MiniMax-M2.7}" + +mkdir -p "$RESULTS_DIR" "$TMUX_ROOT" + +# Per-spec tmux socket — this spec owns its entire tmux server +TMUX_DIR="$(mktemp -d "$TMUX_ROOT/${SPEC_NAME}.XXXXXX")" +TMUX_SOCKET="$TMUX_DIR/tmux.sock" +SESSION_NAME="test" + +# Track ownership for stale sweep +printf '%s\n' "$$" > "$TMUX_DIR/owner.pid" + +# Temp files (initialized for safe cleanup) +PROMPT_FILE="" +RAW_OUTPUT="" + +cleanup() { + # Kill our entire tmux server — takes everything with it + tmux -S "$TMUX_SOCKET" kill-server 2>/dev/null || true + [[ -n "$PROMPT_FILE" ]] && rm -f "$PROMPT_FILE" 2>/dev/null || true + rm -rf "$TMUX_DIR" 2>/dev/null || true + # RAW_OUTPUT is moved to results dir, not cleaned here +} +trap cleanup EXIT INT TERM HUP + +TMUX=(tmux -S "$TMUX_SOCKET") + +# Prerequisites +RESULT_FILE="$RESULTS_DIR/$SPEC_NAME.json" +if ! command -v tmux &>/dev/null; then echo '{"test":"'"$SPEC_NAME"'","result":"skip","checks":[],"notes":"tmux not found"}' > "$RESULT_FILE"; exit 0; fi +if ! command -v pi &>/dev/null; then echo '{"test":"'"$SPEC_NAME"'","result":"skip","checks":[],"notes":"pi not found"}' > "$RESULT_FILE"; exit 0; fi +if [[ ! -x "$HERDR" ]]; then echo '{"test":"'"$SPEC_NAME"'","result":"error","checks":[],"notes":"herdr binary not found"}' > "$RESULT_FILE"; exit 1; fi + +# Start herdr in its own tmux server +"${TMUX[@]}" new-session -d -s "$SESSION_NAME" -x 120 -y 50 +"${TMUX[@]}" send-keys -t "$SESSION_NAME" "$HERDR --no-session" Enter +HERDR_PANE=$("${TMUX[@]}" list-panes -t "$SESSION_NAME" -F '#{pane_id}' | head -1) +"${TMUX[@]}" set-option -p -t "$HERDR_PANE" @pi_name "herdr" + +sleep 1 + +# Build prompt +PROMPT_FILE=$(mktemp /tmp/herdr-test-prompt-XXXXX.md) +RAW_OUTPUT=$(mktemp /tmp/herdr-test-raw-XXXXX.txt) +cat "$SYSTEM_PROMPT_FILE" > "$PROMPT_FILE" +echo -e "\n---\n" >> "$PROMPT_FILE" +cat "$SPEC" >> "$PROMPT_FILE" + +# Run pi in the same tmux server +SESSION_DIR="$RESULTS_DIR/sessions" +mkdir -p "$SESSION_DIR" + +"${TMUX[@]}" split-window -t "$SESSION_NAME" -h \ + "cd $PROJECT_DIR && pi -p \ + --session-dir '$SESSION_DIR' \ + --no-extensions --no-skills --no-prompt-templates \ + -e '$EXTENSION' \ + --no-tools \ + --model '$MODEL' \ + --system-prompt '$PROMPT_FILE' \ + 'Execute the test now. Use the herdr tool to send keys and read the screen. Output the JSON result block when done.' \ + > '$RAW_OUTPUT' 2>&1; \ + echo '___DONE___' >> '$RAW_OUTPUT'; \ + sleep 1" + +# Wait for completion +TIMEOUT=180 +ELAPSED=0 +while [ $ELAPSED -lt $TIMEOUT ]; do + if grep -q '___DONE___' "$RAW_OUTPUT" 2>/dev/null; then + break + fi + sleep 2 + ELAPSED=$((ELAPSED + 2)) +done + +if [ $ELAPSED -ge $TIMEOUT ]; then + echo '{"test":"'"$SPEC_NAME"'","result":"error","checks":[],"notes":"pi timed out after '"$TIMEOUT"'s"}' > "$RESULT_FILE" + cp "$RAW_OUTPUT" "$RESULTS_DIR/$SPEC_NAME.raw.txt" 2>/dev/null || true + rm -f "$RAW_OUTPUT" + exit 1 +fi + +# Always save raw output +cp "$RAW_OUTPUT" "$RESULTS_DIR/$SPEC_NAME.raw.txt" 2>/dev/null || true +rm -f "$RAW_OUTPUT" + +# Extract JSON +JSON_RESULT=$(awk '/^```json$/,/^```$/' "$RESULTS_DIR/$SPEC_NAME.raw.txt" | grep -v '^```') + +if [[ -z "$JSON_RESULT" ]]; then + echo '{"test":"'"$SPEC_NAME"'","result":"error","checks":[],"notes":"no JSON in pi output"}' > "$RESULT_FILE" + exit 1 +fi + +if echo "$JSON_RESULT" | python3 -m json.tool > "$RESULT_FILE" 2>/dev/null; then + RESULT=$(python3 -c "import sys,json; print(json.load(sys.stdin).get('result','unknown'))" < "$RESULT_FILE") + if [[ "$RESULT" == "pass" ]]; then + exit 0 + else + exit 1 + fi +else + echo '{"test":"'"$SPEC_NAME"'","result":"error","checks":[],"notes":"invalid JSON from pi"}' > "$RESULT_FILE" + exit 1 +fi diff --git a/tests/integration/specs/create_workspace.md b/tests/integration/specs/create_workspace.md new file mode 100644 index 00000000..0f588c69 --- /dev/null +++ b/tests/integration/specs/create_workspace.md @@ -0,0 +1,12 @@ +# Test: Create workspace + +## Steps + +1. Read the screen to verify herdr started in empty state (should show "No active workspace" or "Press n to create one") +2. Press `n` to enter create mode +3. Type `test-workspace` +4. Press Enter to create +5. Read the screen and verify: + - The sidebar shows `1` followed by `test-workspace` + - A shell prompt is visible in the main pane area + - The active marker `▸` is next to the workspace diff --git a/tests/integration/specs/sidebar_collapse.md b/tests/integration/specs/sidebar_collapse.md new file mode 100644 index 00000000..b5d42c25 --- /dev/null +++ b/tests/integration/specs/sidebar_collapse.md @@ -0,0 +1,13 @@ +# Test: Sidebar collapse and expand + +## Steps + +1. Press `n` to create a workspace named `test` +2. Read the screen and verify the expanded sidebar shows the full workspace name `test` +3. Press prefix (ctrl+s), then `b` to collapse the sidebar +4. Read the screen and verify: + - The sidebar is now narrow (just numbers and state dots, no workspace name visible) + - The `»` expand icon is visible at the bottom of the sidebar + - The terminal pane takes up more horizontal space +5. Press prefix (ctrl+s), then `b` again to expand +6. Verify the full workspace name `test` is visible again in the sidebar diff --git a/tests/integration/specs/split_and_navigate.md b/tests/integration/specs/split_and_navigate.md new file mode 100644 index 00000000..cb75d72d --- /dev/null +++ b/tests/integration/specs/split_and_navigate.md @@ -0,0 +1,11 @@ +# Test: Split pane and navigate + +## Steps + +1. Press `n` to create a workspace named `dev` +2. Press prefix (ctrl+s), then `v` to split vertically +3. Read the screen and verify two panes are visible side by side, each with a shell prompt +4. Verify the sidebar shows a pane count indicator like `[2]` +5. Press prefix (ctrl+s), then `h` to navigate to the left pane +6. Press prefix (ctrl+s), then `l` to navigate back to the right pane +7. Verify the thick border (focused indicator) moved between panes diff --git a/tests/integration/specs/workspace_switching.md b/tests/integration/specs/workspace_switching.md new file mode 100644 index 00000000..0711c575 --- /dev/null +++ b/tests/integration/specs/workspace_switching.md @@ -0,0 +1,12 @@ +# Test: Workspace switching with prefix+N + +## Steps + +1. Press `n` to create a workspace named `first` +2. Press prefix (ctrl+s), then `n` to create a second workspace named `second` +3. Read the screen and verify the sidebar shows two workspaces numbered 1 and 2 +4. Verify `second` is the active workspace (has the `▸` marker) +5. Press prefix (ctrl+s), then `1` to switch to the first workspace +6. Read the screen and verify `first` is now active (has the `▸` marker) +7. Press prefix (ctrl+s), then `2` to switch back to `second` +8. Verify `second` is active again diff --git a/tests/integration/system.md b/tests/integration/system.md new file mode 100644 index 00000000..d6ccb05b --- /dev/null +++ b/tests/integration/system.md @@ -0,0 +1,48 @@ +# herdr Integration Test Agent + +You are a test executor for herdr, a terminal workspace manager. Your job is to execute a test specification by interacting with herdr through tmux, then report whether the test passed or failed. + +## Environment + +- herdr is running in tmux pane `herdr` with `--no-session` (clean state, no saved workspaces) +- You have the `tmux` tool to interact with it: `send` keys, `read` screen output +- herdr starts in Navigate mode with no workspaces (empty state) + +## herdr Controls + +- **Navigate mode**: press prefix key (ctrl+s) from Terminal mode, or start with no workspaces +- **Create workspace**: press `n`, type name, press Enter +- **Switch workspace**: press `1`-`9` (by number) or arrow keys + Enter +- **Split pane**: press `v` (vertical split) or `-` (horizontal split) +- **Close pane**: press `x` +- **Close workspace**: press `d` (then `y` to confirm) +- **Collapse sidebar**: press `b` +- **Resize mode**: press `r`, then h/j/k/l to resize, Esc to exit +- **Back to terminal**: press Esc or prefix key again +- **Quit**: press `q` in Navigate mode + +After creating a workspace or pressing Enter on one, you enter Terminal mode. Press ctrl+s (prefix) to go back to Navigate mode. + +## How to Execute + +1. Read the test specification carefully +2. Execute each step using tmux send/read +3. After each action, wait briefly then read the screen to verify +4. When done, output your result as a JSON object + +## Output Format + +Your final message MUST be a JSON code block with this structure: + +```json +{ + "test": "<test name>", + "result": "pass" | "fail", + "checks": [ + {"name": "<what was checked>", "pass": true|false, "detail": "<what you saw>"} + ], + "notes": "<any additional observations>" +} +``` + +Be precise. Read the screen carefully. Don't assume — verify by reading. |
