path: root/tests/pi-live.sh

#!/bin/bash
# Live check of dingger against a real pi binary, on a tmux server and
# dingger daemon of its own, with a fake local model (tests/fake-model.py):
# no network, no credentials, nothing of the user's pi touched (pi gets its
# own agent directory).
#
#   tests/pi-live.sh <pi binary> [dingger binary]
#
# Covers: identification, the extension's state and session reports, a turn
# (working, then done), /reload, an extension's blocked event, /importance, /unread,
# screen detection for a pi without the extension, restore (`pi --session`,
# importance included), reloading an agent on an old extension (refused
# while busy or with a draft), exit.
set -u
PI_BIN=$(cd "$(dirname "$1")" && pwd)/$(basename "$1")
ROOT=$(cd "$(dirname "$0")/.." && pwd)
DINGGER="${2:-$ROOT/target/release/dingger}"
case "$DINGGER" in /*) ;; *) DINGGER="$PWD/$DINGGER" ;; esac
MENU="$ROOT/target/dingger-menu"
D=$(mktemp -d /tmp/dingger-pi-live.XXXXXX)
R="dingger-pi-live-$$"
PASSES=0; FAILS=0
pass() { PASSES=$((PASSES + 1)); echo "  ok   $1"; }
fail() { FAILS=$((FAILS + 1)); echo "  FAIL $1"; }
expect() { # description command expected [tries of 0.1s, default 50]
  local desc=$1 cmd=$2 want=$3 tries=${4:-50} got
  for _ in $(seq "$tries"); do
    got=$(eval "$cmd")
    [ "$got" = "$want" ] && { pass "$desc"; return; }
    sleep 0.1
  done
  fail "$desc: want [$want] got [$got]"
}
note() { echo "  note $1"; }
t() { tmux -u -L "$R" "$@"; }
cleanup() {
  t kill-server 2>/dev/null
  [ -n "${DAEMON_PID:-}" ] && kill "$DAEMON_PID" 2>/dev/null
  [ -n "${FAKE_PID:-}" ] && kill "$FAKE_PID" 2>/dev/null
  if [ -n "${PI_LIVE_KEEP:-}" ]; then echo "kept $D"; else rm -rf "$D"; fi
}
trap cleanup EXIT

echo "pi $("$PI_BIN" --version 2>&1 | head -1), dingger $DINGGER"
python3 "$ROOT/tests/fake-model.py" >"$D/port" 2>"$D/fake.err" &
FAKE_PID=$!
for _ in $(seq 50); do [ -s "$D/port" ] && break; sleep 0.1; done
PORT=$(cat "$D/port")
# Two agent directories: one with dingger's extension (and a test extension
# that blocks the way extensions do), one without (screen detection).
for agent in agent agent-plain; do
  mkdir -p "$D/$agent/extensions"
  cat >"$D/$agent/models.json" <<EOF
{"providers": {"fake": {"baseUrl": "http://127.0.0.1:$PORT/v1", "api": "openai-completions", "apiKey": "fake", "models": [{"id": "fake-model", "contextWindow": 400, "cost": {"input": 100000, "output": 1000000, "cacheRead": 0, "cacheWrite": 0}}]}}}
EOF
  # A small window, so a few replies fill it measurably; nothing compacts
  # on its own, and /compact keeps almost nothing.
  echo '{"defaultProvider": "fake", "defaultModel": "fake-model", "compaction": {"enabled": false, "keepRecentTokens": 1}}' >"$D/$agent/settings.json"
done
PI_CODING_AGENT_DIR="$D/agent" "$DINGGER" install pi >/dev/null
cat >"$D/agent/extensions/test-blocked.ts" <<'EOF'
export default function (pi) {
  pi.registerCommand("block", { description: "test: emit blocked", handler: async () => { pi.events.emit("blocked", { active: true, label: "test block" }); } });
  pi.registerCommand("unblock", { description: "test: clear blocked", handler: async () => { pi.events.emit("blocked", { active: false }); } });
  pi.registerCommand("ask", { description: "test: a confirm dialog", handler: async (_args, ctx) => { await ctx.ui.confirm("Test dialog", "Proceed?"); } });
}
EOF
mkdir -p "$D/bin" "$D/work" "$D/work2"
ln -s "$PI_BIN" "$D/bin/pi"
# Plain shells: a login shell would read the user's profile, which puts the
# user's own pi first on PATH.
printf 'set -g default-shell /bin/sh\nset -g default-command /bin/sh\n' >"$D/tmux.conf"
# Every pi here uses the test's agent directory, including one a restore
# starts (`pi --session`, without the command line's own variables).
export PATH="$D/bin:$PATH" PI_CODING_AGENT_DIR="$D/agent" PI_OFFLINE=1 PI_SKIP_VERSION_CHECK=1 PI_TELEMETRY=0
# The fake model's prices make each reply cost $4.00 (10 input tokens at
# $100,000 per million, 3 output at $1,000,000); the daemon reads costs
# from session files every 0.3 s.
export DINGGER_COST_SCAN_SECS=0.3
export DINGGER_TMUX_SOCKET="$R" DINGGER_AGENT_SOCKET="$D/hook.sock" DINGGER_STATE_DIR="$D/state" DINGGER_TMUX_CONFIG="$D/tmux.conf"
t -f "$D/tmux.conf" new-session -d -s main -n agent -x 140 -y 40 -c "$D/work"
t new-window -d -t main: -n plain -c "$D/work2"
EXT="$D/agent/extensions/dingger-agent-state.ts"
stamp() { grep -o 'extensionVersion = "[0-9a-f]*"' "$EXT" | cut -d'"' -f2; }
CURRENT=$(stamp)
# An installed extension of another version: the daemon updates it as it
# starts (a pi started after that loads the current one).
sed -i '' "s/extensionVersion = \"$CURRENT\"/extensionVersion = \"0000000000000000\"/" "$EXT"
"$DINGGER" watch --interval-secs 1 --no-sounds --no-notifications >"$D/daemon.log" 2>&1 &
DAEMON_PID=$!
t send-keys -t main:agent "command -v pi > $D/which-pi" Enter
for _ in $(seq 30); do [ -s "$D/which-pi" ] && break; sleep 0.1; done
if [ "$(cat "$D/which-pi")" != "$D/bin/pi" ]; then
  echo "the test panes would run $(cat "$D/which-pi"), not $PI_BIN; stopping"; exit 1
fi
# --no-mcp (pi 1.0.4+): no MCP servers, whatever the user configured.
NO_MCP=$("$PI_BIN" --help 2>&1 | grep -q -- --no-mcp && echo --no-mcp)
st() { t show -pv -t "main:$1" @agent_state 2>/dev/null; }
type_line() { t send-keys -t "main:$1" -l "$2"; sleep 0.3; t send-keys -t "main:$1" Enter; }

echo "0. the daemon updates the installed extension"
expect "to its own version" "stamp" "$CURRENT" 30
[ ${#CURRENT} -eq 16 ] && pass "a 16-digit version" || fail "version [$CURRENT]"

echo "1. the extension's reports"
t send-keys -t main:agent "clear; PI_CODING_AGENT_DIR=$D/agent pi $NO_MCP" Enter
expect "identified as pi" "t show -pv -t main:agent @agent 2>/dev/null" pi 150
expect "idle at start" "st agent" idle 100
expect "the extension reported (state from the hook)" "t show -pv -t main:agent @dingger_hook 2>/dev/null" idle 100
expect "its session file is on the pane" "t show -pv -t main:agent @agent_session 2>/dev/null | grep -c '^$D/agent/sessions/.*\.jsonl$'" 1 100
type_line agent "slow 3 hello"
expect "a turn: working" "st agent" working 50
expect "then done (nobody watching)" "st agent" done 80
expect "the session file is written" "[ -s \"\$(t show -pv -t main:agent @agent_session)\" ] && echo yes" yes
type_line agent "/reload"
sleep 2
type_line agent "slow 2 again"
expect "after /reload: working" "st agent" working 50
expect "and done" "st agent" done 60
type_line agent "/block"
expect "an extension's blocked event: blocked" "st agent" blocked 50
type_line agent "/unblock"
expect "cleared: back to done" "st agent" done 50
type_line agent "/ask"
sleep 1.5
note "during a pi confirm dialog dingger shows: $(st agent) (pi gives extensions no dialog event)"
t send-keys -t main:agent Escape
sleep 0.5
type_line agent "/importance high"
expect "/importance high: the pane option" "t show -pv -t main:agent @dingger_importance 2>/dev/null" high 50
expect "pi says so" "t capture-pane -p -t main:agent | grep -c 'importance: high'" 1 30
expect "and adds no footer line" "t capture-pane -p -t main:agent | grep -v '^ *$' | tail -1 | grep -c 'importance'" 0
expect "saved in the session file" "grep -c '\"customType\":\"dingger-importance\"' \"\$(t show -pv -t main:agent @agent_session)\"" 1 30
# Seen (what a jump does; no client is attached here), then marked unread.
t set -p -t main:agent @dingger_ack 1; "$DINGGER" poke
expect "seen: idle" "st agent" idle 50
type_line agent "/unread"
expect "/unread: done again" "st agent" done 50
# Not a turn (an unknown command would be sent to the model, and also end
# done): the daemon applied a mark, and pi said so.
expect "by a mark" "sed 's/\x1b\[[0-9;]*m//g' $D/daemon.log | grep -c 'mark unread target=main:agent applied=true'" 1 30
expect "pi says so" "t capture-pane -p -t main:agent | grep -c 'marked unread: dingger shows it as done'" 1 30

echo "1b. reloading an agent on an old extension"
ctl() { t show -pv -t main:agent @dingger_control 2>/dev/null; }
ask_reload() { python3 -c '
import socket, sys
s = socket.socket(socket.AF_UNIX); s.settimeout(3); s.connect(sys.argv[1])
s.sendall(b"{\"method\":\"reload\"}\n"); print(s.makefile().readline().strip())' "$(ctl)"; }
reloads() { sed 's/\x1b\[[0-9;]*m//g' $D/daemon.log | grep -c "reload: $1"; }
expect "the agent reports the current version" "t show -pv -t main:agent @dingger_extension 2>/dev/null" "$CURRENT"
expect "and a control socket" "[ -S \"\$(ctl)\" ] && echo yes" yes
expect "nothing to reload" "\"$DINGGER\" reload-agents" "every pi agent runs the current dingger extension" 1
# Run an old one: change the installed file, /reload by hand, then install
# the current one again (what a dingger update does).
sed -i '' "s/extensionVersion = \"$CURRENT\"/extensionVersion = \"0000000000000000\"/" "$EXT"
type_line agent "/reload"
expect "after /reload, it reports the old version" "t show -pv -t main:agent @dingger_extension 2>/dev/null" 0000000000000000 80
expect "its new instance has a socket of its own" "[ -S \"\$(ctl)\" ] && echo yes" yes
PI_CODING_AGENT_DIR="$D/agent" "$DINGGER" install pi >/dev/null
expect "the daemon counts it outdated" "t show -gv @dingger_outdated 2>/dev/null" 1 30
SOCK1=$(ctl)
# Reloading cannot help while the installed file is not the current one.
cp "$EXT" "$D/current.ts"; echo "// edited" >>"$EXT"
expect "reload-agents refuses: install first" "\"$DINGGER\" reload-agents 2>&1 | grep -c 'run .dingger install pi. first'" 1 1
python3 -c 'import socket,sys; s=socket.socket(socket.AF_UNIX); s.connect(sys.argv[1]); s.sendall(b"{\"method\":\"agents.reload_outdated\"}\n"); s.recv(64)' "$D/hook.sock"
expect "and so does the daemon, asked directly" "sed 's/\x1b\[[0-9;]*m//g' $D/daemon.log | grep -c 'reload agents: the installed pi extension is not'" 1 30
sleep 1.5
expect "without reloading it" "reloads reloading" 0 1
cp "$D/current.ts" "$EXT"
# The extension's own checks, asked directly.
type_line agent "slow 4 busy turn"
expect "working" "st agent" working 50
expect "busy: it refuses" "ask_reload" '{"ok":false,"reason":"busy"}'
expect "and the turn ends" "st agent" done 80
t send-keys -t main:agent -l "half typed"
sleep 0.5
expect "a draft: it refuses" "ask_reload" '{"ok":false,"reason":"draft"}'
draft() { t show -pv -t main:agent @dingger_draft 2>/dev/null; }
draft_file() { echo "$D/drafts/$(basename "$(t show -pv -t main:agent @agent_session)").draft"; }
expect "the draft reaches dingger" "draft" "half typed" 30
expect "and is kept for the session" "cat \"\$(draft_file)\"" "half typed" 30
# Through dingger: refused for the draft, retried until it can.
"$DINGGER" reload-agents >"$D/reload.out"
expect "reload-agents: not yet" "head -1 $D/reload.out" "reloaded 0 of 1 agent on an old dingger extension"
expect "it says why it waits" "sed -n 2p $D/reload.out" "the other 1 reload once idle and off screen, with an empty input box"
expect "the daemon was told no: a draft" "sed 's/\x1b\[[0-9;]*m//g' $D/daemon.log | grep -c 'reload: not now target=main:agent reason=draft'" 1 30
expect "the draft is untouched" "t capture-pane -p -t main:agent | grep -c 'half typed'" 1
for _ in $(seq 10); do t send-keys -t main:agent BSpace; done
expect "deleted: no draft" "draft" "" 30
expect "nor its file" "[ -e \"\$(draft_file)\" ] && echo left || echo gone" gone 30
expect "once the input box is empty: reloaded" "t show -pv -t main:agent @dingger_extension 2>/dev/null" "$CURRENT" 150
expect "by the daemon" "reloads reloading" 1 30
expect "a new instance (a new socket)" "[ \"\$(ctl)\" != \"$SOCK1\" ] && [ -S \"\$(ctl)\" ] && echo yes" yes
expect "the old socket is gone" "[ -e \"$SOCK1\" ] && echo left || echo gone" gone
expect "nothing left to reload" "t show -gv @dingger_outdated 2>/dev/null" 0 30
expect "still done (unseen)" "st agent" done
expect "no message reached the model" "grep -c dingger-reload \"\$(t show -pv -t main:agent @agent_session)\"" 0
t send-keys -t main:agent -l "slow 1 after"
expect "typed, then a pause: a draft" "draft" "slow 1 after" 30
t send-keys -t main:agent Enter
expect "it still works: working" "st agent" working 50
expect "and done" "st agent" done 60
expect "a message sent is no draft" "draft" "" 30
expect "nor kept" "[ -e \"\$(draft_file)\" ] && echo left || echo gone" gone 30
# What it cost: pi prices each reply; dingger counts them.
replies() { grep -c '"role":"assistant"' "$1"; }
SESSION_FILE=$(t show -pv -t main:agent @agent_session)
N=$(replies "$SESSION_FILE")
[ "$N" -ge 3 ] && pass "$N replies so far" || fail "only $N replies"
WANT=$(python3 -c "print(f'{4*$N:.2f}')" )
expect "the agent's spend today" "t show -pv -t main:agent @dingger_cost 2>/dev/null" "$WANT" 30
expect "in total" "t show -gv @dingger_cost_today 2>/dev/null" "$WANT" 30
expect "and this week and month" "t show -gv @dingger_cost_week; t show -gv @dingger_cost_month" "$WANT
$WANT"
# A fork copies the session's replies into a new file, then adds its own.
(cd "$D/work" && pi --fork "$SESSION_FILE" -p "slow 1 forked" $NO_MCP >"$D/fork.out" 2>&1)
FORK=$(ls -t "$(dirname "$SESSION_FILE")"/*.jsonl | head -1)
[ "$FORK" != "$SESSION_FILE" ] && [ "$(replies "$FORK")" -eq $((N + 1)) ] \
  && pass "pi --fork: a new file with the $N replies copied and its own" || fail "fork: $FORK ($(replies "$FORK") replies)"
expect "the fork adds only its own reply" "t show -gv @dingger_cost_today 2>/dev/null" "$(python3 -c "print(f'{4*($N+1):.2f}')")" 30
expect "the agent's own spend is unchanged" "t show -pv -t main:agent @dingger_cost 2>/dev/null" "$WANT"
# How full its context is: the number pi's footer shows ("3.3%/400"), in
# an agent of its own (compacting would hide this one's conversation from
# the restore below).
mkdir -p "$D/work3"
t new-window -d -t main: -n ctx -c "$D/work3"
t send-keys -t main:ctx "pi $NO_MCP" Enter
expect "context: an agent of its own" "st ctx" idle 150
footer() { t capture-pane -p -t main:ctx | grep -oE '(\?|[0-9.]+%)/400' | tail -1 | sed 's|/400||'; }
as_footer() { python3 -c "import sys; f=sys.argv[1]; print('' if f in ('?', '') else round(float(f.rstrip('%'))))" "$(footer)"; }
ctx() { t show -pv -t main:ctx @dingger_context 2>/dev/null; }
same_as_footer() { [ "$(ctx)" = "$(as_footer)" ] && echo same || echo "dingger [$(ctx)] footer [$(footer)]"; }
expect "context: none yet" "footer; ctx" "0.0%
0" 30
for msg in "slow 1 one" "slow 1 two"; do
  t send-keys -t main:ctx -l "$msg" && t send-keys -t main:ctx Enter
  expect "and done" "st ctx" done 60
done
expect "context: after replies, pi's footer" "footer" "3.3%" 30
expect "context: dingger has pi's number" same_as_footer same 30
t send-keys -t main:ctx -l "/compact" && t send-keys -t main:ctx Enter
expect "context: after compacting, pi does not know" "footer" "?" 60
expect "context: nor does dingger" same_as_footer same 30
expect "the menu shows no percentage" "\"$MENU\" --print | grep -F 'main · ctx' | grep -c '%'" 0
t send-keys -t main:ctx -l "slow 1 after compacting" && t send-keys -t main:ctx Enter
expect "and done" "st ctx" done 60
expect "context: known again after a reply" "ctx | grep -cE '^[0-9]+$'" 1 30
expect "context: pi's number again" same_as_footer same 30
expect "the menu: the percentage last" "\"$MENU\" --print | grep -F 'main · ctx' | cut -f3 | grep -oE ' [0-9]+%$'" " $(ctx)%"
expect "the panel: cost, then context" "\"$MENU\" --print-panel --full | grep -F 'main · ctx' | cut -f4 | grep -oE '^\\\$[0-9.]+ [0-9]+%$' | grep -c ' $(ctx)%'" 1
# Its cost, compaction included, as pi's footer counts it ("$16.000").
footer_cost() { t capture-pane -p -t main:ctx | grep -E '/400' | grep -oE '\$[0-9.]+' | tail -1 | tr -d '$'; }
same_cost() { python3 -c "import sys; a,b=sys.argv[1:]; print('same' if a and b and abs(float(a)-float(b))<0.005 else f'dingger [{a}] footer [{b}]')" "$(t show -pv -t main:ctx @dingger_cost 2>/dev/null)" "$(footer_cost)"; }
expect "cost: compaction counted, as pi's footer counts" same_cost same 30
t kill-window -t main:ctx

echo "2. screen detection (pi without the extension)"
t send-keys -t main:plain "PI_CODING_AGENT_DIR=$D/agent-plain pi $NO_MCP" Enter
expect "identified as pi" "t show -pv -t main:plain @agent 2>/dev/null" pi 150
expect "idle at start" "st plain" idle 100
expect "no hook state" "t show -pv -t main:plain @dingger_hook 2>/dev/null" ""
type_line plain "slow 4 hello"
expect "a turn: working, from the screen" "st plain" working 50
expect "then done" "st plain" done 80

echo "3. restore"
SESSION=$(t show -pv -t main:agent @agent_session)
latest() { ls "$D"/state/snapshots/*.json 2>/dev/null | tail -1; }
expect "the snapshot has the agent's session" "grep -c \"\\\"session\\\": \\\"$SESSION\\\"\" \"\$(latest)\" 2>/dev/null" 1 80
# A draft left behind when tmux goes away.
# (`\;`: send-keys would end its command at a trailing ;)
t send-keys -t main:agent -l 'an unsent idea\;'
expect "a draft before the crash" "t show -pv -t main:agent @dingger_draft 2>/dev/null" "an unsent idea;" 30
t kill-server
expect "the layout becomes the restore candidate" "python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[\"status\"])' $D/state/restore.json 2>/dev/null" offered 80
"$DINGGER" restore >"$D/restore.out" 2>&1
expect "restore resumes it by its session file" "grep -c -- \"resume .*pi --session\" $D/restore.out" 1
expect "resumed 1 of 1 agents" "grep -c '^resumed 1 of 1 agents' $D/restore.out" 1
expect "it runs again, reporting the same session" "t show -pv -t main:agent @agent_session 2>/dev/null" "$SESSION" 150
expect "with its conversation" "t capture-pane -p -t main:agent -S - | grep -c 'slow 3 hello'" 1 50
expect "and its importance" "t show -pv -t main:agent @dingger_importance 2>/dev/null" high 50
expect "and its draft, back in the input box" "t capture-pane -p -t main:agent | grep -c '^an unsent idea;'" 1 50
expect "reported again" "t show -pv -t main:agent @dingger_draft 2>/dev/null" "an unsent idea;" 30

echo "4. exit"
LAST_DRAFT=$(draft_file)
expect "its draft is kept" "cat $LAST_DRAFT" "an unsent idea;"
# Deleted, then quit at once: before the input box is read again.
for _ in $(seq 20); do t send-keys -t main:agent BSpace; done
t send-keys -t main:agent C-d
expect "the agent is gone from the pane" "t show -pv -t main:agent @agent 2>/dev/null" "" 100
expect "a draft deleted before quitting is not kept" "[ -e $LAST_DRAFT ] && echo left || echo gone" gone 30

echo
echo "$PASSES passed, $FAILS failed"
[ "$FAILS" -eq 0 ] || { echo "--- restore output:"; cat "$D/restore.out" 2>/dev/null; echo "--- daemon log tail:"; tail -15 "$D/daemon.log" | sed 's/\x1b\[[0-9;]*m//g' | cut -c29-160; exit 1; }