aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-11-01 10:22:01 +0200
committerIgor Soarez <igor@soarez.org>2026-11-01 10:22:01 +0200
commit8146d1db419e8bdacd75920da6f3761abf724853 (patch)
tree921a1d609d96bb2129f7ac2d80635fd3c0c41691 /tests
parent959f0d46cac1a09d0821da599d8dff54d1147892 (diff)
show what pi agents cost today, this week and this month, counting forked replies once
Diffstat (limited to 'tests')
-rwxr-xr-xtests/e2e.sh49
-rwxr-xr-xtests/pi-live.sh23
2 files changed, 69 insertions, 3 deletions
diff --git a/tests/e2e.sh b/tests/e2e.sh
index e73dc73..e5b5865 100755
--- a/tests/e2e.sh
+++ b/tests/e2e.sh
@@ -32,6 +32,8 @@ PASSES=0
export DINGGER_TMUX_SOCKET="$SOCK" DINGGER_AGENT_SOCKET="$HOOK" DINGGER_STATE_DIR="$DIR/state"
# The daemon updates an installed pi extension when it starts: not the user's.
export PI_CODING_AGENT_DIR="$DIR/pi-agent"
+# What pi agents cost: the daemons read session files here every 0.3 s.
+export DINGGER_COST_SCAN_SECS=0.3
t() { tmux -u -L "$SOCK" "$@"; }
@@ -260,14 +262,14 @@ if [ -x "$MENUBAR" ]; then
expect "A1 done" "opt $A1 @agent_state" done
expect "menu title" "mb --print | head -1" "title: ×1 ✓1"
expect "menu rows: herdr priority; idle most recent first (C1 since its jump in step 4, B1 since the start)" \
- "mb --print | tail -n +2 | grep -v '^⟳' | cut -f1,2 | tr '\t\n' '|;'" "× blocked|pi;✓ done|Deploy the widget;○ idle|pi;○ idle|Review PR 42;"
+ "mb --print | tail -n +2 | grep -v '^⟳\|^\$ ' | cut -f1,2 | tr '\t\n' '|;'" "× blocked|pi;✓ done|Deploy the widget;○ idle|pi;○ idle|Review PR 42;"
# Wait for working: B1 was idle before, so idle alone would not show the turn.
hook "$B1" working
expect "B1 working" "opt $B1 @agent_state" working
hook "$B1" idle
expect "B1 finishes a turn while watched: idle" "opt $B1 @agent_state" idle
expect "and is now the most recent idle agent: listed first" \
- "mb --print | tail -n +2 | grep -v '^⟳' | cut -f1,2 | tr '\t\n' '|;'" "× blocked|pi;✓ done|Deploy the widget;○ idle|Review PR 42;○ idle|pi;"
+ "mb --print | tail -n +2 | grep -v '^⟳\|^\$ ' | cut -f1,2 | tr '\t\n' '|;'" "× blocked|pi;✓ done|Deploy the widget;○ idle|Review PR 42;○ idle|pi;"
expect "subtitle: location, harness, waiting time" "mb --print | grep 'Deploy the widget' | cut -f3" "a · wa1 · pi · done now"
expect "idle subtitle has no waiting time" "mb --print | grep 'Review PR 42' | cut -f3" "b · wb1 · pi"
mb --click "$A1"
@@ -516,6 +518,7 @@ in_snapshot() { python3 -c 'import sys; print(sys.argv[2] in open(sys.argv[1]).r
expect "snapshot: the agent's session" "in_snapshot '\"session\": \"$W/sessions/a b.jsonl\"'" True 80
expect "snapshot: a running command's exact arguments" "in_snapshot \"'sleep 100000; :' 'two words'\"" True 80
expect "snapshot: the zsh hook's text" "in_snapshot '\"command\": \"slp 200000 | cat\"'" True 80
+expect "snapshot: the other agent's session" "in_snapshot gone.jsonl" True 80
# An agent that exits just before tmux dies (a shutdown stops programs first).
kill "$(ps -o tpgid= -p "$(r display -p -t "$GONE" '#{pane_pid}')" | tr -d ' ')"
expect "the exited agent is noticed" "r show -pv -t $GONE @agent 2>/dev/null" ""
@@ -1099,6 +1102,48 @@ if command -v node >/dev/null; then
fi
t kill-session -t dr
+# --- 25. what agents cost -----------------------------------------------------
+# The daemon reads pi's session files: each reply once, for the session it
+# first appeared in (a fork copies its parent's), closed agents included.
+echo "25. what agents cost"
+SESS="$PI_CODING_AGENT_DIR/sessions/--work--"
+mkdir -p "$SESS"
+reply() { # id cost [days ago]
+ python3 -c 'import json,sys,time; print(json.dumps({"type":"message","id":sys.argv[1],"message":{"role":"assistant","timestamp":int((time.time()-86400*float(sys.argv[3]))*1000),"usage":{"cost":{"total":float(sys.argv[2])}}}}))' "$1" "$2" "${3:-0}"
+}
+header() { echo "{\"type\":\"session\",\"timestamp\":\"$1\"}"; }
+{ header 2026-01-01T00:00:00Z; reply p0 100 40; reply p1 1.25; reply p2 0.25; } >"$SESS/parent.jsonl"
+# The fork: its parent's entries, then its own.
+{ header 2026-01-02T00:00:00Z; tail -n +2 "$SESS/parent.jsonl"; reply f1 0.5; } >"$SESS/fork.jsonl"
+expect "today, each reply once (not 40 days ago)" "gopt @dingger_cost_today" 2.00 50
+expect "this week" "gopt @dingger_cost_week" 2.00
+expect "this month" "gopt @dingger_cost_month" 2.00
+session_report() { # pane session-file
+ python3 - "$HOOK" "$1" "$2" <<'PY'
+import json, socket, sys
+params = {"pane_id": sys.argv[2], "agent": "pi", "state": "idle", "agent_session_path": sys.argv[3]}
+s = socket.socket(socket.AF_UNIX)
+s.connect(sys.argv[1])
+s.sendall((json.dumps({"method": "pane.report_agent", "params": params}) + "\n").encode())
+s.close()
+PY
+}
+t new-session -d -s cost -n c1 -c "$DIR" "$DIR/bin/pi"
+t new-window -d -t cost: -n c2 -c "$DIR" "$DIR/bin/pi"
+C1=$(t display -p -t cost:c1 '#{pane_id}'); C2=$(t display -p -t cost:c2 '#{pane_id}')
+session_report "$C1" "$SESS/parent.jsonl"; session_report "$C2" "$SESS/fork.jsonl"
+expect "an agent: what its session spent today" "opt $C1 @dingger_cost" 1.50
+expect "a fork: only its own replies" "opt $C2 @dingger_cost" 0.50
+expect "the menu's top line" "mb --print | grep '^\\$ '" '$ $2.00 today · $2.00 this week · $2.00 this month'
+expect "an agent's line" "mb --print | grep -F 'cost · c1' | cut -f3" "cost · c1 · pi · \$1.50"
+expect "dingger status: per agent" "$DINGGER status | grep -F 'cost · c2' | grep -c ' \$0.50 '" 1
+expect "and in total" "$DINGGER status | tail -1" 'spent: $2.00 today · $2.00 this week · $2.00 this month (UTC)'
+reply f2 0.75 >>"$SESS/fork.jsonl"
+expect "a new reply: counted" "opt $C2 @dingger_cost" 1.25 50
+expect "in the total too" "gopt @dingger_cost_today" 2.75
+t kill-session -t cost
+expect "closed agents still count" "gopt @dingger_cost_today" 2.75 10
+
# Every daemon this run started must stop with it.
kill "$DAEMON_PID" "$DAEMON2_PID" 2>/dev/null
expect "the test daemons stop" "ps -o pid= -p $DAEMON_PID,$DAEMON2_PID | wc -l | tr -d ' '" 0
diff --git a/tests/pi-live.sh b/tests/pi-live.sh
index 6dff42f..95c5885 100755
--- a/tests/pi-live.sh
+++ b/tests/pi-live.sh
@@ -50,7 +50,7 @@ PORT=$(cat "$D/port")
for agent in agent agent-plain; do
mkdir -p "$D/$agent/extensions"
cat >"$D/$agent/models.json" <<EOF
-{"providers": {"fake": {"baseUrl": "http://127.0.0.1:$PORT/v1", "api": "openai-completions", "apiKey": "fake", "models": [{"id": "fake-model"}]}}}
+{"providers": {"fake": {"baseUrl": "http://127.0.0.1:$PORT/v1", "api": "openai-completions", "apiKey": "fake", "models": [{"id": "fake-model", "cost": {"input": 100000, "output": 1000000, "cacheRead": 0, "cacheWrite": 0}}]}}}
EOF
echo '{"defaultProvider": "fake", "defaultModel": "fake-model"}' >"$D/$agent/settings.json"
done
@@ -70,6 +70,10 @@ printf 'set -g default-shell /bin/sh\nset -g default-command /bin/sh\n' >"$D/tmu
# Every pi here uses the test's agent directory, including one a restore
# starts (`pi --session`, without the command line's own variables).
export PATH="$D/bin:$PATH" PI_CODING_AGENT_DIR="$D/agent" PI_OFFLINE=1 PI_SKIP_VERSION_CHECK=1 PI_TELEMETRY=0
+# The fake model's prices make each reply cost $4.00 (10 input tokens at
+# $100,000 per million, 3 output at $1,000,000); the daemon reads costs
+# from session files every 0.3 s.
+export DINGGER_COST_SCAN_SECS=0.3
export DINGGER_TMUX_SOCKET="$R" DINGGER_AGENT_SOCKET="$D/hook.sock" DINGGER_STATE_DIR="$D/state" DINGGER_TMUX_CONFIG="$D/tmux.conf"
t -f "$D/tmux.conf" new-session -d -s main -n agent -x 140 -y 40 -c "$D/work"
t new-window -d -t main: -n plain -c "$D/work2"
@@ -196,6 +200,23 @@ expect "it still works: working" "st agent" working 50
expect "and done" "st agent" done 60
expect "a message sent is no draft" "draft" "" 30
expect "nor kept" "[ -e \"\$(draft_file)\" ] && echo left || echo gone" gone 30
+# What it cost: pi prices each reply; dingger counts them.
+replies() { grep -c '"role":"assistant"' "$1"; }
+SESSION_FILE=$(t show -pv -t main:agent @agent_session)
+N=$(replies "$SESSION_FILE")
+[ "$N" -ge 3 ] && pass "$N replies so far" || fail "only $N replies"
+WANT=$(python3 -c "print(f'{4*$N:.2f}')" )
+expect "the agent's spend today" "t show -pv -t main:agent @dingger_cost 2>/dev/null" "$WANT" 30
+expect "in total" "t show -gv @dingger_cost_today 2>/dev/null" "$WANT" 30
+expect "and this week and month" "t show -gv @dingger_cost_week; t show -gv @dingger_cost_month" "$WANT
+$WANT"
+# A fork copies the session's replies into a new file, then adds its own.
+(cd "$D/work" && pi --fork "$SESSION_FILE" -p "slow 1 forked" $NO_MCP >"$D/fork.out" 2>&1)
+FORK=$(ls -t "$(dirname "$SESSION_FILE")"/*.jsonl | head -1)
+[ "$FORK" != "$SESSION_FILE" ] && [ "$(replies "$FORK")" -eq $((N + 1)) ] \
+ && pass "pi --fork: a new file with the $N replies copied and its own" || fail "fork: $FORK ($(replies "$FORK") replies)"
+expect "the fork adds only its own reply" "t show -gv @dingger_cost_today 2>/dev/null" "$(python3 -c "print(f'{4*($N+1):.2f}')")" 30
+expect "the agent's own spend is unchanged" "t show -pv -t main:agent @dingger_cost 2>/dev/null" "$WANT"
echo "2. screen detection (pi without the extension)"
t send-keys -t main:plain "PI_CODING_AGENT_DIR=$D/agent-plain pi $NO_MCP" Enter