diff options
| author | Igor Soarez <igor@soarez.org> | 2026-11-01 10:22:01 +0200 |
|---|---|---|
| committer | Igor Soarez <igor@soarez.org> | 2026-11-01 10:22:01 +0200 |
| commit | 8146d1db419e8bdacd75920da6f3761abf724853 (patch) | |
| tree | 921a1d609d96bb2129f7ac2d80635fd3c0c41691 /tests | |
| parent | 959f0d46cac1a09d0821da599d8dff54d1147892 (diff) | |
show what pi agents cost today, this week and this month, counting forked replies once
Diffstat (limited to 'tests')
| -rwxr-xr-x | tests/e2e.sh | 49 | ||||
| -rwxr-xr-x | tests/pi-live.sh | 23 |
2 files changed, 69 insertions, 3 deletions
diff --git a/tests/e2e.sh b/tests/e2e.sh index e73dc73..e5b5865 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -32,6 +32,8 @@ PASSES=0 export DINGGER_TMUX_SOCKET="$SOCK" DINGGER_AGENT_SOCKET="$HOOK" DINGGER_STATE_DIR="$DIR/state" # The daemon updates an installed pi extension when it starts: not the user's. export PI_CODING_AGENT_DIR="$DIR/pi-agent" +# What pi agents cost: the daemons read session files here every 0.3 s. +export DINGGER_COST_SCAN_SECS=0.3 t() { tmux -u -L "$SOCK" "$@"; } @@ -260,14 +262,14 @@ if [ -x "$MENUBAR" ]; then expect "A1 done" "opt $A1 @agent_state" done expect "menu title" "mb --print | head -1" "title: ×1 ✓1" expect "menu rows: herdr priority; idle most recent first (C1 since its jump in step 4, B1 since the start)" \ - "mb --print | tail -n +2 | grep -v '^⟳' | cut -f1,2 | tr '\t\n' '|;'" "× blocked|pi;✓ done|Deploy the widget;○ idle|pi;○ idle|Review PR 42;" + "mb --print | tail -n +2 | grep -v '^⟳\|^\$ ' | cut -f1,2 | tr '\t\n' '|;'" "× blocked|pi;✓ done|Deploy the widget;○ idle|pi;○ idle|Review PR 42;" # Wait for working: B1 was idle before, so idle alone would not show the turn. hook "$B1" working expect "B1 working" "opt $B1 @agent_state" working hook "$B1" idle expect "B1 finishes a turn while watched: idle" "opt $B1 @agent_state" idle expect "and is now the most recent idle agent: listed first" \ - "mb --print | tail -n +2 | grep -v '^⟳' | cut -f1,2 | tr '\t\n' '|;'" "× blocked|pi;✓ done|Deploy the widget;○ idle|Review PR 42;○ idle|pi;" + "mb --print | tail -n +2 | grep -v '^⟳\|^\$ ' | cut -f1,2 | tr '\t\n' '|;'" "× blocked|pi;✓ done|Deploy the widget;○ idle|Review PR 42;○ idle|pi;" expect "subtitle: location, harness, waiting time" "mb --print | grep 'Deploy the widget' | cut -f3" "a · wa1 · pi · done now" expect "idle subtitle has no waiting time" "mb --print | grep 'Review PR 42' | cut -f3" "b · wb1 · pi" mb --click "$A1" @@ -516,6 +518,7 @@ in_snapshot() { python3 -c 'import sys; print(sys.argv[2] in open(sys.argv[1]).r expect "snapshot: the agent's session" "in_snapshot '\"session\": \"$W/sessions/a b.jsonl\"'" True 80 expect "snapshot: a running command's exact arguments" "in_snapshot \"'sleep 100000; :' 'two words'\"" True 80 expect "snapshot: the zsh hook's text" "in_snapshot '\"command\": \"slp 200000 | cat\"'" True 80 +expect "snapshot: the other agent's session" "in_snapshot gone.jsonl" True 80 # An agent that exits just before tmux dies (a shutdown stops programs first). kill "$(ps -o tpgid= -p "$(r display -p -t "$GONE" '#{pane_pid}')" | tr -d ' ')" expect "the exited agent is noticed" "r show -pv -t $GONE @agent 2>/dev/null" "" @@ -1099,6 +1102,48 @@ if command -v node >/dev/null; then fi t kill-session -t dr +# --- 25. what agents cost ----------------------------------------------------- +# The daemon reads pi's session files: each reply once, for the session it +# first appeared in (a fork copies its parent's), closed agents included. +echo "25. what agents cost" +SESS="$PI_CODING_AGENT_DIR/sessions/--work--" +mkdir -p "$SESS" +reply() { # id cost [days ago] + python3 -c 'import json,sys,time; print(json.dumps({"type":"message","id":sys.argv[1],"message":{"role":"assistant","timestamp":int((time.time()-86400*float(sys.argv[3]))*1000),"usage":{"cost":{"total":float(sys.argv[2])}}}}))' "$1" "$2" "${3:-0}" +} +header() { echo "{\"type\":\"session\",\"timestamp\":\"$1\"}"; } +{ header 2026-01-01T00:00:00Z; reply p0 100 40; reply p1 1.25; reply p2 0.25; } >"$SESS/parent.jsonl" +# The fork: its parent's entries, then its own. +{ header 2026-01-02T00:00:00Z; tail -n +2 "$SESS/parent.jsonl"; reply f1 0.5; } >"$SESS/fork.jsonl" +expect "today, each reply once (not 40 days ago)" "gopt @dingger_cost_today" 2.00 50 +expect "this week" "gopt @dingger_cost_week" 2.00 +expect "this month" "gopt @dingger_cost_month" 2.00 +session_report() { # pane session-file + python3 - "$HOOK" "$1" "$2" <<'PY' +import json, socket, sys +params = {"pane_id": sys.argv[2], "agent": "pi", "state": "idle", "agent_session_path": sys.argv[3]} +s = socket.socket(socket.AF_UNIX) +s.connect(sys.argv[1]) +s.sendall((json.dumps({"method": "pane.report_agent", "params": params}) + "\n").encode()) +s.close() +PY +} +t new-session -d -s cost -n c1 -c "$DIR" "$DIR/bin/pi" +t new-window -d -t cost: -n c2 -c "$DIR" "$DIR/bin/pi" +C1=$(t display -p -t cost:c1 '#{pane_id}'); C2=$(t display -p -t cost:c2 '#{pane_id}') +session_report "$C1" "$SESS/parent.jsonl"; session_report "$C2" "$SESS/fork.jsonl" +expect "an agent: what its session spent today" "opt $C1 @dingger_cost" 1.50 +expect "a fork: only its own replies" "opt $C2 @dingger_cost" 0.50 +expect "the menu's top line" "mb --print | grep '^\\$ '" '$ $2.00 today · $2.00 this week · $2.00 this month' +expect "an agent's line" "mb --print | grep -F 'cost · c1' | cut -f3" "cost · c1 · pi · \$1.50" +expect "dingger status: per agent" "$DINGGER status | grep -F 'cost · c2' | grep -c ' \$0.50 '" 1 +expect "and in total" "$DINGGER status | tail -1" 'spent: $2.00 today · $2.00 this week · $2.00 this month (UTC)' +reply f2 0.75 >>"$SESS/fork.jsonl" +expect "a new reply: counted" "opt $C2 @dingger_cost" 1.25 50 +expect "in the total too" "gopt @dingger_cost_today" 2.75 +t kill-session -t cost +expect "closed agents still count" "gopt @dingger_cost_today" 2.75 10 + # Every daemon this run started must stop with it. kill "$DAEMON_PID" "$DAEMON2_PID" 2>/dev/null expect "the test daemons stop" "ps -o pid= -p $DAEMON_PID,$DAEMON2_PID | wc -l | tr -d ' '" 0 diff --git a/tests/pi-live.sh b/tests/pi-live.sh index 6dff42f..95c5885 100755 --- a/tests/pi-live.sh +++ b/tests/pi-live.sh @@ -50,7 +50,7 @@ PORT=$(cat "$D/port") for agent in agent agent-plain; do mkdir -p "$D/$agent/extensions" cat >"$D/$agent/models.json" <<EOF -{"providers": {"fake": {"baseUrl": "http://127.0.0.1:$PORT/v1", "api": "openai-completions", "apiKey": "fake", "models": [{"id": "fake-model"}]}}} +{"providers": {"fake": {"baseUrl": "http://127.0.0.1:$PORT/v1", "api": "openai-completions", "apiKey": "fake", "models": [{"id": "fake-model", "cost": {"input": 100000, "output": 1000000, "cacheRead": 0, "cacheWrite": 0}}]}}} EOF echo '{"defaultProvider": "fake", "defaultModel": "fake-model"}' >"$D/$agent/settings.json" done @@ -70,6 +70,10 @@ printf 'set -g default-shell /bin/sh\nset -g default-command /bin/sh\n' >"$D/tmu # Every pi here uses the test's agent directory, including one a restore # starts (`pi --session`, without the command line's own variables). export PATH="$D/bin:$PATH" PI_CODING_AGENT_DIR="$D/agent" PI_OFFLINE=1 PI_SKIP_VERSION_CHECK=1 PI_TELEMETRY=0 +# The fake model's prices make each reply cost $4.00 (10 input tokens at +# $100,000 per million, 3 output at $1,000,000); the daemon reads costs +# from session files every 0.3 s. +export DINGGER_COST_SCAN_SECS=0.3 export DINGGER_TMUX_SOCKET="$R" DINGGER_AGENT_SOCKET="$D/hook.sock" DINGGER_STATE_DIR="$D/state" DINGGER_TMUX_CONFIG="$D/tmux.conf" t -f "$D/tmux.conf" new-session -d -s main -n agent -x 140 -y 40 -c "$D/work" t new-window -d -t main: -n plain -c "$D/work2" @@ -196,6 +200,23 @@ expect "it still works: working" "st agent" working 50 expect "and done" "st agent" done 60 expect "a message sent is no draft" "draft" "" 30 expect "nor kept" "[ -e \"\$(draft_file)\" ] && echo left || echo gone" gone 30 +# What it cost: pi prices each reply; dingger counts them. +replies() { grep -c '"role":"assistant"' "$1"; } +SESSION_FILE=$(t show -pv -t main:agent @agent_session) +N=$(replies "$SESSION_FILE") +[ "$N" -ge 3 ] && pass "$N replies so far" || fail "only $N replies" +WANT=$(python3 -c "print(f'{4*$N:.2f}')" ) +expect "the agent's spend today" "t show -pv -t main:agent @dingger_cost 2>/dev/null" "$WANT" 30 +expect "in total" "t show -gv @dingger_cost_today 2>/dev/null" "$WANT" 30 +expect "and this week and month" "t show -gv @dingger_cost_week; t show -gv @dingger_cost_month" "$WANT +$WANT" +# A fork copies the session's replies into a new file, then adds its own. +(cd "$D/work" && pi --fork "$SESSION_FILE" -p "slow 1 forked" $NO_MCP >"$D/fork.out" 2>&1) +FORK=$(ls -t "$(dirname "$SESSION_FILE")"/*.jsonl | head -1) +[ "$FORK" != "$SESSION_FILE" ] && [ "$(replies "$FORK")" -eq $((N + 1)) ] \ + && pass "pi --fork: a new file with the $N replies copied and its own" || fail "fork: $FORK ($(replies "$FORK") replies)" +expect "the fork adds only its own reply" "t show -gv @dingger_cost_today 2>/dev/null" "$(python3 -c "print(f'{4*($N+1):.2f}')")" 30 +expect "the agent's own spend is unchanged" "t show -pv -t main:agent @dingger_cost 2>/dev/null" "$WANT" echo "2. screen detection (pi without the extension)" t send-keys -t main:plain "PI_CODING_AGENT_DIR=$D/agent-plain pi $NO_MCP" Enter |
