diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/release_perf_case.sh | 149 | ||||
| -rwxr-xr-x | scripts/release_perf_producer.pl | 26 | ||||
| -rwxr-xr-x | scripts/release_perf_smoke.sh | 92 | ||||
| -rw-r--r-- | scripts/test_ui_hot_path_architecture.py | 65 |
4 files changed, 314 insertions, 18 deletions
diff --git a/scripts/release_perf_case.sh b/scripts/release_perf_case.sh new file mode 100755 index 00000000..47d72259 --- /dev/null +++ b/scripts/release_perf_case.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 8 ]]; then + echo "usage: $0 <binary> <variant> <scenario> <round> <seconds> <warmup> <output-root> <platform>" >&2 + exit 2 +fi + +bin_dir=$(cd "$(dirname "$1")" && pwd) +bin="$bin_dir/$(basename "$1")" +variant=$2 +scenario=$3 +round=$4 +seconds=$5 +warmup=$6 +out_root=$(cd "$7" && pwd) +platform=$8 +script_dir=$(cd "$(dirname "$0")" && pwd) +producer="$script_dir/release_perf_producer.pl" +cols=86 +rows=47 + +case "$scenario" in + visible30) scenario_tag=v3; total_panes=1; writers=visible; rate=30 ;; + hidden50) scenario_tag=h5; total_panes=50; writers=hidden; rate=60 ;; + *) echo "unknown scenario: $scenario" >&2; exit 2 ;; +esac +case "$platform" in linux) platform_tag=l ;; macos) platform_tag=m ;; *) exit 2 ;; esac +variant_tag=${variant:0:1} +name="rps${platform_tag}${variant_tag}${scenario_tag}r${round}x$$" +state="/var/tmp/herdr-release-perf-$name" +xdg="$state/xdg" +runtime="$state/run" +gate="$state/start-output" +out="$out_root/$variant/$scenario/r$round" +mkdir -p "$xdg" "$runtime" "$out" + +launch_env=(env -u HERDR_BIN_PATH -u HERDR_ENV -u HERDR_SOCKET_PATH -u HERDR_CLIENT_SOCKET_PATH -u HERDR_SESSION -u HERDR_STARTUP_CWD -u HERDR_WORKSPACE_ID -u HERDR_TAB_ID -u HERDR_PANE_ID XDG_CONFIG_HOME="$xdg" XDG_RUNTIME_DIR="$runtime" HERDR_DISABLE_SOUND=1 SHELL=/bin/sh) +control_env=(env -u HERDR_BIN_PATH -u HERDR_ENV -u HERDR_SOCKET_PATH -u HERDR_CLIENT_SOCKET_PATH -u HERDR_STARTUP_CWD -u HERDR_WORKSPACE_ID -u HERDR_TAB_ID -u HERDR_PANE_ID XDG_CONFIG_HOME="$xdg" XDG_RUNTIME_DIR="$runtime" HERDR_DISABLE_SOUND=1 SHELL=/bin/sh HERDR_SESSION="$name") + +cleaned=0 +cleanup() { + if [[ $cleaned -eq 1 ]]; then return; fi + cleaned=1 + "${control_env[@]}" "$bin" session stop "$name" >/dev/null 2>&1 || true + for _ in $(seq 1 50); do + if "${control_env[@]}" "$bin" session delete "$name" >/dev/null 2>&1; then break; fi + sleep 0.1 + done + tmux kill-session -t "$name" >/dev/null 2>&1 || true + rm -rf "$state" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +printf -v launch 'exec ' +printf -v quoted '%q ' "${launch_env[@]}" "$bin" --session "$name" +launch+=$quoted +tmux new-session -d -s "$name" -x "$cols" -y "$rows" "$launch" + +panes_json= +for _ in $(seq 1 150); do + if panes_json=$("${control_env[@]}" "$bin" pane list 2>/dev/null); then break; fi + sleep 0.1 +done +[[ -n "$panes_json" ]] || { echo "session API did not become ready" >&2; exit 1; } +root_pane=$(printf '%s\n' "$panes_json" | jq -r '.result.panes[0].pane_id') +workspace_id=$("${control_env[@]}" "$bin" workspace list | jq -r '.result.workspaces[0].workspace_id') +[[ -n "$root_pane" && "$root_pane" != null ]] || { echo "session did not report a root pane" >&2; exit 1; } +[[ -n "$workspace_id" && "$workspace_id" != null ]] || { echo "session did not report a workspace" >&2; exit 1; } +pane_file="$state/pane-ids.txt" +printf '%s\n' "$root_pane" > "$pane_file" + +for ((index = 2; index <= total_panes; index++)); do + created=$("${control_env[@]}" "$bin" tab create --workspace "$workspace_id" --label "bench-$index" --no-focus) + pane_id=$(printf '%s\n' "$created" | jq -r '.result.root_pane.pane_id') + [[ -n "$pane_id" && "$pane_id" != null ]] || { echo "tab $index did not return a pane" >&2; exit 1; } + printf '%s\n' "$pane_id" >> "$pane_file" +done + +index=0 +while IFS= read -r pane_id; do + index=$((index + 1)) + if [[ $writers == visible && $index -eq 1 ]] || [[ $writers == hidden && $index -gt 1 ]]; then + "${control_env[@]}" "$bin" pane run "$pane_id" "$producer" "$rate" "$gate" "p$index" >/dev/null + fi +done < "$pane_file" +touch "$gate" + +socket="$xdg/herdr/sessions/$name/herdr.sock" +server_pid= +for _ in $(seq 1 80); do + server_pid=$(lsof -t "$socket" 2>/dev/null | head -n1 || true) + [[ -n "$server_pid" ]] && break + sleep 0.1 +done +[[ -n "$server_pid" ]] || { echo "could not find server pid" >&2; exit 1; } +client_pid=$(tmux list-panes -s -t "$name" -F '#{pane_pid}') +[[ -n "$client_pid" ]] || { echo "could not find client pid" >&2; exit 1; } +all_pids=("$server_pid" "$client_pid") + +sleep "$warmup" +raw="$out/cpu-raw.txt" +if [[ $platform == linux ]]; then + pid_csv=$(IFS=,; echo "${all_pids[*]}") + LC_ALL=C pidstat -h -u -p "$pid_csv" 1 "$seconds" > "$raw" +else + top_args=(top -l $((seconds + 1)) -s 1 -stats pid,cpu,time -n 2) + for pid in "${all_pids[@]}"; do top_args+=(-pid "$pid"); done + LC_ALL=C "${top_args[@]}" > "$raw" +fi + +mean_linux() { + awk -v target="$2" ' + /^Linux/ || /^#/ || NF < 5 { next } + $3 == target && $(NF-2) ~ /^[0-9]+([.][0-9]+)?$/ { sum += $(NF-2); count++ } + END { if (!count) exit 1; printf "%.6f,%d", sum/count, count } + ' "$1" +} +mean_macos() { + awk -v target="$2" ' + $1 == target && $2 ~ /^[0-9]+([.][0-9]+)?%?$/ { + seen++; if (seen == 1) next; value=$2; gsub(/%/, "", value); sum += value; count++ } + END { if (!count) exit 1; printf "%.6f,%d", sum/count, count } + ' "$1" +} + +total=0 +for pid in "${all_pids[@]}"; do + if [[ $platform == linux ]]; then parsed=$(mean_linux "$raw" "$pid"); else parsed=$(mean_macos "$raw" "$pid"); fi + mean=${parsed%,*} + samples=${parsed#*,} + [[ $samples -eq $seconds ]] || { echo "expected $seconds samples for pid $pid, got $samples" >&2; exit 1; } + total=$(awk -v total="$total" -v mean="$mean" 'BEGIN { printf "%.6f", total + mean }') +done + +index=0 +while IFS= read -r pane_id; do + index=$((index + 1)) + if [[ $writers == visible && $index -eq 1 ]] || [[ $writers == hidden && $index -gt 1 ]]; then + read_file="$out/pane-$index.txt" + "${control_env[@]}" "$bin" pane read "$pane_id" --source visible --format text > "$read_file" + grep -q 'bench-output-' "$read_file" || { echo "writer pane $index produced no output" >&2; exit 1; } + fi +done < "$pane_file" + +printf '%s\n' "$total" > "$out/total-cpu.txt" +printf '%s,%s,%s,%s\n' "$variant" "$scenario" "$round" "$total" diff --git a/scripts/release_perf_producer.pl b/scripts/release_perf_producer.pl new file mode 100755 index 00000000..e99973cc --- /dev/null +++ b/scripts/release_perf_producer.pl @@ -0,0 +1,26 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use Time::HiRes qw(clock_gettime sleep CLOCK_MONOTONIC); + +my ($rate, $gate, $label) = @ARGV; +die "usage: $0 <rate-hz> <gate-file> <label>\n" + unless defined $rate && defined $gate && defined $label + && length $gate && length $label; +die "rate must be a positive number\n" + unless $rate =~ /\A(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)\z/ && $rate > 0; + +sleep 0.01 until -e $gate; +$| = 1; +my $period = 1 / $rate; +my $next = clock_gettime(CLOCK_MONOTONIC); +my $sequence = 0; +while (1) { + $sequence++; + printf "\rbench-output-%08d-%s", $sequence, $label; + $next += $period; + my $now = clock_gettime(CLOCK_MONOTONIC); + $next = $now + $period if $next < $now - $period; + my $remaining = $next - $now; + sleep $remaining if $remaining > 0; +} diff --git a/scripts/release_perf_smoke.sh b/scripts/release_perf_smoke.sh new file mode 100755 index 00000000..7ce1f736 --- /dev/null +++ b/scripts/release_perf_smoke.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 <candidate-binary>" >&2 + exit 2 +fi + +candidate=$(cd "$(dirname "$1")" && pwd)/$(basename "$1") +[[ -x "$candidate" ]] || { echo "candidate binary is not executable: $candidate" >&2; exit 1; } +script_dir=$(cd "$(dirname "$0")" && pwd) +repo_root=$(cd "$script_dir/.." && pwd) +baseline=${HERDR_PERF_BASELINE_BIN:-} +for command in jq lsof perl tmux; do + command -v "$command" >/dev/null || { echo "required command not found: $command" >&2; exit 1; } +done +if [[ -z "$baseline" ]]; then + command -v curl >/dev/null || { echo "required command not found: curl" >&2; exit 1; } +fi + +case "$(uname -s)" in + Linux) platform=linux; command -v pidstat >/dev/null || { echo "required command not found: pidstat" >&2; exit 1; } ;; + Darwin) platform=macos ;; + *) echo "release performance smoke supports Linux and macOS" >&2; exit 1 ;; +esac +case "$(uname -m)" in + x86_64|amd64) arch=x86_64 ;; + arm64|aarch64) arch=aarch64 ;; + *) echo "unsupported architecture: $(uname -m)" >&2; exit 1 ;; +esac + +root=$(mktemp -d /var/tmp/herdr-release-perf-smoke.XXXXXX) +cleanup() { rm -rf "$root"; } +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +mkdir -p "$root/results" + +if [[ -z "$baseline" ]]; then + baseline_version=$(jq -er '.version' "$repo_root/website/latest.json") + baseline="$root/herdr-baseline" + curl -fL --retry 3 \ + "https://github.com/herdrdev/herdr/releases/download/v${baseline_version}/herdr-${platform}-${arch}" \ + -o "$baseline" + chmod +x "$baseline" +else + baseline=$(cd "$(dirname "$baseline")" && pwd)/$(basename "$baseline") +fi +[[ -x "$baseline" ]] || { echo "baseline binary is not executable: $baseline" >&2; exit 1; } + +case_script="$script_dir/release_perf_case.sh" +seconds=${HERDR_PERF_SAMPLE_SECONDS:-10} +warmup=${HERDR_PERF_WARMUP_SECONDS:-3} + +for round in 1 2; do + if [[ $round -eq 1 ]]; then variants="baseline candidate"; else variants="candidate baseline"; fi + for scenario in hidden50 visible30; do + for variant in $variants; do + if [[ $variant == baseline ]]; then binary=$baseline; else binary=$candidate; fi + "$case_script" "$binary" "$variant" "$scenario" "$round" "$seconds" "$warmup" "$root/results" "$platform" + done + done +done + +mean_total() { + awk '{ sum += $1; count++ } END { if (!count) exit 1; printf "%.3f", sum/count }' \ + "$root/results/$1/$2/r1/total-cpu.txt" \ + "$root/results/$1/$2/r2/total-cpu.txt" +} + +failed=0 +printf '\nrelease performance smoke (%s/%s, two rounds, %ss samples)\n' "$platform" "$arch" "$seconds" +printf '%-12s %12s %12s %12s\n' scenario baseline candidate change +for scenario in hidden50 visible30; do + baseline_total=$(mean_total baseline "$scenario") + candidate_total=$(mean_total candidate "$scenario") + if awk -v before="$baseline_total" -v after="$candidate_total" 'BEGIN { exit !(before <= 0 || after <= 0) }'; then + echo "error: $scenario measured no CPU usage; the benchmark did not exercise the binaries" >&2 + failed=1 + fi + change=$(awk -v before="$baseline_total" -v after="$candidate_total" 'BEGIN { if (before == 0) print "n/a"; else printf "%+.1f%%", (after-before)/before*100 }') + printf '%-12s %12s %12s %12s\n' "$scenario" "$baseline_total" "$candidate_total" "$change" + if awk -v before="$baseline_total" -v after="$candidate_total" 'BEGIN { exit !(after > before * 1.25 && after - before > 0.5) }'; then + echo "error: $scenario candidate CPU exceeds baseline by more than 25% and 0.5 CPU points" >&2 + failed=1 + fi +done + +if [[ $failed -ne 0 ]]; then + exit 1 +fi +echo "release performance smoke passed" diff --git a/scripts/test_ui_hot_path_architecture.py b/scripts/test_ui_hot_path_architecture.py index 44412197..be3d35f0 100644 --- a/scripts/test_ui_hot_path_architecture.py +++ b/scripts/test_ui_hot_path_architecture.py @@ -11,16 +11,21 @@ HOT_PATH_SOURCES = ( *sorted((PROJECT_ROOT / "src" / "ui").rglob("*.rs")), PROJECT_ROOT / "src" / "server" / "render_stream.rs", ) +APP_SERVER_SOURCES = ( + *sorted((PROJECT_ROOT / "src" / "app").rglob("*.rs")), + *sorted((PROJECT_ROOT / "src" / "server").rglob("*.rs")), +) TEST_MODULE = re.compile(r"(?m)^#\[cfg\(test\)\]\s*\nmod\s+\w+\s*\{") +INPUT_STATE_CALL = re.compile(r"(?:\.|::)input_state\b") +KEYBOARD_STATE_ANSI_CALL = re.compile( + r"(?:\.|::)(?:keyboard_state_ansi|kitty_keyboard_state_ansi)\b" +) +AGGREGATE_STATE_CALLS = ( + (INPUT_STATE_CALL, "aggregate terminal input state; add a narrow accessor"), + (KEYBOARD_STATE_ANSI_CALL, "formatted keyboard state"), +) FORBIDDEN_CALLS = ( - ( - re.compile(r"(?:\.|::)input_state\b"), - "aggregate terminal input state; add a narrow accessor", - ), - ( - re.compile(r"(?:\.|::)(?:keyboard_state_ansi|kitty_keyboard_state_ansi)\b"), - "formatted keyboard state", - ), + *AGGREGATE_STATE_CALLS, ( re.compile(r"(?:\.|::)screen_text_snapshot\b"), "formatted terminal screen snapshot", @@ -130,18 +135,21 @@ def production_code(source: str) -> str: return code +def find_violations(paths, rules) -> list[str]: + violations: list[str] = [] + for path in paths: + code = production_code(path.read_text(encoding="utf-8")) + for pattern, description in rules: + for match in pattern.finditer(code): + line = code.count("\n", 0, match.start()) + 1 + relative_path = path.relative_to(PROJECT_ROOT) + violations.append(f"{relative_path}:{line}: {description}") + return violations + + class UiHotPathArchitectureTests(unittest.TestCase): def test_render_hot_paths_avoid_known_expensive_runtime_queries(self) -> None: - violations: list[str] = [] - - for path in HOT_PATH_SOURCES: - source = path.read_text(encoding="utf-8") - code = production_code(source) - for pattern, description in FORBIDDEN_CALLS: - for match in pattern.finditer(code): - line = code.count("\n", 0, match.start()) + 1 - relative_path = path.relative_to(PROJECT_ROOT) - violations.append(f"{relative_path}:{line}: {description}") + violations = find_violations(HOT_PATH_SOURCES, FORBIDDEN_CALLS) self.assertEqual( violations, @@ -150,6 +158,17 @@ class UiHotPathArchitectureTests(unittest.TestCase): + "\n".join(violations), ) + def test_app_and_server_avoid_aggregate_terminal_state(self) -> None: + self.assertTrue(APP_SERVER_SOURCES, "No app/server Rust sources were discovered") + violations = find_violations(APP_SERVER_SOURCES, AGGREGATE_STATE_CALLS) + + self.assertEqual( + violations, + [], + "App/server code must use narrow terminal-state accessors:\n" + + "\n".join(violations), + ) + def test_scanner_ignores_non_production_references(self) -> None: source = ''' // runtime.input_state() @@ -175,6 +194,16 @@ fn render() { TerminalRuntime::input_state; } ''' self.assertRegex(production_code(source), FORBIDDEN_CALLS[0][0]) + def test_scanner_catches_each_aggregate_state_call(self) -> None: + cases = ( + ("fn render() { runtime.input_state(); }", INPUT_STATE_CALL), + ("fn render() { runtime.keyboard_state_ansi(); }", KEYBOARD_STATE_ANSI_CALL), + ("fn render() { runtime.kitty_keyboard_state_ansi(); }", KEYBOARD_STATE_ANSI_CALL), + ) + for source, pattern in cases: + with self.subTest(source=source): + self.assertRegex(production_code(source), pattern) + def test_scanner_catches_imported_process_query(self) -> None: source = "fn render() { foreground_job(pid); }" self.assertRegex(production_code(source), FORBIDDEN_CALLS[3][0]) |