aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authoramankumarsingh77 <amankumarsingh7702@gmal.com>2026-05-03 00:43:43 +0530
committeramankumarsingh77 <amankumarsingh7702@gmal.com>2026-05-03 00:43:43 +0530
commitcb9a51eb557ae67f9d7c65f20e600c278cf7f6ab (patch)
tree5753da56bc99f49c625cbf3ebd91d4d484672329
parentff4859236e2de4844d5231768f57cbfbe3f937e1 (diff)
feat: render screenshots inline in TUI via Image component; add parallelization guidance to SKILL.md
- browser_screenshot now has renderResult that creates an Image component for inline display in Kitty/iTerm2/Ghostty/WezTerm terminals - Falls back to dim text placeholder on terminals without image support - Updated system prompt table description to mention TUI inline rendering - Added Parallelization section to SKILL.md teaching the agent to batch independent operations (browser_open_urls, browser_http_get, etc.)
-rw-r--r--skills/pi-browser-harness/SKILL.md56
-rw-r--r--src/domains/screenshot.ts25
-rw-r--r--src/prompt.ts2
3 files changed, 81 insertions, 2 deletions
diff --git a/skills/pi-browser-harness/SKILL.md b/skills/pi-browser-harness/SKILL.md
index 81a233c..60cd069 100644
--- a/skills/pi-browser-harness/SKILL.md
+++ b/skills/pi-browser-harness/SKILL.md
@@ -32,6 +32,60 @@ browser_screenshot() → browser_click(x, y) → browser_screenshot()
Compositor-level clicks (`Input.dispatchMouseEvent`) work through iframes, shadow DOM,
and cross-origin content. No selectors needed.
+## Parallelization
+
+**Always parallelize when possible.** Never sequence independent operations.
+
+### What can run in parallel
+
+- **Opening result pages:** Use `browser_open_urls` to open multiple URLs in parallel tabs
+ instead of navigating one at a time. After opening, use `browser_list_tabs` to
+ see the new tabs and `browser_switch_tab` to visit each one.
+
+- **API calls:** `browser_http_get` calls are independent of browser state — fire
+ multiple GETs in parallel to fetch APIs, static pages, or search results.
+
+- **JS extraction across tabs:** Once tabs are loaded, extract data from each tab
+ with `browser_execute_js` — these can run in parallel across different tabs.
+
+- **Screenshots across tabs:** After opening multiple pages, capture screenshots
+ of each tab in parallel. Each `browser_screenshot` targets the currently active
+ tab, so switch tabs first, then screenshot.
+
+- **Read + browser tools:** Use `read` and `browser_*` tools in the same turn —
+ reading files or previous results is independent of browser actions.
+
+### What must be sequential
+
+- **Page interactions:** Clicks, typing, scrolling, and form submissions on the
+ same page depend on prior state and must be sequential.
+
+- **Tab switching then acting:** `browser_switch_tab` must complete before acting
+ on the switched-to tab.
+
+### Pattern: Parallel research across tabs
+
+```
+# Step 1: Open all result pages at once
+browser_open_urls(["url1", "url2", "url3"])
+
+# Step 2: Wait for them to load, then extract in parallel
+browser_list_tabs() # get targetIds
+
+# Step 3: Extract data from each tab (parallel)
+browser_switch_tab(targetId1) → browser_screenshot() # These are sequential per tab
+browser_switch_tab(targetId2) → browser_execute_js(...) # but you batch switch+act
+```
+
+### Pattern: Parallel API + browser
+
+```
+# Fire API calls while interacting with a page
+browser_http_get("https://api.example.com/data") # parallel
+browser_click(x, y) # parallel (different target)
+browser_type("search term") # parallel (different target)
+```
+
## Available Tools
### Navigation
@@ -49,7 +103,7 @@ and cross-origin content. No selectors needed.
- `browser_scroll` — scroll the page at coordinates
### Visual
-- `browser_screenshot` — capture PNG screenshot
+- `browser_screenshot` — capture PNG or JPEG screenshot (rendered inline in the TUI)
### Data Extraction
- `browser_execute_js` — execute JavaScript and return result
diff --git a/src/domains/screenshot.ts b/src/domains/screenshot.ts
index 03b49f0..60d4b26 100644
--- a/src/domains/screenshot.ts
+++ b/src/domains/screenshot.ts
@@ -1,5 +1,7 @@
+import { readFileSync } from "node:fs";
import { writeFile, rename } from "node:fs/promises";
import { Type } from "typebox";
+import { Image, type ImageTheme, Text } from "@mariozechner/pi-tui";
import type { BrowserClient } from "../client";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
@@ -82,6 +84,29 @@ export const screenshotTool = defineBrowserTool({
details: { path: cap.data.path, format: cap.data.format, attached: false },
});
},
+
+ renderResult(result, _expanded, theme) {
+ const details = result.details as { path?: string; format?: string } | undefined;
+ const filePath = details?.path;
+ if (!filePath) {
+ return new Text(theme.fg("error", "Screenshot path missing"), 0, 0);
+ }
+ try {
+ const buf = readFileSync(filePath);
+ const b64 = buf.toString("base64");
+ const mimeType = details?.format === "jpeg" ? "image/jpeg" : "image/png";
+ const imageTheme: ImageTheme = {
+ fallbackColor: (str: string) => theme.fg("dim", str),
+ };
+ return new Image(b64, mimeType, imageTheme, {
+ maxWidthCells: 80,
+ maxHeightCells: 24,
+ filename: filePath,
+ });
+ } catch {
+ return new Text(theme.fg("warning", `Screenshot saved: ${filePath}`), 0, 0);
+ }
+ },
});
export const captureWithCrosshair = async (
diff --git a/src/prompt.ts b/src/prompt.ts
index d4ac8a3..6d2b2ab 100644
--- a/src/prompt.ts
+++ b/src/prompt.ts
@@ -21,7 +21,7 @@ to interact with pages visually, and other tools alongside them.
| Tool | Purpose |
|------|---------|
-| browser_screenshot | Capture a PNG screenshot of the current page (result details include path and attached: false — the LLM reads the image via the saved file path) |
+| browser_screenshot | Capture a PNG or JPEG screenshot of the current page (rendered inline in the TUI; the LLM reads the image via the saved file path) |
| browser_click | Click at viewport coordinates |
| browser_type | Type text into the focused element |
| browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) |