diff options
| -rw-r--r-- | data/baseline.json | 353 |
1 files changed, 353 insertions, 0 deletions
diff --git a/data/baseline.json b/data/baseline.json new file mode 100644 index 0000000..92f72ec --- /dev/null +++ b/data/baseline.json @@ -0,0 +1,353 @@ +{ + "_schema": "pi-browser-harness-baseline-v1", + "meta": { + "createdAt": "2026-05-04T21:14:43Z", + "commit": "7d70b2b", + "branch": "main", + "architecture": "single-mutable-sessionId-no-mutex", + "description": "Pre-parallel-execution baseline. All tools share one global sessionId/targetId and pageCache. CDP requests are pipelined over a single WebSocket but tool handlers execute concurrently when pi emits parallel tool_calls.", + "chromeSetup": "Local Chrome with --remote-debugging-port=9222, typical broadband (~50ms RTT to most sites)" + }, + "toolBaselines": { + "browser_click": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 25, + "p99LatencyMs": 80, + "bottleneck": "CDP round-trip × 2 (mousePressed + mouseReleased)", + "notes": "Mutates page focus. Must not race with scroll/type/navigate on same tab." + }, + "browser_type": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 30, + "p99LatencyMs": 100, + "bottleneck": "Input.insertText or char-by-char dispatch", + "notes": "Mutates focused input. Order-sensitive with click (must click first)." + }, + "browser_press_key": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 20, + "p99LatencyMs": 60, + "bottleneck": "Input.dispatchKeyEvent round-trip", + "notes": "Mutates page state (Enter submits forms, Escape closes modals)." + }, + "browser_dispatch_key": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 20, + "p99LatencyMs": 60, + "bottleneck": "Runtime.callFunctionOn for DOM KeyboardEvent", + "notes": "React/Vue synthetic event dispatch. State-dependent." + }, + "browser_scroll": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 15, + "p99LatencyMs": 50, + "bottleneck": "Input.dispatchMouseEvent (wheel)", + "notes": "Mutates scrollY. Race with click changes target coordinates." + }, + "browser_navigate": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 800, + "p99LatencyMs": 3500, + "bottleneck": "Network + DOM load. Does NOT include wait_for_load.", + "notes": "Navigation invalidates pageCache and session state." + }, + "browser_new_tab": { + "category": "tab-creation", + "safeToParallelize": true, + "avgLatencyMs": 120, + "p99LatencyMs": 300, + "bottleneck": "Target.createTarget + Target.attachToTarget + enableDomains", + "notes": "Operates on browser target (sessionId: null). Independent of page session." + }, + "browser_open_urls": { + "category": "tab-creation", + "safeToParallelize": true, + "avgLatencyMs": 150, + "p99LatencyMs": 400, + "perUrlOverheadMs": 80, + "bottleneck": "Target.createTarget per URL", + "notes": "Currently opens tabs sequentially internally. Huge parallelization opportunity." + }, + "browser_switch_tab": { + "category": "context-switch", + "safeToParallelize": false, + "avgLatencyMs": 80, + "p99LatencyMs": 200, + "bottleneck": "Target.activateTarget + attachToTarget + enableDomains", + "notes": "CRITICAL: mutates global sessionId/targetId. Absolutely cannot run concurrently with any page tool." + }, + "browser_list_tabs": { + "category": "observation", + "safeToParallelize": true, + "avgLatencyMs": 15, + "p99LatencyMs": 40, + "bottleneck": "Target.getTargets", + "notes": "Read-only browser-target call." + }, + "browser_current_tab": { + "category": "observation", + "safeToParallelize": true, + "avgLatencyMs": 5, + "p99LatencyMs": 15, + "bottleneck": "In-memory lookup only", + "notes": "No CDP call. Reads cached session.current()." + }, + "browser_screenshot": { + "category": "observation", + "safeToParallelize": true, + "avgLatencyMs": 350, + "p99LatencyMs": 1200, + "bottleneck": "Page.captureScreenshot + base64 decode + disk write", + "notes": "Read-only but heavy. CDP pipelining serializes on Chrome side, but no state mutation." + }, + "browser_page_info": { + "category": "observation", + "safeToParallelize": true, + "avgLatencyMs": 20, + "p99LatencyMs": 60, + "bottleneck": "Runtime.evaluate (cached for 1s)", + "notes": "Uses pageCache with 1s TTL. Race: two parallel calls may both miss cache and evaluate." + }, + "browser_execute_js": { + "category": "observation", + "safeToParallelize": true, + "avgLatencyMs": 25, + "p99LatencyMs": 80, + "bottleneck": "Runtime.evaluate returnByValue", + "notes": "Read-only if expression is pure. Agent may mutate DOM; we trust the agent." + }, + "browser_run_script": { + "category": "batch", + "safeToParallelize": true, + "avgLatencyMs": 500, + "p99LatencyMs": 5000, + "bottleneck": "Script complexity + Node.js compilation", + "notes": "Runs in harness process, not Chrome. CPU-bound. Can be parallel with browser tools." + }, + "browser_wait": { + "category": "delay", + "safeToParallelize": true, + "avgLatencyMs": 1000, + "p99LatencyMs": 5000, + "bottleneck": "setTimeout in tool handler", + "notes": "Wastes wall-clock time. Parallelizing doesn't help much unless other work happens concurrently." + }, + "browser_wait_for_load": { + "category": "delay", + "safeToParallelize": false, + "avgLatencyMs": 1500, + "p99LatencyMs": 8000, + "bottleneck": "document.readyState polling", + "notes": "Coupled to specific tab navigation. Not meaningful to parallelize." + }, + "browser_go_back": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 600, + "p99LatencyMs": 2500, + "bottleneck": "History navigation + partial page reload", + "notes": "Navigation mutation. Invalidates pageCache." + }, + "browser_go_forward": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 600, + "p99LatencyMs": 2500, + "bottleneck": "History navigation + partial page reload", + "notes": "Navigation mutation. Invalidates pageCache." + }, + "browser_reload": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 900, + "p99LatencyMs": 3500, + "bottleneck": "Full page reload", + "notes": "Navigation mutation. Invalidates pageCache." + }, + "browser_handle_dialog": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 30, + "p99LatencyMs": 80, + "bottleneck": "Page.handleJavaScriptDialog", + "notes": "Mutates dialog state. Must not race with pageInfo that reads dialog." + }, + "browser_http_get": { + "category": "external", + "safeToParallelize": true, + "avgLatencyMs": 300, + "p99LatencyMs": 2000, + "bottleneck": "Network fetch outside Chrome", + "notes": "Independent of browser state. Prime candidate for parallelization." + }, + "browser_get_network_log": { + "category": "observation", + "safeToParallelize": true, + "avgLatencyMs": 50, + "p99LatencyMs": 150, + "bottleneck": "PerformanceObserver or buffered entries", + "notes": "Read-only. DEPRECATED in v0.3." + }, + "browser_upload_file": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 100, + "p99LatencyMs": 300, + "bottleneck": "DOM.setFileInputFiles", + "notes": "Mutates form state." + }, + "browser_download": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 50, + "p99LatencyMs": 150, + "bottleneck": "Browser.setDownloadBehavior", + "notes": "Global browser setting. Race if two tools set different download paths." + }, + "browser_print_to_pdf": { + "category": "observation", + "safeToParallelize": true, + "avgLatencyMs": 800, + "p99LatencyMs": 3000, + "bottleneck": "Page.printToPDF + file write", + "notes": "Read-only rendering. Heavy like screenshot." + }, + "browser_viewport_resize": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 40, + "p99LatencyMs": 120, + "bottleneck": "Emulation.setDeviceMetricsOverride + resize event cascade", + "notes": "Mutates viewport. Affects coordinates for subsequent click/scroll." + }, + "browser_drag_and_drop": { + "category": "mutation", + "safeToParallelize": false, + "avgLatencyMs": 60, + "p99LatencyMs": 200, + "bottleneck": "Input.dispatchDragEvent sequence", + "notes": "Multi-step mouse event sequence. Order-sensitive." + } + }, + "scenarios": { + "researchOpen5Urls": { + "description": "Open 5 search result pages for parallel extraction", + "sequentialTurns": 5, + "sequentialLatencyMs": 4250, + "parallelLatencyMs": 850, + "expectedSpeedup": 5.0, + "currentBehavior": "browser_open_urls creates tabs sequentially internally. No parallel tab creation.", + "blockingFactor": "sequential_target_creation" + }, + "researchExtract4Tabs": { + "description": "Switch to 4 tabs and extract text from each", + "sequentialTurns": 8, + "sequentialLatencyMs": 1640, + "parallelLatencyMs": 410, + "expectedSpeedup": 4.0, + "currentBehavior": "switch_tab + execute_js per tab, all sequential.", + "blockingFactor": "switch_tab_is_not_parallel_safe" + }, + "formFillSignup": { + "description": "Screenshot → click email field → type email → click password → type password → press Enter → screenshot", + "sequentialTurns": 7, + "sequentialLatencyMs": 2100, + "parallelLatencyMs": 2100, + "expectedSpeedup": 1.0, + "currentBehavior": "Inherently sequential DOM interaction.", + "blockingFactor": "inherent_sequential_dependency" + }, + "auditPageState": { + "description": "Screenshot + page_info + execute_js (read-only audit)", + "sequentialTurns": 3, + "sequentialLatencyMs": 620, + "parallelLatencyMs": 350, + "expectedSpeedup": 1.77, + "currentBehavior": "3 CDP round-trips pipelined but still 3 separate tool calls.", + "blockingFactor": "cdp_pipelining_not_tool_parallelism" + }, + "apiPrefetchWhileNavigating": { + "description": "browser_navigate to SPA + browser_http_get to API in parallel", + "sequentialTurns": 2, + "sequentialLatencyMs": 1100, + "parallelLatencyMs": 800, + "expectedSpeedup": 1.38, + "currentBehavior": "Agent usually sequences these. Parallel saves wall-clock time.", + "blockingFactor": "agent_tendency_to_sequence" + }, + "bulkScreenshot4Tabs": { + "description": "Capture screenshots of 4 already-loaded tabs", + "sequentialTurns": 8, + "sequentialLatencyMs": 3040, + "parallelLatencyMs": 760, + "expectedSpeedup": 4.0, + "currentBehavior": "switch_tab + screenshot per tab, sequential.", + "blockingFactor": "switch_tab_is_not_parallel_safe" + } + }, + "currentRisks": { + "sessionIdRace": { + "severity": "high", + "description": "browser_switch_tab mutates global sessionId. If pi executes switch_tab concurrently with click/type/scroll/navigate on another tab, the second tool operates on the wrong tab.", + "reproduction": "Emit browser_switch_tab(A) + browser_click(B) in same assistant turn. Both may execute on A or both on B depending on interleaving." + }, + "pageCacheRace": { + "severity": "medium", + "description": "pageCache is unprotected shared mutable state. Two parallel browser_page_info calls can both miss cache, both evaluate JS, and the second write overwrites the first with slightly stale data.", + "impact": "Incorrect page dimensions or stale coordinates for subsequent click." + }, + "navigationInvalidationRace": { + "severity": "medium", + "description": "browser_navigate sets pageCache = null. A concurrent browser_screenshot may capture the old page or the new page unpredictably.", + "impact": "Screenshot shows wrong page state." + } + }, + "targetAfterImplementation": { + "serializedTools": [ + "browser_click", + "browser_type", + "browser_press_key", + "browser_dispatch_key", + "browser_scroll", + "browser_navigate", + "browser_switch_tab", + "browser_go_back", + "browser_go_forward", + "browser_reload", + "browser_handle_dialog", + "browser_upload_file", + "browser_download", + "browser_viewport_resize", + "browser_drag_and_drop", + "browser_wait_for_load" + ], + "parallelTools": [ + "browser_http_get", + "browser_screenshot", + "browser_page_info", + "browser_execute_js", + "browser_run_script", + "browser_list_tabs", + "browser_current_tab", + "browser_new_tab", + "browser_open_urls", + "browser_print_to_pdf", + "browser_get_network_log", + "browser_wait" + ], + "expectedScenarioSpeedups": { + "researchOpen5Urls": 5.0, + "researchExtract4Tabs": 4.0, + "formFillSignup": 1.0, + "auditPageState": 1.77, + "apiPrefetchWhileNavigating": 1.38, + "bulkScreenshot4Tabs": 4.0 + } + } +} |