Replace C browser client with headless Chrome

src/agent.rs

@@ -1,12 +1,14 @@
+mod web;
+
+use self::web::Browser;
 use crate::model::ModelChoice;
 use serde_json::{Map, Value};
 use std::collections::HashMap;
-use std::ffi::{CStr, CString, c_char, c_int, c_void};
 use std::fs::{self, File};
 use std::os::unix::process::CommandExt;
 use std::path::{Path, PathBuf};
 use std::process::{Child, Command, Stdio};
-use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender, TryRecvError};
 use std::sync::{Arc, Mutex};
 use std::thread;
@@ -30,158 +32,8 @@ const SHELL_ENV_ALLOWLIST: &[&str] = &[
 ];
 pub(crate) const COMPACTION_OBSERVATION_PREFIX: &str = "Bash job update after context compaction.";
 
-#[repr(C)]
-struct WebConfig {
-    profile_dir: *const c_char,
-    port: c_int,
-    confirm: Option<unsafe extern "C" fn(*mut c_void, *const c_char, *mut c_char, usize) -> c_int>,
-    confirm_data: *mut c_void,
-    log: Option<unsafe extern "C" fn(*mut c_void, *const c_char)>,
-    log_data: *mut c_void,
-    cancel: Option<unsafe extern "C" fn(*mut c_void) -> bool>,
-    cancel_data: *mut c_void,
-}
-
-unsafe extern "C" {
-    fn ds4_web_create(config: *const WebConfig) -> *mut c_void;
-    fn ds4_web_free(web: *mut c_void);
-    fn ds4_web_google_search(
-        web: *mut c_void,
-        query: *const c_char,
-        error: *mut c_char,
-        error_len: usize,
-    ) -> *mut c_char;
-    fn ds4_web_visit_page(
-        web: *mut c_void,
-        url: *const c_char,
-        error: *mut c_char,
-        error_len: usize,
-    ) -> *mut c_char;
-    fn free(pointer: *mut c_void);
-}
-
-struct WebCallbacks {
-    cancel: AtomicPtr<AtomicBool>,
-}
-
-struct Browser {
-    web: *mut c_void,
-    callbacks: Box<WebCallbacks>,
-}
-
-// The C browser is used by one agent-tools worker at a time under Tools' mutex.
-unsafe impl Send for Browser {}
-
-impl Browser {
-    fn new() -> Result<Self, String> {
-        let mut callbacks = Box::new(WebCallbacks {
-            cancel: AtomicPtr::new(std::ptr::null_mut()),
-        });
-        let profile = CString::new(
-            crate::app::browser_profile_path()
-                .to_string_lossy()
-                .as_bytes(),
-        )
-        .map_err(|_| "The browser profile path contains a NUL byte.".to_owned())?;
-        let data = (&mut *callbacks) as *mut WebCallbacks as *mut c_void;
-        let config = WebConfig {
-            profile_dir: profile.as_ptr(),
-            port: 9333,
-            confirm: Some(web_confirm),
-            confirm_data: data,
-            log: None,
-            log_data: std::ptr::null_mut(),
-            cancel: Some(web_cancel),
-            cancel_data: data,
-        };
-        let web = unsafe { ds4_web_create(&config) };
-        if web.is_null() {
-            Err("Could not initialize browser tools.".into())
-        } else {
-            Ok(Self { web, callbacks })
-        }
-    }
-
-    fn google_search(&self, query: &str, cancel: &AtomicBool) -> Result<String, String> {
-        self.call(query, cancel, ds4_web_google_search)
-    }
-
-    fn visit_page(&self, url: &str, cancel: &AtomicBool) -> Result<String, String> {
-        self.call(url, cancel, ds4_web_visit_page)
-    }
-
-    fn call(
-        &self,
-        value: &str,
-        cancel: &AtomicBool,
-        operation: unsafe extern "C" fn(
-            *mut c_void,
-            *const c_char,
-            *mut c_char,
-            usize,
-        ) -> *mut c_char,
-    ) -> Result<String, String> {
-        let value = CString::new(value).map_err(|_| "web input contains a NUL byte".to_owned())?;
-        let mut error = [0 as c_char; 256];
-        self.callbacks.cancel.store(
-            cancel as *const AtomicBool as *mut AtomicBool,
-            Ordering::Relaxed,
-        );
-        let result =
-            unsafe { operation(self.web, value.as_ptr(), error.as_mut_ptr(), error.len()) };
-        self.callbacks
-            .cancel
-            .store(std::ptr::null_mut(), Ordering::Relaxed);
-        if result.is_null() {
-            let error = unsafe { CStr::from_ptr(error.as_ptr()) }
-                .to_string_lossy()
-                .into_owned();
-            Err(if error.is_empty() {
-                "browser tool failed".into()
-            } else {
-                error
-            })
-        } else {
-            let output = unsafe { CStr::from_ptr(result) }
-                .to_string_lossy()
-                .into_owned();
-            unsafe { free(result.cast()) };
-            Ok(output)
-        }
-    }
-}
-
-impl Drop for Browser {
-    fn drop(&mut self) {
-        unsafe { ds4_web_free(self.web) };
-    }
-}
-
-unsafe extern "C" fn web_confirm(
-    _data: *mut c_void,
-    message: *const c_char,
-    error: *mut c_char,
-    error_len: usize,
-) -> c_int {
-    let _ = message;
-    let _ = error;
-    let _ = error_len;
-    // Approval is obtained by the shared tool worker before entering C. Keeping
-    // this callback lets the reference browser retain its launch guard.
-    1
-}
-
-unsafe extern "C" fn web_cancel(data: *mut c_void) -> bool {
-    if data.is_null() {
-        return false;
-    }
-    let callbacks = unsafe { &*(data.cast::<WebCallbacks>()) };
-    let cancel = callbacks.cancel.load(Ordering::Relaxed);
-    !cancel.is_null() && unsafe { (*cancel).load(Ordering::Relaxed) }
-}
-
-const TOOL_SCHEMAS: &str = r#"{"type":"function","function":{"name":"google_search","description":"Search Google in a visible browser and return compact Markdown links.","parameters":{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}}}
-{"type":"function","function":{"name":"visit_page","description":"Open a URL in a visible browser and return rendered page text.","parameters":{"type":"object","properties":{"url":{"type":"string"}},"required":["url"]}}}
+const TOOL_SCHEMAS: &str = r#"{"type":"function","function":{"name":"google_search","description":"Search Google in a browser and return compact Markdown links.","parameters":{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}}}
+{"type":"function","function":{"name":"visit_page","description":"Open a URL in a browser and return rendered page text.","parameters":{"type":"object","properties":{"url":{"type":"string"}},"required":["url"]}}}
 {"type":"function","function":{"name":"bash","description":"Run a shell command.","parameters":{"type":"object","properties":{"command":{"type":"string"},"timeout_sec":{"type":"number"},"refresh_sec":{"type":"number"}},"required":["command"]}}}
 {"type":"function","function":{"name":"bash_status","description":"Report current status and new output for a bash job.","parameters":{"type":"object","properties":{"job":{"type":"number"},"pid":{"type":"number"},"refresh_sec":{"type":"number"}},"required":["job"]}}}
 {"type":"function","function":{"name":"bash_stop","description":"Terminate a running bash job and report its final output.","parameters":{"type":"object","properties":{"job":{"type":"number"},"pid":{"type":"number"},"refresh_sec":{"type":"number"}},"required":["job"]}}}
@@ -315,7 +167,7 @@ impl Tools {
             more_text: None,
             jobs: HashMap::new(),
             next_job: 1,
-            browser: Browser::new()?,
+            browser: Browser::new(),
             dev_brain: None,
         })
     }
@@ -944,8 +796,8 @@ impl Tools {
     fn approval(&self, call: &ToolCall) -> Option<ApprovalPrompt> {
         if matches!(call.name.as_str(), "google_search" | "visit_page") {
             return Some(ApprovalPrompt {
-                title: "Allow visible browser?".into(),
-                detail: "Start a visible Chrome browser for this tool call.".into(),
+                title: "Allow browser?".into(),
+                detail: "Start headless Chrome for this tool call.".into(),
                 working_directory: self.root.clone(),
             });
         }
@@ -1345,7 +1197,7 @@ pub(crate) fn system_prompt(model: ModelChoice, extra: &str, dev_brain: bool) ->
         )
     } else {
         format!(
-            "You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n## Tools\n\nInvoke native DSML tools exactly as:\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\n\nTool calls are not allowed inside <think></think>. String parameters use raw text and string=\"true\"; numbers and booleans use JSON text and string=\"false\". Read defaults to a bounded chunk; use more to continue and whole=true only when needed. Use write for new files or whole-file replacement. Use edit with path first and exact unique old text; old may contain one [upto] marker between unique head and tail anchors. For long bash commands pass refresh_sec, then use bash_status or bash_stop. The first web call asks permission to start visible Chrome.\n\n### Available Tool Schemas\n\n{schemas}\n\n# Rules\n- Always use strict DSML syntax.\n- Use read/search to get anchors before editing.\n- Preserve the current system configuration unless explicitly asked otherwise."
+            "You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n## Tools\n\nInvoke native DSML tools exactly as:\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\n\nTool calls are not allowed inside <think></think>. String parameters use raw text and string=\"true\"; numbers and booleans use JSON text and string=\"false\". Read defaults to a bounded chunk; use more to continue and whole=true only when needed. Use write for new files or whole-file replacement. Use edit with path first and exact unique old text; old may contain one [upto] marker between unique head and tail anchors. For long bash commands pass refresh_sec, then use bash_status or bash_stop. The first web call asks permission to start headless Chrome.\n\n### Available Tool Schemas\n\n{schemas}\n\n# Rules\n- Always use strict DSML syntax.\n- Use read/search to get anchors before editing.\n- Preserve the current system configuration unless explicitly asked otherwise."
         )
     };
     let tools = if dev_brain {

src/agent/web.rs

@@ -0,0 +1,480 @@
+use headless_chrome::browser::{default_executable, tab::Tab};
+use headless_chrome::protocol::cdp::{Emulation, Runtime};
+use headless_chrome::{Browser as Chrome, LaunchOptions};
+use serde_json::Value;
+use std::ffi::OsStr;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, mpsc};
+use std::thread;
+use std::time::Duration;
+
+const PORT: u16 = 9333;
+const CONNECT_TIMEOUT: Duration = Duration::from_secs(3);
+const CDP_TIMEOUT: Duration = Duration::from_secs(20);
+const POLL_INTERVAL: Duration = Duration::from_millis(50);
+
+const CLICK_GOOGLE_CONSENT_JS: &str = r#"(() => {
+const clean=s=>(s||'').replace(/\s+/g,' ').trim();
+const pats=[/accept all/i,/i agree/i,/agree/i,/accetta tutto/i,/tout accepter/i,/aceptar todo/i,/alle akzeptieren/i];
+const els=[...document.querySelectorAll('button,[role=button],input[type=submit],a')];
+for (const el of els){const t=clean(el.innerText||el.value||el.textContent);if(!t)continue;if(pats.some(p=>p.test(t))){el.click();return 'clicked '+t;}}
+return '';
+})()"#;
+
+const EXTRACT_SEARCH_JS: &str = r#"(() => {
+const clean=s=>(s||'').replace(/\s+/g,' ').trim();
+const esc=s=>clean(s).replace(/\\/g,'\\\\').replace(/\[/g,'\\[').replace(/\]/g,'\\]').replace(/\n/g,' ');
+const visible=el=>{const r=el.getBoundingClientRect();const st=getComputedStyle(el);return r.width>0&&r.height>0&&st.display!=='none'&&st.visibility!=='hidden'&&st.opacity!=='0';};
+const bad=h=>(/(^|\.)google\./.test(h)||/(^|\.)gstatic\./.test(h)||/(^|\.)googleusercontent\./.test(h));
+const lines=['# Google search results','',`URL: ${location.href}`,'','## Visible links'];
+const seen=new Set();
+for(const a of document.querySelectorAll('a[href]')){if(!visible(a))continue;let href=a.href||'';try{const u=new URL(href);if(u.pathname==='/url'&&u.searchParams.get('q'))href=u.searchParams.get('q');}catch{}let u;try{u=new URL(href);}catch{continue;}if(!/^https?:$/.test(u.protocol))continue;if(bad(u.hostname))continue;const text=esc(a.innerText||a.textContent);if(text.length<3)continue;if(seen.has(u.href))continue;seen.add(u.href);lines.push(`- [${text.slice(0,180)}](${u.href})`);if(seen.size>=20)break;}
+lines.push('','## Text snapshot',clean(document.body.innerText).slice(0,1200));
+return lines.join('\n');
+})()"#;
+
+const EXTRACT_PAGE_JS: &str = r#"(() => {
+const clean=s=>(s||'').replace(/\s+/g,' ').trim();
+const esc=s=>clean(s).replace(/\\/g,'\\\\').replace(/\[/g,'\\[').replace(/\]/g,'\\]').replace(/\n/g,' ');
+const visible=el=>{const r=el.getBoundingClientRect();const st=getComputedStyle(el);return r.width>0&&r.height>0&&st.display!=='none'&&st.visibility!=='hidden'&&st.opacity!=='0';};
+const inline=n=>{if(!n)return'';if(n.nodeType===3)return n.nodeValue;if(n.nodeType!==1)return'';const el=n;if(el.tagName==='SCRIPT'||el.tagName==='STYLE'||el.tagName==='NOSCRIPT')return'';if(el.tagName==='A'){const t=esc(el.innerText||el.textContent);const h=el.href||'';return t&&h?`[${t}](${h})`:t;}if(el.tagName==='CODE')return '`'+clean(el.innerText||el.textContent).replace(/`/g,'\\`')+'`';return [...el.childNodes].map(inline).join('');};
+const lines=[`# ${clean(document.title)||location.href}`,'',`URL: ${location.href}`,'','## Content'];
+const blocks=[...document.body.querySelectorAll('h1,h2,h3,h4,h5,h6,p,li,pre,blockquote,td,th,[id="content-text"],[class*="comment-body"],[class*="comment-content"],[data-testid*="comment-text"]')];
+const seen=new Set();
+for(const el of blocks){if(!visible(el))continue;let s='';const tag=el.tagName;if(/^H[1-6]$/.test(tag)){s='#'.repeat(Number(tag[1]))+' '+inline(el);}else if(tag==='LI'){s='- '+inline(el);}else if(tag==='PRE'){s='```\n'+(el.innerText||el.textContent||'').trimEnd()+'\n```';}else if(tag==='BLOCKQUOTE'){s='> '+clean(el.innerText||el.textContent);}else{s=inline(el);}s=s.trim();if(!s||seen.has(s))continue;seen.add(s);lines.push('',s);if(lines.join('\n').length>900000){lines.push('','[Content truncated by browser extractor.]');break;}}
+lines.push('','## Visible links');let n=0;const linkSeen=new Set();
+for(const a of document.querySelectorAll('a[href]')){if(!visible(a))continue;const t=esc(a.innerText||a.textContent);if(t.length<3)continue;let u;try{u=new URL(a.href);}catch{continue;}if(!/^https?:$/.test(u.protocol)||linkSeen.has(u.href))continue;linkSeen.add(u.href);lines.push(`- [${t.slice(0,160)}](${u.href})`);if(++n>=80)break;}
+return lines.join('\n');
+})()"#;
+
+const SCROLL_DYNAMIC_PAGE_JS: &str = r#"(() => new Promise(resolve => {
+const root=()=>document.scrollingElement||document.documentElement||document.body;
+const blockSel='h1,h2,h3,h4,h5,h6,p,li,pre,blockquote,td,th,[id="content-text"],[class*="comment-body"],[class*="comment-content"],[data-testid*="comment-text"]';
+const lazySel='[onscroll],[loading="lazy"],[data-src],[data-lazy],[class*="lazy"],[class*="infinite"],[class*="virtual"],[role="feed"],[id*="comment"],[class*="comment"],[data-testid*="comment"]';
+const hookCount=()=>{let n=0;try{if(window.onscroll)n++;if(document.onscroll)n++;if(document.body&&document.body.onscroll)n++;}catch(e){}try{if(typeof getEventListeners==='function'){for(const o of [window,document,document.body]){if(!o)continue;const ev=getEventListeners(o);if(ev&&ev.scroll)n+=ev.scroll.length;}}catch(e){}try{n+=document.querySelectorAll(lazySel).length;}catch(e){}return n;};
+const metrics=()=>{const r=root();return {height:r?r.scrollHeight:0,view:innerHeight||900,y:scrollY||(r&&r.scrollTop)||0,text:((document.body&&document.body.innerText)||'').length,links:document.links?document.links.length:0,blocks:document.body?document.body.querySelectorAll(blockSel).length:0,hooks:hookCount()};};
+const sig=m=>[m.height,m.text,m.links,m.blocks].join('|');
+const grew=(a,b)=>b.height>a.height+20||b.text>a.text+200||b.links>a.links+2||b.blocks>a.blocks+2;
+const scrollOnce=()=>{const r=root();if(!r)return;const h=Math.max(700,Math.floor((innerHeight||900)*0.85));window.scrollTo(0,Math.min(r.scrollHeight,(scrollY||r.scrollTop||0)+h));};
+let last=metrics(),lastSig=sig(last),same=0,steps=0;
+const scrollable=last.height>last.view*1.35;
+if(!scrollable||last.hooks===0){resolve('scroll skipped hooks='+last.hooks+' text='+last.text);return;}
+const tick=()=>{if(steps>=28){resolve('scrolled '+steps+' text='+last.text);return;}const before=last;scrollOnce();steps++;setTimeout(()=>{const now=metrics(),nowSig=sig(now);if(nowSig===lastSig)same++;else same=0;const loaded=grew(before,now);last=now;lastSig=nowSig;if(steps===1&&!loaded){resolve('scroll probe unchanged text='+now.text);return;}const atBottom=now.y+now.view+20>=now.height;if(same>=4||(atBottom&&same>=1)){resolve('scrolled '+steps+' text='+now.text);return;}tick();},900);};
+tick();
+}))()"#;
+
+pub(super) struct Browser {
+    chrome: Option<Chrome>,
+    profile_dir: PathBuf,
+}
+
+impl Browser {
+    pub(super) fn new() -> Self {
+        Self {
+            chrome: None,
+            profile_dir: crate::app::browser_profile_path(),
+        }
+    }
+
+    pub(super) fn google_search(
+        &mut self,
+        query: &str,
+        cancel: &AtomicBool,
+    ) -> Result<String, String> {
+        if query.is_empty() {
+            return Err("google_search requires query".into());
+        }
+        self.run_page(&google_search_url(query), EXTRACT_SEARCH_JS, false, cancel)
+    }
+
+    pub(super) fn visit_page(&mut self, url: &str, cancel: &AtomicBool) -> Result<String, String> {
+        if url.is_empty() {
+            return Err("visit_page requires url".into());
+        }
+        self.run_page(url, EXTRACT_PAGE_JS, true, cancel)
+    }
+
+    fn run_page(
+        &mut self,
+        url: &str,
+        extract: &str,
+        dynamic_scroll: bool,
+        cancel: &AtomicBool,
+    ) -> Result<String, String> {
+        let chrome = self.ensure_browser(cancel)?.clone();
+        let tab = new_tab(chrome, cancel)?;
+        tab.set_default_timeout(CDP_TIMEOUT);
+
+        let result = (|| {
+            prepare_page(&tab, cancel)?;
+            check_cancel(cancel)?;
+            tab.navigate_to(url)
+                .map_err(|error| format!("page navigation failed: {error}"))?;
+            wait_navigated_ready(&tab, cancel)?;
+
+            if let Ok(clicked) = evaluate_string(&tab, CLICK_GOOGLE_CONSENT_JS, cancel)
+                && !clicked.is_empty()
+            {
+                sleep(cancel, Duration::from_millis(1500))?;
+                wait_navigated_ready(&tab, cancel)?;
+            }
+            if dynamic_scroll {
+                scroll_dynamic_page(&tab, cancel)?;
+            }
+            evaluate_string(&tab, extract, cancel)
+        })();
+
+        close_tab(tab, cancel.load(Ordering::Relaxed));
+        result
+    }
+
+    fn ensure_browser(&mut self, cancel: &AtomicBool) -> Result<&Chrome, String> {
+        if self.chrome.is_none() {
+            let profile_dir = self.profile_dir.clone();
+            self.chrome = Some(background(cancel, move || start_browser(profile_dir))?);
+        }
+        Ok(self.chrome.as_ref().unwrap())
+    }
+}
+
+fn start_browser(profile_dir: PathBuf) -> Result<Chrome, String> {
+    if let Some(browser) = connect_existing() {
+        return Ok(browser);
+    }
+    if profile_dir.to_str().is_none() {
+        return Err("The browser profile path is not valid UTF-8.".into());
+    }
+    std::fs::create_dir_all(&profile_dir)
+        .map_err(|error| format!("failed to create Chrome profile dir: {error}"))?;
+    let executable = std::env::var_os("DS4_CHROME")
+        .filter(|path| !path.is_empty())
+        .map(PathBuf::from)
+        .map_or_else(default_executable, Ok)
+        .map_err(|error| format!("could not find Chrome: {error}"))?;
+    let args = [
+        OsStr::new("--remote-allow-origins=*"),
+        OsStr::new("--no-default-browser-check"),
+        OsStr::new("--disable-sync"),
+        OsStr::new("--use-mock-keychain"),
+        OsStr::new("--password-store=basic"),
+        OsStr::new("--mute-audio"),
+        OsStr::new("about:blank"),
+    ];
+    let options = LaunchOptions::default_builder()
+        .headless(true)
+        .path(Some(executable))
+        .user_data_dir(Some(profile_dir))
+        .port(Some(PORT))
+        .idle_browser_timeout(CDP_TIMEOUT)
+        .ignore_certificate_errors(false)
+        .enable_gpu(true)
+        .disable_default_args(true)
+        .args(args.to_vec())
+        .build()
+        .map_err(|error| format!("could not configure Chrome: {error}"))?;
+    Chrome::new(options).map_err(|error| format!("could not start Chrome: {error}"))
+}
+
+fn connect_existing() -> Option<Chrome> {
+    let agent: ureq::Agent = ureq::Agent::config_builder()
+        .timeout_connect(Some(CONNECT_TIMEOUT))
+        .timeout_recv_response(Some(CONNECT_TIMEOUT))
+        .timeout_recv_body(Some(CONNECT_TIMEOUT))
+        .build()
+        .into();
+    let mut response = agent
+        .get(&format!("http://127.0.0.1:{PORT}/json/version"))
+        .call()
+        .ok()?;
+    let body = response.body_mut().read_to_string().ok()?;
+    let websocket = serde_json::from_str::<Value>(&body)
+        .ok()?
+        .get("webSocketDebuggerUrl")?
+        .as_str()?
+        .to_owned();
+    Chrome::connect_with_timeout(websocket, CDP_TIMEOUT).ok()
+}
+
+fn new_tab(chrome: Chrome, cancel: &AtomicBool) -> Result<Arc<Tab>, String> {
+    let (sender, receiver) = mpsc::sync_channel(1);
+    thread::spawn(move || {
+        let result = chrome
+            .new_tab()
+            .map_err(|error| format!("could not open browser tab: {error}"));
+        if let Err(mpsc::SendError(Ok(tab))) = sender.send(result) {
+            let _ = tab.close_target();
+        }
+    });
+    wait(receiver, cancel)
+}
+
+fn prepare_page(tab: &Arc<Tab>, cancel: &AtomicBool) -> Result<(), String> {
+    check_cancel(cancel)?;
+    tab.call_method(Runtime::Enable(None))
+        .map_err(|error| format!("could not enable page runtime: {error}"))?;
+    let _ = tab.call_method(Emulation::SetFocusEmulationEnabled { enabled: true });
+    let _ = tab.call_method(Emulation::SetDeviceMetricsOverride {
+        width: 1365,
+        height: 900,
+        device_scale_factor: 1.0,
+        mobile: false,
+        scale: None,
+        screen_width: None,
+        screen_height: None,
+        position_x: None,
+        position_y: None,
+        dont_set_visible_size: None,
+        screen_orientation: None,
+        viewport: None,
+        display_feature: None,
+        device_posture: None,
+    });
+    wait_ready(tab, cancel)
+}
+
+fn wait_ready(tab: &Arc<Tab>, cancel: &AtomicBool) -> Result<(), String> {
+    for _ in 0..80 {
+        check_cancel(cancel)?;
+        if let Ok(state) = evaluate_string(tab, "document.readyState", cancel)
+            && matches!(state.as_str(), "complete" | "interactive")
+        {
+            return sleep(cancel, Duration::from_millis(800));
+        }
+        sleep(cancel, Duration::from_millis(250))?;
+    }
+    Ok(())
+}
+
+fn wait_navigated_ready(tab: &Arc<Tab>, cancel: &AtomicBool) -> Result<(), String> {
+    let mut last_len = -1;
+    let mut stable = 0;
+    let mut saw_real_url = false;
+    for iteration in 0..100 {
+        check_cancel(cancel)?;
+        let Ok(probe) = evaluate_string(
+            tab,
+            "location.href+'\\n'+document.readyState+'\\n'+((document.body&&document.body.innerText)||'').length",
+            cancel,
+        ) else {
+            sleep(cancel, Duration::from_millis(250))?;
+            continue;
+        };
+        let Ok((href, ready, text_len)) = parse_probe(&probe) else {
+            sleep(cancel, Duration::from_millis(250))?;
+            continue;
+        };
+        let real_url = !href.is_empty() && href != "about:blank" && !href.starts_with("chrome://");
+        let ready = matches!(ready, "complete" | "interactive");
+        saw_real_url |= real_url;
+        if text_len > 0 && text_len == last_len {
+            stable += 1;
+        } else {
+            stable = 0;
+        }
+        last_len = text_len;
+        if saw_real_url && ready && text_len > 0 && stable >= 2 {
+            return sleep(cancel, Duration::from_millis(500));
+        }
+        if saw_real_url && ready && iteration >= 24 {
+            return Ok(());
+        }
+        sleep(cancel, Duration::from_millis(250))?;
+    }
+    Ok(())
+}
+
+fn scroll_dynamic_page(tab: &Arc<Tab>, cancel: &AtomicBool) -> Result<(), String> {
+    check_cancel(cancel)?;
+    let result = evaluate_string(tab, SCROLL_DYNAMIC_PAGE_JS, cancel);
+    check_cancel(cancel)?;
+    match result {
+        Err(error) if error == "interrupted" => Err(error),
+        _ => Ok(()),
+    }
+}
+
+fn evaluate_string(
+    tab: &Arc<Tab>,
+    expression: &str,
+    cancel: &AtomicBool,
+) -> Result<String, String> {
+    let tab = Arc::clone(tab);
+    let expression = expression.to_owned();
+    background(cancel, move || {
+        let response = tab
+            .call_method(Runtime::Evaluate {
+                expression,
+                object_group: None,
+                include_command_line_api: Some(true),
+                silent: None,
+                context_id: None,
+                return_by_value: Some(true),
+                generate_preview: None,
+                user_gesture: None,
+                await_promise: Some(true),
+                throw_on_side_effect: None,
+                timeout: None,
+                disable_breaks: None,
+                repl_mode: None,
+                allow_unsafe_eval_blocked_by_csp: None,
+                unique_context_id: None,
+                serialization_options: None,
+            })
+            .map_err(|error| format!("JavaScript evaluation failed: {error}"))?;
+        if response.exception_details.is_some() {
+            return Err("JavaScript evaluation failed".into());
+        }
+        response
+            .result
+            .value
+            .and_then(|value| value.as_str().map(str::to_owned))
+            .ok_or_else(|| "Runtime.evaluate did not return a string".into())
+    })
+}
+
+fn background<T, F>(cancel: &AtomicBool, operation: F) -> Result<T, String>
+where
+    T: Send + 'static,
+    F: FnOnce() -> Result<T, String> + Send + 'static,
+{
+    let (sender, receiver) = mpsc::sync_channel(1);
+    thread::spawn(move || {
+        let _ = sender.send(operation());
+    });
+    wait(receiver, cancel)
+}
+
+fn wait<T>(receiver: mpsc::Receiver<Result<T, String>>, cancel: &AtomicBool) -> Result<T, String> {
+    loop {
+        match receiver.recv_timeout(POLL_INTERVAL) {
+            Ok(result) => return result,
+            Err(mpsc::RecvTimeoutError::Timeout) => check_cancel(cancel)?,
+            Err(mpsc::RecvTimeoutError::Disconnected) => {
+                return Err("browser worker stopped unexpectedly".into());
+            }
+        }
+    }
+}
+
+fn close_tab(tab: Arc<Tab>, background: bool) {
+    if background {
+        thread::spawn(move || {
+            let _ = tab.close_target();
+        });
+    } else {
+        let _ = tab.close_target();
+    }
+}
+
+fn check_cancel(cancel: &AtomicBool) -> Result<(), String> {
+    if cancel.load(Ordering::Relaxed) {
+        Err("interrupted".into())
+    } else {
+        Ok(())
+    }
+}
+
+fn sleep(cancel: &AtomicBool, duration: Duration) -> Result<(), String> {
+    let mut remaining = duration;
+    while !remaining.is_zero() {
+        check_cancel(cancel)?;
+        let step = remaining.min(POLL_INTERVAL);
+        thread::sleep(step);
+        remaining = remaining.saturating_sub(step);
+    }
+    check_cancel(cancel)
+}
+
+fn parse_probe(probe: &str) -> Result<(&str, &str, i64), String> {
+    let mut parts = probe.splitn(3, '\n');
+    let href = parts.next().unwrap_or_default();
+    let ready = parts
+        .next()
+        .ok_or_else(|| "page readiness probe returned malformed data".to_owned())?;
+    let text_len = parts
+        .next()
+        .ok_or_else(|| "page readiness probe returned malformed data".to_owned())?
+        .parse()
+        .unwrap_or(0);
+    Ok((href, ready, text_len))
+}
+
+fn google_search_url(query: &str) -> String {
+    const HEX: &[u8; 16] = b"0123456789ABCDEF";
+    let mut url = String::from("https://www.google.com/search?q=");
+    for byte in query.bytes() {
+        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
+            url.push(char::from(byte));
+        } else {
+            url.push('%');
+            url.push(char::from(HEX[(byte >> 4) as usize]));
+            url.push(char::from(HEX[(byte & 15) as usize]));
+        }
+    }
+    url
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::{Read, Write};
+    use std::net::TcpListener;
+
+    #[test]
+    fn browser_helpers_preserve_ds4_wire_values() {
+        assert_eq!(
+            google_search_url("café & Rust~"),
+            "https://www.google.com/search?q=caf%C3%A9%20%26%20Rust~"
+        );
+        assert_eq!(
+            parse_probe("https://example.com\ncomplete\n42").unwrap(),
+            ("https://example.com", "complete", 42)
+        );
+        assert!(parse_probe("incomplete").is_err());
+    }
+
+    #[test]
+    #[ignore = "requires an installed Chrome browser"]
+    fn headless_browser_extracts_a_local_page() {
+        let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = thread::spawn(move || {
+            let (mut stream, _) = listener.accept().unwrap();
+            let mut request = [0; 4096];
+            let _ = stream.read(&mut request);
+            let body = "<title>Fixture</title><h1>Hello</h1><p>Browser works.</p>";
+            write!(
+                stream,
+                "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                body.len()
+            )
+            .unwrap();
+        });
+        let profile_dir = std::env::temp_dir().join(format!(
+            "ds4-browser-test-{}",
+            std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_nanos()
+        ));
+        let mut browser = Browser {
+            chrome: None,
+            profile_dir: profile_dir.clone(),
+        };
+        let output = browser
+            .run_page(
+                &format!("http://{address}"),
+                EXTRACT_PAGE_JS,
+                false,
+                &AtomicBool::new(false),
+            )
+            .unwrap();
+        assert!(output.contains("# Fixture"));
+        assert!(output.contains("# Hello"));
+        assert!(output.contains("Browser works."));
+        drop(browser);
+        server.join().unwrap();
+        std::fs::remove_dir_all(profile_dir).unwrap();
+    }
+}