Replace C browser client with headless Chrome
This commit is contained in:
Georg Bauer
289
Cargo.lock
generated
289
Cargo.lock
generated
@@ -89,6 +89,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "arrayref"
|
||||
version = "0.3.9"
|
||||
@@ -127,7 +133,7 @@ dependencies = [
|
||||
"enumflags2",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"rand",
|
||||
"rand 0.9.5",
|
||||
"raw-window-handle",
|
||||
"serde",
|
||||
"serde_repr",
|
||||
@@ -320,6 +326,20 @@ version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
|
||||
|
||||
[[package]]
|
||||
name = "auto_generate_cdp"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "359220d0b9360b79d17d648d0a3ba1e792ec36bdbc227c8fd0351df3a0415704"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
@@ -377,6 +397,15 @@ version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.12.1"
|
||||
@@ -443,6 +472,12 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder-lite"
|
||||
version = "0.1.0"
|
||||
@@ -565,6 +600,17 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
|
||||
|
||||
[[package]]
|
||||
name = "chacha20"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.0",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.45"
|
||||
@@ -660,6 +706,15 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
|
||||
|
||||
[[package]]
|
||||
name = "convert_case"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.4"
|
||||
@@ -754,6 +809,15 @@ dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.3.0"
|
||||
@@ -806,6 +870,16 @@ dependencies = [
|
||||
"wgpu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.2.2"
|
||||
@@ -830,14 +904,38 @@ version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f"
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
|
||||
dependencies = [
|
||||
"darling_core 0.20.11",
|
||||
"darling_macro 0.20.11",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.21.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
"darling_core 0.21.3",
|
||||
"darling_macro 0.21.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -854,17 +952,34 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
|
||||
dependencies = [
|
||||
"darling_core 0.20.11",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.21.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_core 0.21.3",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "data-encoding"
|
||||
version = "2.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
|
||||
|
||||
[[package]]
|
||||
name = "data-url"
|
||||
version = "0.3.2"
|
||||
@@ -877,6 +992,37 @@ version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder"
|
||||
version = "0.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
|
||||
dependencies = [
|
||||
"derive_builder_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_core"
|
||||
version = "0.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
|
||||
dependencies = [
|
||||
"darling 0.20.11",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_macro"
|
||||
version = "0.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
|
||||
dependencies = [
|
||||
"derive_builder_core",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "diesel"
|
||||
version = "2.3.11"
|
||||
@@ -923,15 +1069,25 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"crypto-common 0.1.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"block-buffer 0.12.1",
|
||||
"const-oid",
|
||||
"crypto-common",
|
||||
"crypto-common 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1028,6 +1184,7 @@ dependencies = [
|
||||
"diesel",
|
||||
"diesel_migrations",
|
||||
"git2",
|
||||
"headless_chrome",
|
||||
"iced",
|
||||
"image",
|
||||
"memmap2",
|
||||
@@ -1051,7 +1208,7 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd122633e4bef06db27737f21d3738fb89c8f6d5360d6d9d7635dda142a7757e"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"darling 0.21.3",
|
||||
"either",
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
@@ -1472,6 +1629,16 @@ dependencies = [
|
||||
"system-deps",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gethostname"
|
||||
version = "1.1.0"
|
||||
@@ -1523,6 +1690,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi 6.0.0",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1850,6 +2018,29 @@ version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
|
||||
[[package]]
|
||||
name = "headless_chrome"
|
||||
version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5754ca220578ad74a5d5174c8ca2ff956fd3b94025f870844de1e910b47dfee5"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"auto_generate_cdp",
|
||||
"base64",
|
||||
"derive_builder",
|
||||
"log",
|
||||
"rand 0.10.2",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"tungstenite",
|
||||
"url",
|
||||
"which",
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
@@ -3663,7 +3854,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
|
||||
dependencies = [
|
||||
"chacha20",
|
||||
"getrandom 0.4.3",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3673,7 +3875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3685,6 +3887,12 @@ dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||
|
||||
[[package]]
|
||||
name = "range-alloc"
|
||||
version = "0.1.5"
|
||||
@@ -4122,6 +4330,17 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"digest 0.10.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.11.0"
|
||||
@@ -4129,8 +4348,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
"cpufeatures 0.3.0",
|
||||
"digest 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4308,6 +4527,17 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socks"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "softbuffer"
|
||||
version = "0.4.8"
|
||||
@@ -4828,6 +5058,22 @@ dependencies = [
|
||||
"core_maths",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"data-encoding",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.9.5",
|
||||
"sha1",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "turbovault-core"
|
||||
version = "1.6.0"
|
||||
@@ -4976,10 +5222,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"flate2",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"socks",
|
||||
"ureq-proto",
|
||||
"utf8-zero",
|
||||
"webpki-roots",
|
||||
@@ -5500,6 +5748,15 @@ dependencies = [
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "8.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f3ef584124b911bcc3875c2f1472e80f24361ceb789bd1c62b3e9a3df9ff43c"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
@@ -5886,6 +6143,16 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.56.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d6f32a0ff4a9f6f01231eb2059cc85479330739333e0e58cadf03b6af2cca10"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
version = "0.57.1"
|
||||
|
||||
@@ -15,6 +15,7 @@ cc = "1.3.0"
|
||||
diesel = { version = "2.3.11", features = ["sqlite", "returning_clauses_for_sqlite_3_35", "64-column-tables"] }
|
||||
diesel_migrations = "2.3.2"
|
||||
git2 = { version = "0.20.4", features = ["vendored-libgit2", "vendored-openssl"] }
|
||||
headless_chrome = "1.0.22"
|
||||
iced = { version = "0.14.0", features = ["advanced", "highlighter", "image-without-codecs", "markdown", "svg", "tokio"] }
|
||||
image = { version = "0.25.10", default-features = false, features = ["gif", "jpeg", "png", "webp"] }
|
||||
memmap2 = "0.9.11"
|
||||
|
||||
6
build.rs
6
build.rs
@@ -18,12 +18,6 @@ fn main() {
|
||||
.file("native/media/ds4_media.m")
|
||||
.flag("-fobjc-arc")
|
||||
.compile("ds4_media");
|
||||
println!("cargo:rerun-if-changed=native/web");
|
||||
cc::Build::new()
|
||||
.include("native/web")
|
||||
.file("native/web/ds4_web.c")
|
||||
.opt_level(2)
|
||||
.compile("ds4_web");
|
||||
println!("cargo:rustc-link-lib=framework=Foundation");
|
||||
println!("cargo:rustc-link-lib=framework=AppKit");
|
||||
println!("cargo:rustc-link-lib=framework=AVFoundation");
|
||||
|
||||
1382
native/web/ds4_web.c
1382
native/web/ds4_web.c
File diff suppressed because it is too large
Load Diff
@@ -1,33 +0,0 @@
|
||||
#ifndef DS4_WEB_H
|
||||
#define DS4_WEB_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
typedef int (*ds4_web_confirm_fn)(void *privdata, const char *message,
|
||||
char *err, size_t err_len);
|
||||
typedef void (*ds4_web_log_fn)(void *privdata, const char *message);
|
||||
typedef bool (*ds4_web_cancel_fn)(void *privdata);
|
||||
|
||||
typedef struct {
|
||||
const char *profile_dir;
|
||||
int port;
|
||||
ds4_web_confirm_fn confirm;
|
||||
void *confirm_privdata;
|
||||
ds4_web_log_fn log;
|
||||
void *log_privdata;
|
||||
ds4_web_cancel_fn cancel;
|
||||
void *cancel_privdata;
|
||||
} ds4_web_config;
|
||||
|
||||
typedef struct ds4_web ds4_web;
|
||||
|
||||
ds4_web *ds4_web_create(const ds4_web_config *cfg);
|
||||
void ds4_web_free(ds4_web *web);
|
||||
|
||||
char *ds4_web_google_search(ds4_web *web, const char *query,
|
||||
char *err, size_t err_len);
|
||||
char *ds4_web_visit_page(ds4_web *web, const char *url,
|
||||
char *err, size_t err_len);
|
||||
|
||||
#endif
|
||||
168
src/agent.rs
168
src/agent.rs
@@ -1,12 +1,14 @@
|
||||
mod web;
|
||||
|
||||
use self::web::Browser;
|
||||
use crate::model::ModelChoice;
|
||||
use serde_json::{Map, Value};
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::{CStr, CString, c_char, c_int, c_void};
|
||||
use std::fs::{self, File};
|
||||
use std::os::unix::process::CommandExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender, TryRecvError};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
@@ -30,158 +32,8 @@ const SHELL_ENV_ALLOWLIST: &[&str] = &[
|
||||
];
|
||||
pub(crate) const COMPACTION_OBSERVATION_PREFIX: &str = "Bash job update after context compaction.";
|
||||
|
||||
#[repr(C)]
|
||||
struct WebConfig {
|
||||
profile_dir: *const c_char,
|
||||
port: c_int,
|
||||
confirm: Option<unsafe extern "C" fn(*mut c_void, *const c_char, *mut c_char, usize) -> c_int>,
|
||||
confirm_data: *mut c_void,
|
||||
log: Option<unsafe extern "C" fn(*mut c_void, *const c_char)>,
|
||||
log_data: *mut c_void,
|
||||
cancel: Option<unsafe extern "C" fn(*mut c_void) -> bool>,
|
||||
cancel_data: *mut c_void,
|
||||
}
|
||||
|
||||
unsafe extern "C" {
|
||||
fn ds4_web_create(config: *const WebConfig) -> *mut c_void;
|
||||
fn ds4_web_free(web: *mut c_void);
|
||||
fn ds4_web_google_search(
|
||||
web: *mut c_void,
|
||||
query: *const c_char,
|
||||
error: *mut c_char,
|
||||
error_len: usize,
|
||||
) -> *mut c_char;
|
||||
fn ds4_web_visit_page(
|
||||
web: *mut c_void,
|
||||
url: *const c_char,
|
||||
error: *mut c_char,
|
||||
error_len: usize,
|
||||
) -> *mut c_char;
|
||||
fn free(pointer: *mut c_void);
|
||||
}
|
||||
|
||||
struct WebCallbacks {
|
||||
cancel: AtomicPtr<AtomicBool>,
|
||||
}
|
||||
|
||||
struct Browser {
|
||||
web: *mut c_void,
|
||||
callbacks: Box<WebCallbacks>,
|
||||
}
|
||||
|
||||
// The C browser is used by one agent-tools worker at a time under Tools' mutex.
|
||||
unsafe impl Send for Browser {}
|
||||
|
||||
impl Browser {
|
||||
fn new() -> Result<Self, String> {
|
||||
let mut callbacks = Box::new(WebCallbacks {
|
||||
cancel: AtomicPtr::new(std::ptr::null_mut()),
|
||||
});
|
||||
let profile = CString::new(
|
||||
crate::app::browser_profile_path()
|
||||
.to_string_lossy()
|
||||
.as_bytes(),
|
||||
)
|
||||
.map_err(|_| "The browser profile path contains a NUL byte.".to_owned())?;
|
||||
let data = (&mut *callbacks) as *mut WebCallbacks as *mut c_void;
|
||||
let config = WebConfig {
|
||||
profile_dir: profile.as_ptr(),
|
||||
port: 9333,
|
||||
confirm: Some(web_confirm),
|
||||
confirm_data: data,
|
||||
log: None,
|
||||
log_data: std::ptr::null_mut(),
|
||||
cancel: Some(web_cancel),
|
||||
cancel_data: data,
|
||||
};
|
||||
let web = unsafe { ds4_web_create(&config) };
|
||||
if web.is_null() {
|
||||
Err("Could not initialize browser tools.".into())
|
||||
} else {
|
||||
Ok(Self { web, callbacks })
|
||||
}
|
||||
}
|
||||
|
||||
fn google_search(&self, query: &str, cancel: &AtomicBool) -> Result<String, String> {
|
||||
self.call(query, cancel, ds4_web_google_search)
|
||||
}
|
||||
|
||||
fn visit_page(&self, url: &str, cancel: &AtomicBool) -> Result<String, String> {
|
||||
self.call(url, cancel, ds4_web_visit_page)
|
||||
}
|
||||
|
||||
fn call(
|
||||
&self,
|
||||
value: &str,
|
||||
cancel: &AtomicBool,
|
||||
operation: unsafe extern "C" fn(
|
||||
*mut c_void,
|
||||
*const c_char,
|
||||
*mut c_char,
|
||||
usize,
|
||||
) -> *mut c_char,
|
||||
) -> Result<String, String> {
|
||||
let value = CString::new(value).map_err(|_| "web input contains a NUL byte".to_owned())?;
|
||||
let mut error = [0 as c_char; 256];
|
||||
self.callbacks.cancel.store(
|
||||
cancel as *const AtomicBool as *mut AtomicBool,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
let result =
|
||||
unsafe { operation(self.web, value.as_ptr(), error.as_mut_ptr(), error.len()) };
|
||||
self.callbacks
|
||||
.cancel
|
||||
.store(std::ptr::null_mut(), Ordering::Relaxed);
|
||||
if result.is_null() {
|
||||
let error = unsafe { CStr::from_ptr(error.as_ptr()) }
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
Err(if error.is_empty() {
|
||||
"browser tool failed".into()
|
||||
} else {
|
||||
error
|
||||
})
|
||||
} else {
|
||||
let output = unsafe { CStr::from_ptr(result) }
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
unsafe { free(result.cast()) };
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Browser {
|
||||
fn drop(&mut self) {
|
||||
unsafe { ds4_web_free(self.web) };
|
||||
}
|
||||
}
|
||||
|
||||
unsafe extern "C" fn web_confirm(
|
||||
_data: *mut c_void,
|
||||
message: *const c_char,
|
||||
error: *mut c_char,
|
||||
error_len: usize,
|
||||
) -> c_int {
|
||||
let _ = message;
|
||||
let _ = error;
|
||||
let _ = error_len;
|
||||
// Approval is obtained by the shared tool worker before entering C. Keeping
|
||||
// this callback lets the reference browser retain its launch guard.
|
||||
1
|
||||
}
|
||||
|
||||
unsafe extern "C" fn web_cancel(data: *mut c_void) -> bool {
|
||||
if data.is_null() {
|
||||
return false;
|
||||
}
|
||||
let callbacks = unsafe { &*(data.cast::<WebCallbacks>()) };
|
||||
let cancel = callbacks.cancel.load(Ordering::Relaxed);
|
||||
!cancel.is_null() && unsafe { (*cancel).load(Ordering::Relaxed) }
|
||||
}
|
||||
|
||||
const TOOL_SCHEMAS: &str = r#"{"type":"function","function":{"name":"google_search","description":"Search Google in a visible browser and return compact Markdown links.","parameters":{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}}}
|
||||
{"type":"function","function":{"name":"visit_page","description":"Open a URL in a visible browser and return rendered page text.","parameters":{"type":"object","properties":{"url":{"type":"string"}},"required":["url"]}}}
|
||||
const TOOL_SCHEMAS: &str = r#"{"type":"function","function":{"name":"google_search","description":"Search Google in a browser and return compact Markdown links.","parameters":{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}}}
|
||||
{"type":"function","function":{"name":"visit_page","description":"Open a URL in a browser and return rendered page text.","parameters":{"type":"object","properties":{"url":{"type":"string"}},"required":["url"]}}}
|
||||
{"type":"function","function":{"name":"bash","description":"Run a shell command.","parameters":{"type":"object","properties":{"command":{"type":"string"},"timeout_sec":{"type":"number"},"refresh_sec":{"type":"number"}},"required":["command"]}}}
|
||||
{"type":"function","function":{"name":"bash_status","description":"Report current status and new output for a bash job.","parameters":{"type":"object","properties":{"job":{"type":"number"},"pid":{"type":"number"},"refresh_sec":{"type":"number"}},"required":["job"]}}}
|
||||
{"type":"function","function":{"name":"bash_stop","description":"Terminate a running bash job and report its final output.","parameters":{"type":"object","properties":{"job":{"type":"number"},"pid":{"type":"number"},"refresh_sec":{"type":"number"}},"required":["job"]}}}
|
||||
@@ -315,7 +167,7 @@ impl Tools {
|
||||
more_text: None,
|
||||
jobs: HashMap::new(),
|
||||
next_job: 1,
|
||||
browser: Browser::new()?,
|
||||
browser: Browser::new(),
|
||||
dev_brain: None,
|
||||
})
|
||||
}
|
||||
@@ -944,8 +796,8 @@ impl Tools {
|
||||
fn approval(&self, call: &ToolCall) -> Option<ApprovalPrompt> {
|
||||
if matches!(call.name.as_str(), "google_search" | "visit_page") {
|
||||
return Some(ApprovalPrompt {
|
||||
title: "Allow visible browser?".into(),
|
||||
detail: "Start a visible Chrome browser for this tool call.".into(),
|
||||
title: "Allow browser?".into(),
|
||||
detail: "Start headless Chrome for this tool call.".into(),
|
||||
working_directory: self.root.clone(),
|
||||
});
|
||||
}
|
||||
@@ -1345,7 +1197,7 @@ pub(crate) fn system_prompt(model: ModelChoice, extra: &str, dev_brain: bool) ->
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n## Tools\n\nInvoke native DSML tools exactly as:\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\n\nTool calls are not allowed inside <think></think>. String parameters use raw text and string=\"true\"; numbers and booleans use JSON text and string=\"false\". Read defaults to a bounded chunk; use more to continue and whole=true only when needed. Use write for new files or whole-file replacement. Use edit with path first and exact unique old text; old may contain one [upto] marker between unique head and tail anchors. For long bash commands pass refresh_sec, then use bash_status or bash_stop. The first web call asks permission to start visible Chrome.\n\n### Available Tool Schemas\n\n{schemas}\n\n# Rules\n- Always use strict DSML syntax.\n- Use read/search to get anchors before editing.\n- Preserve the current system configuration unless explicitly asked otherwise."
|
||||
"You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n## Tools\n\nInvoke native DSML tools exactly as:\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\n\nTool calls are not allowed inside <think></think>. String parameters use raw text and string=\"true\"; numbers and booleans use JSON text and string=\"false\". Read defaults to a bounded chunk; use more to continue and whole=true only when needed. Use write for new files or whole-file replacement. Use edit with path first and exact unique old text; old may contain one [upto] marker between unique head and tail anchors. For long bash commands pass refresh_sec, then use bash_status or bash_stop. The first web call asks permission to start headless Chrome.\n\n### Available Tool Schemas\n\n{schemas}\n\n# Rules\n- Always use strict DSML syntax.\n- Use read/search to get anchors before editing.\n- Preserve the current system configuration unless explicitly asked otherwise."
|
||||
)
|
||||
};
|
||||
let tools = if dev_brain {
|
||||
|
||||
480
src/agent/web.rs
Normal file
480
src/agent/web.rs
Normal file
@@ -0,0 +1,480 @@
|
||||
use headless_chrome::browser::{default_executable, tab::Tab};
|
||||
use headless_chrome::protocol::cdp::{Emulation, Runtime};
|
||||
use headless_chrome::{Browser as Chrome, LaunchOptions};
|
||||
use serde_json::Value;
|
||||
use std::ffi::OsStr;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, mpsc};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
const PORT: u16 = 9333;
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(3);
|
||||
const CDP_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
const CLICK_GOOGLE_CONSENT_JS: &str = r#"(() => {
|
||||
const clean=s=>(s||'').replace(/\s+/g,' ').trim();
|
||||
const pats=[/accept all/i,/i agree/i,/agree/i,/accetta tutto/i,/tout accepter/i,/aceptar todo/i,/alle akzeptieren/i];
|
||||
const els=[...document.querySelectorAll('button,[role=button],input[type=submit],a')];
|
||||
for (const el of els){const t=clean(el.innerText||el.value||el.textContent);if(!t)continue;if(pats.some(p=>p.test(t))){el.click();return 'clicked '+t;}}
|
||||
return '';
|
||||
})()"#;
|
||||
|
||||
const EXTRACT_SEARCH_JS: &str = r#"(() => {
|
||||
const clean=s=>(s||'').replace(/\s+/g,' ').trim();
|
||||
const esc=s=>clean(s).replace(/\\/g,'\\\\').replace(/\[/g,'\\[').replace(/\]/g,'\\]').replace(/\n/g,' ');
|
||||
const visible=el=>{const r=el.getBoundingClientRect();const st=getComputedStyle(el);return r.width>0&&r.height>0&&st.display!=='none'&&st.visibility!=='hidden'&&st.opacity!=='0';};
|
||||
const bad=h=>(/(^|\.)google\./.test(h)||/(^|\.)gstatic\./.test(h)||/(^|\.)googleusercontent\./.test(h));
|
||||
const lines=['# Google search results','',`URL: ${location.href}`,'','## Visible links'];
|
||||
const seen=new Set();
|
||||
for(const a of document.querySelectorAll('a[href]')){if(!visible(a))continue;let href=a.href||'';try{const u=new URL(href);if(u.pathname==='/url'&&u.searchParams.get('q'))href=u.searchParams.get('q');}catch{}let u;try{u=new URL(href);}catch{continue;}if(!/^https?:$/.test(u.protocol))continue;if(bad(u.hostname))continue;const text=esc(a.innerText||a.textContent);if(text.length<3)continue;if(seen.has(u.href))continue;seen.add(u.href);lines.push(`- [${text.slice(0,180)}](${u.href})`);if(seen.size>=20)break;}
|
||||
lines.push('','## Text snapshot',clean(document.body.innerText).slice(0,1200));
|
||||
return lines.join('\n');
|
||||
})()"#;
|
||||
|
||||
const EXTRACT_PAGE_JS: &str = r#"(() => {
|
||||
const clean=s=>(s||'').replace(/\s+/g,' ').trim();
|
||||
const esc=s=>clean(s).replace(/\\/g,'\\\\').replace(/\[/g,'\\[').replace(/\]/g,'\\]').replace(/\n/g,' ');
|
||||
const visible=el=>{const r=el.getBoundingClientRect();const st=getComputedStyle(el);return r.width>0&&r.height>0&&st.display!=='none'&&st.visibility!=='hidden'&&st.opacity!=='0';};
|
||||
const inline=n=>{if(!n)return'';if(n.nodeType===3)return n.nodeValue;if(n.nodeType!==1)return'';const el=n;if(el.tagName==='SCRIPT'||el.tagName==='STYLE'||el.tagName==='NOSCRIPT')return'';if(el.tagName==='A'){const t=esc(el.innerText||el.textContent);const h=el.href||'';return t&&h?`[${t}](${h})`:t;}if(el.tagName==='CODE')return '`'+clean(el.innerText||el.textContent).replace(/`/g,'\\`')+'`';return [...el.childNodes].map(inline).join('');};
|
||||
const lines=[`# ${clean(document.title)||location.href}`,'',`URL: ${location.href}`,'','## Content'];
|
||||
const blocks=[...document.body.querySelectorAll('h1,h2,h3,h4,h5,h6,p,li,pre,blockquote,td,th,[id="content-text"],[class*="comment-body"],[class*="comment-content"],[data-testid*="comment-text"]')];
|
||||
const seen=new Set();
|
||||
for(const el of blocks){if(!visible(el))continue;let s='';const tag=el.tagName;if(/^H[1-6]$/.test(tag)){s='#'.repeat(Number(tag[1]))+' '+inline(el);}else if(tag==='LI'){s='- '+inline(el);}else if(tag==='PRE'){s='```\n'+(el.innerText||el.textContent||'').trimEnd()+'\n```';}else if(tag==='BLOCKQUOTE'){s='> '+clean(el.innerText||el.textContent);}else{s=inline(el);}s=s.trim();if(!s||seen.has(s))continue;seen.add(s);lines.push('',s);if(lines.join('\n').length>900000){lines.push('','[Content truncated by browser extractor.]');break;}}
|
||||
lines.push('','## Visible links');let n=0;const linkSeen=new Set();
|
||||
for(const a of document.querySelectorAll('a[href]')){if(!visible(a))continue;const t=esc(a.innerText||a.textContent);if(t.length<3)continue;let u;try{u=new URL(a.href);}catch{continue;}if(!/^https?:$/.test(u.protocol)||linkSeen.has(u.href))continue;linkSeen.add(u.href);lines.push(`- [${t.slice(0,160)}](${u.href})`);if(++n>=80)break;}
|
||||
return lines.join('\n');
|
||||
})()"#;
|
||||
|
||||
const SCROLL_DYNAMIC_PAGE_JS: &str = r#"(() => new Promise(resolve => {
|
||||
const root=()=>document.scrollingElement||document.documentElement||document.body;
|
||||
const blockSel='h1,h2,h3,h4,h5,h6,p,li,pre,blockquote,td,th,[id="content-text"],[class*="comment-body"],[class*="comment-content"],[data-testid*="comment-text"]';
|
||||
const lazySel='[onscroll],[loading="lazy"],[data-src],[data-lazy],[class*="lazy"],[class*="infinite"],[class*="virtual"],[role="feed"],[id*="comment"],[class*="comment"],[data-testid*="comment"]';
|
||||
const hookCount=()=>{let n=0;try{if(window.onscroll)n++;if(document.onscroll)n++;if(document.body&&document.body.onscroll)n++;}catch(e){}try{if(typeof getEventListeners==='function'){for(const o of [window,document,document.body]){if(!o)continue;const ev=getEventListeners(o);if(ev&&ev.scroll)n+=ev.scroll.length;}}catch(e){}try{n+=document.querySelectorAll(lazySel).length;}catch(e){}return n;};
|
||||
const metrics=()=>{const r=root();return {height:r?r.scrollHeight:0,view:innerHeight||900,y:scrollY||(r&&r.scrollTop)||0,text:((document.body&&document.body.innerText)||'').length,links:document.links?document.links.length:0,blocks:document.body?document.body.querySelectorAll(blockSel).length:0,hooks:hookCount()};};
|
||||
const sig=m=>[m.height,m.text,m.links,m.blocks].join('|');
|
||||
const grew=(a,b)=>b.height>a.height+20||b.text>a.text+200||b.links>a.links+2||b.blocks>a.blocks+2;
|
||||
const scrollOnce=()=>{const r=root();if(!r)return;const h=Math.max(700,Math.floor((innerHeight||900)*0.85));window.scrollTo(0,Math.min(r.scrollHeight,(scrollY||r.scrollTop||0)+h));};
|
||||
let last=metrics(),lastSig=sig(last),same=0,steps=0;
|
||||
const scrollable=last.height>last.view*1.35;
|
||||
if(!scrollable||last.hooks===0){resolve('scroll skipped hooks='+last.hooks+' text='+last.text);return;}
|
||||
const tick=()=>{if(steps>=28){resolve('scrolled '+steps+' text='+last.text);return;}const before=last;scrollOnce();steps++;setTimeout(()=>{const now=metrics(),nowSig=sig(now);if(nowSig===lastSig)same++;else same=0;const loaded=grew(before,now);last=now;lastSig=nowSig;if(steps===1&&!loaded){resolve('scroll probe unchanged text='+now.text);return;}const atBottom=now.y+now.view+20>=now.height;if(same>=4||(atBottom&&same>=1)){resolve('scrolled '+steps+' text='+now.text);return;}tick();},900);};
|
||||
tick();
|
||||
}))()"#;
|
||||
|
||||
pub(super) struct Browser {
|
||||
chrome: Option<Chrome>,
|
||||
profile_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl Browser {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
chrome: None,
|
||||
profile_dir: crate::app::browser_profile_path(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn google_search(
|
||||
&mut self,
|
||||
query: &str,
|
||||
cancel: &AtomicBool,
|
||||
) -> Result<String, String> {
|
||||
if query.is_empty() {
|
||||
return Err("google_search requires query".into());
|
||||
}
|
||||
self.run_page(&google_search_url(query), EXTRACT_SEARCH_JS, false, cancel)
|
||||
}
|
||||
|
||||
pub(super) fn visit_page(&mut self, url: &str, cancel: &AtomicBool) -> Result<String, String> {
|
||||
if url.is_empty() {
|
||||
return Err("visit_page requires url".into());
|
||||
}
|
||||
self.run_page(url, EXTRACT_PAGE_JS, true, cancel)
|
||||
}
|
||||
|
||||
fn run_page(
|
||||
&mut self,
|
||||
url: &str,
|
||||
extract: &str,
|
||||
dynamic_scroll: bool,
|
||||
cancel: &AtomicBool,
|
||||
) -> Result<String, String> {
|
||||
let chrome = self.ensure_browser(cancel)?.clone();
|
||||
let tab = new_tab(chrome, cancel)?;
|
||||
tab.set_default_timeout(CDP_TIMEOUT);
|
||||
|
||||
let result = (|| {
|
||||
prepare_page(&tab, cancel)?;
|
||||
check_cancel(cancel)?;
|
||||
tab.navigate_to(url)
|
||||
.map_err(|error| format!("page navigation failed: {error}"))?;
|
||||
wait_navigated_ready(&tab, cancel)?;
|
||||
|
||||
if let Ok(clicked) = evaluate_string(&tab, CLICK_GOOGLE_CONSENT_JS, cancel)
|
||||
&& !clicked.is_empty()
|
||||
{
|
||||
sleep(cancel, Duration::from_millis(1500))?;
|
||||
wait_navigated_ready(&tab, cancel)?;
|
||||
}
|
||||
if dynamic_scroll {
|
||||
scroll_dynamic_page(&tab, cancel)?;
|
||||
}
|
||||
evaluate_string(&tab, extract, cancel)
|
||||
})();
|
||||
|
||||
close_tab(tab, cancel.load(Ordering::Relaxed));
|
||||
result
|
||||
}
|
||||
|
||||
fn ensure_browser(&mut self, cancel: &AtomicBool) -> Result<&Chrome, String> {
|
||||
if self.chrome.is_none() {
|
||||
let profile_dir = self.profile_dir.clone();
|
||||
self.chrome = Some(background(cancel, move || start_browser(profile_dir))?);
|
||||
}
|
||||
Ok(self.chrome.as_ref().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
fn start_browser(profile_dir: PathBuf) -> Result<Chrome, String> {
|
||||
if let Some(browser) = connect_existing() {
|
||||
return Ok(browser);
|
||||
}
|
||||
if profile_dir.to_str().is_none() {
|
||||
return Err("The browser profile path is not valid UTF-8.".into());
|
||||
}
|
||||
std::fs::create_dir_all(&profile_dir)
|
||||
.map_err(|error| format!("failed to create Chrome profile dir: {error}"))?;
|
||||
let executable = std::env::var_os("DS4_CHROME")
|
||||
.filter(|path| !path.is_empty())
|
||||
.map(PathBuf::from)
|
||||
.map_or_else(default_executable, Ok)
|
||||
.map_err(|error| format!("could not find Chrome: {error}"))?;
|
||||
let args = [
|
||||
OsStr::new("--remote-allow-origins=*"),
|
||||
OsStr::new("--no-default-browser-check"),
|
||||
OsStr::new("--disable-sync"),
|
||||
OsStr::new("--use-mock-keychain"),
|
||||
OsStr::new("--password-store=basic"),
|
||||
OsStr::new("--mute-audio"),
|
||||
OsStr::new("about:blank"),
|
||||
];
|
||||
let options = LaunchOptions::default_builder()
|
||||
.headless(true)
|
||||
.path(Some(executable))
|
||||
.user_data_dir(Some(profile_dir))
|
||||
.port(Some(PORT))
|
||||
.idle_browser_timeout(CDP_TIMEOUT)
|
||||
.ignore_certificate_errors(false)
|
||||
.enable_gpu(true)
|
||||
.disable_default_args(true)
|
||||
.args(args.to_vec())
|
||||
.build()
|
||||
.map_err(|error| format!("could not configure Chrome: {error}"))?;
|
||||
Chrome::new(options).map_err(|error| format!("could not start Chrome: {error}"))
|
||||
}
|
||||
|
||||
fn connect_existing() -> Option<Chrome> {
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.timeout_connect(Some(CONNECT_TIMEOUT))
|
||||
.timeout_recv_response(Some(CONNECT_TIMEOUT))
|
||||
.timeout_recv_body(Some(CONNECT_TIMEOUT))
|
||||
.build()
|
||||
.into();
|
||||
let mut response = agent
|
||||
.get(&format!("http://127.0.0.1:{PORT}/json/version"))
|
||||
.call()
|
||||
.ok()?;
|
||||
let body = response.body_mut().read_to_string().ok()?;
|
||||
let websocket = serde_json::from_str::<Value>(&body)
|
||||
.ok()?
|
||||
.get("webSocketDebuggerUrl")?
|
||||
.as_str()?
|
||||
.to_owned();
|
||||
Chrome::connect_with_timeout(websocket, CDP_TIMEOUT).ok()
|
||||
}
|
||||
|
||||
fn new_tab(chrome: Chrome, cancel: &AtomicBool) -> Result<Arc<Tab>, String> {
|
||||
let (sender, receiver) = mpsc::sync_channel(1);
|
||||
thread::spawn(move || {
|
||||
let result = chrome
|
||||
.new_tab()
|
||||
.map_err(|error| format!("could not open browser tab: {error}"));
|
||||
if let Err(mpsc::SendError(Ok(tab))) = sender.send(result) {
|
||||
let _ = tab.close_target();
|
||||
}
|
||||
});
|
||||
wait(receiver, cancel)
|
||||
}
|
||||
|
||||
fn prepare_page(tab: &Arc<Tab>, cancel: &AtomicBool) -> Result<(), String> {
|
||||
check_cancel(cancel)?;
|
||||
tab.call_method(Runtime::Enable(None))
|
||||
.map_err(|error| format!("could not enable page runtime: {error}"))?;
|
||||
let _ = tab.call_method(Emulation::SetFocusEmulationEnabled { enabled: true });
|
||||
let _ = tab.call_method(Emulation::SetDeviceMetricsOverride {
|
||||
width: 1365,
|
||||
height: 900,
|
||||
device_scale_factor: 1.0,
|
||||
mobile: false,
|
||||
scale: None,
|
||||
screen_width: None,
|
||||
screen_height: None,
|
||||
position_x: None,
|
||||
position_y: None,
|
||||
dont_set_visible_size: None,
|
||||
screen_orientation: None,
|
||||
viewport: None,
|
||||
display_feature: None,
|
||||
device_posture: None,
|
||||
});
|
||||
wait_ready(tab, cancel)
|
||||
}
|
||||
|
||||
fn wait_ready(tab: &Arc<Tab>, cancel: &AtomicBool) -> Result<(), String> {
|
||||
for _ in 0..80 {
|
||||
check_cancel(cancel)?;
|
||||
if let Ok(state) = evaluate_string(tab, "document.readyState", cancel)
|
||||
&& matches!(state.as_str(), "complete" | "interactive")
|
||||
{
|
||||
return sleep(cancel, Duration::from_millis(800));
|
||||
}
|
||||
sleep(cancel, Duration::from_millis(250))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn wait_navigated_ready(tab: &Arc<Tab>, cancel: &AtomicBool) -> Result<(), String> {
|
||||
let mut last_len = -1;
|
||||
let mut stable = 0;
|
||||
let mut saw_real_url = false;
|
||||
for iteration in 0..100 {
|
||||
check_cancel(cancel)?;
|
||||
let Ok(probe) = evaluate_string(
|
||||
tab,
|
||||
"location.href+'\\n'+document.readyState+'\\n'+((document.body&&document.body.innerText)||'').length",
|
||||
cancel,
|
||||
) else {
|
||||
sleep(cancel, Duration::from_millis(250))?;
|
||||
continue;
|
||||
};
|
||||
let Ok((href, ready, text_len)) = parse_probe(&probe) else {
|
||||
sleep(cancel, Duration::from_millis(250))?;
|
||||
continue;
|
||||
};
|
||||
let real_url = !href.is_empty() && href != "about:blank" && !href.starts_with("chrome://");
|
||||
let ready = matches!(ready, "complete" | "interactive");
|
||||
saw_real_url |= real_url;
|
||||
if text_len > 0 && text_len == last_len {
|
||||
stable += 1;
|
||||
} else {
|
||||
stable = 0;
|
||||
}
|
||||
last_len = text_len;
|
||||
if saw_real_url && ready && text_len > 0 && stable >= 2 {
|
||||
return sleep(cancel, Duration::from_millis(500));
|
||||
}
|
||||
if saw_real_url && ready && iteration >= 24 {
|
||||
return Ok(());
|
||||
}
|
||||
sleep(cancel, Duration::from_millis(250))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn scroll_dynamic_page(tab: &Arc<Tab>, cancel: &AtomicBool) -> Result<(), String> {
|
||||
check_cancel(cancel)?;
|
||||
let result = evaluate_string(tab, SCROLL_DYNAMIC_PAGE_JS, cancel);
|
||||
check_cancel(cancel)?;
|
||||
match result {
|
||||
Err(error) if error == "interrupted" => Err(error),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn evaluate_string(
|
||||
tab: &Arc<Tab>,
|
||||
expression: &str,
|
||||
cancel: &AtomicBool,
|
||||
) -> Result<String, String> {
|
||||
let tab = Arc::clone(tab);
|
||||
let expression = expression.to_owned();
|
||||
background(cancel, move || {
|
||||
let response = tab
|
||||
.call_method(Runtime::Evaluate {
|
||||
expression,
|
||||
object_group: None,
|
||||
include_command_line_api: Some(true),
|
||||
silent: None,
|
||||
context_id: None,
|
||||
return_by_value: Some(true),
|
||||
generate_preview: None,
|
||||
user_gesture: None,
|
||||
await_promise: Some(true),
|
||||
throw_on_side_effect: None,
|
||||
timeout: None,
|
||||
disable_breaks: None,
|
||||
repl_mode: None,
|
||||
allow_unsafe_eval_blocked_by_csp: None,
|
||||
unique_context_id: None,
|
||||
serialization_options: None,
|
||||
})
|
||||
.map_err(|error| format!("JavaScript evaluation failed: {error}"))?;
|
||||
if response.exception_details.is_some() {
|
||||
return Err("JavaScript evaluation failed".into());
|
||||
}
|
||||
response
|
||||
.result
|
||||
.value
|
||||
.and_then(|value| value.as_str().map(str::to_owned))
|
||||
.ok_or_else(|| "Runtime.evaluate did not return a string".into())
|
||||
})
|
||||
}
|
||||
|
||||
fn background<T, F>(cancel: &AtomicBool, operation: F) -> Result<T, String>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: FnOnce() -> Result<T, String> + Send + 'static,
|
||||
{
|
||||
let (sender, receiver) = mpsc::sync_channel(1);
|
||||
thread::spawn(move || {
|
||||
let _ = sender.send(operation());
|
||||
});
|
||||
wait(receiver, cancel)
|
||||
}
|
||||
|
||||
fn wait<T>(receiver: mpsc::Receiver<Result<T, String>>, cancel: &AtomicBool) -> Result<T, String> {
|
||||
loop {
|
||||
match receiver.recv_timeout(POLL_INTERVAL) {
|
||||
Ok(result) => return result,
|
||||
Err(mpsc::RecvTimeoutError::Timeout) => check_cancel(cancel)?,
|
||||
Err(mpsc::RecvTimeoutError::Disconnected) => {
|
||||
return Err("browser worker stopped unexpectedly".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn close_tab(tab: Arc<Tab>, background: bool) {
|
||||
if background {
|
||||
thread::spawn(move || {
|
||||
let _ = tab.close_target();
|
||||
});
|
||||
} else {
|
||||
let _ = tab.close_target();
|
||||
}
|
||||
}
|
||||
|
||||
fn check_cancel(cancel: &AtomicBool) -> Result<(), String> {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
Err("interrupted".into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn sleep(cancel: &AtomicBool, duration: Duration) -> Result<(), String> {
|
||||
let mut remaining = duration;
|
||||
while !remaining.is_zero() {
|
||||
check_cancel(cancel)?;
|
||||
let step = remaining.min(POLL_INTERVAL);
|
||||
thread::sleep(step);
|
||||
remaining = remaining.saturating_sub(step);
|
||||
}
|
||||
check_cancel(cancel)
|
||||
}
|
||||
|
||||
fn parse_probe(probe: &str) -> Result<(&str, &str, i64), String> {
|
||||
let mut parts = probe.splitn(3, '\n');
|
||||
let href = parts.next().unwrap_or_default();
|
||||
let ready = parts
|
||||
.next()
|
||||
.ok_or_else(|| "page readiness probe returned malformed data".to_owned())?;
|
||||
let text_len = parts
|
||||
.next()
|
||||
.ok_or_else(|| "page readiness probe returned malformed data".to_owned())?
|
||||
.parse()
|
||||
.unwrap_or(0);
|
||||
Ok((href, ready, text_len))
|
||||
}
|
||||
|
||||
fn google_search_url(query: &str) -> String {
|
||||
const HEX: &[u8; 16] = b"0123456789ABCDEF";
|
||||
let mut url = String::from("https://www.google.com/search?q=");
|
||||
for byte in query.bytes() {
|
||||
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
|
||||
url.push(char::from(byte));
|
||||
} else {
|
||||
url.push('%');
|
||||
url.push(char::from(HEX[(byte >> 4) as usize]));
|
||||
url.push(char::from(HEX[(byte & 15) as usize]));
|
||||
}
|
||||
}
|
||||
url
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
|
||||
#[test]
|
||||
fn browser_helpers_preserve_ds4_wire_values() {
|
||||
assert_eq!(
|
||||
google_search_url("café & Rust~"),
|
||||
"https://www.google.com/search?q=caf%C3%A9%20%26%20Rust~"
|
||||
);
|
||||
assert_eq!(
|
||||
parse_probe("https://example.com\ncomplete\n42").unwrap(),
|
||||
("https://example.com", "complete", 42)
|
||||
);
|
||||
assert!(parse_probe("incomplete").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires an installed Chrome browser"]
|
||||
fn headless_browser_extracts_a_local_page() {
|
||||
let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let mut request = [0; 4096];
|
||||
let _ = stream.read(&mut request);
|
||||
let body = "<title>Fixture</title><h1>Hello</h1><p>Browser works.</p>";
|
||||
write!(
|
||||
stream,
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
.unwrap();
|
||||
});
|
||||
let profile_dir = std::env::temp_dir().join(format!(
|
||||
"ds4-browser-test-{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let mut browser = Browser {
|
||||
chrome: None,
|
||||
profile_dir: profile_dir.clone(),
|
||||
};
|
||||
let output = browser
|
||||
.run_page(
|
||||
&format!("http://{address}"),
|
||||
EXTRACT_PAGE_JS,
|
||||
false,
|
||||
&AtomicBool::new(false),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(output.contains("# Fixture"));
|
||||
assert!(output.contains("# Hello"));
|
||||
assert!(output.contains("Browser works."));
|
||||
drop(browser);
|
||||
server.join().unwrap();
|
||||
std::fs::remove_dir_all(profile_dir).unwrap();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user