diff --git a/README.md b/README.md index cea4d5e..91262b9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The project is under active development. Core blogging workflows are broadly ava - Post and translation authoring with change-aware draft/published/archive lifecycle, file-backed change discard, canonical draft reopening after manual translation edits, non-disruptive automatic translation, desktop archive/unarchive actions, in-place published-frontmatter updates, metadata, tags, categories, cursor-preserving link and media insertion, live link/backlink graphs, media, and batch gallery-image import. - Media import including HEIC/HEIF decoding, q80 WebP thumbnails (plus q85 AI JPEGs), metadata translations, filters, validation, post assignment, and sequential drag-and-drop insertion into post editors. - WordPress WXR migration with saved analyses, HTML-to-Markdown and shortcode conversion, conflict/taxonomy review, recoverable 500-item execution batches, media-parent linking, progress reporting, and optional AI-assisted taxonomy mapping. -- Post, Liquid template, and Lua script editing with dedicated syntax highlighting, explicit syntax-check feedback, change-aware published/draft lifecycle, normalized collision-safe template/script slug changes, reference-safe template renames, and publish-time enforcement of the bDS2 Liquid tag/filter/operator subset, using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms, including airplane-gated Git sync. +- Post, Liquid template, and Lua script editing with dedicated syntax highlighting, explicit syntax-check feedback, change-aware published/draft lifecycle, normalized collision-safe template/script slug changes, reference-safe template renames, publish-time enforcement of the bDS2 Liquid tag/filter/operator subset, and bounded process-local compiled Lua reuse with a fresh sandbox per invocation, using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms, including airplane-gated Git sync. - SQLite and filesystem persistence with byte-canonical bDS2 frontmatter, media sidecars, metadata JSON, and OPML menus; rebuild; bidirectional metadata diff/repair including project categories and publishing preferences; stale post-path cleanup on republish; bDS2-compatible checksums and NFD slug generation; and FTS5 search. - Optional on-device multilingual semantic search and similar-post tag suggestions backed by a persistent USearch index, plus always-available partial-name tag autocomplete and duplicate-post review in the desktop workspace. - Read-only in-app browsers in the Help menu for the bundled global `DOCUMENTATION.md`, the generated Lua API reference with public types and runnable examples, the [CLI/server/TUI documentation](CLI.md), and the [MCP server documentation](MCP.md), with safe GFM rendering and confirmed external links. diff --git a/crates/bds-core/src/scripting/mod.rs b/crates/bds-core/src/scripting/mod.rs index 3914db2..bb7a3e1 100644 --- a/crates/bds-core/src/scripting/mod.rs +++ b/crates/bds-core/src/scripting/mod.rs @@ -1,7 +1,9 @@ +use std::collections::{HashMap, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; use std::time::{Duration, Instant}; +use mlua::chunk::ChunkMode; use mlua::{ Function, HookTriggers, Lua, LuaOptions, LuaSerdeExt, MultiValue, StdLib, Value, VmState, }; @@ -15,9 +17,69 @@ pub use manifest::{ApiManifest, api_manifest}; const MACRO_TIMEOUT: Duration = Duration::from_secs(10); const MEMORY_LIMIT_BYTES: usize = 32 * 1024 * 1024; +const BYTECODE_CACHE_MAX_ENTRIES: usize = 32; const MAX_TOASTS_PER_SCRIPT: usize = 5; const MAX_TOAST_LENGTH: usize = 300; +struct BytecodeCache { + entries: HashMap>, + order: VecDeque, + max_entries: usize, +} + +impl BytecodeCache { + fn new(max_entries: usize) -> Self { + assert!(max_entries > 0); + Self { + entries: HashMap::new(), + order: VecDeque::new(), + max_entries, + } + } + + fn get_or_compile( + &mut self, + source: &str, + compile: impl FnOnce() -> Result, String>, + ) -> Result, String> { + if let Some(bytecode) = self.entries.get(source) { + return Ok(Arc::clone(bytecode)); + } + + let bytecode: Arc<[u8]> = compile()?.into(); + if self.entries.len() == self.max_entries + && let Some(oldest) = self.order.pop_front() + { + self.entries.remove(&oldest); + } + let key = source.to_string(); + self.order.push_back(key.clone()); + self.entries.insert(key, Arc::clone(&bytecode)); + Ok(bytecode) + } +} + +fn bytecode_cache() -> &'static Mutex { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(BytecodeCache::new(BYTECODE_CACHE_MAX_ENTRIES))) +} + +fn lock_bytecode_cache(cache: &Mutex) -> MutexGuard<'_, BytecodeCache> { + cache + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn compiled_chunk(lua: &Lua, source: &str) -> Result, String> { + lock_bytecode_cache(bytecode_cache()).get_or_compile(source, || { + lua.load(source) + .set_name("script") + .into_function() + .map(|function| function.dump(false)) + .map_err(|error| error.to_string()) + }) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ExecutionKind { Macro, @@ -93,11 +155,7 @@ impl ExecutionControl { /// Parse Lua source with the same runtime used for execution. pub fn validate(source: &str) -> Result<(), String> { let lua = sandboxed_lua()?; - lua.load(source) - .set_name("script") - .into_function() - .map(|_| ()) - .map_err(|error| error.to_string()) + compiled_chunk(&lua, source).map(|_| ()) } /// Execute a named Lua entrypoint with one JSON-compatible argument. @@ -192,8 +250,10 @@ pub fn execute_many_with_host( ) .map_err(|error| error.to_string())?; - lua.load(source) + let bytecode = compiled_chunk(&lua, source)?; + lua.load(bytecode.as_ref()) .set_name("script") + .set_mode(ChunkMode::Binary) .exec() .map_err(|error| error.to_string())?; let function: Function = lua @@ -800,6 +860,156 @@ mod tests { assert!(validate("function main( return 1 end").is_err()); } + #[test] + fn bytecode_cache_reuses_exact_source_and_misses_edits() { + let mut cache = BytecodeCache::new(2); + let mut compilations = 0; + let first = cache + .get_or_compile("return 1", || { + compilations += 1; + Ok(vec![1]) + }) + .unwrap(); + let repeated = cache + .get_or_compile("return 1", || { + compilations += 1; + Ok(vec![2]) + }) + .unwrap(); + let edited = cache + .get_or_compile("return 2", || { + compilations += 1; + Ok(vec![2]) + }) + .unwrap(); + + assert!(Arc::ptr_eq(&first, &repeated)); + assert!(!Arc::ptr_eq(&first, &edited)); + assert_eq!(compilations, 2); + } + + #[test] + fn bytecode_cache_excludes_failures_and_evicts_oldest() { + let mut cache = BytecodeCache::new(2); + assert!( + cache + .get_or_compile("invalid", || Err("invalid source".to_string())) + .is_err() + ); + assert!(!cache.entries.contains_key("invalid")); + + cache.get_or_compile("one", || Ok(vec![1])).unwrap(); + cache.get_or_compile("two", || Ok(vec![2])).unwrap(); + cache.get_or_compile("three", || Ok(vec![3])).unwrap(); + + assert!(!cache.entries.contains_key("one")); + assert!(cache.entries.contains_key("two")); + assert!(cache.entries.contains_key("three")); + } + + #[test] + fn bytecode_cache_lock_recovers_from_poisoning() { + let cache = Arc::new(Mutex::new(BytecodeCache::new(2))); + let poisoned = Arc::clone(&cache); + let _ = std::thread::spawn(move || { + let _guard = poisoned.lock().unwrap(); + panic!("poison cache"); + }) + .join(); + + lock_bytecode_cache(&cache) + .get_or_compile("return 1", || Ok(vec![1])) + .unwrap(); + } + + #[test] + fn bytecode_cache_compiles_once_under_concurrent_access() { + let cache = Arc::new(Mutex::new(BytecodeCache::new(2))); + let compilations = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let workers = (0..8) + .map(|_| { + let cache = Arc::clone(&cache); + let compilations = Arc::clone(&compilations); + std::thread::spawn(move || { + lock_bytecode_cache(&cache) + .get_or_compile("return 1", || { + compilations.fetch_add(1, Ordering::Relaxed); + Ok(vec![1]) + }) + .unwrap() + }) + }) + .collect::>(); + + for worker in workers { + worker.join().unwrap(); + } + assert_eq!(compilations.load(Ordering::Relaxed), 1); + } + + #[test] + fn cached_execution_preserves_results_errors_and_fresh_globals() { + let source = r#" + counter = (counter or 0) + 1 + function main(input) + if input.fail then error("expected failure") end + return { counter = counter, value = input.value } + end + "#; + validate(source).unwrap(); + for value in ["first", "second"] { + let result = execute( + source, + "main", + &json!({"value": value}), + ExecutionKind::Utility, + &ExecutionControl::default(), + ) + .unwrap(); + assert_eq!(result.value, json!({"counter": 1, "value": value})); + } + + let error = execute( + source, + "main", + &json!({"fail": true}), + ExecutionKind::Utility, + &ExecutionControl::default(), + ) + .unwrap_err(); + assert!(error.contains("expected failure")); + assert!(error.contains("script")); + assert!(error.contains(":4:"), "{error}"); + } + + #[test] + fn validation_warms_the_execution_cache() { + let source = "-- validation cache sharing\nfunction main() return 7 end"; + validate(source).unwrap(); + let validated = lock_bytecode_cache(bytecode_cache()) + .entries + .get(source) + .cloned() + .unwrap(); + + let result = execute( + source, + "main", + &JsonValue::Null, + ExecutionKind::Utility, + &ExecutionControl::default(), + ) + .unwrap(); + let executed = lock_bytecode_cache(bytecode_cache()) + .entries + .get(source) + .cloned() + .unwrap(); + + assert_eq!(result.value, json!(7)); + assert!(Arc::ptr_eq(&validated, &executed)); + } + #[test] fn enforces_per_script_toast_budget_and_length() { let result = execute( diff --git a/specs/script.allium b/specs/script.allium index 0780720..f04d338 100644 --- a/specs/script.allium +++ b/specs/script.allium @@ -89,6 +89,12 @@ surface ScriptRuntimeSurface { -- unrestricted host capabilities are unavailable unless explicitly -- re-exposed by the host application. + @guarantee CompiledChunkCache + -- Validation and execution share a bounded, process-local cache of Lua + -- bytecode keyed by exact source. Cache hits still execute in a fresh + -- sandbox with the current host capabilities, limits, hooks, and globals. + -- Failed compilation is not cached; bytecode is never persisted. + @guarantee ExplicitHostCapabilities -- Host-provided functions are exposed only through an explicit bds.* -- capability table, never through ambient global access.