From 5c1b333cce68911aab87883bd20a814741d4a602 Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Sat, 18 Jul 2026 22:21:02 +0200 Subject: [PATCH] chore: reworked some specs and did small addition to database schema based on gaps in spec, also rechecked the core plan --- RUST_PLAN_CORE.md | 41 +++++------ .../down.sql | 3 + .../up.sql | 3 + crates/bds-core/src/db/migrations.rs | 12 ++-- crates/bds-core/src/db/schema.rs | 2 + crates/bds-ui/src/views/media_editor.rs | 12 ++-- specs/ai.allium | 27 +++++-- specs/frontmatter.allium | 8 +-- specs/media.allium | 45 ++++++++++-- specs/publishing.allium | 6 ++ specs/schema.allium | 6 +- specs/script.allium | 72 +++++++++++++++++-- 12 files changed, 183 insertions(+), 54 deletions(-) create mode 100644 crates/bds-core/migrations/20260718220000_add_chat_message_token_usage/down.sql create mode 100644 crates/bds-core/migrations/20260718220000_add_chat_message_token_usage/up.sql diff --git a/RUST_PLAN_CORE.md b/RUST_PLAN_CORE.md index 5f4f9ed..d9f6806 100644 --- a/RUST_PLAN_CORE.md +++ b/RUST_PLAN_CORE.md @@ -32,7 +32,7 @@ Status in this document describes the current source code as of 2026-07-18. It d ## Current Core Status -### Project, persistence, and integrity — Mostly done +### Project, persistence, and integrity — Done Available: @@ -45,11 +45,9 @@ Available: - Database rebuild from filesystem and directional metadata diff/repair. - Structured site, media, and translation validation. - Unbounded unique slug allocation. - -Open: - -- Watch project files for external changes and reconcile open editors, preview, and database state. -- Make the script file extension project-configurable instead of hard-coding `.lua`. +- Explicit rebuild-from-filesystem paths for manual file changes. bDS2 does + not live-watch arbitrary project files; its external-change watcher is the + extension CLI/database-notification contract. ### Native desktop shell — Done @@ -75,11 +73,11 @@ Available: - Template and Lua script creation, editing, validation, publication, and deletion. - Rope-based editing with syntax highlighting, selection, clipboard, undo/redo, word/line/page movement, line numbers, soft wrapping, mouse selection, and committed IME input. - Automatic translation flows with airplane-mode gating and media translation propagation. +- Functional post-links panel with backlinks, outlinks, and navigation to linked posts. Open: -- Replace the post-links placeholder with the specified functional linked-post view. -- Add the specified batch workflow for images linked from post content. +- Add the specified post-editor "Add Gallery Images" batch workflow. ### Rendering, preview, and generation — Mostly done @@ -107,26 +105,26 @@ Available: - Model catalog discovery and model selection. - Post translation, media translation, image alt text, post analysis, taxonomy analysis, and language detection. - Explicit offline gating and user-visible errors. +- SQLite fields for input, output, cache-read, and cache-write token usage. Open: -- Persist actual input, output, cache-read, and cache-write token usage where the schema provides those fields. +- Parse actual input, output, cache-read, and cache-write token usage from AI + responses and return it with each one-shot result. Chat persistence of those + counters belongs to the extension chat workflow. Interactive chat, tools, agents, and MCP belong to the extension plan. -### Publishing — Mostly done +### Publishing — Done Available: - SCP and rsync publishing through system commands. - SSH-agent-only authentication with no password prompts. - Separate HTML, thumbnail, and media targets. -- `.meta` exclusion, changed-file skipping for SCP, progress reporting, cancellation, and UI commands. - -Open: - -- Run the three upload targets as parallel tasks instead of processing them sequentially. -- Integrate external-file watching with publish/integrity workflows. +- `.meta` exclusion, changed-file skipping for SCP, progress reporting, and UI commands. +- One managed publish job processes the HTML, thumbnail, and media targets in + sequence, matching bDS2. Parallel target uploads are not a parity requirement. ### Lua scripting — Partly done @@ -136,20 +134,19 @@ Available: - Application log, progress, and toast functions. - User macro and transform execution, including Blogmark transforms. - Lua script persistence, rebuild, metadata diff, validation, and editor support. +- Fixed `.lua` script file contract. Open: -- Expose the specified `bds.*` API for post, media, tag, project, and other script-visible data. +- Expose the bDS2-compatible core `bds.*` API for post, media, tag, project, and other script-visible data. - Generate and bundle the Lua API reference, canonical type reference, and macro/transform/utility examples under `docs/scripting/`. - Add a documentation-sync check tied to the exposed API. ## Remaining Core Blocks 1. Complete the Lua host API and scripting documentation. -2. Add filesystem change watching and reconciliation. -3. Parallelize publishing targets. -4. Finish the linked-post and linked-image authoring workflows. -5. Add generation section-task grouping and AI token accounting. -6. Remove the hard-coded Lua script extension. +2. Finish the linked-image authoring workflow. +3. Add generation section-task grouping. +4. Return normalized token accounting from one-shot AI calls. Core is feature-complete when these blocks are closed and the implementation continues to satisfy the Allium and bDS2 compatibility contracts. diff --git a/crates/bds-core/migrations/20260718220000_add_chat_message_token_usage/down.sql b/crates/bds-core/migrations/20260718220000_add_chat_message_token_usage/down.sql new file mode 100644 index 0000000..84ffaad --- /dev/null +++ b/crates/bds-core/migrations/20260718220000_add_chat_message_token_usage/down.sql @@ -0,0 +1,3 @@ +-- This file should undo anything in `up.sql` +ALTER TABLE chat_messages DROP COLUMN token_usage_output; +ALTER TABLE chat_messages DROP COLUMN token_usage_input; diff --git a/crates/bds-core/migrations/20260718220000_add_chat_message_token_usage/up.sql b/crates/bds-core/migrations/20260718220000_add_chat_message_token_usage/up.sql new file mode 100644 index 0000000..a416d0c --- /dev/null +++ b/crates/bds-core/migrations/20260718220000_add_chat_message_token_usage/up.sql @@ -0,0 +1,3 @@ +-- Your SQL goes here +ALTER TABLE chat_messages ADD COLUMN token_usage_input INTEGER; +ALTER TABLE chat_messages ADD COLUMN token_usage_output INTEGER; diff --git a/crates/bds-core/src/db/migrations.rs b/crates/bds-core/src/db/migrations.rs index d34d2b2..170a1cd 100644 --- a/crates/bds-core/src/db/migrations.rs +++ b/crates/bds-core/src/db/migrations.rs @@ -35,7 +35,7 @@ mod tests { let applied = db .conn() .with_migrations(|conn| conn.applied_migrations().unwrap().len()); - assert_eq!(applied, 1); + assert_eq!(applied, 2); } #[test] @@ -82,7 +82,7 @@ mod tests { #[test] fn migrations_expose_current_ai_catalog_and_usage_columns() { let db = migrated_database(); - let (provider_ref, model_ref, cache_tokens) = db + let (provider_ref, model_ref, usage_tokens) = db .conn() .with(|conn| { diesel::insert_into(ai_providers::table) @@ -124,6 +124,8 @@ mod tests { chat_messages::conversation_id.eq("conversation"), chat_messages::role.eq("assistant"), chat_messages::created_at.eq(1_i64), + chat_messages::token_usage_input.eq(Some(56)), + chat_messages::token_usage_output.eq(Some(78)), chat_messages::cache_read_tokens.eq(Some(12)), chat_messages::cache_write_tokens.eq(Some(34)), )) @@ -138,16 +140,18 @@ mod tests { .first::>(conn)?, chat_messages::table .select(( + chat_messages::token_usage_input, + chat_messages::token_usage_output, chat_messages::cache_read_tokens, chat_messages::cache_write_tokens, )) - .first::<(Option, Option)>(conn)?, + .first::<(Option, Option, Option, Option)>(conn)?, )) }) .unwrap(); assert_eq!(provider_ref.as_deref(), Some("provider-package")); assert_eq!(model_ref.as_deref(), Some("model-package")); - assert_eq!(cache_tokens, (Some(12), Some(34))); + assert_eq!(usage_tokens, (Some(56), Some(78), Some(12), Some(34))); } } diff --git a/crates/bds-core/src/db/schema.rs b/crates/bds-core/src/db/schema.rs index 7ced59b..6c6a0b8 100644 --- a/crates/bds-core/src/db/schema.rs +++ b/crates/bds-core/src/db/schema.rs @@ -80,6 +80,8 @@ diesel::table! { created_at -> BigInt, cache_read_tokens -> Nullable, cache_write_tokens -> Nullable, + token_usage_input -> Nullable, + token_usage_output -> Nullable, } } diff --git a/crates/bds-ui/src/views/media_editor.rs b/crates/bds-ui/src/views/media_editor.rs index 98c476f..c7fda5e 100644 --- a/crates/bds-ui/src/views/media_editor.rs +++ b/crates/bds-ui/src/views/media_editor.rs @@ -240,8 +240,7 @@ pub fn view<'a>( vec![ button(text(t(locale, "editor.quickActions")).size(13)) .on_press_maybe( - ai_enabled - .then_some(Message::MediaEditor(MediaEditorMsg::ToggleQuickActions)), + ai_enabled.then_some(Message::MediaEditor(MediaEditorMsg::ToggleQuickActions)), ) .style(inputs::secondary_button) .padding([6, 16]) @@ -549,7 +548,11 @@ pub fn view<'a>( .into() } -fn quick_action_item<'a>(label: String, msg: MediaEditorMsg, enabled: bool) -> Element<'a, Message> { +fn quick_action_item<'a>( + label: String, + msg: MediaEditorMsg, + enabled: bool, +) -> Element<'a, Message> { button(text(label).size(12).shaping(Shaping::Advanced)) .on_press_maybe(enabled.then_some(Message::MediaEditor(msg))) .padding([6, 12]) @@ -612,7 +615,8 @@ mod tests { #[test] fn quick_actions_menu_starts_closed() { - let state = MediaEditorState::from_media(&make_media(), &["en".to_string()], &[], Vec::new()); + let state = + MediaEditorState::from_media(&make_media(), &["en".to_string()], &[], Vec::new()); assert!(!state.quick_actions_open); } diff --git a/specs/ai.allium b/specs/ai.allium index 6923319..3722086 100644 --- a/specs/ai.allium +++ b/specs/ai.allium @@ -192,6 +192,17 @@ surface OneShotAiSurface { DetectLanguageRequested(text) TranslatePostRequested(post, target_language) TranslateMediaRequested(media, target_language) + + @guarantee UsageReturnedWithSuccessfulResult + -- Every successful one-shot result includes a usage value with nullable + -- input_tokens, output_tokens, cache_read_tokens, and + -- cache_write_tokens. Missing provider counters remain null. One-shot + -- calls return this accounting to their caller; they do not create a + -- chat conversation or persist a ChatMessage. + -- OpenAI-compatible responses normalize prompt_tokens to input_tokens, + -- completion_tokens to output_tokens, prompt_tokens_details.cached_tokens + -- to cache_read_tokens, and completion_tokens_details.cached_tokens to + -- cache_write_tokens. } surface AiChatSurface { @@ -236,7 +247,7 @@ rule AnalyzeTaxonomy { when: AnalyzeTaxonomyRequested(post) requires: active_endpoint_configured -- Suggests tags and categories for a post - ensures: TaxonomySuggestion(tags, categories) + ensures: TaxonomySuggestion(tags, categories, usage) } rule AnalyzeImage { @@ -244,34 +255,34 @@ rule AnalyzeImage { requires: active_endpoint_configured requires: is_image(media.mime_type) -- Vision model generates alt text and caption - ensures: ImageAnalysisResult(alt, caption) + ensures: ImageAnalysisResult(title, alt, caption, usage) } rule AnalyzePost { when: AnalyzePostRequested(post) requires: active_endpoint_configured -- Generates title, excerpt, slug suggestions - ensures: PostAnalysisResult(title, excerpt, slug) + ensures: PostAnalysisResult(title, excerpt, slug, usage) } rule DetectLanguage { when: DetectLanguageRequested(text) requires: active_endpoint_configured - ensures: LanguageDetectionResult(language_code) + ensures: LanguageDetectionResult(language_code, usage) } rule TranslatePost { when: TranslatePostRequested(post, target_language) requires: active_endpoint_configured -- Translates title, excerpt, content to target language - ensures: TranslationResult(title, excerpt, content) + ensures: TranslationResult(title, excerpt, content, usage) } rule TranslateMedia { when: TranslateMediaRequested(media, target_language) requires: active_endpoint_configured -- Translates title, alt, caption to target language - ensures: MediaTranslationResult(title, alt, caption) + ensures: MediaTranslationResult(title, alt, caption, usage) } -- Chat (extension Bucket C scope, with streaming and tool use) @@ -303,6 +314,10 @@ rule SendChatMessage { -- Blog data tools for post/media querying and mutation during chat. -- Render tools may emit structured chart/table/form payloads. -- Token usage tracking includes input, output, cache read, cache write. + ensures: AssistantChatMessagesPersistNormalizedUsage(conversation) + -- Each persisted assistant response stores its normalized input, + -- output, cache-read, and cache-write counters. User and tool messages + -- leave those nullable fields empty. } rule CancelChat { diff --git a/specs/frontmatter.allium b/specs/frontmatter.allium index f535efb..8abf253 100644 --- a/specs/frontmatter.allium +++ b/specs/frontmatter.allium @@ -78,10 +78,6 @@ surface MenuOpmlSurface { item.slug } -config { - script_extension: String = "lua" -} - -- ============================================================================ -- POST FILE FORMAT -- ============================================================================ @@ -239,7 +235,7 @@ rule WriteTemplateFile { -- ============================================================================ value ScriptFrontmatter { - -- File path: scripts/{slug}.{extension} + -- File path: scripts/{slug}.lua -- YAML frontmatter delimited by --- markers -- All keys serialized as camelCase in YAML frontmatter id: String -- UUID v4 @@ -258,7 +254,7 @@ rule WriteScriptFile { when: PublishScriptRequested(script) requires: ValidateScript(script.content) = valid ensures: FileWritten( - path: format("scripts/{slug}.{extension}", slug: script.slug, extension: config.script_extension), + path: format("scripts/{slug}.lua", slug: script.slug), content: format_script_file(script) ) ensures: script.content = null diff --git a/specs/media.allium b/specs/media.allium index 56f378f..54fcb02 100644 --- a/specs/media.allium +++ b/specs/media.allium @@ -17,6 +17,24 @@ surface MediaControlSurface { RebuildMediaFromFilesRequested(project) } +surface PostEditorGalleryImportSurface { + facing _: MediaOperator + + provides: + BatchImportImagesRequested(project, post, file_paths, language) + + @guarantee NativeImageSelection + -- The post-editor "Add Gallery Images" quick action opens the native + -- multi-file picker restricted to images. Cancelling the picker is a + -- no-op; picker failures are reported in the Output panel. + + @guarantee AirplaneModeGate + -- Online mode uses the configured online endpoint. Airplane mode may + -- proceed only when its local endpoint is configured. If it is not, + -- no picker opens and the Output panel explains that automatic AI + -- actions remain gated by airplane mode. +} + value ThumbnailSet { small: String -- 150px width (binary path) medium: String -- 400px width (binary path) @@ -229,20 +247,33 @@ invariant SidecarRoundtrip { rule BatchImportProcessLinkImages { when: BatchImportImagesRequested(project, post, file_paths, language) - requires: not OfflineMode for source_path in file_paths where is_image(source_path): let media = ImportMedia(source_path, project) - let analysis = AnalyzeImage(media, language) - ensures: MediaUpdated(media, analysis) ensures: PostMediaLinked(media, post) + -- Import and linking are mandatory and happen before enrichment. + if ActiveAiEndpointConfigured: + let analysis = AnalyzeImage(media, language) + ensures: MediaUpdated(media, analysis) + for target_language in unique( + {project.main_language} + project.blog_languages - {language} + ): + ensures: MediaTranslationUpserted(media, target_language) + if not ActiveAiEndpointConfigured or AiEnrichmentFailed(media): + ensures: PostMediaLinked(media, post) + -- AI analysis, metadata update, and translation are + -- best-effort and never undo a successful import/link. @guidance -- Triggered from post editor quick action "Add Gallery Images". -- AI results auto-applied without user confirmation. - -- After metadata is set, media is auto-translated to all configured blog languages. - -- Non-image files skipped entirely. + -- After metadata is set, media is auto-translated to the project main + -- language and all configured blog languages except the source + -- language, with duplicate targets removed. + -- Non-image files are not linked or enriched. -- Concurrency limit from project metadata image_import_concurrency (default 4, min 1, max 8). - -- Toast per completed image + final summary toast. - -- On completion: [[gallery]] macro inserted into post content and post editor refreshed. + -- Each successful image and each failed image emits its own Output + -- panel entry. The final entry reports the selected image count. + -- After every worker has finished, including partial failures, + -- \n[[gallery]]\n is inserted into post content and the editor refreshes. } config { diff --git a/specs/publishing.allium b/specs/publishing.allium index a618b06..cf1803b 100644 --- a/specs/publishing.allium +++ b/specs/publishing.allium @@ -65,6 +65,12 @@ surface PublishingRuntimeSurface { provides: PublishJobStarted(project, job, credentials) PublishTargetFailed(job, target, error) + + @guarantee SequentialTargetProcessing + -- One managed publish job processes html, thumbnails, then media in + -- that order, matching bDS2. It reports progress at each target and + -- stops on the first failed target. Parallel target upload is not part + -- of the compatibility contract. } rule UploadSite { diff --git a/specs/schema.allium b/specs/schema.allium index 817e2a7..ef9cc4b 100644 --- a/specs/schema.allium +++ b/specs/schema.allium @@ -154,7 +154,7 @@ entity Script { entrypoint: String -- Default: "render" for macros, "main" otherwise enabled: Boolean version: Integer -- Incremented on each update - file_path: String -- scripts/{slug}.{extension} + file_path: String -- scripts/{slug}.lua status: ScriptStatus content: String? -- Draft body (null when published) created_at: Timestamp @@ -239,6 +239,8 @@ entity ChatMessage { content: String? tool_call_id: String? -- For tool responses tool_calls: String? -- JSON array of tool calls + token_usage_input: Integer? + token_usage_output: Integer? cache_read_tokens: Integer? cache_write_tokens: Integer? created_at: Timestamp @@ -524,6 +526,8 @@ surface ChatMessageRecordSurface { message.content when message.content != null message.tool_call_id when message.tool_call_id != null message.tool_calls when message.tool_calls != null + message.token_usage_input when message.token_usage_input != null + message.token_usage_output when message.token_usage_output != null message.cache_read_tokens when message.cache_read_tokens != null message.cache_write_tokens when message.cache_write_tokens != null message.created_at diff --git a/specs/script.allium b/specs/script.allium index 4ce208f..6ad77e0 100644 --- a/specs/script.allium +++ b/specs/script.allium @@ -7,7 +7,6 @@ -- only the behavioural contract is normative here. config { - script_extension: String = "lua" macro_timeout: Duration = 10.seconds transform_max_toasts_per_script: Integer = 5 transform_max_toasts_total: Integer = 20 @@ -94,6 +93,71 @@ surface ScriptRuntimeSurface { -- Host-provided functions are exposed only through an explicit bds.* -- capability table, never through ambient global access. + @guarantee CoreHostApiCompatibility + -- The core API matches the bDS2 capability names below. Methods are + -- bound to the active project unless an explicit project id is part of + -- the method. Arguments and return tables use the canonical types in + -- the generated scripting reference. + -- + -- bds.app: copy_to_clipboard, get_data_paths, + -- get_blogmark_bookmarklet, get_system_language, + -- get_default_project_path, get_title_bar_metrics, log, + -- notify_renderer_ready, open_folder, read_project_metadata, + -- select_folder, set_preview_post_target, show_item_in_folder, + -- trigger_menu_action. + -- bds.projects: create, delete, delete_with_data, get, get_all, + -- get_active, set_active, update. + -- bds.meta: get_project_metadata, update_project_metadata, + -- set_project_metadata, add_category, remove_category, add_tag, + -- remove_tag, get_categories, get_tags, + -- get_publishing_preferences, set_publishing_preferences, + -- clear_publishing_preferences, sync_on_startup. + -- bds.posts: create, discard, filter, generate_unique_slug, + -- get_by_status, get_by_year_month, get_dashboard_stats, + -- get_linked_by, get_links_to, get_preview_url, update, delete, get, + -- get_all, get_by_slug, get_categories, get_categories_with_counts, + -- get_tags, get_tags_with_counts, get_translation, get_translations, + -- has_published_version, is_slug_available, publish, + -- publish_translation, rebuild_from_files, rebuild_links, + -- reindex_text, search. + -- bds.media: delete_translation, filter, import, get_by_year_month, + -- get_file_path, update, delete, get, get_all, get_tags, + -- get_tags_with_counts, get_thumbnail, get_translation, + -- get_translations, get_url, rebuild_from_files, + -- regenerate_missing_thumbnails, regenerate_thumbnails, reindex_text, + -- replace_file, search, upsert_translation. + -- bds.scripts: create, update, delete, get, get_all, publish, + -- rebuild_from_files. + -- bds.templates: create, update, delete, get, get_all, publish, + -- get_enabled_by_kind, rebuild_from_files, validate. + -- bds.tags: create, update, delete, get, get_all, get_by_name, + -- get_posts_with_tag, get_with_counts, merge, rename, + -- sync_from_posts. + -- bds.tasks: get, status_snapshot, cancel, get_all, get_running, + -- clear_completed. + -- bds.publish: upload_site. + -- bds.chat: detect_post_language, analyze_post, translate_post, + -- analyze_media_image, detect_media_language, + -- translate_media_metadata. Despite its historical namespace, this is + -- the core one-shot AI bridge; conversational chat remains extension. + -- bds.report_progress(payload) reports managed-job progress. The Rust + -- host may retain bds.app.progress and bds.app.toast as additive APIs. + -- bds.sync and bds.embeddings belong to the extension contract. + + @guarantee HostApiFailureValues + -- Host failures do not expose implementation exceptions across the Lua + -- boundary. Lookup/create/update methods return nil on failure and + -- boolean command methods return false, as declared by the generated + -- method signature. Returned records contain only serializable public + -- fields; timestamps are ISO-8601 strings and enum atoms are strings. + + @guarantee GeneratedApiReferenceStaysInSync + -- One machine-readable method/type manifest generates the bundled Lua + -- API reference and editor completion data. A verification test fails + -- when the exposed capability table, manifest, or generated reference + -- diverge. Canonical macro, transform, and utility examples are bundled + -- under docs/scripting/ and execute against the same host API. + @guarantee MacroTimeout -- Macro execution has a short timeout budget of config.macro_timeout. @@ -130,7 +194,7 @@ invariant UniqueScriptSlug { invariant ScriptFileLayout { for s in Scripts where file_path != "": - s.file_path = format("scripts/{slug}.{extension}", slug: s.slug, extension: config.script_extension) + s.file_path = format("scripts/{slug}.lua", slug: s.slug) } -- Script files use standard --- YAML frontmatter @@ -185,7 +249,7 @@ rule CreateAndPublishScript { status: published, enabled: true, version: 1, - file_path: format("scripts/{slug}.{extension}", slug: slug, extension: config.script_extension) + file_path: format("scripts/{slug}.lua", slug: slug) ) ScriptFileWritten(new_script) } @@ -311,7 +375,7 @@ rule ExecuteTransform { rule RebuildScriptsFromFiles { when: RebuildScriptsFromFilesRequested(project) - for file in scan_directory(project.public_dir + "/scripts", "*." + config.script_extension): + for file in scan_directory(project.public_dir + "/scripts", "*.lua"): let parsed = parse_script_file(file) ensures: Script.created(parsed) }