diff --git a/AGENTS.md b/AGENTS.md index 475368f..951f4a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,15 @@ This document provides context and best practices for GitHub Copilot when workin - commit messages are short - one sentence. do not write long articles. - pull requests are more verbose and especially give reasoning for changes +## Important facts + +- published posts don't have body in the database, the body content is only in the file +- functionality you implement have to be tied to UI +- UI you implement has to be tied to functionality +- you must use drizzle to generate migrations and snapshots +- we use an sqlite database. use sqlite semantics in snapshots and other artifacts +- oh MacOS we use native menus and you have to hook them into the intercept for new menu items + --- ## ⚠️ MANDATORY: Test-First Development diff --git a/API.md b/API.md index 69d037b..e874f32 100644 --- a/API.md +++ b/API.md @@ -1,6 +1,6 @@ # API Documentation -Contract version: 1.11.0 +Contract version: 1.12.0 This reference documents all Python runtime API calls available through `bds_api` in embedded Pyodide. @@ -29,6 +29,7 @@ project = await bds.meta.get_project_metadata() - [chat](#chat) - [sync](#sync) - [publish](#publish) +- [embeddings](#embeddings) - [Data Structures](#data-structures) ## projects @@ -325,6 +326,7 @@ None # or - [posts.update](#postsupdate) - [posts.delete](#postsdelete) - [posts.get](#postsget) +- [posts.getBySlug](#postsgetbyslug) - [posts.getPreviewUrl](#postsgetpreviewurl) - [posts.getAll](#postsgetall) - [posts.getByStatus](#postsgetbystatus) @@ -500,6 +502,49 @@ None # or } ``` +### posts.getBySlug + +Fetch one post by slug. + +**Parameters** + +- slug (str, required) + +**Response specification** + +- Return type: `PostData | null` +- Nullability: Returns `None` when no matching value exists. +- Data structures: `PostData` + +**Example call** + +```python +from bds_api import bds +result = await bds.posts.get_by_slug(slug='slug') +``` + +**Example response** + +```python +None # or +{ + 'id': 'value', + 'projectId': 'value', + 'title': 'value', + 'slug': 'value', + 'excerpt': 'value', + 'content': 'value', + 'status': 'draft', + 'author': 'value', + 'language': 'value', + 'createdAt': 'value', + 'updatedAt': 'value', + 'publishedAt': 'value', + 'tags': 'value', + 'categories': 'value' +} +``` + ### posts.getPreviewUrl Get preview URL for post. @@ -2946,7 +2991,8 @@ result = await bds.meta.sync_on_startup() 'pythonRuntimeMode': 'webworker', 'picoTheme': 'value', 'categoryMetadata': {}, - 'categorySettings': {} + 'categorySettings': {}, + 'semanticSimilarityEnabled': False } ] ``` @@ -2988,7 +3034,8 @@ None # or 'pythonRuntimeMode': 'webworker', 'picoTheme': 'value', 'categoryMetadata': {}, - 'categorySettings': {} + 'categorySettings': {}, + 'semanticSimilarityEnabled': False } ``` @@ -3029,7 +3076,8 @@ None # or 'pythonRuntimeMode': 'webworker', 'picoTheme': 'value', 'categoryMetadata': {}, - 'categorySettings': {} + 'categorySettings': {}, + 'semanticSimilarityEnabled': False } ``` @@ -3070,7 +3118,8 @@ None # or 'pythonRuntimeMode': 'webworker', 'picoTheme': 'value', 'categoryMetadata': {}, - 'categorySettings': {} + 'categorySettings': {}, + 'semanticSimilarityEnabled': False } ``` @@ -3851,6 +3900,218 @@ result = await bds.publish.upload_site(credentials={}) [↑ Back to Table of contents](#table-of-contents) +## embeddings + +**Module APIs** + +- [embeddings.findSimilar](#embeddingsfindsimilar) +- [embeddings.computeSimilarities](#embeddingscomputesimilarities) +- [embeddings.getProgress](#embeddingsgetprogress) +- [embeddings.suggestTags](#embeddingssuggesttags) +- [embeddings.findDuplicates](#embeddingsfindduplicates) +- [embeddings.dismissPair](#embeddingsdismisspair) +- [embeddings.indexUnindexedPosts](#embeddingsindexunindexedposts) + +### embeddings.findSimilar + +Find posts semantically similar to the given post. Requires semantic similarity to be enabled in project settings. + +**Parameters** + +- postId (str, required) +- k (int | float, optional) + +**Response specification** + +- Return type: `SimilarPost[]` +- Data structures: `SimilarPost` + +**Example call** + +```python +from bds_api import bds +result = await bds.embeddings.find_similar(post_id='post-1') +``` + +**Example response** + +```python +[ +{ + 'postId': 'value', + 'similarity': 0 +} +] +``` + +### embeddings.computeSimilarities + +Compute cosine similarity between a source post and a list of target posts. Returns a mapping of target post IDs to similarity scores (0.0-1.0). Posts without embeddings are omitted. + +**Parameters** + +- sourcePostId (str, required) +- targetPostIds (list, required) + +**Response specification** + +- Return type: `Record` + +**Example call** + +```python +from bds_api import bds +result = await bds.embeddings.compute_similarities(source_post_id='source_post-1', target_post_ids=[]) +``` + +**Example response** + +```python +{} +``` + +### embeddings.getProgress + +Get the embedding indexing progress for the active project. + +**Parameters** + +- None + +**Response specification** + +- Return type: `{ indexed: number; total: number }` + +**Example call** + +```python +from bds_api import bds +result = await bds.embeddings.get_progress() +``` + +**Example response** + +```python +{} +``` + +### embeddings.suggestTags + +Suggest tags for a post based on tags used by semantically similar posts. + +**Parameters** + +- postId (str, required) +- excludeTags (list, required) + +**Response specification** + +- Return type: `TagSuggestion[]` +- Data structures: `TagSuggestion` + +**Example call** + +```python +from bds_api import bds +result = await bds.embeddings.suggest_tags(post_id='post-1', exclude_tags=[]) +``` + +**Example response** + +```python +[ +{ + 'name': 'value', + 'score': 0 +} +] +``` + +### embeddings.findDuplicates + +Find post pairs with high content similarity (potential duplicates). Threshold is a similarity value from 0.0 to 1.0 (default 0.85). + +**Parameters** + +- threshold (int | float, optional) + +**Response specification** + +- Return type: `DuplicatePair[]` +- Data structures: `DuplicatePair` + +**Example call** + +```python +from bds_api import bds +result = await bds.embeddings.find_duplicates() +``` + +**Example response** + +```python +[ +{ + 'postA': 'value', + 'postB': 'value', + 'similarity': 0 +} +] +``` + +### embeddings.dismissPair + +Dismiss a duplicate pair so it no longer appears in results. + +**Parameters** + +- postIdA (str, required) +- postIdB (str, required) + +**Response specification** + +- Return type: `void` + +**Example call** + +```python +from bds_api import bds +result = await bds.embeddings.dismiss_pair(post_id_a='post_id_a-1', post_id_b='post_id_b-1') +``` + +**Example response** + +```python +None +``` + +### embeddings.indexUnindexedPosts + +Trigger background indexing of all posts not yet embedded. + +**Parameters** + +- None + +**Response specification** + +- Return type: `void` + +**Example call** + +```python +from bds_api import bds +result = await bds.embeddings.index_unindexed_posts() +``` + +**Example response** + +```python +None +``` + +[↑ Back to Table of contents](#table-of-contents) + ## Data Structures Shared structures referenced by response types are defined once here. @@ -4021,6 +4282,7 @@ Extended project metadata from project settings. - picoTheme (`string`, optional): Preferred Pico theme token. - categoryMetadata (`object`, optional): Category metadata keyed by category slug. - categorySettings (`object`, optional): Category render settings keyed by category slug. +- semanticSimilarityEnabled (`boolean`, optional): Enable local ONNX embedding-based semantic similarity features. [↑ Back to Table of contents](#table-of-contents) @@ -4113,6 +4375,40 @@ Result from AI image analysis containing generated title, alt text, and caption. [↑ Back to Table of contents](#table-of-contents) +### SimilarPost + +A post with its semantic similarity score relative to a reference post. + +**Fields** + +- postId (`string`, required): Post identifier. +- similarity (`number`, required): Cosine similarity score from 0.0 to 1.0. + +[↑ Back to Table of contents](#table-of-contents) + +### TagSuggestion + +A tag suggested based on semantic similarity to similar posts. + +**Fields** + +- name (`string`, required): Tag name. +- score (`number`, required): Aggregated suggestion score. + +[↑ Back to Table of contents](#table-of-contents) + +### DuplicatePair + +A pair of posts with high content similarity that may be duplicates. + +**Fields** + +- postA (`{ id: string; title: string; slug: string; publishedAt?: string }`, required): First post in the pair. +- postB (`{ id: string; title: string; slug: string; publishedAt?: string }`, required): Second post in the pair. +- similarity (`number`, required): Cosine similarity score from 0.0 to 1.0. + +[↑ Back to Table of contents](#table-of-contents) + --- -Generated from contract at 2026-02-27T00:00:00.000Z. +Generated from contract at 2026-03-05T00:00:00.000Z. diff --git a/BDS_SEMANTIC_SIMILARITY.md b/BDS_SEMANTIC_SIMILARITY.md deleted file mode 100644 index ab319d2..0000000 --- a/BDS_SEMANTIC_SIMILARITY.md +++ /dev/null @@ -1,370 +0,0 @@ -# Semantic Similarity in bDS - -Surface thematically related posts as an impulse — "Have I written something similar?" — inspired by Luhmann's Zettelkasten. Cross-domain connections across 10k+ posts over 20 years are the point, not a flaw. The algorithm finds the surface. The human finds the depth. - -**Status: Not yet implemented.** No packages installed, no engine, no IPC, no UI integration. - ---- - -## Integration Point - -**InsertModal** (`src/renderer/components/InsertModal/InsertModal.tsx`), link mode. - -When the search field is empty (`query.length < 2`), instead of showing "type at least 2 characters", show 3–5 semantically similar posts to the currently edited post. These are default suggestions — "posts you might want to link to." - -The InsertModal currently accepts these props: -```ts -interface InsertModalProps { - mode: InsertMode; - onInsertLink: (url: string, text?: string) => void; - onInsertImage: (url: string, alt: string, mediaId?: string) => void; - onClose: () => void; - initialText?: string; - currentPostTags?: string[]; - currentPostCategories?: string[]; - // currentPostId is NOT yet threaded through — needs to be added -} -``` - -`currentPostId` must be added to this interface and threaded from `Editor.tsx`. - -Note: The InsertModal now also has a **"Create post" option** — when `query.length >= 2` and no exact title match exists, it shows a `+` button to create a new post with that title and inherit the current post's tags/categories. This is the only UI addition since the original spec; it doesn't conflict with the semantic similarity integration. - ---- - -## Stack - -| Purpose | Library | npm | Notes | -|---|---|---|---| -| Embeddings | Hugging Face Transformers.js | `@huggingface/transformers` | ONNX, local, no API key | -| Vector index | USearch | `usearch` | HNSW, native C++ via N-API, prebuilt binaries | - -Neither package is installed yet. - -**Embedding model:** `multilingual-e5-small` — 384 dimensions, 512-token context, ~470 MB on disk, ~200–300 MB RAM, ~100ms/post inference. Natively multilingual (100+ languages incl. DE/EN) — critical for a mixed-language blog. `all-MiniLM-L6-v2` (~90 MB) was considered but is EN-trained with weak DE transfer; not suitable for nuanced cross-language similarity. - -**Why USearch over alternatives:** -- `sqlite-vec` — requires `loadExtension()` on the SQLite driver; bDS uses `@libsql/client` which doesn't expose it. Eliminated. -- `hnswlib-node` — no prebuilt binaries, requires `node-gyp` compile. Last published 2 years ago. Risk with Electron packaging. -- `vectra` — pure JS, zero build issues, but JSON storage (~30 MB for 10k posts). Acceptable fallback. -- Brute-force in JS — works at 10k (~15ms for the math) but requires loading all embeddings from DB first. DB read overhead with `@libsql/client` FFI is unknown and potentially dominant. -- **USearch** — prebuilt binaries via `prebuildify` (matches `sharp`, `@libsql/client` pattern), actively maintained, HNSW with SIMD, <1ms queries, binary persistence (~6 MB for 10k×384). - -**USearch specifics:** -- Keys are `BigUint64Array` — need a `Map` (numeric label → post UUID) persisted in a small Drizzle table (`embedding_keys`) -- `index.load()` loads everything into RAM (~6 MB). `index.save()` is a full rewrite. Fine for this scale. -- No incremental flush / WAL — acceptable since mutations are one-at-a-time post edits - -**Electron packaging risk:** USearch uses N-API, but verify that its `prebuildify` targets include the Electron ABI for all platforms (macOS arm64/x64, Windows x64/arm64, Linux x64) before committing. Spike this first — if binaries are missing, fall back to `vectra`. - ---- - -## Architecture - -### Files on disk - -``` -{userData}/projects/{projectId}/ - embeddings.usearch # USearch binary index -``` - -The `bigint → postId` key mapping lives in a Drizzle table (`embedding_keys`), not a JSON file — avoids `bigint` JSON serialization issues and stays atomic with the existing DB. - -### Engine: `EmbeddingEngine` (`src/main/engine/EmbeddingEngine.ts`) - -File does not exist yet. Create it. - -Responsibilities: -- Load/save USearch index + key map on startup/shutdown -- Embed post content via `@huggingface/transformers` -- Add/update/remove embeddings when posts change -- Query: given a post ID, return top-k similar post IDs with distances - -Key interface: -```ts -class EmbeddingEngine { - async initialize(): Promise // load index + model - async embedPost(postId: string, content: string): Promise - async removePost(postId: string): Promise - async findSimilar(postId: string, k?: number): Promise - async getIndexingProgress(): Promise<{ indexed: number; total: number }> - async reindexAll(): Promise // after databaseRebuilt - async setProjectContext(projectId: string): Promise // load/unload on switch - async save(): Promise -} -``` - -### EngineBundle (`src/main/engine/EngineBundle.ts`) - -Add `embeddingEngine: EmbeddingEngine` to the `EngineBundle` interface. The current bundle contains: -`postEngine`, `mediaEngine`, `scriptEngine`, `templateEngine`, `metaEngine`, `menuEngine`, `tagEngine`, `postMediaEngine`, `projectEngine`, `gitEngine`, `gitApiAdapter`, `blogGenerationEngine`, `publishEngine`, `metadataDiffEngine`, `taskManager`, `blogmarkTransformService`, `mcpServer`, `blogmarkPythonWorkerRuntime`, `pythonMacroWorkerRuntime`, `publishApiAdapter`, `appApiAdapter`. - -### IPC layer (`src/main/ipc/`) - -The IPC layer now has five files: -- `handlers.ts` — main handler file (posts, media, project, meta, tags, templates, scripts, blog generation, publishing, preview, site validation, import, settings, model catalog, tasks, notifications) -- `chatHandlers.ts` — AI chat streaming & tool use -- `blogHandlers.ts` — blog generation & publishing -- `publishHandlers.ts` — publishing -- `metadataDiffHandlers.ts` — metadata diff -- `index.ts` — module exports - -Add the embedding IPC handlers to `handlers.ts` (they're small; no need for a new file): - -``` -embeddings:findSimilar(postId: string, k?: number) → SimilarPost[] -embeddings:getProgress() → { indexed: number; total: number } -embeddings:suggestTags(postId: string, excludeTags: string[]) → TagSuggestion[] -embeddings:findDuplicates(threshold?: number) → DuplicatePair[] -embeddings:dismissPair(postIdA: string, postIdB: string) → void -``` - -### Database: `embedding_keys` table - -Add to `src/main/database/schema.ts`. The current schema has: `projects`, `posts`, `media`, `settings`, `generatedFileHashes`, `postLinks`, `postMedia`, `tags`, `chatConversations`, `chatMessages`, `importDefinitions`, `scripts`, `templates`, `dbNotifications`, `modelCatalogProviders`, `modelCatalog`, `modelCatalogModalities`, `modelCatalogMeta`. - -New tables: -```ts -export const embeddingKeys = sqliteTable('embedding_keys', { - label: integer('label', { mode: 'bigint' }).primaryKey(), // USearch bigint key - postId: text('post_id').notNull(), - projectId: text('project_id').notNull(), -}); - -export const dismissedDuplicatePairs = sqliteTable('dismissed_duplicate_pairs', { - id: text('id').primaryKey(), - projectId: text('project_id').notNull(), - postIdA: text('post_id_a').notNull(), - postIdB: text('post_id_b').notNull(), - dismissedAt: integer('dismissed_at', { mode: 'timestamp' }).notNull(), -}, (table) => ({ - pairIdx: uniqueIndex('dismissed_pairs_idx').on(table.projectId, table.postIdA, table.postIdB), -})); -``` - -Create a Drizzle migration (`db:generate` / `db:migrate`) after adding the tables. - -### Project switching - -The app supports multiple projects. On project switch (`setProjectContext`), the engine must save and unload the current index, then load (or create) the index for the new project. Each project has its own `embeddings.usearch` file and `embedding_keys` table rows (filtered by `projectId`). - -### Embedding content - -Embed the raw markdown body of each post (title + content). Markdown's lightweight markup (headers, links, emphasis) adds minimal noise and preserves semantic structure well enough for transformer models. No stripping needed. - -**Chunking for long posts:** The model's 512-token context (~400 words) covers most posts. For posts exceeding 512 tokens: -1. Split into 512-token chunks with ~50 token overlap -2. Embed each chunk independently -3. Mean-pool the chunk vectors into a single 384-dim embedding -4. Store the single pooled vector in the index - -This keeps the index simple (one vector per post, one lookup per query) while preserving semantic coverage of long-form content. The overlap prevents losing context at chunk boundaries. - -### Hook into existing post lifecycle - -`PostEngine` emits these events (confirmed in current codebase): -- `postCreated` — on create and on import -- `postUpdated` — on update, publish, revert -- `postDeleted` — on delete -- `databaseRebuilt` — emitted after `reconcileFromDisk()` (e.g., git sync replaces entire DB) -- `rebuildStarted` — emitted just before `databaseRebuilt` - -On post content change → call `embeddingEngine.embedPost()`. On delete → call `embeddingEngine.removePost()`. On `databaseRebuilt` → trigger a full reindex. - -Save strategy: debounce `index.save()` on a timer (e.g., 5s after last mutation). During bulk indexing, batch-save every N posts (e.g., 100) instead of after each one — avoids 10k full file rewrites. - -### Initial indexing (10k+ posts) - -- ~100ms per post × 10k = **~17 minutes** one-time background job -- Must run as a low-priority background task after app startup (use `TaskManager` for queuing) -- Emit progress events so UI can show "Indexing 3,421 / 10,247…" -- On git sync to new machine, file watchers fire for all posts → triggers full reindex automatically -- Model download (~470 MB) on first run — needs progress indicator or opt-in preference - ---- - -## UI Changes - -### InsertModal (link mode, internal tab) - -**When `query.length < 2` and `currentPostId` is set:** -1. Call `embeddings:findSimilar(currentPostId, 5)` on mount -2. Show results in the same result list format, with a subtle header like "Related posts" -3. Clicking a suggestion works identically to a search result — inserts the link - -**When `query.length >= 2`:** existing search behavior, unchanged. (This includes the "Create post" option for link mode.) - -**Fallback:** if embeddings aren't ready (indexing in progress, feature disabled), show the existing "type at least 2 characters" message. - ---- - -### TagInput (post tag editing) - -`TagInput` (`src/renderer/components/TagInput/TagInput.tsx`) already shows a suggestions dropdown driven by text input. Add a second suggestion source: tags inferred from semantically similar posts. - -Add an optional `postId?: string` prop. `PostEditor` already has `postId` and renders `TagInput`, so threading is a one-liner. - -**When `inputValue.length === 0` and the input is focused and `postId` is set:** -1. Call `embeddings:suggestTags(postId, currentTags)` once on focus (cache the result for the session) -2. Show a "Suggested" section at the top of the dropdown, above the regular tag list -3. Clicking a suggested tag adds it identically to any other tag - -**When `inputValue.length > 0`:** existing text-filter behavior, suggested section hidden. - -**Fallback:** if embeddings aren't ready or `postId` is absent, the dropdown behaves exactly as today — no visible change. - -**Algorithm** (in `EmbeddingEngine.suggestTags`): -1. Find top-10 similar posts via `findSimilar(postId, 10)` -2. Collect tags from each neighbour, weighted by similarity score -3. Sum weights per tag (tag appearing in 3 posts at 0.9 similarity scores higher than tag in 1 post at 0.95) -4. Filter out tags the current post already has -5. Return top 5 by weighted score - -```ts -interface TagSuggestion { - name: string; - score: number; // weighted frequency, for ranking only — not shown in UI -} - -async suggestTags(postId: string, excludeTags: string[]): Promise -``` - -New IPC endpoint (add to `handlers.ts`): -``` -embeddings:suggestTags(postId: string, excludeTags: string[]) → TagSuggestion[] -``` - -No new DB table needed — this is a pure read from the existing index. - ---- - -## Duplication Analysis - -A periodic audit tool to surface posts that are so semantically similar they might be unintentional duplicates — the same topic written twice years apart, a post and its draft that both got published, a cross-post that was forgotten. The goal is human review and action, not automated deletion. - -### Algorithm - -For each indexed post, query the index for its top-k nearest neighbours (k=20). Filter pairs where cosine similarity exceeds a threshold (default: 0.92). Deduplicate symmetric pairs (A→B = B→A). Sort descending by similarity. - -This is O(n) queries against the HNSW index — fast even at 10k posts (~20ms total). Run on demand, not continuously. - -```ts -interface DuplicatePair { - postA: { id: string; title: string; slug: string; publishedAt?: Date }; - postB: { id: string; title: string; slug: string; publishedAt?: Date }; - similarity: number; // cosine similarity, 0–1 -} -``` - -New engine method: -```ts -async findDuplicates(threshold?: number): Promise -// threshold default: 0.92. Lower = more results, more noise. Higher = only near-identical posts. -``` - -### Dismissed pairs - -When the user reviews a pair and decides it's intentional (e.g., two posts on the same topic that are meaningfully different), they can dismiss it. Store dismissed pairs in a new DB table so they don't reappear: - -```ts -export const dismissedDuplicatePairs = sqliteTable('dismissed_duplicate_pairs', { - id: text('id').primaryKey(), - projectId: text('project_id').notNull(), - postIdA: text('post_id_a').notNull(), - postIdB: text('post_id_b').notNull(), - dismissedAt: integer('dismissed_at', { mode: 'timestamp' }).notNull(), -}, (table) => ({ - pairIdx: uniqueIndex('dismissed_pairs_idx').on(table.projectId, table.postIdA, table.postIdB), -})); -``` - -`findDuplicates` filters out any pair where both (A, B) and (B, A) appear in `dismissed_duplicate_pairs`. - -### New IPC endpoints - -Add to `handlers.ts`: -``` -embeddings:findDuplicates(threshold?: number) → DuplicatePair[] -embeddings:dismissPair(postIdA: string, postIdB: string) → void -``` - -### UI placement - -The duplication analysis is a periodic audit task — the same category as `validateSite` and `metadataDiff`, both of which already live in the **Blog menu** in `src/main/shared/menuCommands.ts`. Add `findDuplicates` there, next to `validateSite`. - -Required changes to `menuCommands.ts`: -- Add `'findDuplicates'` to `AppMenuAction` union -- Add menu item to the Blog group next to `validateSite`: `{ label: 'menu.item.findDuplicates', action: 'findDuplicates' }` -- Add to `APP_MENU_ACTION_EVENT_MAP`: `findDuplicates: 'menu:findDuplicates'` - -The renderer listens for `menu:findDuplicates` and opens the duplicates tab (same pattern as `menu:validateSite` → `SiteValidationView`). - -Results open as a **dedicated tab** (new tab type: `duplicates`) in the main editor area, so the user can keep it open while navigating to individual posts. Tab type should be added to the `Tab` union in `appStore`. - -**Duplicates tab layout:** -``` -┌─────────────────────────────────────────────────────────┐ -│ Potential Duplicates [Threshold: ▼ 92%] [Re-run] │ -├─────────────────────────────────────────────────────────┤ -│ 97% "My trip to Berlin" (2019-03) │ -│ "Berlin travel notes" (2023-08) [Open both] │ -│ [Dismiss] │ -├─────────────────────────────────────────────────────────┤ -│ 94% "Bullet journaling setup" (2018-11) │ -│ "How I use a bullet journal" (2021-02)[Open both] │ -│ [Dismiss] │ -└─────────────────────────────────────────────────────────┘ -``` - -- **Threshold slider** — adjustable 80–99%, results update on re-run -- **"Open both"** — opens both posts as sequential editor tabs -- **"Dismiss"** — calls `embeddings:dismissPair`, removes the row from the list -- Results show similarity %, both post titles, and published dates -- If no duplicates found at the current threshold: "No duplicates found above X% similarity" -- If index not ready: "Semantic index is still building…" with progress - -### Python API - -Add to `bds_api` and `API.md`: -```python -posts.find_duplicates(threshold=0.92) # → list of DuplicatePair -posts.dismiss_duplicate_pair(post_id_a, post_id_b) # → None -``` - ---- - -## Settings: Opt-In Preference - -The feature must be opt-in (model download + 17 min indexing is not a silent default). - -Store as a project-level metadata field via `meta:updateProjectMetadata`. Add `semanticSimilarityEnabled: boolean` to `ProjectMetadata`. When the user enables it in Project Settings, start the background indexer. When disabled, skip embedding hooks and hide the UI section. - -The model download itself (~470 MB) should show a progress indicator before the indexer starts. - ---- - -## Implementation Steps - -1. **Spike USearch packaging** — verify prebuilt binaries exist for all target Electron ABIs before committing. Fall back to `vectra` if they don't. -2. **Test + implement `EmbeddingEngine`** — model loading, embed, add/remove/query against USearch index, save/load persistence -3. **Drizzle key map table** — add `embedding_keys` to `schema.ts`, run `db:generate` + `db:migrate` -4. **Add `semanticSimilarityEnabled` to project metadata** — `ProjectMetadata` type + `meta:updateProjectMetadata` handler + Project Settings UI toggle -5. **Wire into post lifecycle** — hook `postCreated`/`postUpdated`/`postDeleted`/`databaseRebuilt` → embedding updates (guarded by opt-in flag) -6. **Background indexer** — on startup (if enabled), diff indexed vs. existing posts, queue unindexed for background embedding via `TaskManager` with progress events -7. **IPC endpoints** — `embeddings:findSimilar`, `embeddings:getProgress`, `embeddings:findDuplicates`, `embeddings:dismissPair` in `handlers.ts` -8. **Add `embeddingEngine` to `EngineBundle`** — update `EngineBundle.ts` interface and `main.ts` construction -9. **InsertModal integration** — add `currentPostId` prop, thread from `Editor.tsx`, fetch similar on mount, render as default suggestions -10. **Duplicates tab** — add `'findDuplicates'` to `AppMenuAction` + Blog menu group + `APP_MENU_ACTION_EVENT_MAP` in `menuCommands.ts`; add `duplicates` to `Tab` union in `appStore`; implement `DuplicatesView` component wired to `menu:findDuplicates` event -11. **I18n** — all new UI strings through locale files (no hardcoded text) -12. **Python API** — add `posts.findRelated(postId, k)`, `posts.find_duplicates(threshold)`, `posts.dismiss_duplicate_pair(a, b)` to `bds_api`, regenerate `API.md` - ---- - -## Constraints - -- Feature must be opt-in (model download + 17 min indexing is not a silent default) -- No external API calls — fully local -- Model cached in `~/.cache/huggingface/`, index in internal project directory -- Total added footprint: ~520 MB on disk (onnxruntime-node ~50 MB + model ~470 MB), ~300 MB RAM at runtime for model + index -- Graceful degradation: if USearch native module fails to load (unsupported platform), disable the feature silently — never crash the app -- Follow test-first mandate: write failing tests before implementing `EmbeddingEngine` diff --git a/drizzle/0011_loving_alex_wilder.sql b/drizzle/0010_loving_alex_wilder.sql similarity index 100% rename from drizzle/0011_loving_alex_wilder.sql rename to drizzle/0010_loving_alex_wilder.sql diff --git a/drizzle/0011_embedding_tables.sql b/drizzle/0011_embedding_tables.sql new file mode 100644 index 0000000..e5b1f2d --- /dev/null +++ b/drizzle/0011_embedding_tables.sql @@ -0,0 +1,15 @@ +CREATE TABLE `dismissed_duplicate_pairs` ( + `id` text PRIMARY KEY NOT NULL, + `project_id` text NOT NULL, + `post_id_a` text NOT NULL, + `post_id_b` text NOT NULL, + `dismissed_at` integer NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `dismissed_pairs_idx` ON `dismissed_duplicate_pairs` (`project_id`,`post_id_a`,`post_id_b`);--> statement-breakpoint +CREATE TABLE `embedding_keys` ( + `label` integer PRIMARY KEY NOT NULL, + `post_id` text NOT NULL, + `project_id` text NOT NULL, + `content_hash` text NOT NULL +); diff --git a/drizzle/0012_flimsy_meteorite.sql b/drizzle/0012_flimsy_meteorite.sql new file mode 100644 index 0000000..b5ef5bb --- /dev/null +++ b/drizzle/0012_flimsy_meteorite.sql @@ -0,0 +1 @@ +ALTER TABLE `embedding_keys` ADD `vector` blob; \ No newline at end of file diff --git a/drizzle/meta/0010_snapshot.json b/drizzle/meta/0010_snapshot.json new file mode 100644 index 0000000..2bf91f7 --- /dev/null +++ b/drizzle/meta/0010_snapshot.json @@ -0,0 +1,1439 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "3312ee41-2b01-4579-aa9b-7fd4f0f3c7a4", + "prevId": "b3e7e63c-d906-48af-98a0-e3d0741ff13a", + "tables": { + "chat_conversations": { + "name": "chat_conversations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "copilot_session_id": { + "name": "copilot_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_messages": { + "name": "chat_messages", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "db_notifications": { + "name": "db_notifications", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "entity": { + "name": "entity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_cli": { + "name": "from_cli", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "seen_at": { + "name": "seen_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "generated_file_hashes": { + "name": "generated_file_hashes", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relative_path": { + "name": "relative_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "generated_file_hashes_project_path_idx": { + "name": "generated_file_hashes_project_path_idx", + "columns": [ + "project_id", + "relative_path" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "import_definitions": { + "name": "import_definitions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "wxr_file_path": { + "name": "wxr_file_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploads_folder_path": { + "name": "uploads_folder_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_analysis_result": { + "name": "last_analysis_result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "media": { + "name": "media", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "alt": { + "name": "alt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "caption": { + "name": "caption", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sidecar_path": { + "name": "sidecar_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_models": { + "name": "ai_models", + "columns": { + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "family": { + "name": "family", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attachment": { + "name": "attachment", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "reasoning": { + "name": "reasoning", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "tool_call": { + "name": "tool_call", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "structured_output": { + "name": "structured_output", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "temperature": { + "name": "temperature", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "knowledge": { + "name": "knowledge", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "release_date": { + "name": "release_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_updated_date": { + "name": "last_updated_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "open_weights": { + "name": "open_weights", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "input_price": { + "name": "input_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_price": { + "name": "output_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cache_read_price": { + "name": "cache_read_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cache_write_price": { + "name": "cache_write_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window": { + "name": "context_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_input_tokens": { + "name": "max_input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_output_tokens": { + "name": "max_output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interleaved": { + "name": "interleaved", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_npm": { + "name": "provider_npm", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ai_models_provider_model_id_pk": { + "columns": [ + "provider", + "model_id" + ], + "name": "ai_models_provider_model_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_catalog_meta": { + "name": "ai_catalog_meta", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_model_modalities": { + "name": "ai_model_modalities", + "columns": { + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "modality": { + "name": "modality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ai_model_modalities_provider_model_id_direction_modality_pk": { + "columns": [ + "provider", + "model_id", + "direction", + "modality" + ], + "name": "ai_model_modalities_provider_model_id_direction_modality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_providers": { + "name": "ai_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "env": { + "name": "env", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "npm": { + "name": "npm", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api": { + "name": "api", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "doc": { + "name": "doc", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "post_links": { + "name": "post_links", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_post_id": { + "name": "source_post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_post_id": { + "name": "target_post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "link_text": { + "name": "link_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "post_media": { + "name": "post_media", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "media_id": { + "name": "media_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "post_media_post_media_idx": { + "name": "post_media_post_media_idx", + "columns": [ + "post_id", + "media_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "posts": { + "name": "posts", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "excerpt": { + "name": "excerpt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "categories": { + "name": "categories", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "template_slug": { + "name": "template_slug", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_title": { + "name": "published_title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_content": { + "name": "published_content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_tags": { + "name": "published_tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_categories": { + "name": "published_categories", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_excerpt": { + "name": "published_excerpt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "posts_project_slug_idx": { + "name": "posts_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data_path": { + "name": "data_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "projects_slug_unique": { + "name": "projects_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scripts": { + "name": "scripts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'utility'" + }, + "entrypoint": { + "name": "entrypoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'render'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'published'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "scripts_project_slug_idx": { + "name": "scripts_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tags": { + "name": "tags", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "post_template_slug": { + "name": "post_template_slug", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tags_project_name_idx": { + "name": "tags_project_name_idx", + "columns": [ + "project_id", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'post'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'published'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "templates_project_slug_idx": { + "name": "templates_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/0011_snapshot.json b/drizzle/meta/0011_snapshot.json index d1aa865..613c025 100644 --- a/drizzle/meta/0011_snapshot.json +++ b/drizzle/meta/0011_snapshot.json @@ -1,8 +1,8 @@ { "version": "6", "dialect": "sqlite", - "id": "3312ee41-2b01-4579-aa9b-7fd4f0f3c7a4", - "prevId": "b3e7e63c-d906-48af-98a0-e3d0741ff13a", + "id": "ac24604e-89e5-4def-95a4-1d175c4126f8", + "prevId": "3312ee41-2b01-4579-aa9b-7fd4f0f3c7a4", "tables": { "chat_conversations": { "name": "chat_conversations", @@ -971,13 +971,6 @@ "notNull": false, "autoincrement": false }, - "language": { - "name": "language", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, "published_title": { "name": "published_title", "type": "text", @@ -1012,6 +1005,13 @@ "primaryKey": false, "notNull": false, "autoincrement": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false } }, "indexes": { @@ -1424,6 +1424,99 @@ "compositePrimaryKeys": {}, "uniqueConstraints": {}, "checkConstraints": {} + }, + "embedding_keys": { + "name": "embedding_keys", + "columns": { + "label": { + "name": "label", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dismissed_duplicate_pairs": { + "name": "dismissed_duplicate_pairs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "post_id_a": { + "name": "post_id_a", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "post_id_b": { + "name": "post_id_b", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dismissed_at": { + "name": "dismissed_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "dismissed_pairs_idx": { + "name": "dismissed_pairs_idx", + "columns": [ + "project_id", + "post_id_a", + "post_id_b" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} } }, "views": {}, diff --git a/drizzle/meta/0012_snapshot.json b/drizzle/meta/0012_snapshot.json new file mode 100644 index 0000000..fbea3bc --- /dev/null +++ b/drizzle/meta/0012_snapshot.json @@ -0,0 +1,1539 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "dad2558b-2619-4afd-9e67-dd3b1393df28", + "prevId": "ac24604e-89e5-4def-95a4-1d175c4126f8", + "tables": { + "chat_conversations": { + "name": "chat_conversations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "copilot_session_id": { + "name": "copilot_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_messages": { + "name": "chat_messages", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "db_notifications": { + "name": "db_notifications", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "entity": { + "name": "entity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_cli": { + "name": "from_cli", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "seen_at": { + "name": "seen_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dismissed_duplicate_pairs": { + "name": "dismissed_duplicate_pairs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "post_id_a": { + "name": "post_id_a", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "post_id_b": { + "name": "post_id_b", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dismissed_at": { + "name": "dismissed_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "dismissed_pairs_idx": { + "name": "dismissed_pairs_idx", + "columns": [ + "project_id", + "post_id_a", + "post_id_b" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "embedding_keys": { + "name": "embedding_keys", + "columns": { + "label": { + "name": "label", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "vector": { + "name": "vector", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "generated_file_hashes": { + "name": "generated_file_hashes", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relative_path": { + "name": "relative_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "generated_file_hashes_project_path_idx": { + "name": "generated_file_hashes_project_path_idx", + "columns": [ + "project_id", + "relative_path" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "import_definitions": { + "name": "import_definitions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "wxr_file_path": { + "name": "wxr_file_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploads_folder_path": { + "name": "uploads_folder_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_analysis_result": { + "name": "last_analysis_result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "media": { + "name": "media", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "alt": { + "name": "alt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "caption": { + "name": "caption", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sidecar_path": { + "name": "sidecar_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_models": { + "name": "ai_models", + "columns": { + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "family": { + "name": "family", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attachment": { + "name": "attachment", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "reasoning": { + "name": "reasoning", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "tool_call": { + "name": "tool_call", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "structured_output": { + "name": "structured_output", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "temperature": { + "name": "temperature", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "knowledge": { + "name": "knowledge", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "release_date": { + "name": "release_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_updated_date": { + "name": "last_updated_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "open_weights": { + "name": "open_weights", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "input_price": { + "name": "input_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_price": { + "name": "output_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cache_read_price": { + "name": "cache_read_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cache_write_price": { + "name": "cache_write_price", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window": { + "name": "context_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_input_tokens": { + "name": "max_input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_output_tokens": { + "name": "max_output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interleaved": { + "name": "interleaved", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_npm": { + "name": "provider_npm", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ai_models_provider_model_id_pk": { + "columns": [ + "provider", + "model_id" + ], + "name": "ai_models_provider_model_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_catalog_meta": { + "name": "ai_catalog_meta", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_model_modalities": { + "name": "ai_model_modalities", + "columns": { + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "modality": { + "name": "modality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ai_model_modalities_provider_model_id_direction_modality_pk": { + "columns": [ + "provider", + "model_id", + "direction", + "modality" + ], + "name": "ai_model_modalities_provider_model_id_direction_modality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ai_providers": { + "name": "ai_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "env": { + "name": "env", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "npm": { + "name": "npm", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api": { + "name": "api", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "doc": { + "name": "doc", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "post_links": { + "name": "post_links", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_post_id": { + "name": "source_post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_post_id": { + "name": "target_post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "link_text": { + "name": "link_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "post_media": { + "name": "post_media", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "media_id": { + "name": "media_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "post_media_post_media_idx": { + "name": "post_media_post_media_idx", + "columns": [ + "post_id", + "media_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "posts": { + "name": "posts", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "excerpt": { + "name": "excerpt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "categories": { + "name": "categories", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "template_slug": { + "name": "template_slug", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_title": { + "name": "published_title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_content": { + "name": "published_content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_tags": { + "name": "published_tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_categories": { + "name": "published_categories", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_excerpt": { + "name": "published_excerpt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "posts_project_slug_idx": { + "name": "posts_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data_path": { + "name": "data_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "projects_slug_unique": { + "name": "projects_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scripts": { + "name": "scripts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'utility'" + }, + "entrypoint": { + "name": "entrypoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'render'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'published'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "scripts_project_slug_idx": { + "name": "scripts_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tags": { + "name": "tags", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "post_template_slug": { + "name": "post_template_slug", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tags_project_name_idx": { + "name": "tags_project_name_idx", + "columns": [ + "project_id", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "templates": { + "name": "templates", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'post'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'published'" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "templates_project_slug_idx": { + "name": "templates_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index c590da5..89c3101 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -76,7 +76,21 @@ "idx": 10, "version": "6", "when": 1772462693094, - "tag": "0011_loving_alex_wilder", + "tag": "0010_loving_alex_wilder", + "breakpoints": true + }, + { + "idx": 11, + "version": "6", + "when": 1772711489237, + "tag": "0011_embedding_tables", + "breakpoints": true + }, + { + "idx": 12, + "version": "6", + "when": 1772738580546, + "tag": "0012_flimsy_meteorite", "breakpoints": true } ] diff --git a/package-lock.json b/package-lock.json index 187f2f1..67907a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@dagrejs/dagre": "^2.0.4", "@floating-ui/dom": "^1.7.5", "@highlightjs/cdn-assets": "^11.11.1", + "@huggingface/transformers": "^3.8.1", "@libsql/client": "^0.17.0", "@milkdown/kit": "^7.18.0", "@milkdown/plugin-block": "^7.18.0", @@ -57,6 +58,7 @@ "snowball-stemmers": "^0.6.0", "transliteration": "^2.6.1", "turndown": "^7.2.2", + "usearch": "^2.21.4", "uuid": "^13.0.0", "vanilla-calendar-pro": "^3.1.0", "zod": "^4.3.6", @@ -290,6 +292,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -868,6 +871,7 @@ "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.12.1.tgz", "integrity": "sha512-Fa6xkSiuGKc8XC8Cn96T+TQHYj4ZZ7RdFmXA3i9xe/3hLHfwPZdM+dqfX0Cp0zQklBKhVD8Yzc8LS45rkqcwpQ==", "license": "MIT", + "peer": true, "dependencies": { "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.23.0", @@ -944,6 +948,7 @@ "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.4.tgz", "integrity": "sha512-8y7xqG/hpB53l25CIoit9/ngxdfoG+fx+V3SHBrinnhOtLvKHRyAJJuHzkWrR4YXXLX8eXBsejgAAxHUOdW1yw==", "license": "MIT", + "peer": true, "dependencies": { "@marijn/find-cluster-break": "^1.0.0" } @@ -965,6 +970,7 @@ "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.39.13.tgz", "integrity": "sha512-QBO8ZsgJLCbI28KdY0/oDy5NQLqOQVZCozBknxc2/7L98V+TVYFHnfaCsnGh1U+alpd2LOkStVwYY7nW2R1xbw==", "license": "MIT", + "peer": true, "dependencies": { "@codemirror/state": "^6.5.0", "crelt": "^1.0.6", @@ -1060,6 +1066,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=20.19.0" }, @@ -1100,6 +1107,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=20.19.0" } @@ -1469,7 +1477,6 @@ "dev": true, "license": "BSD-2-Clause", "optional": true, - "peer": true, "dependencies": { "cross-dirname": "^0.1.0", "debug": "^4.3.4", @@ -1491,7 +1498,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -1508,7 +1514,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -1523,7 +1528,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "engines": { "node": ">= 10.0.0" } @@ -2783,6 +2787,27 @@ "hono": "^4" } }, + "node_modules/@huggingface/jinja": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.5.tgz", + "integrity": "sha512-xRlzazC+QZwr6z4ixEqYHo9fgwhTZ3xNSdljlKfUFGZSdlvt166DljRELFUfFytlYOYvo3vTisA/AFOuOAzFQQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/transformers": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.8.1.tgz", + "integrity": "sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.3", + "onnxruntime-node": "1.21.0", + "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4", + "sharp": "^0.34.1" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -3407,7 +3432,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", - "dev": true, "license": "ISC", "dependencies": { "minipass": "^7.0.4" @@ -4092,6 +4116,7 @@ "resolved": "https://registry.npmjs.org/@libsql/client/-/client-0.17.0.tgz", "integrity": "sha512-TLjSU9Otdpq0SpKHl1tD1Nc9MKhrsZbCFGot3EbCxRa8m1E5R1mMwoOjKMMM31IyF7fr+hPNHLpYfwbMKNusmg==", "license": "MIT", + "peer": true, "dependencies": { "@libsql/core": "^0.17.0", "@libsql/hrana-client": "^0.9.0", @@ -4731,6 +4756,7 @@ "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.27.1.tgz", "integrity": "sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA==", "license": "MIT", + "peer": true, "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", @@ -5052,6 +5078,70 @@ "dev": true, "license": "MIT" }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, "node_modules/@react-dnd/asap": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@react-dnd/asap/-/asap-4.0.1.tgz", @@ -5548,8 +5638,7 @@ "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/babel__core": { "version": "7.20.5", @@ -5791,6 +5880,7 @@ "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -5801,6 +5891,7 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "dev": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -5908,6 +5999,7 @@ "integrity": "sha512-IgSWvLobTDOjnaxAfDTIHaECbkNlAlKv2j5SjpB2v7QHKv1FIfjwMy8FsDbVfDX/KjmCmYICcw7uGaXLhtsLNg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.0", "@typescript-eslint/types": "8.56.0", @@ -6296,6 +6388,7 @@ "integrity": "sha512-CGJ25bc8fRi8Lod/3GHSvXRKi7nBo3kxh0ApW4yCjmrWmRmlT53B5E08XRSZRliygG0aVNxLrBEqPYdz/KcCtQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/utils": "4.0.18", "fflate": "^0.8.2", @@ -6520,6 +6613,7 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -6571,6 +6665,7 @@ "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -7110,6 +7205,15 @@ "require-from-string": "^2.0.2" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -7167,9 +7271,7 @@ "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", - "dev": true, - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/brace-expansion": { "version": "1.1.12", @@ -7202,6 +7304,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -7600,7 +7703,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" @@ -8002,8 +8104,7 @@ "integrity": "sha512-+R08/oI0nl3vfPcqftZRpytksBXDzOUveBq/NBVx0sUp1axwzPQrKinNx5yd5sxPu8j1wIy8AfnVQ+5eFdha6Q==", "dev": true, "license": "MIT", - "optional": true, - "peer": true + "optional": true }, "node_modules/cross-env": { "version": "10.1.0", @@ -8139,7 +8240,8 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/d3-cloud": { "version": "1.2.8", @@ -8305,9 +8407,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -8324,9 +8424,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", @@ -8379,9 +8477,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", - "dev": true, - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/devlop": { "version": "1.1.0", @@ -8426,6 +8522,7 @@ "integrity": "sha512-uOOBA3f+kW3o4KpSoMQ6SNpdXU7WtxlJRb9vCZgOvqhTz4b3GjcoWKstdisizNZLsylhTMv8TLHFPFW0Uxsj/g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "app-builder-lib": "26.7.0", "builder-util": "26.4.1", @@ -8527,8 +8624,7 @@ "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/dompurify": { "version": "3.3.1", @@ -9007,7 +9103,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@electron/asar": "^3.2.1", "debug": "^4.1.1", @@ -9028,7 +9123,6 @@ "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "graceful-fs": "^4.1.2", "jsonfile": "^4.0.0", @@ -9064,16 +9158,6 @@ "node": ">= 0.8" } }, - "node_modules/encoding": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", - "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", - "license": "MIT", - "optional": true, - "dependencies": { - "iconv-lite": "^0.6.2" - } - }, "node_modules/end-of-stream": { "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", @@ -9170,9 +9254,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", - "dev": true, - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/esbuild": { "version": "0.25.12", @@ -9181,6 +9263,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -9249,7 +9332,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -9264,6 +9346,7 @@ "integrity": "sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -9805,6 +9888,12 @@ "node": ">=16.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -9890,6 +9979,12 @@ "node": ">=16" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/flatted": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", @@ -10208,9 +10303,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", - "dev": true, "license": "BSD-3-Clause", - "optional": true, "dependencies": { "boolean": "^3.0.1", "es6-error": "^4.1.1", @@ -10227,9 +10320,7 @@ "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, "license": "ISC", - "optional": true, "bin": { "semver": "bin/semver.js" }, @@ -10254,9 +10345,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "define-properties": "^1.2.1", "gopd": "^1.0.1" @@ -10365,6 +10454,12 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "license": "BSD-3-Clause" }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -10379,9 +10474,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "es-define-property": "^1.0.0" }, @@ -10448,6 +10541,7 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.5.tgz", "integrity": "sha512-3qq+FUBtlTHhtYxbxheZgY8NIFnkkC/MR8u5TTsr7YZ3wixryQ3cCwn3iZbg8p8B88iDBBAYSfZDS75t8MN7Vg==", "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -10985,6 +11079,7 @@ "integrity": "sha512-KDYJgZ6T2TKdU8yBfYueq5EPG/EylMsBvCaenWMJb2OXmjgczzwveRCoJ+Hgj1lXPDyasvrgneSn4GBuR1hYyA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@acemir/cssom": "^0.9.31", "@asamuzakjp/dom-selector": "^6.7.6", @@ -11069,9 +11164,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "dev": true, - "license": "ISC", - "optional": true + "license": "ISC" }, "node_modules/json5": { "version": "2.2.3", @@ -11289,6 +11382,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -11325,7 +11424,6 @@ "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "lz-string": "bin/bin.js" } @@ -11441,9 +11539,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "escape-string-regexp": "^4.0.0" }, @@ -12461,7 +12557,6 @@ "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, "license": "ISC", "engines": { "node": ">=16 || 14 >=14.17" @@ -12601,7 +12696,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", - "dev": true, "license": "MIT", "dependencies": { "minipass": "^7.1.2" @@ -12616,7 +12710,6 @@ "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "minimist": "^1.2.6" }, @@ -12629,6 +12722,7 @@ "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.55.1.tgz", "integrity": "sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==", "license": "MIT", + "peer": true, "dependencies": { "dompurify": "3.2.7", "marked": "14.0.0" @@ -12832,6 +12926,17 @@ "node": "^18.17.0 || >=20.5.0" } }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-gyp/node_modules/isexe": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.5.tgz", @@ -12941,9 +13046,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, "license": "MIT", - "optional": true, "engines": { "node": ">= 0.4" } @@ -13006,6 +13109,49 @@ "node": ">=6" } }, + "node_modules/onnxruntime-common": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz", + "integrity": "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz", + "integrity": "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "global-agent": "^3.0.0", + "onnxruntime-common": "1.21.0", + "tar": "^7.0.1" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==", + "license": "MIT" + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -13255,6 +13401,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13271,6 +13418,12 @@ "node": ">=16.20.0" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/plist": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/plist/-/plist-3.1.0.tgz", @@ -13384,7 +13537,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "commander": "^9.4.0" }, @@ -13402,7 +13554,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -13423,7 +13574,6 @@ "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -13439,7 +13589,6 @@ "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -13594,6 +13743,7 @@ "resolved": "https://registry.npmjs.org/prosemirror-model/-/prosemirror-model-1.25.4.tgz", "integrity": "sha512-PIM7E43PBxKce8OQeezAs9j4TP+5yDpZVbuurd1h5phUxEKIu+G2a+EUZzIC5nS1mJktDJWzbqS23n1tsAf5QA==", "license": "MIT", + "peer": true, "dependencies": { "orderedmap": "^2.0.0" } @@ -13627,6 +13777,7 @@ "resolved": "https://registry.npmjs.org/prosemirror-state/-/prosemirror-state-1.4.4.tgz", "integrity": "sha512-6jiYHH2CIGbCfnxdHbXZ12gySFY/fz/ulZE333G6bPqIZ4F+TXo9ifiR86nAHpWnfoNjOb3o5ESi7J8Uz1jXHw==", "license": "MIT", + "peer": true, "dependencies": { "prosemirror-model": "^1.0.0", "prosemirror-transform": "^1.0.0", @@ -13660,6 +13811,7 @@ "resolved": "https://registry.npmjs.org/prosemirror-view/-/prosemirror-view-1.41.6.tgz", "integrity": "sha512-mxpcDG4hNQa/CPtzxjdlir5bJFDlm0/x5nGBbStB2BWX+XOQ9M8ekEG+ojqB5BcVu2Rc80/jssCMZzSstJuSYg==", "license": "MIT", + "peer": true, "dependencies": { "prosemirror-model": "^1.20.0", "prosemirror-state": "^1.0.0", @@ -13691,6 +13843,30 @@ } } }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -13818,6 +13994,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -13883,6 +14060,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -13912,8 +14090,7 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/react-refresh": { "version": "0.18.0", @@ -14217,7 +14394,6 @@ "deprecated": "Rimraf versions prior to v4 are no longer supported", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -14229,9 +14405,7 @@ "version": "2.15.4", "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", - "dev": true, "license": "BSD-3-Clause", - "optional": true, "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", @@ -14423,9 +14597,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", - "dev": true, - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/send": { "version": "1.2.1", @@ -14482,9 +14654,7 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "type-fest": "^0.13.1" }, @@ -14499,9 +14669,7 @@ "version": "0.13.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", - "dev": true, "license": "(MIT OR CC0-1.0)", - "optional": true, "engines": { "node": ">=10" }, @@ -14875,9 +15043,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", - "dev": true, - "license": "BSD-3-Clause", - "optional": true + "license": "BSD-3-Clause" }, "node_modules/ssh2": { "version": "1.17.0", @@ -15139,7 +15305,6 @@ "version": "7.5.9", "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.9.tgz", "integrity": "sha512-BTLcK0xsDh2+PUe9F6c2TlRp4zOOBMTkoQHQIWSIzI0R7KG46uEwq4OPk2W7bZcprBMsuaeFsqwYr7pjh6CuHg==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", @@ -15156,7 +15321,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" @@ -15168,7 +15332,6 @@ "integrity": "sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "mkdirp": "^0.5.1", "rimraf": "~2.6.2" @@ -15470,7 +15633,8 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "devOptional": true, - "license": "0BSD" + "license": "0BSD", + "peer": true }, "node_modules/tsx": { "version": "4.21.0", @@ -16065,6 +16229,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -16285,6 +16450,30 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/usearch": { + "version": "2.21.4", + "resolved": "https://registry.npmjs.org/usearch/-/usearch-2.21.4.tgz", + "integrity": "sha512-AzavmhAfGubKOLdR3S6Rh/6dvgXqxL+6Fzs1fsgKneQG8i7oLX2Gpqsc4EfdSyKb4sQXhavIiKIguMA2R3cRaA==", + "hasInstallScript": true, + "license": "Apache 2.0", + "dependencies": { + "bindings": "^1.5.0", + "node-addon-api": "^8.5.0", + "node-gyp-build": "^4.8.4" + }, + "engines": { + "node": "~10 >=10.20 || >=12.17" + } + }, + "node_modules/usearch/node_modules/node-addon-api": { + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.6.0.tgz", + "integrity": "sha512-gBVjCaqDlRUk0EwoPNKzIr9KkS9041G/q31IBShPs1Xz6UTA+EXdZADbzqAJQrpDRq71CIMnOP5VMut3SL0z5Q==", + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, "node_modules/utf8-byte-length": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.5.tgz", @@ -16381,6 +16570,7 @@ "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -16940,6 +17130,7 @@ "integrity": "sha512-hOQuK7h0FGKgBAas7v0mSAsnvrIgAvWmRFjmzpJ7SwFHH3g1k2u37JtYwOwmEKhK6ZO3v9ggDBBm0La1LCK4uQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/expect": "4.0.18", "@vitest/mocker": "4.0.18", @@ -17017,6 +17208,7 @@ "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.28.tgz", "integrity": "sha512-BRdrNfeoccSoIZeIhyPBfvWSLFP4q8J3u8Ju8Ug5vu3LdD+yTM13Sg4sKtljxozbnuMu1NB1X5HBHRYUzFocKg==", "license": "MIT", + "peer": true, "dependencies": { "@vue/compiler-dom": "3.5.28", "@vue/compiler-sfc": "3.5.28", @@ -17341,6 +17533,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index c25bb19..3b8fe73 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "@dagrejs/dagre": "^2.0.4", "@floating-ui/dom": "^1.7.5", "@highlightjs/cdn-assets": "^11.11.1", + "@huggingface/transformers": "^3.8.1", "@libsql/client": "^0.17.0", "@milkdown/kit": "^7.18.0", "@milkdown/plugin-block": "^7.18.0", @@ -119,6 +120,7 @@ "snowball-stemmers": "^0.6.0", "transliteration": "^2.6.1", "turndown": "^7.2.2", + "usearch": "^2.21.4", "uuid": "^13.0.0", "vanilla-calendar-pro": "^3.1.0", "zod": "^4.3.6", diff --git a/src/main/database/schema.ts b/src/main/database/schema.ts index d48aecc..6f3296f 100644 --- a/src/main/database/schema.ts +++ b/src/main/database/schema.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, integer, real, uniqueIndex, primaryKey } from 'drizzle-orm/sqlite-core'; +import { sqliteTable, text, integer, real, blob, uniqueIndex, primaryKey } from 'drizzle-orm/sqlite-core'; // Projects table - stores blog projects/websites export const projects = sqliteTable('projects', { @@ -269,6 +269,26 @@ export const modelCatalogMeta = sqliteTable('ai_catalog_meta', { value: text('value').notNull(), }); +// Embedding keys table - maps USearch bigint labels to post IDs for semantic similarity +export const embeddingKeys = sqliteTable('embedding_keys', { + label: integer('label').primaryKey(), // USearch bigint key (stored as number, cast to bigint at runtime) + postId: text('post_id').notNull(), + projectId: text('project_id').notNull(), + contentHash: text('content_hash').notNull(), // SHA-256 of title+content, for change detection + vector: blob('vector', { mode: 'buffer' }), // Raw Float32Array bytes (384 × 4 = 1536 bytes) +}); + +// Dismissed duplicate pairs - user has reviewed and dismissed these near-duplicates +export const dismissedDuplicatePairs = sqliteTable('dismissed_duplicate_pairs', { + id: text('id').primaryKey(), + projectId: text('project_id').notNull(), + postIdA: text('post_id_a').notNull(), + postIdB: text('post_id_b').notNull(), + dismissedAt: integer('dismissed_at', { mode: 'timestamp' }).notNull(), +}, (table) => ({ + pairIdx: uniqueIndex('dismissed_pairs_idx').on(table.projectId, table.postIdA, table.postIdB), +})); + // Types for TypeScript export type Project = typeof projects.$inferSelect; export type NewProject = typeof projects.$inferInsert; @@ -306,3 +326,7 @@ export type ModelCatalogModalityEntry = typeof modelCatalogModalities.$inferSele export type NewModelCatalogModalityEntry = typeof modelCatalogModalities.$inferInsert; export type ModelCatalogMetaEntry = typeof modelCatalogMeta.$inferSelect; export type NewModelCatalogMetaEntry = typeof modelCatalogMeta.$inferInsert; +export type EmbeddingKey = typeof embeddingKeys.$inferSelect; +export type NewEmbeddingKey = typeof embeddingKeys.$inferInsert; +export type DismissedDuplicatePair = typeof dismissedDuplicatePairs.$inferSelect; +export type NewDismissedDuplicatePair = typeof dismissedDuplicatePairs.$inferInsert; diff --git a/src/main/engine/EmbeddingEngine.ts b/src/main/engine/EmbeddingEngine.ts new file mode 100644 index 0000000..c8b01dd --- /dev/null +++ b/src/main/engine/EmbeddingEngine.ts @@ -0,0 +1,771 @@ +/** + * EmbeddingEngine + * + * Provides semantic similarity features using local ONNX embeddings (multilingual-e5-small) + * and HNSW vector search via USearch. All processing is fully local — no external API calls. + * + * Features: + * - findSimilar: Find thematically related posts (InsertModal, "have I written this?") + * - suggestTags: Infer tags from similar posts + * - findDuplicates: Audit tool for near-duplicate post detection + * + * Architecture: + * - Model stays loaded across project switches (one model, multiple indexes) + * - USearch index file per project: {userData}/projects/{projectId}/embeddings.usearch + * - Label→postId mapping in `embedding_keys` DB table (avoids bigint JSON issues) + * - Vector cache persisted in `embedding_keys.vector` DB column as BLOB for instant reload + */ + +import { EventEmitter } from 'events'; +import * as path from 'path'; +import * as fs from 'fs/promises'; +import * as crypto from 'crypto'; +import { v4 as uuidv4 } from 'uuid'; +import { eq, and, inArray } from 'drizzle-orm'; +import { getDatabase } from '../database'; +import { embeddingKeys, dismissedDuplicatePairs, posts } from '../database/schema'; + +export interface SimilarPost { + postId: string; + similarity: number; // cosine similarity 0-1 +} + +export interface TagSuggestion { + name: string; + score: number; // weighted frequency +} + +export interface DuplicatePair { + postA: { id: string; title: string; slug: string; publishedAt?: Date }; + postB: { id: string; title: string; slug: string; publishedAt?: Date }; + similarity: number; + exactMatch?: boolean; +} + +// Injected dependencies for testability +export interface EmbeddingEngineDeps { + /** Return the path to the USearch index file for a project */ + getIndexPath: (projectId: string) => string; + /** Create the embedding pipeline (dependency-injected for tests) */ + createPipeline?: () => Promise; +} + +export interface EmbeddingPipeline { + embed(text: string): Promise; +} + +export class EmbeddingEngine extends EventEmitter { + private deps: EmbeddingEngineDeps; + private pipeline: EmbeddingPipeline | null = null; + private pipelineLoadPromise: Promise | null = null; + + // USearch index (lazily loaded per-project) + private index: import('usearch').Index | null = null; + private currentProjectId: string | null = null; + + // Label->postId map (backed by DB, kept in memory for fast lookup) + private labelToPostId: Map = new Map(); + private postIdToLabel: Map = new Map(); + private nextLabel: bigint = 1n; + + // In-memory vector cache -- loaded from DB on startup, updated during embedding. + private vectorCache: Map = new Map(); // postId -> vector + + // Debounced save timer + private saveTimer: ReturnType | null = null; + private readonly SAVE_DEBOUNCE_MS = 5000; + + // Model dimensions + private readonly DIMENSIONS = 384; + private readonly MODEL_ID = 'Xenova/multilingual-e5-small'; + + constructor(deps: EmbeddingEngineDeps) { + super(); + this.deps = deps; + } + + // Lifecycle + + async initialize(): Promise { + if (this.pipeline) return; + if (this.pipelineLoadPromise) { + await this.pipelineLoadPromise; + return; + } + this.pipelineLoadPromise = this.loadPipeline(); + this.pipeline = await this.pipelineLoadPromise; + } + + private async loadPipeline(): Promise { + if (this.deps.createPipeline) { + return this.deps.createPipeline(); + } + + // Dynamic import to avoid loading heavy ONNX runtime at startup + const { pipeline, env } = await import('@huggingface/transformers'); + + // Configure cache for Electron -- use ~/.cache/huggingface + env.useFSCache = true; + + const extractor = await pipeline('feature-extraction', this.MODEL_ID, { + dtype: 'fp32', + }); + + return { + embed: async (text: string): Promise => { + const output = await extractor(text, { pooling: 'mean', normalize: true }); + // v3: output.data is Float32Array + return output.data as Float32Array; + }, + }; + } + + async shutdown(): Promise { + if (this.saveTimer) { + clearTimeout(this.saveTimer); + this.saveTimer = null; + } + if (this.index && this.currentProjectId) { + await this.save(); + } + this.index = null; + this.currentProjectId = null; + this.labelToPostId.clear(); + this.postIdToLabel.clear(); + this.vectorCache.clear(); + this.nextLabel = 1n; + this.pipeline = null; + this.pipelineLoadPromise = null; + } + + // Project switching + + async setProjectContext(projectId: string): Promise { + if (this.currentProjectId === projectId) return; + + // Save and unload current index + if (this.index && this.currentProjectId) { + await this.save(); + } + this.index = null; + this.labelToPostId.clear(); + this.postIdToLabel.clear(); + this.vectorCache.clear(); + this.nextLabel = 1n; + + this.currentProjectId = projectId; + + // Load (or create) index for new project + await this.ensureIndexLoaded(); + } + + private async ensureIndexLoaded(): Promise { + if (this.index) return; + if (!this.currentProjectId) return; + + const { Index, MetricKind, ScalarKind } = await import('usearch'); + this.index = new Index({ + metric: MetricKind.Cos, + quantization: ScalarKind.F32, + dimensions: this.DIMENSIONS, + connectivity: 16, + expansion_add: 128, + expansion_search: 64, + multi: false, + }); + + const indexPath = this.deps.getIndexPath(this.currentProjectId); + try { + await fs.access(indexPath); + this.index.load(indexPath); + } catch { + // No index file yet -- start fresh + } + + // Load key mapping and vectors from DB + await this.loadKeyMapFromDb(this.currentProjectId); + } + + private async loadKeyMapFromDb(projectId: string): Promise { + const db = getDatabase().getLocal(); + const rows = await db + .select() + .from(embeddingKeys) + .where(eq(embeddingKeys.projectId, projectId)); + + this.labelToPostId.clear(); + this.postIdToLabel.clear(); + this.vectorCache.clear(); + this.nextLabel = 1n; + + for (const row of rows) { + const label = BigInt(row.label); + this.labelToPostId.set(label, row.postId); + this.postIdToLabel.set(row.postId, label); + if (label >= this.nextLabel) { + this.nextLabel = label + 1n; + } + if (row.vector) { + const buf = row.vector as Buffer; + this.vectorCache.set(row.postId, new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4)); + } + } + } + + // Core operations + + async embedPost(postId: string, title: string, content: string): Promise { + await this.initialize(); + await this.ensureIndexLoaded(); + + if (!this.index || !this.pipeline || !this.currentProjectId) return; + + const rawText = `${title}\n\n${content}`; + const hash = this.computeHash(rawText); + + // Check if already indexed with same hash (no-op) + const db = getDatabase().getLocal(); + const existing = await db + .select() + .from(embeddingKeys) + .where( + and( + eq(embeddingKeys.postId, postId), + eq(embeddingKeys.projectId, this.currentProjectId), + ), + ); + + if (existing.length > 0 && existing[0]!.contentHash === hash) { + return; // Unchanged, skip re-embedding + } + + // Remove old vector if exists + if (existing.length > 0) { + const oldLabel = BigInt(existing[0]!.label); + try { + this.index.remove(oldLabel); + } catch { + // Ignore remove errors -- label may not be in index + } + this.labelToPostId.delete(oldLabel); + this.postIdToLabel.delete(postId); + this.vectorCache.delete(postId); + await db.delete(embeddingKeys).where( + and( + eq(embeddingKeys.postId, postId), + eq(embeddingKeys.projectId, this.currentProjectId), + ), + ); + } + + // Compute embedding + const text = `query: ${rawText}`; + const vector = await this.embedText(text); + + // Assign new label + const label = this.nextLabel++; + this.index.add(label, vector); + this.labelToPostId.set(label, postId); + this.postIdToLabel.set(postId, label); + this.vectorCache.set(postId, vector); + + // Persist key mapping + vector (label is bigint in-memory, stored as number in SQLite) + await db.insert(embeddingKeys).values({ + label: Number(label), + postId, + projectId: this.currentProjectId, + contentHash: hash, + vector: Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength), + }); + + this.scheduleSave(); + } + + async removePost(postId: string): Promise { + await this.ensureIndexLoaded(); + if (!this.index || !this.currentProjectId) return; + + const label = this.postIdToLabel.get(postId); + if (label === undefined) return; + + try { + this.index.remove(label); + } catch { + // Ignore remove errors + } + this.labelToPostId.delete(label); + this.postIdToLabel.delete(postId); + this.vectorCache.delete(postId); + + const db = getDatabase().getLocal(); + await db.delete(embeddingKeys).where( + and( + eq(embeddingKeys.postId, postId), + eq(embeddingKeys.projectId, this.currentProjectId), + ), + ); + + this.scheduleSave(); + } + + async findSimilar(postId: string, k = 5): Promise { + await this.ensureIndexLoaded(); + if (!this.index || !this.currentProjectId) return []; + + if (!this.postIdToLabel.has(postId)) return []; + + // Guard against empty index (USearch throws on empty index search) + if (this.postIdToLabel.size < 2) return []; + + // Get or compute vector for this post + const vector = await this.getOrComputeVector(postId); + if (!vector) return []; + + // Search for k+1 (to exclude self) with HNSW + const result = this.index.search(vector, k + 1, 0); + if (!result) return []; + + const results: SimilarPost[] = []; + for (let i = 0; i < result.keys.length; i++) { + const foundLabel = result.keys[i]!; + const foundPostId = this.labelToPostId.get(foundLabel); + if (!foundPostId || foundPostId === postId) continue; + + const distance = result.distances[i]!; + // USearch cosine metric returns distance (0=identical), convert to similarity + const similarity = Math.max(0, 1 - distance); + results.push({ postId: foundPostId, similarity }); + } + + return results.sort((a, b) => b.similarity - a.similarity).slice(0, k); + } + + /** + * Compute cosine similarity between a source post and a list of target posts. + * Returns a map of targetPostId → similarity (0-1). Posts without embeddings are omitted. + */ + async computeSimilarities(sourcePostId: string, targetPostIds: string[]): Promise> { + await this.ensureIndexLoaded(); + if (!this.index || !this.currentProjectId || targetPostIds.length === 0) return {}; + + const sourceVec = await this.getOrComputeVector(sourcePostId); + if (!sourceVec) return {}; + + const result: Record = {}; + for (const targetId of targetPostIds) { + if (targetId === sourcePostId) continue; + const targetVec = await this.getOrComputeVector(targetId); + if (!targetVec) continue; + result[targetId] = this.cosineSimilarity(sourceVec, targetVec); + } + return result; + } + + private cosineSimilarity(a: Float32Array, b: Float32Array): number { + let dot = 0, normA = 0, normB = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i]! * b[i]!; + normA += a[i]! * a[i]!; + normB += b[i]! * b[i]!; + } + const denom = Math.sqrt(normA) * Math.sqrt(normB); + return denom === 0 ? 0 : Math.max(0, dot / denom); + } + + // Derived features + + async suggestTags(postId: string, excludeTags: string[]): Promise { + const similar = await this.findSimilar(postId, 10); + if (similar.length === 0) return []; + + if (!this.currentProjectId) return []; + + // Get tags for similar posts + const similarPostIds = similar.map((s) => s.postId); + const db = getDatabase().getLocal(); + const postRows = await db + .select({ id: posts.id, tags: posts.tags }) + .from(posts) + .where(inArray(posts.id, similarPostIds)); + + const excludeSet = new Set(excludeTags.map((t) => t.toLowerCase())); + const tagScores = new Map(); + + for (const row of postRows) { + const simItem = similar.find((s) => s.postId === row.id); + if (!simItem) continue; + + const postTags: string[] = JSON.parse(row.tags || '[]'); + for (const tag of postTags) { + if (excludeSet.has(tag.toLowerCase())) continue; + const current = tagScores.get(tag) || 0; + tagScores.set(tag, current + simItem.similarity); + } + } + + return Array.from(tagScores.entries()) + .map(([name, score]) => ({ name, score })) + .sort((a, b) => b.score - a.score) + .slice(0, 5); + } + + async findDuplicates(threshold = 0.92, onProgress?: (checked: number, total: number) => void): Promise { + await this.ensureIndexLoaded(); + if (!this.index || !this.currentProjectId) return []; + + const projectId = this.currentProjectId; + const db = getDatabase().getLocal(); + + // Get dismissed pairs + const dismissed = await db + .select() + .from(dismissedDuplicatePairs) + .where(eq(dismissedDuplicatePairs.projectId, projectId)); + + const dismissedSet = new Set(); + for (const d of dismissed) { + dismissedSet.add(this.pairKey(d.postIdA, d.postIdB)); + } + + // Get post info for all indexed posts + const allPostIds = Array.from(this.postIdToLabel.keys()); + if (allPostIds.length === 0) return []; + + const postRows = await db + .select({ + id: posts.id, + title: posts.title, + slug: posts.slug, + content: posts.content, + status: posts.status, + filePath: posts.filePath, + publishedAt: posts.publishedAt, + }) + .from(posts) + .where(inArray(posts.id, allPostIds)); + + const postMap = new Map(postRows.map((p) => [p.id, p])); + + // Cache for lazily-loaded post bodies (needed for exact-match detection) + const bodyCache = new Map(); + const getBody = async (postId: string): Promise => { + const cached = bodyCache.get(postId); + if (cached !== undefined) return cached; + const post = postMap.get(postId); + if (!post) { bodyCache.set(postId, ''); return ''; } + // Draft content is in the DB; published content is on the filesystem + if (post.content) { + bodyCache.set(postId, post.content); + return post.content; + } + if (post.filePath) { + try { + const raw = await fs.readFile(post.filePath, 'utf-8'); + const { content: body } = (await import('gray-matter')).default(raw); + bodyCache.set(postId, body); + return body; + } catch { + bodyCache.set(postId, ''); + return ''; + } + } + bodyCache.set(postId, ''); + return ''; + }; + + const pairs: DuplicatePair[] = []; + const seenPairs = new Set(); + + for (let idx = 0; idx < allPostIds.length; idx++) { + const postId = allPostIds[idx]!; + onProgress?.(idx + 1, allPostIds.length); + const vector = await this.getOrComputeVector(postId); + if (!vector) continue; + + const result = this.index.search(vector, 21, 0); + if (!result) continue; + + for (let i = 0; i < result.keys.length; i++) { + const otherLabel = result.keys[i]!; + const otherPostId = this.labelToPostId.get(otherLabel); + if (!otherPostId || otherPostId === postId) continue; + + const distance = result.distances[i]!; + const similarity = Math.max(0, 1 - distance); + if (similarity < threshold) continue; + + const key = this.pairKey(postId, otherPostId); + if (seenPairs.has(key) || dismissedSet.has(key)) continue; + seenPairs.add(key); + + const postA = postMap.get(postId); + const postB = postMap.get(otherPostId); + if (!postA || !postB) continue; + + pairs.push({ + postA: { + id: postA.id, + title: postA.title, + slug: postA.slug, + publishedAt: postA.publishedAt ?? undefined, + }, + postB: { + id: postB.id, + title: postB.title, + slug: postB.slug, + publishedAt: postB.publishedAt ?? undefined, + }, + similarity, + }); + } + } + + // For pairs at 100% embedding similarity, compare actual bodies to find true exact duplicates + for (const pair of pairs) { + if (Math.round(pair.similarity * 100) >= 100) { + const bodyA = await getBody(pair.postA.id); + const bodyB = await getBody(pair.postB.id); + const postA = postMap.get(pair.postA.id); + const postB = postMap.get(pair.postB.id); + if (postA && postB && postA.title === postB.title && bodyA === bodyB) { + pair.exactMatch = true; + } + } + } + + return pairs.sort((a, b) => { + if (a.exactMatch && !b.exactMatch) return -1; + if (!a.exactMatch && b.exactMatch) return 1; + return b.similarity - a.similarity; + }); + } + + async dismissPair(postIdA: string, postIdB: string): Promise { + if (!this.currentProjectId) return; + const db = getDatabase().getLocal(); + const [a, b] = this.sortedPairIds(postIdA, postIdB); + await db.insert(dismissedDuplicatePairs).values({ + id: uuidv4(), + projectId: this.currentProjectId, + postIdA: a, + postIdB: b, + dismissedAt: new Date(), + }).onConflictDoNothing(); + } + + async dismissPairs(pairIds: Array<[string, string]>): Promise { + if (!this.currentProjectId) return; + const db = getDatabase().getLocal(); + const now = new Date(); + const rows = pairIds.map(([idA, idB]) => { + const [a, b] = this.sortedPairIds(idA, idB); + return { id: uuidv4(), projectId: this.currentProjectId!, postIdA: a, postIdB: b, dismissedAt: now }; + }); + // Insert in batches of 100 to avoid SQLite variable limits + for (let i = 0; i < rows.length; i += 100) { + await db.insert(dismissedDuplicatePairs).values(rows.slice(i, i + 100)).onConflictDoNothing(); + } + } + + // Indexing management + + async getIndexingProgress(): Promise<{ indexed: number; total: number }> { + if (!this.currentProjectId) return { indexed: 0, total: 0 }; + await this.ensureIndexLoaded(); + + const db = getDatabase().getLocal(); + const indexed = this.labelToPostId.size; + + const allPosts = await db + .select({ id: posts.id }) + .from(posts) + .where(eq(posts.projectId, this.currentProjectId)); + + return { indexed, total: allPosts.length }; + } + + async reindexAll(onProgress?: (indexed: number, total: number) => void): Promise { + await this.ensureIndexLoaded(); + if (!this.currentProjectId) return; + + const db = getDatabase().getLocal(); + + // Clear existing index + await db.delete(embeddingKeys).where(eq(embeddingKeys.projectId, this.currentProjectId)); + const { Index, MetricKind, ScalarKind } = await import('usearch'); + this.index = new Index({ + metric: MetricKind.Cos, + quantization: ScalarKind.F32, + dimensions: this.DIMENSIONS, + connectivity: 16, + expansion_add: 128, + expansion_search: 64, + multi: false, + }); + this.labelToPostId.clear(); + this.postIdToLabel.clear(); + this.vectorCache.clear(); + this.nextLabel = 1n; + + await this.indexUnindexedPosts(onProgress); + } + + async indexUnindexedPosts(onProgress?: (indexed: number, total: number) => void): Promise { + await this.initialize(); + await this.ensureIndexLoaded(); + if (!this.currentProjectId) return; + + const db = getDatabase().getLocal(); + const allPosts = await db + .select({ + id: posts.id, + title: posts.title, + content: posts.content, + filePath: posts.filePath, + }) + .from(posts) + .where(eq(posts.projectId, this.currentProjectId)); + + // Resolve actual content for each post (read from file for published posts) + const resolvedPosts: Array<{ id: string; title: string; content: string }> = []; + for (const p of allPosts) { + let body = p.content || ''; + if (!p.content && p.filePath) { + try { + const raw = await fs.readFile(p.filePath, 'utf-8'); + const matter = (await import('gray-matter')).default; + const { content: fileBody } = matter(raw); + body = fileBody; + } catch { + // File not found — use empty + } + } + resolvedPosts.push({ id: p.id, title: p.title, content: body }); + } + + // Get current hashes from DB for change detection + const keyRows = await db + .select() + .from(embeddingKeys) + .where(eq(embeddingKeys.projectId, this.currentProjectId)); + + const hashMap = new Map(keyRows.map((r) => [r.postId, r.contentHash])); + + const toIndex = resolvedPosts.filter((p) => { + const raw = `${p.title}\n\n${p.content}`; + const hash = this.computeHash(raw); + return hashMap.get(p.id) !== hash; + }); + + let count = 0; + let batchCount = 0; + const BATCH_SAVE_INTERVAL = 100; + + for (const post of toIndex) { + await this.embedPost(post.id, post.title, post.content); + count++; + batchCount++; + onProgress?.(count, toIndex.length); + + if (batchCount >= BATCH_SAVE_INTERVAL) { + await this.save(); + batchCount = 0; + } + } + + if (batchCount > 0) { + await this.save(); + } + } + + // Persistence + + async save(): Promise { + if (this.saveTimer) { + clearTimeout(this.saveTimer); + this.saveTimer = null; + } + if (!this.index || !this.currentProjectId) return; + + const indexPath = this.deps.getIndexPath(this.currentProjectId); + const dir = path.dirname(indexPath); + await fs.mkdir(dir, { recursive: true }); + this.index.save(indexPath); + } + + private scheduleSave(): void { + if (this.saveTimer) clearTimeout(this.saveTimer); + this.saveTimer = setTimeout(() => { + this.save().catch((err) => console.error('[EmbeddingEngine] save error:', err)); + }, this.SAVE_DEBOUNCE_MS); + } + + // Helpers + + /** + * Get vector for a postId from in-memory cache (loaded from DB at startup). + * Falls back to re-computing from post content only if not in cache. + */ + private async getOrComputeVector(postId: string): Promise { + const cached = this.vectorCache.get(postId); + if (cached) return cached; + + // Re-embed from post content + await this.initialize(); + if (!this.pipeline || !this.currentProjectId) return null; + + const resolved = await this.resolvePostContent(postId); + if (!resolved) return null; + + const rawText = `${resolved.title}\n\n${resolved.content}`; + const text = `query: ${rawText}`; + const vector = await this.embedText(text); + this.vectorCache.set(postId, vector); + return vector; + } + + private async embedText(text: string): Promise { + if (!this.pipeline) throw new Error('EmbeddingEngine not initialized'); + return this.pipeline.embed(text); + } + + /** + * Resolve the actual body text for a post. + * Draft posts have content in the DB; published posts have it on the filesystem. + */ + private async resolvePostContent(postId: string): Promise<{ title: string; content: string } | null> { + if (!this.currentProjectId) return null; + const db = getDatabase().getLocal(); + const rows = await db + .select({ title: posts.title, content: posts.content, filePath: posts.filePath }) + .from(posts) + .where(and(eq(posts.id, postId), eq(posts.projectId, this.currentProjectId))); + if (rows.length === 0) return null; + const post = rows[0]!; + if (post.content) return { title: post.title, content: post.content }; + if (post.filePath) { + try { + const raw = await fs.readFile(post.filePath, 'utf-8'); + const matter = (await import('gray-matter')).default; + const { content: body } = matter(raw); + return { title: post.title, content: body }; + } catch { + // File not found or unreadable — fall back to empty + } + } + return { title: post.title, content: '' }; + } + + private computeHash(text: string): string { + return crypto.createHash('sha256').update(text).digest('hex'); + } + + private pairKey(idA: string, idB: string): string { + const [a, b] = this.sortedPairIds(idA, idB); + return `${a}::${b}`; + } + + private sortedPairIds(idA: string, idB: string): [string, string] { + return idA < idB ? [idA, idB] : [idB, idA]; + } +} diff --git a/src/main/engine/EngineBundle.ts b/src/main/engine/EngineBundle.ts index cbf2f86..ab77924 100644 --- a/src/main/engine/EngineBundle.ts +++ b/src/main/engine/EngineBundle.ts @@ -29,6 +29,7 @@ import type { BlogmarkPythonWorkerRuntime } from './BlogmarkPythonWorkerRuntime' import type { PythonMacroWorkerRuntime } from './PythonMacroWorkerRuntime'; import type { PublishApiAdapter } from './PublishApiAdapter'; import type { AppApiAdapter } from './AppApiAdapter'; +import type { EmbeddingEngine } from './EmbeddingEngine'; export interface EngineBundle { postEngine: PostEngine; @@ -52,4 +53,5 @@ export interface EngineBundle { pythonMacroWorkerRuntime: PythonMacroWorkerRuntime; publishApiAdapter: PublishApiAdapter; appApiAdapter: AppApiAdapter; + embeddingEngine: EmbeddingEngine; } diff --git a/src/main/engine/MCPServer.ts b/src/main/engine/MCPServer.ts index 09d3ed0..27d1af5 100644 --- a/src/main/engine/MCPServer.ts +++ b/src/main/engine/MCPServer.ts @@ -52,6 +52,7 @@ export function decodeCursor(cursor: string): number { interface PostEngineContract { getAllPosts: (options?: PaginationOptions) => Promise>; getPost: (id: string) => Promise; + getPostBySlug: (slug: string) => Promise; searchPosts: (query: string) => Promise; searchPostsFiltered: (query: string, filter: PostFilter, pagination?: PaginationOptions) => Promise<{ posts: PostData[]; total: number }>; createPost: (data: Partial) => Promise; @@ -606,6 +607,42 @@ export class MCPServer { content: [{ type: 'text' as const, text: JSON.stringify(result) }], }; }); + + // ── read_post_by_slug ── + server.registerTool('read_post_by_slug', { + title: 'Read Post by Slug', + description: 'Read the full content and metadata of a specific blog post by its slug. Includes backlinks and outlinks. Useful when you know the slug but not the ID.', + inputSchema: { + slug: z.string().describe('The slug of the post to read'), + }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, async (args) => { + const post = await this.deps.postEngine.getPostBySlug(args.slug); + if (!post) { + return { + content: [{ type: 'text' as const, text: JSON.stringify({ error: `Post with slug "${args.slug}" not found` }) }], + isError: true, + }; + } + const [backlinks, linksTo] = await Promise.all([ + this.deps.postEngine.getLinkedBy(post.id), + this.deps.postEngine.getLinksTo(post.id), + ]); + return { + content: [{ type: 'text' as const, text: JSON.stringify({ + post: { + id: post.id, title: post.title, slug: post.slug, + content: post.content, excerpt: post.excerpt, + status: post.status, author: post.author, + categories: post.categories, tags: post.tags, + createdAt: post.createdAt, updatedAt: post.updatedAt, + publishedAt: post.publishedAt, + backlinks: backlinks.map(b => ({ id: b.id, title: b.title, slug: b.slug })), + linksTo: linksTo.map(l => ({ id: l.id, title: l.title, slug: l.slug })), + }, + }) }], + }; + }); } private registerProposalTools(server: McpServer): void { diff --git a/src/main/engine/MetaEngine.ts b/src/main/engine/MetaEngine.ts index abe7dd2..fe8f3b2 100644 --- a/src/main/engine/MetaEngine.ts +++ b/src/main/engine/MetaEngine.ts @@ -28,6 +28,7 @@ export interface ProjectMetadata { picoTheme?: PicoThemeName; // Selected Pico CSS theme for preview/rendering categoryMetadata?: Record; // Per-category metadata for UI/rendering categorySettings?: Record; // Per-category list rendering preferences + semanticSimilarityEnabled?: boolean; // Enable local ONNX embedding-based semantic similarity } export interface CategoryRenderSettings { @@ -347,6 +348,7 @@ export class MetaEngine extends EventEmitter { pythonRuntimeMode: normalizedUpdates.pythonRuntimeMode, picoTheme: normalizedUpdates.picoTheme, categoryMetadata: normalizedUpdates.categoryMetadata, + semanticSimilarityEnabled: normalizedUpdates.semanticSimilarityEnabled, }); } else { this.projectMetadata = normalizeProjectMetadata({ diff --git a/src/main/engine/PostEngine.ts b/src/main/engine/PostEngine.ts index 1242fe0..a5ec21c 100644 --- a/src/main/engine/PostEngine.ts +++ b/src/main/engine/PostEngine.ts @@ -596,7 +596,23 @@ export class PostEngine extends EventEmitter { async getPost(id: string): Promise { const db = getDatabase().getLocal(); const dbPost = await db.select().from(posts).where(eq(posts.id, id)).get(); - + return this.resolvePostData(dbPost); + } + + async getPostBySlug(slug: string): Promise { + const db = getDatabase().getLocal(); + const dbPost = await db + .select() + .from(posts) + .where(and( + eq(posts.slug, slug), + eq(posts.projectId, this.currentProjectId) + )) + .get(); + return this.resolvePostData(dbPost); + } + + private async resolvePostData(dbPost: typeof posts.$inferSelect | undefined): Promise { if (!dbPost) { return null; } diff --git a/src/main/engine/ai/blog-tools.ts b/src/main/engine/ai/blog-tools.ts index eb08b4c..c4fdd87 100644 --- a/src/main/engine/ai/blog-tools.ts +++ b/src/main/engine/ai/blog-tools.ts @@ -19,6 +19,7 @@ import type { PostMediaLinkData } from '../PostMediaEngine'; export interface BlogToolDeps { postEngine: { getPost: (id: string) => Promise; + getPostBySlug: (slug: string) => Promise; getAllPosts: (options?: PaginationOptions) => Promise<{ items: PostData[]; total: number }>; getPostsFiltered: (filter: PostFilter) => Promise; searchPostsFiltered: (query: string, filter: PostFilter, pagination?: PaginationOptions) => Promise<{ posts: PostData[]; total: number }>; @@ -240,6 +241,34 @@ export function createBlogTools(deps: BlogToolDeps) { }, }), + read_post_by_slug: tool({ + description: 'Read the full content and metadata of a specific blog post by its slug. Includes backlinks (posts linking to this post). Useful when you know the slug but not the ID.', + inputSchema: z.object({ + slug: z.string().describe('The slug of the post to read'), + }), + execute: async ({ slug }) => { + const post = await postEngine.getPostBySlug(slug); + if (!post) return { success: false, error: 'Post not found' }; + const [backlinks, linksTo] = await Promise.all([ + postEngine.getLinkedBy(post.id), + postEngine.getLinksTo(post.id), + ]); + return { + success: true, + post: { + id: post.id, title: post.title, slug: post.slug, + content: post.content, excerpt: post.excerpt, + status: post.status, author: post.author, + categories: post.categories, tags: post.tags, + createdAt: post.createdAt, updatedAt: post.updatedAt, + publishedAt: post.publishedAt, + backlinks: backlinks.map(b => ({ id: b.id, title: b.title, slug: b.slug })), + linksTo: linksTo.map(l => ({ id: l.id, title: l.title, slug: l.slug })), + }, + }; + }, + }), + list_posts: tool({ description: 'List blog posts with optional filtering by status, category, tags, year, or month. Returns paginated results. Each post includes backlinks. The response includes "total" (global post count) and "filteredTotal" (count matching current filters). Use year/month filters to efficiently narrow to a time period. Use check_term first if unsure whether a term is a category or tag.', inputSchema: z.object({ diff --git a/src/main/engine/mainProcessPythonApiInvoker.ts b/src/main/engine/mainProcessPythonApiInvoker.ts index b2de2a0..322d531 100644 --- a/src/main/engine/mainProcessPythonApiInvoker.ts +++ b/src/main/engine/mainProcessPythonApiInvoker.ts @@ -98,6 +98,7 @@ export const ENGINE_MAP: Record = { // Map API method names to engine method names where they differ const METHOD_NAME_MAP: Record = { 'posts.get': 'getPost', + 'posts.getBySlug': 'getPostBySlug', 'posts.create': 'createPost', 'posts.update': 'updatePost', 'posts.delete': 'deletePost', diff --git a/src/main/ipc/embeddingHandlers.ts b/src/main/ipc/embeddingHandlers.ts new file mode 100644 index 0000000..6412ffd --- /dev/null +++ b/src/main/ipc/embeddingHandlers.ts @@ -0,0 +1,65 @@ +import type { EngineBundle } from '../engine/EngineBundle'; +import { startDuplicateSearchTask } from './handlers'; +import { resolveUiLanguageFromSystemLocale, translateMenu } from '../shared/i18n'; +import { app } from 'electron'; + +type SafeHandle = (channel: string, handler: (...args: any[]) => Promise) => void; + +function tr(key: string): string { + const systemLocale = typeof app.getLocale === 'function' ? app.getLocale() : 'en'; + const lang = resolveUiLanguageFromSystemLocale(systemLocale); + return translateMenu(lang, key); +} + +export function registerEmbeddingHandlers(safeHandle: SafeHandle, bundle: EngineBundle): void { + const engine = () => bundle.embeddingEngine; + + safeHandle('embeddings:findSimilar', async (_, postId: string, k?: number) => { + return engine().findSimilar(postId, k); + }); + + safeHandle('embeddings:computeSimilarities', async (_, sourcePostId: string, targetPostIds: string[]) => { + return engine().computeSimilarities(sourcePostId, targetPostIds); + }); + + safeHandle('embeddings:getProgress', async () => { + return engine().getIndexingProgress(); + }); + + safeHandle('embeddings:suggestTags', async (_, postId: string, excludeTags: string[]) => { + return engine().suggestTags(postId, excludeTags ?? []); + }); + + safeHandle('embeddings:findDuplicates', async (_, threshold?: number) => { + return engine().findDuplicates(threshold); + }); + + safeHandle('embeddings:dismissPair', async (_, postIdA: string, postIdB: string) => { + return engine().dismissPair(postIdA, postIdB); + }); + + safeHandle('embeddings:dismissPairs', async (_, pairIds: Array<[string, string]>) => { + return engine().dismissPairs(pairIds); + }); + + safeHandle('embeddings:runDuplicateSearch', async (_, threshold?: number) => { + startDuplicateSearchTask(bundle, threshold ?? 0.92); + }); + + safeHandle('embeddings:indexUnindexedPosts', async () => { + const taskId = `embedding-index-${Date.now()}`; + return bundle.taskManager.runTask({ + id: taskId, + name: tr('task.embeddingIndex.name'), + execute: async (onProgress) => { + await engine().indexUnindexedPosts((indexed, total) => { + const pct = total > 0 ? (indexed / total) * 100 : 0; + onProgress(pct, tr('task.embeddingIndex.indexing') + .replace('{indexed}', String(indexed)) + .replace('{total}', String(total))); + }); + return engine().getIndexingProgress(); + }, + }); + }); +} diff --git a/src/main/ipc/handlers.ts b/src/main/ipc/handlers.ts index 0dc0d3d..1f05a06 100644 --- a/src/main/ipc/handlers.ts +++ b/src/main/ipc/handlers.ts @@ -17,8 +17,10 @@ import { generateBlogmarkBookmarkletSource } from '../shared/blogmark'; import { registerMetadataDiffHandlers } from './metadataDiffHandlers'; import { registerBlogHandlers } from './blogHandlers'; import { registerPublishHandlers } from './publishHandlers'; +import { registerEmbeddingHandlers } from './embeddingHandlers'; import { isOfflineModeActive } from './chatHandlers'; import type { EngineBundle } from '../engine/EngineBundle'; +import { resolveUiLanguageFromSystemLocale, translateMenu } from '../shared/i18n'; /** * Wrap an IPC handler so that "Database is closing" errors during shutdown @@ -136,6 +138,82 @@ function buildMcpAgentConfigOptions(bundle: EngineBundle): import('../engine/MCP }; } +/** + * Start a background task that indexes all unindexed posts for semantic similarity. + * Shows progress in the TaskPopup so the user can see model loading and indexing progress. + */ +export function startEmbeddingIndexTask(bundle: EngineBundle): void { + const systemLocale = typeof app.getLocale === 'function' ? app.getLocale() : 'en'; + const lang = resolveUiLanguageFromSystemLocale(systemLocale); + const tr = (key: string) => translateMenu(lang, key); + + bundle.taskManager.runTask({ + id: `embedding-index-${Date.now()}`, + name: tr('task.embeddingIndex.name'), + execute: async (onProgress) => { + onProgress(0, tr('task.embeddingIndex.loading')); + await bundle.embeddingEngine.indexUnindexedPosts((indexed, total) => { + const pct = total > 0 ? Math.round((indexed / total) * 100) : 0; + onProgress(pct, tr('task.embeddingIndex.indexing') + .replace('{indexed}', String(indexed)) + .replace('{total}', String(total))); + }); + }, + }).catch(() => {}); +} + +/** + * Start a background task that fully rebuilds the embedding index. + * Clears existing embeddings and re-indexes all posts with progress reporting. + */ +export function startRebuildEmbeddingIndexTask(bundle: EngineBundle): void { + const systemLocale = typeof app.getLocale === 'function' ? app.getLocale() : 'en'; + const lang = resolveUiLanguageFromSystemLocale(systemLocale); + const tr = (key: string) => translateMenu(lang, key); + + bundle.taskManager.runTask({ + id: `rebuild-embedding-index-${Date.now()}`, + name: tr('task.rebuildEmbeddingIndex.name'), + execute: async (onProgress) => { + onProgress(0, tr('task.rebuildEmbeddingIndex.clearing')); + await bundle.embeddingEngine.reindexAll((indexed, total) => { + const pct = total > 0 ? Math.round((indexed / total) * 100) : 0; + onProgress(pct, tr('task.embeddingIndex.indexing') + .replace('{indexed}', String(indexed)) + .replace('{total}', String(total))); + }); + }, + }).catch(() => {}); +} + +/** + * Start a background task that searches for duplicate posts using semantic similarity. + * Once complete, the results are forwarded to the renderer via 'embeddings:duplicateSearchResult'. + */ +export function startDuplicateSearchTask(bundle: EngineBundle, threshold = 0.92): void { + const systemLocale = typeof app.getLocale === 'function' ? app.getLocale() : 'en'; + const lang = resolveUiLanguageFromSystemLocale(systemLocale); + const tr = (key: string) => translateMenu(lang, key); + + bundle.taskManager.runTask({ + id: `duplicate-search-${Date.now()}`, + name: tr('task.duplicateSearch.name'), + execute: async (onProgress) => { + onProgress(0, tr('task.duplicateSearch.searching').replace('{checked}', '0').replace('{total}', '…')); + const pairs = await bundle.embeddingEngine.findDuplicates(threshold, (checked, total) => { + const pct = total > 0 ? Math.round((checked / total) * 100) : 0; + onProgress(pct, tr('task.duplicateSearch.searching') + .replace('{checked}', String(checked)) + .replace('{total}', String(total))); + }); + onProgress(100, tr('task.duplicateSearch.name')); + return pairs; + }, + }).then((pairs) => { + ipcMain.emit('forward-to-renderer', 'embeddings:duplicateSearchResult', pairs); + }).catch(() => {}); +} + export function registerIpcHandlers(bundle: EngineBundle): void { // ============ Git Handlers ============ @@ -454,6 +532,11 @@ export function registerIpcHandlers(bundle: EngineBundle): void { return engine.getPost(id); }); + safeHandle('posts:getBySlug', async (_, slug: string) => { + const engine = bundle.postEngine; + return engine.getPostBySlug(slug); + }); + safeHandle('posts:getPreviewUrl', async (_, id: string, options?: { draft?: boolean }) => { const engine = bundle.postEngine; const post = await engine.getPost(id); @@ -1065,6 +1148,11 @@ export function registerIpcHandlers(bundle: EngineBundle): void { return; } + if (typedAction === 'rebuildEmbeddingIndex') { + startRebuildEmbeddingIndexTask(bundle); + return; + } + const handledByWebContents = runWebContentsMenuAction((event as any)?.sender, typedAction); if (handledByWebContents) { return; @@ -1189,10 +1277,15 @@ export function registerIpcHandlers(bundle: EngineBundle): void { return engine.getProjectMetadata(); }); - safeHandle('meta:updateProjectMetadata', async (_, updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; pythonRuntimeMode?: 'webworker' | 'main-thread'; picoTheme?: import('../shared/picoThemes').PicoThemeName; categoryMetadata?: Record; categorySettings?: Record }) => { + safeHandle('meta:updateProjectMetadata', async (_, updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; pythonRuntimeMode?: 'webworker' | 'main-thread'; picoTheme?: import('../shared/picoThemes').PicoThemeName; categoryMetadata?: Record; categorySettings?: Record; semanticSimilarityEnabled?: boolean }) => { const engine = bundle.metaEngine; await ensureMetaContext(engine); + const previousMetadata = await engine.getProjectMetadata(); + const wasEnabled = previousMetadata?.semanticSimilarityEnabled === true; await engine.updateProjectMetadata(updates); + if (updates.semanticSimilarityEnabled === true && !wasEnabled) { + startEmbeddingIndexTask(bundle); + } return engine.getProjectMetadata(); }); @@ -1612,6 +1705,7 @@ export function registerIpcHandlers(bundle: EngineBundle): void { registerMetadataDiffHandlers(safeHandle, bundle); registerBlogHandlers(safeHandle, bundle); registerPublishHandlers(safeHandle, bundle); + registerEmbeddingHandlers(safeHandle, bundle); // ============ MCP Config Handlers ============ @@ -1661,6 +1755,37 @@ export function registerEventForwarding(bundle: EngineBundle): void { const metaEngine = bundle.metaEngine; const tagEngine = bundle.tagEngine; const postMediaEngine = bundle.postMediaEngine; + const embeddingEngine = bundle.embeddingEngine; + + // Wire PostEngine events → EmbeddingEngine (opt-in via semanticSimilarityEnabled) + const ifSemanticEnabled = (fn: () => void): void => { + metaEngine.getProjectMetadata().then((metadata) => { + if (metadata?.semanticSimilarityEnabled === true) { + fn(); + } + }).catch(() => {}); + }; + + postEngine.on('postCreated', (post: { id: string; title: string; content: string }) => { + ifSemanticEnabled(() => { + embeddingEngine.embedPost(post.id, post.title, post.content ?? '').catch(() => {}); + }); + }); + postEngine.on('postUpdated', (post: { id: string; title: string; content: string }) => { + ifSemanticEnabled(() => { + embeddingEngine.embedPost(post.id, post.title, post.content ?? '').catch(() => {}); + }); + }); + postEngine.on('postDeleted', (id: string) => { + ifSemanticEnabled(() => { + embeddingEngine.removePost(id).catch(() => {}); + }); + }); + postEngine.on('databaseRebuilt', () => { + ifSemanticEnabled(() => { + embeddingEngine.reindexAll().catch(() => {}); + }); + }); const forwardEvent = (eventName: string) => { return (...args: unknown[]) => { @@ -1673,6 +1798,11 @@ export function registerEventForwarding(bundle: EngineBundle): void { projectEngine.on('projectUpdated', forwardEvent('project:updated')); projectEngine.on('projectDeleted', forwardEvent('project:deleted')); projectEngine.on('activeProjectChanged', forwardEvent('project:activeChanged')); + projectEngine.on('activeProjectChanged', (project: { id: string } | null) => { + if (project?.id) { + embeddingEngine.setProjectContext(project.id).catch(() => {}); + } + }); postEngine.on('postCreated', forwardEvent('post:created')); postEngine.on('postUpdated', forwardEvent('post:updated')); diff --git a/src/main/ipc/index.ts b/src/main/ipc/index.ts index 63d5753..a30d6d4 100644 --- a/src/main/ipc/index.ts +++ b/src/main/ipc/index.ts @@ -1,2 +1,2 @@ -export { registerIpcHandlers, registerEventForwarding } from './handlers'; +export { registerIpcHandlers, registerEventForwarding, startEmbeddingIndexTask, startDuplicateSearchTask, startRebuildEmbeddingIndexTask } from './handlers'; export { registerChatHandlers, initializeChatHandlers, cleanupChatHandlers } from './chatHandlers'; diff --git a/src/main/main.ts b/src/main/main.ts index f5f6ace..d768c22 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -2,7 +2,7 @@ import { app, BrowserWindow, Menu, MenuItemConstructorOptions, ipcMain, protocol import * as path from 'path'; import * as fs from 'fs'; import { getDatabase, initDatabase } from './database'; -import { registerIpcHandlers, registerEventForwarding, registerChatHandlers, initializeChatHandlers, cleanupChatHandlers } from './ipc'; +import { registerIpcHandlers, registerEventForwarding, registerChatHandlers, initializeChatHandlers, cleanupChatHandlers, startEmbeddingIndexTask, startRebuildEmbeddingIndexTask } from './ipc'; import { media } from './database/schema'; import { eq } from 'drizzle-orm'; import { MediaEngine } from './engine/MediaEngine'; @@ -26,6 +26,7 @@ import { BlogmarkPythonWorkerRuntime } from './engine/BlogmarkPythonWorkerRuntim import { PythonMacroWorkerRuntime } from './engine/PythonMacroWorkerRuntime'; import { AppApiAdapter } from './engine/AppApiAdapter'; import { PublishApiAdapter } from './engine/PublishApiAdapter'; +import { EmbeddingEngine } from './engine/EmbeddingEngine'; import { NoopNotifier } from './engine/CliNotifier'; import { NotificationWatcher } from './engine/NotificationWatcher'; import { setEngineBundle } from './engine/mainProcessPythonApiInvoker'; @@ -536,6 +537,9 @@ async function initializeActiveProjectContext(): Promise { mediaEngine.setProjectContext?.(project.id, dataDir, dataDir); metaEngine.setProjectContext?.(project.id, dataDir); + const embeddingEngineInstance = bundle!.embeddingEngine; + await embeddingEngineInstance.setProjectContext(project.id); + const templateEngine = bundle!.templateEngine as { setProjectContext?: (projectId: string, dataDir?: string) => void; }; @@ -645,6 +649,11 @@ function createApplicationMenu(): Menu { return; } + if (action === 'rebuildEmbeddingIndex') { + startRebuildEmbeddingIndexTask(bundle!); + return; + } + const channel = APP_MENU_ACTION_EVENT_MAP[action]; if (channel) { mainWindow?.webContents.send(channel); @@ -928,6 +937,10 @@ app.whenReady().then(async () => { const blogmarkPythonWorkerRuntime = new BlogmarkPythonWorkerRuntime(); const pythonMacroWorkerRuntime = new PythonMacroWorkerRuntime(); const blogmarkTransformService = new BlogmarkTransformService({ scriptEngine, metaEngine, blogmarkWorkerRuntime: blogmarkPythonWorkerRuntime }); + const embeddingEngine = new EmbeddingEngine({ + getIndexPath: (projectId: string) => + path.join(userData, 'projects', projectId, 'embeddings.usearch'), + }); const appApiAdapter = new AppApiAdapter(projectEngine); const publishApiAdapter = new PublishApiAdapter(projectEngine, publishEngine, taskManager); const mcpServer = new MCPServer({ @@ -961,6 +974,7 @@ app.whenReady().then(async () => { pythonMacroWorkerRuntime, publishApiAdapter, appApiAdapter, + embeddingEngine, }; setEngineBundle(bundle); @@ -1000,6 +1014,16 @@ app.whenReady().then(async () => { await activeProjectContextReady; appInitialized = true; + // If semantic similarity was already enabled when the app started, kick off indexing. + if (bundle) { + const startupBundle = bundle; + startupBundle.metaEngine.getProjectMetadata().then((metadata) => { + if (metadata?.semanticSimilarityEnabled === true) { + startEmbeddingIndexTask(startupBundle); + } + }).catch(() => {}); + } + const startupDeepLinks = extractBlogmarkDeepLinks(process.argv); for (const deepLink of startupDeepLinks) { enqueueBlogmarkDeepLink(deepLink); @@ -1038,6 +1062,12 @@ app.on('before-quit', async () => { console.error('Failed to cleanup MCP server:', error); } + try { + await bundle?.embeddingEngine.shutdown(); + } catch (error) { + console.error('Failed to shutdown embedding engine:', error); + } + const db = getDatabase(); await db.close(); }); diff --git a/src/main/preload.ts b/src/main/preload.ts index d041a2a..aae0b10 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -54,6 +54,7 @@ export const electronAPI: ElectronAPI = { update: (id: string, data: unknown) => ipcRenderer.invoke('posts:update', id, data), delete: (id: string) => ipcRenderer.invoke('posts:delete', id), get: (id: string) => ipcRenderer.invoke('posts:get', id), + getBySlug: (slug: string) => ipcRenderer.invoke('posts:getBySlug', slug), getPreviewUrl: (id: string, options?: { draft?: boolean }) => ipcRenderer.invoke('posts:getPreviewUrl', id, options), getAll: (options?: { limit?: number; offset?: number }) => ipcRenderer.invoke('posts:getAll', options), getByStatus: (status: string) => ipcRenderer.invoke('posts:getByStatus', status), @@ -191,7 +192,7 @@ export const electronAPI: ElectronAPI = { syncOnStartup: () => ipcRenderer.invoke('meta:syncOnStartup'), getProjectMetadata: () => ipcRenderer.invoke('meta:getProjectMetadata'), setProjectMetadata: (metadata: { name: string; description?: string }) => ipcRenderer.invoke('meta:setProjectMetadata', metadata), - updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; pythonRuntimeMode?: 'webworker' | 'main-thread'; picoTheme?: import('./shared/picoThemes').PicoThemeName; categoryMetadata?: Record; categorySettings?: Record }) => ipcRenderer.invoke('meta:updateProjectMetadata', updates), + updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; pythonRuntimeMode?: 'webworker' | 'main-thread'; picoTheme?: import('./shared/picoThemes').PicoThemeName; categoryMetadata?: Record; categorySettings?: Record; semanticSimilarityEnabled?: boolean }) => ipcRenderer.invoke('meta:updateProjectMetadata', updates), getPublishingPreferences: () => ipcRenderer.invoke('meta:getPublishingPreferences'), setPublishingPreferences: (prefs: { sshHost: string; sshUser: string; sshRemotePath: string; sshMode: 'scp' | 'rsync' }) => ipcRenderer.invoke('meta:setPublishingPreferences', prefs), clearPublishingPreferences: () => ipcRenderer.invoke('meta:clearPublishingPreferences'), @@ -452,6 +453,19 @@ export const electronAPI: ElectronAPI = { isConfigured: (agentId: string) => ipcRenderer.invoke('mcp:isConfigured', agentId), getPort: () => ipcRenderer.invoke('mcp:getPort'), }, + + // Semantic similarity / embeddings + embeddings: { + findSimilar: (postId: string, k?: number) => ipcRenderer.invoke('embeddings:findSimilar', postId, k), + computeSimilarities: (sourcePostId: string, targetPostIds: string[]) => ipcRenderer.invoke('embeddings:computeSimilarities', sourcePostId, targetPostIds), + getProgress: () => ipcRenderer.invoke('embeddings:getProgress'), + suggestTags: (postId: string, excludeTags: string[]) => ipcRenderer.invoke('embeddings:suggestTags', postId, excludeTags), + findDuplicates: (threshold?: number) => ipcRenderer.invoke('embeddings:findDuplicates', threshold), + runDuplicateSearch: (threshold?: number) => ipcRenderer.invoke('embeddings:runDuplicateSearch', threshold), + dismissPair: (postIdA: string, postIdB: string) => ipcRenderer.invoke('embeddings:dismissPair', postIdA, postIdB), + dismissPairs: (pairIds: Array<[string, string]>) => ipcRenderer.invoke('embeddings:dismissPairs', pairIds), + indexUnindexedPosts: () => ipcRenderer.invoke('embeddings:indexUnindexedPosts'), + }, }; contextBridge.exposeInMainWorld('electronAPI', electronAPI); diff --git a/src/main/shared/electronApi.ts b/src/main/shared/electronApi.ts index a32a470..57b6fe0 100644 --- a/src/main/shared/electronApi.ts +++ b/src/main/shared/electronApi.ts @@ -54,6 +54,7 @@ export interface ProjectMetadata { picoTheme?: import('./picoThemes').PicoThemeName; categoryMetadata?: Record; categorySettings?: Record; + semanticSimilarityEnabled?: boolean; } export interface CategoryRenderSettings { @@ -505,6 +506,23 @@ export interface ChatSendMetadata { import type { A2UIServerMessage, A2UIClientAction } from '../a2ui/types'; export type { A2UIServerMessage, A2UIClientAction }; +export interface SimilarPost { + postId: string; + similarity: number; +} + +export interface TagSuggestion { + name: string; + score: number; +} + +export interface DuplicatePair { + postA: { id: string; title: string; slug: string; publishedAt?: Date }; + postB: { id: string; title: string; slug: string; publishedAt?: Date }; + similarity: number; + exactMatch?: boolean; +} + export interface SiteValidationReport { sitemapPath: string; sitemapChanged: boolean; @@ -577,6 +595,7 @@ export interface ElectronAPI { update: (id: string, data: Partial) => Promise; delete: (id: string) => Promise; get: (id: string) => Promise; + getBySlug: (slug: string) => Promise; getPreviewUrl: (id: string, options?: { draft?: boolean }) => Promise; getAll: (options?: { limit?: number; offset?: number }) => Promise; getByStatus: (status: string) => Promise; @@ -728,7 +747,7 @@ export interface ElectronAPI { syncOnStartup: () => Promise<{ tags: string[]; categories: string[]; projectMetadata: ProjectMetadata | null }>; getProjectMetadata: () => Promise; setProjectMetadata: (metadata: { name: string; description?: string }) => Promise; - updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; pythonRuntimeMode?: 'webworker' | 'main-thread'; picoTheme?: import('./picoThemes').PicoThemeName; categoryMetadata?: Record; categorySettings?: Record }) => Promise; + updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; pythonRuntimeMode?: 'webworker' | 'main-thread'; picoTheme?: import('./picoThemes').PicoThemeName; categoryMetadata?: Record; categorySettings?: Record; semanticSimilarityEnabled?: boolean }) => Promise; getPublishingPreferences: () => Promise; setPublishingPreferences: (prefs: PublishingPreferences) => Promise; clearPublishingPreferences: () => Promise; @@ -986,6 +1005,17 @@ export interface ElectronAPI { onA2UIMessage: (callback: (data: { conversationId: string; message: A2UIServerMessage }) => void) => () => void; dispatchA2UIAction: (action: A2UIClientAction) => Promise<{ success: boolean; error?: string }>; }; + embeddings: { + findSimilar: (postId: string, k?: number) => Promise; + computeSimilarities: (sourcePostId: string, targetPostIds: string[]) => Promise>; + getProgress: () => Promise<{ indexed: number; total: number }>; + suggestTags: (postId: string, excludeTags: string[]) => Promise; + findDuplicates: (threshold?: number) => Promise; + runDuplicateSearch: (threshold?: number) => Promise; + dismissPair: (postIdA: string, postIdB: string) => Promise; + dismissPairs: (pairIds: Array<[string, string]>) => Promise; + indexUnindexedPosts: () => Promise; + }; on: (channel: string, callback: (...args: unknown[]) => void) => () => void; once: (channel: string, callback: (...args: unknown[]) => void) => void; /** Subscribe to entity-changed events fired by the CLI NotificationWatcher. */ diff --git a/src/main/shared/i18n/locales/de.json b/src/main/shared/i18n/locales/de.json index bafec48..031579b 100644 --- a/src/main/shared/i18n/locales/de.json +++ b/src/main/shared/i18n/locales/de.json @@ -36,11 +36,13 @@ "menu.item.previewPost": "Beitragsvorschau", "menu.item.rebuildDatabase": "Datenbank aus Dateien neu aufbauen", "menu.item.reindexText": "Suchtext neu indizieren", + "menu.item.rebuildEmbeddingIndex": "Embedding-Index neu aufbauen", "menu.item.metadataDiff": "Metadaten-Diff-Werkzeug", "menu.item.editMenu": "Blog-Menü bearbeiten", "menu.item.generateSitemap": "Site rendern", "menu.item.regenerateCalendar": "Kalender neu erzeugen", "menu.item.validateSite": "Website validieren", + "menu.item.findDuplicates": "Doppelte Beiträge finden", "menu.item.uploadSite": "Website hochladen", "menu.item.about": "Über Blogging Desktop Server", "menu.item.openDocumentation": "Dokumentation öffnen", @@ -80,5 +82,12 @@ "render.month.11": "Nov.", "render.month.12": "Dezember", "ai.imageAnalysis.system": "Du erzeugst Bild-Metadaten. Schreibe alle Werte auf Deutsch.\n\nRegeln:\n- \"title\": kurzer beschreibender Titel (3-8 Wörter)\n- \"alt\": sachliche Beschreibung des Sichtbaren (5-12 Wörter). Keine Interpretationen. Kein Präfix \"Bild von\".\n- \"caption\": ansprechende Blog-Bildunterschrift (5-20 Wörter)\n\nAntworte ausschließlich mit JSON: {\"title\": \"...\", \"alt\": \"...\", \"caption\": \"...\"}", - "ai.imageAnalysis.user": "Analysiere dieses Bild. Antworte mit JSON auf Deutsch." + "ai.imageAnalysis.user": "Analysiere dieses Bild. Antworte mit JSON auf Deutsch.", + "task.embeddingIndex.name": "Beiträge für semantische Ähnlichkeit indexieren", + "task.embeddingIndex.loading": "Modell wird geladen…", + "task.embeddingIndex.indexing": "Indexierung: {indexed}/{total}", + "task.rebuildEmbeddingIndex.name": "Embedding-Index neu aufbauen", + "task.rebuildEmbeddingIndex.clearing": "Index wird geleert…", + "task.duplicateSearch.name": "Doppelte Beiträge finden", + "task.duplicateSearch.searching": "Prüfe: {checked}/{total}" } diff --git a/src/main/shared/i18n/locales/en.json b/src/main/shared/i18n/locales/en.json index eb16698..5ff9841 100644 --- a/src/main/shared/i18n/locales/en.json +++ b/src/main/shared/i18n/locales/en.json @@ -36,11 +36,13 @@ "menu.item.previewPost": "Preview Post", "menu.item.rebuildDatabase": "Rebuild Database from Files", "menu.item.reindexText": "Reindex Search Text", + "menu.item.rebuildEmbeddingIndex": "Rebuild Embedding Index", "menu.item.metadataDiff": "Metadata Diff Tool", "menu.item.editMenu": "Edit Blog Menu", "menu.item.generateSitemap": "Render Site", "menu.item.regenerateCalendar": "Regenerate Calendar", "menu.item.validateSite": "Validate Site", + "menu.item.findDuplicates": "Find Duplicate Posts", "menu.item.uploadSite": "Upload Site", "menu.item.about": "About Blogging Desktop Server", "menu.item.openDocumentation": "Open Documentation", @@ -80,5 +82,12 @@ "render.month.11": "November", "render.month.12": "December", "ai.imageAnalysis.system": "You generate image metadata. Write all values in English.\n\nRules:\n- \"title\": short descriptive title (3-8 words)\n- \"alt\": factual description of what is visible (5-12 words). No interpretations. No \"Image of\" prefix.\n- \"caption\": engaging blog caption (5-20 words)\n\nRespond with JSON only: {\"title\": \"...\", \"alt\": \"...\", \"caption\": \"...\"}", - "ai.imageAnalysis.user": "Analyze this image. Respond with JSON in English." + "ai.imageAnalysis.user": "Analyze this image. Respond with JSON in English.", + "task.embeddingIndex.name": "Index posts for Semantic Similarity", + "task.embeddingIndex.loading": "Loading model…", + "task.embeddingIndex.indexing": "Indexing: {indexed}/{total}", + "task.rebuildEmbeddingIndex.name": "Rebuild Embedding Index", + "task.rebuildEmbeddingIndex.clearing": "Clearing index…", + "task.duplicateSearch.name": "Find Duplicate Posts", + "task.duplicateSearch.searching": "Checking: {checked}/{total}" } diff --git a/src/main/shared/i18n/locales/es.json b/src/main/shared/i18n/locales/es.json index a60da8b..7162f1f 100644 --- a/src/main/shared/i18n/locales/es.json +++ b/src/main/shared/i18n/locales/es.json @@ -36,11 +36,13 @@ "menu.item.previewPost": "Vista previa de entrada", "menu.item.rebuildDatabase": "Reconstruir Database from Files", "menu.item.reindexText": "Reindex Buscar Text", + "menu.item.rebuildEmbeddingIndex": "Reconstruir índice de embeddings", "menu.item.metadataDiff": "Herramienta diff de metadatos", "menu.item.editMenu": "Editar menú del blog", "menu.item.generateSitemap": "Renderizar sitio", "menu.item.regenerateCalendar": "Regenerar calendario", "menu.item.validateSite": "Validar sitio", + "menu.item.findDuplicates": "Buscar entradas duplicadas", "menu.item.uploadSite": "Subir sitio", "menu.item.about": "Acerca de Blogging Desktop Server", "menu.item.openDocumentation": "Abrir documentación", @@ -80,5 +82,12 @@ "render.month.11": "noviembre", "render.month.12": "diciembre", "ai.imageAnalysis.system": "Generas metadatos de imagen. Escribe todos los valores en español.\n\nReglas:\n- \"title\": título descriptivo corto (3-8 palabras)\n- \"alt\": descripción factual de lo visible (5-12 palabras). Sin interpretaciones. Sin prefijo \"Imagen de\".\n- \"caption\": pie de foto atractivo para blog (5-20 palabras)\n\nResponde solo con JSON: {\"title\": \"...\", \"alt\": \"...\", \"caption\": \"...\"}", - "ai.imageAnalysis.user": "Analiza esta imagen. Responde con JSON en español." + "ai.imageAnalysis.user": "Analiza esta imagen. Responde con JSON en español.", + "task.embeddingIndex.name": "Indexar entradas para similitud semántica", + "task.embeddingIndex.loading": "Cargando modelo…", + "task.embeddingIndex.indexing": "Indexando: {indexed}/{total}", + "task.rebuildEmbeddingIndex.name": "Reconstruir índice de embeddings", + "task.rebuildEmbeddingIndex.clearing": "Vaciando índice…", + "task.duplicateSearch.name": "Buscar entradas duplicadas", + "task.duplicateSearch.searching": "Comprobando: {checked}/{total}" } diff --git a/src/main/shared/i18n/locales/fr.json b/src/main/shared/i18n/locales/fr.json index cb2b0dc..6adbc3b 100644 --- a/src/main/shared/i18n/locales/fr.json +++ b/src/main/shared/i18n/locales/fr.json @@ -36,11 +36,13 @@ "menu.item.previewPost": "Aperçu de l’article", "menu.item.rebuildDatabase": "Reconstruire Database from Files", "menu.item.reindexText": "Reindex Recherche Text", + "menu.item.rebuildEmbeddingIndex": "Reconstruire l'index d'embeddings", "menu.item.metadataDiff": "Outil de diff des métadonnées", "menu.item.editMenu": "Modifier le menu du blog", "menu.item.generateSitemap": "Rendre le site", "menu.item.regenerateCalendar": "Régénérer le calendrier", "menu.item.validateSite": "Valider le site", + "menu.item.findDuplicates": "Trouver les articles en double", "menu.item.uploadSite": "Publier le site", "menu.item.about": "À propos de Blogging Desktop Server", "menu.item.openDocumentation": "Ouvrir la documentation", @@ -80,5 +82,12 @@ "render.month.11": "novembre", "render.month.12": "décembre", "ai.imageAnalysis.system": "Tu génères des métadonnées d'image. Écris toutes les valeurs en français.\n\nRègles :\n- \"title\" : titre descriptif court (3-8 mots)\n- \"alt\" : description factuelle de ce qui est visible (5-12 mots). Pas d'interprétations. Pas de préfixe \"Image de\".\n- \"caption\" : légende de blog engageante (5-20 mots)\n\nRéponds uniquement en JSON : {\"title\": \"...\", \"alt\": \"...\", \"caption\": \"...\"}", - "ai.imageAnalysis.user": "Analyse cette image. Réponds en JSON en français." + "ai.imageAnalysis.user": "Analyse cette image. Réponds en JSON en français.", + "task.embeddingIndex.name": "Indexer les articles pour la similarité sémantique", + "task.embeddingIndex.loading": "Chargement du modèle…", + "task.embeddingIndex.indexing": "Indexation : {indexed}/{total}", + "task.rebuildEmbeddingIndex.name": "Reconstruire l'index d'embeddings", + "task.rebuildEmbeddingIndex.clearing": "Vidage de l'index…", + "task.duplicateSearch.name": "Trouver les articles en double", + "task.duplicateSearch.searching": "Vérification : {checked}/{total}" } diff --git a/src/main/shared/i18n/locales/it.json b/src/main/shared/i18n/locales/it.json index c66b2dd..a744d30 100644 --- a/src/main/shared/i18n/locales/it.json +++ b/src/main/shared/i18n/locales/it.json @@ -36,11 +36,13 @@ "menu.item.previewPost": "Anteprima post", "menu.item.rebuildDatabase": "Ricostruisci Database from Files", "menu.item.reindexText": "Reindex Ricerca Text", + "menu.item.rebuildEmbeddingIndex": "Ricostruisci indice embeddings", "menu.item.metadataDiff": "Strumento diff metadati", "menu.item.editMenu": "Modifica menu blog", "menu.item.generateSitemap": "Renderizza sito", "menu.item.regenerateCalendar": "Rigenera calendario", "menu.item.validateSite": "Valida sito", + "menu.item.findDuplicates": "Trova post duplicati", "menu.item.uploadSite": "Carica sito", "menu.item.about": "Informazioni su Blogging Desktop Server", "menu.item.openDocumentation": "Apri documentazione", @@ -80,5 +82,12 @@ "render.month.11": "novembre", "render.month.12": "dicembre", "ai.imageAnalysis.system": "Generi metadati per immagini. Scrivi tutti i valori in italiano.\n\nRegole:\n- \"title\": titolo descrittivo breve (3-8 parole)\n- \"alt\": descrizione fattuale di ciò che è visibile (5-12 parole). Nessuna interpretazione. Nessun prefisso \"Immagine di\".\n- \"caption\": didascalia blog coinvolgente (5-20 parole)\n\nRispondi solo con JSON: {\"title\": \"...\", \"alt\": \"...\", \"caption\": \"...\"}", - "ai.imageAnalysis.user": "Analizza questa immagine. Rispondi con JSON in italiano." + "ai.imageAnalysis.user": "Analizza questa immagine. Rispondi con JSON in italiano.", + "task.embeddingIndex.name": "Indicizza i post per la similarità semantica", + "task.embeddingIndex.loading": "Caricamento modello…", + "task.embeddingIndex.indexing": "Indicizzazione: {indexed}/{total}", + "task.rebuildEmbeddingIndex.name": "Ricostruisci indice embeddings", + "task.rebuildEmbeddingIndex.clearing": "Svuotamento indice…", + "task.duplicateSearch.name": "Trova post duplicati", + "task.duplicateSearch.searching": "Controllo: {checked}/{total}" } diff --git a/src/main/shared/menuCommands.ts b/src/main/shared/menuCommands.ts index 5a991a2..9afa5cf 100644 --- a/src/main/shared/menuCommands.ts +++ b/src/main/shared/menuCommands.ts @@ -31,6 +31,7 @@ export type AppMenuAction = | 'previewPost' | 'rebuildDatabase' | 'reindexText' + | 'rebuildEmbeddingIndex' | 'metadataDiff' | 'editMenu' | 'generateSitemap' @@ -39,6 +40,7 @@ export type AppMenuAction = | 'uploadSite' | 'openDocumentation' | 'openApiDocumentation' + | 'findDuplicates' | 'about' | 'viewOnGitHub' | 'reportIssue'; @@ -127,12 +129,14 @@ export const APP_MENU_GROUPS: AppMenuGroupDefinition[] = [ { label: '', action: 'blog-separator-2', separator: true }, { label: 'menu.item.rebuildDatabase', action: 'rebuildDatabase' }, { label: 'menu.item.reindexText', action: 'reindexText' }, + { label: 'menu.item.rebuildEmbeddingIndex', action: 'rebuildEmbeddingIndex' }, { label: '', action: 'blog-separator-3', separator: true }, { label: 'menu.item.metadataDiff', action: 'metadataDiff' }, { label: 'menu.item.editMenu', action: 'editMenu' }, { label: 'menu.item.generateSitemap', action: 'generateSitemap', accelerator: 'CmdOrCtrl+R' }, { label: 'menu.item.regenerateCalendar', action: 'regenerateCalendar' }, { label: 'menu.item.validateSite', action: 'validateSite', accelerator: 'CmdOrCtrl+Shift+L' }, + { label: 'menu.item.findDuplicates', action: 'findDuplicates' }, { label: '', action: 'blog-separator-4', separator: true }, { label: 'menu.item.uploadSite', action: 'uploadSite', accelerator: 'CmdOrCtrl+Shift+U' }, ], @@ -167,11 +171,13 @@ export const APP_MENU_ACTION_EVENT_MAP: Partial> = publishSelected: 'menu:publishSelected', rebuildDatabase: 'menu:rebuildDatabase', reindexText: 'menu:reindexText', + rebuildEmbeddingIndex: 'menu:rebuildEmbeddingIndex', metadataDiff: 'menu:metadataDiff', editMenu: 'menu:editMenu', generateSitemap: 'menu:generateSitemap', regenerateCalendar: 'menu:regenerateCalendar', validateSite: 'menu:validateSite', + findDuplicates: 'menu:findDuplicates', uploadSite: 'menu:uploadSite', openDocumentation: 'menu:openDocumentation', openApiDocumentation: 'menu:openApiDocumentation', diff --git a/src/main/shared/pythonApiContractV1.ts b/src/main/shared/pythonApiContractV1.ts index 20f50ed..d95d09a 100644 --- a/src/main/shared/pythonApiContractV1.ts +++ b/src/main/shared/pythonApiContractV1.ts @@ -81,6 +81,7 @@ const METHODS_V1: PythonApiMethodContractV1[] = [ method('posts.update', 'Update a post by id.', [requiredString('id'), requiredObject('data')], 'PostData | null'), method('posts.delete', 'Delete a post by id.', [requiredString('id')], 'boolean'), method('posts.get', 'Fetch one post by id.', [requiredString('postId')], 'PostData | null'), + method('posts.getBySlug', 'Fetch one post by slug.', [requiredString('slug')], 'PostData | null'), method('posts.getPreviewUrl', 'Get preview URL for post.', [requiredString('id'), optionalObject('options')], 'string | null'), method('posts.getAll', 'Fetch posts with pagination.', [optionalObject('options')], 'PaginatedPostsResult'), method('posts.getByStatus', 'Fetch posts by status.', [requiredString('status')], 'PostData[]'), @@ -202,6 +203,14 @@ const METHODS_V1: PythonApiMethodContractV1[] = [ method('sync.commitAll', 'Stage all changes and commit for active project.', [requiredString('message')], 'GitActionResult'), method('publish.uploadSite', 'Upload rendered site to remote server via SSH.', [requiredObject('credentials')], 'PublishSiteResult'), + + method('embeddings.findSimilar', 'Find posts semantically similar to the given post. Requires semantic similarity to be enabled in project settings.', [requiredString('postId'), optionalNumber('k')], 'SimilarPost[]'), + method('embeddings.computeSimilarities', 'Compute cosine similarity between a source post and a list of target posts. Returns a mapping of target post IDs to similarity scores (0.0-1.0). Posts without embeddings are omitted.', [requiredString('sourcePostId'), requiredArray('targetPostIds')], 'Record'), + method('embeddings.getProgress', 'Get the embedding indexing progress for the active project.', [], '{ indexed: number; total: number }'), + method('embeddings.suggestTags', 'Suggest tags for a post based on tags used by semantically similar posts.', [requiredString('postId'), requiredArray('excludeTags')], 'TagSuggestion[]'), + method('embeddings.findDuplicates', 'Find post pairs with high content similarity (potential duplicates). Threshold is a similarity value from 0.0 to 1.0 (default 0.85).', [optionalNumber('threshold')], 'DuplicatePair[]'), + method('embeddings.dismissPair', 'Dismiss a duplicate pair so it no longer appears in results.', [requiredString('postIdA'), requiredString('postIdB')], 'void'), + method('embeddings.indexUnindexedPosts', 'Trigger background indexing of all posts not yet embedded.', [], 'void'), ]; const DATA_STRUCTURES_V1: PythonApiDataStructureContractV1[] = [ @@ -345,6 +354,7 @@ const DATA_STRUCTURES_V1: PythonApiDataStructureContractV1[] = [ { name: 'picoTheme', type: 'string', required: false, description: 'Preferred Pico theme token.' }, { name: 'categoryMetadata', type: 'object', required: false, description: 'Category metadata keyed by category slug.' }, { name: 'categorySettings', type: 'object', required: false, description: 'Category render settings keyed by category slug.' }, + { name: 'semanticSimilarityEnabled', type: 'boolean', required: false, description: 'Enable local ONNX embedding-based semantic similarity features.' }, ], }, { @@ -415,11 +425,36 @@ const DATA_STRUCTURES_V1: PythonApiDataStructureContractV1[] = [ { name: 'error', type: 'string', required: false, description: 'Error message when analysis failed.' }, ], }, + { + name: 'SimilarPost', + description: 'A post with its semantic similarity score relative to a reference post.', + fields: [ + { name: 'postId', type: 'string', required: true, description: 'Post identifier.' }, + { name: 'similarity', type: 'number', required: true, description: 'Cosine similarity score from 0.0 to 1.0.' }, + ], + }, + { + name: 'TagSuggestion', + description: 'A tag suggested based on semantic similarity to similar posts.', + fields: [ + { name: 'name', type: 'string', required: true, description: 'Tag name.' }, + { name: 'score', type: 'number', required: true, description: 'Aggregated suggestion score.' }, + ], + }, + { + name: 'DuplicatePair', + description: 'A pair of posts with high content similarity that may be duplicates.', + fields: [ + { name: 'postA', type: '{ id: string; title: string; slug: string; publishedAt?: string }', required: true, description: 'First post in the pair.' }, + { name: 'postB', type: '{ id: string; title: string; slug: string; publishedAt?: string }', required: true, description: 'Second post in the pair.' }, + { name: 'similarity', type: 'number', required: true, description: 'Cosine similarity score from 0.0 to 1.0.' }, + ], + }, ]; export const BDS_PYTHON_API_CONTRACT_V1: PythonApiContractV1 = { - version: '1.11.0', - generatedAt: '2026-02-27T00:00:00.000Z', + version: '1.12.0', + generatedAt: '2026-03-05T00:00:00.000Z', methods: METHODS_V1, dataStructures: DATA_STRUCTURES_V1, }; diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index dba8472..122e353 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -4,6 +4,7 @@ import { useAppStore, PostData, MediaData, TaskProgress } from './store'; import { loadTabsForProject, saveTabsForProject } from './utils'; import { openSingletonToolTab } from './navigation/tabPolicy'; import { persistSiteValidationReport } from './navigation/siteValidationPersistence'; +import { persistDuplicatesResult } from './navigation/duplicatesPersistence'; import { executeActivityClick } from './navigation/activityExecution'; import { handleBlogmarkCreatedEvent } from './navigation/blogmarkHandling'; import { @@ -444,6 +445,25 @@ const App: React.FC = () => { }) || (() => {}) ); + unsubscribers.push( + window.electronAPI?.on('menu:findDuplicates', () => { + openSingletonToolTab(openTab, 'find-duplicates'); + }) || (() => {}) + ); + + unsubscribers.push( + window.electronAPI?.on('embeddings:duplicateSearchResult', (...args: unknown[]) => { + const pairs = args[0] as import('../main/shared/electronApi').DuplicatePair[]; + const projectId = useAppStore.getState().activeProject?.id; + if (projectId && pairs) { + persistDuplicatesResult(projectId, pairs); + window.dispatchEvent(new CustomEvent('bds:duplicates-updated', { + detail: { projectId }, + })); + } + }) || (() => {}) + ); + unsubscribers.push( window.electronAPI?.on('menu:generateSitemap', async () => { try { diff --git a/src/renderer/components/ConfirmDeleteModal/ConfirmDeleteModal.tsx b/src/renderer/components/ConfirmDeleteModal/ConfirmDeleteModal.tsx index b7d2a24..cc20f51 100644 --- a/src/renderer/components/ConfirmDeleteModal/ConfirmDeleteModal.tsx +++ b/src/renderer/components/ConfirmDeleteModal/ConfirmDeleteModal.tsx @@ -22,10 +22,9 @@ interface ConfirmDeleteModalProps { export const ConfirmDeleteModal: React.FC = ({ details, onClose }) => { const { t: tr } = useI18n(); - if (!details) return null; const handleConfirm = useCallback(async () => { - await details.onConfirm(); + await details?.onConfirm(); onClose(); }, [details, onClose]); @@ -35,6 +34,8 @@ export const ConfirmDeleteModal: React.FC = ({ details, } }, [onClose]); + if (!details) return null; + const hasReferences = details.references.length > 0; return ( diff --git a/src/renderer/components/DuplicatesView/DuplicatesView.css b/src/renderer/components/DuplicatesView/DuplicatesView.css new file mode 100644 index 0000000..8250d5b --- /dev/null +++ b/src/renderer/components/DuplicatesView/DuplicatesView.css @@ -0,0 +1,190 @@ +.duplicates-view { + display: flex; + flex-direction: column; + gap: 16px; + padding: 16px; + height: 100%; + overflow: auto; + background: var(--vscode-editor-background); + color: var(--vscode-editor-foreground); +} + +.duplicates-view-header h2 { + margin: 0 0 4px 0; + font-size: 1.1rem; +} + +.duplicates-view-header p { + margin: 0; + color: var(--vscode-descriptionForeground); + font-size: 0.875rem; +} + +.duplicates-view-actions { + display: flex; + gap: 8px; +} + +.duplicates-view-refresh { + background-color: var(--vscode-button-secondaryBackground); + color: var(--vscode-button-secondaryForeground); + border: none; + padding: 5px 12px; + border-radius: 4px; + cursor: pointer; + font-size: 0.875rem; +} + +.duplicates-view-refresh:hover:not(:disabled) { + background-color: var(--vscode-button-secondaryHoverBackground); +} + +.duplicates-view-refresh:disabled { + opacity: 0.6; + cursor: not-allowed; +} + +.duplicates-view-status { + margin: 0; + color: var(--vscode-descriptionForeground); + font-size: 0.875rem; +} + +.duplicates-view-list { + display: flex; + flex-direction: column; + gap: 8px; +} + +.duplicate-pair { + display: flex; + align-items: center; + gap: 12px; + padding: 10px 12px; + background: var(--vscode-editorWidget-background); + border: 1px solid var(--vscode-editorWidget-border, var(--vscode-panel-border)); + border-radius: 6px; +} + +.duplicate-pair-posts { + flex: 1; + display: flex; + flex-direction: column; + gap: 4px; + min-width: 0; +} + +.duplicate-pair-post { + background: none; + border: none; + padding: 0; + cursor: pointer; + color: var(--vscode-textLink-foreground); + font-size: 0.875rem; + text-align: left; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.duplicate-pair-post:hover { + color: var(--vscode-textLink-activeForeground); + text-decoration: underline; +} + +.duplicate-pair-separator { + font-size: 0.75rem; + color: var(--vscode-descriptionForeground); + padding-left: 2px; +} + +.duplicate-pair-meta { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 6px; + flex-shrink: 0; +} + +.duplicate-pair-score { + font-size: 0.8rem; + font-weight: 600; + color: var(--vscode-charts-orange, #f8ae4a); + white-space: nowrap; +} + +.duplicate-pair-score--exact { + color: var(--vscode-errorForeground, #f44747); +} + +.duplicate-pair--exact { + border-color: var(--vscode-errorForeground, #f44747); +} + +.duplicate-pair-dismiss { + background: none; + border: 1px solid var(--vscode-button-border, var(--vscode-panel-border)); + padding: 2px 8px; + border-radius: 4px; + cursor: pointer; + font-size: 0.75rem; + color: var(--vscode-descriptionForeground); + white-space: nowrap; +} + +.duplicate-pair-dismiss:hover { + background-color: var(--vscode-list-hoverBackground); + color: var(--vscode-foreground); +} + +.duplicates-view-not-enabled { + padding: 32px; + text-align: center; + color: var(--vscode-descriptionForeground); +} + +.duplicates-view-count { + margin: 0; + font-size: 0.8rem; + color: var(--vscode-descriptionForeground); +} + +.duplicates-view-show-more { + align-self: center; + background-color: var(--vscode-button-secondaryBackground); + color: var(--vscode-button-secondaryForeground); + border: none; + padding: 6px 20px; + border-radius: 4px; + cursor: pointer; + font-size: 0.875rem; +} + +.duplicates-view-show-more:hover { + background-color: var(--vscode-button-secondaryHoverBackground); +} + +.duplicates-view-dismiss-checked { + background-color: var(--vscode-button-background); + color: var(--vscode-button-foreground); + border: none; + padding: 5px 12px; + border-radius: 4px; + cursor: pointer; + font-size: 0.875rem; +} + +.duplicates-view-dismiss-checked:hover:not(:disabled) { + background-color: var(--vscode-button-hoverBackground); +} + +.duplicates-view-dismiss-checked:disabled { + opacity: 0.6; + cursor: not-allowed; +} + +.duplicate-pair-checkbox { + flex-shrink: 0; + cursor: pointer; + accent-color: var(--vscode-focusBorder); +} diff --git a/src/renderer/components/DuplicatesView/DuplicatesView.tsx b/src/renderer/components/DuplicatesView/DuplicatesView.tsx new file mode 100644 index 0000000..d92600c --- /dev/null +++ b/src/renderer/components/DuplicatesView/DuplicatesView.tsx @@ -0,0 +1,249 @@ +import React, { useState, useEffect, useCallback, useMemo } from 'react'; +import { useAppStore } from '../../store'; +import { openEntityTab } from '../../navigation/tabPolicy'; +import { useI18n } from '../../i18n'; +import { getPersistedDuplicatesResult, removeDismissedPair, removeDismissedPairs } from '../../navigation/duplicatesPersistence'; +import type { DuplicatePair } from '../../../main/shared/electronApi'; +import './DuplicatesView.css'; + +const PAGE_SIZE = 500; + +function pairKey(pair: DuplicatePair): string { + return `${pair.postA.id}::${pair.postB.id}`; +} + +export const DuplicatesView: React.FC = () => { + const { t } = useI18n(); + const { openTab, activeProject } = useAppStore(); + const [pairs, setPairs] = useState([]); + const [isSearching, setIsSearching] = useState(false); + const [checkedKeys, setCheckedKeys] = useState>(new Set()); + const [isDismissing, setIsDismissing] = useState(false); + const [notEnabled, setNotEnabled] = useState(false); + const [checked, setChecked] = useState(false); + const [visibleCount, setVisibleCount] = useState(PAGE_SIZE); + + const projectId = activeProject?.id; + + // Load persisted results (or check if feature is enabled) + useEffect(() => { + if (!projectId) return; + let cancelled = false; + (async () => { + const metadata = await window.electronAPI?.meta.getProjectMetadata(); + if (cancelled) return; + if (!metadata?.semanticSimilarityEnabled) { + setNotEnabled(true); + setPairs([]); + setChecked(true); + return; + } + setNotEnabled(false); + const persisted = getPersistedDuplicatesResult(projectId); + if (persisted) { + setPairs(persisted); + } + setChecked(true); + })(); + return () => { cancelled = true; }; + }, [projectId]); + + // Listen for search result updates from the background task + useEffect(() => { + const handler = (event: Event) => { + const detail = (event as CustomEvent<{ projectId?: string }>).detail; + if (!projectId || detail?.projectId !== projectId) return; + const persisted = getPersistedDuplicatesResult(projectId); + setPairs(persisted ?? []); + setIsSearching(false); + setVisibleCount(PAGE_SIZE); + setCheckedKeys(new Set()); + }; + window.addEventListener('bds:duplicates-updated', handler); + return () => window.removeEventListener('bds:duplicates-updated', handler); + }, [projectId]); + + const visiblePairs = useMemo(() => pairs.slice(0, visibleCount), [pairs, visibleCount]); + const hasMore = visibleCount < pairs.length; + + const handleRunSearch = useCallback(() => { + setIsSearching(true); + window.electronAPI?.embeddings.runDuplicateSearch(0.92); + }, []); + + const handleDismiss = useCallback(async (postIdA: string, postIdB: string) => { + try { + await window.electronAPI?.embeddings.dismissPair(postIdA, postIdB); + setPairs(prev => prev.filter(p => !(p.postA.id === postIdA && p.postB.id === postIdB))); + setCheckedKeys(prev => { const next = new Set(prev); next.delete(`${postIdA}::${postIdB}`); return next; }); + if (projectId) { + removeDismissedPair(projectId, postIdA, postIdB); + } + } catch (err) { + console.error('Failed to dismiss duplicate pair:', err); + } + }, [projectId]); + + const handleDismissChecked = useCallback(async () => { + if (checkedKeys.size === 0) return; + setIsDismissing(true); + const pairIds: Array<[string, string]> = []; + for (const key of checkedKeys) { + const [a, b] = key.split('::') as [string, string]; + pairIds.push([a, b]); + } + try { + await window.electronAPI?.embeddings.dismissPairs(pairIds); + setPairs(prev => prev.filter(p => !checkedKeys.has(pairKey(p)))); + if (projectId) { + removeDismissedPairs(projectId, pairIds); + } + setCheckedKeys(new Set()); + } catch (err) { + console.error('Failed to dismiss pairs:', err); + } finally { + setIsDismissing(false); + } + }, [checkedKeys, projectId]); + + const handleToggleCheck = useCallback((key: string) => { + setCheckedKeys(prev => { + const next = new Set(prev); + if (next.has(key)) next.delete(key); else next.add(key); + return next; + }); + }, []); + + const handleCheckAll = useCallback(() => { + setCheckedKeys(new Set(visiblePairs.map(pairKey))); + }, [visiblePairs]); + + const handleUncheckAll = useCallback(() => { + setCheckedKeys(new Set()); + }, []); + + const handleOpenPost = useCallback((postId: string) => { + openEntityTab(openTab, 'post', postId, 'pin'); + }, [openTab]); + + const handleShowMore = useCallback(() => { + setVisibleCount(prev => prev + PAGE_SIZE); + }, []); + + const hasCachedResults = pairs.length > 0 || (checked && getPersistedDuplicatesResult(projectId ?? '') !== null); + + return ( +
+
+

{t('duplicatesView.title')}

+

{t('duplicatesView.description')}

+
+ + {notEnabled && ( +

{t('duplicatesView.notEnabled')}

+ )} + + {!notEnabled && checked && ( +
+ + {pairs.length > 0 && ( + <> + + + + + )} +
+ )} + + {!notEnabled && isSearching && ( +

{t('duplicatesView.loading')}

+ )} + + {!notEnabled && !isSearching && checked && !hasCachedResults && ( +

{t('duplicatesView.empty')}

+ )} + + {!notEnabled && !isSearching && pairs.length > 0 && ( +
+

+ {t('duplicatesView.count', { count: pairs.length })} +

+ {visiblePairs.map(pair => { + const key = pairKey(pair); + return ( +
+ handleToggleCheck(key)} + /> +
+ + + +
+
+ + {pair.exactMatch + ? t('duplicatesView.exactMatch') + : t('duplicatesView.similarity', { value: Math.round(pair.similarity * 100) })} + + +
+
+ ); + })} + {hasMore && ( + + )} +
+ )} +
+ ); +}; diff --git a/src/renderer/components/Editor/Editor.tsx b/src/renderer/components/Editor/Editor.tsx index 994406c..1bcf5c9 100644 --- a/src/renderer/components/Editor/Editor.tsx +++ b/src/renderer/components/Editor/Editor.tsx @@ -21,6 +21,7 @@ import { DocumentationView } from '../DocumentationView/DocumentationView'; import { SiteValidationView } from '../SiteValidationView'; import { ScriptsView } from '../ScriptsView/ScriptsView'; import { TemplatesView } from '../TemplatesView/TemplatesView'; +import { DuplicatesView } from '../DuplicatesView/DuplicatesView'; import { AutoSaveManager, getContrastColor, loadTagColorMap } from '../../utils'; import { InsertModal } from '../InsertModal'; import { AISuggestionsModal, AISuggestions } from '../AISuggestionsModal/AISuggestionsModal'; @@ -794,6 +795,7 @@ export const PostEditor: React.FC = ({ postId }) => { value={tags} onChange={setTags} placeholder={tr('editor.placeholder.tags')} + postId={postId} />
@@ -1024,6 +1026,7 @@ export const PostEditor: React.FC = ({ postId }) => { onClose={() => setShowPostSearch(false)} currentPostTags={tags} currentPostCategories={selectedCategories} + currentPostId={postId} /> )} @@ -1903,6 +1906,7 @@ export const Editor: React.FC = () => { /> ), 'site-validation': () => , + 'find-duplicates': () => , scripts: () => , templates: () => , post: () => (editorRoute.tabId ? : ), diff --git a/src/renderer/components/InsertModal/InsertModal.css b/src/renderer/components/InsertModal/InsertModal.css index ade5ff2..162325b 100644 --- a/src/renderer/components/InsertModal/InsertModal.css +++ b/src/renderer/components/InsertModal/InsertModal.css @@ -140,6 +140,18 @@ font-family: 'Cascadia Code', 'Consolas', 'Courier New', monospace; } +.insert-modal-similarity-badge { + display: inline-block; + margin-left: 8px; + padding: 1px 6px; + font-size: 11px; + font-weight: 500; + border-radius: 4px; + background: var(--color-bg-muted, rgba(255, 255, 255, 0.08)); + color: var(--color-text-muted, #888); + vertical-align: middle; +} + .insert-modal-external { padding: 20px; display: flex; diff --git a/src/renderer/components/InsertModal/InsertModal.tsx b/src/renderer/components/InsertModal/InsertModal.tsx index 09fad95..c3971e8 100644 --- a/src/renderer/components/InsertModal/InsertModal.tsx +++ b/src/renderer/components/InsertModal/InsertModal.tsx @@ -42,6 +42,7 @@ interface InsertModalProps { initialText?: string; // Selected text in editor currentPostTags?: string[]; currentPostCategories?: string[]; + currentPostId?: string; // For semantic "related posts" suggestions } function isPostResult(result: SearchResult): result is PostSearchResult { @@ -60,6 +61,7 @@ export const InsertModal: React.FC = ({ initialText = '', currentPostTags, currentPostCategories, + currentPostId, }) => { const { t: tr } = useI18n(); const openTabInBackground = useAppStore((s) => s.openTabInBackground); @@ -74,6 +76,42 @@ export const InsertModal: React.FC = ({ const [isCreating, setIsCreating] = useState(false); const inputRef = useRef(null); const externalUrlRef = useRef(null); + const [relatedPosts, setRelatedPosts] = useState([]); + const [isLoadingRelated, setIsLoadingRelated] = useState(false); + const [similarityMap, setSimilarityMap] = useState>({}); + + // Load related posts via semantic similarity when idle (query < 2 chars) + useEffect(() => { + if (mode !== 'link' || !currentPostId || activeTab !== 'internal' || query.length >= 2) { + setRelatedPosts([]); + return; + } + let cancelled = false; + setIsLoadingRelated(true); + (async () => { + try { + const similar = await window.electronAPI.embeddings.findSimilar(currentPostId, 5); + if (cancelled || similar.length === 0) { setRelatedPosts([]); return; } + const posts = await Promise.all(similar.map(s => window.electronAPI.posts.get(s.postId))); + if (!cancelled) { + // Store similarity scores + const simMap: Record = {}; + for (const s of similar) { simMap[s.postId] = s.similarity; } + setSimilarityMap(simMap); + setRelatedPosts( + posts.filter((p): p is NonNullable => p != null).map(p => ({ + id: p.id, title: p.title, slug: p.slug, excerpt: p.excerpt, + })), + ); + } + } catch { + if (!cancelled) setRelatedPosts([]); + } finally { + if (!cancelled) setIsLoadingRelated(false); + } + })(); + return () => { cancelled = true; }; + }, [currentPostId, mode, activeTab, query]); // Whether to show the "Create post" option const showCreateOption = mode === 'link' && @@ -124,6 +162,22 @@ export const InsertModal: React.FC = ({ return () => clearTimeout(timeoutId); }, [query, mode, activeTab]); + // Fetch similarity scores for search results relative to current post + useEffect(() => { + if (mode !== 'link' || !currentPostId || results.length === 0) return; + const postResults = results.filter(isPostResult); + if (postResults.length === 0) return; + let cancelled = false; + (async () => { + try { + const targetIds = postResults.map(r => r.id); + const sims = await window.electronAPI.embeddings.computeSimilarities(currentPostId, targetIds); + if (!cancelled) setSimilarityMap(prev => ({ ...prev, ...sims })); + } catch { /* ignore */ } + })(); + return () => { cancelled = true; }; + }, [results, currentPostId, mode]); + // Handle creating a new post from the search query const handleCreatePost = useCallback(async () => { const title = query.trim(); @@ -284,12 +338,43 @@ export const InsertModal: React.FC = ({
{tr('insert.status.searching')}
)} - {!isSearching && query.length < 2 && ( + {!isSearching && query.length < 2 && relatedPosts.length === 0 && !isLoadingRelated && (
{tr('insert.status.typeMore')}
)} + {!isSearching && query.length < 2 && isLoadingRelated && ( +
{tr('insert.status.loadingRelated')}
+ )} + + {!isSearching && query.length < 2 && relatedPosts.length > 0 && ( + <> +
{tr('insert.section.relatedPosts')}
+ {relatedPosts.map((result, index) => ( +
handleSelectResult(result)} + onMouseEnter={() => setSelectedIndex(index)} + > +
+ {result.title} + {similarityMap[result.id] != null && ( + {Math.round(similarityMap[result.id]! * 100)}% + )} +
+ {result.excerpt && ( +
+ {result.excerpt.length > 120 ? result.excerpt.substring(0, 120) + '...' : result.excerpt} +
+ )} +
/posts/{result.slug}
+
+ ))} + + )} + {!isSearching && query.length >= 2 && results.length === 0 && !showCreateOption && (
{tr('insert.status.noResults', { kind: mode === 'link' ? tr('activity.posts').toLowerCase() : tr('activity.media').toLowerCase(), query })} @@ -305,7 +390,12 @@ export const InsertModal: React.FC = ({ > {isPostResult(result) ? ( <> -
{result.title}
+
+ {result.title} + {currentPostId && similarityMap[result.id] != null && ( + {Math.round(similarityMap[result.id]! * 100)}% + )} +
{result.excerpt && (
{result.excerpt.length > 120 diff --git a/src/renderer/components/SettingsView/SettingsView.tsx b/src/renderer/components/SettingsView/SettingsView.tsx index 1476cbc..3a72e2c 100644 --- a/src/renderer/components/SettingsView/SettingsView.tsx +++ b/src/renderer/components/SettingsView/SettingsView.tsx @@ -226,6 +226,7 @@ export const SettingsView: React.FC = () => { const [projectMaxPostsPerPage, setProjectMaxPostsPerPage] = useState(50); const [projectBlogmarkCategory, setProjectBlogmarkCategory] = useState('article'); const [projectPythonRuntimeMode, setProjectPythonRuntimeMode] = useState<'webworker' | 'main-thread'>('webworker'); + const [semanticSimilarityEnabled, setSemanticSimilarityEnabled] = useState(false); // Post categories management const [postCategories, setPostCategories] = useState(DEFAULT_POST_CATEGORIES); @@ -314,6 +315,9 @@ export const SettingsView: React.FC = () => { const incomingPythonRuntimeMode = (metadata as { pythonRuntimeMode?: unknown } | null)?.pythonRuntimeMode; setProjectPythonRuntimeMode(incomingPythonRuntimeMode === 'main-thread' ? 'main-thread' : 'webworker'); + const incomingSemanticSimilarity = (metadata as { semanticSimilarityEnabled?: unknown } | null)?.semanticSimilarityEnabled; + setSemanticSimilarityEnabled(incomingSemanticSimilarity === true); + const incomingCategoryMetadata = (metadata as any)?.categoryMetadata as Record | undefined; const incomingLegacyCategorySettings = (metadata as any)?.categorySettings as Record | undefined; setCategoryMetadata((current) => { @@ -545,6 +549,7 @@ export const SettingsView: React.FC = () => { maxPostsPerPage: Math.min(500, Math.max(1, Math.floor(projectMaxPostsPerPage || 50))), blogmarkCategory: normalizeBlogmarkCategory(projectBlogmarkCategory) || undefined, pythonRuntimeMode: projectPythonRuntimeMode, + semanticSimilarityEnabled, categoryMetadata, }); } @@ -592,7 +597,7 @@ export const SettingsView: React.FC = () => { const editorKeywords = ['editor', 'mode', 'wysiwyg', 'markdown', 'preview', 'visual']; const contentKeywords = ['content', 'categories', 'post', 'article', 'picture', 'aside', 'page']; const aiKeywords = ['ai', 'assistant', 'chat', 'model', 'prompt', 'system', 'api', 'key', 'claude', 'gpt', 'opencode', 'ollama', 'lmstudio', 'lm studio', 'local']; - const technologyKeywords = ['technology', 'python', 'runtime', 'worker', 'webworker', 'main thread', 'execution']; + const technologyKeywords = ['technology', 'python', 'runtime', 'worker', 'webworker', 'main thread', 'execution', 'semantic', 'similarity', 'embedding', 'ai', 'search', 'duplicate']; const publishingKeywords = ['publishing', 'ssh', 'deploy', 'server', 'host', 'upload', 'scp', 'rsync']; const dataKeywords = ['data', 'database', 'rebuild', 'maintenance', 'posts', 'media', 'scripts', 'links', 'folder', 'filesystem']; const mcpKeywords = ['mcp', 'server', 'agent', 'claude', 'copilot', 'gemini', 'opencode', 'model context protocol', 'coding', 'configuration']; @@ -1823,6 +1828,23 @@ export const SettingsView: React.FC = () => { + + + { + const checked = e.target.checked; + setSemanticSimilarityEnabled(checked); + window.electronAPI?.meta.updateProjectMetadata({ semanticSimilarityEnabled: checked }).catch(() => {}); + }} + /> + ); diff --git a/src/renderer/components/TabBar/TabBar.tsx b/src/renderer/components/TabBar/TabBar.tsx index 2c1240a..66a0ac5 100644 --- a/src/renderer/components/TabBar/TabBar.tsx +++ b/src/renderer/components/TabBar/TabBar.tsx @@ -87,6 +87,10 @@ const getTabTitle = ( return tr('siteValidation.tabTitle'); } + if (tab.type === 'find-duplicates') { + return tr('duplicatesView.tabTitle'); + } + if (tab.type === 'scripts') { return scriptTitles.get(tab.id) || tr('tabBar.scripts'); } @@ -180,6 +184,12 @@ const getTabIcon = (tab: Tab): React.ReactNode => { ); + case 'find-duplicates': + return ( + + + + ); case 'scripts': return ( diff --git a/src/renderer/components/TagInput/TagInput.tsx b/src/renderer/components/TagInput/TagInput.tsx index c3dc6a5..62af9e2 100644 --- a/src/renderer/components/TagInput/TagInput.tsx +++ b/src/renderer/components/TagInput/TagInput.tsx @@ -22,6 +22,8 @@ interface TagInputProps { disabled?: boolean; /** Input mode (tags or categories) */ mode?: 'tag' | 'category'; + /** Post ID for AI-based tag suggestions (semantic similarity) */ + postId?: string; } export const TagInput: React.FC = ({ @@ -30,6 +32,7 @@ export const TagInput: React.FC = ({ placeholder = 'Add tags...', disabled = false, mode = 'tag', + postId, }) => { const { t } = useI18n(); const [inputValue, setInputValue] = useState(''); @@ -38,7 +41,8 @@ export const TagInput: React.FC = ({ const [showSuggestions, setShowSuggestions] = useState(false); const [selectedIndex, setSelectedIndex] = useState(-1); const [isCreating, setIsCreating] = useState(false); - + const [aiSuggestedTags, setAiSuggestedTags] = useState([]); + const inputRef = useRef(null); const containerRef = useRef(null); @@ -92,6 +96,26 @@ export const TagInput: React.FC = ({ setSelectedIndex(-1); }, [inputValue, allTags, value]); + // Load AI tag suggestions when focused on an empty input (mode=tag only) + useEffect(() => { + if (mode !== 'tag' || !postId || inputValue.trim()) { + setAiSuggestedTags([]); + return; + } + let cancelled = false; + (async () => { + try { + const suggestions = await window.electronAPI.embeddings.suggestTags(postId, value); + if (!cancelled) { + setAiSuggestedTags(suggestions.map(s => s.name)); + } + } catch { + if (!cancelled) setAiSuggestedTags([]); + } + })(); + return () => { cancelled = true; }; + }, [postId, mode, value, inputValue]); + // Close suggestions when clicking outside useEffect(() => { const handleClickOutside = (event: MouseEvent) => { @@ -273,8 +297,25 @@ export const TagInput: React.FC = ({
{/* Suggestions dropdown */} - {showSuggestions && (suggestions.length > 0 || showCreateOption) && ( + {showSuggestions && (suggestions.length > 0 || showCreateOption || aiSuggestedTags.length > 0) && (
+ {/* AI-suggested tags shown when input is empty */} + {!inputValue.trim() && aiSuggestedTags.length > 0 && ( + <> +
{t('tagInput.aiSuggestedLabel')}
+ {aiSuggestedTags.map((tagName) => ( + + ))} + {suggestions.length > 0 &&
{t('tagInput.allTagsLabel')}
} + + )} {suggestions.map((tag, index) => { const hasColor = !!tag.color; const style: React.CSSProperties = hasColor diff --git a/src/renderer/i18n/locales/de.json b/src/renderer/i18n/locales/de.json index fe58b6d..aa96968 100644 --- a/src/renderer/i18n/locales/de.json +++ b/src/renderer/i18n/locales/de.json @@ -139,6 +139,8 @@ "settings.technology.pythonRuntimeModeDescription": "Lege fest, wo Python-Skripte für Transformationspipelines ausgeführt werden.", "settings.technology.pythonRuntimeMode.webworker": "Web Worker (empfohlen)", "settings.technology.pythonRuntimeMode.mainThread": "Hauptthread (Legacy)", + "settings.technology.semanticSimilarityLabel": "Semantische Ähnlichkeit", + "settings.technology.semanticSimilarityDescription": "Aktiviert lokale KI-Einbettungen für Vorschläge zu verwandten Beiträgen, Tag-Hinweise und Erkennung von Duplikaten. Lädt beim ersten Start ein ~100 MB großes Modell herunter.", "settings.publishing.sshTitle": "SSH-Veröffentlichung", "settings.data.title": "Datenbankwartung", "settings.data.fileSystemTitle": "Dateisystem", @@ -237,6 +239,8 @@ "insert.searchPlaceholder.image": "Medien nach Name, Titel oder Alt-Text durchsuchen...", "insert.status.searching": "Suche...", "insert.status.typeMore": "Zum Suchen mindestens 2 Zeichen eingeben", + "insert.status.loadingRelated": "Ähnliche Beiträge werden geladen...", + "insert.section.relatedPosts": "Verwandte Beiträge", "insert.status.noResults": "Keine {kind} für \"{query}\" gefunden", "insert.label.url": "Webadresse", "insert.label.linkTextOptional": "Linktext (optional)", @@ -977,6 +981,8 @@ "assistantSidebar.conversationTitle": "Assistent-Sitzung", "assistantSidebar.error.startFailed": "Assistent-Sitzung konnte nicht gestartet werden", "assistantSidebar.error.actionFailed": "Assistent-Aktion konnte nicht ausgeführt werden", + "tagInput.aiSuggestedLabel": "KI-Vorschläge", + "tagInput.allTagsLabel": "Alle Tags", "tagInput.alreadyAdded": "Tag bereits hinzugefügt", "tagInput.remove": "{tag} entfernen", "tagInput.createdTag": "Tag \"{name}\" erstellt", @@ -1092,5 +1098,22 @@ "settings.toast.mcpConfigRemoveSuccess": "bDS MCP-Server aus der {agent}-Konfiguration entfernt", "settings.toast.mcpConfigFailed": "Konfiguration von {agent} fehlgeschlagen: {error}", "settings.toast.mcpConfigRemoveFailed": "Entfernen aus {agent} fehlgeschlagen: {error}", - "settings.toast.mcpConfigPath": "Konfiguration geschrieben nach {path}" + "settings.toast.mcpConfigPath": "Konfiguration geschrieben nach {path}", + "duplicatesView.tabTitle": "Duplikate finden", + "duplicatesView.title": "Doppelte Beiträge", + "duplicatesView.description": "Beiträge mit hoher inhaltlicher Ähnlichkeit, die möglicherweise Duplikate sind.", + "duplicatesView.loading": "Suche nach Duplikaten...", + "duplicatesView.empty": "Keine doppelten Beiträge gefunden.", + "duplicatesView.error": "Duplikate konnten nicht geladen werden", + "duplicatesView.refresh": "Aktualisieren", + "duplicatesView.dismiss": "Ignorieren", + "duplicatesView.similarity": "{value}% ähnlich", + "duplicatesView.exactMatch": "Exaktes Duplikat", + "duplicatesView.openPost": "Beitrag öffnen", + "duplicatesView.count": "{count} Paare gefunden", + "duplicatesView.showMore": "Mehr anzeigen", + "duplicatesView.checkAll": "Alle auswählen", + "duplicatesView.uncheckAll": "Alle abwählen", + "duplicatesView.dismissChecked": "Ausgewählte ignorieren ({count})", + "duplicatesView.notEnabled": "Semantische Ähnlichkeit ist nicht aktiviert. Aktivieren Sie sie unter Einstellungen → Technologie." } diff --git a/src/renderer/i18n/locales/en.json b/src/renderer/i18n/locales/en.json index d43d7c8..66e1489 100644 --- a/src/renderer/i18n/locales/en.json +++ b/src/renderer/i18n/locales/en.json @@ -139,6 +139,8 @@ "settings.technology.pythonRuntimeModeDescription": "Choose where Python scripts execute for transform pipelines.", "settings.technology.pythonRuntimeMode.webworker": "Web Worker (Recommended)", "settings.technology.pythonRuntimeMode.mainThread": "Main Thread (Legacy)", + "settings.technology.semanticSimilarityLabel": "Semantic Similarity", + "settings.technology.semanticSimilarityDescription": "Enable local AI embeddings for related-post suggestions, tag hints, and duplicate detection. Downloads a ~100 MB model on first use.", "settings.publishing.sshTitle": "SSH Publishing", "settings.data.title": "Database Maintenance", "settings.data.fileSystemTitle": "File System", @@ -237,6 +239,8 @@ "insert.searchPlaceholder.image": "Search media by name, title, or alt text...", "insert.status.searching": "Searching...", "insert.status.typeMore": "Type at least 2 characters to search", + "insert.status.loadingRelated": "Loading related posts...", + "insert.section.relatedPosts": "Related Posts", "insert.status.noResults": "No {kind} found for \"{query}\"", "insert.label.url": "URL", "insert.label.linkTextOptional": "Link Text (optional)", @@ -977,6 +981,8 @@ "assistantSidebar.conversationTitle": "Assistant Session", "assistantSidebar.error.startFailed": "Failed to start assistant session", "assistantSidebar.error.actionFailed": "Assistant action could not be executed", + "tagInput.aiSuggestedLabel": "AI Suggestions", + "tagInput.allTagsLabel": "All Tags", "tagInput.alreadyAdded": "Tag already added", "tagInput.remove": "Remove {tag}", "tagInput.createdTag": "Tag \"{name}\" created", @@ -1092,5 +1098,22 @@ "settings.toast.mcpConfigRemoveSuccess": "bDS MCP server removed from {agent} configuration", "settings.toast.mcpConfigFailed": "Failed to configure {agent}: {error}", "settings.toast.mcpConfigRemoveFailed": "Failed to remove from {agent}: {error}", - "settings.toast.mcpConfigPath": "Config written to {path}" + "settings.toast.mcpConfigPath": "Config written to {path}", + "duplicatesView.tabTitle": "Find Duplicates", + "duplicatesView.title": "Duplicate Posts", + "duplicatesView.description": "Posts with high content similarity that may be duplicates.", + "duplicatesView.loading": "Searching for duplicates...", + "duplicatesView.empty": "No duplicate posts found.", + "duplicatesView.error": "Failed to load duplicates", + "duplicatesView.refresh": "Refresh", + "duplicatesView.dismiss": "Dismiss", + "duplicatesView.similarity": "{value}% similar", + "duplicatesView.exactMatch": "Exact duplicate", + "duplicatesView.openPost": "Open post", + "duplicatesView.count": "{count} pairs found", + "duplicatesView.showMore": "Show more", + "duplicatesView.checkAll": "Check All", + "duplicatesView.uncheckAll": "Uncheck All", + "duplicatesView.dismissChecked": "Dismiss Checked ({count})", + "duplicatesView.notEnabled": "Semantic similarity is not enabled. Enable it in Settings → Technology." } diff --git a/src/renderer/i18n/locales/es.json b/src/renderer/i18n/locales/es.json index d3c9f14..3adb952 100644 --- a/src/renderer/i18n/locales/es.json +++ b/src/renderer/i18n/locales/es.json @@ -139,6 +139,8 @@ "settings.technology.pythonRuntimeModeDescription": "Elige dónde se ejecutan los scripts de Python para los flujos de transformación.", "settings.technology.pythonRuntimeMode.webworker": "Web Worker (recomendado)", "settings.technology.pythonRuntimeMode.mainThread": "Hilo principal (heredado)", + "settings.technology.semanticSimilarityLabel": "Similitud semántica", + "settings.technology.semanticSimilarityDescription": "Activa incrustaciones de IA locales para sugerencias de publicaciones relacionadas, sugerencias de etiquetas y detección de duplicados. Descarga un modelo de ~100 MB en el primer uso.", "settings.publishing.sshTitle": "Publicación SSH", "settings.data.title": "Mantenimiento de base de datos", "settings.data.fileSystemTitle": "Sistema de archivos", @@ -237,6 +239,8 @@ "insert.searchPlaceholder.image": "Buscar medios por nombre, título o texto alternativo...", "insert.status.searching": "Buscando...", "insert.status.typeMore": "Escribe al menos 2 caracteres para buscar", + "insert.status.loadingRelated": "Cargando publicaciones relacionadas...", + "insert.section.relatedPosts": "Publicaciones relacionadas", "insert.status.noResults": "No se encontró {kind} para \"{query}\"", "insert.label.url": "Dirección URL", "insert.label.linkTextOptional": "Texto del enlace (opcional)", @@ -977,6 +981,8 @@ "assistantSidebar.conversationTitle": "Sesión de asistente", "assistantSidebar.error.startFailed": "No se pudo iniciar la sesión del asistente", "assistantSidebar.error.actionFailed": "No se pudo ejecutar la acción del asistente", + "tagInput.aiSuggestedLabel": "Sugerencias IA", + "tagInput.allTagsLabel": "Todos los tags", "tagInput.alreadyAdded": "La etiqueta “{tag}” ya está añadida", "tagInput.remove": "Quitar", "tagInput.createdTag": "Etiqueta “{tag}” creada", @@ -1092,5 +1098,22 @@ "settings.toast.mcpConfigRemoveSuccess": "Servidor MCP de bDS eliminado de la configuración de {agent}", "settings.toast.mcpConfigFailed": "Error al configurar {agent}: {error}", "settings.toast.mcpConfigRemoveFailed": "Error al eliminar de {agent}: {error}", - "settings.toast.mcpConfigPath": "Configuración escrita en {path}" + "settings.toast.mcpConfigPath": "Configuración escrita en {path}", + "duplicatesView.tabTitle": "Buscar duplicados", + "duplicatesView.title": "Entradas duplicadas", + "duplicatesView.description": "Entradas con alta similitud de contenido que pueden ser duplicadas.", + "duplicatesView.loading": "Buscando duplicados...", + "duplicatesView.empty": "No se encontraron entradas duplicadas.", + "duplicatesView.error": "Error al cargar duplicados", + "duplicatesView.refresh": "Actualizar", + "duplicatesView.dismiss": "Descartar", + "duplicatesView.similarity": "{value}% similar", + "duplicatesView.exactMatch": "Duplicado exacto", + "duplicatesView.openPost": "Abrir entrada", + "duplicatesView.count": "{count} pares encontrados", + "duplicatesView.showMore": "Mostrar más", + "duplicatesView.checkAll": "Seleccionar todo", + "duplicatesView.uncheckAll": "Deseleccionar todo", + "duplicatesView.dismissChecked": "Descartar seleccionados ({count})", + "duplicatesView.notEnabled": "La similitud semántica no está activada. Actívela en Configuración → Tecnología." } diff --git a/src/renderer/i18n/locales/fr.json b/src/renderer/i18n/locales/fr.json index 5648fb0..3cbded1 100644 --- a/src/renderer/i18n/locales/fr.json +++ b/src/renderer/i18n/locales/fr.json @@ -139,6 +139,8 @@ "settings.technology.pythonRuntimeModeDescription": "Choisissez où les scripts Python s’exécutent pour les pipelines de transformation.", "settings.technology.pythonRuntimeMode.webworker": "Web Worker (recommandé)", "settings.technology.pythonRuntimeMode.mainThread": "Thread principal (hérité)", + "settings.technology.semanticSimilarityLabel": "Similarité sémantique", + "settings.technology.semanticSimilarityDescription": "Active les embeddings IA locaux pour les suggestions de publications similaires, les suggestions de tags et la détection de doublons. Télécharge un modèle d'environ 100 Mo lors du premier usage.", "settings.publishing.sshTitle": "Publication SSH", "settings.data.title": "Maintenance de la base de données", "settings.data.fileSystemTitle": "Système de fichiers", @@ -237,6 +239,8 @@ "insert.searchPlaceholder.image": "Rechercher des médias par nom, titre ou texte alternatif...", "insert.status.searching": "Recherche...", "insert.status.typeMore": "Saisissez au moins 2 caractères pour rechercher", + "insert.status.loadingRelated": "Chargement des publications connexes...", + "insert.section.relatedPosts": "Publications connexes", "insert.status.noResults": "Aucun(e) {kind} trouvé(e) pour \"{query}\"", "insert.label.url": "Adresse URL", "insert.label.linkTextOptional": "Texte du lien (optionnel)", @@ -977,6 +981,8 @@ "assistantSidebar.conversationTitle": "Session Assistant", "assistantSidebar.error.startFailed": "Impossible de démarrer la session assistant", "assistantSidebar.error.actionFailed": "L’action assistant n’a pas pu être exécutée", + "tagInput.aiSuggestedLabel": "Suggestions IA", + "tagInput.allTagsLabel": "Tous les tags", "tagInput.alreadyAdded": "Le tag « {tag} » est déjà ajouté", "tagInput.remove": "Supprimer", "tagInput.createdTag": "Tag « {tag} » créé", @@ -1090,5 +1096,22 @@ "settings.toast.mcpConfigRemoveSuccess": "Serveur MCP bDS retiré de la configuration de {agent}", "settings.toast.mcpConfigFailed": "Échec de la configuration de {agent}: {error}", "settings.toast.mcpConfigRemoveFailed": "Échec du retrait de {agent}: {error}", - "settings.toast.mcpConfigPath": "Configuration écrite dans {path}" + "settings.toast.mcpConfigPath": "Configuration écrite dans {path}", + "duplicatesView.tabTitle": "Trouver les doublons", + "duplicatesView.title": "Articles en double", + "duplicatesView.description": "Articles avec une grande similarité de contenu pouvant être des doublons.", + "duplicatesView.loading": "Recherche des doublons...", + "duplicatesView.empty": "Aucun article en double trouvé.", + "duplicatesView.error": "Impossible de charger les doublons", + "duplicatesView.refresh": "Actualiser", + "duplicatesView.dismiss": "Ignorer", + "duplicatesView.similarity": "{value}% similaire", + "duplicatesView.exactMatch": "Doublon exact", + "duplicatesView.openPost": "Ouvrir l'article", + "duplicatesView.count": "{count} paires trouvées", + "duplicatesView.showMore": "Afficher plus", + "duplicatesView.checkAll": "Tout cocher", + "duplicatesView.uncheckAll": "Tout décocher", + "duplicatesView.dismissChecked": "Ignorer cochés ({count})", + "duplicatesView.notEnabled": "La similarité sémantique n'est pas activée. Activez-la dans Paramètres → Technologie." } diff --git a/src/renderer/i18n/locales/it.json b/src/renderer/i18n/locales/it.json index 7a5d0ed..815688f 100644 --- a/src/renderer/i18n/locales/it.json +++ b/src/renderer/i18n/locales/it.json @@ -139,6 +139,8 @@ "settings.technology.pythonRuntimeModeDescription": "Scegli dove eseguire gli script Python per le pipeline di trasformazione.", "settings.technology.pythonRuntimeMode.webworker": "Web Worker (consigliato)", "settings.technology.pythonRuntimeMode.mainThread": "Thread principale (legacy)", + "settings.technology.semanticSimilarityLabel": "Similarità semantica", + "settings.technology.semanticSimilarityDescription": "Abilita gli embedding AI locali per suggerimenti di post correlati, suggerimenti di tag e rilevamento di duplicati. Scarica un modello di circa 100 MB al primo utilizzo.", "settings.publishing.sshTitle": "Pubblicazione SSH", "settings.data.title": "Manutenzione database", "settings.data.fileSystemTitle": "Sistema file", @@ -237,6 +239,8 @@ "insert.searchPlaceholder.image": "Cerca media per nome, titolo o testo alternativo...", "insert.status.searching": "Ricerca...", "insert.status.typeMore": "Digita almeno 2 caratteri per cercare", + "insert.status.loadingRelated": "Caricamento post correlati...", + "insert.section.relatedPosts": "Post correlati", "insert.status.noResults": "Nessun {kind} trovato per \"{query}\"", "insert.label.url": "Indirizzo URL", "insert.label.linkTextOptional": "Testo link (opzionale)", @@ -977,6 +981,8 @@ "assistantSidebar.conversationTitle": "Sessione assistente", "assistantSidebar.error.startFailed": "Impossibile avviare la sessione assistente", "assistantSidebar.error.actionFailed": "Impossibile eseguire l’azione dell’assistente", + "tagInput.aiSuggestedLabel": "Suggerimenti IA", + "tagInput.allTagsLabel": "Tutti i tag", "tagInput.alreadyAdded": "Il tag “{tag}” è già stato aggiunto", "tagInput.remove": "Rimuovi", "tagInput.createdTag": "Tag “{tag}” creato", @@ -1090,5 +1096,22 @@ "settings.toast.mcpConfigRemoveSuccess": "Server MCP bDS rimosso dalla configurazione di {agent}", "settings.toast.mcpConfigFailed": "Configurazione di {agent} non riuscita: {error}", "settings.toast.mcpConfigRemoveFailed": "Rimozione da {agent} non riuscita: {error}", - "settings.toast.mcpConfigPath": "Configurazione scritta in {path}" + "settings.toast.mcpConfigPath": "Configurazione scritta in {path}", + "duplicatesView.tabTitle": "Trova duplicati", + "duplicatesView.title": "Post duplicati", + "duplicatesView.description": "Post con elevata similitudine di contenuto che potrebbero essere duplicati.", + "duplicatesView.loading": "Ricerca duplicati...", + "duplicatesView.empty": "Nessun post duplicato trovato.", + "duplicatesView.error": "Impossibile caricare i duplicati", + "duplicatesView.refresh": "Aggiorna", + "duplicatesView.dismiss": "Ignora", + "duplicatesView.similarity": "{value}% simile", + "duplicatesView.exactMatch": "Duplicato esatto", + "duplicatesView.openPost": "Apri post", + "duplicatesView.count": "{count} coppie trovate", + "duplicatesView.showMore": "Mostra altri", + "duplicatesView.checkAll": "Seleziona tutto", + "duplicatesView.uncheckAll": "Deseleziona tutto", + "duplicatesView.dismissChecked": "Ignora selezionati ({count})", + "duplicatesView.notEnabled": "La similarità semantica non è abilitata. Abilitala in Impostazioni → Tecnologia." } diff --git a/src/renderer/navigation/duplicatesPersistence.ts b/src/renderer/navigation/duplicatesPersistence.ts new file mode 100644 index 0000000..6e362d1 --- /dev/null +++ b/src/renderer/navigation/duplicatesPersistence.ts @@ -0,0 +1,30 @@ +import type { DuplicatePair } from '../../main/shared/electronApi'; + +const store = new Map(); + +export function persistDuplicatesResult(projectId: string, pairs: DuplicatePair[]): void { + store.set(projectId, pairs); +} + +export function getPersistedDuplicatesResult(projectId: string): DuplicatePair[] | null { + return store.get(projectId) ?? null; +} + +export function removeDismissedPair(projectId: string, postIdA: string, postIdB: string): void { + const pairs = store.get(projectId); + if (!pairs) return; + store.set( + projectId, + pairs.filter(p => !(p.postA.id === postIdA && p.postB.id === postIdB)), + ); +} + +export function removeDismissedPairs(projectId: string, pairIds: Array<[string, string]>): void { + const pairs = store.get(projectId); + if (!pairs) return; + const keySet = new Set(pairIds.map(([a, b]) => `${a}::${b}`)); + store.set( + projectId, + pairs.filter(p => !keySet.has(`${p.postA.id}::${p.postB.id}`)), + ); +} diff --git a/src/renderer/navigation/editorRouting.ts b/src/renderer/navigation/editorRouting.ts index 4797396..f102a3f 100644 --- a/src/renderer/navigation/editorRouting.ts +++ b/src/renderer/navigation/editorRouting.ts @@ -17,7 +17,8 @@ export type EditorRoute = | 'api-documentation' | 'site-validation' | 'scripts' - | 'templates'; + | 'templates' + | 'find-duplicates'; export const EDITOR_TAB_ROUTE_REGISTRY: Record> = { post: 'post', @@ -35,6 +36,7 @@ export const EDITOR_TAB_ROUTE_REGISTRY: Record 'api-documentation': { type: 'api-documentation', id: 'api-documentation', isTransient: false }, 'metadata-diff': { type: 'metadata-diff', id: 'metadata-diff', isTransient: false }, 'site-validation': { type: 'site-validation', id: 'site-validation', isTransient: false }, + 'find-duplicates': { type: 'find-duplicates', id: 'find-duplicates', isTransient: false }, }; export function getSingletonToolTabSpec(key: SingletonToolTabKey): CanonicalTabSpec { diff --git a/src/renderer/store/appStore.ts b/src/renderer/store/appStore.ts index e970298..4f32756 100644 --- a/src/renderer/store/appStore.ts +++ b/src/renderer/store/appStore.ts @@ -13,7 +13,7 @@ import type { const STORAGE_KEY = 'bds-app-state'; // Tab types -export type TabType = 'post' | 'media' | 'settings' | 'style' | 'tags' | 'chat' | 'import' | 'menu-editor' | 'metadata-diff' | 'git-diff' | 'documentation' | 'api-documentation' | 'site-validation' | 'scripts' | 'templates'; +export type TabType = 'post' | 'media' | 'settings' | 'style' | 'tags' | 'chat' | 'import' | 'menu-editor' | 'metadata-diff' | 'git-diff' | 'documentation' | 'api-documentation' | 'site-validation' | 'scripts' | 'templates' | 'find-duplicates'; export interface Tab { type: TabType; diff --git a/tests/engine/EmbeddingEngine.test.ts b/tests/engine/EmbeddingEngine.test.ts new file mode 100644 index 0000000..6d559f5 --- /dev/null +++ b/tests/engine/EmbeddingEngine.test.ts @@ -0,0 +1,404 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import * as path from 'path'; +import * as os from 'os'; +import * as fs from 'fs/promises'; +import { EmbeddingEngine, type EmbeddingPipeline } from '../../src/main/engine/EmbeddingEngine'; + +// ── In-memory DB store ───────────────────────────────────────────────────── + +interface KeyRow { + label: bigint; + postId: string; + projectId: string; + contentHash: string; +} + +interface DismissedRow { + id: string; + projectId: string; + postIdA: string; + postIdB: string; + dismissedAt: Date; +} + +interface PostRow { + id: string; + title: string; + slug?: string; + content: string | null; + tags?: string; + publishedAt?: Date | null; +} + +let keyRowsStore: KeyRow[] = []; +let dismissedRowsStore: DismissedRow[] = []; +let postRowsStore: PostRow[] = []; + +// Drizzle stores the SQL table name at this symbol +const DRIZZLE_NAME = Symbol.for('drizzle:Name'); +const DRIZZLE_BASE_NAME = Symbol.for('drizzle:BaseName'); + +function getTableName(table: unknown): string { + if (table && typeof table === 'object') { + const t = table as Record; + return (t[DRIZZLE_NAME] as string) || (t[DRIZZLE_BASE_NAME] as string) || ''; + } + return ''; +} + +const mockDb = { + selectFn: vi.fn(), + insertFn: vi.fn(), + deleteFn: vi.fn(), + + select() { + // Returns a drizzle-like query chain + let tableName = ''; + + const chain: Record = { + from: vi.fn((table: unknown) => { + tableName = getTableName(table); + return chain; + }), + where: vi.fn((_cond: unknown) => { + // Return the appropriate store based on table + let rows: unknown[] = []; + if (tableName === 'embedding_keys') { + rows = keyRowsStore; + } else if (tableName === 'dismissed_duplicate_pairs') { + rows = dismissedRowsStore; + } else if (tableName === 'posts') { + rows = postRowsStore; + } + return Promise.resolve(rows); + }), + }; + return chain; + }, + + insert(_table: unknown) { + const tableName = getTableName(_table); + return { + values: vi.fn((row: unknown) => { + if (tableName === 'embedding_keys') { + keyRowsStore.push(row as KeyRow); + } else if (tableName === 'dismissed_duplicate_pairs') { + dismissedRowsStore.push(row as DismissedRow); + } + return { onConflictDoNothing: vi.fn().mockResolvedValue([]) }; + }), + }; + }, + + delete(_table: unknown) { + return { + where: vi.fn((_cond: unknown) => { + return Promise.resolve([]); + }), + }; + }, +}; + +vi.mock('../../src/main/database', () => ({ + getDatabase: () => ({ + getLocal: () => mockDb, + }), +})); + +// ── Deterministic mock pipeline ──────────────────────────────────────────── + +let embedCallCount = 0; + +function makeEmbedFn() { + return vi.fn().mockImplementation(async (text: string): Promise => { + embedCallCount++; + const arr = new Float32Array(384).fill(0); + // Produce unique vector per text + let hash = 5381; + for (let i = 0; i < text.length; i++) { + hash = ((hash << 5) + hash + text.charCodeAt(i)) | 0; + } + arr[Math.abs(hash) % 384] = 1; + arr[(Math.abs(hash * 31) % 383 + 1) % 384] = 0.7; + // Normalize + const norm = Math.sqrt(arr.reduce((s, v) => s + v * v, 0)); + for (let i = 0; i < arr.length; i++) { + arr[i] = arr[i]! / norm; + } + return arr; + }); +} + +function createMockPipeline(): EmbeddingPipeline { + return { embed: makeEmbedFn() }; +} + +// ── Helpers ──────────────────────────────────────────────────────────────── + +function makeEngine(tmpDir: string): EmbeddingEngine { + return new EmbeddingEngine({ + getIndexPath: (projectId: string) => path.join(tmpDir, `${projectId}.usearch`), + createPipeline: async () => createMockPipeline(), + }); +} + +// Manually replicate embedPost logic in tests (insert key row, update in-memory state) +// so we can set up test scenarios without relying on DB mock filtering +async function addKeyRow(row: KeyRow): Promise { + keyRowsStore.push(row); +} + +// ── Tests ────────────────────────────────────────────────────────────────── + +describe('EmbeddingEngine', () => { + let tmpDir: string; + let engine: EmbeddingEngine; + + beforeEach(async () => { + keyRowsStore = []; + dismissedRowsStore = []; + postRowsStore = []; + embedCallCount = 0; + vi.clearAllMocks(); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'embedding-test-')); + engine = makeEngine(tmpDir); + await engine.setProjectContext('proj1'); + }); + + afterEach(async () => { + await engine.shutdown(); + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + describe('embedPost', () => { + it('adds vector to index and persists key row', async () => { + await engine.embedPost('post-1', 'Hello World', 'This is my first post'); + + expect(keyRowsStore.length).toBe(1); + expect(keyRowsStore[0]!.postId).toBe('post-1'); + expect(keyRowsStore[0]!.projectId).toBe('proj1'); + expect(keyRowsStore[0]!.contentHash).toMatch(/^[a-f0-9]{64}$/); + }); + + it('skips re-embedding when content hash unchanged', async () => { + await engine.embedPost('post-1', 'Hello', 'Content'); + const countBefore = embedCallCount; + + // Embed same content again — engine uses in-memory hash check after first embed + await engine.embedPost('post-1', 'Hello', 'Content'); + + // Should not have called embed again (no re-embed on unchanged content) + expect(embedCallCount).toBe(countBefore); + }); + + it('does not skip re-embedding when content changes', async () => { + await engine.embedPost('post-1', 'Hello', 'Original'); + const countAfterFirst = embedCallCount; + + // Update content (simulating second call with different content; engine detects hash change) + // We need to trick the engine by clearing the internal keyRowsStore entry so the + // DB mock returns empty for the second lookup + keyRowsStore = []; + + await engine.embedPost('post-1', 'Hello', 'Updated content'); + + expect(embedCallCount).toBeGreaterThan(countAfterFirst); + }); + }); + + describe('removePost', () => { + it('removes post from index and key map', async () => { + await engine.embedPost('post-1', 'Hello', 'Content'); + expect(keyRowsStore.length).toBe(1); + + await engine.removePost('post-1'); + + // Key map should not have post-1 anymore + // (The delete mock doesn't clear keyRowsStore, but the in-memory map should be cleared) + const results = await engine.findSimilar('post-1'); + expect(results).toEqual([]); + }); + + it('is a no-op for non-existent post', async () => { + await engine.removePost('non-existent'); // should not throw + }); + }); + + describe('findSimilar', () => { + it('returns empty array for non-indexed post', async () => { + const results = await engine.findSimilar('not-indexed'); + expect(results).toEqual([]); + }); + + it('returns empty when only one post indexed', async () => { + await engine.embedPost('post-1', 'Only post', 'Content'); + const results = await engine.findSimilar('post-1'); + expect(results).toEqual([]); + }); + + it('returns similar posts ranked by similarity', async () => { + await engine.embedPost('post-1', 'Machine learning basics', 'Intro to ML and neural nets'); + await engine.embedPost('post-2', 'Deep learning tutorial', 'Advanced ML techniques'); + await engine.embedPost('post-3', 'Cooking recipes', 'How to make pasta'); + + const results = await engine.findSimilar('post-1', 5); + + expect(Array.isArray(results)).toBe(true); + expect(results.every((r) => r.postId !== 'post-1')).toBe(true); + expect(results.every((r) => r.similarity >= 0 && r.similarity <= 1)).toBe(true); + // Results should be sorted by similarity descending + for (let i = 1; i < results.length; i++) { + expect(results[i]!.similarity).toBeLessThanOrEqual(results[i - 1]!.similarity); + } + }); + }); + + describe('computeSimilarities', () => { + it('returns empty object for non-indexed source post', async () => { + const result = await engine.computeSimilarities('not-indexed', ['post-1']); + expect(result).toEqual({}); + }); + + it('returns empty object for empty target list', async () => { + await engine.embedPost('post-1', 'Title', 'Content'); + const result = await engine.computeSimilarities('post-1', []); + expect(result).toEqual({}); + }); + + it('returns similarity scores for indexed target posts', async () => { + await engine.embedPost('post-1', 'Machine learning basics', 'Intro to ML'); + await engine.embedPost('post-2', 'Deep learning tutorial', 'Advanced ML'); + await engine.embedPost('post-3', 'Cooking recipes', 'How to make pasta'); + + const result = await engine.computeSimilarities('post-1', ['post-2', 'post-3']); + + expect(Object.keys(result)).toHaveLength(2); + expect(result['post-2']).toBeGreaterThanOrEqual(0); + expect(result['post-2']).toBeLessThanOrEqual(1); + expect(result['post-3']).toBeGreaterThanOrEqual(0); + expect(result['post-3']).toBeLessThanOrEqual(1); + }); + + it('omits targets without embeddings', async () => { + await engine.embedPost('post-1', 'Title', 'Content'); + + const result = await engine.computeSimilarities('post-1', ['not-indexed']); + expect(result).toEqual({}); + }); + + it('excludes self from results', async () => { + await engine.embedPost('post-1', 'Title', 'Content'); + + const result = await engine.computeSimilarities('post-1', ['post-1']); + expect(result).toEqual({}); + }); + }); + + describe('getIndexingProgress', () => { + it('returns zero indexed and total when no posts', async () => { + postRowsStore = []; + const progress = await engine.getIndexingProgress(); + expect(progress.indexed).toBe(0); + expect(progress.total).toBe(0); + }); + + it('returns indexed from key map and total from posts table', async () => { + await engine.embedPost('post-1', 'Title 1', 'Content 1'); + + // Set up posts DB to return 3 posts (only 1 indexed) + postRowsStore = [ + { id: 'post-1', title: 'T1', content: 'C1' }, + { id: 'post-2', title: 'T2', content: 'C2' }, + { id: 'post-3', title: 'T3', content: 'C3' }, + ]; + + const progress = await engine.getIndexingProgress(); + expect(progress.indexed).toBe(1); // only post-1 in key map + expect(progress.total).toBe(3); // 3 posts in DB + }); + }); + + describe('setProjectContext', () => { + it('clears key map when switching projects', async () => { + await engine.embedPost('post-1', 'Title', 'Content'); + + // Switch to new project — should clear key map + keyRowsStore = []; // No keys for proj2 + await engine.setProjectContext('proj2'); + + // post-1 is no longer in the key map for proj2 + const results = await engine.findSimilar('post-1'); + expect(results).toEqual([]); + }); + + it('is a no-op when called with same project', async () => { + await engine.embedPost('post-1', 'Title', 'Content'); + + await engine.setProjectContext('proj1'); // same project + + // Key map should still have post-1 + expect(engine['postIdToLabel'].has('post-1')).toBe(true); + }); + }); + + describe('save and load', () => { + it('persists USearch index file to disk', async () => { + await engine.embedPost('post-1', 'Hello', 'World'); + await engine.save(); + + const indexPath = path.join(tmpDir, 'proj1.usearch'); + const stat = await fs.stat(indexPath); + expect(stat.isFile()).toBe(true); + expect(stat.size).toBeGreaterThan(0); + }); + + it('loads persisted index after restart', async () => { + await engine.embedPost('post-1', 'Hello', 'World'); + await engine.embedPost('post-2', 'Goodbye', 'World two'); + await engine.save(); + const savedKeyRows = [...keyRowsStore]; + + // Create new engine instance simulating restart + const engine2 = makeEngine(tmpDir); + keyRowsStore = savedKeyRows; // Restore DB state + await engine2.setProjectContext('proj1'); + + // Should have loaded the key map + expect(engine2['postIdToLabel'].has('post-1')).toBe(true); + expect(engine2['postIdToLabel'].has('post-2')).toBe(true); + + await engine2.shutdown(); + }); + }); + + describe('dismissPair', () => { + it('inserts dismissed pair with canonical ordering', async () => { + await engine.dismissPair('zzz-post', 'aaa-post'); + + expect(dismissedRowsStore.length).toBe(1); + const row = dismissedRowsStore[0]!; + // Should be stored with canonical (alphabetical) ordering + expect(row.postIdA).toBe('aaa-post'); + expect(row.postIdB).toBe('zzz-post'); + expect(row.projectId).toBe('proj1'); + }); + + it('stores pair in both orderings consistently', async () => { + await engine.dismissPair('post-b', 'post-a'); + const row = dismissedRowsStore[0]!; + expect(row.postIdA).toBe('post-a'); // canonical order + expect(row.postIdB).toBe('post-b'); + }); + }); + + describe('content hash change detection', () => { + it('detects unchanged content and skips re-embedding', async () => { + await engine.embedPost('post-1', 'Title', 'Content'); + const embedsAfterFirst = embedCallCount; + + // Second call with same content — in-memory cache should prevent re-embed + await engine.embedPost('post-1', 'Title', 'Content'); + expect(embedCallCount).toBe(embedsAfterFirst); + }); + }); +}); diff --git a/tests/engine/MCPServer.test.ts b/tests/engine/MCPServer.test.ts index f7fbcd3..2c419c7 100644 --- a/tests/engine/MCPServer.test.ts +++ b/tests/engine/MCPServer.test.ts @@ -7,6 +7,7 @@ function createMockPostEngine() { return { getAllPosts: vi.fn().mockResolvedValue({ items: [], hasMore: false, total: 0 }), getPost: vi.fn().mockResolvedValue(null), + getPostBySlug: vi.fn().mockResolvedValue(null), searchPosts: vi.fn().mockResolvedValue([]), searchPostsFiltered: vi.fn().mockResolvedValue({ posts: [], total: 0 }), createPost: vi.fn().mockResolvedValue({ @@ -205,6 +206,11 @@ describe('MCPServer', () => { const mcpServer = server.createMcpServer(); expect(hasRegistered(mcpServer, '_registeredTools', 'discard_proposal')).toBe(true); }); + + it('registers read_post_by_slug tool', () => { + const mcpServer = server.createMcpServer(); + expect(hasRegistered(mcpServer, '_registeredTools', 'read_post_by_slug')).toBe(true); + }); }); describe('registered resources', () => { @@ -1165,6 +1171,33 @@ describe('MCPServer', () => { const result = await tool.handler({ category: 'tech' }, {}) as { content: Array<{ text: string }> }; expect(result.content).toHaveLength(1); }); + + // ── read_post_by_slug tool ─────────────────────────────────────── + + it('read_post_by_slug returns post with backlinks when found', async () => { + const post = { id: 'p1', title: 'Found', slug: 'found-post', content: 'body', status: 'published', tags: [], categories: [], createdAt: new Date(), updatedAt: new Date() }; + mockPostEngine.getPostBySlug.mockResolvedValue(post); + mockPostEngine.getLinkedBy.mockResolvedValue([{ id: 'px', title: 'Ref', slug: 'ref' }]); + mockPostEngine.getLinksTo.mockResolvedValue([]); + const mcpServer = server.createMcpServer(); + const tool = getTool(mcpServer, 'read_post_by_slug'); + const result = await tool.handler({ slug: 'found-post' }, {}) as { content: Array<{ text: string }> }; + const parsed = JSON.parse(result.content[0].text); + expect(parsed.post.id).toBe('p1'); + expect(parsed.post.slug).toBe('found-post'); + expect(parsed.post.backlinks).toEqual([{ id: 'px', title: 'Ref', slug: 'ref' }]); + expect(mockPostEngine.getPostBySlug).toHaveBeenCalledWith('found-post'); + }); + + it('read_post_by_slug returns error for nonexistent slug', async () => { + mockPostEngine.getPostBySlug.mockResolvedValue(null); + const mcpServer = server.createMcpServer(); + const tool = getTool(mcpServer, 'read_post_by_slug'); + const result = await tool.handler({ slug: 'no-such-slug' }, {}) as { content: Array<{ text: string }>; isError?: boolean }; + expect(result.isError).toBe(true); + const parsed = JSON.parse(result.content[0].text); + expect(parsed.error).toContain('not found'); + }); }); // ── Prompt handler behavior ──────────────────────────────────────── diff --git a/tests/engine/MetaEngine.test.ts b/tests/engine/MetaEngine.test.ts index 35f33e6..913bc81 100644 --- a/tests/engine/MetaEngine.test.ts +++ b/tests/engine/MetaEngine.test.ts @@ -450,6 +450,36 @@ describe('MetaEngine', () => { })); }); + it('should preserve semanticSimilarityEnabled when updating from null metadata', async () => { + // projectMetadata is null (fresh engine, no syncOnStartup called) + // Simulates the case where setProjectContext was just called (e.g., dataPath change) + expect(await metaEngine.getProjectMetadata()).toBeNull(); + + await metaEngine.updateProjectMetadata({ name: 'Blog', semanticSimilarityEnabled: true }); + + const metadata = await metaEngine.getProjectMetadata(); + expect(metadata?.semanticSimilarityEnabled).toBe(true); + }); + + it('should preserve semanticSimilarityEnabled when merging into existing metadata', async () => { + await metaEngine.setProjectMetadata({ name: 'Blog', semanticSimilarityEnabled: true }); + + // Update an unrelated field — should not lose semanticSimilarityEnabled + await metaEngine.updateProjectMetadata({ name: 'Renamed Blog' }); + + const metadata = await metaEngine.getProjectMetadata(); + expect(metadata?.semanticSimilarityEnabled).toBe(true); + }); + + it('should persist semanticSimilarityEnabled to project.json', async () => { + await metaEngine.updateProjectMetadata({ name: 'Blog', semanticSimilarityEnabled: true }); + + const metaDir = metaEngine.getMetaDir(); + const projectPath = normalizePath(`${metaDir}/project.json`); + const parsed = JSON.parse(mockFiles.get(projectPath)!); + expect(parsed.semanticSimilarityEnabled).toBe(true); + }); + it('should update project name only', async () => { await metaEngine.setProjectMetadata({ name: 'Original Name', diff --git a/tests/engine/PostEngine.test.ts b/tests/engine/PostEngine.test.ts index ec7fc16..59d0445 100644 --- a/tests/engine/PostEngine.test.ts +++ b/tests/engine/PostEngine.test.ts @@ -619,6 +619,45 @@ Content for retrieval test`); }); }); + describe('getPostBySlug', () => { + it('should return null for non-existent slug', async () => { + const result = await postEngine.getPostBySlug('no-such-slug'); + expect(result).toBeNull(); + }); + + it('should retrieve post by slug', async () => { + const created = await postEngine.createPost({ + title: 'Slug Lookup Post', + content: 'Content for slug test', + }); + + vi.mocked(mockLocalDb.select).mockImplementation(() => { + const chain = createSelectChain(); + chain.where = vi.fn().mockReturnValue({ + ...chain, + get: vi.fn().mockResolvedValue({ + id: created.id, + projectId: created.projectId, + title: created.title, + slug: created.slug, + status: created.status, + content: 'Content for slug test', + tags: '[]', + categories: '[]', + createdAt: created.createdAt, + updatedAt: created.updatedAt, + }), + }); + return chain; + }); + + const result = await postEngine.getPostBySlug(created.slug); + expect(result).not.toBeNull(); + expect(result?.title).toBe('Slug Lookup Post'); + expect(result?.content).toBe('Content for slug test'); + }); + }); + describe('updatePost', () => { it('should return null when updating non-existent post', async () => { const result = await postEngine.updatePost('non-existent-id', { title: 'New Title' }); diff --git a/tests/engine/blog-tools.test.ts b/tests/engine/blog-tools.test.ts index 2d0f47d..5f0d1ac 100644 --- a/tests/engine/blog-tools.test.ts +++ b/tests/engine/blog-tools.test.ts @@ -14,6 +14,7 @@ function createMockDeps(): BlogToolDeps { return { postEngine: { getPost: vi.fn(), + getPostBySlug: vi.fn(), getAllPosts: vi.fn(), getPostsFiltered: vi.fn(), searchPostsFiltered: vi.fn(), @@ -72,12 +73,13 @@ describe('Blog Tools — createBlogTools', () => { tools = createBlogTools(deps); }); - it('returns all 17 tools', () => { + it('returns all 18 tools', () => { const names = Object.keys(tools); - expect(names).toHaveLength(17); + expect(names).toHaveLength(18); expect(names).toContain('check_term'); expect(names).toContain('search_posts'); expect(names).toContain('read_post'); + expect(names).toContain('read_post_by_slug'); expect(names).toContain('list_posts'); expect(names).toContain('get_media'); expect(names).toContain('list_media'); @@ -243,6 +245,52 @@ describe('Blog Tools — read_post', () => { }); }); +// --------------------------------------------------------------------------- +// read_post_by_slug +// --------------------------------------------------------------------------- + +describe('Blog Tools — read_post_by_slug', () => { + let deps: BlogToolDeps; + let tools: ReturnType; + + beforeEach(() => { + deps = createMockDeps(); + tools = createBlogTools(deps); + }); + + it('returns post with backlinks and outlinks when found by slug', async () => { + vi.mocked(deps.postEngine.getPostBySlug).mockResolvedValueOnce(samplePost); + vi.mocked(deps.postEngine.getLinkedBy).mockResolvedValueOnce([ + { id: 'post-2', title: 'Related', slug: 'related' }, + ]); + + const result = await tools.read_post_by_slug.execute!( + { slug: 'hello-world' }, + { toolCallId: 'tc1', messages: [], abortSignal: new AbortController().signal }, + ); + expect(result).toMatchObject({ + success: true, + post: { + id: 'post-1', + title: 'Hello World', + slug: 'hello-world', + content: '# Hello\n\nWorld', + backlinks: [{ id: 'post-2', title: 'Related' }], + }, + }); + expect(deps.postEngine.getPostBySlug).toHaveBeenCalledWith('hello-world'); + }); + + it('returns error for nonexistent slug', async () => { + vi.mocked(deps.postEngine.getPostBySlug).mockResolvedValueOnce(null); + const result = await tools.read_post_by_slug.execute!( + { slug: 'nope' }, + { toolCallId: 'tc1', messages: [], abortSignal: new AbortController().signal }, + ); + expect(result).toMatchObject({ success: false, error: 'Post not found' }); + }); +}); + // --------------------------------------------------------------------------- // list_posts // --------------------------------------------------------------------------- diff --git a/tests/engine/mainStartup.test.ts b/tests/engine/mainStartup.test.ts index 4efc158..2827ddc 100644 --- a/tests/engine/mainStartup.test.ts +++ b/tests/engine/mainStartup.test.ts @@ -1295,6 +1295,26 @@ describe('main bootstrap preview behavior', () => { }; }), })); + vi.doMock('../../src/main/engine/EmbeddingEngine', () => ({ + EmbeddingEngine: vi.fn().mockImplementation(function() { return { + setProjectContext: vi.fn().mockResolvedValue(undefined), + initialize: vi.fn().mockResolvedValue(undefined), + shutdown: vi.fn().mockResolvedValue(undefined), + indexUnindexedPosts: vi.fn().mockResolvedValue(undefined), + }; }), + })); + + vi.doMock('../../src/main/engine/BlogmarkTransformService', () => ({ + BlogmarkTransformService: vi.fn().mockImplementation(function() { return { + applyTransforms: vi.fn(async (input: { post: { title: string; content: string; categories: string[]; tags: string[] } }) => ({ + post: input.post, + appliedScriptIds: [], + errors: [], + toasts: [], + })), + }; }), + })); + await import('../../src/main/main'); await new Promise((resolve) => setTimeout(resolve, 0)); diff --git a/tests/ipc/handlers.test.ts b/tests/ipc/handlers.test.ts index f7eec28..6564112 100644 --- a/tests/ipc/handlers.test.ts +++ b/tests/ipc/handlers.test.ts @@ -371,6 +371,7 @@ describe('IPC Handlers', () => { gitEngine: mockGitEngine, gitApiAdapter: {}, taskManager: mockTaskManager, + embeddingEngine: { reindexAll: vi.fn(), indexUnindexedPosts: vi.fn(), setProjectContext: vi.fn(), embedPost: vi.fn(), removePost: vi.fn() }, blogGenerationEngine: null, // set in beforeEach publishEngine: { setProjectContext: vi.fn(), uploadHtml: vi.fn(), uploadThumbnails: vi.fn(), uploadMedia: vi.fn() }, metadataDiffEngine: { setProjectContext: vi.fn(), comparePostMetadata: vi.fn(), scanAllPublishedPosts: vi.fn(), syncDbToFile: vi.fn(), syncFileToDb: vi.fn(), groupDifferencesByField: vi.fn() }, @@ -1939,6 +1940,21 @@ describe('IPC Handlers', () => { expect(BrowserWindow.fromWebContents).toHaveBeenCalledWith(sender); expect(ownerWindow.setFullScreen).toHaveBeenCalledWith(true); }); + + it('should start rebuild embedding index task when action is rebuildEmbeddingIndex', async () => { + const send = vi.fn(); + const event = { sender: { send } }; + mockTaskManager.runTask.mockResolvedValue(undefined); + + await invokeHandlerWithEvent(event, 'app:triggerMenuAction', 'rebuildEmbeddingIndex'); + + expect(mockTaskManager.runTask).toHaveBeenCalledWith( + expect.objectContaining({ + id: expect.stringContaining('rebuild-embedding-index-'), + }) + ); + expect(send).not.toHaveBeenCalled(); + }); }); }); diff --git a/tests/renderer/components/InsertModal.test.tsx b/tests/renderer/components/InsertModal.test.tsx index 984fe20..3bc4465 100644 --- a/tests/renderer/components/InsertModal.test.tsx +++ b/tests/renderer/components/InsertModal.test.tsx @@ -4,6 +4,61 @@ import { render, screen, fireEvent, act } from '@testing-library/react'; import { InsertModal } from '../../../src/renderer/components/InsertModal/InsertModal'; import { useAppStore } from '../../../src/renderer/store/appStore'; +describe('InsertModal related posts confidence', () => { + it('shows similarity confidence percentage for each related post', async () => { + (window.electronAPI.embeddings.findSimilar as ReturnType).mockResolvedValue([ + { postId: 'p1', similarity: 0.92 }, + { postId: 'p2', similarity: 0.67 }, + ]); + (window.electronAPI.posts.get as ReturnType) + .mockResolvedValueOnce({ id: 'p1', title: 'Close Match', slug: 'close-match', excerpt: '' }) + .mockResolvedValueOnce({ id: 'p2', title: 'Loose Match', slug: 'loose-match', excerpt: '' }); + + render( + + ); + + // Wait for related posts to render + expect(await screen.findByText('Close Match')).toBeInTheDocument(); + expect(screen.getByText('Loose Match')).toBeInTheDocument(); + + // Confidence percentages should be displayed + expect(screen.getByText('92%')).toBeInTheDocument(); + expect(screen.getByText('67%')).toBeInTheDocument(); + }); + + it('shows similarity confidence for search results', async () => { + (window.electronAPI.posts.search as ReturnType).mockResolvedValue([ + { id: 'p1', title: 'Found Post', slug: 'found-post', excerpt: 'Some text' }, + ]); + (window.electronAPI.embeddings.computeSimilarities as ReturnType).mockResolvedValue({ + 'p1': 0.85, + }); + + render( + + ); + + const input = screen.getByPlaceholderText('Search posts by title or content...'); + fireEvent.input(input, { target: { value: 'Found Post' } }); + + expect(await screen.findByText('Found Post')).toBeInTheDocument(); + expect(await screen.findByText('85%')).toBeInTheDocument(); + }); +}); + describe('InsertModal format hints', () => { it('shows canonical post link format hint in internal link mode', () => { render( diff --git a/tests/renderer/components/SettingsView.test.tsx b/tests/renderer/components/SettingsView.test.tsx index f3cb911..198cba5 100644 --- a/tests/renderer/components/SettingsView.test.tsx +++ b/tests/renderer/components/SettingsView.test.tsx @@ -370,4 +370,28 @@ describe('SettingsView Diff Preferences', () => { }) ); }); + + it('auto-saves semanticSimilarityEnabled immediately when toggled without requiring a Save click', async () => { + (window as any).electronAPI.meta.getProjectMetadata = vi.fn().mockResolvedValue({ + maxPostsPerPage: 50, + semanticSimilarityEnabled: false, + categorySettings: { + article: { renderInLists: true, showTitle: true }, + }, + }); + (window as any).electronAPI.meta.updateProjectMetadata = vi.fn().mockResolvedValue({}); + + render(); + + const checkbox = await screen.findByLabelText(/semantic similarity/i); + expect((checkbox as HTMLInputElement).checked).toBe(false); + + await act(async () => { + fireEvent.click(checkbox); + }); + + expect((window as any).electronAPI.meta.updateProjectMetadata).toHaveBeenCalledWith( + expect.objectContaining({ semanticSimilarityEnabled: true }) + ); + }); }); diff --git a/tests/renderer/navigation/editorRouting.test.ts b/tests/renderer/navigation/editorRouting.test.ts index 11f5f58..0908c95 100644 --- a/tests/renderer/navigation/editorRouting.test.ts +++ b/tests/renderer/navigation/editorRouting.test.ts @@ -23,6 +23,7 @@ describe('editorRouting', () => { 'site-validation': 'site-validation', scripts: 'scripts', templates: 'templates', + 'find-duplicates': 'find-duplicates', }); }); diff --git a/tests/renderer/python/pythonApiContractV1.test.ts b/tests/renderer/python/pythonApiContractV1.test.ts index 933eaaf..1c7f01a 100644 --- a/tests/renderer/python/pythonApiContractV1.test.ts +++ b/tests/renderer/python/pythonApiContractV1.test.ts @@ -79,7 +79,7 @@ describe('pythonApiContractV1', () => { it('contains semantic version metadata for compatibility checks', () => { expect(BDS_PYTHON_API_CONTRACT_V1).toMatchObject({ - version: '1.11.0', + version: '1.12.0', generatedAt: expect.any(String), }); }); diff --git a/tests/setup.ts b/tests/setup.ts index d1f0c2a..414ab13 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -156,6 +156,10 @@ Object.defineProperty(globalThis, 'window', { validate: vi.fn().mockResolvedValue({ valid: true, errors: [] }), rebuildFromFiles: vi.fn().mockResolvedValue(undefined), }, + embeddings: { + findSimilar: vi.fn().mockResolvedValue([]), + computeSimilarities: vi.fn().mockResolvedValue({}), + }, on: vi.fn(() => () => {}), }, },