Implement semantic embeddings and duplicate review
This commit is contained in:
hugo committed
1 parent
fd4fd744e0
commit
e8764262a3
42 files changed
+3808
-79
No files matched your search
@@ -32,6 +32,7 @@ Macro scripts expose `render(input, context)` and transform scripts expose `main
|
||||
- [Root helpers](#root-helpers)
|
||||
- [`bds.app`](#bdsapp)
|
||||
- [`bds.chat`](#bdschat)
|
||||
- [`bds.embeddings`](#bdsembeddings)
|
||||
- [`bds.media`](#bdsmedia)
|
||||
- [`bds.meta`](#bdsmeta)
|
||||
- [`bds.posts`](#bdsposts)
|
||||
@@ -795,6 +796,230 @@ local result = bds.chat.translate_media_metadata("example-id", "en")
|
||||
{ key = "value" }
|
||||
```
|
||||
|
||||
## `bds.embeddings`
|
||||
|
||||
### `bds.embeddings.compute_similarities`
|
||||
|
||||
Compute similarity scores from one source post to target posts.
|
||||
|
||||
**Signature**
|
||||
|
||||
```text
|
||||
bds.embeddings.compute_similarities(post_id: string, target_ids: table) -> table | nil
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
| Name | Type | Required | Example |
|
||||
| --- | --- | --- | --- |
|
||||
| `post_id` | `string` | Yes | `"example-id"` |
|
||||
| `target_ids` | `table` | Yes | `{ title = "Example" }` |
|
||||
|
||||
**Returns**
|
||||
|
||||
`table | nil`. `nil` means no value was available or the host operation failed.
|
||||
|
||||
**Example call**
|
||||
|
||||
```lua
|
||||
local result = bds.embeddings.compute_similarities("example-id", { title = "Example" })
|
||||
```
|
||||
|
||||
**Example response**
|
||||
|
||||
```lua
|
||||
{ key = "value" }
|
||||
```
|
||||
|
||||
### `bds.embeddings.dismiss_pair`
|
||||
|
||||
Dismiss a duplicate candidate pair.
|
||||
|
||||
**Signature**
|
||||
|
||||
```text
|
||||
bds.embeddings.dismiss_pair(post_id_a: string, post_id_b: string) -> boolean
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
| Name | Type | Required | Example |
|
||||
| --- | --- | --- | --- |
|
||||
| `post_id_a` | `string` | Yes | `"example"` |
|
||||
| `post_id_b` | `string` | Yes | `"example"` |
|
||||
|
||||
**Returns**
|
||||
|
||||
`boolean`. `false` means the operation was rejected or failed.
|
||||
|
||||
**Example call**
|
||||
|
||||
```lua
|
||||
local result = bds.embeddings.dismiss_pair("example", "example")
|
||||
```
|
||||
|
||||
**Example response**
|
||||
|
||||
```lua
|
||||
true
|
||||
```
|
||||
|
||||
### `bds.embeddings.find_duplicates`
|
||||
|
||||
Find duplicate post candidates for the current project.
|
||||
|
||||
**Signature**
|
||||
|
||||
```text
|
||||
bds.embeddings.find_duplicates() -> table | nil
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
None.
|
||||
|
||||
**Returns**
|
||||
|
||||
`table | nil`. `nil` means no value was available or the host operation failed.
|
||||
|
||||
**Example call**
|
||||
|
||||
```lua
|
||||
local result = bds.embeddings.find_duplicates()
|
||||
```
|
||||
|
||||
**Example response**
|
||||
|
||||
```lua
|
||||
{ key = "value" }
|
||||
```
|
||||
|
||||
### `bds.embeddings.find_similar`
|
||||
|
||||
Find posts similar to the given post id.
|
||||
|
||||
**Signature**
|
||||
|
||||
```text
|
||||
bds.embeddings.find_similar(post_id: string, limit?: integer) -> table | nil
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
| Name | Type | Required | Example |
|
||||
| --- | --- | --- | --- |
|
||||
| `post_id` | `string` | Yes | `"example-id"` |
|
||||
| `limit` | `integer` | No | `nil` |
|
||||
|
||||
**Returns**
|
||||
|
||||
`table | nil`. `nil` means no value was available or the host operation failed.
|
||||
|
||||
**Example call**
|
||||
|
||||
```lua
|
||||
local result = bds.embeddings.find_similar("example-id", nil)
|
||||
```
|
||||
|
||||
**Example response**
|
||||
|
||||
```lua
|
||||
{ key = "value" }
|
||||
```
|
||||
|
||||
### `bds.embeddings.get_progress`
|
||||
|
||||
Get embedding index progress for the current project.
|
||||
|
||||
**Signature**
|
||||
|
||||
```text
|
||||
bds.embeddings.get_progress() -> table | nil
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
None.
|
||||
|
||||
**Returns**
|
||||
|
||||
`table | nil`. `nil` means no value was available or the host operation failed.
|
||||
|
||||
**Example call**
|
||||
|
||||
```lua
|
||||
local result = bds.embeddings.get_progress()
|
||||
```
|
||||
|
||||
**Example response**
|
||||
|
||||
```lua
|
||||
{ key = "value" }
|
||||
```
|
||||
|
||||
### `bds.embeddings.index_unindexed_posts`
|
||||
|
||||
Index posts missing embeddings for the current project.
|
||||
|
||||
**Signature**
|
||||
|
||||
```text
|
||||
bds.embeddings.index_unindexed_posts() -> table | nil
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
None.
|
||||
|
||||
**Returns**
|
||||
|
||||
`table | nil`. `nil` means no value was available or the host operation failed.
|
||||
|
||||
**Example call**
|
||||
|
||||
```lua
|
||||
local result = bds.embeddings.index_unindexed_posts()
|
||||
```
|
||||
|
||||
**Example response**
|
||||
|
||||
```lua
|
||||
{ key = "value" }
|
||||
```
|
||||
|
||||
### `bds.embeddings.suggest_tags`
|
||||
|
||||
Suggest tags for a post from semantic similarity.
|
||||
|
||||
**Signature**
|
||||
|
||||
```text
|
||||
bds.embeddings.suggest_tags(post_id: string, exclude_tags?: table) -> table | nil
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
| Name | Type | Required | Example |
|
||||
| --- | --- | --- | --- |
|
||||
| `post_id` | `string` | Yes | `"example-id"` |
|
||||
| `exclude_tags` | `table` | No | `{ title = "Example" }` |
|
||||
|
||||
**Returns**
|
||||
|
||||
`table | nil`. `nil` means no value was available or the host operation failed.
|
||||
|
||||
**Example call**
|
||||
|
||||
```lua
|
||||
local result = bds.embeddings.suggest_tags("example-id", { title = "Example" })
|
||||
```
|
||||
|
||||
**Example response**
|
||||
|
||||
```lua
|
||||
{ key = "value" }
|
||||
```
|
||||
|
||||
## `bds.media`
|
||||
|
||||
### `bds.media.delete_translation`
|
||||
|
||||
@@ -1543,6 +1543,99 @@
|
||||
}
|
||||
],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Compute similarity scores from one source post to target posts.",
|
||||
"name": "compute_similarities",
|
||||
"namespace": "embeddings",
|
||||
"params": [
|
||||
{
|
||||
"name": "post_id",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "target_ids",
|
||||
"required": true,
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Dismiss a duplicate candidate pair.",
|
||||
"name": "dismiss_pair",
|
||||
"namespace": "embeddings",
|
||||
"params": [
|
||||
{
|
||||
"name": "post_id_a",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "post_id_b",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"returns": "boolean"
|
||||
},
|
||||
{
|
||||
"description": "Find duplicate post candidates for the current project.",
|
||||
"name": "find_duplicates",
|
||||
"namespace": "embeddings",
|
||||
"params": [],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Find posts similar to the given post id.",
|
||||
"name": "find_similar",
|
||||
"namespace": "embeddings",
|
||||
"params": [
|
||||
{
|
||||
"name": "post_id",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "limit",
|
||||
"required": false,
|
||||
"type": "integer"
|
||||
}
|
||||
],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Get embedding index progress for the current project.",
|
||||
"name": "get_progress",
|
||||
"namespace": "embeddings",
|
||||
"params": [],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Index posts missing embeddings for the current project.",
|
||||
"name": "index_unindexed_posts",
|
||||
"namespace": "embeddings",
|
||||
"params": [],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Suggest tags for a post from semantic similarity.",
|
||||
"name": "suggest_tags",
|
||||
"namespace": "embeddings",
|
||||
"params": [
|
||||
{
|
||||
"name": "post_id",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "exclude_tags",
|
||||
"required": false,
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"returns": "table | nil"
|
||||
}
|
||||
],
|
||||
"root_methods": [
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"version":"0.4.0","methods":[{"namespace":"app","name":"copy_to_clipboard","params":[{"name":"text","type":"string","required":true}],"returns":"boolean"},{"namespace":"app","name":"get_blogmark_bookmarklet","params":[],"returns":"string"},{"namespace":"app","name":"get_data_paths","params":[],"returns":"table"},{"namespace":"app","name":"get_default_project_path","params":[],"returns":"string | nil"},{"namespace":"app","name":"get_system_language","params":[],"returns":"string | nil"},{"namespace":"app","name":"get_title_bar_metrics","params":[],"returns":"table | nil"},{"namespace":"app","name":"log","params":[{"name":"text","type":"string","required":true}],"returns":"boolean"},{"namespace":"app","name":"notify_renderer_ready","params":[],"returns":"boolean"},{"namespace":"app","name":"open_folder","params":[{"name":"folder_path","type":"string","required":true}],"returns":"string"},{"namespace":"app","name":"read_project_metadata","params":[{"name":"folder_path","type":"string","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"app","name":"select_folder","params":[{"name":"title","type":"string","required":false}],"returns":"string | nil"},{"namespace":"app","name":"set_preview_post_target","params":[{"name":"post_id","type":"string","required":false}],"returns":"boolean"},{"namespace":"app","name":"show_item_in_folder","params":[{"name":"item_path","type":"string","required":true}],"returns":"nil"},{"namespace":"app","name":"trigger_menu_action","params":[{"name":"action","type":"string","required":true}],"returns":"nil"},{"namespace":"projects","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"projects","name":"delete_with_data","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"projects","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"get_all","params":[],"returns":"ProjectData[]"},{"namespace":"projects","name":"get_active","params":[],"returns":"ProjectData | nil"},{"namespace":"projects","name":"set_active","params":[{"name":"id","type":"string","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"ProjectData | nil"},{"namespace":"posts","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"posts","name":"discard","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"filter","params":[{"name":"filters","type":"table","required":true}],"returns":"PostData[] | nil"},{"namespace":"posts","name":"generate_unique_slug","params":[{"name":"title","type":"string","required":true},{"name":"exclude_post_id","type":"string","required":false}],"returns":"string"},{"namespace":"posts","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"get_all","params":[],"returns":"PostData[]"},{"namespace":"posts","name":"get_by_slug","params":[{"name":"slug","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"get_by_status","params":[{"name":"status","type":"string","required":true}],"returns":"PostData[]"},{"namespace":"posts","name":"get_by_year_month","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_categories","params":[],"returns":"string[]"},{"namespace":"posts","name":"get_categories_with_counts","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_dashboard_stats","params":[],"returns":"table"},{"namespace":"posts","name":"get_linked_by","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"get_links_to","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"get_preview_url","params":[{"name":"post_id","type":"string","required":true},{"name":"options","type":"table","required":false}],"returns":"string | nil"},{"namespace":"posts","name":"get_tags","params":[],"returns":"string[]"},{"namespace":"posts","name":"get_tags_with_counts","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_translation","params":[{"name":"post_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"posts","name":"get_translatioLine truncated
|
||||
{"version":"0.4.0","methods":[{"namespace":"app","name":"copy_to_clipboard","params":[{"name":"text","type":"string","required":true}],"returns":"boolean"},{"namespace":"app","name":"get_blogmark_bookmarklet","params":[],"returns":"string"},{"namespace":"app","name":"get_data_paths","params":[],"returns":"table"},{"namespace":"app","name":"get_default_project_path","params":[],"returns":"string | nil"},{"namespace":"app","name":"get_system_language","params":[],"returns":"string | nil"},{"namespace":"app","name":"get_title_bar_metrics","params":[],"returns":"table | nil"},{"namespace":"app","name":"log","params":[{"name":"text","type":"string","required":true}],"returns":"boolean"},{"namespace":"app","name":"notify_renderer_ready","params":[],"returns":"boolean"},{"namespace":"app","name":"open_folder","params":[{"name":"folder_path","type":"string","required":true}],"returns":"string"},{"namespace":"app","name":"read_project_metadata","params":[{"name":"folder_path","type":"string","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"app","name":"select_folder","params":[{"name":"title","type":"string","required":false}],"returns":"string | nil"},{"namespace":"app","name":"set_preview_post_target","params":[{"name":"post_id","type":"string","required":false}],"returns":"boolean"},{"namespace":"app","name":"show_item_in_folder","params":[{"name":"item_path","type":"string","required":true}],"returns":"nil"},{"namespace":"app","name":"trigger_menu_action","params":[{"name":"action","type":"string","required":true}],"returns":"nil"},{"namespace":"projects","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"projects","name":"delete_with_data","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"projects","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"get_all","params":[],"returns":"ProjectData[]"},{"namespace":"projects","name":"get_active","params":[],"returns":"ProjectData | nil"},{"namespace":"projects","name":"set_active","params":[{"name":"id","type":"string","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"ProjectData | nil"},{"namespace":"posts","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"posts","name":"discard","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"filter","params":[{"name":"filters","type":"table","required":true}],"returns":"PostData[] | nil"},{"namespace":"posts","name":"generate_unique_slug","params":[{"name":"title","type":"string","required":true},{"name":"exclude_post_id","type":"string","required":false}],"returns":"string"},{"namespace":"posts","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"get_all","params":[],"returns":"PostData[]"},{"namespace":"posts","name":"get_by_slug","params":[{"name":"slug","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"get_by_status","params":[{"name":"status","type":"string","required":true}],"returns":"PostData[]"},{"namespace":"posts","name":"get_by_year_month","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_categories","params":[],"returns":"string[]"},{"namespace":"posts","name":"get_categories_with_counts","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_dashboard_stats","params":[],"returns":"table"},{"namespace":"posts","name":"get_linked_by","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"get_links_to","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"get_preview_url","params":[{"name":"post_id","type":"string","required":true},{"name":"options","type":"table","required":false}],"returns":"string | nil"},{"namespace":"posts","name":"get_tags","params":[],"returns":"string[]"},{"namespace":"posts","name":"get_tags_with_counts","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_translation","params":[{"name":"post_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"posts","name":"get_translatioLine truncated
|
||||
@@ -1669,6 +1669,106 @@
|
||||
],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Compute similarity scores from one source post to target posts.",
|
||||
"detail": "bds.embeddings.compute_similarities(post_id: string, target_ids: table) -> table | nil",
|
||||
"insert_text": "bds.embeddings.compute_similarities(\"example-id\", { title = \"Example\" })",
|
||||
"label": "bds.embeddings.compute_similarities",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "post_id",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "target_ids",
|
||||
"required": true,
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Dismiss a duplicate candidate pair.",
|
||||
"detail": "bds.embeddings.dismiss_pair(post_id_a: string, post_id_b: string) -> boolean",
|
||||
"insert_text": "bds.embeddings.dismiss_pair(\"example\", \"example\")",
|
||||
"label": "bds.embeddings.dismiss_pair",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "post_id_a",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "post_id_b",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"returns": "boolean"
|
||||
},
|
||||
{
|
||||
"description": "Find duplicate post candidates for the current project.",
|
||||
"detail": "bds.embeddings.find_duplicates() -> table | nil",
|
||||
"insert_text": "bds.embeddings.find_duplicates()",
|
||||
"label": "bds.embeddings.find_duplicates",
|
||||
"parameters": [],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Find posts similar to the given post id.",
|
||||
"detail": "bds.embeddings.find_similar(post_id: string, limit?: integer) -> table | nil",
|
||||
"insert_text": "bds.embeddings.find_similar(\"example-id\", nil)",
|
||||
"label": "bds.embeddings.find_similar",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "post_id",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "limit",
|
||||
"required": false,
|
||||
"type": "integer"
|
||||
}
|
||||
],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Get embedding index progress for the current project.",
|
||||
"detail": "bds.embeddings.get_progress() -> table | nil",
|
||||
"insert_text": "bds.embeddings.get_progress()",
|
||||
"label": "bds.embeddings.get_progress",
|
||||
"parameters": [],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Index posts missing embeddings for the current project.",
|
||||
"detail": "bds.embeddings.index_unindexed_posts() -> table | nil",
|
||||
"insert_text": "bds.embeddings.index_unindexed_posts()",
|
||||
"label": "bds.embeddings.index_unindexed_posts",
|
||||
"parameters": [],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Suggest tags for a post from semantic similarity.",
|
||||
"detail": "bds.embeddings.suggest_tags(post_id: string, exclude_tags?: table) -> table | nil",
|
||||
"insert_text": "bds.embeddings.suggest_tags(\"example-id\", { title = \"Example\" })",
|
||||
"label": "bds.embeddings.suggest_tags",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "post_id",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "exclude_tags",
|
||||
"required": false,
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"returns": "table | nil"
|
||||
},
|
||||
{
|
||||
"description": "Report progress for the current managed job.",
|
||||
"detail": "bds.report_progress(payload: table) -> boolean",
|
||||
|
||||
Reference in new issue
Block a user