hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement semantic embeddings and duplicate review

Browse Source
This commit is contained in:
Chili Palmer
2026-07-19 18:06:47 +02:00
parent fd4fd744e0
commit e8764262a3
42 changed files with 3808 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
crates/bds-core/src/scripting/core_host.rs
View File

@@ -1382,6 +1382,95 @@ impl CoreHost {
(_, response) => one_shot_json(response),
}
}
fn embeddings(&self, method: &str, args: &[Value]) -> HostResult<Value> {
let db = self.database()?;
let service = engine::embedding::EmbeddingService::production(db.conn(), &self.data_dir);
match method {
"get_progress" => {
let (indexed, total) = service.indexing_progress(&self.project_id)?;
Ok(json!({"indexed": indexed, "total": total}))
}
"find_similar" => {
let post_id = string_arg(args, 0)?;
self.scoped(
|conn| crate::db::queries::post::get_post_by_id(conn, post_id),
|post| post.project_id.as_str(),
)?;
let limit = args.get(1).and_then(Value::as_u64).unwrap_or(5) as usize;
Ok(Value::Array(
service
.find_similar(post_id, limit)?
.into_iter()
.map(|post| {
json!({
"post_id": post.post_id,
"title": post.title,
"score": post.similarity,
})
})
.collect(),
))
}
"compute_similarities" => {
let post_id = string_arg(args, 0)?;
self.scoped(
|conn| crate::db::queries::post::get_post_by_id(conn, post_id),
|post| post.project_id.as_str(),
)?;
let target_ids = string_array_arg(args, 1)?;
json_value(service.compute_similarities(post_id, &target_ids))
}
"suggest_tags" => {
let post_id = string_arg(args, 0)?;
self.scoped(
|conn| crate::db::queries::post::get_post_by_id(conn, post_id),
|post| post.project_id.as_str(),
)?;
json_value(service.suggest_tags(post_id))
}
"find_duplicates" => {
let mut page = 0;
let pairs = loop {
let result = service.find_duplicates(&self.project_id, page)?;
if !result.has_more {
break result.pairs;
}
page += 1;
};
Ok(Value::Array(
pairs
.into_iter()
.map(|pair| {
json!({
"post_id_a": pair.post_id_a,
"title_a": pair.title_a,
"post_id_b": pair.post_id_b,
"title_b": pair.title_b,
"score": pair.similarity,
"similarity": pair.similarity,
"exact_match": pair.exact_match,
})
})
.collect(),
))
}
"dismiss_pair" => {
let post_id_a = string_arg(args, 0)?;
let post_id_b = string_arg(args, 1)?;
for post_id in [post_id_a, post_id_b] {
self.scoped(
|conn| crate::db::queries::post::get_post_by_id(conn, post_id),
|post| post.project_id.as_str(),
)?;
}
service.dismiss_duplicate_pair(post_id_a, post_id_b)?;
Ok(Value::Bool(true))
}
"index_unindexed_posts" => json_value(service.index_unindexed(&self.project_id)),
_ => Err(format!("unknown embeddings capability: {method}").into()),
}
}
}
impl HostApi for CoreHost {
@@ -1398,6 +1487,7 @@ impl HostApi for CoreHost {
"tasks" => self.tasks(method, &arguments),
"publish" => self.publish(method, &arguments),
"chat" => self.chat(method, &arguments),
"embeddings" => self.embeddings(method, &arguments),
"bds" if method == "report_progress" => self.report_progress(&arguments),
_ => Err(format!("unknown host capability: {namespace}.{method}").into()),
};
@@ -1537,6 +1627,19 @@ fn string_arg(args: &[Value], index: usize) -> HostResult<&str> {
.ok_or_else(|| format!("argument {} must be a string", index + 1).into())
}
fn string_array_arg(args: &[Value], index: usize) -> HostResult<Vec<String>> {
args.get(index)
.and_then(Value::as_array)
.map(|items| {
items
.iter()
.filter_map(Value::as_str)
.map(str::to_owned)
.collect()
})
.ok_or_else(|| format!("argument {} must be a table", index + 1).into())
}
fn string_field<'a>(value: &'a Map<String, Value>, field: &str) -> HostResult<&'a str> {
value
.get(field)
@@ -1821,6 +1924,9 @@ mod tests {
upload = upload,
timestamp = post.created_at,
project_path = bds.app.get_default_project_path(),
embedding_progress = bds.embeddings.get_progress(),
embedding_backfill = bds.embeddings.index_unindexed_posts(),
foreign_embedding = bds.embeddings.find_similar(input.foreign_post, 5),
}
end
"#,
@@ -1842,5 +1948,11 @@ mod tests {
assert!(result.value["upload"].is_null());
assert!(result.value["timestamp"].as_str().unwrap().contains('T'));
assert_eq!(manager.progress(task_id), Some(0.5));
assert_eq!(
result.value["embedding_progress"],
json!({"indexed": 0, "total": 1})
);
assert_eq!(result.value["embedding_backfill"], json!([]));
assert!(result.value["foreign_embedding"].is_null());
}
}

9
crates/bds-core/src/scripting/mod.rs
View File

@@ -456,6 +456,7 @@ mod tests {
[
"app",
"chat",
"embeddings",
"media",
"meta",
"posts",
@@ -473,7 +474,7 @@ mod tests {
function main()
return {
sync = bds.sync,
embeddings = bds.embeddings,
embeddings = type(bds.embeddings),
report_progress = type(bds.report_progress),
post_search = type(bds.posts.search),
app_toast = type(bds.app.toast),
@@ -492,6 +493,7 @@ mod tests {
"report_progress": "function",
"post_search": "function",
"app_toast": "function",
"embeddings": "table",
})
);
}
@@ -514,6 +516,7 @@ mod tests {
bds.tasks.status_snapshot(),
bds.publish.upload_site({}),
bds.chat.detect_post_language("title", "body"),
bds.embeddings.get_progress(),
}
end
"#,
@@ -525,8 +528,8 @@ mod tests {
)
.unwrap();
assert_eq!(execution.value.as_array().unwrap().len(), 11);
assert_eq!(host.0.lock().unwrap().len(), 11);
assert_eq!(execution.value.as_array().unwrap().len(), 12);
assert_eq!(host.0.lock().unwrap().len(), 12);
}
#[test]