Implement semantic embeddings and duplicate review

crates/bds-core/migrations/20260718000000_initial_schema/up.sql

@@ -258,9 +258,12 @@ CREATE TABLE IF NOT EXISTS embedding_keys (
     post_id TEXT NOT NULL,
     project_id TEXT NOT NULL,
     content_hash TEXT NOT NULL,
-    vector TEXT NOT NULL
+    vector BLOB NOT NULL
 );
 
+CREATE UNIQUE INDEX IF NOT EXISTS embedding_keys_project_post_idx
+    ON embedding_keys(project_id, post_id);
+
 CREATE TABLE IF NOT EXISTS dismissed_duplicate_pairs (
     id TEXT NOT NULL PRIMARY KEY,
     project_id TEXT NOT NULL REFERENCES projects(id),

crates/bds-core/migrations/20260719090000_embedding_vector_blob/down.sql

@@ -0,0 +1,15 @@
+-- This file should undo anything in `up.sql`
+CREATE TABLE embedding_keys_old (
+    label INTEGER NOT NULL PRIMARY KEY,
+    post_id TEXT NOT NULL,
+    project_id TEXT NOT NULL,
+    content_hash TEXT NOT NULL,
+    vector TEXT NOT NULL
+);
+
+INSERT INTO embedding_keys_old (label, post_id, project_id, content_hash, vector)
+SELECT label, post_id, project_id, content_hash, CAST(vector AS TEXT)
+FROM embedding_keys;
+
+DROP TABLE embedding_keys;
+ALTER TABLE embedding_keys_old RENAME TO embedding_keys;

crates/bds-core/migrations/20260719090000_embedding_vector_blob/up.sql

@@ -0,0 +1,18 @@
+-- Your SQL goes here
+CREATE TABLE embedding_keys_new (
+    label INTEGER NOT NULL PRIMARY KEY,
+    post_id TEXT NOT NULL,
+    project_id TEXT NOT NULL,
+    content_hash TEXT NOT NULL,
+    vector BLOB NOT NULL
+);
+
+INSERT INTO embedding_keys_new (label, post_id, project_id, content_hash, vector)
+SELECT label, post_id, project_id, content_hash, CAST(vector AS BLOB)
+FROM embedding_keys;
+
+DROP TABLE embedding_keys;
+ALTER TABLE embedding_keys_new RENAME TO embedding_keys;
+
+CREATE UNIQUE INDEX embedding_keys_project_post_idx
+    ON embedding_keys(project_id, post_id);