Feature/semantic similarity (#36)

* fix: mixed up migrations

* feat: semantic similarity first take

* feat: semantic similarity first round of fixes

* feat: more work on making semantic similarity work properly

* feat: getPostBySlug for the AI

* feat: show similarity in post-link-insert-modal

* chore: remove done doc

---------

Co-authored-by: hugo <hugoms@me.com>

drizzle/0011_embedding_tables.sql

@@ -0,0 +1,15 @@
+CREATE TABLE `dismissed_duplicate_pairs` (
+	`id` text PRIMARY KEY NOT NULL,
+	`project_id` text NOT NULL,
+	`post_id_a` text NOT NULL,
+	`post_id_b` text NOT NULL,
+	`dismissed_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `dismissed_pairs_idx` ON `dismissed_duplicate_pairs` (`project_id`,`post_id_a`,`post_id_b`);--> statement-breakpoint
+CREATE TABLE `embedding_keys` (
+	`label` integer PRIMARY KEY NOT NULL,
+	`post_id` text NOT NULL,
+	`project_id` text NOT NULL,
+	`content_hash` text NOT NULL
+);

drizzle/0012_flimsy_meteorite.sql

@@ -0,0 +1 @@
+ALTER TABLE `embedding_keys` ADD `vector` blob;

drizzle/meta/0011_snapshot.json

@@ -1,8 +1,8 @@
 {
   "version": "6",
   "dialect": "sqlite",
-  "id": "3312ee41-2b01-4579-aa9b-7fd4f0f3c7a4",
-  "prevId": "b3e7e63c-d906-48af-98a0-e3d0741ff13a",
+  "id": "ac24604e-89e5-4def-95a4-1d175c4126f8",
+  "prevId": "3312ee41-2b01-4579-aa9b-7fd4f0f3c7a4",
   "tables": {
     "chat_conversations": {
       "name": "chat_conversations",
@@ -971,13 +971,6 @@
           "notNull": false,
           "autoincrement": false
         },
-        "language": {
-          "name": "language",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
         "published_title": {
           "name": "published_title",
           "type": "text",
@@ -1012,6 +1005,13 @@
           "primaryKey": false,
           "notNull": false,
           "autoincrement": false
+        },
+        "language": {
+          "name": "language",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
         }
       },
       "indexes": {
@@ -1424,6 +1424,99 @@
       "compositePrimaryKeys": {},
       "uniqueConstraints": {},
       "checkConstraints": {}
+    },
+    "embedding_keys": {
+      "name": "embedding_keys",
+      "columns": {
+        "label": {
+          "name": "label",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "post_id": {
+          "name": "post_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "project_id": {
+          "name": "project_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "content_hash": {
+          "name": "content_hash",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "checkConstraints": {}
+    },
+    "dismissed_duplicate_pairs": {
+      "name": "dismissed_duplicate_pairs",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "project_id": {
+          "name": "project_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "post_id_a": {
+          "name": "post_id_a",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "post_id_b": {
+          "name": "post_id_b",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "dismissed_at": {
+          "name": "dismissed_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {
+        "dismissed_pairs_idx": {
+          "name": "dismissed_pairs_idx",
+          "columns": [
+            "project_id",
+            "post_id_a",
+            "post_id_b"
+          ],
+          "isUnique": true
+        }
+      },
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "checkConstraints": {}
     }
   },
   "views": {},

drizzle/meta/_journal.json

@@ -76,7 +76,21 @@
       "idx": 10,
       "version": "6",
       "when": 1772462693094,
-      "tag": "0011_loving_alex_wilder",
+      "tag": "0010_loving_alex_wilder",
+      "breakpoints": true
+    },
+    {
+      "idx": 11,
+      "version": "6",
+      "when": 1772711489237,
+      "tag": "0011_embedding_tables",
+      "breakpoints": true
+    },
+    {
+      "idx": 12,
+      "version": "6",
+      "when": 1772738580546,
+      "tag": "0012_flimsy_meteorite",
       "breakpoints": true
     }
   ]