add package reference embedding database

Author: breadchris

lunatrace/bsl/hasura/metadata/databases/lunatrace/tables/package_content_embedding.yaml

@@ -0,0 +1,7 @@
+table:
+  name: content_embedding
+  schema: package
+object_relationships:
+  - name: reference_content
+    using:
+      foreign_key_constraint_on: reference_content_id

lunatrace/bsl/hasura/metadata/databases/lunatrace/tables/package_package.yaml

@@ -30,6 +30,13 @@ array_relationships:
         table:
           name: package_maintainer
           schema: package
+  - name: reference_contents
+    using:
+      foreign_key_constraint_on:
+        column: package_id
+        table:
+          name: reference_content
+          schema: package
   - name: releases
     using:
       foreign_key_constraint_on:

lunatrace/bsl/hasura/metadata/databases/lunatrace/tables/package_reference_content.yaml

@@ -0,0 +1,15 @@
+table:
+  name: reference_content
+  schema: package
+object_relationships:
+  - name: package
+    using:
+      foreign_key_constraint_on: package_id
+array_relationships:
+  - name: content_embeddings
+    using:
+      foreign_key_constraint_on:
+        column: reference_content_id
+        table:
+          name: content_embedding
+          schema: package

lunatrace/bsl/hasura/migrations/lunatrace/1677850286590_package_reference_embeddings/down.sql

@@ -0,0 +1,2 @@
+DROP TABLE "package"."content_embedding";
+DROP TABLE "package"."reference_content";

lunatrace/bsl/hasura/migrations/lunatrace/1677850286590_package_reference_embeddings/up.sql

@@ -0,0 +1,25 @@
+CREATE TABLE "package"."reference_content" (
+    "id" uuid NOT NULL DEFAULT gen_random_uuid(),
+    "package_id" uuid NOT NULL REFERENCES "package"."package"("id") ON UPDATE cascade ON DELETE cascade,
+    "url" text NOT NULL,
+    "content" text NOT NULL,
+    "normalized_content" text NOT NULL,
+    "content_type" text NOT NULL,
+    "last_successful_fetch" timestamptz DEFAULT NULL,
+    PRIMARY KEY ("id"),
+    UNIQUE ("package_id", "url")
+);
+
+CREATE TABLE "package"."content_embedding" (
+    "id" uuid NOT NULL DEFAULT gen_random_uuid(),
+    "content_hash" text NOT NULL,
+    "reference_content_id" uuid NOT NULL REFERENCES "package"."reference_content"("id") ON UPDATE cascade ON DELETE cascade,
+    "content" text NOT NULL,
+    "embedding" vector (1536) NOT NULL,
+    PRIMARY KEY ("id"),
+    UNIQUE ("content_hash")
+);
+
+CREATE INDEX ON "package"."content_embedding"
+    USING ivfflat (embedding vector_cosine_ops)
+    WITH (lists = 100);