serde for undefined

packages/convex-helpers/server/pagination.ts

@@ -16,7 +16,7 @@ import {
   streamIndexRange,
 } from "./stream.js";
 
-export type IndexKey = Value[];
+export type IndexKey = (Value | undefined)[];
 
 export type PageRequest<
   DataModel extends GenericDataModel,

packages/convex-helpers/server/stream.test.ts

@@ -748,4 +748,69 @@ describe("stream", () => {
       expect(page1.page.map(stripSystemFields)).toEqual([{ a: 1, b: 3, c: 5 }]);
     });
   });
+  test("undefined cursor serialization roundtrips", async () => {
+    const schema = defineSchema({
+      foo: defineTable({
+        a: v.optional(v.number()),
+        b: v.number(),
+      }).index("ab", ["a", "b"]),
+    });
+    const t = convexTest(schema, modules);
+    await t.run(async (ctx) => {
+      await ctx.db.insert("foo", { a: 1, b: 2 });
+      await ctx.db.insert("foo", { a: undefined, b: 3 });
+      await ctx.db.insert("foo", { a: 2, b: 4 });
+      await ctx.db.insert("foo", { a: undefined, b: 5 });
+      const query = stream(ctx.db, schema).query("foo").withIndex("ab");
+      const result = await query.paginate({ numItems: 1, cursor: null });
+      expect(result.continueCursor).toMatch('["undefined",');
+      expect(result.page.map(stripSystemFields)).toEqual([
+        { a: undefined, b: 3 },
+      ]);
+      expect(result.isDone).toBe(false);
+      const page1 = await query.paginate({
+        numItems: 2,
+        cursor: result.continueCursor,
+      });
+      expect(page1.page.map(stripSystemFields)).toEqual([
+        { b: 5 },
+        { a: 1, b: 2 },
+      ]);
+      expect(page1.isDone).toBe(false);
+      const page2 = await query.paginate({
+        numItems: 2,
+        cursor: page1.continueCursor,
+      });
+      expect(page2.page.map(stripSystemFields)).toEqual([{ a: 2, b: 4 }]);
+      expect(page2.isDone).toBe(true);
+    });
+  });
+  test("literal undefined string works", async () => {
+    const schema = defineSchema({
+      foo: defineTable({
+        a: v.optional(v.string()),
+        b: v.number(),
+      }).index("ab", ["a", "b"]),
+    });
+    const t = convexTest(schema, modules);
+    await t.run(async (ctx) => {
+      await ctx.db.insert("foo", { a: undefined, b: 1 });
+      await ctx.db.insert("foo", { a: "undefined", b: 2 });
+      const query = stream(ctx.db, schema).query("foo").withIndex("ab");
+      const result = await query.paginate({ numItems: 1, cursor: null });
+      expect(result.continueCursor).toMatch('["undefined",');
+      expect(result.page.map(stripSystemFields)).toEqual([
+        { a: undefined, b: 1 },
+      ]);
+      expect(result.isDone).toBe(false);
+      const page1 = await query.paginate({
+        numItems: 1,
+        cursor: result.continueCursor,
+      });
+      expect(page1.continueCursor).toMatch('["_undefined",');
+      expect(page1.page.map(stripSystemFields)).toEqual([
+        { a: "undefined", b: 2 },
+      ]);
+    });
+  });
 });

packages/convex-helpers/server/stream.ts

@@ -1,6 +1,6 @@
 /* eslint-disable no-unexpected-multiline */
 import type { Value } from "convex/values";
-import { convexToJson, jsonToConvex } from "convex/values";
+import { convexToJson, compareValues, jsonToConvex } from "convex/values";
 import type {
   DataModelFromSchemaDefinition,
   DocumentByInfo,
@@ -21,9 +21,8 @@ import type {
   SystemDataModel,
   TableNamesInDataModel,
 } from "convex/server";
-import { compareValues } from "./compare.js";
 
-export type IndexKey = Value[];
+export type IndexKey = (Value | undefined)[];
 
 //
 // Helper functions
@@ -349,7 +348,7 @@ abstract class QueryStream<T extends GenericStreamItem>
     };
     if (opts.cursor !== null) {
       newStartKey = {
-        key: jsonToConvex(JSON.parse(opts.cursor)) as IndexKey,
+        key: deserializeCursor(opts.cursor),
         inclusive: false,
       };
     }
@@ -362,7 +361,7 @@ abstract class QueryStream<T extends GenericStreamItem>
     let maxRows: number | undefined = opts.numItems;
     if (opts.endCursor) {
       newEndKey = {
-        key: jsonToConvex(JSON.parse(opts.endCursor)) as IndexKey,
+        key: deserializeCursor(opts.endCursor),
         inclusive: true,
       };
       // If there's an endCursor, continue until we get there even if it's more
@@ -391,7 +390,7 @@ abstract class QueryStream<T extends GenericStreamItem>
         (maxRowsToRead !== undefined && indexKeys.length >= maxRowsToRead)
       ) {
         hasMore = true;
-        continueCursor = JSON.stringify(convexToJson(indexKey as Value));
+        continueCursor = serializeCursor(indexKey);
         break;
       }
     }
@@ -410,9 +409,7 @@ abstract class QueryStream<T extends GenericStreamItem>
       isDone: !hasMore,
       continueCursor,
       pageStatus,
-      splitCursor: splitCursor
-        ? JSON.stringify(convexToJson(splitCursor as Value))
-        : undefined,
+      splitCursor: splitCursor ? serializeCursor(splitCursor) : undefined,
     };
   }
   async collect() {
@@ -1851,3 +1848,41 @@ function compareKeys(key1: Key, key2: Key): number {
   // of key2.kind is valid...
   throw new Error(`Unexpected key kind: ${key1.kind as any}`);
 }
+
+function serializeCursor(key: IndexKey): string {
+  return JSON.stringify(
+    convexToJson(
+      key.map(
+        (v): Value =>
+          v === undefined
+            ? "undefined"
+            : typeof v === "string" && v.endsWith("undefined")
+              ? // in the unlikely case their string was "undefined"
+                // or "_undefined" etc, we escape it.
+                "_" + v
+              : v,
+      ),
+    ),
+  );
+}
+
+function deserializeCursor(cursor: string): IndexKey {
+  return (jsonToConvex(JSON.parse(cursor)) as Value[]).map((v) => {
+    if (typeof v === "string") {
+      if (v === "undefined") {
+        // This is a special case for the undefined value.
+        // It's not a valid value in the index, but it's a valid value in the
+        // cursor.
+        return undefined;
+      }
+      if (v.endsWith("undefined")) {
+        // in the unlikely case their string was "undefined" it was changed to
+        // "_undefined" in the serialization process.
+        // NB: if their string was "_undefined" it was changed to
+        // "__undefined" in the serialization process, and so on.
+        return v.slice(1);
+      }
+    }
+    return v;
+  });
+}