diff --git a/packages/convex-helpers/server/pagination.ts b/packages/convex-helpers/server/pagination.ts index 25ccb3bc..b1f49f03 100644 --- a/packages/convex-helpers/server/pagination.ts +++ b/packages/convex-helpers/server/pagination.ts @@ -16,7 +16,7 @@ import { streamIndexRange, } from "./stream.js"; -export type IndexKey = Value[]; +export type IndexKey = (Value | undefined)[]; export type PageRequest< DataModel extends GenericDataModel, diff --git a/packages/convex-helpers/server/stream.test.ts b/packages/convex-helpers/server/stream.test.ts index ddf3e54d..609fea28 100644 --- a/packages/convex-helpers/server/stream.test.ts +++ b/packages/convex-helpers/server/stream.test.ts @@ -748,4 +748,69 @@ describe("stream", () => { expect(page1.page.map(stripSystemFields)).toEqual([{ a: 1, b: 3, c: 5 }]); }); }); + test("undefined cursor serialization roundtrips", async () => { + const schema = defineSchema({ + foo: defineTable({ + a: v.optional(v.number()), + b: v.number(), + }).index("ab", ["a", "b"]), + }); + const t = convexTest(schema, modules); + await t.run(async (ctx) => { + await ctx.db.insert("foo", { a: 1, b: 2 }); + await ctx.db.insert("foo", { a: undefined, b: 3 }); + await ctx.db.insert("foo", { a: 2, b: 4 }); + await ctx.db.insert("foo", { a: undefined, b: 5 }); + const query = stream(ctx.db, schema).query("foo").withIndex("ab"); + const result = await query.paginate({ numItems: 1, cursor: null }); + expect(result.continueCursor).toMatch('["undefined",'); + expect(result.page.map(stripSystemFields)).toEqual([ + { a: undefined, b: 3 }, + ]); + expect(result.isDone).toBe(false); + const page1 = await query.paginate({ + numItems: 2, + cursor: result.continueCursor, + }); + expect(page1.page.map(stripSystemFields)).toEqual([ + { b: 5 }, + { a: 1, b: 2 }, + ]); + expect(page1.isDone).toBe(false); + const page2 = await query.paginate({ + numItems: 2, + cursor: page1.continueCursor, + }); + expect(page2.page.map(stripSystemFields)).toEqual([{ a: 2, b: 4 }]); + expect(page2.isDone).toBe(true); + }); + }); + test("literal undefined string works", async () => { + const schema = defineSchema({ + foo: defineTable({ + a: v.optional(v.string()), + b: v.number(), + }).index("ab", ["a", "b"]), + }); + const t = convexTest(schema, modules); + await t.run(async (ctx) => { + await ctx.db.insert("foo", { a: undefined, b: 1 }); + await ctx.db.insert("foo", { a: "undefined", b: 2 }); + const query = stream(ctx.db, schema).query("foo").withIndex("ab"); + const result = await query.paginate({ numItems: 1, cursor: null }); + expect(result.continueCursor).toMatch('["undefined",'); + expect(result.page.map(stripSystemFields)).toEqual([ + { a: undefined, b: 1 }, + ]); + expect(result.isDone).toBe(false); + const page1 = await query.paginate({ + numItems: 1, + cursor: result.continueCursor, + }); + expect(page1.continueCursor).toMatch('["_undefined",'); + expect(page1.page.map(stripSystemFields)).toEqual([ + { a: "undefined", b: 2 }, + ]); + }); + }); }); diff --git a/packages/convex-helpers/server/stream.ts b/packages/convex-helpers/server/stream.ts index ef9cdc0a..a169d0f8 100644 --- a/packages/convex-helpers/server/stream.ts +++ b/packages/convex-helpers/server/stream.ts @@ -1,6 +1,6 @@ /* eslint-disable no-unexpected-multiline */ import type { Value } from "convex/values"; -import { convexToJson, jsonToConvex } from "convex/values"; +import { convexToJson, compareValues, jsonToConvex } from "convex/values"; import type { DataModelFromSchemaDefinition, DocumentByInfo, @@ -21,9 +21,8 @@ import type { SystemDataModel, TableNamesInDataModel, } from "convex/server"; -import { compareValues } from "./compare.js"; -export type IndexKey = Value[]; +export type IndexKey = (Value | undefined)[]; // // Helper functions @@ -349,7 +348,7 @@ abstract class QueryStream }; if (opts.cursor !== null) { newStartKey = { - key: jsonToConvex(JSON.parse(opts.cursor)) as IndexKey, + key: deserializeCursor(opts.cursor), inclusive: false, }; } @@ -362,7 +361,7 @@ abstract class QueryStream let maxRows: number | undefined = opts.numItems; if (opts.endCursor) { newEndKey = { - key: jsonToConvex(JSON.parse(opts.endCursor)) as IndexKey, + key: deserializeCursor(opts.endCursor), inclusive: true, }; // If there's an endCursor, continue until we get there even if it's more @@ -391,7 +390,7 @@ abstract class QueryStream (maxRowsToRead !== undefined && indexKeys.length >= maxRowsToRead) ) { hasMore = true; - continueCursor = JSON.stringify(convexToJson(indexKey as Value)); + continueCursor = serializeCursor(indexKey); break; } } @@ -410,9 +409,7 @@ abstract class QueryStream isDone: !hasMore, continueCursor, pageStatus, - splitCursor: splitCursor - ? JSON.stringify(convexToJson(splitCursor as Value)) - : undefined, + splitCursor: splitCursor ? serializeCursor(splitCursor) : undefined, }; } async collect() { @@ -1851,3 +1848,41 @@ function compareKeys(key1: Key, key2: Key): number { // of key2.kind is valid... throw new Error(`Unexpected key kind: ${key1.kind as any}`); } + +function serializeCursor(key: IndexKey): string { + return JSON.stringify( + convexToJson( + key.map( + (v): Value => + v === undefined + ? "undefined" + : typeof v === "string" && v.endsWith("undefined") + ? // in the unlikely case their string was "undefined" + // or "_undefined" etc, we escape it. + "_" + v + : v, + ), + ), + ); +} + +function deserializeCursor(cursor: string): IndexKey { + return (jsonToConvex(JSON.parse(cursor)) as Value[]).map((v) => { + if (typeof v === "string") { + if (v === "undefined") { + // This is a special case for the undefined value. + // It's not a valid value in the index, but it's a valid value in the + // cursor. + return undefined; + } + if (v.endsWith("undefined")) { + // in the unlikely case their string was "undefined" it was changed to + // "_undefined" in the serialization process. + // NB: if their string was "_undefined" it was changed to + // "__undefined" in the serialization process, and so on. + return v.slice(1); + } + } + return v; + }); +}