From 279f1779657e7efe48078ccf0ba28427e1669c29 Mon Sep 17 00:00:00 2001 From: Ian Macartney Date: Thu, 5 Jun 2025 14:55:12 -0700 Subject: [PATCH 1/7] serde for undefined --- packages/convex-helpers/server/stream.ts | 38 ++++++++++++++++++------ 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/packages/convex-helpers/server/stream.ts b/packages/convex-helpers/server/stream.ts index ef9cdc0a..6ca2622d 100644 --- a/packages/convex-helpers/server/stream.ts +++ b/packages/convex-helpers/server/stream.ts @@ -1,6 +1,6 @@ /* eslint-disable no-unexpected-multiline */ import type { Value } from "convex/values"; -import { convexToJson, jsonToConvex } from "convex/values"; +import { convexToJson, compareValues, jsonToConvex } from "convex/values"; import type { DataModelFromSchemaDefinition, DocumentByInfo, @@ -21,9 +21,8 @@ import type { SystemDataModel, TableNamesInDataModel, } from "convex/server"; -import { compareValues } from "./compare.js"; -export type IndexKey = Value[]; +export type IndexKey = (Value | undefined)[]; // // Helper functions @@ -349,7 +348,7 @@ abstract class QueryStream }; if (opts.cursor !== null) { newStartKey = { - key: jsonToConvex(JSON.parse(opts.cursor)) as IndexKey, + key: deserializeCursor(opts.cursor), inclusive: false, }; } @@ -362,7 +361,7 @@ abstract class QueryStream let maxRows: number | undefined = opts.numItems; if (opts.endCursor) { newEndKey = { - key: jsonToConvex(JSON.parse(opts.endCursor)) as IndexKey, + key: deserializeCursor(opts.endCursor), inclusive: true, }; // If there's an endCursor, continue until we get there even if it's more @@ -391,7 +390,7 @@ abstract class QueryStream (maxRowsToRead !== undefined && indexKeys.length >= maxRowsToRead) ) { hasMore = true; - continueCursor = JSON.stringify(convexToJson(indexKey as Value)); + continueCursor = serializeCursor(indexKey); break; } } @@ -410,9 +409,7 @@ abstract class QueryStream isDone: !hasMore, continueCursor, pageStatus, - splitCursor: splitCursor - ? JSON.stringify(convexToJson(splitCursor as Value)) - : undefined, + splitCursor: splitCursor ? serializeCursor(splitCursor) : undefined, }; } async collect() { @@ -1851,3 +1848,26 @@ function compareKeys(key1: Key, key2: Key): number { // of key2.kind is valid... throw new Error(`Unexpected key kind: ${key1.kind as any}`); } + +function serializeCursor(key: IndexKey): string { + return JSON.stringify( + convexToJson( + key.map((v): Value => (v === undefined ? { $undefined: true } : v)), + ), + ); +} + +function deserializeCursor(cursor: string): IndexKey { + return (jsonToConvex(JSON.parse(cursor)) as Value[]).map((v) => { + if (typeof v === "object" && !Array.isArray(v) && v !== null) { + const entries = Object.entries(v); + if (entries.length === 1 && entries[0]![0] === "$undefined") { + // This is a special case for the undefined value. + // It's not a valid value in the index, but it's a valid value in the + // cursor. + return undefined; + } + } + return v; + }); +} From d7e317c81a75749dd779873dd82aac3817eb03d2 Mon Sep 17 00:00:00 2001 From: Ian Macartney Date: Thu, 5 Jun 2025 15:31:02 -0700 Subject: [PATCH 2/7] can't use convex strat - $ is reserved --- packages/convex-helpers/server/stream.ts | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/convex-helpers/server/stream.ts b/packages/convex-helpers/server/stream.ts index 6ca2622d..6522c01d 100644 --- a/packages/convex-helpers/server/stream.ts +++ b/packages/convex-helpers/server/stream.ts @@ -1852,21 +1852,35 @@ function compareKeys(key1: Key, key2: Key): number { function serializeCursor(key: IndexKey): string { return JSON.stringify( convexToJson( - key.map((v): Value => (v === undefined ? { $undefined: true } : v)), + key.map( + (v): Value => + v === undefined + ? "$_" + : typeof v === "string" && v.endsWith("$_") + ? // in the unlikely case their string was "$_" or "$$_" etc. + // we need to escape it. Always add a $ so "$$_" becomes "$$$_" + "$" + v + : v, + ), ), ); } function deserializeCursor(cursor: string): IndexKey { return (jsonToConvex(JSON.parse(cursor)) as Value[]).map((v) => { - if (typeof v === "object" && !Array.isArray(v) && v !== null) { - const entries = Object.entries(v); - if (entries.length === 1 && entries[0]![0] === "$undefined") { + if (typeof v === "string") { + if (v === "$_") { // This is a special case for the undefined value. // It's not a valid value in the index, but it's a valid value in the // cursor. return undefined; } + if (v.endsWith("$_")) { + // in the unlikely case their string was "$_" it was changed to "$$_" + // in the serialization process. If it was "$$_", it was changed to + // "$$$_" and so on. + return v.slice(1); + } } return v; }); From 0e3a60cdb245a2d2aad987ccf29247a8eff5f929 Mon Sep 17 00:00:00 2001 From: Ian Macartney Date: Thu, 5 Jun 2025 15:31:24 -0700 Subject: [PATCH 3/7] still doesn't work b/c convex .gt/lt doesn't do undefined --- packages/convex-helpers/server/stream.test.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/packages/convex-helpers/server/stream.test.ts b/packages/convex-helpers/server/stream.test.ts index ddf3e54d..bca8c8e3 100644 --- a/packages/convex-helpers/server/stream.test.ts +++ b/packages/convex-helpers/server/stream.test.ts @@ -748,4 +748,52 @@ describe("stream", () => { expect(page1.page.map(stripSystemFields)).toEqual([{ a: 1, b: 3, c: 5 }]); }); }); + test("undefined cursor serialization roundtrips", async () => { + const schema = defineSchema({ + foo: defineTable({ + a: v.optional(v.number()), + b: v.number(), + }).index("ab", ["a", "b"]), + }); + }); + + test("undefined cursor serialization roundtrips", async () => { + const schema = defineSchema({ + foo: defineTable({ + a: v.optional(v.number()), + b: v.number(), + }).index("ab", ["a", "b"]), + }); + const t = convexTest(schema, modules); + await t.run(async (ctx) => { + await ctx.db.insert("foo", { a: 1, b: 2 }); + await ctx.db.insert("foo", { a: undefined, b: 3 }); + await ctx.db.insert("foo", { a: 2, b: 4 }); + await ctx.db.insert("foo", { a: undefined, b: 5 }); + const query = stream(ctx.db, schema).query("foo").withIndex("ab"); + const result = await query.paginate({ numItems: 1, cursor: null }); + expect(result.continueCursor).toMatch('["$_",'); + expect(result.page.map(stripSystemFields)).toEqual([ + { a: undefined, b: 3 }, + ]); + expect(result.isDone).toBe(false); + const page1 = await query.paginate({ + numItems: 2, + cursor: result.continueCursor, + }); + expect(page1.page.map(stripSystemFields)).toEqual([ + { b: 5 }, + { a: 1, b: 2 }, + ]); + expect(page1.isDone).toBe(false); + const page2 = await query.paginate({ + numItems: 1, + cursor: page1.continueCursor, + }); + expect(page2.page.map(stripSystemFields)).toEqual([ + { a: undefined, b: 5 }, + ]); + expect(page2.isDone).toBe(true); + }); + }); }); From 242ffbaccbb103d8ade9a1ff77f3805f0942b221 Mon Sep 17 00:00:00 2001 From: Ian Macartney Date: Mon, 7 Jul 2025 16:34:14 -0700 Subject: [PATCH 4/7] pagination type --- packages/convex-helpers/server/pagination.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/convex-helpers/server/pagination.ts b/packages/convex-helpers/server/pagination.ts index 25ccb3bc..b1f49f03 100644 --- a/packages/convex-helpers/server/pagination.ts +++ b/packages/convex-helpers/server/pagination.ts @@ -16,7 +16,7 @@ import { streamIndexRange, } from "./stream.js"; -export type IndexKey = Value[]; +export type IndexKey = (Value | undefined)[]; export type PageRequest< DataModel extends GenericDataModel, From 78fdf7579033814d8150fa4c56bb5d3af7bce382 Mon Sep 17 00:00:00 2001 From: Ian Macartney Date: Thu, 24 Jul 2025 17:22:00 -0700 Subject: [PATCH 5/7] fix tests --- packages/convex-helpers/server/stream.test.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/convex-helpers/server/stream.test.ts b/packages/convex-helpers/server/stream.test.ts index bca8c8e3..fdb6fb92 100644 --- a/packages/convex-helpers/server/stream.test.ts +++ b/packages/convex-helpers/server/stream.test.ts @@ -748,15 +748,6 @@ describe("stream", () => { expect(page1.page.map(stripSystemFields)).toEqual([{ a: 1, b: 3, c: 5 }]); }); }); - test("undefined cursor serialization roundtrips", async () => { - const schema = defineSchema({ - foo: defineTable({ - a: v.optional(v.number()), - b: v.number(), - }).index("ab", ["a", "b"]), - }); - }); - test("undefined cursor serialization roundtrips", async () => { const schema = defineSchema({ foo: defineTable({ @@ -787,12 +778,10 @@ describe("stream", () => { ]); expect(page1.isDone).toBe(false); const page2 = await query.paginate({ - numItems: 1, + numItems: 2, cursor: page1.continueCursor, }); - expect(page2.page.map(stripSystemFields)).toEqual([ - { a: undefined, b: 5 }, - ]); + expect(page2.page.map(stripSystemFields)).toEqual([{ a: 2, b: 4 }]); expect(page2.isDone).toBe(true); }); }); From 23bf3e075573928bcbaca5bae4c80429ce68aff4 Mon Sep 17 00:00:00 2001 From: Ian Macartney Date: Thu, 24 Jul 2025 19:07:48 -0700 Subject: [PATCH 6/7] change to more readable --- packages/convex-helpers/server/stream.test.ts | 2 +- packages/convex-helpers/server/stream.ts | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/convex-helpers/server/stream.test.ts b/packages/convex-helpers/server/stream.test.ts index fdb6fb92..6bca1359 100644 --- a/packages/convex-helpers/server/stream.test.ts +++ b/packages/convex-helpers/server/stream.test.ts @@ -763,7 +763,7 @@ describe("stream", () => { await ctx.db.insert("foo", { a: undefined, b: 5 }); const query = stream(ctx.db, schema).query("foo").withIndex("ab"); const result = await query.paginate({ numItems: 1, cursor: null }); - expect(result.continueCursor).toMatch('["$_",'); + expect(result.continueCursor).toMatch('["undefined",'); expect(result.page.map(stripSystemFields)).toEqual([ { a: undefined, b: 3 }, ]); diff --git a/packages/convex-helpers/server/stream.ts b/packages/convex-helpers/server/stream.ts index 6522c01d..a169d0f8 100644 --- a/packages/convex-helpers/server/stream.ts +++ b/packages/convex-helpers/server/stream.ts @@ -1855,11 +1855,11 @@ function serializeCursor(key: IndexKey): string { key.map( (v): Value => v === undefined - ? "$_" - : typeof v === "string" && v.endsWith("$_") - ? // in the unlikely case their string was "$_" or "$$_" etc. - // we need to escape it. Always add a $ so "$$_" becomes "$$$_" - "$" + v + ? "undefined" + : typeof v === "string" && v.endsWith("undefined") + ? // in the unlikely case their string was "undefined" + // or "_undefined" etc, we escape it. + "_" + v : v, ), ), @@ -1869,16 +1869,17 @@ function serializeCursor(key: IndexKey): string { function deserializeCursor(cursor: string): IndexKey { return (jsonToConvex(JSON.parse(cursor)) as Value[]).map((v) => { if (typeof v === "string") { - if (v === "$_") { + if (v === "undefined") { // This is a special case for the undefined value. // It's not a valid value in the index, but it's a valid value in the // cursor. return undefined; } - if (v.endsWith("$_")) { - // in the unlikely case their string was "$_" it was changed to "$$_" - // in the serialization process. If it was "$$_", it was changed to - // "$$$_" and so on. + if (v.endsWith("undefined")) { + // in the unlikely case their string was "undefined" it was changed to + // "_undefined" in the serialization process. + // NB: if their string was "_undefined" it was changed to + // "__undefined" in the serialization process, and so on. return v.slice(1); } } From 67d342abe7620303f412b353f9e4f7ad086a38c9 Mon Sep 17 00:00:00 2001 From: Ian Macartney Date: Thu, 24 Jul 2025 19:09:51 -0700 Subject: [PATCH 7/7] test escaping --- packages/convex-helpers/server/stream.test.ts | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/convex-helpers/server/stream.test.ts b/packages/convex-helpers/server/stream.test.ts index 6bca1359..609fea28 100644 --- a/packages/convex-helpers/server/stream.test.ts +++ b/packages/convex-helpers/server/stream.test.ts @@ -785,4 +785,32 @@ describe("stream", () => { expect(page2.isDone).toBe(true); }); }); + test("literal undefined string works", async () => { + const schema = defineSchema({ + foo: defineTable({ + a: v.optional(v.string()), + b: v.number(), + }).index("ab", ["a", "b"]), + }); + const t = convexTest(schema, modules); + await t.run(async (ctx) => { + await ctx.db.insert("foo", { a: undefined, b: 1 }); + await ctx.db.insert("foo", { a: "undefined", b: 2 }); + const query = stream(ctx.db, schema).query("foo").withIndex("ab"); + const result = await query.paginate({ numItems: 1, cursor: null }); + expect(result.continueCursor).toMatch('["undefined",'); + expect(result.page.map(stripSystemFields)).toEqual([ + { a: undefined, b: 1 }, + ]); + expect(result.isDone).toBe(false); + const page1 = await query.paginate({ + numItems: 1, + cursor: result.continueCursor, + }); + expect(page1.continueCursor).toMatch('["_undefined",'); + expect(page1.page.map(stripSystemFields)).toEqual([ + { a: "undefined", b: 2 }, + ]); + }); + }); });