Skip to content

serde for undefined #635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 25, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/convex-helpers/server/pagination.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
streamIndexRange,
} from "./stream.js";

export type IndexKey = Value[];
export type IndexKey = (Value | undefined)[];

export type PageRequest<
DataModel extends GenericDataModel,
Expand Down
65 changes: 65 additions & 0 deletions packages/convex-helpers/server/stream.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -748,4 +748,69 @@ describe("stream", () => {
expect(page1.page.map(stripSystemFields)).toEqual([{ a: 1, b: 3, c: 5 }]);
});
});
test("undefined cursor serialization roundtrips", async () => {
const schema = defineSchema({
foo: defineTable({
a: v.optional(v.number()),
b: v.number(),
}).index("ab", ["a", "b"]),
});
const t = convexTest(schema, modules);
await t.run(async (ctx) => {
await ctx.db.insert("foo", { a: 1, b: 2 });
await ctx.db.insert("foo", { a: undefined, b: 3 });
await ctx.db.insert("foo", { a: 2, b: 4 });
await ctx.db.insert("foo", { a: undefined, b: 5 });
const query = stream(ctx.db, schema).query("foo").withIndex("ab");
const result = await query.paginate({ numItems: 1, cursor: null });
expect(result.continueCursor).toMatch('["undefined",');
expect(result.page.map(stripSystemFields)).toEqual([
{ a: undefined, b: 3 },
]);
expect(result.isDone).toBe(false);
const page1 = await query.paginate({
numItems: 2,
cursor: result.continueCursor,
});
expect(page1.page.map(stripSystemFields)).toEqual([
{ b: 5 },
{ a: 1, b: 2 },
]);
expect(page1.isDone).toBe(false);
const page2 = await query.paginate({
numItems: 2,
cursor: page1.continueCursor,
});
expect(page2.page.map(stripSystemFields)).toEqual([{ a: 2, b: 4 }]);
expect(page2.isDone).toBe(true);
});
});
test("literal undefined string works", async () => {
const schema = defineSchema({
foo: defineTable({
a: v.optional(v.string()),
b: v.number(),
}).index("ab", ["a", "b"]),
});
const t = convexTest(schema, modules);
await t.run(async (ctx) => {
await ctx.db.insert("foo", { a: undefined, b: 1 });
await ctx.db.insert("foo", { a: "undefined", b: 2 });
const query = stream(ctx.db, schema).query("foo").withIndex("ab");
const result = await query.paginate({ numItems: 1, cursor: null });
expect(result.continueCursor).toMatch('["undefined",');
expect(result.page.map(stripSystemFields)).toEqual([
{ a: undefined, b: 1 },
]);
expect(result.isDone).toBe(false);
const page1 = await query.paginate({
numItems: 1,
cursor: result.continueCursor,
});
expect(page1.continueCursor).toMatch('["_undefined",');
expect(page1.page.map(stripSystemFields)).toEqual([
{ a: "undefined", b: 2 },
]);
});
});
});
53 changes: 44 additions & 9 deletions packages/convex-helpers/server/stream.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/* eslint-disable no-unexpected-multiline */
import type { Value } from "convex/values";
import { convexToJson, jsonToConvex } from "convex/values";
import { convexToJson, compareValues, jsonToConvex } from "convex/values";
import type {
DataModelFromSchemaDefinition,
DocumentByInfo,
Expand All @@ -21,9 +21,8 @@ import type {
SystemDataModel,
TableNamesInDataModel,
} from "convex/server";
import { compareValues } from "./compare.js";

export type IndexKey = Value[];
export type IndexKey = (Value | undefined)[];

//
// Helper functions
Expand Down Expand Up @@ -349,7 +348,7 @@ abstract class QueryStream<T extends GenericStreamItem>
};
if (opts.cursor !== null) {
newStartKey = {
key: jsonToConvex(JSON.parse(opts.cursor)) as IndexKey,
key: deserializeCursor(opts.cursor),
inclusive: false,
};
}
Expand All @@ -362,7 +361,7 @@ abstract class QueryStream<T extends GenericStreamItem>
let maxRows: number | undefined = opts.numItems;
if (opts.endCursor) {
newEndKey = {
key: jsonToConvex(JSON.parse(opts.endCursor)) as IndexKey,
key: deserializeCursor(opts.endCursor),
inclusive: true,
};
// If there's an endCursor, continue until we get there even if it's more
Expand Down Expand Up @@ -391,7 +390,7 @@ abstract class QueryStream<T extends GenericStreamItem>
(maxRowsToRead !== undefined && indexKeys.length >= maxRowsToRead)
) {
hasMore = true;
continueCursor = JSON.stringify(convexToJson(indexKey as Value));
continueCursor = serializeCursor(indexKey);
break;
}
}
Expand All @@ -410,9 +409,7 @@ abstract class QueryStream<T extends GenericStreamItem>
isDone: !hasMore,
continueCursor,
pageStatus,
splitCursor: splitCursor
? JSON.stringify(convexToJson(splitCursor as Value))
: undefined,
splitCursor: splitCursor ? serializeCursor(splitCursor) : undefined,
};
}
async collect() {
Expand Down Expand Up @@ -1851,3 +1848,41 @@ function compareKeys(key1: Key, key2: Key): number {
// of key2.kind is valid...
throw new Error(`Unexpected key kind: ${key1.kind as any}`);
}

function serializeCursor(key: IndexKey): string {
return JSON.stringify(
convexToJson(
key.map(
(v): Value =>
v === undefined
? "undefined"
: typeof v === "string" && v.endsWith("undefined")
? // in the unlikely case their string was "undefined"
// or "_undefined" etc, we escape it.
"_" + v
: v,
),
),
);
}

function deserializeCursor(cursor: string): IndexKey {
return (jsonToConvex(JSON.parse(cursor)) as Value[]).map((v) => {
if (typeof v === "string") {
if (v === "undefined") {
// This is a special case for the undefined value.
// It's not a valid value in the index, but it's a valid value in the
// cursor.
return undefined;
}
if (v.endsWith("undefined")) {
// in the unlikely case their string was "undefined" it was changed to
// "_undefined" in the serialization process.
// NB: if their string was "_undefined" it was changed to
// "__undefined" in the serialization process, and so on.
return v.slice(1);
}
}
return v;
});
}