diff --git a/convex/pagination.test.ts b/convex/pagination.test.ts index 4e5126c..d97a542 100644 --- a/convex/pagination.test.ts +++ b/convex/pagination.test.ts @@ -55,3 +55,158 @@ test("paginate", async () => { expect(page3).toMatchObject([]); expect(isDone3).toEqual(true); }); + +test("paginate with deletes", async () => { + const t = convexTest(schema); + await t.run(async (ctx) => { + await ctx.db.insert("messages", { author: "sarah", body: "hello1" }); + await ctx.db.insert("messages", { author: "michal", body: "boo" }); + await ctx.db.insert("messages", { author: "sarah", body: "hello2" }); + await ctx.db.insert("messages", { author: "sarah", body: "hello3" }); + await ctx.db.insert("messages", { author: "sarah", body: "hello4" }); + await ctx.db.insert("messages", { author: "michal", body: "boing" }); + await ctx.db.insert("messages", { author: "sarah", body: "hello5" }); + }); + const { continueCursor, isDone, page } = await t.query(api.pagination.list, { + author: "sarah", + paginationOptions: { + cursor: null, + numItems: 2, + }, + }); + expect(page).toMatchObject([ + { author: "sarah", body: "hello1" }, + { author: "sarah", body: "hello2" }, + ]); + expect(isDone).toEqual(false); + await t.run(async (ctx) => { + await ctx.db.delete(page[1]._id); + }); + const { isDone: isDone2, page: page2 } = await t.query(api.pagination.list, { + author: "sarah", + paginationOptions: { + cursor: continueCursor, + numItems: 4, + }, + }); + expect(page2).toMatchObject([ + { author: "sarah", body: "hello3" }, + { author: "sarah", body: "hello4" }, + { author: "sarah", body: "hello5" }, + ]); + expect(isDone2).toEqual(true); +}); + +test("paginate with deletes and indexes", async () => { + const t = convexTest(schema); + await t.run(async (ctx) => { + await ctx.db.insert("messages", { author: "sarah", body: "hello1" }); + await ctx.db.insert("messages", { author: "jordan", body: "hello1" }); + await ctx.db.insert("messages", { author: "jordan", body: "hello2" }); + await ctx.db.insert("messages", { author: "jordan", body: "hello3" }); + await ctx.db.insert("messages", { author: "jordan", body: "hello4" }); + }); + const { continueCursor, isDone, page } = await t.query( + api.pagination.listWithIndex, + { + author: "jordan", + paginationOptions: { + cursor: null, + numItems: 1, + }, + }, + ); + expect(page).toMatchObject([{ author: "jordan", body: "hello1" }]); + expect(isDone).toEqual(false); + await t.run(async (ctx) => { + await ctx.db.delete(page[0]._id); + }); + const { + isDone: isDone2, + page: page2, + continueCursor: continueCursor2, + } = await t.query(api.pagination.listWithIndex, { + author: "jordan", + paginationOptions: { + cursor: continueCursor, + numItems: 2, + }, + }); + expect(page2).toMatchObject([ + { author: "jordan", body: "hello2" }, + { author: "jordan", body: "hello3" }, + ]); + expect(isDone2).toEqual(false); + await t.run(async (ctx) => { + await ctx.db.delete(page2[1]._id); + }); + const { isDone: isDone3, page: page3 } = await t.query( + api.pagination.listWithIndex, + { + author: "jordan", + paginationOptions: { + cursor: continueCursor2, + numItems: 1, + }, + }, + ); + expect(page3).toMatchObject([{ author: "jordan", body: "hello4" }]); + expect(isDone3).toEqual(true); +}); + +test("paginate with deletes and composite index", async () => { + const t = convexTest(schema); + await t.run(async (ctx) => { + await ctx.db.insert("messages", { author: "sarah", body: "hello1" }); + await ctx.db.insert("messages", { author: "jordan", body: "hello2" }); + await ctx.db.insert("messages", { author: "jordan", body: "hello4" }); + await ctx.db.insert("messages", { author: "jordan", body: "hello3" }); + await ctx.db.insert("messages", { author: "jordan", body: "hello1" }); + }); + const { continueCursor, isDone, page } = await t.query( + api.pagination.listWithCompositeIndex, + { + author: "jordan", + paginationOptions: { + cursor: null, + numItems: 1, + }, + }, + ); + expect(page).toMatchObject([{ author: "jordan", body: "hello1" }]); + expect(isDone).toEqual(false); + await t.run(async (ctx) => { + await ctx.db.delete(page[0]._id); + }); + const { + isDone: isDone2, + page: page2, + continueCursor: continueCursor2, + } = await t.query(api.pagination.listWithCompositeIndex, { + author: "jordan", + paginationOptions: { + cursor: continueCursor, + numItems: 2, + }, + }); + expect(page2).toMatchObject([ + { author: "jordan", body: "hello2" }, + { author: "jordan", body: "hello3" }, + ]); + expect(isDone2).toEqual(false); + await t.run(async (ctx) => { + await ctx.db.delete(page2[1]._id); + }); + const { isDone: isDone3, page: page3 } = await t.query( + api.pagination.listWithCompositeIndex, + { + author: "jordan", + paginationOptions: { + cursor: continueCursor2, + numItems: 1, + }, + }, + ); + expect(page3).toMatchObject([{ author: "jordan", body: "hello4" }]); + expect(isDone3).toEqual(true); +}); diff --git a/convex/pagination.ts b/convex/pagination.ts index e142431..730c378 100644 --- a/convex/pagination.ts +++ b/convex/pagination.ts @@ -16,3 +16,29 @@ export const list = query({ .paginate(args.paginationOptions); }, }); + +export const listWithIndex = query({ + args: { + author: v.string(), + paginationOptions: paginationOptsValidator, + }, + handler: async (ctx, args) => { + return await ctx.db + .query("messages") + .withIndex("author", (q) => q.eq("author", args.author)) + .paginate(args.paginationOptions); + }, +}); + +export const listWithCompositeIndex = query({ + args: { + author: v.string(), + paginationOptions: paginationOptsValidator, + }, + handler: async (ctx, args) => { + return await ctx.db + .query("messages") + .withIndex("author_body", (q) => q.eq("author", args.author)) + .paginate(args.paginationOptions); + }, +}); diff --git a/convex/schema.ts b/convex/schema.ts index 636f68f..d9eeb75 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -9,6 +9,7 @@ export default defineSchema({ score: v.optional(v.number()), }) .index("author", ["author"]) + .index("author_body", ["author", "body"]) .searchIndex("body", { searchField: "body", filterFields: ["author"], diff --git a/index.ts b/index.ts index 8fbf86f..7a678b1 100644 --- a/index.ts +++ b/index.ts @@ -372,6 +372,35 @@ class DatabaseFake { let isInPage = cursor === null; let isDone = false; let continueCursor = null; + + // Parse cursor to get the index key values + let cursorIndexKeys: any[] | null = null; + if (cursor && cursor !== "_end_cursor") { + try { + // Parse as JSON array (index keys) + const parsed = JSON.parse(cursor); + if (Array.isArray(parsed)) { + cursorIndexKeys = parsed; + } else { + throw new Error("Invalid cursor format: expected JSON array"); + } + } catch (e) { + if (e instanceof Error && e.message.includes("expected JSON array")) { + throw e; + } + throw new Error("Invalid cursor format: must be a JSON array or null"); + } + } + + // If cursor is "_end_cursor", we're already at the end + if (cursor === "_end_cursor") { + return { + page: [], + isDone: true, + continueCursor: "_end_cursor", + }; + } + for (;;) { const { value, done } = this.queryNext(queryId); if (done) { @@ -381,16 +410,43 @@ class DatabaseFake { continueCursor = "_end_cursor"; break; } + + // First determine if we should start including documents in the page + if (cursor === null) { + isInPage = true; + } else if (cursor === "_end_cursor") { + // Already at end, should not happen + break; + } else if (cursorIndexKeys) { + // Compare based on index key values - start from the next doc after cursor + if (!isInPage) { + const shouldStart = this._shouldStartFromDocument( + query, + value!, + cursorIndexKeys, + ); + if (shouldStart) { + isInPage = true; + } + } + } + + // Then, if we're in the page, add the document if (isInPage) { page.push(value); if (page.length >= pageSize) { - continueCursor = value!._id; + // Generate cursor based on index key values + continueCursor = this._generateIndexCursor(query, value!); + + // Check if there are more documents after this one + const { done: nextDone } = this.queryNext(queryId); + if (nextDone) { + isDone = true; + continueCursor = "_end_cursor"; + } break; } } - if (value!._id === cursor) { - isInPage = true; - } } return { page, @@ -399,6 +455,157 @@ class DatabaseFake { }; } + private _generateIndexCursor( + query: SerializedQuery, + document: GenericDocument, + ): string { + const source = query.source; + + if (source.type === "IndexRange") { + const [tableName, indexName] = source.indexName.split("."); + let fields: string[]; + + if (indexName === "by_creation_time") { + fields = ["_creationTime", "_id"]; + } else if (indexName === "by_id") { + fields = ["_id"]; + } else { + const indexes = this._schema?.tables.get(tableName)?.indexes; + const index = indexes?.find( + ({ indexDescriptor }: { indexDescriptor: string }) => + indexDescriptor === indexName, + ); + if (index) { + fields = index.fields.concat(["_creationTime", "_id"]); + } else { + throw new Error( + `Unknown index "${indexName}" for table "${tableName}"`, + ); + } + } + + // Extract the index key values from the document + const indexValues = fields.map((field) => + evaluateFieldPath(field, document), + ); + return JSON.stringify(indexValues); + } + + if (source.type === "FullTableScan") { + // For FullTableScan, use the natural sort order which is by _creationTime, _id + const fields = ["_creationTime", "_id"]; + const indexValues = fields.map((field) => + evaluateFieldPath(field, document), + ); + return JSON.stringify(indexValues); + } + + // For other query types, use creation time and ID + const fields = ["_creationTime", "_id"]; + const indexValues = fields.map((field) => + evaluateFieldPath(field, document), + ); + return JSON.stringify(indexValues); + } + + private _shouldStartFromDocument( + query: SerializedQuery, + document: GenericDocument, + cursorIndexKeys: any[], + ): boolean { + const source = query.source; + + if (source.type === "IndexRange") { + const [tableName, indexName] = source.indexName.split("."); + let fields: string[]; + + if (indexName === "by_creation_time") { + fields = ["_creationTime", "_id"]; + } else if (indexName === "by_id") { + fields = ["_id"]; + } else { + const indexes = this._schema?.tables.get(tableName)?.indexes; + const index = indexes?.find( + ({ indexDescriptor }: { indexDescriptor: string }) => + indexDescriptor === indexName, + ); + if (index) { + fields = index.fields.concat(["_creationTime", "_id"]); + } else { + throw new Error( + `Unknown index "${indexName}" for table "${tableName}"`, + ); + } + } + + // Extract the index key values from the document + const documentIndexValues = fields.map((field) => + evaluateFieldPath(field, document), + ); + + // Compare the index values according to the sort order + const order = source.order ?? "asc"; + const comparison = this._compareIndexValues( + documentIndexValues, + cursorIndexKeys, + order, + ); + + return comparison > 0; + } + + if (source.type === "FullTableScan") { + // For FullTableScan, use the natural sort order which is by _creationTime, _id + const fields = ["_creationTime", "_id"]; + const documentIndexValues = fields.map((field) => + evaluateFieldPath(field, document), + ); + + // Compare the index values according to the sort order + const order = source.order ?? "asc"; + const comparison = this._compareIndexValues( + documentIndexValues, + cursorIndexKeys, + order, + ); + + return comparison > 0; + } + + // For other query types, use creation time and ID comparison + const fields = ["_creationTime", "_id"]; + const documentIndexValues = fields.map((field) => + evaluateFieldPath(field, document), + ); + + const order = "asc"; + const comparison = this._compareIndexValues( + documentIndexValues, + cursorIndexKeys, + order, + ); + + return comparison > 0; + } + + private _compareIndexValues( + a: any[], + b: any[], + order: "asc" | "desc", + ): number { + const orderMultiplier = order === "asc" ? 1 : -1; + + for (let i = 0; i < Math.min(a.length, b.length); i++) { + const comparison = compareValues(a[i], b[i]); + if (comparison !== 0) { + return comparison * orderMultiplier; + } + } + + // If all compared values are equal, shorter array comes first + return (a.length - b.length) * orderMultiplier; + } + private _iterateDocs( tableName: string, callback: (doc: GenericDocument) => void,