Skip to content

Commit d3ced3a

Browse files
depombodepombo
authored
use manual pagination as async iterables cannot be autoretried (#77)
* use manual pagination as async iterables cannot be autoretried * update tests
1 parent d1a78a2 commit d3ced3a

File tree

6 files changed

+514
-213
lines changed

6 files changed

+514
-213
lines changed

‎packages/fastify-app/src/test/helpers/mockStripe.ts‎

Lines changed: 77 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -21,74 +21,79 @@ export const mockStripe = {
2121
subscription: null,
2222
})
2323
),
24-
listLineItems: vitest.fn(() => ({
25-
async *[Symbol.asyncIterator]() {
26-
yield {
27-
id: 'li_123',
28-
object: 'item',
29-
description: 'T-shirt',
30-
currency: 'usd',
31-
amount_discount: 0,
32-
amount_subtotal: 2198,
33-
amount_tax: 0,
34-
amount_total: 2198,
35-
price: {
36-
id: 'price_1IDQm5JDPojXS6LNM31hxKzp',
37-
object: 'price',
38-
active: true,
39-
billing_scheme: 'per_unit',
40-
created: 1683237782,
24+
listLineItems: vitest.fn(() =>
25+
Promise.resolve({
26+
object: 'list',
27+
data: [
28+
{
29+
id: 'li_123',
30+
object: 'item',
31+
description: 'T-shirt',
4132
currency: 'usd',
42-
custom_unit_amount: null,
43-
livemode: false,
44-
lookup_key: null,
45-
metadata: {},
46-
nickname: null,
47-
product: 'prod_NppuJWzzNnD5Ut',
48-
recurring: null,
49-
tax_behavior: 'unspecified',
50-
tiers_mode: null,
51-
transform_quantity: null,
52-
type: 'one_time',
53-
unit_amount: 1099,
54-
unit_amount_decimal: '1099',
33+
amount_discount: 0,
34+
amount_subtotal: 2198,
35+
amount_tax: 0,
36+
amount_total: 2198,
37+
price: {
38+
id: 'price_1IDQm5JDPojXS6LNM31hxKzp',
39+
object: 'price',
40+
active: true,
41+
billing_scheme: 'per_unit',
42+
created: 1683237782,
43+
currency: 'usd',
44+
custom_unit_amount: null,
45+
livemode: false,
46+
lookup_key: null,
47+
metadata: {},
48+
nickname: null,
49+
product: 'prod_NppuJWzzNnD5Ut',
50+
recurring: null,
51+
tax_behavior: 'unspecified',
52+
tiers_mode: null,
53+
transform_quantity: null,
54+
type: 'one_time',
55+
unit_amount: 1099,
56+
unit_amount_decimal: '1099',
57+
},
58+
quantity: 1,
5559
},
56-
quantity: 1,
57-
}
58-
yield {
59-
id: 'li_456',
60-
object: 'item',
61-
description: 'Hoodie',
62-
currency: 'usd',
63-
amount_discount: 0,
64-
amount_subtotal: 2198,
65-
amount_tax: 0,
66-
amount_total: 2198,
67-
price: {
68-
id: 'price_1IDQm5JDPojXS6LNM31hxKzp',
69-
object: 'price',
70-
active: true,
71-
billing_scheme: 'per_unit',
72-
created: 1683237782,
60+
{
61+
id: 'li_456',
62+
object: 'item',
63+
description: 'Hoodie',
7364
currency: 'usd',
74-
custom_unit_amount: null,
75-
livemode: false,
76-
lookup_key: null,
77-
metadata: {},
78-
nickname: null,
79-
product: 'prod_NppuJWzzNnD5Ut',
80-
recurring: null,
81-
tax_behavior: 'unspecified',
82-
tiers_mode: null,
83-
transform_quantity: null,
84-
type: 'one_time',
85-
unit_amount: 1099,
86-
unit_amount_decimal: '1099',
65+
amount_discount: 0,
66+
amount_subtotal: 2198,
67+
amount_tax: 0,
68+
amount_total: 2198,
69+
price: {
70+
id: 'price_1IDQm5JDPojXS6LNM31hxKzp',
71+
object: 'price',
72+
active: true,
73+
billing_scheme: 'per_unit',
74+
created: 1683237782,
75+
currency: 'usd',
76+
custom_unit_amount: null,
77+
livemode: false,
78+
lookup_key: null,
79+
metadata: {},
80+
nickname: null,
81+
product: 'prod_NppuJWzzNnD5Ut',
82+
recurring: null,
83+
tax_behavior: 'unspecified',
84+
tiers_mode: null,
85+
transform_quantity: null,
86+
type: 'one_time',
87+
unit_amount: 1099,
88+
unit_amount_decimal: '1099',
89+
},
90+
quantity: 2,
8791
},
88-
quantity: 2,
89-
}
90-
},
91-
})),
92+
],
93+
has_more: false,
94+
url: '/v1/checkout/sessions/cs_test/line_items',
95+
})
96+
),
9297
list: vitest.fn((id) => ({
9398
async *[Symbol.asyncIterator]() {
9499
yield {
@@ -278,12 +283,14 @@ export const mockStripe = {
278283
webhooks_delivered_at: 1642595537,
279284
})
280285
),
281-
listLineItems: vitest.fn(() => ({
282-
async *[Symbol.asyncIterator]() {
283-
yield { id: 'li_123' }
284-
yield { id: 'li_1234' }
285-
},
286-
})),
286+
listLineItems: vitest.fn(() =>
287+
Promise.resolve({
288+
object: 'list',
289+
data: [{ id: 'li_123' }, { id: 'li_1234' }],
290+
has_more: false,
291+
url: '/v1/invoices/in_test/lines',
292+
})
293+
),
287294
},
288295
subscriptions: {
289296
retrieve: vitest.fn((id) =>

‎packages/fastify-app/src/test/incremental-sync.test.ts‎

Lines changed: 53 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -102,16 +102,13 @@ describe('Incremental Sync', () => {
102102
]
103103

104104
// Mock Stripe API for first sync - returns all products
105-
const mockList = vitest.fn(async function* () {
106-
for (const product of allProducts) {
107-
yield product
108-
}
105+
const listSpy = vitest.spyOn(stripeSync.stripe.products, 'list').mockResolvedValue({
106+
object: 'list',
107+
data: allProducts,
108+
has_more: false,
109+
url: '/v1/products',
109110
})
110111

111-
const listSpy = vitest
112-
.spyOn(stripeSync.stripe.products, 'list')
113-
.mockReturnValue(mockList() as unknown)
114-
115112
// First sync - no cursor, should fetch all
116113
await stripeSync.syncProducts()
117114

@@ -129,16 +126,13 @@ describe('Incremental Sync', () => {
129126
expect(cursor).toBe(1705075200) // Max created from first sync
130127

131128
// Mock Stripe API for second sync - only returns new products
132-
const mockListIncremental = vitest.fn(async function* () {
133-
for (const product of newProducts) {
134-
yield product
135-
}
129+
const listSpyIncremental = vitest.spyOn(stripeSync.stripe.products, 'list').mockResolvedValue({
130+
object: 'list',
131+
data: newProducts,
132+
has_more: false,
133+
url: '/v1/products',
136134
})
137135

138-
const listSpyIncremental = vitest
139-
.spyOn(stripeSync.stripe.products, 'list')
140-
.mockReturnValue(mockListIncremental() as unknown)
141-
142136
// Second sync - should use cursor
143137
await stripeSync.syncProducts()
144138

@@ -167,14 +161,13 @@ describe('Incremental Sync', () => {
167161
name: `Product ${i}`,
168162
})) as Stripe.Product[]
169163

170-
const mockList = vitest.fn(async function* () {
171-
for (const product of products) {
172-
yield product
173-
}
164+
vitest.spyOn(stripeSync.stripe.products, 'list').mockResolvedValue({
165+
object: 'list',
166+
data: products,
167+
has_more: false,
168+
url: '/v1/products',
174169
})
175170

176-
vitest.spyOn(stripeSync.stripe.products, 'list').mockReturnValue(mockList() as unknown)
177-
178171
// Spy on updateObjectCursor (new observability method)
179172
const updateSpy = vitest.spyOn(stripeSync.postgresClient, 'updateObjectCursor')
180173

@@ -220,14 +213,13 @@ describe('Incremental Sync', () => {
220213
} as Stripe.Product,
221214
]
222215

223-
const mockList = vitest.fn(async function* () {
224-
for (const product of products) {
225-
yield product
226-
}
216+
vitest.spyOn(stripeSync.stripe.products, 'list').mockResolvedValue({
217+
object: 'list',
218+
data: products,
219+
has_more: false,
220+
url: '/v1/products',
227221
})
228222

229-
vitest.spyOn(stripeSync.stripe.products, 'list').mockReturnValue(mockList() as unknown)
230-
231223
await stripeSync.syncProducts()
232224

233225
const initialCursor = await getCursor('products')
@@ -267,13 +259,12 @@ describe('Incremental Sync', () => {
267259
} as Stripe.Product,
268260
]
269261

270-
const mockSetupList = vitest.fn(async function* () {
271-
for (const product of setupProducts) {
272-
yield product
273-
}
262+
vitest.spyOn(stripeSync.stripe.products, 'list').mockResolvedValue({
263+
object: 'list',
264+
data: setupProducts,
265+
has_more: false,
266+
url: '/v1/products',
274267
})
275-
276-
vitest.spyOn(stripeSync.stripe.products, 'list').mockReturnValue(mockSetupList() as unknown)
277268
await stripeSync.syncProducts()
278269

279270
// Verify cursor was set
@@ -292,16 +283,13 @@ describe('Incremental Sync', () => {
292283
} as Stripe.Product,
293284
]
294285

295-
const mockList = vitest.fn(async function* () {
296-
for (const product of products) {
297-
yield product
298-
}
286+
const listSpy = vitest.spyOn(stripeSync.stripe.products, 'list').mockResolvedValue({
287+
object: 'list',
288+
data: products,
289+
has_more: false,
290+
url: '/v1/products',
299291
})
300292

301-
const listSpy = vitest
302-
.spyOn(stripeSync.stripe.products, 'list')
303-
.mockReturnValue(mockList() as unknown)
304-
305293
// Call with explicit filter (earlier than cursor)
306294
await stripeSync.syncProducts({
307295
created: { gte: 1672531200 },
@@ -325,23 +313,36 @@ describe('Incremental Sync', () => {
325313
]
326314

327315
let callCount = 0
328-
const mockList = vitest.fn(async function* () {
329-
for (const product of products) {
330-
yield product
331-
}
332-
// Simulate error after yielding products
316+
let hasThrown = false
317+
vitest.spyOn(stripeSync.stripe.products, 'list').mockImplementation(async () => {
333318
callCount++
334-
if (callCount === 1) {
319+
if (callCount === 1 && !hasThrown) {
320+
// First page succeeds with products
321+
return {
322+
object: 'list',
323+
data: products,
324+
has_more: true, // Indicate there's more (but second fetch will fail)
325+
url: '/v1/products',
326+
}
327+
} else if (callCount === 2 && !hasThrown) {
328+
// Second fetch fails (only on first sync)
329+
hasThrown = true
335330
throw new Error('Simulated sync error')
331+
} else {
332+
// After first sync, just return products without more pages
333+
return {
334+
object: 'list',
335+
data: callCount === 1 ? products : [],
336+
has_more: false,
337+
url: '/v1/products',
338+
}
336339
}
337340
})
338341

339-
vitest.spyOn(stripeSync.stripe.products, 'list').mockReturnValue(mockList() as unknown)
340-
341342
// First attempt should fail but save checkpoint
342343
await expect(stripeSync.syncProducts()).rejects.toThrow('Simulated sync error')
343344

344-
// Cursor should be saved up to checkpoint
345+
// Cursor should be saved up to checkpoint (first product was processed successfully)
345346
const cursor = await getCursor('products')
346347
expect(cursor).toBe(1704902400)
347348

@@ -368,7 +369,7 @@ describe('Incremental Sync', () => {
368369
expect(objStatus.rows[0].status).toBe('error')
369370
expect(objStatus.rows[0].error_message).toContain('Simulated sync error')
370371

371-
// Second attempt should succeed
372+
// Second attempt should succeed - reset callCount
372373
callCount = 0
373374
await stripeSync.syncProducts()
374375

‎packages/sync-engine/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "stripe-experiment-sync",
3-
"version": "1.0.13",
3+
"version": "1.0.14",
44
"private": false,
55
"description": "Stripe Sync Engine to sync Stripe data to Postgres",
66
"type": "module",

0 commit comments

Comments
 (0)