OpenSourceArk/Ghost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved handling large amounts of email events (#16189)

Browse Source 
refs https://github.com/TryGhost/Team/issues/2486

Stop the event fetching loop as soon as we receive events that were
created later then when we started the loop. This ensures that we don't
miss events if we receive a giant batch of events that take a long time
to process.
This commit is contained in:
Simon Backx 2023-01-26 16:06:15 +01:00 committed by GitHub
parent 672d6b3f90
commit 6631071dd3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ghost/mailgun-client/lib/mailgun-client.js
View File

@ -119,13 +119,16 @@ module.exports = class MailgunClient {
debug(`fetchEvents: starting fetching first events page`);
const mailgunConfig = this.#getConfig();
let startTime = Date.now();
const startDate = new Date();
try {
let page = await mailgunInstance.events.get(mailgunConfig.domain, mailgunOptions);
metrics.metric('mailgun-get-events', {
value: Date.now() - startTime,
statusCode: 200
});
let events = (page?.items?.map(this.normalizeEvent) || []).filter(e => !!e);
// By limiting the processed events to ones created before this job started we cancel early ready for the next job run.
// Avoids chance of events being missed in long job runs due to mailgun's eventual-consistency creating events outside of our 30min sliding re-check window
let events = (page?.items?.map(this.normalizeEvent) || []).filter(e => !!e && e.timestamp <= startDate);
debug(`fetchEvents: finished fetching first page with ${events.length} events`);
let eventCount = 0;
@ -152,7 +155,8 @@ module.exports = class MailgunClient {
value: Date.now() - startTime,
statusCode: 200
});
events = (page?.items?.map(this.normalizeEvent) || []).filter(e => !!e);
// We need to cap events at the time we started fetching them (see comment above)
events = (page?.items?.map(this.normalizeEvent) || []).filter(e => !!e && e.timestamp <= startDate);
debug(`fetchEvents: finished fetching next page with ${events.length} events`);
}