From 8a01e7a0763121f71ef2acb49bceffa4636cf7e5 Mon Sep 17 00:00:00 2001
From: Simon Backx <simon@ghost.org>
Date: Tue, 12 Dec 2023 17:37:36 +0100
Subject: [PATCH] Improved batch and recipient data generation

ref PROD-244

Instead of generating one batch with all recipients, we now will generate
one batch per 1000 members and distribute the recipients across them.
---
 .../lib/importers/EmailBatchesImporter.js     |  8 +++--
 .../lib/importers/EmailRecipientsImporter.js  | 30 ++++++++++++-------
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/ghost/data-generator/lib/importers/EmailBatchesImporter.js b/ghost/data-generator/lib/importers/EmailBatchesImporter.js
index b09da9a6f8..b75f904f78 100644
--- a/ghost/data-generator/lib/importers/EmailBatchesImporter.js
+++ b/ghost/data-generator/lib/importers/EmailBatchesImporter.js
@@ -11,10 +11,12 @@ class EmailBatchesImporter extends TableImporter {
     }
 
     async import(quantity) {
-        const emails = await this.transaction.select('id', 'created_at').from('emails');
+        const emails = await this.transaction.select('id', 'created_at', 'email_count').from('emails');
 
-        // TODO: Generate >1 batch per email
-        await this.importForEach(emails, quantity ?? emails.length);
+        // 1 batch per 1000 recipients
+        await this.importForEach(emails, quantity ?? (() => {
+            return Math.ceil(this.model.email_count / 1000);
+        }));
     }
 
     generate() {
diff --git a/ghost/data-generator/lib/importers/EmailRecipientsImporter.js b/ghost/data-generator/lib/importers/EmailRecipientsImporter.js
index b51383aa1a..865af87bdb 100644
--- a/ghost/data-generator/lib/importers/EmailRecipientsImporter.js
+++ b/ghost/data-generator/lib/importers/EmailRecipientsImporter.js
@@ -28,25 +28,31 @@ class EmailRecipientsImporter extends TableImporter {
                 'opened_count',
                 'failed_count')
             .from('emails');
+        this.emails = emails;
         this.emailBatches = await this.transaction.select('id', 'email_id', 'updated_at').from('email_batches');
         this.members = await this.transaction.select('id', 'uuid', 'email', 'name').from('members');
         this.membersSubscribeEvents = await this.transaction.select('id', 'newsletter_id', 'created_at', 'member_id').from('members_subscribe_events');
 
-        await this.importForEach(emails, quantity ? quantity / emails.length : this.members.length);
+        await this.importForEach(this.emailBatches, quantity ? quantity / emails.length : 1000);
     }
 
     setReferencedModel(model) {
-        this.model = model;
-        this.batch = this.emailBatches.find(batch => batch.email_id === model.id);
+        this.batch = model;
+        this.model = this.emails.find(email => email.id === this.batch.email_id);
+        this.batchIndex = this.emailBatches.filter(b => b.email_id === this.model.id).findIndex(batch => batch.id === this.batch.id);
+
         // Shallow clone members list so we can shuffle and modify it
         const earliestOpenTime = new Date(this.batch.updated_at);
         const latestOpenTime = new Date(this.batch.updated_at);
         latestOpenTime.setDate(latestOpenTime.getDate() + 14);
         const currentTime = new Date();
+
         this.membersList = this.membersSubscribeEvents
             .filter(entry => entry.newsletter_id === this.model.newsletter_id)
             .filter(entry => new Date(entry.created_at) < earliestOpenTime)
-            .map(memberSubscribeEvent => memberSubscribeEvent.member_id);
+            .map(memberSubscribeEvent => memberSubscribeEvent.member_id)
+            .slice(this.batchIndex * 1000, (this.batchIndex + 1) * 1000);
+
         this.events = this.membersList.length > 0 ? generateEvents({
             shape: 'ease-out',
             trend: 'negative',
@@ -54,21 +60,25 @@ class EmailRecipientsImporter extends TableImporter {
             startTime: earliestOpenTime,
             endTime: currentTime < latestOpenTime ? currentTime : latestOpenTime
         }) : [];
+
         this.emailMeta = {
-            emailCount: this.model.email_count,
             // delievered and not opened
             deliveredCount: this.model.delivered_count - this.model.opened_count,
             openedCount: this.model.opened_count,
             failedCount: this.model.failed_count
         };
+
+        let offset = this.batchIndex * 1000;
+
+        // We always first create the failures, then the opened, then the delivered, so we need to remove those from the meta so we don't generate them multiple times
+        this.emailMeta = {
+            failedCount: Math.max(0, this.emailMeta.failedCount - offset),
+            openedCount: Math.max(0, this.emailMeta.openedCount - Math.max(0, offset - this.emailMeta.failedCount)),
+            deliveredCount: Math.max(0, this.emailMeta.deliveredCount - Math.max(0, offset - this.emailMeta.failedCount - this.emailMeta.openedCount))
+        };
     }
 
     generate() {
-        if (this.emailMeta.emailCount <= 0) {
-            return;
-        }
-        this.emailMeta.emailCount -= 1;
-
         const timestamp = this.events.shift();
         if (!timestamp) {
             return;