Add endpoint to record mail events (#16990)

refs https://github.com/TryGhost/Team/issues/3319

.github/dev.js

@@ -77,6 +77,16 @@ if (DASH_DASH_ARGS.includes('collections') || DASH_DASH_ARGS.includes('all')) {
     });
 }
 
+if (DASH_DASH_ARGS.includes('mail-events') || DASH_DASH_ARGS.includes('all')) {
+    commands.push({
+        name: 'collections',
+        command: 'yarn dev',
+        cwd: path.resolve(__dirname, '../ghost/mail-events'),
+        prefixColor: 'pink',
+        env: {}
+    });
+}
+
 if (DASH_DASH_ARGS.includes('admin-x') || DASH_DASH_ARGS.includes('adminx') || DASH_DASH_ARGS.includes('adminX') || DASH_DASH_ARGS.includes('all')) {
     commands.push({
         name: 'adminX',

ghost/admin/app/services/feature.js

@@ -73,6 +73,7 @@ export default class FeatureService extends Service {
     @feature('signupForm') signupForm;
     @feature('collections') collections;
     @feature('adminXSettings') adminXSettings;
+    @feature('mailEvents') mailEvents;
 
     _user = null;
 

ghost/admin/app/templates/settings/labs.hbs

@@ -310,6 +310,20 @@
                         </div>
                     </div>
                 </div>
+
+                <div class="gh-expandable-block">
+                    <div class="gh-expandable-header">
+                        <div>
+                            <h4 class="gh-expandable-title">Mail Events</h4>
+                            <p class="gh-expandable-description">
+                                Enables processing of mail events
+                            </p>
+                        </div>
+                        <div class="for-switch">
+                            <GhFeatureFlag @flag="mailEvents" />
+                        </div>
+                    </div>
+                </div>
             </div>
         </div>
         {{/if}}

ghost/core/core/boot.js

@@ -326,6 +326,7 @@ async function initServices({config}) {
     const slackNotifications = require('./server/services/slack-notifications');
     const mediaInliner = require('./server/services/media-inliner');
     const collections = require('./server/services/collections');
+    const mailEvents = require('./server/services/mail-events');
 
     const urlUtils = require('./shared/url-utils');
 
@@ -363,7 +364,8 @@ async function initServices({config}) {
         emailSuppressionList.init(),
         slackNotifications.init(),
         collections.init(),
-        mediaInliner.init()
+        mediaInliner.init(),
+        mailEvents.init()
     ]);
     debug('End: Services');
 

ghost/core/core/server/api/endpoints/index.js

@@ -201,6 +201,10 @@ module.exports = {
         return apiFramework.pipeline(require('./links'), localUtils);
     },
 
+    get mailEvents() {
+        return apiFramework.pipeline(require('./mail-events'), localUtils);
+    },
+
     /**
      * Content API Controllers
      *

ghost/core/core/server/api/endpoints/mail-events.js

@@ -0,0 +1,14 @@
+const mailEvents = require('../../services/mail-events');
+
+module.exports = {
+    docName: 'mail_events',
+    add: {
+        headers: {
+            cacheInvalidate: false
+        },
+        permissions: false,
+        async query(frame) {
+            return mailEvents.service.processPayload(frame.data);
+        }
+    }
+};

ghost/core/core/server/api/endpoints/utils/serializers/output/index.js

@@ -139,5 +139,9 @@ module.exports = {
 
     get links() {
         return require('./links');
+    },
+
+    get mail_events() {
+        return require('./mail-events');
     }
 };

ghost/core/core/server/api/endpoints/utils/serializers/output/mail-events.js

@@ -0,0 +1,9 @@
+const debug = require('@tryghost/debug')('api:endpoints:utils:serializers:output:mail-events');
+
+module.exports = {
+    add(response, apiConfig, frame) {
+        debug('add');
+
+        frame.response = {};
+    }
+};

ghost/core/core/server/api/endpoints/utils/validators/input/index.js

@@ -75,5 +75,9 @@ module.exports = {
 
     get snippets() {
         return require('./snippets');
+    },
+
+    get mail_events() {
+        return require('./mail-events');
     }
 };

ghost/core/core/server/api/endpoints/utils/validators/input/mail-events.js

@@ -0,0 +1,7 @@
+const mailEvents = require('../../../../../services/mail-events');
+
+module.exports = {
+    add(apiConfig, frame) {
+        mailEvents.service.validatePayload(frame.data);
+    }
+};

ghost/core/core/server/data/migrations/versions/5.53/2023-06-13-12-24-add-temp-mail-events-table.js

@@ -0,0 +1,9 @@
+const {addTable} = require('../../utils');
+
+module.exports = addTable('temp_mail_events', {
+    id: {type: 'string', maxlength: 100, nullable: false, primary: true},
+    type: {type: 'string', maxlength: 50, nullable: false},
+    message_id: {type: 'string', maxlength: 150, nullable: false},
+    recipient: {type: 'string', maxlength: 191, nullable: false},
+    occurred_at: {type: 'dateTime', nullable: false}
+});

ghost/core/core/server/data/schema/schema.js

@@ -1021,5 +1021,12 @@ module.exports = {
         currency: {type: 'string', maxlength: 24, nullable: true},
         created_at: {type: 'dateTime', nullable: false},
         email_sent_at: {type: 'dateTime', nullable: true}
+    },
+    temp_mail_events: {
+        id: {type: 'string', maxlength: 100, nullable: false, primary: true},
+        type: {type: 'string', maxlength: 50, nullable: false},
+        message_id: {type: 'string', maxlength: 150, nullable: false},
+        recipient: {type: 'string', maxlength: 191, nullable: false},
+        occurred_at: {type: 'dateTime', nullable: false}
     }
 };

ghost/core/core/server/models/mail-event.js

@@ -0,0 +1,12 @@
+const ghostBookshelf = require('./base');
+
+const MailEvent = ghostBookshelf.Model.extend({
+    tableName: 'temp_mail_events',
+    defaults() {
+        return {};
+    }
+}, {});
+
+module.exports = {
+    MailEvent: ghostBookshelf.model('MailEvent', MailEvent)
+};

ghost/core/core/server/services/mail-events/BookshelfMailEventRepository.js

@@ -0,0 +1,40 @@
+/**
+ * @typedef {import('@tryghost/mail-events').MailEventRepository} MailEventRepository
+ * @typedef {import('@tryghost/mail-events').MailEvent} MailEvent
+ */
+
+/**
+ * @typedef {object} MailEventModel
+ * @property {function} add
+ */
+
+/**
+ * @implements MailEventRepository
+ */
+module.exports = class BookshelfMailEventRepository {
+    /**
+     * @type {MailEventModel}
+     */
+    #MailEventModel;
+
+    /**
+     * @param {object} MailEventModel
+     */
+    constructor(MailEventModel) {
+        this.#MailEventModel = MailEventModel;
+    }
+
+    /**
+     * @param {MailEvent} mailEvent
+     * @returns {Promise<void>}
+     */
+    async save(mailEvent) {
+        await this.#MailEventModel.add({
+            id: mailEvent.id,
+            type: mailEvent.type,
+            message_id: mailEvent.messageId,
+            recipient: mailEvent.recipient,
+            occurred_at: new Date(mailEvent.timestampMs)
+        });
+    }
+};

ghost/core/core/server/services/mail-events/index.js

@@ -0,0 +1,21 @@
+const {MailEventService} = require('@tryghost/mail-events');
+const MailEventRepository = require('./BookshelfMailEventRepository.js');
+
+class MailEventsServiceWrapper {
+    /**
+     * @type {MailEventService}
+     */
+    service;
+
+    async init() {
+        const config = require('../../../shared/config');
+        const labs = require('../../../shared/labs');
+        const models = require('../../models');
+
+        const repository = new MailEventRepository(models.MailEvent);
+
+        this.service = new MailEventService(repository, config, labs);
+    }
+}
+
+module.exports = new MailEventsServiceWrapper();

ghost/core/core/server/web/api/endpoints/admin/routes.js

@@ -17,6 +17,7 @@ module.exports = function apiRoutes() {
 
     // ## Public
     router.get('/site', mw.publicAdminApi, http(api.site.read));
+    router.post('/mail_events', mw.publicAdminApi, http(api.mailEvents.add));
 
     // ## Collections
     router.get('/collections', mw.authAdminApi, labs.enabledMiddleware('collections'), http(api.collections.browse));

ghost/core/core/shared/labs.js

@@ -39,7 +39,8 @@ const ALPHA_FEATURES = [
     'emailCustomization',
     'signupCard',
     'collections',
-    'adminXSettings'
+    'adminXSettings',
+    'mailEvents'
 ];
 
 module.exports.GA_KEYS = [...GA_FEATURES];

ghost/core/package.json

@@ -113,6 +113,7 @@
     "@tryghost/link-tracking": "0.0.0",
     "@tryghost/logging": "2.4.4",
     "@tryghost/magic-link": "0.0.0",
+    "@tryghost/mail-events": "0.0.0",
     "@tryghost/mailgun-client": "0.0.0",
     "@tryghost/member-attribution": "0.0.0",
     "@tryghost/member-events": "0.0.0",

ghost/core/test/e2e-api/admin/__snapshots__/mail-events.test.js.snap

@@ -0,0 +1,16 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Mail Events API Can add a mail event 1: [body] 1`] = `Object {}`;
+
+exports[`Mail Events API Can add a mail event 2: [headers] 1`] = `
+Object {
+  "access-control-allow-origin": "http://127.0.0.1:2369",
+  "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
+  "content-length": "2",
+  "content-type": "application/json; charset=utf-8",
+  "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
+  "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
+  "vary": "Accept-Version, Origin, Accept-Encoding",
+  "x-powered-by": "Express",
+}
+`;

ghost/core/test/e2e-api/admin/mail-events.test.js

@@ -0,0 +1,59 @@
+const assert = require('assert/strict');
+const {MailEventService} = require('@tryghost/mail-events');
+const {agentProvider, matchers, mockManager} = require('../../utils/e2e-framework');
+const configUtils = require('../../utils/configUtils');
+const models = require('../../../core/server/models');
+
+const {anyContentVersion, anyEtag} = matchers;
+
+describe('Mail Events API', function () {
+    let agent;
+
+    before(async function () {
+        agent = await agentProvider.getAdminAPIAgent();
+
+        mockManager.mockLabsEnabled(MailEventService.LABS_KEY);
+    });
+
+    it('Can add a mail event', async function () {
+        configUtils.set(MailEventService.CONFIG_KEY_PAYLOAD_SIGNING_KEY, 'foobarbaz');
+
+        const payload = {
+            // The signature is based on the previous config value as well as the
+            // "mail_events" array below. If you change any of these values, you will need to
+            // update the signature otherwise the request will fail
+            signature: '51ab01400f9a78669733d85fcf344401f5da648f8c95707bc06da0456cb99fbc',
+            mail_events: [
+                {
+                    id: 'Ase7i2zsRYeDXztHGENqRA',
+                    timestamp: 1521243339.873676,
+                    event: 'opened',
+                    message: {
+                        headers: {
+                            'message-id': '20130503182626.18666.16540@sandboxb052085d6a7b401bb117d3a432d1d659.mailgun.org'
+                        }
+                    },
+                    recipient: 'alice@example.com'
+                }
+            ]
+        };
+
+        await agent
+            .post('/mail_events/', {
+                headers: {
+                    'content-type': 'application/json'
+                }
+            })
+            .body(payload)
+            .expectStatus(200)
+            .matchBodySnapshot({})
+            .matchHeaderSnapshot({
+                'content-version': anyContentVersion,
+                etag: anyEtag
+            });
+
+        const storedMailEvent = await models.MailEvent.findOne({id: 'Ase7i2zsRYeDXztHGENqRA'});
+
+        assert.ok(storedMailEvent, 'Expected mail event was not found in the database');
+    });
+});

ghost/core/test/integration/exporter/exporter.test.js

@@ -89,7 +89,8 @@ describe('Exporter', function () {
                 'tokens',
                 'users',
                 'webhooks',
-                'milestones'
+                'milestones',
+                'temp_mail_events'
             ];
 
             should.exist(exportData);
@@ -117,7 +118,8 @@ describe('Exporter', function () {
                 'members_email_change_events',
                 'members_status_events',
                 'members_paid_subscription_events',
-                'members_subscribe_events'
+                'members_subscribe_events',
+                'temp_mail_events'
             ];
 
             excludedTables.forEach((tableName) => {

ghost/core/test/unit/server/data/exporter/index.test.js

@@ -214,7 +214,7 @@ describe('Exporter', function () {
             const nonSchemaTables = ['migrations', 'migrations_lock'];
             const requiredTables = schemaTables.concat(nonSchemaTables);
             // NOTE: You should not add tables to this list unless they are temporary
-            const ignoredTables = ['temp_member_analytic_events'];
+            const ignoredTables = ['temp_member_analytic_events', 'temp_mail_events'];
 
             const expectedTables = requiredTables.filter(table => !ignoredTables.includes(table)).sort();
             const actualTables = BACKUP_TABLES.concat(TABLES_ALLOWLIST).sort();

ghost/core/test/unit/server/data/schema/integrity.test.js

@@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route
  */
 describe('DB version integrity', function () {
     // Only these variables should need updating
-    const currentSchemaHash = '2445c734ffb514d11b56e74591bcde4e';
+    const currentSchemaHash = 'b174080d5df4de9b0f4bc9b786396217';
     const currentFixturesHash = '93c3b3cb8bca34a733634e74ee514172';
     const currentSettingsHash = '4f23a583335dcb4cb3fae553122ea200';
     const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';

ghost/core/test/unit/server/services/mail-events/BookshelfMailEventRepository.test.js

@@ -0,0 +1,27 @@
+const assert = require('assert/strict');
+const sinon = require('sinon');
+const {MailEvent} = require('@tryghost/mail-events');
+const BookshelfMailEventRepository = require('../../../../../core/server/services/mail-events/BookshelfMailEventRepository');
+
+describe('BookshelfMailEventRepository', function () {
+    describe('save', function () {
+        it('should store a mail event', async function () {
+            const modelStub = {
+                add: sinon.stub().resolves()
+            };
+            const event = new MailEvent('abc123', 'opened', '987def', 'foo@bar.baz', Date.now());
+            const repository = new BookshelfMailEventRepository(modelStub);
+
+            await repository.save(event);
+
+            assert.ok(modelStub.add.calledOnce);
+            assert.ok(modelStub.add.calledWith({
+                id: event.id,
+                type: event.type,
+                message_id: event.messageId,
+                recipient: event.recipient,
+                occurred_at: new Date(event.timestampMs)
+            }));
+        });
+    });
+});

ghost/mail-events/.eslintrc.js

@@ -0,0 +1,13 @@
+module.exports = {
+    parser: '@typescript-eslint/parser',
+    plugins: ['ghost', '@typescript-eslint'],
+    extends: [
+        'plugin:ghost/node'
+    ],
+    rules: {
+        'no-unused-vars': 'off',
+        '@typescript-eslint/no-unused-vars': ['error'],
+        'no-shadow': 'off',
+        '@typescript-eslint/no-shadow': ['error']
+    }
+};

ghost/mail-events/README.md

@@ -0,0 +1,17 @@
+# Mail Events
+
+Processes mail events
+
+## Develop
+
+This is a monorepo package.
+
+Follow the instructions for the top-level repo.
+
+1. `git clone` this repo & `cd` into it as usual
+2. Run `yarn` to install top-level dependencies
+
+## Test
+
+- `yarn lint` run just eslint
+- `yarn test` run lint and tests

ghost/mail-events/package.json

@@ -0,0 +1,44 @@
+{
+    "name": "@tryghost/mail-events",
+    "version": "0.0.0",
+    "repository": "https://github.com/TryGhost/Ghost/tree/main/packages/mail-events",
+    "author": "Ghost Foundation",
+    "private": true,
+    "main": "build/index.js",
+    "types": "build/index.d.ts",
+    "scripts": {
+        "dev": "tsc --watch --preserveWatchOutput --sourceMap",
+        "build": "tsc",
+        "prepare": "tsc",
+        "test:unit": "NODE_ENV=testing c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura mocha -r ts-node/register './test/**/*.test.ts'",
+        "test": "yarn test:types && yarn test:unit",
+        "test:types": "tsc --noEmit",
+        "lint:code": "eslint src/ --ext .ts --cache",
+        "lint": "yarn lint:code && yarn lint:test",
+        "lint:test": "eslint -c test/.eslintrc.js test/ --ext .ts --cache"
+    },
+    "files": [
+        "build"
+    ],
+    "devDependencies": {
+        "c8": "7.13.0",
+        "mocha": "10.2.0",
+        "sinon": "15.0.4",
+        "ts-node": "10.9.1",
+        "typescript": "5.1.3"
+    },
+    "dependencies": {
+        "@tryghost/errors": "^1.2.25",
+        "@tryghost/in-memory-repository": "0.0.0",
+        "@tryghost/tpl": "^0.1.25"
+    },
+    "c8": {
+        "exclude": [
+            "src/index.ts",
+            "src/MailEventRepository.ts",
+            "src/InMemoryMailEventRepository.ts",
+            "src/**/*.d.ts",
+            "test/**/*.ts"
+        ]
+    }
+}

ghost/mail-events/src/InMemoryMailEventRepository.ts

@@ -0,0 +1,8 @@
+import {InMemoryRepository} from '@tryghost/in-memory-repository';
+import {MailEvent} from './MailEvent';
+
+export class InMemoryMailEventRepository extends InMemoryRepository<string, MailEvent> {
+    protected toPrimitive(): object {
+        return {};
+    }
+}

ghost/mail-events/src/MailEvent.ts

@@ -0,0 +1,10 @@
+export class MailEvent {
+    constructor(
+        readonly id: string,
+        readonly type: string,
+        readonly messageId: string,
+        readonly recipient: string,
+        readonly timestampMs: number,
+        readonly deleted: boolean = false
+    ) {}
+}

ghost/mail-events/src/MailEventRepository.ts

@@ -0,0 +1,5 @@
+import {MailEvent} from './MailEvent';
+
+export interface MailEventRepository {
+    save(event: MailEvent): Promise<void>;
+}

ghost/mail-events/src/MailEventService.ts

@@ -0,0 +1,168 @@
+const crypto = require('crypto');
+const errors = require('@tryghost/errors');
+const tpl = require('@tryghost/tpl');
+
+import {MailEvent} from './MailEvent';
+import {MailEventRepository} from './MailEventRepository';
+
+/**
+ * @see https://documentation.mailgun.com/en/latest/user_manual.html#events-1
+ */
+enum EventType {
+    CLICKED = 'clicked',
+    COMPLAINED = 'complained',
+    DELIVERED = 'delivered',
+    FAILED = 'failed',
+    OPENED = 'opened',
+    UNSUBSCRIBED = 'unsubscribed'
+}
+
+interface PayloadEvent {
+    id: string;
+    timestamp: number; // Unix timestamp in seconds
+    event: string;
+    message: {
+        headers: {
+            'message-id': string;
+        }
+    },
+    recipient: string;
+}
+
+interface Payload {
+    signature: string;
+    mail_events: PayloadEvent[];
+}
+
+interface Labs {
+    isSet(key: string): boolean;
+}
+
+interface Config {
+    get(key: string): any;
+}
+
+const VALIDATION_MESSAGES = {
+    signingKeyNotConfigured: 'payload signing key is not configured',
+    payloadSignatureMissing: 'Payload is missing "signature"',
+    payloadSignatureInvalid: '"signature" is invalid',
+    payloadEventsMissing: 'Payload is missing "mail_events"',
+    payloadEventsInvalid: '"mail_events" is not an array',
+    payloadEventKeyMissing: 'Event [{idx}] is missing "{key}"'
+};
+
+export class MailEventService {
+    static readonly LABS_KEY = 'mailEvents';
+    static readonly CONFIG_KEY_PAYLOAD_SIGNING_KEY = 'hostSettings:mailEventsPayloadSigningKey';
+
+    constructor(
+        private mailEventRepository: MailEventRepository,
+        private config: Config,
+        private labs: Labs
+    ) {}
+
+    async processPayload(payload: Payload) {
+        if (this.labs.isSet(MailEventService.LABS_KEY) === false) {
+            throw new errors.NotFoundError();
+        }
+
+        const payloadSigningKey = this.config.get(MailEventService.CONFIG_KEY_PAYLOAD_SIGNING_KEY);
+
+        // Verify that the service is configured correctly - We expect a string
+        // for the payload signing key but as a safeguard we check the type here
+        // to prevent any unexpected behaviour
+        if (typeof payloadSigningKey !== 'string') {
+            throw new errors.InternalServerError({
+                message: tpl(VALIDATION_MESSAGES.signingKeyNotConfigured)
+            });
+        }
+
+        // Verify the payload
+        this.verifyPayload(payload, payloadSigningKey);
+
+        // Store known events
+        const eventTypes = new Set<string>(Object.values(EventType) as string[]);
+
+        for (const payloadEvent of payload.mail_events) {
+            if (eventTypes.has(payloadEvent.event) === false) {
+                continue;
+            }
+
+            try {
+                await this.mailEventRepository.save(
+                    new MailEvent(
+                        payloadEvent.id,
+                        payloadEvent.event,
+                        payloadEvent.message.headers['message-id'],
+                        payloadEvent.recipient,
+                        payloadEvent.timestamp * 1000
+                    )
+                );
+            } catch (err) {
+                throw new errors.InternalServerError({
+                    message: 'Event could not be stored',
+                    err: err
+                });
+            }
+        }
+    }
+
+    validatePayload(payload: Payload) {
+        if (payload.signature === undefined) {
+            throw new errors.ValidationError({
+                message: tpl(VALIDATION_MESSAGES.payloadSignatureMissing)
+            });
+        }
+
+        if (typeof payload.signature !== 'string') {
+            throw new errors.ValidationError({
+                message: tpl(VALIDATION_MESSAGES.payloadSignatureInvalid)
+            });
+        }
+
+        if (payload.mail_events === undefined) {
+            throw new errors.ValidationError({
+                message: tpl(VALIDATION_MESSAGES.payloadEventsMissing)
+            });
+        }
+
+        if (Array.isArray(payload.mail_events) === false) {
+            throw new errors.ValidationError({
+                message: tpl(VALIDATION_MESSAGES.payloadEventsInvalid)
+            });
+        }
+
+        const expectedKeys: (keyof PayloadEvent)[] = ['id', 'timestamp', 'event', 'message', 'recipient'];
+
+        payload.mail_events.forEach((payloadEvent, idx) => {
+            expectedKeys.forEach((key) => {
+                if (payloadEvent[key] === undefined) {
+                    throw new errors.ValidationError({
+                        message: tpl(VALIDATION_MESSAGES.payloadEventKeyMissing, {idx, key})
+                    });
+                }
+
+                if (key === 'message' && payloadEvent.message.headers?.['message-id'] === undefined) {
+                    throw new errors.ValidationError({
+                        message: tpl(VALIDATION_MESSAGES.payloadEventKeyMissing, {idx, key: 'message.headers.message-id'})
+                    });
+                }
+            });
+        });
+    }
+
+    private verifyPayload(payload: Payload, payloadSigningKey: string) {
+        const data = JSON.stringify(payload.mail_events);
+
+        const signature = crypto
+            .createHmac('sha256', payloadSigningKey)
+            .update(data)
+            .digest('hex');
+
+        if (signature !== payload.signature) {
+            throw new errors.UnauthorizedError({
+                message: tpl(VALIDATION_MESSAGES.payloadSignatureInvalid)
+            });
+        }
+    }
+}

ghost/mail-events/src/index.ts

@@ -0,0 +1,3 @@
+export * from './MailEvent';
+export * from './MailEventRepository';
+export * from './MailEventService';

ghost/mail-events/test/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+    parser: '@typescript-eslint/parser',
+    plugins: ['ghost'],
+    extends: [
+        'plugin:ghost/test'
+    ]
+};

ghost/mail-events/test/MailEventService.test.ts

@@ -0,0 +1,366 @@
+import assert from 'assert';
+import sinon from 'sinon';
+
+import {MailEvent} from '../src/MailEvent';
+import {InMemoryMailEventRepository as MailEventRepository} from '../src/InMemoryMailEventRepository';
+import {MailEventService} from '../src/MailEventService';
+
+const makePayloadEvent = (
+    type: string,
+    timestamp = Date.now()
+) => ({
+    id: 'event-id',
+    timestamp: timestamp / 1000,
+    event: type,
+    message: {
+        headers: {
+            'message-id': 'message-id'
+        }
+    },
+    recipient: 'message-recipient'
+});
+
+const PAYLOAD_SIGNING_KEY = 'abc123';
+
+describe('MailEventService', function () {
+    let repository: sinon.SinonStubbedInstance<MailEventRepository>;
+    let config: sinon.SinonStubbedInstance<any>;
+    let labs: sinon.SinonStubbedInstance<any>;
+    let service: MailEventService;
+
+    beforeEach(function () {
+        repository = sinon.createStubInstance(MailEventRepository);
+        labs = {
+            isSet: sinon.stub()
+                .withArgs(MailEventService.LABS_KEY)
+                .returns(true)
+        };
+        config = {
+            get: sinon.stub()
+                .withArgs(MailEventService.CONFIG_KEY_PAYLOAD_SIGNING_KEY)
+                .returns(PAYLOAD_SIGNING_KEY)
+        };
+        service = new MailEventService(repository, config, labs);
+    });
+
+    describe('processPayload', function () {
+        it('should reject if labs flag is false', async function () {
+            labs.isSet.withArgs(MailEventService.LABS_KEY).returns(false);
+
+            await assert.rejects(
+                service.processPayload({} as any),
+                {
+                    name: 'NotFoundError',
+                    message: 'Resource could not be found.'
+                }
+            );
+        });
+
+        it('should reject if payload signing key is invalid', async function () {
+            config.get.withArgs(MailEventService.CONFIG_KEY_PAYLOAD_SIGNING_KEY).returns(undefined);
+
+            await assert.rejects(
+                service.processPayload({} as any),
+                {
+                    name: 'InternalServerError',
+                    message: 'payload signing key is not configured'
+                }
+            );
+        });
+
+        it('should reject if payload verification fails', async function () {
+            await assert.rejects(
+                service.processPayload({
+                    signature: 'foobarbaz',
+                    mail_events: [
+                        makePayloadEvent('opened')
+                    ]
+                } as any),
+                {
+                    name: 'UnauthorizedError',
+                    message: '"signature" is invalid'
+                }
+            );
+        });
+
+        it('should store a single event', async function () {
+            const payloadEvent = makePayloadEvent('opened');
+
+            // Ensure a fixed timestamp is used so that we know the signature up front
+            payloadEvent.timestamp = 1686665992511 / 1000;
+
+            const payload = {
+                signature: '9f2567330688b82759600fad93c93b3e8f571d397c33688a8620400af20b79b3',
+                mail_events: [
+                    payloadEvent
+                ]
+            };
+
+            await service.processPayload(payload);
+
+            const storedEvent = repository.save.getCall(0).args[0];
+
+            assert.ok(storedEvent instanceof MailEvent);
+            assert.equal(storedEvent.id, payloadEvent.id);
+        });
+
+        it('should store multiple events', async function () {
+            const events = [
+                makePayloadEvent('opened'),
+                makePayloadEvent('opened')
+            ];
+
+            // Ensure fixed timestamps are used so that we know the signature up front
+            events[0].timestamp = 1686665992511 / 1000;
+            events[1].timestamp = 1686665992512 / 1000;
+
+            const payload = {
+                signature: '02959cc9731ee575b66969a508f545d19c5968b42a03fa398ce9c93d8e7df0a5',
+                mail_events: events
+            };
+
+            await service.processPayload(payload);
+
+            assert.ok(repository.save.calledTwice);
+        });
+
+        it('should ignore unknown events', async function () {
+            const events = [
+                makePayloadEvent('unknown-event'),
+                makePayloadEvent('opened')
+            ];
+
+            // Ensure fixed timestamps are used so that we know the signature up front
+            events[0].timestamp = 1686665992511 / 1000;
+            events[1].timestamp = 1686665992512 / 1000;
+
+            const payload = {
+                signature: 'd6de350faa9ec56d739ec7ffd5cb4230f90f583df05fe59a6c1a41afac7048df',
+                mail_events: events
+            };
+
+            await service.processPayload(payload);
+
+            assert.ok(repository.save.calledOnce);
+            assert.equal(repository.save.getCall(0).args[0].type, 'opened');
+        });
+
+        it('should ensure event timestamps are converted to ms', async function () {
+            const payloadEvent = makePayloadEvent('opened');
+
+            // Ensure a fixed timestamp is used so that we know the signature up front
+            payloadEvent.timestamp = 1686665992511 / 1000;
+
+            const payload = {
+                signature: '9f2567330688b82759600fad93c93b3e8f571d397c33688a8620400af20b79b3',
+                mail_events: [
+                    payloadEvent
+                ]
+            };
+
+            await service.processPayload(payload);
+
+            assert.ok(repository.save.calledOnce);
+
+            const storedEvent = repository.save.getCall(0).args[0];
+
+            assert.equal(storedEvent.timestampMs, payloadEvent.timestamp * 1000);
+        });
+
+        it('should reject if an event can not be stored', async function () {
+            const payloadEvent = makePayloadEvent('opened');
+
+            // Ensure a fixed timestamp is used so that we know the signature up front
+            payloadEvent.timestamp = 1686665992511 / 1000;
+
+            const payload = {
+                signature: '9f2567330688b82759600fad93c93b3e8f571d397c33688a8620400af20b79b3',
+                mail_events: [
+                    payloadEvent
+                ]
+            };
+
+            repository.save.rejects(new Error('foobarbaz'));
+
+            await assert.rejects(
+                service.processPayload(payload),
+                {
+                    name: 'InternalServerError',
+                    message: 'Event could not be stored'
+                }
+            );
+        });
+    });
+
+    describe('validatePayload', function () {
+        it('should validate that the payload contains a signature', function () {
+            assert.throws(
+                () => service.validatePayload({} as any),
+                {
+                    name: 'ValidationError',
+                    message: 'Payload is missing "signature"'
+                }
+            );
+        });
+
+        it('should validate that the payload contains a valid signature', function () {
+            assert.throws(() => {
+                service.validatePayload({
+                    signature: {}
+                } as any);
+            }, {
+                name: 'ValidationError',
+                message: '"signature" is invalid'
+            });
+        });
+
+        it('should validate that the payload contains events', function () {
+            assert.throws(() => {
+                service.validatePayload({
+                    signature: 'foobarbaz'
+                } as any);
+            }, {
+                name: 'ValidationError',
+                message: 'Payload is missing "mail_events"'
+            });
+        });
+
+        it('should validate that the payload contains valid events', function () {
+            assert.throws(() => {
+                service.validatePayload({
+                    signature: 'foobarbaz',
+                    mail_events: {}
+                } as any);
+            }, {
+                name: 'ValidationError',
+                message: '"mail_events" is not an array'
+            });
+        });
+
+        it('should validate that events in the payload have an id', function () {
+            const malformedPayloadEvent = makePayloadEvent('opened') as any;
+            delete malformedPayloadEvent.id;
+
+            const payload = {
+                signature: 'foobarbaz',
+                mail_events: [
+                    makePayloadEvent('opened'),
+                    malformedPayloadEvent
+                ]
+            };
+
+            assert.throws(
+                () => service.validatePayload(payload),
+                {
+                    name: 'ValidationError',
+                    message: 'Event [1] is missing "id"'
+                }
+            );
+        });
+
+        it('should validate that events in the payload have an timestamp', function () {
+            const malformedPayloadEvent = makePayloadEvent('opened') as any;
+            delete malformedPayloadEvent.timestamp;
+
+            const payload = {
+                signature: 'foobarbaz',
+                mail_events: [
+                    makePayloadEvent('opened'),
+                    malformedPayloadEvent
+                ]
+            };
+
+            assert.throws(
+                () => service.validatePayload(payload),
+                {
+                    name: 'ValidationError',
+                    message: 'Event [1] is missing "timestamp"'
+                }
+            );
+        });
+
+        it('should validate that events in the payload have an event', function () {
+            const malformedPayloadEvent = makePayloadEvent('opened') as any;
+            delete malformedPayloadEvent.event;
+
+            const payload = {
+                signature: 'foobarbaz',
+                mail_events: [
+                    makePayloadEvent('opened'),
+                    malformedPayloadEvent
+                ]
+            };
+
+            assert.throws(
+                () => service.validatePayload(payload),
+                {
+                    name: 'ValidationError',
+                    message: 'Event [1] is missing "event"'
+                }
+            );
+        });
+
+        it('should validate that events in the payload have a message', function () {
+            const malformedPayloadEvent = makePayloadEvent('opened') as any;
+            delete malformedPayloadEvent.message;
+
+            const payload = {
+                signature: 'foobarbaz',
+                mail_events: [
+                    makePayloadEvent('opened'),
+                    malformedPayloadEvent
+                ]
+            };
+
+            assert.throws(
+                () => service.validatePayload(payload),
+                {
+                    name: 'ValidationError',
+                    message: 'Event [1] is missing "message"'
+                }
+            );
+        });
+
+        it('should validate that events in the payload have a recipient', function () {
+            const malformedPayloadEvent = makePayloadEvent('opened') as any;
+            delete malformedPayloadEvent.recipient;
+
+            const payload = {
+                signature: 'foobarbaz',
+                mail_events: [
+                    makePayloadEvent('opened'),
+                    malformedPayloadEvent
+                ]
+            };
+
+            assert.throws(
+                () => service.validatePayload(payload),
+                {
+                    name: 'ValidationError',
+                    message: 'Event [1] is missing "recipient"'
+                }
+            );
+        });
+
+        it('should validate that "message.headers.message-id" is present on an event', function () {
+            const malformedPayloadEvent = makePayloadEvent('opened') as any;
+            delete malformedPayloadEvent.message.headers;
+
+            const payload = {
+                signature: 'foobarbaz',
+                mail_events: [
+                    makePayloadEvent('opened'),
+                    malformedPayloadEvent
+                ]
+            };
+
+            assert.throws(
+                () => service.validatePayload(payload),
+                {
+                    name: 'ValidationError',
+                    message: 'Event [1] is missing "message.headers.message-id"'
+                }
+            );
+        });
+    });
+});

ghost/mail-events/tsconfig.json

@@ -0,0 +1,113 @@
+{
+  "ts-node": {
+    "files": true
+  },
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2022",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    // "lib": ["es2019"],                                      /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "commonjs",                                /* Specify what module code is generated. */
+    "rootDir": "src",                                    /* Specify the root folder within your source files. */
+    // "moduleResolution": "node10",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
+    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
+    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
+    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
+    "resolveJsonModule": true,                           /* Enable importing .json files. */
+    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    "outDir": "build",                                   /* Specify an output folder for all emitted files. */
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+    // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */
+
+    /* Type Checking */
+    "strict": true,                                      /* Enable all strict type-checking options. */
+    "noImplicitAny": true,                               /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
+  },
+  "include": ["src/**/*"]
+}