diff --git a/.github/dev.js b/.github/dev.js index d14b75e4e2..45493fb18c 100644 --- a/.github/dev.js +++ b/.github/dev.js @@ -77,6 +77,16 @@ if (DASH_DASH_ARGS.includes('collections') || DASH_DASH_ARGS.includes('all')) { }); } +if (DASH_DASH_ARGS.includes('mail-events') || DASH_DASH_ARGS.includes('all')) { + commands.push({ + name: 'collections', + command: 'yarn dev', + cwd: path.resolve(__dirname, '../ghost/mail-events'), + prefixColor: 'pink', + env: {} + }); +} + if (DASH_DASH_ARGS.includes('admin-x') || DASH_DASH_ARGS.includes('adminx') || DASH_DASH_ARGS.includes('adminX') || DASH_DASH_ARGS.includes('all')) { commands.push({ name: 'adminX', diff --git a/ghost/admin/app/services/feature.js b/ghost/admin/app/services/feature.js index 68377db847..fbbf35bcfe 100644 --- a/ghost/admin/app/services/feature.js +++ b/ghost/admin/app/services/feature.js @@ -73,6 +73,7 @@ export default class FeatureService extends Service { @feature('signupForm') signupForm; @feature('collections') collections; @feature('adminXSettings') adminXSettings; + @feature('mailEvents') mailEvents; _user = null; diff --git a/ghost/admin/app/templates/settings/labs.hbs b/ghost/admin/app/templates/settings/labs.hbs index a91b68a310..3b1d754b24 100644 --- a/ghost/admin/app/templates/settings/labs.hbs +++ b/ghost/admin/app/templates/settings/labs.hbs @@ -310,6 +310,20 @@ + +
+
+
+

Mail Events

+

+ Enables processing of mail events +

+
+
+ +
+
+
{{/if}} diff --git a/ghost/core/core/boot.js b/ghost/core/core/boot.js index 9c42a3c5a9..233c3e2b49 100644 --- a/ghost/core/core/boot.js +++ b/ghost/core/core/boot.js @@ -326,6 +326,7 @@ async function initServices({config}) { const slackNotifications = require('./server/services/slack-notifications'); const mediaInliner = require('./server/services/media-inliner'); const collections = require('./server/services/collections'); + const mailEvents = require('./server/services/mail-events'); const urlUtils = require('./shared/url-utils'); @@ -363,7 +364,8 @@ async function initServices({config}) { emailSuppressionList.init(), slackNotifications.init(), collections.init(), - mediaInliner.init() + mediaInliner.init(), + mailEvents.init() ]); debug('End: Services'); diff --git a/ghost/core/core/server/api/endpoints/index.js b/ghost/core/core/server/api/endpoints/index.js index b837838e38..7fe11c12da 100644 --- a/ghost/core/core/server/api/endpoints/index.js +++ b/ghost/core/core/server/api/endpoints/index.js @@ -201,6 +201,10 @@ module.exports = { return apiFramework.pipeline(require('./links'), localUtils); }, + get mailEvents() { + return apiFramework.pipeline(require('./mail-events'), localUtils); + }, + /** * Content API Controllers * diff --git a/ghost/core/core/server/api/endpoints/mail-events.js b/ghost/core/core/server/api/endpoints/mail-events.js new file mode 100644 index 0000000000..7e44595825 --- /dev/null +++ b/ghost/core/core/server/api/endpoints/mail-events.js @@ -0,0 +1,14 @@ +const mailEvents = require('../../services/mail-events'); + +module.exports = { + docName: 'mail_events', + add: { + headers: { + cacheInvalidate: false + }, + permissions: false, + async query(frame) { + return mailEvents.service.processPayload(frame.data); + } + } +}; diff --git a/ghost/core/core/server/api/endpoints/utils/serializers/output/index.js b/ghost/core/core/server/api/endpoints/utils/serializers/output/index.js index 2777ae101e..dfd3ed643e 100644 --- a/ghost/core/core/server/api/endpoints/utils/serializers/output/index.js +++ b/ghost/core/core/server/api/endpoints/utils/serializers/output/index.js @@ -139,5 +139,9 @@ module.exports = { get links() { return require('./links'); + }, + + get mail_events() { + return require('./mail-events'); } }; diff --git a/ghost/core/core/server/api/endpoints/utils/serializers/output/mail-events.js b/ghost/core/core/server/api/endpoints/utils/serializers/output/mail-events.js new file mode 100644 index 0000000000..a7519e435b --- /dev/null +++ b/ghost/core/core/server/api/endpoints/utils/serializers/output/mail-events.js @@ -0,0 +1,9 @@ +const debug = require('@tryghost/debug')('api:endpoints:utils:serializers:output:mail-events'); + +module.exports = { + add(response, apiConfig, frame) { + debug('add'); + + frame.response = {}; + } +}; diff --git a/ghost/core/core/server/api/endpoints/utils/validators/input/index.js b/ghost/core/core/server/api/endpoints/utils/validators/input/index.js index 40e1bfb702..9f39b22d0b 100644 --- a/ghost/core/core/server/api/endpoints/utils/validators/input/index.js +++ b/ghost/core/core/server/api/endpoints/utils/validators/input/index.js @@ -75,5 +75,9 @@ module.exports = { get snippets() { return require('./snippets'); + }, + + get mail_events() { + return require('./mail-events'); } }; diff --git a/ghost/core/core/server/api/endpoints/utils/validators/input/mail-events.js b/ghost/core/core/server/api/endpoints/utils/validators/input/mail-events.js new file mode 100644 index 0000000000..6e26353689 --- /dev/null +++ b/ghost/core/core/server/api/endpoints/utils/validators/input/mail-events.js @@ -0,0 +1,7 @@ +const mailEvents = require('../../../../../services/mail-events'); + +module.exports = { + add(apiConfig, frame) { + mailEvents.service.validatePayload(frame.data); + } +}; diff --git a/ghost/core/core/server/data/migrations/versions/5.53/2023-06-13-12-24-add-temp-mail-events-table.js b/ghost/core/core/server/data/migrations/versions/5.53/2023-06-13-12-24-add-temp-mail-events-table.js new file mode 100644 index 0000000000..c2b334ddc7 --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.53/2023-06-13-12-24-add-temp-mail-events-table.js @@ -0,0 +1,9 @@ +const {addTable} = require('../../utils'); + +module.exports = addTable('temp_mail_events', { + id: {type: 'string', maxlength: 100, nullable: false, primary: true}, + type: {type: 'string', maxlength: 50, nullable: false}, + message_id: {type: 'string', maxlength: 150, nullable: false}, + recipient: {type: 'string', maxlength: 191, nullable: false}, + occurred_at: {type: 'dateTime', nullable: false} +}); diff --git a/ghost/core/core/server/data/schema/schema.js b/ghost/core/core/server/data/schema/schema.js index 4da6128dfd..44eb36b64c 100644 --- a/ghost/core/core/server/data/schema/schema.js +++ b/ghost/core/core/server/data/schema/schema.js @@ -1021,5 +1021,12 @@ module.exports = { currency: {type: 'string', maxlength: 24, nullable: true}, created_at: {type: 'dateTime', nullable: false}, email_sent_at: {type: 'dateTime', nullable: true} + }, + temp_mail_events: { + id: {type: 'string', maxlength: 100, nullable: false, primary: true}, + type: {type: 'string', maxlength: 50, nullable: false}, + message_id: {type: 'string', maxlength: 150, nullable: false}, + recipient: {type: 'string', maxlength: 191, nullable: false}, + occurred_at: {type: 'dateTime', nullable: false} } }; diff --git a/ghost/core/core/server/models/mail-event.js b/ghost/core/core/server/models/mail-event.js new file mode 100644 index 0000000000..a7b2131df9 --- /dev/null +++ b/ghost/core/core/server/models/mail-event.js @@ -0,0 +1,12 @@ +const ghostBookshelf = require('./base'); + +const MailEvent = ghostBookshelf.Model.extend({ + tableName: 'temp_mail_events', + defaults() { + return {}; + } +}, {}); + +module.exports = { + MailEvent: ghostBookshelf.model('MailEvent', MailEvent) +}; diff --git a/ghost/core/core/server/services/mail-events/BookshelfMailEventRepository.js b/ghost/core/core/server/services/mail-events/BookshelfMailEventRepository.js new file mode 100644 index 0000000000..84950fd23e --- /dev/null +++ b/ghost/core/core/server/services/mail-events/BookshelfMailEventRepository.js @@ -0,0 +1,40 @@ +/** + * @typedef {import('@tryghost/mail-events').MailEventRepository} MailEventRepository + * @typedef {import('@tryghost/mail-events').MailEvent} MailEvent + */ + +/** + * @typedef {object} MailEventModel + * @property {function} add + */ + +/** + * @implements MailEventRepository + */ +module.exports = class BookshelfMailEventRepository { + /** + * @type {MailEventModel} + */ + #MailEventModel; + + /** + * @param {object} MailEventModel + */ + constructor(MailEventModel) { + this.#MailEventModel = MailEventModel; + } + + /** + * @param {MailEvent} mailEvent + * @returns {Promise} + */ + async save(mailEvent) { + await this.#MailEventModel.add({ + id: mailEvent.id, + type: mailEvent.type, + message_id: mailEvent.messageId, + recipient: mailEvent.recipient, + occurred_at: new Date(mailEvent.timestampMs) + }); + } +}; diff --git a/ghost/core/core/server/services/mail-events/index.js b/ghost/core/core/server/services/mail-events/index.js new file mode 100644 index 0000000000..003cf6876f --- /dev/null +++ b/ghost/core/core/server/services/mail-events/index.js @@ -0,0 +1,21 @@ +const {MailEventService} = require('@tryghost/mail-events'); +const MailEventRepository = require('./BookshelfMailEventRepository.js'); + +class MailEventsServiceWrapper { + /** + * @type {MailEventService} + */ + service; + + async init() { + const config = require('../../../shared/config'); + const labs = require('../../../shared/labs'); + const models = require('../../models'); + + const repository = new MailEventRepository(models.MailEvent); + + this.service = new MailEventService(repository, config, labs); + } +} + +module.exports = new MailEventsServiceWrapper(); diff --git a/ghost/core/core/server/web/api/endpoints/admin/routes.js b/ghost/core/core/server/web/api/endpoints/admin/routes.js index 931dc0da96..d7e07be8a1 100644 --- a/ghost/core/core/server/web/api/endpoints/admin/routes.js +++ b/ghost/core/core/server/web/api/endpoints/admin/routes.js @@ -17,6 +17,7 @@ module.exports = function apiRoutes() { // ## Public router.get('/site', mw.publicAdminApi, http(api.site.read)); + router.post('/mail_events', mw.publicAdminApi, http(api.mailEvents.add)); // ## Collections router.get('/collections', mw.authAdminApi, labs.enabledMiddleware('collections'), http(api.collections.browse)); diff --git a/ghost/core/core/shared/labs.js b/ghost/core/core/shared/labs.js index 68c26881f4..756b7fd8de 100644 --- a/ghost/core/core/shared/labs.js +++ b/ghost/core/core/shared/labs.js @@ -39,7 +39,8 @@ const ALPHA_FEATURES = [ 'emailCustomization', 'signupCard', 'collections', - 'adminXSettings' + 'adminXSettings', + 'mailEvents' ]; module.exports.GA_KEYS = [...GA_FEATURES]; diff --git a/ghost/core/package.json b/ghost/core/package.json index 0d96f87f15..ff45deb8a7 100644 --- a/ghost/core/package.json +++ b/ghost/core/package.json @@ -113,6 +113,7 @@ "@tryghost/link-tracking": "0.0.0", "@tryghost/logging": "2.4.4", "@tryghost/magic-link": "0.0.0", + "@tryghost/mail-events": "0.0.0", "@tryghost/mailgun-client": "0.0.0", "@tryghost/member-attribution": "0.0.0", "@tryghost/member-events": "0.0.0", diff --git a/ghost/core/test/e2e-api/admin/__snapshots__/mail-events.test.js.snap b/ghost/core/test/e2e-api/admin/__snapshots__/mail-events.test.js.snap new file mode 100644 index 0000000000..151726411f --- /dev/null +++ b/ghost/core/test/e2e-api/admin/__snapshots__/mail-events.test.js.snap @@ -0,0 +1,16 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Mail Events API Can add a mail event 1: [body] 1`] = `Object {}`; + +exports[`Mail Events API Can add a mail event 2: [headers] 1`] = ` +Object { + "access-control-allow-origin": "http://127.0.0.1:2369", + "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", + "content-length": "2", + "content-type": "application/json; charset=utf-8", + "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/, + "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/, + "vary": "Accept-Version, Origin, Accept-Encoding", + "x-powered-by": "Express", +} +`; diff --git a/ghost/core/test/e2e-api/admin/mail-events.test.js b/ghost/core/test/e2e-api/admin/mail-events.test.js new file mode 100644 index 0000000000..9d7b71b8b8 --- /dev/null +++ b/ghost/core/test/e2e-api/admin/mail-events.test.js @@ -0,0 +1,59 @@ +const assert = require('assert/strict'); +const {MailEventService} = require('@tryghost/mail-events'); +const {agentProvider, matchers, mockManager} = require('../../utils/e2e-framework'); +const configUtils = require('../../utils/configUtils'); +const models = require('../../../core/server/models'); + +const {anyContentVersion, anyEtag} = matchers; + +describe('Mail Events API', function () { + let agent; + + before(async function () { + agent = await agentProvider.getAdminAPIAgent(); + + mockManager.mockLabsEnabled(MailEventService.LABS_KEY); + }); + + it('Can add a mail event', async function () { + configUtils.set(MailEventService.CONFIG_KEY_PAYLOAD_SIGNING_KEY, 'foobarbaz'); + + const payload = { + // The signature is based on the previous config value as well as the + // "mail_events" array below. If you change any of these values, you will need to + // update the signature otherwise the request will fail + signature: '51ab01400f9a78669733d85fcf344401f5da648f8c95707bc06da0456cb99fbc', + mail_events: [ + { + id: 'Ase7i2zsRYeDXztHGENqRA', + timestamp: 1521243339.873676, + event: 'opened', + message: { + headers: { + 'message-id': '20130503182626.18666.16540@sandboxb052085d6a7b401bb117d3a432d1d659.mailgun.org' + } + }, + recipient: 'alice@example.com' + } + ] + }; + + await agent + .post('/mail_events/', { + headers: { + 'content-type': 'application/json' + } + }) + .body(payload) + .expectStatus(200) + .matchBodySnapshot({}) + .matchHeaderSnapshot({ + 'content-version': anyContentVersion, + etag: anyEtag + }); + + const storedMailEvent = await models.MailEvent.findOne({id: 'Ase7i2zsRYeDXztHGENqRA'}); + + assert.ok(storedMailEvent, 'Expected mail event was not found in the database'); + }); +}); diff --git a/ghost/core/test/integration/exporter/exporter.test.js b/ghost/core/test/integration/exporter/exporter.test.js index f2a303b783..82fa5ec952 100644 --- a/ghost/core/test/integration/exporter/exporter.test.js +++ b/ghost/core/test/integration/exporter/exporter.test.js @@ -89,7 +89,8 @@ describe('Exporter', function () { 'tokens', 'users', 'webhooks', - 'milestones' + 'milestones', + 'temp_mail_events' ]; should.exist(exportData); @@ -117,7 +118,8 @@ describe('Exporter', function () { 'members_email_change_events', 'members_status_events', 'members_paid_subscription_events', - 'members_subscribe_events' + 'members_subscribe_events', + 'temp_mail_events' ]; excludedTables.forEach((tableName) => { diff --git a/ghost/core/test/unit/server/data/exporter/index.test.js b/ghost/core/test/unit/server/data/exporter/index.test.js index 945af49ef0..5dd93ca653 100644 --- a/ghost/core/test/unit/server/data/exporter/index.test.js +++ b/ghost/core/test/unit/server/data/exporter/index.test.js @@ -214,7 +214,7 @@ describe('Exporter', function () { const nonSchemaTables = ['migrations', 'migrations_lock']; const requiredTables = schemaTables.concat(nonSchemaTables); // NOTE: You should not add tables to this list unless they are temporary - const ignoredTables = ['temp_member_analytic_events']; + const ignoredTables = ['temp_member_analytic_events', 'temp_mail_events']; const expectedTables = requiredTables.filter(table => !ignoredTables.includes(table)).sort(); const actualTables = BACKUP_TABLES.concat(TABLES_ALLOWLIST).sort(); diff --git a/ghost/core/test/unit/server/data/schema/integrity.test.js b/ghost/core/test/unit/server/data/schema/integrity.test.js index 848d33d7e5..c53c77a8cf 100644 --- a/ghost/core/test/unit/server/data/schema/integrity.test.js +++ b/ghost/core/test/unit/server/data/schema/integrity.test.js @@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route */ describe('DB version integrity', function () { // Only these variables should need updating - const currentSchemaHash = '2445c734ffb514d11b56e74591bcde4e'; + const currentSchemaHash = 'b174080d5df4de9b0f4bc9b786396217'; const currentFixturesHash = '93c3b3cb8bca34a733634e74ee514172'; const currentSettingsHash = '4f23a583335dcb4cb3fae553122ea200'; const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01'; diff --git a/ghost/core/test/unit/server/services/mail-events/BookshelfMailEventRepository.test.js b/ghost/core/test/unit/server/services/mail-events/BookshelfMailEventRepository.test.js new file mode 100644 index 0000000000..927cfa95b2 --- /dev/null +++ b/ghost/core/test/unit/server/services/mail-events/BookshelfMailEventRepository.test.js @@ -0,0 +1,27 @@ +const assert = require('assert/strict'); +const sinon = require('sinon'); +const {MailEvent} = require('@tryghost/mail-events'); +const BookshelfMailEventRepository = require('../../../../../core/server/services/mail-events/BookshelfMailEventRepository'); + +describe('BookshelfMailEventRepository', function () { + describe('save', function () { + it('should store a mail event', async function () { + const modelStub = { + add: sinon.stub().resolves() + }; + const event = new MailEvent('abc123', 'opened', '987def', 'foo@bar.baz', Date.now()); + const repository = new BookshelfMailEventRepository(modelStub); + + await repository.save(event); + + assert.ok(modelStub.add.calledOnce); + assert.ok(modelStub.add.calledWith({ + id: event.id, + type: event.type, + message_id: event.messageId, + recipient: event.recipient, + occurred_at: new Date(event.timestampMs) + })); + }); + }); +}); diff --git a/ghost/mail-events/.eslintrc.js b/ghost/mail-events/.eslintrc.js new file mode 100644 index 0000000000..85e844e154 --- /dev/null +++ b/ghost/mail-events/.eslintrc.js @@ -0,0 +1,13 @@ +module.exports = { + parser: '@typescript-eslint/parser', + plugins: ['ghost', '@typescript-eslint'], + extends: [ + 'plugin:ghost/node' + ], + rules: { + 'no-unused-vars': 'off', + '@typescript-eslint/no-unused-vars': ['error'], + 'no-shadow': 'off', + '@typescript-eslint/no-shadow': ['error'] + } +}; diff --git a/ghost/mail-events/README.md b/ghost/mail-events/README.md new file mode 100644 index 0000000000..db173f7f5c --- /dev/null +++ b/ghost/mail-events/README.md @@ -0,0 +1,17 @@ +# Mail Events + +Processes mail events + +## Develop + +This is a monorepo package. + +Follow the instructions for the top-level repo. + +1. `git clone` this repo & `cd` into it as usual +2. Run `yarn` to install top-level dependencies + +## Test + +- `yarn lint` run just eslint +- `yarn test` run lint and tests diff --git a/ghost/mail-events/package.json b/ghost/mail-events/package.json new file mode 100644 index 0000000000..db47c0d625 --- /dev/null +++ b/ghost/mail-events/package.json @@ -0,0 +1,44 @@ +{ + "name": "@tryghost/mail-events", + "version": "0.0.0", + "repository": "https://github.com/TryGhost/Ghost/tree/main/packages/mail-events", + "author": "Ghost Foundation", + "private": true, + "main": "build/index.js", + "types": "build/index.d.ts", + "scripts": { + "dev": "tsc --watch --preserveWatchOutput --sourceMap", + "build": "tsc", + "prepare": "tsc", + "test:unit": "NODE_ENV=testing c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura mocha -r ts-node/register './test/**/*.test.ts'", + "test": "yarn test:types && yarn test:unit", + "test:types": "tsc --noEmit", + "lint:code": "eslint src/ --ext .ts --cache", + "lint": "yarn lint:code && yarn lint:test", + "lint:test": "eslint -c test/.eslintrc.js test/ --ext .ts --cache" + }, + "files": [ + "build" + ], + "devDependencies": { + "c8": "7.13.0", + "mocha": "10.2.0", + "sinon": "15.0.4", + "ts-node": "10.9.1", + "typescript": "5.1.3" + }, + "dependencies": { + "@tryghost/errors": "^1.2.25", + "@tryghost/in-memory-repository": "0.0.0", + "@tryghost/tpl": "^0.1.25" + }, + "c8": { + "exclude": [ + "src/index.ts", + "src/MailEventRepository.ts", + "src/InMemoryMailEventRepository.ts", + "src/**/*.d.ts", + "test/**/*.ts" + ] + } +} diff --git a/ghost/mail-events/src/InMemoryMailEventRepository.ts b/ghost/mail-events/src/InMemoryMailEventRepository.ts new file mode 100644 index 0000000000..6125bd13ff --- /dev/null +++ b/ghost/mail-events/src/InMemoryMailEventRepository.ts @@ -0,0 +1,8 @@ +import {InMemoryRepository} from '@tryghost/in-memory-repository'; +import {MailEvent} from './MailEvent'; + +export class InMemoryMailEventRepository extends InMemoryRepository { + protected toPrimitive(): object { + return {}; + } +} diff --git a/ghost/mail-events/src/MailEvent.ts b/ghost/mail-events/src/MailEvent.ts new file mode 100644 index 0000000000..d2fbb53c7e --- /dev/null +++ b/ghost/mail-events/src/MailEvent.ts @@ -0,0 +1,10 @@ +export class MailEvent { + constructor( + readonly id: string, + readonly type: string, + readonly messageId: string, + readonly recipient: string, + readonly timestampMs: number, + readonly deleted: boolean = false + ) {} +} diff --git a/ghost/mail-events/src/MailEventRepository.ts b/ghost/mail-events/src/MailEventRepository.ts new file mode 100644 index 0000000000..00a1206451 --- /dev/null +++ b/ghost/mail-events/src/MailEventRepository.ts @@ -0,0 +1,5 @@ +import {MailEvent} from './MailEvent'; + +export interface MailEventRepository { + save(event: MailEvent): Promise; +} diff --git a/ghost/mail-events/src/MailEventService.ts b/ghost/mail-events/src/MailEventService.ts new file mode 100644 index 0000000000..e00fd5df11 --- /dev/null +++ b/ghost/mail-events/src/MailEventService.ts @@ -0,0 +1,168 @@ +const crypto = require('crypto'); +const errors = require('@tryghost/errors'); +const tpl = require('@tryghost/tpl'); + +import {MailEvent} from './MailEvent'; +import {MailEventRepository} from './MailEventRepository'; + +/** + * @see https://documentation.mailgun.com/en/latest/user_manual.html#events-1 + */ +enum EventType { + CLICKED = 'clicked', + COMPLAINED = 'complained', + DELIVERED = 'delivered', + FAILED = 'failed', + OPENED = 'opened', + UNSUBSCRIBED = 'unsubscribed' +} + +interface PayloadEvent { + id: string; + timestamp: number; // Unix timestamp in seconds + event: string; + message: { + headers: { + 'message-id': string; + } + }, + recipient: string; +} + +interface Payload { + signature: string; + mail_events: PayloadEvent[]; +} + +interface Labs { + isSet(key: string): boolean; +} + +interface Config { + get(key: string): any; +} + +const VALIDATION_MESSAGES = { + signingKeyNotConfigured: 'payload signing key is not configured', + payloadSignatureMissing: 'Payload is missing "signature"', + payloadSignatureInvalid: '"signature" is invalid', + payloadEventsMissing: 'Payload is missing "mail_events"', + payloadEventsInvalid: '"mail_events" is not an array', + payloadEventKeyMissing: 'Event [{idx}] is missing "{key}"' +}; + +export class MailEventService { + static readonly LABS_KEY = 'mailEvents'; + static readonly CONFIG_KEY_PAYLOAD_SIGNING_KEY = 'hostSettings:mailEventsPayloadSigningKey'; + + constructor( + private mailEventRepository: MailEventRepository, + private config: Config, + private labs: Labs + ) {} + + async processPayload(payload: Payload) { + if (this.labs.isSet(MailEventService.LABS_KEY) === false) { + throw new errors.NotFoundError(); + } + + const payloadSigningKey = this.config.get(MailEventService.CONFIG_KEY_PAYLOAD_SIGNING_KEY); + + // Verify that the service is configured correctly - We expect a string + // for the payload signing key but as a safeguard we check the type here + // to prevent any unexpected behaviour + if (typeof payloadSigningKey !== 'string') { + throw new errors.InternalServerError({ + message: tpl(VALIDATION_MESSAGES.signingKeyNotConfigured) + }); + } + + // Verify the payload + this.verifyPayload(payload, payloadSigningKey); + + // Store known events + const eventTypes = new Set(Object.values(EventType) as string[]); + + for (const payloadEvent of payload.mail_events) { + if (eventTypes.has(payloadEvent.event) === false) { + continue; + } + + try { + await this.mailEventRepository.save( + new MailEvent( + payloadEvent.id, + payloadEvent.event, + payloadEvent.message.headers['message-id'], + payloadEvent.recipient, + payloadEvent.timestamp * 1000 + ) + ); + } catch (err) { + throw new errors.InternalServerError({ + message: 'Event could not be stored', + err: err + }); + } + } + } + + validatePayload(payload: Payload) { + if (payload.signature === undefined) { + throw new errors.ValidationError({ + message: tpl(VALIDATION_MESSAGES.payloadSignatureMissing) + }); + } + + if (typeof payload.signature !== 'string') { + throw new errors.ValidationError({ + message: tpl(VALIDATION_MESSAGES.payloadSignatureInvalid) + }); + } + + if (payload.mail_events === undefined) { + throw new errors.ValidationError({ + message: tpl(VALIDATION_MESSAGES.payloadEventsMissing) + }); + } + + if (Array.isArray(payload.mail_events) === false) { + throw new errors.ValidationError({ + message: tpl(VALIDATION_MESSAGES.payloadEventsInvalid) + }); + } + + const expectedKeys: (keyof PayloadEvent)[] = ['id', 'timestamp', 'event', 'message', 'recipient']; + + payload.mail_events.forEach((payloadEvent, idx) => { + expectedKeys.forEach((key) => { + if (payloadEvent[key] === undefined) { + throw new errors.ValidationError({ + message: tpl(VALIDATION_MESSAGES.payloadEventKeyMissing, {idx, key}) + }); + } + + if (key === 'message' && payloadEvent.message.headers?.['message-id'] === undefined) { + throw new errors.ValidationError({ + message: tpl(VALIDATION_MESSAGES.payloadEventKeyMissing, {idx, key: 'message.headers.message-id'}) + }); + } + }); + }); + } + + private verifyPayload(payload: Payload, payloadSigningKey: string) { + const data = JSON.stringify(payload.mail_events); + + const signature = crypto + .createHmac('sha256', payloadSigningKey) + .update(data) + .digest('hex'); + + if (signature !== payload.signature) { + throw new errors.UnauthorizedError({ + message: tpl(VALIDATION_MESSAGES.payloadSignatureInvalid) + }); + } + } +} diff --git a/ghost/mail-events/src/index.ts b/ghost/mail-events/src/index.ts new file mode 100644 index 0000000000..52e56a659e --- /dev/null +++ b/ghost/mail-events/src/index.ts @@ -0,0 +1,3 @@ +export * from './MailEvent'; +export * from './MailEventRepository'; +export * from './MailEventService'; diff --git a/ghost/mail-events/test/.eslintrc.js b/ghost/mail-events/test/.eslintrc.js new file mode 100644 index 0000000000..6fe6dc1504 --- /dev/null +++ b/ghost/mail-events/test/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + parser: '@typescript-eslint/parser', + plugins: ['ghost'], + extends: [ + 'plugin:ghost/test' + ] +}; diff --git a/ghost/mail-events/test/MailEventService.test.ts b/ghost/mail-events/test/MailEventService.test.ts new file mode 100644 index 0000000000..00a0351a50 --- /dev/null +++ b/ghost/mail-events/test/MailEventService.test.ts @@ -0,0 +1,366 @@ +import assert from 'assert'; +import sinon from 'sinon'; + +import {MailEvent} from '../src/MailEvent'; +import {InMemoryMailEventRepository as MailEventRepository} from '../src/InMemoryMailEventRepository'; +import {MailEventService} from '../src/MailEventService'; + +const makePayloadEvent = ( + type: string, + timestamp = Date.now() +) => ({ + id: 'event-id', + timestamp: timestamp / 1000, + event: type, + message: { + headers: { + 'message-id': 'message-id' + } + }, + recipient: 'message-recipient' +}); + +const PAYLOAD_SIGNING_KEY = 'abc123'; + +describe('MailEventService', function () { + let repository: sinon.SinonStubbedInstance; + let config: sinon.SinonStubbedInstance; + let labs: sinon.SinonStubbedInstance; + let service: MailEventService; + + beforeEach(function () { + repository = sinon.createStubInstance(MailEventRepository); + labs = { + isSet: sinon.stub() + .withArgs(MailEventService.LABS_KEY) + .returns(true) + }; + config = { + get: sinon.stub() + .withArgs(MailEventService.CONFIG_KEY_PAYLOAD_SIGNING_KEY) + .returns(PAYLOAD_SIGNING_KEY) + }; + service = new MailEventService(repository, config, labs); + }); + + describe('processPayload', function () { + it('should reject if labs flag is false', async function () { + labs.isSet.withArgs(MailEventService.LABS_KEY).returns(false); + + await assert.rejects( + service.processPayload({} as any), + { + name: 'NotFoundError', + message: 'Resource could not be found.' + } + ); + }); + + it('should reject if payload signing key is invalid', async function () { + config.get.withArgs(MailEventService.CONFIG_KEY_PAYLOAD_SIGNING_KEY).returns(undefined); + + await assert.rejects( + service.processPayload({} as any), + { + name: 'InternalServerError', + message: 'payload signing key is not configured' + } + ); + }); + + it('should reject if payload verification fails', async function () { + await assert.rejects( + service.processPayload({ + signature: 'foobarbaz', + mail_events: [ + makePayloadEvent('opened') + ] + } as any), + { + name: 'UnauthorizedError', + message: '"signature" is invalid' + } + ); + }); + + it('should store a single event', async function () { + const payloadEvent = makePayloadEvent('opened'); + + // Ensure a fixed timestamp is used so that we know the signature up front + payloadEvent.timestamp = 1686665992511 / 1000; + + const payload = { + signature: '9f2567330688b82759600fad93c93b3e8f571d397c33688a8620400af20b79b3', + mail_events: [ + payloadEvent + ] + }; + + await service.processPayload(payload); + + const storedEvent = repository.save.getCall(0).args[0]; + + assert.ok(storedEvent instanceof MailEvent); + assert.equal(storedEvent.id, payloadEvent.id); + }); + + it('should store multiple events', async function () { + const events = [ + makePayloadEvent('opened'), + makePayloadEvent('opened') + ]; + + // Ensure fixed timestamps are used so that we know the signature up front + events[0].timestamp = 1686665992511 / 1000; + events[1].timestamp = 1686665992512 / 1000; + + const payload = { + signature: '02959cc9731ee575b66969a508f545d19c5968b42a03fa398ce9c93d8e7df0a5', + mail_events: events + }; + + await service.processPayload(payload); + + assert.ok(repository.save.calledTwice); + }); + + it('should ignore unknown events', async function () { + const events = [ + makePayloadEvent('unknown-event'), + makePayloadEvent('opened') + ]; + + // Ensure fixed timestamps are used so that we know the signature up front + events[0].timestamp = 1686665992511 / 1000; + events[1].timestamp = 1686665992512 / 1000; + + const payload = { + signature: 'd6de350faa9ec56d739ec7ffd5cb4230f90f583df05fe59a6c1a41afac7048df', + mail_events: events + }; + + await service.processPayload(payload); + + assert.ok(repository.save.calledOnce); + assert.equal(repository.save.getCall(0).args[0].type, 'opened'); + }); + + it('should ensure event timestamps are converted to ms', async function () { + const payloadEvent = makePayloadEvent('opened'); + + // Ensure a fixed timestamp is used so that we know the signature up front + payloadEvent.timestamp = 1686665992511 / 1000; + + const payload = { + signature: '9f2567330688b82759600fad93c93b3e8f571d397c33688a8620400af20b79b3', + mail_events: [ + payloadEvent + ] + }; + + await service.processPayload(payload); + + assert.ok(repository.save.calledOnce); + + const storedEvent = repository.save.getCall(0).args[0]; + + assert.equal(storedEvent.timestampMs, payloadEvent.timestamp * 1000); + }); + + it('should reject if an event can not be stored', async function () { + const payloadEvent = makePayloadEvent('opened'); + + // Ensure a fixed timestamp is used so that we know the signature up front + payloadEvent.timestamp = 1686665992511 / 1000; + + const payload = { + signature: '9f2567330688b82759600fad93c93b3e8f571d397c33688a8620400af20b79b3', + mail_events: [ + payloadEvent + ] + }; + + repository.save.rejects(new Error('foobarbaz')); + + await assert.rejects( + service.processPayload(payload), + { + name: 'InternalServerError', + message: 'Event could not be stored' + } + ); + }); + }); + + describe('validatePayload', function () { + it('should validate that the payload contains a signature', function () { + assert.throws( + () => service.validatePayload({} as any), + { + name: 'ValidationError', + message: 'Payload is missing "signature"' + } + ); + }); + + it('should validate that the payload contains a valid signature', function () { + assert.throws(() => { + service.validatePayload({ + signature: {} + } as any); + }, { + name: 'ValidationError', + message: '"signature" is invalid' + }); + }); + + it('should validate that the payload contains events', function () { + assert.throws(() => { + service.validatePayload({ + signature: 'foobarbaz' + } as any); + }, { + name: 'ValidationError', + message: 'Payload is missing "mail_events"' + }); + }); + + it('should validate that the payload contains valid events', function () { + assert.throws(() => { + service.validatePayload({ + signature: 'foobarbaz', + mail_events: {} + } as any); + }, { + name: 'ValidationError', + message: '"mail_events" is not an array' + }); + }); + + it('should validate that events in the payload have an id', function () { + const malformedPayloadEvent = makePayloadEvent('opened') as any; + delete malformedPayloadEvent.id; + + const payload = { + signature: 'foobarbaz', + mail_events: [ + makePayloadEvent('opened'), + malformedPayloadEvent + ] + }; + + assert.throws( + () => service.validatePayload(payload), + { + name: 'ValidationError', + message: 'Event [1] is missing "id"' + } + ); + }); + + it('should validate that events in the payload have an timestamp', function () { + const malformedPayloadEvent = makePayloadEvent('opened') as any; + delete malformedPayloadEvent.timestamp; + + const payload = { + signature: 'foobarbaz', + mail_events: [ + makePayloadEvent('opened'), + malformedPayloadEvent + ] + }; + + assert.throws( + () => service.validatePayload(payload), + { + name: 'ValidationError', + message: 'Event [1] is missing "timestamp"' + } + ); + }); + + it('should validate that events in the payload have an event', function () { + const malformedPayloadEvent = makePayloadEvent('opened') as any; + delete malformedPayloadEvent.event; + + const payload = { + signature: 'foobarbaz', + mail_events: [ + makePayloadEvent('opened'), + malformedPayloadEvent + ] + }; + + assert.throws( + () => service.validatePayload(payload), + { + name: 'ValidationError', + message: 'Event [1] is missing "event"' + } + ); + }); + + it('should validate that events in the payload have a message', function () { + const malformedPayloadEvent = makePayloadEvent('opened') as any; + delete malformedPayloadEvent.message; + + const payload = { + signature: 'foobarbaz', + mail_events: [ + makePayloadEvent('opened'), + malformedPayloadEvent + ] + }; + + assert.throws( + () => service.validatePayload(payload), + { + name: 'ValidationError', + message: 'Event [1] is missing "message"' + } + ); + }); + + it('should validate that events in the payload have a recipient', function () { + const malformedPayloadEvent = makePayloadEvent('opened') as any; + delete malformedPayloadEvent.recipient; + + const payload = { + signature: 'foobarbaz', + mail_events: [ + makePayloadEvent('opened'), + malformedPayloadEvent + ] + }; + + assert.throws( + () => service.validatePayload(payload), + { + name: 'ValidationError', + message: 'Event [1] is missing "recipient"' + } + ); + }); + + it('should validate that "message.headers.message-id" is present on an event', function () { + const malformedPayloadEvent = makePayloadEvent('opened') as any; + delete malformedPayloadEvent.message.headers; + + const payload = { + signature: 'foobarbaz', + mail_events: [ + makePayloadEvent('opened'), + malformedPayloadEvent + ] + }; + + assert.throws( + () => service.validatePayload(payload), + { + name: 'ValidationError', + message: 'Event [1] is missing "message.headers.message-id"' + } + ); + }); + }); +}); diff --git a/ghost/mail-events/tsconfig.json b/ghost/mail-events/tsconfig.json new file mode 100644 index 0000000000..8467c9ff7e --- /dev/null +++ b/ghost/mail-events/tsconfig.json @@ -0,0 +1,113 @@ +{ + "ts-node": { + "files": true + }, + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": ["es2019"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + "rootDir": "src", /* Specify the root folder within your source files. */ + // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ + // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ + // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ + // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ + "resolveJsonModule": true, /* Enable importing .json files. */ + // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + "outDir": "build", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "include": ["src/**/*"] +}