From da24d13601f828308571fbe463dbbf2fdd3aadd9 Mon Sep 17 00:00:00 2001 From: Simon Backx Date: Thu, 18 Aug 2022 17:38:42 +0200 Subject: [PATCH] Added member attribution events and storage (#15243) refs https://github.com/TryGhost/Team/issues/1808 refs https://github.com/TryGhost/Team/issues/1809 refs https://github.com/TryGhost/Team/issues/1820 refs https://github.com/TryGhost/Team/issues/1814 ### Changes in `member-events` package - Added MemberCreatedEvent (event, not model) - Added SubscriptionCreatedEvent (event, not model) ### Added `member-attribution` package (new) - Added the AttributionBuilder class which is able to convert a url history to an attribution object (exposed as getAttribution on the service itself, which handles the dependencies) ``` [{ "path": "/", "time": 123 }] ``` to ``` { "url": "/", "id": null, "type": "url" } ``` - event handler listens for MemberCreatedEvent and SubscriptionCreatedEvent and creates the corresponding models in the database. ### Changes in `members-api` package - Added urlHistory to `sendMagicLink` endpoint body + convert the urlHistory to an attribution object that is stored in the tokenData of the magic link (sent by Portal in this PR: https://github.com/TryGhost/Portal/pull/256). - Added urlHistory to `createCheckoutSession` endpoint + convert the urlHistory to attribution keys that are saved in the Stripe Session metadata (sent by Portal in this PR: https://github.com/TryGhost/Portal/pull/256). - Added attribution data property to member repository's create method (when a member is created) - Dispatch MemberCreatedEvent with attribution ### Changes in `members-stripe-service` package (`ghost/stripe`) - Dispatch SubscriptionCreatedEvent in WebhookController on subscription checkout (with attribution from session metadata) --- ghost/core/core/boot.js | 2 + .../services/member-attribution/index.js | 24 ++ .../core/core/server/services/members/api.js | 4 +- ghost/core/package.json | 1 + ...reate-stripe-checkout-session.test.js.snap | 102 ++++++++ .../create-stripe-checkout-session.test.js | 139 ++++++++++- .../e2e-api/members/send-magic-link.test.js | 97 ++++++++ .../core/test/e2e-api/members/signin.test.js | 86 +++++++ .../test/e2e-api/members/webhooks.test.js | 173 +++++++++++++- .../services/member-attribution.test.js | 96 ++++++++ .../api/admin/members-importer.test.js | 1 - .../api/admin/members-signin-url.test.js | 4 - .../test/utils/e2e-framework-mock-manager.js | 2 + ghost/member-attribution/.eslintrc.js | 6 + ghost/member-attribution/README.md | 21 ++ ghost/member-attribution/index.js | 1 + ghost/member-attribution/lib/attribution.js | 71 ++++++ ghost/member-attribution/lib/event-handler.js | 52 ++++ ghost/member-attribution/lib/history.js | 46 ++++ ghost/member-attribution/lib/service.js | 27 +++ .../member-attribution/lib/url-translator.js | 56 +++++ ghost/member-attribution/package.json | 26 ++ ghost/member-attribution/test/.eslintrc.js | 6 + .../test/attribution.test.js | 86 +++++++ .../test/event-handler.test.js | 223 ++++++++++++++++++ ghost/member-attribution/test/history.test.js | 75 ++++++ ghost/member-attribution/test/service.test.js | 12 + .../test/url-translator.test.js | 74 ++++++ .../test/utils/assertions.js | 11 + ghost/member-attribution/test/utils/index.js | 11 + .../test/utils/overrides.js | 10 + ghost/member-events/index.js | 1 + ghost/member-events/lib/MemberCreatedEvent.js | 25 ++ .../lib/SubscriptionCreatedEvent.js | 3 +- ghost/member-events/package.json | 3 +- ghost/members-api/lib/MembersAPI.js | 21 +- ghost/members-api/lib/controllers/router.js | 43 +++- ghost/members-api/lib/repositories/member.js | 24 +- ghost/members-api/package.json | 3 +- ghost/stripe/lib/WebhookController.js | 17 +- 40 files changed, 1662 insertions(+), 23 deletions(-) create mode 100644 ghost/core/core/server/services/member-attribution/index.js create mode 100644 ghost/core/test/e2e-api/members/send-magic-link.test.js create mode 100644 ghost/core/test/e2e-server/services/member-attribution.test.js create mode 100644 ghost/member-attribution/.eslintrc.js create mode 100644 ghost/member-attribution/README.md create mode 100644 ghost/member-attribution/index.js create mode 100644 ghost/member-attribution/lib/attribution.js create mode 100644 ghost/member-attribution/lib/event-handler.js create mode 100644 ghost/member-attribution/lib/history.js create mode 100644 ghost/member-attribution/lib/service.js create mode 100644 ghost/member-attribution/lib/url-translator.js create mode 100644 ghost/member-attribution/package.json create mode 100644 ghost/member-attribution/test/.eslintrc.js create mode 100644 ghost/member-attribution/test/attribution.test.js create mode 100644 ghost/member-attribution/test/event-handler.test.js create mode 100644 ghost/member-attribution/test/history.test.js create mode 100644 ghost/member-attribution/test/service.test.js create mode 100644 ghost/member-attribution/test/url-translator.test.js create mode 100644 ghost/member-attribution/test/utils/assertions.js create mode 100644 ghost/member-attribution/test/utils/index.js create mode 100644 ghost/member-attribution/test/utils/overrides.js create mode 100644 ghost/member-events/lib/MemberCreatedEvent.js diff --git a/ghost/core/core/boot.js b/ghost/core/core/boot.js index d0cf88a5be..2ba585e206 100644 --- a/ghost/core/core/boot.js +++ b/ghost/core/core/boot.js @@ -279,6 +279,7 @@ async function initServices({config}) { const apiVersionCompatibility = require('./server/services/api-version-compatibility'); const scheduling = require('./server/adapters/scheduling'); const comments = require('./server/services/comments'); + const memberAttribution = require('./server/services/member-attribution'); const urlUtils = require('./shared/url-utils'); @@ -291,6 +292,7 @@ async function initServices({config}) { await stripe.init(); await Promise.all([ + memberAttribution.init(), members.init(), permissions.init(), xmlrpc.listen(), diff --git a/ghost/core/core/server/services/member-attribution/index.js b/ghost/core/core/server/services/member-attribution/index.js new file mode 100644 index 0000000000..e9bd5a8737 --- /dev/null +++ b/ghost/core/core/server/services/member-attribution/index.js @@ -0,0 +1,24 @@ +const urlService = require('../url'); +const labsService = require('../../../shared/labs'); + +class MemberAttributionServiceWrapper { + init() { + if (this.service) { + // Prevent creating duplicate DomainEvents subscribers + return; + } + + const MemberAttributionService = require('@tryghost/member-attribution'); + const models = require('../../models'); + + // For now we don't need to expose anything (yet) + this.service = new MemberAttributionService({ + MemberCreatedEvent: models.MemberCreatedEvent, + SubscriptionCreatedEvent: models.SubscriptionCreatedEvent, + urlService, + labsService + }); + } +} + +module.exports = new MemberAttributionServiceWrapper(); diff --git a/ghost/core/core/server/services/members/api.js b/ghost/core/core/server/services/members/api.js index 0a35b80eee..00d7c0ca25 100644 --- a/ghost/core/core/server/services/members/api.js +++ b/ghost/core/core/server/services/members/api.js @@ -14,6 +14,7 @@ const urlUtils = require('../../../shared/url-utils'); const labsService = require('../../../shared/labs'); const offersService = require('../offers'); const newslettersService = require('../newsletters'); +const memberAttributionService = require('../member-attribution'); const MAGIC_LINK_TOKEN_VALIDITY = 24 * 60 * 60 * 1000; @@ -195,7 +196,8 @@ function createApiInstance(config) { stripeAPIService: stripeService.api, offersAPI: offersService.api, labsService: labsService, - newslettersService: newslettersService + newslettersService: newslettersService, + memberAttributionService: memberAttributionService.service }); return membersApiInstance; diff --git a/ghost/core/package.json b/ghost/core/package.json index 4bea580ebc..5c80651efe 100644 --- a/ghost/core/package.json +++ b/ghost/core/package.json @@ -83,6 +83,7 @@ "@tryghost/logging": "2.2.4", "@tryghost/magic-link": "0.0.0", "@tryghost/mailgun-client": "0.0.0", + "@tryghost/member-attribution": "0.0.0", "@tryghost/member-events": "0.0.0", "@tryghost/members-api": "0.0.0", "@tryghost/members-csv": "0.0.0", diff --git a/ghost/core/test/e2e-api/members/__snapshots__/create-stripe-checkout-session.test.js.snap b/ghost/core/test/e2e-api/members/__snapshots__/create-stripe-checkout-session.test.js.snap index ee94c61da7..faf94942e4 100644 --- a/ghost/core/test/e2e-api/members/__snapshots__/create-stripe-checkout-session.test.js.snap +++ b/ghost/core/test/e2e-api/members/__snapshots__/create-stripe-checkout-session.test.js.snap @@ -46,3 +46,105 @@ Object { "x-powered-by": "Express", } `; + +exports[`Create Stripe Checkout Session Does pass attribution source to session metadata 1: [body] 1`] = ` +Object { + "publicKey": "pk_test_for_stripe", + "sessionId": "cs_123", +} +`; + +exports[`Create Stripe Checkout Session Does pass attribution source to session metadata 2: [headers] 1`] = ` +Object { + "access-control-allow-origin": "*", + "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", + "content-type": "application/json", + "vary": "Accept-Encoding", + "x-powered-by": "Express", +} +`; + +exports[`Create Stripe Checkout Session Does pass urlHistory 1: [body] 1`] = ` +Object { + "publicKey": "pk_test_for_stripe", + "sessionId": "cs_123", +} +`; + +exports[`Create Stripe Checkout Session Does pass urlHistory 2: [headers] 1`] = ` +Object { + "access-control-allow-origin": "*", + "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", + "content-type": "application/json", + "vary": "Accept-Encoding", + "x-powered-by": "Express", +} +`; + +exports[`Create Stripe Checkout Session Ignores attribution_* values in metadata 1: [body] 1`] = ` +Object { + "publicKey": "pk_test_for_stripe", + "sessionId": "cs_123", +} +`; + +exports[`Create Stripe Checkout Session Ignores attribution_* values in metadata 2: [headers] 1`] = ` +Object { + "access-control-allow-origin": "*", + "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", + "content-type": "application/json", + "vary": "Accept-Encoding", + "x-powered-by": "Express", +} +`; + +exports[`Create Stripe Checkout Session Member attribution Does pass post attribution source to session metadata 1: [body] 1`] = ` +Object { + "publicKey": "pk_test_for_stripe", + "sessionId": "cs_123", +} +`; + +exports[`Create Stripe Checkout Session Member attribution Does pass post attribution source to session metadata 2: [headers] 1`] = ` +Object { + "access-control-allow-origin": "*", + "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", + "content-type": "application/json", + "vary": "Accept-Encoding", + "x-powered-by": "Express", +} +`; + +exports[`Create Stripe Checkout Session Member attribution Does pass url attribution source to session metadata 1: [body] 1`] = ` +Object { + "publicKey": "pk_test_for_stripe", + "sessionId": "cs_123", +} +`; + +exports[`Create Stripe Checkout Session Member attribution Does pass url attribution source to session metadata 2: [headers] 1`] = ` +Object { + "access-control-allow-origin": "*", + "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", + "content-type": "application/json", + "vary": "Accept-Encoding", + "x-powered-by": "Express", +} +`; + +exports[`Create Stripe Checkout Session Member attribution Ignores attribution_* values in metadata 1: [body] 1`] = ` +Object { + "publicKey": "pk_test_for_stripe", + "sessionId": "cs_123", +} +`; + +exports[`Create Stripe Checkout Session Member attribution Ignores attribution_* values in metadata 2: [headers] 1`] = ` +Object { + "access-control-allow-origin": "*", + "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", + "content-type": "application/json", + "vary": "Accept-Encoding", + "x-powered-by": "Express", +} +`; diff --git a/ghost/core/test/e2e-api/members/create-stripe-checkout-session.test.js b/ghost/core/test/e2e-api/members/create-stripe-checkout-session.test.js index 63f8f5f60b..70be213a7c 100644 --- a/ghost/core/test/e2e-api/members/create-stripe-checkout-session.test.js +++ b/ghost/core/test/e2e-api/members/create-stripe-checkout-session.test.js @@ -1,8 +1,16 @@ const {agentProvider, mockManager, fixtureManager, matchers} = require('../../utils/e2e-framework'); const nock = require('nock'); +const should = require('should'); +const models = require('../../../core/server/models'); +const urlService = require('../../../core/server/services/url'); let membersAgent, adminAgent, membersService; +async function getPost(id) { + // eslint-disable-next-line dot-notation + return await models['Post'].where('id', id).fetch({require: true}); +} + describe('Create Stripe Checkout Session', function () { before(async function () { const agents = await agentProvider.getAgentsForMembers(); @@ -11,7 +19,7 @@ describe('Create Stripe Checkout Session', function () { membersService = require('../../../core/server/services/members'); - await fixtureManager.init('members'); + await fixtureManager.init('posts', 'members'); await adminAgent.loginAsOwner(); }); @@ -73,4 +81,133 @@ describe('Create Stripe Checkout Session', function () { .matchBodySnapshot() .matchHeaderSnapshot(); }); + + /** + * When a checkout session is created with an urlHistory, we should convert it to an + * attribution and check if that is set in the metadata of the stripe session + */ + describe('Member attribution', function () { + it('Does pass url attribution source to session metadata', async function () { + const {body: {tiers}} = await adminAgent.get('/tiers/?include=monthly_price&yearly_price'); + + const paidTier = tiers.find(tier => tier.type === 'paid'); + + const scope = nock('https://api.stripe.com') + .persist() + .post(/v1\/.*/) + .reply((uri, body) => { + if (uri === '/v1/checkout/sessions') { + const parsed = new URLSearchParams(body); + should(parsed.get('metadata[attribution_url]')).eql('/test'); + should(parsed.get('metadata[attribution_type]')).eql('url'); + should(parsed.get('metadata[attribution_id]')).be.null(); + return [200, {id: 'cs_123'}]; + } + + throw new Error('Should not get called'); + }); + + await membersAgent.post('/api/create-stripe-checkout-session/') + .body({ + customerEmail: 'attribution@test.com', + tierId: paidTier.id, + cadence: 'month', + metadata: { + urlHistory: [ + { + path: '/test', + time: 123 + } + ] + } + }) + .expectStatus(200) + .matchBodySnapshot() + .matchHeaderSnapshot(); + + should(scope.isDone()).eql(true); + }); + + it('Does pass post attribution source to session metadata', async function () { + const post = await getPost(fixtureManager.get('posts', 0).id); + const url = urlService.getUrlByResourceId(post.id, {absolute: false}); + + const {body: {tiers}} = await adminAgent.get('/tiers/?include=monthly_price&yearly_price'); + + const paidTier = tiers.find(tier => tier.type === 'paid'); + + const scope = nock('https://api.stripe.com') + .persist() + .post(/v1\/.*/) + .reply((uri, body) => { + if (uri === '/v1/checkout/sessions') { + const parsed = new URLSearchParams(body); + should(parsed.get('metadata[attribution_url]')).eql(url); + should(parsed.get('metadata[attribution_type]')).eql('post'); + should(parsed.get('metadata[attribution_id]')).eql(post.id); + return [200, {id: 'cs_123'}]; + } + + throw new Error('Should not get called'); + }); + + await membersAgent.post('/api/create-stripe-checkout-session/') + .body({ + customerEmail: 'attribution-post@test.com', + tierId: paidTier.id, + cadence: 'month', + metadata: { + urlHistory: [ + { + path: url, + time: 123 + } + ] + } + }) + .expectStatus(200) + .matchBodySnapshot() + .matchHeaderSnapshot(); + + should(scope.isDone()).eql(true); + }); + + it('Ignores attribution_* values in metadata', async function () { + const {body: {tiers}} = await adminAgent.get('/tiers/?include=monthly_price&yearly_price'); + + const paidTier = tiers.find(tier => tier.type === 'paid'); + + const scope = nock('https://api.stripe.com') + .persist() + .post(/v1\/.*/) + .reply((uri, body) => { + if (uri === '/v1/checkout/sessions') { + const parsed = new URLSearchParams(body); + should(parsed.get('metadata[attribution_url]')).be.null(); + should(parsed.get('metadata[attribution_type]')).be.null(); + should(parsed.get('metadata[attribution_id]')).be.null(); + return [200, {id: 'cs_123'}]; + } + + throw new Error('Should not get called'); + }); + + await membersAgent.post('/api/create-stripe-checkout-session/') + .body({ + customerEmail: 'attribution-2@test.com', + tierId: paidTier.id, + cadence: 'month', + metadata: { + attribution_type: 'url', + attribution_url: '/', + attribution_id: null + } + }) + .expectStatus(200) + .matchBodySnapshot() + .matchHeaderSnapshot(); + + should(scope.isDone()).eql(true); + }); + }); }); diff --git a/ghost/core/test/e2e-api/members/send-magic-link.test.js b/ghost/core/test/e2e-api/members/send-magic-link.test.js new file mode 100644 index 0000000000..b490573397 --- /dev/null +++ b/ghost/core/test/e2e-api/members/send-magic-link.test.js @@ -0,0 +1,97 @@ +const {agentProvider, mockManager, fixtureManager} = require('../../utils/e2e-framework'); +const should = require('should'); + +let membersAgent, membersService; + +describe('sendMagicLink', function () { + before(async function () { + const agents = await agentProvider.getAgentsForMembers(); + membersAgent = agents.membersAgent; + + membersService = require('../../../core/server/services/members'); + + await fixtureManager.init('members'); + }); + + beforeEach(function () { + mockManager.mockMail(); + }); + + afterEach(function () { + mockManager.restore(); + }); + + it('Creates a valid magic link with tokenData, and without urlHistory', async function () { + const email = 'newly-created-user-magic-link-test@test.com'; + await membersAgent.post('/api/send-magic-link') + .body({ + email, + emailType: 'signup' + }) + .expectEmptyBody() + .expectStatus(201); + + // Check email is sent + const mail = mockManager.assert.sentEmail({ + to: email, + subject: /Complete your sign up to Ghost!/ + }); + + // Get link from email + const [url] = mail.text.match(/https?:\/\/[^\s]+/); + const parsed = new URL(url); + const token = parsed.searchParams.get('token'); + + // Get data + const data = await membersService.api.getTokenDataFromMagicLinkToken(token); + + should(data).match({ + email, + attribution: { + id: null, + url: null, + type: null + } + }); + }); + + it('Converts the urlHistory to the attribution and stores it in the token', async function () { + const email = 'newly-created-user-magic-link-test-2@test.com'; + await membersAgent.post('/api/send-magic-link') + .body({ + email, + emailType: 'signup', + urlHistory: [ + { + path: '/test-path', + time: 123 + } + ] + }) + .expectEmptyBody() + .expectStatus(201); + + // Check email is sent + const mail = mockManager.assert.sentEmail({ + to: email, + subject: /Complete your sign up to Ghost!/ + }); + + // Get link from email + const [url] = mail.text.match(/https?:\/\/[^\s]+/); + const parsed = new URL(url); + const token = parsed.searchParams.get('token'); + + // Get data + const data = await membersService.api.getTokenDataFromMagicLinkToken(token); + + should(data).match({ + email, + attribution: { + id: null, + url: '/test-path', + type: 'url' + } + }); + }); +}); diff --git a/ghost/core/test/e2e-api/members/signin.test.js b/ghost/core/test/e2e-api/members/signin.test.js index 701538f43c..9ffe2f148c 100644 --- a/ghost/core/test/e2e-api/members/signin.test.js +++ b/ghost/core/test/e2e-api/members/signin.test.js @@ -1,7 +1,27 @@ const {agentProvider, mockManager, fixtureManager} = require('../../utils/e2e-framework'); +const models = require('../../../core/server/models'); +const assert = require('assert'); +require('should'); +const labsService = require('../../../core/shared/labs'); let membersAgent, membersService; +async function assertMemberEvents({eventType, memberId, asserts}) { + const events = await models[eventType].where('member_id', memberId).fetchAll(); + const eventsJSON = events.map(e => e.toJSON()); + + // Order shouldn't matter here + for (const a of asserts) { + eventsJSON.should.matchAny(a); + } + assert.equal(events.length, asserts.length, `Only ${asserts.length} ${eventType} should have been added.`); +} + +async function getMemberByEmail(email) { + // eslint-disable-next-line dot-notation + return await models['Member'].where('email', email).fetch({require: true}); +} + describe('Members Signin', function () { before(async function () { const agents = await agentProvider.getAgentsForMembers(); @@ -69,4 +89,70 @@ describe('Members Signin', function () { .expectHeader('Location', /\/welcome-free\/$/) .expectHeader('Set-Cookie', /members-ssr.*/); }); + + it('Will create a new member on signup', async function () { + const email = 'not-existent-member@test.com'; + const magicLink = await membersService.api.getMagicLink(email); + const magicLinkUrl = new URL(magicLink); + const token = magicLinkUrl.searchParams.get('token'); + + await membersAgent.get(`/?token=${token}&action=signup`) + .expectStatus(302) + .expectHeader('Location', /\/welcome-free\/$/) + .expectHeader('Set-Cookie', /members-ssr.*/); + + const member = await getMemberByEmail(email); + + // Check event created + await assertMemberEvents({ + eventType: 'MemberCreatedEvent', + memberId: member.id, + asserts: [ + { + created_at: member.get('created_at'), + attribution_url: null, + attribution_id: null, + attribution_type: null, + source: 'member' + } + ] + }); + }); + + describe('Member attribution', function () { + it('Will create a member attribution if magic link contains an attribution source', async function () { + const email = 'non-existent-member@test.com'; + const magicLink = await membersService.api.getMagicLink(email, { + attribution: { + id: 'test_source_id', + url: '/test-source-url/', + type: 'post' + } + }); + const magicLinkUrl = new URL(magicLink); + const token = magicLinkUrl.searchParams.get('token'); + + await membersAgent.get(`/?token=${token}&action=signup`) + .expectStatus(302) + .expectHeader('Location', /\/welcome-free\/$/) + .expectHeader('Set-Cookie', /members-ssr.*/); + + const member = await getMemberByEmail(email); + + // Check event created + await assertMemberEvents({ + eventType: 'MemberCreatedEvent', + memberId: member.id, + asserts: [ + { + created_at: member.get('created_at'), + attribution_id: 'test_source_id', + attribution_url: '/test-source-url/', + attribution_type: 'post', + source: 'member' + } + ] + }); + }); + }); }); diff --git a/ghost/core/test/e2e-api/members/webhooks.test.js b/ghost/core/test/e2e-api/members/webhooks.test.js index 6f967fc02a..3ec47e074a 100644 --- a/ghost/core/test/e2e-api/members/webhooks.test.js +++ b/ghost/core/test/e2e-api/members/webhooks.test.js @@ -19,6 +19,15 @@ async function getPaidProduct() { return await Product.findOne({type: 'paid'}); } +async function getSubscription(subscriptionId) { + // eslint-disable-next-line dot-notation + return await models['StripeCustomerSubscription'].where('subscription_id', subscriptionId).fetch({require: true}); +} +async function getMember(memberId) { + // eslint-disable-next-line dot-notation + return await models['Member'].where('id', memberId).fetch({require: true}); +} + async function assertMemberEvents({eventType, memberId, asserts}) { const events = (await models[eventType].where('member_id', memberId).fetchAll()).toJSON(); events.should.match(asserts); @@ -26,8 +35,7 @@ async function assertMemberEvents({eventType, memberId, asserts}) { } async function assertSubscription(subscriptionId, asserts) { - // eslint-disable-next-line dot-notation - const subscription = await models['StripeCustomerSubscription'].where('subscription_id', subscriptionId).fetch({require: true}); + const subscription = await getSubscription(subscriptionId); // We use the native toJSON to prevent calling the overriden serialize method models.Base.Model.prototype.serialize.call(subscription).should.match(asserts); @@ -1674,5 +1682,166 @@ describe('Members API', function () { }); }); }); + + // Test if the session metadata is processed correctly + describe('Member attribution', function () { + beforeEach(function () { + mockManager.mockLabsEnabled('memberAttribution'); + }); + + // The subscription that we got from Stripe was created 2 seconds earlier (used for testing events) + const beforeNow = Math.floor((Date.now() - 2000) / 1000) * 1000; + + async function testWithAttribution(attribution) { + const customer_id = createStripeID('cust'); + const subscription_id = createStripeID('sub'); + + const interval = 'month'; + const unit_amount = 150; + + set(subscription, { + id: subscription_id, + customer: customer_id, + status: 'active', + items: { + type: 'list', + data: [{ + id: 'item_123', + price: { + id: 'price_123', + product: 'product_123', + active: true, + nickname: interval, + currency: 'usd', + recurring: { + interval + }, + unit_amount, + type: 'recurring' + } + }] + }, + start_date: beforeNow / 1000, + current_period_end: Math.floor(beforeNow / 1000) + (60 * 60 * 24 * 31), + cancel_at_period_end: false, + metadata: {} + }); + + set(customer, { + id: customer_id, + name: 'Test Member', + email: `${customer_id}@email.com`, + subscriptions: { + type: 'list', + data: [subscription] + } + }); + + let webhookPayload = JSON.stringify({ + type: 'checkout.session.completed', + data: { + object: { + mode: 'subscription', + customer: customer.id, + subscription: subscription.id, + metadata: attribution ? { + attribution_id: attribution.id, + attribution_url: attribution.url, + attribution_type: attribution.type + } : {} + } + } + }); + + let webhookSignature = stripe.webhooks.generateTestHeaderString({ + payload: webhookPayload, + secret: process.env.WEBHOOK_SECRET + }); + + await membersAgent.post('/webhooks/stripe/') + .body(webhookPayload) + .header('stripe-signature', webhookSignature) + .expectStatus(200); + + const {body} = await adminAgent.get(`/members/?search=${customer_id}@email.com`); + assert.equal(body.members.length, 1, 'The member was not created'); + const member = body.members[0]; + + assert.equal(member.status, 'paid', 'The member should be "paid"'); + assert.equal(member.subscriptions.length, 1, 'The member should have a single subscription'); + + // Convert Stripe ID to internal model ID + const subscriptionModel = await getSubscription(member.subscriptions[0].id); + + await assertMemberEvents({ + eventType: 'SubscriptionCreatedEvent', + memberId: member.id, + asserts: [ + { + member_id: member.id, + subscription_id: subscriptionModel.id, + + // Defaults if attribution is not set + attribution_id: attribution?.id ?? null, + attribution_url: attribution?.url ?? null, + attribution_type: attribution?.type ?? null + } + ] + }); + + const memberModel = await getMember(member.id); + + // It also should have created a new member, and a MemberCreatedEvent + // With the same attributions + await assertMemberEvents({ + eventType: 'MemberCreatedEvent', + memberId: member.id, + asserts: [ + { + member_id: member.id, + created_at: memberModel.get('created_at'), + + // Defaults if attribution is not set + attribution_id: attribution?.id ?? null, + attribution_url: attribution?.url ?? null, + attribution_type: attribution?.type ?? null, + source: 'member' + } + ] + }); + } + + it('Creates a SubscriptionCreatedEvent with url attribution', async function () { + // This mainly tests for nullable fields being set to null and handled correctly + const attribution = { + id: null, + url: '/', + type: 'url' + }; + + await testWithAttribution(attribution); + }); + + it('Creates a SubscriptionCreatedEvent with post attribution', async function () { + const attribution = { + id: 'my-post-id', + url: '/my-post-slug', + type: 'post' + }; + + await testWithAttribution(attribution); + }); + + it('Creates a SubscriptionCreatedEvent without attribution', async function () { + const attribution = undefined; + await testWithAttribution(attribution); + }); + + it('Creates a SubscriptionCreatedEvent with empty attribution object', async function () { + // Shouldn't happen, but to make sure we handle it + const attribution = {}; + await testWithAttribution(attribution); + }); + }); }); }); diff --git a/ghost/core/test/e2e-server/services/member-attribution.test.js b/ghost/core/test/e2e-server/services/member-attribution.test.js new file mode 100644 index 0000000000..e8f2f4c7c7 --- /dev/null +++ b/ghost/core/test/e2e-server/services/member-attribution.test.js @@ -0,0 +1,96 @@ +const {agentProvider, fixtureManager} = require('../../utils/e2e-framework'); +const should = require('should'); +const nock = require('nock'); +const models = require('../../../core/server/models'); +const urlService = require('../../../core/server/services/url'); +const memberAttributionService = require('../../../core/server/services/member-attribution'); + +describe('Member Attribution Service', function () { + before(async function () { + await agentProvider.getAdminAPIAgent(); + await fixtureManager.init('posts'); + }); + + afterEach(function () { + nock.cleanAll(); + }); + + /** + * Test that getAttribution correctly resolves all model types that are supported + */ + describe('getAttribution for models', function () { + it('resolves posts', async function () { + const id = fixtureManager.get('posts', 0).id; + const post = await models.Post.where('id', id).fetch({require: true}); + const url = urlService.getUrlByResourceId(post.id, {absolute: false}); + + const attribution = memberAttributionService.service.getAttribution([ + { + path: url, + time: 123 + } + ]); + attribution.should.eql(({ + id: post.id, + url, + type: 'post' + })); + }); + + it('resolves pages', async function () { + const id = fixtureManager.get('posts', 5).id; + const post = await models.Post.where('id', id).fetch({require: true}); + should(post.get('type')).eql('page'); + + const url = urlService.getUrlByResourceId(post.id, {absolute: false}); + + const attribution = memberAttributionService.service.getAttribution([ + { + path: url, + time: 123 + } + ]); + attribution.should.eql(({ + id: post.id, + url, + type: 'page' + })); + }); + + it('resolves tags', async function () { + const id = fixtureManager.get('tags', 0).id; + const tag = await models.Tag.where('id', id).fetch({require: true}); + const url = urlService.getUrlByResourceId(tag.id, {absolute: false}); + + const attribution = memberAttributionService.service.getAttribution([ + { + path: url, + time: 123 + } + ]); + attribution.should.eql(({ + id: tag.id, + url, + type: 'tag' + })); + }); + + it('resolves authors', async function () { + const id = fixtureManager.get('users', 0).id; + const author = await models.User.where('id', id).fetch({require: true}); + const url = urlService.getUrlByResourceId(author.id, {absolute: false}); + + const attribution = memberAttributionService.service.getAttribution([ + { + path: url, + time: 123 + } + ]); + attribution.should.eql(({ + id: author.id, + url, + type: 'author' + })); + }); + }); +}); diff --git a/ghost/core/test/regression/api/admin/members-importer.test.js b/ghost/core/test/regression/api/admin/members-importer.test.js index 571c5d29ff..a6ea865bf4 100644 --- a/ghost/core/test/regression/api/admin/members-importer.test.js +++ b/ghost/core/test/regression/api/admin/members-importer.test.js @@ -19,7 +19,6 @@ describe('Members Importer API', function () { beforeEach(function () { mockManager.mockMail(); - mockManager.mockLabsEnabled('members'); }); afterEach(function () { diff --git a/ghost/core/test/regression/api/admin/members-signin-url.test.js b/ghost/core/test/regression/api/admin/members-signin-url.test.js index a0f89d344c..a9d27bda5a 100644 --- a/ghost/core/test/regression/api/admin/members-signin-url.test.js +++ b/ghost/core/test/regression/api/admin/members-signin-url.test.js @@ -9,10 +9,6 @@ const {mockManager} = require('../../../utils/e2e-framework'); let request; describe('Members Sigin URL API', function () { - beforeEach(function () { - mockManager.mockLabsEnabled('members'); - }); - afterEach(function () { mockManager.restore(); }); diff --git a/ghost/core/test/utils/e2e-framework-mock-manager.js b/ghost/core/test/utils/e2e-framework-mock-manager.js index 560e42583e..9c278a408b 100644 --- a/ghost/core/test/utils/e2e-framework-mock-manager.js +++ b/ghost/core/test/utils/e2e-framework-mock-manager.js @@ -93,6 +93,8 @@ const sentEmail = (matchers) => { assert.equal(spyCall.args[0][key], value, `Expected Email ${emailCount} to have ${key} of ${value}`); }); + + return spyCall.args[0]; }; /** diff --git a/ghost/member-attribution/.eslintrc.js b/ghost/member-attribution/.eslintrc.js new file mode 100644 index 0000000000..c9c1bcb522 --- /dev/null +++ b/ghost/member-attribution/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/node' + ] +}; diff --git a/ghost/member-attribution/README.md b/ghost/member-attribution/README.md new file mode 100644 index 0000000000..e42b1f12cc --- /dev/null +++ b/ghost/member-attribution/README.md @@ -0,0 +1,21 @@ +# Member Attribution + + +## Usage + + +## Develop + +This is a monorepo package. + +Follow the instructions for the top-level repo. +1. `git clone` this repo & `cd` into it as usual +2. Run `yarn` to install top-level dependencies. + + + +## Test + +- `yarn lint` run just eslint +- `yarn test` run lint and tests + diff --git a/ghost/member-attribution/index.js b/ghost/member-attribution/index.js new file mode 100644 index 0000000000..ff50ccb46e --- /dev/null +++ b/ghost/member-attribution/index.js @@ -0,0 +1 @@ +module.exports = require('./lib/service'); diff --git a/ghost/member-attribution/lib/attribution.js b/ghost/member-attribution/lib/attribution.js new file mode 100644 index 0000000000..ea1b8703d3 --- /dev/null +++ b/ghost/member-attribution/lib/attribution.js @@ -0,0 +1,71 @@ +/** + * @typedef {object} Attribution + * @prop {string|null} [id] + * @prop {string|null} [url] + * @prop {string} [type] + */ + +/** + * Convert a UrlHistory to an attribution object + */ +class AttributionBuilder { + /** + */ + constructor({urlTranslator}) { + this.urlTranslator = urlTranslator; + } + + /** + * Last Post Algorithm™️ + * @param {UrlHistory} history + * @returns {Attribution} + */ + getAttribution(history) { + if (history.length === 0) { + return { + id: null, + url: null, + type: null + }; + } + + // TODO: if something is wrong with the attribution script, and it isn't loading + // we might get out of date URLs + // so we need to check the time of each item and ignore items that are older than 24u here! + + // Start at the end. Return the first post we find + for (const item of history) { + const typeId = this.urlTranslator.getTypeAndId(item.path); + + if (typeId && typeId.type === 'post') { + return { + url: item.path, + ...typeId + }; + } + } + + // No post found? + // Try page or tag or author + for (const item of history) { + const typeId = this.urlTranslator.getTypeAndId(item.path); + + if (typeId) { + return { + url: item.path, + ...typeId + }; + } + } + + // Default to last URL + // In the future we might decide to exclude certain URLs, that can happen here + return { + id: null, + url: history.last.path, + type: 'url' + }; + } +} + +module.exports = AttributionBuilder; diff --git a/ghost/member-attribution/lib/event-handler.js b/ghost/member-attribution/lib/event-handler.js new file mode 100644 index 0000000000..e34d6853d3 --- /dev/null +++ b/ghost/member-attribution/lib/event-handler.js @@ -0,0 +1,52 @@ +const {MemberCreatedEvent, SubscriptionCreatedEvent} = require('@tryghost/member-events'); + +class MemberAttributionEventHandler { + constructor({MemberCreatedEvent: MemberCreatedEventModel, SubscriptionCreatedEvent: SubscriptionCreatedEventModel, DomainEvents, labsService}) { + this._MemberCreatedEventModel = MemberCreatedEventModel; + this._SubscriptionCreatedEvent = SubscriptionCreatedEventModel; + this.DomainEvents = DomainEvents; + this.labsService = labsService; + } + + subscribe() { + this.DomainEvents.subscribe(MemberCreatedEvent, async (event) => { + let attribution = event.data.attribution; + + if (!this.labsService.isSet('memberAttribution')){ + // Prevent storing attribution + // Can replace this later with a privacy toggle + attribution = {}; + } + + await this._MemberCreatedEventModel.add({ + member_id: event.data.memberId, + created_at: event.timestamp, + attribution_id: attribution?.id ?? null, + attribution_url: attribution?.url ?? null, + attribution_type: attribution?.type ?? null, + source: event.data.source + }); + }); + + this.DomainEvents.subscribe(SubscriptionCreatedEvent, async (event) => { + let attribution = event.data.attribution; + + if (!this.labsService.isSet('memberAttribution')){ + // Prevent storing attribution + // Can replace this later with a privacy toggle + attribution = {}; + } + + await this._SubscriptionCreatedEvent.add({ + member_id: event.data.memberId, + subscription_id: event.data.subscriptionId, + created_at: event.timestamp, + attribution_id: attribution?.id ?? null, + attribution_url: attribution?.url ?? null, + attribution_type: attribution?.type ?? null + }); + }); + } +} + +module.exports = MemberAttributionEventHandler; diff --git a/ghost/member-attribution/lib/history.js b/ghost/member-attribution/lib/history.js new file mode 100644 index 0000000000..4061fa595c --- /dev/null +++ b/ghost/member-attribution/lib/history.js @@ -0,0 +1,46 @@ +/** + * @typedef {UrlHistoryItem[]} UrlHistoryArray + */ + +/** + * @typedef {Object} UrlHistoryItem + * @prop {string} path + * @prop {number} time + */ + +/** + * Represents a validated history + */ +class UrlHistory { + constructor(urlHistory) { + this.history = urlHistory && UrlHistory.isValidHistory(urlHistory) ? urlHistory : []; + } + + get length() { + return this.history.length; + } + + get last() { + if (this.length === 0) { + return undefined; + } + return this.history[this.history.length - 1]; + } + + /** + * Iterate from latest item to newest item (reversed!) + */ + *[Symbol.iterator]() { + yield* this.history.slice().reverse(); + } + + static isValidHistory(history) { + return Array.isArray(history) && !history.find(item => !this.isValidHistoryItem(item)); + } + + static isValidHistoryItem(item) { + return !!item && !!item.path && !!item.time && typeof item.path === 'string' && typeof item.time === 'number' && Number.isSafeInteger(item.time); + } +} + +module.exports = UrlHistory; diff --git a/ghost/member-attribution/lib/service.js b/ghost/member-attribution/lib/service.js new file mode 100644 index 0000000000..9878d778e0 --- /dev/null +++ b/ghost/member-attribution/lib/service.js @@ -0,0 +1,27 @@ +const MemberAttributionEventHandler = require('./event-handler'); +const DomainEvents = require('@tryghost/domain-events'); +const UrlTranslator = require('./url-translator'); +const AttributionBuilder = require('./attribution'); +const UrlHistory = require('./history'); + +class MemberAttributionService { + constructor({MemberCreatedEvent, SubscriptionCreatedEvent, urlService, labsService}) { + const eventHandler = new MemberAttributionEventHandler({MemberCreatedEvent, SubscriptionCreatedEvent, DomainEvents, labsService}); + eventHandler.subscribe(); + + const urlTranslator = new UrlTranslator({urlService}); + this.attributionBuilder = new AttributionBuilder({urlTranslator}); + } + + /** + * + * @param {import('./history').UrlHistoryArray} historyArray + * @returns {import('./attribution').Attribution} + */ + getAttribution(historyArray) { + const history = new UrlHistory(historyArray); + return this.attributionBuilder.getAttribution(history); + } +} + +module.exports = MemberAttributionService; diff --git a/ghost/member-attribution/lib/url-translator.js b/ghost/member-attribution/lib/url-translator.js new file mode 100644 index 0000000000..1ccde85de7 --- /dev/null +++ b/ghost/member-attribution/lib/url-translator.js @@ -0,0 +1,56 @@ +/** + * @typedef {Object} UrlService + * @prop {(resourceId: string) => Object} getResource + * + */ + +/** + * Translate a url into a type and id + */ +class UrlTranslator { + /** + * + * @param {Object} deps + * @param {UrlService} deps.urlService + */ + constructor({urlService}) { + this.urlService = urlService; + } + + getTypeAndId(url) { + const resource = this.urlService.getResource(url); + if (!resource) { + return; + } + + if (resource.config.type === 'posts') { + return { + type: 'post', + id: resource.data.id + }; + } + + if (resource.config.type === 'pages') { + return { + type: 'page', + id: resource.data.id + }; + } + + if (resource.config.type === 'tags') { + return { + type: 'tag', + id: resource.data.id + }; + } + + if (resource.config.type === 'authors') { + return { + type: 'author', + id: resource.data.id + }; + } + } +} + +module.exports = UrlTranslator; diff --git a/ghost/member-attribution/package.json b/ghost/member-attribution/package.json new file mode 100644 index 0000000000..f426decbb3 --- /dev/null +++ b/ghost/member-attribution/package.json @@ -0,0 +1,26 @@ +{ + "name": "@tryghost/member-attribution", + "version": "0.0.0", + "repository": "https://github.com/TryGhost/Ghost/tree/main/packages/member-attribution", + "author": "Ghost Foundation", + "private": true, + "main": "index.js", + "scripts": { + "dev": "echo \"Implement me!\"", + "test:unit": "NODE_ENV=testing c8 --all --check-coverage --reporter text --reporter cobertura mocha './test/**/*.test.js'", + "test": "yarn test:unit", + "lint:code": "eslint *.js lib/ --ext .js --cache", + "lint": "yarn lint:code && yarn lint:test", + "lint:test": "eslint -c test/.eslintrc.js test/ --ext .js --cache" + }, + "devDependencies": { + "c8": "7.12.0", + "mocha": "10.0.0", + "should": "13.2.3", + "sinon": "14.0.0" + }, + "dependencies": { + "@tryghost/domain-events": "0.0.0", + "@tryghost/member-events": "0.0.0" + } +} diff --git a/ghost/member-attribution/test/.eslintrc.js b/ghost/member-attribution/test/.eslintrc.js new file mode 100644 index 0000000000..829b601eb0 --- /dev/null +++ b/ghost/member-attribution/test/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/test' + ] +}; diff --git a/ghost/member-attribution/test/attribution.test.js b/ghost/member-attribution/test/attribution.test.js new file mode 100644 index 0000000000..74ce2fc5ad --- /dev/null +++ b/ghost/member-attribution/test/attribution.test.js @@ -0,0 +1,86 @@ +// Switch these lines once there are useful utils +// const testUtils = require('./utils'); +require('./utils'); +const UrlHistory = require('../lib/history'); +const AttributionBuilder = require('../lib/attribution'); + +describe('AttributionBuilder', function () { + let attributionBuilder; + + before(function () { + attributionBuilder = new AttributionBuilder({ + urlTranslator: { + getTypeAndId(path) { + if (path === '/my-post') { + return { + id: 123, + type: 'post' + }; + } + if (path === '/my-page') { + return { + id: 845, + type: 'page' + }; + } + return; + } + } + }); + }); + + it('Returns empty if empty history', function () { + const history = new UrlHistory([]); + should(attributionBuilder.getAttribution(history)).eql({id: null, type: null, url: null}); + }); + + it('Returns last url', function () { + const history = new UrlHistory([{path: '/not-last', time: 123}, {path: '/test', time: 123}]); + should(attributionBuilder.getAttribution(history)).eql({type: 'url', id: null, url: '/test'}); + }); + + it('Returns last post', function () { + const history = new UrlHistory([ + {path: '/my-post', time: 123}, + {path: '/test', time: 124}, + {path: '/unknown-page', time: 125} + ]); + should(attributionBuilder.getAttribution(history)).eql({type: 'post', id: 123, url: '/my-post'}); + }); + + it('Returns last post even when it found pages', function () { + const history = new UrlHistory([ + {path: '/my-post', time: 123}, + {path: '/my-page', time: 124}, + {path: '/unknown-page', time: 125} + ]); + should(attributionBuilder.getAttribution(history)).eql({type: 'post', id: 123, url: '/my-post'}); + }); + + it('Returns last page if no posts', function () { + const history = new UrlHistory([ + {path: '/other', time: 123}, + {path: '/my-page', time: 124}, + {path: '/unknown-page', time: 125} + ]); + should(attributionBuilder.getAttribution(history)).eql({type: 'page', id: 845, url: '/my-page'}); + }); + + it('Returns all null for invalid histories', function () { + const history = new UrlHistory('invalid'); + should(attributionBuilder.getAttribution(history)).eql({ + type: null, + id: null, + url: null + }); + }); + + it('Returns all null for empty histories', function () { + const history = new UrlHistory([]); + should(attributionBuilder.getAttribution(history)).eql({ + type: null, + id: null, + url: null + }); + }); +}); diff --git a/ghost/member-attribution/test/event-handler.test.js b/ghost/member-attribution/test/event-handler.test.js new file mode 100644 index 0000000000..81e0ca2cdc --- /dev/null +++ b/ghost/member-attribution/test/event-handler.test.js @@ -0,0 +1,223 @@ +// Switch these lines once there are useful utils +// const testUtils = require('./utils'); +require('./utils'); +const {MemberCreatedEvent, SubscriptionCreatedEvent} = require('@tryghost/member-events'); +const MemberAttributionEventHandler = require('../lib/event-handler'); + +describe('MemberAttributionEventHandler', function () { + describe('Constructor', function () { + it('doesn\'t throw', function () { + new MemberAttributionEventHandler({}); + }); + }); + + describe('MemberCreatedEvent handling', function () { + it('defaults to external for missing attributions', function () { + const DomainEvents = { + subscribe: (type, handler) => { + if (type === MemberCreatedEvent) { + handler(MemberCreatedEvent.create({ + memberId: '123', + source: 'test' + }, new Date(0))); + } + } + }; + + const MemberCreatedEventModel = {add: sinon.stub()}; + const labsService = {isSet: sinon.stub().returns(true)}; + const subscribeSpy = sinon.spy(DomainEvents, 'subscribe'); + const eventHandler = new MemberAttributionEventHandler({ + DomainEvents, + MemberCreatedEvent: MemberCreatedEventModel, + labsService + }); + eventHandler.subscribe(); + sinon.assert.calledOnceWithMatch(MemberCreatedEventModel.add, { + member_id: '123', + created_at: new Date(0), + source: 'test' + }); + sinon.assert.calledTwice(subscribeSpy); + }); + + it('passes custom attributions', function () { + const DomainEvents = { + subscribe: (type, handler) => { + if (type === MemberCreatedEvent) { + handler(MemberCreatedEvent.create({ + memberId: '123', + source: 'test', + attribution: { + id: '123', + type: 'post', + url: 'url' + } + }, new Date(0))); + } + } + }; + + const MemberCreatedEventModel = {add: sinon.stub()}; + const labsService = {isSet: sinon.stub().returns(true)}; + const subscribeSpy = sinon.spy(DomainEvents, 'subscribe'); + const eventHandler = new MemberAttributionEventHandler({ + DomainEvents, + MemberCreatedEvent: MemberCreatedEventModel, + labsService + }); + eventHandler.subscribe(); + sinon.assert.calledOnceWithMatch(MemberCreatedEventModel.add, { + member_id: '123', + created_at: new Date(0), + attribution_id: '123', + attribution_type: 'post', + attribution_url: 'url', + source: 'test' + }); + sinon.assert.calledTwice(subscribeSpy); + }); + + it('filters if disabled', function () { + const DomainEvents = { + subscribe: (type, handler) => { + if (type === MemberCreatedEvent) { + handler(MemberCreatedEvent.create({ + memberId: '123', + source: 'test', + attribution: { + id: '123', + type: 'post', + url: 'url' + } + }, new Date(0))); + } + } + }; + + const MemberCreatedEventModel = {add: sinon.stub()}; + const labsService = {isSet: sinon.stub().returns(false)}; + const subscribeSpy = sinon.spy(DomainEvents, 'subscribe'); + const eventHandler = new MemberAttributionEventHandler({ + DomainEvents, + MemberCreatedEvent: MemberCreatedEventModel, + labsService + }); + eventHandler.subscribe(); + sinon.assert.calledOnceWithMatch(MemberCreatedEventModel.add, { + member_id: '123', + created_at: new Date(0), + attribution_id: null, + attribution_type: null, + attribution_url: null, + source: 'test' + }); + sinon.assert.calledTwice(subscribeSpy); + }); + }); + + describe('SubscriptionCreatedEvent handling', function () { + it('defaults to external for missing attributions', function () { + const DomainEvents = { + subscribe: (type, handler) => { + if (type === SubscriptionCreatedEvent) { + handler(SubscriptionCreatedEvent.create({ + memberId: '123', + subscriptionId: '456' + }, new Date(0))); + } + } + }; + + const SubscriptionCreatedEventModel = {add: sinon.stub()}; + const labsService = {isSet: sinon.stub().returns(true)}; + const subscribeSpy = sinon.spy(DomainEvents, 'subscribe'); + const eventHandler = new MemberAttributionEventHandler({ + DomainEvents, + SubscriptionCreatedEvent: SubscriptionCreatedEventModel, + labsService + }); + eventHandler.subscribe(); + sinon.assert.calledOnceWithMatch(SubscriptionCreatedEventModel.add, { + member_id: '123', + subscription_id: '456', + created_at: new Date(0) + }); + sinon.assert.calledTwice(subscribeSpy); + }); + + it('passes custom attributions', function () { + const DomainEvents = { + subscribe: (type, handler) => { + if (type === SubscriptionCreatedEvent) { + handler(SubscriptionCreatedEvent.create({ + memberId: '123', + subscriptionId: '456', + attribution: { + id: '123', + type: 'post', + url: 'url' + } + }, new Date(0))); + } + } + }; + + const SubscriptionCreatedEventModel = {add: sinon.stub()}; + const labsService = {isSet: sinon.stub().returns(true)}; + const subscribeSpy = sinon.spy(DomainEvents, 'subscribe'); + const eventHandler = new MemberAttributionEventHandler({ + DomainEvents, + SubscriptionCreatedEvent: SubscriptionCreatedEventModel, + labsService + }); + eventHandler.subscribe(); + sinon.assert.calledOnceWithMatch(SubscriptionCreatedEventModel.add, { + member_id: '123', + subscription_id: '456', + created_at: new Date(0), + attribution_id: '123', + attribution_type: 'post', + attribution_url: 'url' + }); + sinon.assert.calledTwice(subscribeSpy); + }); + + it('filters if disabled', function () { + const DomainEvents = { + subscribe: (type, handler) => { + if (type === SubscriptionCreatedEvent) { + handler(SubscriptionCreatedEvent.create({ + memberId: '123', + subscriptionId: '456', + attribution: { + id: '123', + type: 'post', + url: 'url' + } + }, new Date(0))); + } + } + }; + + const SubscriptionCreatedEventModel = {add: sinon.stub()}; + const labsService = {isSet: sinon.stub().returns(false)}; + const subscribeSpy = sinon.spy(DomainEvents, 'subscribe'); + const eventHandler = new MemberAttributionEventHandler({ + DomainEvents, + SubscriptionCreatedEvent: SubscriptionCreatedEventModel, + labsService + }); + eventHandler.subscribe(); + sinon.assert.calledOnceWithMatch(SubscriptionCreatedEventModel.add, { + member_id: '123', + subscription_id: '456', + created_at: new Date(0), + attribution_id: null, + attribution_type: null, + attribution_url: null + }); + sinon.assert.calledTwice(subscribeSpy); + }); + }); +}); diff --git a/ghost/member-attribution/test/history.test.js b/ghost/member-attribution/test/history.test.js new file mode 100644 index 0000000000..383fa0a648 --- /dev/null +++ b/ghost/member-attribution/test/history.test.js @@ -0,0 +1,75 @@ +// Switch these lines once there are useful utils +// const testUtils = require('./utils'); +require('./utils'); +const UrlHistory = require('../lib/history'); + +describe('UrlHistory', function () { + describe('Constructor', function () { + it('sets history to empty array if invalid', function () { + const history = new UrlHistory('invalid'); + should(history.history).eql([]); + }); + it('sets history to empty array if missing', function () { + const history = new UrlHistory(); + should(history.history).eql([]); + }); + }); + + describe('Validation', function () { + it('isValidHistory returns false for non arrays', function () { + should(UrlHistory.isValidHistory('string')).eql(false); + should(UrlHistory.isValidHistory()).eql(false); + should(UrlHistory.isValidHistory(12)).eql(false); + should(UrlHistory.isValidHistory(null)).eql(false); + should(UrlHistory.isValidHistory({})).eql(false); + should(UrlHistory.isValidHistory(NaN)).eql(false); + + should(UrlHistory.isValidHistory([ + { + time: 1, + path: '/test' + }, + 't' + ])).eql(false); + }); + + it('isValidHistory returns true for valid arrays', function () { + should(UrlHistory.isValidHistory([])).eql(true); + should(UrlHistory.isValidHistory([ + { + time: 1, + path: '/test' + } + ])).eql(true); + }); + + it('isValidHistoryItem returns false for invalid objects', function () { + should(UrlHistory.isValidHistoryItem({})).eql(false); + should(UrlHistory.isValidHistoryItem('test')).eql(false); + should(UrlHistory.isValidHistoryItem(0)).eql(false); + should(UrlHistory.isValidHistoryItem()).eql(false); + should(UrlHistory.isValidHistoryItem(NaN)).eql(false); + should(UrlHistory.isValidHistoryItem([])).eql(false); + + should(UrlHistory.isValidHistoryItem({ + time: 'test', + path: 'test' + })).eql(false); + + should(UrlHistory.isValidHistoryItem({ + path: 'test' + })).eql(false); + + should(UrlHistory.isValidHistoryItem({ + time: 123 + })).eql(false); + }); + + it('isValidHistoryItem returns true for valid objects', function () { + should(UrlHistory.isValidHistoryItem({ + time: 123, + path: '/time' + })).eql(true); + }); + }); +}); diff --git a/ghost/member-attribution/test/service.test.js b/ghost/member-attribution/test/service.test.js new file mode 100644 index 0000000000..3abea1eb0b --- /dev/null +++ b/ghost/member-attribution/test/service.test.js @@ -0,0 +1,12 @@ +// Switch these lines once there are useful utils +// const testUtils = require('./utils'); +require('./utils'); +const MemberAttributionService = require('../lib/service'); + +describe('MemberAttributionService', function () { + describe('Constructor', function () { + it('doesn\'t throw', function () { + new MemberAttributionService({}); + }); + }); +}); diff --git a/ghost/member-attribution/test/url-translator.test.js b/ghost/member-attribution/test/url-translator.test.js new file mode 100644 index 0000000000..836da4899e --- /dev/null +++ b/ghost/member-attribution/test/url-translator.test.js @@ -0,0 +1,74 @@ +// Switch these lines once there are useful utils +// const testUtils = require('./utils'); +require('./utils'); +const UrlTranslator = require('../lib/url-translator'); + +describe('UrlTranslator', function () { + describe('Constructor', function () { + it('doesn\'t throw', function () { + new UrlTranslator({}); + }); + }); + + describe('getTypeAndId', function () { + let translator; + before(function () { + translator = new UrlTranslator({ + urlService: { + getResource: (path) => { + switch (path) { + case '/post': return { + config: {type: 'posts'}, + data: {id: 'post'} + }; + case '/tag': return { + config: {type: 'tags'}, + data: {id: 'tag'} + }; + case '/page': return { + config: {type: 'pages'}, + data: {id: 'page'} + }; + case '/author': return { + config: {type: 'authors'}, + data: {id: 'author'} + }; + } + } + } + }); + }); + + it('returns posts', function () { + should(translator.getTypeAndId('/post')).eql({ + type: 'post', + id: 'post' + }); + }); + + it('returns pages', function () { + should(translator.getTypeAndId('/page')).eql({ + type: 'page', + id: 'page' + }); + }); + + it('returns authors', function () { + should(translator.getTypeAndId('/author')).eql({ + type: 'author', + id: 'author' + }); + }); + + it('returns tags', function () { + should(translator.getTypeAndId('/tag')).eql({ + type: 'tag', + id: 'tag' + }); + }); + + it('returns undefined', function () { + should(translator.getTypeAndId('/other')).eql(undefined); + }); + }); +}); diff --git a/ghost/member-attribution/test/utils/assertions.js b/ghost/member-attribution/test/utils/assertions.js new file mode 100644 index 0000000000..7364ee8aa1 --- /dev/null +++ b/ghost/member-attribution/test/utils/assertions.js @@ -0,0 +1,11 @@ +/** + * Custom Should Assertions + * + * Add any custom assertions to this file. + */ + +// Example Assertion +// should.Assertion.add('ExampleAssertion', function () { +// this.params = {operator: 'to be a valid Example Assertion'}; +// this.obj.should.be.an.Object; +// }); diff --git a/ghost/member-attribution/test/utils/index.js b/ghost/member-attribution/test/utils/index.js new file mode 100644 index 0000000000..0d67d86ff8 --- /dev/null +++ b/ghost/member-attribution/test/utils/index.js @@ -0,0 +1,11 @@ +/** + * Test Utilities + * + * Shared utils for writing tests + */ + +// Require overrides - these add globals for tests +require('./overrides'); + +// Require assertions - adds custom should assertions +require('./assertions'); diff --git a/ghost/member-attribution/test/utils/overrides.js b/ghost/member-attribution/test/utils/overrides.js new file mode 100644 index 0000000000..90203424ee --- /dev/null +++ b/ghost/member-attribution/test/utils/overrides.js @@ -0,0 +1,10 @@ +// This file is required before any test is run + +// Taken from the should wiki, this is how to make should global +// Should is a global in our eslint test config +global.should = require('should').noConflict(); +should.extend(); + +// Sinon is a simple case +// Sinon is a global in our eslint test config +global.sinon = require('sinon'); diff --git a/ghost/member-events/index.js b/ghost/member-events/index.js index 904828b217..a028f44adc 100644 --- a/ghost/member-events/index.js +++ b/ghost/member-events/index.js @@ -1,4 +1,5 @@ module.exports = { + MemberCreatedEvent: require('./lib/MemberCreatedEvent'), MemberEntryViewEvent: require('./lib/MemberEntryViewEvent'), MemberSubscribeEvent: require('./lib/MemberSubscribeEvent'), MemberUnsubscribeEvent: require('./lib/MemberUnsubscribeEvent'), diff --git a/ghost/member-events/lib/MemberCreatedEvent.js b/ghost/member-events/lib/MemberCreatedEvent.js new file mode 100644 index 0000000000..9368b4d4a9 --- /dev/null +++ b/ghost/member-events/lib/MemberCreatedEvent.js @@ -0,0 +1,25 @@ +/** + * @typedef {object} MemberCreatedEventData + * @prop {string} memberId + * @prop {string} source + * @prop {import('@tryghost/member-attribution/lib/attribution').Attribution} [attribution] Attribution + */ + +module.exports = class MemberCreatedEvent { + /** + * @param {MemberCreatedEventData} data + * @param {Date} timestamp + */ + constructor(data, timestamp) { + this.data = data; + this.timestamp = timestamp; + } + + /** + * @param {MemberCreatedEventData} data + * @param {Date} [timestamp] + */ + static create(data, timestamp) { + return new MemberCreatedEvent(data, timestamp ?? new Date); + } +}; diff --git a/ghost/member-events/lib/SubscriptionCreatedEvent.js b/ghost/member-events/lib/SubscriptionCreatedEvent.js index 1ecf614452..30f242a35e 100644 --- a/ghost/member-events/lib/SubscriptionCreatedEvent.js +++ b/ghost/member-events/lib/SubscriptionCreatedEvent.js @@ -3,6 +3,7 @@ * @prop {string} memberId * @prop {string} subscriptionId * @prop {string} offerId + * @prop {import('@tryghost/member-attribution/lib/attribution').Attribution} [attribution] */ module.exports = class SubscriptionCreatedEvent { @@ -20,6 +21,6 @@ module.exports = class SubscriptionCreatedEvent { * @param {Date} [timestamp] */ static create(data, timestamp) { - return new SubscriptionCreatedEvent(data, timestamp || new Date); + return new SubscriptionCreatedEvent(data, timestamp ?? new Date); } }; diff --git a/ghost/member-events/package.json b/ghost/member-events/package.json index 7a1eb8ac82..b2040ee66b 100644 --- a/ghost/member-events/package.json +++ b/ghost/member-events/package.json @@ -19,6 +19,7 @@ "c8": "7.12.0", "mocha": "10.0.0", "should": "13.2.3", - "sinon": "14.0.0" + "sinon": "14.0.0", + "@tryghost/member-attribution": "0.0.0" } } diff --git a/ghost/members-api/lib/MembersAPI.js b/ghost/members-api/lib/MembersAPI.js index d85d0df47f..4ebd8548f7 100644 --- a/ghost/members-api/lib/MembersAPI.js +++ b/ghost/members-api/lib/MembersAPI.js @@ -59,7 +59,8 @@ module.exports = function MembersAPI({ stripeAPIService, offersAPI, labsService, - newslettersService + newslettersService, + memberAttributionService }) { const tokenService = new TokenService({ privateKey, @@ -162,6 +163,7 @@ module.exports = function MembersAPI({ stripeAPIService, tokenService, sendEmailWithMagicLink, + memberAttributionService, labsService }); @@ -184,12 +186,16 @@ module.exports = function MembersAPI({ return magicLinkService.sendMagicLink({email, type, tokenData: Object.assign({email}, tokenData), referrer}); } - function getMagicLink(email) { - return magicLinkService.getMagicLink({tokenData: {email}, type: 'signin'}); + function getMagicLink(email, tokenData = {}) { + return magicLinkService.getMagicLink({tokenData: {email, ...tokenData}, type: 'signin'}); + } + + async function getTokenDataFromMagicLinkToken(token) { + return await magicLinkService.getDataFromToken(token); } async function getMemberDataFromMagicLinkToken(token) { - const {email, labels = [], name = '', oldEmail, newsletters} = await magicLinkService.getDataFromToken(token); + const {email, labels = [], name = '', oldEmail, newsletters, attribution} = await getTokenDataFromMagicLinkToken(token); if (!email) { return null; } @@ -205,7 +211,7 @@ module.exports = function MembersAPI({ } return member; } - const newMember = await users.create({name, email, labels, newsletters}); + const newMember = await users.create({name, email, labels, newsletters, attribution}); await MemberLoginEvent.add({member_id: newMember.id}); return getMemberIdentityData(email); } @@ -311,6 +317,9 @@ module.exports = function MembersAPI({ members: users, memberBREADService, events: eventRepository, - productRepository + productRepository, + + // Test helpers + getTokenDataFromMagicLinkToken }; }; diff --git a/ghost/members-api/lib/controllers/router.js b/ghost/members-api/lib/controllers/router.js index 613cec4ebf..171acb7ef7 100644 --- a/ghost/members-api/lib/controllers/router.js +++ b/ghost/members-api/lib/controllers/router.js @@ -25,6 +25,7 @@ module.exports = class RouterController { * @param {() => boolean} deps.allowSelfSignup * @param {any} deps.magicLinkService * @param {import('@tryghost/members-stripe-service')} deps.stripeAPIService + * @param {import('@tryghost/member-attribution')} deps.memberAttributionService * @param {any} deps.tokenService * @param {{isSet(name: string): boolean}} deps.labsService */ @@ -38,6 +39,7 @@ module.exports = class RouterController { magicLinkService, stripeAPIService, tokenService, + memberAttributionService, sendEmailWithMagicLink, labsService }) { @@ -51,6 +53,7 @@ module.exports = class RouterController { this._stripeAPIService = stripeAPIService; this._tokenService = tokenService; this._sendEmailWithMagicLink = sendEmailWithMagicLink; + this._memberAttributionService = memberAttributionService; this.labsService = labsService; } @@ -135,7 +138,7 @@ module.exports = class RouterController { const cadence = req.body.cadence; const identity = req.body.identity; const offerId = req.body.offerId; - const metadata = req.body.metadata; + const metadata = req.body.metadata ?? {}; if (!ghostPriceId && !offerId && !tierId && !cadence) { throw new BadRequestError({ @@ -195,6 +198,33 @@ module.exports = class RouterController { metadata.offer = offer.id; } + // Don't allow to set the source manually + delete metadata.attribution_id; + delete metadata.attribution_url; + delete metadata.attribution_type; + + if (metadata.urlHistory) { + // The full attribution history doesn't fit in the Stripe metadata (can't store objects + limited to 50 keys and 500 chars values) + // So we need to add top-level attributes with string values + const urlHistory = metadata.urlHistory; + delete metadata.urlHistory; + + const attribution = this._memberAttributionService.getAttribution(urlHistory); + + // Don't set null properties + if (attribution.id) { + metadata.attribution_id = attribution.id; + } + + if (attribution.url) { + metadata.attribution_url = attribution.url; + } + + if (attribution.type) { + metadata.attribution_type = attribution.type; + } + } + if (!ghostPriceId) { const tier = await this._productRepository.get({id: tierId}); if (tier) { @@ -253,7 +283,12 @@ module.exports = class RouterController { if (!memberExistsForCustomer) { successUrl = await this._magicLinkService.getMagicLink({ tokenData: { - email: req.body.customerEmail + email: req.body.customerEmail, + attribution: { + id: metadata.attribution_id ?? null, + type: metadata.attribution_type ?? null, + url: metadata.attribution_url ?? null + } }, type: 'signup' }); @@ -360,6 +395,10 @@ module.exports = class RouterController { } } else { const tokenData = _.pick(req.body, ['labels', 'name', 'newsletters']); + + // Save attribution data in the tokenData + tokenData.attribution = this._memberAttributionService.getAttribution(req.body.urlHistory); + await this._sendEmailWithMagicLink({email, tokenData, requestedType: emailType, referrer: req.get('referer')}); } res.writeHead(201); diff --git a/ghost/members-api/lib/repositories/member.js b/ghost/members-api/lib/repositories/member.js index ebc9a4bb01..1a383625d2 100644 --- a/ghost/members-api/lib/repositories/member.js +++ b/ghost/members-api/lib/repositories/member.js @@ -3,7 +3,7 @@ const errors = require('@tryghost/errors'); const logging = require('@tryghost/logging'); const tpl = require('@tryghost/tpl'); const DomainEvents = require('@tryghost/domain-events'); -const {SubscriptionCreatedEvent, MemberSubscribeEvent} = require('@tryghost/member-events'); +const {MemberCreatedEvent, SubscriptionCreatedEvent, MemberSubscribeEvent} = require('@tryghost/member-events'); const ObjectId = require('bson-objectid'); const {NotFoundError} = require('@tryghost/errors'); @@ -167,6 +167,22 @@ module.exports = class MemberRepository { }, options); } + /** + * Create a member + * @param {Object} data + * @param {string} data.email + * @param {string} [data.name] + * @param {string} [data.note] + * @param {(string|Object)[]} [data.labels] + * @param {boolean} [data.subscribed] (deprecated) + * @param {string} [data.geolocation] + * @param {Date} [data.created_at] + * @param {Object[]} [data.products] + * @param {Object[]} [data.newsletters] + * @param {import('@tryghost/member-attribution/lib/history').Attribution} [data.attribution] + * @param {*} options + * @returns + */ async create(data, options) { if (!options) { options = {}; @@ -280,6 +296,12 @@ module.exports = class MemberRepository { }, eventData.created_at)); } + DomainEvents.dispatch(MemberCreatedEvent.create({ + memberId: member.id, + attribution: data.attribution, + source + }, eventData.created_at)); + return member; } diff --git a/ghost/members-api/package.json b/ghost/members-api/package.json index 07ea3bc170..57579d253b 100644 --- a/ghost/members-api/package.json +++ b/ghost/members-api/package.json @@ -24,7 +24,8 @@ "mocha": "10.0.0", "nock": "13.2.9", "should": "13.2.3", - "sinon": "14.0.0" + "sinon": "14.0.0", + "@tryghost/member-attribution": "0.0.0" }, "dependencies": { "@tryghost/domain-events": "0.0.0", diff --git a/ghost/stripe/lib/WebhookController.js b/ghost/stripe/lib/WebhookController.js index a71582107d..bb9cd3ac7f 100644 --- a/ghost/stripe/lib/WebhookController.js +++ b/ghost/stripe/lib/WebhookController.js @@ -225,9 +225,16 @@ module.exports = class WebhookController { if (!member) { const metadataName = _.get(session, 'metadata.name'); const metadataNewsletters = _.get(session, 'metadata.newsletters'); + const attribution = { + id: session.metadata.attribution_id ?? null, + url: session.metadata.attribution_url ?? null, + type: session.metadata.attribution_type ?? null + }; + const payerName = _.get(customer, 'subscriptions.data[0].default_payment_method.billing_details.name'); const name = metadataName || payerName || null; - const memberData = {email: customer.email, name}; + + const memberData = {email: customer.email, name, attribution}; if (metadataNewsletters) { try { memberData.newsletters = JSON.parse(metadataNewsletters); @@ -272,10 +279,16 @@ module.exports = class WebhookController { const subscription = await this.deps.memberRepository.getSubscriptionByStripeID(session.subscription); + // TODO: should we check if we don't send this event multiple times if Stripe calls the webhook multiple times? const event = SubscriptionCreatedEvent.create({ memberId: member.id, subscriptionId: subscription.id, - offerId: session.metadata.offer || null + offerId: session.metadata.offer || null, + attribution: { + id: session.metadata.attribution_id ?? null, + url: session.metadata.attribution_url ?? null, + type: session.metadata.attribution_type ?? null + } }); DomainEvents.dispatch(event);