From 076e3c02b25827e715a6018ee0babfcd8ab332f3 Mon Sep 17 00:00:00 2001 From: Simon Backx Date: Thu, 27 Oct 2022 11:44:19 +0200 Subject: [PATCH] Added linking between member and subscription created events (#15693) fixes https://github.com/TryGhost/Team/issues/2160 - Adds a `batch_id` to both events that contain the same ID if they were created at the same time. - Removes duplicate signup/conversion events using the batch_id - Requires an update in mongo-knex to work (refs https://ghost.slack.com/archives/C02G9E68C/p1666773313272409?thread_ts=1666767872.375009&cid=C02G9E68C) - Some dependencies needed an update to load the latest mongo-knex - Added tiers to membersUtils, loaded on startup (we can start to use this instead of fetching it every time) --- ghost/admin/app/components/modal-tier.js | 4 ++ .../components/posts/post-activity-feed.hbs | 11 ++- .../settings/members/stripe-settings-form.js | 3 + ghost/admin/app/helpers/parse-member-event.js | 28 +++++--- ghost/admin/app/services/members-utils.js | 22 ++++++ ghost/admin/app/services/session.js | 4 +- ghost/admin/package.json | 6 +- .../output/mappers/activity-feed-events.js | 7 ++ ...-49-add-batch-id-members-created-events.js | 7 ++ ...dd-batch-id-subscription-created-events.js | 7 ++ ...50-member-subscription-created-batch-id.js | 72 +++++++++++++++++++ ghost/core/core/server/data/schema/schema.js | 6 +- .../server/models/member-created-event.js | 23 ++++++ .../models/member-paid-subscription-event.js | 4 ++ .../models/subscription-created-event.js | 7 ++ ghost/core/package.json | 4 +- .../__snapshots__/activity-feed.test.js.snap | 4 +- .../__snapshots__/webhooks.test.js.snap | 2 +- .../unit/server/data/schema/integrity.test.js | 2 +- .../package.json | 2 +- ghost/link-tracking/package.json | 2 +- ghost/member-events/lib/MemberCreatedEvent.js | 1 + .../lib/SubscriptionCreatedEvent.js | 1 + ghost/members-api/lib/repositories/event.js | 32 +++++++-- ghost/members-api/lib/repositories/member.js | 16 ++++- ghost/members-api/package.json | 2 +- .../lib/event-storage.js | 6 +- ghost/offers/package.json | 2 +- yarn.lock | 58 +++++++-------- 29 files changed, 278 insertions(+), 67 deletions(-) create mode 100644 ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-49-add-batch-id-members-created-events.js create mode 100644 ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-49-add-batch-id-subscription-created-events.js create mode 100644 ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-50-member-subscription-created-batch-id.js diff --git a/ghost/admin/app/components/modal-tier.js b/ghost/admin/app/components/modal-tier.js index 4a9a59724d..be40af19a5 100644 --- a/ghost/admin/app/components/modal-tier.js +++ b/ghost/admin/app/components/modal-tier.js @@ -23,6 +23,7 @@ export default class ModalTierPrice extends ModalBase { @service feature; @service settings; @service config; + @service membersUtils; @tracked model; @tracked tier; @tracked periodVal; @@ -185,6 +186,9 @@ export default class ModalTierPrice extends ModalBase { this.hasSaved = true; yield this.confirm(); this.send('closeModal'); + + // Reload in the background (no await here) + this.membersUtils.reload(); } catch (error) { if (error === undefined) { // Validation error diff --git a/ghost/admin/app/components/posts/post-activity-feed.hbs b/ghost/admin/app/components/posts/post-activity-feed.hbs index 3992c2852c..b7dc532ba2 100644 --- a/ghost/admin/app/components/posts/post-activity-feed.hbs +++ b/ghost/admin/app/components/posts/post-activity-feed.hbs @@ -53,9 +53,14 @@ {{capitalize-first-letter parsedEvent.action}} - {{#if parsedEvent.description}} - – - {{parsedEvent.description}} + {{#if parsedEvent.info}} + {{parsedEvent.info}} + {{/if}} + {{#if (eq this.eventType "clicked")}} + {{#if (and parsedEvent.description parsedEvent.url) }} + – + {{parsedEvent.description}} + {{/if}} {{/if}} diff --git a/ghost/admin/app/components/settings/members/stripe-settings-form.js b/ghost/admin/app/components/settings/members/stripe-settings-form.js index 37b963dbf0..3921b091d3 100644 --- a/ghost/admin/app/components/settings/members/stripe-settings-form.js +++ b/ghost/admin/app/components/settings/members/stripe-settings-form.js @@ -277,6 +277,9 @@ export default class StripeSettingsForm extends Component { try { const updatedTier = yield tier.save(); + + // Reload in the background (no await here) + this.membersUtils.reload(); return updatedTier; } catch (error) { if (error.payload?.errors && error.payload.errors[0].code === 'STRIPE_NOT_CONFIGURED') { diff --git a/ghost/admin/app/helpers/parse-member-event.js b/ghost/admin/app/helpers/parse-member-event.js index 45174c4be3..1ebd9a12c3 100644 --- a/ghost/admin/app/helpers/parse-member-event.js +++ b/ghost/admin/app/helpers/parse-member-event.js @@ -6,6 +6,7 @@ import {inject as service} from '@ember/service'; export default class ParseMemberEventHelper extends Helper { @service feature; @service utils; + @service membersUtils; compute([event, hasMultipleNewsletters]) { const subject = event.data.member.name || event.data.member.email; @@ -42,10 +43,6 @@ export default class ParseMemberEventHelper extends Helper { getIcon(event) { let icon; - if (event.type === 'signup_event') { - icon = 'signed-up'; - } - if (event.type === 'login_event') { icon = 'logged-in'; } @@ -70,6 +67,10 @@ export default class ParseMemberEventHelper extends Helper { } } + if (event.type === 'signup_event' || (event.type === 'subscription_event' && event.data.type === 'created' && event.data.signup)) { + icon = 'signed-up'; + } + if (event.type === 'email_opened_event') { icon = 'opened-email'; } @@ -102,7 +103,7 @@ export default class ParseMemberEventHelper extends Helper { } getAction(event, hasMultipleNewsletters) { - if (event.type === 'signup_event') { + if (event.type === 'signup_event' || (event.type === 'subscription_event' && event.data.type === 'created' && event.data.signup)) { return 'signed up'; } @@ -248,10 +249,21 @@ export default class ParseMemberEventHelper extends Helper { if (mrrDelta === 0) { return; } - let sign = mrrDelta > 0 ? '+' : '-'; - let symbol = getSymbol(event.data.currency); - return `(MRR ${sign}${symbol}${Math.abs(mrrDelta)})`; + const symbol = getSymbol(event.data.currency); + + if (event.data.type === 'created') { + const sign = mrrDelta > 0 ? '' : '-'; + const tierName = this.membersUtils.hasMultipleTiers ? (event.data.tierName ?? 'MRR') : 'paid'; + return `(${tierName} - ${sign}${symbol}${Math.abs(mrrDelta)}/month)`; + } + const sign = mrrDelta > 0 ? '+' : '-'; + return `(MRR - ${sign}${symbol}${Math.abs(mrrDelta)})`; } + + if (event.type === 'signup_event' && this.membersUtils.paidMembersEnabled) { + return '(free)'; + } + return; } diff --git a/ghost/admin/app/services/members-utils.js b/ghost/admin/app/services/members-utils.js index eabe4df7cf..7a659f63a7 100644 --- a/ghost/admin/app/services/members-utils.js +++ b/ghost/admin/app/services/members-utils.js @@ -6,6 +6,8 @@ export default class MembersUtilsService extends Service { @service feature; @service store; + paidTiers = null; + get isMembersEnabled() { return this.settings.membersEnabled; } @@ -18,6 +20,26 @@ export default class MembersUtilsService extends Service { return this.settings.membersInviteOnly; } + get hasMultipleTiers() { + return this.paidMembersEnabled && this.paidTiers && this.paidTiers.length > 1; + } + + async fetch() { + if (this.paidTiers !== null) { + return; + } + + return this.store.query('tier', {filter: 'type:paid+active:true', limit: 'all'}).then((tiers) => { + this.paidTiers = tiers; + }); + } + + async reload() { + return this.store.query('tier', {filter: 'type:paid+active:true', limit: 'all'}).then((tiers) => { + this.paidTiers = tiers; + }); + } + /** * Note: always use paidMembersEnabled! Only use this getter for the Stripe Connection UI. */ diff --git a/ghost/admin/app/services/session.js b/ghost/admin/app/services/session.js index 078f59cec7..beaed9fecf 100644 --- a/ghost/admin/app/services/session.js +++ b/ghost/admin/app/services/session.js @@ -18,6 +18,7 @@ export default class SessionService extends ESASessionService { @service ui; @service upgradeStatus; @service whatsNew; + @service membersUtils; @tracked user = null; @@ -37,7 +38,8 @@ export default class SessionService extends ESASessionService { await RSVP.all([ this.config.fetchAuthenticated(), this.feature.fetch(), - this.settings.fetch() + this.settings.fetch(), + this.membersUtils.fetch() ]); await this.frontend.loginIfNeeded(); diff --git a/ghost/admin/package.json b/ghost/admin/package.json index 65ee83c23a..7ea90797f2 100644 --- a/ghost/admin/package.json +++ b/ghost/admin/package.json @@ -49,8 +49,8 @@ "@tryghost/limit-service": "1.2.3", "@tryghost/members-csv": "0.0.0", "@tryghost/mobiledoc-kit": "0.12.5-ghost.2", - "@tryghost/nql": "0.9.2", - "@tryghost/nql-lang": "0.3.2", + "@tryghost/nql": "0.11.0", + "@tryghost/nql-lang": "0.5.0", "@tryghost/string": "0.2.1", "@tryghost/timezone-data": "0.2.73", "autoprefixer": "9.8.6", @@ -181,4 +181,4 @@ "path-browserify": "1.0.1", "webpack": "5.74.0" } -} \ No newline at end of file +} diff --git a/ghost/core/core/server/api/endpoints/utils/serializers/output/mappers/activity-feed-events.js b/ghost/core/core/server/api/endpoints/utils/serializers/output/mappers/activity-feed-events.js index f2e1ff24fc..26c13fa2bf 100644 --- a/ghost/core/core/server/api/endpoints/utils/serializers/output/mappers/activity-feed-events.js +++ b/ghost/core/core/server/api/endpoints/utils/serializers/output/mappers/activity-feed-events.js @@ -117,6 +117,13 @@ const activityFeedMapper = (event, frame) => { if (event.data?.attribution) { event.data.attribution = serializeAttribution(event.data.attribution); } + // TODO: add dedicated mappers for other event types + if (event.data?.batch_id) { + delete event.data.batch_id; + } + if (event.data?.subscriptionCreatedEvent) { + delete event.data.subscriptionCreatedEvent; + } return event; }; diff --git a/ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-49-add-batch-id-members-created-events.js b/ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-49-add-batch-id-members-created-events.js new file mode 100644 index 0000000000..acf34fbfe2 --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-49-add-batch-id-members-created-events.js @@ -0,0 +1,7 @@ +const {createAddColumnMigration} = require('../../utils'); + +module.exports = createAddColumnMigration('members_created_events', 'batch_id', { + type: 'string', + maxlength: 24, + nullable: true +}); diff --git a/ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-49-add-batch-id-subscription-created-events.js b/ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-49-add-batch-id-subscription-created-events.js new file mode 100644 index 0000000000..a1c9792cbd --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-49-add-batch-id-subscription-created-events.js @@ -0,0 +1,7 @@ +const {createAddColumnMigration} = require('../../utils'); + +module.exports = createAddColumnMigration('members_subscription_created_events', 'batch_id', { + type: 'string', + maxlength: 24, + nullable: true +}); diff --git a/ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-50-member-subscription-created-batch-id.js b/ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-50-member-subscription-created-batch-id.js new file mode 100644 index 0000000000..5565e895f2 --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.21/2022-10-26-04-50-member-subscription-created-batch-id.js @@ -0,0 +1,72 @@ +const _ = require('lodash'); +const logging = require('@tryghost/logging'); +const ObjectId = require('bson-objectid').default; +const {createTransactionalMigration} = require('../../utils'); +const DatabaseInfo = require('@tryghost/database-info'); + +// This migration links together members_created_events and members_subscription_created_events + +module.exports = createTransactionalMigration( + async function up(knex) { + if (DatabaseInfo.isSQLite(knex)) { + logging.info('Skipped linking members_created_events and members_subscription_created_events on SQLite'); + return; + } + + // All events that happened within 15 minutes of each other will be linked + const rows = await knex('members_created_events as m') + .select('m.id as m_id', 's.id as s_id', 'm.member_id as member_id', 's.subscription_id as subscription_id') + .join('members_subscription_created_events AS s', 's.member_id', 'm.member_id') + .whereRaw('TIMESTAMPDIFF(MINUTE, s.created_at, m.created_at) between -15 and 15'); + + if (!rows.length) { + logging.info('Did not find linkable members_created_events and members_subscription_created_events'); + return; + } + + // Attach a unique id to each row + for (const row of rows) { // eslint-disable-line no-restricted-syntax + row.batch_id = ObjectId().toHexString(); + } + + // Create batches (insertBatch doesn't support the onConflict option) + const batches = _.chunk(rows, 1000); + + for (const batch of batches) { // eslint-disable-line no-restricted-syntax + // Update the members_created_events table using INSERT ON DUPLICATE KEY UPDATE trick + const response1 = await knex('members_created_events').insert(batch.map((r) => { + return { + id: r.m_id, + batch_id: r.batch_id, + member_id: r.member_id, // added to make the insert work + source: '', // added to make the insert work + created_at: knex.raw('NOW()') // added to make the insert work + }; + })).onConflict('id').merge(['batch_id']); + + if (response1[0] !== 0) { + logging.error(`Inserted ${response1[0]} members_created_events, expected 0`); + throw new Error('Rolling back'); + } + + const response2 = await knex('members_subscription_created_events').insert(batch.map((r) => { + return { + id: r.s_id, + batch_id: r.batch_id, + member_id: r.member_id, // added to make the insert work + subscription_id: r.subscription_id, // added to make the insert work + created_at: knex.raw('NOW()') // added to make the insert work + }; + })).onConflict('id').merge(['batch_id']); + + if (response2[0] !== 0) { + logging.error(`Inserted ${response1[0]} members_subscription_created_events, expected 0`); + throw new Error('Rolling back'); + } + } + logging.info(`Linked ${rows.length} members_created_events and members_subscription_created_events`); + }, + async function down() { + // noop + } +); diff --git a/ghost/core/core/server/data/schema/schema.js b/ghost/core/core/server/data/schema/schema.js index 09614af37d..3c3a6be60c 100644 --- a/ghost/core/core/server/data/schema/schema.js +++ b/ghost/core/core/server/data/schema/schema.js @@ -517,7 +517,8 @@ module.exports = { type: 'string', maxlength: 50, nullable: false, validations: { isIn: [['member', 'import', 'system', 'api', 'admin']] } - } + }, + batch_id: {type: 'string', maxlength: 24, nullable: true} }, members_cancel_events: { id: {type: 'string', maxlength: 24, nullable: false, primary: true}, @@ -697,7 +698,8 @@ module.exports = { attribution_url: {type: 'string', maxlength: 2000, nullable: true}, referrer_source: {type: 'string', maxlength: 191, nullable: true}, referrer_medium: {type: 'string', maxlength: 191, nullable: true}, - referrer_url: {type: 'string', maxlength: 2000, nullable: true} + referrer_url: {type: 'string', maxlength: 2000, nullable: true}, + batch_id: {type: 'string', maxlength: 24, nullable: true} }, offer_redemptions: { id: {type: 'string', maxlength: 24, nullable: false, primary: true}, diff --git a/ghost/core/core/server/models/member-created-event.js b/ghost/core/core/server/models/member-created-event.js index 3871f8763f..04cb008c17 100644 --- a/ghost/core/core/server/models/member-created-event.js +++ b/ghost/core/core/server/models/member-created-event.js @@ -8,6 +8,13 @@ const MemberCreatedEvent = ghostBookshelf.Model.extend({ return this.belongsTo('Member', 'member_id', 'id'); }, + /** + * The subscription created event that happend at the same time (if any) + */ + subscriptionCreatedEvent() { + return this.belongsTo('SubscriptionCreatedEvent', 'batch_id', 'batch_id'); + }, + postAttribution() { return this.belongsTo('Post', 'attribution_id', 'id'); }, @@ -18,6 +25,22 @@ const MemberCreatedEvent = ghostBookshelf.Model.extend({ tagAttribution() { return this.belongsTo('Tag', 'attribution_id', 'id'); + }, + + filterRelations() { + return { + subscriptionCreatedEvent: { + // Mongo-knex doesn't support belongsTo relations + tableName: 'members_subscription_created_events', + tableNameAs: 'subscriptionCreatedEvent', + type: 'manyToMany', + joinTable: 'members_created_events', + joinFrom: 'id', + joinToForeign: 'batch_id', + joinTo: 'batch_id', + joinType: 'leftJoin' + } + }; } }, { async edit() { diff --git a/ghost/core/core/server/models/member-paid-subscription-event.js b/ghost/core/core/server/models/member-paid-subscription-event.js index d331564812..fc5777f9b9 100644 --- a/ghost/core/core/server/models/member-paid-subscription-event.js +++ b/ghost/core/core/server/models/member-paid-subscription-event.js @@ -8,6 +8,10 @@ const MemberPaidSubscriptionEvent = ghostBookshelf.Model.extend({ return this.belongsTo('Member', 'member_id', 'id'); }, + stripeSubscription() { + return this.belongsTo('StripeCustomerSubscription', 'subscription_id', 'id'); + }, + subscriptionCreatedEvent() { return this.belongsTo('SubscriptionCreatedEvent', 'subscription_id', 'subscription_id'); }, diff --git a/ghost/core/core/server/models/subscription-created-event.js b/ghost/core/core/server/models/subscription-created-event.js index 0079cce424..7dd1dc26ad 100644 --- a/ghost/core/core/server/models/subscription-created-event.js +++ b/ghost/core/core/server/models/subscription-created-event.js @@ -8,6 +8,13 @@ const SubscriptionCreatedEvent = ghostBookshelf.Model.extend({ return this.belongsTo('Member', 'member_id', 'id'); }, + /** + * The member created event that happend at the same time (if any) + */ + memberCreatedEvent() { + return this.belongsTo('MemberCreatedEvent', 'batch_id', 'batch_id'); + }, + subscription() { return this.belongsTo('StripeCustomerSubscription', 'subscription_id', 'id'); }, diff --git a/ghost/core/package.json b/ghost/core/package.json index a85067402d..d1125e9c32 100644 --- a/ghost/core/package.json +++ b/ghost/core/package.json @@ -59,7 +59,7 @@ "@tryghost/api-framework": "0.0.0", "@tryghost/api-version-compatibility-service": "0.0.0", "@tryghost/audience-feedback": "0.0.0", - "@tryghost/bookshelf-plugins": "0.5.4", + "@tryghost/bookshelf-plugins": "0.6.0", "@tryghost/bootstrap-socket": "0.0.0", "@tryghost/color-utils": "0.1.21", "@tryghost/config-url-helpers": "1.0.3", @@ -107,7 +107,7 @@ "@tryghost/mw-session-from-token": "0.0.0", "@tryghost/mw-vhost": "0.0.0", "@tryghost/nodemailer": "0.3.29", - "@tryghost/nql": "0.9.2", + "@tryghost/nql": "0.11.0", "@tryghost/oembed-service": "0.0.0", "@tryghost/package-json": "0.0.0", "@tryghost/pretty-cli": "1.2.31", diff --git a/ghost/core/test/e2e-api/admin/__snapshots__/activity-feed.test.js.snap b/ghost/core/test/e2e-api/admin/__snapshots__/activity-feed.test.js.snap index 207254b08f..96518d38f5 100644 --- a/ghost/core/test/e2e-api/admin/__snapshots__/activity-feed.test.js.snap +++ b/ghost/core/test/e2e-api/admin/__snapshots__/activity-feed.test.js.snap @@ -81,7 +81,7 @@ exports[`Activity Feed API Can filter events by post id 2: [headers] 1`] = ` Object { "access-control-allow-origin": "http://127.0.0.1:2369", "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", - "content-length": "23206", + "content-length": "20770", "content-type": "application/json; charset=utf-8", "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/, "vary": "Accept-Version, Origin, Accept-Encoding", @@ -741,7 +741,7 @@ exports[`Activity Feed API Returns signup events in activity feed 2: [headers] 1 Object { "access-control-allow-origin": "http://127.0.0.1:2369", "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", - "content-length": "23027", + "content-length": "23155", "content-type": "application/json; charset=utf-8", "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/, "vary": "Accept-Version, Origin, Accept-Encoding", diff --git a/ghost/core/test/e2e-api/members/__snapshots__/webhooks.test.js.snap b/ghost/core/test/e2e-api/members/__snapshots__/webhooks.test.js.snap index 79df7855ee..4d32c218c7 100644 --- a/ghost/core/test/e2e-api/members/__snapshots__/webhooks.test.js.snap +++ b/ghost/core/test/e2e-api/members/__snapshots__/webhooks.test.js.snap @@ -401,7 +401,7 @@ exports[`Members API Member attribution Returns subscription created attribution Object { "access-control-allow-origin": "http://127.0.0.1:2369", "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", - "content-length": "14813", + "content-length": "7962", "content-type": "application/json; charset=utf-8", "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/, "vary": "Accept-Version, Origin, Accept-Encoding", diff --git a/ghost/core/test/unit/server/data/schema/integrity.test.js b/ghost/core/test/unit/server/data/schema/integrity.test.js index 4f08100757..09bdb7f955 100644 --- a/ghost/core/test/unit/server/data/schema/integrity.test.js +++ b/ghost/core/test/unit/server/data/schema/integrity.test.js @@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route */ describe('DB version integrity', function () { // Only these variables should need updating - const currentSchemaHash = 'f94be1265fce0bfbe5c98cb02610f9de'; + const currentSchemaHash = 'bb48c3c754f74f02a6ce020af6b0e6a7'; const currentFixturesHash = 'dcb7ba7c66b4b98d6c26a722985e756a'; const currentSettingsHash = '2978a5684a2d5fcf089f61f5d368a0c0'; const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01'; diff --git a/ghost/custom-theme-settings-service/package.json b/ghost/custom-theme-settings-service/package.json index ddd8dae6d9..2f31e70650 100644 --- a/ghost/custom-theme-settings-service/package.json +++ b/ghost/custom-theme-settings-service/package.json @@ -16,7 +16,7 @@ "lib" ], "devDependencies": { - "@tryghost/nql-lang": "0.3.2", + "@tryghost/nql-lang": "0.5.0", "c8": "7.12.0", "mocha": "10.1.0", "should": "13.2.3", diff --git a/ghost/link-tracking/package.json b/ghost/link-tracking/package.json index 2f7398ec1e..8c775804a7 100644 --- a/ghost/link-tracking/package.json +++ b/ghost/link-tracking/package.json @@ -28,7 +28,7 @@ "bson-objectid": "2.0.3", "@tryghost/errors": "1.2.18", "@tryghost/tpl": "0.1.19", - "@tryghost/nql": "0.9.2", + "@tryghost/nql": "0.11.0", "lodash": "4.17.21", "moment": "2.29.4" } diff --git a/ghost/member-events/lib/MemberCreatedEvent.js b/ghost/member-events/lib/MemberCreatedEvent.js index 66b6e6320f..7bd9c05434 100644 --- a/ghost/member-events/lib/MemberCreatedEvent.js +++ b/ghost/member-events/lib/MemberCreatedEvent.js @@ -1,6 +1,7 @@ /** * @typedef {object} MemberCreatedEventData * @prop {string} memberId + * @prop {string} batchId * @prop {'import' | 'system' | 'api' | 'admin' | 'member'} source * @prop {import('@tryghost/member-attribution/lib/attribution').Attribution} [attribution] Attribution */ diff --git a/ghost/member-events/lib/SubscriptionCreatedEvent.js b/ghost/member-events/lib/SubscriptionCreatedEvent.js index afa3479b40..e8f303fa51 100644 --- a/ghost/member-events/lib/SubscriptionCreatedEvent.js +++ b/ghost/member-events/lib/SubscriptionCreatedEvent.js @@ -2,6 +2,7 @@ * @typedef {object} SubscriptionCreatedEventData * @prop {string} source * @prop {string} memberId + * @prop {string} batchId * @prop {string} tierId * @prop {string} subscriptionId * @prop {string} offerId diff --git a/ghost/members-api/lib/repositories/event.js b/ghost/members-api/lib/repositories/event.js index d9ba3d4bbf..7137f4e72f 100644 --- a/ghost/members-api/lib/repositories/event.js +++ b/ghost/members-api/lib/repositories/event.js @@ -173,7 +173,16 @@ module.exports = class EventRepository { options = { ...options, - withRelated: ['member', 'subscriptionCreatedEvent.postAttribution', 'subscriptionCreatedEvent.userAttribution', 'subscriptionCreatedEvent.tagAttribution'], + withRelated: [ + 'member', + 'subscriptionCreatedEvent.postAttribution', + 'subscriptionCreatedEvent.userAttribution', + 'subscriptionCreatedEvent.tagAttribution', + 'subscriptionCreatedEvent.memberCreatedEvent', + + // This is rediculous, but we need the tier name (we'll be able to shorten this later when we switch to the subscriptions table) + 'stripeSubscription.stripePrice.stripeProduct.product' + ], filter: [] }; if (filters['data.created_at']) { @@ -191,12 +200,16 @@ module.exports = class EventRepository { const {data: models, meta} = await this._MemberPaidSubscriptionEvent.findPage(options); const data = models.map((model) => { + const d = { + ...model.toJSON(options), + attribution: model.get('type') === 'created' && model.related('subscriptionCreatedEvent') && model.related('subscriptionCreatedEvent').id ? this._memberAttributionService.getEventAttribution(model.related('subscriptionCreatedEvent')) : null, + signup: model.get('type') === 'created' && model.related('subscriptionCreatedEvent') && model.related('subscriptionCreatedEvent').id && model.related('subscriptionCreatedEvent').related('memberCreatedEvent') && model.related('subscriptionCreatedEvent').related('memberCreatedEvent').id ? true : false, + tierName: model.related('stripeSubscription') && model.related('stripeSubscription').related('stripePrice') && model.related('stripeSubscription').related('stripePrice').related('stripeProduct') && model.related('stripeSubscription').related('stripePrice').related('stripeProduct').related('product') ? model.related('stripeSubscription').related('stripePrice').related('stripeProduct').related('product').get('name') : null + }; + delete d.stripeSubscription; return { type: 'subscription_event', - data: { - ...model.toJSON(options), - attribution: model.get('type') === 'created' && model.related('subscriptionCreatedEvent') && model.related('subscriptionCreatedEvent').id ? this._memberAttributionService.getEventAttribution(model.related('subscriptionCreatedEvent')) : null - } + data: d }; }); @@ -300,8 +313,13 @@ module.exports = class EventRepository { async getCreatedEvents(options = {}, filters = {}) { options = { ...options, - withRelated: ['member', 'postAttribution', 'userAttribution', 'tagAttribution'], - filter: [] + withRelated: [ + 'member', + 'postAttribution', + 'userAttribution', + 'tagAttribution' + ], + filter: ['subscriptionCreatedEvent.id:null'] }; if (filters['data.created_at']) { options.filter.push(filters['data.created_at'].replace(/data.created_at:/g, 'created_at:')); diff --git a/ghost/members-api/lib/repositories/member.js b/ghost/members-api/lib/repositories/member.js index a29b73c4be..75c0afc6d2 100644 --- a/ghost/members-api/lib/repositories/member.js +++ b/ghost/members-api/lib/repositories/member.js @@ -227,6 +227,11 @@ module.exports = class MemberRepository { options = {}; } + if (!options.batch_id) { + // We'll use this to link related events + options.batch_id = ObjectId().toHexString(); + } + const {labels, stripeCustomer, offerId, attribution} = data; if (labels) { @@ -339,7 +344,7 @@ module.exports = class MemberRepository { subscription, offerId, attribution - }); + }, {batch_id: options.batch_id}); } catch (err) { if (err.code !== 'ER_DUP_ENTRY' && err.code !== 'SQLITE_CONSTRAINT') { throw err; @@ -352,6 +357,7 @@ module.exports = class MemberRepository { } this.dispatchEvent(MemberCreatedEvent.create({ memberId: member.id, + batchId: options.batch_id, attribution: data.attribution, source }, eventData.created_at), options); @@ -807,6 +813,11 @@ module.exports = class MemberRepository { }); }); } + + if (!options.batch_id) { + options.batch_id = ObjectId().toHexString(); + } + const member = await this._Member.findOne({ id: data.id }, {...options, forUpdate: true}); @@ -1010,7 +1021,8 @@ module.exports = class MemberRepository { memberId: member.id, subscriptionId: subscriptionModel.get('id'), offerId: data.offerId, - attribution: data.attribution + attribution: data.attribution, + batchId: options.batch_id }); this.dispatchEvent(event, options); } diff --git a/ghost/members-api/package.json b/ghost/members-api/package.json index 9c7fd72065..b4ae6460ae 100644 --- a/ghost/members-api/package.json +++ b/ghost/members-api/package.json @@ -36,7 +36,7 @@ "@tryghost/member-events": "0.0.0", "@tryghost/members-analytics-ingress": "0.0.0", "@tryghost/members-payments": "0.0.0", - "@tryghost/nql": "0.9.2", + "@tryghost/nql": "0.11.0", "@tryghost/tpl": "0.1.19", "@types/jsonwebtoken": "8.5.9", "body-parser": "1.20.1", diff --git a/ghost/members-events-service/lib/event-storage.js b/ghost/members-events-service/lib/event-storage.js index 82d714c594..7f98794757 100644 --- a/ghost/members-events-service/lib/event-storage.js +++ b/ghost/members-events-service/lib/event-storage.js @@ -34,7 +34,8 @@ class EventStorage { source: event.data.source, referrer_source: attribution?.referrerSource ?? null, referrer_medium: attribution?.referrerMedium ?? null, - referrer_url: attribution?.referrerUrl ?? null + referrer_url: attribution?.referrerUrl ?? null, + batch_id: event.data.batchId ?? null }); }); @@ -50,7 +51,8 @@ class EventStorage { attribution_type: attribution?.type ?? null, referrer_source: attribution?.referrerSource ?? null, referrer_medium: attribution?.referrerMedium ?? null, - referrer_url: attribution?.referrerUrl ?? null + referrer_url: attribution?.referrerUrl ?? null, + batch_id: event.data.batchId ?? null }); }); } diff --git a/ghost/offers/package.json b/ghost/offers/package.json index 2d2f1fb228..5c499ddf7b 100644 --- a/ghost/offers/package.json +++ b/ghost/offers/package.json @@ -25,7 +25,7 @@ "dependencies": { "@tryghost/domain-events": "0.0.0", "@tryghost/errors": "1.2.18", - "@tryghost/mongo-utils": "0.3.5", + "@tryghost/mongo-utils": "0.5.0", "@tryghost/string": "0.2.1", "lodash": "4.17.21" } diff --git a/yarn.lock b/yarn.lock index 0cff3f00ba..b267cadd9c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4181,14 +4181,14 @@ "@tryghost/debug" "^0.1.19" lodash "^4.17.21" -"@tryghost/bookshelf-filter@^0.4.15": - version "0.4.15" - resolved "https://registry.yarnpkg.com/@tryghost/bookshelf-filter/-/bookshelf-filter-0.4.15.tgz#1291e1754d4bc704457f791972450d303cc9855d" - integrity sha512-rpB5yR2XR3QDgbRtYzXrKlYuyHQvT5J6bZ0YSjsMieRKKNP/WigyhH/M6zKCSvtU4jHYEcv7SqldFtmf+htCyQ== +"@tryghost/bookshelf-filter@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@tryghost/bookshelf-filter/-/bookshelf-filter-0.5.0.tgz#ed9535eebec0e8362fdcc4653c54d6a3f8bc127d" + integrity sha512-OiKIuzMXFFbUo6pra+HV02wSilBC0P6ymDcx/QAN0z7rboZwEEZJsF6cULihs5YQmaKHWPNyMi3kOM3jyQLJcw== dependencies: "@tryghost/debug" "^0.1.19" "@tryghost/errors" "^1.2.18" - "@tryghost/nql" "^0.9.0" + "@tryghost/nql" "^0.11.0" "@tryghost/tpl" "^0.1.19" "@tryghost/bookshelf-has-posts@^0.1.19": @@ -4223,15 +4223,15 @@ "@tryghost/tpl" "^0.1.19" lodash "^4.17.21" -"@tryghost/bookshelf-plugins@0.5.4": - version "0.5.4" - resolved "https://registry.yarnpkg.com/@tryghost/bookshelf-plugins/-/bookshelf-plugins-0.5.4.tgz#f75c7099f4219eb73778aa8c6b8486d979edd3c4" - integrity sha512-p8Gi6E4JSg+wqtaats13BmTe6Z+CjQ/2OV3u0E+BDGvOX/Z7sifUZfBQpaQAj6G9z5WDj9DesmGXTzDBnq3BsA== +"@tryghost/bookshelf-plugins@0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@tryghost/bookshelf-plugins/-/bookshelf-plugins-0.6.0.tgz#6f037714cd381e90192bd8436a020f3299fce1e5" + integrity sha512-WK0+Ap/cSImDbka2sksNcTZ3EGgB1g7YVGhSn6wblBu8p8JEEa7rIj6XiyOWKTaINoVmdX4mbUCMiZrGoHqW6g== dependencies: "@tryghost/bookshelf-collision" "^0.1.28" "@tryghost/bookshelf-custom-query" "^0.1.15" "@tryghost/bookshelf-eager-load" "^0.1.18" - "@tryghost/bookshelf-filter" "^0.4.15" + "@tryghost/bookshelf-filter" "^0.5.0" "@tryghost/bookshelf-has-posts" "^0.1.19" "@tryghost/bookshelf-include-count" "^0.3.1" "@tryghost/bookshelf-order" "^0.1.15" @@ -4545,18 +4545,18 @@ mobiledoc-dom-renderer "0.7.0" mobiledoc-text-renderer "0.4.0" -"@tryghost/mongo-knex@^0.6.4": - version "0.6.4" - resolved "https://registry.yarnpkg.com/@tryghost/mongo-knex/-/mongo-knex-0.6.4.tgz#760d91d794cf3bf65336a00f6329fe26849f863b" - integrity sha512-249oNobgZvf2e3SI2x8F9AuaPsydgt003HbtKaMtLEGfobU9RsNlmaZNDJGXyaecukO4swymtt3aeHFTjWekyg== +"@tryghost/mongo-knex@^0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@tryghost/mongo-knex/-/mongo-knex-0.8.0.tgz#089a4948bf915a108baf62ead9f2d25d945a756c" + integrity sha512-WTsLW7Q6L/mmX9jB7DDHDIR3k7SADjdXTP0dsJAiEwevMFByEQZmGba5bc1FhkTrbNWaVgk0wEnzE/r4SwhXvw== dependencies: debug "^4.3.3" lodash "^4.17.21" -"@tryghost/mongo-utils@0.3.5", "@tryghost/mongo-utils@^0.3.5": - version "0.3.5" - resolved "https://registry.yarnpkg.com/@tryghost/mongo-utils/-/mongo-utils-0.3.5.tgz#85167cbdefaaa4924261a3a9cd6cbe94a8e5875d" - integrity sha512-ycxWoC5D1t3Us5qFK8jN7cCKPfxEwmqhbK2T1hcb7fMXOv2WBZNfbQ3MAU3uICeT8QfyM6dWHPM18QLxycIhrw== +"@tryghost/mongo-utils@0.5.0", "@tryghost/mongo-utils@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@tryghost/mongo-utils/-/mongo-utils-0.5.0.tgz#4d697b3374ec8dd3ebe206a269907aff661fa3fb" + integrity sha512-7eRb+pQdTe+NK+wQ44WUNQxC8DIPdCYg8jTp8Mk0G6BqlvaYdRtbrZU/uqeR7Z6efLiG4uPfLG7BNkNx5MjvMg== dependencies: lodash "^4.17.11" @@ -4572,21 +4572,21 @@ nodemailer-mailgun-transport "^2.1.5" nodemailer-stub-transport "^1.1.0" -"@tryghost/nql-lang@0.3.2", "@tryghost/nql-lang@^0.3.2": - version "0.3.2" - resolved "https://registry.yarnpkg.com/@tryghost/nql-lang/-/nql-lang-0.3.2.tgz#618bff56963d58b211873a4d3f86d9ff04ceb26a" - integrity sha512-KlO+x32nTi7q7HAinF1e2XyH0zeDBQOPL6TCDjV+bjDGaJ4h0GzR406/79MosoYbPE3kiMGVIrIdMeQcysGoyg== +"@tryghost/nql-lang@0.5.0", "@tryghost/nql-lang@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@tryghost/nql-lang/-/nql-lang-0.5.0.tgz#e34082997eca361a71b9d2de6ea6f7dd5a0654b6" + integrity sha512-Nxmz82Zm0uRXy0GL+Bd7G2mvjakoYbT6QDiyj6W1v90zXyzMedRfb9289lMzYX8tUG4VkoVOj9rWzFm5iSdpbw== dependencies: date-fns "^2.28.0" -"@tryghost/nql@0.9.2", "@tryghost/nql@^0.9.0": - version "0.9.2" - resolved "https://registry.yarnpkg.com/@tryghost/nql/-/nql-0.9.2.tgz#76e645ecf5927b84f4c1ef60953811e8324d404d" - integrity sha512-XFpZ/1bBGpYtoDA1bTgtjxmq//aimhngeYnKu3anKQ9Ri/UxzHP7ML9NnRhvIPAfrGKaSSb+epv6Zi41LA/d9Q== +"@tryghost/nql@0.11.0", "@tryghost/nql@^0.11.0": + version "0.11.0" + resolved "https://registry.yarnpkg.com/@tryghost/nql/-/nql-0.11.0.tgz#137a05aa3e3c733c8f646afc59dcb9eeb2c6269a" + integrity sha512-B8AZUC97NVp6atZ2Jg6mL+03AbQf5gzCkqhCgVjhHeeyQh5xvU1uIMyvuio2wq9MFWxHPD8w8cSltidpY9wTuA== dependencies: - "@tryghost/mongo-knex" "^0.6.4" - "@tryghost/mongo-utils" "^0.3.5" - "@tryghost/nql-lang" "^0.3.2" + "@tryghost/mongo-knex" "^0.8.0" + "@tryghost/mongo-utils" "^0.5.0" + "@tryghost/nql-lang" "^0.5.0" mingo "^2.2.2" "@tryghost/pretty-cli@1.2.29":