From b51e12d90f985e185ee0da8b4d05af847427887d Mon Sep 17 00:00:00 2001 From: Simon Backx Date: Tue, 26 Sep 2023 17:29:17 +0200 Subject: [PATCH] Added emails for recommendations (#18361) fixes https://github.com/TryGhost/Product/issues/3938 --- .github/scripts/dev.js | 5 +- .../core/core/server/lib/request-external.js | 9 + .../mentions/BookshelfMentionRepository.js | 3 +- .../core/server/services/mentions/service.js | 4 + .../RecommendationEnablerService.js | 2 +- .../RecommendationServiceWrapper.js | 76 ++- .../recommendation-emails.test.js.snap | 646 ++++++++++++++++++ .../services/recommendation-emails.test.js | 180 +++++ ghost/oembed-service/lib/OEmbedService.js | 7 +- .../IncomingRecommendationEmailRenderer.ts | 36 + .../src/IncomingRecommendationService.ts | 135 ++++ .../src/RecommendationService.ts | 28 +- ghost/recommendations/src/index.ts | 2 + ghost/staff-service/lib/StaffService.js | 1 - .../recommendation-received.hbs | 105 +++ .../recommendation-received.txt.js | 13 + .../lib/InMemoryMentionRepository.js | 2 +- ghost/webmentions/lib/Mention.js | 20 +- ghost/webmentions/lib/MentionCreatedEvent.js | 1 + .../lib/MentionDiscoveryService.js | 2 + ghost/webmentions/lib/MentionsAPI.js | 17 +- ghost/webmentions/test/Mention.test.js | 21 + ghost/webmentions/test/MentionsAPI.test.js | 88 ++- 23 files changed, 1349 insertions(+), 54 deletions(-) create mode 100644 ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap create mode 100644 ghost/core/test/e2e-server/services/recommendation-emails.test.js create mode 100644 ghost/recommendations/src/IncomingRecommendationEmailRenderer.ts create mode 100644 ghost/recommendations/src/IncomingRecommendationService.ts create mode 100644 ghost/staff-service/lib/email-templates/recommendation-received.hbs create mode 100644 ghost/staff-service/lib/email-templates/recommendation-received.txt.js diff --git a/.github/scripts/dev.js b/.github/scripts/dev.js index 75098599d5..bdd2bdb643 100644 --- a/.github/scripts/dev.js +++ b/.github/scripts/dev.js @@ -28,7 +28,10 @@ const COMMAND_GHOST = { command: 'nx run ghost:dev', cwd: path.resolve(__dirname, '../../ghost/core'), prefixColor: 'blue', - env: {} + env: { + // In development mode, we allow self-signed certificates (for sending webmentions and oembeds) + NODE_TLS_REJECT_UNAUTHORIZED: '0', + } }; const COMMAND_ADMIN = { diff --git a/ghost/core/core/server/lib/request-external.js b/ghost/core/core/server/lib/request-external.js index 56a7c51f1f..fc652e41b6 100644 --- a/ghost/core/core/server/lib/request-external.js +++ b/ghost/core/core/server/lib/request-external.js @@ -18,6 +18,11 @@ function isPrivateIp(addr) { } async function errorIfHostnameResolvesToPrivateIp(options) { + // Allow all requests if we are in development mode + if (config.get('env') === 'development') { + return Promise.resolve(); + } + // allow requests through to local Ghost instance const siteUrl = new URL(config.get('url')); const requestUrl = new URL(options.url.href); @@ -37,6 +42,10 @@ async function errorIfHostnameResolvesToPrivateIp(options) { } async function errorIfInvalidUrl(options) { + if (config.get('env') === 'development') { + return Promise.resolve(); + } + if (!options.url.hostname || !validator.isURL(options.url.hostname)) { throw new errors.InternalServerError({ message: 'URL invalid.', diff --git a/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js b/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js index 71b9bf6bf3..7f77f764f9 100644 --- a/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js +++ b/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js @@ -57,7 +57,8 @@ module.exports = class BookshelfMentionRepository { sourceExcerpt: model.get('source_excerpt'), sourceFavicon: model.get('source_favicon'), sourceFeaturedImage: model.get('source_featured_image'), - verified: model.get('verified') + verified: model.get('verified'), + deleted: model.get('deleted') }); } diff --git a/ghost/core/core/server/services/mentions/service.js b/ghost/core/core/server/services/mentions/service.js index 04dca0fc7b..7bf8ea9a7e 100644 --- a/ghost/core/core/server/services/mentions/service.js +++ b/ghost/core/core/server/services/mentions/service.js @@ -27,6 +27,8 @@ function getPostUrl(post) { module.exports = { /** @type {import('@tryghost/webmentions/lib/MentionsAPI')} */ api: null, + /** @type {import('./BookshelfMentionRepository')} */ + repository: null, controller: new MentionController(), metadata: new WebmentionMetadata(), /** @type {import('@tryghost/webmentions/lib/MentionSendingService')} */ @@ -41,6 +43,8 @@ module.exports = { MentionModel: models.Mention, DomainEvents }); + this.repository = repository; + const webmentionMetadata = this.metadata; const discoveryService = new MentionDiscoveryService({externalRequest}); const resourceService = new ResourceService({ diff --git a/ghost/core/core/server/services/recommendations/RecommendationEnablerService.js b/ghost/core/core/server/services/recommendations/RecommendationEnablerService.js index 69a6bfd488..ebbf84b2f3 100644 --- a/ghost/core/core/server/services/recommendations/RecommendationEnablerService.js +++ b/ghost/core/core/server/services/recommendations/RecommendationEnablerService.js @@ -14,7 +14,7 @@ module.exports = class RecommendationEnablerService { * @returns {string} */ getSetting() { - this.#settingsService.read('recommendations_enabled'); + return this.#settingsService.read('recommendations_enabled'); } /** diff --git a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js index 88139781e3..13854285ab 100644 --- a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js +++ b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js @@ -1,3 +1,7 @@ +const DomainEvents = require('@tryghost/domain-events'); +const {MentionCreatedEvent} = require('@tryghost/webmentions'); +const logging = require('@tryghost/logging'); + class RecommendationServiceWrapper { /** * @type {import('@tryghost/recommendations').RecommendationRepository} @@ -24,6 +28,11 @@ class RecommendationServiceWrapper { */ service; + /** + * @type {import('@tryghost/recommendations').IncomingRecommendationService} + */ + incomingRecommendationService; + init() { if (this.repository) { return; @@ -40,7 +49,9 @@ class RecommendationServiceWrapper { RecommendationService, RecommendationController, WellknownService, - BookshelfClickEventRepository + BookshelfClickEventRepository, + IncomingRecommendationService, + IncomingRecommendationEmailRenderer } = require('@tryghost/recommendations'); const mentions = require('../mentions'); @@ -75,26 +86,75 @@ class RecommendationServiceWrapper { wellknownService, mentionSendingService: mentions.sendingService, clickEventRepository: this.clickEventRepository, - subscribeEventRepository: this.subscribeEventRepository, - mentionsApi: mentions.api + subscribeEventRepository: this.subscribeEventRepository }); + + const mail = require('../mail'); + const mailer = new mail.GhostMailer(); + const emailService = { + async send(to, subject, html, text) { + return mailer.send({ + to, + subject, + html, + text + }); + } + }; + + this.incomingRecommendationService = new IncomingRecommendationService({ + mentionsApi: mentions.api, + recommendationService: this.service, + emailService, + async getEmailRecipients() { + const users = await models.User.getEmailAlertUsers('recommendation-received'); + return users.map((model) => { + return { + email: model.email, + slug: model.slug + }; + }); + }, + emailRenderer: new IncomingRecommendationEmailRenderer({ + staffService: require('../staff') + }) + }); + this.controller = new RecommendationController({ service: this.service }); - // eslint-disable-next-line no-console - this.service.init().catch(console.error); + this.service.init().catch(logging.error); + this.incomingRecommendationService.init().catch(logging.error); + + const PATH_SUFFIX = '/.well-known/recommendations.json'; + + function isRecommendationUrl(url) { + return url.pathname.endsWith(PATH_SUFFIX); + } // Add mapper to WebmentionMetadata mentions.metadata.addMapper((url) => { - const p = '/.well-known/recommendations.json'; - if (url.pathname.endsWith(p)) { + if (isRecommendationUrl(url)) { // Strip p const newUrl = new URL(url.toString()); - newUrl.pathname = newUrl.pathname.slice(0, -p.length); + newUrl.pathname = newUrl.pathname.slice(0, -PATH_SUFFIX.length); return newUrl; } }); + + const labs = require('../../../shared/labs'); + + // Listen for incoming webmentions + DomainEvents.subscribe(MentionCreatedEvent, async (event) => { + if (labs.isSet('recommendations')) { + // Check if this is a recommendation + if (event.data.mention.verified && isRecommendationUrl(event.data.mention.source)) { + logging.info('[INCOMING RECOMMENDATION] Received recommendation from ' + event.data.mention.source); + await this.incomingRecommendationService.sendRecommendationEmail(event.data.mention); + } + } + }); } } diff --git a/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap b/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap new file mode 100644 index 0000000000..23af02bac2 --- /dev/null +++ b/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap @@ -0,0 +1,646 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Incoming Recommendation Emails Sends a different email if we receive a recommendation back 1: [html 1] 1`] = ` +" + + + + + 💌 New recommenation + + + + + + + + + + +
  +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + +
+

Good news!

+ +

One of the sites you're recommending is now recommending you back:

+ +
+ +
+
Other Ghost Site
+
+
+ + Other Ghost Site +
+
+ +
+ + + + + + + +
+ + + + + + +
View recommendations
+
+
+

You can also copy & paste this URL into your browser:

+

http://127.0.0.1:2369/ghost/#/settings-x/recommendations

+
+

This message was sent from 127.0.0.1 to jbloggs@example.com

+
+

Don’t want to receive these emails? Manage your preferences here.

+
+
+ + + +
+ 		 
+ + +" +`; + +exports[`Incoming Recommendation Emails Sends a different email if we receive a recommendation back 2: [text 1] 1`] = ` +" +You have been recommended by Other Ghost Site. + +--- + +Sent to jbloggs@example.com from 127.0.0.1. +If you would no longer like to receive these notifications you can adjust your settings at http://127.0.0.1:2369/ghost/#/settings-x/users/show/joe-bloggs. + " +`; + +exports[`Incoming Recommendation Emails Sends a different email if we receive a recommendation back 3: [metadata 1] 1`] = ` +Object { + "subject": "Other Ghost Site recommended you", + "to": "jbloggs@example.com", +} +`; + +exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 1: [html 1] 1`] = ` +" + + + + + 💌 New recommenation + + + + + + + + + + +
  +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + +
+

Good news!

+ +

A new site is recommending you to their audience:

+ +
+ +
+
Other Ghost Site
+
+
+ + Other Ghost Site +
+
+ +
+ + + + + + + +
+ + + + + + +
Recommend back
+
+
+

You can also copy & paste this URL into your browser:

+

http://127.0.0.1:2369/ghost/#/settings-x/recommendations

+
+

This message was sent from 127.0.0.1 to jbloggs@example.com

+
+

Don’t want to receive these emails? Manage your preferences here.

+
+
+ + + +
+ 		 
+ + +" +`; + +exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 2: [html 2] 1`] = ` +" + + + + + 💌 New recommenation + + + + + + + + + + +
  +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + +
+

Good news!

+ +

A new site is recommending you to their audience:

+ +
+ +
+
Other Ghost Site
+
+
+ + Other Ghost Site +
+
+ +
+ + + + + + + +
+ + + + + + +
Recommend back
+
+
+

You can also copy & paste this URL into your browser:

+

http://127.0.0.1:2369/ghost/#/settings-x/recommendations

+
+

This message was sent from 127.0.0.1 to swellingsworth@example.com

+
+

Don’t want to receive these emails? Manage your preferences here.

+
+
+ + + +
+ 		 
+ + +" +`; + +exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 2: [text 1] 1`] = ` +" +You have been recommended by Other Ghost Site. + +--- + +Sent to jbloggs@example.com from 127.0.0.1. +If you would no longer like to receive these notifications you can adjust your settings at http://127.0.0.1:2369/ghost/#/settings-x/users/show/joe-bloggs. + " +`; + +exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 3: [metadata 1] 1`] = ` +Object { + "subject": "Other Ghost Site recommended you", + "to": "jbloggs@example.com", +} +`; + +exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 3: [text 1] 1`] = ` +" +You have been recommended by Other Ghost Site. + +--- + +Sent to jbloggs@example.com from 127.0.0.1. +If you would no longer like to receive these notifications you can adjust your settings at http://127.0.0.1:2369/ghost/#/settings-x/users/show/joe-bloggs. + " +`; + +exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 4: [metadata 1] 1`] = ` +Object { + "subject": "Other Ghost Site recommended you", + "to": "jbloggs@example.com", +} +`; diff --git a/ghost/core/test/e2e-server/services/recommendation-emails.test.js b/ghost/core/test/e2e-server/services/recommendation-emails.test.js new file mode 100644 index 0000000000..baabe37dff --- /dev/null +++ b/ghost/core/test/e2e-server/services/recommendation-emails.test.js @@ -0,0 +1,180 @@ +const {agentProvider, fixtureManager, mockManager, dbUtils} = require('../../utils/e2e-framework'); +const assert = require('assert/strict'); +const mentionsService = require('../../../core/server/services/mentions'); +const recommendationsService = require('../../../core/server/services/recommendations'); + +let agent; +const DomainEvents = require('@tryghost/domain-events'); +const {Mention} = require('@tryghost/webmentions'); +const {Recommendation} = require('@tryghost/recommendations'); + +describe('Incoming Recommendation Emails', function () { + let emailMockReceiver; + + before(async function () { + agent = await agentProvider.getAdminAPIAgent(); + await fixtureManager.init('users'); + await agent.loginAsAdmin(); + }); + + beforeEach(async function () { + emailMockReceiver = mockManager.mockMail(); + }); + + afterEach(async function () { + mockManager.restore(); + }); + + it('Sends an email if we receive a recommendation', async function () { + const webmention = await Mention.create({ + source: 'https://www.otherghostsite.com/.well-known/recommendations.json', + target: 'https://www.mysite.com/', + timestamp: new Date(), + payload: null, + resourceId: null, + resourceType: null, + sourceTitle: 'Other Ghost Site', + sourceSiteTitle: 'Other Ghost Site', + sourceAuthor: null, + sourceExcerpt: null, + sourceFavicon: null, + sourceFeaturedImage: null + }); + + // Mark it as verified + webmention.verify('{"url": "https://www.mysite.com/"}', 'application/json'); + assert.ok(webmention.verified); + + // Save to repository + await mentionsService.repository.save(webmention); + + await DomainEvents.allSettled(); + + emailMockReceiver + .assertSentEmailCount(2) + .matchHTMLSnapshot([{}], 0) + .matchHTMLSnapshot([{}], 1) + .matchPlaintextSnapshot([{}]) + .matchMetadataSnapshot(); + + const email = emailMockReceiver.getSentEmail(0); + + // Check if the site title is visible in the email + assert(email.html.includes('Other Ghost Site')); + assert(email.html.includes('Recommend back')); + assert(email.html.includes('https://www.otherghostsite.com')); + }); + + it('Sends a different email if we receive a recommendation back', async function () { + if (dbUtils.isSQLite()) { + this.skip(); + } + + // Create a recommendation to otherghostsite.com + const recommendation = Recommendation.create({ + title: `Recommendation`, + reason: `Reason`, + url: new URL(`https://www.otherghostsite.com/`), + favicon: null, + featuredImage: null, + excerpt: 'Test excerpt', + oneClickSubscribe: true, + createdAt: new Date(5000) + }); + + await recommendationsService.repository.save(recommendation); + + const webmention = await Mention.create({ + source: 'https://www.otherghostsite.com/.well-known/recommendations.json', + target: 'https://www.mysite.com/', + timestamp: new Date(), + payload: null, + resourceId: null, + resourceType: null, + sourceTitle: 'Other Ghost Site', + sourceSiteTitle: 'Other Ghost Site', + sourceAuthor: null, + sourceExcerpt: null, + sourceFavicon: null, + sourceFeaturedImage: null + }); + + // Mark it as verified + webmention.verify('{"url": "https://www.mysite.com/"}', 'application/json'); + assert.ok(webmention.verified); + + // Save to repository + await mentionsService.repository.save(webmention); + + await DomainEvents.allSettled(); + + emailMockReceiver + .assertSentEmailCount(2) + .matchHTMLSnapshot([{}]) + .matchPlaintextSnapshot([{}]) + .matchMetadataSnapshot(); + + const email = emailMockReceiver.getSentEmail(0); + + // Check if the site title is visible in the email + assert(email.html.includes('Other Ghost Site')); + assert(email.html.includes('View recommendations')); + assert(email.html.includes('https://www.otherghostsite.com')); + }); + + it('Does not send an email if we receive a normal mention', async function () { + const webmention = await Mention.create({ + source: 'https://www.otherghostsite.com/recommendations.json', + target: 'https://www.mysite.com/', + timestamp: new Date(), + payload: null, + resourceId: null, + resourceType: null, + sourceTitle: 'Other Ghost Site', + sourceSiteTitle: 'Other Ghost Site', + sourceAuthor: null, + sourceExcerpt: null, + sourceFavicon: null, + sourceFeaturedImage: null + }); + + // Mark it as verified + webmention.verify('{"url": "https://www.mysite.com/"}', 'application/json'); + assert.ok(webmention.verified); + + // Save to repository + await mentionsService.repository.save(webmention); + + await DomainEvents.allSettled(); + + mockManager.assert.sentEmailCount(0); + }); + + it('Does not send an email for an unverified webmention', async function () { + const webmention = await Mention.create({ + source: 'https://www.otherghostsite.com/.well-known/recommendations.json', + target: 'https://www.mysite.com/', + timestamp: new Date(), + payload: null, + resourceId: null, + resourceType: null, + sourceTitle: 'Other Ghost Site', + sourceSiteTitle: 'Other Ghost Site', + sourceAuthor: null, + sourceExcerpt: null, + sourceFavicon: null, + sourceFeaturedImage: null + }); + + // Mark it as verified + webmention.verify('{"url": "https://www.myste.com/"}', 'application/json'); + assert.ok(!webmention.verified); + + // Save to repository + await mentionsService.repository.save(webmention); + + await DomainEvents.allSettled(); + + mockManager.assert.sentEmailCount(0); + }); +}); diff --git a/ghost/oembed-service/lib/OEmbedService.js b/ghost/oembed-service/lib/OEmbedService.js index 5255e8eb28..064069ae18 100644 --- a/ghost/oembed-service/lib/OEmbedService.js +++ b/ghost/oembed-service/lib/OEmbedService.js @@ -232,7 +232,12 @@ class OEmbedService { let scraperResponse; try { - scraperResponse = await metascraper({html, url}); + scraperResponse = await metascraper({ + html, + url, + // In development, allow non-standard tlds + validateUrl: this.config.get('env') !== 'development' + }); } catch (err) { // Log to avoid being blind to errors happenning in metascraper logging.error(err); diff --git a/ghost/recommendations/src/IncomingRecommendationEmailRenderer.ts b/ghost/recommendations/src/IncomingRecommendationEmailRenderer.ts new file mode 100644 index 0000000000..bece0fa35c --- /dev/null +++ b/ghost/recommendations/src/IncomingRecommendationEmailRenderer.ts @@ -0,0 +1,36 @@ +import {IncomingRecommendation, EmailRecipient} from './IncomingRecommendationService'; + +type StaffService = { + api: { + emails: { + renderHTML(template: string, data: unknown): Promise, + renderText(template: string, data: unknown): Promise + } + } +} + +export class IncomingRecommendationEmailRenderer { + #staffService: StaffService; + + constructor({staffService}: {staffService: StaffService}) { + this.#staffService = staffService; + } + + async renderSubject(recommendation: IncomingRecommendation) { + return `${recommendation.siteTitle} recommended you`; + } + + async renderHTML(recommendation: IncomingRecommendation, recipient: EmailRecipient) { + return this.#staffService.api.emails.renderHTML('recommendation-received', { + recommendation, + recipient + }); + } + + async renderText(recommendation: IncomingRecommendation, recipient: EmailRecipient) { + return this.#staffService.api.emails.renderText('recommendation-received', { + recommendation, + recipient + }); + } +}; diff --git a/ghost/recommendations/src/IncomingRecommendationService.ts b/ghost/recommendations/src/IncomingRecommendationService.ts new file mode 100644 index 0000000000..08ecc1a070 --- /dev/null +++ b/ghost/recommendations/src/IncomingRecommendationService.ts @@ -0,0 +1,135 @@ +import {IncomingRecommendationEmailRenderer} from './IncomingRecommendationEmailRenderer'; +import {RecommendationService} from './RecommendationService'; +import logging from '@tryghost/logging'; + +export type IncomingRecommendation = { + title: string; + siteTitle: string|null; + url: URL; + excerpt: string|null; + favicon: URL|null; + featuredImage: URL|null; + recommendingBack: boolean; +} + +export type Report = { + startDate: Date, + endDate: Date, + recommendations: IncomingRecommendation[] +} + +type Mention = { + source: URL, + sourceTitle: string, + sourceSiteTitle: string|null, + sourceAuthor: string|null, + sourceExcerpt: string|null, + sourceFavicon: URL|null, + sourceFeaturedImage: URL|null +} + +type MentionsAPI = { + refreshMentions(options: {filter: string, limit: number|'all'}): Promise + listMentions(options: {filter: string, limit: number|'all'}): Promise<{data: Mention[]}> +} + +export type EmailRecipient = { + email: string +} + +type EmailService = { + send(to: string, subject: string, html: string, text: string): Promise +} + +export class IncomingRecommendationService { + #mentionsApi: MentionsAPI; + #recommendationService: RecommendationService; + + #emailService: EmailService; + #emailRenderer: IncomingRecommendationEmailRenderer; + #getEmailRecipients: () => Promise; + + constructor(deps: { + recommendationService: RecommendationService, + mentionsApi: MentionsAPI, + emailService: EmailService, + emailRenderer: IncomingRecommendationEmailRenderer, + getEmailRecipients: () => Promise, + }) { + this.#recommendationService = deps.recommendationService; + this.#mentionsApi = deps.mentionsApi; + this.#emailService = deps.emailService; + this.#emailRenderer = deps.emailRenderer; + this.#getEmailRecipients = deps.getEmailRecipients; + } + + async init() { + // When we boot, it is possible that we missed some webmentions from other sites recommending you + // More importantly, we might have missed some deletes which we can detect. + // So we do a slow revalidation of all incoming recommendations + // This also prevents doing multiple external fetches when doing quick reboots of Ghost after each other (requires Ghost to be up for at least 15 seconds) + if (!process.env.NODE_ENV?.startsWith('test')) { + setTimeout(() => { + logging.info('Updating incoming recommendations on boot'); + this.#updateIncomingRecommendations().catch((err) => { + logging.error('Failed to update incoming recommendations on boot', err); + }); + }, 15 * 1000 + Math.random() * 5 * 60 * 1000); + } + } + + #getMentionFilter({verified = true} = {}) { + const base = `source:~$'/.well-known/recommendations.json'`; + if (verified) { + return `${base}+verified:true`; + } + return base; + } + + async #updateIncomingRecommendations() { + // Note: we also recheck recommendations that were not verified (verification could have failed) + const filter = this.#getMentionFilter({verified: false}); + await this.#mentionsApi.refreshMentions({filter, limit: 100}); + } + + async #mentionToIncomingRecommendation(mention: Mention): Promise { + try { + const url = new URL(mention.source.toString().replace(/\/.well-known\/recommendations\.json$/, '')); + + // Check if we are also recommending this URL + const existing = await this.#recommendationService.countRecommendations({ + filter: `url:~^'${url}'` + }); + const recommendingBack = existing > 0; + + return { + title: mention.sourceTitle, + siteTitle: mention.sourceSiteTitle, + url, + excerpt: mention.sourceExcerpt, + favicon: mention.sourceFavicon, + featuredImage: mention.sourceFeaturedImage, + recommendingBack + }; + } catch (e) { + logging.error('Failed to parse mention to incoming recommendation data type', e); + } + return null; + } + + async sendRecommendationEmail(mention: Mention) { + const recommendation = await this.#mentionToIncomingRecommendation(mention); + if (!recommendation) { + return; + } + const recipients = await this.#getEmailRecipients(); + + for (const recipient of recipients) { + const subject = await this.#emailRenderer.renderSubject(recommendation); + const html = await this.#emailRenderer.renderHTML(recommendation, recipient); + const text = await this.#emailRenderer.renderText(recommendation, recipient); + + await this.#emailService.send(recipient.email, subject, html, text); + } + } +} diff --git a/ghost/recommendations/src/RecommendationService.ts b/ghost/recommendations/src/RecommendationService.ts index 9c6ef60699..fd2841ae9d 100644 --- a/ghost/recommendations/src/RecommendationService.ts +++ b/ghost/recommendations/src/RecommendationService.ts @@ -27,10 +27,6 @@ type MentionSendingService = { sendAll(options: {url: URL, links: URL[]}): Promise } -type MentionsAPI = { - refreshMentions(options: {filter: string, limit: number|'all'}): Promise -} - type RecommendationEnablerService = { getSetting(): string, setSetting(value: string): Promise @@ -48,7 +44,6 @@ export class RecommendationService { wellknownService: WellknownService; mentionSendingService: MentionSendingService; recommendationEnablerService: RecommendationEnablerService; - mentionsApi: MentionsAPI; constructor(deps: { repository: RecommendationRepository, @@ -56,8 +51,7 @@ export class RecommendationService { subscribeEventRepository: BookshelfRepository, wellknownService: WellknownService, mentionSendingService: MentionSendingService, - recommendationEnablerService: RecommendationEnablerService, - mentionsApi: MentionsAPI + recommendationEnablerService: RecommendationEnablerService }) { this.repository = deps.repository; this.wellknownService = deps.wellknownService; @@ -65,31 +59,11 @@ export class RecommendationService { this.recommendationEnablerService = deps.recommendationEnablerService; this.clickEventRepository = deps.clickEventRepository; this.subscribeEventRepository = deps.subscribeEventRepository; - this.mentionsApi = deps.mentionsApi; } async init() { const recommendations = await this.#listRecommendations(); await this.updateWellknown(recommendations); - - // When we boot, it is possible that we missed some webmentions from other sites recommending you - // More importantly, we might have missed some deletes which we can detect. - // So we do a slow revalidation of all incoming recommendations - // This also prevents doing multiple external fetches when doing quick reboots of Ghost after each other (requires Ghost to be up for at least 15 seconds) - if (!process.env.NODE_ENV?.startsWith('test')) { - setTimeout(() => { - logging.info('Updating incoming recommendations on boot'); - this.#updateIncomingRecommendations().catch((err) => { - logging.error('Failed to update incoming recommendations on boot', err); - }); - }, 15 * 1000 + Math.random() * 5 * 60 * 1000); - } - } - - async #updateIncomingRecommendations() { - // Note: we also recheck recommendations that were not verified (verification could have failed) - const filter = `source:~$'/.well-known/recommendations.json'`; - await this.mentionsApi.refreshMentions({filter, limit: 100}); } async updateWellknown(recommendations: Recommendation[]) { diff --git a/ghost/recommendations/src/index.ts b/ghost/recommendations/src/index.ts index 82cba44a2b..6f6467d81a 100644 --- a/ghost/recommendations/src/index.ts +++ b/ghost/recommendations/src/index.ts @@ -9,3 +9,5 @@ export * from './ClickEvent'; export * from './BookshelfClickEventRepository'; export * from './SubscribeEvent'; export * from './BookshelfSubscribeEventRepository'; +export * from './IncomingRecommendationService'; +export * from './IncomingRecommendationEmailRenderer'; diff --git a/ghost/staff-service/lib/StaffService.js b/ghost/staff-service/lib/StaffService.js index c4becf8369..24e67e4d21 100644 --- a/ghost/staff-service/lib/StaffService.js +++ b/ghost/staff-service/lib/StaffService.js @@ -15,7 +15,6 @@ class StaffService { const Emails = require('./StaffServiceEmails'); - /** @private */ this.emails = new Emails({ logging, models, diff --git a/ghost/staff-service/lib/email-templates/recommendation-received.hbs b/ghost/staff-service/lib/email-templates/recommendation-received.hbs new file mode 100644 index 0000000000..975116d5a7 --- /dev/null +++ b/ghost/staff-service/lib/email-templates/recommendation-received.hbs @@ -0,0 +1,105 @@ + + + + + + 💌 New recommenation + {{> styles}} + + + + + + + + + +
  +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + +
+

Good news!

+ + {{#if recommendation.recommendingBack}} +

One of the sites you're recommending is now recommending you back:

+ {{else}} +

A new site is recommending you to their audience:

+ {{/if}} + +
+ +
+
{{recommendation.title}}
+
{{recommendation.excerpt}}
+
+ {{#if recommendation.favicon}}{{/if}} + {{#if recommendation.siteTitle}}{{recommendation.siteTitle}}{{/if}} +
+
+ {{#if recommendation.featuredImage}} +
+ +
+ {{/if}} + +
+ + + + + + + +
+ + + + {{#if recommendation.recommendingBack}} + + {{else}} + + {{/if}} + + +
View recommendations Recommend back
+
+
+

You can also copy & paste this URL into your browser:

+

{{siteUrl}}ghost/#/settings-x/recommendations

+
+

This message was sent from {{siteDomain}} to {{toEmail}}

+
+

Don’t want to receive these emails? Manage your preferences here.

+
+
+ + + +
+ 		 
+ + diff --git a/ghost/staff-service/lib/email-templates/recommendation-received.txt.js b/ghost/staff-service/lib/email-templates/recommendation-received.txt.js new file mode 100644 index 0000000000..f2af8a002c --- /dev/null +++ b/ghost/staff-service/lib/email-templates/recommendation-received.txt.js @@ -0,0 +1,13 @@ +module.exports = function (data) { + const {recommendation} = data; + + // Be careful when you indent the email, because whitespaces are visible in emails! + return ` +You have been recommended by ${recommendation.siteTitle || recommendation.title || recommendation.url}. + +--- + +Sent to ${data.toEmail} from ${data.siteDomain}. +If you would no longer like to receive these notifications you can adjust your settings at ${data.staffUrl}. + `; +}; diff --git a/ghost/webmentions/lib/InMemoryMentionRepository.js b/ghost/webmentions/lib/InMemoryMentionRepository.js index b901ecde4b..bff186f3a2 100644 --- a/ghost/webmentions/lib/InMemoryMentionRepository.js +++ b/ghost/webmentions/lib/InMemoryMentionRepository.js @@ -56,7 +56,7 @@ module.exports = class InMemoryMentionRepository { */ async getBySourceAndTarget(source, target) { return this.#store.find((item) => { - return item.source.href === source.href && item.target.href === target.href && !Mention.isDeleted(item); + return item.source.href === source.href && item.target.href === target.href; }); } diff --git a/ghost/webmentions/lib/Mention.js b/ghost/webmentions/lib/Mention.js index 7c97edc9fe..5924510ce1 100644 --- a/ghost/webmentions/lib/Mention.js +++ b/ghost/webmentions/lib/Mention.js @@ -30,6 +30,14 @@ module.exports = class Mention { this.#deleted = true; } + #undelete() { + // When an earlier mention is deleted, but then it gets verified again, we need to undelete it + if (this.#deleted) { + this.#deleted = false; + this.events.push(MentionCreatedEvent.create({mention: this})); + } + } + /** * @param {string} html * @param {string} contentType @@ -44,9 +52,11 @@ module.exports = class Mention { this.#verified = hasTargetUrl; if (wasVerified && !this.#verified) { - // Delete the mention + // Delete the mention, but keep it verified (it was just deleted, because it was verified earlier, so now it is removed from the site according to the spec) this.#deleted = true; this.#verified = true; + } else { + this.#undelete(); } } catch (e) { this.#verified = false; @@ -62,9 +72,11 @@ module.exports = class Mention { this.#verified = !!html.includes(JSON.stringify(this.target.href)); if (wasVerified && !this.#verified) { - // Delete the mention + // Delete the mention, but keep it verified (it was just deleted, because it was verified earlier, so now it is removed from the site according to the spec) this.#deleted = true; this.#verified = true; + } else { + this.#undelete(); } } catch (e) { this.#verified = false; @@ -217,6 +229,7 @@ module.exports = class Mention { this.#resourceId = data.resourceId; this.#resourceType = data.resourceType; this.#verified = data.verified; + this.#deleted = data.deleted || false; } /** @@ -302,7 +315,8 @@ module.exports = class Mention { payload, resourceId, resourceType, - verified + verified, + deleted: isNew ? false : !!data.deleted }); mention.setSourceMetadata(data); diff --git a/ghost/webmentions/lib/MentionCreatedEvent.js b/ghost/webmentions/lib/MentionCreatedEvent.js index 1b6483a8d1..ae3d102549 100644 --- a/ghost/webmentions/lib/MentionCreatedEvent.js +++ b/ghost/webmentions/lib/MentionCreatedEvent.js @@ -1,5 +1,6 @@ /** * @typedef {object} MentionCreatedEventData + * @property {import('./Mention')} mention */ module.exports = class MentionCreatedEvent { diff --git a/ghost/webmentions/lib/MentionDiscoveryService.js b/ghost/webmentions/lib/MentionDiscoveryService.js index 991d578116..4d4c8b04da 100644 --- a/ghost/webmentions/lib/MentionDiscoveryService.js +++ b/ghost/webmentions/lib/MentionDiscoveryService.js @@ -1,4 +1,5 @@ const cheerio = require('cheerio'); +const logging = require('@tryghost/logging'); module.exports = class MentionDiscoveryService { #externalRequest; @@ -26,6 +27,7 @@ module.exports = class MentionDiscoveryService { }); return this.getEndpointFromResponse(response); } catch (error) { + logging.error(`Error fetching ${url.href} to discover webmention endpoint`, error); return null; } } diff --git a/ghost/webmentions/lib/MentionsAPI.js b/ghost/webmentions/lib/MentionsAPI.js index 165d918d3b..05d26815c6 100644 --- a/ghost/webmentions/lib/MentionsAPI.js +++ b/ghost/webmentions/lib/MentionsAPI.js @@ -183,6 +183,7 @@ module.exports = class MentionsAPI { async #updateWebmention(mention, webmention) { const isNew = !mention; + const wasDeleted = mention?.deleted ?? false; const targetExists = await this.#routingService.pageExists(webmention.target); if (!targetExists) { @@ -235,23 +236,23 @@ module.exports = class MentionsAPI { } if (metadata?.body) { - try { - mention.verify(metadata.body, metadata.contentType); - } catch (e) { - logging.error(e); - } + mention.verify(metadata.body, metadata.contentType); } } await this.#repository.save(mention); if (isNew) { - logging.info('[Webmention] Created ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified); + logging.info('[Webmention] Created ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified + ', deleted: ' + mention.deleted); } else { - if (mention.deleted) { + if (mention.deleted && !wasDeleted) { logging.info('[Webmention] Deleted ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified); } else { - logging.info('[Webmention] Updated ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified); + if (!mention.deleted && wasDeleted) { + logging.info('[Webmention] Restored ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified); + } else { + logging.info('[Webmention] Updated ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified + ', deleted: ' + mention.deleted); + } } } diff --git a/ghost/webmentions/test/Mention.test.js b/ghost/webmentions/test/Mention.test.js index e384e88f75..1d8d9a5f48 100644 --- a/ghost/webmentions/test/Mention.test.js +++ b/ghost/webmentions/test/Mention.test.js @@ -126,6 +126,27 @@ describe('Mention', function () { }); }); + describe('undelete', function () { + afterEach(function () { + sinon.restore(); + }); + + it('can undelete a verified mention', async function () { + const mention = await Mention.create({ + ...validInput, + id: new ObjectID(), + deleted: true, + verified: true + }); + assert(mention.verified); + assert(mention.deleted); + + mention.verify('{"url": "https://target.com/"}', 'application/json'); + assert(mention.verified); + assert(!mention.isDeleted()); + }); + }); + describe('create', function () { it('Will error with invalid inputs', async function () { const invalidInputs = [ diff --git a/ghost/webmentions/test/MentionsAPI.test.js b/ghost/webmentions/test/MentionsAPI.test.js index 037d5ff0ab..aa1d062e80 100644 --- a/ghost/webmentions/test/MentionsAPI.test.js +++ b/ghost/webmentions/test/MentionsAPI.test.js @@ -29,7 +29,8 @@ const mockWebmentionMetadata = { author: 'Dr Egg Man', image: new URL('https://unsplash.com/photos/QAND9huzD04'), favicon: new URL('https://ghost.org/favicon.ico'), - body: `

Some HTML and a mentioned url

` + body: `

Some HTML and a mentioned url

`, + contentType: 'text/html' }; } }; @@ -432,7 +433,7 @@ describe('MentionsAPI', function () { } }); - it('Will delete an existing mention if the source page does not exist', async function () { + it('Will delete and restore an existing mention if the source page does not exist', async function () { const repository = new InMemoryMentionRepository(); const api = new MentionsAPI({ repository, @@ -449,6 +450,7 @@ describe('MentionsAPI', function () { fetch: sinon.stub() .onFirstCall().resolves(mockWebmentionMetadata.fetch()) .onSecondCall().rejects() + .onThirdCall().resolves(mockWebmentionMetadata.fetch()) } }); @@ -481,6 +483,88 @@ describe('MentionsAPI', function () { assert.equal(page.data.length, 0); break checkMentionDeleted; } + + checkRestored: { + const mention = await api.processWebmention({ + source: new URL('https://source.com'), + target: new URL('https://target.com'), + payload: {} + }); + + const page = await api.listMentions({ + limit: 'all' + }); + + assert.equal(page.data[0].id, mention.id); + break checkRestored; + } + }); + + it('Will delete and restore an existing mention if the target url is not present on the source page', async function () { + const repository = new InMemoryMentionRepository(); + const api = new MentionsAPI({ + repository, + routingService: mockRoutingService, + resourceService: { + async getByURL() { + return { + type: 'post', + id: new ObjectID + }; + } + }, + webmentionMetadata: { + fetch: sinon.stub() + .onFirstCall().resolves(mockWebmentionMetadata.fetch()) + .onSecondCall().resolves({...(await mockWebmentionMetadata.fetch()), body: 'test'}) + .onThirdCall().resolves(mockWebmentionMetadata.fetch()) + } + }); + + checkFirstMention: { + const mention = await api.processWebmention({ + source: new URL('https://source.com'), + target: new URL('https://target.com'), + payload: {} + }); + + const page = await api.listMentions({ + limit: 'all' + }); + + assert.equal(page.data[0].id, mention.id); + break checkFirstMention; + } + + checkMentionDeleted: { + await api.processWebmention({ + source: new URL('https://source.com'), + target: new URL('https://target.com'), + payload: {} + }); + + const page = await api.listMentions({ + limit: 'all' + }); + + assert.equal(page.data.length, 0); + break checkMentionDeleted; + } + + checkRestored: { + const mention = await api.processWebmention({ + source: new URL('https://source.com'), + target: new URL('https://target.com'), + payload: {} + }); + + const page = await api.listMentions({ + limit: 'all' + }); + + assert.equal(page.data[0].id, mention.id); + break checkRestored; + } }); it('Will throw for new mentions if the source page is not found', async function () {