diff --git a/.github/scripts/dev.js b/.github/scripts/dev.js
index 75098599d5..bdd2bdb643 100644
--- a/.github/scripts/dev.js
+++ b/.github/scripts/dev.js
@@ -28,7 +28,10 @@ const COMMAND_GHOST = {
command: 'nx run ghost:dev',
cwd: path.resolve(__dirname, '../../ghost/core'),
prefixColor: 'blue',
- env: {}
+ env: {
+ // In development mode, we allow self-signed certificates (for sending webmentions and oembeds)
+ NODE_TLS_REJECT_UNAUTHORIZED: '0',
+ }
};
const COMMAND_ADMIN = {
diff --git a/ghost/core/core/server/lib/request-external.js b/ghost/core/core/server/lib/request-external.js
index 56a7c51f1f..fc652e41b6 100644
--- a/ghost/core/core/server/lib/request-external.js
+++ b/ghost/core/core/server/lib/request-external.js
@@ -18,6 +18,11 @@ function isPrivateIp(addr) {
}
async function errorIfHostnameResolvesToPrivateIp(options) {
+ // Allow all requests if we are in development mode
+ if (config.get('env') === 'development') {
+ return Promise.resolve();
+ }
+
// allow requests through to local Ghost instance
const siteUrl = new URL(config.get('url'));
const requestUrl = new URL(options.url.href);
@@ -37,6 +42,10 @@ async function errorIfHostnameResolvesToPrivateIp(options) {
}
async function errorIfInvalidUrl(options) {
+ if (config.get('env') === 'development') {
+ return Promise.resolve();
+ }
+
if (!options.url.hostname || !validator.isURL(options.url.hostname)) {
throw new errors.InternalServerError({
message: 'URL invalid.',
diff --git a/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js b/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js
index 71b9bf6bf3..7f77f764f9 100644
--- a/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js
+++ b/ghost/core/core/server/services/mentions/BookshelfMentionRepository.js
@@ -57,7 +57,8 @@ module.exports = class BookshelfMentionRepository {
sourceExcerpt: model.get('source_excerpt'),
sourceFavicon: model.get('source_favicon'),
sourceFeaturedImage: model.get('source_featured_image'),
- verified: model.get('verified')
+ verified: model.get('verified'),
+ deleted: model.get('deleted')
});
}
diff --git a/ghost/core/core/server/services/mentions/service.js b/ghost/core/core/server/services/mentions/service.js
index 04dca0fc7b..7bf8ea9a7e 100644
--- a/ghost/core/core/server/services/mentions/service.js
+++ b/ghost/core/core/server/services/mentions/service.js
@@ -27,6 +27,8 @@ function getPostUrl(post) {
module.exports = {
/** @type {import('@tryghost/webmentions/lib/MentionsAPI')} */
api: null,
+ /** @type {import('./BookshelfMentionRepository')} */
+ repository: null,
controller: new MentionController(),
metadata: new WebmentionMetadata(),
/** @type {import('@tryghost/webmentions/lib/MentionSendingService')} */
@@ -41,6 +43,8 @@ module.exports = {
MentionModel: models.Mention,
DomainEvents
});
+ this.repository = repository;
+
const webmentionMetadata = this.metadata;
const discoveryService = new MentionDiscoveryService({externalRequest});
const resourceService = new ResourceService({
diff --git a/ghost/core/core/server/services/recommendations/RecommendationEnablerService.js b/ghost/core/core/server/services/recommendations/RecommendationEnablerService.js
index 69a6bfd488..ebbf84b2f3 100644
--- a/ghost/core/core/server/services/recommendations/RecommendationEnablerService.js
+++ b/ghost/core/core/server/services/recommendations/RecommendationEnablerService.js
@@ -14,7 +14,7 @@ module.exports = class RecommendationEnablerService {
* @returns {string}
*/
getSetting() {
- this.#settingsService.read('recommendations_enabled');
+ return this.#settingsService.read('recommendations_enabled');
}
/**
diff --git a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js
index 88139781e3..13854285ab 100644
--- a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js
+++ b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js
@@ -1,3 +1,7 @@
+const DomainEvents = require('@tryghost/domain-events');
+const {MentionCreatedEvent} = require('@tryghost/webmentions');
+const logging = require('@tryghost/logging');
+
class RecommendationServiceWrapper {
/**
* @type {import('@tryghost/recommendations').RecommendationRepository}
@@ -24,6 +28,11 @@ class RecommendationServiceWrapper {
*/
service;
+ /**
+ * @type {import('@tryghost/recommendations').IncomingRecommendationService}
+ */
+ incomingRecommendationService;
+
init() {
if (this.repository) {
return;
@@ -40,7 +49,9 @@ class RecommendationServiceWrapper {
RecommendationService,
RecommendationController,
WellknownService,
- BookshelfClickEventRepository
+ BookshelfClickEventRepository,
+ IncomingRecommendationService,
+ IncomingRecommendationEmailRenderer
} = require('@tryghost/recommendations');
const mentions = require('../mentions');
@@ -75,26 +86,75 @@ class RecommendationServiceWrapper {
wellknownService,
mentionSendingService: mentions.sendingService,
clickEventRepository: this.clickEventRepository,
- subscribeEventRepository: this.subscribeEventRepository,
- mentionsApi: mentions.api
+ subscribeEventRepository: this.subscribeEventRepository
});
+
+ const mail = require('../mail');
+ const mailer = new mail.GhostMailer();
+ const emailService = {
+ async send(to, subject, html, text) {
+ return mailer.send({
+ to,
+ subject,
+ html,
+ text
+ });
+ }
+ };
+
+ this.incomingRecommendationService = new IncomingRecommendationService({
+ mentionsApi: mentions.api,
+ recommendationService: this.service,
+ emailService,
+ async getEmailRecipients() {
+ const users = await models.User.getEmailAlertUsers('recommendation-received');
+ return users.map((model) => {
+ return {
+ email: model.email,
+ slug: model.slug
+ };
+ });
+ },
+ emailRenderer: new IncomingRecommendationEmailRenderer({
+ staffService: require('../staff')
+ })
+ });
+
this.controller = new RecommendationController({
service: this.service
});
- // eslint-disable-next-line no-console
- this.service.init().catch(console.error);
+ this.service.init().catch(logging.error);
+ this.incomingRecommendationService.init().catch(logging.error);
+
+ const PATH_SUFFIX = '/.well-known/recommendations.json';
+
+ function isRecommendationUrl(url) {
+ return url.pathname.endsWith(PATH_SUFFIX);
+ }
// Add mapper to WebmentionMetadata
mentions.metadata.addMapper((url) => {
- const p = '/.well-known/recommendations.json';
- if (url.pathname.endsWith(p)) {
+ if (isRecommendationUrl(url)) {
// Strip p
const newUrl = new URL(url.toString());
- newUrl.pathname = newUrl.pathname.slice(0, -p.length);
+ newUrl.pathname = newUrl.pathname.slice(0, -PATH_SUFFIX.length);
return newUrl;
}
});
+
+ const labs = require('../../../shared/labs');
+
+ // Listen for incoming webmentions
+ DomainEvents.subscribe(MentionCreatedEvent, async (event) => {
+ if (labs.isSet('recommendations')) {
+ // Check if this is a recommendation
+ if (event.data.mention.verified && isRecommendationUrl(event.data.mention.source)) {
+ logging.info('[INCOMING RECOMMENDATION] Received recommendation from ' + event.data.mention.source);
+ await this.incomingRecommendationService.sendRecommendationEmail(event.data.mention);
+ }
+ }
+ });
}
}
diff --git a/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap b/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap
new file mode 100644
index 0000000000..23af02bac2
--- /dev/null
+++ b/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap
@@ -0,0 +1,646 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Incoming Recommendation Emails Sends a different email if we receive a recommendation back 1: [html 1] 1`] = `
+"
+
+
+
+
+ 💌 New recommenation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Good news!
+
+ One of the sites you're recommending is now recommending you back :
+
+
+
+
+
Other Ghost Site
+
+
+
+ Other Ghost Site
+
+
+
+
+
+
+
+ You can also copy & paste this URL into your browser:
+ http://127.0.0.1:2369/ghost/#/settings-x/recommendations
+
+
+
+
+
+
+ This message was sent from 127.0.0.1 to jbloggs@example.com
+
+
+
+
+ Don’t want to receive these emails? Manage your preferences here .
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"
+`;
+
+exports[`Incoming Recommendation Emails Sends a different email if we receive a recommendation back 2: [text 1] 1`] = `
+"
+You have been recommended by Other Ghost Site.
+
+---
+
+Sent to jbloggs@example.com from 127.0.0.1.
+If you would no longer like to receive these notifications you can adjust your settings at http://127.0.0.1:2369/ghost/#/settings-x/users/show/joe-bloggs.
+ "
+`;
+
+exports[`Incoming Recommendation Emails Sends a different email if we receive a recommendation back 3: [metadata 1] 1`] = `
+Object {
+ "subject": "Other Ghost Site recommended you",
+ "to": "jbloggs@example.com",
+}
+`;
+
+exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 1: [html 1] 1`] = `
+"
+
+
+
+
+ 💌 New recommenation
+
+
+
+
+
+
+
+"
+`;
+
+exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 2: [html 2] 1`] = `
+"
+
+
+
+
+ 💌 New recommenation
+
+
+
+
+
+
+
+"
+`;
+
+exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 2: [text 1] 1`] = `
+"
+You have been recommended by Other Ghost Site.
+
+---
+
+Sent to jbloggs@example.com from 127.0.0.1.
+If you would no longer like to receive these notifications you can adjust your settings at http://127.0.0.1:2369/ghost/#/settings-x/users/show/joe-bloggs.
+ "
+`;
+
+exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 3: [metadata 1] 1`] = `
+Object {
+ "subject": "Other Ghost Site recommended you",
+ "to": "jbloggs@example.com",
+}
+`;
+
+exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 3: [text 1] 1`] = `
+"
+You have been recommended by Other Ghost Site.
+
+---
+
+Sent to jbloggs@example.com from 127.0.0.1.
+If you would no longer like to receive these notifications you can adjust your settings at http://127.0.0.1:2369/ghost/#/settings-x/users/show/joe-bloggs.
+ "
+`;
+
+exports[`Incoming Recommendation Emails Sends an email if we receive a recommendation 4: [metadata 1] 1`] = `
+Object {
+ "subject": "Other Ghost Site recommended you",
+ "to": "jbloggs@example.com",
+}
+`;
diff --git a/ghost/core/test/e2e-server/services/recommendation-emails.test.js b/ghost/core/test/e2e-server/services/recommendation-emails.test.js
new file mode 100644
index 0000000000..baabe37dff
--- /dev/null
+++ b/ghost/core/test/e2e-server/services/recommendation-emails.test.js
@@ -0,0 +1,180 @@
+const {agentProvider, fixtureManager, mockManager, dbUtils} = require('../../utils/e2e-framework');
+const assert = require('assert/strict');
+const mentionsService = require('../../../core/server/services/mentions');
+const recommendationsService = require('../../../core/server/services/recommendations');
+
+let agent;
+const DomainEvents = require('@tryghost/domain-events');
+const {Mention} = require('@tryghost/webmentions');
+const {Recommendation} = require('@tryghost/recommendations');
+
+describe('Incoming Recommendation Emails', function () {
+ let emailMockReceiver;
+
+ before(async function () {
+ agent = await agentProvider.getAdminAPIAgent();
+ await fixtureManager.init('users');
+ await agent.loginAsAdmin();
+ });
+
+ beforeEach(async function () {
+ emailMockReceiver = mockManager.mockMail();
+ });
+
+ afterEach(async function () {
+ mockManager.restore();
+ });
+
+ it('Sends an email if we receive a recommendation', async function () {
+ const webmention = await Mention.create({
+ source: 'https://www.otherghostsite.com/.well-known/recommendations.json',
+ target: 'https://www.mysite.com/',
+ timestamp: new Date(),
+ payload: null,
+ resourceId: null,
+ resourceType: null,
+ sourceTitle: 'Other Ghost Site',
+ sourceSiteTitle: 'Other Ghost Site',
+ sourceAuthor: null,
+ sourceExcerpt: null,
+ sourceFavicon: null,
+ sourceFeaturedImage: null
+ });
+
+ // Mark it as verified
+ webmention.verify('{"url": "https://www.mysite.com/"}', 'application/json');
+ assert.ok(webmention.verified);
+
+ // Save to repository
+ await mentionsService.repository.save(webmention);
+
+ await DomainEvents.allSettled();
+
+ emailMockReceiver
+ .assertSentEmailCount(2)
+ .matchHTMLSnapshot([{}], 0)
+ .matchHTMLSnapshot([{}], 1)
+ .matchPlaintextSnapshot([{}])
+ .matchMetadataSnapshot();
+
+ const email = emailMockReceiver.getSentEmail(0);
+
+ // Check if the site title is visible in the email
+ assert(email.html.includes('Other Ghost Site'));
+ assert(email.html.includes('Recommend back'));
+ assert(email.html.includes('https://www.otherghostsite.com'));
+ });
+
+ it('Sends a different email if we receive a recommendation back', async function () {
+ if (dbUtils.isSQLite()) {
+ this.skip();
+ }
+
+ // Create a recommendation to otherghostsite.com
+ const recommendation = Recommendation.create({
+ title: `Recommendation`,
+ reason: `Reason`,
+ url: new URL(`https://www.otherghostsite.com/`),
+ favicon: null,
+ featuredImage: null,
+ excerpt: 'Test excerpt',
+ oneClickSubscribe: true,
+ createdAt: new Date(5000)
+ });
+
+ await recommendationsService.repository.save(recommendation);
+
+ const webmention = await Mention.create({
+ source: 'https://www.otherghostsite.com/.well-known/recommendations.json',
+ target: 'https://www.mysite.com/',
+ timestamp: new Date(),
+ payload: null,
+ resourceId: null,
+ resourceType: null,
+ sourceTitle: 'Other Ghost Site',
+ sourceSiteTitle: 'Other Ghost Site',
+ sourceAuthor: null,
+ sourceExcerpt: null,
+ sourceFavicon: null,
+ sourceFeaturedImage: null
+ });
+
+ // Mark it as verified
+ webmention.verify('{"url": "https://www.mysite.com/"}', 'application/json');
+ assert.ok(webmention.verified);
+
+ // Save to repository
+ await mentionsService.repository.save(webmention);
+
+ await DomainEvents.allSettled();
+
+ emailMockReceiver
+ .assertSentEmailCount(2)
+ .matchHTMLSnapshot([{}])
+ .matchPlaintextSnapshot([{}])
+ .matchMetadataSnapshot();
+
+ const email = emailMockReceiver.getSentEmail(0);
+
+ // Check if the site title is visible in the email
+ assert(email.html.includes('Other Ghost Site'));
+ assert(email.html.includes('View recommendations'));
+ assert(email.html.includes('https://www.otherghostsite.com'));
+ });
+
+ it('Does not send an email if we receive a normal mention', async function () {
+ const webmention = await Mention.create({
+ source: 'https://www.otherghostsite.com/recommendations.json',
+ target: 'https://www.mysite.com/',
+ timestamp: new Date(),
+ payload: null,
+ resourceId: null,
+ resourceType: null,
+ sourceTitle: 'Other Ghost Site',
+ sourceSiteTitle: 'Other Ghost Site',
+ sourceAuthor: null,
+ sourceExcerpt: null,
+ sourceFavicon: null,
+ sourceFeaturedImage: null
+ });
+
+ // Mark it as verified
+ webmention.verify('{"url": "https://www.mysite.com/"}', 'application/json');
+ assert.ok(webmention.verified);
+
+ // Save to repository
+ await mentionsService.repository.save(webmention);
+
+ await DomainEvents.allSettled();
+
+ mockManager.assert.sentEmailCount(0);
+ });
+
+ it('Does not send an email for an unverified webmention', async function () {
+ const webmention = await Mention.create({
+ source: 'https://www.otherghostsite.com/.well-known/recommendations.json',
+ target: 'https://www.mysite.com/',
+ timestamp: new Date(),
+ payload: null,
+ resourceId: null,
+ resourceType: null,
+ sourceTitle: 'Other Ghost Site',
+ sourceSiteTitle: 'Other Ghost Site',
+ sourceAuthor: null,
+ sourceExcerpt: null,
+ sourceFavicon: null,
+ sourceFeaturedImage: null
+ });
+
+ // Mark it as verified
+ webmention.verify('{"url": "https://www.myste.com/"}', 'application/json');
+ assert.ok(!webmention.verified);
+
+ // Save to repository
+ await mentionsService.repository.save(webmention);
+
+ await DomainEvents.allSettled();
+
+ mockManager.assert.sentEmailCount(0);
+ });
+});
diff --git a/ghost/oembed-service/lib/OEmbedService.js b/ghost/oembed-service/lib/OEmbedService.js
index 5255e8eb28..064069ae18 100644
--- a/ghost/oembed-service/lib/OEmbedService.js
+++ b/ghost/oembed-service/lib/OEmbedService.js
@@ -232,7 +232,12 @@ class OEmbedService {
let scraperResponse;
try {
- scraperResponse = await metascraper({html, url});
+ scraperResponse = await metascraper({
+ html,
+ url,
+ // In development, allow non-standard tlds
+ validateUrl: this.config.get('env') !== 'development'
+ });
} catch (err) {
// Log to avoid being blind to errors happenning in metascraper
logging.error(err);
diff --git a/ghost/recommendations/src/IncomingRecommendationEmailRenderer.ts b/ghost/recommendations/src/IncomingRecommendationEmailRenderer.ts
new file mode 100644
index 0000000000..bece0fa35c
--- /dev/null
+++ b/ghost/recommendations/src/IncomingRecommendationEmailRenderer.ts
@@ -0,0 +1,36 @@
+import {IncomingRecommendation, EmailRecipient} from './IncomingRecommendationService';
+
+type StaffService = {
+ api: {
+ emails: {
+ renderHTML(template: string, data: unknown): Promise,
+ renderText(template: string, data: unknown): Promise
+ }
+ }
+}
+
+export class IncomingRecommendationEmailRenderer {
+ #staffService: StaffService;
+
+ constructor({staffService}: {staffService: StaffService}) {
+ this.#staffService = staffService;
+ }
+
+ async renderSubject(recommendation: IncomingRecommendation) {
+ return `${recommendation.siteTitle} recommended you`;
+ }
+
+ async renderHTML(recommendation: IncomingRecommendation, recipient: EmailRecipient) {
+ return this.#staffService.api.emails.renderHTML('recommendation-received', {
+ recommendation,
+ recipient
+ });
+ }
+
+ async renderText(recommendation: IncomingRecommendation, recipient: EmailRecipient) {
+ return this.#staffService.api.emails.renderText('recommendation-received', {
+ recommendation,
+ recipient
+ });
+ }
+};
diff --git a/ghost/recommendations/src/IncomingRecommendationService.ts b/ghost/recommendations/src/IncomingRecommendationService.ts
new file mode 100644
index 0000000000..08ecc1a070
--- /dev/null
+++ b/ghost/recommendations/src/IncomingRecommendationService.ts
@@ -0,0 +1,135 @@
+import {IncomingRecommendationEmailRenderer} from './IncomingRecommendationEmailRenderer';
+import {RecommendationService} from './RecommendationService';
+import logging from '@tryghost/logging';
+
+export type IncomingRecommendation = {
+ title: string;
+ siteTitle: string|null;
+ url: URL;
+ excerpt: string|null;
+ favicon: URL|null;
+ featuredImage: URL|null;
+ recommendingBack: boolean;
+}
+
+export type Report = {
+ startDate: Date,
+ endDate: Date,
+ recommendations: IncomingRecommendation[]
+}
+
+type Mention = {
+ source: URL,
+ sourceTitle: string,
+ sourceSiteTitle: string|null,
+ sourceAuthor: string|null,
+ sourceExcerpt: string|null,
+ sourceFavicon: URL|null,
+ sourceFeaturedImage: URL|null
+}
+
+type MentionsAPI = {
+ refreshMentions(options: {filter: string, limit: number|'all'}): Promise
+ listMentions(options: {filter: string, limit: number|'all'}): Promise<{data: Mention[]}>
+}
+
+export type EmailRecipient = {
+ email: string
+}
+
+type EmailService = {
+ send(to: string, subject: string, html: string, text: string): Promise
+}
+
+export class IncomingRecommendationService {
+ #mentionsApi: MentionsAPI;
+ #recommendationService: RecommendationService;
+
+ #emailService: EmailService;
+ #emailRenderer: IncomingRecommendationEmailRenderer;
+ #getEmailRecipients: () => Promise;
+
+ constructor(deps: {
+ recommendationService: RecommendationService,
+ mentionsApi: MentionsAPI,
+ emailService: EmailService,
+ emailRenderer: IncomingRecommendationEmailRenderer,
+ getEmailRecipients: () => Promise,
+ }) {
+ this.#recommendationService = deps.recommendationService;
+ this.#mentionsApi = deps.mentionsApi;
+ this.#emailService = deps.emailService;
+ this.#emailRenderer = deps.emailRenderer;
+ this.#getEmailRecipients = deps.getEmailRecipients;
+ }
+
+ async init() {
+ // When we boot, it is possible that we missed some webmentions from other sites recommending you
+ // More importantly, we might have missed some deletes which we can detect.
+ // So we do a slow revalidation of all incoming recommendations
+ // This also prevents doing multiple external fetches when doing quick reboots of Ghost after each other (requires Ghost to be up for at least 15 seconds)
+ if (!process.env.NODE_ENV?.startsWith('test')) {
+ setTimeout(() => {
+ logging.info('Updating incoming recommendations on boot');
+ this.#updateIncomingRecommendations().catch((err) => {
+ logging.error('Failed to update incoming recommendations on boot', err);
+ });
+ }, 15 * 1000 + Math.random() * 5 * 60 * 1000);
+ }
+ }
+
+ #getMentionFilter({verified = true} = {}) {
+ const base = `source:~$'/.well-known/recommendations.json'`;
+ if (verified) {
+ return `${base}+verified:true`;
+ }
+ return base;
+ }
+
+ async #updateIncomingRecommendations() {
+ // Note: we also recheck recommendations that were not verified (verification could have failed)
+ const filter = this.#getMentionFilter({verified: false});
+ await this.#mentionsApi.refreshMentions({filter, limit: 100});
+ }
+
+ async #mentionToIncomingRecommendation(mention: Mention): Promise {
+ try {
+ const url = new URL(mention.source.toString().replace(/\/.well-known\/recommendations\.json$/, ''));
+
+ // Check if we are also recommending this URL
+ const existing = await this.#recommendationService.countRecommendations({
+ filter: `url:~^'${url}'`
+ });
+ const recommendingBack = existing > 0;
+
+ return {
+ title: mention.sourceTitle,
+ siteTitle: mention.sourceSiteTitle,
+ url,
+ excerpt: mention.sourceExcerpt,
+ favicon: mention.sourceFavicon,
+ featuredImage: mention.sourceFeaturedImage,
+ recommendingBack
+ };
+ } catch (e) {
+ logging.error('Failed to parse mention to incoming recommendation data type', e);
+ }
+ return null;
+ }
+
+ async sendRecommendationEmail(mention: Mention) {
+ const recommendation = await this.#mentionToIncomingRecommendation(mention);
+ if (!recommendation) {
+ return;
+ }
+ const recipients = await this.#getEmailRecipients();
+
+ for (const recipient of recipients) {
+ const subject = await this.#emailRenderer.renderSubject(recommendation);
+ const html = await this.#emailRenderer.renderHTML(recommendation, recipient);
+ const text = await this.#emailRenderer.renderText(recommendation, recipient);
+
+ await this.#emailService.send(recipient.email, subject, html, text);
+ }
+ }
+}
diff --git a/ghost/recommendations/src/RecommendationService.ts b/ghost/recommendations/src/RecommendationService.ts
index 9c6ef60699..fd2841ae9d 100644
--- a/ghost/recommendations/src/RecommendationService.ts
+++ b/ghost/recommendations/src/RecommendationService.ts
@@ -27,10 +27,6 @@ type MentionSendingService = {
sendAll(options: {url: URL, links: URL[]}): Promise
}
-type MentionsAPI = {
- refreshMentions(options: {filter: string, limit: number|'all'}): Promise
-}
-
type RecommendationEnablerService = {
getSetting(): string,
setSetting(value: string): Promise
@@ -48,7 +44,6 @@ export class RecommendationService {
wellknownService: WellknownService;
mentionSendingService: MentionSendingService;
recommendationEnablerService: RecommendationEnablerService;
- mentionsApi: MentionsAPI;
constructor(deps: {
repository: RecommendationRepository,
@@ -56,8 +51,7 @@ export class RecommendationService {
subscribeEventRepository: BookshelfRepository,
wellknownService: WellknownService,
mentionSendingService: MentionSendingService,
- recommendationEnablerService: RecommendationEnablerService,
- mentionsApi: MentionsAPI
+ recommendationEnablerService: RecommendationEnablerService
}) {
this.repository = deps.repository;
this.wellknownService = deps.wellknownService;
@@ -65,31 +59,11 @@ export class RecommendationService {
this.recommendationEnablerService = deps.recommendationEnablerService;
this.clickEventRepository = deps.clickEventRepository;
this.subscribeEventRepository = deps.subscribeEventRepository;
- this.mentionsApi = deps.mentionsApi;
}
async init() {
const recommendations = await this.#listRecommendations();
await this.updateWellknown(recommendations);
-
- // When we boot, it is possible that we missed some webmentions from other sites recommending you
- // More importantly, we might have missed some deletes which we can detect.
- // So we do a slow revalidation of all incoming recommendations
- // This also prevents doing multiple external fetches when doing quick reboots of Ghost after each other (requires Ghost to be up for at least 15 seconds)
- if (!process.env.NODE_ENV?.startsWith('test')) {
- setTimeout(() => {
- logging.info('Updating incoming recommendations on boot');
- this.#updateIncomingRecommendations().catch((err) => {
- logging.error('Failed to update incoming recommendations on boot', err);
- });
- }, 15 * 1000 + Math.random() * 5 * 60 * 1000);
- }
- }
-
- async #updateIncomingRecommendations() {
- // Note: we also recheck recommendations that were not verified (verification could have failed)
- const filter = `source:~$'/.well-known/recommendations.json'`;
- await this.mentionsApi.refreshMentions({filter, limit: 100});
}
async updateWellknown(recommendations: Recommendation[]) {
diff --git a/ghost/recommendations/src/index.ts b/ghost/recommendations/src/index.ts
index 82cba44a2b..6f6467d81a 100644
--- a/ghost/recommendations/src/index.ts
+++ b/ghost/recommendations/src/index.ts
@@ -9,3 +9,5 @@ export * from './ClickEvent';
export * from './BookshelfClickEventRepository';
export * from './SubscribeEvent';
export * from './BookshelfSubscribeEventRepository';
+export * from './IncomingRecommendationService';
+export * from './IncomingRecommendationEmailRenderer';
diff --git a/ghost/staff-service/lib/StaffService.js b/ghost/staff-service/lib/StaffService.js
index c4becf8369..24e67e4d21 100644
--- a/ghost/staff-service/lib/StaffService.js
+++ b/ghost/staff-service/lib/StaffService.js
@@ -15,7 +15,6 @@ class StaffService {
const Emails = require('./StaffServiceEmails');
- /** @private */
this.emails = new Emails({
logging,
models,
diff --git a/ghost/staff-service/lib/email-templates/recommendation-received.hbs b/ghost/staff-service/lib/email-templates/recommendation-received.hbs
new file mode 100644
index 0000000000..975116d5a7
--- /dev/null
+++ b/ghost/staff-service/lib/email-templates/recommendation-received.hbs
@@ -0,0 +1,105 @@
+
+
+
+
+
+ 💌 New recommenation
+ {{> styles}}
+
+
+
+
+
+
diff --git a/ghost/staff-service/lib/email-templates/recommendation-received.txt.js b/ghost/staff-service/lib/email-templates/recommendation-received.txt.js
new file mode 100644
index 0000000000..f2af8a002c
--- /dev/null
+++ b/ghost/staff-service/lib/email-templates/recommendation-received.txt.js
@@ -0,0 +1,13 @@
+module.exports = function (data) {
+ const {recommendation} = data;
+
+ // Be careful when you indent the email, because whitespaces are visible in emails!
+ return `
+You have been recommended by ${recommendation.siteTitle || recommendation.title || recommendation.url}.
+
+---
+
+Sent to ${data.toEmail} from ${data.siteDomain}.
+If you would no longer like to receive these notifications you can adjust your settings at ${data.staffUrl}.
+ `;
+};
diff --git a/ghost/webmentions/lib/InMemoryMentionRepository.js b/ghost/webmentions/lib/InMemoryMentionRepository.js
index b901ecde4b..bff186f3a2 100644
--- a/ghost/webmentions/lib/InMemoryMentionRepository.js
+++ b/ghost/webmentions/lib/InMemoryMentionRepository.js
@@ -56,7 +56,7 @@ module.exports = class InMemoryMentionRepository {
*/
async getBySourceAndTarget(source, target) {
return this.#store.find((item) => {
- return item.source.href === source.href && item.target.href === target.href && !Mention.isDeleted(item);
+ return item.source.href === source.href && item.target.href === target.href;
});
}
diff --git a/ghost/webmentions/lib/Mention.js b/ghost/webmentions/lib/Mention.js
index 7c97edc9fe..5924510ce1 100644
--- a/ghost/webmentions/lib/Mention.js
+++ b/ghost/webmentions/lib/Mention.js
@@ -30,6 +30,14 @@ module.exports = class Mention {
this.#deleted = true;
}
+ #undelete() {
+ // When an earlier mention is deleted, but then it gets verified again, we need to undelete it
+ if (this.#deleted) {
+ this.#deleted = false;
+ this.events.push(MentionCreatedEvent.create({mention: this}));
+ }
+ }
+
/**
* @param {string} html
* @param {string} contentType
@@ -44,9 +52,11 @@ module.exports = class Mention {
this.#verified = hasTargetUrl;
if (wasVerified && !this.#verified) {
- // Delete the mention
+ // Delete the mention, but keep it verified (it was just deleted, because it was verified earlier, so now it is removed from the site according to the spec)
this.#deleted = true;
this.#verified = true;
+ } else {
+ this.#undelete();
}
} catch (e) {
this.#verified = false;
@@ -62,9 +72,11 @@ module.exports = class Mention {
this.#verified = !!html.includes(JSON.stringify(this.target.href));
if (wasVerified && !this.#verified) {
- // Delete the mention
+ // Delete the mention, but keep it verified (it was just deleted, because it was verified earlier, so now it is removed from the site according to the spec)
this.#deleted = true;
this.#verified = true;
+ } else {
+ this.#undelete();
}
} catch (e) {
this.#verified = false;
@@ -217,6 +229,7 @@ module.exports = class Mention {
this.#resourceId = data.resourceId;
this.#resourceType = data.resourceType;
this.#verified = data.verified;
+ this.#deleted = data.deleted || false;
}
/**
@@ -302,7 +315,8 @@ module.exports = class Mention {
payload,
resourceId,
resourceType,
- verified
+ verified,
+ deleted: isNew ? false : !!data.deleted
});
mention.setSourceMetadata(data);
diff --git a/ghost/webmentions/lib/MentionCreatedEvent.js b/ghost/webmentions/lib/MentionCreatedEvent.js
index 1b6483a8d1..ae3d102549 100644
--- a/ghost/webmentions/lib/MentionCreatedEvent.js
+++ b/ghost/webmentions/lib/MentionCreatedEvent.js
@@ -1,5 +1,6 @@
/**
* @typedef {object} MentionCreatedEventData
+ * @property {import('./Mention')} mention
*/
module.exports = class MentionCreatedEvent {
diff --git a/ghost/webmentions/lib/MentionDiscoveryService.js b/ghost/webmentions/lib/MentionDiscoveryService.js
index 991d578116..4d4c8b04da 100644
--- a/ghost/webmentions/lib/MentionDiscoveryService.js
+++ b/ghost/webmentions/lib/MentionDiscoveryService.js
@@ -1,4 +1,5 @@
const cheerio = require('cheerio');
+const logging = require('@tryghost/logging');
module.exports = class MentionDiscoveryService {
#externalRequest;
@@ -26,6 +27,7 @@ module.exports = class MentionDiscoveryService {
});
return this.getEndpointFromResponse(response);
} catch (error) {
+ logging.error(`Error fetching ${url.href} to discover webmention endpoint`, error);
return null;
}
}
diff --git a/ghost/webmentions/lib/MentionsAPI.js b/ghost/webmentions/lib/MentionsAPI.js
index 165d918d3b..05d26815c6 100644
--- a/ghost/webmentions/lib/MentionsAPI.js
+++ b/ghost/webmentions/lib/MentionsAPI.js
@@ -183,6 +183,7 @@ module.exports = class MentionsAPI {
async #updateWebmention(mention, webmention) {
const isNew = !mention;
+ const wasDeleted = mention?.deleted ?? false;
const targetExists = await this.#routingService.pageExists(webmention.target);
if (!targetExists) {
@@ -235,23 +236,23 @@ module.exports = class MentionsAPI {
}
if (metadata?.body) {
- try {
- mention.verify(metadata.body, metadata.contentType);
- } catch (e) {
- logging.error(e);
- }
+ mention.verify(metadata.body, metadata.contentType);
}
}
await this.#repository.save(mention);
if (isNew) {
- logging.info('[Webmention] Created ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
+ logging.info('[Webmention] Created ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified + ', deleted: ' + mention.deleted);
} else {
- if (mention.deleted) {
+ if (mention.deleted && !wasDeleted) {
logging.info('[Webmention] Deleted ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
} else {
- logging.info('[Webmention] Updated ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
+ if (!mention.deleted && wasDeleted) {
+ logging.info('[Webmention] Restored ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
+ } else {
+ logging.info('[Webmention] Updated ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified + ', deleted: ' + mention.deleted);
+ }
}
}
diff --git a/ghost/webmentions/test/Mention.test.js b/ghost/webmentions/test/Mention.test.js
index e384e88f75..1d8d9a5f48 100644
--- a/ghost/webmentions/test/Mention.test.js
+++ b/ghost/webmentions/test/Mention.test.js
@@ -126,6 +126,27 @@ describe('Mention', function () {
});
});
+ describe('undelete', function () {
+ afterEach(function () {
+ sinon.restore();
+ });
+
+ it('can undelete a verified mention', async function () {
+ const mention = await Mention.create({
+ ...validInput,
+ id: new ObjectID(),
+ deleted: true,
+ verified: true
+ });
+ assert(mention.verified);
+ assert(mention.deleted);
+
+ mention.verify('{"url": "https://target.com/"}', 'application/json');
+ assert(mention.verified);
+ assert(!mention.isDeleted());
+ });
+ });
+
describe('create', function () {
it('Will error with invalid inputs', async function () {
const invalidInputs = [
diff --git a/ghost/webmentions/test/MentionsAPI.test.js b/ghost/webmentions/test/MentionsAPI.test.js
index 037d5ff0ab..aa1d062e80 100644
--- a/ghost/webmentions/test/MentionsAPI.test.js
+++ b/ghost/webmentions/test/MentionsAPI.test.js
@@ -29,7 +29,8 @@ const mockWebmentionMetadata = {
author: 'Dr Egg Man',
image: new URL('https://unsplash.com/photos/QAND9huzD04'),
favicon: new URL('https://ghost.org/favicon.ico'),
- body: `Some HTML and a mentioned url
`
+ body: `Some HTML and a mentioned url
`,
+ contentType: 'text/html'
};
}
};
@@ -432,7 +433,7 @@ describe('MentionsAPI', function () {
}
});
- it('Will delete an existing mention if the source page does not exist', async function () {
+ it('Will delete and restore an existing mention if the source page does not exist', async function () {
const repository = new InMemoryMentionRepository();
const api = new MentionsAPI({
repository,
@@ -449,6 +450,7 @@ describe('MentionsAPI', function () {
fetch: sinon.stub()
.onFirstCall().resolves(mockWebmentionMetadata.fetch())
.onSecondCall().rejects()
+ .onThirdCall().resolves(mockWebmentionMetadata.fetch())
}
});
@@ -481,6 +483,88 @@ describe('MentionsAPI', function () {
assert.equal(page.data.length, 0);
break checkMentionDeleted;
}
+
+ checkRestored: {
+ const mention = await api.processWebmention({
+ source: new URL('https://source.com'),
+ target: new URL('https://target.com'),
+ payload: {}
+ });
+
+ const page = await api.listMentions({
+ limit: 'all'
+ });
+
+ assert.equal(page.data[0].id, mention.id);
+ break checkRestored;
+ }
+ });
+
+ it('Will delete and restore an existing mention if the target url is not present on the source page', async function () {
+ const repository = new InMemoryMentionRepository();
+ const api = new MentionsAPI({
+ repository,
+ routingService: mockRoutingService,
+ resourceService: {
+ async getByURL() {
+ return {
+ type: 'post',
+ id: new ObjectID
+ };
+ }
+ },
+ webmentionMetadata: {
+ fetch: sinon.stub()
+ .onFirstCall().resolves(mockWebmentionMetadata.fetch())
+ .onSecondCall().resolves({...(await mockWebmentionMetadata.fetch()), body: 'test'})
+ .onThirdCall().resolves(mockWebmentionMetadata.fetch())
+ }
+ });
+
+ checkFirstMention: {
+ const mention = await api.processWebmention({
+ source: new URL('https://source.com'),
+ target: new URL('https://target.com'),
+ payload: {}
+ });
+
+ const page = await api.listMentions({
+ limit: 'all'
+ });
+
+ assert.equal(page.data[0].id, mention.id);
+ break checkFirstMention;
+ }
+
+ checkMentionDeleted: {
+ await api.processWebmention({
+ source: new URL('https://source.com'),
+ target: new URL('https://target.com'),
+ payload: {}
+ });
+
+ const page = await api.listMentions({
+ limit: 'all'
+ });
+
+ assert.equal(page.data.length, 0);
+ break checkMentionDeleted;
+ }
+
+ checkRestored: {
+ const mention = await api.processWebmention({
+ source: new URL('https://source.com'),
+ target: new URL('https://target.com'),
+ payload: {}
+ });
+
+ const page = await api.listMentions({
+ limit: 'all'
+ });
+
+ assert.equal(page.data[0].id, mention.id);
+ break checkRestored;
+ }
});
it('Will throw for new mentions if the source page is not found', async function () {