From a596acf7d27f237617319356e8ac80e10b2f4c95 Mon Sep 17 00:00:00 2001 From: Simon Backx Date: Thu, 19 Jan 2023 17:35:10 +0100 Subject: [PATCH] Added MentionSendingService (#16151) fixes https://github.com/TryGhost/Team/issues/2409 The MentionSendingService listens for post changes and sends webmentions for outbound links in the post. --- .../core/server/services/mentions/service.js | 26 +- .../webmentions/lib/MentionSendingService.js | 163 +++++++ ghost/webmentions/lib/webmentions.js | 1 + ghost/webmentions/package.json | 9 +- .../test/MentionSendingService.test.js | 440 ++++++++++++++++++ ghost/webmentions/test/utils/index.js | 48 ++ 6 files changed, 684 insertions(+), 3 deletions(-) create mode 100644 ghost/webmentions/lib/MentionSendingService.js create mode 100644 ghost/webmentions/test/MentionSendingService.test.js create mode 100644 ghost/webmentions/test/utils/index.js diff --git a/ghost/core/core/server/services/mentions/service.js b/ghost/core/core/server/services/mentions/service.js index 024e7481f6..f02148e583 100644 --- a/ghost/core/core/server/services/mentions/service.js +++ b/ghost/core/core/server/services/mentions/service.js @@ -2,9 +2,20 @@ const MentionController = require('./MentionController'); const WebmentionMetadata = require('./WebmentionMetadata'); const { InMemoryMentionRepository, - MentionsAPI + MentionsAPI, + MentionSendingService } = require('@tryghost/webmentions'); +const events = require('../../lib/common/events'); +const externalRequest = require('../../../server/lib/request-external.js'); +const urlUtils = require('../../../shared/url-utils'); +const url = require('../../../server/api/endpoints/utils/serializers/output/utils/url'); +const labs = require('../../../shared/labs'); +function getPostUrl(post) { + const jsonModel = {}; + url.forPost(post.id, jsonModel, {options: {}}); + return jsonModel.url; +} module.exports = { controller: new MentionController(), async init() { @@ -53,5 +64,18 @@ module.exports = { extra: 'data' } }); + + const sendingService = new MentionSendingService({ + discoveryService: { + getEndpoint: async () => { + return new URL('https://site.ghost/webmentions/receive'); + } + }, + externalRequest, + getSiteUrl: () => urlUtils.urlFor('home', true), + getPostUrl: post => getPostUrl(post), + isEnabled: () => labs.isSet('webmentions') + }); + sendingService.listen(events); } }; diff --git a/ghost/webmentions/lib/MentionSendingService.js b/ghost/webmentions/lib/MentionSendingService.js new file mode 100644 index 0000000000..f55b8ae46e --- /dev/null +++ b/ghost/webmentions/lib/MentionSendingService.js @@ -0,0 +1,163 @@ +const errors = require('@tryghost/errors'); +const logging = require('@tryghost/logging'); + +module.exports = class MentionSendingService { + #discoveryService; + #externalRequest; + #getSiteUrl; + #getPostUrl; + #isEnabled; + + constructor({discoveryService, externalRequest, getSiteUrl, getPostUrl, isEnabled}) { + this.#discoveryService = discoveryService; + this.#externalRequest = externalRequest; + this.#getSiteUrl = getSiteUrl; + this.#getPostUrl = getPostUrl; + this.#isEnabled = isEnabled; + } + + get siteUrl() { + try { + return new URL(this.#getSiteUrl()); + } catch (e) { + return null; + } + } + + /** + * Listen for changes in posts and automatically send webmentions. + * @param {*} events + */ + listen(events) { + // Note: we don't need to listen for post.published (post.edited is also called at that time) + events.on('post.edited', this.sendForEditedPost.bind(this)); + } + + async sendForEditedPost(post) { + try { + if (!this.#isEnabled()) { + return; + } + // TODO: we need to check old url and send webmentions in case the url changed of a post + if (post.get('status') === post.previous('status') && post.get('html') === post.previous('html')) { + // Not changed + return; + } + if (post.get('status') !== 'published' && post.previous('status') !== 'published') { + // Post should be or should have been published + return; + } + await this.sendAll({ + url: new URL(this.#getPostUrl(post)), + html: post.get('html'), + previousHtml: post.previous('status') === 'published' ? post.previous('html') : null + }); + } catch (e) { + logging.error('Error in webmention sending service post.added event handler:'); + logging.error(e); + } + } + + async send({source, target, endpoint}) { + logging.info('[Webmention] Sending webmention from ' + source.href + ' to ' + target.href + ' via ' + endpoint.href); + const response = await this.#externalRequest.post(endpoint.href, { + body: { + source: source.href, + target: target.href + }, + form: true, + throwHttpErrors: false, + maxRedirects: 10, + followRedirect: true, + methodRewriting: false, // WARNING! this setting has a different meaning in got v12! + timeout: { + lookup: 200, + connect: 200, + secureConnect: 200, + socket: 1000, + send: 5000, + response: 1000 + } + }); + if (response.statusCode >= 200 && response.statusCode < 300) { + return; + } + throw new errors.BadRequestError({ + message: 'Webmention sending failed with status code ' + response.statusCode, + statusCode: response.statusCode + }); + } + + /** + * Send a webmention call for the links in a resource. + * @param {object} resource + * @param {URL} resource.url + * @param {string} resource.html + * @param {string|null} [resource.previousHtml] + */ + async sendAll(resource) { + const links = this.getLinks(resource.html); + if (resource.previousHtml) { + // We also need to send webmentions for removed links + const oldLinks = this.getLinks(resource.previousHtml); + for (const link of oldLinks) { + if (!links.find(l => l.href === link.href)) { + links.push(link); + } + } + } + + if (links.length) { + logging.info('[Webmention] Sending all webmentions for ' + resource.url.href); + } + + for (const target of links) { + const endpoint = await this.#discoveryService.getEndpoint(target); + if (endpoint) { + // Send webmention call + try { + await this.send({source: resource.url, target, endpoint}); + } catch (e) { + logging.error('[Webmention] Failed sending via ' + endpoint.href + ': ' + e.message); + } + } + } + } + + /** + * @private + * Get all external links in a HTML document. + * Excludes the site's own domain. + * @param {string} html + * @returns {URL[]} + */ + getLinks(html) { + const cheerio = require('cheerio'); + const $ = cheerio.load(html); + const urls = []; + const siteUrl = this.siteUrl; + + for (const el of $('a').toArray()) { + const href = $(el).attr('href'); + if (href) { + let url; + try { + url = new URL(href); + + if (siteUrl && url.hostname === siteUrl.hostname) { + // Ignore links to the site's own domain + continue; + } + + if (['http:', 'https:'].includes(url.protocol) && !urls.find(u => u.href === url.href)) { + // Ignore duplicate URLs + urls.push(url); + } + } catch (e) { + // Ignore invalid URLs + } + } + } + return urls; + } +}; diff --git a/ghost/webmentions/lib/webmentions.js b/ghost/webmentions/lib/webmentions.js index 1ad9f78e89..6cd5c1a813 100644 --- a/ghost/webmentions/lib/webmentions.js +++ b/ghost/webmentions/lib/webmentions.js @@ -1,3 +1,4 @@ module.exports.InMemoryMentionRepository = require('./InMemoryMentionRepository'); module.exports.MentionsAPI = require('./MentionsAPI'); module.exports.Mention = require('./Mention'); +module.exports.MentionSendingService = require('./MentionSendingService'); diff --git a/ghost/webmentions/package.json b/ghost/webmentions/package.json index 15ca9b0366..36ac9eddd8 100644 --- a/ghost/webmentions/package.json +++ b/ghost/webmentions/package.json @@ -20,7 +20,12 @@ "devDependencies": { "c8": "7.12.0", "mocha": "10.2.0", - "sinon": "15.0.1" + "nock": "13.3.0", + "sinon": "15.0.1", + "bson-objectid": "2.0.4" }, - "dependencies": {} + "dependencies": { + "@tryghost/errors": "1.2.20", + "@tryghost/logging": "2.3.6" + } } diff --git a/ghost/webmentions/test/MentionSendingService.test.js b/ghost/webmentions/test/MentionSendingService.test.js new file mode 100644 index 0000000000..48a81a44c4 --- /dev/null +++ b/ghost/webmentions/test/MentionSendingService.test.js @@ -0,0 +1,440 @@ +const MentionSendingService = require('../lib/MentionSendingService.js'); +const assert = require('assert'); +const nock = require('nock'); +// non-standard to use externalRequest here, but this is required for the overrides in the libary, which we want to test for security reasons in combination with the package +const externalRequest = require('../../core/core/server/lib/request-external.js'); +const sinon = require('sinon'); +const logging = require('@tryghost/logging'); +const {createModel} = require('./utils/index.js'); + +describe('MentionSendingService', function () { + let errorLogStub; + + beforeEach(function () { + nock.disableNetConnect(); + sinon.stub(logging, 'info'); + errorLogStub = sinon.stub(logging, 'error'); + }); + + afterEach(function () { + nock.cleanAll(); + sinon.restore(); + }); + + describe('listen', function () { + it('Calls on post.edited', async function () { + const service = new MentionSendingService({}); + const stub = sinon.stub(service, 'sendForEditedPost').resolves(); + let callback; + const events = { + on: sinon.stub().callsFake((event, c) => { + callback = c; + }) + }; + service.listen(events); + sinon.assert.calledOnce(events.on); + await callback({}); + sinon.assert.calledOnce(stub); + }); + }); + + describe('sendForEditedPost', function () { + it('Ignores if disabled', async function () { + const service = new MentionSendingService({ + isEnabled: () => false + }); + const stub = sinon.stub(service, 'sendAll'); + await service.sendForEditedPost({}); + sinon.assert.notCalled(stub); + }); + + it('Ignores draft posts', async function () { + const service = new MentionSendingService({ + isEnabled: () => true + }); + const stub = sinon.stub(service, 'sendAll'); + await service.sendForEditedPost(createModel({ + status: 'draft', + html: 'changed', + previous: { + status: 'draft', + html: '' + } + })); + sinon.assert.notCalled(stub); + }); + + it('Ignores if html was not changed', async function () { + const service = new MentionSendingService({ + isEnabled: () => true + }); + const stub = sinon.stub(service, 'sendAll'); + await service.sendForEditedPost(createModel({ + status: 'published', + html: 'same', + previous: { + status: 'published', + html: 'same' + } + })); + sinon.assert.notCalled(stub); + }); + + it('Ignores email only posts', async function () { + const service = new MentionSendingService({ + isEnabled: () => true + }); + const stub = sinon.stub(service, 'sendAll'); + await service.sendForEditedPost(createModel({ + status: 'send', + html: 'changed', + previous: { + status: 'draft', + html: 'same' + } + })); + sinon.assert.notCalled(stub); + }); + + it('Sends on publish', async function () { + const service = new MentionSendingService({ + isEnabled: () => true, + getPostUrl: () => 'https://site.com/post/' + }); + const stub = sinon.stub(service, 'sendAll'); + await service.sendForEditedPost(createModel({ + status: 'published', + html: 'same', + previous: { + status: 'draft', + html: 'same' + } + })); + sinon.assert.calledOnce(stub); + const firstCall = stub.getCall(0).args[0]; + assert.strictEqual(firstCall.url.toString(), 'https://site.com/post/'); + assert.strictEqual(firstCall.html, 'same'); + assert.strictEqual(firstCall.previousHtml, null); + }); + + it('Sends on html change', async function () { + const service = new MentionSendingService({ + isEnabled: () => true, + getPostUrl: () => 'https://site.com/post/' + }); + const stub = sinon.stub(service, 'sendAll'); + await service.sendForEditedPost(createModel({ + status: 'published', + html: 'updated', + previous: { + status: 'published', + html: 'same' + } + })); + sinon.assert.calledOnce(stub); + const firstCall = stub.getCall(0).args[0]; + assert.strictEqual(firstCall.url.toString(), 'https://site.com/post/'); + assert.strictEqual(firstCall.html, 'updated'); + assert.strictEqual(firstCall.previousHtml, 'same'); + }); + + it('Catches and logs errors', async function () { + const service = new MentionSendingService({ + isEnabled: () => true, + getPostUrl: () => 'https://site.com/post/' + }); + sinon.stub(service, 'sendAll').rejects(new Error('Internal error test')); + await service.sendForEditedPost(createModel({ + status: 'published', + html: 'same', + previous: { + status: 'draft', + html: 'same' + } + })); + assert(errorLogStub.calledTwice); + }); + }); + + describe('sendAll', function () { + it('Sends to all links', async function () { + let counter = 0; + const scope = nock('https://example.org') + .persist() + .post('/webmentions-test') + .reply(() => { + counter += 1; + return [202]; + }); + + const service = new MentionSendingService({ + externalRequest, + getSiteUrl: () => new URL('https://site.com'), + discoveryService: { + getEndpoint: async () => new URL('https://example.org/webmentions-test') + } + }); + await service.sendAll({url: new URL('https://site.com'), + html: ` + + + Example + Example repeated + Example + Example 2 + + + `}); + assert.strictEqual(scope.isDone(), true); + assert.equal(counter, 3); + }); + + it('Catches and logs errors', async function () { + let counter = 0; + const scope = nock('https://example.org') + .persist() + .post('/webmentions-test') + .reply(() => { + counter += 1; + if (counter === 2) { + return [500]; + } + return [202]; + }); + + const service = new MentionSendingService({ + externalRequest, + getSiteUrl: () => new URL('https://site.com'), + discoveryService: { + getEndpoint: async () => new URL('https://example.org/webmentions-test') + } + }); + await service.sendAll({url: new URL('https://site.com'), + html: ` + + + Example + Example repeated + Example + Example 2 + + + `}); + assert.strictEqual(scope.isDone(), true); + assert.equal(counter, 3); + assert(errorLogStub.calledOnce); + }); + + it('Sends to deleted links', async function () { + let counter = 0; + const scope = nock('https://example.org') + .persist() + .post('/webmentions-test') + .reply(() => { + counter += 1; + return [202]; + }); + + const service = new MentionSendingService({ + externalRequest, + getSiteUrl: () => new URL('https://site.com'), + discoveryService: { + getEndpoint: async () => new URL('https://example.org/webmentions-test') + } + }); + await service.sendAll({url: new URL('https://site.com'), + html: `Example`, + previousHtml: `Example`}); + assert.strictEqual(scope.isDone(), true); + assert.equal(counter, 2); + }); + }); + + describe('getLinks', function () { + it('Returns all unique links in a HTML-document', async function () { + const service = new MentionSendingService({ + getSiteUrl: () => new URL('https://site.com') + }); + const links = service.getLinks(` + + + Example + Example repeated + Example + Example 2 + + + `); + assert.deepStrictEqual(links, [ + new URL('https://example.com'), + new URL('https://example.org#fragment'), + new URL('http://example2.org') + ]); + }); + + it('Does not include invalid or local URLs', async function () { + const service = new MentionSendingService({ + getSiteUrl: () => new URL('https://site.com') + }); + const links = service.getLinks(`Example`); + assert.deepStrictEqual(links, []); + }); + + it('Does not include non-http protocols', async function () { + const service = new MentionSendingService({ + getSiteUrl: () => new URL('https://site.com') + }); + const links = service.getLinks(`Example`); + assert.deepStrictEqual(links, []); + }); + + it('Does not include invalid urls', async function () { + const service = new MentionSendingService({ + getSiteUrl: () => new URL('https://site.com') + }); + const links = service.getLinks(`Example`); + assert.deepStrictEqual(links, []); + }); + + it('Does not include urls from site domain', async function () { + const service = new MentionSendingService({ + getSiteUrl: () => new URL('https://site.com') + }); + const links = service.getLinks(`Example`); + assert.deepStrictEqual(links, []); + }); + + it('Ignores invalid site urls', async function () { + const service = new MentionSendingService({ + getSiteUrl: () => new URL('invalid()') + }); + const links = service.getLinks(`Example`); + assert.deepStrictEqual(links, [ + new URL('http://site.com/test?123') + ]); + }); + }); + + describe('send', function () { + it('Can handle 202 accepted responses', async function () { + const scope = nock('https://example.org') + .persist() + .post('/webmentions-test', `source=${encodeURIComponent('https://example.com/source')}&target=${encodeURIComponent('https://target.com/target')}`) + .reply(202); + + const service = new MentionSendingService({externalRequest}); + await service.send({ + source: new URL('https://example.com/source'), + target: new URL('https://target.com/target'), + endpoint: new URL('https://example.org/webmentions-test') + }); + assert(scope.isDone()); + }); + + it('Can handle 201 created responses', async function () { + const scope = nock('https://example.org') + .persist() + .post('/webmentions-test', `source=${encodeURIComponent('https://example.com/source')}&target=${encodeURIComponent('https://target.com/target')}`) + .reply(201); + + const service = new MentionSendingService({externalRequest}); + await service.send({ + source: new URL('https://example.com/source'), + target: new URL('https://target.com/target'), + endpoint: new URL('https://example.org/webmentions-test') + }); + assert(scope.isDone()); + }); + + it('Can handle 400 responses', async function () { + const scope = nock('https://example.org') + .persist() + .post('/webmentions-test') + .reply(400); + + const service = new MentionSendingService({externalRequest}); + await assert.rejects(service.send({ + source: new URL('https://example.com/source'), + target: new URL('https://target.com/target'), + endpoint: new URL('https://example.org/webmentions-test') + }), /sending failed/); + assert(scope.isDone()); + }); + + it('Can handle 500 responses', async function () { + const scope = nock('https://example.org') + .persist() + .post('/webmentions-test') + .reply(500); + + const service = new MentionSendingService({externalRequest}); + await assert.rejects(service.send({ + source: new URL('https://example.com/source'), + target: new URL('https://target.com/target'), + endpoint: new URL('https://example.org/webmentions-test') + }), /sending failed/); + assert(scope.isDone()); + }); + + it('Can handle network errors', async function () { + const scope = nock('https://example.org') + .persist() + .post('/webmentions-test') + .replyWithError('network error'); + + const service = new MentionSendingService({externalRequest}); + await assert.rejects(service.send({ + source: new URL('https://example.com/source'), + target: new URL('https://target.com/target'), + endpoint: new URL('https://example.org/webmentions-test') + }), /network error/); + assert(scope.isDone()); + }); + + // Redirects are currently not supported by got for POST requests! + //it('Can handle redirect responses', async function () { + // const scope = nock('https://example.org') + // .persist() + // .post('/webmentions-test') + // .reply(302, '', { + // headers: { + // Location: 'https://example.org/webmentions-test-2' + // } + // }); + // const scope2 = nock('https://example.org') + // .persist() + // .post('/webmentions-test-2') + // .reply(201); + // + // const service = new MentionSendingService({externalRequest}); + // await service.send({ + // source: new URL('https://example.com'), + // target: new URL('https://example.com'), + // endpoint: new URL('https://example.org/webmentions-test') + // }); + // assert(scope.isDone()); + // assert(scope2.isDone()); + //}); + // TODO: also check if we don't follow private IPs after redirects + + it('Does not send to private IP', async function () { + const service = new MentionSendingService({externalRequest}); + await assert.rejects(service.send({ + source: new URL('https://example.com/source'), + target: new URL('https://target.com/target'), + endpoint: new URL('http://localhost/webmentions') + }), /non-permitted private IP/); + }); + + it('Does not send to private IP behind DNS', async function () { + // Test that we don't make a request when a domain resolves to a private IP + // domaincontrol.com -> 127.0.0.1 + const service = new MentionSendingService({externalRequest}); + await assert.rejects(service.send({ + source: new URL('https://example.com/source'), + target: new URL('https://target.com/target'), + endpoint: new URL('http://domaincontrol.com/webmentions') + }), /non-permitted private IP/); + }); + }); +}); diff --git a/ghost/webmentions/test/utils/index.js b/ghost/webmentions/test/utils/index.js new file mode 100644 index 0000000000..1b62a3e416 --- /dev/null +++ b/ghost/webmentions/test/utils/index.js @@ -0,0 +1,48 @@ +const ObjectId = require('bson-objectid').default; + +const createModel = (propertiesAndRelations) => { + const id = propertiesAndRelations.id ?? ObjectId().toHexString(); + return { + id, + getLazyRelation: (relation) => { + propertiesAndRelations.loaded = propertiesAndRelations.loaded ?? []; + if (!propertiesAndRelations.loaded.includes(relation)) { + propertiesAndRelations.loaded.push(relation); + } + if (Array.isArray(propertiesAndRelations[relation])) { + return Promise.resolve({ + models: propertiesAndRelations[relation] + }); + } + return Promise.resolve(propertiesAndRelations[relation]); + }, + related: (relation) => { + if (!Object.keys(propertiesAndRelations).includes('loaded')) { + throw new Error(`Model.related('${relation}'): When creating a test model via createModel you must include 'loaded' to specify which relations are already loaded and useable via Model.related.`); + } + if (!propertiesAndRelations.loaded.includes(relation)) { + throw new Error(`Model.related('${relation}') was used on a test model that didn't explicitly loaded that relation.`); + } + return propertiesAndRelations[relation]; + }, + get: (property) => { + return propertiesAndRelations[property]; + }, + previous: (property) => { + return propertiesAndRelations.previous[property]; + }, + save: (properties) => { + Object.assign(propertiesAndRelations, properties); + return Promise.resolve(); + }, + toJSON: () => { + return { + id, + ...propertiesAndRelations + }; + } + }; +}; +module.exports = { + createModel +};