Added MentionSendingService (#16151)

fixes https://github.com/TryGhost/Team/issues/2409

The MentionSendingService listens for post changes and sends webmentions
for outbound links in the post.
This commit is contained in:
Simon Backx 2023-01-19 17:35:10 +01:00 committed by GitHub
parent 1f32a7be8a
commit a596acf7d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 684 additions and 3 deletions

View File

@ -2,9 +2,20 @@ const MentionController = require('./MentionController');
const WebmentionMetadata = require('./WebmentionMetadata');
const {
InMemoryMentionRepository,
MentionsAPI
MentionsAPI,
MentionSendingService
} = require('@tryghost/webmentions');
const events = require('../../lib/common/events');
const externalRequest = require('../../../server/lib/request-external.js');
const urlUtils = require('../../../shared/url-utils');
const url = require('../../../server/api/endpoints/utils/serializers/output/utils/url');
const labs = require('../../../shared/labs');
function getPostUrl(post) {
const jsonModel = {};
url.forPost(post.id, jsonModel, {options: {}});
return jsonModel.url;
}
module.exports = {
controller: new MentionController(),
async init() {
@ -53,5 +64,18 @@ module.exports = {
extra: 'data'
}
});
const sendingService = new MentionSendingService({
discoveryService: {
getEndpoint: async () => {
return new URL('https://site.ghost/webmentions/receive');
}
},
externalRequest,
getSiteUrl: () => urlUtils.urlFor('home', true),
getPostUrl: post => getPostUrl(post),
isEnabled: () => labs.isSet('webmentions')
});
sendingService.listen(events);
}
};

View File

@ -0,0 +1,163 @@
const errors = require('@tryghost/errors');
const logging = require('@tryghost/logging');
module.exports = class MentionSendingService {
#discoveryService;
#externalRequest;
#getSiteUrl;
#getPostUrl;
#isEnabled;
constructor({discoveryService, externalRequest, getSiteUrl, getPostUrl, isEnabled}) {
this.#discoveryService = discoveryService;
this.#externalRequest = externalRequest;
this.#getSiteUrl = getSiteUrl;
this.#getPostUrl = getPostUrl;
this.#isEnabled = isEnabled;
}
get siteUrl() {
try {
return new URL(this.#getSiteUrl());
} catch (e) {
return null;
}
}
/**
* Listen for changes in posts and automatically send webmentions.
* @param {*} events
*/
listen(events) {
// Note: we don't need to listen for post.published (post.edited is also called at that time)
events.on('post.edited', this.sendForEditedPost.bind(this));
}
async sendForEditedPost(post) {
try {
if (!this.#isEnabled()) {
return;
}
// TODO: we need to check old url and send webmentions in case the url changed of a post
if (post.get('status') === post.previous('status') && post.get('html') === post.previous('html')) {
// Not changed
return;
}
if (post.get('status') !== 'published' && post.previous('status') !== 'published') {
// Post should be or should have been published
return;
}
await this.sendAll({
url: new URL(this.#getPostUrl(post)),
html: post.get('html'),
previousHtml: post.previous('status') === 'published' ? post.previous('html') : null
});
} catch (e) {
logging.error('Error in webmention sending service post.added event handler:');
logging.error(e);
}
}
async send({source, target, endpoint}) {
logging.info('[Webmention] Sending webmention from ' + source.href + ' to ' + target.href + ' via ' + endpoint.href);
const response = await this.#externalRequest.post(endpoint.href, {
body: {
source: source.href,
target: target.href
},
form: true,
throwHttpErrors: false,
maxRedirects: 10,
followRedirect: true,
methodRewriting: false, // WARNING! this setting has a different meaning in got v12!
timeout: {
lookup: 200,
connect: 200,
secureConnect: 200,
socket: 1000,
send: 5000,
response: 1000
}
});
if (response.statusCode >= 200 && response.statusCode < 300) {
return;
}
throw new errors.BadRequestError({
message: 'Webmention sending failed with status code ' + response.statusCode,
statusCode: response.statusCode
});
}
/**
* Send a webmention call for the links in a resource.
* @param {object} resource
* @param {URL} resource.url
* @param {string} resource.html
* @param {string|null} [resource.previousHtml]
*/
async sendAll(resource) {
const links = this.getLinks(resource.html);
if (resource.previousHtml) {
// We also need to send webmentions for removed links
const oldLinks = this.getLinks(resource.previousHtml);
for (const link of oldLinks) {
if (!links.find(l => l.href === link.href)) {
links.push(link);
}
}
}
if (links.length) {
logging.info('[Webmention] Sending all webmentions for ' + resource.url.href);
}
for (const target of links) {
const endpoint = await this.#discoveryService.getEndpoint(target);
if (endpoint) {
// Send webmention call
try {
await this.send({source: resource.url, target, endpoint});
} catch (e) {
logging.error('[Webmention] Failed sending via ' + endpoint.href + ': ' + e.message);
}
}
}
}
/**
* @private
* Get all external links in a HTML document.
* Excludes the site's own domain.
* @param {string} html
* @returns {URL[]}
*/
getLinks(html) {
const cheerio = require('cheerio');
const $ = cheerio.load(html);
const urls = [];
const siteUrl = this.siteUrl;
for (const el of $('a').toArray()) {
const href = $(el).attr('href');
if (href) {
let url;
try {
url = new URL(href);
if (siteUrl && url.hostname === siteUrl.hostname) {
// Ignore links to the site's own domain
continue;
}
if (['http:', 'https:'].includes(url.protocol) && !urls.find(u => u.href === url.href)) {
// Ignore duplicate URLs
urls.push(url);
}
} catch (e) {
// Ignore invalid URLs
}
}
}
return urls;
}
};

View File

@ -1,3 +1,4 @@
module.exports.InMemoryMentionRepository = require('./InMemoryMentionRepository');
module.exports.MentionsAPI = require('./MentionsAPI');
module.exports.Mention = require('./Mention');
module.exports.MentionSendingService = require('./MentionSendingService');

View File

@ -20,7 +20,12 @@
"devDependencies": {
"c8": "7.12.0",
"mocha": "10.2.0",
"sinon": "15.0.1"
"nock": "13.3.0",
"sinon": "15.0.1",
"bson-objectid": "2.0.4"
},
"dependencies": {}
"dependencies": {
"@tryghost/errors": "1.2.20",
"@tryghost/logging": "2.3.6"
}
}

View File

@ -0,0 +1,440 @@
const MentionSendingService = require('../lib/MentionSendingService.js');
const assert = require('assert');
const nock = require('nock');
// non-standard to use externalRequest here, but this is required for the overrides in the libary, which we want to test for security reasons in combination with the package
const externalRequest = require('../../core/core/server/lib/request-external.js');
const sinon = require('sinon');
const logging = require('@tryghost/logging');
const {createModel} = require('./utils/index.js');
describe('MentionSendingService', function () {
let errorLogStub;
beforeEach(function () {
nock.disableNetConnect();
sinon.stub(logging, 'info');
errorLogStub = sinon.stub(logging, 'error');
});
afterEach(function () {
nock.cleanAll();
sinon.restore();
});
describe('listen', function () {
it('Calls on post.edited', async function () {
const service = new MentionSendingService({});
const stub = sinon.stub(service, 'sendForEditedPost').resolves();
let callback;
const events = {
on: sinon.stub().callsFake((event, c) => {
callback = c;
})
};
service.listen(events);
sinon.assert.calledOnce(events.on);
await callback({});
sinon.assert.calledOnce(stub);
});
});
describe('sendForEditedPost', function () {
it('Ignores if disabled', async function () {
const service = new MentionSendingService({
isEnabled: () => false
});
const stub = sinon.stub(service, 'sendAll');
await service.sendForEditedPost({});
sinon.assert.notCalled(stub);
});
it('Ignores draft posts', async function () {
const service = new MentionSendingService({
isEnabled: () => true
});
const stub = sinon.stub(service, 'sendAll');
await service.sendForEditedPost(createModel({
status: 'draft',
html: 'changed',
previous: {
status: 'draft',
html: ''
}
}));
sinon.assert.notCalled(stub);
});
it('Ignores if html was not changed', async function () {
const service = new MentionSendingService({
isEnabled: () => true
});
const stub = sinon.stub(service, 'sendAll');
await service.sendForEditedPost(createModel({
status: 'published',
html: 'same',
previous: {
status: 'published',
html: 'same'
}
}));
sinon.assert.notCalled(stub);
});
it('Ignores email only posts', async function () {
const service = new MentionSendingService({
isEnabled: () => true
});
const stub = sinon.stub(service, 'sendAll');
await service.sendForEditedPost(createModel({
status: 'send',
html: 'changed',
previous: {
status: 'draft',
html: 'same'
}
}));
sinon.assert.notCalled(stub);
});
it('Sends on publish', async function () {
const service = new MentionSendingService({
isEnabled: () => true,
getPostUrl: () => 'https://site.com/post/'
});
const stub = sinon.stub(service, 'sendAll');
await service.sendForEditedPost(createModel({
status: 'published',
html: 'same',
previous: {
status: 'draft',
html: 'same'
}
}));
sinon.assert.calledOnce(stub);
const firstCall = stub.getCall(0).args[0];
assert.strictEqual(firstCall.url.toString(), 'https://site.com/post/');
assert.strictEqual(firstCall.html, 'same');
assert.strictEqual(firstCall.previousHtml, null);
});
it('Sends on html change', async function () {
const service = new MentionSendingService({
isEnabled: () => true,
getPostUrl: () => 'https://site.com/post/'
});
const stub = sinon.stub(service, 'sendAll');
await service.sendForEditedPost(createModel({
status: 'published',
html: 'updated',
previous: {
status: 'published',
html: 'same'
}
}));
sinon.assert.calledOnce(stub);
const firstCall = stub.getCall(0).args[0];
assert.strictEqual(firstCall.url.toString(), 'https://site.com/post/');
assert.strictEqual(firstCall.html, 'updated');
assert.strictEqual(firstCall.previousHtml, 'same');
});
it('Catches and logs errors', async function () {
const service = new MentionSendingService({
isEnabled: () => true,
getPostUrl: () => 'https://site.com/post/'
});
sinon.stub(service, 'sendAll').rejects(new Error('Internal error test'));
await service.sendForEditedPost(createModel({
status: 'published',
html: 'same',
previous: {
status: 'draft',
html: 'same'
}
}));
assert(errorLogStub.calledTwice);
});
});
describe('sendAll', function () {
it('Sends to all links', async function () {
let counter = 0;
const scope = nock('https://example.org')
.persist()
.post('/webmentions-test')
.reply(() => {
counter += 1;
return [202];
});
const service = new MentionSendingService({
externalRequest,
getSiteUrl: () => new URL('https://site.com'),
discoveryService: {
getEndpoint: async () => new URL('https://example.org/webmentions-test')
}
});
await service.sendAll({url: new URL('https://site.com'),
html: `
<html>
<body>
<a href="https://example.com">Example</a>
<a href="https://example.com">Example repeated</a>
<a href="https://example.org#fragment">Example</a>
<a href="http://example2.org">Example 2</a>
</body>
</html>
`});
assert.strictEqual(scope.isDone(), true);
assert.equal(counter, 3);
});
it('Catches and logs errors', async function () {
let counter = 0;
const scope = nock('https://example.org')
.persist()
.post('/webmentions-test')
.reply(() => {
counter += 1;
if (counter === 2) {
return [500];
}
return [202];
});
const service = new MentionSendingService({
externalRequest,
getSiteUrl: () => new URL('https://site.com'),
discoveryService: {
getEndpoint: async () => new URL('https://example.org/webmentions-test')
}
});
await service.sendAll({url: new URL('https://site.com'),
html: `
<html>
<body>
<a href="https://example.com">Example</a>
<a href="https://example.com">Example repeated</a>
<a href="https://example.org#fragment">Example</a>
<a href="http://example2.org">Example 2</a>
</body>
</html>
`});
assert.strictEqual(scope.isDone(), true);
assert.equal(counter, 3);
assert(errorLogStub.calledOnce);
});
it('Sends to deleted links', async function () {
let counter = 0;
const scope = nock('https://example.org')
.persist()
.post('/webmentions-test')
.reply(() => {
counter += 1;
return [202];
});
const service = new MentionSendingService({
externalRequest,
getSiteUrl: () => new URL('https://site.com'),
discoveryService: {
getEndpoint: async () => new URL('https://example.org/webmentions-test')
}
});
await service.sendAll({url: new URL('https://site.com'),
html: `<a href="https://example.com">Example</a>`,
previousHtml: `<a href="https://typo.com">Example</a>`});
assert.strictEqual(scope.isDone(), true);
assert.equal(counter, 2);
});
});
describe('getLinks', function () {
it('Returns all unique links in a HTML-document', async function () {
const service = new MentionSendingService({
getSiteUrl: () => new URL('https://site.com')
});
const links = service.getLinks(`
<html>
<body>
<a href="https://example.com">Example</a>
<a href="https://example.com">Example repeated</a>
<a href="https://example.org#fragment">Example</a>
<a href="http://example2.org">Example 2</a>
</body>
</html>
`);
assert.deepStrictEqual(links, [
new URL('https://example.com'),
new URL('https://example.org#fragment'),
new URL('http://example2.org')
]);
});
it('Does not include invalid or local URLs', async function () {
const service = new MentionSendingService({
getSiteUrl: () => new URL('https://site.com')
});
const links = service.getLinks(`<a href="/">Example</a>`);
assert.deepStrictEqual(links, []);
});
it('Does not include non-http protocols', async function () {
const service = new MentionSendingService({
getSiteUrl: () => new URL('https://site.com')
});
const links = service.getLinks(`<a href="ftp://invalid.com">Example</a>`);
assert.deepStrictEqual(links, []);
});
it('Does not include invalid urls', async function () {
const service = new MentionSendingService({
getSiteUrl: () => new URL('https://site.com')
});
const links = service.getLinks(`<a href="()">Example</a>`);
assert.deepStrictEqual(links, []);
});
it('Does not include urls from site domain', async function () {
const service = new MentionSendingService({
getSiteUrl: () => new URL('https://site.com')
});
const links = service.getLinks(`<a href="http://site.com/test?123">Example</a>`);
assert.deepStrictEqual(links, []);
});
it('Ignores invalid site urls', async function () {
const service = new MentionSendingService({
getSiteUrl: () => new URL('invalid()')
});
const links = service.getLinks(`<a href="http://site.com/test?123">Example</a>`);
assert.deepStrictEqual(links, [
new URL('http://site.com/test?123')
]);
});
});
describe('send', function () {
it('Can handle 202 accepted responses', async function () {
const scope = nock('https://example.org')
.persist()
.post('/webmentions-test', `source=${encodeURIComponent('https://example.com/source')}&target=${encodeURIComponent('https://target.com/target')}`)
.reply(202);
const service = new MentionSendingService({externalRequest});
await service.send({
source: new URL('https://example.com/source'),
target: new URL('https://target.com/target'),
endpoint: new URL('https://example.org/webmentions-test')
});
assert(scope.isDone());
});
it('Can handle 201 created responses', async function () {
const scope = nock('https://example.org')
.persist()
.post('/webmentions-test', `source=${encodeURIComponent('https://example.com/source')}&target=${encodeURIComponent('https://target.com/target')}`)
.reply(201);
const service = new MentionSendingService({externalRequest});
await service.send({
source: new URL('https://example.com/source'),
target: new URL('https://target.com/target'),
endpoint: new URL('https://example.org/webmentions-test')
});
assert(scope.isDone());
});
it('Can handle 400 responses', async function () {
const scope = nock('https://example.org')
.persist()
.post('/webmentions-test')
.reply(400);
const service = new MentionSendingService({externalRequest});
await assert.rejects(service.send({
source: new URL('https://example.com/source'),
target: new URL('https://target.com/target'),
endpoint: new URL('https://example.org/webmentions-test')
}), /sending failed/);
assert(scope.isDone());
});
it('Can handle 500 responses', async function () {
const scope = nock('https://example.org')
.persist()
.post('/webmentions-test')
.reply(500);
const service = new MentionSendingService({externalRequest});
await assert.rejects(service.send({
source: new URL('https://example.com/source'),
target: new URL('https://target.com/target'),
endpoint: new URL('https://example.org/webmentions-test')
}), /sending failed/);
assert(scope.isDone());
});
it('Can handle network errors', async function () {
const scope = nock('https://example.org')
.persist()
.post('/webmentions-test')
.replyWithError('network error');
const service = new MentionSendingService({externalRequest});
await assert.rejects(service.send({
source: new URL('https://example.com/source'),
target: new URL('https://target.com/target'),
endpoint: new URL('https://example.org/webmentions-test')
}), /network error/);
assert(scope.isDone());
});
// Redirects are currently not supported by got for POST requests!
//it('Can handle redirect responses', async function () {
// const scope = nock('https://example.org')
// .persist()
// .post('/webmentions-test')
// .reply(302, '', {
// headers: {
// Location: 'https://example.org/webmentions-test-2'
// }
// });
// const scope2 = nock('https://example.org')
// .persist()
// .post('/webmentions-test-2')
// .reply(201);
//
// const service = new MentionSendingService({externalRequest});
// await service.send({
// source: new URL('https://example.com'),
// target: new URL('https://example.com'),
// endpoint: new URL('https://example.org/webmentions-test')
// });
// assert(scope.isDone());
// assert(scope2.isDone());
//});
// TODO: also check if we don't follow private IPs after redirects
it('Does not send to private IP', async function () {
const service = new MentionSendingService({externalRequest});
await assert.rejects(service.send({
source: new URL('https://example.com/source'),
target: new URL('https://target.com/target'),
endpoint: new URL('http://localhost/webmentions')
}), /non-permitted private IP/);
});
it('Does not send to private IP behind DNS', async function () {
// Test that we don't make a request when a domain resolves to a private IP
// domaincontrol.com -> 127.0.0.1
const service = new MentionSendingService({externalRequest});
await assert.rejects(service.send({
source: new URL('https://example.com/source'),
target: new URL('https://target.com/target'),
endpoint: new URL('http://domaincontrol.com/webmentions')
}), /non-permitted private IP/);
});
});
});

View File

@ -0,0 +1,48 @@
const ObjectId = require('bson-objectid').default;
const createModel = (propertiesAndRelations) => {
const id = propertiesAndRelations.id ?? ObjectId().toHexString();
return {
id,
getLazyRelation: (relation) => {
propertiesAndRelations.loaded = propertiesAndRelations.loaded ?? [];
if (!propertiesAndRelations.loaded.includes(relation)) {
propertiesAndRelations.loaded.push(relation);
}
if (Array.isArray(propertiesAndRelations[relation])) {
return Promise.resolve({
models: propertiesAndRelations[relation]
});
}
return Promise.resolve(propertiesAndRelations[relation]);
},
related: (relation) => {
if (!Object.keys(propertiesAndRelations).includes('loaded')) {
throw new Error(`Model.related('${relation}'): When creating a test model via createModel you must include 'loaded' to specify which relations are already loaded and useable via Model.related.`);
}
if (!propertiesAndRelations.loaded.includes(relation)) {
throw new Error(`Model.related('${relation}') was used on a test model that didn't explicitly loaded that relation.`);
}
return propertiesAndRelations[relation];
},
get: (property) => {
return propertiesAndRelations[property];
},
previous: (property) => {
return propertiesAndRelations.previous[property];
},
save: (properties) => {
Object.assign(propertiesAndRelations, properties);
return Promise.resolve();
},
toJSON: () => {
return {
id,
...propertiesAndRelations
};
}
};
};
module.exports = {
createModel
};