Improved webmention stability in case of slow external servers (#18242)

refs https://github.com/TryGhost/Product/issues/3850

- Added a recheck for recommendation related webmentions after boot (to
check missed webmentions during down time)
- Increased general timeouts to 15s for all webmention related HTTP
requests. Instead, increased retries to 3.
- Increased timeout for fetching webmention metadata from 2s to 15s
- Added more logging about verification and deletion status of
webmentions
This commit is contained in:
Simon Backx 2023-09-20 13:09:47 +02:00 committed by GitHub
parent b4bcc193a4
commit 1e3232cf82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 223 additions and 86 deletions

View File

@ -34,7 +34,13 @@ module.exports = class WebmentionMetadata {
*/
async fetch(url) {
const mappedUrl = this.#getMappedUrl(url);
const data = await oembedService.fetchOembedDataFromUrl(mappedUrl.href, 'mention');
const data = await oembedService.fetchOembedDataFromUrl(mappedUrl.href, 'mention', {
timeout: 15000,
retry: {
// Only retry on network issues, or specific HTTP status codes
limit: 3
}
});
const result = {
siteTitle: data.metadata.publisher,
@ -50,7 +56,13 @@ module.exports = class WebmentionMetadata {
if (mappedUrl.href !== url.href) {
// Still need to fetch body and contentType separately now
// For verification
const {body, contentType} = await oembedService.fetchPageHtml(url);
const {body, contentType} = await oembedService.fetchPageHtml(url, {
timeout: 15000,
retry: {
// Only retry on network issues, or specific HTTP status codes
limit: 3
}
});
result.body = body;
result.contentType = contentType;
}

View File

@ -45,7 +45,7 @@ class RecommendationServiceWrapper {
const mentions = require('../mentions');
if (!mentions.sendingService) {
if (!mentions.sendingService || !mentions.api) {
// eslint-disable-next-line ghost/ghost-custom/no-native-error
throw new Error('MentionSendingService not intialized, but this is a dependency of RecommendationServiceWrapper. Check boot order.');
}
@ -75,7 +75,8 @@ class RecommendationServiceWrapper {
wellknownService,
mentionSendingService: mentions.sendingService,
clickEventRepository: this.clickEventRepository,
subscribeEventRepository: this.subscribeEventRepository
subscribeEventRepository: this.subscribeEventRepository,
mentionsApi: mentions.api
});
this.controller = new RecommendationController({
service: this.service

View File

@ -29,7 +29,7 @@
"Expertise": "Foglalkozás",
"Founder @ Acme Inc": "Acme Kft. alapító",
"Full-time parent": "Főállású szülő",
"Head of Marketing at Acme, Inc": "Marketing vezető — Acme Kft.",
"Head of Marketing at Acme, Inc": "Marketing vezető —\u00a0Acme Kft.",
"Hide": "Elrejtés",
"Hide comment": "Hozzászólás elrejtése",
"Jamie Larson": "Kiss Sára",

View File

@ -1,6 +1,6 @@
{
"All the best!": "Üdvözlettel,",
"Complete signup for {{siteTitle}}!": "{{siteTitle}} — Regisztráció",
"Complete signup for {{siteTitle}}!": "{{siteTitle}} —\u00a0Regisztráció",
"Complete your sign up to {{siteTitle}}!": "{{siteTitle}} — Regisztráció",
"Confirm email address": "Kérjük hagyja jóvá meg email címét",
"Confirm signup": "Regisztráció jóváhagyása",

View File

@ -131,17 +131,19 @@ class OEmbedService {
/**
* @param {string} url
* @param {Object} options
*
* @returns {Promise<{url: string, body: string, contentType: string|undefined}>}
*/
async fetchPageHtml(url) {
async fetchPageHtml(url, options = {}) {
// Fetch url and get response as binary buffer to
// avoid implicit cast
let {headers, body, url: responseUrl} = await this.fetchPage(
url,
{
encoding: 'binary',
responseType: 'buffer'
responseType: 'buffer',
...options
});
try {
@ -328,10 +330,12 @@ class OEmbedService {
/**
* @param {string} url - oembed URL
* @param {string} type - card type
* @param {Object} [options] Specific fetch options
* @param {number} [options.timeout] Change the default timeout for fetching html
*
* @returns {Promise<Object>}
*/
async fetchOembedDataFromUrl(url, type) {
async fetchOembedDataFromUrl(url, type, options = {}) {
try {
const urlObject = new URL(url);
@ -358,7 +362,7 @@ class OEmbedService {
}
// Not in the list, we need to fetch the content
const {url: pageUrl, body, contentType} = await this.fetchPageHtml(url);
const {url: pageUrl, body, contentType} = await this.fetchPageHtml(url, options);
// fetch only bookmark when explicitly requested
if (type === 'bookmark') {

View File

@ -6,6 +6,7 @@ import errors from '@tryghost/errors';
import tpl from '@tryghost/tpl';
import {ClickEvent} from './ClickEvent';
import {SubscribeEvent} from './SubscribeEvent';
import logging from '@tryghost/logging';
export type RecommendationIncludeTypes = {
'count.clicks': number,
@ -26,6 +27,10 @@ type MentionSendingService = {
sendAll(options: {url: URL, links: URL[]}): Promise<void>
}
type MentionsAPI = {
refreshMentions(options: {filter: string, limit: number|'all'}): Promise<void>
}
type RecommendationEnablerService = {
getSetting(): string,
setSetting(value: string): Promise<void>
@ -43,6 +48,7 @@ export class RecommendationService {
wellknownService: WellknownService;
mentionSendingService: MentionSendingService;
recommendationEnablerService: RecommendationEnablerService;
mentionsApi: MentionsAPI;
constructor(deps: {
repository: RecommendationRepository,
@ -51,6 +57,7 @@ export class RecommendationService {
wellknownService: WellknownService,
mentionSendingService: MentionSendingService,
recommendationEnablerService: RecommendationEnablerService,
mentionsApi: MentionsAPI
}) {
this.repository = deps.repository;
this.wellknownService = deps.wellknownService;
@ -58,11 +65,31 @@ export class RecommendationService {
this.recommendationEnablerService = deps.recommendationEnablerService;
this.clickEventRepository = deps.clickEventRepository;
this.subscribeEventRepository = deps.subscribeEventRepository;
this.mentionsApi = deps.mentionsApi;
}
async init() {
const recommendations = await this.#listRecommendations();
await this.updateWellknown(recommendations);
// When we boot, it is possible that we missed some webmentions from other sites recommending you
// More importantly, we might have missed some deletes which we can detect.
// So we do a slow revalidation of all incoming recommendations
// This also prevents doing multiple external fetches when doing quick reboots of Ghost after each other (requires Ghost to be up for at least 15 seconds)
if (!process.env.NODE_ENV?.startsWith('test')) {
setTimeout(() => {
logging.info('Updating incoming recommendations on boot');
this.#updateIncomingRecommendations().catch((err) => {
logging.error('Failed to update incoming recommendations on boot', err);
});
}, 15 * 1000 + Math.random() * 5 * 60 * 1000);
}
}
async #updateIncomingRecommendations() {
// Note: we also recheck recommendations that were not verified (verification could have failed)
const filter = `source:~$'/.well-known/recommendations.json'`;
await this.mentionsApi.refreshMentions({filter, limit: 100});
}
async updateWellknown(recommendations: Recommendation[]) {
@ -86,7 +113,9 @@ export class RecommendationService {
links: [
recommendation.url
]
}).catch(console.error); // eslint-disable-line no-console
}).catch((err) => {
logging.error('Failed to send mention to recommendation', err);
});
}
async readRecommendation(id: string): Promise<RecommendationPlain> {

View File

@ -19,6 +19,17 @@ module.exports = class Mention {
return this.#verified;
}
/** @type {boolean} */
#deleted = false;
get deleted() {
return this.#deleted;
}
delete() {
this.#deleted = true;
}
/**
* @param {string} html
* @param {string} contentType
@ -177,11 +188,6 @@ module.exports = class Mention {
this.#sourceFeaturedImage = sourceFeaturedImage;
}
#deleted = false;
delete() {
this.#deleted = true;
}
toJSON() {
return {
id: this.id,

View File

@ -1,5 +1,4 @@
const cheerio = require('cheerio');
const errors = require('@tryghost/errors');
module.exports = class MentionDiscoveryService {
#externalRequest;
@ -16,16 +15,15 @@ module.exports = class MentionDiscoveryService {
async getEndpoint(url) {
try {
const response = await this.#externalRequest(url.href, {
throwHttpErrors: false,
throwHttpErrors: true,
followRedirect: true,
maxRedirects: 10
maxRedirects: 10,
timeout: 15000,
retry: {
// Only retry on network issues, or specific HTTP status codes
limit: 3
}
});
if (response.statusCode === 404) {
throw new errors.BadRequestError({
message: 'Webmention discovery service could not find target site',
statusCode: response.statusCode
});
}
return this.getEndpointFromResponse(response);
} catch (error) {
return null;

View File

@ -97,7 +97,11 @@ module.exports = class MentionSendingService {
throwHttpErrors: false,
maxRedirects: 10,
followRedirect: true,
timeout: 10000
timeout: 15000,
retry: {
// Only retry on network issues, or specific HTTP status codes
limit: 3
}
});
if (response.statusCode >= 200 && response.statusCode < 300) {

View File

@ -163,6 +163,101 @@ module.exports = class MentionsAPI {
return page;
}
/**
* Update the metadata of the webmentions in the database, and delete them if they are no longer valid.
* @param {object} options
* @param {number|'all'} [options.limit]
* @param {number} [options.page]
* @param {string} [options.filter]
*/
async refreshMentions(options) {
const mentions = await this.#repository.getAll(options);
for (const mention of mentions) {
await this.#updateWebmention(mention, {
source: mention.source,
target: mention.target
});
}
}
async #updateWebmention(mention, webmention) {
const isNew = !mention;
const targetExists = await this.#routingService.pageExists(webmention.target);
if (!targetExists) {
if (!mention) {
throw new errors.BadRequestError({
message: `${webmention.target} is not a valid URL for this site.`
});
} else {
mention.delete();
}
}
if (targetExists) {
const resourceInfo = await this.#resourceService.getByURL(webmention.target);
let metadata;
try {
metadata = await this.#webmentionMetadata.fetch(webmention.source);
if (mention) {
mention.setSourceMetadata({
sourceTitle: metadata.title,
sourceSiteTitle: metadata.siteTitle,
sourceAuthor: metadata.author,
sourceExcerpt: metadata.excerpt,
sourceFavicon: metadata.favicon,
sourceFeaturedImage: metadata.image
});
}
} catch (err) {
if (!mention) {
throw err;
}
mention.delete();
}
if (!mention) {
mention = await Mention.create({
source: webmention.source,
target: webmention.target,
timestamp: new Date(),
payload: webmention.payload,
resourceId: resourceInfo.id ? resourceInfo.id.toHexString() : null,
resourceType: resourceInfo.type,
sourceTitle: metadata.title,
sourceSiteTitle: metadata.siteTitle,
sourceAuthor: metadata.author,
sourceExcerpt: metadata.excerpt,
sourceFavicon: metadata.favicon,
sourceFeaturedImage: metadata.image
});
}
if (metadata?.body) {
try {
mention.verify(metadata.body, metadata.contentType);
} catch (e) {
logging.error(e);
}
}
}
await this.#repository.save(mention);
if (isNew) {
logging.info('[Webmention] Created ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
} else {
if (mention.deleted) {
logging.info('[Webmention] Deleted ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
} else {
logging.info('[Webmention] Updated ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
}
}
return mention;
}
/**
* @param {object} webmention
* @param {URL} webmention.source
@ -177,65 +272,6 @@ module.exports = class MentionsAPI {
webmention.target
);
const targetExists = await this.#routingService.pageExists(webmention.target);
if (!targetExists) {
if (!mention) {
throw new errors.BadRequestError({
message: `${webmention.target} is not a valid URL for this site.`
});
} else {
mention.delete();
}
}
const resourceInfo = await this.#resourceService.getByURL(webmention.target);
let metadata;
try {
metadata = await this.#webmentionMetadata.fetch(webmention.source);
if (mention) {
mention.setSourceMetadata({
sourceTitle: metadata.title,
sourceSiteTitle: metadata.siteTitle,
sourceAuthor: metadata.author,
sourceExcerpt: metadata.excerpt,
sourceFavicon: metadata.favicon,
sourceFeaturedImage: metadata.image
});
}
} catch (err) {
if (!mention) {
throw err;
}
mention.delete();
}
if (!mention) {
mention = await Mention.create({
source: webmention.source,
target: webmention.target,
timestamp: new Date(),
payload: webmention.payload,
resourceId: resourceInfo.id ? resourceInfo.id.toHexString() : null,
resourceType: resourceInfo.type,
sourceTitle: metadata.title,
sourceSiteTitle: metadata.siteTitle,
sourceAuthor: metadata.author,
sourceExcerpt: metadata.excerpt,
sourceFavicon: metadata.favicon,
sourceFeaturedImage: metadata.image
});
}
if (metadata?.body) {
try {
mention.verify(metadata.body, metadata.contentType);
} catch (e) {
logging.error(e);
}
}
await this.#repository.save(mention);
return mention;
return await this.#updateWebmention(mention, webmention);
}
};

View File

@ -19,6 +19,7 @@ const mockResourceService = {
};
}
};
const mockWebmentionMetadata = {
async fetch() {
return {
@ -223,6 +224,52 @@ describe('MentionsAPI', function () {
assert(page.data[1].id === mentionTwo.id, 'Second mention should be the second one in ascending order');
});
it('Can update recommendations', async function () {
const repository = new InMemoryMentionRepository();
const api = new MentionsAPI({
repository,
routingService: mockRoutingService,
resourceService: mockResourceService,
webmentionMetadata: {
fetch: sinon.stub()
.onFirstCall().resolves(mockWebmentionMetadata.fetch())
.onSecondCall().resolves(mockWebmentionMetadata.fetch())
.onThirdCall().resolves(mockWebmentionMetadata.fetch())
.onCall(3).rejects()
}
});
await api.processWebmention({
source: new URL('https://source.com'),
target: new URL('https://target.com'),
payload: {}
});
sinon.useFakeTimers(addMinutes(new Date(), 10).getTime());
await api.processWebmention({
source: new URL('https://source2.com'),
target: new URL('https://target.com'),
payload: {}
});
let page = await api.listMentions({
limit: 'all'
});
assert(page.meta.pagination.total === 2);
// Now we invalidate the second mention
await api.refreshMentions({
limit: 'all'
});
page = await api.listMentions({
limit: 'all'
});
assert(page.meta.pagination.total === 1);
});
it('Can handle updating mentions', async function () {
const repository = new InMemoryMentionRepository();
const api = new MentionsAPI({