Improved webmention stability in case of slow external servers (#18242)
refs https://github.com/TryGhost/Product/issues/3850 - Added a recheck for recommendation related webmentions after boot (to check missed webmentions during down time) - Increased general timeouts to 15s for all webmention related HTTP requests. Instead, increased retries to 3. - Increased timeout for fetching webmention metadata from 2s to 15s - Added more logging about verification and deletion status of webmentions
This commit is contained in:
parent
b4bcc193a4
commit
1e3232cf82
@ -34,7 +34,13 @@ module.exports = class WebmentionMetadata {
|
||||
*/
|
||||
async fetch(url) {
|
||||
const mappedUrl = this.#getMappedUrl(url);
|
||||
const data = await oembedService.fetchOembedDataFromUrl(mappedUrl.href, 'mention');
|
||||
const data = await oembedService.fetchOembedDataFromUrl(mappedUrl.href, 'mention', {
|
||||
timeout: 15000,
|
||||
retry: {
|
||||
// Only retry on network issues, or specific HTTP status codes
|
||||
limit: 3
|
||||
}
|
||||
});
|
||||
|
||||
const result = {
|
||||
siteTitle: data.metadata.publisher,
|
||||
@ -50,7 +56,13 @@ module.exports = class WebmentionMetadata {
|
||||
if (mappedUrl.href !== url.href) {
|
||||
// Still need to fetch body and contentType separately now
|
||||
// For verification
|
||||
const {body, contentType} = await oembedService.fetchPageHtml(url);
|
||||
const {body, contentType} = await oembedService.fetchPageHtml(url, {
|
||||
timeout: 15000,
|
||||
retry: {
|
||||
// Only retry on network issues, or specific HTTP status codes
|
||||
limit: 3
|
||||
}
|
||||
});
|
||||
result.body = body;
|
||||
result.contentType = contentType;
|
||||
}
|
||||
|
@ -45,7 +45,7 @@ class RecommendationServiceWrapper {
|
||||
|
||||
const mentions = require('../mentions');
|
||||
|
||||
if (!mentions.sendingService) {
|
||||
if (!mentions.sendingService || !mentions.api) {
|
||||
// eslint-disable-next-line ghost/ghost-custom/no-native-error
|
||||
throw new Error('MentionSendingService not intialized, but this is a dependency of RecommendationServiceWrapper. Check boot order.');
|
||||
}
|
||||
@ -75,7 +75,8 @@ class RecommendationServiceWrapper {
|
||||
wellknownService,
|
||||
mentionSendingService: mentions.sendingService,
|
||||
clickEventRepository: this.clickEventRepository,
|
||||
subscribeEventRepository: this.subscribeEventRepository
|
||||
subscribeEventRepository: this.subscribeEventRepository,
|
||||
mentionsApi: mentions.api
|
||||
});
|
||||
this.controller = new RecommendationController({
|
||||
service: this.service
|
||||
|
@ -29,7 +29,7 @@
|
||||
"Expertise": "Foglalkozás",
|
||||
"Founder @ Acme Inc": "Acme Kft. alapító",
|
||||
"Full-time parent": "Főállású szülő",
|
||||
"Head of Marketing at Acme, Inc": "Marketing vezető — Acme Kft.",
|
||||
"Head of Marketing at Acme, Inc": "Marketing vezető —\u00a0Acme Kft.",
|
||||
"Hide": "Elrejtés",
|
||||
"Hide comment": "Hozzászólás elrejtése",
|
||||
"Jamie Larson": "Kiss Sára",
|
||||
|
@ -1,6 +1,6 @@
|
||||
{
|
||||
"All the best!": "Üdvözlettel,",
|
||||
"Complete signup for {{siteTitle}}!": "{{siteTitle}} — Regisztráció",
|
||||
"Complete signup for {{siteTitle}}!": "{{siteTitle}} —\u00a0Regisztráció",
|
||||
"Complete your sign up to {{siteTitle}}!": "{{siteTitle}} — Regisztráció",
|
||||
"Confirm email address": "Kérjük hagyja jóvá meg email címét",
|
||||
"Confirm signup": "Regisztráció jóváhagyása",
|
||||
|
@ -131,17 +131,19 @@ class OEmbedService {
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @param {Object} options
|
||||
*
|
||||
* @returns {Promise<{url: string, body: string, contentType: string|undefined}>}
|
||||
*/
|
||||
async fetchPageHtml(url) {
|
||||
async fetchPageHtml(url, options = {}) {
|
||||
// Fetch url and get response as binary buffer to
|
||||
// avoid implicit cast
|
||||
let {headers, body, url: responseUrl} = await this.fetchPage(
|
||||
url,
|
||||
{
|
||||
encoding: 'binary',
|
||||
responseType: 'buffer'
|
||||
responseType: 'buffer',
|
||||
...options
|
||||
});
|
||||
|
||||
try {
|
||||
@ -328,10 +330,12 @@ class OEmbedService {
|
||||
/**
|
||||
* @param {string} url - oembed URL
|
||||
* @param {string} type - card type
|
||||
* @param {Object} [options] Specific fetch options
|
||||
* @param {number} [options.timeout] Change the default timeout for fetching html
|
||||
*
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
async fetchOembedDataFromUrl(url, type) {
|
||||
async fetchOembedDataFromUrl(url, type, options = {}) {
|
||||
try {
|
||||
const urlObject = new URL(url);
|
||||
|
||||
@ -358,7 +362,7 @@ class OEmbedService {
|
||||
}
|
||||
|
||||
// Not in the list, we need to fetch the content
|
||||
const {url: pageUrl, body, contentType} = await this.fetchPageHtml(url);
|
||||
const {url: pageUrl, body, contentType} = await this.fetchPageHtml(url, options);
|
||||
|
||||
// fetch only bookmark when explicitly requested
|
||||
if (type === 'bookmark') {
|
||||
|
@ -6,6 +6,7 @@ import errors from '@tryghost/errors';
|
||||
import tpl from '@tryghost/tpl';
|
||||
import {ClickEvent} from './ClickEvent';
|
||||
import {SubscribeEvent} from './SubscribeEvent';
|
||||
import logging from '@tryghost/logging';
|
||||
|
||||
export type RecommendationIncludeTypes = {
|
||||
'count.clicks': number,
|
||||
@ -26,6 +27,10 @@ type MentionSendingService = {
|
||||
sendAll(options: {url: URL, links: URL[]}): Promise<void>
|
||||
}
|
||||
|
||||
type MentionsAPI = {
|
||||
refreshMentions(options: {filter: string, limit: number|'all'}): Promise<void>
|
||||
}
|
||||
|
||||
type RecommendationEnablerService = {
|
||||
getSetting(): string,
|
||||
setSetting(value: string): Promise<void>
|
||||
@ -43,6 +48,7 @@ export class RecommendationService {
|
||||
wellknownService: WellknownService;
|
||||
mentionSendingService: MentionSendingService;
|
||||
recommendationEnablerService: RecommendationEnablerService;
|
||||
mentionsApi: MentionsAPI;
|
||||
|
||||
constructor(deps: {
|
||||
repository: RecommendationRepository,
|
||||
@ -51,6 +57,7 @@ export class RecommendationService {
|
||||
wellknownService: WellknownService,
|
||||
mentionSendingService: MentionSendingService,
|
||||
recommendationEnablerService: RecommendationEnablerService,
|
||||
mentionsApi: MentionsAPI
|
||||
}) {
|
||||
this.repository = deps.repository;
|
||||
this.wellknownService = deps.wellknownService;
|
||||
@ -58,11 +65,31 @@ export class RecommendationService {
|
||||
this.recommendationEnablerService = deps.recommendationEnablerService;
|
||||
this.clickEventRepository = deps.clickEventRepository;
|
||||
this.subscribeEventRepository = deps.subscribeEventRepository;
|
||||
this.mentionsApi = deps.mentionsApi;
|
||||
}
|
||||
|
||||
async init() {
|
||||
const recommendations = await this.#listRecommendations();
|
||||
await this.updateWellknown(recommendations);
|
||||
|
||||
// When we boot, it is possible that we missed some webmentions from other sites recommending you
|
||||
// More importantly, we might have missed some deletes which we can detect.
|
||||
// So we do a slow revalidation of all incoming recommendations
|
||||
// This also prevents doing multiple external fetches when doing quick reboots of Ghost after each other (requires Ghost to be up for at least 15 seconds)
|
||||
if (!process.env.NODE_ENV?.startsWith('test')) {
|
||||
setTimeout(() => {
|
||||
logging.info('Updating incoming recommendations on boot');
|
||||
this.#updateIncomingRecommendations().catch((err) => {
|
||||
logging.error('Failed to update incoming recommendations on boot', err);
|
||||
});
|
||||
}, 15 * 1000 + Math.random() * 5 * 60 * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
async #updateIncomingRecommendations() {
|
||||
// Note: we also recheck recommendations that were not verified (verification could have failed)
|
||||
const filter = `source:~$'/.well-known/recommendations.json'`;
|
||||
await this.mentionsApi.refreshMentions({filter, limit: 100});
|
||||
}
|
||||
|
||||
async updateWellknown(recommendations: Recommendation[]) {
|
||||
@ -86,7 +113,9 @@ export class RecommendationService {
|
||||
links: [
|
||||
recommendation.url
|
||||
]
|
||||
}).catch(console.error); // eslint-disable-line no-console
|
||||
}).catch((err) => {
|
||||
logging.error('Failed to send mention to recommendation', err);
|
||||
});
|
||||
}
|
||||
|
||||
async readRecommendation(id: string): Promise<RecommendationPlain> {
|
||||
|
@ -19,6 +19,17 @@ module.exports = class Mention {
|
||||
return this.#verified;
|
||||
}
|
||||
|
||||
/** @type {boolean} */
|
||||
#deleted = false;
|
||||
|
||||
get deleted() {
|
||||
return this.#deleted;
|
||||
}
|
||||
|
||||
delete() {
|
||||
this.#deleted = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} html
|
||||
* @param {string} contentType
|
||||
@ -177,11 +188,6 @@ module.exports = class Mention {
|
||||
this.#sourceFeaturedImage = sourceFeaturedImage;
|
||||
}
|
||||
|
||||
#deleted = false;
|
||||
delete() {
|
||||
this.#deleted = true;
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return {
|
||||
id: this.id,
|
||||
|
@ -1,5 +1,4 @@
|
||||
const cheerio = require('cheerio');
|
||||
const errors = require('@tryghost/errors');
|
||||
|
||||
module.exports = class MentionDiscoveryService {
|
||||
#externalRequest;
|
||||
@ -16,16 +15,15 @@ module.exports = class MentionDiscoveryService {
|
||||
async getEndpoint(url) {
|
||||
try {
|
||||
const response = await this.#externalRequest(url.href, {
|
||||
throwHttpErrors: false,
|
||||
throwHttpErrors: true,
|
||||
followRedirect: true,
|
||||
maxRedirects: 10
|
||||
maxRedirects: 10,
|
||||
timeout: 15000,
|
||||
retry: {
|
||||
// Only retry on network issues, or specific HTTP status codes
|
||||
limit: 3
|
||||
}
|
||||
});
|
||||
if (response.statusCode === 404) {
|
||||
throw new errors.BadRequestError({
|
||||
message: 'Webmention discovery service could not find target site',
|
||||
statusCode: response.statusCode
|
||||
});
|
||||
}
|
||||
return this.getEndpointFromResponse(response);
|
||||
} catch (error) {
|
||||
return null;
|
||||
|
@ -97,7 +97,11 @@ module.exports = class MentionSendingService {
|
||||
throwHttpErrors: false,
|
||||
maxRedirects: 10,
|
||||
followRedirect: true,
|
||||
timeout: 10000
|
||||
timeout: 15000,
|
||||
retry: {
|
||||
// Only retry on network issues, or specific HTTP status codes
|
||||
limit: 3
|
||||
}
|
||||
});
|
||||
|
||||
if (response.statusCode >= 200 && response.statusCode < 300) {
|
||||
|
@ -163,6 +163,101 @@ module.exports = class MentionsAPI {
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the metadata of the webmentions in the database, and delete them if they are no longer valid.
|
||||
* @param {object} options
|
||||
* @param {number|'all'} [options.limit]
|
||||
* @param {number} [options.page]
|
||||
* @param {string} [options.filter]
|
||||
*/
|
||||
async refreshMentions(options) {
|
||||
const mentions = await this.#repository.getAll(options);
|
||||
|
||||
for (const mention of mentions) {
|
||||
await this.#updateWebmention(mention, {
|
||||
source: mention.source,
|
||||
target: mention.target
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async #updateWebmention(mention, webmention) {
|
||||
const isNew = !mention;
|
||||
const targetExists = await this.#routingService.pageExists(webmention.target);
|
||||
|
||||
if (!targetExists) {
|
||||
if (!mention) {
|
||||
throw new errors.BadRequestError({
|
||||
message: `${webmention.target} is not a valid URL for this site.`
|
||||
});
|
||||
} else {
|
||||
mention.delete();
|
||||
}
|
||||
}
|
||||
|
||||
if (targetExists) {
|
||||
const resourceInfo = await this.#resourceService.getByURL(webmention.target);
|
||||
let metadata;
|
||||
try {
|
||||
metadata = await this.#webmentionMetadata.fetch(webmention.source);
|
||||
if (mention) {
|
||||
mention.setSourceMetadata({
|
||||
sourceTitle: metadata.title,
|
||||
sourceSiteTitle: metadata.siteTitle,
|
||||
sourceAuthor: metadata.author,
|
||||
sourceExcerpt: metadata.excerpt,
|
||||
sourceFavicon: metadata.favicon,
|
||||
sourceFeaturedImage: metadata.image
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
if (!mention) {
|
||||
throw err;
|
||||
}
|
||||
mention.delete();
|
||||
}
|
||||
|
||||
if (!mention) {
|
||||
mention = await Mention.create({
|
||||
source: webmention.source,
|
||||
target: webmention.target,
|
||||
timestamp: new Date(),
|
||||
payload: webmention.payload,
|
||||
resourceId: resourceInfo.id ? resourceInfo.id.toHexString() : null,
|
||||
resourceType: resourceInfo.type,
|
||||
sourceTitle: metadata.title,
|
||||
sourceSiteTitle: metadata.siteTitle,
|
||||
sourceAuthor: metadata.author,
|
||||
sourceExcerpt: metadata.excerpt,
|
||||
sourceFavicon: metadata.favicon,
|
||||
sourceFeaturedImage: metadata.image
|
||||
});
|
||||
}
|
||||
|
||||
if (metadata?.body) {
|
||||
try {
|
||||
mention.verify(metadata.body, metadata.contentType);
|
||||
} catch (e) {
|
||||
logging.error(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await this.#repository.save(mention);
|
||||
|
||||
if (isNew) {
|
||||
logging.info('[Webmention] Created ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
|
||||
} else {
|
||||
if (mention.deleted) {
|
||||
logging.info('[Webmention] Deleted ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
|
||||
} else {
|
||||
logging.info('[Webmention] Updated ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified);
|
||||
}
|
||||
}
|
||||
|
||||
return mention;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} webmention
|
||||
* @param {URL} webmention.source
|
||||
@ -177,65 +272,6 @@ module.exports = class MentionsAPI {
|
||||
webmention.target
|
||||
);
|
||||
|
||||
const targetExists = await this.#routingService.pageExists(webmention.target);
|
||||
|
||||
if (!targetExists) {
|
||||
if (!mention) {
|
||||
throw new errors.BadRequestError({
|
||||
message: `${webmention.target} is not a valid URL for this site.`
|
||||
});
|
||||
} else {
|
||||
mention.delete();
|
||||
}
|
||||
}
|
||||
|
||||
const resourceInfo = await this.#resourceService.getByURL(webmention.target);
|
||||
let metadata;
|
||||
try {
|
||||
metadata = await this.#webmentionMetadata.fetch(webmention.source);
|
||||
if (mention) {
|
||||
mention.setSourceMetadata({
|
||||
sourceTitle: metadata.title,
|
||||
sourceSiteTitle: metadata.siteTitle,
|
||||
sourceAuthor: metadata.author,
|
||||
sourceExcerpt: metadata.excerpt,
|
||||
sourceFavicon: metadata.favicon,
|
||||
sourceFeaturedImage: metadata.image
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
if (!mention) {
|
||||
throw err;
|
||||
}
|
||||
mention.delete();
|
||||
}
|
||||
|
||||
if (!mention) {
|
||||
mention = await Mention.create({
|
||||
source: webmention.source,
|
||||
target: webmention.target,
|
||||
timestamp: new Date(),
|
||||
payload: webmention.payload,
|
||||
resourceId: resourceInfo.id ? resourceInfo.id.toHexString() : null,
|
||||
resourceType: resourceInfo.type,
|
||||
sourceTitle: metadata.title,
|
||||
sourceSiteTitle: metadata.siteTitle,
|
||||
sourceAuthor: metadata.author,
|
||||
sourceExcerpt: metadata.excerpt,
|
||||
sourceFavicon: metadata.favicon,
|
||||
sourceFeaturedImage: metadata.image
|
||||
});
|
||||
}
|
||||
|
||||
if (metadata?.body) {
|
||||
try {
|
||||
mention.verify(metadata.body, metadata.contentType);
|
||||
} catch (e) {
|
||||
logging.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
await this.#repository.save(mention);
|
||||
return mention;
|
||||
return await this.#updateWebmention(mention, webmention);
|
||||
}
|
||||
};
|
||||
|
@ -19,6 +19,7 @@ const mockResourceService = {
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const mockWebmentionMetadata = {
|
||||
async fetch() {
|
||||
return {
|
||||
@ -223,6 +224,52 @@ describe('MentionsAPI', function () {
|
||||
assert(page.data[1].id === mentionTwo.id, 'Second mention should be the second one in ascending order');
|
||||
});
|
||||
|
||||
it('Can update recommendations', async function () {
|
||||
const repository = new InMemoryMentionRepository();
|
||||
const api = new MentionsAPI({
|
||||
repository,
|
||||
routingService: mockRoutingService,
|
||||
resourceService: mockResourceService,
|
||||
webmentionMetadata: {
|
||||
fetch: sinon.stub()
|
||||
.onFirstCall().resolves(mockWebmentionMetadata.fetch())
|
||||
.onSecondCall().resolves(mockWebmentionMetadata.fetch())
|
||||
.onThirdCall().resolves(mockWebmentionMetadata.fetch())
|
||||
.onCall(3).rejects()
|
||||
}
|
||||
});
|
||||
|
||||
await api.processWebmention({
|
||||
source: new URL('https://source.com'),
|
||||
target: new URL('https://target.com'),
|
||||
payload: {}
|
||||
});
|
||||
|
||||
sinon.useFakeTimers(addMinutes(new Date(), 10).getTime());
|
||||
|
||||
await api.processWebmention({
|
||||
source: new URL('https://source2.com'),
|
||||
target: new URL('https://target.com'),
|
||||
payload: {}
|
||||
});
|
||||
|
||||
let page = await api.listMentions({
|
||||
limit: 'all'
|
||||
});
|
||||
assert(page.meta.pagination.total === 2);
|
||||
|
||||
// Now we invalidate the second mention
|
||||
|
||||
await api.refreshMentions({
|
||||
limit: 'all'
|
||||
});
|
||||
|
||||
page = await api.listMentions({
|
||||
limit: 'all'
|
||||
});
|
||||
assert(page.meta.pagination.total === 1);
|
||||
});
|
||||
|
||||
it('Can handle updating mentions', async function () {
|
||||
const repository = new InMemoryMentionRepository();
|
||||
const api = new MentionsAPI({
|
||||
|
Loading…
Reference in New Issue
Block a user