Added well known recommendations service (#17895)
fixes https://github.com/TryGhost/Product/issues/3797 fixes https://github.com/TryGhost/Product/issues/3776 fixes https://github.com/TryGhost/Product/issues/3798 - Added support for storing json webmentions - Improved handling deleted webmentions (set deleted to true instead of verified to false)
This commit is contained in:
Simon Backx
GitHub
parent
e97b71dc52
commit
96fefaea69
@ -26,7 +26,7 @@ function matchCacheKey(req, cache) {
|
||||
return true;
|
||||
}
|
||||
|
||||
function createPublicFileMiddleware(location, file, mime, maxAge) {
|
||||
function createPublicFileMiddleware(location, file, mime, maxAge, options = {}) {
|
||||
let cache;
|
||||
// These files are provided by Ghost, and therefore live inside of the core folder
|
||||
const staticFilePath = config.get('paths').publicFilePath;
|
||||
@ -45,7 +45,7 @@ function createPublicFileMiddleware(location, file, mime, maxAge) {
|
||||
}
|
||||
|
||||
// send image files directly and let express handle content-length, etag, etc
|
||||
if (mime.match(/^image/)) {
|
||||
if (mime.match(/^image/) || options.disableServerCache) {
|
||||
return res.sendFile(filePath, (err) => {
|
||||
if (err && err.status === 404) {
|
||||
// ensure we're triggering basic asset 404 and not a templated 404
|
||||
@ -101,8 +101,8 @@ function createPublicFileMiddleware(location, file, mime, maxAge) {
|
||||
|
||||
// ### servePublicFile Middleware
|
||||
// Handles requests to robots.txt and favicon.ico (and caches them)
|
||||
function servePublicFile(location, file, type, maxAge) {
|
||||
const publicFileMiddleware = createPublicFileMiddleware(location, file, type, maxAge);
|
||||
function servePublicFile(location, file, type, maxAge, options = {}) {
|
||||
const publicFileMiddleware = createPublicFileMiddleware(location, file, type, maxAge, options);
|
||||
|
||||
return function servePublicFileMiddleware(req, res, next) {
|
||||
if (req.path === '/' + file) {
|
||||
|
||||
@ -98,6 +98,9 @@ module.exports = function setupSiteApp(routerConfig) {
|
||||
(req, res, next) => membersService.api.middleware.wellKnown(req, res, next)
|
||||
);
|
||||
|
||||
// Recommendations well-known
|
||||
siteApp.use(mw.servePublicFile('built', '.well-known/recommendations.json', 'application/json', config.get('caching:publicAssets:maxAge'), {disableServerCache: true}));
|
||||
|
||||
// setup middleware for internal apps
|
||||
// @TODO: refactor this to be a proper app middleware hook for internal apps
|
||||
config.get('apps:internal').forEach((appName) => {
|
||||
|
||||
@ -14,7 +14,8 @@ module.exports = class WebmentionMetadata {
|
||||
author: data.metadata.author,
|
||||
image: data.metadata.thumbnail ? new URL(data.metadata.thumbnail) : null,
|
||||
favicon: data.metadata.icon ? new URL(data.metadata.icon) : null,
|
||||
body: data.body
|
||||
body: data.body,
|
||||
contentType: data.contentType
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -28,6 +28,8 @@ module.exports = {
|
||||
/** @type {import('@tryghost/webmentions/lib/MentionsAPI')} */
|
||||
api: null,
|
||||
controller: new MentionController(),
|
||||
/** @type {import('@tryghost/webmentions/lib/MentionSendingService')} */
|
||||
sendingService: null,
|
||||
didInit: false,
|
||||
async init() {
|
||||
if (this.didInit) {
|
||||
@ -107,5 +109,7 @@ module.exports = {
|
||||
}
|
||||
});
|
||||
sendingService.listen(events);
|
||||
|
||||
this.sendingService = sendingService;
|
||||
}
|
||||
};
|
||||
|
||||
@ -19,15 +19,34 @@ class RecommendationServiceWrapper {
|
||||
return;
|
||||
}
|
||||
|
||||
const {InMemoryRecommendationRepository, RecommendationService, RecommendationController} = require('@tryghost/recommendations');
|
||||
const config = require('../../../shared/config');
|
||||
const urlUtils = require('../../../shared/url-utils');
|
||||
const {InMemoryRecommendationRepository, RecommendationService, RecommendationController, WellknownService} = require('@tryghost/recommendations');
|
||||
|
||||
const mentions = require('../mentions');
|
||||
|
||||
if (!mentions.sendingService) {
|
||||
// eslint-disable-next-line ghost/ghost-custom/no-native-error
|
||||
throw new Error('MentionSendingService not intialized, but this is a dependency of RecommendationServiceWrapper. Check boot order.');
|
||||
}
|
||||
|
||||
const wellknownService = new WellknownService({
|
||||
dir: config.getContentPath('public'),
|
||||
urlUtils
|
||||
});
|
||||
|
||||
this.repository = new InMemoryRecommendationRepository();
|
||||
this.service = new RecommendationService({
|
||||
repository: this.repository
|
||||
repository: this.repository,
|
||||
wellknownService,
|
||||
mentionSendingService: mentions.sendingService
|
||||
});
|
||||
this.controller = new RecommendationController({
|
||||
service: this.service
|
||||
});
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
this.service.init().catch(console.error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ Object {
|
||||
"reason": "Because dogs are cute",
|
||||
"title": "Dog Pictures",
|
||||
"updated_at": null,
|
||||
"url": "https://dogpictures.com",
|
||||
"url": "https://dogpictures.com/",
|
||||
},
|
||||
],
|
||||
}
|
||||
@ -23,7 +23,7 @@ exports[`Recommendations Admin API Can add a full recommendation 2: [headers] 1`
|
||||
Object {
|
||||
"access-control-allow-origin": "http://127.0.0.1:2369",
|
||||
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
|
||||
"content-length": "353",
|
||||
"content-length": "354",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
|
||||
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
|
||||
@ -47,7 +47,7 @@ Object {
|
||||
"reason": null,
|
||||
"title": "Dog Pictures",
|
||||
"updated_at": null,
|
||||
"url": "https://dogpictures.com",
|
||||
"url": "https://dogpictures.com/",
|
||||
},
|
||||
],
|
||||
}
|
||||
@ -57,7 +57,7 @@ exports[`Recommendations Admin API Can add a minimal recommendation 2: [headers]
|
||||
Object {
|
||||
"access-control-allow-origin": "http://127.0.0.1:2369",
|
||||
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
|
||||
"content-length": "262",
|
||||
"content-length": "263",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
|
||||
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
|
||||
@ -113,7 +113,7 @@ Object {
|
||||
"reason": "Because dogs are cute",
|
||||
"title": "Dog Pictures",
|
||||
"updated_at": null,
|
||||
"url": "https://dogpictures.com",
|
||||
"url": "https://dogpictures.com/",
|
||||
},
|
||||
],
|
||||
}
|
||||
@ -123,7 +123,7 @@ exports[`Recommendations Admin API Can browse 2: [headers] 1`] = `
|
||||
Object {
|
||||
"access-control-allow-origin": "http://127.0.0.1:2369",
|
||||
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
|
||||
"content-length": "353",
|
||||
"content-length": "354",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
|
||||
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
|
||||
@ -159,7 +159,7 @@ Object {
|
||||
"reason": "Because cats are cute",
|
||||
"title": "Cat Pictures",
|
||||
"updated_at": null,
|
||||
"url": "https://catpictures.com",
|
||||
"url": "https://catpictures.com/",
|
||||
},
|
||||
],
|
||||
}
|
||||
@ -169,7 +169,7 @@ exports[`Recommendations Admin API Can edit recommendation 2: [headers] 1`] = `
|
||||
Object {
|
||||
"access-control-allow-origin": "http://127.0.0.1:2369",
|
||||
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
|
||||
"content-length": "354",
|
||||
"content-length": "355",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/,
|
||||
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
|
||||
|
||||
@ -46,7 +46,7 @@ describe('Recommendations Admin API', function () {
|
||||
|
||||
// Check everything is set correctly
|
||||
assert.equal(body.recommendations[0].title, 'Dog Pictures');
|
||||
assert.equal(body.recommendations[0].url, 'https://dogpictures.com');
|
||||
assert.equal(body.recommendations[0].url, 'https://dogpictures.com/');
|
||||
assert.equal(body.recommendations[0].reason, null);
|
||||
assert.equal(body.recommendations[0].excerpt, null);
|
||||
assert.equal(body.recommendations[0].featured_image, null);
|
||||
@ -84,7 +84,7 @@ describe('Recommendations Admin API', function () {
|
||||
|
||||
// Check everything is set correctly
|
||||
assert.equal(body.recommendations[0].title, 'Dog Pictures');
|
||||
assert.equal(body.recommendations[0].url, 'https://dogpictures.com');
|
||||
assert.equal(body.recommendations[0].url, 'https://dogpictures.com/');
|
||||
assert.equal(body.recommendations[0].reason, 'Because dogs are cute');
|
||||
assert.equal(body.recommendations[0].excerpt, 'Dogs are cute');
|
||||
assert.equal(body.recommendations[0].featured_image, 'https://dogpictures.com/dog.jpg');
|
||||
@ -123,7 +123,7 @@ describe('Recommendations Admin API', function () {
|
||||
// Check everything is set correctly
|
||||
assert.equal(body.recommendations[0].id, id);
|
||||
assert.equal(body.recommendations[0].title, 'Cat Pictures');
|
||||
assert.equal(body.recommendations[0].url, 'https://catpictures.com');
|
||||
assert.equal(body.recommendations[0].url, 'https://catpictures.com/');
|
||||
assert.equal(body.recommendations[0].reason, 'Because cats are cute');
|
||||
assert.equal(body.recommendations[0].excerpt, 'Cats are cute');
|
||||
assert.equal(body.recommendations[0].featured_image, 'https://catpictures.com/cat.jpg');
|
||||
|
||||
@ -132,7 +132,7 @@ class OEmbedService {
|
||||
/**
|
||||
* @param {string} url
|
||||
*
|
||||
* @returns {Promise<{url: string, body: string}>}
|
||||
* @returns {Promise<{url: string, body: string, contentType: string|undefined}>}
|
||||
*/
|
||||
async fetchPageHtml(url) {
|
||||
// Fetch url and get response as binary buffer to
|
||||
@ -154,7 +154,8 @@ class OEmbedService {
|
||||
if (encoding === null) {
|
||||
return {
|
||||
body: body.toString(),
|
||||
url: responseUrl
|
||||
url: responseUrl,
|
||||
contentType: headers['content-type']
|
||||
};
|
||||
}
|
||||
|
||||
@ -163,14 +164,16 @@ class OEmbedService {
|
||||
|
||||
return {
|
||||
body: decodedBody,
|
||||
url: responseUrl
|
||||
url: responseUrl,
|
||||
contentType: headers['content-type']
|
||||
};
|
||||
} catch (err) {
|
||||
logging.error(err);
|
||||
//return non decoded body anyway
|
||||
return {
|
||||
body: body.toString(),
|
||||
url: responseUrl
|
||||
url: responseUrl,
|
||||
contentType: headers['content-type']
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -355,7 +358,7 @@ class OEmbedService {
|
||||
}
|
||||
|
||||
// Not in the list, we need to fetch the content
|
||||
const {url: pageUrl, body} = await this.fetchPageHtml(url);
|
||||
const {url: pageUrl, body, contentType} = await this.fetchPageHtml(url);
|
||||
|
||||
// fetch only bookmark when explicitly requested
|
||||
if (type === 'bookmark') {
|
||||
@ -364,8 +367,26 @@ class OEmbedService {
|
||||
|
||||
// mentions need to return bookmark data (metadata) and body (html) for link verification
|
||||
if (type === 'mention') {
|
||||
if (contentType.includes('application/json')) {
|
||||
// No need to fetch metadata: we have none
|
||||
const bookmark = {
|
||||
version: '1.0',
|
||||
type: 'bookmark',
|
||||
url,
|
||||
metadata: {
|
||||
title: null,
|
||||
description: null,
|
||||
publisher: null,
|
||||
author: null,
|
||||
thumbnail: null,
|
||||
icon: null
|
||||
},
|
||||
contentType
|
||||
};
|
||||
return {...bookmark, body};
|
||||
}
|
||||
const bookmark = await this.fetchBookmarkData(url, body);
|
||||
return {...bookmark, body};
|
||||
return {...bookmark, body, contentType};
|
||||
}
|
||||
|
||||
// attempt to fetch oembed
|
||||
|
||||
@ -22,12 +22,13 @@
|
||||
"build"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@tryghost/errors": "1.2.24",
|
||||
"@types/node": "^20.5.7",
|
||||
"c8": "8.0.1",
|
||||
"mocha": "10.2.0",
|
||||
"sinon": "15.2.0",
|
||||
"ts-node": "10.9.1",
|
||||
"typescript": "5.2.2",
|
||||
"@tryghost/errors": "1.2.24"
|
||||
"typescript": "5.2.2"
|
||||
},
|
||||
"dependencies": {}
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
|
||||
excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.",
|
||||
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
|
||||
favicon: "https://www.shesabeast.co/content/images/size/w256h256/2022/08/transparent-icon-black-copy-gray-bar.png",
|
||||
url: "https://www.thepragmaticprogrammer.com/",
|
||||
url: new URL("https://www.thepragmaticprogrammer.com/"),
|
||||
oneClickSubscribe: false
|
||||
}),
|
||||
new Recommendation({
|
||||
@ -18,7 +18,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
|
||||
excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.",
|
||||
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
|
||||
favicon: "https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7cde267-8f9e-47fa-9aef-5be03bad95ed%2Fapple-touch-icon-1024x1024.png",
|
||||
url: "https://www.thepragmaticprogrammer.com/",
|
||||
url: new URL("https://www.thepragmaticprogrammer.com/"),
|
||||
oneClickSubscribe: false
|
||||
}),
|
||||
new Recommendation({
|
||||
@ -27,7 +27,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
|
||||
excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.",
|
||||
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
|
||||
favicon: "https://clickhole.com/wp-content/uploads/2020/05/cropped-clickhole-icon-180x180.png",
|
||||
url: "https://www.thepragmaticprogrammer.com/",
|
||||
url: new URL("https://www.thepragmaticprogrammer.com/"),
|
||||
oneClickSubscribe: false
|
||||
}),
|
||||
new Recommendation({
|
||||
@ -36,7 +36,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
|
||||
excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.",
|
||||
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
|
||||
favicon: "https://www.theverge.com/icons/apple_touch_icon.png",
|
||||
url: "https://www.thepragmaticprogrammer.com/",
|
||||
url: new URL("https://www.thepragmaticprogrammer.com/"),
|
||||
oneClickSubscribe: false
|
||||
}),
|
||||
new Recommendation({
|
||||
@ -45,7 +45,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor
|
||||
excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.",
|
||||
featuredImage: "https://www.thepragmaticprogrammer.com/image.png",
|
||||
favicon: "https://substackcdn.com/image/fetch/w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff3f2b2ad-681f-45e1-9496-db80f45e853d_403x403.png",
|
||||
url: "https://www.thepragmaticprogrammer.com/",
|
||||
url: new URL("https://www.thepragmaticprogrammer.com/"),
|
||||
oneClickSubscribe: true
|
||||
})
|
||||
];
|
||||
|
||||
@ -7,12 +7,12 @@ export class Recommendation {
|
||||
excerpt: string|null // Fetched from the site meta data
|
||||
featuredImage: string|null // Fetched from the site meta data
|
||||
favicon: string|null // Fetched from the site meta data
|
||||
url: string
|
||||
url: URL
|
||||
oneClickSubscribe: boolean
|
||||
createdAt: Date
|
||||
updatedAt: Date|null
|
||||
|
||||
constructor(data: {id?: string, title: string, reason: string|null, excerpt: string|null, featuredImage: string|null, favicon: string|null, url: string, oneClickSubscribe: boolean, createdAt?: Date, updatedAt?: Date|null}) {
|
||||
constructor(data: {id?: string, title: string, reason: string|null, excerpt: string|null, featuredImage: string|null, favicon: string|null, url: URL, oneClickSubscribe: boolean, createdAt?: Date, updatedAt?: Date|null}) {
|
||||
this.id = data.id ?? ObjectId().toString();
|
||||
this.title = data.title;
|
||||
this.reason = data.reason;
|
||||
|
||||
@ -38,6 +38,18 @@ function validateBoolean(object: any, key: string, {required = true} = {}): bool
|
||||
}
|
||||
}
|
||||
|
||||
function validateURL(object: any, key: string, {required = true} = {}): URL|undefined {
|
||||
const string = validateString(object, key, {required});
|
||||
if (string !== undefined) {
|
||||
try {
|
||||
return new URL(string);
|
||||
} catch (e) {
|
||||
throw new errors.BadRequestError({message: `${key} must be a valid URL`});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export class RecommendationController {
|
||||
service: RecommendationService;
|
||||
|
||||
@ -67,7 +79,7 @@ export class RecommendationController {
|
||||
|
||||
const cleanedRecommendation: Omit<Recommendation, 'id'|'createdAt'|'updatedAt'> = {
|
||||
title: validateString(recommendation, "title") ?? '',
|
||||
url: validateString(recommendation, "url") ?? '',
|
||||
url: validateURL(recommendation, "url")!,
|
||||
|
||||
// Optional fields
|
||||
oneClickSubscribe: validateBoolean(recommendation, "one_click_subscribe", {required: false}) ?? false,
|
||||
@ -89,7 +101,7 @@ export class RecommendationController {
|
||||
const recommendation = frame.data.recommendations[0];
|
||||
const cleanedRecommendation: Partial<Recommendation> = {
|
||||
title: validateString(recommendation, "title", {required: false}),
|
||||
url: validateString(recommendation, "url", {required: false}),
|
||||
url: validateURL(recommendation, "url", {required: false}),
|
||||
oneClickSubscribe: validateBoolean(recommendation, "one_click_subscribe", {required: false}),
|
||||
reason: validateString(recommendation, "reason", {required: false}),
|
||||
excerpt: validateString(recommendation, "excerpt", {required: false}),
|
||||
@ -112,7 +124,7 @@ export class RecommendationController {
|
||||
excerpt: r.excerpt,
|
||||
featured_image: r.featuredImage,
|
||||
favicon: r.favicon,
|
||||
url: r.url,
|
||||
url: r.url.toString(),
|
||||
one_click_subscribe: r.oneClickSubscribe,
|
||||
created_at: r.createdAt,
|
||||
updated_at: r.updatedAt
|
||||
|
||||
@ -1,26 +1,66 @@
|
||||
import {Recommendation} from "./Recommendation";
|
||||
import {RecommendationRepository} from "./RecommendationRepository";
|
||||
import {WellknownService} from "./WellknownService";
|
||||
|
||||
type MentionSendingService = {
|
||||
sendAll(options: {url: URL, links: URL[]}): Promise<void>
|
||||
}
|
||||
|
||||
export class RecommendationService {
|
||||
repository: RecommendationRepository;
|
||||
wellknownService: WellknownService;
|
||||
mentionSendingService: MentionSendingService;
|
||||
|
||||
constructor(deps: {repository: RecommendationRepository}) {
|
||||
constructor(deps: {repository: RecommendationRepository, wellknownService: WellknownService, mentionSendingService: MentionSendingService}) {
|
||||
this.repository = deps.repository;
|
||||
this.wellknownService = deps.wellknownService;
|
||||
this.mentionSendingService = deps.mentionSendingService;
|
||||
}
|
||||
|
||||
async init() {
|
||||
await this.updateWellknown();
|
||||
}
|
||||
|
||||
async updateWellknown() {
|
||||
const recommendations = await this.listRecommendations();
|
||||
await this.wellknownService.set(recommendations);
|
||||
}
|
||||
|
||||
sendMentionToRecommendation(recommendation: Recommendation) {
|
||||
this.mentionSendingService.sendAll({
|
||||
url: this.wellknownService.getURL(),
|
||||
links: [
|
||||
recommendation.url
|
||||
]
|
||||
}).catch(console.error);
|
||||
}
|
||||
|
||||
async addRecommendation(recommendation: Recommendation) {
|
||||
return this.repository.add(recommendation);
|
||||
const r = this.repository.add(recommendation);
|
||||
await this.updateWellknown();
|
||||
|
||||
// Only send an update for the mentioned URL
|
||||
this.sendMentionToRecommendation(recommendation);
|
||||
return r;
|
||||
}
|
||||
|
||||
async editRecommendation(id: string, recommendationEdit: Partial<Recommendation>) {
|
||||
// Check if it exists
|
||||
const existing = await this.repository.getById(id);
|
||||
return this.repository.edit(existing.id, recommendationEdit);
|
||||
const e = await this.repository.edit(existing.id, recommendationEdit);
|
||||
|
||||
await this.updateWellknown();
|
||||
this.sendMentionToRecommendation(e);
|
||||
return e;
|
||||
}
|
||||
|
||||
async deleteRecommendation(id: string) {
|
||||
const existing = await this.repository.getById(id);
|
||||
await this.repository.remove(existing.id);
|
||||
await this.updateWellknown();
|
||||
|
||||
// Send a mention (because it was deleted, according to the webmentions spec)
|
||||
this.sendMentionToRecommendation(existing);
|
||||
}
|
||||
|
||||
async listRecommendations() {
|
||||
|
||||
48
ghost/recommendations/src/WellknownService.ts
Normal file
48
ghost/recommendations/src/WellknownService.ts
Normal file
@ -0,0 +1,48 @@
|
||||
import {Recommendation} from "./Recommendation"
|
||||
import fs from "fs/promises";
|
||||
import _path from 'path';
|
||||
|
||||
type UrlUtils = {
|
||||
relativeToAbsolute(url: string): string
|
||||
}
|
||||
type Options = {
|
||||
/**
|
||||
* Where to publish the wellknown file
|
||||
*/
|
||||
dir: string,
|
||||
urlUtils: UrlUtils
|
||||
}
|
||||
|
||||
export class WellknownService {
|
||||
dir: string
|
||||
urlUtils: UrlUtils
|
||||
|
||||
constructor({dir, urlUtils}: Options) {
|
||||
this.dir = dir;
|
||||
this.urlUtils = urlUtils;
|
||||
}
|
||||
|
||||
#formatRecommendation(recommendation: Recommendation) {
|
||||
return {
|
||||
url: recommendation.url,
|
||||
reason: recommendation.reason,
|
||||
updated_at: (recommendation.updatedAt ?? recommendation.createdAt).toISOString(),
|
||||
created_at: (recommendation.createdAt).toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
getPath() {
|
||||
return _path.join(this.dir, '/.well-known/recommendations.json');
|
||||
}
|
||||
|
||||
getURL(): URL {
|
||||
return new URL(this.urlUtils.relativeToAbsolute('/.well-known/recommendations.json'));
|
||||
}
|
||||
|
||||
async set(recommendations: Recommendation[]) {
|
||||
const content = JSON.stringify(recommendations.map(r => this.#formatRecommendation(r)));
|
||||
const path = this.getPath();
|
||||
await fs.mkdir(_path.dirname(path), {recursive: true});
|
||||
await fs.writeFile(path, content);
|
||||
}
|
||||
}
|
||||
@ -3,3 +3,4 @@ export * from './RecommendationService';
|
||||
export * from './RecommendationRepository';
|
||||
export * from './InMemoryRecommendationRepository';
|
||||
export * from './Recommendation';
|
||||
export * from './WellknownService';
|
||||
|
||||
@ -21,11 +21,44 @@ module.exports = class Mention {
|
||||
|
||||
/**
|
||||
* @param {string} html
|
||||
* @param {string} contentType
|
||||
*/
|
||||
verify(html) {
|
||||
const $ = cheerio.load(html);
|
||||
const hasTargetUrl = $('a[href*="' + this.target.href + '"], img[src*="' + this.target.href + '"], video[src*="' + this.target.href + '"]').length > 0;
|
||||
this.#verified = hasTargetUrl;
|
||||
verify(html, contentType) {
|
||||
const wasVerified = this.#verified;
|
||||
|
||||
if (contentType.includes('text/html')) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
const hasTargetUrl = $('a[href*="' + this.target.href + '"], img[src*="' + this.target.href + '"], video[src*="' + this.target.href + '"]').length > 0;
|
||||
this.#verified = hasTargetUrl;
|
||||
|
||||
if (wasVerified && !this.#verified) {
|
||||
// Delete the mention
|
||||
this.#deleted = true;
|
||||
this.#verified = true;
|
||||
}
|
||||
} catch (e) {
|
||||
this.#verified = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (contentType.includes('application/json')) {
|
||||
try {
|
||||
// Check valid JSON
|
||||
JSON.parse(html);
|
||||
|
||||
// Check full text string is present in the json
|
||||
this.#verified = !!html.includes(JSON.stringify(this.target.href));
|
||||
|
||||
if (wasVerified && !this.#verified) {
|
||||
// Delete the mention
|
||||
this.#deleted = true;
|
||||
this.#verified = true;
|
||||
}
|
||||
} catch (e) {
|
||||
this.#verified = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** @type {URL} */
|
||||
@ -274,12 +307,19 @@ module.exports = class Mention {
|
||||
return mention;
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDeleted() {
|
||||
return this.#deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Mention} mention
|
||||
* @returns {boolean}
|
||||
*/
|
||||
static isDeleted(mention) {
|
||||
return mention.#deleted;
|
||||
return mention.isDeleted();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -67,7 +67,7 @@ module.exports = class MentionSendingService {
|
||||
let previousHtml = post.previous('status') === 'published' ? post.previous('html') : null;
|
||||
if (html || previousHtml) {
|
||||
await this.#jobService.addJob('sendWebmentions', async () => {
|
||||
await this.sendAll({
|
||||
await this.sendForHTMLResource({
|
||||
url: new URL(this.#getPostUrl(post)),
|
||||
html: html,
|
||||
previousHtml: previousHtml
|
||||
@ -80,6 +80,10 @@ module.exports = class MentionSendingService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{source: URL, target: URL, endpoint: URL}} options
|
||||
* @returns
|
||||
*/
|
||||
async send({source, target, endpoint}) {
|
||||
logging.info('[Webmention] Sending webmention from ' + source.href + ' to ' + target.href + ' via ' + endpoint.href);
|
||||
|
||||
@ -113,7 +117,7 @@ module.exports = class MentionSendingService {
|
||||
* @param {string} resource.html
|
||||
* @param {string|null} [resource.previousHtml]
|
||||
*/
|
||||
async sendAll(resource) {
|
||||
async sendForHTMLResource(resource) {
|
||||
const links = resource.html ? this.getLinks(resource.html) : [];
|
||||
if (resource.previousHtml) {
|
||||
// We also need to send webmentions for removed links
|
||||
@ -129,12 +133,25 @@ module.exports = class MentionSendingService {
|
||||
logging.info('[Webmention] Sending all webmentions for ' + resource.url.href);
|
||||
}
|
||||
|
||||
await this.sendAll({
|
||||
url: resource.url,
|
||||
links
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a webmention call for the links in a resource.
|
||||
* @param {object} resource
|
||||
* @param {URL} resource.url
|
||||
* @param {URL[]} resource.links
|
||||
*/
|
||||
async sendAll({url, links}) {
|
||||
for (const target of links) {
|
||||
const endpoint = await this.#discoveryService.getEndpoint(target);
|
||||
if (endpoint) {
|
||||
// Send webmention call
|
||||
try {
|
||||
await this.send({source: resource.url, target, endpoint});
|
||||
await this.send({source: url, target, endpoint});
|
||||
} catch (e) {
|
||||
logging.error('[Webmention] Failed sending via ' + endpoint.href + ': ' + e.message);
|
||||
}
|
||||
|
||||
@ -68,13 +68,14 @@ const Mention = require('./Mention');
|
||||
|
||||
/**
|
||||
* @typedef {object} WebmentionMetadata
|
||||
* @prop {string} siteTitle
|
||||
* @prop {string} title
|
||||
* @prop {string} excerpt
|
||||
* @prop {string} author
|
||||
* @prop {URL} image
|
||||
* @prop {URL} favicon
|
||||
* @prop {string|null} siteTitle
|
||||
* @prop {string|null} title
|
||||
* @prop {string|null} excerpt
|
||||
* @prop {string|null} author
|
||||
* @prop {URL|null} image
|
||||
* @prop {URL|null} favicon
|
||||
* @prop {string} body
|
||||
* @prop {string|undefined} contentType
|
||||
*/
|
||||
|
||||
/**
|
||||
@ -228,7 +229,7 @@ module.exports = class MentionsAPI {
|
||||
|
||||
if (metadata?.body) {
|
||||
try {
|
||||
mention.verify(metadata.body);
|
||||
mention.verify(metadata.body, metadata.contentType);
|
||||
} catch (e) {
|
||||
logging.error(e);
|
||||
}
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
const assert = require('assert/strict');
|
||||
const ObjectID = require('bson-objectid');
|
||||
const Mention = require('../lib/Mention');
|
||||
const cheerio = require('cheerio');
|
||||
const sinon = require('sinon');
|
||||
|
||||
const validInput = {
|
||||
source: 'https://source.com',
|
||||
@ -35,15 +37,40 @@ describe('Mention', function () {
|
||||
});
|
||||
|
||||
describe('verify', function () {
|
||||
afterEach(function () {
|
||||
sinon.restore();
|
||||
});
|
||||
|
||||
it('can handle invalid HTML', async function () {
|
||||
const mention = await Mention.create(validInput);
|
||||
assert(!mention.verified);
|
||||
|
||||
sinon.stub(cheerio, 'load').throws(new Error('Invalid HTML'));
|
||||
|
||||
mention.verify('irrelevant', 'text/html');
|
||||
assert(!mention.verified);
|
||||
assert(!mention.isDeleted());
|
||||
});
|
||||
|
||||
it('Does basic check for the target URL and updates verified property', async function () {
|
||||
const mention = await Mention.create(validInput);
|
||||
assert(!mention.verified);
|
||||
|
||||
mention.verify('<a href="https://target.com/">');
|
||||
mention.verify('<a href="https://target.com/">', 'text/html');
|
||||
assert(mention.verified);
|
||||
assert(!mention.isDeleted());
|
||||
|
||||
mention.verify('<a href="https://not-da-target.com">');
|
||||
mention.verify('something else', 'text/html');
|
||||
assert(mention.verified);
|
||||
assert(mention.isDeleted());
|
||||
});
|
||||
it('detects differences', async function () {
|
||||
const mention = await Mention.create(validInput);
|
||||
assert(!mention.verified);
|
||||
|
||||
mention.verify('<a href="https://not-target.com/">', 'text/html');
|
||||
assert(!mention.verified);
|
||||
assert(!mention.isDeleted());
|
||||
});
|
||||
it('Does check for Image targets', async function () {
|
||||
const mention = await Mention.create({
|
||||
@ -52,11 +79,13 @@ describe('Mention', function () {
|
||||
});
|
||||
assert(!mention.verified);
|
||||
|
||||
mention.verify('<img src="https://target.com/image.jpg">');
|
||||
mention.verify('<img src="https://target.com/image.jpg">', 'text/html');
|
||||
assert(mention.verified);
|
||||
assert(!mention.isDeleted());
|
||||
|
||||
mention.verify('<img src="https://not-da-target.com/image.jpg">');
|
||||
assert(!mention.verified);
|
||||
mention.verify('something else', 'text/html');
|
||||
assert(mention.verified);
|
||||
assert(mention.isDeleted());
|
||||
});
|
||||
it('Does check for Video targets', async function () {
|
||||
const mention = await Mention.create({
|
||||
@ -65,11 +94,35 @@ describe('Mention', function () {
|
||||
});
|
||||
assert(!mention.verified);
|
||||
|
||||
mention.verify('<video src="https://target.com/video.mp4">');
|
||||
mention.verify('<video src="https://target.com/video.mp4">', 'text/html');
|
||||
assert(mention.verified);
|
||||
assert(!mention.isDeleted());
|
||||
|
||||
mention.verify('<video src="https://not-da-target.com/video.mp4">');
|
||||
mention.verify('something else', 'text/html');
|
||||
assert(mention.verified);
|
||||
assert(mention.isDeleted());
|
||||
});
|
||||
|
||||
it('can verify links in JSON', async function () {
|
||||
const mention = await Mention.create(validInput);
|
||||
assert(!mention.verified);
|
||||
|
||||
mention.verify('{"url": "https://target.com/"}', 'application/json');
|
||||
assert(mention.verified);
|
||||
assert(!mention.isDeleted());
|
||||
|
||||
mention.verify('{}', 'application/json');
|
||||
assert(mention.verified);
|
||||
assert(mention.isDeleted());
|
||||
});
|
||||
|
||||
it('can handle invalid JSON', async function () {
|
||||
const mention = await Mention.create(validInput);
|
||||
assert(!mention.verified);
|
||||
|
||||
mention.verify('{"url": "ht', 'application/json');
|
||||
assert(!mention.verified);
|
||||
assert(!mention.isDeleted());
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@ -55,7 +55,7 @@ describe('MentionSendingService', function () {
|
||||
const service = new MentionSendingService({
|
||||
isEnabled: () => false
|
||||
});
|
||||
const stub = sinon.stub(service, 'sendAll');
|
||||
const stub = sinon.stub(service, 'sendForHTMLResource');
|
||||
await service.sendForPost({});
|
||||
sinon.assert.notCalled(stub);
|
||||
});
|
||||
@ -64,7 +64,7 @@ describe('MentionSendingService', function () {
|
||||
const service = new MentionSendingService({
|
||||
isEnabled: () => true
|
||||
});
|
||||
const stub = sinon.stub(service, 'sendAll');
|
||||
const stub = sinon.stub(service, 'sendForHTMLResource');
|
||||
let options = {importing: true};
|
||||
await service.sendForPost({}, options);
|
||||
sinon.assert.notCalled(stub);
|
||||
@ -74,7 +74,7 @@ describe('MentionSendingService', function () {
|
||||
const service = new MentionSendingService({
|
||||
isEnabled: () => true
|
||||
});
|
||||
const stub = sinon.stub(service, 'sendAll');
|
||||
const stub = sinon.stub(service, 'sendForHTMLResource');
|
||||
let options = {context: {internal: true}};
|
||||
await service.sendForPost({}, options);
|
||||
sinon.assert.notCalled(stub);
|
||||
@ -84,7 +84,7 @@ describe('MentionSendingService', function () {
|
||||
const service = new MentionSendingService({
|
||||
isEnabled: () => true
|
||||
});
|
||||
const stub = sinon.stub(service, 'sendAll');
|
||||
const stub = sinon.stub(service, 'sendForHTMLResource');
|
||||
await service.sendForPost(createModel({
|
||||
status: 'draft',
|
||||
html: 'changed',
|
||||
@ -100,7 +100,7 @@ describe('MentionSendingService', function () {
|
||||
const service = new MentionSendingService({
|
||||
isEnabled: () => true
|
||||
});
|
||||
const stub = sinon.stub(service, 'sendAll');
|
||||
const stub = sinon.stub(service, 'sendForHTMLResource');
|
||||
await service.sendForPost(createModel({
|
||||
status: 'published',
|
||||
html: 'same',
|
||||
@ -116,7 +116,7 @@ describe('MentionSendingService', function () {
|
||||
const service = new MentionSendingService({
|
||||
isEnabled: () => true
|
||||
});
|
||||
const stub = sinon.stub(service, 'sendAll');
|
||||
const stub = sinon.stub(service, 'sendForHTMLResource');
|
||||
await service.sendForPost(createModel({
|
||||
status: 'send',
|
||||
html: 'changed',
|
||||
@ -134,7 +134,7 @@ describe('MentionSendingService', function () {
|
||||
getPostUrl: () => 'https://site.com/post/',
|
||||
jobService: jobService
|
||||
});
|
||||
const stub = sinon.stub(service, 'sendAll');
|
||||
const stub = sinon.stub(service, 'sendForHTMLResource');
|
||||
await service.sendForPost(createModel({
|
||||
status: 'published',
|
||||
html: 'same',
|
||||
@ -156,7 +156,7 @@ describe('MentionSendingService', function () {
|
||||
getPostUrl: () => 'https://site.com/post/',
|
||||
jobService: jobService
|
||||
});
|
||||
const stub = sinon.stub(service, 'sendAll');
|
||||
const stub = sinon.stub(service, 'sendForHTMLResource');
|
||||
await service.sendForPost(createModel({
|
||||
status: 'published',
|
||||
html: 'updated',
|
||||
@ -177,7 +177,7 @@ describe('MentionSendingService', function () {
|
||||
isEnabled: () => true,
|
||||
getPostUrl: () => 'https://site.com/post/'
|
||||
});
|
||||
sinon.stub(service, 'sendAll').rejects(new Error('Internal error test'));
|
||||
sinon.stub(service, 'sendForHTMLResource').rejects(new Error('Internal error test'));
|
||||
await service.sendForPost(createModel({
|
||||
status: 'published',
|
||||
html: 'same',
|
||||
@ -195,7 +195,7 @@ describe('MentionSendingService', function () {
|
||||
getPostUrl: () => 'https://site.com/post/',
|
||||
jobService: jobService
|
||||
});
|
||||
const stub = sinon.stub(service, 'sendAll');
|
||||
const stub = sinon.stub(service, 'sendForHTMLResource');
|
||||
await service.sendForPost(createModel({
|
||||
status: 'published',
|
||||
html: '',
|
||||
@ -208,7 +208,7 @@ describe('MentionSendingService', function () {
|
||||
});
|
||||
});
|
||||
|
||||
describe('sendAll', function () {
|
||||
describe('sendForHTMLResource', function () {
|
||||
it('Sends to all links', async function () {
|
||||
this.retries(1);
|
||||
let counter = 0;
|
||||
@ -227,7 +227,7 @@ describe('MentionSendingService', function () {
|
||||
getEndpoint: async () => new URL('https://example.org/webmentions-test')
|
||||
}
|
||||
});
|
||||
await service.sendAll({url: new URL('https://site.com'),
|
||||
await service.sendForHTMLResource({url: new URL('https://site.com'),
|
||||
html: `
|
||||
<html>
|
||||
<body>
|
||||
@ -263,7 +263,7 @@ describe('MentionSendingService', function () {
|
||||
getEndpoint: async () => new URL('https://example.org/webmentions-test')
|
||||
}
|
||||
});
|
||||
await service.sendAll({url: new URL('https://site.com'),
|
||||
await service.sendForHTMLResource({url: new URL('https://site.com'),
|
||||
html: `
|
||||
<html>
|
||||
<body>
|
||||
@ -298,7 +298,7 @@ describe('MentionSendingService', function () {
|
||||
},
|
||||
jobService: jobService
|
||||
});
|
||||
await service.sendAll({url: new URL('https://site.com'),
|
||||
await service.sendForHTMLResource({url: new URL('https://site.com'),
|
||||
html: `<a href="https://example.com">Example</a>`,
|
||||
previousHtml: `<a href="https://typo.com">Example</a>`});
|
||||
assert.equal(scope.isDone(), true);
|
||||
@ -311,7 +311,7 @@ describe('MentionSendingService', function () {
|
||||
isEnabled: () => true
|
||||
});
|
||||
const linksStub = sinon.stub(service, 'getLinks');
|
||||
await service.sendAll({html: ``,previousHtml: ``});
|
||||
await service.sendForHTMLResource({html: ``,previousHtml: ``});
|
||||
sinon.assert.notCalled(linksStub);
|
||||
});
|
||||
});
|
||||
|
||||
@ -8588,7 +8588,7 @@
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.1.0":
|
||||
"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.1.0", "@types/node@^20.5.7":
|
||||
version "20.5.7"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.5.7.tgz#4b8ecac87fbefbc92f431d09c30e176fc0a7c377"
|
||||
integrity sha512-dP7f3LdZIysZnmvP3ANJYTSwg+wLLl8p7RqniVlV7j+oXSXAbt9h0WIBFmJy5inWZoX9wZN6eXx+YXd9Rh3RBA==
|
||||
|
||||
Loading…
Reference in New Issue
Block a user