From 96fefaea69b393c48e62a8de35ee8eef1150ad9e Mon Sep 17 00:00:00 2001 From: Simon Backx Date: Thu, 31 Aug 2023 16:57:18 +0200 Subject: [PATCH] Added well known recommendations service (#17895) fixes https://github.com/TryGhost/Product/issues/3797 fixes https://github.com/TryGhost/Product/issues/3776 fixes https://github.com/TryGhost/Product/issues/3798 - Added support for storing json webmentions - Improved handling deleted webmentions (set deleted to true instead of verified to false) --- .../web/middleware/serve-public-file.js | 8 +-- ghost/core/core/frontend/web/site.js | 3 + .../services/mentions/WebmentionMetadata.js | 3 +- .../core/server/services/mentions/service.js | 4 ++ .../RecommendationServiceWrapper.js | 23 ++++++- .../recommendations.test.js.snap | 16 ++--- .../e2e-api/admin/recommendations.test.js | 6 +- ghost/oembed-service/lib/OEmbedService.js | 33 +++++++-- ghost/recommendations/package.json | 5 +- .../src/InMemoryRecommendationRepository.ts | 10 +-- ghost/recommendations/src/Recommendation.ts | 4 +- .../src/RecommendationController.ts | 18 ++++- .../src/RecommendationService.ts | 46 ++++++++++++- ghost/recommendations/src/WellknownService.ts | 48 +++++++++++++ ghost/recommendations/src/index.ts | 1 + ghost/webmentions/lib/Mention.js | 50 ++++++++++++-- .../webmentions/lib/MentionSendingService.js | 23 ++++++- ghost/webmentions/lib/MentionsAPI.js | 15 +++-- ghost/webmentions/test/Mention.test.js | 67 +++++++++++++++++-- .../test/MentionSendingService.test.js | 30 ++++----- yarn.lock | 2 +- 21 files changed, 338 insertions(+), 77 deletions(-) create mode 100644 ghost/recommendations/src/WellknownService.ts diff --git a/ghost/core/core/frontend/web/middleware/serve-public-file.js b/ghost/core/core/frontend/web/middleware/serve-public-file.js index 3688192674..75faeb6c19 100644 --- a/ghost/core/core/frontend/web/middleware/serve-public-file.js +++ b/ghost/core/core/frontend/web/middleware/serve-public-file.js @@ -26,7 +26,7 @@ function matchCacheKey(req, cache) { return true; } -function createPublicFileMiddleware(location, file, mime, maxAge) { +function createPublicFileMiddleware(location, file, mime, maxAge, options = {}) { let cache; // These files are provided by Ghost, and therefore live inside of the core folder const staticFilePath = config.get('paths').publicFilePath; @@ -45,7 +45,7 @@ function createPublicFileMiddleware(location, file, mime, maxAge) { } // send image files directly and let express handle content-length, etag, etc - if (mime.match(/^image/)) { + if (mime.match(/^image/) || options.disableServerCache) { return res.sendFile(filePath, (err) => { if (err && err.status === 404) { // ensure we're triggering basic asset 404 and not a templated 404 @@ -101,8 +101,8 @@ function createPublicFileMiddleware(location, file, mime, maxAge) { // ### servePublicFile Middleware // Handles requests to robots.txt and favicon.ico (and caches them) -function servePublicFile(location, file, type, maxAge) { - const publicFileMiddleware = createPublicFileMiddleware(location, file, type, maxAge); +function servePublicFile(location, file, type, maxAge, options = {}) { + const publicFileMiddleware = createPublicFileMiddleware(location, file, type, maxAge, options); return function servePublicFileMiddleware(req, res, next) { if (req.path === '/' + file) { diff --git a/ghost/core/core/frontend/web/site.js b/ghost/core/core/frontend/web/site.js index 3a95f39467..347a537027 100644 --- a/ghost/core/core/frontend/web/site.js +++ b/ghost/core/core/frontend/web/site.js @@ -98,6 +98,9 @@ module.exports = function setupSiteApp(routerConfig) { (req, res, next) => membersService.api.middleware.wellKnown(req, res, next) ); + // Recommendations well-known + siteApp.use(mw.servePublicFile('built', '.well-known/recommendations.json', 'application/json', config.get('caching:publicAssets:maxAge'), {disableServerCache: true})); + // setup middleware for internal apps // @TODO: refactor this to be a proper app middleware hook for internal apps config.get('apps:internal').forEach((appName) => { diff --git a/ghost/core/core/server/services/mentions/WebmentionMetadata.js b/ghost/core/core/server/services/mentions/WebmentionMetadata.js index 62417e0221..9f6388d9a6 100644 --- a/ghost/core/core/server/services/mentions/WebmentionMetadata.js +++ b/ghost/core/core/server/services/mentions/WebmentionMetadata.js @@ -14,7 +14,8 @@ module.exports = class WebmentionMetadata { author: data.metadata.author, image: data.metadata.thumbnail ? new URL(data.metadata.thumbnail) : null, favicon: data.metadata.icon ? new URL(data.metadata.icon) : null, - body: data.body + body: data.body, + contentType: data.contentType }; return result; } diff --git a/ghost/core/core/server/services/mentions/service.js b/ghost/core/core/server/services/mentions/service.js index 29ef608b70..1690948f1e 100644 --- a/ghost/core/core/server/services/mentions/service.js +++ b/ghost/core/core/server/services/mentions/service.js @@ -28,6 +28,8 @@ module.exports = { /** @type {import('@tryghost/webmentions/lib/MentionsAPI')} */ api: null, controller: new MentionController(), + /** @type {import('@tryghost/webmentions/lib/MentionSendingService')} */ + sendingService: null, didInit: false, async init() { if (this.didInit) { @@ -107,5 +109,7 @@ module.exports = { } }); sendingService.listen(events); + + this.sendingService = sendingService; } }; diff --git a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js index 8f895fa2be..cbf1c86d55 100644 --- a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js +++ b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js @@ -19,15 +19,34 @@ class RecommendationServiceWrapper { return; } - const {InMemoryRecommendationRepository, RecommendationService, RecommendationController} = require('@tryghost/recommendations'); + const config = require('../../../shared/config'); + const urlUtils = require('../../../shared/url-utils'); + const {InMemoryRecommendationRepository, RecommendationService, RecommendationController, WellknownService} = require('@tryghost/recommendations'); + + const mentions = require('../mentions'); + + if (!mentions.sendingService) { + // eslint-disable-next-line ghost/ghost-custom/no-native-error + throw new Error('MentionSendingService not intialized, but this is a dependency of RecommendationServiceWrapper. Check boot order.'); + } + + const wellknownService = new WellknownService({ + dir: config.getContentPath('public'), + urlUtils + }); this.repository = new InMemoryRecommendationRepository(); this.service = new RecommendationService({ - repository: this.repository + repository: this.repository, + wellknownService, + mentionSendingService: mentions.sendingService }); this.controller = new RecommendationController({ service: this.service }); + + // eslint-disable-next-line no-console + this.service.init().catch(console.error); } } diff --git a/ghost/core/test/e2e-api/admin/__snapshots__/recommendations.test.js.snap b/ghost/core/test/e2e-api/admin/__snapshots__/recommendations.test.js.snap index 86873de1a3..47376a62f4 100644 --- a/ghost/core/test/e2e-api/admin/__snapshots__/recommendations.test.js.snap +++ b/ghost/core/test/e2e-api/admin/__snapshots__/recommendations.test.js.snap @@ -13,7 +13,7 @@ Object { "reason": "Because dogs are cute", "title": "Dog Pictures", "updated_at": null, - "url": "https://dogpictures.com", + "url": "https://dogpictures.com/", }, ], } @@ -23,7 +23,7 @@ exports[`Recommendations Admin API Can add a full recommendation 2: [headers] 1` Object { "access-control-allow-origin": "http://127.0.0.1:2369", "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", - "content-length": "353", + "content-length": "354", "content-type": "application/json; charset=utf-8", "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/, "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/, @@ -47,7 +47,7 @@ Object { "reason": null, "title": "Dog Pictures", "updated_at": null, - "url": "https://dogpictures.com", + "url": "https://dogpictures.com/", }, ], } @@ -57,7 +57,7 @@ exports[`Recommendations Admin API Can add a minimal recommendation 2: [headers] Object { "access-control-allow-origin": "http://127.0.0.1:2369", "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", - "content-length": "262", + "content-length": "263", "content-type": "application/json; charset=utf-8", "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/, "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/, @@ -113,7 +113,7 @@ Object { "reason": "Because dogs are cute", "title": "Dog Pictures", "updated_at": null, - "url": "https://dogpictures.com", + "url": "https://dogpictures.com/", }, ], } @@ -123,7 +123,7 @@ exports[`Recommendations Admin API Can browse 2: [headers] 1`] = ` Object { "access-control-allow-origin": "http://127.0.0.1:2369", "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", - "content-length": "353", + "content-length": "354", "content-type": "application/json; charset=utf-8", "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/, "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/, @@ -159,7 +159,7 @@ Object { "reason": "Because cats are cute", "title": "Cat Pictures", "updated_at": null, - "url": "https://catpictures.com", + "url": "https://catpictures.com/", }, ], } @@ -169,7 +169,7 @@ exports[`Recommendations Admin API Can edit recommendation 2: [headers] 1`] = ` Object { "access-control-allow-origin": "http://127.0.0.1:2369", "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", - "content-length": "354", + "content-length": "355", "content-type": "application/json; charset=utf-8", "content-version": StringMatching /v\\\\d\\+\\\\\\.\\\\d\\+/, "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/, diff --git a/ghost/core/test/e2e-api/admin/recommendations.test.js b/ghost/core/test/e2e-api/admin/recommendations.test.js index 43f6c20862..09f7aa7c7f 100644 --- a/ghost/core/test/e2e-api/admin/recommendations.test.js +++ b/ghost/core/test/e2e-api/admin/recommendations.test.js @@ -46,7 +46,7 @@ describe('Recommendations Admin API', function () { // Check everything is set correctly assert.equal(body.recommendations[0].title, 'Dog Pictures'); - assert.equal(body.recommendations[0].url, 'https://dogpictures.com'); + assert.equal(body.recommendations[0].url, 'https://dogpictures.com/'); assert.equal(body.recommendations[0].reason, null); assert.equal(body.recommendations[0].excerpt, null); assert.equal(body.recommendations[0].featured_image, null); @@ -84,7 +84,7 @@ describe('Recommendations Admin API', function () { // Check everything is set correctly assert.equal(body.recommendations[0].title, 'Dog Pictures'); - assert.equal(body.recommendations[0].url, 'https://dogpictures.com'); + assert.equal(body.recommendations[0].url, 'https://dogpictures.com/'); assert.equal(body.recommendations[0].reason, 'Because dogs are cute'); assert.equal(body.recommendations[0].excerpt, 'Dogs are cute'); assert.equal(body.recommendations[0].featured_image, 'https://dogpictures.com/dog.jpg'); @@ -123,7 +123,7 @@ describe('Recommendations Admin API', function () { // Check everything is set correctly assert.equal(body.recommendations[0].id, id); assert.equal(body.recommendations[0].title, 'Cat Pictures'); - assert.equal(body.recommendations[0].url, 'https://catpictures.com'); + assert.equal(body.recommendations[0].url, 'https://catpictures.com/'); assert.equal(body.recommendations[0].reason, 'Because cats are cute'); assert.equal(body.recommendations[0].excerpt, 'Cats are cute'); assert.equal(body.recommendations[0].featured_image, 'https://catpictures.com/cat.jpg'); diff --git a/ghost/oembed-service/lib/OEmbedService.js b/ghost/oembed-service/lib/OEmbedService.js index 8487ed9591..943de2638e 100644 --- a/ghost/oembed-service/lib/OEmbedService.js +++ b/ghost/oembed-service/lib/OEmbedService.js @@ -132,7 +132,7 @@ class OEmbedService { /** * @param {string} url * - * @returns {Promise<{url: string, body: string}>} + * @returns {Promise<{url: string, body: string, contentType: string|undefined}>} */ async fetchPageHtml(url) { // Fetch url and get response as binary buffer to @@ -154,7 +154,8 @@ class OEmbedService { if (encoding === null) { return { body: body.toString(), - url: responseUrl + url: responseUrl, + contentType: headers['content-type'] }; } @@ -163,14 +164,16 @@ class OEmbedService { return { body: decodedBody, - url: responseUrl + url: responseUrl, + contentType: headers['content-type'] }; } catch (err) { logging.error(err); //return non decoded body anyway return { body: body.toString(), - url: responseUrl + url: responseUrl, + contentType: headers['content-type'] }; } } @@ -355,7 +358,7 @@ class OEmbedService { } // Not in the list, we need to fetch the content - const {url: pageUrl, body} = await this.fetchPageHtml(url); + const {url: pageUrl, body, contentType} = await this.fetchPageHtml(url); // fetch only bookmark when explicitly requested if (type === 'bookmark') { @@ -364,8 +367,26 @@ class OEmbedService { // mentions need to return bookmark data (metadata) and body (html) for link verification if (type === 'mention') { + if (contentType.includes('application/json')) { + // No need to fetch metadata: we have none + const bookmark = { + version: '1.0', + type: 'bookmark', + url, + metadata: { + title: null, + description: null, + publisher: null, + author: null, + thumbnail: null, + icon: null + }, + contentType + }; + return {...bookmark, body}; + } const bookmark = await this.fetchBookmarkData(url, body); - return {...bookmark, body}; + return {...bookmark, body, contentType}; } // attempt to fetch oembed diff --git a/ghost/recommendations/package.json b/ghost/recommendations/package.json index 30fd459f34..36d012d8cc 100644 --- a/ghost/recommendations/package.json +++ b/ghost/recommendations/package.json @@ -22,12 +22,13 @@ "build" ], "devDependencies": { + "@tryghost/errors": "1.2.24", + "@types/node": "^20.5.7", "c8": "8.0.1", "mocha": "10.2.0", "sinon": "15.2.0", "ts-node": "10.9.1", - "typescript": "5.2.2", - "@tryghost/errors": "1.2.24" + "typescript": "5.2.2" }, "dependencies": {} } diff --git a/ghost/recommendations/src/InMemoryRecommendationRepository.ts b/ghost/recommendations/src/InMemoryRecommendationRepository.ts index 187aaa7f21..bd8b77dece 100644 --- a/ghost/recommendations/src/InMemoryRecommendationRepository.ts +++ b/ghost/recommendations/src/InMemoryRecommendationRepository.ts @@ -9,7 +9,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.", featuredImage: "https://www.thepragmaticprogrammer.com/image.png", favicon: "https://www.shesabeast.co/content/images/size/w256h256/2022/08/transparent-icon-black-copy-gray-bar.png", - url: "https://www.thepragmaticprogrammer.com/", + url: new URL("https://www.thepragmaticprogrammer.com/"), oneClickSubscribe: false }), new Recommendation({ @@ -18,7 +18,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.", featuredImage: "https://www.thepragmaticprogrammer.com/image.png", favicon: "https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7cde267-8f9e-47fa-9aef-5be03bad95ed%2Fapple-touch-icon-1024x1024.png", - url: "https://www.thepragmaticprogrammer.com/", + url: new URL("https://www.thepragmaticprogrammer.com/"), oneClickSubscribe: false }), new Recommendation({ @@ -27,7 +27,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.", featuredImage: "https://www.thepragmaticprogrammer.com/image.png", favicon: "https://clickhole.com/wp-content/uploads/2020/05/cropped-clickhole-icon-180x180.png", - url: "https://www.thepragmaticprogrammer.com/", + url: new URL("https://www.thepragmaticprogrammer.com/"), oneClickSubscribe: false }), new Recommendation({ @@ -36,7 +36,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.", featuredImage: "https://www.thepragmaticprogrammer.com/image.png", favicon: "https://www.theverge.com/icons/apple_touch_icon.png", - url: "https://www.thepragmaticprogrammer.com/", + url: new URL("https://www.thepragmaticprogrammer.com/"), oneClickSubscribe: false }), new Recommendation({ @@ -45,7 +45,7 @@ export class InMemoryRecommendationRepository implements RecommendationRepositor excerpt: "The Pragmatic Programmer is one of those rare tech books you’ll read, re-read, and read again over the years. Whether you’re new to the field or an experienced practitioner, you’ll come away with fresh insights each and every time.", featuredImage: "https://www.thepragmaticprogrammer.com/image.png", favicon: "https://substackcdn.com/image/fetch/w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff3f2b2ad-681f-45e1-9496-db80f45e853d_403x403.png", - url: "https://www.thepragmaticprogrammer.com/", + url: new URL("https://www.thepragmaticprogrammer.com/"), oneClickSubscribe: true }) ]; diff --git a/ghost/recommendations/src/Recommendation.ts b/ghost/recommendations/src/Recommendation.ts index 8e7e4d7771..24c2764cdc 100644 --- a/ghost/recommendations/src/Recommendation.ts +++ b/ghost/recommendations/src/Recommendation.ts @@ -7,12 +7,12 @@ export class Recommendation { excerpt: string|null // Fetched from the site meta data featuredImage: string|null // Fetched from the site meta data favicon: string|null // Fetched from the site meta data - url: string + url: URL oneClickSubscribe: boolean createdAt: Date updatedAt: Date|null - constructor(data: {id?: string, title: string, reason: string|null, excerpt: string|null, featuredImage: string|null, favicon: string|null, url: string, oneClickSubscribe: boolean, createdAt?: Date, updatedAt?: Date|null}) { + constructor(data: {id?: string, title: string, reason: string|null, excerpt: string|null, featuredImage: string|null, favicon: string|null, url: URL, oneClickSubscribe: boolean, createdAt?: Date, updatedAt?: Date|null}) { this.id = data.id ?? ObjectId().toString(); this.title = data.title; this.reason = data.reason; diff --git a/ghost/recommendations/src/RecommendationController.ts b/ghost/recommendations/src/RecommendationController.ts index 01567c4ef9..a0fb55dd9b 100644 --- a/ghost/recommendations/src/RecommendationController.ts +++ b/ghost/recommendations/src/RecommendationController.ts @@ -38,6 +38,18 @@ function validateBoolean(object: any, key: string, {required = true} = {}): bool } } +function validateURL(object: any, key: string, {required = true} = {}): URL|undefined { + const string = validateString(object, key, {required}); + if (string !== undefined) { + try { + return new URL(string); + } catch (e) { + throw new errors.BadRequestError({message: `${key} must be a valid URL`}); + } + } +} + + export class RecommendationController { service: RecommendationService; @@ -67,7 +79,7 @@ export class RecommendationController { const cleanedRecommendation: Omit = { title: validateString(recommendation, "title") ?? '', - url: validateString(recommendation, "url") ?? '', + url: validateURL(recommendation, "url")!, // Optional fields oneClickSubscribe: validateBoolean(recommendation, "one_click_subscribe", {required: false}) ?? false, @@ -89,7 +101,7 @@ export class RecommendationController { const recommendation = frame.data.recommendations[0]; const cleanedRecommendation: Partial = { title: validateString(recommendation, "title", {required: false}), - url: validateString(recommendation, "url", {required: false}), + url: validateURL(recommendation, "url", {required: false}), oneClickSubscribe: validateBoolean(recommendation, "one_click_subscribe", {required: false}), reason: validateString(recommendation, "reason", {required: false}), excerpt: validateString(recommendation, "excerpt", {required: false}), @@ -112,7 +124,7 @@ export class RecommendationController { excerpt: r.excerpt, featured_image: r.featuredImage, favicon: r.favicon, - url: r.url, + url: r.url.toString(), one_click_subscribe: r.oneClickSubscribe, created_at: r.createdAt, updated_at: r.updatedAt diff --git a/ghost/recommendations/src/RecommendationService.ts b/ghost/recommendations/src/RecommendationService.ts index c7f5bbcb4b..9b0d581e72 100644 --- a/ghost/recommendations/src/RecommendationService.ts +++ b/ghost/recommendations/src/RecommendationService.ts @@ -1,26 +1,66 @@ import {Recommendation} from "./Recommendation"; import {RecommendationRepository} from "./RecommendationRepository"; +import {WellknownService} from "./WellknownService"; + +type MentionSendingService = { + sendAll(options: {url: URL, links: URL[]}): Promise +} export class RecommendationService { repository: RecommendationRepository; + wellknownService: WellknownService; + mentionSendingService: MentionSendingService; - constructor(deps: {repository: RecommendationRepository}) { + constructor(deps: {repository: RecommendationRepository, wellknownService: WellknownService, mentionSendingService: MentionSendingService}) { this.repository = deps.repository; + this.wellknownService = deps.wellknownService; + this.mentionSendingService = deps.mentionSendingService; + } + + async init() { + await this.updateWellknown(); + } + + async updateWellknown() { + const recommendations = await this.listRecommendations(); + await this.wellknownService.set(recommendations); + } + + sendMentionToRecommendation(recommendation: Recommendation) { + this.mentionSendingService.sendAll({ + url: this.wellknownService.getURL(), + links: [ + recommendation.url + ] + }).catch(console.error); } async addRecommendation(recommendation: Recommendation) { - return this.repository.add(recommendation); + const r = this.repository.add(recommendation); + await this.updateWellknown(); + + // Only send an update for the mentioned URL + this.sendMentionToRecommendation(recommendation); + return r; } async editRecommendation(id: string, recommendationEdit: Partial) { // Check if it exists const existing = await this.repository.getById(id); - return this.repository.edit(existing.id, recommendationEdit); + const e = await this.repository.edit(existing.id, recommendationEdit); + + await this.updateWellknown(); + this.sendMentionToRecommendation(e); + return e; } async deleteRecommendation(id: string) { const existing = await this.repository.getById(id); await this.repository.remove(existing.id); + await this.updateWellknown(); + + // Send a mention (because it was deleted, according to the webmentions spec) + this.sendMentionToRecommendation(existing); } async listRecommendations() { diff --git a/ghost/recommendations/src/WellknownService.ts b/ghost/recommendations/src/WellknownService.ts new file mode 100644 index 0000000000..344fb60b05 --- /dev/null +++ b/ghost/recommendations/src/WellknownService.ts @@ -0,0 +1,48 @@ +import {Recommendation} from "./Recommendation" +import fs from "fs/promises"; +import _path from 'path'; + +type UrlUtils = { + relativeToAbsolute(url: string): string +} +type Options = { + /** + * Where to publish the wellknown file + */ + dir: string, + urlUtils: UrlUtils +} + +export class WellknownService { + dir: string + urlUtils: UrlUtils + + constructor({dir, urlUtils}: Options) { + this.dir = dir; + this.urlUtils = urlUtils; + } + + #formatRecommendation(recommendation: Recommendation) { + return { + url: recommendation.url, + reason: recommendation.reason, + updated_at: (recommendation.updatedAt ?? recommendation.createdAt).toISOString(), + created_at: (recommendation.createdAt).toISOString(), + } + } + + getPath() { + return _path.join(this.dir, '/.well-known/recommendations.json'); + } + + getURL(): URL { + return new URL(this.urlUtils.relativeToAbsolute('/.well-known/recommendations.json')); + } + + async set(recommendations: Recommendation[]) { + const content = JSON.stringify(recommendations.map(r => this.#formatRecommendation(r))); + const path = this.getPath(); + await fs.mkdir(_path.dirname(path), {recursive: true}); + await fs.writeFile(path, content); + } +} diff --git a/ghost/recommendations/src/index.ts b/ghost/recommendations/src/index.ts index da6b0d0cd0..9ae03734fb 100644 --- a/ghost/recommendations/src/index.ts +++ b/ghost/recommendations/src/index.ts @@ -3,3 +3,4 @@ export * from './RecommendationService'; export * from './RecommendationRepository'; export * from './InMemoryRecommendationRepository'; export * from './Recommendation'; +export * from './WellknownService'; diff --git a/ghost/webmentions/lib/Mention.js b/ghost/webmentions/lib/Mention.js index 8ab6a8ff5a..fc1600c53d 100644 --- a/ghost/webmentions/lib/Mention.js +++ b/ghost/webmentions/lib/Mention.js @@ -21,11 +21,44 @@ module.exports = class Mention { /** * @param {string} html + * @param {string} contentType */ - verify(html) { - const $ = cheerio.load(html); - const hasTargetUrl = $('a[href*="' + this.target.href + '"], img[src*="' + this.target.href + '"], video[src*="' + this.target.href + '"]').length > 0; - this.#verified = hasTargetUrl; + verify(html, contentType) { + const wasVerified = this.#verified; + + if (contentType.includes('text/html')) { + try { + const $ = cheerio.load(html); + const hasTargetUrl = $('a[href*="' + this.target.href + '"], img[src*="' + this.target.href + '"], video[src*="' + this.target.href + '"]').length > 0; + this.#verified = hasTargetUrl; + + if (wasVerified && !this.#verified) { + // Delete the mention + this.#deleted = true; + this.#verified = true; + } + } catch (e) { + this.#verified = false; + } + } + + if (contentType.includes('application/json')) { + try { + // Check valid JSON + JSON.parse(html); + + // Check full text string is present in the json + this.#verified = !!html.includes(JSON.stringify(this.target.href)); + + if (wasVerified && !this.#verified) { + // Delete the mention + this.#deleted = true; + this.#verified = true; + } + } catch (e) { + this.#verified = false; + } + } } /** @type {URL} */ @@ -274,12 +307,19 @@ module.exports = class Mention { return mention; } + /** + * @returns {boolean} + */ + isDeleted() { + return this.#deleted; + } + /** * @param {Mention} mention * @returns {boolean} */ static isDeleted(mention) { - return mention.#deleted; + return mention.isDeleted(); } }; diff --git a/ghost/webmentions/lib/MentionSendingService.js b/ghost/webmentions/lib/MentionSendingService.js index dd181a8ece..22fc3dfbc8 100644 --- a/ghost/webmentions/lib/MentionSendingService.js +++ b/ghost/webmentions/lib/MentionSendingService.js @@ -67,7 +67,7 @@ module.exports = class MentionSendingService { let previousHtml = post.previous('status') === 'published' ? post.previous('html') : null; if (html || previousHtml) { await this.#jobService.addJob('sendWebmentions', async () => { - await this.sendAll({ + await this.sendForHTMLResource({ url: new URL(this.#getPostUrl(post)), html: html, previousHtml: previousHtml @@ -80,6 +80,10 @@ module.exports = class MentionSendingService { } } + /** + * @param {{source: URL, target: URL, endpoint: URL}} options + * @returns + */ async send({source, target, endpoint}) { logging.info('[Webmention] Sending webmention from ' + source.href + ' to ' + target.href + ' via ' + endpoint.href); @@ -113,7 +117,7 @@ module.exports = class MentionSendingService { * @param {string} resource.html * @param {string|null} [resource.previousHtml] */ - async sendAll(resource) { + async sendForHTMLResource(resource) { const links = resource.html ? this.getLinks(resource.html) : []; if (resource.previousHtml) { // We also need to send webmentions for removed links @@ -129,12 +133,25 @@ module.exports = class MentionSendingService { logging.info('[Webmention] Sending all webmentions for ' + resource.url.href); } + await this.sendAll({ + url: resource.url, + links + }); + } + + /** + * Send a webmention call for the links in a resource. + * @param {object} resource + * @param {URL} resource.url + * @param {URL[]} resource.links + */ + async sendAll({url, links}) { for (const target of links) { const endpoint = await this.#discoveryService.getEndpoint(target); if (endpoint) { // Send webmention call try { - await this.send({source: resource.url, target, endpoint}); + await this.send({source: url, target, endpoint}); } catch (e) { logging.error('[Webmention] Failed sending via ' + endpoint.href + ': ' + e.message); } diff --git a/ghost/webmentions/lib/MentionsAPI.js b/ghost/webmentions/lib/MentionsAPI.js index 65cb1f9dfc..ad952d3eca 100644 --- a/ghost/webmentions/lib/MentionsAPI.js +++ b/ghost/webmentions/lib/MentionsAPI.js @@ -68,13 +68,14 @@ const Mention = require('./Mention'); /** * @typedef {object} WebmentionMetadata - * @prop {string} siteTitle - * @prop {string} title - * @prop {string} excerpt - * @prop {string} author - * @prop {URL} image - * @prop {URL} favicon + * @prop {string|null} siteTitle + * @prop {string|null} title + * @prop {string|null} excerpt + * @prop {string|null} author + * @prop {URL|null} image + * @prop {URL|null} favicon * @prop {string} body + * @prop {string|undefined} contentType */ /** @@ -228,7 +229,7 @@ module.exports = class MentionsAPI { if (metadata?.body) { try { - mention.verify(metadata.body); + mention.verify(metadata.body, metadata.contentType); } catch (e) { logging.error(e); } diff --git a/ghost/webmentions/test/Mention.test.js b/ghost/webmentions/test/Mention.test.js index cc112aa9a9..e384e88f75 100644 --- a/ghost/webmentions/test/Mention.test.js +++ b/ghost/webmentions/test/Mention.test.js @@ -1,6 +1,8 @@ const assert = require('assert/strict'); const ObjectID = require('bson-objectid'); const Mention = require('../lib/Mention'); +const cheerio = require('cheerio'); +const sinon = require('sinon'); const validInput = { source: 'https://source.com', @@ -35,15 +37,40 @@ describe('Mention', function () { }); describe('verify', function () { + afterEach(function () { + sinon.restore(); + }); + + it('can handle invalid HTML', async function () { + const mention = await Mention.create(validInput); + assert(!mention.verified); + + sinon.stub(cheerio, 'load').throws(new Error('Invalid HTML')); + + mention.verify('irrelevant', 'text/html'); + assert(!mention.verified); + assert(!mention.isDeleted()); + }); + it('Does basic check for the target URL and updates verified property', async function () { const mention = await Mention.create(validInput); assert(!mention.verified); - mention.verify(''); + mention.verify('', 'text/html'); assert(mention.verified); + assert(!mention.isDeleted()); - mention.verify(''); + mention.verify('something else', 'text/html'); + assert(mention.verified); + assert(mention.isDeleted()); + }); + it('detects differences', async function () { + const mention = await Mention.create(validInput); assert(!mention.verified); + + mention.verify('', 'text/html'); + assert(!mention.verified); + assert(!mention.isDeleted()); }); it('Does check for Image targets', async function () { const mention = await Mention.create({ @@ -52,11 +79,13 @@ describe('Mention', function () { }); assert(!mention.verified); - mention.verify(''); + mention.verify('', 'text/html'); assert(mention.verified); + assert(!mention.isDeleted()); - mention.verify(''); - assert(!mention.verified); + mention.verify('something else', 'text/html'); + assert(mention.verified); + assert(mention.isDeleted()); }); it('Does check for Video targets', async function () { const mention = await Mention.create({ @@ -65,11 +94,35 @@ describe('Mention', function () { }); assert(!mention.verified); - mention.verify('Example`, previousHtml: `Example`}); assert.equal(scope.isDone(), true); @@ -311,7 +311,7 @@ describe('MentionSendingService', function () { isEnabled: () => true }); const linksStub = sinon.stub(service, 'getLinks'); - await service.sendAll({html: ``,previousHtml: ``}); + await service.sendForHTMLResource({html: ``,previousHtml: ``}); sinon.assert.notCalled(linksStub); }); }); diff --git a/yarn.lock b/yarn.lock index 9cf4090f70..cec737c65c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8588,7 +8588,7 @@ dependencies: "@types/node" "*" -"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.1.0": +"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.1.0", "@types/node@^20.5.7": version "20.5.7" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.5.7.tgz#4b8ecac87fbefbc92f431d09c30e176fc0a7c377" integrity sha512-dP7f3LdZIysZnmvP3ANJYTSwg+wLLl8p7RqniVlV7j+oXSXAbt9h0WIBFmJy5inWZoX9wZN6eXx+YXd9Rh3RBA==