OpenSourceArk/Ghost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🐛 Fixed bookmark creation for sites that block some user agents

Browse Source 
closes https://linear.app/tryghost/issue/ENG-762

- nytimes.com and other sites return 403 responses when requests do not match typical browser user-agents
- our bookmark fetching requests were using `Ghost(https://github.com/TryGhost/Ghost)` meaning bookmark creation failed for these user-agent-blocking sites
- switched to using a standard browser user-agent string to avoid such blocks
This commit is contained in:
Kevin Ansfield 2024-06-20 22:06:53 +01:00
parent 3bc5eb8cf9
commit 0b4e249037
2 changed files with 26 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ghost/oembed-service/lib/OEmbedService.js
View File

@ -6,6 +6,9 @@ const _ = require('lodash');
const charset = require('charset');
const iconv = require('iconv-lite');
// Some sites block non-standard user agents so we need to mimic a typical browser
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9';
const messages = {
noUrlProvided: 'No url provided.',
insufficientMetadata: 'URL contains insufficient metadata.',
@ -126,6 +129,9 @@ class OEmbedService {
return this.externalRequest(
url,
{
headers: {
'user-agent': USER_AGENT
},
timeout: 2000,
followRedirect: true,
...options
@ -205,7 +211,11 @@ class OEmbedService {
* @returns {Promise<Object>}
*/
async fetchBookmarkData(url, html) {
const gotOpts = {};
const gotOpts = {
headers: {
'User-Agent': USER_AGENT
}
};
if (process.env.NODE_ENV?.startsWith('test')) {
gotOpts.retry = 0;

15
ghost/oembed-service/test/oembed-service.test.js
View File

@ -131,5 +131,20 @@ describe('oembed-service', function () {
assert.equal(response.author_url, 'https://example.com/user/testauthor');
assert.equal(response.html, '<iframe src="https://www.example.com/embed"></iframe>');
});
it('uses a known user-agent for bookmark requests', async function () {
nock('https://www.example.com')
.get('/')
.query(true)
.matchHeader('User-Agent', /Mozilla\/.*/)
.reply(200, `<html><head><title>Example</title></head></html>`);
const response = await oembedService.fetchOembedDataFromUrl('https://www.example.com', 'bookmark');
assert.equal(response.version, '1.0');
assert.equal(response.type, 'bookmark');
assert.equal(response.url, 'https://www.example.com');
assert.equal(response.metadata.title, 'Example');
});
});
});