From 60d81b2003c8cd26e970bc4f43a59623850e2290 Mon Sep 17 00:00:00 2001 From: Michael Barrett Date: Thu, 14 Mar 2024 14:44:54 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20`/p/`=20redirects=20not?= =?UTF-8?q?=20being=20indexed=20by=20search=20engines=20(#19864)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ref [ENG-741](https://linear.app/tryghost/issue/ENG-741/🐛-our-robotstxt-config-causes-indexing-issues-for-customers-who-have) `/p/` has been dropped from the `robots.txt` file so that search engines can index the pages at these locations. In the event that the page at the location is a preview page, the existing robots meta tag on the page will prevent indexing. --- ghost/core/core/frontend/public/robots.txt | 1 - ghost/core/test/e2e-frontend/default_routes.test.js | 3 --- 2 files changed, 4 deletions(-) diff --git a/ghost/core/core/frontend/public/robots.txt b/ghost/core/core/frontend/public/robots.txt index 4654e58f39..8ad5b3f97b 100644 --- a/ghost/core/core/frontend/public/robots.txt +++ b/ghost/core/core/frontend/public/robots.txt @@ -1,7 +1,6 @@ User-agent: * Sitemap: {{blog-url}}/sitemap.xml Disallow: /ghost/ -Disallow: /p/ Disallow: /email/ Disallow: /r/ Disallow: /webmentions/receive/ diff --git a/ghost/core/test/e2e-frontend/default_routes.test.js b/ghost/core/test/e2e-frontend/default_routes.test.js index d4ee04009c..567cf13494 100644 --- a/ghost/core/test/e2e-frontend/default_routes.test.js +++ b/ghost/core/test/e2e-frontend/default_routes.test.js @@ -319,12 +319,9 @@ describe('Default Frontend routing', function () { .expect(200) .expect(assertCorrectFrontendHeaders); - // The response here is a publicly documented format users rely on - // In case it's changed remember to update the docs at https://ghost.org/help/modifying-robots-txt/ res.text.should.equal( 'User-agent: *\n' + 'Sitemap: http://127.0.0.1:2369/sitemap.xml\nDisallow: /ghost/\n' + - 'Disallow: /p/\n' + 'Disallow: /email/\n' + 'Disallow: /r/\n' + 'Disallow: /webmentions/receive/\n'