From 563793c7ed6e424057671171e0be762ce9974f7d Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Fri, 7 Apr 2023 00:37:01 -0700 Subject: [PATCH] Added retries for all known flaky tests (#16582) refs TryGhost/Team#2833 - for mocha tests, we can add `this.retries(1)` to any flaky tests - for playwright tests, we can add `test.describe.configure({ retries: 1})` to any `describe` block - not a long-term solution, but it should help mitigate issues with flaky tests in short term --- ghost/core/test/e2e-api/admin/posts-legacy.test.js | 1 + ghost/core/test/e2e-browser/admin/members.spec.js | 1 + ghost/core/test/e2e-browser/admin/publishing.spec.js | 2 ++ .../test/e2e-browser/portal/member-actions.spec.js | 2 ++ .../services/email-service/batch-sending.test.js | 1 + ghost/webmentions/test/MentionSendingService.test.js | 10 ++++++++++ ghost/webmentions/test/MentionsAPI.test.js | 1 + 7 files changed, 18 insertions(+) diff --git a/ghost/core/test/e2e-api/admin/posts-legacy.test.js b/ghost/core/test/e2e-api/admin/posts-legacy.test.js index dcfc16196e..73bf8a3472 100644 --- a/ghost/core/test/e2e-api/admin/posts-legacy.test.js +++ b/ghost/core/test/e2e-api/admin/posts-legacy.test.js @@ -259,6 +259,7 @@ describe('Posts API', function () { }); it('Can add a post', async function () { + this.retries(1); const post = { title: 'My post', status: 'draft', diff --git a/ghost/core/test/e2e-browser/admin/members.spec.js b/ghost/core/test/e2e-browser/admin/members.spec.js index fdc2f9b334..be44d2acae 100644 --- a/ghost/core/test/e2e-browser/admin/members.spec.js +++ b/ghost/core/test/e2e-browser/admin/members.spec.js @@ -4,6 +4,7 @@ const fs = require('fs'); test.describe('Admin', () => { test.describe('Members', () => { + this.describe.configure({retries: 1}); test('A member can be created', async ({page}) => { await page.goto('/ghost'); await page.locator('.gh-nav a[href="#/members/"]').click(); diff --git a/ghost/core/test/e2e-browser/admin/publishing.spec.js b/ghost/core/test/e2e-browser/admin/publishing.spec.js index e22ebedb98..01d5a268e3 100644 --- a/ghost/core/test/e2e-browser/admin/publishing.spec.js +++ b/ghost/core/test/e2e-browser/admin/publishing.spec.js @@ -280,6 +280,8 @@ test.describe('Publishing', () => { }); test.describe('Update post', () => { + test.describe.configure({retries: 1}); + test('Can update a published post', async ({page: adminPage}) => { await adminPage.goto('/ghost'); diff --git a/ghost/core/test/e2e-browser/portal/member-actions.spec.js b/ghost/core/test/e2e-browser/portal/member-actions.spec.js index 144e4bd37e..1311f092fc 100644 --- a/ghost/core/test/e2e-browser/portal/member-actions.spec.js +++ b/ghost/core/test/e2e-browser/portal/member-actions.spec.js @@ -19,6 +19,8 @@ const addNewsletter = async (page) => { test.describe('Portal', () => { test.describe('Member actions', () => { + test.describe.configure({retries: 1}); + test('can log out', async ({page}) => { // create a new free member await createMember(page, { diff --git a/ghost/core/test/integration/services/email-service/batch-sending.test.js b/ghost/core/test/integration/services/email-service/batch-sending.test.js index c5afdcbb80..7f06bdefb3 100644 --- a/ghost/core/test/integration/services/email-service/batch-sending.test.js +++ b/ghost/core/test/integration/services/email-service/batch-sending.test.js @@ -1047,6 +1047,7 @@ describe('Batch sending tests', function () { }); it('Shows subscription details box for free members', async function () { + this.retries(1); // Create a new member without a first_name await models.Member.add({ email: 'subscription-box-1@example.com', diff --git a/ghost/webmentions/test/MentionSendingService.test.js b/ghost/webmentions/test/MentionSendingService.test.js index aac961d478..137fec382a 100644 --- a/ghost/webmentions/test/MentionSendingService.test.js +++ b/ghost/webmentions/test/MentionSendingService.test.js @@ -210,6 +210,7 @@ describe('MentionSendingService', function () { describe('sendAll', function () { it('Sends to all links', async function () { + this.retries(1); let counter = 0; const scope = nock('https://example.org') .persist() @@ -242,6 +243,7 @@ describe('MentionSendingService', function () { }); it('Catches and logs errors', async function () { + this.retries(1); let counter = 0; const scope = nock('https://example.org') .persist() @@ -278,6 +280,7 @@ describe('MentionSendingService', function () { }); it('Sends to deleted links', async function () { + this.retries(1); let counter = 0; const scope = nock('https://example.org') .persist() @@ -380,6 +383,7 @@ describe('MentionSendingService', function () { describe('send', function () { it('Can handle 202 accepted responses', async function () { + this.retries(1); const source = new URL('https://example.com/source'); const target = new URL('https://target.com/target'); const endpoint = new URL('https://example.org/webmentions-test'); @@ -398,6 +402,7 @@ describe('MentionSendingService', function () { }); it('Can handle 201 created responses', async function () { + this.retries(1); const source = new URL('https://example.com/source'); const target = new URL('https://target.com/target'); const endpoint = new URL('https://example.org/webmentions-test'); @@ -416,6 +421,7 @@ describe('MentionSendingService', function () { }); it('Can handle 400 responses', async function () { + this.retries(1); const scope = nock('https://example.org') .persist() .post('/webmentions-test') @@ -431,6 +437,7 @@ describe('MentionSendingService', function () { }); it('Can handle 500 responses', async function () { + this.retries(1); const scope = nock('https://example.org') .persist() .post('/webmentions-test') @@ -446,6 +453,7 @@ describe('MentionSendingService', function () { }); it('Can handle redirect responses', async function () { + this.retries(1); const scope = nock('https://example.org') .persist() .post('/webmentions-test') @@ -468,6 +476,7 @@ describe('MentionSendingService', function () { }); it('Can handle network errors', async function () { + this.retries(1); const scope = nock('https://example.org') .persist() .post('/webmentions-test') @@ -483,6 +492,7 @@ describe('MentionSendingService', function () { }); it('Does not send to private IP behind DNS', async function () { + this.retries(1); // Test that we don't make a request when a domain resolves to a private IP // domaincontrol.com -> 127.0.0.1 const service = new MentionSendingService({externalRequest}); diff --git a/ghost/webmentions/test/MentionsAPI.test.js b/ghost/webmentions/test/MentionsAPI.test.js index 6d7df5467d..633a255d0d 100644 --- a/ghost/webmentions/test/MentionsAPI.test.js +++ b/ghost/webmentions/test/MentionsAPI.test.js @@ -45,6 +45,7 @@ describe('MentionsAPI', function () { }); it('Can generate a mentions report', async function () { + this.retries(1); const repository = new InMemoryMentionRepository(); const api = new MentionsAPI({ repository,