[test optimization] Fix playwright's quarantine and retry logic (#7024)

Author: juan-fernandez

integration-tests/ci-visibility/playwright-efd-and-retries/flaky-test.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const { test, expect } = require('@playwright/test')
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(process.env.PW_BASE_URL)
+})
+
+test.describe('playwright', () => {
+  test('should not retry new tests', async ({ page }, testInfo) => {
+    // test will always fail because ATR and --retries are disabled for EFD
+    if (testInfo.retry === 0) {
+      throw new Error('Hello Warld')
+    }
+    await expect(page.locator('.hello-world')).toHaveText([
+      'Hello World'
+    ])
+  })
+
+  test('should retry old flaky tests', async ({ page }, testInfo) => {
+    // will eventually pass because the test is not retried by EFD
+    if (testInfo.retry === 0) {
+      throw new Error('Hello Warld')
+    }
+    await expect(page.locator('.hello-world')).toHaveText([
+      'Hello World'
+    ])
+  })
+})

integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js

@@ -31,3 +31,11 @@ test.describe('attempt to fix', () => {
     expect(true).toBe(true)
   })
 })
+
+if (process.env.SHOULD_INCLUDE_FLAKY_TEST) {
+  test('flaky test is retried without attempt to fix', async ({ page }, testInfo) => {
+    await expect(page.locator('.hello-world')).toHaveText([
+      testInfo.retry === 0 ? 'Hello Warld' : 'Hello World'
+    ])
+  })
+}

integration-tests/ci-visibility/playwright-tests-test-management/quarantine-2-test.js

@@ -0,0 +1,12 @@
+'use strict'
+
+const { test, expect } = require('@playwright/test')
+
+test.describe('flaky', () => {
+  test('should be flaky', async ({ page }, testInfo) => {
+    if (testInfo.retry === 0) {
+      throw new Error('I am flaky')
+    }
+    expect(true).toBe(true)
+  })
+})