fix(test-runner): recover beforeAll-skipped tests for --last-failed (#42056)

Co-authored-by: piotrkozlowski <piotr.kozlowski@housecallpro.com>

Author: Pinio31

packages/playwright/src/runner/lastRun.ts

@@ -17,7 +17,7 @@
 import fs from 'fs';
 import path from 'path';
 
-import type { FullResult, Suite } from '../../types/testReporter';
+import type { FullResult, Suite, TestCase } from '../../types/testReporter';
 import type { config as commonConfig } from '../common';
 import type { ReporterV2 } from '../reporters/reporterV2';
 
@@ -26,6 +26,14 @@ type LastRunInfo = {
   failedTests: string[];
 };
 
+function didNotRun(test: TestCase): boolean {
+  if (test.outcome() !== 'skipped')
+    return false;
+  if (test.results.some(result => result.status === 'interrupted'))
+    return false;
+  return !test.results.length || test.expectedStatus !== 'skipped';
+}
+
 export class LastRunReporter implements ReporterV2 {
   private _lastRunFile: string | undefined;
   private _suite: Suite | undefined;
@@ -71,7 +79,7 @@ export class LastRunReporter implements ReporterV2 {
       return;
     const lastRunInfo: LastRunInfo = {
       status: result.status,
-      failedTests: this._suite?.allTests().filter(t => !t.ok()).map(t => t.id) || [],
+      failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [],
     };
     await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true });
     await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2));

tests/playwright-test/runner.spec.ts

@@ -866,6 +866,55 @@ test('should run nothing with --last-failed when previous run had no failures',
   expect(result2.didNotRun).toBe(0);
 });
 
+test('should run last failed tests that did not run because a beforeAll hook failed', async ({ runInlineTest }) => {
+  const workspace = {
+    'a.spec.js': `
+      import fs from 'fs';
+      import { test, expect } from '@playwright/test';
+      test.beforeAll(() => {
+        if (!fs.existsSync('marker.txt')) {
+          fs.writeFileSync('marker.txt', '');
+          throw new Error('from beforeAll');
+        }
+      });
+      test('one', () => {});
+      test('two', () => {});
+    `
+  };
+  const result1 = await runInlineTest(workspace);
+  expect(result1.exitCode).toBe(1);
+  expect(result1.failed).toBe(1);
+  expect(result1.didNotRun).toBe(1);
+
+  const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed'] });
+  expect(result2.exitCode).toBe(0);
+  expect(result2.passed).toBe(2);
+  expect(result2.didNotRun).toBe(0);
+});
+
+test('should not run intentionally skipped tests with --last-failed', async ({ runInlineTest }) => {
+  const workspace = {
+    'a.spec.js': `
+      import { test, expect } from '@playwright/test';
+      test('fail', () => {
+        expect(1).toBe(2);
+      });
+      test('skipped', () => {
+        test.skip();
+      });
+    `
+  };
+  const result1 = await runInlineTest(workspace);
+  expect(result1.exitCode).toBe(1);
+  expect(result1.failed).toBe(1);
+  expect(result1.skipped).toBe(1);
+
+  const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed'] });
+  expect(result2.exitCode).toBe(1);
+  expect(result2.failed).toBe(1);
+  expect(result2.skipped).toBe(0);
+});
+
 test('should run last failed tests in a shard', async ({ runInlineTest }) => {
   const workspace = {
     'a.spec.js': `