Merge pull request #24 from ysfaran/feat/previous-attempts

feat: add previous attempts to Ctrf test results

Author: Ma11hewThomas

src/generate-report.ts

@@ -16,6 +16,7 @@ import {
   type CtrfTest,
   type CtrfEnvironment,
   type CtrfAttachment,
+  type CtrfTestAttempt,
 } from '../types/ctrf'
 
 interface ReporterConfigOptions {
@@ -222,6 +223,22 @@ class GenerateCtrfReport implements Reporter {
       if (this.reporterConfigOptions.annotations !== undefined) {
         test.extra = { annotations: testCase.annotations }
       }
+
+      if (testCase.results.length > 1) {
+        const retryResults = testCase.results.slice(0, -1)
+        test.retryAttempts = []
+
+        for (const retryResult of retryResults) {
+          const retryAttempt: CtrfTestAttempt = {
+            status: this.mapPlaywrightStatusToCtrf(retryResult.status),
+            duration: retryResult.duration,
+            message: this.extractFailureDetails(retryResult).message,
+            trace: this.extractFailureDetails(retryResult).trace,
+            snippet: this.extractFailureDetails(retryResult).snippet,
+          }
+          test.retryAttempts.push(retryAttempt)
+        }
+      }
     }
 
     ctrfReport.results.tests.push(test)

tests/dummy-suites/flaky-test-suite.ts

@@ -0,0 +1,122 @@
+import {
+  type Suite,
+  type TestCase,
+  type Location,
+  type TestResult,
+  type TestError,
+} from '@playwright/test/reporter'
+
+/**
+ * Creates a minimal Suite object with a single flaky test
+ * with 2 failed attempts and one passed attempt
+ */
+export const createFlakyTestSuite = (): Suite => {
+  const testError: TestError = {
+    message: 'test-error-message',
+    stack: 'test-error-stack',
+    snippet: 'test-error-snippet',
+  }
+
+  const failedTestResult: TestResult = {
+    retry: 0,
+    duration: 4444,
+    status: 'failed',
+    startTime: new Date('2023-01-01T00:00:00.000Z'),
+    parallelIndex: 0,
+    workerIndex: 0,
+    attachments: [],
+    errors: [testError],
+    error: testError,
+    steps: [],
+    stdout: [],
+    stderr: [],
+  }
+
+  const testError2: TestError = {
+    message: 'test-error-message2',
+    stack: 'test-error-stack2',
+    snippet: 'test-error-snippet2',
+  }
+
+  const failedTestResult2: TestResult = {
+    retry: 1,
+    duration: 5555,
+    status: 'failed',
+    startTime: new Date('2023-01-01T00:00:00.000Z'),
+    parallelIndex: 0,
+    workerIndex: 0,
+    attachments: [],
+    errors: [testError2],
+    error: testError2,
+    steps: [],
+    stdout: [],
+    stderr: [],
+  }
+
+  const passedTestResult: TestResult = {
+    retry: 2,
+    duration: 888,
+    status: 'passed',
+    startTime: new Date('2023-01-01T00:00:05.200Z'),
+    parallelIndex: 0,
+    workerIndex: 0,
+    attachments: [],
+    errors: [],
+    steps: [],
+    stdout: [],
+    stderr: [],
+  }
+
+  const testCase: TestCase = {
+    title: 'should validate the expected condition',
+    id: 'test-id-123',
+    annotations: [],
+    expectedStatus: 'passed',
+    timeout: 30000,
+    results: [failedTestResult, failedTestResult2, passedTestResult],
+    location: {
+      file: 'flaky-test.spec.ts',
+      line: 42,
+      column: 3,
+    },
+    parent: undefined as any, // Will be set later
+    outcome: () => 'flaky',
+    ok: () => true,
+    titlePath: () => ['Flaky Test Suite', 'should be flaky'],
+    repeatEachIndex: 0,
+    retries: 1,
+  }
+
+  const suite: Suite = {
+    title: 'Flaky Test Suite',
+    titlePath: () => ['Flaky Test Suite'],
+    location: {
+      file: 'flaky-test.spec.ts',
+      line: 10,
+      column: 1,
+    } as Location,
+    project: () => ({
+      name: 'Test Project',
+      outputDir: './test-results',
+      grep: /.*/,
+      grepInvert: null,
+      metadata: {},
+      dependencies: [],
+      repeatEach: 1,
+      retries: 3,
+      timeout: 30000,
+      use: {},
+      testDir: './tests',
+      testIgnore: [],
+      testMatch: [],
+      snapshotDir: './snapshots',
+    }),
+    allTests: () => [testCase],
+    tests: [testCase],
+    suites: [],
+  }
+
+  testCase.parent = suite
+
+  return suite
+}

tests/flaky-tests.spec.ts

@@ -0,0 +1,56 @@
+import { createFlakyTestSuite } from './dummy-suites/flaky-test-suite'
+import GenerateCtrfReport from '../src/generate-report'
+import fs from 'fs'
+import { CtrfReport } from '../types/ctrf'
+
+jest.mock('fs', () => ({
+  writeFileSync: jest.fn(),
+  existsSync: jest.fn(() => true),
+}))
+const nowDateMock = new Date('2023-01-01T00:00:00.000Z')
+jest.useFakeTimers().setSystemTime(nowDateMock)
+
+const mockedFs = fs as jest.Mocked<typeof fs>
+
+describe('Flaky Tests', () => {
+  it('should generate report with retry attempts correctly', async () => {
+    // Arrange
+    const testSuite = createFlakyTestSuite()
+    const report = new GenerateCtrfReport()
+
+    // Act
+    report.onBegin(undefined as any, testSuite)
+    report.onEnd()
+
+    // Assert
+    expect(mockedFs.writeFileSync).toHaveBeenCalledTimes(1)
+
+    const reportJsonContent = mockedFs.writeFileSync.mock.calls[0][1] as string
+    const parsedReport: CtrfReport = JSON.parse(reportJsonContent)
+
+    expect(parsedReport.results.tests).toHaveLength(1)
+
+    const test = parsedReport.results.tests[0]
+
+    expect(test.status).toBe('passed')
+    expect(test.retries).toBe(2)
+    expect(test.flaky).toBe(true)
+    expect(test.duration).toBe(888)
+
+    expect(test.retryAttempts).toHaveLength(2)
+
+    const failedAttempt = test.retryAttempts![0]
+    expect(failedAttempt.status).toBe('failed')
+    expect(failedAttempt.duration).toBe(4444)
+    expect(failedAttempt.message).toBe('test-error-message')
+    expect(failedAttempt.trace).toBe('test-error-stack')
+    expect(failedAttempt.snippet).toBe('test-error-snippet')
+
+    const failedAttempt2 = test.retryAttempts![1]
+    expect(failedAttempt2.status).toBe('failed')
+    expect(failedAttempt2.duration).toBe(5555)
+    expect(failedAttempt2.message).toBe('test-error-message2')
+    expect(failedAttempt2.trace).toBe('test-error-stack2')
+    expect(failedAttempt2.snippet).toBe('test-error-snippet2')
+  })
+})

types/ctrf.d.ts

@@ -47,9 +47,18 @@ export interface CtrfTest {
   screenshot?: string
   parameters?: Record<string, any>
   steps?: Step[]
+  retryAttempts?: CtrfTestAttempt[]
   extra?: Record<string, any>
 }
 
+export interface CtrfTestAttempt {
+  status: CtrfTestState
+  duration: number
+  message?: string
+  trace?: string
+  snippet?: string
+}
+
 export interface CtrfEnvironment {
   appName?: string
   appVersion?: string