diff --git a/doc/api/test.md b/doc/api/test.md index b12e6c9212af3b..e3e4eb97d40dc7 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -245,6 +245,19 @@ it.expectFailure('should do the thing', () => { it('should do the thing', { expectFailure: true }, () => { assert.strictEqual(doTheThing(), true); }); + +it('should do the thing', { expectFailure: 'feature not implemented' }, () => { + assert.strictEqual(doTheThing(), true); +}); + +it('should fail with specific error', { + expectFailure: { + with: /error message/, + message: 'reason for failure', + }, +}, () => { + assert.strictEqual(doTheThing(), true); +}); ``` `skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo` @@ -1677,6 +1690,9 @@ changes: thread. If `false`, only one test runs at a time. If unspecified, subtests inherit this value from their parent. **Default:** `false`. + * `expectFailure` {boolean|string} If truthy, the test is expected to fail. + If a string is provided, that string is displayed in the test results as the + reason why the test is expected to fail. **Default:** `false`. * `only` {boolean} If truthy, and the test context is configured to run `only` tests, then this test will be run. Otherwise, the test is skipped. **Default:** `false`. diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index 01c698871b9134..f92127207ec003 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -77,7 +77,7 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure } else if (todo !== undefined) { line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } else if (expectFailure !== undefined) { - line += ' # EXPECTED FAILURE'; + line += ` # EXPECTED FAILURE${typeof expectFailure === 'string' && expectFailure.length ? ` ${tapEscape(expectFailure)}` : ''}`; } line += '\n'; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index e426438faba75f..824a92c0d38064 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -56,6 +56,7 @@ const { once: runOnce, setOwnProperty, } = require('internal/util'); +const assert = require('assert'); const { isPromise } = require('internal/util/types'); const { validateAbortSignal, @@ -635,7 +636,13 @@ class Test extends AsyncResource { this.plan = null; this.expectedAssertions = plan; this.cancelled = false; - this.expectFailure = expectFailure !== undefined && expectFailure !== false; + if (expectFailure === undefined || expectFailure === false) { + this.expectFailure = false; + } else if (typeof expectFailure === 'string' || typeof expectFailure === 'object') { + this.expectFailure = expectFailure; + } else { + this.expectFailure = true; + } this.skipped = skip !== undefined && skip !== false; this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo; this.startTime = null; @@ -947,7 +954,23 @@ class Test extends AsyncResource { return; } - if (this.expectFailure === true) { + if (this.expectFailure) { + if (typeof this.expectFailure === 'object' && + this.expectFailure.with !== undefined) { + const { with: validation } = this.expectFailure; + try { + const { throws } = assert; + throws(() => { throw err; }, validation); + } catch (e) { + this.passed = false; + this.error = new ERR_TEST_FAILURE( + 'The test failed, but the error did not match the expected validation', + kTestCodeFailure, + ); + this.error.cause = e; + return; + } + } this.passed = true; } else { this.passed = false; @@ -961,6 +984,20 @@ class Test extends AsyncResource { return; } + if (this.skipped || this.isTodo) { + this.passed = true; + return; + } + + if (this.expectFailure) { + this.passed = false; + this.error = new ERR_TEST_FAILURE( + 'Test passed but was expected to fail', + kTestCodeFailure, + ); + return; + } + this.passed = true; } @@ -1350,7 +1387,10 @@ class Test extends AsyncResource { } else if (this.isTodo) { directive = this.reporter.getTodo(this.message); } else if (this.expectFailure) { - directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure + const message = typeof this.expectFailure === 'object' ? + this.expectFailure.message : + this.expectFailure; + directive = this.reporter.getXFail(message); } if (this.reportedType) { diff --git a/test/parallel/test-runner-xfail.js b/test/parallel/test-runner-xfail.js new file mode 100644 index 00000000000000..d774485af63d06 --- /dev/null +++ b/test/parallel/test-runner-xfail.js @@ -0,0 +1,56 @@ +'use strict'; +const common = require('../common'); +const { test } = require('node:test'); +const { spawn } = require('child_process'); +const assert = require('node:assert'); + +if (process.env.CHILD_PROCESS === 'true') { + test('fail with message string', { expectFailure: 'reason string' }, () => { + assert.fail('boom'); + }); + + test('fail with message object', { expectFailure: { message: 'reason object' } }, () => { + assert.fail('boom'); + }); + + test('fail with validation regex', { expectFailure: { with: /boom/ } }, () => { + assert.fail('boom'); + }); + + test('fail with validation object', { expectFailure: { with: { message: 'boom' } } }, () => { + assert.fail('boom'); + }); + + test('fail with validation class', { expectFailure: { with: assert.AssertionError } }, () => { + assert.fail('boom'); + }); + + test('fail with validation error (wrong error)', { expectFailure: { with: /bang/ } }, () => { + assert.fail('boom'); // Should result in real failure because error doesn't match + }); + + test('unexpected pass', { expectFailure: true }, () => { + // Should result in real failure because it didn't fail + }); + +} else { + const child = spawn(process.execPath, ['--test-reporter', 'tap', __filename], { + env: { ...process.env, CHILD_PROCESS: 'true' }, + stdio: 'pipe', + }); + + let stdout = ''; + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { stdout += chunk; }); + + child.on('close', common.mustCall((code) => { + // We expect exit code 1 because 'unexpected pass' and 'wrong error' should fail the test run + assert.strictEqual(code, 1); + + // Check outputs + assert.match(stdout, /# EXPECTED FAILURE reason string/); + assert.match(stdout, /# EXPECTED FAILURE reason object/); + assert.match(stdout, /not ok \d+ - fail with validation error \(wrong error\)/); + assert.match(stdout, /not ok \d+ - unexpected pass/); + })); +}