test_runner: support custom message for expectFailure

doc/api/test.md

@@ -245,6 +245,19 @@ it.expectFailure('should do the thing', () => {
 it('should do the thing', { expectFailure: true }, () => {
   assert.strictEqual(doTheThing(), true);
 });
+
+it('should do the thing', { expectFailure: 'feature not implemented' }, () => {
+  assert.strictEqual(doTheThing(), true);
+});
+
+it('should fail with specific error', {
+  expectFailure: {
+    with: /error message/,
+    message: 'reason for failure',
+  },
+}, () => {
+  assert.strictEqual(doTheThing(), true);
+});
 ```
 
 `skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo`
@@ -1677,6 +1690,9 @@ changes:
     thread. If `false`, only one test runs at a time.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `false`.
+  * `expectFailure` {boolean|string} If truthy, the test is expected to fail.
+    If a string is provided, that string is displayed in the test results as the
+    reason why the test is expected to fail. **Default:** `false`.
   * `only` {boolean} If truthy, and the test context is configured to run
     `only` tests, then this test will be run. Otherwise, the test is skipped.
     **Default:** `false`.

lib/internal/test_runner/reporter/tap.js

@@ -77,7 +77,7 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure
   } else if (todo !== undefined) {
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
   } else if (expectFailure !== undefined) {
-    line += ' # EXPECTED FAILURE';
+    line += ` # EXPECTED FAILURE${typeof expectFailure === 'string' && expectFailure.length ? ` ${tapEscape(expectFailure)}` : ''}`;
   }
 
   line += '\n';

lib/internal/test_runner/test.js

@@ -56,6 +56,7 @@ const {
   once: runOnce,
   setOwnProperty,
 } = require('internal/util');
+const assert = require('assert');
 const { isPromise } = require('internal/util/types');
 const {
   validateAbortSignal,
@@ -635,7 +636,13 @@ class Test extends AsyncResource {
     this.plan = null;
     this.expectedAssertions = plan;
     this.cancelled = false;
-    this.expectFailure = expectFailure !== undefined && expectFailure !== false;
+    if (expectFailure === undefined || expectFailure === false) {
+      this.expectFailure = false;
+    } else if (typeof expectFailure === 'string' || typeof expectFailure === 'object') {
+      this.expectFailure = expectFailure;
+    } else {
+      this.expectFailure = true;
+    }
     this.skipped = skip !== undefined && skip !== false;
     this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo;
     this.startTime = null;
@@ -947,7 +954,23 @@ class Test extends AsyncResource {
       return;
     }
 
-    if (this.expectFailure === true) {
+    if (this.expectFailure) {
+      if (typeof this.expectFailure === 'object' &&
+          this.expectFailure.with !== undefined) {
+        const { with: validation } = this.expectFailure;
+        try {
+          const { throws } = assert;
+          throws(() => { throw err; }, validation);
+        } catch (e) {
+          this.passed = false;
+          this.error = new ERR_TEST_FAILURE(
+            'The test failed, but the error did not match the expected validation',
+            kTestCodeFailure,
+          );
+          this.error.cause = e;
+          return;
+        }
+      }
       this.passed = true;
     } else {
       this.passed = false;
@@ -961,6 +984,20 @@ class Test extends AsyncResource {
       return;
     }
 
+    if (this.skipped || this.isTodo) {
+      this.passed = true;
+      return;
+    }
+
+    if (this.expectFailure) {
+      this.passed = false;
+      this.error = new ERR_TEST_FAILURE(
+        'Test passed but was expected to fail',
+        kTestCodeFailure,
+      );
+      return;
+    }
+
     this.passed = true;
   }
 
@@ -1350,7 +1387,10 @@ class Test extends AsyncResource {
     } else if (this.isTodo) {
       directive = this.reporter.getTodo(this.message);
     } else if (this.expectFailure) {
-      directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure
+      const message = typeof this.expectFailure === 'object' ?
+        this.expectFailure.message :
+        this.expectFailure;
+      directive = this.reporter.getXFail(message);
     }
 
     if (this.reportedType) {

test/parallel/test-runner-xfail.js

@@ -0,0 +1,56 @@
+'use strict';
+const common = require('../common');
+const { test } = require('node:test');
+const { spawn } = require('child_process');
+const assert = require('node:assert');
+
+if (process.env.CHILD_PROCESS === 'true') {
+  test('fail with message string', { expectFailure: 'reason string' }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with message object', { expectFailure: { message: 'reason object' } }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with validation regex', { expectFailure: { with: /boom/ } }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with validation object', { expectFailure: { with: { message: 'boom' } } }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with validation class', { expectFailure: { with: assert.AssertionError } }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with validation error (wrong error)', { expectFailure: { with: /bang/ } }, () => {
+    assert.fail('boom'); // Should result in real failure because error doesn't match
+  });
+
+  test('unexpected pass', { expectFailure: true }, () => {
+    // Should result in real failure because it didn't fail
+  });
+
+} else {
+  const child = spawn(process.execPath, ['--test-reporter', 'tap', __filename], {
+    env: { ...process.env, CHILD_PROCESS: 'true' },
+    stdio: 'pipe',
+  });
+
+  let stdout = '';
+  child.stdout.setEncoding('utf8');
+  child.stdout.on('data', (chunk) => { stdout += chunk; });
+
+  child.on('close', common.mustCall((code) => {
+    // We expect exit code 1 because 'unexpected pass' and 'wrong error' should fail the test run
+    assert.strictEqual(code, 1);
+
+    // Check outputs
+    assert.match(stdout, /# EXPECTED FAILURE reason string/);
+    assert.match(stdout, /# EXPECTED FAILURE reason object/);
+    assert.match(stdout, /not ok \d+ - fail with validation error \(wrong error\)/);
+    assert.match(stdout, /not ok \d+ - unexpected pass/);
+  }));
+}