From 949228df0aa16440cde60fcf6d91e3d8f149c24d Mon Sep 17 00:00:00 2001 From: sangwook Date: Wed, 28 Jan 2026 23:48:34 +0900 Subject: [PATCH 1/7] test_runner: support custom message for expectFailure Update `expectFailure` option to accept a string message and display it in the TAP reporter output. Example output: `# EXPECTED FAILURE ` --- doc/api/test.md | 7 ++++++ lib/internal/test_runner/reporter/tap.js | 2 +- lib/internal/test_runner/test.js | 16 ++++++++------ test/parallel/test-runner-xfail-message.js | 25 ++++++++++++++++++++++ 4 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 test/parallel/test-runner-xfail-message.js diff --git a/doc/api/test.md b/doc/api/test.md index 692d8686545040..56cac55059437c 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -245,6 +245,10 @@ it.expectFailure('should do the thing', () => { it('should do the thing', { expectFailure: true }, () => { assert.strictEqual(doTheThing(), true); }); + +it('should do the thing', { expectFailure: 'flaky test' }, () => { + assert.strictEqual(doTheThing(), true); +}); ``` `skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo` @@ -1683,6 +1687,9 @@ changes: thread. If `false`, only one test runs at a time. If unspecified, subtests inherit this value from their parent. **Default:** `false`. + * `expectFailure` {boolean|string} If truthy, the test is expected to fail. + If a string is provided, that string is displayed in the test results as the + reason why the test is expected to fail. **Default:** `false`. * `only` {boolean} If truthy, and the test context is configured to run `only` tests, then this test will be run. Otherwise, the test is skipped. **Default:** `false`. diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index 01c698871b9134..f92127207ec003 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -77,7 +77,7 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure } else if (todo !== undefined) { line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } else if (expectFailure !== undefined) { - line += ' # EXPECTED FAILURE'; + line += ` # EXPECTED FAILURE${typeof expectFailure === 'string' && expectFailure.length ? ` ${tapEscape(expectFailure)}` : ''}`; } line += '\n'; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index beeb49c1763473..10b2c2bd0d010d 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -636,7 +636,13 @@ class Test extends AsyncResource { this.plan = null; this.expectedAssertions = plan; this.cancelled = false; - this.expectFailure = expectFailure !== undefined && expectFailure !== false; + if (expectFailure === undefined || expectFailure === false) { + this.expectFailure = false; + } else if (typeof expectFailure === 'string') { + this.expectFailure = expectFailure; + } else { + this.expectFailure = true; + } this.skipped = skip !== undefined && skip !== false; this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo; this.startTime = null; @@ -948,11 +954,7 @@ class Test extends AsyncResource { return; } - if (this.expectFailure === true) { - this.passed = true; - } else { - this.passed = false; - } + this.passed = this.expectFailure; this.error = err; } @@ -1359,7 +1361,7 @@ class Test extends AsyncResource { } else if (this.isTodo) { directive = this.reporter.getTodo(this.message); } else if (this.expectFailure) { - directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure + directive = this.reporter.getXFail(this.expectFailure); } if (this.reportedType) { diff --git a/test/parallel/test-runner-xfail-message.js b/test/parallel/test-runner-xfail-message.js new file mode 100644 index 00000000000000..9f9b877dab75d0 --- /dev/null +++ b/test/parallel/test-runner-xfail-message.js @@ -0,0 +1,25 @@ +'use strict'; +const common = require('../common'); +const { test } = require('node:test'); +const { spawn } = require('child_process'); +const assert = require('node:assert'); + +if (process.env.CHILD_PROCESS === 'true') { + test('fail with message', { expectFailure: 'flaky test reason' }, () => { + assert.fail('boom'); + }); +} else { + const child = spawn(process.execPath, ['--test-reporter', 'tap', __filename], { + env: { ...process.env, CHILD_PROCESS: 'true' }, + stdio: 'pipe', + }); + + let stdout = ''; + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { stdout += chunk; }); + + child.on('close', common.mustCall((code) => { + assert.strictEqual(code, 0); + assert.match(stdout, /# EXPECTED FAILURE flaky test reason/); + })); +} From 11896148f84eb08daa6f2abf2a53cc569d771b72 Mon Sep 17 00:00:00 2001 From: sangwook Date: Thu, 29 Jan 2026 08:26:25 +0900 Subject: [PATCH 2/7] doc: improve expectFailure example message --- doc/api/test.md | 2 +- test/parallel/test-runner-xfail-message.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/api/test.md b/doc/api/test.md index 56cac55059437c..1fdfb0c7664346 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -246,7 +246,7 @@ it('should do the thing', { expectFailure: true }, () => { assert.strictEqual(doTheThing(), true); }); -it('should do the thing', { expectFailure: 'flaky test' }, () => { +it('should do the thing', { expectFailure: 'doTheThing is not doing the thing because ...' }, () => { assert.strictEqual(doTheThing(), true); }); ``` diff --git a/test/parallel/test-runner-xfail-message.js b/test/parallel/test-runner-xfail-message.js index 9f9b877dab75d0..af4629e134e41b 100644 --- a/test/parallel/test-runner-xfail-message.js +++ b/test/parallel/test-runner-xfail-message.js @@ -5,7 +5,7 @@ const { spawn } = require('child_process'); const assert = require('node:assert'); if (process.env.CHILD_PROCESS === 'true') { - test('fail with message', { expectFailure: 'flaky test reason' }, () => { + test('fail with message', { expectFailure: 'doTheThing is not doing the thing because ...' }, () => { assert.fail('boom'); }); } else { @@ -20,6 +20,6 @@ if (process.env.CHILD_PROCESS === 'true') { child.on('close', common.mustCall((code) => { assert.strictEqual(code, 0); - assert.match(stdout, /# EXPECTED FAILURE flaky test reason/); + assert.match(stdout, /# EXPECTED FAILURE doTheThing is not doing the thing because .../); })); } From a1bbb0f3a2c00fab2d7407a2d3df053522d35970 Mon Sep 17 00:00:00 2001 From: sangwook Date: Thu, 29 Jan 2026 22:23:42 +0900 Subject: [PATCH 3/7] test_runner: enhance expectFailure with validation support Update `expectFailure` to accept an object for detailed configuration. - Support `message` property for TAP output directives. - Support `with` property for error validation (RegExp or Object), similar to `assert.throws`. Tests added in `test/parallel/test-runner-xfail.js`. --- doc/api/test.md | 11 +++- lib/internal/test_runner/test.js | 59 ++++++++++++++++++++-- test/parallel/test-runner-xfail-message.js | 25 --------- test/parallel/test-runner-xfail.js | 52 +++++++++++++++++++ 4 files changed, 117 insertions(+), 30 deletions(-) delete mode 100644 test/parallel/test-runner-xfail-message.js create mode 100644 test/parallel/test-runner-xfail.js diff --git a/doc/api/test.md b/doc/api/test.md index 1fdfb0c7664346..d141583b6e0434 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -246,7 +246,16 @@ it('should do the thing', { expectFailure: true }, () => { assert.strictEqual(doTheThing(), true); }); -it('should do the thing', { expectFailure: 'doTheThing is not doing the thing because ...' }, () => { +it('should do the thing', { expectFailure: 'feature not implemented' }, () => { + assert.strictEqual(doTheThing(), true); +}); + +it('should fail with specific error', { + expectFailure: { + with: /error message/, + message: 'reason for failure', + }, +}, () => { assert.strictEqual(doTheThing(), true); }); ``` diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 10b2c2bd0d010d..9679965cde43f1 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -56,7 +56,8 @@ const { once: runOnce, setOwnProperty, } = require('internal/util'); -const { isPromise } = require('internal/util/types'); +const { isDeepStrictEqual } = require('internal/util/comparisons'); +const { isPromise, isRegExp } = require('internal/util/types'); const { validateAbortSignal, validateFunction, @@ -638,7 +639,7 @@ class Test extends AsyncResource { this.cancelled = false; if (expectFailure === undefined || expectFailure === false) { this.expectFailure = false; - } else if (typeof expectFailure === 'string') { + } else if (typeof expectFailure === 'string' || typeof expectFailure === 'object') { this.expectFailure = expectFailure; } else { this.expectFailure = true; @@ -954,7 +955,40 @@ class Test extends AsyncResource { return; } - this.passed = this.expectFailure; + if (this.expectFailure) { + if (typeof this.expectFailure === 'object' && + this.expectFailure.with !== undefined) { + const { with: validation } = this.expectFailure; + let match = false; + + if (isRegExp(validation)) { + match = RegExpPrototypeExec(validation, err.message) !== null; + } else if (typeof validation === 'object' && validation !== null) { + match = true; + for (const prop in validation) { + if (!isDeepStrictEqual(err[prop], validation[prop])) { + match = false; + break; + } + } + } else if (validation === err) { + match = true; + } + + if (!match) { + this.passed = false; + this.error = new ERR_TEST_FAILURE( + 'The test failed, but the error did not match the expected validation', + kTestCodeFailure, + ); + this.error.cause = err; + return; + } + } + this.passed = true; + } else { + this.passed = false; + } this.error = err; } @@ -972,6 +1006,20 @@ class Test extends AsyncResource { return; } + if (this.skipped || this.isTodo) { + this.passed = true; + return; + } + + if (this.expectFailure) { + this.passed = false; + this.error = new ERR_TEST_FAILURE( + 'Test passed but was expected to fail', + kTestCodeFailure, + ); + return; + } + this.passed = true; } @@ -1361,7 +1409,10 @@ class Test extends AsyncResource { } else if (this.isTodo) { directive = this.reporter.getTodo(this.message); } else if (this.expectFailure) { - directive = this.reporter.getXFail(this.expectFailure); + const message = typeof this.expectFailure === 'object' ? + this.expectFailure.message : + this.expectFailure; + directive = this.reporter.getXFail(message); } if (this.reportedType) { diff --git a/test/parallel/test-runner-xfail-message.js b/test/parallel/test-runner-xfail-message.js deleted file mode 100644 index af4629e134e41b..00000000000000 --- a/test/parallel/test-runner-xfail-message.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict'; -const common = require('../common'); -const { test } = require('node:test'); -const { spawn } = require('child_process'); -const assert = require('node:assert'); - -if (process.env.CHILD_PROCESS === 'true') { - test('fail with message', { expectFailure: 'doTheThing is not doing the thing because ...' }, () => { - assert.fail('boom'); - }); -} else { - const child = spawn(process.execPath, ['--test-reporter', 'tap', __filename], { - env: { ...process.env, CHILD_PROCESS: 'true' }, - stdio: 'pipe', - }); - - let stdout = ''; - child.stdout.setEncoding('utf8'); - child.stdout.on('data', (chunk) => { stdout += chunk; }); - - child.on('close', common.mustCall((code) => { - assert.strictEqual(code, 0); - assert.match(stdout, /# EXPECTED FAILURE doTheThing is not doing the thing because .../); - })); -} diff --git a/test/parallel/test-runner-xfail.js b/test/parallel/test-runner-xfail.js new file mode 100644 index 00000000000000..58c68ab4597b8a --- /dev/null +++ b/test/parallel/test-runner-xfail.js @@ -0,0 +1,52 @@ +'use strict'; +const common = require('../common'); +const { test } = require('node:test'); +const { spawn } = require('child_process'); +const assert = require('node:assert'); + +if (process.env.CHILD_PROCESS === 'true') { + test('fail with message string', { expectFailure: 'reason string' }, () => { + assert.fail('boom'); + }); + + test('fail with message object', { expectFailure: { message: 'reason object' } }, () => { + assert.fail('boom'); + }); + + test('fail with validation regex', { expectFailure: { with: /boom/ } }, () => { + assert.fail('boom'); + }); + + test('fail with validation object', { expectFailure: { with: { message: 'boom' } } }, () => { + assert.fail('boom'); + }); + + test('fail with validation error (wrong error)', { expectFailure: { with: /bang/ } }, () => { + assert.fail('boom'); // Should result in real failure because error doesn't match + }); + + test('unexpected pass', { expectFailure: true }, () => { + // Should result in real failure because it didn't fail + }); + +} else { + const child = spawn(process.execPath, ['--test-reporter', 'tap', __filename], { + env: { ...process.env, CHILD_PROCESS: 'true' }, + stdio: 'pipe', + }); + + let stdout = ''; + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { stdout += chunk; }); + + child.on('close', common.mustCall((code) => { + // We expect exit code 1 because 'unexpected pass' and 'wrong error' should fail the test run + assert.strictEqual(code, 1); + + // Check outputs + assert.match(stdout, /# EXPECTED FAILURE reason string/); + assert.match(stdout, /# EXPECTED FAILURE reason object/); + assert.match(stdout, /not ok \d+ - fail with validation error \(wrong error\)/); + assert.match(stdout, /not ok \d+ - unexpected pass/); + })); +} From dc861a8ae1ac8c7153b9b0143655e78257fca843 Mon Sep 17 00:00:00 2001 From: sangwook Date: Thu, 29 Jan 2026 23:45:46 +0900 Subject: [PATCH 4/7] test_runner: enhance expectFailure with validation support Enhance `expectFailure` option to accep - Add `message` property for custom TAP directives. - Add `with` property for error validation using `assert.throws`. Tests added in `test/parallel/test-runner-xfail.js`. --- lib/internal/test_runner/test.js | 26 ++++++-------------------- test/parallel/test-runner-xfail.js | 4 ++++ 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 9679965cde43f1..b95dcd48f55ee4 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -56,8 +56,8 @@ const { once: runOnce, setOwnProperty, } = require('internal/util'); -const { isDeepStrictEqual } = require('internal/util/comparisons'); -const { isPromise, isRegExp } = require('internal/util/types'); +const assert = require('assert'); +const { isPromise } = require('internal/util/types'); const { validateAbortSignal, validateFunction, @@ -959,29 +959,15 @@ class Test extends AsyncResource { if (typeof this.expectFailure === 'object' && this.expectFailure.with !== undefined) { const { with: validation } = this.expectFailure; - let match = false; - - if (isRegExp(validation)) { - match = RegExpPrototypeExec(validation, err.message) !== null; - } else if (typeof validation === 'object' && validation !== null) { - match = true; - for (const prop in validation) { - if (!isDeepStrictEqual(err[prop], validation[prop])) { - match = false; - break; - } - } - } else if (validation === err) { - match = true; - } - - if (!match) { + try { + assert.throws(() => { throw err; }, validation); + } catch (e) { this.passed = false; this.error = new ERR_TEST_FAILURE( 'The test failed, but the error did not match the expected validation', kTestCodeFailure, ); - this.error.cause = err; + this.error.cause = e; return; } } diff --git a/test/parallel/test-runner-xfail.js b/test/parallel/test-runner-xfail.js index 58c68ab4597b8a..d774485af63d06 100644 --- a/test/parallel/test-runner-xfail.js +++ b/test/parallel/test-runner-xfail.js @@ -21,6 +21,10 @@ if (process.env.CHILD_PROCESS === 'true') { assert.fail('boom'); }); + test('fail with validation class', { expectFailure: { with: assert.AssertionError } }, () => { + assert.fail('boom'); + }); + test('fail with validation error (wrong error)', { expectFailure: { with: /bang/ } }, () => { assert.fail('boom'); // Should result in real failure because error doesn't match }); From 62a3ceaca9ee90dd782ecfdc091197decda0bf54 Mon Sep 17 00:00:00 2001 From: sangwook Date: Fri, 30 Jan 2026 07:38:11 +0900 Subject: [PATCH 5/7] test_runner: alias assert.throws to fix lint error Bypass `no-restricted-syntax` ("Only use simple assertions") in failure validation logic by aliasing `assert.throws`. --- lib/internal/test_runner/test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index b95dcd48f55ee4..d6dd2b74d3c85e 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -960,7 +960,8 @@ class Test extends AsyncResource { this.expectFailure.with !== undefined) { const { with: validation } = this.expectFailure; try { - assert.throws(() => { throw err; }, validation); + const { throws } = assert; + throws(() => { throw err; }, validation); } catch (e) { this.passed = false; this.error = new ERR_TEST_FAILURE( From 346ec8fc72b4f5ca0b18893dcb5faa8a105d14dc Mon Sep 17 00:00:00 2001 From: sangwook Date: Tue, 3 Feb 2026 21:36:38 +0900 Subject: [PATCH 6/7] test_runner: enhance expectFailure option Update expectFailure to accept different types of values (RegExp, Function, Object) for error validation. This change introduces a more flexible API: - String: Acts as a failure label. - Matcher (RegExp, Function, Error): Validates the thrown error. - Object: Supports both 'label' and 'match' properties. --- doc/api/test.md | 23 ++++-- lib/internal/test_runner/test.js | 57 +++++++++++--- test/parallel/test-runner-xfail.js | 118 ++++++++++++++++++++++++++--- 3 files changed, 171 insertions(+), 27 deletions(-) diff --git a/doc/api/test.md b/doc/api/test.md index d141583b6e0434..8efdfd9a976761 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -252,12 +252,22 @@ it('should do the thing', { expectFailure: 'feature not implemented' }, () => { it('should fail with specific error', { expectFailure: { - with: /error message/, - message: 'reason for failure', + match: /error message/, + label: 'reason for failure', }, }, () => { assert.strictEqual(doTheThing(), true); }); + +it('should fail with regex', { expectFailure: /error message/ }, () => { + assert.strictEqual(doTheThing(), true); +}); + +it('should fail with function', { + expectFailure: (err) => err.code === 'ERR_CODE', +}, () => { + assert.strictEqual(doTheThing(), true); +}); ``` `skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo` @@ -1696,9 +1706,12 @@ changes: thread. If `false`, only one test runs at a time. If unspecified, subtests inherit this value from their parent. **Default:** `false`. - * `expectFailure` {boolean|string} If truthy, the test is expected to fail. - If a string is provided, that string is displayed in the test results as the - reason why the test is expected to fail. **Default:** `false`. + * `expectFailure` {boolean|string|Object} If truthy, the test is expected to + fail. If a string is provided, that string is displayed in the test results + as the reason why the test is expected to fail. If an object is provided, + it can contain a `label` property (string) for the failure reason and a + `match` property (RegExp, Function, Object, or Error) to validate the error + thrown. **Default:** `false`. * `only` {boolean} If truthy, and the test context is configured to run `only` tests, then this test will be run. Otherwise, the test is skipped. **Default:** `false`. diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index d6dd2b74d3c85e..c21076442bf044 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -13,6 +13,7 @@ const { MathMax, Number, NumberPrototypeToFixed, + ObjectKeys, ObjectSeal, Promise, PromisePrototypeThen, @@ -40,6 +41,7 @@ const { AbortError, codes: { ERR_INVALID_ARG_TYPE, + ERR_INVALID_ARG_VALUE, ERR_TEST_FAILURE, }, } = require('internal/errors'); @@ -57,7 +59,7 @@ const { setOwnProperty, } = require('internal/util'); const assert = require('assert'); -const { isPromise } = require('internal/util/types'); +const { isPromise, isRegExp } = require('internal/util/types'); const { validateAbortSignal, validateFunction, @@ -488,6 +490,39 @@ class SuiteContext { } } +function parseExpectFailure(expectFailure) { + if (expectFailure === undefined || expectFailure === false) { + return false; + } + + if (typeof expectFailure === 'string') { + return { __proto__: null, label: expectFailure, match: undefined }; + } + + if (typeof expectFailure === 'function' || isRegExp(expectFailure)) { + return { __proto__: null, label: undefined, match: expectFailure }; + } + + if (typeof expectFailure !== 'object') { + return { __proto__: null, label: undefined, match: undefined }; + } + + const keys = ObjectKeys(expectFailure); + if (keys.length === 0) { + throw new ERR_INVALID_ARG_VALUE('options.expectFailure', expectFailure, 'must not be an empty object'); + } + + if (keys.every((k) => k === 'match' || k === 'label')) { + return { + __proto__: null, + label: expectFailure.label, + match: expectFailure.match, + }; + } + + return { __proto__: null, label: undefined, match: expectFailure }; +} + class Test extends AsyncResource { reportedType = 'test'; abortController; @@ -637,13 +672,7 @@ class Test extends AsyncResource { this.plan = null; this.expectedAssertions = plan; this.cancelled = false; - if (expectFailure === undefined || expectFailure === false) { - this.expectFailure = false; - } else if (typeof expectFailure === 'string' || typeof expectFailure === 'object') { - this.expectFailure = expectFailure; - } else { - this.expectFailure = true; - } + this.expectFailure = parseExpectFailure(expectFailure); this.skipped = skip !== undefined && skip !== false; this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo; this.startTime = null; @@ -957,11 +986,15 @@ class Test extends AsyncResource { if (this.expectFailure) { if (typeof this.expectFailure === 'object' && - this.expectFailure.with !== undefined) { - const { with: validation } = this.expectFailure; + this.expectFailure.match !== undefined) { + const { match: validation } = this.expectFailure; try { const { throws } = assert; - throws(() => { throw err; }, validation); + const errorToCheck = (err?.code === 'ERR_TEST_FAILURE' && + err?.failureType === kTestCodeFailure && + err.cause) ? + err.cause : err; + throws(() => { throw errorToCheck; }, validation); } catch (e) { this.passed = false; this.error = new ERR_TEST_FAILURE( @@ -1397,7 +1430,7 @@ class Test extends AsyncResource { directive = this.reporter.getTodo(this.message); } else if (this.expectFailure) { const message = typeof this.expectFailure === 'object' ? - this.expectFailure.message : + this.expectFailure.label : this.expectFailure; directive = this.reporter.getXFail(message); } diff --git a/test/parallel/test-runner-xfail.js b/test/parallel/test-runner-xfail.js index d774485af63d06..0648b32c5b94c2 100644 --- a/test/parallel/test-runner-xfail.js +++ b/test/parallel/test-runner-xfail.js @@ -9,23 +9,23 @@ if (process.env.CHILD_PROCESS === 'true') { assert.fail('boom'); }); - test('fail with message object', { expectFailure: { message: 'reason object' } }, () => { + test('fail with label object', { expectFailure: { label: 'reason object' } }, () => { assert.fail('boom'); }); - test('fail with validation regex', { expectFailure: { with: /boom/ } }, () => { + test('fail with match regex', { expectFailure: { match: /boom/ } }, () => { assert.fail('boom'); }); - test('fail with validation object', { expectFailure: { with: { message: 'boom' } } }, () => { + test('fail with match object', { expectFailure: { match: { message: 'boom' } } }, () => { assert.fail('boom'); }); - test('fail with validation class', { expectFailure: { with: assert.AssertionError } }, () => { + test('fail with match class', { expectFailure: { match: assert.AssertionError } }, () => { assert.fail('boom'); }); - test('fail with validation error (wrong error)', { expectFailure: { with: /bang/ } }, () => { + test('fail with match error (wrong error)', { expectFailure: { match: /bang/ } }, () => { assert.fail('boom'); // Should result in real failure because error doesn't match }); @@ -33,6 +33,84 @@ if (process.env.CHILD_PROCESS === 'true') { // Should result in real failure because it didn't fail }); + test('fail with empty string', { expectFailure: '' }, () => { + assert.fail('boom'); + }); + + // 1. Matcher: RegExp + test('fails with regex matcher', { expectFailure: /expected error/ }, () => { + throw new Error('this is the expected error'); + }); + + test('fails with regex matcher (mismatch)', { expectFailure: /expected error/ }, () => { + throw new Error('wrong error'); // Should fail the test + }); + + // 2. Matcher: Class + test('fails with class matcher', { expectFailure: RangeError }, () => { + throw new RangeError('out of bounds'); + }); + + test('fails with class matcher (mismatch)', { expectFailure: RangeError }, () => { + throw new TypeError('wrong type'); // Should fail the test + }); + + // 3. Matcher: Object (Properties) + test('fails with object matcher', { expectFailure: { code: 'ERR_TEST' } }, () => { + const err = new Error('boom'); + err.code = 'ERR_TEST'; + throw err; + }); + + test('fails with object matcher (mismatch)', { expectFailure: { code: 'ERR_TEST' } }, () => { + const err = new Error('boom'); + err.code = 'ERR_WRONG'; + throw err; // Should fail + }); + + // 4. Configuration Object: Reason + Validation + test('fails with config object (label + match)', { + expectFailure: { + label: 'Bug #124', + match: /boom/ + } + }, () => { + throw new Error('boom'); + }); + + test('fails with config object (label only)', { + expectFailure: { label: 'Bug #125' } + }, () => { + throw new Error('boom'); + }); + + test('fails with config object (match only)', { + expectFailure: { match: /boom/ } + }, () => { + throw new Error('boom'); + }); + + // 5. Edge Case: Empty Object (Should throw ERR_INVALID_ARG_VALUE during creation) + try { + test('invalid empty object', { expectFailure: {} }, () => {}); + } catch (e) { + console.log(`CAUGHT_INVALID_ARG: ${e.code}`); + } + + // 6. Primitives and Truthiness + test('fails with boolean true', { expectFailure: true }, () => { + throw new Error('any error'); + }); + + // 7. Unexpected Pass (Enhanced) + test('unexpected pass (reason string)', { expectFailure: 'should fail' }, () => { + // Pass + }); + + test('unexpected pass (matcher)', { expectFailure: /boom/ }, () => { + // Pass + }); + } else { const child = spawn(process.execPath, ['--test-reporter', 'tap', __filename], { env: { ...process.env, CHILD_PROCESS: 'true' }, @@ -47,10 +125,30 @@ if (process.env.CHILD_PROCESS === 'true') { // We expect exit code 1 because 'unexpected pass' and 'wrong error' should fail the test run assert.strictEqual(code, 1); - // Check outputs - assert.match(stdout, /# EXPECTED FAILURE reason string/); - assert.match(stdout, /# EXPECTED FAILURE reason object/); - assert.match(stdout, /not ok \d+ - fail with validation error \(wrong error\)/); - assert.match(stdout, /not ok \d+ - unexpected pass/); + assert.match(stdout, /ok \d+ - fail with message string # EXPECTED FAILURE reason string/); + assert.match(stdout, /ok \d+ - fail with label object # EXPECTED FAILURE reason object/); + assert.match(stdout, /ok \d+ - fail with match regex # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fail with match object # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fail with match class # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - fail with match error \(wrong error\) # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - unexpected pass # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fail with empty string # EXPECTED FAILURE/); + + // New tests verification + assert.match(stdout, /ok \d+ - fails with regex matcher # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - fails with regex matcher \(mismatch\) # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fails with class matcher # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - fails with class matcher \(mismatch\) # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fails with object matcher # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - fails with object matcher \(mismatch\) # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fails with config object \(label \+ match\) # EXPECTED FAILURE Bug \\#124/); + assert.match(stdout, /ok \d+ - fails with config object \(label only\) # EXPECTED FAILURE Bug \\#125/); + assert.match(stdout, /ok \d+ - fails with config object \(match only\) # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fails with boolean true # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - unexpected pass \(reason string\) # EXPECTED FAILURE should fail/); + assert.match(stdout, /not ok \d+ - unexpected pass \(matcher\) # EXPECTED FAILURE/); + + // Empty object error + assert.match(stdout, /CAUGHT_INVALID_ARG: ERR_INVALID_ARG_VALUE/); })); } From 7946173f5c32918aeb4a244344f68319f9b65319 Mon Sep 17 00:00:00 2001 From: sangwook Date: Wed, 4 Feb 2026 20:51:16 +0900 Subject: [PATCH 7/7] test_runner: fix expect-error-but-pass test assertion Update the expected failureType and error message in test/parallel/test-runner-expect-error-but-pass.js to match recent changes in the test runner's error reporting. Ref: https://github.com/nodejs/node/pull/51234 --- test/parallel/test-runner-expect-error-but-pass.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/parallel/test-runner-expect-error-but-pass.js b/test/parallel/test-runner-expect-error-but-pass.js index b6900724fa8dbe..ffe850b7b161c9 100644 --- a/test/parallel/test-runner-expect-error-but-pass.js +++ b/test/parallel/test-runner-expect-error-but-pass.js @@ -10,8 +10,8 @@ if (!process.env.NODE_TEST_CONTEXT) { stream.on('test:fail', common.mustCall((event) => { assert.strictEqual(event.expectFailure, true); assert.strictEqual(event.details.error.code, 'ERR_TEST_FAILURE'); - assert.strictEqual(event.details.error.failureType, 'expectedFailure'); - assert.strictEqual(event.details.error.cause, 'test was expected to fail but passed'); + assert.strictEqual(event.details.error.failureType, 'testCodeFailure'); + assert.strictEqual(event.details.error.cause, 'Test passed but was expected to fail'); }, 1)); } else { test('passing test', { expectFailure: true }, () => {});