diff --git a/doc/api/test.md b/doc/api/test.md index b12e6c9212af3b..9150767c24c7af 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -245,6 +245,29 @@ it.expectFailure('should do the thing', () => { it('should do the thing', { expectFailure: true }, () => { assert.strictEqual(doTheThing(), true); }); + +it('should do the thing', { expectFailure: 'feature not implemented' }, () => { + assert.strictEqual(doTheThing(), true); +}); + +it('should fail with specific error', { + expectFailure: { + match: /error message/, + label: 'reason for failure', + }, +}, () => { + assert.strictEqual(doTheThing(), true); +}); + +it('should fail with regex', { expectFailure: /error message/ }, () => { + assert.strictEqual(doTheThing(), true); +}); + +it('should fail with function', { + expectFailure: (err) => err.code === 'ERR_CODE', +}, () => { + assert.strictEqual(doTheThing(), true); +}); ``` `skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo` @@ -1677,6 +1700,12 @@ changes: thread. If `false`, only one test runs at a time. If unspecified, subtests inherit this value from their parent. **Default:** `false`. + * `expectFailure` {boolean|string|Object} If truthy, the test is expected to + fail. If a string is provided, that string is displayed in the test results + as the reason why the test is expected to fail. If an object is provided, + it can contain a `label` property (string) for the failure reason and a + `match` property (RegExp, Function, Object, or Error) to validate the error + thrown. **Default:** `false`. * `only` {boolean} If truthy, and the test context is configured to run `only` tests, then this test will be run. Otherwise, the test is skipped. **Default:** `false`. diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index 01c698871b9134..f92127207ec003 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -77,7 +77,7 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure } else if (todo !== undefined) { line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } else if (expectFailure !== undefined) { - line += ' # EXPECTED FAILURE'; + line += ` # EXPECTED FAILURE${typeof expectFailure === 'string' && expectFailure.length ? ` ${tapEscape(expectFailure)}` : ''}`; } line += '\n'; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index e426438faba75f..4f058a3c1a861b 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -13,6 +13,7 @@ const { MathMax, Number, NumberPrototypeToFixed, + ObjectKeys, ObjectSeal, Promise, PromisePrototypeThen, @@ -40,6 +41,7 @@ const { AbortError, codes: { ERR_INVALID_ARG_TYPE, + ERR_INVALID_ARG_VALUE, ERR_TEST_FAILURE, }, } = require('internal/errors'); @@ -56,7 +58,8 @@ const { once: runOnce, setOwnProperty, } = require('internal/util'); -const { isPromise } = require('internal/util/types'); +const assert = require('assert'); +const { isPromise, isRegExp } = require('internal/util/types'); const { validateAbortSignal, validateFunction, @@ -486,6 +489,39 @@ class SuiteContext { } } +function parseExpectFailure(expectFailure) { + if (expectFailure === undefined || expectFailure === false) { + return false; + } + + if (typeof expectFailure === 'string') { + return { __proto__: null, label: expectFailure, match: undefined }; + } + + if (typeof expectFailure === 'function' || isRegExp(expectFailure)) { + return { __proto__: null, label: undefined, match: expectFailure }; + } + + if (typeof expectFailure !== 'object') { + return { __proto__: null, label: undefined, match: undefined }; + } + + const keys = ObjectKeys(expectFailure); + if (keys.length === 0) { + throw new ERR_INVALID_ARG_VALUE('options.expectFailure', expectFailure, 'must not be an empty object'); + } + + if (keys.every((k) => k === 'match' || k === 'label')) { + return { + __proto__: null, + label: expectFailure.label, + match: expectFailure.match, + }; + } + + return { __proto__: null, label: undefined, match: expectFailure }; +} + class Test extends AsyncResource { reportedType = 'test'; abortController; @@ -635,7 +671,7 @@ class Test extends AsyncResource { this.plan = null; this.expectedAssertions = plan; this.cancelled = false; - this.expectFailure = expectFailure !== undefined && expectFailure !== false; + this.expectFailure = parseExpectFailure(expectFailure); this.skipped = skip !== undefined && skip !== false; this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo; this.startTime = null; @@ -947,7 +983,27 @@ class Test extends AsyncResource { return; } - if (this.expectFailure === true) { + if (this.expectFailure) { + if (typeof this.expectFailure === 'object' && + this.expectFailure.match !== undefined) { + const { match: validation } = this.expectFailure; + try { + const { throws } = assert; + const errorToCheck = (err?.code === 'ERR_TEST_FAILURE' && + err?.failureType === kTestCodeFailure && + err.cause) ? + err.cause : err; + throws(() => { throw errorToCheck; }, validation); + } catch (e) { + this.passed = false; + this.error = new ERR_TEST_FAILURE( + 'The test failed, but the error did not match the expected validation', + kTestCodeFailure, + ); + this.error.cause = e; + return; + } + } this.passed = true; } else { this.passed = false; @@ -961,6 +1017,20 @@ class Test extends AsyncResource { return; } + if (this.skipped || this.isTodo) { + this.passed = true; + return; + } + + if (this.expectFailure) { + this.passed = false; + this.error = new ERR_TEST_FAILURE( + 'Test passed but was expected to fail', + kTestCodeFailure, + ); + return; + } + this.passed = true; } @@ -1350,7 +1420,10 @@ class Test extends AsyncResource { } else if (this.isTodo) { directive = this.reporter.getTodo(this.message); } else if (this.expectFailure) { - directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure + const message = typeof this.expectFailure === 'object' ? + this.expectFailure.label : + this.expectFailure; + directive = this.reporter.getXFail(message); } if (this.reportedType) { diff --git a/test/parallel/test-runner-xfail.js b/test/parallel/test-runner-xfail.js new file mode 100644 index 00000000000000..0648b32c5b94c2 --- /dev/null +++ b/test/parallel/test-runner-xfail.js @@ -0,0 +1,154 @@ +'use strict'; +const common = require('../common'); +const { test } = require('node:test'); +const { spawn } = require('child_process'); +const assert = require('node:assert'); + +if (process.env.CHILD_PROCESS === 'true') { + test('fail with message string', { expectFailure: 'reason string' }, () => { + assert.fail('boom'); + }); + + test('fail with label object', { expectFailure: { label: 'reason object' } }, () => { + assert.fail('boom'); + }); + + test('fail with match regex', { expectFailure: { match: /boom/ } }, () => { + assert.fail('boom'); + }); + + test('fail with match object', { expectFailure: { match: { message: 'boom' } } }, () => { + assert.fail('boom'); + }); + + test('fail with match class', { expectFailure: { match: assert.AssertionError } }, () => { + assert.fail('boom'); + }); + + test('fail with match error (wrong error)', { expectFailure: { match: /bang/ } }, () => { + assert.fail('boom'); // Should result in real failure because error doesn't match + }); + + test('unexpected pass', { expectFailure: true }, () => { + // Should result in real failure because it didn't fail + }); + + test('fail with empty string', { expectFailure: '' }, () => { + assert.fail('boom'); + }); + + // 1. Matcher: RegExp + test('fails with regex matcher', { expectFailure: /expected error/ }, () => { + throw new Error('this is the expected error'); + }); + + test('fails with regex matcher (mismatch)', { expectFailure: /expected error/ }, () => { + throw new Error('wrong error'); // Should fail the test + }); + + // 2. Matcher: Class + test('fails with class matcher', { expectFailure: RangeError }, () => { + throw new RangeError('out of bounds'); + }); + + test('fails with class matcher (mismatch)', { expectFailure: RangeError }, () => { + throw new TypeError('wrong type'); // Should fail the test + }); + + // 3. Matcher: Object (Properties) + test('fails with object matcher', { expectFailure: { code: 'ERR_TEST' } }, () => { + const err = new Error('boom'); + err.code = 'ERR_TEST'; + throw err; + }); + + test('fails with object matcher (mismatch)', { expectFailure: { code: 'ERR_TEST' } }, () => { + const err = new Error('boom'); + err.code = 'ERR_WRONG'; + throw err; // Should fail + }); + + // 4. Configuration Object: Reason + Validation + test('fails with config object (label + match)', { + expectFailure: { + label: 'Bug #124', + match: /boom/ + } + }, () => { + throw new Error('boom'); + }); + + test('fails with config object (label only)', { + expectFailure: { label: 'Bug #125' } + }, () => { + throw new Error('boom'); + }); + + test('fails with config object (match only)', { + expectFailure: { match: /boom/ } + }, () => { + throw new Error('boom'); + }); + + // 5. Edge Case: Empty Object (Should throw ERR_INVALID_ARG_VALUE during creation) + try { + test('invalid empty object', { expectFailure: {} }, () => {}); + } catch (e) { + console.log(`CAUGHT_INVALID_ARG: ${e.code}`); + } + + // 6. Primitives and Truthiness + test('fails with boolean true', { expectFailure: true }, () => { + throw new Error('any error'); + }); + + // 7. Unexpected Pass (Enhanced) + test('unexpected pass (reason string)', { expectFailure: 'should fail' }, () => { + // Pass + }); + + test('unexpected pass (matcher)', { expectFailure: /boom/ }, () => { + // Pass + }); + +} else { + const child = spawn(process.execPath, ['--test-reporter', 'tap', __filename], { + env: { ...process.env, CHILD_PROCESS: 'true' }, + stdio: 'pipe', + }); + + let stdout = ''; + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { stdout += chunk; }); + + child.on('close', common.mustCall((code) => { + // We expect exit code 1 because 'unexpected pass' and 'wrong error' should fail the test run + assert.strictEqual(code, 1); + + assert.match(stdout, /ok \d+ - fail with message string # EXPECTED FAILURE reason string/); + assert.match(stdout, /ok \d+ - fail with label object # EXPECTED FAILURE reason object/); + assert.match(stdout, /ok \d+ - fail with match regex # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fail with match object # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fail with match class # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - fail with match error \(wrong error\) # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - unexpected pass # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fail with empty string # EXPECTED FAILURE/); + + // New tests verification + assert.match(stdout, /ok \d+ - fails with regex matcher # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - fails with regex matcher \(mismatch\) # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fails with class matcher # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - fails with class matcher \(mismatch\) # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fails with object matcher # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - fails with object matcher \(mismatch\) # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fails with config object \(label \+ match\) # EXPECTED FAILURE Bug \\#124/); + assert.match(stdout, /ok \d+ - fails with config object \(label only\) # EXPECTED FAILURE Bug \\#125/); + assert.match(stdout, /ok \d+ - fails with config object \(match only\) # EXPECTED FAILURE/); + assert.match(stdout, /ok \d+ - fails with boolean true # EXPECTED FAILURE/); + assert.match(stdout, /not ok \d+ - unexpected pass \(reason string\) # EXPECTED FAILURE should fail/); + assert.match(stdout, /not ok \d+ - unexpected pass \(matcher\) # EXPECTED FAILURE/); + + // Empty object error + assert.match(stdout, /CAUGHT_INVALID_ARG: ERR_INVALID_ARG_VALUE/); + })); +}